diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 00000000000..92524056522 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,50 @@ +env: + CIRRUS_CLONE_DEPTH: 50 + ELIXIR_ASSERT_TIMEOUT: 2000 + ELIXIRC_OPTS: "--warnings-as-errors" + ERLC_OPTS: "warnings_as_errors" + LANG: C.UTF-8 + +test_template: &DEFAULT_TEST_SETTINGS + # don't cancel the task execution if it's master or a release branch + auto_cancellation: $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH !=~ 'v\d+\.\d+.*' + +test_freebsd_task: + <<: *DEFAULT_TEST_SETTINGS + + name: FreeBSD 13.0 + alias: FreeBSD Stable + + freebsd_instance: + image_family: freebsd-13-0 + cpu: 8 + memory: 7424Mi + + env: + CHECK_REPRODUCIBLE: true + LC_ALL: en_US.UTF-8 + PATH: $PATH:/usr/local/lib/erlang24/bin + + install_script: + - pkg install -y erlang-runtime24 git gmake + - rm -rf .git + - gmake compile + + build_info_script: bin/elixir --version + + test_formatted_script: + - gmake test_formatted && + echo "All Elixir source code files are properly formatted." + + dialyzer_script: dialyzer -pa lib/elixir/ebin --build_plt --output_plt elixir.plt --apps lib/elixir/ebin/elixir.beam lib/elixir/ebin/Elixir.Kernel.beam + + test_erlang_script: gmake test_erlang + + test_elixir_script: gmake test_elixir + + check_reproducible_script: | + if [ -n "$CHECK_REPRODUCIBLE" ]; then + gmake check_reproducible + else + echo "The reproducibility of the build is only checked in the last stable Erlang/OTP version." + fi diff --git a/.formatter.exs b/.formatter.exs new file mode 100644 index 00000000000..026e9e29387 --- /dev/null +++ b/.formatter.exs @@ -0,0 +1,19 @@ +[ + inputs: [ + "lib/*/{lib,unicode,test}/**/*.{ex,exs}", + "lib/*/*.exs", + "lib/ex_unit/examples/*.exs", + ".formatter.exs" + ], + locals_without_parens: [ + # Formatter tests + assert_format: 2, + assert_format: 3, + assert_same: 1, + assert_same: 2, + + # Errors tests + assert_eval_raise: 3 + ], + normalize_bitstring_modifiers: false +] diff --git a/.gitattributes b/.gitattributes index 1a79d0b894c..f0d70e87e1f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,3 @@ lib/elixir/test/elixir/fixtures/*.txt text eol=lf +*.ex diff=elixir +*.exs diff=elixir diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..2772f43485d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +--- +blank_issues_enabled: true + +contact_links: + - name: Discuss proposals + url: https://groups.google.com/g/elixir-lang-core + about: Send proposals for new ideas to our mailing list + + - name: Ask questions and support + url: https://elixirforum.com/ + about: Ask questions, provide support and more on Elixir Forum diff --git a/.github/ISSUE_TEMPLATE/issue.yml b/.github/ISSUE_TEMPLATE/issue.yml new file mode 100644 index 00000000000..160fcae30cf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/issue.yml @@ -0,0 +1,53 @@ +--- +name: Report an issue +description: + Tell us about something that is not working the way we (probably) intend +body: + - type: markdown + attributes: + value: > + Thank you for contributing to Elixir! :heart: + + + Please, do not use this form for guidance, questions or support. + Try instead in [Elixir Forum](https://elixirforum.com), + the [IRC Chat](https://web.libera.chat/#elixir), + [Stack Overflow](https://stackoverflow.com/questions/tagged/elixir), + [Slack](https://elixir-slackin.herokuapp.com), + [Discord](https://discord.gg/elixir) or in other online communities. + + - type: textarea + id: elixir-and-otp-version + attributes: + label: Elixir and Erlang/OTP versions + description: Paste the output of `elixir --version` here. + validations: + required: true + + - type: input + id: os + attributes: + label: Operating system + description: The operating system that this issue is happening on. + validations: + required: true + + - type: textarea + id: current-behavior + attributes: + label: Current behavior + description: > + Include code samples, errors, and stacktraces if appropriate. + + + If reporting a bug, please include the reproducing steps. + validations: + required: true + + - type: textarea + id: expected-behavior + attributes: + label: Expected behavior + description: A short description on how you expect the code to behave. + validations: + required: true diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..5ace4600a1f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..8eb424bdacd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,129 @@ +name: CI + +on: [pull_request, push] + +env: + ELIXIR_ASSERT_TIMEOUT: 2000 + ELIXIRC_OPTS: "--warnings-as-errors" + ERLC_OPTS: "warnings_as_errors" + LANG: C.UTF-8 + +permissions: + contents: read + +jobs: + test_linux: + name: Linux, ${{ matrix.otp_release }}, Ubuntu 18.04 + strategy: + fail-fast: false + matrix: + include: + - otp_release: OTP-25.0 + otp_latest: true + - otp_release: OTP-24.3 + - otp_release: OTP-24.0 + - otp_release: OTP-23.3 + - otp_release: OTP-23.0 + - otp_release: master + development: true + - otp_release: maint + development: true + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 50 + - name: Install Erlang/OTP + run: | + cd $RUNNER_TEMP + wget -O otp.tar.gz https://repo.hex.pm/builds/otp/ubuntu-18.04/${{ matrix.otp_release }}.tar.gz + mkdir -p otp + tar zxf otp.tar.gz -C otp --strip-components=1 + otp/Install -minimal $(pwd)/otp + echo "$(pwd)/otp/bin" >> $GITHUB_PATH + - name: Compile Elixir + run: | + rm -rf .git + make compile + echo "$PWD/bin" >> $GITHUB_PATH + - name: Build info + run: bin/elixir --version + - name: Check format + run: make test_formatted && echo "All Elixir source code files are properly formatted." + - name: Run Dialyzer + run: dialyzer -pa lib/elixir/ebin --build_plt --output_plt elixir.plt --apps lib/elixir/ebin/elixir.beam lib/elixir/ebin/Elixir.Kernel.beam + - name: Erlang test suite + run: make test_erlang + continue-on-error: ${{ matrix.development }} + - name: Elixir test suite + run: make test_elixir + continue-on-error: ${{ matrix.development }} + - name: Check reproducible builds + run: taskset 1 make check_reproducible + if: ${{ matrix.otp_latest }} + - name: Build docs + if: ${{ matrix.otp_latest }} + run: | + git config --global advice.detachedHead false + EX_DOC_LATEST_STABLE_VERSION=$(curl -s https://hex.pm/api/packages/ex_doc | jq --raw-output '.latest_stable_version') + for branch in main v${EX_DOC_LATEST_STABLE_VERSION}; do + echo "Building docs with ExDoc ${branch}" + cd .. + git clone https://github.com/elixir-lang/ex_doc.git --branch ${branch} --depth 1 + cd ex_doc + ../elixir/bin/mix do local.rebar --force + local.hex --force + deps.get + compile + cd ../elixir/ + make docs + rm -rf ../ex_doc/ + done + + test_windows: + name: Windows, OTP-${{ matrix.otp_release }}, Windows Server 2019 + strategy: + matrix: + otp_release: ['23.3'] + runs-on: windows-2019 + steps: + - name: Configure Git + run: git config --global core.autocrlf input + - uses: actions/checkout@v3 + with: + fetch-depth: 50 + - name: Cache Erlang/OTP package + uses: actions/cache@v4 + with: + path: C:\Users\runneradmin\AppData\Local\Temp\chocolatey\erlang + key: OTP-${{ matrix.otp_release }}-windows-2019 + - name: Install Erlang/OTP + run: choco install -y erlang --version ${{ matrix.otp_release }} + - name: Compile Elixir + run: | + remove-item '.git' -recurse -force + make compile + - name: Build info + run: bin/elixir --version + - name: Check format + run: make test_formatted && echo "All Elixir source code files are properly formatted." + - name: Erlang test suite + run: make --keep-going test_erlang + - name: Elixir test suite + run: | + del c:/Windows/System32/drivers/etc/hosts + make --keep-going test_elixir + + check_posix_compliant: + name: Check POSIX-compliant + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 50 + - name: Install Shellcheck + run: | + sudo apt update + sudo apt install -y shellcheck + - name: Check POSIX-compliant + run: | + shellcheck -e SC2039,2086 bin/elixir && echo "bin/elixir is POSIX compliant" + shellcheck bin/elixirc && echo "bin/elixirc is POSIX compliant" + shellcheck bin/iex && echo "bin/iex is POSIX compliant" diff --git a/.github/workflows/notify.exs b/.github/workflows/notify.exs new file mode 100644 index 00000000000..6fd136111b3 --- /dev/null +++ b/.github/workflows/notify.exs @@ -0,0 +1,62 @@ +# #!/usr/bin/env elixir +[tag] = System.argv() + +Mix.install([ + {:req, "~> 0.2.1"}, + {:jason, "~> 1.0"} +]) + +%{status: 200, body: body} = + Req.get!("https://api.github.com/repos/elixir-lang/elixir/releases/tags/#{tag}") + +if body["draft"] do + raise "cannot notify a draft release" +end + +## Notify on elixir-lang-ann + +names_and_checksums = + for asset <- body["assets"], + name = asset["name"], + name =~ ~r/.sha\d+sum$/, + do: {name, Req.get!(asset["browser_download_url"]).body} + +line_items = + for {name, checksum_and_name} <- Enum.sort(names_and_checksums) do + [checksum | _] = String.split(checksum_and_name, " ") + root = Path.rootname(name) + "." <> type = Path.extname(name) + " * #{root} - #{type} - #{checksum}\n" + end + +headers = %{ + "X-Postmark-Server-Token" => System.fetch_env!("ELIXIR_LANG_ANN_TOKEN") +} + +body = %{ + "From" => "jose.valim@dashbit.co", + "To" => "elixir-lang-ann@googlegroups.com", + "Subject" => "Elixir #{tag} released", + "HtmlBody" => "https://github.com/elixir-lang/elixir/releases/tag/#{tag}\n\n#{line_items}", + "MessageStream": "outbound" +} + +resp = Req.post!("https://api.postmarkapp.com/email", {:json, body}, headers: headers) +IO.puts("#{resp.status} elixir-lang-ann\n#{inspect(resp.body)}") + +## Notify on Elixir Forum + +headers = %{ + "api-key" => System.fetch_env!("ELIXIR_FORUM_TOKEN"), + "api-username" => "Elixir" +} + +body = %{ + "title" => "Elixir #{tag} released", + "raw" => "https://github.com/elixir-lang/elixir/releases/tag/#{tag}\n\n#{body["body"]}", + # Elixir News + "category" => 28 +} + +resp = Req.post!("https://elixirforum.com/posts.json", {:json, body}, headers: headers) +IO.puts("#{resp.status} Elixir Forum\n#{inspect(resp.body)}") diff --git a/.github/workflows/notify.yml b/.github/workflows/notify.yml new file mode 100644 index 00000000000..af779483625 --- /dev/null +++ b/.github/workflows/notify.yml @@ -0,0 +1,28 @@ +name: Notify + +on: + release: + types: + - published + +permissions: + contents: read + +jobs: + notify: + runs-on: ubuntu-18.04 + name: Notify + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 50 + - uses: erlef/setup-beam@v1 + with: + otp-version: 24.3 + elixir-version: 1.13.4 + - name: Run Elixir script + env: + ELIXIR_FORUM_TOKEN: ${{ secret.ELIXIR_FORUM_TOKEN }} + ELIXIR_LANG_ANN_TOKEN: ${{ secret.ELIXIR_LANG_ANN_TOKEN }} + run: | + elixir .github./workflows/notify.exs ${{ github.ref_name }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..091d4050dde --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,96 @@ +name: Release + +on: + push: + tags: + - v* + +env: + ELIXIR_OPTS: "--warnings-as-errors" + ERLC_OPTS: "warnings_as_errors" + LANG: C.UTF-8 + +jobs: + create_draft_release: + permissions: + contents: none + runs-on: ubuntu-18.04 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Create draft release + run: | + gh release create \ + --repo ${{ github.repository }} \ + --title ${{ github.ref_name }} \ + --notes '' \ + --draft \ + ${{ github.ref_name }} + release_pre_built: + needs: create_draft_release + strategy: + fail-fast: true + matrix: + include: + - otp: 23 + otp_version: 23.3 + - otp: 24 + otp_version: 24.3 + - otp: 25 + otp_version: 25.0 + build_docs: build_docs + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 50 + - uses: erlef/setup-beam@v1 + with: + otp-version: ${{ matrix.otp_version }} + version-type: strict + - name: Build Elixir Release + run: | + make Precompiled.zip + mv Precompiled.zip elixir-otp-${{ matrix.otp }}.zip + shasum -a 1 elixir-otp-${{ matrix.otp }}.zip > elixir-otp-${{ matrix.otp }}.zip.sha1sum + shasum -a 256 elixir-otp-${{ matrix.otp }}.zip > elixir-otp-${{ matrix.otp }}.zip.sha256sum + echo "$PWD/bin" >> $GITHUB_PATH + - name: Upload Pre-built + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload --clobber "${{ github.ref_name }}" \ + elixir-otp-${{ matrix.otp }}.zip \ + elixir-otp-${{ matrix.otp }}.zip.sha{1,256}sum + - name: Get latest stable ExDoc version + if: ${{ matrix.build_docs }} + run: | + EX_DOC_LATEST_STABLE_VERSION=$(curl -s https://hex.pm/api/packages/ex_doc | jq --raw-output '.latest_stable_version') + echo "EX_DOC_LATEST_STABLE_VERSION=${EX_DOC_LATEST_STABLE_VERSION}" >> $GITHUB_ENV + - uses: actions/checkout@v3 + if: ${{ matrix.build_docs }} + with: + repository: elixir-lang/ex_doc + ref: v${{ env.EX_DOC_LATEST_STABLE_VERSION }} + path: ex_doc + - name: Build ex_doc + if: ${{ matrix.build_docs }} + run: | + mv ex_doc ../ex_doc + cd ../ex_doc + ../elixir/bin/mix do local.rebar --force + local.hex --force + deps.get + compile + cd ../elixir + - name: Build Docs + if: ${{ matrix.build_docs }} + run: | + make Docs.zip + shasum -a 1 Docs.zip > Docs.zip.sha1sum + shasum -a 256 Docs.zip > Docs.zip.sha256sum + - name: Upload Docs + if: ${{ matrix.build_docs }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload --clobber "${{ github.ref_name }}" \ + Docs.zip \ + Docs.zip.sha{1,256}sum diff --git a/.gitignore b/.gitignore index 9f5a6a2479e..0760a308146 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,13 @@ -/.eunit -/.release -/docs -/ebin -/lib/*/ebin/* -/lib/*/tmp -/lib/elixir/src/elixir.app.src -/lib/elixir/src/*_lexer.erl +/doc/ +/lib/*/ebin/ +/lib/*/_build/ +/lib/*/tmp/ /lib/elixir/src/*_parser.erl -/lib/elixir/test/ebin -/rel/elixir +/lib/elixir/test/ebin/ +/man/elixir.1 +/man/iex.1 +/Docs-v*.zip +/Precompiled-v*.zip +/.eunit +.elixir.plt erl_crash.dump -.dialyzer_plt -.dialyzer.base_plt diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index dcb0d1f96b1..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -language: erlang -script: "make compile && rm -rf .git && make test" -notifications: - irc: "irc.freenode.org#elixir-lang" - recipients: - - jose.valim@plataformatec.com.br - - eric.meadows.jonsson@gmail.com -otp_release: - - 17.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 104d62bc78f..69ac7d3fc3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,1133 +1,334 @@ -# Changelog - -## v0.14.3-dev - -* Enhancements - -* Bug fixes - * [Kernel] `|>`, `<<<`, `>>>` and `^^^` were made left associative in operator table - * [Kernel] `<`, `>`, `<=`, `>=` were given higher precedence than comparison ones (`==`, `!=`, etc) in operator table - -* Soft deprecations (no warnings emitted) - -* Deprecations - -* Backwards incompatible changes - -## v0.14.2 (2014-06-29) - -* Enhancements - * [Enum] Improve performance of `Enum.join/2` and `Enum.map_join/3` by using iolists - * [Kernel] Ensure compatibility with Erlang 17.1 - * [Kernel] Support `@external_resource` attribute to external dependencies to a module - * [Mix] Allow built Git dependencies to run on a system without Git by passing `--no-deps-check` - * [Mix] Add `MIX_ARCHIVES` env variable (it is recommended for Elixir build tools to swap this environment) - * [Task] Set `:proc_lib` initial call on task to aid debugging - * [Typespec] Delay typespec compilation to after expansion - * [URI] Allow `parse/1` now accepts `%URI{}` as argument and return the uri itself - -* Bug fixes - * [CLI] Support paths inside archives in `-pa` and `-pz` options - * [IEx] Remove delay when printing data from the an application start callback - * [IEx] Ensure we show a consistent error when we cannot evaluate `.iex.exs` - * [Kernel] Ensure derived protocols are defined with a file - * [Kernel] Change precedence of `&` to not special case `/` - * [Kernel] Ensure we can only use variables and `\\` as arguments of bodyless clause - -* Soft deprecations (no warnings emitted) - * [EEx] Using `EEx.TransformerEngine` and `EEx.AssignsEngine` are deprecated in favor of function composition with `Macro.prewalk/1` (see `EEx.SmartEngine` for an example) - * [Kernel] `Kernel.xor/2` is deprecated - * [Mix] `Mix.Generator.from_file/1` is deprecated in favor of passing `from_file: file` option to `embed_text/2` and `embed_template/2` (note though that `from_file/1` expects a path relative to the current file while the `from_file: file` expects one relative to the current working directory) - -* Deprecations - * [Kernel] `size/1` is deprecated in favor of `byte_size/1` and `tuple_size/1` (this change was soft deprecated two releases ago) - -* Backwards incompatible changes - * [CLI] Remove support for the `--gen-debug` option as its usage is not documented by OTP - * [Kernel] Sigils no longer balance start and end tokens, e.g. the sigil `~s(f(o)o)` is no longer valid as it finishes in the first closing `)` - * [Kernel] Variables set in `cond` clause heads are no longer available outside of that particular `cond` clause (this is the behaviour also found in `case`, `receive` and friends) - * [System] `build_info/0` now returns a map - -## v0.14.1 (2014-06-18) - -* Enhancements - * [Base] Decoding and encoding functions now accept the `:case` as an option - * [ExUnit] The test process now exits with `:shutdown` reason - * [GenEvent] `GenEvent.stream/2` now accepts `:sync` and `:async` modes - * [Node] Add `Node.start/3` and `Node.stop/0` - * [String] Updated Unicode database to 7.0 - * [Task] Log when tasks crash - -* Bug fixes - * [Enum] `Enum.slice/2` and `Enum.slice/3` always returns a list (and never nil) - * [Kernel] Disambiguate (w)erl to (w)erl.exe - * [Mix] Ensure umbrella project is recompiled when a dependency inside an umbrella child changes - * [OptionParser] Do not allow underscores in option names - * [Path] Fix path expansion of `"/.."` - * [Path] Do not match files starting with `.` in `Path.wildcard/2` by default - * [Process] `Process.info(pid, :registered_name)` returns `{:registered_name, nil}` if there is no registered name - * [String] `String.slice/2` and `String.slice/3` always returns a list (and never nil) - * [URI] `encode/1` does not escape reserved/unreserved characters by default nor encodes whitespace as `+` (check `URI.encode_www_form/1` and `URI.decode_www_form/1` for previous behaviour) - -* Deprecations - * [Mix] `:escript_*` options were moved into a single `:escript` group - -* Backwards incompatible changes - * [GenEvent] `GenEvent.stream/2` defaults to `:sync` mode - * [Kernel] Remove `get_in/1` - -## v0.14.0 (2014-06-08) - -* Enhancements - * [ExUnit] Add `on_exit/1` callbacks that are guaranteed to run once the test process exits and always in another process - * [Kernel] Store documentation in the abstract code to avoid loading them when the module is loaded - * [Kernel] Add `get_in/2`, `put_in/3`, `update_in/3` and `get_and_update_in/3` to handle nested data structure operations - * [Kernel] Add `get_in/1`, `put_in/2`, `update_in/2` and `get_and_update_in/2` to handle nested data structure operations via paths - * [Mix] Add `Mix.Config` to ease definition of configuration files - * [Mix] Add `mix loadconfig` task that can be called multiple times to load external configs - * [Mix] Support `--config` option on `mix run` - * [Mix] Support `HTTP_PROXY` and `HTTPS_PROXY` on Mix url commands - * [Mix] Support `--names` options in `mix help` which emit only names (useful for autocompletion) - * [Protocol] Add `Protocol.consolidate/2`, `Protocol.consolidated?/1` and a `mix compile.protocols` task for protocol consolidation - * [Protocol] Add `Protocol.derive/3` for runtime deriving of a struct - * [String] Add `String.chunk/2` - * [Struct] Add support for `@derive` before `defstruct/2` definitions - -* Bug fixes - * [File] `File.rm` now consistently deletes read-only across operating systems - * [Kernel] Ensure Mix `_build` structure works on Windows when copying projects - * [Kernel] Ensure `1.0E10` (with uppercase E) is also valid syntax - * [Mix] Fix `mix do` task for Windows' powershell users - * [Path] Fix `Path.absname("/")` and `Path.expand("/")` to return the absolute path `"/"`. - -* Soft deprecations (no warnings emitted) - * [Kernel] `size/1` is deprecated, please use `byte_size/1` or `tuple_size/1` instead - * [ExUnit] `teardown/2` and `teardown_all/2` are deprecated in favor of `on_exit/1` callbacks - -* Deprecations - * [Access] `Access.access/2` is deprecated in favor of `Access.get/2` - * [Dict] `Dict.Behaviour` is deprecated in favor of `Dict` - * [Kernel] `Application.Behaviour`, `GenEvent.Behaviour`, `GenServer.Behaviour` and `Supervisor.Behaviour` are deprecated in favor of `Application`, `GenEvent`, `GenServer` and `Supervisor` - * [Kernel] `defexception/3` is deprecated in favor of `defexception/1` - * [Kernel] `raise/3` is deprecated in favor of `reraise/2` - * [Kernel] `set_elem/3` is deprecated in favor of `put_elem/3` - * [Kernel] Passing an atom `var!/1` is deprecated, variables can be built dynamically with `Macro.var/2` - * [Mix] Exceptions that define a `:mix_error` field to be compatible with Mix are no longer supported. Instead please provide a `:mix` field and use `Mix.raise/1` and `Mix.raise/2` - -* Backwards incompatible changes - * [Access] `Kernel.access/2` no longer exists and the `Access` protocol now requires `get/2` (instead of `access/2`) and `get_and_update/3` to be implemented - * [Kernel] Retrieving docs as `module.__info__(:docs)` is no longer supported, please use `Code.get_docs/2` instead - * [Kernel] `Code.compiler_options/1` no longer accepts custom options, only the ones specified by Elixir (use mix config instead) - * [Mix] `mix new` no longer generates a supevision tree by default, please pass `--sup` instead - * [Task] Tasks are automatically linked to callers and a failure in the task will crash the caller directly - -## v0.13.3 (2014-05-24) - -* Enhancements - * [OptionParser] Add `:strict` option that only parses known switches - * [OptionParser] Add `next/2` useful for manual parsing of options - * [Macro] Add `Macro.prewalk/2/3` and `Macro.postwalk/2/3` - * [Kernel] `GenEvent`, `GenServer`, `Supervisor`, `Agent` and `Task` modules added - * [Kernel] Make deprecations compiler warnings to avoid the same deprecation being printed multiple times - -* Bug fixes - * [Enum] Fix `Enum.join/2` and `Enum.map_join/3` for empty binaries at the beginning of the collection - * [ExUnit] Ensure the formatter doesn't error when printing :EXITs - * [Kernel] Rename `ELIXIR_ERL_OPTS` to `ELIXIR_ERL_OPTIONS` for consistency with `ERL_COMPILER_OPTIONS` - * [OptionParser] Parse `-` as a plain argument - * [OptionParser] `--` is always removed from argument list on `parse/2` and when it is the leading entry on `parse_head/2` - * [Regex] Properly escape regex (previously regex controls were double escaped) - -* Soft deprecations (no warnings emitted) - * [Dict] `Dict.Behaviour` is deprecated in favor of `Dict` - * [Kernel] `Application.Behaviour`, `GenEvent.Behaviour`, `GenServer.Behaviour` and `Supervisor.Behaviour` are deprecated in favor of `Application`, `GenEvent`, `GenServer` and `Supervisor` - * [Kernel] `defexception/3` is deprecated in favor of `defexception/1` - * [Kernel] `raise/3` is deprecated in favor of `reraise/2` - * [Kernel] `set_elem/3` is deprecated in favor of `put_elem/3` - -* Soft deprecations for conversions (no warnings emitted) - * [Kernel] `atom_to_binary/1` and `atom_to_list/1` are deprecated in favor of `Atom.to_string/1` and `Atom.to_char_list/1` - * [Kernel] `bitstring_to_list/1` and `list_to_bitstring/1` are deprecated in favor of the `:erlang` ones - * [Kernel] `binary_to_atom/1`, `binary_to_existing_atom/1`, `binary_to_float/1`, `binary_to_integer/1` and `binary_to_integer/2` are deprecated in favor of conversion functions in `String` - * [Kernel] `float_to_binary/*` and `float_to_list/*` are deprecated in favor of `Float.to_string/*` and `Float.to_char_list/*` - * [Kernel] `integer_to_binary/*` and `integer_to_list/*` are deprecated in favor of `Integer.to_string/*` and `Integer.to_char_list/*` - * [Kernel] `iodata_to_binary/1` and `iodata_length/1` are deprecated `IO.iodata_to_binary/1` and `IO.iodata_length/1` - * [Kernel] `list_to_atom/1`, `list_to_existing_atom/1`, `list_to_float/1`, `list_to_integer/1`, `list_to_integer/2` and `list_to_tuple/1` are deprecated in favor of conversion functions in `List` - * [Kernel] `tuple_to_list/1` is deprecated in favor of `Tuple.to_list/1` - * [List] `List.from_char_data/1` and `List.from_char_data!/1` deprecated in favor of `String.to_char_list/1` - * [String] `String.from_char_data/1` and `String.from_char_data!/1` deprecated in favor of `List.to_string/1` - -* Deprecations - * [Kernel] `is_exception/1`, `is_record/1` and `is_record/2` are deprecated in favor of `Exception.exception?1`, `Record.record?/1` and `Record.record?/2` - * [Kernel] `defrecord/3` is deprecated in favor of structs - * [Kernel] `:hygiene` in `quote` is deprecated - * [Mix] `Mix.project/0` is deprecated in favor of `Mix.Project.config/0` - * [Process] `Process.spawn/1`, `Process.spawn/3`, `Process.spawn_link/1`, `Process.spawn_link/3`, `Process.spawn_monitor/1`, `Process.spawn_monitor/3`, `Process.send/2` and `Process.self/0` are deprecated in favor of the ones in `Kernel` - -* Backwards incompatible changes - * [Exception] Exceptions now generate structs instead of records - * [OptionParser] Errors on parsing returns the switch and value as binaries (unparsed) - * [String] `String.to_char_list/1` (previously deprecated) no longer returns a tuple but the char list only and raises in case of failure - -## v0.13.2 (2014-05-11) - -* Enhancements - * [Application] Add an Application module with common functions to work with OTP applications - * [Exception] Add `Exception.message/1`, `Exception.format_banner/1`, `Exception.format_exit/1` and `Exception.format/1` - * [File] Add `File.ln_s/1` - * [Mix] `mix deps.clean` now works accross environments - * [Mix] Support line numbers in `mix test`, e.g. test/some/file_test.exs:12 - * [Mix] Use `@file` attributes to detect dependencies in between `.ex` and external files. This means changing an `.eex` file will no longer recompile the whole project only the files that depend directly on it - * [Mix] Support application configurations in `config/config.exs` - * [Mix] Support user-wide configuration with `~/.mix/config.exs` - * [Mix] `mix help` now uses ANSI formatting to print guides - * [Regex] Support functions in `Regex.replace/4` - * [String] Support `:parts` in `String.split/3` - -* Bug fixes - * [Code] Ensure we don't lose the caller stacktrace on code evaluation - * [IEx] Exit signals now exits the IEx evaluator and a new one is spawned on its place - * [IEx] Ensure we don't prune too much stacktrace when reporting failures - * [IEx] Fix an issue where `iex.bat` on Windows was not passing the proper parameters forward - * [Kernel] Ensure modules defined on root respect defined aliases - * [Kernel] Do not wrap single lists in `:__block__` - * [Kernel] Ensure emitted beam code works nicely with dialyzer - * [Kernel] Do not allow a module named `Elixir` to be defined - * [Kernel] Create remote funs even if mod is a variable in capture `&mod.fun/arity` - * [Kernel] Improve compiler message when duplicated modules are detected - * [Mix] Generate `.gitignore` for `--umbrella` projects - * [Mix] Verify if a git dependency in deps has a proper git checkout and clean it automatically when it doesn't - * [Mix] Ensure `mix test` works with `IEx.pry/0` - * [System] Convert remaining functions in System to rely on char data - -* Soft deprecations (no warnings emitted) - * [Exception] `exception.message` is deprecated in favor `Exception.message/1` for retrieving exception messages - * [Kernel] `is_exception/1`, `is_record/1` and `is_record/2` are deprecated in favor of `Exception.exception?1`, `Record.record?/1` and `Record.record?/2` - * [Mix] `Mix.project/0` is deprecated in favor of `Mix.Project.config/0` - * [Process] `Process.spawn/1`, `Process.spawn/3`, `Process.spawn_link/1`, `Process.spawn_link/3`, `Process.spawn_monitor/1`, `Process.spawn_monitor/3`, `Process.send/2` and `Process.self/0` are deprecated in favor of the ones in `Kernel` - -* Deprecations - * [IEx] IEx.Options is deprecated in favor of `IEx.configure/1` and `IEx.configuration/0` - * [Kernel] `lc` and `bc` comprehensions are deprecated in favor of `for` - * [Macro] `Macro.safe_terms/1` is deprecated - * [Process] `Process.delete/0` is deprecated - * [Regex] Deprecate `:global` option in `Regex.split/3` in favor of `parts: :infinity` - * [String] Deprecate `:global` option in `String.split/3` in favor of `parts: :infinity` - -* Backwards incompatible changes - * [ExUnit] `ExUnit.Test` and `ExUnit.TestCase` has been converted to structs - * [ExUnit] The test and callback context has been converted to maps - * [Kernel] `File.Stat`, `HashDict`, `HashSet`, `Inspect.Opts`, `Macro.Env`, `Range`, `Regex` and `Version.Requirement` have been converted to structs. This means `is_record/2` checks will no longer work, instead, you can pattern match on them using `%Range{}` and similar - * [URI] The `URI.Info` record has now become the `URI` struct - * [Version] The `Version.Schema` record has now become the `Version` struct - -## v0.13.1 (2014-04-27) - -* Enhancements - * [Mix] Support `MIX_EXS` as configuration for running the current mix.exs file - * [Mix] Support Hex out of the box. This means users do not need to install Hex directly, instead, Mix will prompt whenever there is a need to have Hex installed - -* Bug fixes - * [ExUnit] Ensure doctest failures are properly reported - * [Kernel] Fix a bug where comprehensions arguments were not properly take into account in the variable scope - * [Mix] Fix issue on rebar install when the endpoint was redirecting to a relative uri - -* Soft deprecations (no warnings emitted) - * [Kernel] `iolist_size` and `iolist_to_binary` are deprecated in favor of `iodata_length` and `iodata_to_binary` - * [String] `String.to_char_list/1` is deprecated in favor of `List.from_char_data/1` - * [String] `String.from_char_list/1` is deprecated in favor of `String.from_char_data/1` - -* Deprecations - * [Mix] `:env` key in project configuration is deprecated - * [Regex] `Regex.groups/1` is deprecated in favor of `Regex.names/1` - -* Backwards incompatible changes - * [Macro] `Macro.unpipe/1` now returns tuples and `Macro.pipe/2` was removed in favor of `Macro.pipe/3` which explicitly expects the second element of the tuple returned by the new `Macro.unpipe/1` - * [Path] The functions in Path now only emit strings as result, regardless if the input was a char list or a string - * [Path] Atoms are no longer supported in Path functions - * [Regex] Regexes are no longer unicode by default. Instead, they must be explicitly marked with the `u` option - -## v0.13.0 (2014-04-20) - -* Enhancements - * [Base] Add `Base` module which does conversions to bases 16, 32, hex32, 64 and url64 - * [Code] Add `Code.eval_file/2` - * [Collectable] Add the `Collectable` protocol that empowers `Enum.into/2` and `Stream.into/2` and the `:into` option in comprehensions - * [Collectable] Implement `Collectable` for lists, dicts, bitstrings, functions and provide both `File.Stream` and `IO.Stream` - * [EEx] Add `handle_body/1` callback to `EEx.Engine` - * [Enum] Add `Enum.group_by/2`, `Enum.into/2`, `Enum.into/3`, `Enum.traverse/2` and `Enum.sum/2` - * [ExUnit] Randomize cases and tests suite runs, allow seed configuration and the `--seed` flag via `mix test` - * [ExUnit] Support `--only` for filtering when running tests with `mix test` - * [ExUnit] Raise an error if another `capture_io` process already captured the device - * [ExUnit] Improve formatter to show source code and rely on lhs and rhs (instead of expected and actual) - * [IEx] Allow prompt configuration with the `:prompt` option - * [IEx] Use werl on Windows - * [Kernel] Support `ERL_PATH` in `bin/elixir` - * [Kernel] Support interpolation in keyword syntax - * [Map] Add a Map module and support 17.0 maps and structs - * [Mix] Add dependency option `:only` to specify the dependency environment. `mix deps.get` and `mix deps.update` works accross all environment unless `--only` is specified - * [Mix] Add `Mix.Shell.prompt/1` - * [Mix] Ensure the project is compiled in case Mix' CLI cannot find a task - * [Node] Add `Node.ping/1` - * [Process] Include `Process.send/3` and support the `--gen-debug` option - * [Regex] Regexes no longer need the "g" option when there is a need to use named captures - * [Stream] Add `Stream.into/2` and `Stream.into/3` - * [StringIO] Add a `StringIO` module that allows a String to be used as IO device - * [System] Add `System.delete_env/1` to remove a variable from the environment - -* Bug fixes - * [CLI] Ensure `--app` is handled as an atom before processing - * [ExUnit] Ensure `ExUnit.Assertions` does not emit compiler warnings for `assert_receive` - * [Kernel] Ensure the same pid is not queued twice in the parallel compiler - * [Macro] `Macro.to_string/2` considers proper precedence when translating `!(foo > bar)` into a string - * [Mix] Automatically recompile on outdated Elixir version and show proper error messages - * [Mix] Ensure generated `.app` file includes core dependencies - * [Mix] Allow a dependency with no SCM to be overridden - * [Mix] Allow queries in `mix local.install` URL - * [OptionParser] Do not recognize undefined aliases as switches - -* Soft deprecations (no warnings emitted) - * [Kernel] `lc` and `bc` comprehensions are deprecated in favor of `for` - * [ListDict] `ListDict` is deprecated in favor of `Map` - * [Record] `defrecord/2`, `defrecordp/3`, `is_record/1` and `is_record/2` macros in Kernel are deprecated. Instead, use the new macros and API defined in the `Record` module - -* Deprecations - * [Dict] `Dict.empty/1`, `Dict.new/1` and `Dict.new/2` are deprecated - * [Exception] `Exception.normalize/1` is deprecated in favor of `Exception.normalize/2` - -* Backwards incompatible changes - * [ExUnit] Formatters are now required to be a GenEvent and `ExUnit.run/2` returns a map with results - -## v0.12.5 (2014-03-09) - -* Bug fixes - * [Kernel] Ensure `try` does not generate an after clause. Generating an after clause forbade clauses in the `else` part from being tail recursive. This should improve performance and memory consumption of `Stream` functions - * [Mix] Automatically recompile on outdated Elixir version and show proper error messages - -* Deprecations - * [File] `File.stream_to!/3` is deprecated - * [GenFSM] `GenFSM` is deprecated - * [Kernel] `%` for sigils is deprecated in favor of `~` - * [Kernel] `is_range/1` and `is_regex/1` are deprecated in favor of `Range.range?/1` and `Regex.regex?/1` - * [Stream] `Stream.after/1` is deprecated - * [URI] `URI.decode_query/1` is deprecated in favor of `URI.decode_query/2` with explicit dict argument - * [URI] Passing lists as key or values in `URI.encode_query/1` is deprecated - -* Backwards incompatible changes - * [Mix] Remove `MIX_GIT_FORCE_HTTPS` as Git itself already provides mechanisms for doing so - -## v0.12.4 (2014-02-12) - -* Enhancements - * [Mix] `mix deps.get` and `mix deps.update` no longer compile dependencies afterwards. Instead, they mark the dependencies which are going to be automatically compiled next time `deps.check` is invoked (which is done automatically by most mix tasks). This means users should have a better workflow when migrating in between environments - -* Deprecations - * [Kernel] `//` for default arguments is deprecated in favor of `\\` - * [Kernel] Using `%` for sigils is deprecated in favor of `~`. This is a soft deprecation, no warnings will be emitted for it in this release - * [Kernel] Using `^` inside function clause heads is deprecated, please use a guard instead - -* Backwards incompatible changes - * [ExUnit] `CaptureIO` returns an empty string instead of nil when there is no capture - * [Version] The `Version` module now only works with SemVer. The functions `Version.parse/1` and `Version.parse_requirement/1` now return `{:ok,res} | :error` for the cases you want to handle non SemVer cases manually. All other functions will trigger errors on non semantics versions - -## v0.12.3 (2014-02-02) - -* Enhancements - * [Kernel] Warnings now are explicitly tagged with "warning:" in messages - * [Kernel] Explicit functions inlined by the compiler, including operators. This means that `Kernel.+/2` will now expand to `:erlang.+/2` and so on - * [Mix] Do not fail if a Mix dependency relies on an outdated Elixir version - * [Process] Add `Process.send/2` and `Process.send_after/3` - * [Version] Add `Version.compare/2` - -* Bug fixes - * [Atom] Inspect `:...` and `:foo@bar` without quoting - * [Keyword] The list `[1, 2, three: :four]` now correctly expands to `[1, 2, {:three, :four}]` - * [Kernel] Ensure undefined `@attributes` shows proper stacktrace in warnings - * [Kernel] Guarantee nullary funs/macros are allowed in guards - * [Process] Ensure monitoring functions are inlined by the compiler - -* Deprecations - * [IEx] The helper `m/0` has been deprecated. The goal is to group all runtime statistic related helpers into a single module - * [Kernel] `binary_to_term/1`, `binary_to_term/2`, `term_to_binary/1` and `term_to_binary/2` are deprecated in favor of their counterparts in the `:erlang` module - * [Kernel] `//` for default arguments is deprecated in favor of `\\`. This is a soft deprecation, no warnings will be emitted for it in this release - * [Kernel] Deprecated `@behavior` in favor of `@behaviour` - * [Record] `to_keywords`, `getter` and `list getter` functionalities in `defrecordp` are deprecated - * [Record] `Record.import/2` is deprecated - -* Backwards incompatible changes - * [Dict] Implementations of `equal?/2` and `merge/2` in `HashDict` and `ListDict` are no longer polymorphic. To get polymorphism, use the functions in `Dict` instead - * [File] `File.cp/3` and `File.cp_r/3` no longer carry Unix semantics where the function behaves differently if the destination is an existing previous directory or not. It now always copies source to destination, doing it recursively in the latter - * [IEx] IEx now loads the `.iex.exs` file instead of `.iex` - * [Kernel] Remove `**` from the list of allowed operators - * [Kernel] Limit sigils delimiters to one of the following: `<>`, `{}`, `[]`, `()`, `||`, `//`, `"` and `'` - * [Range] `Range` is no longer a record, instead use `first .. last` if you need pattern matching - * [Set] Implementations of `difference/2`, `disjoint?/2`, `equal?/2`, `intersection/2`, `subset?/2` and `union/2` in `HashSet` are no longer polymorphic. To get polymorphism, use the functions in `Set` instead - -## v0.12.2 (2014-01-15) - -* Enhancements - * [EEx] Allow `EEx.AssignsEngine` to accept any Dict - * [Enum] Add `Enum.flat_map_reduce/3` - * [ExUnit] Support `@moduletag` in ExUnit cases - * [Kernel] Improve stacktraces to be relative to the compilation path and include the related application - * [Stream] Add `Stream.transform/3` - -* Bug fixes - * [ExUnit] `:include` in ExUnit only has effect if a test was previously excluded with `:exclude` - * [ExUnit] Only run `setup_all` and `teardown_all` if there are tests in the case - * [Kernel] Ensure bitstring modifier arguments are expanded - * [Kernel] Ensure compiler does not block on missing modules - * [Kernel] Ensure `<>/2` works only with binaries - * [Kernel] Fix usage of string literals inside `<<>>` when `utf8`/`utf16`/`utf32` is used as specifier - * [Mix] Ensure mix properly copies _build dependencies on Windows - -* Deprecations - * [Enum] Deprecate `Enum.first/1` in favor of `Enum.at/2` and `List.first/1` - * [Kernel] Deprecate continuable heredocs. In previous versions, Elixir would continue parsing on the same line the heredoc started, this behaviour has been deprecated - * [Kernel] `is_alive/0` is deprecated in favor of `Node.alive?` - * [Kernel] `Kernel.inspect/2` with `Inspect.Opts[]` is deprecated in favor of `Inspect.Algebra.to_doc/2` - * [Kernel] `Kernel.inspect/2` with `:raw` option is deprecated, use `:records` option instead - * [Kernel] Deprecate `<-/2` in favor of `send/2` - -* Backwards incompatible changes - * [String] Change `String.next_grapheme/1` and `String.next_codepoint/1` to return `nil` on string end - -## v0.12.1 (2014-01-04) - -* Enhancements - * [ExUnit] Support `:include` and `:exclude` configuration options to filter which tests should run based on their tags. Those options are also supported via `mix test` as `--include` and `--exclude` - * [ExUnit] Allow doctests to match against `#MyModule<>` - -* Bug fixes - * [CLI] Abort when a pattern given to elixirc does not match any file - * [Float] Fix `Float.parse/1` to handle numbers of the form "-0.x" - * [IEx] Improve error message for `IEx.Helpers.r` when module does not exist - * [Mix] Ensure `deps.get` updates origin if lock origin and dep origin do not match - * [Mix] Use relative symlinks in _build - * [Typespec] Fix conversion of unary ops from typespec format to ast - * [Typespec] Fix handling of `tuple()` and `{}` - -* Deprecations - * [Kernel] Do not leak clause heads. Previously, a variable defined in a case/receive head clauses would leak to the outer scope. This behaviour is deprecated and will be removed in the next release. - * [Kernel] Deprecate `__FILE__` in favor of `__DIR__` or `__ENV__.file` - -* Backwards incompatible changes - * [GenFSM] GenServer now stops on unknown event/sync_event requests - * [GenServer] GenServer now stops on unknown call/cast requests - * [Kernel] Change how `->` is represented in AST. Now each clause is represented by its own AST node which makes composition easier. See commit 51aef55 for more information. - -## v0.12.0 (2013-12-15) - -* Enhancements - * [Exception] Allow `exception/1` to be overridden and promote it as the main mechanism to customize exceptions - * [File] Add `File.stream_to!/3` - * [Float] Add `Float.floor/1`, `Float.ceil/1` and `Float.round/3` - * [Kernel] Add `List.delete_at/2` and `List.updated_at/3` - * [Kernel] Add `Enum.reverse/2` - * [Kernel] Implement `defmodule/2`, `@/1`, `def/2` and friends in Elixir itself. `case/2`, `try/2` and `receive/1` have been made special forms. `var!/1`, `var!/2` and `alias!/1` have also been implemented in Elixir and demoted from special forms - * [Record] Support dynamic fields in `defrecordp` - * [Stream] Add `Stream.resource/3` - * [Stream] Add `Stream.zip/2`, `Stream.filter_map/3`, `Stream.each/2`, `Stream.take_every/2`, `Stream.chunk/2`, `Stream.chunk/3`, `Stream.chunk/4`, `Stream.chunk_by/2`, `Stream.scan/2`, `Stream.scan/3`, `Stream.uniq/2`, `Stream.after/2` and `Stream.run/1` - * [Stream] Support `Stream.take/2` and `Stream.drop/2` with negative counts - -* Bug fixes - * [HashDict] Ensure a `HashDict` stored in an attribute can be accessed via the attribute - * [Enum] Fix bug in `Enum.chunk/4` where you'd get an extra element when the enumerable was a multiple of the counter and a pad was given - * [IEx] Ensure `c/2` helper works with full paths - * [Kernel] `quote location: :keep` now only affects definitions in order to keep the proper trace in definition exceptions - * [Mix] Also symlink `include` directories in _build dependencies - * [Version] Fix `Version.match?/2` with `~>` and versions with alphanumeric build info (like `-dev`) - -* Deprecations - * [Enum] `Enumerable.count/1` and `Enumerable.member?/2` should now return tagged tuples. Please see `Enumerable` docs for more info - * [Enum] Deprecate `Enum.chunks/2`, `Enum.chunks/4` and `Enum.chunks_by/2` in favor of `Enum.chunk/2`, `Enum.chunk/4` and `Enum.chunk_by/2` - * [File] `File.binstream!/3` is deprecated. Simply use `File.stream!/3` which is able to figure out if `stream` or `binstream` operations should be used - * [Macro] `Macro.extract_args/1` is deprecated in favor of `Macro.decompose_call/1` - -* Backwards incompatible changes - * [Enum] Behaviour of `Enum.drop/2` and `Enum.take/2` has been switched when given negative counts - * [Enum] Behaviour of `Enum.zip/2` has been changed to stop as soon as the first enumerable finishes - * [Enum] `Enumerable.reduce/3` protocol has changed to support suspension. Please see `Enumerable` docs for more info - * [Mix] Require `:escript_main_module` to be set before generating escripts - * [Range] `Range.Iterator` protocol has changed in order to work with the new `Enumerable.reduce/3`. Please see `Range.Iterator` docs for more info - * [Stream] The `Stream.Lazy` structure has changed to accumulate functions and accumulators as we go (its inspected representation has also changed) - * [Typespec] `when` clauses were moved to the outer part of the spec and should be in the keywords format. So `add(a, b) when is_subtype(a, integer) and is_subtype(b, integer) :: integer` should now be written as `add(a, b) :: integer when a: integer, b: integer` - -## v0.11.2 (2013-11-14) - -* Enhancements - * [Mix] Add `mix iex` that redirects users to the proper `iex -S mix` command - * [Mix] Support `build_per_environment: true` in project configuration that manages a separete build per environment, useful when you have per-environment behaviour/compilation - -* Backwards incompatible changes - * [Mix] Mix now compiles files to `_build`. Projects should update just fine, however documentation and books may want to update to the latest information - -## v0.11.1 (2013-11-07) - -* Enhancements - * [Mix] Improve dependency convergence by explicitly checking each requirement instead of expecting all requirements to be equal - * [Mix] Support optional dependencies with `optional: true`. Optional dependencies are downloaded for the current project but they are automatically skipped when such project is used as a dependency - -* Bug fixes - * [Kernel] Set compilation status per ParallelCompiler and not globally - * [Mix] Ensure Mix does not load previous dependencies versions before `deps.get`/`deps.update` - * [Mix] Ensure umbrella apps are sorted before running recursive commands - * [Mix] Ensure umbrella apps run in the same environment as the parent project - * [Mix] Ensure dependency tree is topsorted before compiling - * [Mix] Raise error when duplicated projects are pushed into the stack - * [URI] Allow lowercase escapes in URI - -* Backwards incompatible changes - * [Mix] Setting `:load_paths` in your project configuration is deprecated - -## v0.11.0 (2013-11-02) - -* Enhancements - * [Code] Eval now returns variables from other contexts - * [Dict] Document and enforce all dicts use the match operator (`===`) when checking for keys - * [Enum] Add `Enum.slice/2` with a range - * [Enum] Document and enforce `Enum.member?/2` to use the match operator (`===`) - * [IEx] Split `IEx.Evaluator` from `IEx.Server` to allow custom evaluators - * [IEx] Add support for `IEx.pry` which halts a given process for inspection - * [IO] Add specs and allow some IO APIs to receive any data that implements `String.Chars` - * [Kernel] Improve stacktraces on command line interfaces - * [Kernel] Sigils can now handle balanced tokens as in `%s(f(o)o)` - * [Kernel] Emit warnings when an alias is not used - * [Macro] Add `Macro.pipe/3` and `Macro.unpipe/1` for building pipelines - * [Mix] Allow umbrella children to share dependencies between them - * [Mix] Allow mix to be escriptize'd - * [Mix] Speed mix projects compilation by relying on more manifests information - * [Protocol] Protocols now provide `impl_for/1` and `impl_for!/1` functions which receive a structure and returns its respective implementation, otherwise returns nil or an error - * [Set] Document and enforce all sets use the match operator (`===`) when checking for keys - * [String] Update to Unicode 6.3.0 - * [String] Add `String.slice/2` with a range - -* Bug fixes - * [Exception] Ensure `defexception` fields can be set dynamically - * [Kernel] Guarantee aliases hygiene is respected when the current module name is not known upfront - * [Kernel] `Kernel.access/2` no longer flattens lists - * [Mix] Ensure cyclic dependencies are properly handled - * [String] Implement the extended grapheme cluster algorithm for `String` operations - -* Deprecations - * [Kernel] `pid_to_list/1`, `list_to_pid/1`, `binary_to_atom/2`, `binary_to_existing_atom/2` and `atom_to_binary/2` are deprecated in favor of their counterparts in the `:erlang` module - * [Kernel] `insert_elem/3` and `delete_elem/2` are deprecated in favor of `Tuple.insert_at/3` and `Tuple.delete_at/2` - * [Kernel] Use of `in` inside matches (as in `x in [1,2,3] -> x`) is deprecated in favor of the guard syntax (`x when x in [1,2,3]`) - * [Macro] `Macro.expand_all/2` is deprecated - * [Protocol] `@only` and `@except` in protocols are now deprecated - * [Protocol] Protocols no longer fallback to `Any` out of the box (this functionality needs to be explicitly enabled by setting `@fallback_to_any` to true) - * [String] `String.to_integer/1` and `String.to_float/1` are deprecated in favor of `Integer.parse/1` and `Float.parse/1` - -* Backwards incompatible changes - * [CLI] Reading `.elixirrc` has been dropped in favor of setting env vars - * [Kernel] `Kernel.access/2` now expects the second argument to be a compile time list - * [Kernel] `fn -> end` quoted expression is no longer wrapped in a `do` keyword - * [Kernel] Quoted variables from the same module must be explicitly shared. Previously, if a function returned `quote do: a = 1`, another function from the same module could access it as `quote do: a`. This has been fixed and the variables must be explicitly shared with `var!(a, __MODULE__)` - * [Mix] Umbrella apps now treat children apps as dependencies. This means all dependencies will be checked out in the umbrela `deps` directory. On upgrade, child apps need to point to the umbrella project by setting `deps_path: "../../deps_path", lockfile: "../../mix.lock"` in their project config - * [Process] `Process.group_leader/2` args have been reversed so the "subject" comes first - * [Protocol] Protocol no longer dispatches to `Number`, but to `Integer` and `Float` - -## v0.10.3 (2013-10-02) - -* Enhancements - * [Enum] Add `Enum.take_every/2` - * [IEx] IEx now respects signals sent from the Ctrl+G menu - * [Kernel] Allow documentation for types with `@typedoc` - * [Mix] Allow apps to be selected in umbrella projects - * [Record] Generated record functions `new` and `update` also take options with strings as keys - * [Stream] Add `Stream.unfold/1` - -* Bug fixes - * [Dict] Fix a bug when a HashDict was marked as equal when one was actually a subset of the other - * [EEx] Solve issue where `do` blocks inside templates were not properly aligned - * [ExUnit] Improve checks and have better error reports on poorly aligned doctests - * [Kernel] Fix handling of multiple heredocs on the same line - * [Kernel] Provide better error messages for match, guard and quoting errors - * [Kernel] Make `Kernel.raise/2` a macro to avoid messing up stacktraces - * [Kernel] Ensure `&()` works on quoted blocks with only one expression - * [Mix] Address an issue where a dependency was not compiled in the proper order when specified in different projects - * [Mix] Ensure `compile: false` is a valid mechanism for disabling the compilation of dependencies - * [Regex] Fix bug on `Regex.scan/3` when capturing groups and the regex has no groups - * [String] Fix a bug with `String.split/2` when given an empty pattern - * [Typespec] Guarantee typespecs error reports point to the proper line - -* Deprecations - * [Kernel] The previous partial application syntax (without the `&` operator) has now been deprecated - * [Regex] `Regex.captures/3` is deprecated in favor of `Regex.named_captures/3` - * [String] `String.valid_codepoint?/1` is deprecated in favor of pattern matching with `<<_ :: utf8 >>` - -* Backwards incompatible changes - * [IEx] The `r/0` helper has been removed as it caused surprising behaviour when many modules with dependencies were accumulated - * [Mix] `Mix.Version` was renamed to `Version` - * [Mix] `File.IteratorError` was renamed to `IO.StreamError` - * [Mix] `mix new` now defaults to the `--sup` option, use `--bare` to get the previous behaviour - -## v0.10.2 (2013-09-03) - -* Enhancements - * [CLI] Add `--verbose` to elixirc, which now is non-verbose by default - * [Dict] Add `Dict.Behaviour` as a convenience to create your own dictionaries - * [Enum] Add `Enum.split/2`, `Enum.reduce/2`, `Enum.flat_map/2`, `Enum.chunk/2`, `Enum.chunk/4`, `Enum.chunk_by/2`, `Enum.concat/1` and `Enum.concat/2` - * [Enum] Support negative indices in `Enum.at/fetch/fetch!` - * [ExUnit] Show failures on CLIFormatter as soon as they pop up - * [IEx] Allow for strings in `h` helper - * [IEx] Helpers `r` and `c` can handle erlang sources - * [Integer] Add `odd?/1` and `even?/1` - * [IO] Added support to specifying a number of bytes to stream to `IO.stream`, `IO.binstream`, `File.stream!` and `File.binstream!` - * [Kernel] Include file and line on error report for overriding an existing function/macro - * [Kernel] Convert external functions into quoted expressions. This allows record fields to contain functions as long as they point to an `&Mod.fun/arity` - * [Kernel] Allow `foo?` and `bar!` as valid variable names - * [List] Add `List.replace_at/3` - * [Macro] Improve printing of the access protocol on `Macro.to_string/1` - * [Macro] Add `Macro.to_string/2` to support annotations on the converted string - * [Mix] Automatically recompile a project if the Elixir version changes - * [Path] Add `Path.relative_to_cwd/2` - * [Regex] Allow erlang `re` options when compiling Elixir regexes - * [Stream] Add `Stream.concat/1`, `Stream.concat/2` and `Stream.flat_map/2` - * [String] Add regex pattern support to `String.replace/3` - * [String] Add `String.ljust/2`, `String.rjust/2`, `String.ljust/3` and `String.rjust/3` - * [URI] `URI.parse/1` supports IPv6 addresses - -* Bug fixes - * [Behaviour] Do not compile behaviour docs if docs are disabled on compilation - * [ExUnit] Doctests no longer eat too much space and provides detailed reports for poorly indented lines - * [File] Fix a bug where `File.touch(file, datetime)` was not setting the proper datetime when the file did not exist - * [Kernel] Limit `inspect` results to 50 items by default to avoid printing too much data - * [Kernel] Return a readable error on oversized atoms - * [Kernel] Allow functions ending with `?` or `!` to be captured - * [Kernel] Fix default shutdown of child supervisors to `:infinity` - * [Kernel] Fix regression when calling a function/macro ending with bang, followed by `do/end` blocks - * [List] Fix bug on `List.insert_at/3` that added the item at the wrong position for negative indexes - * [Macro] `Macro.escape/2` can now escape improper lists - * [Mix] Fix `Mix.Version` matching on pre-release info - * [Mix] Ensure `watch_exts` trigger full recompilation on change with `mix compile` - * [Mix] Fix regression on `mix clean --all` - * [String] `String.strip/2` now supports removing unicode characters - * [String] `String.slice/3` still returns the proper result when there is no length to be extracted - * [System] `System.get_env/0` now returns a list of tuples as previously advertised - -* Deprecations - * [Dict] `Dict.update/3` is deprecated in favor of `Dict.update!/3` - * [Enum] `Enum.min/2` and `Enum.max/2` are deprecated in favor of `Enum.min_by/2` and `Enum.max_by/2` - * [Enum] `Enum.join/2` and `Enum.map_join/3` with a char list are deprecated - * [IO] `IO.stream(device)` and `IO.binstream(device)` are deprecated in favor of `IO.stream(device, :line)` and `IO.binstream(device, :line)` - * [Kernel] `list_to_binary/1`, `binary_to_list/1` and `binary_to_list/3` are deprecated in favor of `String.from_char_list!/1` and `String.to_char_list!/1` for characters and `:binary.list_to_bin/1`, `:binary.bin_to_list/1` and `:binary.bin_to_list/3` for bytes - * [Kernel] `to_binary/1` is deprecated in favor of `to_string/1` - * [Kernel] Deprecate `def/4` and friends in favor of `def/2` with unquote and friends - * [Kernel] Deprecate `%b` and `%B` in favor of `%s` and `%S` - * [List] `List.concat/2` is deprecated in favor of `Enum.concat/2` - * [Macro] `Macro.unescape_binary/1` and `Macro.unescape_binary/2` are deprecated in favor of `Macro.unescape_string/1` and `Macro.unescape_string/2` - * [Mix] `:umbrella` option for umbrella paths has been deprecated in favor of `:in_umbrella` - -* Backwards incompatible changes - * [IO] IO functions now only accept iolists as arguments - * [Kernel] `Binary.Chars` was renamed to `String.Chars` - * [Kernel] The previous ambiguous import syntax `import :functions, Foo` was removed in favor of `import Foo, only: :functions` - * [OptionParser] `parse` and `parse_head` now returns a tuple with three elements instead of two - -## v0.10.1 (2013-08-03) - -* Enhancements - * [Behaviour] Add support for `defmacrocallback/1` - * [Enum] Add `Enum.shuffle/1` - * [ExUnit] The `:trace` option now also reports run time for each test - * [ExUnit] Add support for `:color` to enable/disable ANSI coloring - * [IEx] Add the `clear` helper to clear the screen. - * [Kernel] Add the capture operator `&` - * [Kernel] Add support for `GenFSM.Behaviour` - * [Kernel] Functions now points to the module and function they were defined when inspected - * [Kernel] A documentation attached to a function that is never defined now prints warnings - * [List] Add `List.keysort/2` - * [Mix] `:test_helper` project configuration did not affect `mix test` and was therefore removed. A `test/test_helper.exs` file is still necessary albeit it doesn't need to be automatically required in each test file - * [Mix] Add manifests for yecc, leex and Erlang compilers, making it easier to detect dependencies in between compilers and providing a more useful clean behaviour - * [Mix] `mix help` now outputs information about the default mix task - * [Mix] Add `--no-deps-check` option to `mix run`, `mix compile` and friends to not check dependency status - * [Mix] Add support for `MIX_GIT_FORCE_HTTPS` system environment that forces HTTPS for known providers, useful when the regular git port is blocked. This configuration does not affect the `mix.lock` results - * [Mix] Allow coverage tool to be pluggable via the `:test_coverage` configuration - * [Mix] Add `mix cmd` as a convenience to run a command recursively in child apps in an umbrella application - * [Mix] Support `umbrella: true` in dependencies as a convenience for setting up umbrella path deps - * [Mix] `mix run` now behaves closer to the `elixir` command and properly mangles the ARGV - * [String] Add `Regex.scan/3` now supports capturing groups - * [String] Add `String.reverse/1` - -* Bug fixes - * [Behaviour] Ensure callbacks are stored in the definition order - * [CLI] Speed up boot time on Elixir .bat files - * [IEx] Reduce cases where IEx parser can get stuck - * [Kernel] Improve error messages when the use of an operator has no effect - * [Kernel] Fix a bug where warnings were not being generated when imported macros conflicted with local functions or macros - * [Kernel] Document that `on_definition` can only be a function as it is evaluated inside the function context - * [Kernel] Ensure `%w` sigils with no interpolation are fully expanded at compile time - * [Mix] `mix deps.update`, `mix deps.clean` and `mix deps.unlock` no longer change all dependencies unless `--all` is given - * [Mix] Always run ` mix loadpaths` on `mix app.start`, even if `--no-compile` is given - * [OptionParser] Do not add boolean flags to the end result if they were not given - * [OptionParser] Do not parse non-boolean flags as booleans when true or false are given - * [OptionParser] Ensure `:keep` and `:integer`|`:float` can be given together as options - * [OptionParser] Ensure `--no-flag` sets `:flag` to false when `:flag` is a registered boolean switch - -* Deprecations - * [Kernel] `function(Mod.fun/arity)` and `function(fun/arity)` are deprecated in favor of `&Mod.fun/arity` and `&fun/arity` - * [Kernel] `function/3` is deprecated in favor of `Module.function/3` - * [Kernel] `Kernel.ParallelCompiler` now receives a set of callbacks instead of a single one - * [Mix] `:test_coverage` option now expect keywords arguments and the `--cover` flag is now treated as a boolean - -* Backwards incompatible changes - * [Regex] `Regex.scan/3` now always returns a list of lists, normalizing the result, instead of list with mixed lists and binaries - * [System] `System.halt/2` was removed since the current Erlang implementation of such function is bugged - -## v0.10.0 (2013-07-15) - -* Enhancements - * [ExUnit] Support `trace: true` option which gives detailed reporting on test runs - * [HashDict] Optimize `HashDict` to store pairs in a cons cell reducing storage per key by half - * [Kernel] Add pretty printing support for inspect - * [Kernel] Add document algebra library used as the foundation for pretty printing - * [Kernel] Add `defrecordp/3` that enables specifying the first element of the tuple - * [Kernel] Add the `Set` API and a hash based implementation via `HashSet` - * [Kernel] Add `Stream` as composable, lazy-enumerables - * [Mix] `mix archive` now includes the version of the generated archive - * [Mix] Mix now requires explicit dependency overriding to be given with `override: true` - * [Mix] Projects can now define an `:elixir` key to outline supported Elixir versions - * [Typespec] Improve error messages to contain file, line and the typespec itself - -* Bug fixes - * [CLI] Elixir can now run on Unix directories with `:` in its path - * [Kernel] `match?/2` does not leak variables to outer scope - * [Kernel] Keep `head|tail` format when splicing at the tail - * [Kernel] Ensure variables defined in the module body are not passed to callbacks - * [Mix] On dependencies conflict, show from where each source is coming from - * [Mix] Empty projects no longer leave empty ebin files on `mix compile` - * [Module] Calling `Module.register_attribute/3` no longer automatically changes it to persisted or accumulated - -* Deprecations - * [Enum] Receiving the index of iteration in `Enum.map/2` and `Enum.each/2` is deprecated in favor of `Stream.with_index/1` - * [File] `File.iterator/1` and `File.biniterator/1` are deprecated in favor of `IO.stream/1` and `IO.binstream/1` - * [File] `File.iterator!/2` and `File.biniterator!/2` are deprecated in favor of `File.stream!/2` and `File.binstream!/2` - * [Kernel] Deprecate recently added `quote binding: ...` in favor of the clearer `quote bind_quoted: ...` - * [Kernel] Deprecate `Kernel.float/1` in favor of a explicit conversion - * [Mix] Deprecate `mix run EXPR` in favor of `mix run -e EXPR` - * [Record] `Record.__index__/2` deprecated in favor of `Record.__record__(:index, key)` - -* Backwards incompatible changes - * [Kernel] The `Binary.Inspect` protocol has been renamed to `Inspect` - * [Kernel] Tighten up the grammar rules regarding parentheses omission, previously the examples below would compile but now they raise an error message: - - do_something 1, is_list [], 3 - [1, is_atom :foo, 3] - - * [Module] Calling `Module.register_attribute/3` no longer automatically changes it to persisted or accumulated - * [Record] First element of a record via `defrecordp` is now the `defrecordp` name and no longer the current atom - * [URI] Remove custom URI parsers in favor of `URI.default_port/2` - -## v0.9.3 (2013-06-23) - -* Enhancements - * [File] Add `File.chgrp`, `File.chmod` and `File.chown` - * [Kernel] Add `--warnings-as-errors` to Elixir's compiler options - * [Kernel] Print warnings to stderr - * [Kernel] Warn on undefined module attributes - * [Kernel] Emit warning for `x in []` in guards - * [Kernel] Add `binding/0` and `binding/1` for retrieving bindings - * [Kernel] `quote` now allows a binding as an option - * [Macro] Add `Macro.expand_once/2` and `Macro.expand_all/2` - * [Mix] Implement `Mix.Version` for basic versioning semantics - * [Mix] Support creation and installation of archives (.ez files) - * [Mix] `github: ...` shortcut now uses the faster `git` schema instead of `https` - * [Record] Allow types to be given to `defrecordp` - -* Bug fixes - * [Kernel] The elixir executable on Windows now supports the same options as the UNIX one - * [Kernel] Improve error messages on default clauses clash - * [Kernel] `__MODULE__.Foo` now returns `Foo` when outside of a Module - * [Kernel] Improve error messages when default clauses from different definitions collide - * [Kernel] `^x` variables should always refer to the value before the expression - * [Kernel] Allow `(x, y) when z` in function clauses and try expressions - * [Mix] Mix now properly evaluates rebar scripts - -* Deprecations - * [Code] `Code.string_to_ast/1` has been deprecated in favor of `Code.string_to_quoted/1` - * [Macro] `Macro.to_binary/1` has been deprecated in favor of `Macro.to_string/1` - * [Typespec] Deprecate `(fun(...) -> ...)` in favor of `(... -> ...)` - -* Backwards incompatible changes - * [Bitwise] Precedence of operators used by the Bitwise module were changed, check `elixir_parser.yrl` for more information - * [File] `rm_rf` and `cp_r` now returns a tuple with three elements on failures - * [Kernel] The quoted representation for `->` clauses changed from a tuple with two elements to a tuple with three elements to support metadata - * [Kernel] Sigils now dispatch to `sigil_$` instead of `__$__` where `$` is the sigil character - * [Macro] `Macro.expand/2` now expands until final form. Although this is backwards incompatible, it is very likely you do not need to change your code, since expansion until its final form is recommended, particularly if you are expecting an atom out of it - * [Mix] No longer support beam files on `mix local` - -## v0.9.2 (2013-06-13) - -* Enhancements - * [ExUnit] `capture_io` now captures prompt by default - * [Mix] Automatically import git dependencies from Rebar - * [Mix] Support for dependencies directly from the umbrella application - * [Regex] Add `Regex.escape` - * [String] Add `String.contains?` - * [URI] Implement `Binary.Chars` (aka `to_binary`) for `URI.Info` - -* Bug fixes - * [HashDict] Ensure HashDict uses exact match throughout its implementation - * [IEx] Do not interpret ANSI codes in IEx results - * [IEx] Ensure `--cookie` is set before accessing remote shell - * [Kernel] Do not ignore nil when dispatching protocols to avoid infinite loops - * [Mix] Fix usage of shell expressions in `Mix.Shell.cmd` - * [Mix] Start the application by default on escripts - -* Deprecations - * [Regex] `Regex.index/2` is deprecated in favor `Regex.run/3` - * [Kernel] `super` no longer supports implicit arguments - -* Backwards incompatible changes - * [Kernel] The `=~` operator now returns true or false instead of an index - -## v0.9.1 (2013-05-30) - -* Enhancements - * [IEx] Limit the number of entries kept in history and allow it to be configured - * [Kernel] Add `String.start_with?` and `String.end_with?` - * [Typespec] Allow keywords, e.g. `[foo: integer, bar: boolean | module]`, in typespecs - -* Bug fixes - * [Dict] `Enum.to_list` and `Dict.to_list` now return the same results for dicts - * [IEx] Enable shell customization via the `IEx.Options` module - * [Kernel] Fix a bug where `unquote_splicing` did not work on the left side of a stab op - * [Kernel] Unused functions with cyclic dependencies are now also warned as unused - * [Mix] Fix a bug where `mix deps.get` was not retrieving nested dependencies - * [Record] Fix a bug where nested records cannot be defined - * [Record] Fix a bug where a record named Record cannot be defined - -## v0.9.0 (2013-05-23) - -* Enhancements - * [ExUnit] `ExUnit.CaptureIO` now accepts an input to be used during capture - * [IEx] Add support for .iex files that are loaded during shell's boot process - * [IEx] Add `import_file/1` helper - -* Backwards incompatible changes - * [Enum] `Enum.Iterator` was replaced by the more composable and functional `Enumerable` protocol which supports reductions - * [File] `File.iterator/1` and `File.biniterator/1` have been removed in favor of the safe `File.iterator!/1` and `File.biniterator!/1` ones - * [Kernel] Erlang R15 is no longer supported - * [Kernel] Elixir modules are now represented as `Elixir.ModuleName` (using `.` instead of `-` as separator) - -## v0.8.3 (2013-05-22) - -* Enhancements - * [CLI] Flags `-p` and `-pr` fails if pattern match no files - * [CLI] Support `--hidden` and `--cookie` flags for distributed Erlang - * [Enum] Add `Enum.to_list/1`, `Enum.member?/2`, `Enum.uniq/2`, `Enum.max/1`, `Enum.max/2`, `Enum.min/1` and `Enum.min/2` - * [ExUnit] Add `ExUnit.CaptureIO` for IO capturing during tests - * [ExUnit] Consider load time on ExUnit time reports - * [IEx] Support `ls` with colored output - * [IEx] Add `#iex:break` to break incomplete expressions - * [Kernel] Add `Enum.at`, `Enum.fetch` and `Enum.fetch!` - * [Kernel] Add `String.to_integer` and `String.to_float` - * [Kernel] Add `Dict.take`, `Dict.drop`, `Dict.split`, `Dict.pop` and `Dict.fetch!` - * [Kernel] Many optimizations for code compilation - * [Kernel] `in` can be used with right side expression outside guards - * [Kernel] Add `Node.get_cookie/0` and `Node.set_cookie/2` - * [Kernel] Add `__DIR__` - * [Kernel] Expand macros and attributes on quote, import, alias and require - * [Kernel] Improve warnings related to default arguments - * [Keyword] Add `Keyword.delete_first/2` - * [Mix] Add `local.rebar` to download a local copy of rebar, and change `deps.compile` to use it if needed - * [Mix] Support umbrella applications - * [Mix] Load beam files available at `MIX_PATH` on CLI usage - * [String] Add `String.valid?` and `String.valid_character?` - -* Bug fixes - * [ExUnit] Handle exit messages from in ExUnit - * [ExUnit] Failures on ExUnit's setup_all now invalidates all tests - * [Kernel] Ensure we don't splice keyword args unecessarily - * [Kernel] Private functions used by private macros no longer emit an unused warning - * [Kernel] Ensure Elixir won't trip on empty receive blocks - * [Kernel] `String.slice` now returns an empty string when out of range by 1 - * [Mix] Generate manifest files after compilation to avoid depending on directory timestamps and to remove unused .beam files - * [Path] `Path.expand/2` now correctly expands `~` in the second argument - * [Regex] Fix badmatch with `Regex.captures(%r/(.)/g, "cat")` - * [URI] Downcase host and scheme and URIs - -* Deprecations - * [Code] `Code.eval` is deprecated in favor of `Code.eval_string` - * [Exception] `Exception.format_entry` is deprecated in favor of `Exception.format_stacktrace_entry` - * [ExUnit] `assert left inlist right` is deprecated in favor of `assert left in right` - * [IO] `IO.getb` is deprecated in favor of `IO.getn` - * [List] `List.member?/2` is deprecated in favor of `Enum.member?/2` - * [Kernel] `var_context` in quote was deprecated in favor of `context` - * [Kernel] `Enum.at!` and `Dict.get!` is deprecated in favor of `Enum.fetch!` and `Dict.fetch!` - -* Backwards incompatible changes - * [Dict] `List.Dict` was moved to `ListDict` - * [IO] `IO.gets`, `IO.getn` and friends now return binaries when reading from stdio - * [Kernel] Precedence of `|>` has changed to lower to support constructs like `1..5 |> Enum.to_list` - * [Mix] `mix escriptize` now receives arguments as binaries - -## v0.8.2 (2013-04-20) - -* Enhancements - * [ExUnit] Use ANSI escape codes in CLI output - * [ExUnit] Include suite run time on CLI results - * [ExUnit] Add support to doctests, allowing test cases to be generated from code samples - * [File] Add `File.ls` and `File.ls!` - * [IEx] Support `pwd` and `cd` helpers - * [Kernel] Better error reporting for invalid bitstring generators - * [Kernel] Improve meta-programming by allowing `unquote` on `def/2`, `defp/2`, `defmacro/2` and `defmacrop/2` - * [Kernel] Add support to R16B new functions: `insert_elem/3` and `delete_elem/2` - * [Kernel] Import conflicts are now lazily handled. If two modules import the same functions, it will fail only if the function is invoked - * [Mix] Support `--cover` on mix test and `test_coverage` on Mixfiles - * [Record] Each record now provides `Record.options` with the options supported by its `new` and `update` functions - -* Bug fixes - * [Binary] inspect no longer escapes standalone hash `#` - * [IEx] The `h` helper can now retrieve docs for special forms - * [Kernel] Record optimizations were not being triggered in functions inside the record module - * [Kernel] Aliases defined inside macros should be carried over - * [Kernel] Fix a bug where nested records could not use the Record[] syntax - * [Path] Fix a bug on `Path.expand` when expanding paths starting with `~` - -* Deprecations - * [Kernel] `setelem/3` is deprecated in favor of `set_elem/3` - * [Kernel] `function(:is_atom, 1)` is deprecated in favor of `function(is_atom/1)` - -* Backwards incompatible changes - * [Kernel] `unquote` now only applies to the closest quote. If your code contains a quote that contains another quote that calls unquote, it will no longer work. Use `Macro.escape` instead and pass your quoted contents up in steps, for example: - - quote do - quote do: unquote(x) - end - - should become: - - quote do - unquote(Macro.escape(x)) - end - -## v0.8.1 (2013-02-17) - -* Enhancements - * [ExUnit] Tests can now receive metadata set on setup/teardown callbacks - * [ExUnit] Add support to ExUnit.CaseTemplate to share callbacks in between test cases - * [IO] Add `IO.ANSI` to make it easy to write ANSI escape codes - * [Kernel] Better support for Unicode lists - * [Kernel] Reduce variables footprint in `case`/`receive` clauses - * [Kernel] Disable native compilation when on_load attributes is present to work around an Erlang bug - * [Macro] `Macro.expand` also considers macros from the current `__ENV__` module - * [Mix] Improve support for compilation of `.erl` files - * [Mix] Add support for compilation of `.yrl` and `.xrl` files - * [OptionParser] Switches are now overridden by default but can be kept in order if chosen - * [Typespec] Better error reporting for invalid typespecs - -* Bug fixes - * [Mix] Allow Mix projects to be generated with just one letter - -* Backwards incompatible changes - * [Kernel] `before_compile` and `after_compile` callbacks now receive the environment as first argument instead of the module - -* Deprecations - * [ExUnit] Explicitly defined test/setup/teardown functions are deprecated - * [Kernel] Tidy up and clean `quote` API - * [Kernel] Old `:local.(args)` syntax is deprecated - * [Process] `Process.self` is deprecated in favor `Kernel.self` - -## v0.8.0 (2013-01-28) - -* Enhancements - * [Binary] Support `<< "string" :: utf8 >>` as in Erlang - * [Binary] Support `\a` escape character in binaries - * [Binary] Support syntax shortcut for specifying size in bit syntax - * [CLI] Support `--app` option to start an application and its dependencies - * [Dict] Support `put_new` in `Dict` and `Keyword` - * [Dict] Add `ListDict` and a faster `HashDict` implementation - * [ExUnit] ExUnit now supports multiple runs in the same process - * [ExUnit] Failures in ExUnit now shows a tailored stacktrace - * [ExUnit] Introduce `ExUnit.ExpectationError` to provide better error messages - * [Kernel] Introduce `Application.Behaviour` to define application module callbacks - * [Kernel] Introduce `Supervisor.Behaviour` to define supervisors callbacks - * [Kernel] More optimizations were added to Record handling - * [Kernel] `?\x` and `?\` are now supported ways to retrieve a codepoint - * [Kernel] Octal numbers can now be defined as `0777` - * [Kernel] Improve macros hygiene regarding variables, aliases and imports - * [Mix] Mix now starts the current application before run, iex, test and friends - * [Mix] Mix now provides basic support for compiling `.erl` files - * [Mix] `mix escriptize` only generates escript if necessary and accept `--force` and `--no-compile` as options - * [Path] Introduce `Path` module to hold filesystem paths related functions - * [String] Add `String.capitalize` and `String.slice` - * [System] Add `System.tmp_dir`, `System.cwd` and `System.user_home` - -* Bug fixes - * [Kernel] `import` with `only` accepts functions starting with underscore - * [String] `String.first` and `String.last` return nil for empty binaries - * [String] `String.rstrip` and `String.lstrip` now verify if argument is a binary - * [Typespec] Support `...` inside typespec's lists - -* Backwards incompatible changes - * [Kernel] The AST now allows metadata to be attached to each node. This means the second item in the AST is no longer an integer (representing the line), but a keywords list. Code that relies on the line information from AST or that manually generate AST nodes need to be properly updated - -* Deprecations - * [Dict] Deprecate `Binary.Dict` and `OrdDict` in favor of `HashDict` and `ListDict` - * [File] Deprecate path related functions in favor of the module `Path` - * [Kernel] The `/>` operator has been deprecated in favor of `|>` - * [Mix] `Mix.Project.sources` is deprecated in favor of `Mix.Project.config_files` - * [Mix] `mix iex` is no longer functional, please use `iex -S mix` - * [OptionParser] `:flags` option was deprecated in favor of `:switches` to support many types - -## v0.7.2 (2012-12-04) - -* Enhancements - * [CLI] `--debug-info` is now true by default - * [ExUnit] Make ExUnit exit happen in two steps allowing developers to add custom `at_exit` hooks - * [IEx] Many improvements to helpers functions `h/1`, `s/1` and others - * [Kernel] Functions defined with `fn` can now handle many clauses - * [Kernel] Raise an error if clauses with different arities are defined in the same function - * [Kernel] `function` macro now accepts arguments in `M.f/a` and `f/a` formats - * [Macro] Improvements to `Macro.to_binary` - * [Mix] Mix now echoes the output as it comes when executing external commands such as git or rebar - * [Mix] Mix now validates `application` callback's values - * [Record] Record accessors are now optimized and can be up to 6x faster in some cases - * [String] Support `\xXX` and `\x{HEX}` escape sequences in strings, char lists and regexes - -* Bug fixes - * [Bootstrap] Compiling Elixir source no longer fails if environment variables contain utf-8 entries - * [IEx] IEx will now wait for all command line options to be processed before starting - * [Kernel] Ensure proper stacktraces when showing deprecations - -* Deprecations - * [Enum] `Enum.qsort` is deprecated in favor of `Enum.sort` - * [List] `List.sort` and `List.uniq` have been deprecated in favor of their `Enum` counterparts - * [Record] Default-based generated functions are deprecated - * [Typespec] Enhancements and deprecations to the `@spec/@callback` and the fun type syntax - -## v0.7.1 (2012-11-18) - -* Enhancements - * [IEx] Only show documented functions and also show docs for default generated functions - * [IO] Add `IO.binread`, `IO.binwrite` and `IO.binreadline` to handle raw binary file operations - * [ExUnit] Add support for user configuration at `HOME/.ex_unit.exs` - * [ExUnit] Add support for custom formatters via a well-defined behaviour - * [Kernel] Add support for `defrecordp` - * [Kernel] Improved dialyzer support - * [Kernel] Improved error messages when creating functions with aliases names - * [Mix] Improve SCM behaviour to allow more robust integration - * [Mix] Changing deps information on `mix.exs` forces users to fetch new dependencies - * [Mix] Support (parallel) requires on mix run - * [Mix] Support `-q` when running tests to compile only changed files - * [String] Support `String.downcase` and `String.upcase` according to Unicode 6.2.0 - * [String] Add support for graphemes in `String.length`, `String.at` and others - * [Typespec] Support `@opaque` as attribute - * [Typespec] Define a default type `t` for protocols and records - * [Typespec] Add support for the access protocol in typespecs - -* Bug fixes - * [Kernel] Fix an issue where variables inside clauses remained unassigned - * [Kernel] Ensure `defoverridable` functions can be referred in many clauses - * [Kernel] Allow keywords as function names when following a dot (useful when integrating with erlang libraries) - * [File] File is opened by default on binary mode instead of utf-8 - -* Deprecations - * [Behaviour] `defcallback/1` is deprecated in favor of `defcallback/2` which matches erlang `@callbacks` - * [Enum] `Enum.times` is deprecated in favor of using ranges - * [System] `halt` moved to `System` module - -## v0.7.0 (2012-10-20) - -* Enhancements - * [Behaviour] Add Behaviour with a simple callback DSL to define callbacks - * [Binary] Add a Dict binary that converts its keys to binaries on insertion - * [Binary] Optimize `Binary.Inspect` and improve inspect for floats - * [CLI] Support `--detached` option - * [Code] `Code.string_to_ast` supports `:existing_atoms_only` as an option in order to guarantee no new atoms is generated when parsing the code - * [EEx] Support `<%%` and `<%#` tags - * [ExUnit] Support `after_spawn` callbacks which are invoked after each process is spawned - * [ExUnit] Support context data in `setup_all`, `setup`, `teardown` and `teardown_all` callbacks - * [IEx] Support `after_spawn` callbacks which are invoked after each process is spawned - * [Kernel] Better error messages when invalid options are given to `import`, `alias` or `require` - * [Kernel] Allow partial application on literals, for example: `{&1, &2}` to build tuples or `[&1|&2]` to build cons cells - * [Kernel] Added `integer_to_binary` and `binary_to_integer` - * [Kernel] Added `float_to_binary` and `binary_to_float` - * [Kernel] Many improvements to `unquote` and `unquote_splicing`. For example, `unquote(foo).unquote(bar)(args)` is supported and no longer need to be written via `apply` - * [Keyword] Keyword list is no longer ordered according to Erlang terms but the order in which they are specified - * [List] Add `List.keyreplace` and `List.keystore` - * [Macro] Support `Macro.safe_term` which returns `:ok` if an expression does not execute code and is made only of raw data types - * [Mix] Add support for environments - the current environment can be set via `MIX_ENV` - * [Mix] Add support for handling and fetching dependencies' dependencies - * [Module] Support module creation via `Module.create` - * [Range] Support decreasing ranges - * [Record] Improvements to the Record API, added `Record.defmacros` - * [Regex] Add `:return` option to `Regex.run` and `Regex.scan` - * [String] Add a String module responsible for handling UTf-8 binaries - -* Bug fixes - * [File] `File.cp` and `File.cp_r` now preserves the file's mode - * [IEx] Fix a bug where printing to `:stdio` on `IEx` was causing it to hang - * [Macro] Fix a bug where quoted expressions were not behaving the same as their non-quoted counterparts - * [Mix] `mix deps.get [DEPS]` now only gets the specified dependencies - * [Mix] Mix now exits with status 1 in case of failures - * [Protocol] Avoid false positives on protocol dispatch (a bug caused the dispatch to be triggered to an invalid protocol) - -* Backwards incompatible changes - * [ExUnit] `setup` and `teardown` callbacks now receives the test name as second argument - * [Kernel] Raw function definition with `def/4`, `defp/4`, `defmacro/4`, `defmacrop/4` now evaluates all arguments. The previous behaviour was accidental and did not properly evaluate all arguments - * [Kernel] Change tuple-related (`elem` and `setelem`), Enum functions (`find_index`, `nth!` and `times`) and List functions (List.key*) to zero-index - -* Deprecations - * [Code] `Code.require_file` and `Code.load_file` now expect the full name as argument - * [Enum] `List.reverse/1` and `List.zip/2` were moved to `Enum` - * [GenServer] Rename `GenServer.Behavior` to `GenServer.Behaviour` - * [Kernel] Bitstring syntax now uses `::` instead of `|` - * [Kernel] `Erlang.` syntax is deprecated in favor of simply using atoms - * [Module] `Module.read_attribute` and `Module.add_attribute` deprecated in favor of `Module.get_attribute` and `Module.put_attribute` which mimics Dict API - -## v0.6.0 (2012-08-01) - -* Backwards incompatible changes - * [Kernel] Compile files now follow `Elixir-ModuleName` convention to solve issues with Erlang embedded mode. This removes the `__MAIN__` pseudo-variable as modules are now located inside `Elixir` namespace - * [Kernel] `__using__` callback triggered by `use` now receives just one argument. Caller information can be accessed via macros using `__CALLER__` - * [Kernel] Comprehensions syntax changed to be more compatible with Erlang behaviour - * [Kernel] loop and recur are removed in favor of recursion with named functions - * [Module] Removed data functions in favor of unifying the attributes API - -* Deprecations - * [Access] The semantics of the access protocol were reduced from a broad query API to simple data structure key-based access - * [ExUnit] Some assertions are deprecated in favor of simply using `assert()` - * [File] `File.read_info` is deprecated in favor of `File.stat` - * [IO] `IO.print` is deprecated in favor of `IO.write` - * [Kernel] Deprecate `__LINE__` and `__FUNCTION__` in favor of `__ENV__.line` and `__ENV__.function` - * [Kernel] Deprecate `in_guard` in favor of `__CALLER__.in_guard?` - * [Kernel] `refer` is deprecated in favor of `alias` - * [Module] `Module.add_compile_callback(module, target, callback)` is deprecated in favor of `Module.put_attribute(module, :before_compile, {target, callback})` - * [Module] `Module.function_defined?` is deprecated in favor of `Module.defines?` - * [Module] `Module.defined_functions` is deprecated in favor of `Module.definitions_in` - -* Enhancements - * [Enum] Enhance Enum protocol to support `Enum.count` - * [Enum] Optimize functions when a list is given as collection - * [Enum] Add `find_index`, `nth!` and others - * [ExUnit] Support setup and teardown callbacks - * [IEx] IEx now provides autocomplete if the OS supports tty - * [IEx] IEx now supports remsh - * [IEx] Elixir now defaults to compile with documentation and `d` can be used in IEx to print modules and functions documentation - * [IEx] Functions `c` and `m` are available in IEx to compile and print available module information. Functions `h` and `v` are available to show history and print previous commands values - * [IO/File] Many improvements to `File` and `IO` modules - * [Kernel] Operator `!` is now allowed in guard clauses - * [Kernel] Introduce operator `=~` for regular expression matches - * [Kernel] Compiled docs now include the function signature - * [Kernel] `defmodule` do not start a new variable scope, this improves meta-programming capabilities - * [Kernel] quote special form now supports line and unquote as options - * [Kernel] Document the macro `@` and allow attributes to be read inside functions - * [Kernel] Add support to the `%R` sigil. The same as `%r`, but without interpolation or escaping. Both implementations were also optimized to generate the regex at compilation time - * [Kernel] Add `__ENV__` which returns a `Macro.Env` record with information about the compilation environment - * [Kernel] Add `__CALLER__` inside macros which returns a `Macro.Env` record with information about the calling site - * [Macro] Add `Macro.expand`, useful for debugging what a macro expands to - * [Mix] First Mix public release - * [Module] Add support to `@before_compile` and `@after_compile` callbacks. The first receives the module name while the latter receives the module name and its object code - * [OptionParser] Make OptionParser public, add support to flags and improved switch parsing - * [Range] Add a Range module with support to `in` operator (`x in 1..3`) and iterators - * [Record] Allow `Record[_: value]` to set a default value to all records fields, as in Erlang - * [Record] Records now provide a `to_keywords` function - * [Regex] Back references are now properly supported - * [System] Add `System.find_executable` - -## v0.5.0 (2012-05-24) - -* First official release +# Changelog for Elixir v1.14 + +Elixir v1.14 requires Erlang/OTP 23+ with a small batch of new features +and the usual enhancements and bug fixes to Elixir and its standard library. +We cover the most notable changes next. + +## PartitionSupervisor + +`PartitionSupervisor` is a new module that implements a new supervisor type. The +partition supervisor is designed to help with situations where you have a single +supervised process that becomes a bottleneck. If that process's state can be +easily partitioned, then you can use `PartitionSupervisor` to supervise multiple +isolated copies of that process running concurrently, each assigned its own +partition. + +For example, imagine you have a `ErrorReporter` process that you use to report +errors to a monitoring service. + +```elixir +# Application supervisor: +children = [ + # ..., + ErrorReporter +] + +Supervisor.start_link(children, strategy: :one_for_one) +``` + +As the concurrency of your application goes up, the `ErrorReporter` process +might receive requests from many other processes and eventually become a +bottleneck. In a case like this, it could help to spin up multiple copies of the +`ErrorReporter` process under a `PartitionSupervisor`. + +```elixir +# Application supervisor +children = [ + {PartitionSupervisor, child_spec: ErrorReporter, name: Reporters} +] +``` + +The `PartitionSupervisor` will spin up a number of processes equal to +`System.schedulers_online()` by default (most often one per core). Now, when +routing requests to `ErrorReporter` processes we can use a `:via` tuple and +route the requests through the partition supervisor. + +```elixir +partitioning_key = self() +ErrorReporter.report({:via, PartitionSupervisor, {Reporters, partitioning_key}}, error) +``` + +Using `self()` as the partitioning key here means that the same process will +always report errors to the same `ErrorReporter` process, ensuring a form of +back-pressure. You can use any term as the partitioning key. + +### A Common Example + +A common and practical example of a good use case for `PartitionSupervisor` is +partitioning something like a `DynamicSupervisor`. When starting many processes +under it, a dynamic supervisor can be a bottleneck, especially if said processes +take long to initialize. Instead of starting a single `DynamicSupervisor`, +you can start multiple ones: + +```elixir +children = [ + {PartitionSupervisor, child_spec: DynamicSupervisor, name: MyApp.DynamicSupervisors} +] + +Supervisor.start_link(children, strategy: :one_for_one) +``` + +Now you start processes on the dynamic supervisor for the right partition. +For instance, you can partition by PID, like in the previous example: + +```elixir +DynamicSupervisor.start_child( + {:via, PartitionSupervisor, {MyApp.DynamicSupervisors, self()}}, + my_child_specification +) +``` + +## Improved errors on binaries and evaluation + +Erlang/OTP 25 improved errors on binary construction and evaluation. These improvements +apply to Elixir as well. Before v1.14, errors when constructing binaries would +often be hard-to-debug generic "argument errors". With Erlang/OTP 25 and Elixir v1.14, +more detail is provided for easier debugging. This work is part of [EEP +54](https://www.erlang.org/eeps/eep-0054). + +Before: + +```elixir +int = 1 +bin = "foo" +int <> bin +#=> ** (ArgumentError) argument error +``` + +Now: + +```elixir +int = 1 +bin = "foo" +int <> bin +#=> ** (ArgumentError) construction of binary failed: +#=> segment 1 of type 'binary': +#=> expected a binary but got: 1 +``` + +## Slicing with steps + +Elixir v1.12 introduced **stepped ranges**, which are ranges where you can +specify the "step": + +```elixir +Enum.to_list(1..10//3) +#=> [1, 4, 7, 10] +``` + +Stepped ranges are particularly useful for numerical operations involving +vectors and matrices (see [Nx](https://github.com/elixir-nx/nx), for example). +However, the Elixir standard library was not making use of stepped ranges in its +APIs. Elixir v1.14 starts to take advantage of steps with support for stepped +ranges in a couple of functions. One of them is `Enum.slice/2`: + +```elixir +letters = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"] +Enum.slice(letters, 0..5//2) +#=> ["a", "c", "e"] +``` + +`binary_slice/2` (and `binary_slice/3` for completeness) has been added to the +`Kernel` module, that works with bytes and also support stepped ranges: + +```elixir +binary_slice("Elixir", 1..5//2) +#=> "lx" +``` + +## Expression-based inspection and `Inspect` improvements + +In Elixir, it's conventional to implement the `Inspect` protocol for opaque +structs so that they're inspected with a special notation, resembling this: + +```elixir +MapSet.new([:apple, :banana]) +#MapSet<[:apple, :banana]> +``` + +This is generally done when the struct content or part of it is private and the +`%name{...}` representation would reveal fields that are not part of the public +API. + +The downside of the `#name<...>` convention is that *the inspected output is not +valid Elixir code*. You cannot do things such as copying the inspected output +and pasting it into an IEx session or similar. + +Elixir v1.14 changes the convention for some of the standard-library structs. +The `Inspect` implementation for those structs is now a string with a valid +Elixir expression that recreates the struct itself if evaluated. In the +`MapSet` example above, this is what we have now: + +```elixir +fruits = MapSet.new([:apple, :banana]) +MapSet.put(fruits, :pear) +#=> MapSet.new([:apple, :banana, :pear]) +``` + +The `MapSet.new/1` expression evaluates to exactly the struct that we're +inspecting. This allows us to hide the internals of `MapSet`, while keeping +it as valid Elixir code. This expression-based inspection has been +implemented for `Version.Requirement`, `MapSet`, and `Date.Range`. + +Finally, we have also improved the `Inspect` protocol for structs so the +fields are inspected in the order they are declared in `defstruct`. +The option `:optional` has also been added when deriving the `Inspect` +protocol, giving developers more control over the struct representation. +See the updated documentation for `Inspect` for a general rundown on +the approaches and options available. + +## v1.14.0-dev + +### 1. Enhancements + +#### EEx + + * [EEx] Support multi-line comments to EEx via `<%!-- --%>` + * [EEx] Add `EEx.tokenize/2` + +#### Elixir + + * [Calendar] Support ISO8601 basic format parsing with `DateTime.from_iso8601/2` + * [Code] Add `:normalize_bitstring_modifiers` to `Code.format_string!/2` + * [Code] Emit deprecation and type warnings for invalid options in on `Code.compile_string/2` and `Code.compile_quoted/2` + * [Code] Warn if an outdated lexical tracker is given on eval + * [Code] Add `Code.env_for_eval/1` and `Code.eval_quoted_with_env/3` + * [Code] Improve stacktraces from eval operations on Erlang/OTP 25+ + * [Enum] Allow slicing with steps in `Enum.slice/2` + * [Float] Do not show floats in scientific notation if below `1.0e16` and the fractional value is precisely zero + * [Inspect] Improve error reporting when there is a faulty implementation of the `Inspect` protocol + * [Inspect] Allow `:optional` when deriving the Inspect protocol for hiding fields that match their default value + * [Inspect] Inspect struct fields in the order they are declared in `defstruct` + * [Inspect] Use expression-based inspection for `Date.Range`, `MapSet`, and `Version.Requirement` + * [IO] Support `Macro.Env` and keywords as stacktrace definitions in `IO.warn/2` + * [Kernel] Allow any guard expression as the size of a bitstring in a pattern match + * [Kernel] Allow composite types with pins as the map key in a pattern match + * [Kernel] Print escaped version of control chars when they show up as unexpected tokens + * [Kernel] Warn on confusable non-ASCII identifiers + * [Kernel] Add `..` as a nullary operator that returns `0..-1//1` + * [Kernel] Implement Unicode Technical Standard #39 recommendations. In particular, we warn for confusable scripts and restrict identifiers to single-scripts or highly restrictive mixed-scripts + * [Kernel] Automatically perform NFC conversion of identifiers + * [Kernel] Add `binary_slice/2` and `binary_slice/3` + * [Keyword] Add `Keyword.from_keys/2` and `Keyword.replace_lazy/3` + * [List] Add `List.keysort/3` with support for a `sorter` function + * [Macro] Add `Macro.classify_atom/1` and `Macro.inspect_atom/2` + * [Macro.Env] Add `Macro.Env.prune_compile_info/1` + * [Map] Add `Map.from_keys/2` and `Map.replace_lazy/3` + * [MapSet] Add `MapSet.filter/2` and `MapSet.reject/2` + * [Node] Add `Node.spawn_monitor/2` and `Node.spawn_monitor/4` + * [PartitionSupervisor] Add `PartitionSupervisor` that starts multiple isolated partitions of the same child for scalability + * [Path] Add `Path.safe_relative/1` and `Path.safe_relative_to/2` + * [Registry] Add `Registry.count_select/2` + * [Stream] Add `Stream.duplicate/2` and `Stream.transform/5` + * [String] Support empty lookup lists in `String.replace/3`, `String.split/3`, and `String.splitter/3` + * [String] Allow slicing with steps in `String.slice/2` + * [Task] Add `:zip_input_on_exit` option to `Task.async_stream/3` + * [Task] Store `:mfa` in the `Task` struct for reflection purposes + * [URI] Add `URI.append_query/2` + * [Version] Add `Version.to_string/1` + * [Version] Colorize `Version.Requirement` source in the `Inspect` protocol + +#### ExUnit + + * [ExUnit] Add `ExUnit.Callbacks.start_link_supervised!/2` + * [ExUnit] Add `ExUnit.run/1` to rerun test modules + * [ExUnit] Colorize summary in yellow with message when all tests are excluded + +#### IEx + + * [IEx] Evaluate `--dot-iex` line by line + * [IEx.Helpers] Allow an atom to be given to `pid/1` + +#### Logger + + * [Logger] Add `Logger.put_process_level/2` + +#### Mix + + * [mix compile] Add `--no-optional-deps` to skip optional dependencies to test compilation works without optional dependencies + * [mix deps] `Mix.Dep.Converger` now tells which deps formed a cycle + * [mix do] Support `--app` option to restrict recursive tasks in umbrella projects + * [mix do] Allow using `+` as a task separator instead of comma + * [mix format] Support filename in `mix format -` when reading from stdin + * [mix new] Do not allow projects to be created with application names that conflict with multi-arg Erlang VM switches + * [mix profile] Return the return value of the profiled function + * [mix release] Make BEAM compression opt-in + * [mix test] Improve error message when suite fails due to coverage + * [mix test] Support `:test_elixirc_options` and default to not generating docs nor debug info chunk for tests + +### 2. Bug fixes + +#### Elixir + + * [Calendar] Handle widths with "0" in them in `Calendar.strftime/3` + * [CLI] Improve errors on incorrect `--rpc-eval` usage + * [Code] Do not emit warnings when formatting code + * [Enum] Allow slices to overflow on both starting and ending positions + * [Kernel] Do not allow restricted characters in identifiers according to UTS39 + * [Kernel] Define `__exception__` field as `true` when expanding exceptions in typespecs + * [Kernel] Warn if any of `True`, `False`, and `Nil` aliases are used + * [Kernel] Warn on underived `@derive` attributes + * [Protocol] Warn if a protocol has no definitions + * [String] Allow slices to overflow on both starting and ending positions + +#### ExUnit + + * [ExUnit] Do not raise when diffing unknown bindings in guards + * [ExUnit] Properly print diffs when comparing improper lists with strings at the tail position + * [ExUnit] Add short hash to `tmp_dir` in ExUnit to avoid test name collision + * [ExUnit] Do not store logs in the CLI formatter (this reduces memory usage for suites with `capture_log`) + * [ExUnit] Run `ExUnit.after_suite/1` callback even when no tests run + * [ExUnit] Fix scenario where `setup` with imported function from within `describe` failed to compile + +#### Mix + + * [mix compile.elixir] Fix `--warnings-as-errors` when used with `--all-warnings` + * [mix format] Do not add new lines if the formatted file is empty + * [mix release] Only set `RELEASE_MODE` after `env.{sh,bat}` are executed + +#### IEx + + * [IEx] Disallow short-hand pipe after matches + +### 3. Soft deprecations (no warnings emitted) + +#### EEx + + * [EEx] Using `<%# ... %>` for comments is deprecated. Please use `<% # ... %>` or the new multi-line comments with `<%!-- ... --%>` + +#### Logger + + * [Logger] Deprecate `Logger.enable/1` and `Logger.disable/1` in favor of `Logger.put_process_level/2` + +#### Mix + + * [mix cmd] The `--app` option in `mix cmd CMD` is deprecated in favor of the more efficient `mix do --app app cmd CMD` + +### 4. Hard deprecations + +#### Elixir + + * [Application] Calling `Application.get_env/3` and friends in the module body is now discouraged, use `Application.compile_env/3` instead + * [Bitwise] `use Bitwise` is deprecated, use `import Bitwise` instead + * [Bitwise] `~~~` is deprecated in favor of `bnot` for clarity + * [Kernel.ParallelCompiler] Returning a list or two-element tuple from `:each_cycle` is deprecated, return a `{:compile | :runtime, modules, warnings}` tuple instead + * [Kernel] Deprecate the operator `<|>` to avoid ambiguity with upcoming extended numerical operators + * [String] Deprecate passing a binary compiled pattern to `String.starts_with?/2` + +#### Logger + + * [Logger] Deprecate `$levelpad` on message formatting + +#### Mix + + * [Mix] `Mix.Tasks.Xref.calls/1` is deprecated in favor of compilation tracers + +### 5. Backwards incompatible changes + +#### Mix + + * [mix local.rebar] Remove support for rebar2, which has not been updated in 5 years, and is no longer supported on recent Erlang/OTP versions + +## v1.13 + +The CHANGELOG for v1.13 releases can be found [in the v1.13 branch](https://github.com/elixir-lang/elixir/blob/v1.13/CHANGELOG.md). diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..0e0f08f15ec --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,66 @@ +# Code of Conduct + +Contact: elixir-lang-conduct@googlegroups.com + +## Why have a Code of Conduct? + +As contributors and maintainers of this project, we are committed to providing a friendly, safe and welcoming environment for all, regardless of age, disability, gender, nationality, race, religion, sexuality, or similar personal characteristic. + +The goal of the Code of Conduct is to specify a baseline standard of behavior so that people with different social values and communication styles can talk about Elixir effectively, productively, and respectfully, even in face of disagreements. The Code of Conduct also provides a mechanism for resolving conflicts in the community when they arise. + +## Our Values + +These are the values Elixir developers should aspire to: + + * Be friendly and welcoming + * Be kind + * Remember that people have varying communication styles and that not everyone is using their native language. (Meaning and tone can be lost in translation.) + * Interpret the arguments of others in good faith, do not seek to disagree. + * When we do disagree, try to understand why. + * Be thoughtful + * Productive communication requires effort. Think about how your words will be interpreted. + * Remember that sometimes it is best to refrain entirely from commenting. + * Be respectful + * In particular, respect differences of opinion. It is important that we resolve disagreements and differing views constructively. + * Be constructive + * Avoid derailing: stay on topic; if you want to talk about something else, start a new conversation. + * Avoid unconstructive criticism: don't merely decry the current state of affairs; offer — or at least solicit — suggestions as to how things may be improved. + * Avoid harsh words and stern tone: we are all aligned towards the well-being of the community and the progress of the ecosystem. Harsh words exclude, demotivate, and lead to unnecessary conflict. + * Avoid snarking (pithy, unproductive, sniping comments). + * Avoid microaggressions (brief and commonplace verbal, behavioral and environmental indignities that communicate hostile, derogatory or negative slights and insults towards a project, person or group). + * Be responsible + * What you say and do matters. Take responsibility for your words and actions, including their consequences, whether intended or otherwise. + +The following actions are explicitly forbidden: + + * Insulting, demeaning, hateful, or threatening remarks. + * Discrimination based on age, disability, gender, nationality, race, religion, sexuality, or similar personal characteristic. + * Bullying or systematic harassment. + * Unwelcome sexual advances. + * Incitement to any of these. + +## Where does the Code of Conduct apply? + +If you participate in or contribute to the Elixir ecosystem in any way, you are encouraged to follow the Code of Conduct while doing so. + +Explicit enforcement of the Code of Conduct applies to the official mediums operated by the Elixir project: + +* The [official GitHub projects][1] and code reviews. +* The official elixir-lang mailing lists. +* The **[#elixir][2]** IRC channel on [Libera.Chat][3]. + +Other Elixir activities (such as conferences, meetups, and unofficial forums) are encouraged to adopt this Code of Conduct. Such groups must provide their own contact information. + +Project maintainers may block, remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by emailing: elixir-lang-conduct@googlegroups.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. **All reports will be kept confidential**. + +**The goal of the Code of Conduct is to resolve conflicts in the most harmonious way possible**. We hope that in most cases issues may be resolved through polite discussion and mutual agreement. Bannings and other forceful measures are to be employed only as a last resort. **Do not** post about the issue publicly or try to rally sentiment against a particular individual or group. + +## Acknowledgements + +This document was based on the Code of Conduct from the Go project (dated Sep/2021) and the Contributor Covenant (v1.4). + +[1]: https://github.com/elixir-lang/ +[2]: https://web.libera.chat/#elixir +[3]: https://libera.chat/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 7497c70c66a..00000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,286 +0,0 @@ -# Contributing to Elixir - -Please take a moment to review this document in order to make the contribution -process easy and effective for everyone involved! - -## Using the issue tracker - -Use the issues tracker for: - -* [bug reports](#bugs-reports) -* [submitting pull requests](#pull-requests) - -Please **do not** use the issues tracker for personal support requests nor feature requests. Support requests should be send to: - -* [the elixir-talk mailing list](http://groups.google.com/group/elixir-lang-talk) -* [Stack Overflow](http://stackoverflow.com/questions/ask?tags=elixir) -* [#elixir-lang](irc://chat.freenode.net/elixir-lang) - -Feature requests can be discussed on [the elixir-core mailing list](http://groups.google.com/group/elixir-lang-core). - -We do our best to keep the issues tracker tidy and organized, making it useful -for everyone. For example, we classify open issues per application and perceived -difficulty of the issue, making it easier for developers to -[contribute to Elixir](#contributing). - -## Bug reports - -A bug is a _demonstrable problem_ that is caused by the code in the repository. -Good bug reports are extremely helpful - thank you! - -Guidelines for bug reports: - -1. **Use the GitHub issue search** — check if the issue has already been - reported. - -2. **Check if the issue has been fixed** — try to reproduce it using the - `master` branch in the repository. - -3. **Isolate and report the problem** — ideally create a reduced test - case. - -Please try to be as detailed as possible in your report. Include information about -your Operating System, your Erlang and Elixir versions. Please provide steps to -reproduce the issue as well as the outcome you were expecting! All these details -will help developers to fix any potential bugs. - -Example: - -> Short and descriptive example bug report title -> -> A summary of the issue and the environment in which it occurs. If suitable, -> include the steps required to reproduce the bug. -> -> 1. This is the first step -> 2. This is the second step -> 3. Further steps, etc. -> -> `` - a link to the reduced test case (e.g. a GitHub Gist) -> -> Any other information you want to share that is relevant to the issue being -> reported. This might include the lines of code that you have identified as -> causing the bug, and potential solutions (and your opinions on their -> merits). - -## Feature requests - -Feature requests are welcome and should be discussed on [the elixir-core mailing list](http://groups.google.com/group/elixir-lang-core). But take a moment to find -out whether your idea fits with the scope and aims of the project. It's up to *you* -to make a strong case to convince the community of the merits of this feature. -Please provide as much detail and context as possible. - -## Contributing - -We incentivize everyone to contribute to Elixir and help us tackle -existing issues! To do so, there are a few things you need to know -about the code. First, Elixir code is divided in applications inside -the `lib` folder: - -* `elixir` - Contains Elixir's kernel and stdlib - -* `eex` - Template engine that allows you to embed Elixir - -* `ex_unit` - Simple test framework that ships with Elixir - -* `iex` — IEx, Elixir's interactive shell - -* `mix` — Elixir's build tool - -You can run all tests in the root directory with `make test` and you can -also run tests for a specific framework `make test_#{NAME}`, for example, -`make test_ex_unit`. - -In case you are changing a single file, you can compile and run tests only -for that particular file for fast development cycles. For example, if you -are changing the String module, you can compile it and run its tests as: - - $ bin/elixirc lib/elixir/lib/string.ex -o lib/elixir/ebin - $ bin/elixir lib/elixir/test/elixir/string_test.exs - -After your changes are done, please remember to run the full suite with -`make test`. - -From time to time, your tests may fail in an existing Elixir checkout and -may require a clean start by running `make clean compile`. You can always -check [the official build status on Travis-CI](https://travis-ci.org/elixir-lang/elixir). - -With tests running and passing, you are ready to contribute to Elixir and -send your pull requests. - -### Building on Windows - -There are a few extra steps you'll need to take for contributing from Windows. -Basically, once you have Erlang 17, Git, and MSYS from MinGW on your system, -you're all set. Specifically, here's what you need to do to get up and running: - -1. Install [Git](http://www.git-scm.com/download/win), -[Erlang](http://www.erlang.org/download.html), and the -[MinGW Installation Manager](http://sourceforge.net/projects/mingw/files/latest/download?source=files). -2. Use the MinGW Installation Manager to install the msys-bash, msys-make, and -msys-grep packages. -3. Add `;C:\Program Files (x86)\Git\bin;C:\Program Files\erl6.0\bin;C:\Program Files\erl6.0\erts-6.0\bin;C:\MinGW\msys\1.0\bin` -to your "Path" environment variable . (This is under Control Panel > System -and Security > System > Advanced system settings > Environment Variables > -System variables) - -You can now work in the Command Prompt similar to how you would on other OS'es, -except for some things (like creating symlinks) you'll need to run the Command -Prompt as an Administrator. - -## Contributing Documentation - -Code documentation (`@doc`, `@moduledoc`, `@typedoc`) has a special convention: -the first paragraph is considered to be a short summary. - -For functions, macros and callbacks say what it will do. For example write -something like: - -```elixir -@doc """ -Returns only those elements for which `fun` is true. - -... -""" -def filter(collection, fun) ... -``` - -For modules, protocols and types say what it is. For example write -something like: - -```elixir -defmodule File.Stat do - @moduledoc """ - Information about a file. - - ... - """ - - defstruct [...] -end -``` - -Keep in mind that the first paragraph might show up in a summary somewhere, long -texts in the first paragraph create very ugly summaries. As a rule of thumb -anything longer than 80 characters is too long. - -Try to keep unneccesary details out of the first paragraph, it's only there to -give a user a quick idea of what the documented "thing" does/is. The rest of the -documentation string can contain the details, for example when a value and when -`nil` is returned. - -If possible include examples, preferably in a form that works with doctests. For -example: - -```elixir -@doc """ -Return only those elements for which `fun` is true. - -## Examples - - iex> Enum.filter([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) - [2] - -""" -def filter(collection, fun) ... -``` - -This makes it easy to test the examples so that they don't go stale and examples -are often a great help in explaining what a function does. - -## Pull requests - -Good pull requests - patches, improvements, new features - are a fantastic -help. They should remain focused in scope and avoid containing unrelated -commits. - -**IMPORTANT**: By submitting a patch, you agree that your work will be -licensed under the license used by the project. - -If you have any large pull request in mind (e.g. implementing features, -refactoring code, etc), **please ask first** otherwise you risk spending -a lot of time working on something that the project's developers might -not want to merge into the project. - -Please adhere to the coding conventions in the project (indentation, -accurate comments, etc.) and don't forget to add your own tests and -documentation. When working with git, we recommend the following process -in order to craft an excellent pull request: - -1. [Fork](http://help.github.com/fork-a-repo/) the project, clone your fork, - and configure the remotes: - - ```bash - # Clone your fork of the repo into the current directory - git clone https://github.com//elixir - # Navigate to the newly cloned directory - cd elixir - # Assign the original repo to a remote called "upstream" - git remote add upstream https://github.com/elixir-lang/elixir - ``` - -2. If you cloned a while ago, get the latest changes from upstream: - - ```bash - git checkout master - git pull upstream master - ``` - -3. Create a new topic branch (off of `master`) to contain your feature, change, - or fix. - - **IMPORTANT**: Making changes in `master` is discouraged. You should always - keep your local `master` in sync with upstream `master` and make your - changes in topic branches. - - ```bash - git checkout -b - ``` - -4. Commit your changes in logical chunks. Keep your commit messages organized, - with a short description in the first line and more detailed information on - the following lines. Feel free to use Git's - [interactive rebase](https://help.github.com/articles/interactive-rebase) - feature to tidy up your commits before making them public. - -5. Make sure all the tests are still passing. - - ```bash - make test - ``` - - This command will compile the code in your branch and use that - version of Elixir to run the tests. This is needed to ensure your changes can - pass all the tests. - -6. Push your topic branch up to your fork: - - ```bash - git push origin - ``` - -7. [Open a Pull Request](https://help.github.com/articles/using-pull-requests/) - with a clear title and description. - -8. If you haven't updated your pull request for a while, you should consider - rebasing on master and resolving any conflicts. - - **IMPORTANT**: _Never ever_ merge upstream `master` into your branches. You - should always `git rebase` on `master` to bring your changes up to date when - necessary. - - ```bash - git checkout master - git pull upstream master - git checkout - git rebase master - ``` - -We have saved some excellent pull requests we have received in the past in case -you are looking for some examples: - -* https://github.com/elixir-lang/elixir/pull/992 -* https://github.com/elixir-lang/elixir/pull/1041 -* https://github.com/elixir-lang/elixir/pull/1058 -* https://github.com/elixir-lang/elixir/pull/1059 - -Thank you for your contributions! diff --git a/LEGAL b/LEGAL deleted file mode 100644 index 8948b8ad2f0..00000000000 --- a/LEGAL +++ /dev/null @@ -1,8 +0,0 @@ -LEGAL NOTICE INFORMATION ------------------------- - -All the files in this distribution are covered under either Elixir's -license (see the file LICENSE) except the files mentioned below that -contains sections that are under Erlang's License (EPL): - -lib/elixir/src/elixir_parser.erl (generated by build scripts) diff --git a/LICENSE b/LICENSE index d3a92d217c9..d9a10c0d8e8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,13 +1,176 @@ -Copyright 2012-2013 Plataformatec. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - http://www.apache.org/licenses/LICENSE-2.0 + 1. Definitions. - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/Makefile b/Makefile index b660c202c21..31868ecf7c5 100644 --- a/Makefile +++ b/Makefile @@ -1,95 +1,116 @@ -REBAR := rebar -ELIXIRC := bin/elixirc --verbose --ignore-module-conflict +PREFIX ?= /usr/local +TEST_FILES ?= "*_test.exs" +SHARE_PREFIX ?= $(PREFIX)/share +MAN_PREFIX ?= $(SHARE_PREFIX)/man +#CANONICAL := MAJOR.MINOR/ +CANONICAL ?= main/ +DOCS_FORMAT ?= html +ELIXIRC := bin/elixirc --ignore-module-conflict $(ELIXIRC_OPTS) ERLC := erlc -I lib/elixir/include +ERL_MAKE := if [ -n "$(ERLC_OPTS)" ]; then ERL_COMPILER_OPTIONS=$(ERLC_OPTS) erl -make; else erl -make; fi ERL := erl -I lib/elixir/include -noshell -pa lib/elixir/ebin +GENERATE_APP := $(CURDIR)/lib/elixir/generate_app.escript VERSION := $(strip $(shell cat VERSION)) Q := @ -PREFIX := /usr/local LIBDIR := lib +BINDIR := bin INSTALL = install INSTALL_DIR = $(INSTALL) -m755 -d INSTALL_DATA = $(INSTALL) -m644 INSTALL_PROGRAM = $(INSTALL) -m755 +GIT_REVISION = $(strip $(shell git rev-parse HEAD 2> /dev/null )) +GIT_TAG = $(strip $(shell head="$(call GIT_REVISION)"; git tag --points-at $$head 2> /dev/null | tail -1) ) +SOURCE_DATE_EPOCH_PATH = lib/elixir/tmp/ebin_reproducible +SOURCE_DATE_EPOCH_FILE = $(SOURCE_DATE_EPOCH_PATH)/SOURCE_DATE_EPOCH -.PHONY: install compile erlang elixir dialyze test clean docs release_docs release_zip check_erlang_release +.PHONY: install compile erlang elixir unicode app build_plt clean_plt dialyze test check_reproducible clean clean_residual_files format install_man clean_man docs Docs.zip Precompiled.zip zips .NOTPARALLEL: compile #==> Functions -# This check should work for older versions like R16B -# as well as new verions like 17.1 and 18 define CHECK_ERLANG_RELEASE - $(Q) erl -noshell -eval 'io:fwrite("~s", [erlang:system_info(otp_release)])' -s erlang halt | grep -q '^1[789]'; \ - if [ $$? != 0 ]; then \ - echo "At least Erlang 17.0 is required to build Elixir"; \ - exit 1; \ - fi; + erl -noshell -eval '{V,_} = string:to_integer(erlang:system_info(otp_release)), io:fwrite("~s", [is_integer(V) and (V >= 23)])' -s erlang halt | grep -q '^true'; \ + if [ $$? != 0 ]; then \ + echo "At least Erlang/OTP 23.0 is required to build Elixir"; \ + exit 1; \ + fi endef define APP_TEMPLATE $(1): lib/$(1)/ebin/Elixir.$(2).beam lib/$(1)/ebin/$(1).app lib/$(1)/ebin/$(1).app: lib/$(1)/mix.exs - $(Q) mkdir -p lib/$(1)/_build/shared/lib/$(1) - $(Q) cp -R lib/$(1)/ebin lib/$(1)/_build/shared/lib/$(1)/ - $(Q) cd lib/$(1) && ../../bin/elixir -e "Mix.start(:permanent, [])" -r mix.exs -e "Mix.Task.run('compile.app')" - $(Q) cp lib/$(1)/_build/shared/lib/$(1)/ebin/$(1).app lib/$(1)/ebin/$(1).app - $(Q) rm -rf lib/$(1)/_build + $(Q) cd lib/$(1) && ../../bin/elixir -e 'Mix.start(:permanent, [])' -r mix.exs -e 'Mix.Task.run("compile.app", ~w[--compile-path ebin])' lib/$(1)/ebin/Elixir.$(2).beam: $(wildcard lib/$(1)/lib/*.ex) $(wildcard lib/$(1)/lib/*/*.ex) $(wildcard lib/$(1)/lib/*/*/*.ex) @ echo "==> $(1) (compile)" @ rm -rf lib/$(1)/ebin $(Q) cd lib/$(1) && ../../$$(ELIXIRC) "lib/**/*.ex" -o ebin -test_$(1): $(1) - @ echo "==> $(1) (exunit)" - $(Q) cd lib/$(1) && ../../bin/elixir -r "test/test_helper.exs" -pr "test/**/*_test.exs"; +test_$(1): compile $(1) + @ echo "==> $(1) (ex_unit)" + $(Q) cd lib/$(1) && ../../bin/elixir -r "test/test_helper.exs" -pr "test/**/$(TEST_FILES)"; +endef + +define WRITE_SOURCE_DATE_EPOCH +$(shell mkdir -p $(SOURCE_DATE_EPOCH_PATH) && bin/elixir -e \ + 'IO.puts System.build_info()[:date] \ + |> DateTime.from_iso8601() \ + |> elem(1) \ + |> DateTime.to_unix()' > $(SOURCE_DATE_EPOCH_FILE)) +endef + +define READ_SOURCE_DATE_EPOCH +$(strip $(shell cat $(SOURCE_DATE_EPOCH_FILE))) endef #==> Compilation tasks -KERNEL:=lib/elixir/ebin/Elixir.Kernel.beam -UNICODE:=lib/elixir/ebin/Elixir.String.Unicode.beam +APP := lib/elixir/ebin/elixir.app +PARSER := lib/elixir/src/elixir_parser.erl +KERNEL := lib/elixir/ebin/Elixir.Kernel.beam +UNICODE := lib/elixir/ebin/Elixir.String.Unicode.beam default: compile -compile: lib/elixir/src/elixir.app.src erlang elixir +compile: erlang $(APP) elixir -lib/elixir/src/elixir.app.src: src/elixir.app.src - $(Q) $(call CHECK_ERLANG_RELEASE) - $(Q) rm -rf lib/elixir/src/elixir.app.src - $(Q) echo "%% This file is automatically generated from /src/elixir.app.src" \ - >lib/elixir/src/elixir.app.src - $(Q) cat src/elixir.app.src >>lib/elixir/src/elixir.app.src +erlang: $(PARSER) + $(Q) if [ ! -f $(APP) ]; then $(call CHECK_ERLANG_RELEASE); fi + $(Q) cd lib/elixir && mkdir -p ebin && $(ERL_MAKE) -erlang: - $(Q) cd lib/elixir && ../../$(REBAR) compile +$(PARSER): lib/elixir/src/elixir_parser.yrl + $(Q) erlc -o $@ +'{verbose,true}' +'{report,true}' $< -# Since Mix depends on EEx and EEx depends on -# Mix, we first compile EEx without the .app -# file, then mix and then compile EEx fully -elixir: stdlib lib/eex/ebin/Elixir.EEx.beam mix ex_unit eex iex +# Since Mix depends on EEx and EEx depends on Mix, +# we first compile EEx without the .app file, +# then Mix, and then compile EEx fully +elixir: stdlib lib/eex/ebin/Elixir.EEx.beam mix ex_unit logger eex iex stdlib: $(KERNEL) VERSION -$(KERNEL): lib/elixir/lib/*.ex lib/elixir/lib/*/*.ex - $(Q) if [ ! -f $(KERNEL) ]; then \ - echo "==> bootstrap (compile)"; \ - $(ERL) -s elixir_compiler core -s erlang halt; \ +$(KERNEL): lib/elixir/lib/*.ex lib/elixir/lib/*/*.ex lib/elixir/lib/*/*/*.ex + $(Q) if [ ! -f $(KERNEL) ]; then \ + echo "==> bootstrap (compile)"; \ + $(ERL) -s elixir_compiler bootstrap -s erlang halt; \ fi + $(Q) $(MAKE) unicode @ echo "==> elixir (compile)"; - $(Q) cd lib/elixir && ../../$(ELIXIRC) "lib/kernel.ex" -o ebin; $(Q) cd lib/elixir && ../../$(ELIXIRC) "lib/**/*.ex" -o ebin; - $(Q) $(MAKE) unicode - $(Q) rm -rf lib/elixir/ebin/elixir.app - $(Q) cd lib/elixir && ../../$(REBAR) compile + $(Q) $(MAKE) app + +app: $(APP) +$(APP): lib/elixir/src/elixir.app.src lib/elixir/ebin VERSION $(GENERATE_APP) + $(Q) $(GENERATE_APP) $< $@ $(VERSION) unicode: $(UNICODE) $(UNICODE): lib/elixir/unicode/* @ echo "==> unicode (compile)"; - @ echo "This step can take up to a minute to compile in order to embed the Unicode database" - $(Q) cd lib/elixir && ../../$(ELIXIRC) unicode/unicode.ex -o ebin; + $(Q) $(ELIXIRC) lib/elixir/unicode/unicode.ex -o lib/elixir/ebin; + $(Q) $(ELIXIRC) lib/elixir/unicode/security.ex -o lib/elixir/ebin; + $(Q) $(ELIXIRC) lib/elixir/unicode/tokenizer.ex -o lib/elixir/ebin; $(eval $(call APP_TEMPLATE,ex_unit,ExUnit)) +$(eval $(call APP_TEMPLATE,logger,Logger)) $(eval $(call APP_TEMPLATE,eex,EEx)) $(eval $(call APP_TEMPLATE,mix,Mix)) $(eval $(call APP_TEMPLATE,iex,IEx)) @@ -97,64 +118,148 @@ $(eval $(call APP_TEMPLATE,iex,IEx)) install: compile @ echo "==> elixir (install)" $(Q) for dir in lib/*; do \ + rm -rf $(DESTDIR)$(PREFIX)/$(LIBDIR)/elixir/$$dir/ebin; \ $(INSTALL_DIR) "$(DESTDIR)$(PREFIX)/$(LIBDIR)/elixir/$$dir/ebin"; \ $(INSTALL_DATA) $$dir/ebin/* "$(DESTDIR)$(PREFIX)/$(LIBDIR)/elixir/$$dir/ebin"; \ done $(Q) $(INSTALL_DIR) "$(DESTDIR)$(PREFIX)/$(LIBDIR)/elixir/bin" $(Q) $(INSTALL_PROGRAM) $(filter-out %.ps1, $(filter-out %.bat, $(wildcard bin/*))) "$(DESTDIR)$(PREFIX)/$(LIBDIR)/elixir/bin" - $(Q) $(INSTALL_DIR) "$(DESTDIR)$(PREFIX)/bin" - $(Q) for file in "$(DESTDIR)$(PREFIX)"/$(LIBDIR)/elixir/bin/* ; do \ - ln -sf "../$(LIBDIR)/elixir/bin/$${file##*/}" "$(DESTDIR)$(PREFIX)/bin/" ; \ + $(Q) $(INSTALL_DIR) "$(DESTDIR)$(PREFIX)/$(BINDIR)" + $(Q) for file in "$(DESTDIR)$(PREFIX)"/$(LIBDIR)/elixir/bin/*; do \ + ln -sf "../$(LIBDIR)/elixir/bin/$${file##*/}" "$(DESTDIR)$(PREFIX)/$(BINDIR)/"; \ done + $(MAKE) install_man + +check_reproducible: compile + $(Q) echo "==> Checking for reproducible builds..." + $(Q) rm -rf lib/*/tmp/ebin_reproducible/ + $(call WRITE_SOURCE_DATE_EPOCH) + $(Q) mkdir -p lib/elixir/tmp/ebin_reproducible/ \ + lib/eex/tmp/ebin_reproducible/ \ + lib/ex_unit/tmp/ebin_reproducible/ \ + lib/iex/tmp/ebin_reproducible/ \ + lib/logger/tmp/ebin_reproducible/ \ + lib/mix/tmp/ebin_reproducible/ + $(Q) mv lib/elixir/ebin/* lib/elixir/tmp/ebin_reproducible/ + $(Q) mv lib/eex/ebin/* lib/eex/tmp/ebin_reproducible/ + $(Q) mv lib/ex_unit/ebin/* lib/ex_unit/tmp/ebin_reproducible/ + $(Q) mv lib/iex/ebin/* lib/iex/tmp/ebin_reproducible/ + $(Q) mv lib/logger/ebin/* lib/logger/tmp/ebin_reproducible/ + $(Q) mv lib/mix/ebin/* lib/mix/tmp/ebin_reproducible/ + SOURCE_DATE_EPOCH=$(call READ_SOURCE_DATE_EPOCH) $(MAKE) compile + $(Q) echo "Diffing..." + $(Q) bin/elixir lib/elixir/diff.exs lib/elixir/ebin/ lib/elixir/tmp/ebin_reproducible/ + $(Q) bin/elixir lib/elixir/diff.exs lib/eex/ebin/ lib/eex/tmp/ebin_reproducible/ + $(Q) bin/elixir lib/elixir/diff.exs lib/ex_unit/ebin/ lib/ex_unit/tmp/ebin_reproducible/ + $(Q) bin/elixir lib/elixir/diff.exs lib/iex/ebin/ lib/iex/tmp/ebin_reproducible/ + $(Q) bin/elixir lib/elixir/diff.exs lib/logger/ebin/ lib/logger/tmp/ebin_reproducible/ + $(Q) bin/elixir lib/elixir/diff.exs lib/mix/ebin/ lib/mix/tmp/ebin_reproducible/ + $(Q) echo "Builds are reproducible" clean: - cd lib/elixir && ../../$(REBAR) clean rm -rf ebin rm -rf lib/*/ebin - rm -rf lib/elixir/test/ebin - rm -rf lib/*/tmp - rm -rf lib/mix/test/fixtures/git_repo - rm -rf lib/mix/test/fixtures/deps_on_git_repo - rm -rf lib/mix/test/fixtures/git_rebar - rm -rf lib/elixir/src/elixir.app.src - -clean_exbeam: - $(Q) rm -f lib/*/ebin/Elixir.*.beam - -#==> Release tasks + rm -rf $(PARSER) + $(Q) $(MAKE) clean_residual_files -SOURCE_REF = $(shell head="$$(git rev-parse HEAD)" tag="$$(git tag --points-at $$head | tail -1)" ; echo "$${tag:-$$head}\c") -DOCS = bin/elixir ../ex_doc/bin/ex_doc "$(1)" "$(VERSION)" "lib/$(2)/ebin" -m "$(3)" -u "https://github.com/elixir-lang/elixir" --source-ref "$(call SOURCE_REF)" -o docs/$(2) -p http://elixir-lang.org/docs.html +clean_elixir: + $(Q) rm -f lib/*/ebin/Elixir.*.beam -docs: compile ../ex_doc/bin/ex_doc - $(Q) rm -rf docs - $(call DOCS,Elixir,elixir,Kernel) - $(call DOCS,EEx,eex,EEx) - $(call DOCS,Mix,mix,Mix) - $(call DOCS,IEx,iex,IEx) - $(call DOCS,ExUnit,ex_unit,ExUnit) +clean_residual_files: + rm -rf lib/*/_build/ + rm -rf lib/*/tmp/ + rm -rf lib/elixir/test/ebin/ + rm -rf lib/mix/test/fixtures/deps_on_git_repo/ + rm -rf lib/mix/test/fixtures/git_rebar/ + rm -rf lib/mix/test/fixtures/git_repo/ + rm -rf lib/mix/test/fixtures/git_sparse_repo/ + rm -rf lib/mix/test/fixtures/archive/ebin/ + rm -f erl_crash.dump + $(Q) $(MAKE) clean_man + +#==> Documentation tasks + +LOGO_PATH = $(shell test -f ../docs/logo.png && echo "--logo ../docs/logo.png") +SOURCE_REF = $(shell tag="$(call GIT_TAG)" revision="$(call GIT_REVISION)"; echo "$${tag:-$$revision}") +COMPILE_DOCS = CANONICAL=$(CANONICAL) bin/elixir ../ex_doc/bin/ex_doc "$(1)" "$(VERSION)" "lib/$(2)/ebin" --main "$(3)" --source-url "https://github.com/elixir-lang/elixir" --source-ref "$(call SOURCE_REF)" $(call LOGO_PATH) --output doc/$(2) --canonical "https://hexdocs.pm/$(2)/$(CANONICAL)" --homepage-url "https://elixir-lang.org/docs.html" --formatter "$(DOCS_FORMAT)" $(4) + +docs: compile ../ex_doc/bin/ex_doc docs_elixir docs_eex docs_mix docs_iex docs_ex_unit docs_logger + +docs_elixir: compile ../ex_doc/bin/ex_doc + @ echo "==> ex_doc (elixir)" + $(Q) rm -rf doc/elixir + $(call COMPILE_DOCS,Elixir,elixir,Kernel,--config "lib/elixir/docs.exs") + +docs_eex: compile ../ex_doc/bin/ex_doc + @ echo "==> ex_doc (eex)" + $(Q) rm -rf doc/eex + $(call COMPILE_DOCS,EEx,eex,EEx,--config "lib/mix/docs.exs") + +docs_mix: compile ../ex_doc/bin/ex_doc + @ echo "==> ex_doc (mix)" + $(Q) rm -rf doc/mix + $(call COMPILE_DOCS,Mix,mix,Mix,--config "lib/mix/docs.exs") + +docs_iex: compile ../ex_doc/bin/ex_doc + @ echo "==> ex_doc (iex)" + $(Q) rm -rf doc/iex + $(call COMPILE_DOCS,IEx,iex,IEx,--config "lib/mix/docs.exs") + +docs_ex_unit: compile ../ex_doc/bin/ex_doc + @ echo "==> ex_doc (ex_unit)" + $(Q) rm -rf doc/ex_unit + $(call COMPILE_DOCS,ExUnit,ex_unit,ExUnit,--config "lib/mix/docs.exs") + +docs_logger: compile ../ex_doc/bin/ex_doc + @ echo "==> ex_doc (logger)" + $(Q) rm -rf doc/logger + $(call COMPILE_DOCS,Logger,logger,Logger,--config "lib/mix/docs.exs") ../ex_doc/bin/ex_doc: @ echo "ex_doc is not found in ../ex_doc as expected. See README for more information." @ false -release_zip: compile - rm -rf v$(VERSION).zip - zip -9 -r v$(VERSION).zip bin CHANGELOG.md LEGAL lib/*/ebin LICENSE README.md VERSION +#==> Zip tasks + +Docs.zip: docs + rm -f Docs.zip + zip -9 -r Docs.zip CHANGELOG.md doc NOTICE LICENSE README.md + @ echo "Docs file created $(CURDIR)/Docs.zip" + +Precompiled.zip: build_man compile + rm -f Precompiled.zip + zip -9 -r Precompiled.zip bin CHANGELOG.md lib/*/ebin lib/*/lib LICENSE man NOTICE README.md VERSION + @ echo "Precompiled file created $(CURDIR)/Precompiled.zip" -release_docs: docs - cd ../docs - rm -rf ../docs/master - mv docs ../docs/master +#==> Test tasks -#==> Tests tasks +# If you modify this task, please update .cirrus.yml accordingly +test: test_formatted test_erlang test_elixir -test: test_erlang test_elixir +test_windows: test test_taskkill + +test_taskkill: + taskkill //IM erl.exe //F //T //FI "MEMUSAGE gt 0" + taskkill //IM epmd.exe //F //T //FI "MEMUSAGE gt 0" TEST_ERL = lib/elixir/test/erlang TEST_EBIN = lib/elixir/test/ebin TEST_ERLS = $(addprefix $(TEST_EBIN)/, $(addsuffix .beam, $(basename $(notdir $(wildcard $(TEST_ERL)/*.erl))))) +define FORMAT + $(Q) if [ "$(OS)" = "Windows_NT" ]; then \ + cmd //C call ./bin/mix.bat format $(1); \ + else \ + bin/elixir bin/mix format $(1); \ + fi +endef + +format: compile + $(call FORMAT) + +test_formatted: compile + $(call FORMAT,--check-formatted) + test_erlang: compile $(TEST_ERLS) @ echo "==> elixir (eunit)" $(Q) $(ERL) -pa $(TEST_EBIN) -s test_helper test; @@ -164,25 +269,61 @@ $(TEST_EBIN)/%.beam: $(TEST_ERL)/%.erl $(Q) mkdir -p $(TEST_EBIN) $(Q) $(ERLC) -o $(TEST_EBIN) $< -test_elixir: test_stdlib test_ex_unit test_doc_test test_mix test_eex test_iex - -test_doc_test: compile - @ echo "==> doctest (exunit)" - $(Q) cd lib/elixir && ../../bin/elixir -r "test/doc_test.exs"; +test_elixir: test_stdlib test_ex_unit test_logger test_eex test_iex test_mix test_stdlib: compile - @ echo "==> elixir (exunit)" + @ echo "==> elixir (ex_unit)" $(Q) exec epmd & exit - $(Q) cd lib/elixir && ../../bin/elixir -r "test/elixir/test_helper.exs" -pr "test/elixir/**/*_test.exs"; + $(Q) if [ "$(OS)" = "Windows_NT" ]; then \ + cd lib/elixir && cmd //C call ../../bin/elixir.bat -r "test/elixir/test_helper.exs" -pr "test/elixir/**/$(TEST_FILES)"; \ + else \ + cd lib/elixir && ../../bin/elixir -r "test/elixir/test_helper.exs" -pr "test/elixir/**/$(TEST_FILES)"; \ + fi + +#==> Dialyzer tasks + +DIALYZER_OPTS = --no_check_plt --fullpath -Werror_handling -Wunmatched_returns -Wunderspecs +PLT = .elixir.plt + +$(PLT): + @ echo "==> Building PLT with Elixir's dependencies..." + $(Q) dialyzer --output_plt $(PLT) --build_plt --apps erts kernel stdlib compiler syntax_tools parsetools tools ssl inets crypto runtime_tools ftp tftp mnesia public_key asn1 sasl + +clean_plt: + $(Q) rm -f $(PLT) -.dialyzer.base_plt: - @ echo "==> Adding Erlang/OTP basic applications to a new base PLT" - $(Q) dialyzer --output_plt .dialyzer.base_plt --build_plt --apps erts kernel stdlib compiler tools syntax_tools parsetools +build_plt: clean_plt $(PLT) -dialyze: .dialyzer.base_plt - $(Q) rm -f .dialyzer_plt - $(Q) cp .dialyzer.base_plt .dialyzer_plt - @ echo "==> Adding Elixir to PLT..." - $(Q) dialyzer --plt .dialyzer_plt --add_to_plt -r lib/elixir/ebin lib/ex_unit/ebin lib/eex/ebin lib/iex/ebin lib/mix/ebin +dialyze: compile $(PLT) @ echo "==> Dialyzing Elixir..." - $(Q) dialyzer --plt .dialyzer_plt -r lib/elixir/ebin lib/ex_unit/ebin lib/eex/ebin lib/iex/ebin lib/mix/ebin + $(Q) dialyzer -pa lib/elixir/ebin --plt $(PLT) $(DIALYZER_OPTS) lib/*/ebin + +#==> Man page tasks + +build_man: man/iex.1 man/elixir.1 + +man/iex.1: + $(Q) cp man/iex.1.in man/iex.1 + $(Q) sed -i.bak "/{COMMON}/r man/common" man/iex.1 + $(Q) sed -i.bak "/{COMMON}/d" man/iex.1 + $(Q) rm -f man/iex.1.bak + +man/elixir.1: + $(Q) cp man/elixir.1.in man/elixir.1 + $(Q) sed -i.bak "/{COMMON}/r man/common" man/elixir.1 + $(Q) sed -i.bak "/{COMMON}/d" man/elixir.1 + $(Q) rm -f man/elixir.1.bak + +clean_man: + rm -f man/elixir.1 + rm -f man/elixir.1.bak + rm -f man/iex.1 + rm -f man/iex.1.bak + +install_man: build_man + $(Q) mkdir -p $(DESTDIR)$(MAN_PREFIX)/man1 + $(Q) $(INSTALL_DATA) man/elixir.1 $(DESTDIR)$(MAN_PREFIX)/man1 + $(Q) $(INSTALL_DATA) man/elixirc.1 $(DESTDIR)$(MAN_PREFIX)/man1 + $(Q) $(INSTALL_DATA) man/iex.1 $(DESTDIR)$(MAN_PREFIX)/man1 + $(Q) $(INSTALL_DATA) man/mix.1 $(DESTDIR)$(MAN_PREFIX)/man1 + $(MAKE) clean_man diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000000..ac47695cc30 --- /dev/null +++ b/NOTICE @@ -0,0 +1,37 @@ +LEGAL NOTICE INFORMATION +------------------------ + +All the files in this distribution are copyright to the terms below. + +== lib/elixir/src/elixir_parser.erl (generated by build scripts) + +Copyright Ericsson AB 1996-2015 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +== All other files + +Copyright 2012 Plataformatec +Copyright 2021 The Elixir Team + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index 5bfeb70f5ff..60c53f81362 100644 --- a/README.md +++ b/README.md @@ -1,57 +1,235 @@ -![Elixir](https://github.com/elixir-lang/elixir-lang.github.com/raw/master/images/logo/logo.png) -========= -[![Build Status](https://secure.travis-ci.org/elixir-lang/elixir.svg?branch=master "Build Status")](http://travis-ci.org/elixir-lang/elixir) +Elixir +Elixir -For more about Elixir, installation and documentation, [check Elixir's website](http://elixir-lang.org/). +[![CI](https://github.com/elixir-lang/elixir/workflows/CI/badge.svg?branch=main)](https://github.com/elixir-lang/elixir/actions?query=branch%3Amain+workflow%3ACI) [![Build status](https://api.cirrus-ci.com/github/elixir-lang/elixir.svg?branch=main)](https://cirrus-ci.com/github/elixir-lang/elixir) -## Usage +Elixir is a dynamic, functional language designed for building scalable +and maintainable applications. -If you want to contribute to Elixir or run it from source, clone this repository to your machine, compile and test it: +For more about Elixir, installation and documentation, +[check Elixir's website](https://elixir-lang.org/). - $ git clone https://github.com/elixir-lang/elixir.git - $ cd elixir - $ make clean test +## Policies -If Elixir fails to build (specifically when pulling in a new version via git), be sure to remove any previous build artifacts by running `make clean`, then `make test`. +New releases are announced in the [announcement mailing list][8]. +You can subscribe by sending an email to elixir-lang-ann+subscribe@googlegroups.com +and replying to the confirmation email. -If tests pass, you are ready to move on to the [Getting Started guide][1] or to try Interactive Elixir by running: `bin/iex` in your terminal. +All security releases [will be tagged with `[security]`][10]. For more +information, please read our [Security Policy][9]. -However, if tests fail, it is likely you have an outdated Erlang version (Elixir requires Erlang 17.0 or later). You can check your Erlang version by calling `erl` in the command line. You will see some information as follows: +All interactions in our official communication channels follow our +[Code of Conduct][1]. - Erlang/OTP 17 [erts-6.0] [source-07b8f44] [64-bit] [smp:4:4] [async-threads:10] [hipe] [kernel-poll:false] +## Bug reports -If you have the correct version and tests still fail, feel free to [open an issue][2]. +For reporting bugs, [visit our issue tracker][2] and follow the steps +for reporting a new issue. **Please disclose security vulnerabilities +privately at elixir-security@googlegroups.com**. -## Building documentation +## Issues tracker management + +All currently open bugs related to the Elixir repository are listed +in the issues tracker. The Elixir team uses the issues tracker to focus +on *actionable items*, including planned enhancements in the short- and +medium-term. We also do our best to label entries for clarity and to ease +collaboration. + +Our *actionable item policy* has some important consequences, such as: + + * Proposing new features as well as request for support, help, and + guidance must be done in their own spaces, detailed next. + + * Issues where we have identified to be outside of Elixir scope, + such as a bug upstream, will be closed (and requested to be moved + elsewhere if appropriate). + + * We actively close unrelated and non-actionable issues to keep the + issues tracker tidy. However, we may get things wrong from time to + time, so we are glad to revisit issues and reopen if necessary. + +Keep the tone positive and be kind! For more information, see the +[Code of Conduct][1]. + +### Proposing new features + +For proposing new features, please start a discussion in the +[Elixir Core mailing list][3]. Keep in mind that it is your responsibility +to argue and explain why a feature is useful and how it will impact the +codebase and the community. + +Once a proposal is accepted, it will be added to [the issue tracker][2]. +Features and bug fixes that have already been merged and will be included +in the next release are then "closed" and added to the [changelog][7]. + +### Discussions, support, and help + +For general discussions, support, and help, please use many of the community +spaces [listed on the sidebar of the Elixir website](https://elixir-lang.org/), +such as forums, chat platforms, etc, where the wider community will be available +to help you. -Building the documentation requires [ex_doc](https://github.com/elixir-lang/ex_doc) to be installed and built in the same containing folder as elixir. +## Compiling from source - # After cloning and compiling Elixir - $ git clone git://github.com/elixir-lang/ex_doc.git - $ cd ex_doc && ../elixir/bin/mix compile - $ cd ../elixir && make docs +For the many different ways to install Elixir, +[see our installation instructions on the website](https://elixir-lang.org/install.html). +However, if you want to contribute to Elixir, you will need to compile from source. + +First, [install Erlang](https://elixir-lang.org/install.html#installing-erlang). +After that, clone this repository to your machine, compile and test it: + +```sh +git clone https://github.com/elixir-lang/elixir.git +cd elixir +make clean test +``` + +> Note: if you are running on Windows, +[this article includes important notes for compiling Elixir from source +on Windows](https://github.com/elixir-lang/elixir/wiki/Windows). + +In case you want to use this Elixir version as your system version, +you need to add the `bin` directory to [your PATH environment variable](https://elixir-lang.org/install.html#setting-path-environment-variable). + +If Elixir fails to build (specifically when pulling in a new version via +`git`), be sure to remove any previous build artifacts by running +`make clean`, then `make test`. ## Contributing -We appreciate any contribution to Elixir, so check out our [CONTRIBUTING.md](CONTRIBUTING.md) guide for more information. We usually keep a list of features and bugs [in the issue tracker][2]. +We welcome everyone to contribute to Elixir. To do so, there are a few +things you need to know about the code. First, Elixir code is divided +in applications inside the `lib` folder: + +* `elixir` - Elixir's kernel and standard library + +* `eex` - EEx is the template engine that allows you to embed Elixir + +* `ex_unit` - ExUnit is a simple test framework that ships with Elixir + +* `iex` - IEx stands for Interactive Elixir: Elixir's interactive shell + +* `logger` - Logger is the built-in logger + +* `mix` - Mix is Elixir's build tool + +You can run all tests in the root directory with `make test` and you can +also run tests for a specific framework `make test_#{APPLICATION}`, for example, +`make test_ex_unit`. If you just changed something in Elixir's standard +library, you can run only that portion through `make test_stdlib`. + +If you are changing just one file, you can choose to compile and run tests only +for that particular file for fast development cycles. For example, if you +are changing the String module, you can compile it and run its tests as: + +```sh +bin/elixirc lib/elixir/lib/string.ex -o lib/elixir/ebin +bin/elixir lib/elixir/test/elixir/string_test.exs +``` + +To recompile (including Erlang modules): + +```sh +make compile +``` + +After your changes are done, please remember to run `make format` to guarantee +all files are properly formatted and then run the full suite with +`make test`. + +If your contribution fails during the bootstrapping of the language, +you can rebuild the language from scratch with: + +```sh +make clean_elixir compile +``` + +Similarly, if you can't get Elixir to compile or the tests to pass after +updating an existing checkout, run `make clean compile`. You can check +[the official build status](https://github.com/elixir-lang/elixir/actions/workflows/ci.yml). +More tasks can be found by reading the [Makefile](Makefile). + +With tests running and passing, you are ready to contribute to Elixir and +[send a pull request](https://help.github.com/articles/using-pull-requests/). +We have saved some excellent pull requests we have received in the past in +case you are looking for some examples: + +* [Implement Enum.member? - Pull request](https://github.com/elixir-lang/elixir/pull/992) +* [Add String.valid? - Pull request](https://github.com/elixir-lang/elixir/pull/1058) +* [Implement capture_io for ExUnit - Pull request](https://github.com/elixir-lang/elixir/pull/1059) + +### Reviewing changes + +Once a pull request is sent, the Elixir team will review your changes. +We outline our process below to clarify the roles of everyone involved. + +All pull requests must be approved by two committers before being merged into +the repository. If any changes are necessary, the team will leave appropriate +comments requesting changes to the code. Unfortunately, we cannot guarantee a +pull request will be merged, even when modifications are requested, as the Elixir +team will re-evaluate the contribution as it changes. + +Committers may also push style changes directly to your branch. If you would +rather manage all changes yourself, you can disable the "Allow edits from maintainers" +feature when submitting your pull request. + +The Elixir team may optionally assign someone to review a pull request. +If someone is assigned, they must explicitly approve the code before +another team member can merge it. + +When the review finishes, your pull request will be squashed and merged +into the repository. If you have carefully organized your commits and +believe they should be merged without squashing, please mention it in +a comment. + +## Building documentation + +Building the documentation requires [ExDoc](https://github.com/elixir-lang/ex_doc) +to be installed and built alongside Elixir: + +```sh +# After cloning and compiling Elixir, in its parent directory: +git clone https://github.com/elixir-lang/ex_doc.git +cd ex_doc && ../elixir/bin/mix do deps.get + compile +``` + +Now go back to Elixir's root directory and run: + +```sh +make docs # to generate HTML pages +make docs DOCS_FORMAT=epub # to generate EPUB documents +``` + +This will produce documentation sets for `elixir`, `eex`, `ex_unit`, `iex`, `logger`, +and `mix` under the `doc` directory. If you are planning to contribute documentation, +[please check our best practices for writing documentation](https://hexdocs.pm/elixir/writing-documentation.html). -## Important links +## Development links -* \#elixir-lang on freenode IRC -* [Website][1] -* [Issue tracker][2] -* [elixir-talk Mailing list (questions)][3] -* [elixir-core Mailing list (development)][4] + * [Elixir Documentation][6] + * [Elixir Core Mailing list (development)][3] + * [Announcement mailing list][8] + * [Code of Conduct][1] + * [Issue tracker][2] + * [Changelog][7] + * [Security Policy][9] + * **[#elixir][4]** on [Libera.Chat][5] IRC - [1]: http://elixir-lang.org + [1]: CODE_OF_CONDUCT.md [2]: https://github.com/elixir-lang/elixir/issues - [3]: http://groups.google.com/group/elixir-lang-talk - [4]: http://groups.google.com/group/elixir-lang-core + [3]: https://groups.google.com/group/elixir-lang-core + [4]: https://web.libera.chat/#elixir + [5]: https://libera.chat + [6]: https://elixir-lang.org/docs.html + [7]: CHANGELOG.md + [8]: https://groups.google.com/group/elixir-lang-ann + [9]: SECURITY.md + [10]: https://groups.google.com/forum/#!searchin/elixir-lang-ann/%5Bsecurity%5D%7Csort:date ## License -"Elixir" and the Elixir logo are copyright (c) 2012 Plataformatec. +"Elixir" and the Elixir logo are registered trademarks of The Elixir Team. -Elixir source code is released under Apache 2 License with some parts under Erlang's license (EPL). +Elixir source code is released under Apache License 2.0. -Check [LEGAL](LEGAL) and [LICENSE](LICENSE) files for more information. +Check [NOTICE](NOTICE) and [LICENSE](LICENSE) files for more information. diff --git a/RELEASE.md b/RELEASE.md index 965ace69e87..1042fe2ef20 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,31 +1,49 @@ # Release process -This document simply outlines the release process: +## Shipping a new version -1. Remove `-dev` extension from VERSION +1. Ensure you are running on the oldest supported Erlang version -2. Ensure CHANGELOG is updated and timestamp +2. Update version in /VERSION, bin/elixir and bin/elixir.bat -3. Commit changes above with title "Release vVERSION" and generate new tag +3. Ensure /CHANGELOG.md is updated, versioned and add the current date -4. Run `make clean test` to ensure all tests pass from scratch and the CI is green +4. Update "Compatibility and Deprecations" if a new OTP version is supported -5. Push master and tags +5. Commit changes above with title "Release vVERSION", generate a new tag, and push it -6. Release new docs with `make release_docs`, move docs to `docs/stable` +6. Wait until GitHub Actions publish artifacts to the draft release and the CI is green -7. Release new zip with `make release_zip`, push new zip to GitHub Releases +7. Copy the relevant bits from /CHANGELOG.md to the GitHub release and publish it -8. Merge master into stable branch and push it +8. Add the release to `elixir.csv` with the minimum supported OTP version (all releases), update `erlang.csv` to the latest supported OTP version, and `_data/elixir-versions.yml` (except for RCs) files in `elixir-lang/elixir-lang.github.com` -9. After release, bump versions, add `-dev` back and commit +## Creating a new vMAJOR.MINOR branch -10. `make release_docs` once again and push it to `elixir-lang.org` +### In the new branch -11. Also update `release` file in `elixir-lang.org` +1. Set `CANONICAL=` in /Makefile -## Places where version is mentioned +2. Update tables in /SECURITY.md and "Compatibility and Deprecations" -* VERSION -* CHANGELOG -* src/elixir.app.src +3. Commit "Branch out vMAJOR.MINOR" + +### Back in main + +1. Bump /VERSION file, bin/elixir and bin/elixir.bat + +2. Start new /CHANGELOG.md + +3. Update tables in /SECURITY.md and in "Compatibility and Deprecations" + +4. Commit "Start vMAJOR.MINOR+1" + +## Changing supported Erlang/OTP versions + +1. Update the table in Compatibility and Deprecations + +2. Update `otp_release` checks in /Makefile and `/lib/elixir/src/elixir.erl` + +3. Update CI workflows in `/.cirrus.yml`, `/.github/workflows/ci.yml`, and `/.github/workflows/releases.yml` + +4. Remove `otp_release` version checks that are no longer needed diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..128bcd458ba --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,24 @@ +# Security Policy + +## Supported versions + +Elixir applies bug fixes only to the latest minor branch. Security patches are available for the last 5 minor branches: + +Elixir version | Support +:------------- | :----------------------------- +1.14 | Development +1.13 | Bug fixes and security patches +1.12 | Security patches only +1.11 | Security patches only +1.10 | Security patches only +1.9 | Security patches only + +## Announcements + +New releases are announced in the read-only [announcements mailing list](https://groups.google.com/group/elixir-lang-ann). You can subscribe by sending an email to elixir-lang-ann+subscribe@googlegroups.com and replying to the confirmation email. + +Security notifications [will be tagged with `[security]`](https://groups.google.com/forum/#!searchin/elixir-lang-ann/%5Bsecurity%5D%7Csort:date). + +## Reporting a vulnerability + +Please disclose security vulnerabilities privately at elixir-security@googlegroups.com diff --git a/VERSION b/VERSION index 49ccc4f4b86..16772dd8ade 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.14.3-dev \ No newline at end of file +1.14.0-dev \ No newline at end of file diff --git a/bin/elixir b/bin/elixir index d63c1d206fc..85c8e835c15 100755 --- a/bin/elixir +++ b/bin/elixir @@ -1,26 +1,62 @@ #!/bin/sh -if [ $# -eq 0 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ]; then - echo "Usage: `basename $0` [options] [.exs file] [data] - - -v Prints version and exit - -e \"command\" Evaluates the given command (*) - -r \"file\" Requires the given files/patterns (*) - -S \"script\"   Finds and executes the given script - -pr \"file\" Requires the given files/patterns in parallel (*) - -pa \"path\" Prepends the given path to Erlang code path (*) - -pz \"path\" Appends the given path to Erlang code path (*) - --app \"app\" Start the given app and its dependencies (*) - --erl \"switches\" Switches to be passed down to erlang (*) - --name \"name\" Makes and assigns a name to the distributed node - --sname \"name\" Makes and assigns a short name to the distributed node - --cookie \"cookie\" Sets a cookie for this distributed node - --hidden Makes a hidden node - --detached Starts the Erlang VM detached from console - --no-halt Does not halt the Erlang VM after execution - -** Options marked with (*) can be given more than once -** Options given after the .exs file or -- are passed down to the executed code -** Options can be passed to the erlang runtime using ELIXIR_ERL_OPTIONS or --erl" >&2 +set -e + +ELIXIR_VERSION=1.14.0-dev + +if [ $# -eq 0 ] || { [ $# -eq 1 ] && { [ "$1" = "--help" ] || [ "$1" = "-h" ]; }; }; then + cat <&2 +Usage: $(basename "$0") [options] [.exs file] [data] + +## General options + + -e "COMMAND" Evaluates the given command (*) + -h, --help Prints this message (standalone) + -r "FILE" Requires the given files/patterns (*) + -S SCRIPT Finds and executes the given script in \$PATH + -pr "FILE" Requires the given files/patterns in parallel (*) + -pa "PATH" Prepends the given path to Erlang code path (*) + -pz "PATH" Appends the given path to Erlang code path (*) + -v, --version Prints Erlang/OTP and Elixir versions (standalone) + + --app APP Starts the given app and its dependencies (*) + --erl "SWITCHES" Switches to be passed down to Erlang (*) + --eval "COMMAND" Evaluates the given command, same as -e (*) + --logger-otp-reports BOOL Enables or disables OTP reporting + --logger-sasl-reports BOOL Enables or disables SASL reporting + --no-halt Does not halt the Erlang VM after execution + --short-version Prints Elixir version (standalone) + --werl Uses Erlang's Windows shell GUI (Windows only) + +Options given after the .exs file or -- are passed down to the executed code. +Options can be passed to the Erlang runtime using \$ELIXIR_ERL_OPTIONS or --erl. + +## Distribution options + +The following options are related to node distribution. + + --cookie COOKIE Sets a cookie for this distributed node + --hidden Makes a hidden node + --name NAME Makes and assigns a name to the distributed node + --rpc-eval NODE "COMMAND" Evaluates the given command on the given remote node (*) + --sname NAME Makes and assigns a short name to the distributed node + +## Release options + +The following options are generally used under releases. + + --boot "FILE" Uses the given FILE.boot to start the system + --boot-var VAR "VALUE" Makes \$VAR available as VALUE to FILE.boot (*) + --erl-config "FILE" Loads configuration in FILE.config written in Erlang (*) + --pipe-to "PIPEDIR" "LOGDIR" Starts the Erlang VM as a named PIPEDIR and LOGDIR + --vm-args "FILE" Passes the contents in file as arguments to the VM + +--pipe-to starts Elixir detached from console (Unix-like only). +It will attempt to create PIPEDIR and LOGDIR if they don't exist. +See run_erl to learn more. To reattach, run: to_erl PIPEDIR. + +** Options marked with (*) can be given more than once. +** Standalone options can't be combined with other options. +USAGE exit 1 fi @@ -30,64 +66,184 @@ readlink_f () { if [ -h "$filename" ]; then readlink_f "$(readlink "$filename")" else - echo "`pwd -P`/$filename" + echo "$(pwd -P)/$filename" fi } -MODE="elixir" +if [ $# -eq 1 ] && [ "$1" = "--short-version" ]; then + echo "$ELIXIR_VERSION" + exit 0 +fi + +# Stores static Erlang arguments and --erl (which is passed as is) ERL="" + +# Stores erl arguments preserving spaces/quotes (mimics an array) +erl_set () { + eval "E${E}=\$1" + E=$((E + 1)) +} + +# Checks if a string starts with prefix. Usage: starts_with "$STRING" "$PREFIX" +starts_with () { + case $1 in + "$2"*) true;; + *) false;; + esac +} + +ERL_EXEC="erl" +MODE="elixir" I=1 +E=0 +LENGTH=$# +set -- "$@" -extra -while [ $I -le $# ]; do - S=1 - eval "PEEK=\${$I}" - case "$PEEK" in +while [ $I -le $LENGTH ]; do + # S counts to be shifted, C counts to be copied + S=0 + C=0 + case "$1" in +iex) + C=1 MODE="iex" ;; +elixirc) + C=1 MODE="elixirc" ;; - -v|--compile|--no-halt) + -v|--no-halt) + C=1 + ;; + -e|-r|-pr|-pa|-pz|--app|--eval|--remsh|--dot-iex) + C=2 + ;; + --rpc-eval) + C=3 + ;; + --hidden) + S=1 + ERL="$ERL -hidden" ;; - -e|-r|-pr|-pa|-pz|--remsh|--app) + --logger-otp-reports) S=2 + if [ "$2" = 'true' ] || [ "$2" = 'false' ]; then + ERL="$ERL -logger handle_otp_reports $2" + fi ;; - --detached|--hidden) - ERL="$ERL `echo $PEEK | cut -c 2-`" + --logger-sasl-reports) + S=2 + if [ "$2" = 'true' ] || [ "$2" = 'false' ]; then + ERL="$ERL -logger handle_sasl_reports $2" + fi + ;; + --erl) + S=2 + ERL="$ERL $2" ;; --cookie) - I=$(expr $I + 1) - eval "VAL=\${$I}" - ERL="$ERL -setcookie "$VAL"" + S=2 + erl_set "-setcookie" + erl_set "$2" ;; --sname|--name) - I=$(expr $I + 1) - eval "VAL=\${$I}" - ERL="$ERL `echo $PEEK | cut -c 2-` "$VAL"" + S=2 + erl_set "$(echo "$1" | cut -c 2-)" + erl_set "$2" ;; - --erl) - I=$(expr $I + 1) - eval "VAL=\${$I}" - ERL="$ERL "$VAL"" + --erl-config) + S=2 + erl_set "-config" + erl_set "$2" + ;; + --vm-args) + S=2 + erl_set "-args_file" + erl_set "$2" + ;; + --boot) + S=2 + erl_set "-boot" + erl_set "$2" + ;; + --boot-var) + S=3 + erl_set "-boot_var" + erl_set "$2" + erl_set "$3" + ;; + --pipe-to) + S=3 + RUN_ERL_PIPE="$2" + RUN_ERL_LOG="$3" + if [ "$(starts_with "$RUN_ERL_PIPE" "-")" ]; then + echo "--pipe-to : PIPEDIR cannot be a switch" >&2 && exit 1 + elif [ "$(starts_with "$RUN_ERL_LOG" "-")" ]; then + echo "--pipe-to : LOGDIR cannot be a switch" >&2 && exit 1 + fi + ;; + --werl) + S=1 + if [ "$OS" = "Windows_NT" ]; then ERL_EXEC="werl"; fi ;; *) + while [ $I -le $LENGTH ]; do + I=$((I + 1)) + set -- "$@" "$1" + shift + done break ;; esac - I=$(expr $I + $S) + + while [ $I -le $LENGTH ] && [ $C -gt 0 ]; do + C=$((C - 1)) + I=$((I + 1)) + set -- "$@" "$1" + shift + done + + I=$((I + S)) + shift $S +done + +I=$((E - 1)) +while [ $I -ge 0 ]; do + eval "VAL=\$E$I" + set -- "$VAL" "$@" + I=$((I - 1)) done SELF=$(readlink_f "$0") SCRIPT_PATH=$(dirname "$SELF") -if [ "$MODE" != "iex" ]; then ERL="$ERL -noshell -s elixir start_cli"; fi -if [ -z "$ERL_PATH" ]; then - if [ -f "$SCRIPT_PATH/../releases/RELEASES" ] && [ -f "$SCRIPT_PATH/erl" ]; then - ERL_PATH="$SCRIPT_PATH"/erl - else - ERL_PATH=erl - fi +if [ "$OSTYPE" = "cygwin" ]; then SCRIPT_PATH=$(cygpath -m "$SCRIPT_PATH"); fi +if [ "$MODE" != "iex" ]; then ERL="-noshell -s elixir start_cli $ERL"; fi + +if [ "$OS" != "Windows_NT" ] && [ -z "$NO_COLOR" ]; then + if test -t 1 -a -t 2; then ERL="-elixir ansi_enabled true $ERL"; fi +fi + +# One MAY change ERTS_BIN= but you MUST NOT change +# ERTS_BIN=$ERTS_BIN as it is handled by Elixir releases. +ERTS_BIN= +ERTS_BIN="$ERTS_BIN" + +set -- "$ERTS_BIN$ERL_EXEC" -pa "$SCRIPT_PATH"/../lib/*/ebin $ELIXIR_ERL_OPTIONS $ERL "$@" + +if [ -n "$RUN_ERL_PIPE" ]; then + ESCAPED="" + for PART in "$@"; do + ESCAPED="$ESCAPED $(echo "$PART" | sed 's/[^a-zA-Z0-9_\-\/]/\\&/g')" + done + mkdir -p "$RUN_ERL_PIPE" + mkdir -p "$RUN_ERL_LOG" + ERL_EXEC="run_erl" + set -- "$ERTS_BIN$ERL_EXEC" -daemon "$RUN_ERL_PIPE/" "$RUN_ERL_LOG/" "$ESCAPED" fi -exec "$ERL_PATH" -pa "$SCRIPT_PATH"/../lib/*/ebin $ELIXIR_ERL_OPTIONS $ERL -extra "$@" +if [ -n "$ELIXIR_CLI_DRY_RUN" ]; then + echo "$@" +else + exec "$@" +fi diff --git a/bin/elixir.bat b/bin/elixir.bat index 3ed6dda1150..47d761133da 100644 --- a/bin/elixir.bat +++ b/bin/elixir.bat @@ -1,95 +1,191 @@ -@echo off -if "%1"=="" goto documentation -if "%1"=="--help" goto documentation -if "%1"=="-h" goto documentation -if "%1"=="/h" goto documentation +@if defined ELIXIR_CLI_ECHO (@echo on) else (@echo off) + +set ELIXIR_VERSION=1.14.0-dev + +setlocal enabledelayedexpansion +if ""%1""=="""" if ""%2""=="""" goto documentation +if /I ""%1""==""--help"" if ""%2""=="""" goto documentation +if /I ""%1""==""-h"" if ""%2""=="""" goto documentation +if /I ""%1""==""/h"" if ""%2""=="""" goto documentation +if ""%1""==""/?"" if ""%2""=="""" goto documentation +if /I ""%1""==""--short-version"" if ""%2""=="""" goto shortversion goto parseopts :documentation echo Usage: %~nx0 [options] [.exs file] [data] echo. -echo -v Prints version and exit -echo -e command Evaluates the given command (*) -echo -r file Requires the given files/patterns (*) -echo -S script Finds and executes the given script -echo -pr file Requires the given files/patterns in parallel (*) -echo -pa path Prepends the given path to Erlang code path (*) -echo -pz path Appends the given path to Erlang code path (*) -echo --app app Start the given app and its dependencies (*) -echo --erl switches Switches to be passed down to erlang (*) -echo --name name Makes and assigns a name to the distributed node -echo --sname name Makes and assigns a short name to the distributed node -echo --cookie cookie Sets a cookie for this distributed node -echo --hidden Makes a hidden node -echo --detached Starts the Erlang VM detached from console -echo --no-halt Does not halt the Erlang VM after execution +echo ## General options +echo. +echo -e "COMMAND" Evaluates the given command (*) +echo -h, --help Prints this message (standalone) +echo -r "FILE" Requires the given files/patterns (*) +echo -S SCRIPT Finds and executes the given script in $PATH +echo -pr "FILE" Requires the given files/patterns in parallel (*) +echo -pa "PATH" Prepends the given path to Erlang code path (*) +echo -pz "PATH" Appends the given path to Erlang code path (*) +echo -v, --version Prints Erlang/OTP and Elixir versions (standalone) +echo. +echo --app APP Starts the given app and its dependencies (*) +echo --erl "SWITCHES" Switches to be passed down to Erlang (*) +echo --eval "COMMAND" Evaluates the given command, same as -e (*) +echo --logger-otp-reports BOOL Enables or disables OTP reporting +echo --logger-sasl-reports BOOL Enables or disables SASL reporting +echo --no-halt Does not halt the Erlang VM after execution +echo --short-version Prints Elixir version (standalone) +echo --werl Uses Erlang's Windows shell GUI (Windows only) +echo. +echo Options given after the .exs file or -- are passed down to the executed code. +echo Options can be passed to the Erlang runtime using $ELIXIR_ERL_OPTIONS or --erl. +echo. +echo ## Distribution options +echo. +echo The following options are related to node distribution. echo. -echo ** Options marked with (*) can be given more than once -echo ** Options given after the .exs file or -- are passed down to the executed code -echo ** Options can be passed to the erlang runtime using ELIXIR_ERL_OPTIONS or --erl -goto :EOF +echo --cookie COOKIE Sets a cookie for this distributed node +echo --hidden Makes a hidden node +echo --name NAME Makes and assigns a name to the distributed node +echo --rpc-eval NODE "COMMAND" Evaluates the given command on the given remote node (*) +echo --sname NAME Makes and assigns a short name to the distributed node +echo. +echo ## Release options +echo. +echo The following options are generally used under releases. +echo. +echo --boot "FILE" Uses the given FILE.boot to start the system +echo --boot-var VAR "VALUE" Makes $VAR available as VALUE to FILE.boot (*) +echo --erl-config "FILE" Loads configuration in FILE.config written in Erlang (*) +echo --vm-args "FILE" Passes the contents in file as arguments to the VM +echo. +echo --pipe-to is not supported on Windows. If set, Elixir won't boot. +echo. +echo ** Options marked with (*) can be given more than once. +echo ** Standalone options can't be combined with other options. +goto end + +:shortversion +echo !ELIXIR_VERSION! +goto end :parseopts +rem Parameters for Elixir +set parsElixir= + rem Parameters for Erlang set parsErlang= -rem Make sure we keep a copy of all parameters -set allPars=%* - -rem Get the original path name from the batch file -set originPath=%~dp0 - rem Optional parameters before the "-extra" parameter set beforeExtra= -rem Flag which determines whether or not to use werl vs erl -set useWerl=0 +rem Option which determines whether the loop is over +set endLoop=0 + +rem Designates which mode / Elixir component to run as +set runMode="elixir" + +rem Designates the path to the current script +set SCRIPT_PATH=%~dp0 + +rem Designates the path to the ERTS system +set ERTS_BIN= +set ERTS_BIN=!ERTS_BIN! rem Recursive loop called for each parameter that parses the cmd line parameters :startloop -set par="%1" -shift -if "%par%"=="" ( - rem if no parameters defined - goto :expand_erl_libs +set "par=%~1" +if "!par!"=="" ( + rem skip if no parameter + goto expand_erl_libs ) -if "%par%"=="""" ( - rem if no parameters defined - special case for parameter that is already quoted - goto :expand_erl_libs +shift +set par="!par:"=\"!" +if !endLoop! == 1 ( + set parsElixir=!parsElixir! !par! + goto startloop ) rem ******* EXECUTION OPTIONS ********************** -IF "%par%"==""+iex"" (Set useWerl=1) +if !par!=="--werl" (set useWerl=1 && goto startloop) +if !par!=="+iex" (set parsElixir=!parsElixir! +iex && set runMode="iex" && goto startloop) +if !par!=="+elixirc" (set parsElixir=!parsElixir! +elixirc && set runMode="elixirc" && goto startloop) +rem ******* EVAL PARAMETERS ************************ +if ""==!par:-e=! ( + set "VAR=%~1" + if not defined VAR (set VAR= ) + set parsElixir=!parsElixir! -e "!VAR:"=\"!" + shift + goto startloop +) +if ""==!par:--eval=! ( + set "VAR=%~1" + if not defined VAR (set VAR= ) + set parsElixir=!parsElixir! --eval "!VAR:"=\"!" + shift + goto startloop +) +if ""==!par:--rpc-eval=! ( + set "VAR=%~2" + if not defined VAR (set VAR= ) + set parsElixir=!parsElixir! --rpc-eval %1 "!VAR:"=\"!" + shift + shift + goto startloop +) +rem ******* ELIXIR PARAMETERS ********************** +if ""==!par:-r=! (set "parsElixir=!parsElixir! -r %1" && shift && goto startloop) +if ""==!par:-pr=! (set "parsElixir=!parsElixir! -pr %1" && shift && goto startloop) +if ""==!par:-pa=! (set "parsElixir=!parsElixir! -pa %1" && shift && goto startloop) +if ""==!par:-pz=! (set "parsElixir=!parsElixir! -pz %1" && shift && goto startloop) +if ""==!par:-v=! (set "parsElixir=!parsElixir! -v" && goto startloop) +if ""==!par:--version=! (set "parsElixir=!parsElixir! --version" && goto startloop) +if ""==!par:--app=! (set "parsElixir=!parsElixir! --app %1" && shift && goto startloop) +if ""==!par:--no-halt=! (set "parsElixir=!parsElixir! --no-halt" && goto startloop) +if ""==!par:--remsh=! (set "parsElixir=!parsElixir! --remsh %1" && shift && goto startloop) +if ""==!par:--dot-iex=! (set "parsElixir=!parsElixir! --dot-iex %1" && shift && goto startloop) rem ******* ERLANG PARAMETERS ********************** -IF NOT "%par%"=="%par:--detached=%" (Set parsErlang=%parsErlang% -detached) -IF NOT "%par%"=="%par:--hidden=%" (Set parsErlang=%parsErlang% -hidden) -IF NOT "%par%"=="%par:--cookie=%" (Set parsErlang=%parsErlang% -setcookie %1 && shift) -IF NOT "%par%"=="%par:--sname=%" (Set parsErlang=%parsErlang% -sname %1 && shift) -IF NOT "%par%"=="%par:--name=%" (Set parsErlang=%parsErlang% -name %1 && shift) -IF NOT "%par%"=="%par:--erl=%" (Set beforeExtra=%beforeExtra% %~1 && shift) -rem ******* elixir parameters ********************** -rem Note: we don't have to do anything with options that don't take an argument -IF NOT "%par%"=="%par:-e=%" (shift) -IF NOT "%par%"=="%par:-r=%" (shift) -IF NOT "%par%"=="%par:-pr=%" (shift) -IF NOT "%par%"=="%par:-pa=%" (shift) -IF NOT "%par%"=="%par:-pz=%" (shift) -IF NOT "%par%"=="%par:--app=%" (shift) -IF NOT "%par%"=="%par:--remsh=%" (shift) -goto:startloop +if ""==!par:--boot=! (set "parsErlang=!parsErlang! -boot %1" && shift && goto startloop) +if ""==!par:--boot-var=! (set "parsErlang=!parsErlang! -boot_var %1 %2" && shift && shift && goto startloop) +if ""==!par:--cookie=! (set "parsErlang=!parsErlang! -setcookie %1" && shift && goto startloop) +if ""==!par:--hidden=! (set "parsErlang=!parsErlang! -hidden" && goto startloop) +if ""==!par:--detached=! (set "parsErlang=!parsErlang! -detached" && echo warning: the --detached option is deprecated && goto startloop) +if ""==!par:--erl-config=! (set "parsErlang=!parsErlang! -config %1" && shift && goto startloop) +if ""==!par:--logger-otp-reports=! (set "parsErlang=!parsErlang! -logger handle_otp_reports %1" && shift && goto startloop) +if ""==!par:--logger-sasl-reports=! (set "parsErlang=!parsErlang! -logger handle_sasl_reports %1" && shift && goto startloop) +if ""==!par:--name=! (set "parsErlang=!parsErlang! -name %1" && shift && goto startloop) +if ""==!par:--sname=! (set "parsErlang=!parsErlang! -sname %1" && shift && goto startloop) +if ""==!par:--vm-args=! (set "parsErlang=!parsErlang! -args_file %1" && shift && goto startloop) +if ""==!par:--erl=! (set "beforeExtra=!beforeExtra! %~1" && shift && goto startloop) +if ""==!par:--pipe-to=! (echo --pipe-to : Option is not supported on Windows && goto end) +set endLoop=1 +set parsElixir=!parsElixir! !par! +goto startloop -rem ******* assume all pre-params are parsed ******************** :expand_erl_libs -rem ******* expand all ebin paths as Windows does not support the ..\*\ebin wildcard ******************** -SETLOCAL enabledelayedexpansion +rem expand all ebin paths as Windows does not support the ..\*\ebin wildcard set ext_libs= -for /d %%d in ("%originPath%..\lib\*.") do ( +for /d %%d in ("!SCRIPT_PATH!..\lib\*.") do ( set ext_libs=!ext_libs! -pa "%%~fd\ebin" ) -SETLOCAL disabledelayedexpansion + :run -IF %useWerl% EQU 1 ( - werl.exe %ext_libs% %ELIXIR_ERL_OPTIONS% %parsErlang% -s elixir start_cli %beforeExtra% -extra %* -) ELSE ( - erl.exe %ext_libs% -noshell %ELIXIR_ERL_OPTIONS% %parsErlang% -s elixir start_cli %beforeExtra% -extra %* +reg query HKCU\Console /v VirtualTerminalLevel 2>nul | findstr /e "0x1" >nul 2>nul +if %errorlevel% == 0 ( + set beforeExtra=-elixir ansi_enabled true !beforeExtra! +) +if not !runMode! == "iex" ( + set beforeExtra=-noshell -s elixir start_cli !beforeExtra! +) +if defined ELIXIR_CLI_DRY_RUN ( + if defined useWerl ( + echo start "" "!ERTS_BIN!werl.exe" !ext_libs! !ELIXIR_ERL_OPTIONS! !parsErlang! !beforeExtra! -extra !parsElixir! + ) else ( + echo "!ERTS_BIN!erl.exe" !ext_libs! !ELIXIR_ERL_OPTIONS! !parsErlang! !beforeExtra! -extra !parsElixir! + ) +) else ( + if defined useWerl ( + start "" "!ERTS_BIN!werl.exe" !ext_libs! !ELIXIR_ERL_OPTIONS! !parsErlang! !beforeExtra! -extra !parsElixir! + ) else ( + "!ERTS_BIN!erl.exe" !ext_libs! !ELIXIR_ERL_OPTIONS! !parsErlang! !beforeExtra! -extra !parsElixir! + ) ) +:end +endlocal diff --git a/bin/elixirc b/bin/elixirc index 109eca106cc..650e4758b98 100755 --- a/bin/elixirc +++ b/bin/elixirc @@ -1,17 +1,25 @@ #!/bin/sh +set -e + if [ $# -eq 0 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ]; then - echo "Usage: `basename $0` [elixir switches] [compiler switches] [.ex files] + cat <&2 +Usage: $(basename "$0") [elixir switches] [compiler switches] [.ex files] + + -h, --help Prints this message and exits + -o The directory to output compiled files + -v, --version Prints Elixir version and exits (standalone) - -o The directory to output compiled files - --no-docs Do not attach documentation to compiled modules - --no-debug-info Do not attach debug info to compiled modules - --ignore-module-conflict - --warnings-as-errors Treat warnings as errors and return non-zero exit code - --verbose Print informational messages. + --ignore-module-conflict Does not emit warnings if a module was previously defined + --no-debug-info Does not attach debug info to compiled modules + --no-docs Does not attach documentation to compiled modules + --profile time Profile the time to compile modules + --verbose Prints compilation status + --warnings-as-errors Treats warnings as errors and return non-zero exit status -** Options given after -- are passed down to the executed code -** Options can be passed to the erlang runtime using ELIXIR_ERL_OPTIONS -** Options can be passed to the erlang compiler using ERL_COMPILER_OPTIONS" >&2 +Options given after -- are passed down to the executed code. +Options can be passed to the Erlang runtime using \$ELIXIR_ERL_OPTIONS. +Options can be passed to the Erlang compiler using \$ERL_COMPILER_OPTIONS. +USAGE exit 1 fi @@ -21,7 +29,7 @@ readlink_f () { if [ -h "$filename" ]; then readlink_f "$(readlink "$filename")" else - echo "`pwd -P`/$filename" + echo "$(pwd -P)/$filename" fi } diff --git a/bin/elixirc.bat b/bin/elixirc.bat index 9d118975d25..e046f6af282 100644 --- a/bin/elixirc.bat +++ b/bin/elixirc.bat @@ -1,10 +1,12 @@ -@echo off +@if defined ELIXIR_CLI_ECHO (@echo on) else (@echo off) +setlocal set argc=0 for %%A in (%*) do ( - if "%%A"=="--help" goto documentation - if "%%A"=="-h" goto documentation - if "%%A"=="/h" goto documentation - set /A argc+=1 + if /I "%%A"=="--help" goto documentation + if /I "%%A"=="-h" goto documentation + if /I "%%A"=="/h" goto documentation + if "%%A"=="/?" goto documentation + set /A argc+=1 ) if %argc%==0 goto documentation goto run @@ -12,15 +14,24 @@ goto run :documentation echo Usage: %~nx0 [elixir switches] [compiler switches] [.ex files] echo. -echo -o The directory to output compiled files -echo --no-docs Do not attach documentation to compiled modules -echo --no-debug-info Do not attach debug info to compiled modules -echo --ignore-module-conflict -echo --warnings-as-errors Treat warnings as errors and return non-zero exit code -echo --verbose Print informational messages. +echo -h, --help Prints this message and exits +echo -o The directory to output compiled files +echo -v, --version Prints Elixir version and exits (standalone) +echo. +echo --ignore-module-conflict Does not emit warnings if a module was previously defined +echo --no-debug-info Does not attach debug info to compiled modules +echo --no-docs Does not attach documentation to compiled modules +echo --profile time Profile the time to compile modules +echo --verbose Prints compilation status +echo --warnings-as-errors Treats warnings as errors and returns non-zero exit status echo. echo ** Options given after -- are passed down to the executed code -echo ** Options can be passed to the erlang runtime using ELIXIR_ERL_OPTIONS -echo ** Options can be passed to the erlang compiler using ERL_COMPILER_OPTIONS >&2 +echo ** Options can be passed to the Erlang runtime using ELIXIR_ERL_OPTIONS +echo ** Options can be passed to the Erlang compiler using ERL_COMPILER_OPTIONS +goto end + :run call "%~dp0\elixir.bat" +elixirc %* + +:end +endlocal diff --git a/bin/iex b/bin/iex index e5187f2b08d..0eaf166b110 100755 --- a/bin/iex +++ b/bin/iex @@ -1,28 +1,19 @@ #!/bin/sh -if [ $# -gt 0 ] && ([ "$1" = "--help" ] || [ "$1" = "-h" ]); then - echo "Usage: `basename $0` [options] [.exs file] [data] +set -e - -v Prints version - -e \"command\" Evaluates the given command (*) - -r \"file\" Requires the given files/patterns (*) - -S \"script\"   Finds and executes the given script - -pr \"file\" Requires the given files/patterns in parallel (*) - -pa \"path\" Prepends the given path to Erlang code path (*) - -pz \"path\" Appends the given path to Erlang code path (*) - --app \"app\" Start the given app and its dependencies (*) - --erl \"switches\" Switches to be passed down to erlang (*) - --name \"name\" Makes and assigns a name to the distributed node - --sname \"name\" Makes and assigns a short name to the distributed node - --cookie \"cookie\" Sets a cookie for this distributed node - --hidden Makes a hidden node - --detached Starts the Erlang VM detached from console - --remsh \"name\" Connects to a node using a remote shell - --dot-iex \"path\" Overrides default .iex.exs file and uses path instead; - path can be empty, then no file will be loaded +if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then + cat <&2 +Usage: $(basename "$0") [options] [.exs file] [data] -** Options marked with (*) can be given more than once -** Options given after the .exs file or -- are passed down to the executed code -** Options can be passed to the VM using ELIXIR_ERL_OPTIONS or --erl" >&2 +The following options are exclusive to IEx: + + --dot-iex "FILE" Evaluates FILE, line by line, to set up IEx' environment. + Defaults to evaluating .iex.exs or ~/.iex.exs, if any exists. + If FILE is empty, then no file will be loaded. + --remsh NAME Connects to a node using a remote shell + +It accepts all other options listed by "elixir --help". +USAGE exit 1 fi @@ -32,10 +23,10 @@ readlink_f () { if [ -h "$filename" ]; then readlink_f "$(readlink "$filename")" else - echo "`pwd -P`/$filename" + echo "$(pwd -P)/$filename" fi } SELF=$(readlink_f "$0") SCRIPT_PATH=$(dirname "$SELF") -exec "$SCRIPT_PATH"/elixir --no-halt --erl "-user Elixir.IEx.CLI" +iex "$@" +exec "$SCRIPT_PATH"/elixir --no-halt --erl "-noshell -user Elixir.IEx.CLI" +iex "$@" diff --git a/bin/iex.bat b/bin/iex.bat index 717714c3605..137bfd79674 100644 --- a/bin/iex.bat +++ b/bin/iex.bat @@ -1,2 +1,28 @@ -@echo off -call "%~dp0\elixir.bat" +iex --erl "-user Elixir.IEx.CLI" --no-halt %* +@if defined ELIXIR_CLI_ECHO (@echo on) else (@echo off) +setlocal +if /I ""%1""==""--help"" goto documentation +if /I ""%1""==""-h"" goto documentation +if /I ""%1""==""/h"" goto documentation +if ""%1""==""/?"" goto documentation +goto run + +:documentation +echo Usage: %~nx0 [options] [.exs file] [data] +echo. +echo The following options are exclusive to IEx: +echo. +echo --dot-iex "FILE" Evaluates FILE, line by line, to set up IEx' environment. +echo Defaults to evaluating .iex.exs or ~/.iex.exs, if any exists. +echo If FILE is empty, then no file will be loaded. +echo --remsh NAME Connects to a node using a remote shell +echo --werl Uses Erlang's Windows shell GUI (Windows only) +echo. +echo Set the IEX_WITH_WERL environment variable to always use werl. +echo It accepts all other options listed by "elixir --help". +goto end + +:run +if defined IEX_WITH_WERL (set __ELIXIR_IEX_FLAGS=--werl) else (set __ELIXIR_IEX_FLAGS=) +call "%~dp0\elixir.bat" --no-halt --erl "-noshell -user Elixir.IEx.CLI" +iex %__ELIXIR_IEX_FLAGS% %* +:end +endlocal diff --git a/bin/mix b/bin/mix index 9dab2afd5ce..facb1aa2260 100755 --- a/bin/mix +++ b/bin/mix @@ -1,3 +1,3 @@ #!/usr/bin/env elixir -Mix.start -Mix.CLI.main +Mix.start() +Mix.CLI.main() diff --git a/bin/mix.bat b/bin/mix.bat index 435a5257340..7e4a7a65386 100644 --- a/bin/mix.bat +++ b/bin/mix.bat @@ -1,2 +1,2 @@ -@echo off +@if defined ELIXIR_CLI_ECHO (@echo on) else (@echo off) call "%~dp0\elixir.bat" "%~dp0\mix" %* diff --git a/bin/mix.ps1 b/bin/mix.ps1 index 9a4a36005cf..369c4942113 100644 --- a/bin/mix.ps1 +++ b/bin/mix.ps1 @@ -1,27 +1,23 @@ -# Initialize with path to mix.bat relative to caller's working directory -$toCmd = '' + (Resolve-Path -relative (Split-Path $MyInvocation.MyCommand.Path)) + '\mix.bat' +# Store path to mix.bat as a FileInfo object +$mixBatPath = (Get-ChildItem (((Get-ChildItem $MyInvocation.MyCommand.Path).Directory.FullName) + '\mix.bat')) +$newArgs = @() -foreach ($arg in $args) +for ($i = 0; $i -lt $args.length; $i++) { - $toCmd += ' ' - - if ($arg -is [array]) + if ($args[$i] -is [array]) { # Commas created the array so we need to reintroduce those commas - for ($i = 0; $i -lt $arg.length; $i++) + for ($j = 0; $j -lt $args[$i].length - 1; $j++) { - $toCmd += $arg[$i] - if ($i -ne ($arg.length - 1)) - { - $toCmd += ', ' - } + $newArgs += ($args[$i][$j] + ',') } + $newArgs += $args[$i][-1] } else { - $toCmd += $arg + $newArgs += $args[$i] } } # Corrected arguments are ready to pass to batch file -cmd /c $toCmd \ No newline at end of file +& $mixBatPath $newArgs diff --git a/lib/eex/lib/eex.ex b/lib/eex/lib/eex.ex index fd0095db04b..4c53460c672 100644 --- a/lib/eex/lib/eex.ex +++ b/lib/eex/lib/eex.ex @@ -1,120 +1,146 @@ defmodule EEx.SyntaxError do - defexception [:message] + defexception [:message, :file, :line, :column] + + @impl true + def message(exception) do + "#{exception.file}:#{exception.line}:#{exception.column}: #{exception.message}" + end end defmodule EEx do @moduledoc ~S""" - EEx stands for Embedded Elixir. It allows you to embed - Elixir code inside a string in a robust way: + EEx stands for Embedded Elixir. - iex> EEx.eval_string "foo <%= bar %>", [bar: "baz"] - "foo baz" + Embedded Elixir allows you to embed Elixir code inside a string + in a robust way. - ## API + iex> EEx.eval_string("foo <%= bar %>", bar: "baz") + "foo baz" - This module provides 3 main APIs for you to use: + This module provides three main APIs for you to use: - 1. Evaluate a string (`eval_string`) or a file (`eval_file`) + 1. Evaluate a string (`eval_string/3`) or a file (`eval_file/3`) directly. This is the simplest API to use but also the - slowest, since the code is evaluated and not compiled before. + slowest, since the code is evaluated at runtime and not precompiled. - 2. Define a function from a string (`function_from_string`) - or a file (`function_from_file`). This allows you to embed + 2. Define a function from a string (`function_from_string/5`) + or a file (`function_from_file/5`). This allows you to embed the template as a function inside a module which will then be compiled. This is the preferred API if you have access to the template at compilation time. - 3. Compile a string (`compile_string`) or a file (`compile_file`) + 3. Compile a string (`compile_string/2`) or a file (`compile_file/2`) into Elixir syntax tree. This is the API used by both functions above and is available to you if you want to provide your own ways of handling the compiled template. + The APIs above support several options, documented below. You may + also pass an engine which customizes how the EEx code is compiled. + ## Options - All functions in this module accepts EEx-related options. - They are: + All functions in this module, unless otherwise noted, accept EEx-related + options. They are: + + * `:file` - the file to be used in the template. Defaults to the given + file the template is read from or to `"nofile"` when compiling from a string. + + * `:line` - the line to be used as the template start. Defaults to `1`. + + * `:indentation` - (since v1.11.0) an integer added to the column after every + new line. Defaults to `0`. - * `:line` - the line to be used as the template start. Defaults to 1. - * `:file` - the file to be used in the template. Defaults to the given - file the template is read from or to "nofile" when compiling - from a string. * `:engine` - the EEx engine to be used for compilation. - ## Engine + * `:trim` - if `true`, trims whitespace left and right of quotation as + long as at least one newline is present. All subsequent newlines and + spaces are removed but one newline is retained. Defaults to `false`. - EEx has the concept of engines which allows you to modify or - transform the code extracted from the given string or file. + * `:parser_options` - (since: 1.13.0) allow customizing the parsed code + that is generated. See `Code.string_to_quoted/2` for available options. + Note that the options `:file`, `:line` and `:column` are ignored if + passed in. Defaults to `Code.get_compiler_option(:parser_options)` + (which defaults to `[]` if not set). - By default, `EEx` uses the `EEx.SmartEngine` that provides some - conveniences on top of the simple `EEx.Engine`. + ## Tags + + EEx supports multiple tags, declared below: - ### Tags + <% Elixir expression: executes code but discards output %> + <%= Elixir expression: executes code and prints result %> + <%% EEx quotation: returns the contents inside the tag as is %> + <%!-- Comments: they are discarded from source --%> - `EEx.SmartEngine` supports the following tags: + EEx supports additional tags, that may be used by some engines, + but they do not have a meaning by default: - <% Elixir expression - inline with output %> - <%= Elixir expression - replace with result %> - <%% EEx quotation - returns the contents inside %> - <%# Comments - they are discarded from source %> + <%| ... %> + <%/ ... %> - All expressions that output something to the template - **must** use the equals sign (`=`). Since everything in - Elixir is a macro, there are no exceptions for this rule. - For example, while some template languages would special- - case `if` clauses, they are treated the same in EEx and - also require `=` in order to have their result printed: + ## Engine - <%= if true do %> - It is obviously true - <% else %> - This will never appear - <% end %> + EEx has the concept of engines which allows you to modify or + transform the code extracted from the given string or file. - Notice that different engines may have different rules - for each tag. Other tags may be added in future versions. + By default, `EEx` uses the `EEx.SmartEngine` that provides some + conveniences on top of the simple `EEx.Engine`. - ### Macros + ### `EEx.SmartEngine` - `EEx.SmartEngine` also adds some macros to your template. - An example is the `@` macro which allows easy data access - in a template: + The smart engine uses EEx default rules and adds the `@` construct + for reading template assigns: - iex> EEx.eval_string "<%= @foo %>", assigns: [foo: 1] + iex> EEx.eval_string("<%= @foo %>", assigns: [foo: 1]) "1" - In other words, `<%= @foo %>` is simply translated to: + In other words, `<%= @foo %>` translates to: - <%= Dict.get assigns, :foo %> + <%= {:ok, v} = Access.fetch(assigns, :foo); v %> - The assigns extension is useful when the number of variables + The `assigns` extension is useful when the number of variables required by the template is not specified at compilation time. """ + @type line :: non_neg_integer + @type column :: non_neg_integer + @type marker :: '=' | '/' | '|' | '' + @type metadata :: %{column: column, line: line} + @type token :: + {:comment, charlist, metadata} + | {:text, charlist, metadata} + | {:expr | :start_expr | :middle_expr | :end_expr, marker, charlist, metadata} + | {:eof, metadata} + @doc """ - Generates a function definition from the string. + Generates a function definition from the given string. + + The first argument is the kind of the generated function (`:def` or `:defp`). + The `name` argument is the name that the generated function will have. + `template` is the string containing the EEx template. `args` is a list of arguments + that the generated function will accept. They will be available inside the EEx + template. - The kind (`:def` or `:defp`) must be given, the - function name, its arguments and the compilation options. + The supported `options` are described [in the module docs](#module-options). ## Examples iex> defmodule Sample do ...> require EEx - ...> EEx.function_from_string :def, :sample, "<%= a + b %>", [:a, :b] + ...> EEx.function_from_string(:def, :sample, "<%= a + b %>", [:a, :b]) ...> end iex> Sample.sample(1, 2) "3" """ - defmacro function_from_string(kind, name, source, args \\ [], options \\ []) do - quote bind_quoted: binding do - info = Keyword.merge [file: __ENV__.file, line: __ENV__.line], options - args = Enum.map args, fn arg -> {arg, [line: info[:line]], nil} end - compiled = EEx.compile_string(source, info) + defmacro function_from_string(kind, name, template, args \\ [], options \\ []) do + quote bind_quoted: binding() do + info = Keyword.merge([file: __ENV__.file, line: __ENV__.line], options) + args = Enum.map(args, fn arg -> {arg, [line: info[:line]], nil} end) + compiled = EEx.compile_string(template, info) case kind do - :def -> def(unquote(name)(unquote_splicing(args)), do: unquote(compiled)) - :defp -> defp(unquote(name)(unquote_splicing(args)), do: unquote(compiled)) + :def -> def unquote(name)(unquote_splicing(args)), do: unquote(compiled) + :defp -> defp unquote(name)(unquote_splicing(args)), do: unquote(compiled) end end end @@ -122,12 +148,17 @@ defmodule EEx do @doc """ Generates a function definition from the file contents. - The kind (`:def` or `:defp`) must be given, the - function name, its arguments and the compilation options. + The first argument is the kind of the generated function (`:def` or `:defp`). + The `name` argument is the name that the generated function will have. + `file` is the path to the EEx template file. `args` is a list of arguments + that the generated function will accept. They will be available inside the EEx + template. This function is useful in case you have templates but you want to precompile inside a module for speed. + The supported `options` are described [in the module docs](#module-options). + ## Examples # sample.eex @@ -136,77 +167,177 @@ defmodule EEx do # sample.ex defmodule Sample do require EEx - EEx.function_from_file :def, :sample, "sample.eex", [:a, :b] + EEx.function_from_file(:def, :sample, "sample.eex", [:a, :b]) end # iex - Sample.sample(1, 2) #=> "3" + Sample.sample(1, 2) + #=> "3" """ defmacro function_from_file(kind, name, file, args \\ [], options \\ []) do - quote bind_quoted: binding do - info = Keyword.merge options, [file: file, line: 1] - args = Enum.map args, fn arg -> {arg, [line: 1], nil} end + quote bind_quoted: binding() do + info = Keyword.merge([file: IO.chardata_to_string(file), line: 1], options) + args = Enum.map(args, fn arg -> {arg, [line: 1], nil} end) compiled = EEx.compile_file(file, info) @external_resource file @file file case kind do - :def -> def(unquote(name)(unquote_splicing(args)), do: unquote(compiled)) - :defp -> defp(unquote(name)(unquote_splicing(args)), do: unquote(compiled)) + :def -> def unquote(name)(unquote_splicing(args)), do: unquote(compiled) + :defp -> defp unquote(name)(unquote_splicing(args)), do: unquote(compiled) end end end @doc """ - Get a string `source` and generate a quoted expression + Gets a string `source` and generates a quoted expression that can be evaluated by Elixir or compiled to a function. + + This is useful if you want to compile a EEx template into code and inject + that code somewhere or evaluate it at runtime. + + The generated quoted code will use variables defined in the template that + will be taken from the context where the code is evaluated. If you + have a template such as `<%= a + b %>`, then the returned quoted code + will use the `a` and `b` variables in the context where it's evaluated. See + examples below. + + The supported `options` are described [in the module docs](#module-options). + + ## Examples + + iex> quoted = EEx.compile_string("<%= a + b %>") + iex> {result, _bindings} = Code.eval_quoted(quoted, a: 1, b: 2) + iex> result + "3" + """ - def compile_string(source, options \\ []) do - EEx.Compiler.compile(source, options) + @spec compile_string(String.t(), keyword) :: Macro.t() + def compile_string(source, options \\ []) when is_binary(source) and is_list(options) do + case tokenize(source, options) do + {:ok, tokens} -> + EEx.Compiler.compile(tokens, options) + + {:error, message, %{column: column, line: line}} -> + file = options[:file] || "nofile" + raise EEx.SyntaxError, file: file, line: line, column: column, message: message + end end @doc """ - Get a `filename` and generate a quoted expression + Gets a `filename` and generates a quoted expression that can be evaluated by Elixir or compiled to a function. + + This is useful if you want to compile a EEx template into code and inject + that code somewhere or evaluate it at runtime. + + The generated quoted code will use variables defined in the template that + will be taken from the context where the code is evaluated. If you + have a template such as `<%= a + b %>`, then the returned quoted code + will use the `a` and `b` variables in the context where it's evaluated. See + examples below. + + The supported `options` are described [in the module docs](#module-options). + + ## Examples + + # sample.eex + <%= a + b %> + + # In code: + quoted = EEx.compile_file("sample.eex") + {result, _bindings} = Code.eval_quoted(quoted, a: 1, b: 2) + result + #=> "3" + """ - def compile_file(filename, options \\ []) do - options = Keyword.merge options, [file: filename, line: 1] + @spec compile_file(Path.t(), keyword) :: Macro.t() + def compile_file(filename, options \\ []) when is_list(options) do + filename = IO.chardata_to_string(filename) + options = Keyword.merge([file: filename, line: 1], options) compile_string(File.read!(filename), options) end @doc """ - Get a string `source` and evaluate the values using the `bindings`. + Gets a string `source` and evaluate the values using the `bindings`. + + The supported `options` are described [in the module docs](#module-options). ## Examples - iex> EEx.eval_string "foo <%= bar %>", [bar: "baz"] + iex> EEx.eval_string("foo <%= bar %>", bar: "baz") "foo baz" """ - def eval_string(source, bindings \\ [], options \\ []) do + @spec eval_string(String.t(), keyword, keyword) :: String.t() + def eval_string(source, bindings \\ [], options \\ []) + when is_binary(source) and is_list(bindings) and is_list(options) do compiled = compile_string(source, options) do_eval(compiled, bindings, options) end @doc """ - Get a `filename` and evaluate the values using the `bindings`. + Gets a `filename` and evaluate the values using the `bindings`. + + The supported `options` are described [in the module docs](#module-options). ## Examples - # sample.ex + # sample.eex foo <%= bar %> - # iex - EEx.eval_file "sample.ex", [bar: "baz"] #=> "foo baz" + # IEx + EEx.eval_file("sample.eex", bar: "baz") + #=> "foo baz" """ - def eval_file(filename, bindings \\ [], options \\ []) do - options = Keyword.put options, :file, filename + @spec eval_file(Path.t(), keyword, keyword) :: String.t() + def eval_file(filename, bindings \\ [], options \\ []) + when is_list(bindings) and is_list(options) do + filename = IO.chardata_to_string(filename) + options = Keyword.put_new(options, :file, filename) compiled = compile_file(filename, options) do_eval(compiled, bindings, options) end + @doc """ + Tokenize the given contents according to the given options. + + ## Options + + * `:line` - An integer to start as line. Default is 1. + * `:column` - An integer to start as column. Default is 1. + * `:indentation` - An integer that indicates the indentation. Default is 0. + * `:trim` - Tells the tokenizer to either trim the content or not. Default is false. + * `:file` - Can be either a file or a string "nofile". + + ## Examples + + iex> EEx.tokenize('foo', line: 1, column: 1) + {:ok, [{:text, 'foo', %{column: 1, line: 1}}, {:eof, %{column: 4, line: 1}}]} + + ## Result + + It returns `{:ok, [token]}` where a token is one of: + + * `{:text, content, %{column: column, line: line}}` + * `{:expr, marker, content, %{column: column, line: line}}` + * `{:start_expr, marker, content, %{column: column, line: line}}` + * `{:middle_expr, marker, content, %{column: column, line: line}}` + * `{:end_expr, marker, content, %{column: column, line: line}}` + * `{:eof, %{column: column, line: line}}` + + Or `{:error, message, %{column: column, line: line}}` in case of errors. + Note new tokens may be added in the future. + """ + @doc since: "1.14.0" + @spec tokenize(IO.chardata(), opts :: keyword) :: + {:ok, [token()]} | {:error, String.t(), metadata()} + def tokenize(contents, opts \\ []) do + EEx.Compiler.tokenize(contents, opts) + end + ### Helpers defp do_eval(compiled, bindings, options) do diff --git a/lib/eex/lib/eex/compiler.ex b/lib/eex/lib/eex/compiler.ex index c34ef566c60..cd95fbe3ec5 100644 --- a/lib/eex/lib/eex/compiler.ex +++ b/lib/eex/lib/eex/compiler.ex @@ -3,64 +3,415 @@ defmodule EEx.Compiler do # When changing this setting, don't forget to update the docs for EEx @default_engine EEx.SmartEngine + @h_spaces [?\s, ?\t] + @all_spaces [?\s, ?\t, ?\n, ?\r] + + @doc """ + Tokenize EEx contents. + """ + def tokenize(contents, opts) when is_binary(contents) do + tokenize(String.to_charlist(contents), opts) + end + + def tokenize(contents, opts) when is_list(contents) do + file = opts[:file] || "nofile" + line = opts[:line] || 1 + trim = opts[:trim] || false + indentation = opts[:indentation] || 0 + column = indentation + (opts[:column] || 1) + + state = %{trim: trim, indentation: indentation, file: file} + + {contents, line, column} = + (trim && trim_init(contents, line, column, state)) || {contents, line, column} + + tokenize(contents, line, column, state, [{line, column}], []) + end + + defp tokenize('<%%' ++ t, line, column, state, buffer, acc) do + tokenize(t, line, column + 3, state, [?%, ?< | buffer], acc) + end + + defp tokenize('<%!--' ++ t, line, column, state, buffer, acc) do + case comment(t, line, column + 5, state, []) do + {:error, line, column, message} -> + {:error, message, %{line: line, column: column}} + + {:ok, new_line, new_column, rest, comments} -> + token = {:comment, Enum.reverse(comments), %{line: line, column: column}} + trim_and_tokenize(rest, new_line, new_column, state, buffer, acc, &[token | &1]) + end + end + + # TODO: Deprecate this on Elixir v1.18 + defp tokenize('<%#' ++ t, line, column, state, buffer, acc) do + case expr(t, line, column + 3, state, []) do + {:error, line, column, message} -> + {:error, message, %{line: line, column: column}} + + {:ok, _, new_line, new_column, rest} -> + trim_and_tokenize(rest, new_line, new_column, state, buffer, acc, & &1) + end + end + + defp tokenize('<%' ++ t, line, column, state, buffer, acc) do + {marker, t} = retrieve_marker(t) + + case expr(t, line, column + 2 + length(marker), state, []) do + {:error, line, column, message} -> + {:error, message, %{line: line, column: column}} + + {:ok, expr, new_line, new_column, rest} -> + {key, expr} = + case :elixir_tokenizer.tokenize(expr, 1, file: "eex", check_terminators: false) do + {:ok, _line, _column, warnings, tokens} -> + Enum.each(Enum.reverse(warnings), fn {location, file, msg} -> + :elixir_errors.erl_warn(location, file, msg) + end) + + token_key(tokens, expr) + + {:error, _, _, _, _} -> + {:expr, expr} + end + + marker = + if key in [:middle_expr, :end_expr] and marker != '' do + message = + "unexpected beginning of EEx tag \"<%#{marker}\" on \"<%#{marker}#{expr}%>\", " <> + "please remove \"#{marker}\"" + + :elixir_errors.erl_warn({line, column}, state.file, message) + '' + else + marker + end + + token = {key, marker, expr, %{line: line, column: column}} + trim_and_tokenize(rest, new_line, new_column, state, buffer, acc, &[token | &1]) + end + end + + defp tokenize('\n' ++ t, line, _column, state, buffer, acc) do + tokenize(t, line + 1, state.indentation + 1, state, [?\n | buffer], acc) + end + + defp tokenize([h | t], line, column, state, buffer, acc) do + tokenize(t, line, column + 1, state, [h | buffer], acc) + end + + defp tokenize([], line, column, _state, buffer, acc) do + eof = {:eof, %{line: line, column: column}} + {:ok, Enum.reverse([eof | tokenize_text(buffer, acc)])} + end + + defp trim_and_tokenize(rest, line, column, state, buffer, acc, fun) do + {rest, line, column, buffer} = trim_if_needed(rest, line, column, state, buffer) + + acc = tokenize_text(buffer, acc) + tokenize(rest, line, column, state, [{line, column}], fun.(acc)) + end + + # Retrieve marker for <% + + defp retrieve_marker([marker | t]) when marker in [?=, ?/, ?|] do + {[marker], t} + end + + defp retrieve_marker(t) do + {'', t} + end + + # Tokenize a multi-line comment until we find --%> + + defp comment([?-, ?-, ?%, ?> | t], line, column, _state, buffer) do + {:ok, line, column + 4, t, buffer} + end + + defp comment('\n' ++ t, line, _column, state, buffer) do + comment(t, line + 1, state.indentation + 1, state, '\n' ++ buffer) + end + + defp comment([head | t], line, column, state, buffer) do + comment(t, line, column + 1, state, [head | buffer]) + end + + defp comment([], line, column, _state, _buffer) do + {:error, line, column, "missing token '--%>'"} + end + + # Tokenize an expression until we find %> + + defp expr([?%, ?> | t], line, column, _state, buffer) do + {:ok, Enum.reverse(buffer), line, column + 2, t} + end + + defp expr('\n' ++ t, line, _column, state, buffer) do + expr(t, line + 1, state.indentation + 1, state, [?\n | buffer]) + end + + defp expr([h | t], line, column, state, buffer) do + expr(t, line, column + 1, state, [h | buffer]) + end + + defp expr([], line, column, _state, _buffer) do + {:error, line, column, "missing token '%>'"} + end + + # Receives tokens and check if it is a start, middle or an end token. + defp token_key(tokens, expr) do + case {tokens, tokens |> Enum.reverse() |> drop_eol()} do + {[{:end, _} | _], [{:do, _} | _]} -> + {:middle_expr, expr} + + {_, [{:do, _} | _]} -> + {:start_expr, maybe_append_space(expr)} + + {_, [{:block_identifier, _, _} | _]} -> + {:middle_expr, maybe_append_space(expr)} + + {[{:end, _} | _], [{:stab_op, _, _} | _]} -> + {:middle_expr, expr} + + {_, [{:stab_op, _, _} | reverse_tokens]} -> + fn_index = Enum.find_index(reverse_tokens, &match?({:fn, _}, &1)) || :infinity + end_index = Enum.find_index(reverse_tokens, &match?({:end, _}, &1)) || :infinity + + if end_index > fn_index do + {:start_expr, expr} + else + {:middle_expr, expr} + end + + {tokens, _} -> + case Enum.drop_while(tokens, &closing_bracket?/1) do + [{:end, _} | _] -> {:end_expr, expr} + _ -> {:expr, expr} + end + end + end + + defp drop_eol([{:eol, _} | rest]), do: drop_eol(rest) + defp drop_eol(rest), do: rest + + defp maybe_append_space([?\s]), do: [?\s] + defp maybe_append_space([h]), do: [h, ?\s] + defp maybe_append_space([h | t]), do: [h | maybe_append_space(t)] + + defp closing_bracket?({closing, _}) when closing in ~w"( [ {"a, do: true + defp closing_bracket?(_), do: false + + # Tokenize the buffered text by appending + # it to the given accumulator. + + defp tokenize_text([{_line, _column}], acc) do + acc + end + + defp tokenize_text(buffer, acc) do + [{line, column} | buffer] = Enum.reverse(buffer) + [{:text, buffer, %{line: line, column: column}} | acc] + end + + ## Trim + + defp trim_if_needed(rest, line, column, state, buffer) do + if state.trim do + buffer = trim_left(buffer, 0) + {rest, line, column} = trim_right(rest, line, column, 0, state) + {rest, line, column, buffer} + else + {rest, line, column, buffer} + end + end + + defp trim_init([h | t], line, column, state) when h in @h_spaces, + do: trim_init(t, line, column + 1, state) + + defp trim_init([?\r, ?\n | t], line, _column, state), + do: trim_init(t, line + 1, state.indentation + 1, state) + + defp trim_init([?\n | t], line, _column, state), + do: trim_init(t, line + 1, state.indentation + 1, state) + + defp trim_init([?<, ?% | _] = rest, line, column, _state), + do: {rest, line, column} + + defp trim_init(_, _, _, _), do: false + + defp trim_left(buffer, count) do + case trim_whitespace(buffer, 0) do + {[?\n, ?\r | rest], _} -> trim_left(rest, count + 1) + {[?\n | rest], _} -> trim_left(rest, count + 1) + _ when count > 0 -> [?\n | buffer] + _ -> buffer + end + end + + defp trim_right(rest, line, column, last_column, state) do + case trim_whitespace(rest, column) do + {[?\r, ?\n | rest], column} -> + trim_right(rest, line + 1, state.indentation + 1, column + 1, state) + + {[?\n | rest], column} -> + trim_right(rest, line + 1, state.indentation + 1, column, state) + + {[], column} -> + {[], line, column} + + _ when last_column > 0 -> + {[?\n | rest], line - 1, last_column} + + _ -> + {rest, line, column} + end + end + + defp trim_whitespace([h | t], column) when h in @h_spaces, do: trim_whitespace(t, column + 1) + defp trim_whitespace(list, column), do: {list, column} @doc """ This is the compilation entry point. It glues the tokenizer and the engine together by handling the tokens and invoking the engine every time a full expression or text is received. """ - def compile(source, opts) do - file = opts[:file] || "nofile" - line = opts[:line] || 1 - tokens = EEx.Tokenizer.tokenize(source, line) - state = %{engine: opts[:engine] || @default_engine, - file: file, line: line, quoted: [], start_line: nil} - generate_buffer(tokens, "", [], state) + @spec compile([EEx.token()], keyword) :: Macro.t() + def compile(tokens, opts) do + file = opts[:file] || "nofile" + line = opts[:line] || 1 + parser_options = opts[:parser_options] || Code.get_compiler_option(:parser_options) + engine = opts[:engine] || @default_engine + + state = %{ + engine: engine, + file: file, + line: line, + quoted: [], + start_line: nil, + start_column: nil, + parser_options: parser_options + } + + init = state.engine.init(opts) + generate_buffer(tokens, init, [], state) end - # Generates the buffers by handling each expression from the tokenizer + # Ignore tokens related to comment. + defp generate_buffer([{:comment, _chars, _meta} | rest], buffer, scope, state) do + generate_buffer(rest, buffer, scope, state) + end + + # Generates the buffers by handling each expression from the tokenizer. + # It returns Macro.t/0 or it raises. + + defp generate_buffer([{:text, chars, meta} | rest], buffer, scope, state) do + buffer = + if function_exported?(state.engine, :handle_text, 3) do + meta = [line: meta.line, column: meta.column] + state.engine.handle_text(buffer, meta, IO.chardata_to_string(chars)) + else + # TODO: Deprecate this branch on Elixir v1.18. + # We should most likely move this check to init to emit the deprecation once. + state.engine.handle_text(buffer, IO.chardata_to_string(chars)) + end - defp generate_buffer([{:text, chars}|t], buffer, scope, state) do - buffer = state.engine.handle_text(buffer, IO.chardata_to_string(chars)) - generate_buffer(t, buffer, scope, state) + generate_buffer(rest, buffer, scope, state) end - defp generate_buffer([{:expr, line, mark, chars}|t], buffer, scope, state) do - expr = Code.string_to_quoted!(chars, [line: line, file: state.file]) - buffer = state.engine.handle_expr(buffer, mark, expr) - generate_buffer(t, buffer, scope, state) + defp generate_buffer([{:expr, mark, chars, meta} | rest], buffer, scope, state) do + options = + [file: state.file, line: meta.line, column: column(meta.column, mark)] ++ + state.parser_options + + expr = Code.string_to_quoted!(chars, options) + buffer = state.engine.handle_expr(buffer, IO.chardata_to_string(mark), expr) + generate_buffer(rest, buffer, scope, state) + end + + defp generate_buffer( + [{:start_expr, mark, chars, meta} | rest], + buffer, + scope, + state + ) do + if mark == '' do + message = + "the contents of this expression won't be output unless the EEx block starts with \"<%=\"" + + :elixir_errors.erl_warn({meta.line, meta.column}, state.file, message) + end + + {rest, line, contents} = look_ahead_middle(rest, meta.line, chars) || {rest, meta.line, chars} + + {contents, rest} = + generate_buffer( + rest, + state.engine.handle_begin(buffer), + [contents | scope], + %{ + state + | quoted: [], + line: line, + start_line: meta.line, + start_column: column(meta.column, mark) + } + ) + + buffer = state.engine.handle_expr(buffer, IO.chardata_to_string(mark), contents) + generate_buffer(rest, buffer, scope, state) end - defp generate_buffer([{:start_expr, start_line, mark, chars}|t], buffer, scope, state) do - {contents, line, t} = look_ahead_text(t, start_line, chars) - {contents, t} = generate_buffer(t, "", [contents|scope], - %{state | quoted: [], line: line, start_line: start_line}) - buffer = state.engine.handle_expr(buffer, mark, contents) - generate_buffer(t, buffer, scope, state) + defp generate_buffer( + [{:middle_expr, '', chars, meta} | rest], + buffer, + [current | scope], + state + ) do + {wrapped, state} = wrap_expr(current, meta.line, buffer, chars, state) + state = %{state | line: meta.line} + generate_buffer(rest, state.engine.handle_begin(buffer), [wrapped | scope], state) end - defp generate_buffer([{:middle_expr, line, _, chars}|t], buffer, [current|scope], state) do - {wrapped, state} = wrap_expr(current, line, buffer, chars, state) - generate_buffer(t, "", [wrapped|scope], %{state | line: line}) + defp generate_buffer([{:middle_expr, _, chars, meta} | _], _buffer, [], state) do + raise EEx.SyntaxError, + message: "unexpected middle of expression <%#{chars}%>", + file: state.file, + line: meta.line, + column: meta.column end - defp generate_buffer([{:end_expr, line, _, chars}|t], buffer, [current|_], state) do - {wrapped, state} = wrap_expr(current, line, buffer, chars, state) - tuples = Code.string_to_quoted!(wrapped, [line: state.start_line, file: state.file]) + defp generate_buffer( + [{:end_expr, '', chars, meta} | rest], + buffer, + [current | _], + state + ) do + {wrapped, state} = wrap_expr(current, meta.line, buffer, chars, state) + column = state.start_column + options = [file: state.file, line: state.start_line, column: column] ++ state.parser_options + tuples = Code.string_to_quoted!(wrapped, options) buffer = insert_quoted(tuples, state.quoted) - {buffer, t} + {buffer, rest} end - defp generate_buffer([{:end_expr, line, _, chars}|_], _buffer, [], _state) do - raise EEx.SyntaxError, message: "unexpected token: #{inspect chars} at line #{inspect line}" + defp generate_buffer([{:end_expr, _, chars, meta} | _], _buffer, [], state) do + raise EEx.SyntaxError, + message: "unexpected end of expression <%#{chars}%>", + file: state.file, + line: meta.line, + column: meta.column end - defp generate_buffer([], buffer, [], state) do + defp generate_buffer([{:eof, _meta}], buffer, [], state) do state.engine.handle_body(buffer) end - defp generate_buffer([], _buffer, _scope, _state) do - raise EEx.SyntaxError, message: "unexpected end of string. expecting a closing <% end %>." + defp generate_buffer([{:eof, meta}], _buffer, _scope, state) do + raise EEx.SyntaxError, + message: "unexpected end of string, expected a closing '<% end %>'", + file: state.file, + line: meta.line, + column: meta.column end # Creates a placeholder and wrap it inside the expression block @@ -68,33 +419,42 @@ defmodule EEx.Compiler do defp wrap_expr(current, line, buffer, chars, state) do new_lines = List.duplicate(?\n, line - state.line) key = length(state.quoted) - placeholder = '__EEX__(' ++ Integer.to_char_list(key) ++ ');' - {current ++ placeholder ++ new_lines ++ chars, - %{state | quoted: [{key, buffer}|state.quoted]}} + placeholder = '__EEX__(' ++ Integer.to_charlist(key) ++ ');' + count = current ++ placeholder ++ new_lines ++ chars + new_state = %{state | quoted: [{key, state.engine.handle_end(buffer)} | state.quoted]} + + {count, new_state} end - # Look text ahead on expressions + # Look middle expressions that immediately follow a start_expr - defp look_ahead_text([{:text, text}, {:middle_expr, line, _, chars}|t]=list, start, contents) do + defp look_ahead_middle([{:comment, _comment, _meta} | rest], start, contents), + do: look_ahead_middle(rest, start, contents) + + defp look_ahead_middle([{:text, text, _meta} | rest], start, contents) do if only_spaces?(text) do - {contents ++ text ++ chars, line, t} + look_ahead_middle(rest, start, contents ++ text) else - {contents, start, list} + nil end end - defp look_ahead_text(t, start, contents) do - {contents, start, t} + defp look_ahead_middle([{:middle_expr, _, chars, meta} | rest], _start, contents) do + {rest, meta.line, contents ++ chars} + end + + defp look_ahead_middle(_tokens, _start, _contents) do + nil end defp only_spaces?(chars) do - Enum.all?(chars, &(&1 in [?\s, ?\t, ?\r, ?\n])) + Enum.all?(chars, &(&1 in @all_spaces)) end # Changes placeholder to real expression defp insert_quoted({:__EEX__, _, [key]}, quoted) do - {^key, value} = List.keyfind quoted, key, 0 + {^key, value} = List.keyfind(quoted, key, 0) value end @@ -107,10 +467,15 @@ defmodule EEx.Compiler do end defp insert_quoted(list, quoted) when is_list(list) do - Enum.map list, &insert_quoted(&1, quoted) + Enum.map(list, &insert_quoted(&1, quoted)) end defp insert_quoted(other, _quoted) do other end + + defp column(column, mark) do + # length('<%') == 2 + column + 2 + length(mark) + end end diff --git a/lib/eex/lib/eex/engine.ex b/lib/eex/lib/eex/engine.ex index 257f2f46d18..480d9f88de0 100644 --- a/lib/eex/lib/eex/engine.ex +++ b/lib/eex/lib/eex/engine.ex @@ -2,111 +2,220 @@ defmodule EEx.Engine do @moduledoc ~S""" Basic EEx engine that ships with Elixir. - An engine needs to implement three functions: + An engine needs to implement all callbacks below. - * `handle_body(quoted)` - receives the final built quoted - expression, should do final post-processing and return a - quoted expression. + This module also ships with a default engine implementation + you can delegate to. See `EEx.SmartEngine` as an example. + """ + + @type state :: term + + @doc """ + Called at the beginning of every template. + + It must return the initial state. + """ + @callback init(opts :: keyword) :: state + + @doc """ + Called at the end of every template. + + It must return Elixir's quoted expressions for the template. + """ + @callback handle_body(state) :: Macro.t() + + @doc """ + Called for the text/static parts of a template. + + It must return the updated state. + """ + @callback handle_text(state, [line: pos_integer, column: pos_integer], text :: String.t()) :: + state - * `handle_text(buffer, text)` - it receives the buffer, - the text and must return a new quoted expression. + @doc """ + Called for the dynamic/code parts of a template. - * `handle_expr(buffer, marker, expr)` - it receives the buffer, - the marker, the expr and must return a new quoted expression. + The marker is what follows exactly after `<%`. For example, + `<% foo %>` has an empty marker, but `<%= foo %>` has `"="` + as marker. The allowed markers so far are: - The marker is what follows exactly after `<%`. For example, - `<% foo %>` has an empty marker, but `<%= foo %>` has `"="` - as marker. The allowed markers so far are: `""` and `"="`. + * `""` + * `"="` + * `"/"` + * `"|"` - Read `handle_expr/3` below for more information about the markers - implemented by default by this engine. + Markers `"/"` and `"|"` are only for use in custom EEx engines + and are not implemented by default. Using them without an + appropriate implementation raises `EEx.SyntaxError`. - `EEx.Engine` can be used directly if one desires to use the - default implementations for the functions above. + It must return the updated state. """ + @callback handle_expr(state, marker :: String.t(), expr :: Macro.t()) :: state + + @doc """ + Invoked at the beginning of every nesting. - use Behaviour + It must return a new state that is used only inside the nesting. + Once the nesting terminates, the current `state` is resumed. + """ + @callback handle_begin(state) :: state - defcallback handle_body(Macro.t) :: Macro.t - defcallback handle_text(Macro.t, binary) :: Macro.t - defcallback handle_expr(Macro.t, binary, Macro.t) :: Macro.t + @doc """ + Invokes at the end of a nesting. + + It must return Elixir's quoted expressions for the nesting. + """ + @callback handle_end(state) :: Macro.t() @doc false + @deprecated "Use explicit delegation to EEx.Engine instead" defmacro __using__(_) do quote do @behaviour EEx.Engine - def handle_body(body) do - EEx.Engine.handle_body(body) + def init(opts) do + EEx.Engine.init(opts) end - def handle_text(buffer, text) do - EEx.Engine.handle_text(buffer, text) + def handle_body(state) do + EEx.Engine.handle_body(state) end - def handle_expr(buffer, mark, expr) do - EEx.Engine.handle_expr(buffer, mark, expr) + def handle_begin(state) do + EEx.Engine.handle_begin(state) end - defoverridable [handle_body: 1, handle_expr: 3, handle_text: 2] + def handle_end(state) do + EEx.Engine.handle_end(state) + end + + def handle_text(state, text) do + EEx.Engine.handle_text(state, [], text) + end + + def handle_expr(state, marker, expr) do + EEx.Engine.handle_expr(state, marker, expr) + end + + defoverridable EEx.Engine end end @doc """ Handles assigns in quoted expressions. + A warning will be printed on missing assigns. + Future versions will raise. + This can be added to any custom engine by invoking - `handle_assign/3` with `Macro.prewalk/1`: + `handle_assign/1` with `Macro.prewalk/2`: - def handle_expr(buffer, token, expr) do + def handle_expr(state, token, expr) do expr = Macro.prewalk(expr, &EEx.Engine.handle_assign/1) - EEx.Engine.handle_expr(buffer, token, expr) + super(state, token, expr) end """ - def handle_assign({:@, line, [{name, _, atom}]}) when is_atom(name) and is_atom(atom) do - quote line: line, do: Dict.get(var!(assigns), unquote(name)) + @spec handle_assign(Macro.t()) :: Macro.t() + def handle_assign({:@, meta, [{name, _, atom}]}) when is_atom(name) and is_atom(atom) do + line = meta[:line] || 0 + quote(line: line, do: EEx.Engine.fetch_assign!(var!(assigns), unquote(name))) end def handle_assign(arg) do arg end - @doc """ - The default implementation implementation simply returns the - given expression. - """ - def handle_body(quoted) do - quoted + @doc false + # TODO: Raise on v2.0 + @spec fetch_assign!(Access.t(), Access.key()) :: term | nil + def fetch_assign!(assigns, key) do + case Access.fetch(assigns, key) do + {:ok, val} -> + val + + :error -> + keys = Enum.map(assigns, &elem(&1, 0)) + + IO.warn( + "assign @#{key} not available in EEx template. " <> + "Please ensure all assigns are given as options. " <> + "Available assigns: #{inspect(keys)}" + ) + + nil + end end - @doc """ - The default implementation simply concatenates text to the buffer. - """ - def handle_text(buffer, text) do - quote do: unquote(buffer) <> unquote(text) + @doc "Default implementation for `c:init/1`." + def init(_opts) do + %{ + binary: [], + dynamic: [], + vars_count: 0 + } end - @doc """ - Implements expressions according to the markers. + @doc "Default implementation for `c:handle_begin/1`." + def handle_begin(state) do + check_state!(state) + %{state | binary: [], dynamic: []} + end - <% Elixir expression - inline with output %> - <%= Elixir expression - replace with result %> + @doc "Default implementation for `c:handle_end/1`." + def handle_end(quoted) do + handle_body(quoted) + end - All other markers are not implemented by this engine. - """ - def handle_expr(buffer, "=", expr) do - quote do - tmp = unquote(buffer) - tmp <> to_string(unquote(expr)) - end + @doc "Default implementation for `c:handle_body/1`." + def handle_body(state) do + check_state!(state) + %{binary: binary, dynamic: dynamic} = state + binary = {:<<>>, [], Enum.reverse(binary)} + dynamic = [binary | dynamic] + {:__block__, [], Enum.reverse(dynamic)} end - def handle_expr(buffer, "", expr) do - quote do - tmp = unquote(buffer) - unquote(expr) - tmp - end + @doc "Default implementation for `c:handle_text/3`." + def handle_text(state, _meta, text) do + check_state!(state) + %{binary: binary} = state + %{state | binary: [text | binary]} + end + + @doc "Default implementation for `c:handle_expr/3`." + def handle_expr(state, "=", ast) do + check_state!(state) + %{binary: binary, dynamic: dynamic, vars_count: vars_count} = state + var = Macro.var(:"arg#{vars_count}", __MODULE__) + + ast = + quote do + unquote(var) = String.Chars.to_string(unquote(ast)) + end + + segment = + quote do + unquote(var) :: binary + end + + %{state | dynamic: [ast | dynamic], binary: [segment | binary], vars_count: vars_count + 1} + end + + def handle_expr(state, "", ast) do + %{dynamic: dynamic} = state + %{state | dynamic: [ast | dynamic]} + end + + def handle_expr(_state, marker, _ast) when marker in ["/", "|"] do + raise EEx.SyntaxError, + "unsupported EEx syntax <%#{marker} %> (the syntax is valid but not supported by the current EEx engine)" + end + + defp check_state!(%{binary: _, dynamic: _, vars_count: _}), do: :ok + + defp check_state!(state) do + raise "unexpected EEx.Engine state: #{inspect(state)}. " <> + "This typically means a bug or an outdated EEx.Engine or tool" end end diff --git a/lib/eex/lib/eex/smart_engine.ex b/lib/eex/lib/eex/smart_engine.ex index c59db81596f..0d3be7d90d2 100644 --- a/lib/eex/lib/eex/smart_engine.ex +++ b/lib/eex/lib/eex/smart_engine.ex @@ -1,63 +1,3 @@ -defmodule EEx.TransformerEngine do - @moduledoc false - - @doc false - defmacro __using__(_) do - quote do - @behaviour EEx.Engine - - def handle_body(body) do - EEx.Engine.handle_body(body) - end - - def handle_text(buffer, text) do - EEx.Engine.handle_text(buffer, text) - end - - def handle_expr(buffer, mark, expr) do - EEx.Engine.handle_expr(buffer, mark, transform(expr)) - end - - defp transform({a, b, c}) do - {transform(a), b, transform(c)} - end - - defp transform({a, b}) do - {transform(a), transform(b)} - end - - defp transform(list) when is_list(list) do - for i <- list, do: transform(i) - end - - defp transform(other) do - other - end - - defoverridable [transform: 1, handle_body: 1, handle_expr: 3, handle_text: 2] - end - end -end - -defmodule EEx.AssignsEngine do - @moduledoc false - - @doc false - defmacro __using__(_) do - quote unquote: false do - defp transform({:@, line, [{name, _, atom}]}) when is_atom(name) and is_atom(atom) do - quote do: Dict.get(var!(assigns), unquote(name)) - end - - defp transform(arg) do - super(arg) - end - - defoverridable [transform: 1] - end - end -end - defmodule EEx.SmartEngine do @moduledoc """ The default engine used by EEx. @@ -71,9 +11,9 @@ defmodule EEx.SmartEngine do "1" In the example above, we can access the value `foo` under - the binding `assigns` using `@foo`. This is useful when - a template, after compiled, may receive different assigns - and the developer don't want to recompile it for each + the binding `assigns` using `@foo`. This is useful because + a template, after being compiled, can receive different + assigns and would not require recompilation for each variable set. Assigns can also be used when compiled to a function: @@ -84,18 +24,35 @@ defmodule EEx.SmartEngine do # sample.ex defmodule Sample do require EEx - EEx.function_from_file :def, :sample, "sample.eex", [:assigns] + EEx.function_from_file(:def, :sample, "sample.eex", [:assigns]) end # iex - Sample.sample(a: 1, b: 2) #=> "3" + Sample.sample(a: 1, b: 2) + #=> "3" """ - use EEx.Engine + @behaviour EEx.Engine + + @impl true + defdelegate init(opts), to: EEx.Engine + + @impl true + defdelegate handle_body(state), to: EEx.Engine + + @impl true + defdelegate handle_begin(state), to: EEx.Engine + + @impl true + defdelegate handle_end(state), to: EEx.Engine + + @impl true + defdelegate handle_text(state, meta, text), to: EEx.Engine - def handle_expr(buffer, mark, expr) do + @impl true + def handle_expr(state, marker, expr) do expr = Macro.prewalk(expr, &EEx.Engine.handle_assign/1) - super(buffer, mark, expr) + EEx.Engine.handle_expr(state, marker, expr) end end diff --git a/lib/eex/lib/eex/tokenizer.ex b/lib/eex/lib/eex/tokenizer.ex deleted file mode 100644 index 44d5b681eed..00000000000 --- a/lib/eex/lib/eex/tokenizer.ex +++ /dev/null @@ -1,161 +0,0 @@ -defmodule EEx.Tokenizer do - @moduledoc false - - @doc """ - Tokenizes the given char list or binary. - It returns 4 different types of tokens as result: - - * `{:text, contents}` - * `{:expr, line, marker, contents}` - * `{:start_expr, line, marker, contents}` - * `{:middle_expr, line, marker, contents}` - * `{:end_expr, line, marker, contents}` - - """ - def tokenize(bin, line) when is_binary(bin) do - tokenize(String.to_char_list(bin), line) - end - - def tokenize(list, line) do - Enum.reverse(tokenize(list, line, [], [])) - end - - defp tokenize('<%%' ++ t, line, buffer, acc) do - {buffer, new_line, rest} = tokenize_expr t, line, [?%, ?<|buffer] - tokenize rest, new_line, [?>, ?%|buffer], acc - end - - defp tokenize('<%#' ++ t, line, buffer, acc) do - {_, new_line, rest} = tokenize_expr t, line, [] - tokenize rest, new_line, buffer, acc - end - - defp tokenize('<%' ++ t, line, buffer, acc) do - {marker, t} = retrieve_marker(t) - {expr, new_line, rest} = tokenize_expr t, line, [] - - token = token_name(expr) - acc = tokenize_text(buffer, acc) - final = {token, line, marker, Enum.reverse(expr)} - tokenize rest, new_line, [], [final | acc] - end - - defp tokenize('\n' ++ t, line, buffer, acc) do - tokenize t, line + 1, [?\n|buffer], acc - end - - defp tokenize([h|t], line, buffer, acc) do - tokenize t, line, [h|buffer], acc - end - - defp tokenize([], _line, buffer, acc) do - tokenize_text(buffer, acc) - end - - # Retrieve marker for <% - - defp retrieve_marker('=' ++ t) do - {"=", t} - end - - defp retrieve_marker(t) do - {"", t} - end - - # Tokenize an expression until we find %> - - defp tokenize_expr([?%, ?>|t], line, buffer) do - {buffer, line, t} - end - - defp tokenize_expr('\n' ++ t, line, buffer) do - tokenize_expr t, line + 1, [?\n|buffer] - end - - defp tokenize_expr([h|t], line, buffer) do - tokenize_expr t, line, [h|buffer] - end - - defp tokenize_expr([], _line, _buffer) do - raise EEx.SyntaxError, message: "missing token: %>" - end - - # Receive an expression content and check - # if it is a start, middle or an end token. - # - # Start tokens finish with `do` and `fn ->` - # Middle tokens are marked with `->` or keywords - # End tokens contain only the end word - - defp token_name([h|t]) when h in [?\s, ?\t] do - token_name(t) - end - - defp token_name('od' ++ [h|_]) when h in [?\s, ?\t, ?)] do - :start_expr - end - - defp token_name('>-' ++ rest) do - rest = Enum.reverse(rest) - - # Tokenize the remaining passing check_terminators as - # false, which relax the tokenizer to not error on - # unmatched pairs. Then, we check if there is a "fn" - # token and, if so, it is not followed by an "end" - # token. If this is the case, we are on a start expr. - case :elixir_tokenizer.tokenize(rest, 1, file: "eex", check_terminators: false) do - {:ok, _line, tokens} -> - tokens = Enum.reverse(tokens) - fn_index = fn_index(tokens) - - if fn_index && end_index(tokens) > fn_index do - :start_expr - else - :middle_expr - end - _error -> - :middle_expr - end - end - - defp token_name('esle' ++ t), do: check_spaces(t, :middle_expr) - defp token_name('retfa' ++ t), do: check_spaces(t, :middle_expr) - defp token_name('hctac' ++ t), do: check_spaces(t, :middle_expr) - defp token_name('eucser' ++ t), do: check_spaces(t, :middle_expr) - defp token_name('dne' ++ t), do: check_spaces(t, :end_expr) - - defp token_name(_) do - :expr - end - - defp fn_index(tokens) do - Enum.find_index tokens, fn - {:fn_paren, _} -> true - {:fn, _} -> true - _ -> false - end - end - - defp end_index(tokens) do - Enum.find_index(tokens, &match?({:end, _}, &1)) || :infinity - end - - defp check_spaces(string, token) do - if Enum.all?(string, &(&1 in [?\s, ?\t])) do - token - else - :expr - end - end - - # Tokenize the buffered text by appending - # it to the given accumulator. - - defp tokenize_text([], acc) do - acc - end - - defp tokenize_text(buffer, acc) do - [{:text, Enum.reverse(buffer)} | acc] - end -end diff --git a/lib/eex/mix.exs b/lib/eex/mix.exs index 0a2877473c5..259b1707036 100644 --- a/lib/eex/mix.exs +++ b/lib/eex/mix.exs @@ -1,9 +1,11 @@ -defmodule EEx.Mixfile do +defmodule EEx.MixProject do use Mix.Project def project do - [app: :eex, - version: System.version, - build_per_environment: false] + [ + app: :eex, + version: System.version(), + build_per_environment: false + ] end end diff --git a/lib/eex/test/eex/smart_engine_test.exs b/lib/eex/test/eex/smart_engine_test.exs index 1003da8da15..f7a18323e50 100644 --- a/lib/eex/test/eex/smart_engine_test.exs +++ b/lib/eex/test/eex/smart_engine_test.exs @@ -1,26 +1,59 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule EEx.SmartEngineTest do use ExUnit.Case, async: true test "evaluates simple string" do - assert_eval "foo bar", "foo bar" + assert_eval("foo bar", "foo bar") end test "evaluates with assigns as keywords" do - assert_eval "1", "<%= @foo %>", assigns: [foo: 1] + assert_eval("1", "<%= @foo %>", assigns: [foo: 1]) end test "evaluates with assigns as a map" do - assert_eval "1", "<%= @foo %>", assigns: %{foo: 1} + assert_eval("1", "<%= @foo %>", assigns: %{foo: 1}) + end + + test "error with missing assigns" do + stderr = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert_eval("", "<%= @foo %>", assigns: %{}) + end) + + assert stderr =~ "assign @foo not available in EEx template" end test "evaluates with loops" do - assert_eval "1\n2\n3\n", "<%= for x <- [1, 2, 3] do %><%= x %>\n<% end %>" + assert_eval("1\n2\n3\n", "<%= for x <- [1, 2, 3] do %><%= x %>\n<% end %>") + end + + test "preserves line numbers in assignments" do + result = EEx.compile_string("foo\n<%= @hello %>", engine: EEx.SmartEngine) + + Macro.prewalk(result, fn + {_left, meta, [_, :hello]} -> + assert Keyword.get(meta, :line) == 2 + send(self(), :found) + + node -> + node + end) + + assert_received :found + end + + test "error with unused \"do\" block without \"<%=\" modifier" do + stderr = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert_eval("", "<% if true do %>I'm invisible!<% end %>", assigns: %{}) + end) + + assert stderr =~ "the contents of this expression won't be output" end defp assert_eval(expected, actual, binding \\ []) do - result = EEx.eval_string(actual, binding, file: __ENV__.file) + result = EEx.eval_string(actual, binding, file: __ENV__.file, engine: EEx.SmartEngine) assert result == expected end end diff --git a/lib/eex/test/eex/tokenizer_test.exs b/lib/eex/test/eex/tokenizer_test.exs index d58268b0903..6b0e97e61e7 100644 --- a/lib/eex/test/eex/tokenizer_test.exs +++ b/lib/eex/test/eex/tokenizer_test.exs @@ -1,105 +1,395 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule EEx.TokenizerTest do use ExUnit.Case, async: true - require EEx.Tokenizer, as: T + + @opts [indentation: 0, trim: false] test "simple chars lists" do - assert T.tokenize('foo', 1) == [ {:text, 'foo'} ] + assert EEx.tokenize('foo', @opts) == + {:ok, [{:text, 'foo', %{column: 1, line: 1}}, {:eof, %{column: 4, line: 1}}]} end test "simple strings" do - assert T.tokenize("foo", 1) == [ {:text, 'foo'} ] + assert EEx.tokenize("foo", @opts) == + {:ok, [{:text, 'foo', %{column: 1, line: 1}}, {:eof, %{column: 4, line: 1}}]} end test "strings with embedded code" do - assert T.tokenize('foo <% bar %>', 1) == [ {:text, 'foo '}, {:expr, 1, "", ' bar '} ] + assert EEx.tokenize('foo <% bar %>', @opts) == + {:ok, + [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:expr, '', ' bar ', %{column: 5, line: 1}}, + {:eof, %{column: 14, line: 1}} + ]} end test "strings with embedded equals code" do - assert T.tokenize('foo <%= bar %>', 1) == [ {:text, 'foo '}, {:expr, 1, "=", ' bar '} ] + assert EEx.tokenize('foo <%= bar %>', @opts) == + {:ok, + [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:expr, '=', ' bar ', %{column: 5, line: 1}}, + {:eof, %{column: 15, line: 1}} + ]} + end + + test "strings with embedded slash code" do + assert EEx.tokenize('foo <%/ bar %>', @opts) == + {:ok, + [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:expr, '/', ' bar ', %{column: 5, line: 1}}, + {:eof, %{column: 15, line: 1}} + ]} + end + + test "strings with embedded pipe code" do + assert EEx.tokenize('foo <%| bar %>', @opts) == + {:ok, + [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:expr, '|', ' bar ', %{column: 5, line: 1}}, + {:eof, %{column: 15, line: 1}} + ]} end test "strings with more than one line" do - assert T.tokenize('foo\n<%= bar %>', 1) == [ {:text, 'foo\n'}, {:expr, 2, "=", ' bar '} ] + assert EEx.tokenize('foo\n<%= bar %>', @opts) == + {:ok, + [ + {:text, 'foo\n', %{column: 1, line: 1}}, + {:expr, '=', ' bar ', %{column: 1, line: 2}}, + {:eof, %{column: 11, line: 2}} + ]} end test "strings with more than one line and expression with more than one line" do string = ''' -foo <%= bar + foo <%= bar -baz %> -<% foo %> -''' + baz %> + <% foo %> + ''' - assert T.tokenize(string, 1) == [ - {:text, 'foo '}, - {:expr, 1, "=", ' bar\n\nbaz '}, - {:text, '\n'}, - {:expr, 4, "", ' foo '}, - {:text, '\n'} - ] + exprs = [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:expr, '=', ' bar\n\nbaz ', %{column: 5, line: 1}}, + {:text, '\n', %{column: 7, line: 3}}, + {:expr, '', ' foo ', %{column: 1, line: 4}}, + {:text, '\n', %{column: 10, line: 4}}, + {:eof, %{column: 1, line: 5}} + ] + + assert EEx.tokenize(string, @opts) == {:ok, exprs} end test "quotation" do - assert T.tokenize('foo <%% true %>', 1) == [ - {:text, 'foo <% true %>'} + assert EEx.tokenize('foo <%% true %>', @opts) == + {:ok, + [ + {:text, 'foo <% true %>', %{column: 1, line: 1}}, + {:eof, %{column: 16, line: 1}} + ]} + end + + test "quotation with do-end" do + assert EEx.tokenize('foo <%% true do %>bar<%% end %>', @opts) == + {:ok, + [ + {:text, 'foo <% true do %>bar<% end %>', %{column: 1, line: 1}}, + {:eof, %{column: 32, line: 1}} + ]} + end + + test "quotation with interpolation" do + exprs = [ + {:text, 'a <% b ', %{column: 1, line: 1}}, + {:expr, '=', ' c ', %{column: 9, line: 1}}, + {:text, ' ', %{column: 17, line: 1}}, + {:expr, '=', ' d ', %{column: 18, line: 1}}, + {:text, ' e %> f', %{column: 26, line: 1}}, + {:eof, %{column: 33, line: 1}} ] + + assert EEx.tokenize('a <%% b <%= c %> <%= d %> e %> f', @opts) == {:ok, exprs} end - test "quotation with do/end" do - assert T.tokenize('foo <%% true do %>bar<%% end %>', 1) == [ - {:text, 'foo <% true do %>bar<% end %>'} + test "improperly formatted quotation with interpolation" do + exprs = [ + {:text, '<%% a <%= b %> c %>', %{column: 1, line: 1}}, + {:eof, %{column: 22, line: 1}} ] + + assert EEx.tokenize('<%%% a <%%= b %> c %>', @opts) == {:ok, exprs} end - test "comments" do - assert T.tokenize('foo <%# true %>', 1) == [ - {:text, 'foo '} + test "EEx comments" do + exprs = [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:eof, %{column: 16, line: 1}} + ] + + assert EEx.tokenize('foo <%# true %>', @opts) == {:ok, exprs} + + exprs = [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:eof, %{column: 8, line: 2}} ] + + assert EEx.tokenize('foo <%#\ntrue %>', @opts) == {:ok, exprs} end - test "comments with do/end" do - assert T.tokenize('foo <%# true do %>bar<%# end %>', 1) == [ - {:text, 'foo bar'} + test "EEx comments with do-end" do + exprs = [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:text, 'bar', %{column: 19, line: 1}}, + {:eof, %{column: 32, line: 1}} ] + + assert EEx.tokenize('foo <%# true do %>bar<%# end %>', @opts) == {:ok, exprs} + end + + test "EEx comments inside do-end" do + exprs = [ + {:start_expr, '', ' if true do ', %{column: 1, line: 1}}, + {:text, 'bar', %{column: 31, line: 1}}, + {:end_expr, [], ' end ', %{column: 34, line: 1}}, + {:eof, %{column: 43, line: 1}} + ] + + assert EEx.tokenize('<% if true do %><%# comment %>bar<% end %>', @opts) == {:ok, exprs} + + exprs = [ + {:start_expr, [], ' case true do ', %{column: 1, line: 1}}, + {:middle_expr, '', ' true -> ', %{column: 33, line: 1}}, + {:text, 'bar', %{column: 46, line: 1}}, + {:end_expr, [], ' end ', %{column: 49, line: 1}}, + {:eof, %{column: 58, line: 1}} + ] + + assert EEx.tokenize('<% case true do %><%# comment %><% true -> %>bar<% end %>', @opts) == + {:ok, exprs} + end + + test "EEx multi-line comments" do + exprs = [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:comment, ' true ', %{column: 5, line: 1}}, + {:text, ' bar', %{column: 20, line: 1}}, + {:eof, %{column: 24, line: 1}} + ] + + assert EEx.tokenize('foo <%!-- true --%> bar', @opts) == {:ok, exprs} + + exprs = [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:comment, ' \ntrue\n ', %{column: 5, line: 1}}, + {:text, ' bar', %{column: 6, line: 3}}, + {:eof, %{column: 10, line: 3}} + ] + + assert EEx.tokenize('foo <%!-- \ntrue\n --%> bar', @opts) == {:ok, exprs} + + exprs = [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:comment, ' <%= true %> ', %{column: 5, line: 1}}, + {:text, ' bar', %{column: 27, line: 1}}, + {:eof, %{column: 31, line: 1}} + ] + + assert EEx.tokenize('foo <%!-- <%= true %> --%> bar', @opts) == {:ok, exprs} + end + + test "Elixir comments" do + exprs = [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:expr, [], ' true # this is a boolean ', %{column: 5, line: 1}}, + {:eof, %{column: 35, line: 1}} + ] + + assert EEx.tokenize('foo <% true # this is a boolean %>', @opts) == {:ok, exprs} + end + + test "Elixir comments with do-end" do + exprs = [ + {:start_expr, [], ' if true do # startif ', %{column: 1, line: 1}}, + {:text, 'text', %{column: 27, line: 1}}, + {:end_expr, [], ' end # closeif ', %{column: 31, line: 1}}, + {:eof, %{column: 50, line: 1}} + ] + + assert EEx.tokenize('<% if true do # startif %>text<% end # closeif %>', @opts) == + {:ok, exprs} end test "strings with embedded do end" do - assert T.tokenize('foo <% if true do %>bar<% end %>', 1) == [ - {:text, 'foo '}, - {:start_expr, 1, "", ' if true do '}, - {:text, 'bar'}, - {:end_expr, 1, "", ' end '} + exprs = [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:start_expr, '', ' if true do ', %{column: 5, line: 1}}, + {:text, 'bar', %{column: 21, line: 1}}, + {:end_expr, '', ' end ', %{column: 24, line: 1}}, + {:eof, %{column: 33, line: 1}} ] + + assert EEx.tokenize('foo <% if true do %>bar<% end %>', @opts) == {:ok, exprs} end test "strings with embedded -> end" do - assert T.tokenize('foo <% cond do %><% false -> %>bar<% true -> %>baz<% end %>', 1) == [ - {:text, 'foo '}, - {:start_expr, 1, "", ' cond do '}, - {:middle_expr, 1, "", ' false -> '}, - {:text, 'bar'}, - {:middle_expr, 1, "", ' true -> '}, - {:text, 'baz'}, - {:end_expr, 1, "", ' end '} + exprs = [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:start_expr, '', ' cond do ', %{column: 5, line: 1}}, + {:middle_expr, '', ' false -> ', %{column: 18, line: 1}}, + {:text, 'bar', %{column: 32, line: 1}}, + {:middle_expr, '', ' true -> ', %{column: 35, line: 1}}, + {:text, 'baz', %{column: 48, line: 1}}, + {:end_expr, '', ' end ', %{column: 51, line: 1}}, + {:eof, %{column: 60, line: 1}} + ] + + assert EEx.tokenize('foo <% cond do %><% false -> %>bar<% true -> %>baz<% end %>', @opts) == + {:ok, exprs} + end + + test "strings with fn-end with newline" do + exprs = [ + {:start_expr, '=', ' a fn ->\n', %{column: 1, line: 1}}, + {:text, 'foo', %{column: 3, line: 2}}, + {:end_expr, [], ' end ', %{column: 6, line: 2}}, + {:eof, %{column: 15, line: 2}} + ] + + assert EEx.tokenize('<%= a fn ->\n%>foo<% end %>', @opts) == + {:ok, exprs} + end + + test "strings with multiple fn-end" do + exprs = [ + {:start_expr, '=', ' a fn -> ', %{column: 1, line: 1}}, + {:text, 'foo', %{column: 15, line: 1}}, + {:middle_expr, '', ' end, fn -> ', %{column: 18, line: 1}}, + {:text, 'bar', %{column: 34, line: 1}}, + {:end_expr, '', ' end ', %{column: 37, line: 1}}, + {:eof, %{column: 46, line: 1}} + ] + + assert EEx.tokenize('<%= a fn -> %>foo<% end, fn -> %>bar<% end %>', @opts) == + {:ok, exprs} + end + + test "strings with fn-end followed by do block" do + exprs = [ + {:start_expr, '=', ' a fn -> ', %{column: 1, line: 1}}, + {:text, 'foo', %{column: 15, line: 1}}, + {:middle_expr, '', ' end do ', %{column: 18, line: 1}}, + {:text, 'bar', %{column: 30, line: 1}}, + {:end_expr, '', ' end ', %{column: 33, line: 1}}, + {:eof, %{column: 42, line: 1}} ] + + assert EEx.tokenize('<%= a fn -> %>foo<% end do %>bar<% end %>', @opts) == {:ok, exprs} end test "strings with embedded keywords blocks" do - assert T.tokenize('foo <% if true do %>bar<% else %>baz<% end %>', 1) == [ - {:text, 'foo '}, - {:start_expr, 1, "", ' if true do '}, - {:text, 'bar'}, - {:middle_expr, 1, "", ' else '}, - {:text, 'baz'}, - {:end_expr, 1, "", ' end '} + exprs = [ + {:text, 'foo ', %{column: 1, line: 1}}, + {:start_expr, '', ' if true do ', %{column: 5, line: 1}}, + {:text, 'bar', %{column: 21, line: 1}}, + {:middle_expr, '', ' else ', %{column: 24, line: 1}}, + {:text, 'baz', %{column: 34, line: 1}}, + {:end_expr, '', ' end ', %{column: 37, line: 1}}, + {:eof, %{column: 46, line: 1}} ] + + assert EEx.tokenize('foo <% if true do %>bar<% else %>baz<% end %>', @opts) == + {:ok, exprs} end - test "raise syntax error when there is start mark and no end mark" do - assert_raise EEx.SyntaxError, "missing token: %>", fn -> - T.tokenize('foo <% :bar', 1) + test "trim mode" do + template = '\t<%= if true do %> \n TRUE \n <% else %>\n FALSE \n <% end %> \n\n ' + + exprs = [ + {:start_expr, '=', ' if true do ', %{column: 2, line: 1}}, + {:text, '\n TRUE \n', %{column: 20, line: 1}}, + {:middle_expr, '', ' else ', %{column: 3, line: 3}}, + {:text, '\n FALSE \n', %{column: 13, line: 3}}, + {:end_expr, '', ' end ', %{column: 3, line: 5}}, + {:eof, %{column: 3, line: 7}} + ] + + assert EEx.tokenize(template, [trim: true] ++ @opts) == {:ok, exprs} + end + + test "trim mode with comment" do + exprs = [ + {:text, '\n123', %{column: 19, line: 1}}, + {:eof, %{column: 4, line: 2}} + ] + + assert EEx.tokenize(' <%# comment %> \n123', [trim: true] ++ @opts) == {:ok, exprs} + end + + test "trim mode with multi-line comment" do + exprs = [ + {:comment, ' comment ', %{column: 3, line: 1}}, + {:text, '\n123', %{column: 23, line: 1}}, + {:eof, %{column: 4, line: 2}} + ] + + assert EEx.tokenize(' <%!-- comment --%> \n123', [trim: true] ++ @opts) == {:ok, exprs} + end + + test "trim mode with CRLF" do + exprs = [ + {:text, '0\n', %{column: 1, line: 1}}, + {:expr, '=', ' 12 ', %{column: 3, line: 2}}, + {:text, '\n34', %{column: 15, line: 2}}, + {:eof, %{column: 3, line: 3}} + ] + + assert EEx.tokenize('0\r\n <%= 12 %> \r\n34', [trim: true] ++ @opts) == {:ok, exprs} + end + + test "trim mode set to false" do + exprs = [ + {:text, ' ', %{column: 1, line: 1}}, + {:expr, '=', ' 12 ', %{column: 2, line: 1}}, + {:text, ' \n', %{column: 11, line: 1}}, + {:eof, %{column: 1, line: 2}} + ] + + assert EEx.tokenize(' <%= 12 %> \n', [trim: false] ++ @opts) == {:ok, exprs} + end + + test "trim mode no false positives" do + assert_not_trimmed = fn x -> + assert EEx.tokenize(x, [trim: false] ++ @opts) == EEx.tokenize(x, @opts) end + + assert_not_trimmed.('foo <%= "bar" %> ') + assert_not_trimmed.('\n <%= "foo" %>bar') + assert_not_trimmed.(' <%% hello %> ') + assert_not_trimmed.(' <%= 01 %><%= 23 %>\n') + end + + test "returns error when there is start mark and no end mark" do + assert EEx.tokenize('foo <% :bar', @opts) == + {:error, "missing token '%>'", %{column: 12, line: 1}} + + assert EEx.tokenize('<%# true ', @opts) == + {:error, "missing token '%>'", %{column: 10, line: 1}} + + assert EEx.tokenize('<%!-- foo ', @opts) == + {:error, "missing token '--%>'", %{column: 11, line: 1}} + end + + test "marks invalid expressions as regular expressions" do + assert EEx.tokenize('<% 1 $ 2 %>', @opts) == + {:ok, + [ + {:expr, [], ' 1 $ 2 ', %{column: 1, line: 1}}, + {:eof, %{column: 12, line: 1}} + ]} end end diff --git a/lib/eex/test/eex_test.exs b/lib/eex/test/eex_test.exs index 5cd9318b33c..dea891e4894 100644 --- a/lib/eex/test/eex_test.exs +++ b/lib/eex/test/eex_test.exs @@ -1,39 +1,36 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) require EEx defmodule EExTest.Compiled do def before_compile do - fill_in_stacktrace - {__ENV__.line, hd(tl(System.stacktrace))} + {__ENV__.line, hd(tl(get_stacktrace()))} end - EEx.function_from_string :def, :string_sample, "<%= a + b %>", [:a, :b] + EEx.function_from_string(:def, :string_sample, "<%= a + b %>", [:a, :b]) filename = Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex") - EEx.function_from_file :defp, :private_file_sample, filename, [:bar] + EEx.function_from_file(:defp, :private_file_sample, filename, [:bar]) filename = Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex") - EEx.function_from_file :def, :public_file_sample, filename, [:bar] + EEx.function_from_file(:def, :public_file_sample, filename, [:bar]) def file_sample(arg), do: private_file_sample(arg) def after_compile do - fill_in_stacktrace - {__ENV__.line, hd(tl(System.stacktrace))} + {__ENV__.line, hd(tl(get_stacktrace()))} end @file "unknown" def unknown do - fill_in_stacktrace - {__ENV__.line, hd(tl(System.stacktrace))} + {__ENV__.line, hd(tl(get_stacktrace()))} end - defp fill_in_stacktrace do + defp get_stacktrace do try do - :erlang.error "failed" - catch - :error, _ -> System.stacktrace + :erlang.error("failed") + rescue + _ -> __STACKTRACE__ end end end @@ -50,331 +47,665 @@ defmodule EExTest do use ExUnit.Case, async: true doctest EEx - doctest EEx.AssignsEngine + doctest EEx.Engine + doctest EEx.SmartEngine - test "evaluates simple string" do - assert_eval "foo bar", "foo bar" - end + describe "evaluates" do + test "simple string" do + assert_eval("foo bar", "foo bar") + end - test "evaluates with embedded" do - assert_eval "foo bar", "foo <%= :bar %>" - end + test "Unicode" do + template = """ + • <%= "•" %> • + <%= "Jößé Vâlìm" %> Jößé Vâlìm + """ - test "evaluates with embedded and the binding" do - assert EEx.eval_string("foo <%= bar %>", [bar: 1]) == "foo 1" - end + assert_eval(" • • •\n Jößé Vâlìm Jößé Vâlìm\n", template) + end - test "evaluates with embedded do end" do - assert_eval "foo bar", "foo <%= if true do %>bar<% end %>" - end + test "no spaces" do + string = """ + <%=cond do%> + <%false ->%> + this + <%true ->%> + that + <%end%> + """ + + expected = "\n that\n\n" + assert_eval(expected, string, []) + end - test "evaluates with embedded do end and eval the expression" do - assert_eval "foo ", "foo <%= if false do %>bar<% end %>" - end + test "trim mode" do + string = "<%= 123 %> \n \n <%= 789 %>" + expected = "123\n789" + assert_eval(expected, string, [], trim: true) - test "evaluates with embedded do end and nested print expression" do - assert_eval "foo bar", "foo <%= if true do %><%= :bar %><% end %>" - end + string = "<%= 123 %> \n456\n <%= 789 %>" + expected = "123\n456\n789" + assert_eval(expected, string, [], trim: true) - test "evaluates with embedded do end and nested expressions" do - assert_eval "foo bar baz", "foo <%= if true do %>bar <% Process.put(:eex_text, 1) %><%= :baz %><% end %>" - assert Process.get(:eex_text) == 1 - end + string = "<%= 123 %> \n\n456\n\n <%= 789 %>" + expected = "123\n456\n789" + assert_eval(expected, string, [], trim: true) - test "evaluates with embedded middle expression" do - assert_eval "foo bar", "foo <%= if true do %>bar<% else %>baz<% end %>" - end + string = "<%= 123 %> \n \n456\n \n <%= 789 %>" + expected = "123\n456\n789" + assert_eval(expected, string, [], trim: true) - test "evaluates with embedded middle expression and eval the expression" do - assert_eval "foo baz", "foo <%= if false do %>bar<% else %>baz<% end %>" - end + string = "\n <%= 123 %> \n <%= 456 %> \n <%= 789 %> \n" + expected = "123\n456\n789" + assert_eval(expected, string, [], trim: true) - test "evaluates with nested start expression" do - assert_eval "foo bar", "foo <%= if true do %><%= if true do %>bar<% end %><% end %>" - end + string = "\r\n <%= 123 %> \r\n <%= 456 %> \r\n <%= 789 %> \r\n" + expected = "123\n456\n789" + assert_eval(expected, string, [], trim: true) + end - test "evaluates with nested middle expression" do - assert_eval "foo baz", "foo <%= if true do %><%= if false do %>bar<% else %>baz<% end %><% end %>" - end + test "trim mode with middle expression" do + string = """ + <%= cond do %> + <% false -> %> + this + <% true -> %> + that + <% end %> + """ + + expected = "\n that\n" + assert_eval(expected, string, [], trim: true) + end - test "evaluates with defined variable" do - assert_eval "foo 1", "foo <% bar = 1 %><%= bar %>" - end + test "trim mode with multiple lines" do + string = """ + <%= "First line" %> + <%= "Second line" %> + <%= "Third line" %> + <%= "Fourth line" %> + """ - test "evaluates with require code" do - assert_eval "foo 1,2,3", "foo <% require Enum, as: E %><%= E.join [1, 2, 3], \",\" %>" - end + expected = "First line\nSecond line\nThird line\nFourth line" + assert_eval(expected, string, [], trim: true) + end - test "evaluates with end of token" do - assert_eval "foo bar %>", "foo bar %>" - end + test "trim mode with no spaces" do + string = """ + <%=if true do%> + this + <%else%> + that + <%end%> + """ + + expected = "\n this\n" + assert_eval(expected, string, [], trim: true) + + string = """ + <%=cond do%> + <%false ->%> + this + <%true ->%> + that + <%end%> + """ + + expected = "\n that\n" + assert_eval(expected, string, [], trim: true) + end - test "raises a syntax error when the token is invalid" do - assert_raise EEx.SyntaxError, "missing token: %>", fn -> - EEx.compile_string "foo <%= bar" + test "embedded code" do + assert_eval("foo bar", "foo <%= :bar %>") + end + + test "embedded code with binding" do + assert EEx.eval_string("foo <%= bar %>", bar: 1) == "foo 1" + end + + test "embedded code with do end when true" do + assert_eval("foo bar", "foo <%= if true do %>bar<% end %>") + end + + test "embedded code with do end when false" do + assert_eval("foo ", "foo <%= if false do %>bar<% end %>") + end + + test "embedded code with do preceded by bracket" do + assert_eval("foo bar", "foo <%= if {true}do %>bar<% end %>") + assert_eval("foo bar", "foo <%= if (true)do %>bar<% end %>") + assert_eval("foo bar", "foo <%= if [true]do %>bar<% end %>") + end + + test "embedded code with do end and expression" do + assert_eval("foo bar", "foo <%= if true do %><%= :bar %><% end %>") + end + + test "embedded code with do end and multiple expressions" do + assert_eval( + "foo bar baz", + "foo <%= if true do %>bar <% Process.put(:eex_text, 1) %><%= :baz %><% end %>" + ) + + assert Process.get(:eex_text) == 1 + end + + test "embedded code with middle expression" do + assert_eval("foo bar", "foo <%= if true do %>bar<% else %>baz<% end %>") + end + + test "embedded code with evaluated middle expression" do + assert_eval("foo baz", "foo <%= if false do %>bar<% else %>baz<% end %>") + end + + test "embedded code with comments in do end" do + assert_eval("foo bar", "foo <%= case true do %><%# comment %><% true -> %>bar<% end %>") + + assert_eval( + "foo\n\nbar\n", + "foo\n<%= case true do %>\n<%# comment %>\n<% true -> %>\nbar\n<% end %>" + ) end - end - test "raises a syntax error when end expression is found without a start expression" do - assert_raise EEx.SyntaxError, "unexpected token: ' end ' at line 1", fn -> - EEx.compile_string "foo <% end %>" + test "embedded code with multi-line comments in do end" do + assert_eval("foo bar", "foo <%= case true do %><%!-- comment --%><% true -> %>bar<% end %>") + + assert_eval( + "foo\n\nbar\n", + "foo\n<%= case true do %>\n<%!-- comment --%>\n<% true -> %>\nbar\n<% end %>" + ) + end + + test "embedded code with nested do end" do + assert_eval("foo bar", "foo <%= if true do %><%= if true do %>bar<% end %><% end %>") + end + + test "embedded code with nested do end with middle expression" do + assert_eval( + "foo baz", + "foo <%= if true do %><%= if false do %>bar<% else %>baz<% end %><% end %>" + ) + end + + test "embedded code with end followed by bracket" do + assert_eval( + " 101 102 103 ", + "<%= Enum.map([1, 2, 3], fn x -> %> <%= 100 + x %> <% end) %>" + ) + + assert_eval( + " 101 102 103 ", + "<%= Enum.map([1, 2, 3], fn x ->\n%> <%= 100 + x %> <% end) %>" + ) + + assert_eval( + " 101 102 103 ", + "<%= apply Enum, :map, [[1, 2, 3], fn x -> %> <%= 100 + x %> <% end] %>" + ) + + assert_eval( + " 101 102 103 ", + "<%= #{__MODULE__}.tuple_map {[1, 2, 3], fn x -> %> <%= 100 + x %> <% end} %>" + ) + + assert_eval( + " 101 102 103 ", + "<%= apply(Enum, :map, [[1, 2, 3], fn x -> %> <%= 100 + x %> <% end]) %>" + ) + + assert_eval( + " 101 102 103 ", + "<%= Enum.map([1, 2, 3], (fn x -> %> <%= 100 + x %> <% end) ) %>" + ) + end + + test "embedded code with variable definition" do + assert_eval("foo 1", "foo <% bar = 1 %><%= bar %>") + end + + test "embedded code with require" do + assert_eval("foo 1,2,3", "foo <% require Enum, as: E %><%= E.join [1, 2, 3], \",\" %>") + end + + test "with end of token" do + assert_eval("foo bar %>", "foo bar %>") end end - test "raises a syntax error when start expression is found without an end expression" do - assert_raise EEx.SyntaxError, "unexpected end of string. expecting a closing <% end %>.", fn -> - EEx.compile_string "foo <% if true do %>" + describe "raises syntax errors" do + test "when the token is invalid" do + assert_raise EEx.SyntaxError, "nofile:1:12: missing token '%>'", fn -> + EEx.compile_string("foo <%= bar") + end + end + + test "when middle expression is found without a start expression" do + assert_raise EEx.SyntaxError, + "nofile:1:18: unexpected middle of expression <% else %>", + fn -> + EEx.compile_string("<%= if true %>foo<% else %>bar<% end %>") + end + end + + test "when end expression is found without a start expression" do + assert_raise EEx.SyntaxError, "nofile:1:5: unexpected end of expression <% end %>", fn -> + EEx.compile_string("foo <% end %>") + end + end + + test "when start expression is found without an end expression" do + msg = "nofile:2:18: unexpected end of string, expected a closing '<% end %>'" + + assert_raise EEx.SyntaxError, msg, fn -> + EEx.compile_string("foo\n<%= if true do %>") + end + end + + test "when nested end expression is found without a start expression" do + assert_raise EEx.SyntaxError, "nofile:1:31: unexpected end of expression <% end %>", fn -> + EEx.compile_string("foo <%= if true do %><% end %><% end %>") + end + end + + test "when middle expression has a modifier" do + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + EEx.compile_string("foo <%= if true do %>true<%= else %>false<% end %>") + end) =~ ~s[unexpected beginning of EEx tag \"<%=\" on \"<%= else %>\"] + end + + test "when end expression has a modifier" do + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + EEx.compile_string("foo <%= if true do %>true<% else %>false<%= end %>") + end) =~ + ~s[unexpected beginning of EEx tag \"<%=\" on \"<%= end %>\"] + end + + test "when trying to use marker '/' without implementation" do + msg = + ~r/unsupported EEx syntax <%\/ %> \(the syntax is valid but not supported by the current EEx engine\)/ + + assert_raise EEx.SyntaxError, msg, fn -> + EEx.compile_string("<%/ true %>") + end + end + + test "when trying to use marker '|' without implementation" do + msg = + ~r/unsupported EEx syntax <%| %> \(the syntax is valid but not supported by the current EEx engine\)/ + + assert_raise EEx.SyntaxError, msg, fn -> + EEx.compile_string("<%| true %>") + end end end - test "raises a syntax error when nested end expression is found without an start expression" do - assert_raise EEx.SyntaxError, "unexpected token: ' end ' at line 1", fn -> - EEx.compile_string "foo <% if true do %><% end %><% end %>" + describe "error messages" do + test "honor line numbers" do + assert_raise EEx.SyntaxError, "nofile:99:12: missing token '%>'", fn -> + EEx.compile_string("foo <%= bar", line: 99) + end + end + + test "honor file names" do + assert_raise EEx.SyntaxError, "my_file.eex:1:12: missing token '%>'", fn -> + EEx.compile_string("foo <%= bar", file: "my_file.eex") + end end end - test "respects line numbers" do - expected = """ -foo -2 -""" + describe "environment" do + test "respects line numbers" do + expected = """ + foo + 2 + """ - string = """ -foo -<%= __ENV__.line %> -""" + string = """ + foo + <%= __ENV__.line %> + """ - assert_eval expected, string - end + assert_eval(expected, string) + end - test "respects line numbers inside nested expressions" do - expected = """ -foo + test "respects line numbers inside nested expressions" do + expected = """ + foo -3 + 3 -5 -""" + 5 + """ - string = """ -foo -<%= if true do %> -<%= __ENV__.line %> -<% end %> -<%= __ENV__.line %> -""" + string = """ + foo + <%= if true do %> + <%= __ENV__.line %> + <% end %> + <%= __ENV__.line %> + """ - assert_eval expected, string - end + assert_eval(expected, string) + end - test "respects line numbers inside start expression" do - expected = """ -foo + test "respects line numbers inside start expression" do + expected = """ + foo -true + true -5 -""" + 5 + """ - string = """ -foo -<%= if __ENV__.line == 2 do %> -<%= true %> -<% end %> -<%= __ENV__.line %> -""" + string = """ + foo + <%= if __ENV__.line == 2 do %> + <%= true %> + <% end %> + <%= __ENV__.line %> + """ - assert_eval expected, string - end + assert_eval(expected, string) + end - test "respects line numbers inside middle expression with ->" do - expected = """ -foo + test "respects line numbers inside middle expression with ->" do + expected = """ + foo -true + true -7 -""" + 7 + """ - string = """ -foo -<%= cond do %> -<% false -> %> false -<% __ENV__.line == 4 -> %> -<%= true %> -<% end %> -<%= __ENV__.line %> -""" + string = """ + foo + <%= cond do %> + <% false -> %> false + <% __ENV__.line == 4 -> %> + <%= true %> + <% end %> + <%= __ENV__.line %> + """ - assert_eval expected, string - end + assert_eval(expected, string) + end - test "respects line number inside middle expressions with keywords" do - expected = """ -foo + test "respects line number inside middle expressions with keywords" do + expected = """ + foo -5 + 5 -7 -""" + 7 + """ - string = """ -foo -<%= if false do %> -<%= __ENV__.line %> -<% else %> -<%= __ENV__.line %> -<% end %> -<%= __ENV__.line %> -""" + string = """ + foo + <%= if false do %> + <%= __ENV__.line %> + <% else %> + <%= __ENV__.line %> + <% end %> + <%= __ENV__.line %> + """ - assert_eval expected, string + assert_eval(expected, string) + end + + test "respects files" do + assert_eval("sample.ex", "<%= __ENV__.file %>", [], file: "sample.ex") + end end - test "properly handle functions" do - expected = """ + describe "clauses" do + test "inside functions" do + expected = """ -Number 1 + Number 1 -Number 2 + Number 2 -Number 3 + Number 3 -""" + """ - string = """ -<%= Enum.map [1, 2, 3], fn x -> %> -Number <%= x %> -<% end %> -""" + string = """ + <%= Enum.map [1, 2, 3], fn x -> %> + Number <%= x %> + <% end %> + """ - assert_eval expected, string - end + assert_eval(expected, string) + end - test "do not consider already finished functions" do - expected = """ -foo + test "inside multiple functions" do + expected = """ -true + A 1 -""" + B 2 - string = """ -foo -<%= cond do %> -<% false -> %> false -<% fn -> 1 end -> %> -<%= true %> -<% end %> -""" + A 3 - assert_eval expected, string - end + """ - test "evaluates nested do expressions" do - string = """ - <% y = ["a", "b", "c"] %> - <%= cond do %> - <% "a" in y -> %> - Good - <% true -> %> - <% if true do %>true<% else %>false<% end %> - Bad - <% end %> - """ - - assert_eval "\n\n Good\n \n", string - end + string = """ + <%= #{__MODULE__}.switching_map [1, 2, 3], fn x -> %> + A <%= x %> + <% end, fn x -> %> + B <%= x %> + <% end %> + """ - test "for comprehensions" do - string = """ - <%= for _name <- packages || [] do %> - <% end %> - <%= all || :done %> - """ - assert_eval "\ndone\n", string, packages: nil, all: nil - end + assert_eval(expected, string) + end - test "unicode" do - template = """ - • <%= "•" %> • - <%= "Jößé Vâlìm" %> Jößé Vâlìm - """ - result = EEx.eval_string(template) - assert result == " • • •\n Jößé Vâlìm Jößé Vâlìm\n" - end + test "inside callback and do block" do + expected = """ - test "evaluates the source from a given file" do - filename = Path.join(__DIR__, "fixtures/eex_template.eex") - result = EEx.eval_file(filename) - assert result == "foo bar.\n" - end - test "evaluates the source from a given file with bindings" do - filename = Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex") - result = EEx.eval_file(filename, [bar: 1]) - assert result == "foo 1\n" - end + A 1 + + B 2 + + A 3 + + """ + + string = """ + <% require #{__MODULE__} %> + <%= #{__MODULE__}.switching_macro [1, 2, 3], fn x -> %> + A <%= x %> + <% end do %> + B <%= x %> + <% end %> + """ - test "raises an Exception when there's an error with the given file" do - assert_raise File.Error, "could not read file non-existent.eex: no such file or directory", fn -> - filename = "non-existent.eex" - EEx.compile_file(filename) + assert_eval(expected, string) end - end - test "sets external resource attribute" do - assert EExTest.Compiled.__info__(:attributes)[:external_resource] == - [Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex")] + test "inside cond" do + expected = """ + foo + + true + + """ + + string = """ + foo + <%= cond do %> + <% false -> %> false + <% fn -> 1 end -> %> + <%= true %> + <% end %> + """ + + assert_eval(expected, string) + end + + test "inside cond with do end" do + string = """ + <% y = ["a", "b", "c"] %> + <%= cond do %> + <% "a" in y -> %> + Good + <% true -> %> + <%= if true do %>true<% else %>false<% end %> + Bad + <% end %> + """ + + assert_eval("\n\n Good\n \n", string) + end + + test "line and column meta" do + parser_options = Code.get_compiler_option(:parser_options) + Code.put_compiler_option(:parser_options, columns: true) + + try do + indentation = 12 + + ast = + EEx.compile_string( + """ + <%= f() %> <% f() %> + <%= f fn -> %> + <%= f() %> + <% end %> + """, + indentation: indentation + ) + + {_, calls} = + Macro.prewalk(ast, [], fn + {:f, meta, _args} = expr, acc -> {expr, [meta | acc]} + other, acc -> {other, acc} + end) + + assert Enum.reverse(calls) == [ + [line: 1, column: indentation + 5], + [line: 1, column: indentation + 15], + [line: 2, column: indentation + 7], + [line: 3, column: indentation + 9] + ] + after + Code.put_compiler_option(:parser_options, parser_options) + end + end end - test "defined from string" do - assert EExTest.Compiled.string_sample(1, 2) == "3" + describe "buffers" do + test "inside comprehensions" do + string = """ + <%= for _name <- packages || [] do %> + <% end %> + <%= all || :done %> + """ + + assert_eval("\ndone\n", string, packages: nil, all: nil) + end end - test "defined from file" do - assert EExTest.Compiled.file_sample(1) == "foo 1\n" - assert EExTest.Compiled.public_file_sample(1) == "foo 1\n" + describe "from file" do + test "evaluates the source" do + filename = Path.join(__DIR__, "fixtures/eex_template.eex") + result = EEx.eval_file(filename) + assert_normalized_newline_equal("foo bar.\n", result) + end + + test "evaluates the source with bindings" do + filename = Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex") + result = EEx.eval_file(filename, bar: 1) + assert_normalized_newline_equal("foo 1\n", result) + end + + test "raises an Exception when file is missing" do + msg = "could not read file \"non-existent.eex\": no such file or directory" + + assert_raise File.Error, msg, fn -> + filename = "non-existent.eex" + EEx.compile_file(filename) + end + end + + test "sets external resource attribute" do + assert EExTest.Compiled.__info__(:attributes)[:external_resource] == + [Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex")] + end + + test "supports t:Path.t() paths" do + filename = to_charlist(Path.join(__DIR__, "fixtures/eex_template_with_bindings.eex")) + result = EEx.eval_file(filename, bar: 1) + assert_normalized_newline_equal("foo 1\n", result) + end + + assert_raise EEx.SyntaxError, "my_file.eex:1:12: missing token '%>'", fn -> + EEx.compile_string("foo <%= bar", file: "my_file.eex") + end + + test "supports overriding file and line through options" do + filename = Path.join(__DIR__, "fixtures/eex_template_with_syntax_error.eex") + + assert_raise EEx.SyntaxError, "my_file.eex:11:1: missing token '%>'", fn -> + EEx.eval_file(filename, _bindings = [], file: "my_file.eex", line: 10) + end + end end - test "defined from file do not affect backtrace" do - assert EExTest.Compiled.before_compile == - {8, - {EExTest.Compiled, - :before_compile, - 0, - [file: to_char_list(Path.relative_to_cwd(__ENV__.file)), line: 7] - } - } - - assert EExTest.Compiled.after_compile == - {23, - {EExTest.Compiled, - :after_compile, - 0, - [file: to_char_list(Path.relative_to_cwd(__ENV__.file)), line: 22] - } - } - - assert EExTest.Compiled.unknown == - {29, - {EExTest.Compiled, - :unknown, - 0, - [file: 'unknown', line: 28] - } - } + describe "precompiled" do + test "from string" do + assert EExTest.Compiled.string_sample(1, 2) == "3" + end + + test "from file" do + assert_normalized_newline_equal("foo 1\n", EExTest.Compiled.file_sample(1)) + assert_normalized_newline_equal("foo 1\n", EExTest.Compiled.public_file_sample(1)) + end + + test "from file does not affect backtrace" do + file = to_charlist(Path.relative_to_cwd(__ENV__.file)) + + assert EExTest.Compiled.before_compile() == + {7, {EExTest.Compiled, :before_compile, 0, [file: file, line: 7]}} + + assert EExTest.Compiled.after_compile() == + {21, {EExTest.Compiled, :after_compile, 0, [file: file, line: 21]}} + + assert EExTest.Compiled.unknown() == + {26, {EExTest.Compiled, :unknown, 0, [file: 'unknown', line: 26]}} + end end defmodule TestEngine do @behaviour EEx.Engine + def init(_opts) do + "INIT" + end + def handle_body(body) do - {:wrapped, body} + "BODY(#{body})" end - def handle_text(buffer, text) do - EEx.Engine.handle_text(buffer, text) + def handle_begin(_) do + "BEGIN" + end + + def handle_end(buffer) do + buffer <> ":END" + end + + def handle_text(buffer, meta, text) do + buffer <> ":TEXT-#{meta[:line]}-#{meta[:column]}(#{String.trim(text)})" + end + + def handle_expr(buffer, "/", expr) do + buffer <> ":DIV(#{Macro.to_string(expr)})" + end + + def handle_expr(buffer, "=", expr) do + buffer <> ":EQUAL(#{Macro.to_string(expr)})" end def handle_expr(buffer, mark, expr) do @@ -382,12 +713,74 @@ foo end end - test "calls handle_body" do - assert {:wrapped, "foo"} = EEx.eval_string("foo", [], engine: TestEngine) + describe "custom engines" do + test "text" do + assert_eval("BODY(INIT:TEXT-1-1(foo))", "foo", [], engine: TestEngine) + end + + test "custom marker" do + assert_eval("BODY(INIT:TEXT-1-1(foo):DIV(:bar))", "foo <%/ :bar %>", [], engine: TestEngine) + end + + test "begin/end" do + assert_eval( + ~s[BODY(INIT:TEXT-1-1(foo):EQUAL(if do\n "BEGIN:TEXT-1-17(this):END"\nelse\n "BEGIN:TEXT-1-31(that):END"\nend))], + "foo <%= if do %>this<% else %>that<% end %>", + [], + engine: TestEngine + ) + end + + test "not implemented custom marker" do + msg = + ~r/unsupported EEx syntax <%| %> \(the syntax is valid but not supported by the current EEx engine\)/ + + assert_raise EEx.SyntaxError, msg, fn -> + assert_eval({:wrapped, "foo baz"}, "foo <%| :bar %>", [], engine: TestEngine) + end + end + end + + describe "parser options" do + test "customizes parsed code" do + atoms_encoder = fn "not_jose", _ -> {:ok, :jose} end + + assert_eval("valid", "<%= not_jose %>", [jose: "valid"], + parser_options: [static_atoms_encoder: atoms_encoder] + ) + end end - defp assert_eval(expected, actual, binding \\ []) do - result = EEx.eval_string(actual, binding, file: __ENV__.file, engine: EEx.Engine) + defp assert_eval(expected, actual, binding \\ [], opts \\ []) do + opts = Keyword.merge([file: __ENV__.file, engine: opts[:engine] || EEx.Engine], opts) + result = EEx.eval_string(actual, binding, opts) assert result == expected end + + defp assert_normalized_newline_equal(expected, actual) do + assert String.replace(expected, "\r\n", "\n") == String.replace(actual, "\r\n", "\n") + end + + def tuple_map({list, callback}) do + Enum.map(list, callback) + end + + def switching_map(list, a, b) do + list + |> Enum.with_index() + |> Enum.map(fn + {element, index} when rem(index, 2) == 0 -> a.(element) + {element, index} when rem(index, 2) == 1 -> b.(element) + end) + end + + defmacro switching_macro(list, a, do: block) do + quote do + b = fn var!(x) -> + unquote(block) + end + + unquote(__MODULE__).switching_map(unquote(list), unquote(a), b) + end + end end diff --git a/lib/eex/test/fixtures/eex_template_with_syntax_error.eex b/lib/eex/test/fixtures/eex_template_with_syntax_error.eex new file mode 100644 index 00000000000..e8aa61680f4 --- /dev/null +++ b/lib/eex/test/fixtures/eex_template_with_syntax_error.eex @@ -0,0 +1 @@ +foo <%= bar diff --git a/lib/eex/test/test_helper.exs b/lib/eex/test/test_helper.exs index bf1bd1990e9..76a54364006 100644 --- a/lib/eex/test/test_helper.exs +++ b/lib/eex/test/test_helper.exs @@ -1 +1 @@ -ExUnit.start [trace: "--trace" in System.argv] \ No newline at end of file +ExUnit.start(trace: "--trace" in System.argv()) diff --git a/lib/elixir/Emakefile b/lib/elixir/Emakefile new file mode 100644 index 00000000000..84b5a49d23b --- /dev/null +++ b/lib/elixir/Emakefile @@ -0,0 +1,16 @@ +{'src/*', [ + warn_unused_vars, + warn_export_all, + warn_shadow_vars, + warn_unused_import, + warn_unused_function, + warn_bif_clash, + warn_unused_record, + warn_deprecated_function, + warn_obsolete_guard, + warn_exported_vars, + %% warn_missing_spec, + %% warn_untyped_record, + debug_info, + {outdir, "ebin/"} +]}. diff --git a/lib/elixir/diff.exs b/lib/elixir/diff.exs new file mode 100644 index 00000000000..23147790731 --- /dev/null +++ b/lib/elixir/diff.exs @@ -0,0 +1,156 @@ +defmodule Diff do + @moduledoc """ + Utilities for comparing build artifacts. + """ + + @known_chunks ~w( + abstract_code + debug_info + attributes + compile_info + exports + labeled_exports + imports + indexed_imports + locals + labeled_locals + atoms + )a + + @doc """ + Compares the build artifacts of two build directories. + """ + @spec compare_dirs(Path.t(), Path.t()) :: + { + only1_paths :: list(Path.t()), + only2_paths :: list(Path.t()), + diff :: list({Path.t(), diff :: String.t()}) + } + def compare_dirs(dir1, dir2) do + dir1 = Path.expand(dir1) + dir2 = Path.expand(dir2) + + assert_dir!(dir1) + assert_dir!(dir2) + + dir1_paths = relative_paths(dir1) + dir2_paths = relative_paths(dir2) + + only1_paths = dir1_paths -- dir2_paths + only2_paths = dir2_paths -- dir1_paths + common_paths = dir1_paths -- only1_paths + common_files = Enum.reject(common_paths, &File.dir?/1) + + diff = + Enum.flat_map(common_files, fn path -> + file1 = Path.join(dir1, path) + file2 = Path.join(dir2, path) + + case compare_files(file1, file2) do + :eq -> [] + {:diff, diff} -> [{path, diff}] + end + end) + + {only1_paths, only2_paths, diff} + end + + @doc """ + Compares the contents of two files. + + If the files are BEAM files, it performs a more human-friendly + "BEAM-diff". + """ + @spec compare_files(Path.t(), Path.t()) :: :eq | {:diff, diff :: String.t()} + def compare_files(file1, file2) do + content1 = File.read!(file1) + content2 = File.read!(file2) + + if content1 == content2 do + :eq + else + diff = + if String.ends_with?(file1, ".beam") do + beam_diff(file1, content1, file2, content2) + else + file_diff(file1, file2) + end + + {:diff, diff} + end + end + + defp beam_diff(file1, content1, file2, content2) do + with {:ok, {module, chunks1}} <- :beam_lib.chunks(content1, @known_chunks), + {:ok, {^module, chunks2}} <- :beam_lib.chunks(content2, @known_chunks), + true <- chunks1 != chunks2 do + for {chunk1, chunk2} <- Enum.zip(chunks1, chunks2), chunk1 != chunk2 do + tmp_file1 = + chunk1 + |> inspect(pretty: true, limit: :infinity) + |> write_tmp() + + tmp_file2 = + chunk2 + |> inspect(pretty: true, limit: :infinity) + |> write_tmp() + + file_diff(tmp_file1, tmp_file2) + end + else + _ -> + file_diff(file1, file2) + end + end + + defp file_diff(file1, file2) do + {diff, _} = System.cmd("diff", [file1, file2]) + diff + end + + defp relative_paths(dir) do + dir + |> Path.join("**") + |> Path.wildcard() + |> Enum.map(&Path.relative_to(&1, dir)) + end + + defp assert_dir!(dir) do + unless File.dir?(dir) do + raise ArgumentError, "#{inspect(dir)} is not a directory" + end + end + + defp write_tmp(content) do + filename = generate_tmp_filename() + File.mkdir_p!("tmp") + File.write!(Path.join("tmp", filename), content) + Path.join("tmp", filename) + end + + defp generate_tmp_filename do + sec = :os.system_time(:second) + rand = :rand.uniform(999_999_999) + scheduler_id = :erlang.system_info(:scheduler_id) + "tmp-#{sec}-#{rand}-#{scheduler_id}" + end +end + +case System.argv() do + [dir1, dir2] -> + case Diff.compare_dirs(dir1, dir2) do + {[], [], []} -> + IO.puts("#{inspect(dir1)} and #{inspect(dir2)} are equal") + + {only1, only2, diff} -> + for path <- only1, do: IO.puts("Only in #{dir1}: #{path}") + for path <- only2, do: IO.puts("Only in #{dir2}: #{path}") + for {path, diff} <- diff, do: IO.puts("Diff #{path}:\n#{diff}") + + System.halt(1) + end + + _ -> + IO.puts("Please, provide two directories as arguments") + System.halt(1) +end diff --git a/lib/elixir/docs.exs b/lib/elixir/docs.exs new file mode 100644 index 00000000000..ab21210023e --- /dev/null +++ b/lib/elixir/docs.exs @@ -0,0 +1,118 @@ +# Returns config for Elixir docs + +canonical = System.fetch_env!("CANONICAL") + +[ + extras: Path.wildcard("lib/elixir/pages/*.md") ++ ["CHANGELOG.md"], + deps: [ + eex: "https://hexdocs.pm/eex/#{canonical}", + ex_unit: "https://hexdocs.pm/ex_unit/#{canonical}", + iex: "https://hexdocs.pm/iex/#{canonical}", + logger: "https://hexdocs.pm/logger/#{canonical}", + mix: "https://hexdocs.pm/mix/#{canonical}" + ], + groups_for_functions: [ + Guards: &(&1[:guard] == true) + ], + skip_undefined_reference_warnings_on: ["lib/elixir/pages/compatibility-and-deprecations.md"], + groups_for_modules: [ + # [Kernel, Kernel.SpecialForms], + + "Basic Types": [ + Atom, + Base, + Bitwise, + Date, + DateTime, + Exception, + Float, + Function, + Integer, + Module, + NaiveDateTime, + Record, + Regex, + String, + Time, + Tuple, + URI, + Version, + Version.Requirement + ], + "Collections & Enumerables": [ + Access, + Date.Range, + Enum, + Keyword, + List, + Map, + MapSet, + Range, + Stream + ], + "IO & System": [ + File, + File.Stat, + File.Stream, + IO, + IO.ANSI, + IO.Stream, + OptionParser, + Path, + Port, + StringIO, + System + ], + Calendar: [ + Calendar, + Calendar.ISO, + Calendar.TimeZoneDatabase, + Calendar.UTCOnlyTimeZoneDatabase + ], + "Processes & Applications": [ + Agent, + Application, + Config, + Config.Provider, + Config.Reader, + DynamicSupervisor, + GenServer, + Node, + PartitionSupervisor, + Process, + Registry, + Supervisor, + Task, + Task.Supervisor + ], + Protocols: [ + Collectable, + Enumerable, + Inspect, + Inspect.Algebra, + Inspect.Opts, + List.Chars, + Protocol, + String.Chars + ], + "Code & Macros": [ + Code, + Code.Fragment, + Kernel.ParallelCompiler, + Macro, + Macro.Env + ] + + ## Automatically detected groups + + # Deprecated: [ + # Behaviour, + # Dict, + # GenEvent, + # HashDict, + # HashSet, + # Set, + # Supervisor.Spec + # ] + ] +] diff --git a/lib/elixir/generate_app.escript b/lib/elixir/generate_app.escript new file mode 100755 index 00000000000..256497380e0 --- /dev/null +++ b/lib/elixir/generate_app.escript @@ -0,0 +1,13 @@ +#!/usr/bin/env escript +%% -*- erlang -*- + +main([Source, Target, Version]) -> + {ok, [{application, Name, Props0}]} = file:consult(Source), + Ebin = filename:dirname(Target), + Files = filelib:wildcard(filename:join(Ebin, "*.beam")), + Mods = [list_to_atom(filename:basename(F, ".beam")) || F <- Files], + Props1 = lists:keyreplace(modules, 1, Props0, {modules, Mods}), + Props = lists:keyreplace(vsn, 1, Props1, {vsn, Version}), + AppDef = io_lib:format("~tp.~n", [{application, Name, Props}]), + ok = file:write_file(Target, AppDef), + io:format("Generated ~ts app~n", [Name]). diff --git a/lib/elixir/include/elixir.hrl b/lib/elixir/include/elixir.hrl deleted file mode 100644 index 9d536c9184a..00000000000 --- a/lib/elixir/include/elixir.hrl +++ /dev/null @@ -1,68 +0,0 @@ --define(m(M, K), maps:get(K, M)). --define(line(Opts), elixir_utils:get_line(Opts)). - --record(elixir_scope, { - context=nil, %% can be match, guards or nil - extra=nil, %% extra information about the context, like fn_match and map_key - noname=false, %% when true, don't add new names (used by try) - super=false, %% when true, it means super was invoked - caller=false, %% when true, it means caller was invoked - return=true, %% when true, the return value is used - module=nil, %% the current module - function=nil, %% the current function - vars=[], %% a dict of defined variables and their alias - backup_vars=nil, %% a copy of vars to be used on ^var - match_vars=nil, %% a set of all variables defined in a particular match - export_vars=nil, %% a dict of all variables defined in a particular clause - extra_guards=nil, %% extra guards from args expansion - counter=[], %% a dict counting the variables defined - file=(<<"nofile">>) %% the current scope filename -}). - --record(elixir_quote, { - line=false, - keep=false, - context=nil, - vars_hygiene=true, - aliases_hygiene=true, - imports_hygiene=true, - unquote=true, - unquoted=false, - escape=false -}). - --record(elixir_tokenizer, { - file, - terminators=[], - check_terminators=true, - existing_atoms_only=false -}). - -%% Used in tokenization and interpolation - -%% Numbers --define(is_hex(S), ?is_digit(S) orelse (S >= $A andalso S =< $F) orelse (S >= $a andalso S =< $f)). --define(is_bin(S), S >= $0 andalso S =< $1). --define(is_octal(S), S >= $0 andalso S =< $7). --define(is_leading_octal(S), S >= $0 andalso S =< $3). - -%% Digits and letters --define(is_digit(S), S >= $0 andalso S =< $9). --define(is_upcase(S), S >= $A andalso S =< $Z). --define(is_downcase(S), S >= $a andalso S =< $z). - -%% Atoms --define(is_atom_start(S), ?is_quote(S) orelse ?is_upcase(S) orelse ?is_downcase(S) orelse (S == $_)). --define(is_atom(S), ?is_identifier(S) orelse (S == $@)). - --define(is_identifier(S), ?is_digit(S) orelse ?is_upcase(S) orelse ?is_downcase(S) orelse (S == $_)). --define(is_sigil(S), (S == $/) orelse (S == $<) orelse (S == $") orelse (S == $') orelse - (S == $[) orelse (S == $() orelse (S == ${) orelse (S == $|)). - -%% Quotes --define(is_quote(S), S == $" orelse S == $'). - -%% Spaces --define(is_horizontal_space(S), (S == $\s) orelse (S == $\t)). --define(is_vertical_space(S), (S == $\r) orelse (S == $\n)). --define(is_space(S), ?is_horizontal_space(S) orelse ?is_vertical_space(S)). diff --git a/lib/elixir/lib/access.ex b/lib/elixir/lib/access.ex index acf88b370b6..7dfe056ff62 100644 --- a/lib/elixir/lib/access.ex +++ b/lib/elixir/lib/access.ex @@ -1,19 +1,19 @@ -defprotocol Access do +defmodule Access do @moduledoc """ - The Access protocol is used by `foo[bar]` and also - empowers the nested update functions in Kernel. + Key-based access to data structures. - For instance, `foo[bar]` translates `Access.get(foo, bar)`. - `Kernel.get_in/2`, `Kernel.put_in/3`, `Kernel.update_in/3` and - `Kernel.get_and_update_in/3` are also all powered by the Access - protocol. + The `Access` module defines a behaviour for dynamically accessing + keys of any type in a data structure via the `data[key]` syntax. - This protocol is implemented by default for keywords, maps - and dictionary like types: + `Access` supports keyword lists (`Keyword`) and maps (`Map`) out + of the box. Keywords supports only atoms keys, keys for maps can + be of any type. Both return `nil` if the key does not exist: iex> keywords = [a: 1, b: 2] iex> keywords[:a] 1 + iex> keywords[:c] + nil iex> map = %{a: 1, b: 2} iex> map[:a] @@ -23,127 +23,810 @@ defprotocol Access do iex> star_ratings[1.5] "★☆" - The key comparison must be implemented using the `===` operator. + This syntax is very convenient as it can be nested arbitrarily: + + iex> keywords = [a: 1, b: 2] + iex> keywords[:c][:unknown] + nil + + This works because accessing anything on a `nil` value, returns + `nil` itself: + + iex> nil[:a] + nil + + The access syntax can also be used with the `Kernel.put_in/2`, + `Kernel.update_in/2` and `Kernel.get_and_update_in/2` macros + to allow values to be set in nested data structures: + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> put_in(users["john"][:age], 28) + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + > Attention! While the access syntax is allowed in maps via + > `map[key]`, if your map is made of predefined atom keys, + > you should prefer to access those atom keys with `map.key` + > instead of `map[key]`, as `map.key` will raise if the key + > is missing (which is not supposed to happen if the keys are + > predefined). Similarly, since structs are maps and structs + > have predefined keys, they only allow the `struct.key` + > syntax and they do not allow the `struct[key]` access syntax. + > See the `Map` module for more information. + + ## Nested data structures + + Both key-based access syntaxes can be used with the nested update + functions and macros in `Kernel`, such as `Kernel.get_in/2`, + `Kernel.put_in/3`, `Kernel.update_in/3`, `Kernel.pop_in/2`, and + `Kernel.get_and_update_in/3`. + + For example, to update a map inside another map: + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> put_in(users["john"].age, 28) + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + This module provides convenience functions for traversing other + structures, like tuples and lists. These functions can be used + in all the `Access`-related functions and macros in `Kernel`. + + For instance, given a user map with the `:name` and `:languages` + keys, here is how to deeply traverse the map and convert all + language names to uppercase: + + iex> languages = [ + ...> %{name: "elixir", type: :functional}, + ...> %{name: "c", type: :procedural} + ...> ] + iex> user = %{name: "john", languages: languages} + iex> update_in(user, [:languages, Access.all(), :name], &String.upcase/1) + %{ + name: "john", + languages: [ + %{name: "ELIXIR", type: :functional}, + %{name: "C", type: :procedural} + ] + } + + See the functions `key/1`, `key!/1`, `elem/1`, and `all/0` for + some of the available accessors. """ + @type container :: keyword | struct | map + @type nil_container :: nil + @type t :: container | nil_container | any + @type key :: any + @type value :: any + + @type get_fun(data) :: + (:get, data, (term -> term) -> new_data :: container) + + @type get_and_update_fun(data, current_value) :: + (:get_and_update, data, (term -> term) -> + {current_value, new_data :: container} | :pop) + + @type access_fun(data, current_value) :: + get_fun(data) | get_and_update_fun(data, current_value) + @doc """ - Accesses the given key in the container. + Invoked in order to access the value stored under `key` in the given term `term`. + + This function should return `{:ok, value}` where `value` is the value under + `key` if the key exists in the term, or `:error` if the key does not exist in + the term. + + Many of the functions defined in the `Access` module internally call this + function. This function is also used when the square-brackets access syntax + (`structure[key]`) is used: the `fetch/2` callback implemented by the module + that defines the `structure` struct is invoked and if it returns `{:ok, + value}` then `value` is returned, or if it returns `:error` then `nil` is + returned. + + See the `Map.fetch/2` and `Keyword.fetch/2` implementations for examples of + how to implement this callback. """ - @spec get(t, term) :: t - def get(container, key) + @callback fetch(term :: t, key) :: {:ok, value} | :error @doc """ - Gets a value and updates the given `key` in one pass. + Invoked in order to access the value under `key` and update it at the same time. + + The implementation of this callback should invoke `fun` with the value under + `key` in the passed structure `data`, or with `nil` if `key` is not present in it. + This function must return either `{current_value, new_value}` or `:pop`. - The function must receive the value for the given `key` - (or `nil` if the key doesn't exist in `container`) and - the function must return a tuple containing the `get` - value and the new value to be stored in the `container`. + If the passed function returns `{current_value, new_value}`, + the return value of this callback should be `{current_value, new_data}`, where: + + * `current_value` is the retrieved value (which can be operated on before being returned) + + * `new_value` is the new value to be stored under `key` + + * `new_data` is `data` after updating the value of `key` with `new_value`. + + If the passed function returns `:pop`, the return value of this callback + must be `{value, new_data}` where `value` is the value under `key` + (or `nil` if not present) and `new_data` is `data` without `key`. + + See the implementations of `Map.get_and_update/3` or `Keyword.get_and_update/3` + for more examples. """ - @spec get_and_update(t, term, (term -> {get, term})) :: {get, t} when get: var - def get_and_update(container, key, fun) -end + @callback get_and_update(data, key, (value | nil -> {current_value, new_value :: value} | :pop)) :: + {current_value, new_data :: data} + when current_value: value, data: container -defimpl Access, for: List do - def get(dict, key) when is_atom(key) do - case :lists.keyfind(key, 1, dict) do - {^key, value} -> value - false -> nil + @doc """ + Invoked to "pop" the value under `key` out of the given data structure. + + When `key` exists in the given structure `data`, the implementation should + return a `{value, new_data}` tuple where `value` is the value that was under + `key` and `new_data` is `term` without `key`. + + When `key` is not present in the given structure, a tuple `{value, data}` + should be returned, where `value` is implementation-defined. + + See the implementations for `Map.pop/3` or `Keyword.pop/3` for more examples. + """ + @callback pop(data, key) :: {value, data} when data: container + + defmacrop raise_undefined_behaviour(exception, module, top) do + quote do + exception = + case __STACKTRACE__ do + [unquote(top) | _] -> + reason = + "#{inspect(unquote(module))} does not implement the Access behaviour. " <> + "If you are using get_in/put_in/update_in, you can specify the field " <> + "to be accessed using Access.key!/1" + + %{unquote(exception) | reason: reason} + + _ -> + unquote(exception) + end + + reraise exception, __STACKTRACE__ end end - def get(_dict, key) do - raise ArgumentError, - "the access protocol for lists expect the key to be an atom, got: #{inspect key}" + @doc """ + Fetches the value for the given key in a container (a map, keyword + list, or struct that implements the `Access` behaviour). + + Returns `{:ok, value}` where `value` is the value under `key` if there is such + a key, or `:error` if `key` is not found. + + ## Examples + + iex> Access.fetch(%{name: "meg", age: 26}, :name) + {:ok, "meg"} + + iex> Access.fetch([ordered: true, on_timeout: :exit], :timeout) + :error + + """ + @spec fetch(container, term) :: {:ok, term} | :error + @spec fetch(nil_container, any) :: :error + def fetch(container, key) + + def fetch(%module{} = container, key) do + module.fetch(container, key) + rescue + exception in UndefinedFunctionError -> + raise_undefined_behaviour(exception, module, {^module, :fetch, [^container, ^key], _}) end - def get_and_update(dict, key, fun) when is_atom(key) do - get_and_update(dict, [], key, fun) + def fetch(map, key) when is_map(map) do + case map do + %{^key => value} -> {:ok, value} + _ -> :error + end end - defp get_and_update([{key, value}|t], acc, key, fun) do - {get, update} = fun.(value) - {get, :lists.reverse(acc, [{key, update}|t])} + def fetch(list, key) when is_list(list) and is_atom(key) do + case :lists.keyfind(key, 1, list) do + {_, value} -> {:ok, value} + false -> :error + end end - defp get_and_update([h|t], acc, key, fun) do - get_and_update(t, [h|acc], key, fun) + def fetch(list, key) when is_list(list) do + raise ArgumentError, + "the Access calls for keywords expect the key to be an atom, got: " <> inspect(key) end - defp get_and_update([], acc, key, fun) do - {get, update} = fun.(nil) - {get, [{key, update}|:lists.reverse(acc)]} + def fetch(nil, _key) do + :error end -end -defimpl Access, for: Map do - def get(map, key) do - case :maps.find(key, map) do + @doc """ + Same as `fetch/2` but returns the value directly, + or raises a `KeyError` exception if `key` is not found. + + ## Examples + + iex> Access.fetch!(%{name: "meg", age: 26}, :name) + "meg" + + """ + @doc since: "1.10.0" + @spec fetch!(container, term) :: term + def fetch!(container, key) do + case fetch(container, key) do {:ok, value} -> value - :error -> nil + :error -> raise(KeyError, key: key, term: container) end end - def get_and_update(map, key, fun) do - value = - case :maps.find(key, map) do - {:ok, value} -> value - :error -> nil - end + @doc """ + Gets the value for the given key in a container (a map, keyword + list, or struct that implements the `Access` behaviour). - {get, update} = fun.(value) - {get, :maps.put(key, update, map)} - end + Returns the value under `key` if there is such a key, or `default` if `key` is + not found. - def get!(%{} = map, key) do - case :maps.find(key, map) do + ## Examples + + iex> Access.get(%{name: "john"}, :name, "default name") + "john" + iex> Access.get(%{name: "john"}, :age, 25) + 25 + + iex> Access.get([ordered: true], :timeout) + nil + + """ + @spec get(container, term, term) :: term + @spec get(nil_container, any, default) :: default when default: var + def get(container, key, default \\ nil) + + # Reimplementing the same logic as Access.fetch/2 here is done for performance, since + # this is called a lot and calling fetch/2 means introducing some overhead (like + # building the "{:ok, _}" tuple and deconstructing it back right away). + + def get(%module{} = container, key, default) do + try do + module.fetch(container, key) + rescue + exception in UndefinedFunctionError -> + raise_undefined_behaviour(exception, module, {^module, :fetch, [^container, ^key], _}) + else {:ok, value} -> value - :error -> raise KeyError, key: key, term: map + :error -> default end end - def get!(other, key) do - raise ArgumentError, - "could not get key #{inspect key}. Expected map/struct, got: #{inspect other}" + def get(map, key, default) when is_map(map) do + case map do + %{^key => value} -> value + _ -> default + end end - def get_and_update!(%{} = map, key, fun) do - case :maps.find(key, map) do - {:ok, value} -> - {get, update} = fun.(value) - {get, :maps.put(key, update, map)} - :error -> - raise KeyError, key: key, term: map + def get(list, key, default) when is_list(list) and is_atom(key) do + case :lists.keyfind(key, 1, list) do + {_, value} -> value + false -> default end end - def get_and_update!(other, key, _fun) do + def get(list, key, _default) when is_list(list) do raise ArgumentError, - "could not update key #{inspect key}. Expected map/struct, got: #{inspect other}" + "the Access calls for keywords expect the key to be an atom, got: " <> inspect(key) end -end -defimpl Access, for: Atom do - def get(nil, _) do - nil + def get(nil, _key, default) do + default + end + + @doc """ + Gets and updates the given key in a `container` (a map, a keyword list, + a struct that implements the `Access` behaviour). + + The `fun` argument receives the value of `key` (or `nil` if `key` is not + present in `container`) and must return a two-element tuple `{current_value, new_value}`: + the "get" value `current_value` (the retrieved value, which can be operated on before + being returned) and the new value to be stored under `key` (`new_value`). + `fun` may also return `:pop`, which means the current value + should be removed from the container and returned. + + The returned value is a two-element tuple with the "get" value returned by + `fun` and a new container with the updated value under `key`. + + ## Examples + + iex> Access.get_and_update([a: 1], :a, fn current_value -> + ...> {current_value, current_value + 1} + ...> end) + {1, [a: 2]} + + """ + @spec get_and_update(data, key, (value | nil -> {current_value, new_value :: value} | :pop)) :: + {current_value, new_data :: data} + when current_value: var, data: container + def get_and_update(container, key, fun) + + def get_and_update(%module{} = container, key, fun) do + module.get_and_update(container, key, fun) + rescue + exception in UndefinedFunctionError -> + raise_undefined_behaviour( + exception, + module, + {^module, :get_and_update, [^container, ^key, ^fun], _} + ) + end + + def get_and_update(map, key, fun) when is_map(map) do + Map.get_and_update(map, key, fun) + end + + def get_and_update(list, key, fun) when is_list(list) do + Keyword.get_and_update(list, key, fun) + end + + def get_and_update(nil, key, _fun) do + raise ArgumentError, "could not put/update key #{inspect(key)} on a nil value" + end + + @doc """ + Removes the entry with a given key from a container (a map, keyword + list, or struct that implements the `Access` behaviour). + + Returns a tuple containing the value associated with the key and the + updated container. `nil` is returned for the value if the key isn't + in the container. + + ## Examples + + With a map: + + iex> Access.pop(%{name: "Elixir", creator: "Valim"}, :name) + {"Elixir", %{creator: "Valim"}} + + A keyword list: + + iex> Access.pop([name: "Elixir", creator: "Valim"], :name) + {"Elixir", [creator: "Valim"]} + + An unknown key: + + iex> Access.pop(%{name: "Elixir", creator: "Valim"}, :year) + {nil, %{creator: "Valim", name: "Elixir"}} + + """ + @spec pop(data, key) :: {value, data} when data: container + def pop(%module{} = container, key) do + module.pop(container, key) + rescue + exception in UndefinedFunctionError -> + raise_undefined_behaviour(exception, module, {^module, :pop, [^container, ^key], _}) + end + + def pop(map, key) when is_map(map) do + Map.pop(map, key) end - def get(atom, _) do - undefined(atom) + def pop(list, key) when is_list(list) do + Keyword.pop(list, key) end - def get_and_update(nil, _, fun) do - fun.(nil) + def pop(nil, key) do + raise ArgumentError, "could not pop key #{inspect(key)} on a nil value" + end + + ## Accessors + + @doc """ + Returns a function that accesses the given key in a map/struct. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + The returned function uses the default value if the key does not exist. + This can be used to specify defaults and safely traverse missing keys: + + iex> get_in(%{}, [Access.key(:user, %{}), Access.key(:name, "meg")]) + "meg" + + Such is also useful when using update functions, allowing us to introduce + values as we traverse the data structure for updates: + + iex> put_in(%{}, [Access.key(:user, %{}), Access.key(:name)], "Mary") + %{user: %{name: "Mary"}} + + ## Examples + + iex> map = %{user: %{name: "john"}} + iex> get_in(map, [Access.key(:unknown, %{}), Access.key(:name, "john")]) + "john" + iex> get_and_update_in(map, [Access.key(:user), Access.key(:name)], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"john", %{user: %{name: "JOHN"}}} + iex> pop_in(map, [Access.key(:user), Access.key(:name)]) + {"john", %{user: %{}}} + + An error is raised if the accessed structure is not a map or a struct: + + iex> get_in([], [Access.key(:foo)]) + ** (BadMapError) expected a map, got: [] + + """ + @spec key(key, term) :: access_fun(data :: struct | map, current_value :: term) + def key(key, default \\ nil) do + fn + :get, data, next -> + next.(Map.get(data, key, default)) + + :get_and_update, data, next -> + value = Map.get(data, key, default) + + case next.(value) do + {get, update} -> {get, Map.put(data, key, update)} + :pop -> {value, Map.delete(data, key)} + end + end end - def get_and_update(atom, _key, _fun) do - undefined(atom) + @doc """ + Returns a function that accesses the given key in a map/struct. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + Similar to `key/2`, but the returned function raises if the key does not exist. + + ## Examples + + iex> map = %{user: %{name: "john"}} + iex> get_in(map, [Access.key!(:user), Access.key!(:name)]) + "john" + iex> get_and_update_in(map, [Access.key!(:user), Access.key!(:name)], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"john", %{user: %{name: "JOHN"}}} + iex> pop_in(map, [Access.key!(:user), Access.key!(:name)]) + {"john", %{user: %{}}} + iex> get_in(map, [Access.key!(:user), Access.key!(:unknown)]) + ** (KeyError) key :unknown not found in: %{name: \"john\"} + + An error is raised if the accessed structure is not a map/struct: + + iex> get_in([], [Access.key!(:foo)]) + ** (RuntimeError) Access.key!/1 expected a map/struct, got: [] + + """ + @spec key!(key) :: access_fun(data :: struct | map, current_value :: term) + def key!(key) do + fn + :get, %{} = data, next -> + next.(Map.fetch!(data, key)) + + :get_and_update, %{} = data, next -> + value = Map.fetch!(data, key) + + case next.(value) do + {get, update} -> {get, Map.put(data, key, update)} + :pop -> {value, Map.delete(data, key)} + end + + _op, data, _next -> + raise "Access.key!/1 expected a map/struct, got: #{inspect(data)}" + end + end + + @doc ~S""" + Returns a function that accesses the element at the given index in a tuple. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + The returned function raises if `index` is out of bounds. + + Note that popping elements out of tuples is not possible and raises an + error. + + ## Examples + + iex> map = %{user: {"john", 27}} + iex> get_in(map, [:user, Access.elem(0)]) + "john" + iex> get_and_update_in(map, [:user, Access.elem(0)], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"john", %{user: {"JOHN", 27}}} + iex> pop_in(map, [:user, Access.elem(0)]) + ** (RuntimeError) cannot pop data from a tuple + + An error is raised if the accessed structure is not a tuple: + + iex> get_in(%{}, [Access.elem(0)]) + ** (RuntimeError) Access.elem/1 expected a tuple, got: %{} + + """ + @spec elem(non_neg_integer) :: access_fun(data :: tuple, current_value :: term) + def elem(index) when is_integer(index) and index >= 0 do + pos = index + 1 + + fn + :get, data, next when is_tuple(data) -> + next.(:erlang.element(pos, data)) + + :get_and_update, data, next when is_tuple(data) -> + value = :erlang.element(pos, data) + + case next.(value) do + {get, update} -> {get, :erlang.setelement(pos, data, update)} + :pop -> raise "cannot pop data from a tuple" + end + + _op, data, _next -> + raise "Access.elem/1 expected a tuple, got: #{inspect(data)}" + end + end + + @doc ~S""" + Returns a function that accesses all the elements in a list. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + ## Examples + + iex> list = [%{name: "john"}, %{name: "mary"}] + iex> get_in(list, [Access.all(), :name]) + ["john", "mary"] + iex> get_and_update_in(list, [Access.all(), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {["john", "mary"], [%{name: "JOHN"}, %{name: "MARY"}]} + iex> pop_in(list, [Access.all(), :name]) + {["john", "mary"], [%{}, %{}]} + + Here is an example that traverses the list dropping even + numbers and multiplying odd numbers by 2: + + iex> require Integer + iex> get_and_update_in([1, 2, 3, 4, 5], [Access.all()], fn num -> + ...> if Integer.is_even(num), do: :pop, else: {num, num * 2} + ...> end) + {[1, 2, 3, 4, 5], [2, 6, 10]} + + An error is raised if the accessed structure is not a list: + + iex> get_in(%{}, [Access.all()]) + ** (RuntimeError) Access.all/0 expected a list, got: %{} + + """ + @spec all() :: access_fun(data :: list, current_value :: list) + def all() do + &all/3 + end + + defp all(:get, data, next) when is_list(data) do + Enum.map(data, next) + end + + defp all(:get_and_update, data, next) when is_list(data) do + all(data, next, _gets = [], _updates = []) + end + + defp all(_op, data, _next) do + raise "Access.all/0 expected a list, got: #{inspect(data)}" + end + + defp all([head | rest], next, gets, updates) do + case next.(head) do + {get, update} -> all(rest, next, [get | gets], [update | updates]) + :pop -> all(rest, next, [head | gets], updates) + end + end + + defp all([], _next, gets, updates) do + {:lists.reverse(gets), :lists.reverse(updates)} + end + + @doc ~S""" + Returns a function that accesses the element at `index` (zero based) of a list. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + ## Examples + + iex> list = [%{name: "john"}, %{name: "mary"}] + iex> get_in(list, [Access.at(1), :name]) + "mary" + iex> get_in(list, [Access.at(-1), :name]) + "mary" + iex> get_and_update_in(list, [Access.at(0), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"john", [%{name: "JOHN"}, %{name: "mary"}]} + iex> get_and_update_in(list, [Access.at(-1), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"mary", [%{name: "john"}, %{name: "MARY"}]} + + `at/1` can also be used to pop elements out of a list or + a key inside of a list: + + iex> list = [%{name: "john"}, %{name: "mary"}] + iex> pop_in(list, [Access.at(0)]) + {%{name: "john"}, [%{name: "mary"}]} + iex> pop_in(list, [Access.at(0), :name]) + {"john", [%{}, %{name: "mary"}]} + + When the index is out of bounds, `nil` is returned and the update function is never called: + + iex> list = [%{name: "john"}, %{name: "mary"}] + iex> get_in(list, [Access.at(10), :name]) + nil + iex> get_and_update_in(list, [Access.at(10), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {nil, [%{name: "john"}, %{name: "mary"}]} + + An error is raised if the accessed structure is not a list: + + iex> get_in(%{}, [Access.at(1)]) + ** (RuntimeError) Access.at/1 expected a list, got: %{} + + """ + @spec at(integer) :: access_fun(data :: list, current_value :: term) + def at(index) when is_integer(index) do + fn op, data, next -> at(op, data, index, next) end + end + + defp at(:get, data, index, next) when is_list(data) do + data |> Enum.at(index) |> next.() + end + + defp at(:get_and_update, data, index, next) when is_list(data) do + get_and_update_at(data, index, next, [], fn -> nil end) + end + + defp at(_op, data, _index, _next) do + raise "Access.at/1 expected a list, got: #{inspect(data)}" + end + + defp get_and_update_at([head | rest], 0, next, updates, _default_fun) do + case next.(head) do + {get, update} -> {get, :lists.reverse([update | updates], rest)} + :pop -> {head, :lists.reverse(updates, rest)} + end + end + + defp get_and_update_at([_ | _] = list, index, next, updates, default_fun) when index < 0 do + list_length = length(list) + + if list_length + index >= 0 do + get_and_update_at(list, list_length + index, next, updates, default_fun) + else + {default_fun.(), list} + end + end + + defp get_and_update_at([head | rest], index, next, updates, default_fun) when index > 0 do + get_and_update_at(rest, index - 1, next, [head | updates], default_fun) + end + + defp get_and_update_at([], _index, _next, updates, default_fun) do + {default_fun.(), :lists.reverse(updates)} + end + + @doc ~S""" + Same as `at/1` except that it raises `Enum.OutOfBoundsError` + if the given index is out of bounds. + + ## Examples + + iex> get_in([:a, :b, :c], [Access.at!(2)]) + :c + iex> get_in([:a, :b, :c], [Access.at!(3)]) + ** (Enum.OutOfBoundsError) out of bounds error + + """ + @doc since: "1.11.0" + @spec at!(integer) :: access_fun(data :: list, current_value :: term) + def at!(index) when is_integer(index) do + fn op, data, next -> at!(op, data, index, next) end + end + + defp at!(:get, data, index, next) when is_list(data) do + case Enum.fetch(data, index) do + {:ok, value} -> next.(value) + :error -> raise Enum.OutOfBoundsError + end + end + + defp at!(:get_and_update, data, index, next) when is_list(data) do + get_and_update_at(data, index, next, [], fn -> raise Enum.OutOfBoundsError end) + end + + defp at!(_op, data, _index, _next) do + raise "Access.at!/1 expected a list, got: #{inspect(data)}" + end + + @doc ~S""" + Returns a function that accesses all elements of a list that match the provided predicate. + + The returned function is typically passed as an accessor to `Kernel.get_in/2`, + `Kernel.get_and_update_in/3`, and friends. + + ## Examples + + iex> list = [%{name: "john", salary: 10}, %{name: "francine", salary: 30}] + iex> get_in(list, [Access.filter(&(&1.salary > 20)), :name]) + ["francine"] + iex> get_and_update_in(list, [Access.filter(&(&1.salary <= 20)), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {["john"], [%{name: "JOHN", salary: 10}, %{name: "francine", salary: 30}]} + + `filter/1` can also be used to pop elements out of a list or + a key inside of a list: + + iex> list = [%{name: "john", salary: 10}, %{name: "francine", salary: 30}] + iex> pop_in(list, [Access.filter(&(&1.salary >= 20))]) + {[%{name: "francine", salary: 30}], [%{name: "john", salary: 10}]} + iex> pop_in(list, [Access.filter(&(&1.salary >= 20)), :name]) + {["francine"], [%{name: "john", salary: 10}, %{salary: 30}]} + + When no match is found, an empty list is returned and the update function is never called + + iex> list = [%{name: "john", salary: 10}, %{name: "francine", salary: 30}] + iex> get_in(list, [Access.filter(&(&1.salary >= 50)), :name]) + [] + iex> get_and_update_in(list, [Access.filter(&(&1.salary >= 50)), :name], fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {[], [%{name: "john", salary: 10}, %{name: "francine", salary: 30}]} + + An error is raised if the predicate is not a function or is of the incorrect arity: + + iex> get_in([], [Access.filter(5)]) + ** (FunctionClauseError) no function clause matching in Access.filter/1 + + An error is raised if the accessed structure is not a list: + + iex> get_in(%{}, [Access.filter(fn a -> a == 10 end)]) + ** (RuntimeError) Access.filter/1 expected a list, got: %{} + + """ + @doc since: "1.6.0" + @spec filter((term -> boolean)) :: access_fun(data :: list, current_value :: list) + def filter(func) when is_function(func) do + fn op, data, next -> filter(op, data, func, next) end + end + + defp filter(:get, data, func, next) when is_list(data) do + data |> Enum.filter(func) |> Enum.map(next) + end + + defp filter(:get_and_update, data, func, next) when is_list(data) do + get_and_update_filter(data, func, next, [], []) + end + + defp filter(_op, data, _func, _next) do + raise "Access.filter/1 expected a list, got: #{inspect(data)}" + end + + defp get_and_update_filter([head | rest], func, next, updates, gets) do + if func.(head) do + case next.(head) do + {get, update} -> + get_and_update_filter(rest, func, next, [update | updates], [get | gets]) + + :pop -> + get_and_update_filter(rest, func, next, updates, [head | gets]) + end + else + get_and_update_filter(rest, func, next, [head | updates], gets) + end end - defp undefined(atom) do - raise Protocol.UndefinedError, - protocol: @protocol, - value: atom, - description: "only the nil atom is supported" + defp get_and_update_filter([], _func, _next, updates, gets) do + {:lists.reverse(gets), :lists.reverse(updates)} end end diff --git a/lib/elixir/lib/agent.ex b/lib/elixir/lib/agent.ex index 628f0675f79..0ccc4022676 100644 --- a/lib/elixir/lib/agent.ex +++ b/lib/elixir/lib/agent.ex @@ -6,41 +6,55 @@ defmodule Agent do must be accessed from different processes or by the same process at different points in time. - The Agent module provides a basic server implementation that + The `Agent` module provides a basic server implementation that allows state to be retrieved and updated via a simple API. ## Examples - For example, in the Mix tool that ships with Elixir, we need - to keep a set of all tasks executed by a given project. Since - this set is shared, we can implement it with an Agent: + For example, the following agent implements a counter: - defmodule Mix.TasksServer do - def start_link do - Agent.start_link(fn -> HashSet.new end, name: __MODULE__) + defmodule Counter do + use Agent + + def start_link(initial_value) do + Agent.start_link(fn -> initial_value end, name: __MODULE__) end - @doc "Checks if the task has already executed" - def executed?(task, project) do - item = {task, project} - Agent.get(__MODULE__, fn set -> - item in set - end) + def value do + Agent.get(__MODULE__, & &1) end - @doc "Marks a task as executed" - def put_task(task, project) do - item = {task, project} - Agent.update(__MODULE__, &Set.put(&1, item)) + def increment do + Agent.update(__MODULE__, &(&1 + 1)) end end - Note that agents still provide a segregation between the - client and server APIs, as seen in GenServers. In particular, - all code inside the function passed to the agent is executed - by the agent. This distinction is important because you may - want to avoid expensive operations inside the agent, as it will - effectively block the agent until the request is fulfilled. + Usage would be: + + Counter.start_link(0) + #=> {:ok, #PID<0.123.0>} + + Counter.value() + #=> 0 + + Counter.increment() + #=> :ok + + Counter.increment() + #=> :ok + + Counter.value() + #=> 2 + + Thanks to the agent server process, the counter can be safely incremented + concurrently. + + Agents provide a segregation between the client and server APIs (similar to + `GenServer`s). In particular, the functions passed as arguments to the calls to + `Agent` functions are invoked inside the agent (the server). This distinction + is important because you may want to avoid expensive operations inside the + agent, as they will effectively block the agent until the request is + fulfilled. Consider these two examples: @@ -51,51 +65,112 @@ defmodule Agent do # Compute in the agent/client def get_something(agent) do - Agent.get(agent, &(&1)) |> do_something_expensive() + Agent.get(agent, & &1) |> do_something_expensive() + end + + The first function blocks the agent. The second function copies all the state + to the client and then executes the operation in the client. One aspect to + consider is whether the data is large enough to require processing in the server, + at least initially, or small enough to be sent to the client cheaply. Another + factor is whether the data needs to be processed atomically: getting the + state and calling `do_something_expensive(state)` outside of the agent means + that the agent's state can be updated in the meantime. This is specially + important in case of updates as computing the new state in the client rather + than in the server can lead to race conditions if multiple clients are trying + to update the same state to different values. + + ## How to supervise + + An `Agent` is most commonly started under a supervision tree. + When we invoke `use Agent`, it automatically defines a `child_spec/1` + function that allows us to start the agent directly under a supervisor. + To start an agent under a supervisor with an initial counter of 0, + one may do: + + children = [ + {Counter, 0} + ] + + Supervisor.start_link(children, strategy: :one_for_all) + + While one could also simply pass the `Counter` as a child to the supervisor, + such as: + + children = [ + Counter # Same as {Counter, []} + ] + + Supervisor.start_link(children, strategy: :one_for_all) + + The definition above wouldn't work for this particular example, + as it would attempt to start the counter with an initial value + of an empty list. However, this may be a viable option in your + own agents. A common approach is to use a keyword list, as that + would allow setting the initial value and giving a name to the + counter process, for example: + + def start_link(opts) do + {initial_value, opts} = Keyword.pop(opts, :initial_value, 0) + Agent.start_link(fn -> initial_value end, opts) end - The first one blocks the agent while the second one copies - all the state to the client and executes the operation in the client. - The trade-off here is exactly if the data is small enough to be - sent to the client cheaply or large enough to require processing on - the server (or at least some initial processing). + and then you can use `Counter`, `{Counter, name: :my_counter}` or + even `{Counter, initial_value: 0, name: :my_counter}` as a child + specification. + + `use Agent` also accepts a list of options which configures the + child specification and therefore how it runs under a supervisor. + The generated `child_spec/1` can be customized with the following options: + + * `:id` - the child specification identifier, defaults to the current module + * `:restart` - when the child should be restarted, defaults to `:permanent` + * `:shutdown` - how to shut down the child, either immediately or by giving it time to shut down + + For example: - ## Name Registration + use Agent, restart: :transient, shutdown: 10_000 - An Agent is bound to the same name registration rules as GenServers. - Read more about it in the `GenServer` docs. + See the "Child specification" section in the `Supervisor` module for more + detailed information. The `@doc` annotation immediately preceding + `use Agent` will be attached to the generated `child_spec/1` function. + + ## Name registration + + An agent is bound to the same name registration rules as GenServers. + Read more about it in the `GenServer` documentation. ## A word on distributed agents It is important to consider the limitations of distributed agents. Agents - work by sending anonymous functions between the caller and the agent. - In a distributed setup with multiple nodes, agents only work if the caller - (client) and the agent have the same version of a given module. - - This setup may exhibit issues when doing "rolling upgrades". By rolling - upgrades we mean the following situation: you wish to deploy a new version of - your software by *shutting down* some of your nodes and replacing them with - nodes running a new version of the software. In this setup, part of your - environment will have one version of a given module and the other part - another version (the newer one) of the same module; this may cause agents to - crash. That said, if you plan to run in distributed environments, agents - should likely be avoided. - - Note, however, that agents work fine if you want to perform hot code - swapping, as it keeps both the old and new versions of a given module. - We detail how to do hot code swapping with agents in the next section. + provide two APIs, one that works with anonymous functions and another + that expects an explicit module, function, and arguments. + + In a distributed setup with multiple nodes, the API that accepts anonymous + functions only works if the caller (client) and the agent have the same + version of the caller module. + + Keep in mind this issue also shows up when performing "rolling upgrades" + with agents. By rolling upgrades we mean the following situation: you wish + to deploy a new version of your software by *shutting down* some of your + nodes and replacing them with nodes running a new version of the software. + In this setup, part of your environment will have one version of a given + module and the other part another version (the newer one) of the same module. + + The best solution is to simply use the explicit module, function, and arguments + APIs when working with distributed agents. ## Hot code swapping An agent can have its code hot swapped live by simply passing a module, - function and args tuple to the update instruction. For example, imagine + function, and arguments tuple to the update instruction. For example, imagine you have an agent named `:sample` and you want to convert its inner state - from some dict structure to a map. It can be done with the following + from a keyword list to a map. It can be done with the following instruction: {:update, :sample, {:advanced, {Enum, :into, [%{}]}}} - The agent's state will be added to the given list as the first argument. + The agent's state will be added to the given list of arguments (`[%{}]`) as + the first argument. """ @typedoc "Return values of `start*` functions" @@ -111,13 +186,50 @@ defmodule Agent do @type state :: term @doc """ - Starts an agent linked to the current process. + Returns a specification to start an agent under a supervisor. + + See the "Child specification" section in the `Supervisor` module for more detailed information. + """ + @doc since: "1.5.0" + def child_spec(arg) do + %{ + id: Agent, + start: {Agent, :start_link, [arg]} + } + end + + @doc false + defmacro __using__(opts) do + quote location: :keep, bind_quoted: [opts: opts] do + unless Module.has_attribute?(__MODULE__, :doc) do + @doc """ + Returns a specification to start this module under a supervisor. + + See `Supervisor`. + """ + end + + def child_spec(arg) do + default = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [arg]} + } + + Supervisor.child_spec(default, unquote(Macro.escape(opts))) + end + + defoverridable child_spec: 1 + end + end + + @doc """ + Starts an agent linked to the current process with the given function. This is often used to start the agent as part of a supervision tree. - Once the agent is spawned, the given function is invoked and its return - value is used as the agent state. Note that `start_link` does not return - until the given function has returned. + Once the agent is spawned, the given function `fun` is invoked in the server + process, and should return the initial agent state. Note that `start_link/2` + does not return until the given function has returned. ## Options @@ -129,7 +241,7 @@ defmodule Agent do and the start function will return `{:error, :timeout}`. If the `:debug` option is present, the corresponding function in the - [`:sys` module](http://www.erlang.org/doc/man/sys.html) will be invoked. + [`:sys` module](`:sys`) will be invoked. If the `:spawn_opt` option is present, its value will be passed as options to the underlying process as in `Process.spawn/4`. @@ -137,36 +249,86 @@ defmodule Agent do ## Return values If the server is successfully created and initialized, the function returns - `{:ok, pid}`, where `pid` is the pid of the server. If there already exists - an agent with the specified name, the function returns - `{:error, {:already_started, pid}}` with the pid of that process. + `{:ok, pid}`, where `pid` is the PID of the server. If an agent with the + specified name already exists, the function returns + `{:error, {:already_started, pid}}` with the PID of that process. + + If the given function callback fails, the function returns `{:error, reason}`. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.get(pid, fn state -> state end) + 42 + + iex> {:error, {exception, _stacktrace}} = Agent.start(fn -> raise "oops" end) + iex> exception + %RuntimeError{message: "oops"} - If the given function callback fails with `reason`, the function returns - `{:error, reason}`. """ - @spec start_link((() -> term), GenServer.options) :: on_start + @spec start_link((() -> term), GenServer.options()) :: on_start def start_link(fun, options \\ []) when is_function(fun, 0) do GenServer.start_link(Agent.Server, fun, options) end + @doc """ + Starts an agent linked to the current process. + + Same as `start_link/2` but a module, function, and arguments are expected + instead of an anonymous function; `fun` in `module` will be called with the + given arguments `args` to initialize the state. + """ + @spec start_link(module, atom, [any], GenServer.options()) :: on_start + def start_link(module, fun, args, options \\ []) do + GenServer.start_link(Agent.Server, {module, fun, args}, options) + end + @doc """ Starts an agent process without links (outside of a supervision tree). See `start_link/2` for more information. + + ## Examples + + iex> {:ok, pid} = Agent.start(fn -> 42 end) + iex> Agent.get(pid, fn state -> state end) + 42 + """ - @spec start((() -> term), GenServer.options) :: on_start + @spec start((() -> term), GenServer.options()) :: on_start def start(fun, options \\ []) when is_function(fun, 0) do GenServer.start(Agent.Server, fun, options) end @doc """ - Gets the agent value and executes the given function. + Starts an agent without links with the given module, function, and arguments. + + See `start_link/4` for more information. + """ + @spec start(module, atom, [any], GenServer.options()) :: on_start + def start(module, fun, args, options \\ []) do + GenServer.start(Agent.Server, {module, fun, args}, options) + end + + @doc """ + Gets an agent value via the given anonymous function. The function `fun` is sent to the `agent` which invokes the function passing the agent state. The result of the function invocation is - returned. + returned from this function. + + `timeout` is an integer greater than zero which specifies how many + milliseconds are allowed before the agent executes the function and returns + the result value, or the atom `:infinity` to wait indefinitely. If no result + is received within the specified time, the function call fails and the caller + exits. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.get(pid, fn state -> state end) + 42 - A timeout can also be specified (it has a default value of 5000). """ @spec get(agent, (state -> a), timeout) :: a when a: var def get(agent, fun, timeout \\ 5000) when is_function(fun, 1) do @@ -174,14 +336,40 @@ defmodule Agent do end @doc """ - Gets and updates the agent state in one operation. + Gets an agent value via the given function. + + Same as `get/3` but a module, function, and arguments are expected + instead of an anonymous function. The state is added as first + argument to the given list of arguments. + """ + @spec get(agent, module, atom, [term], timeout) :: any + def get(agent, module, fun, args, timeout \\ 5000) do + GenServer.call(agent, {:get, {module, fun, args}}, timeout) + end + + @doc """ + Gets and updates the agent state in one operation via the given anonymous + function. The function `fun` is sent to the `agent` which invokes the function passing the agent state. The function must return a tuple with two - elements, the first being the value to return (i.e. the `get` value) - and the second one is the new state. + elements, the first being the value to return (that is, the "get" value) + and the second one being the new state of the agent. + + `timeout` is an integer greater than zero which specifies how many + milliseconds are allowed before the agent executes the function and returns + the result value, or the atom `:infinity` to wait indefinitely. If no result + is received within the specified time, the function call fails and the caller + exits. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.get_and_update(pid, fn state -> {state, state + 1} end) + 42 + iex> Agent.get(pid, fn state -> state end) + 43 - A timeout can also be specified (it has a default value of 5000). """ @spec get_and_update(agent, (state -> {a, state}), timeout) :: a when a: var def get_and_update(agent, fun, timeout \\ 5000) when is_function(fun, 1) do @@ -189,40 +377,132 @@ defmodule Agent do end @doc """ - Updates the agent state. + Gets and updates the agent state in one operation via the given function. + + Same as `get_and_update/3` but a module, function, and arguments are expected + instead of an anonymous function. The state is added as first + argument to the given list of arguments. + """ + @spec get_and_update(agent, module, atom, [term], timeout) :: any + def get_and_update(agent, module, fun, args, timeout \\ 5000) do + GenServer.call(agent, {:get_and_update, {module, fun, args}}, timeout) + end + + @doc """ + Updates the agent state via the given anonymous function. The function `fun` is sent to the `agent` which invokes the function - passing the agent state. The function must return the new state. + passing the agent state. The return value of `fun` becomes the new + state of the agent. - A timeout can also be specified (it has a default value of 5000). This function always returns `:ok`. + + `timeout` is an integer greater than zero which specifies how many + milliseconds are allowed before the agent executes the function and returns + the result value, or the atom `:infinity` to wait indefinitely. If no result + is received within the specified time, the function call fails and the caller + exits. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.update(pid, fn state -> state + 1 end) + :ok + iex> Agent.get(pid, fn state -> state end) + 43 + """ - @spec update(agent, (state -> state)) :: :ok + @spec update(agent, (state -> state), timeout) :: :ok def update(agent, fun, timeout \\ 5000) when is_function(fun, 1) do GenServer.call(agent, {:update, fun}, timeout) end @doc """ - Performs a cast (fire and forget) operation on the agent state. + Updates the agent state via the given function. + + Same as `update/3` but a module, function, and arguments are expected + instead of an anonymous function. The state is added as first + argument to the given list of arguments. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.update(pid, Kernel, :+, [12]) + :ok + iex> Agent.get(pid, fn state -> state end) + 54 + + """ + @spec update(agent, module, atom, [term], timeout) :: :ok + def update(agent, module, fun, args, timeout \\ 5000) do + GenServer.call(agent, {:update, {module, fun, args}}, timeout) + end + + @doc """ + Performs a cast (*fire and forget*) operation on the agent state. The function `fun` is sent to the `agent` which invokes the function - passing the agent state. The function must return the new state. + passing the agent state. The return value of `fun` becomes the new + state of the agent. + + Note that `cast` returns `:ok` immediately, regardless of whether `agent` (or + the node it should live on) exists. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.cast(pid, fn state -> state + 1 end) + :ok + iex> Agent.get(pid, fn state -> state end) + 43 - Note that `cast` returns `:ok` immediately, regardless of whether the - destination node or agent exists. """ @spec cast(agent, (state -> state)) :: :ok def cast(agent, fun) when is_function(fun, 1) do - GenServer.cast(agent, fun) + GenServer.cast(agent, {:cast, fun}) end @doc """ - Stops the agent. + Performs a cast (*fire and forget*) operation on the agent state. + + Same as `cast/2` but a module, function, and arguments are expected + instead of an anonymous function. The state is added as first + argument to the given list of arguments. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.cast(pid, Kernel, :+, [12]) + :ok + iex> Agent.get(pid, fn state -> state end) + 54 + + """ + @spec cast(agent, module, atom, [term]) :: :ok + def cast(agent, module, fun, args) do + GenServer.cast(agent, {:cast, {module, fun, args}}) + end + + @doc """ + Synchronously stops the agent with the given `reason`. + + It returns `:ok` if the agent terminates with the given + reason. If the agent terminates with another reason, the call will + exit. + + This function keeps OTP semantics regarding error reporting. + If the reason is any other than `:normal`, `:shutdown` or + `{:shutdown, _}`, an error report will be logged. + + ## Examples + + iex> {:ok, pid} = Agent.start_link(fn -> 42 end) + iex> Agent.stop(pid) + :ok - Returns `:ok` if the agent is stopped within the given `timeout`. """ - @spec stop(agent, timeout) :: :ok - def stop(agent, timeout \\ 5000) do - GenServer.call(agent, :stop, timeout) + @spec stop(agent, reason :: term, timeout) :: :ok + def stop(agent, reason \\ :normal, timeout \\ :infinity) do + GenServer.stop(agent, reason, timeout) end end diff --git a/lib/elixir/lib/agent/server.ex b/lib/elixir/lib/agent/server.ex index 0b06e4cddbe..03cb763ea2b 100644 --- a/lib/elixir/lib/agent/server.ex +++ b/lib/elixir/lib/agent/server.ex @@ -4,50 +4,48 @@ defmodule Agent.Server do use GenServer def init(fun) do - {:ok, fun.()} + _ = initial_call(fun) + {:ok, run(fun, [])} end def handle_call({:get, fun}, _from, state) do - {:reply, fun.(state), state} + {:reply, run(fun, [state]), state} end def handle_call({:get_and_update, fun}, _from, state) do - {reply, state} = fun.(state) - {:reply, reply, state} + case run(fun, [state]) do + {reply, state} -> {:reply, reply, state} + other -> {:stop, {:bad_return_value, other}, state} + end end def handle_call({:update, fun}, _from, state) do - {:reply, :ok, fun.(state)} + {:reply, :ok, run(fun, [state])} end - def handle_call(:stop, _from, state) do - {:stop, :normal, :ok, state} + def handle_cast({:cast, fun}, state) do + {:noreply, run(fun, [state])} end - def handle_call(msg, from, state) do - super(msg, from, state) + def code_change(_old, state, fun) do + {:ok, run(fun, [state])} end - def handle_cast(fun, state) when is_function(fun, 1) do - {:noreply, fun.(state)} + defp initial_call(mfa) do + _ = Process.put(:"$initial_call", get_initial_call(mfa)) + :ok end - def handle_cast(msg, state) do - super(msg, state) + defp get_initial_call(fun) when is_function(fun, 0) do + {:module, module} = Function.info(fun, :module) + {:name, name} = Function.info(fun, :name) + {module, name, 0} end - def code_change(_old, state, { m, f, a }) do - {:ok, apply(m, f, [state|a])} + defp get_initial_call({mod, fun, args}) do + {mod, fun, length(args)} end - def terminate(_reason, _state) do - # There is a race condition if the agent is - # restarted too fast and it is registered. - try do - self |> :erlang.process_info(:registered_name) |> elem(1) |> Process.unregister - rescue - _ -> :ok - end - :ok - end + defp run({m, f, a}, extra), do: apply(m, f, extra ++ a) + defp run(fun, extra), do: apply(fun, extra) end diff --git a/lib/elixir/lib/application.ex b/lib/elixir/lib/application.ex index 691704e0258..c6535ee29de 100644 --- a/lib/elixir/lib/application.ex +++ b/lib/elixir/lib/application.ex @@ -2,183 +2,793 @@ defmodule Application do @moduledoc """ A module for working with applications and defining application callbacks. - In Elixir (actually, in Erlang/OTP), an application is a component - implementing some specific functionality, that can be started and stopped - as a unit, and which can be re-used in other systems as well. - - Applications are defined with an application file named `APP.app` where - `APP` is the APP name, usually in `underscore_case` convention. The - application file must reside in the same `ebin` directory as the - application's modules bytecode. - - In Elixir, Mix is responsible for compiling your source code and - generating your application `.app` file. Furthermore, Mix is also - responsible for configuring, starting and stopping your application - and its dependencies. For this reason, this documentation will focus - on the remaining aspects of your application: the application environment, - and the application callback module. - - You can learn more about Mix compilation of `.app` files by typing - `mix help compile.app`. + Applications are the idiomatic way to package software in Erlang/OTP. To get + the idea, they are similar to the "library" concept common in other + programming languages, but with some additional characteristics. + + An application is a component implementing some specific functionality, with a + standardized directory structure, configuration, and life cycle. Applications + are *loaded*, *started*, and *stopped*. Each application also has its own + environment, which provides a unified API for configuring each application. - ## Application environment + Developers typically interact with the application environment and its + callback module. Therefore those will be the topics we will cover first + before jumping into details about the application resource file and life-cycle. - Once an application is started, OTP provides an application environment - that can be used to configure applications. + ## The application environment - Assuming you are inside a Mix project, you can edit your application - function in the `mix.exs` file to the following: + Each application has its own environment. The environment is a keyword list + that maps atoms to terms. Note that this environment is unrelated to the + operating system environment. + + By default, the environment of an application is an empty list. In a Mix + project's `mix.exs` file, you can set the `:env` key in `application/0`: def application do - [env: [hello: :world]] + [env: [db_host: "localhost"]] + end + + Now, in your application, you can read this environment by using functions + such as `fetch_env!/2` and friends: + + defmodule MyApp.DBClient do + def start_link() do + SomeLib.DBClient.start_link(host: db_host()) + end + + defp db_host do + Application.fetch_env!(:my_app, :db_host) + end + end + + In Mix projects, the environment of the application and its dependencies can + be overridden via the `config/config.exs` and `config/runtime.exs` files. The + former is loaded at build-time, before your code compiles, and the latter at + runtime, just before your app starts. For example, someone using your application + can override its `:db_host` environment variable as follows: + + import Config + config :my_app, :db_host, "db.local" + + See the "Configuration" section in the `Mix` module for more information. + You can also change the application environment dynamically by using functions + such as `put_env/3` and `delete_env/2`. + + > Note: The config files `config/config.exs` and `config/runtime.exs` + > are rarely used by libraries. Libraries typically define their environment + > in the `def application` function of their `mix.exs`. Configuration files + > are rather used by applications to configure their libraries. + + > Note: Each application is responsible for its own environment. Do not + > use the functions in this module for directly accessing or modifying + > the environment of other applications. Whenever you change the application + > environment, Elixir's build tool will only recompile the files that + > belong to that application. So if you read the application environment + > of another application, there is a chance you will be depending on + > outdated configuration, as your file won't be recompiled as it changes. + + ## Compile-time environment + + In the previous example, we read the application environment at runtime: + + defmodule MyApp.DBClient do + def start_link() do + SomeLib.DBClient.start_link(host: db_host()) + end + + defp db_host do + Application.fetch_env!(:my_app, :db_host) + end + end + + In other words, the environment key `:db_host` for application `:my_app` + will only be read when `MyApp.DBClient` effectively starts. While reading + the application environment at runtime is the preferred approach, in some + rare occasions you may want to use the application environment to configure + the compilation of a certain project. However, if you try to access + `Application.fetch_env!/2` outside of a function: + + defmodule MyApp.DBClient do + @db_host Application.fetch_env!(:my_app, :db_host) + + def start_link() do + SomeLib.DBClient.start_link(host: @db_host) + end + end + + You might see warnings and errors: + + warning: Application.fetch_env!/2 is discouraged in the module body, + use Application.compile_env/3 instead + iex:3: MyApp.DBClient + + ** (ArgumentError) could not fetch application environment :db_host + for application :my_app because the application was not loaded nor + configured + + This happens because, when defining modules, the application environment + is not yet available. Luckily, the warning tells us how to solve this + issue, by using `Application.compile_env/3` instead: + + defmodule MyApp.DBClient do + @db_host Application.compile_env(:my_app, :db_host, "db.local") + + def start_link() do + SomeLib.DBClient.start_link(host: @db_host) + end end - In the application function, we can define the default environment values - for our application. By starting your application with `iex -S mix`, you - can access the default value: + The difference here is that `compile_env` expects the default value to be + given as an argument, instead of using the `def application` function of + your `mix.exs`. Furthermore, by using `compile_env/3`, tools like Mix will + store the values used during compilation and compare the compilation values + with the runtime values whenever your system starts, raising an error in + case they differ. - Application.get_env(:APP_NAME, :hello) - #=> {:ok, :hello} + In any case, compile-time environments should be avoided. Whenever possible, + reading the application environment at runtime should be the first choice. - It is also possible to put and delete values from the application value, - including new values that are not defined in the environment file (although - those should be avoided). + ## The application callback module - In the future, we plan to support configuration files which allows - developers to configure the environment of their dependencies. + Applications can be loaded, started, and stopped. Generally, build tools + like Mix take care of starting an application and all of its dependencies + for you, but you can also do it manually by calling: - Keep in mind that each application is responsible for its environment. - Do not use the functions in this module for directly access or modify - the environment of other application (as it may lead to inconsistent - data in the application environment). + {:ok, _} = Application.ensure_all_started(:some_app) - ## Application module callback + When an application starts, developers may configure a callback module + that executes custom code. Developers use this callback to start the + application supervision tree. - Often times, an application defines a supervision tree that must be started - and stopped when the application starts and stops. For such, we need to - define an application module callback. The first step is to define the - module callback in the application definition in the `mix.exs` file: + The first step to do so is to add a `:mod` key to the `application/0` + definition in your `mix.exs` file. It expects a tuple, with the application + callback module and start argument (commonly an empty list): def application do [mod: {MyApp, []}] end - Our application now requires the `MyApp` module to provide an application - callback. This can be done by invoking `use Application` in that module - and defining a `start/2` callback, for example: + The `MyApp` module given to `:mod` needs to implement the `Application` behaviour. + This can be done by putting `use Application` in that module and implementing the + `c:start/2` callback, for example: defmodule MyApp do use Application def start(_type, _args) do - MyApp.Supervisor.start_link() + children = [] + Supervisor.start_link(children, strategy: :one_for_one) end end - `start/2` most commonly returns `{:ok, pid}` or `{:ok, pid, state}` where - `pid` identifies the supervision tree and the state is the application state. - `args` is second element of the tuple given to the `:mod` option. + The `c:start/2` callback has to spawn and link a supervisor and return `{:ok, + pid}` or `{:ok, pid, state}`, where `pid` is the PID of the supervisor, and + `state` is an optional application state. `args` is the second element of the + tuple given to the `:mod` option. + + The `type` argument passed to `c:start/2` is usually `:normal` unless in a + distributed setup where application takeovers and failovers are configured. + Distributed applications are beyond the scope of this documentation. + + When an application is shutting down, its `c:stop/1` callback is called after + the supervision tree has been stopped by the runtime. This callback allows the + application to do any final cleanup. The argument is the state returned by + `c:start/2`, if it did, or `[]` otherwise. The return value of `c:stop/1` is + ignored. + + By using `Application`, modules get a default implementation of `c:stop/1` + that ignores its argument and returns `:ok`, but it can be overridden. + + Application callback modules may also implement the optional callback + `c:prep_stop/1`. If present, `c:prep_stop/1` is invoked before the supervision + tree is terminated. Its argument is the state returned by `c:start/2`, if it did, + or `[]` otherwise, and its return value is passed to `c:stop/1`. + + ## The application resource file + + In the sections above, we have configured an application in the + `application/0` section of the `mix.exs` file. Ultimately, Mix will use + this configuration to create an [*application resource + file*](https://www.erlang.org/doc/man/application.html), which is a file called + `APP_NAME.app`. For example, the application resource file of the OTP + application `ex_unit` is called `ex_unit.app`. + + You can learn more about the generation of application resource files in + the documentation of `Mix.Tasks.Compile.App`, available as well by running + `mix help compile.app`. + + ## The application life cycle + + ### Loading applications + + Applications are *loaded*, which means that the runtime finds and processes + their resource files: + + Application.load(:ex_unit) + #=> :ok + + When an application is loaded, the environment specified in its resource file + is merged with any overrides from config files. + + Loading an application *does not* load its modules. + + In practice, you rarely load applications by hand because that is part of the + start process, explained next. + + ### Starting applications + + Applications are also *started*: + + Application.start(:ex_unit) + #=> :ok + + Once your application is compiled, running your system is a matter of starting + your current application and its dependencies. Differently from other languages, + Elixir does not have a `main` procedure that is responsible for starting your + system. Instead, you start one or more applications, each with their own + initialization and termination logic. + + When an application is started, the `Application.load/1` is automatically + invoked if it hasn't been done yet. Then, it checks if the dependencies listed + in the `applications` key of the resource file are already started. Having at + least one dependency not started is an error condition. Functions like + `ensure_all_started/1` takes care of starting an application and all of its + dependencies for you. + + If the application does not have a callback module configured, starting is + done at this point. Otherwise, its `c:start/2` callback is invoked. The PID of + the top-level supervisor returned by this function is stored by the runtime + for later use, and the returned application state is saved too, if any. + + ### Stopping applications + + Started applications are, finally, *stopped*: + + Application.stop(:ex_unit) + #=> :ok - The `type` passed into `start/2` is usually `:normal` unless in a distributed - setup where applications takeover and failovers are configured. This particular - aspect of applications can be read with more detail in the OTP documentation: + Stopping an application without a callback module is defined, but except for + some system tracing, it is in practice a no-op. - * http://www.erlang.org/doc/man/application.html - * http://www.erlang.org/doc/design_principles/applications.html + Stopping an application with a callback module has three steps: - A developer may also implement the `stop/1` callback (automatically defined - by `use Application`) which does any application cleanup. It receives the - application state and can return any value. Notice that shutting down the - supervisor is automatically handled by the VM; + 1. If present, invoke the optional callback `c:prep_stop/1`. + 2. Terminate the top-level supervisor. + 3. Invoke the required callback `c:stop/1`. + + The arguments passed to the callbacks are related to the state optionally + returned by `c:start/2`, and are documented in the section about the callback + module above. + + It is important to highlight that step 2 is a blocking one. Termination of a + supervisor triggers a recursive chain of children terminations, therefore + orderly shutting down all descendant processes. The `c:stop/1` callback is + invoked only after termination of the whole supervision tree. + + Shutting down a live system cleanly can be done by calling `System.stop/1`. It + will shut down every application in the opposite order they had been started. + + By default, a SIGTERM from the operating system will automatically translate to + `System.stop/0`. You can also have more explicit control over operating system + signals via the `:os.set_signal/2` function. + + ## Tooling + + The Mix build tool automates most of the application management tasks. For example, + `mix test` automatically starts your application dependencies and your application + itself before your test runs. `mix run --no-halt` boots your current project and + can be used to start a long running system. See `mix help run`. + + Developers can also use `mix release` to build **releases**. Releases are able to + package all of your source code as well as the Erlang VM into a single directory. + Releases also give you explicit control over how each application is started and in + which order. They also provide a more streamlined mechanism for starting and + stopping systems, debugging, logging, as well as system monitoring. + + Finally, Elixir provides tools such as escripts and archives, which are + different mechanisms for packaging your application. Those are typically used + when tools must be shared between developers and not as deployment options. + See `mix help archive.build` and `mix help escript.build` for more detail. + + ## Further information + + For further details on applications please check the documentation of the + [`:application` Erlang module](`:application`), and the + [Applications](https://www.erlang.org/doc/design_principles/applications.html) + section of the [OTP Design Principles User's + Guide](https://www.erlang.org/doc/design_principles/users_guide.html). + """ + + @doc """ + Called when an application is started. + + This function is called when an application is started using + `Application.start/2` (and functions on top of that, such as + `Application.ensure_started/2`). This function should start the top-level + process of the application (which should be the top supervisor of the + application's supervision tree if the application follows the OTP design + principles around supervision). + + `start_type` defines how the application is started: + + * `:normal` - used if the startup is a normal startup or if the application + is distributed and is started on the current node because of a failover + from another node and the application specification key `:start_phases` + is `:undefined`. + * `{:takeover, node}` - used if the application is distributed and is + started on the current node because of a failover on the node `node`. + * `{:failover, node}` - used if the application is distributed and is + started on the current node because of a failover on node `node`, and the + application specification key `:start_phases` is not `:undefined`. + + `start_args` are the arguments passed to the application in the `:mod` + specification key (for example, `mod: {MyApp, [:my_args]}`). + + This function should either return `{:ok, pid}` or `{:ok, pid, state}` if + startup is successful. `pid` should be the PID of the top supervisor. `state` + can be an arbitrary term, and if omitted will default to `[]`; if the + application is later stopped, `state` is passed to the `stop/1` callback (see + the documentation for the `c:stop/1` callback for more information). + + `use Application` provides no default implementation for the `start/2` + callback. + """ + @callback start(start_type, start_args :: term) :: + {:ok, pid} + | {:ok, pid, state} + | {:error, reason :: term} + + @doc """ + Called before stopping the application. + + This function is called before the top-level supervisor is terminated. It + receives the state returned by `c:start/2`, if it did, or `[]` otherwise. + The return value is later passed to `c:stop/1`. + """ + @callback prep_stop(state) :: state + + @doc """ + Called after an application has been stopped. + + This function is called after an application has been stopped, i.e., after its + supervision tree has been stopped. It should do the opposite of what the + `c:start/2` callback did, and should perform any necessary cleanup. The return + value of this callback is ignored. + + `state` is the state returned by `c:start/2`, if it did, or `[]` otherwise. + If the optional callback `c:prep_stop/1` is present, `state` is its return + value instead. + + `use Application` defines a default implementation of this function which does + nothing and just returns `:ok`. + """ + @callback stop(state) :: term + + @doc """ + Starts an application in synchronous phases. + + This function is called after `start/2` finishes but before + `Application.start/2` returns. It will be called once for every start phase + defined in the application's (and any included applications') specification, + in the order they are listed in. + """ + @callback start_phase(phase :: term, start_type, phase_args :: term) :: + :ok | {:error, reason :: term} + + @doc """ + Callback invoked after code upgrade, if the application environment + has changed. + + `changed` is a keyword list of keys and their changed values in the + application environment. `new` is a keyword list with all new keys + and their values. `removed` is a list with all removed keys. """ + @callback config_change(changed, new, removed) :: :ok + when changed: keyword, new: keyword, removed: [atom] + + @optional_callbacks start_phase: 3, prep_stop: 1, config_change: 3 @doc false defmacro __using__(_) do quote location: :keep do - @behaviour :application + @behaviour Application @doc false def stop(_state) do :ok end - defoverridable [stop: 1] + defoverridable Application end end + @application_keys [ + :description, + :id, + :vsn, + :modules, + :maxP, + :maxT, + :registered, + :included_applications, + :optional_applications, + :applications, + :mod, + :start_phases + ] + + application_key_specs = Enum.reduce(@application_keys, &{:|, [], [&1, &2]}) + @type app :: atom @type key :: atom + @type application_key :: unquote(application_key_specs) @type value :: term - @type start_type :: :permanent | :transient | :temporary + @type state :: term + @type start_type :: :normal | {:takeover, node} | {:failover, node} + @type restart_type :: :permanent | :transient | :temporary + + @doc """ + Returns the spec for `app`. + + The following keys are returned: + + * #{Enum.map_join(@application_keys, "\n * ", &"`#{inspect(&1)}`")} + + Note the environment is not returned as it can be accessed via + `fetch_env/2`. Returns `nil` if the application is not loaded. + """ + @spec spec(app) :: [{application_key, value}] | nil + def spec(app) when is_atom(app) do + case :application.get_all_key(app) do + {:ok, info} -> :lists.keydelete(:env, 1, info) + :undefined -> nil + end + end + + @doc """ + Returns the value for `key` in `app`'s specification. + + See `spec/1` for the supported keys. If the given + specification parameter does not exist, this function + will raise. Returns `nil` if the application is not loaded. + """ + @spec spec(app, application_key) :: value | nil + def spec(app, key) when is_atom(app) and key in @application_keys do + case :application.get_key(app, key) do + {:ok, value} -> value + :undefined -> nil + end + end + + @doc """ + Gets the application for the given module. + + The application is located by analyzing the spec + of all loaded applications. Returns `nil` if + the module is not listed in any application spec. + """ + @spec get_application(atom) :: atom | nil + def get_application(module) when is_atom(module) do + case :application.get_application(module) do + {:ok, app} -> app + :undefined -> nil + end + end @doc """ Returns all key-value pairs for `app`. """ - @spec get_all_env(app) :: [{key,value}] - def get_all_env(app) do + @spec get_all_env(app) :: [{key, value}] + def get_all_env(app) when is_atom(app) do :application.get_all_env(app) end @doc """ - Returns the value for `key` in `app`'s environment. + Reads the application environment at compilation time. + + Similar to `get_env/3`, except it must be used to read values + at compile time. This allows Elixir to track when configuration + values change between compile time and runtime. + + The first argument is the application name. The second argument + `key_or_path` is either an atom key or a path to traverse in + search of the configuration, starting with an atom key. + + For example, imagine the following configuration: - If the specified application is not loaded, or the configuration parameter - does not exist, the function returns the `default` value. + config :my_app, :key, [foo: [bar: :baz]] + + We can access it during compile time as: + + Application.compile_env(:my_app, :key) + #=> [foo: [bar: :baz]] + + Application.compile_env(:my_app, [:key, :foo]) + #=> [bar: :baz] + + Application.compile_env(:my_app, [:key, :foo, :bar]) + #=> :baz + + A default value can also be given as third argument. If + any of the keys in the path along the way is missing, the + default value is used: + + Application.compile_env(:my_app, [:unknown, :foo, :bar], :default) + #=> :default + + Application.compile_env(:my_app, [:key, :unknown, :bar], :default) + #=> :default + + Application.compile_env(:my_app, [:key, :foo, :unknown], :default) + #=> :default + + Giving a path is useful to let Elixir know that only certain paths + in a large configuration are compile time dependent. """ - @spec get_env(app, key, value) :: value - def get_env(app, key, default \\ nil) do - case :application.get_env(app, key) do + @doc since: "1.10.0" + @spec compile_env(app, key | list, value) :: value + defmacro compile_env(app, key_or_path, default \\ nil) do + if __CALLER__.function do + raise "Application.compile_env/3 cannot be called inside functions, only in the module body" + end + + key_or_path = expand_key_or_path(key_or_path, __CALLER__) + + quote do + Application.__compile_env__(unquote(app), unquote(key_or_path), unquote(default), __ENV__) + end + end + + defp expand_key_or_path({:__aliases__, _, _} = alias, env), + do: Macro.expand(alias, %{env | function: {:__info__, 1}}) + + defp expand_key_or_path(list, env) when is_list(list), + do: Enum.map(list, &expand_key_or_path(&1, env)) + + defp expand_key_or_path(other, _env), + do: other + + @doc false + def __compile_env__(app, key_or_path, default, env) do + case fetch_compile_env(app, key_or_path, env) do {:ok, value} -> value - :undefined -> default + :error -> default + end + end + + @doc """ + Reads the application environment at compilation time or raises. + + This is the same as `compile_env/3` but it raises an + `ArgumentError` if the configuration is not available. + """ + @doc since: "1.10.0" + @spec compile_env!(app, key | list) :: value + defmacro compile_env!(app, key_or_path) do + if __CALLER__.function do + raise "Application.compile_env!/2 cannot be called inside functions, only in the module body" + end + + key_or_path = expand_key_or_path(key_or_path, __CALLER__) + + quote do + Application.__compile_env__!(unquote(app), unquote(key_or_path), __ENV__) + end + end + + @doc false + def __compile_env__!(app, key_or_path, env) do + case fetch_compile_env(app, key_or_path, env) do + {:ok, value} -> + value + + :error -> + raise ArgumentError, + "could not fetch application environment #{inspect(key_or_path)} for application " <> + "#{inspect(app)} #{fetch_env_failed_reason(app, key_or_path)}" end end + defp fetch_compile_env(app, key, env) when is_atom(key), + do: fetch_compile_env(app, key, [], env) + + defp fetch_compile_env(app, [key | paths], env) when is_atom(key), + do: fetch_compile_env(app, key, paths, env) + + defp fetch_compile_env(app, key, path, env) do + return = traverse_env(fetch_env(app, key), path) + + for tracer <- env.tracers do + tracer.trace({:compile_env, app, [key | path], return}, env) + end + + return + end + + defp traverse_env(return, []), do: return + defp traverse_env(:error, _paths), do: :error + defp traverse_env({:ok, value}, [key | keys]), do: traverse_env(Access.fetch(value, key), keys) + + @doc """ + Returns the value for `key` in `app`'s environment. + + If the configuration parameter does not exist, the function returns the + `default` value. + + > **Important:** you must use this function to read only your own application + > environment. Do not read the environment of other applications. + + > **Important:** if you are writing a library to be used by other developers, + > it is generally recommended to avoid the application environment, as the + > application environment is effectively a global storage. For more information, + > read our [library guidelines](library-guidelines.md). + + ## Examples + + `get_env/3` is commonly used to read the configuration of your OTP applications. + Since Mix configurations are commonly used to configure applications, we will use + this as a point of illustration. + + Consider a new application `:my_app`. `:my_app` contains a database engine which + supports a pool of databases. The database engine needs to know the configuration for + each of those databases, and that configuration is supplied by key-value pairs in + environment of `:my_app`. + + config :my_app, Databases.RepoOne, + # A database configuration + ip: "localhost", + port: 5433 + + config :my_app, Databases.RepoTwo, + # Another database configuration (for the same OTP app) + ip: "localhost", + port: 20717 + + config :my_app, my_app_databases: [Databases.RepoOne, Databases.RepoTwo] + + Our database engine used by `:my_app` needs to know what databases exist, and + what the database configurations are. The database engine can make a call to + `Application.get_env(:my_app, :my_app_databases, [])` to retrieve the list of + databases (specified by module names). + + The engine can then traverse each repository in the list and call + `Application.get_env(:my_app, Databases.RepoOne)` and so forth to retrieve the + configuration of each one. In this case, each configuration will be a keyword + list, so you can use the functions in the `Keyword` module or even the `Access` + module to traverse it, for example: + + config = Application.get_env(:my_app, Databases.RepoOne) + config[:ip] + + """ + @spec get_env(app, key, value) :: value + def get_env(app, key, default \\ nil) when is_atom(app) do + maybe_warn_on_app_env_key(app, key) + :application.get_env(app, key, default) + end + @doc """ Returns the value for `key` in `app`'s environment in a tuple. - If the specified application is not loaded, or the configuration parameter - does not exist, the function returns `:error`. + If the configuration parameter does not exist, the function returns `:error`. + + > **Important:** you must use this function to read only your own application + > environment. Do not read the environment of other applications. + + > **Important:** if you are writing a library to be used by other developers, + > it is generally recommended to avoid the application environment, as the + > application environment is effectively a global storage. For more information, + > read our [library guidelines](library-guidelines.md). """ @spec fetch_env(app, key) :: {:ok, value} | :error - def fetch_env(app, key) do + def fetch_env(app, key) when is_atom(app) do + maybe_warn_on_app_env_key(app, key) + case :application.get_env(app, key) do {:ok, value} -> {:ok, value} :undefined -> :error end end + @doc """ + Returns the value for `key` in `app`'s environment. + + If the configuration parameter does not exist, raises `ArgumentError`. + + > **Important:** you must use this function to read only your own application + > environment. Do not read the environment of other applications. + + > **Important:** if you are writing a library to be used by other developers, + > it is generally recommended to avoid the application environment, as the + > application environment is effectively a global storage. For more information, + > read our [library guidelines](library-guidelines.md). + """ + @spec fetch_env!(app, key) :: value + def fetch_env!(app, key) when is_atom(app) do + case fetch_env(app, key) do + {:ok, value} -> + value + + :error -> + raise ArgumentError, + "could not fetch application environment #{inspect(key)} for application " <> + "#{inspect(app)} #{fetch_env_failed_reason(app, key)}" + end + end + + defp fetch_env_failed_reason(app, key) do + vsn = :application.get_key(app, :vsn) + + case vsn do + {:ok, _} -> + "because configuration at #{inspect(key)} was not set" + + :undefined -> + "because the application was not loaded nor configured" + end + end + @doc """ Puts the `value` in `key` for the given `app`. ## Options - * `:timeout` - the timeout for the change (defaults to 5000ms) + * `:timeout` - the timeout for the change (defaults to `5_000` milliseconds) * `:persistent` - persists the given value on application load and reloads If `put_env/4` is called before the application is loaded, the application environment values specified in the `.app` file will override the ones previously set. - The persistent option can be set to true when there is a need to guarantee + The `:persistent` option can be set to `true` when there is a need to guarantee parameters set with this function will not be overridden by the ones defined in the application resource file on load. This means persistent values will stick after the application is loaded and also on application reload. """ - @spec put_env(app, key, value, [timeout: timeout, persistent: boolean]) :: :ok - def put_env(app, key, value, opts \\ []) do + @spec put_env(app, key, value, timeout: timeout, persistent: boolean) :: :ok + def put_env(app, key, value, opts \\ []) when is_atom(app) do + maybe_warn_on_app_env_key(app, key) :application.set_env(app, key, value, opts) end + @doc """ + Puts the environment for multiple apps at the same time. + + The given config should not: + + * have the same application listed more than once + * have the same key inside the same application listed more than once + + If those conditions are not met, it will raise. + + It receives the same options as `put_env/4`. Returns `:ok`. + """ + @doc since: "1.9.0" + @spec put_all_env([{app, [{key, value}]}], timeout: timeout, persistent: boolean) :: :ok + def put_all_env(config, opts \\ []) when is_list(config) and is_list(opts) do + :application.set_env(config, opts) + end + @doc """ Deletes the `key` from the given `app` environment. - See `put_env/4` for a description of the options. + It receives the same options as `put_env/4`. Returns `:ok`. """ - @spec delete_env(app, key, [timeout: timeout, persistent: boolean]) :: :ok - def delete_env(app, key, opts \\ []) do + @spec delete_env(app, key, timeout: timeout, persistent: boolean) :: :ok + def delete_env(app, key, opts \\ []) when is_atom(app) do + maybe_warn_on_app_env_key(app, key) :application.unset_env(app, key, opts) end + defp maybe_warn_on_app_env_key(_app, key) when is_atom(key), + do: :ok + + # TODO: Remove this deprecation warning on 2.0+ and allow list lookups as in compile_env. + defp maybe_warn_on_app_env_key(app, key) do + message = "passing non-atom as application env key is deprecated, got: #{inspect(key)}" + IO.warn_once({Application, :key, app, key}, message, _stacktrace_drop_levels = 2) + end + @doc """ Ensures the given `app` is started. @@ -189,11 +799,27 @@ defmodule Application do :ok = Application.ensure_started(:my_test_dep) """ - @spec ensure_started(app, start_type) :: :ok | {:error, term} + @spec ensure_started(app, restart_type) :: :ok | {:error, term} def ensure_started(app, type \\ :temporary) when is_atom(app) do :application.ensure_started(app, type) end + @doc """ + Ensures the given `app` is loaded. + + Same as `load/1` but returns `:ok` if the application was already + loaded. + """ + @doc since: "1.10.0" + @spec ensure_loaded(app) :: :ok | {:error, term} + def ensure_loaded(app) when is_atom(app) do + case :application.load(app) do + :ok -> :ok + {:error, {:already_loaded, ^app}} -> :ok + {:error, _} = error -> error + end + end + @doc """ Ensures the given `app` and its applications are started. @@ -201,7 +827,7 @@ defmodule Application do `:applications` in the `.app` file in case they were not previously started. """ - @spec ensure_all_started(app, start_type) :: {:ok, [app]} | {:error, term} + @spec ensure_all_started(app, restart_type) :: {:ok, [app]} | {:error, {app, term}} def ensure_all_started(app, type \\ :temporary) when is_atom(app) do :application.ensure_all_started(app, type) end @@ -217,7 +843,7 @@ defmodule Application do started before this application is. If not, `{:error, {:not_started, app}}` is returned, where `app` is the name of the missing application. - In case you want to automatically load **and start** all of `app`'s dependencies, + In case you want to automatically load **and start** all of `app`'s dependencies, see `ensure_all_started/2`. The `type` argument specifies the type of the application: @@ -240,7 +866,7 @@ defmodule Application do Note also that the `:transient` type is of little practical use, since when a supervision tree terminates, the reason is set to `:shutdown`, not `:normal`. """ - @spec start(app, start_type) :: :ok | {:error, term} + @spec start(app, restart_type) :: :ok | {:error, term} def start(app, type \\ :temporary) when is_atom(app) do :application.start(app, type) end @@ -251,7 +877,7 @@ defmodule Application do When stopped, the application is still loaded. """ @spec stop(app) :: :ok | {:error, term} - def stop(app) do + def stop(app) when is_atom(app) do :application.stop(app) end @@ -302,35 +928,73 @@ defmodule Application do #=> "bar-123" For more information on code paths, check the `Code` module in - Elixir and also Erlang's `:code` module. + Elixir and also Erlang's [`:code` module](`:code`). """ - @spec app_dir(app) :: String.t + @spec app_dir(app) :: String.t() def app_dir(app) when is_atom(app) do case :code.lib_dir(app) do lib when is_list(lib) -> IO.chardata_to_string(lib) - {:error, :bad_name} -> raise ArgumentError, "unknown application: #{inspect app}" + {:error, :bad_name} -> raise ArgumentError, "unknown application: #{inspect(app)}" end end @doc """ Returns the given path inside `app_dir/1`. + + If `path` is a string, then it will be used as the path inside `app_dir/1`. If + `path` is a list of strings, it will be joined (see `Path.join/1`) and the result + will be used as the path inside `app_dir/1`. + + ## Examples + + File.mkdir_p!("foo/ebin") + Code.prepend_path("foo/ebin") + + Application.app_dir(:foo, "my_path") + #=> "foo/my_path" + + Application.app_dir(:foo, ["my", "nested", "path"]) + #=> "foo/my/nested/path" + """ - @spec app_dir(app, String.t) :: String.t - def app_dir(app, path) when is_binary(path) do + @spec app_dir(app, String.t() | [String.t()]) :: String.t() + def app_dir(app, path) + + def app_dir(app, path) when is_atom(app) and is_binary(path) do Path.join(app_dir(app), path) end + def app_dir(app, path) when is_atom(app) and is_list(path) do + Path.join([app_dir(app) | path]) + end + + @doc """ + Returns a list with information about the applications which are currently running. + """ + @spec started_applications(timeout) :: [{app, description :: charlist(), vsn :: charlist()}] + def started_applications(timeout \\ 5000) do + :application.which_applications(timeout) + end + + @doc """ + Returns a list with information about the applications which have been loaded. + """ + @spec loaded_applications :: [{app, description :: charlist(), vsn :: charlist()}] + def loaded_applications do + :application.loaded_applications() + end + @doc """ Formats the error reason returned by `start/2`, - `ensure_started/2, `stop/1`, `load/1` and `unload/1`, + `ensure_started/2`, `stop/1`, `load/1` and `unload/1`, returns a string. """ - @spec format_error(any) :: String.t + @spec format_error(any) :: String.t() def format_error(reason) do try do - impl_format_error(reason) + do_format_error(reason) catch - # A user could create an error that looks like a builtin one + # A user could create an error that looks like a built-in one # causing an error. :error, _ -> inspect(reason) @@ -338,68 +1002,67 @@ defmodule Application do end # exit(:normal) call is special cased, undo the special case. - defp impl_format_error({{:EXIT, :normal}, {mod, :start, args}}) do + defp do_format_error({{:EXIT, :normal}, {mod, :start, args}}) do Exception.format_exit({:normal, {mod, :start, args}}) end # {:error, reason} return value - defp impl_format_error({reason, {mod, :start, args}}) do - Exception.format_mfa(mod, :start, args) <> " returned an error: " <> - Exception.format_exit(reason) + defp do_format_error({reason, {mod, :start, args}}) do + Exception.format_mfa(mod, :start, args) <> + " returned an error: " <> Exception.format_exit(reason) end # error or exit(reason) call, use exit reason as reason. - defp impl_format_error({:bad_return, {{mod, :start, args}, {:EXIT, reason}}}) do + defp do_format_error({:bad_return, {{mod, :start, args}, {:EXIT, reason}}}) do Exception.format_exit({reason, {mod, :start, args}}) end # bad return value - defp impl_format_error({:bad_return, {{mod, :start, args}, return}}) do - Exception.format_mfa(mod, :start, args) <> - " returned a bad value: " <> inspect(return) + defp do_format_error({:bad_return, {{mod, :start, args}, return}}) do + Exception.format_mfa(mod, :start, args) <> " returned a bad value: " <> inspect(return) end - defp impl_format_error({:already_started, app}) when is_atom(app) do + defp do_format_error({:already_started, app}) when is_atom(app) do "already started application #{app}" end - defp impl_format_error({:not_started, app}) when is_atom(app) do + defp do_format_error({:not_started, app}) when is_atom(app) do "not started application #{app}" end - defp impl_format_error({:bad_application, app}) do + defp do_format_error({:bad_application, app}) do "bad application: #{inspect(app)}" end - defp impl_format_error({:already_loaded, app}) when is_atom(app) do + defp do_format_error({:already_loaded, app}) when is_atom(app) do "already loaded application #{app}" end - defp impl_format_error({:not_loaded, app}) when is_atom(app) do + defp do_format_error({:not_loaded, app}) when is_atom(app) do "not loaded application #{app}" end - defp impl_format_error({:invalid_restart_type, restart}) do + defp do_format_error({:invalid_restart_type, restart}) do "invalid application restart type: #{inspect(restart)}" end - defp impl_format_error({:invalid_name, name}) do + defp do_format_error({:invalid_name, name}) do "invalid application name: #{inspect(name)}" end - defp impl_format_error({:invalid_options, opts}) do + defp do_format_error({:invalid_options, opts}) do "invalid application options: #{inspect(opts)}" end - defp impl_format_error({:badstartspec, spec}) do + defp do_format_error({:badstartspec, spec}) do "bad application start specs: #{inspect(spec)}" end - defp impl_format_error({'no such file or directory', file}) do + defp do_format_error({'no such file or directory', file}) do "could not find application file: #{file}" end - defp impl_format_error(reason) do + defp do_format_error(reason) do Exception.format_exit(reason) end end diff --git a/lib/elixir/lib/atom.ex b/lib/elixir/lib/atom.ex index 36e62f9f416..39418cc390e 100644 --- a/lib/elixir/lib/atom.ex +++ b/lib/elixir/lib/atom.ex @@ -1,25 +1,82 @@ defmodule Atom do - @doc """ - Convenience functions for working with atoms. + @moduledoc """ + Atoms are constants whose values are their own name. + + They are often useful to enumerate over distinct values, such as: + + iex> :apple + :apple + iex> :orange + :orange + iex> :watermelon + :watermelon + + Atoms are equal if their names are equal. + + iex> :apple == :apple + true + iex> :apple == :orange + false + + Often they are used to express the state of an operation, by using + values such as `:ok` and `:error`. + + The booleans `true` and `false` are also atoms: + + iex> true == :true + true + iex> is_atom(false) + true + iex> is_boolean(:false) + true + + Elixir allows you to skip the leading `:` for the atoms `false`, `true`, + and `nil`. + + Atoms must be composed of Unicode characters such as letters, numbers, + underscore, and `@`. If the keyword has a character that does not + belong to the category above, such as spaces, you can wrap it in + quotes: + + iex> :"this is an atom with spaces" + :"this is an atom with spaces" + """ @doc """ - Converts an atom to string. + Converts an atom to a string. Inlined by the compiler. + + ## Examples + + iex> Atom.to_string(:foo) + "foo" + """ - @spec to_string(atom) :: String.t + @spec to_string(atom) :: String.t() def to_string(atom) do - :erlang.atom_to_binary(atom, :utf8) + :erlang.atom_to_binary(atom) end @doc """ - Converts an atom to a char list. + Converts an atom to a charlist. Inlined by the compiler. + + ## Examples + + iex> Atom.to_charlist(:"An atom") + 'An atom' + """ - @spec to_char_list(atom) :: char_list - def to_char_list(atom) do + @spec to_charlist(atom) :: charlist + def to_charlist(atom) do :erlang.atom_to_list(atom) end + + @doc false + @deprecated "Use Atom.to_charlist/1 instead" + @spec to_char_list(atom) :: charlist + def to_char_list(atom), do: Atom.to_charlist(atom) end diff --git a/lib/elixir/lib/base.ex b/lib/elixir/lib/base.ex index ff01c991e73..5363e152044 100644 --- a/lib/elixir/lib/base.ex +++ b/lib/elixir/lib/base.ex @@ -3,149 +3,263 @@ defmodule Base do @moduledoc """ This module provides data encoding and decoding functions - according to [RFC 4648](http://tools.ietf.org/html/rfc4648). + according to [RFC 4648](https://tools.ietf.org/html/rfc4648). This document defines the commonly used base 16, base 32, and base 64 encoding schemes. ## Base 16 alphabet - | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | - |------:|---------:|------:|---------:|------:|---------:|------:|---------:| - | 0| 0| 4| 4| 8| 8| 12| C| - | 1| 1| 5| 5| 9| 9| 13| D| - | 2| 2| 6| 6| 10| A| 14| E| - | 3| 3| 7| 7| 11| B| 15| F| + | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | + |------:|:---------|------:|:---------|------:|:---------|------:|:---------| + | 0 | 0 | 4 | 4 | 8 | 8 | 12 | C | + | 1 | 1 | 5 | 5 | 9 | 9 | 13 | D | + | 2 | 2 | 6 | 6 | 10 | A | 14 | E | + | 3 | 3 | 7 | 7 | 11 | B | 15 | F | ## Base 32 alphabet - | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | - |------:|---------:|------:|---------:|------:|---------:|------:|---------:| - | 0| A| 9| J| 18| S| 27| 3| - | 1| B| 10| K| 19| T| 28| 4| - | 2| C| 11| L| 20| U| 29| 5| - | 3| D| 12| M| 21| V| 30| 6| - | 4| E| 13| N| 22| W| 31| 7| - | 5| F| 14| O| 23| X| | | - | 6| G| 15| P| 24| Y| (pad)| =| - | 7| H| 16| Q| 25| Z| | | - | 8| I| 17| R| 26| 2| | | + | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | + |------:|:---------|------:|:---------|------:|:---------|------:|:---------| + | 0 | A | 9 | J | 18 | S | 27 | 3 | + | 1 | B | 10 | K | 19 | T | 28 | 4 | + | 2 | C | 11 | L | 20 | U | 29 | 5 | + | 3 | D | 12 | M | 21 | V | 30 | 6 | + | 4 | E | 13 | N | 22 | W | 31 | 7 | + | 5 | F | 14 | O | 23 | X | | | + | 6 | G | 15 | P | 24 | Y | (pad) | = | + | 7 | H | 16 | Q | 25 | Z | | | + | 8 | I | 17 | R | 26 | 2 | | | ## Base 32 (extended hex) alphabet - | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | - |------:|---------:|------:|---------:|------:|---------:|------:|---------:| - | 0| 0| 9| 9| 18| I| 27| R| - | 1| 1| 10| A| 19| J| 28| S| - | 2| 2| 11| B| 20| K| 29| T| - | 3| 3| 12| C| 21| L| 30| U| - | 4| 4| 13| D| 22| M| 31| V| - | 5| 5| 14| E| 23| N| | | - | 6| 6| 15| F| 24| O| (pad)| =| - | 7| 7| 16| G| 25| P| | | - | 8| 8| 17| H| 26| Q| | | + | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | + |------:|:---------|------:|:---------|------:|:---------|------:|:---------| + | 0 | 0 | 9 | 9 | 18 | I | 27 | R | + | 1 | 1 | 10 | A | 19 | J | 28 | S | + | 2 | 2 | 11 | B | 20 | K | 29 | T | + | 3 | 3 | 12 | C | 21 | L | 30 | U | + | 4 | 4 | 13 | D | 22 | M | 31 | V | + | 5 | 5 | 14 | E | 23 | N | | | + | 6 | 6 | 15 | F | 24 | O | (pad) | = | + | 7 | 7 | 16 | G | 25 | P | | | + | 8 | 8 | 17 | H | 26 | Q | | | ## Base 64 alphabet - | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | - |------:|---------:|------:|---------:|------:|---------:|------:|---------:| - | 0| A| 17| R| 34| i| 51| z| - | 1| B| 18| S| 35| j| 52| 0| - | 2| C| 19| T| 36| k| 53| 1| - | 3| D| 20| U| 37| l| 54| 2| - | 4| E| 21| V| 38| m| 55| 3| - | 5| F| 22| W| 39| n| 56| 4| - | 6| G| 23| X| 40| o| 57| 5| - | 7| H| 24| Y| 41| p| 58| 6| - | 8| I| 25| Z| 42| q| 59| 7| - | 9| J| 26| a| 43| r| 60| 8| - | 10| K| 27| b| 44| s| 61| 9| - | 11| L| 28| c| 45| t| 62| +| - | 12| M| 29| d| 46| u| 63| /| - | 13| N| 30| e| 47| v| | | - | 14| O| 31| f| 48| w| (pad)| =| - | 15| P| 32| g| 49| x| | | - | 16| Q| 33| h| 50| y| | | + | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | + |------:|:----------|------:|:---------|------:|:---------|------:|:---------| + | 0 | A | 17 | R | 34 | i | 51 | z | + | 1 | B | 18 | S | 35 | j | 52 | 0 | + | 2 | C | 19 | T | 36 | k | 53 | 1 | + | 3 | D | 20 | U | 37 | l | 54 | 2 | + | 4 | E | 21 | V | 38 | m | 55 | 3 | + | 5 | F | 22 | W | 39 | n | 56 | 4 | + | 6 | G | 23 | X | 40 | o | 57 | 5 | + | 7 | H | 24 | Y | 41 | p | 58 | 6 | + | 8 | I | 25 | Z | 42 | q | 59 | 7 | + | 9 | J | 26 | a | 43 | r | 60 | 8 | + | 10 | K | 27 | b | 44 | s | 61 | 9 | + | 11 | L | 28 | c | 45 | t | 62 | + | + | 12 | M | 29 | d | 46 | u | 63 | / | + | 13 | N | 30 | e | 47 | v | | | + | 14 | O | 31 | f | 48 | w | (pad) | = | + | 15 | P | 32 | g | 49 | x | | | + | 16 | Q | 33 | h | 50 | y | | | ## Base 64 (URL and filename safe) alphabet - | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | - |------:|---------:|------:|---------:|------:|---------:|------:|---------:| - | 0| A| 17| R| 34| i| 51| z| - | 1| B| 18| S| 35| j| 52| 0| - | 2| C| 19| T| 36| k| 53| 1| - | 3| D| 20| U| 37| l| 54| 2| - | 4| E| 21| V| 38| m| 55| 3| - | 5| F| 22| W| 39| n| 56| 4| - | 6| G| 23| X| 40| o| 57| 5| - | 7| H| 24| Y| 41| p| 58| 6| - | 8| I| 25| Z| 42| q| 59| 7| - | 9| J| 26| a| 43| r| 60| 8| - | 10| K| 27| b| 44| s| 61| 9| - | 11| L| 28| c| 45| t| 62| -| - | 12| M| 29| d| 46| u| 63| _| - | 13| N| 30| e| 47| v| | | - | 14| O| 31| f| 48| w| (pad)| =| - | 15| P| 32| g| 49| x| | | - | 16| Q| 33| h| 50| y| | | + | Value | Encoding | Value | Encoding | Value | Encoding | Value | Encoding | + |------:|:---------|------:|:---------|------:|:---------|------:|:---------| + | 0 | A | 17 | R | 34 | i | 51 | z | + | 1 | B | 18 | S | 35 | j | 52 | 0 | + | 2 | C | 19 | T | 36 | k | 53 | 1 | + | 3 | D | 20 | U | 37 | l | 54 | 2 | + | 4 | E | 21 | V | 38 | m | 55 | 3 | + | 5 | F | 22 | W | 39 | n | 56 | 4 | + | 6 | G | 23 | X | 40 | o | 57 | 5 | + | 7 | H | 24 | Y | 41 | p | 58 | 6 | + | 8 | I | 25 | Z | 42 | q | 59 | 7 | + | 9 | J | 26 | a | 43 | r | 60 | 8 | + | 10 | K | 27 | b | 44 | s | 61 | 9 | + | 11 | L | 28 | c | 45 | t | 62 | - | + | 12 | M | 29 | d | 46 | u | 63 | _ | + | 13 | N | 30 | e | 47 | v | | | + | 14 | O | 31 | f | 48 | w | (pad) | = | + | 15 | P | 32 | g | 49 | x | | | + | 16 | Q | 33 | h | 50 | y | | | """ - b16_alphabet = Enum.with_index '0123456789ABCDEF' - b64_alphabet = Enum.with_index 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' - b64url_alphabet = Enum.with_index 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_' - b32_alphabet = Enum.with_index 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' - b32hex_alphabet = Enum.with_index '0123456789ABCDEFGHIJKLMNOPQRSTUV' - - Enum.each [ {:enc16, :dec16, b16_alphabet}, - {:enc64, :dec64, b64_alphabet}, - {:enc32, :dec32, b32_alphabet}, - {:enc64url, :dec64url, b64url_alphabet}, - {:enc32hex, :dec32hex, b32hex_alphabet} ], fn({enc, dec, alphabet}) -> - for {encoding, value} <- alphabet do - defp unquote(enc)(unquote(value)), do: unquote(encoding) - defp unquote(dec)(unquote(encoding)), do: unquote(value) + @type encode_case :: :upper | :lower + @type decode_case :: :upper | :lower | :mixed + + b16_alphabet = '0123456789ABCDEF' + b64_alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + b64url_alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_' + b32_alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' + b32hex_alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUV' + + defmacrop encode_pair(alphabet, case, value) do + quote do + case unquote(value) do + unquote(encode_pair_clauses(alphabet, case)) + end + end + end + + defp encode_pair_clauses(alphabet, case) when case in [:sensitive, :upper] do + shift = shift(alphabet) + + alphabet + |> Enum.with_index() + |> encode_clauses(shift) + end + + defp encode_pair_clauses(alphabet, :lower) do + shift = shift(alphabet) + + alphabet + |> Stream.map(fn c -> if c in ?A..?Z, do: c - ?A + ?a, else: c end) + |> Enum.with_index() + |> encode_clauses(shift) + end + + defp shift(alphabet) do + alphabet + |> length() + |> :math.log2() + |> round() + end + + defp encode_clauses(alphabet, shift) do + for {encoding1, value1} <- alphabet, + {encoding2, value2} <- alphabet do + encoding = bsl(encoding1, 8) + encoding2 + value = bsl(value1, shift) + value2 + [clause] = quote(do: (unquote(value) -> unquote(encoding))) + clause + end + end + + defmacrop decode_char(alphabet, case, encoding) do + quote do + case unquote(encoding) do + unquote(decode_char_clauses(alphabet, case)) + end end - defp unquote(dec)(c) do - raise ArgumentError, "non-alphabet digit found: #{<>}" + end + + defp decode_char_clauses(alphabet, case) when case in [:sensitive, :upper] do + clauses = + alphabet + |> Enum.with_index() + |> decode_clauses() + + clauses ++ bad_digit_clause() + end + + defp decode_char_clauses(alphabet, :lower) do + {uppers, rest} = + alphabet + |> Stream.with_index() + |> Enum.split_with(fn {encoding, _} -> encoding in ?A..?Z end) + + lowers = Enum.map(uppers, fn {encoding, value} -> {encoding - ?A + ?a, value} end) + + if length(uppers) > length(rest) do + decode_mixed_clauses(lowers, rest) + else + decode_mixed_clauses(rest, lowers) end end - defp encode_case(:upper, func), - do: func - defp encode_case(:lower, func), - do: &to_lower(func.(&1)) + defp decode_char_clauses(alphabet, :mixed) when length(alphabet) == 16 do + alphabet = Enum.with_index(alphabet) + + lowers = + alphabet + |> Stream.filter(fn {encoding, _} -> encoding in ?A..?Z end) + |> Enum.map(fn {encoding, value} -> {encoding - ?A + ?a, value} end) + + decode_mixed_clauses(alphabet, lowers) + end + + defp decode_char_clauses(alphabet, :mixed) when length(alphabet) == 32 do + clauses = + alphabet + |> Stream.with_index() + |> Enum.flat_map(fn {encoding, value} = pair -> + if encoding in ?A..?Z do + [pair, {encoding - ?A + ?a, value}] + else + [pair] + end + end) + |> decode_clauses() + + clauses ++ bad_digit_clause() + end + + defp decode_mixed_clauses(first, second) do + first_clauses = decode_clauses(first) + second_clauses = decode_clauses(second) ++ bad_digit_clause() - defp decode_case(:upper, func), - do: func - defp decode_case(:lower, func), - do: &func.(from_lower(&1)) - defp decode_case(:mixed, func), - do: &func.(from_mixed(&1)) + join_clause = + quote do + encoding -> + case encoding do + unquote(second_clauses) + end + end - defp to_lower(char) when char in ?A..?Z, - do: char + (?a - ?A) - defp to_lower(char), - do: char + first_clauses ++ join_clause + end + + defp decode_clauses(alphabet) do + for {encoding, value} <- alphabet do + [clause] = quote(do: (unquote(encoding) -> unquote(value))) + clause + end + end - defp from_lower(char) when char in ?a..?z, - do: char - (?a - ?A) - defp from_lower(char) when not char in ?A..?Z, - do: char - defp from_lower(char), - do: raise(ArgumentError, "non-alphabet digit found: #{<>}") + defp bad_digit_clause() do + quote do + c -> + raise ArgumentError, + "non-alphabet digit found: #{inspect(<>, binaries: :as_strings)} (byte #{c})" + end + end - defp from_mixed(char) when char in ?a..?z, - do: char - (?a - ?A) - defp from_mixed(char), - do: char + defp maybe_pad(body, "", _, _), do: body + defp maybe_pad(body, tail, false, _), do: body <> tail + + defp maybe_pad(body, tail, _, group_size) do + case group_size - rem(byte_size(tail), group_size) do + ^group_size -> body <> tail + 6 -> body <> tail <> "======" + 5 -> body <> tail <> "=====" + 4 -> body <> tail <> "====" + 3 -> body <> tail <> "===" + 2 -> body <> tail <> "==" + 1 -> body <> tail <> "=" + end + end @doc """ Encodes a binary string into a base 16 encoded string. - Accepts an atom `:upper` (default) for encoding to upper case characters or - `:lower` for lower case characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to use when encoding + + The values for `:case` can be: + + * `:upper` - uses upper case characters (default) + * `:lower` - uses lower case characters ## Examples @@ -156,20 +270,26 @@ defmodule Base do "666f6f626172" """ - @spec encode16(binary) :: binary - @spec encode16(binary, Keyword.t) :: binary + @spec encode16(binary, case: encode_case) :: binary def encode16(data, opts \\ []) when is_binary(data) do case = Keyword.get(opts, :case, :upper) - do_encode16(data, encode_case(case, &enc16/1)) + do_encode16(case, data) end - @doc """ Decodes a base 16 encoded string into a binary string. - Accepts an atom `:upper` (default) for decoding from upper case characters or - `:lower` for lower case characters. `:mixed` can be given for mixed case - characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to accept when decoding + + The values for `:case` can be: + + * `:upper` - only allows upper case characters (default) + * `:lower` - only allows lower case characters + * `:mixed` - allows mixed case characters ## Examples @@ -183,11 +303,9 @@ defmodule Base do {:ok, "foobar"} """ - @spec decode16(binary) :: {:ok, binary} | :error - @spec decode16(binary, Keyword.t) :: {:ok, binary} | :error - def decode16(string, opts \\ []) when is_binary(string) do - case = Keyword.get(opts, :case, :upper) - {:ok, do_decode16(string, decode_case(case, &dec16/1))} + @spec decode16(binary, case: decode_case) :: {:ok, binary} | :error + def decode16(string, opts \\ []) do + {:ok, decode16!(string, opts)} rescue ArgumentError -> :error end @@ -195,9 +313,17 @@ defmodule Base do @doc """ Decodes a base 16 encoded string into a binary string. - Accepts an atom `:upper` (default) for decoding from upper case characters or - `:lower` for lower case characters. `:mixed` can be given for mixed case - characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to accept when decoding + + The values for `:case` can be: + + * `:upper` - only allows upper case characters (default) + * `:lower` - only allows lower case characters + * `:mixed` - allows mixed case characters An `ArgumentError` exception is raised if the padding is incorrect or a non-alphabet character is present in the string. @@ -214,39 +340,71 @@ defmodule Base do "foobar" """ - @spec decode16!(binary) :: binary - @spec decode16!(binary, Keyword.t) :: binary - def decode16!(string, opts \\ []) when is_binary(string) do + @spec decode16!(binary, case: decode_case) :: binary + def decode16!(string, opts \\ []) + + def decode16!(string, opts) when is_binary(string) and rem(byte_size(string), 2) == 0 do case = Keyword.get(opts, :case, :upper) - do_decode16(string, decode_case(case, &dec16/1)) + do_decode16(case, string) + end + + def decode16!(string, _opts) when is_binary(string) do + raise ArgumentError, + "string given to decode has wrong length. An even number of bytes was expected, got: #{byte_size(string)}. " <> + "Double check your string for unwanted characters or pad it accordingly" end @doc """ Encodes a binary string into a base 64 encoded string. + Accepts `padding: false` option which will omit padding from + the output string. + ## Examples iex> Base.encode64("foobar") "Zm9vYmFy" + iex> Base.encode64("foob") + "Zm9vYg==" + + iex> Base.encode64("foob", padding: false) + "Zm9vYg" + """ - @spec encode64(binary) :: binary - def encode64(data) when is_binary(data) do - do_encode64(data, &enc64/1) + @spec encode64(binary, padding: boolean) :: binary + def encode64(data, opts \\ []) when is_binary(data) do + pad? = Keyword.get(opts, :padding, true) + do_encode64(data, pad?) end @doc """ Decodes a base 64 encoded string into a binary string. + Accepts `ignore: :whitespace` option which will ignore all the + whitespace characters in the input string. + + Accepts `padding: false` option which will ignore padding from + the input string. + ## Examples iex> Base.decode64("Zm9vYmFy") {:ok, "foobar"} + iex> Base.decode64("Zm9vYmFy\\n", ignore: :whitespace) + {:ok, "foobar"} + + iex> Base.decode64("Zm9vYg==") + {:ok, "foob"} + + iex> Base.decode64("Zm9vYg", padding: false) + {:ok, "foob"} + """ - @spec decode64(binary) :: {:ok, binary} | :error - def decode64(string) when is_binary(string) do - {:ok, do_decode64(string, &dec64/1)} + @spec decode64(binary, ignore: :whitespace, padding: boolean) :: {:ok, binary} | :error + def decode64(string, opts \\ []) when is_binary(string) do + {:ok, decode64!(string, opts)} rescue ArgumentError -> :error end @@ -254,7 +412,11 @@ defmodule Base do @doc """ Decodes a base 64 encoded string into a binary string. - The following alphabet is used both for encoding and decoding: + Accepts `ignore: :whitespace` option which will ignore all the + whitespace characters in the input string. + + Accepts `padding: false` option which will ignore padding from + the input string. An `ArgumentError` exception is raised if the padding is incorrect or a non-alphabet character is present in the string. @@ -264,40 +426,69 @@ defmodule Base do iex> Base.decode64!("Zm9vYmFy") "foobar" + iex> Base.decode64!("Zm9vYmFy\\n", ignore: :whitespace) + "foobar" + + iex> Base.decode64!("Zm9vYg==") + "foob" + + iex> Base.decode64!("Zm9vYg", padding: false) + "foob" + """ - @spec decode64!(binary) :: binary - def decode64!(string) when is_binary(string) do - do_decode64(string, &dec64/1) + @spec decode64!(binary, ignore: :whitespace, padding: boolean) :: binary + def decode64!(string, opts \\ []) when is_binary(string) do + pad? = Keyword.get(opts, :padding, true) + string |> remove_ignored(opts[:ignore]) |> do_decode64(pad?) end @doc """ Encodes a binary string into a base 64 encoded string with URL and filename safe alphabet. + Accepts `padding: false` option which will omit padding from + the output string. + ## Examples - iex> Base.url_encode64(<<255,127,254,252>>) + iex> Base.url_encode64(<<255, 127, 254, 252>>) "_3_-_A==" + iex> Base.url_encode64(<<255, 127, 254, 252>>, padding: false) + "_3_-_A" + """ - @spec url_encode64(binary) :: binary - def url_encode64(data) when is_binary(data) do - do_encode64(data, &enc64url/1) + @spec url_encode64(binary, padding: boolean) :: binary + def url_encode64(data, opts \\ []) when is_binary(data) do + pad? = Keyword.get(opts, :padding, true) + do_encode64url(data, pad?) end @doc """ Decodes a base 64 encoded string with URL and filename safe alphabet into a binary string. + Accepts `ignore: :whitespace` option which will ignore all the + whitespace characters in the input string. + + Accepts `padding: false` option which will ignore padding from + the input string. + ## Examples iex> Base.url_decode64("_3_-_A==") - {:ok, <<255,127,254,252>>} + {:ok, <<255, 127, 254, 252>>} + + iex> Base.url_decode64("_3_-_A==\\n", ignore: :whitespace) + {:ok, <<255, 127, 254, 252>>} + + iex> Base.url_decode64("_3_-_A", padding: false) + {:ok, <<255, 127, 254, 252>>} """ - @spec url_decode64(binary) :: {:ok, binary} | :error - def url_decode64(string) when is_binary(string) do - {:ok, do_decode64(string, &dec64url/1)} + @spec url_decode64(binary, ignore: :whitespace, padding: boolean) :: {:ok, binary} | :error + def url_decode64(string, opts \\ []) when is_binary(string) do + {:ok, url_decode64!(string, opts)} rescue ArgumentError -> :error end @@ -306,25 +497,52 @@ defmodule Base do Decodes a base 64 encoded string with URL and filename safe alphabet into a binary string. + Accepts `ignore: :whitespace` option which will ignore all the + whitespace characters in the input string. + + Accepts `padding: false` option which will ignore padding from + the input string. + An `ArgumentError` exception is raised if the padding is incorrect or a non-alphabet character is present in the string. ## Examples iex> Base.url_decode64!("_3_-_A==") - <<255,127,254,252>> + <<255, 127, 254, 252>> + + iex> Base.url_decode64!("_3_-_A==\\n", ignore: :whitespace) + <<255, 127, 254, 252>> + + iex> Base.url_decode64!("_3_-_A", padding: false) + <<255, 127, 254, 252>> """ - @spec url_decode64!(binary) :: binary - def url_decode64!(string) when is_binary(string) do - do_decode64(string, &dec64url/1) + @spec url_decode64!(binary, ignore: :whitespace, padding: boolean) :: binary + def url_decode64!(string, opts \\ []) when is_binary(string) do + pad? = Keyword.get(opts, :padding, true) + string |> remove_ignored(opts[:ignore]) |> do_decode64url(pad?) end @doc """ Encodes a binary string into a base 32 encoded string. - Accepts an atom `:upper` (default) for encoding to upper case characters or - `:lower` for lower case characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to use when encoding + * `:padding` - specifies whether to apply padding + + The values for `:case` can be: + + * `:upper` - uses upper case characters (default) + * `:lower` - uses lower case characters + + The values for `:padding` can be: + + * `true` - pad the output string to the nearest multiple of 8 (default) + * `false` - omit padding from the output string ## Examples @@ -334,20 +552,37 @@ defmodule Base do iex> Base.encode32("foobar", case: :lower) "mzxw6ytboi======" + iex> Base.encode32("foobar", padding: false) + "MZXW6YTBOI" + """ - @spec encode32(binary) :: binary - @spec encode32(binary, Keyword.t) :: binary + @spec encode32(binary, case: encode_case, padding: boolean) :: binary def encode32(data, opts \\ []) when is_binary(data) do case = Keyword.get(opts, :case, :upper) - do_encode32(data, encode_case(case, &enc32/1)) + pad? = Keyword.get(opts, :padding, true) + do_encode32(case, data, pad?) end @doc """ Decodes a base 32 encoded string into a binary string. - Accepts an atom `:upper` (default) for decoding from upper case characters or - `:lower` for lower case characters. `:mixed` can be given for mixed case - characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to accept when decoding + * `:padding` - specifies whether to require padding + + The values for `:case` can be: + + * `:upper` - only allows upper case characters (default) + * `:lower` - only allows lower case characters + * `:mixed` - allows mixed case characters + + The values for `:padding` can be: + + * `true` - requires the input string to be padded to the nearest multiple of 8 (default) + * `false` - ignores padding from the input string ## Examples @@ -360,12 +595,13 @@ defmodule Base do iex> Base.decode32("mzXW6ytBOi======", case: :mixed) {:ok, "foobar"} + iex> Base.decode32("MZXW6YTBOI", padding: false) + {:ok, "foobar"} + """ - @spec decode32(binary) :: {:ok, binary} | :error - @spec decode32(binary, Keyword.t) :: {:ok, binary} | :error + @spec decode32(binary, case: decode_case, padding: boolean) :: {:ok, binary} | :error def decode32(string, opts \\ []) do - case = Keyword.get(opts, :case, :upper) - {:ok, do_decode32(string, decode_case(case, &dec32/1))} + {:ok, decode32!(string, opts)} rescue ArgumentError -> :error end @@ -373,13 +609,27 @@ defmodule Base do @doc """ Decodes a base 32 encoded string into a binary string. - Accepts an atom `:upper` (default) for decoding from upper case characters or - `:lower` for lower case characters. `:mixed` can be given for mixed case - characters. - An `ArgumentError` exception is raised if the padding is incorrect or a non-alphabet character is present in the string. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to accept when decoding + * `:padding` - specifies whether to require padding + + The values for `:case` can be: + + * `:upper` - only allows upper case characters (default) + * `:lower` - only allows lower case characters + * `:mixed` - allows mixed case characters + + The values for `:padding` can be: + + * `true` - requires the input string to be padded to the nearest multiple of 8 (default) + * `false` - ignores padding from the input string + ## Examples iex> Base.decode32!("MZXW6YTBOI======") @@ -391,20 +641,37 @@ defmodule Base do iex> Base.decode32!("mzXW6ytBOi======", case: :mixed) "foobar" + iex> Base.decode32!("MZXW6YTBOI", padding: false) + "foobar" + """ - @spec decode32!(binary) :: binary - @spec decode32!(binary, Keyword.t) :: binary - def decode32!(string, opts \\ []) do + @spec decode32!(binary, case: decode_case, padding: boolean) :: binary + def decode32!(string, opts \\ []) when is_binary(string) do case = Keyword.get(opts, :case, :upper) - do_decode32(string, decode_case(case, &dec32/1)) + pad? = Keyword.get(opts, :padding, true) + do_decode32(case, string, pad?) end @doc """ Encodes a binary string into a base 32 encoded string with an extended hexadecimal alphabet. - Accepts an atom `:upper` (default) for encoding to upper case characters or - `:lower` for lower case characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to use when encoding + * `:padding` - specifies whether to apply padding + + The values for `:case` can be: + + * `:upper` - uses upper case characters (default) + * `:lower` - uses lower case characters + + The values for `:padding` can be: + + * `true` - pad the output string to the nearest multiple of 8 (default) + * `false` - omit padding from the output string ## Examples @@ -414,21 +681,38 @@ defmodule Base do iex> Base.hex_encode32("foobar", case: :lower) "cpnmuoj1e8======" + iex> Base.hex_encode32("foobar", padding: false) + "CPNMUOJ1E8" + """ - @spec hex_encode32(binary) :: binary - @spec hex_encode32(binary, Keyword.t) :: binary + @spec hex_encode32(binary, case: encode_case, padding: boolean) :: binary def hex_encode32(data, opts \\ []) when is_binary(data) do case = Keyword.get(opts, :case, :upper) - do_encode32(data, encode_case(case, &enc32hex/1)) + pad? = Keyword.get(opts, :padding, true) + do_encode32hex(case, data, pad?) end @doc """ Decodes a base 32 encoded string with extended hexadecimal alphabet into a binary string. - Accepts an atom `:upper` (default) for decoding from upper case characters or - `:lower` for lower case characters. `:mixed` can be given for mixed case - characters. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to accept when decoding + * `:padding` - specifies whether to require padding + + The values for `:case` can be: + + * `:upper` - only allows upper case characters (default) + * `:lower` - only allows lower case characters + * `:mixed` - allows mixed case characters + + The values for `:padding` can be: + + * `true` - requires the input string to be padded to the nearest multiple of 8 (default) + * `false` - ignores padding from the input string ## Examples @@ -441,12 +725,13 @@ defmodule Base do iex> Base.hex_decode32("cpnMuOJ1E8======", case: :mixed) {:ok, "foobar"} + iex> Base.hex_decode32("CPNMUOJ1E8", padding: false) + {:ok, "foobar"} + """ - @spec hex_decode32(binary) :: {:ok, binary} | :error - @spec hex_decode32(binary, Keyword.t) :: {:ok, binary} | :error - def hex_decode32(string, opts \\ []) when is_binary(string) do - case = Keyword.get(opts, :case, :upper) - {:ok, do_decode32(string, decode_case(case, &dec32hex/1))} + @spec hex_decode32(binary, case: decode_case, padding: boolean) :: {:ok, binary} | :error + def hex_decode32(string, opts \\ []) do + {:ok, hex_decode32!(string, opts)} rescue ArgumentError -> :error end @@ -455,13 +740,27 @@ defmodule Base do Decodes a base 32 encoded string with extended hexadecimal alphabet into a binary string. - Accepts an atom `:upper` (default) for decoding from upper case characters or - `:lower` for lower case characters. `:mixed` can be given for mixed case - characters. - An `ArgumentError` exception is raised if the padding is incorrect or a non-alphabet character is present in the string. + ## Options + + The accepted options are: + + * `:case` - specifies the character case to accept when decoding + * `:padding` - specifies whether to require padding + + The values for `:case` can be: + + * `:upper` - only allows upper case characters (default) + * `:lower` - only allows lower case characters + * `:mixed` - allows mixed case characters + + The values for `:padding` can be: + + * `true` - requires the input string to be padded to the nearest multiple of 8 (default) + * `false` - ignores padding from the input string + ## Examples iex> Base.hex_decode32!("CPNMUOJ1E8======") @@ -473,120 +772,523 @@ defmodule Base do iex> Base.hex_decode32!("cpnMuOJ1E8======", case: :mixed) "foobar" + iex> Base.hex_decode32!("CPNMUOJ1E8", padding: false) + "foobar" + """ - @spec hex_decode32!(binary) :: binary - @spec hex_decode32!(binary, Keyword.t) :: binary + @spec hex_decode32!(binary, case: decode_case, padding: boolean) :: binary def hex_decode32!(string, opts \\ []) when is_binary(string) do case = Keyword.get(opts, :case, :upper) - do_decode32(string, decode_case(case, &dec32hex/1)) + pad? = Keyword.get(opts, :padding, true) + do_decode32hex(case, string, pad?) + end + + defp remove_ignored(string, nil), do: string + + defp remove_ignored(string, :whitespace) do + for <>, char not in '\s\t\r\n', into: <<>>, do: <> end - defp do_encode16(<<>>, _), do: <<>> - defp do_encode16(data, enc) do - for <>, into: <<>>, do: <> + enc16 = [upper: :enc16_upper, lower: :enc16_lower] + + for {case, fun} <- enc16 do + defp unquote(fun)(char) do + encode_pair(unquote(b16_alphabet), unquote(case), char) + end end - defp do_decode16(<<>>, _), do: <<>> - defp do_decode16(string, dec) when rem(byte_size(string), 2) == 0 do - for <>, into: <<>> do - <> + defp do_encode16(_, <<>>), do: <<>> + + for {case, fun} <- enc16 do + defp do_encode16(unquote(case), data) do + split = 8 * div(byte_size(data), 8) + <> = data + + main = + for <>, into: <<>> do + << + unquote(fun)(c1)::16, + unquote(fun)(c2)::16, + unquote(fun)(c3)::16, + unquote(fun)(c4)::16, + unquote(fun)(c5)::16, + unquote(fun)(c6)::16, + unquote(fun)(c7)::16, + unquote(fun)(c8)::16 + >> + end + + case rest do + <> -> + << + main::binary, + unquote(fun)(c1)::16, + unquote(fun)(c2)::16, + unquote(fun)(c3)::16, + unquote(fun)(c4)::16, + unquote(fun)(c5)::16, + unquote(fun)(c6)::16, + unquote(fun)(c7)::16 + >> + + <> -> + << + main::binary, + unquote(fun)(c1)::16, + unquote(fun)(c2)::16, + unquote(fun)(c3)::16, + unquote(fun)(c4)::16, + unquote(fun)(c5)::16, + unquote(fun)(c6)::16 + >> + + <> -> + << + main::binary, + unquote(fun)(c1)::16, + unquote(fun)(c2)::16, + unquote(fun)(c3)::16, + unquote(fun)(c4)::16, + unquote(fun)(c5)::16 + >> + + <> -> + << + main::binary, + unquote(fun)(c1)::16, + unquote(fun)(c2)::16, + unquote(fun)(c3)::16, + unquote(fun)(c4)::16 + >> + + <> -> + <> + + <> -> + <> + + <> -> + <> + + <<>> -> + main + end end end - defp do_decode16(_, _) do - raise ArgumentError, "odd-length string" - end - - defp do_encode64(<<>>, _), do: <<>> - defp do_encode64(data, enc) do - split = 3 * div(byte_size(data), 3) - <> = data - main = for <>, into: <<>>, do: <> - case rest do - <> -> - <> - <> -> - <> - <<>> -> - main + + dec16 = [upper: :dec16_upper, lower: :dec16_lower, mixed: :dec16_mixed] + + for {case, fun} <- dec16 do + defp unquote(fun)(encoding) do + decode_char(unquote(b16_alphabet), unquote(case), encoding) end end - defp do_decode64(<<>>, _), do: <<>> - defp do_decode64(string, dec) when rem(byte_size(string), 4) == 0 do - split = byte_size(string) - 4 - <> = string - main = for <>, into: <<>>, do: <> - case rest do - <> -> - <> - <> -> - <> - <> -> - <> - <<>> -> - main + defp do_decode16(_, <<>>), do: <<>> + + for {case, fun} <- dec16 do + defp do_decode16(unquote(case), string) do + split = 8 * div(byte_size(string), 8) + <> = string + + main = + for <>, into: <<>> do + << + unquote(fun)(c1)::4, + unquote(fun)(c2)::4, + unquote(fun)(c3)::4, + unquote(fun)(c4)::4, + unquote(fun)(c5)::4, + unquote(fun)(c6)::4, + unquote(fun)(c7)::4, + unquote(fun)(c8)::4 + >> + end + + case rest do + <> -> + << + main::bits, + unquote(fun)(c1)::4, + unquote(fun)(c2)::4, + unquote(fun)(c3)::4, + unquote(fun)(c4)::4, + unquote(fun)(c5)::4, + unquote(fun)(c6)::4 + >> + + <> -> + << + main::bits, + unquote(fun)(c1)::4, + unquote(fun)(c2)::4, + unquote(fun)(c3)::4, + unquote(fun)(c4)::4 + >> + + <> -> + <> + + <<_::8>> -> + raise ArgumentError, + "string given to decode has wrong length. An even number of bytes was expected, got: #{byte_size(string)}. " <> + "Double check your string for unwanted characters or pad it accordingly" + + <<>> -> + main + end end end - defp do_decode64(_, _) do - raise ArgumentError, "incorrect padding" - end - - defp do_encode32(<<>>, _), do: <<>> - defp do_encode32(data, enc) do - split = 5 * div(byte_size(data), 5) - <> = data - main = for <>, into: <<>>, do: <> - case rest do - <> -> - <> - <> -> - <> - <> -> - <> - <> -> - <> - <<>> -> - main + + for {base, alphabet} <- ["64": b64_alphabet, "64url": b64url_alphabet] do + pair = :"enc#{base}_pair" + char = :"enc#{base}_char" + do_encode = :"do_encode#{base}" + + defp unquote(pair)(value) do + encode_pair(unquote(alphabet), :sensitive, value) + end + + defp unquote(char)(value) do + value + |> unquote(pair)() + |> band(0x00FF) + end + + defp unquote(do_encode)(<<>>, _), do: <<>> + + defp unquote(do_encode)(data, pad?) do + split = 6 * div(byte_size(data), 6) + <> = data + + main = + for <>, into: <<>> do + << + unquote(pair)(c1)::16, + unquote(pair)(c2)::16, + unquote(pair)(c3)::16, + unquote(pair)(c4)::16 + >> + end + + tail = + case rest do + <> -> + << + unquote(pair)(c1)::16, + unquote(pair)(c2)::16, + unquote(pair)(c3)::16, + unquote(char)(bsl(c, 2))::8 + >> + + <> -> + <> + + <> -> + <> + + <> -> + <> + + <> -> + <> + + <<>> -> + <<>> + end + + maybe_pad(main, tail, pad?, 4) end end - defp do_decode32(<<>>, _), do: <<>> - defp do_decode32(string, dec) when rem(byte_size(string), 8) == 0 do - split = byte_size(string) - 8 - <> = string - main = for <>, into: <<>>, do: <> - case rest do - <> -> - <> - <> -> - <> - <> -> - <> - <> -> - <> - <> -> - <> - <<>> -> - main + for {base, alphabet} <- ["64": b64_alphabet, "64url": b64url_alphabet] do + fun = :"dec#{base}" + do_decode = :"do_decode#{base}" + + defp unquote(fun)(encoding) do + decode_char(unquote(alphabet), :sensitive, encoding) + end + + defp unquote(do_decode)(<<>>, _), do: <<>> + + defp unquote(do_decode)(string, pad?) do + segs = div(byte_size(string) + 7, 8) - 1 + <> = string + + main = + for <>, into: <<>> do + << + unquote(fun)(c1)::6, + unquote(fun)(c2)::6, + unquote(fun)(c3)::6, + unquote(fun)(c4)::6, + unquote(fun)(c5)::6, + unquote(fun)(c6)::6, + unquote(fun)(c7)::6, + unquote(fun)(c8)::6 + >> + end + + case rest do + <> -> + <> + + <> -> + <> + + <> -> + << + main::bits, + unquote(fun)(c1)::6, + unquote(fun)(c2)::6, + unquote(fun)(c3)::6, + unquote(fun)(c4)::6 + >> + + <> -> + << + main::bits, + unquote(fun)(c1)::6, + unquote(fun)(c2)::6, + unquote(fun)(c3)::6, + unquote(fun)(c4)::6, + unquote(fun)(c5)::6, + bsr(unquote(fun)(c6), 4)::2 + >> + + <> -> + << + main::bits, + unquote(fun)(c1)::6, + unquote(fun)(c2)::6, + unquote(fun)(c3)::6, + unquote(fun)(c4)::6, + unquote(fun)(c5)::6, + unquote(fun)(c6)::6, + bsr(unquote(fun)(c7), 2)::4 + >> + + <> -> + << + main::bits, + unquote(fun)(c1)::6, + unquote(fun)(c2)::6, + unquote(fun)(c3)::6, + unquote(fun)(c4)::6, + unquote(fun)(c5)::6, + unquote(fun)(c6)::6, + unquote(fun)(c7)::6, + unquote(fun)(c8)::6 + >> + + <> when not pad? -> + <> + + <> when not pad? -> + <> + + <> when not pad? -> + << + main::bits, + unquote(fun)(c1)::6, + unquote(fun)(c2)::6, + unquote(fun)(c3)::6, + unquote(fun)(c4)::6, + unquote(fun)(c5)::6, + bsr(unquote(fun)(c6), 4)::2 + >> + + <> when not pad? -> + << + main::bits, + unquote(fun)(c1)::6, + unquote(fun)(c2)::6, + unquote(fun)(c3)::6, + unquote(fun)(c4)::6, + unquote(fun)(c5)::6, + unquote(fun)(c6)::6, + bsr(unquote(fun)(c7), 2)::4 + >> + + _ -> + raise ArgumentError, "incorrect padding" + end end end - defp do_decode32(_, _) do - raise ArgumentError, "incorrect padding" + + for {base, alphabet} <- ["32": b32_alphabet, "32hex": b32hex_alphabet], + case <- [:upper, :lower] do + pair = :"enc#{base}_#{case}_pair" + char = :"enc#{base}_#{case}_char" + do_encode = :"do_encode#{base}" + + defp unquote(pair)(value) do + encode_pair(unquote(alphabet), unquote(case), value) + end + + defp unquote(char)(value) do + value + |> unquote(pair)() + |> band(0x00FF) + end + + defp unquote(do_encode)(_, <<>>, _), do: <<>> + + defp unquote(do_encode)(unquote(case), data, pad?) do + split = 5 * div(byte_size(data), 5) + <> = data + + main = + for <>, into: <<>> do + << + unquote(pair)(c1)::16, + unquote(pair)(c2)::16, + unquote(pair)(c3)::16, + unquote(pair)(c4)::16 + >> + end + + tail = + case rest do + <> -> + << + unquote(pair)(c1)::16, + unquote(pair)(c2)::16, + unquote(pair)(c3)::16, + unquote(char)(bsl(c4, 3))::8 + >> + + <> -> + <> + + <> -> + <> + + <> -> + <> + + <<>> -> + <<>> + end + + maybe_pad(main, tail, pad?, 8) + end end + for {base, alphabet} <- ["32": b32_alphabet, "32hex": b32hex_alphabet], + case <- [:upper, :lower, :mixed] do + fun = :"dec#{base}_#{case}" + do_decode = :"do_decode#{base}" + + defp unquote(fun)(encoding) do + decode_char(unquote(alphabet), unquote(case), encoding) + end + + defp unquote(do_decode)(_, <<>>, _), do: <<>> + + defp unquote(do_decode)(unquote(case), string, pad?) do + segs = div(byte_size(string) + 7, 8) - 1 + <> = string + + main = + for <>, into: <<>> do + << + unquote(fun)(c1)::5, + unquote(fun)(c2)::5, + unquote(fun)(c3)::5, + unquote(fun)(c4)::5, + unquote(fun)(c5)::5, + unquote(fun)(c6)::5, + unquote(fun)(c7)::5, + unquote(fun)(c8)::5 + >> + end + + case rest do + <> -> + <> + + <> -> + << + main::bits, + unquote(fun)(c1)::5, + unquote(fun)(c2)::5, + unquote(fun)(c3)::5, + bsr(unquote(fun)(c4), 4)::1 + >> + + <> -> + << + main::bits, + unquote(fun)(c1)::5, + unquote(fun)(c2)::5, + unquote(fun)(c3)::5, + unquote(fun)(c4)::5, + bsr(unquote(fun)(c5), 1)::4 + >> + + <> -> + << + main::bits, + unquote(fun)(c1)::5, + unquote(fun)(c2)::5, + unquote(fun)(c3)::5, + unquote(fun)(c4)::5, + unquote(fun)(c5)::5, + unquote(fun)(c6)::5, + bsr(unquote(fun)(c7), 3)::2 + >> + + <> -> + << + main::bits, + unquote(fun)(c1)::5, + unquote(fun)(c2)::5, + unquote(fun)(c3)::5, + unquote(fun)(c4)::5, + unquote(fun)(c5)::5, + unquote(fun)(c6)::5, + unquote(fun)(c7)::5, + unquote(fun)(c8)::5 + >> + + <> when not pad? -> + <> + + <> when not pad? -> + << + main::bits, + unquote(fun)(c1)::5, + unquote(fun)(c2)::5, + unquote(fun)(c3)::5, + bsr(unquote(fun)(c4), 4)::1 + >> + + <> when not pad? -> + << + main::bits, + unquote(fun)(c1)::5, + unquote(fun)(c2)::5, + unquote(fun)(c3)::5, + unquote(fun)(c4)::5, + bsr(unquote(fun)(c5), 1)::4 + >> + + <> when not pad? -> + << + main::bits, + unquote(fun)(c1)::5, + unquote(fun)(c2)::5, + unquote(fun)(c3)::5, + unquote(fun)(c4)::5, + unquote(fun)(c5)::5, + unquote(fun)(c6)::5, + bsr(unquote(fun)(c7), 3)::2 + >> + + _ -> + raise ArgumentError, "incorrect padding" + end + end + end end diff --git a/lib/elixir/lib/behaviour.ex b/lib/elixir/lib/behaviour.ex index 8f882a00294..bfbacce7202 100644 --- a/lib/elixir/lib/behaviour.ex +++ b/lib/elixir/lib/behaviour.ex @@ -1,63 +1,36 @@ defmodule Behaviour do @moduledoc """ - Utilities for defining behaviour interfaces. + Mechanism for handling behaviours. - Behaviours can be referenced by other modules - to ensure they implement required callbacks. + This module is deprecated. Instead of `defcallback/1` and + `defmacrocallback/1`, the `@callback` and `@macrocallback` + module attributes can be used respectively. See the + documentation for `Module` for more information on these + attributes. - For example, you can specify the `URI.Parser` - behaviour as follows: - - defmodule URI.Parser do - use Behaviour - - @doc "Parses the given URL" - defcallback parse(uri_info :: URI.t) :: URI.t - - @doc "Defines a default port" - defcallback default_port() :: integer - end - - And then a module may use it as: - - defmodule URI.HTTP do - @behaviour URI.Parser - def default_port(), do: 80 - def parse(info), do: info - end - - If the behaviour changes or `URI.HTTP` does - not implement one of the callbacks, a warning - will be raised. - - ## Implementation - - Since Erlang R15, behaviours must be defined via - `@callback` attributes. `defcallback` is a simple - mechanism that defines the `@callback` attribute - according to the given type specification. `defcallback` allows - documentation to be created for the callback and defines - a custom function signature. - - The callbacks and their documentation can be retrieved - via the `__behaviour__` callback function. + Instead of `MyModule.__behaviour__(:callbacks)`, + `MyModule.behaviour_info(:callbacks)` can be used. """ + @moduledoc deprecated: "Use @callback and @macrocallback attributes instead" + @doc """ - Define a function callback according to the given type specification. + Defines a function callback according to the given type specification. """ + @deprecated "Use the @callback module attribute instead" defmacro defcallback(spec) do - do_defcallback(split_spec(spec, quote(do: term)), __CALLER__) + do_defcallback(:def, split_spec(spec, quote(do: term))) end @doc """ - Define a macro callback according to the given type specification. + Defines a macro callback according to the given type specification. """ + @deprecated "Use the @macrocallback module attribute instead" defmacro defmacrocallback(spec) do - do_defmacrocallback(split_spec(spec, quote(do: Macro.t)), __CALLER__) + do_defcallback(:defmacro, split_spec(spec, quote(do: Macro.t()))) end - defp split_spec({:when, _, [{:::, _, [spec, return]}, guard]}, _default) do + defp split_spec({:when, _, [{:"::", _, [spec, return]}, guard]}, _default) do {spec, return, guard} end @@ -65,7 +38,7 @@ defmodule Behaviour do {spec, default, guard} end - defp split_spec({:::, _, [spec, return]}, _default) do + defp split_spec({:"::", _, [spec, return]}, _default) do {spec, return, []} end @@ -73,40 +46,38 @@ defmodule Behaviour do {spec, default, []} end - defp do_defcallback({spec, return, guards}, caller) do + defp do_defcallback(kind, {spec, return, guards}) do case Macro.decompose_call(spec) do {name, args} -> - do_callback(:def, name, args, name, length(args), args, return, guards, caller) - _ -> - raise ArgumentError, "invalid syntax in defcallback #{Macro.to_string(spec)}" - end - end + do_callback(kind, name, args, return, guards) - defp do_defmacrocallback({spec, return, guards}, caller) do - case Macro.decompose_call(spec) do - {name, args} -> - do_callback(:defmacro, :"MACRO-#{name}", [quote(do: env :: Macro.Env.t)|args], - name, length(args), args, return, guards, caller) _ -> - raise ArgumentError, "invalid syntax in defmacrocallback #{Macro.to_string(spec)}" + raise ArgumentError, "invalid syntax in #{kind}callback #{Macro.to_string(spec)}" end end - defp do_callback(kind, name, args, docs_name, docs_arity, _docs_args, return, guards, caller) do - Enum.each args, fn - {:::, _, [left, right]} -> + defp do_callback(kind, name, args, return, guards) do + fun = fn + {:"::", _, [left, right]} -> ensure_not_default(left) ensure_not_default(right) left + other -> ensure_not_default(other) other end - quote do - @callback unquote(name)(unquote_splicing(args)) :: unquote(return) when unquote(guards) - Behaviour.store_docs(__MODULE__, unquote(caller.line), unquote(kind), - unquote(docs_name), unquote(docs_arity)) + :lists.foreach(fun, args) + + spec = + quote do + unquote(name)(unquote_splicing(args)) :: unquote(return) when unquote(guards) + end + + case kind do + :def -> quote(do: @callback(unquote(spec))) + :defmacro -> quote(do: @macrocallback(unquote(spec))) end end @@ -116,37 +87,37 @@ defmodule Behaviour do defp ensure_not_default(_), do: :ok - @doc false - def store_docs(module, line, kind, name, arity) do - doc = Module.get_attribute module, :doc - Module.delete_attribute module, :doc - Module.put_attribute module, :behaviour_docs, {{name, arity}, line, kind, doc} - end - @doc false defmacro __using__(_) do quote do - Module.register_attribute(__MODULE__, :behaviour_docs, accumulate: true) - @before_compile unquote(__MODULE__) - import unquote(__MODULE__) - end - end + warning = + "the Behaviour module is deprecated. Instead of using this module, " <> + "use the @callback and @macrocallback module attributes. See the " <> + "documentation for Module for more information on these attributes" - @doc false - defmacro __before_compile__(env) do - docs = if Code.compiler_options[:docs] do - Enum.reverse Module.get_attribute(env.module, :behaviour_docs) - end + IO.warn(warning) - quote do @doc false def __behaviour__(:callbacks) do __MODULE__.behaviour_info(:callbacks) end def __behaviour__(:docs) do - unquote(Macro.escape(docs)) + {:docs_v1, _, :elixir, _, _, _, docs} = Code.fetch_docs(__MODULE__) + + for {{kind, name, arity}, line, _, doc, _} <- docs, kind in [:callback, :macrocallback] do + case kind do + :callback -> {{name, arity}, line, :def, __behaviour__doc_value(doc)} + :macrocallback -> {{name, arity}, line, :defmacro, __behaviour__doc_value(doc)} + end + end end + + defp __behaviour__doc_value(%{"en" => doc}), do: doc + defp __behaviour__doc_value(:hidden), do: false + defp __behaviour__doc_value(_), do: nil + + import unquote(__MODULE__) end end end diff --git a/lib/elixir/lib/bitwise.ex b/lib/elixir/lib/bitwise.ex index 9d74a608983..c59950be068 100644 --- a/lib/elixir/lib/bitwise.ex +++ b/lib/elixir/lib/bitwise.ex @@ -1,41 +1,40 @@ defmodule Bitwise do @moduledoc """ - This module provides macros and operators for bitwise operators. - These macros can be used in guards. + A set of functions that perform calculations on bits. - The easiest way to use is to simply import them into - your module: + All bitwise functions work only on integers; otherwise an + `ArithmeticError` is raised. The functions `band/2`, + `bor/2`, `bsl/2`, and `bsr/2` also have operators, + respectively: `&&&/2`, `|||/2`, `<<>>/2`. - iex> use Bitwise - iex> bnot 1 - -2 - iex> 1 &&& 1 - 1 + ## Guards - You can select to include only or skip operators by passing options: + All bitwise functions can be used in guards: - iex> use Bitwise, only_operators: true - iex> 1 &&& 1 - 1 + iex> odd? = fn + ...> int when Bitwise.band(int, 1) == 1 -> true + ...> _ -> false + ...> end + iex> odd?.(1) + true + All functions in this module are inlined by the compiler. """ - @doc """ - Allow a developer to use this module in their programs with - the following options: + @doc false + @deprecated "import Bitwise instead" + defmacro __using__(options) do + except = + cond do + Keyword.get(options, :only_operators) -> + [bnot: 1, band: 2, bor: 2, bxor: 2, bsl: 2, bsr: 2] - * `:only_operators` - include only operators - * `:skip_operators` - skip operators + Keyword.get(options, :skip_operators) -> + ["~~~": 1, &&&: 2, |||: 2, "^^^": 2, <<<: 2, >>>: 2] - """ - defmacro __using__(options) do - except = cond do - Keyword.get(options, :only_operators) -> - [bnot: 1, band: 2, bor: 2, bxor: 2, bsl: 2, bsr: 2] - Keyword.get(options, :skip_operators) -> - [~~~: 1, &&&: 2, |||: 2, ^^^: 2, <<<: 2, >>>: 2] - true -> [] - end + true -> + [] + end quote do import Bitwise, except: unquote(except) @@ -43,86 +42,229 @@ defmodule Bitwise do end @doc """ - Bitwise not. + Calculates the bitwise NOT of the argument. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> bnot(2) + -3 + + iex> bnot(2) &&& 3 + 1 + """ - defmacro bnot(expr) do - quote do: :erlang.bnot(unquote(expr)) + @doc guard: true + @spec bnot(integer) :: integer + def bnot(expr) do + :erlang.bnot(expr) end - @doc """ - Bitwise not as operator. - """ - defmacro ~~~expr do - quote do: :erlang.bnot(unquote(expr)) + @doc false + def unquote(:"~~~")(expr) do + :erlang.bnot(expr) end @doc """ - Bitwise and. + Calculates the bitwise AND of its arguments. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> band(9, 3) + 1 + """ - defmacro band(left, right) do - quote do: :erlang.band(unquote(left), unquote(right)) + @doc guard: true + @spec band(integer, integer) :: integer + def band(left, right) do + :erlang.band(left, right) end @doc """ - Bitwise and as operator. + Bitwise AND operator. + + Calculates the bitwise AND of its arguments. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> 9 &&& 3 + 1 + """ - defmacro left &&& right do - quote do: :erlang.band(unquote(left), unquote(right)) + @doc guard: true + @spec integer &&& integer :: integer + def left &&& right do + :erlang.band(left, right) end @doc """ - Bitwise or. + Calculates the bitwise OR of its arguments. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> bor(9, 3) + 11 + """ - defmacro bor(left, right) do - quote do: :erlang.bor(unquote(left), unquote(right)) + @doc guard: true + @spec bor(integer, integer) :: integer + def bor(left, right) do + :erlang.bor(left, right) end @doc """ - Bitwise or as operator. + Bitwise OR operator. + + Calculates the bitwise OR of its arguments. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> 9 ||| 3 + 11 + """ - defmacro left ||| right do - quote do: :erlang.bor(unquote(left), unquote(right)) + @doc guard: true + @spec integer ||| integer :: integer + def left ||| right do + :erlang.bor(left, right) end @doc """ - Bitwise xor. + Calculates the bitwise XOR of its arguments. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> bxor(9, 3) + 10 + """ - defmacro bxor(left, right) do - quote do: :erlang.bxor(unquote(left), unquote(right)) + @doc guard: true + @spec bxor(integer, integer) :: integer + def bxor(left, right) do + :erlang.bxor(left, right) end - @doc """ - Bitwise xor as operator. - """ - defmacro left ^^^ right do - quote do: :erlang.bxor(unquote(left), unquote(right)) + @doc false + def unquote(:"^^^")(left, right) do + :erlang.bxor(left, right) end @doc """ - Arithmetic bitshift left. + Calculates the result of an arithmetic left bitshift. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> bsl(1, 2) + 4 + + iex> bsl(1, -2) + 0 + + iex> bsl(-1, 2) + -4 + + iex> bsl(-1, -2) + -1 + """ - defmacro bsl(left, right) do - quote do: :erlang.bsl(unquote(left), unquote(right)) + @doc guard: true + @spec bsl(integer, integer) :: integer + def bsl(left, right) do + :erlang.bsl(left, right) end @doc """ - Arithmetic bitshift left as operator. + Arithmetic left bitshift operator. + + Calculates the result of an arithmetic left bitshift. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> 1 <<< 2 + 4 + + iex> 1 <<< -2 + 0 + + iex> -1 <<< 2 + -4 + + iex> -1 <<< -2 + -1 + """ - defmacro left <<< right do - quote do: :erlang.bsl(unquote(left), unquote(right)) + @doc guard: true + @spec integer <<< integer :: integer + def left <<< right do + :erlang.bsl(left, right) end @doc """ - Arithmetic bitshift right. + Calculates the result of an arithmetic right bitshift. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> bsr(1, 2) + 0 + + iex> bsr(1, -2) + 4 + + iex> bsr(-1, 2) + -1 + + iex> bsr(-1, -2) + -4 + """ - defmacro bsr(left, right) do - quote do: :erlang.bsr(unquote(left), unquote(right)) + @doc guard: true + @spec bsr(integer, integer) :: integer + def bsr(left, right) do + :erlang.bsr(left, right) end @doc """ - Arithmetic bitshift right as operator. + Arithmetic right bitshift operator. + + Calculates the result of an arithmetic right bitshift. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> 1 >>> 2 + 0 + + iex> 1 >>> -2 + 4 + + iex> -1 >>> 2 + -1 + + iex> -1 >>> -2 + -4 + """ - defmacro left >>> right do - quote do: :erlang.bsr(unquote(left), unquote(right)) + @doc guard: true + @spec integer >>> integer :: integer + def left >>> right do + :erlang.bsr(left, right) end end diff --git a/lib/elixir/lib/calendar.ex b/lib/elixir/lib/calendar.ex new file mode 100644 index 00000000000..55dd53c8682 --- /dev/null +++ b/lib/elixir/lib/calendar.ex @@ -0,0 +1,886 @@ +defmodule Calendar do + @moduledoc """ + This module defines the responsibilities for working with + calendars, dates, times and datetimes in Elixir. + + Currently it defines types and the minimal implementation + for a calendar behaviour in Elixir. The goal of the Calendar + features in Elixir is to provide a base for interoperability + instead of full-featured datetime API. + + For the actual date, time and datetime structures, see `Date`, + `Time`, `NaiveDateTime` and `DateTime`. + + Note designations for year, month, day, and the like, are overspecified + (i.e. an integer instead of `1..12` for months) because different + calendars may have a different number of days per month, months per year and so on. + """ + + @type year :: integer + @type month :: pos_integer + @type day :: pos_integer + @type week :: pos_integer + @type day_of_week :: non_neg_integer + @type era :: non_neg_integer + + @typedoc """ + A tuple representing the `day` and the `era`. + """ + @type day_of_era :: {day :: non_neg_integer(), era} + + @type hour :: non_neg_integer + @type minute :: non_neg_integer + @type second :: non_neg_integer + + @typedoc """ + The internal time format is used when converting between calendars. + + It represents time as a fraction of a day (starting from midnight). + `parts_in_day` specifies how much of the day is already passed, + while `parts_per_day` signifies how many parts there fit in a day. + """ + @type day_fraction :: {parts_in_day :: non_neg_integer, parts_per_day :: pos_integer} + + @typedoc """ + The internal date format that is used when converting between calendars. + + This is the number of days including the fractional part that has passed of + the last day since 0000-01-01+00:00T00:00.000000 in ISO 8601 notation (also + known as midnight 1 January BC 1 of the proleptic Gregorian calendar). + """ + @type iso_days :: {days :: integer, day_fraction} + + @typedoc """ + Microseconds with stored precision. + + The precision represents the number of digits that must be used when + representing the microseconds to external format. If the precision is 0, + it means microseconds must be skipped. + """ + @type microsecond :: {value :: non_neg_integer, precision :: non_neg_integer} + + @typedoc "A calendar implementation" + @type calendar :: module + + @typedoc "The time zone ID according to the IANA tz database (for example, Europe/Zurich)" + @type time_zone :: String.t() + + @typedoc "The time zone abbreviation (for example, CET or CEST or BST, and such)" + @type zone_abbr :: String.t() + + @typedoc """ + The time zone UTC offset in seconds for standard time. + + See also `t:std_offset/0`. + """ + @type utc_offset :: integer + + @typedoc """ + The time zone standard offset in seconds (typically not zero in summer times). + + It must be added to `t:utc_offset/0` to get the total offset from UTC used for "wall time". + """ + @type std_offset :: integer + + @typedoc "Any map/struct that contains the date fields" + @type date :: %{optional(any) => any, calendar: calendar, year: year, month: month, day: day} + + @typedoc "Any map/struct that contains the time fields" + @type time :: %{ + optional(any) => any, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + + @typedoc "Any map/struct that contains the naive_datetime fields" + @type naive_datetime :: %{ + optional(any) => any, + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + + @typedoc "Any map/struct that contains the datetime fields" + @type datetime :: %{ + optional(any) => any, + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: time_zone, + zone_abbr: zone_abbr, + utc_offset: utc_offset, + std_offset: std_offset + } + + @typedoc """ + Specifies the time zone database for calendar operations. + + Many functions in the `DateTime` module require a time zone database. + By default, it uses the default time zone database returned by + `Calendar.get_time_zone_database/0`, which defaults to + `Calendar.UTCOnlyTimeZoneDatabase` which only handles "Etc/UTC" + datetimes and returns `{:error, :utc_only_time_zone_database}` + for any other time zone. + + Other time zone databases (including ones provided by packages) + can be configured as default either via configuration: + + config :elixir, :time_zone_database, CustomTimeZoneDatabase + + or by calling `Calendar.put_time_zone_database/1`. + + See `Calendar.TimeZoneDatabase` for more information on custom + time zone databases. + """ + @type time_zone_database :: module() + + @doc """ + Returns how many days there are in the given year-month. + """ + @callback days_in_month(year, month) :: day + + @doc """ + Returns how many months there are in the given year. + """ + @callback months_in_year(year) :: month + + @doc """ + Returns `true` if the given year is a leap year. + + A leap year is a year of a longer length than normal. The exact meaning + is up to the calendar. A calendar must return `false` if it does not support + the concept of leap years. + """ + @callback leap_year?(year) :: boolean + + @doc """ + Calculates the day of the week from the given `year`, `month`, and `day`. + + The `starting_on` represents the starting day of the week. All + calendars must support at least the `:default` value. They may + also support other values representing their days of the week. + """ + @callback day_of_week(year, month, day, starting_on :: :default | atom) :: + {day_of_week(), first_day_of_week :: non_neg_integer(), + last_day_of_week :: non_neg_integer()} + + @doc """ + Calculates the day of the year from the given `year`, `month`, and `day`. + """ + @callback day_of_year(year, month, day) :: non_neg_integer() + + @doc """ + Calculates the quarter of the year from the given `year`, `month`, and `day`. + """ + @callback quarter_of_year(year, month, day) :: non_neg_integer() + + @doc """ + Calculates the year and era from the given `year`. + """ + @callback year_of_era(year, month, day) :: {year, era} + + @doc """ + Calculates the day and era from the given `year`, `month`, and `day`. + """ + @callback day_of_era(year, month, day) :: day_of_era() + + @doc """ + Converts the date into a string according to the calendar. + """ + @callback date_to_string(year, month, day) :: String.t() + + @doc """ + Converts the datetime (without time zone) into a string according to the calendar. + """ + @callback naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) :: + String.t() + + @doc """ + Converts the datetime (with time zone) into a string according to the calendar. + """ + @callback datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + zone_abbr, + utc_offset, + std_offset + ) :: String.t() + + @doc """ + Converts the time into a string according to the calendar. + """ + @callback time_to_string(hour, minute, second, microsecond) :: String.t() + + @doc """ + Converts the given datetime (without time zone) into the `t:iso_days/0` format. + """ + @callback naive_datetime_to_iso_days(year, month, day, hour, minute, second, microsecond) :: + iso_days + + @doc """ + Converts `t:iso_days/0` to the Calendar's datetime format. + """ + @callback naive_datetime_from_iso_days(iso_days) :: + {year, month, day, hour, minute, second, microsecond} + + @doc """ + Converts the given time to the `t:day_fraction/0` format. + """ + @callback time_to_day_fraction(hour, minute, second, microsecond) :: day_fraction + + @doc """ + Converts `t:day_fraction/0` to the Calendar's time format. + """ + @callback time_from_day_fraction(day_fraction) :: {hour, minute, second, microsecond} + + @doc """ + Define the rollover moment for the given calendar. + + This is the moment, in your calendar, when the current day ends + and the next day starts. + + The result of this function is used to check if two calendars rollover at + the same time of day. If they do not, we can only convert datetimes and times + between them. If they do, this means that we can also convert dates as well + as naive datetimes between them. + + This day fraction should be in its most simplified form possible, to make comparisons fast. + + ## Examples + + * If, in your Calendar, a new day starts at midnight, return {0, 1}. + * If, in your Calendar, a new day starts at sunrise, return {1, 4}. + * If, in your Calendar, a new day starts at noon, return {1, 2}. + * If, in your Calendar, a new day starts at sunset, return {3, 4}. + + """ + @callback day_rollover_relative_to_midnight_utc() :: day_fraction + + @doc """ + Should return `true` if the given date describes a proper date in the calendar. + """ + @callback valid_date?(year, month, day) :: boolean + + @doc """ + Should return `true` if the given time describes a proper time in the calendar. + """ + @callback valid_time?(hour, minute, second, microsecond) :: boolean + + @doc """ + Parses the string representation for a time returned by `c:time_to_string/4` + into a time-tuple. + """ + @doc since: "1.10.0" + @callback parse_time(String.t()) :: + {:ok, {hour, minute, second, microsecond}} + | {:error, atom} + + @doc """ + Parses the string representation for a date returned by `c:date_to_string/3` + into a date-tuple. + """ + @doc since: "1.10.0" + @callback parse_date(String.t()) :: + {:ok, {year, month, day}} + | {:error, atom} + + @doc """ + Parses the string representation for a naive datetime returned by + `c:naive_datetime_to_string/7` into a naive-datetime-tuple. + + The given string may contain a timezone offset but it is ignored. + """ + @doc since: "1.10.0" + @callback parse_naive_datetime(String.t()) :: + {:ok, {year, month, day, hour, minute, second, microsecond}} + | {:error, atom} + + @doc """ + Parses the string representation for a datetime returned by + `c:datetime_to_string/11` into a datetime-tuple. + + The returned datetime must be in UTC. The original `utc_offset` + it was written in must be returned in the result. + """ + @doc since: "1.10.0" + @callback parse_utc_datetime(String.t()) :: + {:ok, {year, month, day, hour, minute, second, microsecond}, utc_offset} + | {:error, atom} + + # General Helpers + + @doc """ + Returns `true` if two calendars have the same moment of starting a new day, + `false` otherwise. + + If two calendars are not compatible, we can only convert datetimes and times + between them. If they are compatible, this means that we can also convert + dates as well as naive datetimes between them. + """ + @doc since: "1.5.0" + @spec compatible_calendars?(Calendar.calendar(), Calendar.calendar()) :: boolean + def compatible_calendars?(calendar, calendar), do: true + + def compatible_calendars?(calendar1, calendar2) do + calendar1.day_rollover_relative_to_midnight_utc() == + calendar2.day_rollover_relative_to_midnight_utc() + end + + @doc """ + Returns a microsecond tuple truncated to a given precision (`:microsecond`, + `:millisecond` or `:second`). + """ + @doc since: "1.6.0" + @spec truncate(Calendar.microsecond(), :microsecond | :millisecond | :second) :: + Calendar.microsecond() + def truncate(microsecond_tuple, :microsecond), do: microsecond_tuple + + def truncate({microsecond, precision}, :millisecond) do + output_precision = min(precision, 3) + {div(microsecond, 1000) * 1000, output_precision} + end + + def truncate(_, :second), do: {0, 0} + + @doc """ + Sets the current time zone database. + """ + @doc since: "1.8.0" + @spec put_time_zone_database(time_zone_database()) :: :ok + def put_time_zone_database(database) do + Application.put_env(:elixir, :time_zone_database, database) + end + + @doc """ + Gets the current time zone database. + """ + @doc since: "1.8.0" + @spec get_time_zone_database() :: time_zone_database() + def get_time_zone_database() do + Application.get_env(:elixir, :time_zone_database, Calendar.UTCOnlyTimeZoneDatabase) + end + + @doc """ + Formats received datetime into a string. + + The datetime can be any of the Calendar types (`Time`, `Date`, + `NaiveDateTime`, and `DateTime`) or any map, as long as they + contain all of the relevant fields necessary for formatting. + For example, if you use `%Y` to format the year, the datetime + must have the `:year` field. Therefore, if you pass a `Time`, + or a map without the `:year` field to a format that expects `%Y`, + an error will be raised. + + ## Options + + * `:preferred_datetime` - a string for the preferred format to show datetimes, + it can't contain the `%c` format and defaults to `"%Y-%m-%d %H:%M:%S"` + if the option is not received + + * `:preferred_date` - a string for the preferred format to show dates, + it can't contain the `%x` format and defaults to `"%Y-%m-%d"` + if the option is not received + + * `:preferred_time` - a string for the preferred format to show times, + it can't contain the `%X` format and defaults to `"%H:%M:%S"` + if the option is not received + + * `:am_pm_names` - a function that receives either `:am` or `:pm` and returns + the name of the period of the day, if the option is not received it defaults + to a function that returns `"am"` and `"pm"`, respectively + + * `:month_names` - a function that receives a number and returns the name of + the corresponding month, if the option is not received it defaults to a + function that returns the month names in English + + * `:abbreviated_month_names` - a function that receives a number and returns the + abbreviated name of the corresponding month, if the option is not received it + defaults to a function that returns the abbreviated month names in English + + * `:day_of_week_names` - a function that receives a number and returns the name of + the corresponding day of week, if the option is not received it defaults to a + function that returns the day of week names in English + + * `:abbreviated_day_of_week_names` - a function that receives a number and returns + the abbreviated name of the corresponding day of week, if the option is not received + it defaults to a function that returns the abbreviated day of week names in English + + ## Formatting syntax + + The formatting syntax for strftime is a sequence of characters in the following format: + + % + + where: + + * `%`: indicates the start of a formatted section + * ``: set the padding (see below) + * ``: a number indicating the minimum size of the formatted section + * ``: the format itself (see below) + + ### Accepted padding options + + * `-`: no padding, removes all padding from the format + * `_`: pad with spaces + * `0`: pad with zeroes + + ### Accepted formats + + The accepted formats are: + + Format | Description | Examples (in ISO) + :----- | :-----------------------------------------------------------------------| :------------------------ + a | Abbreviated name of day | Mon + A | Full name of day | Monday + b | Abbreviated month name | Jan + B | Full month name | January + c | Preferred date+time representation | 2018-10-17 12:34:56 + d | Day of the month | 01, 31 + f | Microseconds *(does not support width and padding modifiers)* | 000000, 999999, 0123 + H | Hour using a 24-hour clock | 00, 23 + I | Hour using a 12-hour clock | 01, 12 + j | Day of the year | 001, 366 + m | Month | 01, 12 + M | Minute | 00, 59 + p | "AM" or "PM" (noon is "PM", midnight as "AM") | AM, PM + P | "am" or "pm" (noon is "pm", midnight as "am") | am, pm + q | Quarter | 1, 2, 3, 4 + S | Second | 00, 59, 60 + u | Day of the week | 1 (Monday), 7 (Sunday) + x | Preferred date (without time) representation | 2018-10-17 + X | Preferred time (without date) representation | 12:34:56 + y | Year as 2-digits | 01, 01, 86, 18 + Y | Year | -0001, 0001, 1986 + z | +hhmm/-hhmm time zone offset from UTC (empty string if naive) | +0300, -0530 + Z | Time zone abbreviation (empty string if naive) | CET, BRST + % | Literal "%" character | % + + Any other character will be interpreted as an invalid format and raise an error + + ## Examples + + Without options: + + iex> Calendar.strftime(~U[2019-08-26 13:52:06.0Z], "%y-%m-%d %I:%M:%S %p") + "19-08-26 01:52:06 PM" + + iex> Calendar.strftime(~U[2019-08-26 13:52:06.0Z], "%a, %B %d %Y") + "Mon, August 26 2019" + + iex> Calendar.strftime(~U[2020-04-02 13:52:06.0Z], "%B %-d, %Y") + "April 2, 2020" + + iex> Calendar.strftime(~U[2019-08-26 13:52:06.0Z], "%c") + "2019-08-26 13:52:06" + + With options: + + iex> Calendar.strftime(~U[2019-08-26 13:52:06.0Z], "%c", preferred_datetime: "%H:%M:%S %d-%m-%y") + "13:52:06 26-08-19" + + iex> Calendar.strftime( + ...> ~U[2019-08-26 13:52:06.0Z], + ...> "%A", + ...> day_of_week_names: fn day_of_week -> + ...> {"segunda-feira", "terça-feira", "quarta-feira", "quinta-feira", + ...> "sexta-feira", "sábado", "domingo"} + ...> |> elem(day_of_week - 1) + ...> end + ...>) + "segunda-feira" + + iex> Calendar.strftime( + ...> ~U[2019-08-26 13:52:06.0Z], + ...> "%B", + ...> month_names: fn month -> + ...> {"январь", "февраль", "март", "апрель", "май", "июнь", + ...> "июль", "август", "сентябрь", "октябрь", "ноябрь", "декабрь"} + ...> |> elem(month - 1) + ...> end + ...>) + "август" + """ + @doc since: "1.11.0" + @spec strftime(map(), String.t(), keyword()) :: String.t() + def strftime(date_or_time_or_datetime, string_format, user_options \\ []) + when is_map(date_or_time_or_datetime) and is_binary(string_format) do + parse( + string_format, + date_or_time_or_datetime, + options(user_options), + [] + ) + |> IO.iodata_to_binary() + end + + defp parse("", _datetime, _format_options, acc), + do: Enum.reverse(acc) + + defp parse("%" <> rest, datetime, format_options, acc), + do: parse_modifiers(rest, nil, nil, {datetime, format_options, acc}) + + defp parse(<>, datetime, format_options, acc), + do: parse(rest, datetime, format_options, [char | acc]) + + defp parse_modifiers("-" <> rest, width, nil, parser_data) do + parse_modifiers(rest, width, "", parser_data) + end + + defp parse_modifiers("0" <> rest, nil, nil, parser_data) do + parse_modifiers(rest, nil, ?0, parser_data) + end + + defp parse_modifiers("_" <> rest, width, nil, parser_data) do + parse_modifiers(rest, width, ?\s, parser_data) + end + + defp parse_modifiers(<>, width, pad, parser_data) when digit in ?0..?9 do + new_width = (width || 0) * 10 + (digit - ?0) + + parse_modifiers(rest, new_width, pad, parser_data) + end + + # set default padding if none was specified + defp parse_modifiers(<> = rest, width, nil, parser_data) do + parse_modifiers(rest, width, default_pad(format), parser_data) + end + + # set default width if none was specified + defp parse_modifiers(<> = rest, nil, pad, parser_data) do + parse_modifiers(rest, default_width(format), pad, parser_data) + end + + defp parse_modifiers(rest, width, pad, {datetime, format_options, acc}) do + format_modifiers(rest, width, pad, datetime, format_options, acc) + end + + defp am_pm(hour, format_options) when hour > 11 do + format_options.am_pm_names.(:pm) + end + + defp am_pm(hour, format_options) when hour <= 11 do + format_options.am_pm_names.(:am) + end + + defp default_pad(format) when format in 'aAbBpPZ', do: ?\s + defp default_pad(_format), do: ?0 + + defp default_width(format) when format in 'dHImMSy', do: 2 + defp default_width(?j), do: 3 + defp default_width(format) when format in 'Yz', do: 4 + defp default_width(_format), do: 0 + + # Literally just % + defp format_modifiers("%" <> rest, width, pad, datetime, format_options, acc) do + parse(rest, datetime, format_options, [pad_leading("%", width, pad) | acc]) + end + + # Abbreviated name of day + defp format_modifiers("a" <> rest, width, pad, datetime, format_options, acc) do + result = + datetime + |> Date.day_of_week() + |> format_options.abbreviated_day_of_week_names.() + |> pad_leading(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Full name of day + defp format_modifiers("A" <> rest, width, pad, datetime, format_options, acc) do + result = + datetime + |> Date.day_of_week() + |> format_options.day_of_week_names.() + |> pad_leading(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Abbreviated month name + defp format_modifiers("b" <> rest, width, pad, datetime, format_options, acc) do + result = + datetime.month + |> format_options.abbreviated_month_names.() + |> pad_leading(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Full month name + defp format_modifiers("B" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.month |> format_options.month_names.() |> pad_leading(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Preferred date+time representation + defp format_modifiers( + "c" <> _rest, + _width, + _pad, + _datetime, + %{preferred_datetime_invoked: true}, + _acc + ) do + raise ArgumentError, + "tried to format preferred_datetime within another preferred_datetime format" + end + + defp format_modifiers("c" <> rest, width, pad, datetime, format_options, acc) do + result = + format_options.preferred_datetime + |> parse(datetime, %{format_options | preferred_datetime_invoked: true}, []) + |> pad_preferred(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Day of the month + defp format_modifiers("d" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.day |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Microseconds + defp format_modifiers("f" <> rest, _width, _pad, datetime, format_options, acc) do + {microsecond, precision} = datetime.microsecond + + result = + microsecond + |> Integer.to_string() + |> String.pad_leading(6, "0") + |> binary_part(0, max(precision, 1)) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Hour using a 24-hour clock + defp format_modifiers("H" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.hour |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Hour using a 12-hour clock + defp format_modifiers("I" <> rest, width, pad, datetime, format_options, acc) do + result = (rem(datetime.hour() + 23, 12) + 1) |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Day of the year + defp format_modifiers("j" <> rest, width, pad, datetime, format_options, acc) do + result = datetime |> Date.day_of_year() |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Month + defp format_modifiers("m" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.month |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Minute + defp format_modifiers("M" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.minute |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # "AM" or "PM" (noon is "PM", midnight as "AM") + defp format_modifiers("p" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.hour |> am_pm(format_options) |> String.upcase() |> pad_leading(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # "am" or "pm" (noon is "pm", midnight as "am") + defp format_modifiers("P" <> rest, width, pad, datetime, format_options, acc) do + result = + datetime.hour + |> am_pm(format_options) + |> String.downcase() + |> pad_leading(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Quarter + defp format_modifiers("q" <> rest, width, pad, datetime, format_options, acc) do + result = datetime |> Date.quarter_of_year() |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Second + defp format_modifiers("S" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.second |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Day of the week + defp format_modifiers("u" <> rest, width, pad, datetime, format_options, acc) do + result = datetime |> Date.day_of_week() |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Preferred date (without time) representation + defp format_modifiers( + "x" <> _rest, + _width, + _pad, + _datetime, + %{preferred_date_invoked: true}, + _acc + ) do + raise ArgumentError, + "tried to format preferred_date within another preferred_date format" + end + + defp format_modifiers("x" <> rest, width, pad, datetime, format_options, acc) do + result = + format_options.preferred_date + |> parse(datetime, %{format_options | preferred_date_invoked: true}, []) + |> pad_preferred(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Preferred time (without date) representation + defp format_modifiers( + "X" <> _rest, + _width, + _pad, + _datetime, + %{preferred_time_invoked: true}, + _acc + ) do + raise ArgumentError, + "tried to format preferred_time within another preferred_time format" + end + + defp format_modifiers("X" <> rest, width, pad, datetime, format_options, acc) do + result = + format_options.preferred_time + |> parse(datetime, %{format_options | preferred_time_invoked: true}, []) + |> pad_preferred(width, pad) + + parse(rest, datetime, format_options, [result | acc]) + end + + # Year as 2-digits + defp format_modifiers("y" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.year |> rem(100) |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # Year + defp format_modifiers("Y" <> rest, width, pad, datetime, format_options, acc) do + result = datetime.year |> Integer.to_string() |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + # +hhmm/-hhmm time zone offset from UTC (empty string if naive) + defp format_modifiers( + "z" <> rest, + width, + pad, + datetime = %{utc_offset: utc_offset, std_offset: std_offset}, + format_options, + acc + ) do + absolute_offset = abs(utc_offset + std_offset) + + offset_number = + Integer.to_string(div(absolute_offset, 3600) * 100 + rem(div(absolute_offset, 60), 60)) + + sign = if utc_offset + std_offset >= 0, do: "+", else: "-" + result = "#{sign}#{pad_leading(offset_number, width, pad)}" + parse(rest, datetime, format_options, [result | acc]) + end + + defp format_modifiers("z" <> rest, _width, _pad, datetime, format_options, acc) do + parse(rest, datetime, format_options, ["" | acc]) + end + + # Time zone abbreviation (empty string if naive) + defp format_modifiers("Z" <> rest, width, pad, datetime, format_options, acc) do + result = datetime |> Map.get(:zone_abbr, "") |> pad_leading(width, pad) + parse(rest, datetime, format_options, [result | acc]) + end + + defp format_modifiers(rest, _width, _pad, _datetime, _format_options, _acc) do + {next, _rest} = String.next_grapheme(rest) || {"", ""} + raise ArgumentError, "invalid strftime format: %#{next}" + end + + defp pad_preferred(result, width, pad) when length(result) < width do + pad_preferred([pad | result], width, pad) + end + + defp pad_preferred(result, _width, _pad), do: result + + defp pad_leading(string, count, padding) do + to_pad = count - byte_size(string) + if to_pad > 0, do: do_pad_leading(to_pad, padding, string), else: string + end + + defp do_pad_leading(0, _, acc), do: acc + + defp do_pad_leading(count, padding, acc), + do: do_pad_leading(count - 1, padding, [padding | acc]) + + defp options(user_options) do + default_options = %{ + preferred_date: "%Y-%m-%d", + preferred_time: "%H:%M:%S", + preferred_datetime: "%Y-%m-%d %H:%M:%S", + am_pm_names: fn + :am -> "am" + :pm -> "pm" + end, + month_names: fn month -> + {"January", "February", "March", "April", "May", "June", "July", "August", "September", + "October", "November", "December"} + |> elem(month - 1) + end, + day_of_week_names: fn day_of_week -> + {"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"} + |> elem(day_of_week - 1) + end, + abbreviated_month_names: fn month -> + {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"} + |> elem(month - 1) + end, + abbreviated_day_of_week_names: fn day_of_week -> + {"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"} |> elem(day_of_week - 1) + end, + preferred_datetime_invoked: false, + preferred_date_invoked: false, + preferred_time_invoked: false + } + + Enum.reduce(user_options, default_options, fn {key, value}, acc -> + if Map.has_key?(acc, key) do + %{acc | key => value} + else + raise ArgumentError, "unknown option #{inspect(key)} given to Calendar.strftime/3" + end + end) + end +end diff --git a/lib/elixir/lib/calendar/date.ex b/lib/elixir/lib/calendar/date.ex new file mode 100644 index 00000000000..ea77f22728b --- /dev/null +++ b/lib/elixir/lib/calendar/date.ex @@ -0,0 +1,1044 @@ +defmodule Date do + @moduledoc """ + A Date struct and functions. + + The Date struct contains the fields year, month, day and calendar. + New dates can be built with the `new/3` function or using the + `~D` (see `sigil_D/2`) sigil: + + iex> ~D[2000-01-01] + ~D[2000-01-01] + + Both `new/3` and sigil return a struct where the date fields can + be accessed directly: + + iex> date = ~D[2000-01-01] + iex> date.year + 2000 + iex> date.month + 1 + + The functions on this module work with the `Date` struct as well + as any struct that contains the same fields as the `Date` struct, + such as `NaiveDateTime` and `DateTime`. Such functions expect + `t:Calendar.date/0` in their typespecs (instead of `t:t/0`). + + Developers should avoid creating the Date structs directly + and instead rely on the functions provided by this module as well + as the ones in third-party calendar libraries. + + ## Comparing dates + + Comparisons in Elixir using `==/2`, `>/2`, ` Enum.min([~D[2017-03-31], ~D[2017-04-01]], Date) + ~D[2017-03-31] + + ## Using epochs + + The `add/2` and `diff/2` functions can be used for computing dates + or retrieving the number of days between instants. For example, if there + is an interest in computing the number of days from the Unix epoch + (1970-01-01): + + iex> Date.diff(~D[2010-04-17], ~D[1970-01-01]) + 14716 + + iex> Date.add(~D[1970-01-01], 14716) + ~D[2010-04-17] + + Those functions are optimized to deal with common epochs, such + as the Unix Epoch above or the Gregorian Epoch (0000-01-01). + """ + + @enforce_keys [:year, :month, :day] + defstruct [:year, :month, :day, calendar: Calendar.ISO] + + @type t :: %__MODULE__{ + year: Calendar.year(), + month: Calendar.month(), + day: Calendar.day(), + calendar: Calendar.calendar() + } + + @doc """ + Returns a range of dates. + + A range of dates represents a discrete number of dates where + the first and last values are dates with matching calendars. + + Ranges of dates can be either increasing (`first <= last`) or + decreasing (`first > last`). They are also always inclusive. + + ## Examples + + iex> Date.range(~D[1999-01-01], ~D[2000-01-01]) + Date.range(~D[1999-01-01], ~D[2000-01-01]) + + A range of dates implements the `Enumerable` protocol, which means + functions in the `Enum` module can be used to work with + ranges: + + iex> range = Date.range(~D[2001-01-01], ~D[2002-01-01]) + iex> range + Date.range(~D[2001-01-01], ~D[2002-01-01]) + iex> Enum.count(range) + 366 + iex> ~D[2001-02-01] in range + true + iex> Enum.take(range, 3) + [~D[2001-01-01], ~D[2001-01-02], ~D[2001-01-03]] + + """ + @doc since: "1.5.0" + @spec range(Calendar.date(), Calendar.date()) :: Date.Range.t() + def range(%{calendar: calendar} = first, %{calendar: calendar} = last) do + {first_days, _} = to_iso_days(first) + {last_days, _} = to_iso_days(last) + # TODO: Deprecate inferring a range with a step of -1 on Elixir v1.16 + step = if first_days <= last_days, do: 1, else: -1 + range(first, first_days, last, last_days, calendar, step) + end + + def range(%{calendar: _, year: _, month: _, day: _}, %{calendar: _, year: _, month: _, day: _}) do + raise ArgumentError, "both dates must have matching calendars" + end + + @doc """ + Returns a range of dates with a step. + + ## Examples + + iex> range = Date.range(~D[2001-01-01], ~D[2002-01-01], 2) + iex> range + Date.range(~D[2001-01-01], ~D[2002-01-01], 2) + iex> Enum.count(range) + 183 + iex> ~D[2001-01-03] in range + true + iex> Enum.take(range, 3) + [~D[2001-01-01], ~D[2001-01-03], ~D[2001-01-05]] + + """ + @doc since: "1.12.0" + @spec range(Calendar.date(), Calendar.date(), step :: pos_integer | neg_integer) :: + Date.Range.t() + def range(%{calendar: calendar} = first, %{calendar: calendar} = last, step) + when is_integer(step) and step != 0 do + {first_days, _} = to_iso_days(first) + {last_days, _} = to_iso_days(last) + range(first, first_days, last, last_days, calendar, step) + end + + def range( + %{calendar: _, year: _, month: _, day: _} = first, + %{calendar: _, year: _, month: _, day: _} = last, + step + ) do + raise ArgumentError, + "both dates must have matching calendar and the step must be a " <> + "non-zero integer, got: #{inspect(first)}, #{inspect(last)}, #{step}" + end + + defp range(first, first_days, last, last_days, calendar, step) do + %Date.Range{ + first: %Date{calendar: calendar, year: first.year, month: first.month, day: first.day}, + last: %Date{calendar: calendar, year: last.year, month: last.month, day: last.day}, + first_in_iso_days: first_days, + last_in_iso_days: last_days, + step: step + } + end + + @doc """ + Returns the current date in UTC. + + ## Examples + + iex> date = Date.utc_today() + iex> date.year >= 2016 + true + + """ + @doc since: "1.4.0" + @spec utc_today(Calendar.calendar()) :: t + def utc_today(calendar \\ Calendar.ISO) + + def utc_today(Calendar.ISO) do + {:ok, {year, month, day}, _, _} = Calendar.ISO.from_unix(System.os_time(), :native) + %Date{year: year, month: month, day: day} + end + + def utc_today(calendar) do + calendar + |> DateTime.utc_now() + |> DateTime.to_date() + end + + @doc """ + Returns `true` if the year in the given `date` is a leap year. + + ## Examples + + iex> Date.leap_year?(~D[2000-01-01]) + true + iex> Date.leap_year?(~D[2001-01-01]) + false + iex> Date.leap_year?(~D[2004-01-01]) + true + iex> Date.leap_year?(~D[1900-01-01]) + false + iex> Date.leap_year?(~N[2004-01-01 01:23:45]) + true + + """ + @doc since: "1.4.0" + @spec leap_year?(Calendar.date()) :: boolean() + def leap_year?(date) + + def leap_year?(%{calendar: calendar, year: year}) do + calendar.leap_year?(year) + end + + @doc """ + Returns the number of days in the given `date` month. + + ## Examples + + iex> Date.days_in_month(~D[1900-01-13]) + 31 + iex> Date.days_in_month(~D[1900-02-09]) + 28 + iex> Date.days_in_month(~N[2000-02-20 01:23:45]) + 29 + + """ + @doc since: "1.4.0" + @spec days_in_month(Calendar.date()) :: Calendar.day() + def days_in_month(date) + + def days_in_month(%{calendar: calendar, year: year, month: month}) do + calendar.days_in_month(year, month) + end + + @doc """ + Returns the number of months in the given `date` year. + + ## Example + + iex> Date.months_in_year(~D[1900-01-13]) + 12 + + """ + @doc since: "1.7.0" + @spec months_in_year(Calendar.date()) :: Calendar.month() + def months_in_year(date) + + def months_in_year(%{calendar: calendar, year: year}) do + calendar.months_in_year(year) + end + + @doc """ + Builds a new ISO date. + + Expects all values to be integers. Returns `{:ok, date}` if each + entry fits its appropriate range, returns `{:error, reason}` otherwise. + + ## Examples + + iex> Date.new(2000, 1, 1) + {:ok, ~D[2000-01-01]} + iex> Date.new(2000, 13, 1) + {:error, :invalid_date} + iex> Date.new(2000, 2, 29) + {:ok, ~D[2000-02-29]} + + iex> Date.new(2000, 2, 30) + {:error, :invalid_date} + iex> Date.new(2001, 2, 29) + {:error, :invalid_date} + + """ + @spec new(Calendar.year(), Calendar.month(), Calendar.day(), Calendar.calendar()) :: + {:ok, t} | {:error, atom} + def new(year, month, day, calendar \\ Calendar.ISO) do + if calendar.valid_date?(year, month, day) do + {:ok, %Date{year: year, month: month, day: day, calendar: calendar}} + else + {:error, :invalid_date} + end + end + + @doc """ + Builds a new ISO date. + + Expects all values to be integers. Returns `date` if each + entry fits its appropriate range, raises if the date is invalid. + + ## Examples + + iex> Date.new!(2000, 1, 1) + ~D[2000-01-01] + iex> Date.new!(2000, 13, 1) + ** (ArgumentError) cannot build date, reason: :invalid_date + iex> Date.new!(2000, 2, 29) + ~D[2000-02-29] + """ + @doc since: "1.11.0" + @spec new!(Calendar.year(), Calendar.month(), Calendar.day(), Calendar.calendar()) :: t + def new!(year, month, day, calendar \\ Calendar.ISO) do + case new(year, month, day, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, "cannot build date, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts the given date to a string according to its calendar. + + ### Examples + + iex> Date.to_string(~D[2000-02-28]) + "2000-02-28" + iex> Date.to_string(~N[2000-02-28 01:23:45]) + "2000-02-28" + iex> Date.to_string(~D[-0100-12-15]) + "-0100-12-15" + + """ + @spec to_string(Calendar.date()) :: String.t() + def to_string(date) + + def to_string(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.date_to_string(year, month, day) + end + + @doc """ + Parses the extended "Dates" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + The year parsed by this function is limited to four digits. + + ## Examples + + iex> Date.from_iso8601("2015-01-23") + {:ok, ~D[2015-01-23]} + + iex> Date.from_iso8601("2015:01:23") + {:error, :invalid_format} + + iex> Date.from_iso8601("2015-01-32") + {:error, :invalid_date} + + """ + @spec from_iso8601(String.t(), Calendar.calendar()) :: {:ok, t} | {:error, atom} + def from_iso8601(string, calendar \\ Calendar.ISO) do + with {:ok, {year, month, day}} <- Calendar.ISO.parse_date(string) do + convert(%Date{year: year, month: month, day: day}, calendar) + end + end + + @doc """ + Parses the extended "Dates" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + Raises if the format is invalid. + + ## Examples + + iex> Date.from_iso8601!("2015-01-23") + ~D[2015-01-23] + iex> Date.from_iso8601!("2015:01:23") + ** (ArgumentError) cannot parse "2015:01:23" as date, reason: :invalid_format + + """ + @spec from_iso8601!(String.t(), Calendar.calendar()) :: t + def from_iso8601!(string, calendar \\ Calendar.ISO) do + case from_iso8601(string, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, "cannot parse #{inspect(string)} as date, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts the given `date` to + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + By default, `Date.to_iso8601/2` returns dates formatted in the "extended" + format, for human readability. It also supports the "basic" format through passing the `:basic` option. + + Only supports converting dates which are in the ISO calendar, + or other calendars in which the days also start at midnight. + Attempting to convert dates from other calendars will raise an `ArgumentError`. + + ### Examples + + iex> Date.to_iso8601(~D[2000-02-28]) + "2000-02-28" + + iex> Date.to_iso8601(~D[2000-02-28], :basic) + "20000228" + + iex> Date.to_iso8601(~N[2000-02-28 00:00:00]) + "2000-02-28" + + """ + @spec to_iso8601(Calendar.date(), :extended | :basic) :: String.t() + def to_iso8601(date, format \\ :extended) + + def to_iso8601(%{calendar: Calendar.ISO} = date, format) when format in [:basic, :extended] do + %{year: year, month: month, day: day} = date + Calendar.ISO.date_to_string(year, month, day, format) + end + + def to_iso8601(%{calendar: _} = date, format) when format in [:basic, :extended] do + date + |> convert!(Calendar.ISO) + |> to_iso8601() + end + + @doc """ + Converts the given `date` to an Erlang date tuple. + + Only supports converting dates which are in the ISO calendar, + or other calendars in which the days also start at midnight. + Attempting to convert dates from other calendars will raise. + + ## Examples + + iex> Date.to_erl(~D[2000-01-01]) + {2000, 1, 1} + + iex> Date.to_erl(~N[2000-01-01 00:00:00]) + {2000, 1, 1} + + """ + @spec to_erl(Calendar.date()) :: :calendar.date() + def to_erl(date) do + %{year: year, month: month, day: day} = convert!(date, Calendar.ISO) + {year, month, day} + end + + @doc """ + Converts an Erlang date tuple to a `Date` struct. + + Only supports converting dates which are in the ISO calendar, + or other calendars in which the days also start at midnight. + Attempting to convert dates from other calendars will return an error tuple. + + ## Examples + + iex> Date.from_erl({2000, 1, 1}) + {:ok, ~D[2000-01-01]} + iex> Date.from_erl({2000, 13, 1}) + {:error, :invalid_date} + + """ + @spec from_erl(:calendar.date(), Calendar.calendar()) :: {:ok, t} | {:error, atom} + def from_erl(tuple, calendar \\ Calendar.ISO) + + def from_erl({year, month, day}, calendar) do + with {:ok, date} <- new(year, month, day, Calendar.ISO), do: convert(date, calendar) + end + + @doc """ + Converts an Erlang date tuple but raises for invalid dates. + + ## Examples + + iex> Date.from_erl!({2000, 1, 1}) + ~D[2000-01-01] + iex> Date.from_erl!({2000, 13, 1}) + ** (ArgumentError) cannot convert {2000, 13, 1} to date, reason: :invalid_date + + """ + @spec from_erl!(:calendar.date(), Calendar.calendar()) :: t + def from_erl!(tuple, calendar \\ Calendar.ISO) do + case from_erl(tuple, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, + "cannot convert #{inspect(tuple)} to date, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts a number of gregorian days to a `Date` struct. + + ## Examples + + iex> Date.from_gregorian_days(1) + ~D[0000-01-02] + iex> Date.from_gregorian_days(730_485) + ~D[2000-01-01] + iex> Date.from_gregorian_days(-1) + ~D[-0001-12-31] + + """ + @doc since: "1.11.0" + @spec from_gregorian_days(integer(), Calendar.calendar()) :: t + def from_gregorian_days(days, calendar \\ Calendar.ISO) when is_integer(days) do + from_iso_days({days, 0}, calendar) + end + + @doc """ + Converts a `date` struct to a number of gregorian days. + + ## Examples + + iex> Date.to_gregorian_days(~D[0000-01-02]) + 1 + iex> Date.to_gregorian_days(~D[2000-01-01]) + 730_485 + iex> Date.to_gregorian_days(~N[2000-01-01 00:00:00]) + 730_485 + + """ + @doc since: "1.11.0" + @spec to_gregorian_days(Calendar.date()) :: integer() + def to_gregorian_days(date) do + {days, _} = to_iso_days(date) + days + end + + @doc """ + Compares two date structs. + + Returns `:gt` if first date is later than the second + and `:lt` for vice versa. If the two dates are equal + `:eq` is returned. + + ## Examples + + iex> Date.compare(~D[2016-04-16], ~D[2016-04-28]) + :lt + + This function can also be used to compare across more + complex calendar types by considering only the date fields: + + iex> Date.compare(~D[2016-04-16], ~N[2016-04-28 01:23:45]) + :lt + iex> Date.compare(~D[2016-04-16], ~N[2016-04-16 01:23:45]) + :eq + iex> Date.compare(~N[2016-04-16 12:34:56], ~N[2016-04-16 01:23:45]) + :eq + + """ + @doc since: "1.4.0" + @spec compare(Calendar.date(), Calendar.date()) :: :lt | :eq | :gt + def compare(%{calendar: calendar} = date1, %{calendar: calendar} = date2) do + %{year: year1, month: month1, day: day1} = date1 + %{year: year2, month: month2, day: day2} = date2 + + case {{year1, month1, day1}, {year2, month2, day2}} do + {first, second} when first > second -> :gt + {first, second} when first < second -> :lt + _ -> :eq + end + end + + def compare(%{calendar: calendar1} = date1, %{calendar: calendar2} = date2) do + if Calendar.compatible_calendars?(calendar1, calendar2) do + case {to_iso_days(date1), to_iso_days(date2)} do + {first, second} when first > second -> :gt + {first, second} when first < second -> :lt + _ -> :eq + end + else + raise ArgumentError, """ + cannot compare #{inspect(date1)} with #{inspect(date2)}. + + This comparison would be ambiguous as their calendars have incompatible day rollover moments. + Specify an exact time of day (using DateTime) to resolve this ambiguity + """ + end + end + + @doc """ + Converts the given `date` from its calendar to the given `calendar`. + + Returns `{:ok, date}` if the calendars are compatible, + or `{:error, :incompatible_calendars}` if they are not. + + See also `Calendar.compatible_calendars?/2`. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10,000 years to the current Gregorian + year: + + iex> Date.convert(~D[2000-01-01], Calendar.Holocene) + {:ok, %Date{calendar: Calendar.Holocene, year: 12000, month: 1, day: 1}} + + """ + @doc since: "1.5.0" + @spec convert(Calendar.date(), Calendar.calendar()) :: + {:ok, t} | {:error, :incompatible_calendars} + def convert(%{calendar: calendar, year: year, month: month, day: day}, calendar) do + {:ok, %Date{calendar: calendar, year: year, month: month, day: day}} + end + + def convert(%{calendar: calendar} = date, target_calendar) do + if Calendar.compatible_calendars?(calendar, target_calendar) do + result_date = + date + |> to_iso_days() + |> from_iso_days(target_calendar) + + {:ok, result_date} + else + {:error, :incompatible_calendars} + end + end + + @doc """ + Similar to `Date.convert/2`, but raises an `ArgumentError` + if the conversion between the two calendars is not possible. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10,000 years to the current Gregorian + year: + + iex> Date.convert!(~D[2000-01-01], Calendar.Holocene) + %Date{calendar: Calendar.Holocene, year: 12000, month: 1, day: 1} + + """ + @doc since: "1.5.0" + @spec convert!(Calendar.date(), Calendar.calendar()) :: t + def convert!(date, calendar) do + case convert(date, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, + "cannot convert #{inspect(date)} to target calendar #{inspect(calendar)}, " <> + "reason: #{inspect(reason)}" + end + end + + @doc """ + Adds the number of days to the given `date`. + + The days are counted as Gregorian days. The date is returned in the same + calendar as it was given in. + + ## Examples + + iex> Date.add(~D[2000-01-03], -2) + ~D[2000-01-01] + iex> Date.add(~D[2000-01-01], 2) + ~D[2000-01-03] + iex> Date.add(~N[2000-01-01 09:00:00], 2) + ~D[2000-01-03] + iex> Date.add(~D[-0010-01-01], -2) + ~D[-0011-12-30] + + """ + @doc since: "1.5.0" + @spec add(Calendar.date(), integer()) :: t + def add(%{calendar: Calendar.ISO} = date, days) do + %{year: year, month: month, day: day} = date + + {year, month, day} = + Calendar.ISO.date_to_iso_days(year, month, day) + |> Kernel.+(days) + |> Calendar.ISO.date_from_iso_days() + + %Date{calendar: Calendar.ISO, year: year, month: month, day: day} + end + + def add(%{calendar: calendar} = date, days) do + {base_days, fraction} = to_iso_days(date) + from_iso_days({base_days + days, fraction}, calendar) + end + + @doc """ + Calculates the difference between two dates, in a full number of days. + + It returns the number of Gregorian days between the dates. Only `Date` + structs that follow the same or compatible calendars can be compared + this way. If two calendars are not compatible, it will raise. + + ## Examples + + iex> Date.diff(~D[2000-01-03], ~D[2000-01-01]) + 2 + iex> Date.diff(~D[2000-01-01], ~D[2000-01-03]) + -2 + iex> Date.diff(~D[0000-01-02], ~D[-0001-12-30]) + 3 + iex> Date.diff(~D[2000-01-01], ~N[2000-01-03 09:00:00]) + -2 + + """ + @doc since: "1.5.0" + @spec diff(Calendar.date(), Calendar.date()) :: integer + def diff(%{calendar: Calendar.ISO} = date1, %{calendar: Calendar.ISO} = date2) do + %{year: year1, month: month1, day: day1} = date1 + %{year: year2, month: month2, day: day2} = date2 + + Calendar.ISO.date_to_iso_days(year1, month1, day1) - + Calendar.ISO.date_to_iso_days(year2, month2, day2) + end + + def diff(%{calendar: calendar1} = date1, %{calendar: calendar2} = date2) do + if Calendar.compatible_calendars?(calendar1, calendar2) do + {days1, _} = to_iso_days(date1) + {days2, _} = to_iso_days(date2) + days1 - days2 + else + raise ArgumentError, + "cannot calculate the difference between #{inspect(date1)} and #{inspect(date2)} because their calendars are not compatible and thus the result would be ambiguous" + end + end + + @doc false + def to_iso_days(%{calendar: Calendar.ISO, year: year, month: month, day: day}) do + {Calendar.ISO.date_to_iso_days(year, month, day), {0, 86_400_000_000}} + end + + def to_iso_days(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.naive_datetime_to_iso_days(year, month, day, 0, 0, 0, {0, 0}) + end + + defp from_iso_days({days, _}, Calendar.ISO) do + {year, month, day} = Calendar.ISO.date_from_iso_days(days) + %Date{year: year, month: month, day: day, calendar: Calendar.ISO} + end + + defp from_iso_days(iso_days, target_calendar) do + {year, month, day, _, _, _, _} = target_calendar.naive_datetime_from_iso_days(iso_days) + %Date{year: year, month: month, day: day, calendar: target_calendar} + end + + @doc """ + Calculates the day of the week of a given `date`. + + Returns the day of the week as an integer. For the ISO 8601 + calendar (the default), it is an integer from 1 to 7, where + 1 is Monday and 7 is Sunday. + + An optional `starting_on` value may be supplied, which + configures the weekday the week starts on. The default value + for it is `:default`, which translates to `:monday` for the + built-in ISO calendar. Any other weekday may be given to. + + ## Examples + + iex> Date.day_of_week(~D[2016-10-31]) + 1 + iex> Date.day_of_week(~D[2016-11-01]) + 2 + iex> Date.day_of_week(~N[2016-11-01 01:23:45]) + 2 + iex> Date.day_of_week(~D[-0015-10-30]) + 3 + + iex> Date.day_of_week(~D[2016-10-31], :sunday) + 2 + iex> Date.day_of_week(~D[2016-11-01], :sunday) + 3 + iex> Date.day_of_week(~N[2016-11-01 01:23:45], :sunday) + 3 + iex> Date.day_of_week(~D[-0015-10-30], :sunday) + 4 + + """ + @doc since: "1.4.0" + @spec day_of_week(Calendar.date(), starting_on :: :default | atom) :: Calendar.day_of_week() + def day_of_week(date, starting_on \\ :default) + + def day_of_week(%{calendar: calendar, year: year, month: month, day: day}, starting_on) do + {day_of_week, _first, _last} = calendar.day_of_week(year, month, day, starting_on) + day_of_week + end + + @doc """ + Calculates a date that is the first day of the week for the given `date`. + + If the day is already the first day of the week, it returns the + day itself. For the built-in ISO calendar, the week starts on Monday. + A weekday rather than `:default` can be given as `starting_on`. + + ## Examples + + iex> Date.beginning_of_week(~D[2020-07-11]) + ~D[2020-07-06] + iex> Date.beginning_of_week(~D[2020-07-06]) + ~D[2020-07-06] + iex> Date.beginning_of_week(~D[2020-07-11], :sunday) + ~D[2020-07-05] + iex> Date.beginning_of_week(~D[2020-07-11], :saturday) + ~D[2020-07-11] + iex> Date.beginning_of_week(~N[2020-07-11 01:23:45]) + ~D[2020-07-06] + + """ + @doc since: "1.11.0" + @spec beginning_of_week(Calendar.date(), starting_on :: :default | atom) :: Date.t() + def beginning_of_week(date, starting_on \\ :default) + + def beginning_of_week(%{calendar: Calendar.ISO} = date, starting_on) do + %{year: year, month: month, day: day} = date + iso_days = Calendar.ISO.date_to_iso_days(year, month, day) + + {year, month, day} = + case Calendar.ISO.iso_days_to_day_of_week(iso_days, starting_on) do + 1 -> + {year, month, day} + + day_of_week -> + Calendar.ISO.date_from_iso_days(iso_days - day_of_week + 1) + end + + %Date{calendar: Calendar.ISO, year: year, month: month, day: day} + end + + def beginning_of_week(%{calendar: calendar} = date, starting_on) do + %{year: year, month: month, day: day} = date + + case calendar.day_of_week(year, month, day, starting_on) do + {day_of_week, day_of_week, _} -> + %Date{calendar: calendar, year: year, month: month, day: day} + + {day_of_week, first_day_of_week, _} -> + add(date, -(day_of_week - first_day_of_week)) + end + end + + @doc """ + Calculates a date that is the last day of the week for the given `date`. + + If the day is already the last day of the week, it returns the + day itself. For the built-in ISO calendar, the week ends on Sunday. + A weekday rather than `:default` can be given as `starting_on`. + + ## Examples + + iex> Date.end_of_week(~D[2020-07-11]) + ~D[2020-07-12] + iex> Date.end_of_week(~D[2020-07-05]) + ~D[2020-07-05] + iex> Date.end_of_week(~D[2020-07-06], :sunday) + ~D[2020-07-11] + iex> Date.end_of_week(~D[2020-07-06], :saturday) + ~D[2020-07-10] + iex> Date.end_of_week(~N[2020-07-11 01:23:45]) + ~D[2020-07-12] + + """ + @doc since: "1.11.0" + @spec end_of_week(Calendar.date(), starting_on :: :default | atom) :: Date.t() + def end_of_week(date, starting_on \\ :default) + + def end_of_week(%{calendar: Calendar.ISO} = date, starting_on) do + %{year: year, month: month, day: day} = date + iso_days = Calendar.ISO.date_to_iso_days(year, month, day) + + {year, month, day} = + case Calendar.ISO.iso_days_to_day_of_week(iso_days, starting_on) do + 7 -> + {year, month, day} + + day_of_week -> + Calendar.ISO.date_from_iso_days(iso_days + 7 - day_of_week) + end + + %Date{calendar: Calendar.ISO, year: year, month: month, day: day} + end + + def end_of_week(%{calendar: calendar} = date, starting_on) do + %{year: year, month: month, day: day} = date + + case calendar.day_of_week(year, month, day, starting_on) do + {day_of_week, _, day_of_week} -> + %Date{calendar: calendar, year: year, month: month, day: day} + + {day_of_week, _, last_day_of_week} -> + add(date, last_day_of_week - day_of_week) + end + end + + @doc """ + Calculates the day of the year of a given `date`. + + Returns the day of the year as an integer. For the ISO 8601 + calendar (the default), it is an integer from 1 to 366. + + ## Examples + + iex> Date.day_of_year(~D[2016-01-01]) + 1 + iex> Date.day_of_year(~D[2016-11-01]) + 306 + iex> Date.day_of_year(~D[-0015-10-30]) + 303 + iex> Date.day_of_year(~D[2004-12-31]) + 366 + + """ + @doc since: "1.8.0" + @spec day_of_year(Calendar.date()) :: Calendar.day() + def day_of_year(date) + + def day_of_year(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.day_of_year(year, month, day) + end + + @doc """ + Calculates the quarter of the year of a given `date`. + + Returns the day of the year as an integer. For the ISO 8601 + calendar (the default), it is an integer from 1 to 4. + + ## Examples + + iex> Date.quarter_of_year(~D[2016-10-31]) + 4 + iex> Date.quarter_of_year(~D[2016-01-01]) + 1 + iex> Date.quarter_of_year(~N[2016-04-01 01:23:45]) + 2 + iex> Date.quarter_of_year(~D[-0015-09-30]) + 3 + + """ + @doc since: "1.8.0" + @spec quarter_of_year(Calendar.date()) :: non_neg_integer() + def quarter_of_year(date) + + def quarter_of_year(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.quarter_of_year(year, month, day) + end + + @doc """ + Calculates the year-of-era and era for a given + calendar year. + + Returns a tuple `{year, era}` representing the + year within the era and the era number. + + ## Examples + + iex> Date.year_of_era(~D[0001-01-01]) + {1, 1} + iex> Date.year_of_era(~D[0000-12-31]) + {1, 0} + iex> Date.year_of_era(~D[-0001-01-01]) + {2, 0} + + """ + @doc since: "1.8.0" + @spec year_of_era(Calendar.date()) :: {Calendar.year(), non_neg_integer()} + def year_of_era(date) + + def year_of_era(%{calendar: calendar, year: year, month: month, day: day}) do + # TODO: Remove me on 1.17 + # The behaviour implementation already warns on missing callback. + if function_exported?(calendar, :year_of_era, 3) do + calendar.year_of_era(year, month, day) + else + calendar.year_of_era(year) + end + end + + @doc """ + Calculates the day-of-era and era for a given + calendar `date`. + + Returns a tuple `{day, era}` representing the + day within the era and the era number. + + ## Examples + + iex> Date.day_of_era(~D[0001-01-01]) + {1, 1} + + iex> Date.day_of_era(~D[0000-12-31]) + {1, 0} + + """ + @doc since: "1.8.0" + @spec day_of_era(Calendar.date()) :: {Calendar.day(), non_neg_integer()} + def day_of_era(date) + + def day_of_era(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.day_of_era(year, month, day) + end + + @doc """ + Calculates a date that is the first day of the month for the given `date`. + + ## Examples + + iex> Date.beginning_of_month(~D[2000-01-31]) + ~D[2000-01-01] + iex> Date.beginning_of_month(~D[2000-01-01]) + ~D[2000-01-01] + iex> Date.beginning_of_month(~N[2000-01-31 01:23:45]) + ~D[2000-01-01] + + """ + @doc since: "1.11.0" + @spec beginning_of_month(Calendar.date()) :: t() + def beginning_of_month(date) + + def beginning_of_month(%{year: year, month: month, calendar: calendar}) do + %Date{year: year, month: month, day: 1, calendar: calendar} + end + + @doc """ + Calculates a date that is the last day of the month for the given `date`. + + ## Examples + + iex> Date.end_of_month(~D[2000-01-01]) + ~D[2000-01-31] + iex> Date.end_of_month(~D[2000-01-31]) + ~D[2000-01-31] + iex> Date.end_of_month(~N[2000-01-01 01:23:45]) + ~D[2000-01-31] + + """ + @doc since: "1.11.0" + @spec end_of_month(Calendar.date()) :: t() + def end_of_month(date) + + def end_of_month(%{year: year, month: month, calendar: calendar} = date) do + day = Date.days_in_month(date) + %Date{year: year, month: month, day: day, calendar: calendar} + end + + ## Helpers + + defimpl String.Chars do + def to_string(%{calendar: calendar, year: year, month: month, day: day}) do + calendar.date_to_string(year, month, day) + end + end + + defimpl Inspect do + def inspect(%{calendar: calendar, year: year, month: month, day: day}, _) do + "~D[" <> calendar.date_to_string(year, month, day) <> suffix(calendar) <> "]" + end + + defp suffix(Calendar.ISO), do: "" + defp suffix(calendar), do: " " <> inspect(calendar) + end +end diff --git a/lib/elixir/lib/calendar/date_range.ex b/lib/elixir/lib/calendar/date_range.ex new file mode 100644 index 00000000000..7076e32fa69 --- /dev/null +++ b/lib/elixir/lib/calendar/date_range.ex @@ -0,0 +1,227 @@ +defmodule Date.Range do + @moduledoc """ + Returns an inclusive range between dates. + + Ranges must be created with the `Date.range/2` or `Date.range/3` function. + + The following fields are public: + + * `:first` - the initial date on the range + * `:last` - the last date on the range + * `:step` - (since v1.12.0) the step + + The remaining fields are private and should not be accessed. + """ + + @type t :: %__MODULE__{ + first: Date.t(), + last: Date.t(), + first_in_iso_days: iso_days(), + last_in_iso_days: iso_days(), + step: pos_integer | neg_integer + } + + @typep iso_days() :: Calendar.iso_days() + + @enforce_keys [:first, :last, :first_in_iso_days, :last_in_iso_days, :step] + defstruct [:first, :last, :first_in_iso_days, :last_in_iso_days, :step] + + defimpl Enumerable do + def member?( + %Date.Range{ + first: %{calendar: calendar}, + first_in_iso_days: first_days, + last_in_iso_days: last_days, + step: step + } = range, + %Date{calendar: calendar} = date + ) do + {days, _} = Date.to_iso_days(date) + + cond do + empty?(range) -> + {:ok, false} + + first_days <= last_days -> + {:ok, first_days <= days and days <= last_days and rem(days - first_days, step) == 0} + + true -> + {:ok, last_days <= days and days <= first_days and rem(days - first_days, step) == 0} + end + end + + def member?(%Date.Range{step: _}, _) do + {:ok, false} + end + + # TODO: Remove me on v2.0 + def member?( + %{__struct__: Date.Range, first_in_iso_days: first_days, last_in_iso_days: last_days} = + date_range, + date + ) do + step = if first_days <= last_days, do: 1, else: -1 + member?(Map.put(date_range, :step, step), date) + end + + def count(range) do + {:ok, size(range)} + end + + def slice( + %Date.Range{ + first_in_iso_days: first, + first: %{calendar: calendar}, + step: step + } = range + ) do + {:ok, size(range), &slice(first + &1 * step, step + &3 - 1, &2, calendar)} + end + + # TODO: Remove me on v2.0 + def slice( + %{__struct__: Date.Range, first_in_iso_days: first_days, last_in_iso_days: last_days} = + date_range + ) do + step = if first_days <= last_days, do: 1, else: -1 + slice(Map.put(date_range, :step, step)) + end + + defp slice(current, _step, 1, calendar) do + [date_from_iso_days(current, calendar)] + end + + defp slice(current, step, remaining, calendar) do + [ + date_from_iso_days(current, calendar) + | slice(current + step, step, remaining - 1, calendar) + ] + end + + def reduce( + %Date.Range{ + first_in_iso_days: first_days, + last_in_iso_days: last_days, + first: %{calendar: calendar}, + step: step + }, + acc, + fun + ) do + reduce(first_days, last_days, acc, fun, step, calendar) + end + + # TODO: Remove me on v2.0 + def reduce( + %{__struct__: Date.Range, first_in_iso_days: first_days, last_in_iso_days: last_days} = + date_range, + acc, + fun + ) do + step = if first_days <= last_days, do: 1, else: -1 + reduce(Map.put(date_range, :step, step), acc, fun) + end + + defp reduce(_first_days, _last_days, {:halt, acc}, _fun, _step, _calendar) do + {:halted, acc} + end + + defp reduce(first_days, last_days, {:suspend, acc}, fun, step, calendar) do + {:suspended, acc, &reduce(first_days, last_days, &1, fun, step, calendar)} + end + + defp reduce(first_days, last_days, {:cont, acc}, fun, step, calendar) + when step > 0 and first_days <= last_days + when step < 0 and first_days >= last_days do + reduce( + first_days + step, + last_days, + fun.(date_from_iso_days(first_days, calendar), acc), + fun, + step, + calendar + ) + end + + defp reduce(_, _, {:cont, acc}, _fun, _step, _calendar) do + {:done, acc} + end + + defp date_from_iso_days(days, Calendar.ISO) do + {year, month, day} = Calendar.ISO.date_from_iso_days(days) + %Date{year: year, month: month, day: day, calendar: Calendar.ISO} + end + + defp date_from_iso_days(days, calendar) do + {year, month, day, _, _, _, _} = + calendar.naive_datetime_from_iso_days({days, {0, 86_400_000_000}}) + + %Date{year: year, month: month, day: day, calendar: calendar} + end + + defp size(%Date.Range{first_in_iso_days: first_days, last_in_iso_days: last_days, step: step}) + when step > 0 and first_days > last_days, + do: 0 + + defp size(%Date.Range{first_in_iso_days: first_days, last_in_iso_days: last_days, step: step}) + when step < 0 and first_days < last_days, + do: 0 + + defp size(%Date.Range{first_in_iso_days: first_days, last_in_iso_days: last_days, step: step}), + do: abs(div(last_days - first_days, step)) + 1 + + # TODO: Remove me on v2.0 + defp size( + %{__struct__: Date.Range, first_in_iso_days: first_days, last_in_iso_days: last_days} = + date_range + ) do + step = if first_days <= last_days, do: 1, else: -1 + size(Map.put(date_range, :step, step)) + end + + defp empty?(%Date.Range{ + first_in_iso_days: first_days, + last_in_iso_days: last_days, + step: step + }) + when step > 0 and first_days > last_days, + do: true + + defp empty?(%Date.Range{ + first_in_iso_days: first_days, + last_in_iso_days: last_days, + step: step + }) + when step < 0 and first_days < last_days, + do: true + + defp empty?(%Date.Range{step: _}), do: false + + # TODO: Remove me on v2.0 + defp empty?( + %{__struct__: Date.Range, first_in_iso_days: first_days, last_in_iso_days: last_days} = + date_range + ) do + step = if first_days <= last_days, do: 1, else: -1 + empty?(Map.put(date_range, :step, step)) + end + end + + defimpl Inspect do + import Kernel, except: [inspect: 2] + + def inspect(%Date.Range{first: first, last: last, step: 1}, _) do + "Date.range(" <> inspect(first) <> ", " <> inspect(last) <> ")" + end + + def inspect(%Date.Range{first: first, last: last, step: step}, _) do + "Date.range(" <> inspect(first) <> ", " <> inspect(last) <> ", #{step})" + end + + # TODO: Remove me on v2.0 + def inspect(%{__struct__: Date.Range, first: first, last: last} = date_range, opts) do + step = if first <= last, do: 1, else: -1 + inspect(Map.put(date_range, :step, step), opts) + end + end +end diff --git a/lib/elixir/lib/calendar/datetime.ex b/lib/elixir/lib/calendar/datetime.ex new file mode 100644 index 00000000000..1bc53a7c95f --- /dev/null +++ b/lib/elixir/lib/calendar/datetime.ex @@ -0,0 +1,1761 @@ +defmodule DateTime do + @moduledoc """ + A datetime implementation with a time zone. + + This datetime can be seen as a snapshot of a date and time + at a given time zone. For such purposes, it also includes both + UTC and Standard offsets, as well as the zone abbreviation + field used exclusively for formatting purposes. Note future + datetimes are not necessarily guaranteed to exist, as time + zones may change any time in the future due to geopolitical + reasons. See the "Datetimes as snapshots" section for more + information. + + Remember, comparisons in Elixir using `==/2`, `>/2`, ` Enum.min([~U[2022-01-12 00:01:00.00Z], ~U[2021-01-12 00:01:00.00Z]], DateTime) + ~U[2021-01-12 00:01:00.00Z] + + Developers should avoid creating the `DateTime` struct directly + and instead rely on the functions provided by this module as + well as the ones in third-party calendar libraries. + + ## Time zone database + + Many functions in this module require a time zone database. + By default, it uses the default time zone database returned by + `Calendar.get_time_zone_database/0`, which defaults to + `Calendar.UTCOnlyTimeZoneDatabase` which only handles "Etc/UTC" + datetimes and returns `{:error, :utc_only_time_zone_database}` + for any other time zone. + + Other time zone databases can also be configured. Here are some + available options and libraries: + + * [`tz`](https://github.com/mathieuprog/tz) + * [`tzdata`](https://github.com/lau/tzdata) + * [`zoneinfo`](https://github.com/smartrent/zoneinfo) - + recommended for embedded devices + + To use them, first make sure it is added as a dependency in `mix.exs`. + It can then be configured either via configuration: + + config :elixir, :time_zone_database, Tz.TimeZoneDatabase + + or by calling `Calendar.put_time_zone_database/1`: + + Calendar.put_time_zone_database(Tz.TimeZoneDatabase) + + See the proper names in the library installation instructions. + + ## Datetimes as snapshots + + In the first section, we described datetimes as a "snapshot of + a date and time at a given time zone". To understand precisely + what we mean, let's see an example. + + Imagine someone in Poland wants to schedule a meeting with someone + in Brazil in the next year. The meeting will happen at 2:30 AM + in the Polish time zone. At what time will the meeting happen in + Brazil? + + You can consult the time zone database today, one year before, + using the API in this module and it will give you an answer that + is valid right now. However, this answer may not be valid in the + future. Why? Because both Brazil and Poland may change their timezone + rules, ultimately affecting the result. For example, a country may + choose to enter or abandon "Daylight Saving Time", which is a + process where we adjust the clock one hour forward or one hour + back once per year. Whenener the rules change, the exact instant + that 2:30 AM in Polish time will be in Brazil may change. + + In other words, whenever working with future DateTimes, there is + no guarantee the results you get will always be correct, until + the event actually happens. Therefore, when you ask for a future + time, the answers you get are a snapshot that reflects the current + state of the time zone rules. For datetimes in the past, this is + not a problem, because time zone rules do not change for past + events. + + To make matters worse, it may be that the 2:30 AM in Polish time + does not actually even exist or it is ambiguous. If a certain + time zone observes "Daylight Saving Time", they will move their + clock forward once a year. When this happens, there is a whole + hour that does not exist. Then, when they move the clock back, + there is a certain hour that will happen twice. So if you want + to schedule a meeting when this shift back happens, you would + need to explicitly say which of the 2:30 AM you precisely mean. + Applications that are date and time sensitive, need to take + these scenarios into account and correctly communicate them to + users. + + The good news is: Elixir contains all of the building blocks + necessary to tackle those problems. The default timezone database + used by Elixir, `Calendar.UTCOnlyTimeZoneDatabase`, only works + with UTC, which does not observe those issues. Once you bring + a proper time zone database, the functions in this module will + query the database and return the relevant information. For + example, look at how `DateTime.new/4` returns different results + based on the scenarios described in this section. + """ + + @enforce_keys [:year, :month, :day, :hour, :minute, :second] ++ + [:time_zone, :zone_abbr, :utc_offset, :std_offset] + + defstruct [ + :year, + :month, + :day, + :hour, + :minute, + :second, + :time_zone, + :zone_abbr, + :utc_offset, + :std_offset, + microsecond: {0, 0}, + calendar: Calendar.ISO + ] + + @type t :: %__MODULE__{ + year: Calendar.year(), + month: Calendar.month(), + day: Calendar.day(), + calendar: Calendar.calendar(), + hour: Calendar.hour(), + minute: Calendar.minute(), + second: Calendar.second(), + microsecond: Calendar.microsecond(), + time_zone: Calendar.time_zone(), + zone_abbr: Calendar.zone_abbr(), + utc_offset: Calendar.utc_offset(), + std_offset: Calendar.std_offset() + } + + @unix_days :calendar.date_to_gregorian_days({1970, 1, 1}) + @seconds_per_day 24 * 60 * 60 + + @doc """ + Returns the current datetime in UTC. + + ## Examples + + iex> datetime = DateTime.utc_now() + iex> datetime.time_zone + "Etc/UTC" + + """ + @spec utc_now(Calendar.calendar()) :: t + def utc_now(calendar \\ Calendar.ISO) do + System.os_time() |> from_unix!(:native, calendar) + end + + @doc """ + Builds a datetime from date and time structs. + + It expects a time zone to put the `DateTime` in. + If the time zone is not passed it will default to `"Etc/UTC"`, + which always succeeds. Otherwise, the `DateTime` is checked against the time zone database + given as `time_zone_database`. See the "Time zone database" + section in the module documentation. + + ## Examples + + iex> DateTime.new(~D[2016-05-24], ~T[13:26:08.003], "Etc/UTC") + {:ok, ~U[2016-05-24 13:26:08.003Z]} + + When the datetime is ambiguous - for instance during changing from summer + to winter time - the two possible valid datetimes are returned in a tuple. + The first datetime is also the one which comes first chronologically, while + the second one comes last. + + iex> {:ambiguous, first_dt, second_dt} = DateTime.new(~D[2018-10-28], ~T[02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> first_dt + #DateTime<2018-10-28 02:30:00+02:00 CEST Europe/Copenhagen> + iex> second_dt + #DateTime<2018-10-28 02:30:00+01:00 CET Europe/Copenhagen> + + When there is a gap in wall time - for instance in spring when the clocks are + turned forward - the latest valid datetime just before the gap and the first + valid datetime just after the gap. + + iex> {:gap, just_before, just_after} = DateTime.new(~D[2019-03-31], ~T[02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> just_before + #DateTime<2019-03-31 01:59:59.999999+01:00 CET Europe/Copenhagen> + iex> just_after + #DateTime<2019-03-31 03:00:00+02:00 CEST Europe/Copenhagen> + + Most of the time there is one, and just one, valid datetime for a certain + date and time in a certain time zone. + + iex> {:ok, datetime} = DateTime.new(~D[2018-07-28], ~T[12:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> datetime + #DateTime<2018-07-28 12:30:00+02:00 CEST Europe/Copenhagen> + + """ + @doc since: "1.11.0" + @spec new(Date.t(), Time.t(), Calendar.time_zone(), Calendar.time_zone_database()) :: + {:ok, t} + | {:ambiguous, first_datetime :: t, second_datetime :: t} + | {:gap, t, t} + | {:error, + :incompatible_calendars | :time_zone_not_found | :utc_only_time_zone_database} + def new( + date, + time, + time_zone \\ "Etc/UTC", + time_zone_database \\ Calendar.get_time_zone_database() + ) + + def new(%Date{calendar: calendar} = date, %Time{calendar: calendar} = time, "Etc/UTC", _db) do + %{year: year, month: month, day: day} = date + %{hour: hour, minute: minute, second: second, microsecond: microsecond} = time + + datetime = %DateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + std_offset: 0, + utc_offset: 0, + zone_abbr: "UTC", + time_zone: "Etc/UTC" + } + + {:ok, datetime} + end + + def new(date, time, time_zone, time_zone_database) do + with {:ok, naive_datetime} <- NaiveDateTime.new(date, time) do + from_naive(naive_datetime, time_zone, time_zone_database) + end + end + + @doc """ + Builds a datetime from date and time structs, raising on errors. + + It expects a time zone to put the `DateTime` in. + If the time zone is not passed it will default to `"Etc/UTC"`, + which always succeeds. Otherwise, the DateTime is checked against the time zone database + given as `time_zone_database`. See the "Time zone database" + section in the module documentation. + + ## Examples + + iex> DateTime.new!(~D[2016-05-24], ~T[13:26:08.003], "Etc/UTC") + ~U[2016-05-24 13:26:08.003Z] + + When the datetime is ambiguous - for instance during changing from summer + to winter time - an error will be raised. + + iex> DateTime.new!(~D[2018-10-28], ~T[02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + ** (ArgumentError) cannot build datetime with ~D[2018-10-28] and ~T[02:30:00] because such instant is ambiguous in time zone Europe/Copenhagen as there is an overlap between #DateTime<2018-10-28 02:30:00+02:00 CEST Europe/Copenhagen> and #DateTime<2018-10-28 02:30:00+01:00 CET Europe/Copenhagen> + + When there is a gap in wall time - for instance in spring when the clocks are + turned forward - an error will be raised. + + iex> DateTime.new!(~D[2019-03-31], ~T[02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + ** (ArgumentError) cannot build datetime with ~D[2019-03-31] and ~T[02:30:00] because such instant does not exist in time zone Europe/Copenhagen as there is a gap between #DateTime<2019-03-31 01:59:59.999999+01:00 CET Europe/Copenhagen> and #DateTime<2019-03-31 03:00:00+02:00 CEST Europe/Copenhagen> + + Most of the time there is one, and just one, valid datetime for a certain + date and time in a certain time zone. + + iex> datetime = DateTime.new!(~D[2018-07-28], ~T[12:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> datetime + #DateTime<2018-07-28 12:30:00+02:00 CEST Europe/Copenhagen> + + """ + @doc since: "1.11.0" + @spec new!(Date.t(), Time.t(), Calendar.time_zone(), Calendar.time_zone_database()) :: t + def new!( + date, + time, + time_zone \\ "Etc/UTC", + time_zone_database \\ Calendar.get_time_zone_database() + ) + + def new!(date, time, time_zone, time_zone_database) do + case new(date, time, time_zone, time_zone_database) do + {:ok, datetime} -> + datetime + + {:ambiguous, dt1, dt2} -> + raise ArgumentError, + "cannot build datetime with #{inspect(date)} and #{inspect(time)} because such " <> + "instant is ambiguous in time zone #{time_zone} as there is an overlap " <> + "between #{inspect(dt1)} and #{inspect(dt2)}" + + {:gap, dt1, dt2} -> + raise ArgumentError, + "cannot build datetime with #{inspect(date)} and #{inspect(time)} because such " <> + "instant does not exist in time zone #{time_zone} as there is a gap " <> + "between #{inspect(dt1)} and #{inspect(dt2)}" + + {:error, reason} -> + raise ArgumentError, + "cannot build datetime with #{inspect(date)} and #{inspect(time)}, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts the given Unix time to `DateTime`. + + The integer can be given in different unit + according to `System.convert_time_unit/3` and it will + be converted to microseconds internally. Up to + 253402300799 seconds is supported. + + Unix times are always in UTC and therefore the DateTime + will be returned in UTC. + + ## Examples + + iex> {:ok, datetime} = DateTime.from_unix(1_464_096_368) + iex> datetime + ~U[2016-05-24 13:26:08Z] + + iex> {:ok, datetime} = DateTime.from_unix(1_432_560_368_868_569, :microsecond) + iex> datetime + ~U[2015-05-25 13:26:08.868569Z] + + iex> {:ok, datetime} = DateTime.from_unix(253_402_300_799) + iex> datetime + ~U[9999-12-31 23:59:59Z] + + iex> {:error, :invalid_unix_time} = DateTime.from_unix(253_402_300_800) + + The unit can also be an integer as in `t:System.time_unit/0`: + + iex> {:ok, datetime} = DateTime.from_unix(143_256_036_886_856, 1024) + iex> datetime + ~U[6403-03-17 07:05:22.320312Z] + + Negative Unix times are supported up to -377705116800 seconds: + + iex> {:ok, datetime} = DateTime.from_unix(-377_705_116_800) + iex> datetime + ~U[-9999-01-01 00:00:00Z] + + iex> {:error, :invalid_unix_time} = DateTime.from_unix(-377_705_116_801) + + """ + @spec from_unix(integer, :native | System.time_unit(), Calendar.calendar()) :: + {:ok, t} | {:error, atom} + def from_unix(integer, unit \\ :second, calendar \\ Calendar.ISO) when is_integer(integer) do + case Calendar.ISO.from_unix(integer, unit) do + {:ok, {year, month, day}, {hour, minute, second}, microsecond} -> + iso_datetime = %DateTime{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + std_offset: 0, + utc_offset: 0, + zone_abbr: "UTC", + time_zone: "Etc/UTC" + } + + convert(iso_datetime, calendar) + + {:error, _} = error -> + error + end + end + + @doc """ + Converts the given Unix time to `DateTime`. + + The integer can be given in different unit + according to `System.convert_time_unit/3` and it will + be converted to microseconds internally. + + Unix times are always in UTC and therefore the DateTime + will be returned in UTC. + + ## Examples + + # An easy way to get the Unix epoch is passing 0 to this function + iex> DateTime.from_unix!(0) + ~U[1970-01-01 00:00:00Z] + + iex> DateTime.from_unix!(1_464_096_368) + ~U[2016-05-24 13:26:08Z] + + iex> DateTime.from_unix!(1_432_560_368_868_569, :microsecond) + ~U[2015-05-25 13:26:08.868569Z] + + iex> DateTime.from_unix!(143_256_036_886_856, 1024) + ~U[6403-03-17 07:05:22.320312Z] + + """ + @spec from_unix!(integer, :native | System.time_unit(), Calendar.calendar()) :: t + def from_unix!(integer, unit \\ :second, calendar \\ Calendar.ISO) do + case from_unix(integer, unit, calendar) do + {:ok, datetime} -> + datetime + + {:error, :invalid_unix_time} -> + raise ArgumentError, "invalid Unix time #{integer}" + end + end + + @doc """ + Converts the given `NaiveDateTime` to `DateTime`. + + It expects a time zone to put the `NaiveDateTime` in. + If the time zone is "Etc/UTC", it always succeeds. Otherwise, + the NaiveDateTime is checked against the time zone database + given as `time_zone_database`. See the "Time zone database" + section in the module documentation. + + ## Examples + + iex> DateTime.from_naive(~N[2016-05-24 13:26:08.003], "Etc/UTC") + {:ok, ~U[2016-05-24 13:26:08.003Z]} + + When the datetime is ambiguous - for instance during changing from summer + to winter time - the two possible valid datetimes are returned in a tuple. + The first datetime is also the one which comes first chronologically, while + the second one comes last. + + iex> {:ambiguous, first_dt, second_dt} = DateTime.from_naive(~N[2018-10-28 02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> first_dt + #DateTime<2018-10-28 02:30:00+02:00 CEST Europe/Copenhagen> + iex> second_dt + #DateTime<2018-10-28 02:30:00+01:00 CET Europe/Copenhagen> + + When there is a gap in wall time - for instance in spring when the clocks are + turned forward - the latest valid datetime just before the gap and the first + valid datetime just after the gap. + + iex> {:gap, just_before, just_after} = DateTime.from_naive(~N[2019-03-31 02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> just_before + #DateTime<2019-03-31 01:59:59.999999+01:00 CET Europe/Copenhagen> + iex> just_after + #DateTime<2019-03-31 03:00:00+02:00 CEST Europe/Copenhagen> + + Most of the time there is one, and just one, valid datetime for a certain + date and time in a certain time zone. + + iex> {:ok, datetime} = DateTime.from_naive(~N[2018-07-28 12:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> datetime + #DateTime<2018-07-28 12:30:00+02:00 CEST Europe/Copenhagen> + + This function accepts any map or struct that contains at least the same fields as a `NaiveDateTime` + struct. The most common example of that is a `DateTime`. In this case the information about the time + zone of that `DateTime` is completely ignored. This is the same principle as passing a `DateTime` to + `Date.to_iso8601/2`. `Date.to_iso8601/2` extracts only the date-specific fields (calendar, year, + month and day) of the given structure and ignores all others. + + This way if you have a `DateTime` in one time zone, you can get the same wall time in another time zone. + For instance if you have 2018-08-24 10:00:00 in Copenhagen and want a `DateTime` for 2018-08-24 10:00:00 + in UTC you can do: + + iex> cph_datetime = DateTime.from_naive!(~N[2018-08-24 10:00:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> {:ok, utc_datetime} = DateTime.from_naive(cph_datetime, "Etc/UTC", FakeTimeZoneDatabase) + iex> utc_datetime + ~U[2018-08-24 10:00:00Z] + + If instead you want a `DateTime` for the same point time in a different time zone see the + `DateTime.shift_zone/3` function which would convert 2018-08-24 10:00:00 in Copenhagen + to 2018-08-24 08:00:00 in UTC. + """ + @doc since: "1.4.0" + @spec from_naive( + Calendar.naive_datetime(), + Calendar.time_zone(), + Calendar.time_zone_database() + ) :: + {:ok, t} + | {:ambiguous, first_datetime :: t, second_datetime :: t} + | {:gap, t, t} + | {:error, + :incompatible_calendars | :time_zone_not_found | :utc_only_time_zone_database} + + def from_naive( + naive_datetime, + time_zone, + time_zone_database \\ Calendar.get_time_zone_database() + ) + + def from_naive(naive_datetime, "Etc/UTC", _) do + utc_period = %{std_offset: 0, utc_offset: 0, zone_abbr: "UTC"} + {:ok, from_naive_with_period(naive_datetime, "Etc/UTC", utc_period)} + end + + def from_naive(%{calendar: Calendar.ISO} = naive_datetime, time_zone, time_zone_database) do + case time_zone_database.time_zone_periods_from_wall_datetime(naive_datetime, time_zone) do + {:ok, period} -> + {:ok, from_naive_with_period(naive_datetime, time_zone, period)} + + {:ambiguous, first_period, second_period} -> + first_datetime = from_naive_with_period(naive_datetime, time_zone, first_period) + second_datetime = from_naive_with_period(naive_datetime, time_zone, second_period) + {:ambiguous, first_datetime, second_datetime} + + {:gap, {first_period, first_period_until_wall}, {second_period, second_period_from_wall}} -> + # `until_wall` is not valid, but any time just before is. + # So by subtracting a second and adding .999999 seconds + # we get the last microsecond just before. + before_naive = + first_period_until_wall + |> Map.replace!(:microsecond, {999_999, 6}) + |> NaiveDateTime.add(-1) + + after_naive = second_period_from_wall + + latest_datetime_before = from_naive_with_period(before_naive, time_zone, first_period) + first_datetime_after = from_naive_with_period(after_naive, time_zone, second_period) + {:gap, latest_datetime_before, first_datetime_after} + + {:error, _} = error -> + error + end + end + + def from_naive(%{calendar: calendar} = naive_datetime, time_zone, time_zone_database) + when calendar != Calendar.ISO do + # For non-ISO calendars, convert to ISO, create ISO DateTime, and then + # convert to original calendar + iso_result = + with {:ok, in_iso} <- NaiveDateTime.convert(naive_datetime, Calendar.ISO) do + from_naive(in_iso, time_zone, time_zone_database) + end + + case iso_result do + {:ok, dt} -> + convert(dt, calendar) + + {:ambiguous, dt1, dt2} -> + with {:ok, dt1converted} <- convert(dt1, calendar), + {:ok, dt2converted} <- convert(dt2, calendar), + do: {:ambiguous, dt1converted, dt2converted} + + {:gap, dt1, dt2} -> + with {:ok, dt1converted} <- convert(dt1, calendar), + {:ok, dt2converted} <- convert(dt2, calendar), + do: {:gap, dt1converted, dt2converted} + + {:error, _} = error -> + error + end + end + + defp from_naive_with_period(naive_datetime, time_zone, period) do + %{std_offset: std_offset, utc_offset: utc_offset, zone_abbr: zone_abbr} = period + + %{ + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + year: year, + month: month, + day: day + } = naive_datetime + + %DateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + std_offset: std_offset, + utc_offset: utc_offset, + zone_abbr: zone_abbr, + time_zone: time_zone + } + end + + @doc """ + Converts the given `NaiveDateTime` to `DateTime`. + + It expects a time zone to put the NaiveDateTime in. + If the time zone is "Etc/UTC", it always succeeds. Otherwise, + the NaiveDateTime is checked against the time zone database + given as `time_zone_database`. See the "Time zone database" + section in the module documentation. + + ## Examples + + iex> DateTime.from_naive!(~N[2016-05-24 13:26:08.003], "Etc/UTC") + ~U[2016-05-24 13:26:08.003Z] + + iex> DateTime.from_naive!(~N[2018-05-24 13:26:08.003], "Europe/Copenhagen", FakeTimeZoneDatabase) + #DateTime<2018-05-24 13:26:08.003+02:00 CEST Europe/Copenhagen> + + """ + @doc since: "1.4.0" + @spec from_naive!( + NaiveDateTime.t(), + Calendar.time_zone(), + Calendar.time_zone_database() + ) :: t + def from_naive!( + naive_datetime, + time_zone, + time_zone_database \\ Calendar.get_time_zone_database() + ) do + case from_naive(naive_datetime, time_zone, time_zone_database) do + {:ok, datetime} -> + datetime + + {:ambiguous, dt1, dt2} -> + raise ArgumentError, + "cannot convert #{inspect(naive_datetime)} to datetime because such " <> + "instant is ambiguous in time zone #{time_zone} as there is an overlap " <> + "between #{inspect(dt1)} and #{inspect(dt2)}" + + {:gap, dt1, dt2} -> + raise ArgumentError, + "cannot convert #{inspect(naive_datetime)} to datetime because such " <> + "instant does not exist in time zone #{time_zone} as there is a gap " <> + "between #{inspect(dt1)} and #{inspect(dt2)}" + + {:error, reason} -> + raise ArgumentError, + "cannot convert #{inspect(naive_datetime)} to datetime, reason: #{inspect(reason)}" + end + end + + @doc """ + Changes the time zone of a `DateTime`. + + Returns a `DateTime` for the same point in time, but instead at + the time zone provided. It assumes that `DateTime` is valid and + exists in the given time zone and calendar. + + By default, it uses the default time zone database returned by + `Calendar.get_time_zone_database/0`, which defaults to + `Calendar.UTCOnlyTimeZoneDatabase` which only handles "Etc/UTC" datetimes. + Other time zone databases can be passed as argument or set globally. + See the "Time zone database" section in the module docs. + + ## Examples + + iex> {:ok, pacific_datetime} = DateTime.shift_zone(~U[2018-07-16 10:00:00Z], "America/Los_Angeles", FakeTimeZoneDatabase) + iex> pacific_datetime + #DateTime<2018-07-16 03:00:00-07:00 PDT America/Los_Angeles> + + iex> DateTime.shift_zone(~U[2018-07-16 10:00:00Z], "bad timezone", FakeTimeZoneDatabase) + {:error, :time_zone_not_found} + + """ + @doc since: "1.8.0" + @spec shift_zone(t, Calendar.time_zone(), Calendar.time_zone_database()) :: + {:ok, t} | {:error, :time_zone_not_found | :utc_only_time_zone_database} + def shift_zone(datetime, time_zone, time_zone_database \\ Calendar.get_time_zone_database()) + + def shift_zone(%{time_zone: time_zone} = datetime, time_zone, _) do + {:ok, datetime} + end + + def shift_zone(datetime, time_zone, time_zone_database) do + %{ + std_offset: std_offset, + utc_offset: utc_offset, + calendar: calendar, + microsecond: {_, precision} + } = datetime + + datetime + |> to_iso_days() + |> apply_tz_offset(utc_offset + std_offset) + |> shift_zone_for_iso_days_utc(calendar, precision, time_zone, time_zone_database) + end + + defp shift_zone_for_iso_days_utc(iso_days_utc, calendar, precision, time_zone, time_zone_db) do + case time_zone_db.time_zone_period_from_utc_iso_days(iso_days_utc, time_zone) do + {:ok, %{std_offset: std_offset, utc_offset: utc_offset, zone_abbr: zone_abbr}} -> + {year, month, day, hour, minute, second, {microsecond_without_precision, _}} = + iso_days_utc + |> apply_tz_offset(-(utc_offset + std_offset)) + |> calendar.naive_datetime_from_iso_days() + + datetime = %DateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond_without_precision, precision}, + std_offset: std_offset, + utc_offset: utc_offset, + zone_abbr: zone_abbr, + time_zone: time_zone + } + + {:ok, datetime} + + {:error, _} = error -> + error + end + end + + @doc """ + Changes the time zone of a `DateTime` or raises on errors. + + See `shift_zone/3` for more information. + + ## Examples + + iex> DateTime.shift_zone!(~U[2018-07-16 10:00:00Z], "America/Los_Angeles", FakeTimeZoneDatabase) + #DateTime<2018-07-16 03:00:00-07:00 PDT America/Los_Angeles> + + iex> DateTime.shift_zone!(~U[2018-07-16 10:00:00Z], "bad timezone", FakeTimeZoneDatabase) + ** (ArgumentError) cannot shift ~U[2018-07-16 10:00:00Z] to "bad timezone" time zone, reason: :time_zone_not_found + + """ + @doc since: "1.10.0" + @spec shift_zone!(t, Calendar.time_zone(), Calendar.time_zone_database()) :: t + def shift_zone!(datetime, time_zone, time_zone_database \\ Calendar.get_time_zone_database()) do + case shift_zone(datetime, time_zone, time_zone_database) do + {:ok, datetime} -> + datetime + + {:error, reason} -> + raise ArgumentError, + "cannot shift #{inspect(datetime)} to #{inspect(time_zone)} time zone" <> + ", reason: #{inspect(reason)}" + end + end + + @doc """ + Returns the current datetime in the provided time zone. + + By default, it uses the default time_zone returned by + `Calendar.get_time_zone_database/0`, which defaults to + `Calendar.UTCOnlyTimeZoneDatabase` which only handles "Etc/UTC" datetimes. + Other time zone databases can be passed as argument or set globally. + See the "Time zone database" section in the module docs. + + ## Examples + + iex> {:ok, datetime} = DateTime.now("Etc/UTC") + iex> datetime.time_zone + "Etc/UTC" + + iex> DateTime.now("Europe/Copenhagen") + {:error, :utc_only_time_zone_database} + + iex> DateTime.now("bad timezone", FakeTimeZoneDatabase) + {:error, :time_zone_not_found} + + """ + @doc since: "1.8.0" + @spec now(Calendar.time_zone(), Calendar.time_zone_database()) :: + {:ok, t} | {:error, :time_zone_not_found | :utc_only_time_zone_database} + def now(time_zone, time_zone_database \\ Calendar.get_time_zone_database()) + + def now("Etc/UTC", _) do + {:ok, utc_now()} + end + + def now(time_zone, time_zone_database) do + shift_zone(utc_now(), time_zone, time_zone_database) + end + + @doc """ + Returns the current datetime in the provided time zone or raises on errors + + See `now/2` for more information. + + ## Examples + + iex> datetime = DateTime.now!("Etc/UTC") + iex> datetime.time_zone + "Etc/UTC" + + iex> DateTime.now!("Europe/Copenhagen") + ** (ArgumentError) cannot get current datetime in "Europe/Copenhagen" time zone, reason: :utc_only_time_zone_database + + iex> DateTime.now!("bad timezone", FakeTimeZoneDatabase) + ** (ArgumentError) cannot get current datetime in "bad timezone" time zone, reason: :time_zone_not_found + + """ + @doc since: "1.10.0" + @spec now!(Calendar.time_zone(), Calendar.time_zone_database()) :: t + def now!(time_zone, time_zone_database \\ Calendar.get_time_zone_database()) do + case now(time_zone, time_zone_database) do + {:ok, datetime} -> + datetime + + {:error, reason} -> + raise ArgumentError, + "cannot get current datetime in #{inspect(time_zone)} time zone, reason: " <> + inspect(reason) + end + end + + @doc """ + Converts the given `datetime` to Unix time. + + The `datetime` is expected to be using the ISO calendar + with a year greater than or equal to 0. + + It will return the integer with the given unit, + according to `System.convert_time_unit/3`. + + If you want to get the current time in Unix seconds, + do not do `DateTime.utc_now() |> DateTime.to_unix()`. + Simply call `System.os_time(:second)` instead. + + ## Examples + + iex> 1_464_096_368 |> DateTime.from_unix!() |> DateTime.to_unix() + 1464096368 + + iex> dt = %DateTime{calendar: Calendar.ISO, day: 20, hour: 18, microsecond: {273806, 6}, + ...> minute: 58, month: 11, second: 19, time_zone: "America/Montevideo", + ...> utc_offset: -10800, std_offset: 3600, year: 2014, zone_abbr: "UYST"} + iex> DateTime.to_unix(dt) + 1416517099 + + iex> flamel = %DateTime{calendar: Calendar.ISO, day: 22, hour: 8, microsecond: {527771, 6}, + ...> minute: 2, month: 3, second: 25, std_offset: 0, time_zone: "Etc/UTC", + ...> utc_offset: 0, year: 1418, zone_abbr: "UTC"} + iex> DateTime.to_unix(flamel) + -17412508655 + + """ + @spec to_unix(Calendar.datetime(), System.time_unit()) :: integer + def to_unix(datetime, unit \\ :second) + + def to_unix(%{utc_offset: utc_offset, std_offset: std_offset} = datetime, unit) do + {days, fraction} = to_iso_days(datetime) + unix_units = Calendar.ISO.iso_days_to_unit({days - @unix_days, fraction}, unit) + offset_units = System.convert_time_unit(utc_offset + std_offset, :second, unit) + unix_units - offset_units + end + + @doc """ + Converts the given `datetime` into a `NaiveDateTime`. + + Because `NaiveDateTime` does not hold time zone information, + any time zone related data will be lost during the conversion. + + ## Examples + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 1}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.to_naive(dt) + ~N[2000-02-29 23:00:07.0] + + """ + @spec to_naive(Calendar.datetime()) :: NaiveDateTime.t() + def to_naive(datetime) + + def to_naive(%{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: _ + }) do + %NaiveDateTime{ + year: year, + month: month, + day: day, + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + end + + @doc """ + Converts a `DateTime` into a `Date`. + + Because `Date` does not hold time nor time zone information, + data will be lost during the conversion. + + ## Examples + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.to_date(dt) + ~D[2000-02-29] + + """ + @spec to_date(Calendar.datetime()) :: Date.t() + def to_date(datetime) + + def to_date(%{ + year: year, + month: month, + day: day, + calendar: calendar, + hour: _, + minute: _, + second: _, + microsecond: _, + time_zone: _ + }) do + %Date{year: year, month: month, day: day, calendar: calendar} + end + + @doc """ + Converts a `DateTime` into `Time`. + + Because `Time` does not hold date nor time zone information, + data will be lost during the conversion. + + ## Examples + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 1}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.to_time(dt) + ~T[23:00:07.0] + + """ + @spec to_time(Calendar.datetime()) :: Time.t() + def to_time(datetime) + + def to_time(%{ + year: _, + month: _, + day: _, + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: _ + }) do + %Time{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + } + end + + @doc """ + Converts the given datetime to + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601) format. + + By default, `DateTime.to_iso8601/2` returns datetimes formatted in the "extended" + format, for human readability. It also supports the "basic" format through passing the `:basic` option. + + Only supports converting datetimes which are in the ISO calendar, + attempting to convert datetimes from other calendars will raise. + You can also optionally specify an offset for the formatted string. + + WARNING: the ISO 8601 datetime format does not contain the time zone nor + its abbreviation, which means information is lost when converting to such + format. + + ### Examples + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.to_iso8601(dt) + "2000-02-29T23:00:07+01:00" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "UTC", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 0, std_offset: 0, time_zone: "Etc/UTC"} + iex> DateTime.to_iso8601(dt) + "2000-02-29T23:00:07Z" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.to_iso8601(dt, :extended) + "2000-02-29T23:00:07-04:00" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.to_iso8601(dt, :basic) + "20000229T230007-0400" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.to_iso8601(dt, :extended, 3600) + "2000-03-01T04:00:07+01:00" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.to_iso8601(dt, :extended, 0) + "2000-03-01T03:00:07+00:00" + + iex> dt = %DateTime{year: 2000, month: 3, day: 01, zone_abbr: "UTC", + ...> hour: 03, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 0, std_offset: 0, time_zone: "Etc/UTC"} + iex> DateTime.to_iso8601(dt, :extended, 0) + "2000-03-01T03:00:07Z" + + iex> {:ok, dt, offset} = DateTime.from_iso8601("2000-03-01T03:00:07Z") + iex> "2000-03-01T03:00:07Z" = DateTime.to_iso8601(dt, :extended, offset) + """ + @spec to_iso8601(Calendar.datetime(), :basic | :extended, nil | integer()) :: String.t() + def to_iso8601(datetime, format \\ :extended, offset \\ nil) + + def to_iso8601(%{calendar: Calendar.ISO} = datetime, format, nil) + when format in [:extended, :basic] do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: time_zone, + utc_offset: utc_offset, + std_offset: std_offset + } = datetime + + datetime_to_string(year, month, day, hour, minute, second, microsecond, format) <> + Calendar.ISO.offset_to_string(utc_offset, std_offset, time_zone, format) + end + + def to_iso8601( + %{calendar: Calendar.ISO, microsecond: {_, precision}, time_zone: "Etc/UTC"} = datetime, + format, + 0 + ) + when format in [:extended, :basic] do + {year, month, day, hour, minute, second, {microsecond, _}} = shift_by_offset(datetime, 0) + + datetime_to_string(year, month, day, hour, minute, second, {microsecond, precision}, format) <> + "Z" + end + + def to_iso8601(%{calendar: Calendar.ISO} = datetime, format, offset) + when format in [:extended, :basic] do + {_, precision} = datetime.microsecond + {year, month, day, hour, minute, second, {microsecond, _}} = shift_by_offset(datetime, offset) + + datetime_to_string(year, month, day, hour, minute, second, {microsecond, precision}, format) <> + Calendar.ISO.offset_to_string(offset, 0, nil, format) + end + + def to_iso8601(%{calendar: _} = datetime, format, offset) when format in [:extended, :basic] do + datetime + |> convert!(Calendar.ISO) + |> to_iso8601(format, offset) + end + + defp shift_by_offset(%{calendar: calendar} = datetime, offset) do + total_offset = datetime.utc_offset + datetime.std_offset + + datetime + |> to_iso_days() + # Subtract total original offset in order to get UTC and add the new offset + |> Calendar.ISO.add_day_fraction_to_iso_days(offset - total_offset, 86400) + |> calendar.naive_datetime_from_iso_days() + end + + defp datetime_to_string(year, month, day, hour, minute, second, microsecond, format) do + Calendar.ISO.date_to_string(year, month, day, format) <> + "T" <> + Calendar.ISO.time_to_string(hour, minute, second, microsecond, format) + end + + @doc """ + Parses the extended "Date and time of day" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + Since ISO 8601 does not include the proper time zone, the given + string will be converted to UTC and its offset in seconds will be + returned as part of this function. Therefore offset information + must be present in the string. + + As specified in the standard, the separator "T" may be omitted if + desired as there is no ambiguity within this function. + + Note leap seconds are not supported by the built-in Calendar.ISO. + + ## Examples + + iex> {:ok, datetime, 0} = DateTime.from_iso8601("2015-01-23T23:50:07Z") + iex> datetime + ~U[2015-01-23 23:50:07Z] + + iex> {:ok, datetime, 9000} = DateTime.from_iso8601("2015-01-23T23:50:07.123+02:30") + iex> datetime + ~U[2015-01-23 21:20:07.123Z] + + iex> {:ok, datetime, 9000} = DateTime.from_iso8601("2015-01-23T23:50:07,123+02:30") + iex> datetime + ~U[2015-01-23 21:20:07.123Z] + + iex> {:ok, datetime, 0} = DateTime.from_iso8601("-2015-01-23T23:50:07Z") + iex> datetime + ~U[-2015-01-23 23:50:07Z] + + iex> {:ok, datetime, 9000} = DateTime.from_iso8601("-2015-01-23T23:50:07,123+02:30") + iex> datetime + ~U[-2015-01-23 21:20:07.123Z] + + iex> {:ok, datetime, 9000} = DateTime.from_iso8601("20150123T235007.123+0230", :basic) + iex> datetime + ~U[2015-01-23 21:20:07.123Z] + + iex> DateTime.from_iso8601("2015-01-23P23:50:07") + {:error, :invalid_format} + iex> DateTime.from_iso8601("2015-01-23T23:50:07") + {:error, :missing_offset} + iex> DateTime.from_iso8601("2015-01-23 23:50:61") + {:error, :invalid_time} + iex> DateTime.from_iso8601("2015-01-32 23:50:07") + {:error, :invalid_date} + iex> DateTime.from_iso8601("2015-01-23T23:50:07.123-00:00") + {:error, :invalid_format} + + """ + @doc since: "1.4.0" + @spec from_iso8601(String.t(), Calendar.calendar(), :extended | :basic) :: + {:ok, t, Calendar.utc_offset()} | {:error, atom} + + def from_iso8601(string, format_or_calendar \\ Calendar.ISO) + + def from_iso8601(string, format) when format in [:basic, :extended] do + from_iso8601(string, Calendar.ISO, format) + end + + def from_iso8601(string, calendar) when is_atom(calendar) do + from_iso8601(string, calendar, :extended) + end + + @doc """ + Converts to ISO8601 specifying both a calendar and a mode. + + See `from_iso8601/2` for more information. + + ## Examples + + iex> {:ok, datetime, 9000} = DateTime.from_iso8601("2015-01-23T23:50:07,123+02:30", Calendar.ISO, :extended) + iex> datetime + ~U[2015-01-23 21:20:07.123Z] + + iex> {:ok, datetime, 9000} = DateTime.from_iso8601("20150123T235007.123+0230", Calendar.ISO, :basic) + iex> datetime + ~U[2015-01-23 21:20:07.123Z] + """ + def from_iso8601(string, calendar, format) do + with {:ok, {year, month, day, hour, minute, second, microsecond}, offset} <- + Calendar.ISO.parse_utc_datetime(string, format) do + datetime = %DateTime{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + std_offset: 0, + utc_offset: 0, + zone_abbr: "UTC", + time_zone: "Etc/UTC" + } + + with {:ok, converted} <- convert(datetime, calendar) do + {:ok, converted, offset} + end + end + end + + @doc """ + Converts a number of gregorian seconds to a `DateTime` struct. + + The returned `DateTime` will have `UTC` timezone, if you want other timezone, please use + `DateTime.shift_zone/3`. + + ## Examples + + iex> DateTime.from_gregorian_seconds(1) + ~U[0000-01-01 00:00:01Z] + iex> DateTime.from_gregorian_seconds(63_755_511_991, {5000, 3}) + ~U[2020-05-01 00:26:31.005Z] + iex> DateTime.from_gregorian_seconds(-1) + ~U[-0001-12-31 23:59:59Z] + + """ + @doc since: "1.11.0" + @spec from_gregorian_seconds(integer(), Calendar.microsecond(), Calendar.calendar()) :: t + def from_gregorian_seconds( + seconds, + {microsecond, precision} \\ {0, 0}, + calendar \\ Calendar.ISO + ) + when is_integer(seconds) do + iso_days = Calendar.ISO.gregorian_seconds_to_iso_days(seconds, microsecond) + + {year, month, day, hour, minute, second, {microsecond, _}} = + calendar.naive_datetime_from_iso_days(iso_days) + + %DateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision}, + std_offset: 0, + utc_offset: 0, + zone_abbr: "UTC", + time_zone: "Etc/UTC" + } + end + + @doc """ + Converts a `DateTime` struct to a number of gregorian seconds and microseconds. + + ## Examples + + iex> dt = %DateTime{year: 0000, month: 1, day: 1, zone_abbr: "UTC", + ...> hour: 0, minute: 0, second: 1, microsecond: {0, 0}, + ...> utc_offset: 0, std_offset: 0, time_zone: "Etc/UTC"} + iex> DateTime.to_gregorian_seconds(dt) + {1, 0} + + iex> dt = %DateTime{year: 2020, month: 5, day: 1, zone_abbr: "UTC", + ...> hour: 0, minute: 26, second: 31, microsecond: {5000, 0}, + ...> utc_offset: 0, std_offset: 0, time_zone: "Etc/UTC"} + iex> DateTime.to_gregorian_seconds(dt) + {63_755_511_991, 5000} + + iex> dt = %DateTime{year: 2020, month: 5, day: 1, zone_abbr: "CET", + ...> hour: 1, minute: 26, second: 31, microsecond: {5000, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.to_gregorian_seconds(dt) + {63_755_511_991, 5000} + + """ + @doc since: "1.11.0" + @spec to_gregorian_seconds(Calendar.datetime()) :: {integer(), non_neg_integer()} + def to_gregorian_seconds( + %{ + std_offset: std_offset, + utc_offset: utc_offset, + microsecond: {microsecond, _} + } = datetime + ) do + {days, day_fraction} = + datetime + |> to_iso_days() + |> apply_tz_offset(utc_offset + std_offset) + + seconds_in_day = seconds_from_day_fraction(day_fraction) + {days * @seconds_per_day + seconds_in_day, microsecond} + end + + @doc """ + Converts the given `datetime` to a string according to its calendar. + + ### Examples + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.to_string(dt) + "2000-02-29 23:00:07+01:00 CET Europe/Warsaw" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "UTC", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 0, std_offset: 0, time_zone: "Etc/UTC"} + iex> DateTime.to_string(dt) + "2000-02-29 23:00:07Z" + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.to_string(dt) + "2000-02-29 23:00:07-04:00 AMT America/Manaus" + + iex> dt = %DateTime{year: -100, month: 12, day: 19, zone_abbr: "CET", + ...> hour: 3, minute: 20, second: 31, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Stockholm"} + iex> DateTime.to_string(dt) + "-0100-12-19 03:20:31+01:00 CET Europe/Stockholm" + + """ + @spec to_string(Calendar.datetime()) :: String.t() + def to_string(%{calendar: calendar} = datetime) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: time_zone, + zone_abbr: zone_abbr, + utc_offset: utc_offset, + std_offset: std_offset + } = datetime + + calendar.datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + zone_abbr, + utc_offset, + std_offset + ) + end + + @doc """ + Compares two datetime structs. + + Returns `:gt` if the first datetime is later than the second + and `:lt` for vice versa. If the two datetimes are equal + `:eq` is returned. + + Note that both UTC and Standard offsets will be taken into + account when comparison is done. + + ## Examples + + iex> dt1 = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> dt2 = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.compare(dt1, dt2) + :gt + + """ + @doc since: "1.4.0" + @spec compare(Calendar.datetime(), Calendar.datetime()) :: :lt | :eq | :gt + def compare( + %{utc_offset: utc_offset1, std_offset: std_offset1} = datetime1, + %{utc_offset: utc_offset2, std_offset: std_offset2} = datetime2 + ) do + {days1, {parts1, ppd1}} = + datetime1 + |> to_iso_days() + |> apply_tz_offset(utc_offset1 + std_offset1) + + {days2, {parts2, ppd2}} = + datetime2 + |> to_iso_days() + |> apply_tz_offset(utc_offset2 + std_offset2) + + # Ensure fraction tuples have same denominator. + first = {days1, parts1 * ppd2} + second = {days2, parts2 * ppd1} + + cond do + first > second -> :gt + first < second -> :lt + true -> :eq + end + end + + @doc """ + Subtracts `datetime2` from `datetime1`. + + The answer can be returned in any `unit` available from `t:System.time_unit/0`. + + Leap seconds are not taken into account. + + This function returns the difference in seconds where seconds are measured + according to `Calendar.ISO`. + + ## Examples + + iex> dt1 = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> dt2 = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> DateTime.diff(dt1, dt2) + 18000 + iex> DateTime.diff(dt2, dt1) + -18000 + + """ + @doc since: "1.5.0" + @spec diff(Calendar.datetime(), Calendar.datetime(), System.time_unit()) :: integer() + def diff( + %{utc_offset: utc_offset1, std_offset: std_offset1} = datetime1, + %{utc_offset: utc_offset2, std_offset: std_offset2} = datetime2, + unit \\ :second + ) do + naive_diff = + (datetime1 |> to_iso_days() |> Calendar.ISO.iso_days_to_unit(unit)) - + (datetime2 |> to_iso_days() |> Calendar.ISO.iso_days_to_unit(unit)) + + offset_diff = utc_offset2 + std_offset2 - (utc_offset1 + std_offset1) + naive_diff + System.convert_time_unit(offset_diff, :second, unit) + end + + @doc """ + Adds a specified amount of time to a `DateTime`. + + Accepts an `amount_to_add` in any `unit` available from `t:System.time_unit/0`. + Negative values will move backwards in time. + + Takes changes such as summer time/DST into account. This means that adding time + can cause the wall time to "go backwards" during "fall back" during autumn. + Adding just a few seconds to a datetime just before "spring forward" can cause wall + time to increase by more than an hour. + + Fractional second precision stays the same in a similar way to `NaiveDateTime.add/2`. + + ### Examples + + iex> dt = DateTime.from_naive!(~N[2018-11-15 10:00:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> dt |> DateTime.add(3600, :second, FakeTimeZoneDatabase) + #DateTime<2018-11-15 11:00:00+01:00 CET Europe/Copenhagen> + + iex> DateTime.add(~U[2018-11-15 10:00:00Z], 3600, :second) + ~U[2018-11-15 11:00:00Z] + + When adding 3 seconds just before "spring forward" we go from 1:59:59 to 3:00:02 + + iex> dt = DateTime.from_naive!(~N[2019-03-31 01:59:59.123], "Europe/Copenhagen", FakeTimeZoneDatabase) + iex> dt |> DateTime.add(3, :second, FakeTimeZoneDatabase) + #DateTime<2019-03-31 03:00:02.123+02:00 CEST Europe/Copenhagen> + + """ + @doc since: "1.8.0" + @spec add(Calendar.datetime(), integer, System.time_unit(), Calendar.time_zone_database()) :: + t() + def add( + datetime, + amount_to_add, + unit \\ :second, + time_zone_database \\ Calendar.get_time_zone_database() + ) + when is_integer(amount_to_add) do + %{ + utc_offset: utc_offset, + std_offset: std_offset, + calendar: calendar, + microsecond: {_, precision} + } = datetime + + ppd = System.convert_time_unit(86400, :second, unit) + total_offset = System.convert_time_unit(utc_offset + std_offset, :second, unit) + + result = + datetime + |> to_iso_days() + # Subtract total offset in order to get UTC and add the integer for the addition + |> Calendar.ISO.add_day_fraction_to_iso_days(amount_to_add - total_offset, ppd) + |> shift_zone_for_iso_days_utc(calendar, precision, datetime.time_zone, time_zone_database) + + case result do + {:ok, result_datetime} -> + result_datetime + + {:error, error} -> + raise ArgumentError, + "cannot add #{amount_to_add} #{unit} to #{inspect(datetime)} (with time zone " <> + "database #{inspect(time_zone_database)}), reason: #{inspect(error)}" + end + end + + @doc """ + Returns the given datetime with the microsecond field truncated to the given + precision (`:microsecond`, `:millisecond` or `:second`). + + The given datetime is returned unchanged if it already has lower precision than + the given precision. + + ## Examples + + iex> dt1 = %DateTime{year: 2017, month: 11, day: 7, zone_abbr: "CET", + ...> hour: 11, minute: 45, second: 18, microsecond: {123456, 6}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Paris"} + iex> DateTime.truncate(dt1, :microsecond) + #DateTime<2017-11-07 11:45:18.123456+01:00 CET Europe/Paris> + + iex> dt2 = %DateTime{year: 2017, month: 11, day: 7, zone_abbr: "CET", + ...> hour: 11, minute: 45, second: 18, microsecond: {123456, 6}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Paris"} + iex> DateTime.truncate(dt2, :millisecond) + #DateTime<2017-11-07 11:45:18.123+01:00 CET Europe/Paris> + + iex> dt3 = %DateTime{year: 2017, month: 11, day: 7, zone_abbr: "CET", + ...> hour: 11, minute: 45, second: 18, microsecond: {123456, 6}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Paris"} + iex> DateTime.truncate(dt3, :second) + #DateTime<2017-11-07 11:45:18+01:00 CET Europe/Paris> + + """ + @doc since: "1.6.0" + @spec truncate(Calendar.datetime(), :microsecond | :millisecond | :second) :: t() + def truncate(%DateTime{microsecond: microsecond} = datetime, precision) do + %{datetime | microsecond: Calendar.truncate(microsecond, precision)} + end + + def truncate(%{} = datetime_map, precision) do + truncate(from_map(datetime_map), precision) + end + + @doc """ + Converts a given `datetime` from one calendar to another. + + If it is not possible to convert unambiguously between the calendars + (see `Calendar.compatible_calendars?/2`), an `{:error, :incompatible_calendars}` tuple + is returned. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10,000 years to the current Gregorian + year: + + iex> dt1 = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.convert(dt1, Calendar.Holocene) + {:ok, %DateTime{calendar: Calendar.Holocene, day: 29, hour: 23, + microsecond: {0, 0}, minute: 0, month: 2, second: 7, std_offset: 0, + time_zone: "America/Manaus", utc_offset: -14400, year: 12000, + zone_abbr: "AMT"}} + + """ + @doc since: "1.5.0" + @spec convert(Calendar.datetime(), Calendar.calendar()) :: + {:ok, t} | {:error, :incompatible_calendars} + + def convert(%DateTime{calendar: calendar} = datetime, calendar) do + {:ok, datetime} + end + + def convert(%{calendar: calendar} = datetime, calendar) do + {:ok, from_map(datetime)} + end + + def convert(%{calendar: dt_calendar, microsecond: {_, precision}} = datetime, calendar) do + if Calendar.compatible_calendars?(dt_calendar, calendar) do + result_datetime = + datetime + |> to_iso_days + |> from_iso_days(datetime, calendar, precision) + + {:ok, result_datetime} + else + {:error, :incompatible_calendars} + end + end + + @doc """ + Converts a given `datetime` from one calendar to another. + + If it is not possible to convert unambiguously between the calendars + (see `Calendar.compatible_calendars?/2`), an ArgumentError is raised. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10,000 years to the current Gregorian + year: + + iex> dt1 = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "AMT", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: -14400, std_offset: 0, time_zone: "America/Manaus"} + iex> DateTime.convert!(dt1, Calendar.Holocene) + %DateTime{calendar: Calendar.Holocene, day: 29, hour: 23, + microsecond: {0, 0}, minute: 0, month: 2, second: 7, std_offset: 0, + time_zone: "America/Manaus", utc_offset: -14400, year: 12000, + zone_abbr: "AMT"} + + """ + @doc since: "1.5.0" + @spec convert!(Calendar.datetime(), Calendar.calendar()) :: t + def convert!(datetime, calendar) do + case convert(datetime, calendar) do + {:ok, value} -> + value + + {:error, :incompatible_calendars} -> + raise ArgumentError, + "cannot convert #{inspect(datetime)} to target calendar #{inspect(calendar)}, " <> + "reason: #{inspect(datetime.calendar)} and #{inspect(calendar)} have different " <> + "day rollover moments, making this conversion ambiguous" + end + end + + # Keep it multiline for proper function clause errors. + defp to_iso_days(%{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }) do + calendar.naive_datetime_to_iso_days(year, month, day, hour, minute, second, microsecond) + end + + defp from_iso_days(iso_days, datetime, calendar, precision) do + %{time_zone: time_zone, zone_abbr: zone_abbr, utc_offset: utc_offset, std_offset: std_offset} = + datetime + + {year, month, day, hour, minute, second, {microsecond, _}} = + calendar.naive_datetime_from_iso_days(iso_days) + + %DateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision}, + time_zone: time_zone, + zone_abbr: zone_abbr, + utc_offset: utc_offset, + std_offset: std_offset + } + end + + defp apply_tz_offset(iso_days, 0) do + iso_days + end + + defp apply_tz_offset(iso_days, offset) do + Calendar.ISO.add_day_fraction_to_iso_days(iso_days, -offset, 86400) + end + + defp from_map(%{} = datetime_map) do + %DateTime{ + year: datetime_map.year, + month: datetime_map.month, + day: datetime_map.day, + hour: datetime_map.hour, + minute: datetime_map.minute, + second: datetime_map.second, + microsecond: datetime_map.microsecond, + time_zone: datetime_map.time_zone, + zone_abbr: datetime_map.zone_abbr, + utc_offset: datetime_map.utc_offset, + std_offset: datetime_map.std_offset + } + end + + defp seconds_from_day_fraction({parts_in_day, @seconds_per_day}), + do: parts_in_day + + defp seconds_from_day_fraction({parts_in_day, parts_per_day}), + do: div(parts_in_day * @seconds_per_day, parts_per_day) + + defimpl String.Chars do + def to_string(datetime) do + %{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: time_zone, + zone_abbr: zone_abbr, + utc_offset: utc_offset, + std_offset: std_offset + } = datetime + + calendar.datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + zone_abbr, + utc_offset, + std_offset + ) + end + end + + defimpl Inspect do + def inspect(datetime, _) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: time_zone, + zone_abbr: zone_abbr, + utc_offset: utc_offset, + std_offset: std_offset, + calendar: calendar + } = datetime + + formatted = + calendar.datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + zone_abbr, + utc_offset, + std_offset + ) + + case datetime do + %{utc_offset: 0, std_offset: 0, time_zone: "Etc/UTC"} -> + "~U[" <> formatted <> suffix(calendar) <> "]" + + _ -> + "#DateTime<" <> formatted <> suffix(calendar) <> ">" + end + end + + defp suffix(Calendar.ISO), do: "" + defp suffix(calendar), do: " " <> inspect(calendar) + end +end diff --git a/lib/elixir/lib/calendar/iso.ex b/lib/elixir/lib/calendar/iso.ex new file mode 100644 index 00000000000..98d78bc2a31 --- /dev/null +++ b/lib/elixir/lib/calendar/iso.ex @@ -0,0 +1,1657 @@ +defmodule Calendar.ISO do + @moduledoc """ + The default calendar implementation, a Gregorian calendar following ISO 8601. + + This calendar implements a proleptic Gregorian calendar and + is therefore compatible with the calendar used in most countries + today. The proleptic means the Gregorian rules for leap years are + applied for all time, consequently the dates give different results + before the year 1583 from when the Gregorian calendar was adopted. + + ## ISO 8601 compliance + + The ISO 8601 specification is feature-rich, but allows applications + to selectively implement most parts of it. The choices Elixir makes + are catalogued below. + + ### Features + + The standard library supports a minimal set of possible ISO 8601 features. + Specifically, the parser only supports calendar dates and does not support + ordinal and week formats. + + By default Elixir only parses extended-formatted date/times. You can opt-in + to parse basic-formatted date/times. + + `NaiveDateTime.to_iso8601/2` and `DateTime.to_iso8601/2` allow you to produce + either basic or extended formatted strings, and `Calendar.strftime/2` allows + you to format datetimes however else you desire. + + Elixir does not support reduced accuracy formats (for example, a date without + the day component) nor decimal precisions in the lowest component (such as + `10:01:25,5`). No functions exist to parse ISO 8601 durations or time intervals. + + #### Examples + + Elixir expects the extended format by default when parsing: + + iex> Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_naive_datetime("20150123T235007") + {:error, :invalid_format} + + Parsing can be restricted to basic if desired: + + iex> Calendar.ISO.parse_naive_datetime("20150123T235007Z", :basic) + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_naive_datetime("20150123T235007Z", :extended) + {:error, :invalid_format} + + Only calendar dates are supported in parsing; ordinal and week dates are not. + + iex> Calendar.ISO.parse_date("2015-04-15") + {:ok, {2015, 4, 15}} + iex> Calendar.ISO.parse_date("2015-105") + {:error, :invalid_format} + iex> Calendar.ISO.parse_date("2015-W16") + {:error, :invalid_format} + iex> Calendar.ISO.parse_date("2015-W016-3") + {:error, :invalid_format} + + Years, months, days, hours, minutes, and seconds must be fully specified: + + iex> Calendar.ISO.parse_date("2015-04-15") + {:ok, {2015, 4, 15}} + iex> Calendar.ISO.parse_date("2015-04") + {:error, :invalid_format} + iex> Calendar.ISO.parse_date("2015") + {:error, :invalid_format} + + iex> Calendar.ISO.parse_time("23:50:07.0123456") + {:ok, {23, 50, 7, {12345, 6}}} + iex> Calendar.ISO.parse_time("23:50:07") + {:ok, {23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_time("23:50") + {:error, :invalid_format} + iex> Calendar.ISO.parse_time("23") + {:error, :invalid_format} + + ### Extensions + + The parser and formatter adopt one ISO 8601 extension: extended year notation. + + This allows dates to be prefixed with a `+` or `-` sign, extending the range of + expressible years from the default (`0000..9999`) to `-9999..9999`. Elixir still + restricts years in this format to four digits. + + #### Examples + + iex> Calendar.ISO.parse_date("-2015-01-23") + {:ok, {-2015, 1, 23}} + iex> Calendar.ISO.parse_date("+2015-01-23") + {:ok, {2015, 1, 23}} + + iex> Calendar.ISO.parse_naive_datetime("-2015-01-23 23:50:07") + {:ok, {-2015, 1, 23, 23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_naive_datetime("+2015-01-23 23:50:07") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + + iex> Calendar.ISO.parse_utc_datetime("-2015-01-23 23:50:07Z") + {:ok, {-2015, 1, 23, 23, 50, 7, {0, 0}}, 0} + iex> Calendar.ISO.parse_utc_datetime("+2015-01-23 23:50:07Z") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}, 0} + + ### Additions + + ISO 8601 does not allow a whitespace instead of `T` as a separator + between date and times, both when parsing and formatting. + This is a common enough representation, Elixir allows it during parsing. + + The formatting of dates in `NaiveDateTime.to_iso8601/1` and `DateTime.to_iso8601/1` + do produce specification-compliant string representations using the `T` separator. + + #### Examples + + iex> Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.0123456") + {:ok, {2015, 1, 23, 23, 50, 7, {12345, 6}}} + iex> Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.0123456") + {:ok, {2015, 1, 23, 23, 50, 7, {12345, 6}}} + + iex> Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07.0123456Z") + {:ok, {2015, 1, 23, 23, 50, 7, {12345, 6}}, 0} + iex> Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.0123456Z") + {:ok, {2015, 1, 23, 23, 50, 7, {12345, 6}}, 0} + + """ + + @behaviour Calendar + + @unix_epoch 62_167_219_200 + unix_start = (315_537_897_600 + @unix_epoch) * -1_000_000 + unix_end = 315_569_519_999_999_999 - @unix_epoch * 1_000_000 + @unix_range_microseconds unix_start..unix_end + + defguardp is_format(term) when term in [:basic, :extended] + + @typedoc """ + "Before the Current Era" or "Before the Common Era" (BCE), for those years less than `1`. + """ + @type bce :: 0 + + @typedoc """ + The "Current Era" or the "Common Era" (CE) which starts in year `1`. + """ + @type ce :: 1 + + @typedoc """ + The calendar era. + + The ISO calendar has two eras: + * [CE](`t:ce/0`) - which starts in year `1` and is defined as era `1`. + * [BCE](`t:bce/0`) - for those years less than `1` and is defined as era `0`. + """ + @type era :: bce | ce + @type year :: -9999..9999 + @type month :: 1..12 + @type day :: 1..31 + @type hour :: 0..23 + @type minute :: 0..59 + @type second :: 0..59 + @type weekday :: :monday | :tuesday | :wednesday | :thursday | :friday | :saturday | :sunday + @type utc_offset :: integer + @type format :: :basic | :extended + + @typedoc """ + Microseconds with stored precision. + + The precision represents the number of digits that must be used when + representing the microseconds to external format. If the precision is 0, + it means microseconds must be skipped. + """ + @type microsecond :: {0..999_999, 0..6} + + @typedoc """ + Integer that represents the day of the week, where 1 is Monday and 7 is Sunday. + """ + @type day_of_week :: 1..7 + + @type day_of_year :: 1..366 + @type quarter_of_year :: 1..4 + @type year_of_era :: {1..10000, era} + + @seconds_per_minute 60 + @seconds_per_hour 60 * 60 + # Note that this does *not* handle leap seconds. + @seconds_per_day 24 * 60 * 60 + @last_second_of_the_day @seconds_per_day - 1 + @microseconds_per_second 1_000_000 + @parts_per_day @seconds_per_day * @microseconds_per_second + + @datetime_seps [?\s, ?T] + @ext_date_sep ?- + @ext_time_sep ?: + + @days_per_nonleap_year 365 + @days_per_leap_year 366 + + # The ISO epoch starts, in this implementation, + # with ~D[0000-01-01]. Era "1" starts + # on ~D[0001-01-01] which is 366 days later. + @iso_epoch 366 + + [match_basic_date, match_ext_date, guard_date, read_date] = + quote do + [ + <>, + <>, + y1 >= ?0 and y1 <= ?9 and y2 >= ?0 and y2 <= ?9 and y3 >= ?0 and y3 <= ?9 and y4 >= ?0 and + y4 <= ?9 and m1 >= ?0 and m1 <= ?9 and m2 >= ?0 and m2 <= ?9 and d1 >= ?0 and d1 <= ?9 and + d2 >= ?0 and d2 <= ?9, + { + (y1 - ?0) * 1000 + (y2 - ?0) * 100 + (y3 - ?0) * 10 + (y4 - ?0), + (m1 - ?0) * 10 + (m2 - ?0), + (d1 - ?0) * 10 + (d2 - ?0) + } + ] + end + + [match_basic_time, match_ext_time, guard_time, read_time] = + quote do + [ + <>, + <>, + h1 >= ?0 and h1 <= ?9 and h2 >= ?0 and h2 <= ?9 and i1 >= ?0 and i1 <= ?9 and i2 >= ?0 and + i2 <= ?9 and s1 >= ?0 and s1 <= ?9 and s2 >= ?0 and s2 <= ?9, + { + (h1 - ?0) * 10 + (h2 - ?0), + (i1 - ?0) * 10 + (i2 - ?0), + (s1 - ?0) * 10 + (s2 - ?0) + } + ] + end + + defguardp is_year(year) when year in -9999..9999 + defguardp is_year_BCE(year) when year in -9999..0 + defguardp is_year_CE(year) when year in 1..9999 + defguardp is_month(month) when month in 1..12 + defguardp is_day(day) when day in 1..31 + defguardp is_hour(hour) when hour in 0..23 + defguardp is_minute(minute) when minute in 0..59 + defguardp is_second(second) when second in 0..59 + + defguardp is_microsecond(microsecond, precision) + when microsecond in 0..999_999 and precision in 0..6 + + defguardp is_time_zone(term) when is_binary(term) + defguardp is_zone_abbr(term) when is_binary(term) + defguardp is_utc_offset(offset) when is_integer(offset) + defguardp is_std_offset(offset) when is_integer(offset) + + @doc """ + Parses a time `string` in the `:extended` format. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_time("23:50:07") + {:ok, {23, 50, 7, {0, 0}}} + + iex> Calendar.ISO.parse_time("23:50:07Z") + {:ok, {23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_time("T23:50:07Z") + {:ok, {23, 50, 7, {0, 0}}} + + """ + @doc since: "1.10.0" + @impl true + @spec parse_time(String.t()) :: + {:ok, {hour, minute, second, microsecond}} + | {:error, atom} + def parse_time(string) when is_binary(string), + do: parse_time(string, :extended) + + @doc """ + Parses a time `string` according to a given `format`. + + The `format` can either be `:basic` or `:extended`. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_time("235007", :basic) + {:ok, {23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_time("235007", :extended) + {:error, :invalid_format} + + """ + @doc since: "1.12.0" + @spec parse_time(String.t(), format) :: + {:ok, {hour, minute, second, microsecond}} + | {:error, atom} + def parse_time(string, format) when is_binary(string) and is_format(format) do + case string do + "T" <> rest -> do_parse_time(rest, format) + _ -> do_parse_time(string, format) + end + end + + defp do_parse_time(<>, :basic) + when unquote(guard_time) do + {hour, minute, second} = unquote(read_time) + parse_formatted_time(hour, minute, second, rest) + end + + defp do_parse_time(<>, :extended) + when unquote(guard_time) do + {hour, minute, second} = unquote(read_time) + parse_formatted_time(hour, minute, second, rest) + end + + defp do_parse_time(_, _) do + {:error, :invalid_format} + end + + defp parse_formatted_time(hour, minute, second, rest) do + with {microsecond, rest} <- parse_microsecond(rest), + {_offset, ""} <- parse_offset(rest) do + if valid_time?(hour, minute, second, microsecond) do + {:ok, {hour, minute, second, microsecond}} + else + {:error, :invalid_time} + end + else + _ -> {:error, :invalid_format} + end + end + + @doc """ + Parses a date `string` in the `:extended` format. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_date("2015-01-23") + {:ok, {2015, 1, 23}} + + iex> Calendar.ISO.parse_date("2015:01:23") + {:error, :invalid_format} + iex> Calendar.ISO.parse_date("2015-01-32") + {:error, :invalid_date} + + """ + @doc since: "1.10.0" + @impl true + @spec parse_date(String.t()) :: + {:ok, {year, month, day}} + | {:error, atom} + def parse_date(string) when is_binary(string), + do: parse_date(string, :extended) + + @doc """ + Parses a date `string` according to a given `format`. + + The `format` can either be `:basic` or `:extended`. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_date("20150123", :basic) + {:ok, {2015, 1, 23}} + iex> Calendar.ISO.parse_date("20150123", :extended) + {:error, :invalid_format} + + """ + @doc since: "1.12.0" + @spec parse_date(String.t(), format) :: + {:ok, {year, month, day}} + | {:error, atom} + def parse_date(string, format) when is_binary(string) and is_format(format), + do: parse_date_guarded(string, format) + + defp parse_date_guarded("-" <> string, format), + do: do_parse_date(string, -1, format) + + defp parse_date_guarded("+" <> string, format), + do: do_parse_date(string, 1, format) + + defp parse_date_guarded(string, format), + do: do_parse_date(string, 1, format) + + defp do_parse_date(unquote(match_basic_date), multiplier, :basic) when unquote(guard_date) do + {year, month, day} = unquote(read_date) + parse_formatted_date(year, month, day, multiplier) + end + + defp do_parse_date(unquote(match_ext_date), multiplier, :extended) when unquote(guard_date) do + {year, month, day} = unquote(read_date) + parse_formatted_date(year, month, day, multiplier) + end + + defp do_parse_date(_, _, _) do + {:error, :invalid_format} + end + + defp parse_formatted_date(year, month, day, multiplier) do + year = multiplier * year + + if valid_date?(year, month, day) do + {:ok, {year, month, day}} + else + {:error, :invalid_date} + end + end + + @doc """ + Parses a naive datetime `string` in the `:extended` format. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07Z") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07-02:30") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + + iex> Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.0") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 1}}} + iex> Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07,0123456") + {:ok, {2015, 1, 23, 23, 50, 7, {12345, 6}}} + + """ + @doc since: "1.10.0" + @impl true + @spec parse_naive_datetime(String.t()) :: + {:ok, {year, month, day, hour, minute, second, microsecond}} + | {:error, atom} + def parse_naive_datetime(string) when is_binary(string), + do: parse_naive_datetime(string, :extended) + + @doc """ + Parses a naive datetime `string` according to a given `format`. + + The `format` can either be `:basic` or `:extended`. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_naive_datetime("20150123 235007", :basic) + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + iex> Calendar.ISO.parse_naive_datetime("20150123 235007", :extended) + {:error, :invalid_format} + + """ + @doc since: "1.12.0" + @spec parse_naive_datetime(String.t(), format) :: + {:ok, {year, month, day, hour, minute, second, microsecond}} + | {:error, atom} + def parse_naive_datetime(string, format) when is_binary(string) and is_format(format), + do: parse_naive_datetime_guarded(string, format) + + defp parse_naive_datetime_guarded("-" <> string, format), + do: do_parse_naive_datetime(string, -1, format) + + defp parse_naive_datetime_guarded("+" <> string, format), + do: do_parse_naive_datetime(string, 1, format) + + defp parse_naive_datetime_guarded(string, format), + do: do_parse_naive_datetime(string, 1, format) + + defp do_parse_naive_datetime( + <>, + multiplier, + :basic + ) + when unquote(guard_date) and datetime_sep in @datetime_seps and unquote(guard_time) do + {year, month, day} = unquote(read_date) + {hour, minute, second} = unquote(read_time) + parse_formatted_naive_datetime(year, month, day, hour, minute, second, rest, multiplier) + end + + defp do_parse_naive_datetime( + <>, + multiplier, + :extended + ) + when unquote(guard_date) and datetime_sep in @datetime_seps and unquote(guard_time) do + {year, month, day} = unquote(read_date) + {hour, minute, second} = unquote(read_time) + parse_formatted_naive_datetime(year, month, day, hour, minute, second, rest, multiplier) + end + + defp do_parse_naive_datetime(_, _, _) do + {:error, :invalid_format} + end + + defp parse_formatted_naive_datetime(year, month, day, hour, minute, second, rest, multiplier) do + year = multiplier * year + + with {microsecond, rest} <- parse_microsecond(rest), + {_offset, ""} <- parse_offset(rest) do + cond do + not valid_date?(year, month, day) -> + {:error, :invalid_date} + + not valid_time?(hour, minute, second, microsecond) -> + {:error, :invalid_time} + + true -> + {:ok, {year, month, day, hour, minute, second, microsecond}} + end + else + _ -> {:error, :invalid_format} + end + end + + @doc """ + Parses a UTC datetime `string` in the `:extended` format. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07Z") + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}, 0} + + iex> Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07+02:30") + {:ok, {2015, 1, 23, 21, 20, 7, {0, 0}}, 9000} + + iex> Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07") + {:error, :missing_offset} + + """ + @doc since: "1.10.0" + @impl true + @spec parse_utc_datetime(String.t()) :: + {:ok, {year, month, day, hour, minute, second, microsecond}, utc_offset} + | {:error, atom} + def parse_utc_datetime(string) when is_binary(string), + do: parse_utc_datetime(string, :extended) + + @doc """ + Parses a UTC datetime `string` according to a given `format`. + + The `format` can either be `:basic` or `:extended`. + + For more information on supported strings, see how this + module implements [ISO 8601](#module-iso-8601-compliance). + + ## Examples + + iex> Calendar.ISO.parse_utc_datetime("20150123 235007Z", :basic) + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}, 0} + iex> Calendar.ISO.parse_utc_datetime("20150123 235007Z", :extended) + {:error, :invalid_format} + + """ + @doc since: "1.12.0" + @spec parse_utc_datetime(String.t(), format) :: + {:ok, {year, month, day, hour, minute, second, microsecond}, utc_offset} + | {:error, atom} + def parse_utc_datetime(string, format) when is_binary(string) and is_format(format), + do: parse_utc_datetime_guarded(string, format) + + defp parse_utc_datetime_guarded("-" <> string, format), + do: do_parse_utc_datetime(string, -1, format) + + defp parse_utc_datetime_guarded("+" <> string, format), + do: do_parse_utc_datetime(string, 1, format) + + defp parse_utc_datetime_guarded(string, format), + do: do_parse_utc_datetime(string, 1, format) + + defp do_parse_utc_datetime( + <>, + multiplier, + :basic + ) + when unquote(guard_date) and datetime_sep in @datetime_seps and unquote(guard_time) do + {year, month, day} = unquote(read_date) + {hour, minute, second} = unquote(read_time) + parse_formatted_utc_datetime(year, month, day, hour, minute, second, rest, multiplier) + end + + defp do_parse_utc_datetime( + <>, + multiplier, + :extended + ) + when unquote(guard_date) and datetime_sep in @datetime_seps and unquote(guard_time) do + {year, month, day} = unquote(read_date) + {hour, minute, second} = unquote(read_time) + parse_formatted_utc_datetime(year, month, day, hour, minute, second, rest, multiplier) + end + + defp do_parse_utc_datetime(_, _, _) do + {:error, :invalid_format} + end + + defp parse_formatted_utc_datetime(year, month, day, hour, minute, second, rest, multiplier) do + year = multiplier * year + + with {microsecond, rest} <- parse_microsecond(rest), + {offset, ""} <- parse_offset(rest) do + cond do + not valid_date?(year, month, day) -> + {:error, :invalid_date} + + not valid_time?(hour, minute, second, microsecond) -> + {:error, :invalid_time} + + offset == 0 -> + {:ok, {year, month, day, hour, minute, second, microsecond}, offset} + + is_nil(offset) -> + {:error, :missing_offset} + + true -> + day_fraction = time_to_day_fraction(hour, minute, second, {0, 0}) + + {{year, month, day}, {hour, minute, second, _}} = + case add_day_fraction_to_iso_days({0, day_fraction}, -offset, 86400) do + {0, day_fraction} -> + {{year, month, day}, time_from_day_fraction(day_fraction)} + + {extra_days, day_fraction} -> + base_days = date_to_iso_days(year, month, day) + {date_from_iso_days(base_days + extra_days), time_from_day_fraction(day_fraction)} + end + + {:ok, {year, month, day, hour, minute, second, microsecond}, offset} + end + else + _ -> {:error, :invalid_format} + end + end + + @doc """ + Returns the `t:Calendar.iso_days/0` format of the specified date. + + ## Examples + + iex> Calendar.ISO.naive_datetime_to_iso_days(0, 1, 1, 0, 0, 0, {0, 6}) + {0, {0, 86400000000}} + iex> Calendar.ISO.naive_datetime_to_iso_days(2000, 1, 1, 12, 0, 0, {0, 6}) + {730485, {43200000000, 86400000000}} + iex> Calendar.ISO.naive_datetime_to_iso_days(2000, 1, 1, 13, 0, 0, {0, 6}) + {730485, {46800000000, 86400000000}} + iex> Calendar.ISO.naive_datetime_to_iso_days(-1, 1, 1, 0, 0, 0, {0, 6}) + {-365, {0, 86400000000}} + + """ + @doc since: "1.5.0" + @impl true + @spec naive_datetime_to_iso_days( + Calendar.year(), + Calendar.month(), + Calendar.day(), + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() + ) :: Calendar.iso_days() + def naive_datetime_to_iso_days(year, month, day, hour, minute, second, microsecond) do + {date_to_iso_days(year, month, day), time_to_day_fraction(hour, minute, second, microsecond)} + end + + @doc """ + Converts the `t:Calendar.iso_days/0` format to the datetime format specified by this calendar. + + ## Examples + + iex> Calendar.ISO.naive_datetime_from_iso_days({0, {0, 86400}}) + {0, 1, 1, 0, 0, 0, {0, 6}} + iex> Calendar.ISO.naive_datetime_from_iso_days({730_485, {0, 86400}}) + {2000, 1, 1, 0, 0, 0, {0, 6}} + iex> Calendar.ISO.naive_datetime_from_iso_days({730_485, {43200, 86400}}) + {2000, 1, 1, 12, 0, 0, {0, 6}} + iex> Calendar.ISO.naive_datetime_from_iso_days({-365, {0, 86400000000}}) + {-1, 1, 1, 0, 0, 0, {0, 6}} + + """ + @doc since: "1.5.0" + @spec naive_datetime_from_iso_days(Calendar.iso_days()) :: { + Calendar.year(), + Calendar.month(), + Calendar.day(), + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() + } + @impl true + def naive_datetime_from_iso_days({days, day_fraction}) do + {year, month, day} = date_from_iso_days(days) + {hour, minute, second, microsecond} = time_from_day_fraction(day_fraction) + {year, month, day, hour, minute, second, microsecond} + end + + @doc """ + Returns the normalized day fraction of the specified time. + + ## Examples + + iex> Calendar.ISO.time_to_day_fraction(0, 0, 0, {0, 6}) + {0, 86400000000} + iex> Calendar.ISO.time_to_day_fraction(12, 34, 56, {123, 6}) + {45296000123, 86400000000} + + """ + @doc since: "1.5.0" + @impl true + @spec time_to_day_fraction( + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() + ) :: Calendar.day_fraction() + def time_to_day_fraction(0, 0, 0, {0, _}) do + {0, @parts_per_day} + end + + def time_to_day_fraction(hour, minute, second, {microsecond, _}) do + combined_seconds = hour * @seconds_per_hour + minute * @seconds_per_minute + second + {combined_seconds * @microseconds_per_second + microsecond, @parts_per_day} + end + + @doc """ + Converts a day fraction to this Calendar's representation of time. + + ## Examples + + iex> Calendar.ISO.time_from_day_fraction({1, 2}) + {12, 0, 0, {0, 6}} + iex> Calendar.ISO.time_from_day_fraction({13, 24}) + {13, 0, 0, {0, 6}} + + """ + @doc since: "1.5.0" + @impl true + @spec time_from_day_fraction(Calendar.day_fraction()) :: + {hour(), minute(), second(), microsecond()} + def time_from_day_fraction({0, _}) do + {0, 0, 0, {0, 6}} + end + + def time_from_day_fraction({parts_in_day, parts_per_day}) do + total_microseconds = divide_by_parts_per_day(parts_in_day, parts_per_day) + + {hours, rest_microseconds1} = + div_rem(total_microseconds, @seconds_per_hour * @microseconds_per_second) + + {minutes, rest_microseconds2} = + div_rem(rest_microseconds1, @seconds_per_minute * @microseconds_per_second) + + {seconds, microseconds} = div_rem(rest_microseconds2, @microseconds_per_second) + {hours, minutes, seconds, {microseconds, 6}} + end + + defp divide_by_parts_per_day(parts_in_day, @parts_per_day), do: parts_in_day + + defp divide_by_parts_per_day(parts_in_day, parts_per_day), + do: div(parts_in_day * @parts_per_day, parts_per_day) + + # Converts year, month, day to count of days since 0000-01-01. + @doc false + def date_to_iso_days(0, 1, 1) do + 0 + end + + def date_to_iso_days(1970, 1, 1) do + 719_528 + end + + def date_to_iso_days(year, month, day) do + ensure_day_in_month!(year, month, day) + + days_in_previous_years(year) + days_before_month(month) + leap_day_offset(year, month) + day - + 1 + end + + # Converts count of days since 0000-01-01 to {year, month, day} tuple. + @doc false + def date_from_iso_days(days) when days in -3_652_059..3_652_424 do + {year, day_of_year} = days_to_year(days) + extra_day = if leap_year?(year), do: 1, else: 0 + {month, day_in_month} = year_day_to_year_date(extra_day, day_of_year) + {year, month, day_in_month + 1} + end + + defp div_rem(int1, int2) do + div = div(int1, int2) + rem = int1 - div * int2 + + if rem >= 0 do + {div, rem} + else + {div - 1, rem + int2} + end + end + + @doc """ + Returns how many days there are in the given year-month. + + ## Examples + + iex> Calendar.ISO.days_in_month(1900, 1) + 31 + iex> Calendar.ISO.days_in_month(1900, 2) + 28 + iex> Calendar.ISO.days_in_month(2000, 2) + 29 + iex> Calendar.ISO.days_in_month(2001, 2) + 28 + iex> Calendar.ISO.days_in_month(2004, 2) + 29 + iex> Calendar.ISO.days_in_month(2004, 4) + 30 + iex> Calendar.ISO.days_in_month(-1, 5) + 31 + + """ + @doc since: "1.4.0" + @spec days_in_month(year, month) :: 28..31 + @impl true + def days_in_month(year, month) when is_year(year) and is_month(month) do + days_in_month_guarded(year, month) + end + + defp days_in_month_guarded(year, 2) do + if leap_year?(year), do: 29, else: 28 + end + + defp days_in_month_guarded(_, month) when month in [4, 6, 9, 11], do: 30 + defp days_in_month_guarded(_, _), do: 31 + + @doc """ + Returns how many months there are in the given year. + + ## Example + + iex> Calendar.ISO.months_in_year(2004) + 12 + + """ + @doc since: "1.7.0" + @impl true + @spec months_in_year(year) :: 12 + def months_in_year(year) when is_year(year) do + 12 + end + + @doc """ + Returns if the given year is a leap year. + + ## Examples + + iex> Calendar.ISO.leap_year?(2000) + true + iex> Calendar.ISO.leap_year?(2001) + false + iex> Calendar.ISO.leap_year?(2004) + true + iex> Calendar.ISO.leap_year?(1900) + false + iex> Calendar.ISO.leap_year?(-4) + true + + """ + @doc since: "1.3.0" + @spec leap_year?(year) :: boolean() + @impl true + def leap_year?(year) when is_year(year) do + rem(year, 4) === 0 and (rem(year, 100) !== 0 or rem(year, 400) === 0) + end + + # TODO: Deprecate me on v1.15 + @doc false + def day_of_week(year, month, day) do + day_of_week(year, month, day, :default) |> elem(0) + end + + @doc """ + Calculates the day of the week from the given `year`, `month`, and `day`. + + It is an integer from 1 to 7, where 1 is the given `starting_on` weekday. + For example, if `starting_on` is set to `:monday`, then 1 is Monday and + 7 is Sunday. + + `starting_on` can also be `:default`, which is equivalent to `:monday`. + + ## Examples + + iex> Calendar.ISO.day_of_week(2016, 10, 31, :monday) + {1, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 1, :monday) + {2, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 2, :monday) + {3, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 3, :monday) + {4, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 4, :monday) + {5, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 5, :monday) + {6, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 6, :monday) + {7, 1, 7} + iex> Calendar.ISO.day_of_week(-99, 1, 31, :monday) + {4, 1, 7} + + iex> Calendar.ISO.day_of_week(2016, 10, 31, :sunday) + {2, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 1, :sunday) + {3, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 2, :sunday) + {4, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 3, :sunday) + {5, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 4, :sunday) + {6, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 5, :sunday) + {7, 1, 7} + iex> Calendar.ISO.day_of_week(2016, 11, 6, :sunday) + {1, 1, 7} + iex> Calendar.ISO.day_of_week(-99, 1, 31, :sunday) + {5, 1, 7} + + iex> Calendar.ISO.day_of_week(2016, 10, 31, :saturday) + {3, 1, 7} + + """ + @doc since: "1.11.0" + @spec day_of_week(year, month, day, :default | weekday) :: {day_of_week(), 1, 7} + @impl true + def day_of_week(year, month, day, starting_on) do + iso_days = date_to_iso_days(year, month, day) + {iso_days_to_day_of_week(iso_days, starting_on), 1, 7} + end + + @doc false + def iso_days_to_day_of_week(iso_days, starting_on) do + Integer.mod(iso_days + day_of_week_offset(starting_on), 7) + 1 + end + + defp day_of_week_offset(:default), do: 5 + defp day_of_week_offset(:wednesday), do: 3 + defp day_of_week_offset(:thursday), do: 2 + defp day_of_week_offset(:friday), do: 1 + defp day_of_week_offset(:saturday), do: 0 + defp day_of_week_offset(:sunday), do: 6 + defp day_of_week_offset(:monday), do: 5 + defp day_of_week_offset(:tuesday), do: 4 + + @doc """ + Calculates the day of the year from the given `year`, `month`, and `day`. + + It is an integer from 1 to 366. + + ## Examples + + iex> Calendar.ISO.day_of_year(2016, 1, 31) + 31 + iex> Calendar.ISO.day_of_year(-99, 2, 1) + 32 + iex> Calendar.ISO.day_of_year(2018, 2, 28) + 59 + + """ + @doc since: "1.8.0" + @spec day_of_year(year, month, day) :: day_of_year() + @impl true + def day_of_year(year, month, day) do + ensure_day_in_month!(year, month, day) + days_before_month(month) + leap_day_offset(year, month) + day + end + + @doc """ + Calculates the quarter of the year from the given `year`, `month`, and `day`. + + It is an integer from 1 to 4. + + ## Examples + + iex> Calendar.ISO.quarter_of_year(2016, 1, 31) + 1 + iex> Calendar.ISO.quarter_of_year(2016, 4, 3) + 2 + iex> Calendar.ISO.quarter_of_year(-99, 9, 31) + 3 + iex> Calendar.ISO.quarter_of_year(2018, 12, 28) + 4 + + """ + @doc since: "1.8.0" + @spec quarter_of_year(year, month, day) :: quarter_of_year() + @impl true + def quarter_of_year(year, month, day) + when is_year(year) and is_month(month) and is_day(day) do + div(month - 1, 3) + 1 + end + + @doc """ + Calculates the year and era from the given `year`. + + The ISO calendar has two eras: the "current era" (CE) which + starts in year `1` and is defined as era `1`. And "before the current + era" (BCE) for those years less than `1`, defined as era `0`. + + ## Examples + + iex> Calendar.ISO.year_of_era(1) + {1, 1} + iex> Calendar.ISO.year_of_era(2018) + {2018, 1} + iex> Calendar.ISO.year_of_era(0) + {1, 0} + iex> Calendar.ISO.year_of_era(-1) + {2, 0} + + """ + @doc since: "1.8.0" + @spec year_of_era(year) :: {1..10000, era} + def year_of_era(year) when is_year_CE(year), do: {year, 1} + def year_of_era(year) when is_year_BCE(year), do: {abs(year) + 1, 0} + + @doc """ + Calendar callback to compute the year and era from the + given `year`, `month` and `day`. + + In the ISO calendar, the new year coincides with the new era, + so the `month` and `day` arguments are discarded. If you only + have the year available, you can `year_of_era/1` instead. + + ## Examples + + iex> Calendar.ISO.year_of_era(1, 1, 1) + {1, 1} + iex> Calendar.ISO.year_of_era(2018, 12, 1) + {2018, 1} + iex> Calendar.ISO.year_of_era(0, 1, 1) + {1, 0} + iex> Calendar.ISO.year_of_era(-1, 12, 1) + {2, 0} + + """ + @doc since: "1.13.0" + @impl true + @spec year_of_era(year, month, day) :: {1..10000, era} + def year_of_era(year, _month, _day), do: year_of_era(year) + + @doc """ + Calculates the day and era from the given `year`, `month`, and `day`. + + ## Examples + + iex> Calendar.ISO.day_of_era(0, 1, 1) + {366, 0} + iex> Calendar.ISO.day_of_era(1, 1, 1) + {1, 1} + iex> Calendar.ISO.day_of_era(0, 12, 31) + {1, 0} + iex> Calendar.ISO.day_of_era(0, 12, 30) + {2, 0} + iex> Calendar.ISO.day_of_era(-1, 12, 31) + {367, 0} + + """ + @doc since: "1.8.0" + @spec day_of_era(year, month, day) :: Calendar.day_of_era() + @impl true + def day_of_era(year, month, day) when is_year_CE(year) do + day = date_to_iso_days(year, month, day) - @iso_epoch + 1 + {day, 1} + end + + def day_of_era(year, month, day) when is_year_BCE(year) do + day = abs(date_to_iso_days(year, month, day) - @iso_epoch) + {day, 0} + end + + @doc """ + Converts the given time into a string. + + By default, returns times formatted in the "extended" format, + for human readability. It also supports the "basic" format + by passing the `:basic` option. + + ## Examples + + iex> Calendar.ISO.time_to_string(2, 2, 2, {2, 6}) + "02:02:02.000002" + iex> Calendar.ISO.time_to_string(2, 2, 2, {2, 2}) + "02:02:02.00" + iex> Calendar.ISO.time_to_string(2, 2, 2, {2, 0}) + "02:02:02" + + iex> Calendar.ISO.time_to_string(2, 2, 2, {2, 6}, :basic) + "020202.000002" + iex> Calendar.ISO.time_to_string(2, 2, 2, {2, 6}, :extended) + "02:02:02.000002" + + """ + @impl true + @doc since: "1.5.0" + @spec time_to_string( + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond(), + :basic | :extended + ) :: String.t() + def time_to_string( + hour, + minute, + second, + {ms_value, ms_precision} = microsecond, + format \\ :extended + ) + when is_hour(hour) and is_minute(minute) and is_second(second) and + is_microsecond(ms_value, ms_precision) and format in [:basic, :extended] do + time_to_string_guarded(hour, minute, second, microsecond, format) + end + + defp time_to_string_guarded(hour, minute, second, {_, 0}, format) do + time_to_string_format(hour, minute, second, format) + end + + defp time_to_string_guarded(hour, minute, second, {microsecond, precision}, format) do + time_to_string_format(hour, minute, second, format) <> + "." <> (microsecond |> zero_pad(6) |> binary_part(0, precision)) + end + + defp time_to_string_format(hour, minute, second, :extended) do + zero_pad(hour, 2) <> ":" <> zero_pad(minute, 2) <> ":" <> zero_pad(second, 2) + end + + defp time_to_string_format(hour, minute, second, :basic) do + zero_pad(hour, 2) <> zero_pad(minute, 2) <> zero_pad(second, 2) + end + + @doc """ + Converts the given date into a string. + + By default, returns dates formatted in the "extended" format, + for human readability. It also supports the "basic" format + by passing the `:basic` option. + + ## Examples + + iex> Calendar.ISO.date_to_string(2015, 2, 28) + "2015-02-28" + iex> Calendar.ISO.date_to_string(2017, 8, 1) + "2017-08-01" + iex> Calendar.ISO.date_to_string(-99, 1, 31) + "-0099-01-31" + + iex> Calendar.ISO.date_to_string(2015, 2, 28, :basic) + "20150228" + iex> Calendar.ISO.date_to_string(-99, 1, 31, :basic) + "-00990131" + + """ + @doc since: "1.4.0" + @spec date_to_string(year, month, day, :basic | :extended) :: String.t() + @impl true + def date_to_string(year, month, day, format \\ :extended) + when is_integer(year) and is_integer(month) and is_integer(day) and + format in [:basic, :extended] do + date_to_string_guarded(year, month, day, format) + end + + defp date_to_string_guarded(year, month, day, :extended) do + zero_pad(year, 4) <> "-" <> zero_pad(month, 2) <> "-" <> zero_pad(day, 2) + end + + defp date_to_string_guarded(year, month, day, :basic) do + zero_pad(year, 4) <> zero_pad(month, 2) <> zero_pad(day, 2) + end + + @doc """ + Converts the datetime (without time zone) into a string. + + By default, returns datetimes formatted in the "extended" format, + for human readability. It also supports the "basic" format + by passing the `:basic` option. + + ## Examples + + iex> Calendar.ISO.naive_datetime_to_string(2015, 2, 28, 1, 2, 3, {4, 6}) + "2015-02-28 01:02:03.000004" + iex> Calendar.ISO.naive_datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}) + "2017-08-01 01:02:03.00000" + + iex> Calendar.ISO.naive_datetime_to_string(2015, 2, 28, 1, 2, 3, {4, 6}, :basic) + "20150228 010203.000004" + + """ + @doc since: "1.4.0" + @impl true + @spec naive_datetime_to_string( + year, + month, + day, + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond(), + :basic | :extended + ) :: String.t() + def naive_datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + format \\ :extended + ) do + date_to_string(year, month, day, format) <> + " " <> time_to_string(hour, minute, second, microsecond, format) + end + + @doc """ + Converts the datetime (with time zone) into a string. + + By default, returns datetimes formatted in the "extended" format, + for human readability. It also supports the "basic" format + by passing the `:basic` option. + + ## Examples + + iex> time_zone = "Etc/UTC" + iex> Calendar.ISO.datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "UTC", 0, 0) + "2017-08-01 01:02:03.00000Z" + iex> Calendar.ISO.datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "UTC", 3600, 0) + "2017-08-01 01:02:03.00000+01:00" + iex> Calendar.ISO.datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "UTC", 3600, 3600) + "2017-08-01 01:02:03.00000+02:00" + + iex> time_zone = "Europe/Berlin" + iex> Calendar.ISO.datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "CET", 3600, 0) + "2017-08-01 01:02:03.00000+01:00 CET Europe/Berlin" + iex> Calendar.ISO.datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "CDT", 3600, 3600) + "2017-08-01 01:02:03.00000+02:00 CDT Europe/Berlin" + + iex> time_zone = "America/Los_Angeles" + iex> Calendar.ISO.datetime_to_string(2015, 2, 28, 1, 2, 3, {4, 5}, time_zone, "PST", -28800, 0) + "2015-02-28 01:02:03.00000-08:00 PST America/Los_Angeles" + iex> Calendar.ISO.datetime_to_string(2015, 2, 28, 1, 2, 3, {4, 5}, time_zone, "PDT", -28800, 3600) + "2015-02-28 01:02:03.00000-07:00 PDT America/Los_Angeles" + + iex> time_zone = "Europe/Berlin" + iex> Calendar.ISO.datetime_to_string(2017, 8, 1, 1, 2, 3, {4, 5}, time_zone, "CET", 3600, 0, :basic) + "20170801 010203.00000+0100 CET Europe/Berlin" + + """ + @doc since: "1.4.0" + @impl true + @spec datetime_to_string( + year, + month, + day, + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond(), + Calendar.time_zone(), + Calendar.zone_abbr(), + Calendar.utc_offset(), + Calendar.std_offset(), + :basic | :extended + ) :: String.t() + def datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + zone_abbr, + utc_offset, + std_offset, + format \\ :extended + ) + when is_time_zone(time_zone) and is_zone_abbr(zone_abbr) and is_utc_offset(utc_offset) and + is_std_offset(std_offset) do + date_to_string(year, month, day, format) <> + " " <> + time_to_string(hour, minute, second, microsecond, format) <> + offset_to_string(utc_offset, std_offset, time_zone, format) <> + zone_to_string(utc_offset, std_offset, zone_abbr, time_zone) + end + + @doc false + def offset_to_string(0, 0, "Etc/UTC", _format), do: "Z" + + def offset_to_string(utc, std, _zone, format) do + total = utc + std + second = abs(total) + minute = second |> rem(3600) |> div(60) + hour = div(second, 3600) + format_offset(total, hour, minute, format) + end + + defp format_offset(total, hour, minute, :extended) do + sign(total) <> zero_pad(hour, 2) <> ":" <> zero_pad(minute, 2) + end + + defp format_offset(total, hour, minute, :basic) do + sign(total) <> zero_pad(hour, 2) <> zero_pad(minute, 2) + end + + defp zone_to_string(_, _, _, "Etc/UTC"), do: "" + defp zone_to_string(_, _, abbr, zone), do: " " <> abbr <> " " <> zone + + @doc """ + Determines if the date given is valid according to the proleptic Gregorian calendar. + + ## Examples + + iex> Calendar.ISO.valid_date?(2015, 2, 28) + true + iex> Calendar.ISO.valid_date?(2015, 2, 30) + false + iex> Calendar.ISO.valid_date?(-1, 12, 31) + true + iex> Calendar.ISO.valid_date?(-1, 12, 32) + false + + """ + @doc since: "1.5.0" + @impl true + @spec valid_date?(year, month, day) :: boolean + def valid_date?(year, month, day) + when is_integer(year) and is_integer(month) and is_integer(day) do + is_year(year) and is_month(month) and day in 1..days_in_month(year, month) + end + + @doc """ + Determines if the date given is valid according to the proleptic Gregorian calendar. + + Leap seconds are not supported by the built-in Calendar.ISO. + + ## Examples + + iex> Calendar.ISO.valid_time?(10, 50, 25, {3006, 6}) + true + iex> Calendar.ISO.valid_time?(23, 59, 60, {0, 0}) + false + iex> Calendar.ISO.valid_time?(24, 0, 0, {0, 0}) + false + + """ + @doc since: "1.5.0" + @impl true + @spec valid_time?(Calendar.hour(), Calendar.minute(), Calendar.second(), Calendar.microsecond()) :: + boolean + def valid_time?(hour, minute, second, {ms_value, ms_precision} = _microsecond) + when is_integer(hour) and is_integer(minute) and is_integer(second) and is_integer(ms_value) and + is_integer(ms_value) do + is_hour(hour) and is_minute(minute) and is_second(second) and + is_microsecond(ms_value, ms_precision) + end + + @doc """ + See `c:Calendar.day_rollover_relative_to_midnight_utc/0` for documentation. + """ + @doc since: "1.5.0" + @impl true + @spec day_rollover_relative_to_midnight_utc() :: {0, 1} + def day_rollover_relative_to_midnight_utc() do + {0, 1} + end + + defp sign(total) when total < 0, do: "-" + defp sign(_), do: "+" + + defp zero_pad(val, count) when val >= 0 do + num = Integer.to_string(val) + :binary.copy("0", max(count - byte_size(num), 0)) <> num + end + + defp zero_pad(val, count) do + "-" <> zero_pad(-val, count) + end + + ## Helpers + + @doc false + def from_unix(integer, unit) when is_integer(integer) do + total = System.convert_time_unit(integer, unit, :microsecond) + + if total in @unix_range_microseconds do + microseconds = Integer.mod(total, @microseconds_per_second) + seconds = @unix_epoch + Integer.floor_div(total, @microseconds_per_second) + precision = precision_for_unit(unit) + {date, time} = iso_seconds_to_datetime(seconds) + {:ok, date, time, {microseconds, precision}} + else + {:error, :invalid_unix_time} + end + end + + defp precision_for_unit(unit) do + case System.convert_time_unit(1, :second, unit) do + 1 -> 0 + 10 -> 1 + 100 -> 2 + 1_000 -> 3 + 10_000 -> 4 + 100_000 -> 5 + _ -> 6 + end + end + + defp parse_microsecond("." <> rest) do + case parse_microsecond(rest, 0, "") do + {"", 0, _} -> + :error + + {microsecond, precision, rest} when precision in 1..6 -> + pad = String.duplicate("0", 6 - byte_size(microsecond)) + {{String.to_integer(microsecond <> pad), precision}, rest} + + {microsecond, _precision, rest} -> + {{String.to_integer(binary_part(microsecond, 0, 6)), 6}, rest} + end + end + + defp parse_microsecond("," <> rest) do + parse_microsecond("." <> rest) + end + + defp parse_microsecond(rest) do + {{0, 0}, rest} + end + + defp parse_microsecond(<>, precision, acc) when head in ?0..?9, + do: parse_microsecond(tail, precision + 1, <>) + + defp parse_microsecond(rest, precision, acc), do: {acc, precision, rest} + + defp parse_offset(""), do: {nil, ""} + defp parse_offset("Z"), do: {0, ""} + defp parse_offset("-00:00"), do: :error + + defp parse_offset(<>), + do: parse_offset(1, hour, min, rest) + + defp parse_offset(<>), + do: parse_offset(-1, hour, min, rest) + + defp parse_offset(<>), + do: parse_offset(1, hour, min, rest) + + defp parse_offset(<>), + do: parse_offset(-1, hour, min, rest) + + defp parse_offset(<>), do: parse_offset(1, hour, "00", rest) + defp parse_offset(<>), do: parse_offset(-1, hour, "00", rest) + defp parse_offset(_), do: :error + + defp parse_offset(sign, hour, min, rest) do + with {hour, ""} when hour < 24 <- Integer.parse(hour), + {min, ""} when min < 60 <- Integer.parse(min) do + {(hour * 60 + min) * 60 * sign, rest} + else + _ -> :error + end + end + + @doc false + def gregorian_seconds_to_iso_days(seconds, microsecond) do + {days, rest_seconds} = div_rem(seconds, @seconds_per_day) + microseconds_in_day = rest_seconds * @microseconds_per_second + microsecond + day_fraction = {microseconds_in_day, @parts_per_day} + {days, day_fraction} + end + + @doc false + def iso_days_to_unit({days, {parts, ppd}}, unit) do + day_microseconds = days * @parts_per_day + microseconds = divide_by_parts_per_day(parts, ppd) + System.convert_time_unit(day_microseconds + microseconds, :microsecond, unit) + end + + @doc false + def add_day_fraction_to_iso_days({days, {parts, ppd}}, add, ppd) do + normalize_iso_days(days, parts + add, ppd) + end + + def add_day_fraction_to_iso_days({days, {parts, ppd}}, add, add_ppd) do + parts = parts * add_ppd + add = add * ppd + gcd = Integer.gcd(ppd, add_ppd) + result_parts = div(parts + add, gcd) + result_ppd = div(ppd * add_ppd, gcd) + normalize_iso_days(days, result_parts, result_ppd) + end + + defp normalize_iso_days(days, parts, ppd) do + days_offset = div(parts, ppd) + parts = rem(parts, ppd) + + if parts < 0 do + {days + days_offset - 1, {parts + ppd, ppd}} + else + {days + days_offset, {parts, ppd}} + end + end + + # Note that this function does not add the extra leap day for a leap year. + # If you want to add that leap day when appropriate, + # add the result of leap_day_offset/2 to the result of days_before_month/1. + defp days_before_month(1), do: 0 + defp days_before_month(2), do: 31 + defp days_before_month(3), do: 59 + defp days_before_month(4), do: 90 + defp days_before_month(5), do: 120 + defp days_before_month(6), do: 151 + defp days_before_month(7), do: 181 + defp days_before_month(8), do: 212 + defp days_before_month(9), do: 243 + defp days_before_month(10), do: 273 + defp days_before_month(11), do: 304 + defp days_before_month(12), do: 334 + + defp leap_day_offset(_year, month) when month < 3, do: 0 + + defp leap_day_offset(year, _month) do + if leap_year?(year), do: 1, else: 0 + end + + defp days_to_year(days) when days < 0 do + year_estimate = -div(-days, @days_per_nonleap_year) - 1 + + {year, days_before_year} = + days_to_year(year_estimate, days, days_to_end_of_epoch(year_estimate)) + + leap_year_pad = if leap_year?(year), do: 1, else: 0 + {year, leap_year_pad + @days_per_nonleap_year + days - days_before_year} + end + + defp days_to_year(days) do + year_estimate = div(days, @days_per_nonleap_year) + + {year, days_before_year} = + days_to_year(year_estimate, days, days_in_previous_years(year_estimate)) + + {year, days - days_before_year} + end + + defp days_to_year(year, days1, days2) when year < 0 and days1 >= days2 do + days_to_year(year + 1, days1, days_to_end_of_epoch(year + 1)) + end + + defp days_to_year(year, days1, days2) when year >= 0 and days1 < days2 do + days_to_year(year - 1, days1, days_in_previous_years(year - 1)) + end + + defp days_to_year(year, _days1, days2) do + {year, days2} + end + + defp days_to_end_of_epoch(year) when year < 0 do + previous_year = year + 1 + + div(previous_year, 4) - div(previous_year, 100) + div(previous_year, 400) + + previous_year * @days_per_nonleap_year + end + + defp days_in_previous_years(0), do: 0 + + defp days_in_previous_years(year) do + previous_year = year - 1 + + Integer.floor_div(previous_year, 4) - Integer.floor_div(previous_year, 100) + + Integer.floor_div(previous_year, 400) + previous_year * @days_per_nonleap_year + + @days_per_leap_year + end + + # Note that 0 is the first day of the month. + defp year_day_to_year_date(_extra_day, day_of_year) when day_of_year < 31 do + {1, day_of_year} + end + + defp year_day_to_year_date(extra_day, day_of_year) when day_of_year < 59 + extra_day do + {2, day_of_year - 31} + end + + defp year_day_to_year_date(extra_day, day_of_year) when day_of_year < 90 + extra_day do + {3, day_of_year - (59 + extra_day)} + end + + defp year_day_to_year_date(extra_day, day_of_year) when day_of_year < 120 + extra_day do + {4, day_of_year - (90 + extra_day)} + end + + defp year_day_to_year_date(extra_day, day_of_year) when day_of_year < 151 + extra_day do + {5, day_of_year - (120 + extra_day)} + end + + defp year_day_to_year_date(extra_day, day_of_year) when day_of_year < 181 + extra_day do + {6, day_of_year - (151 + extra_day)} + end + + defp year_day_to_year_date(extra_day, day_of_year) when day_of_year < 212 + extra_day do + {7, day_of_year - (181 + extra_day)} + end + + defp year_day_to_year_date(extra_day, day_of_year) when day_of_year < 243 + extra_day do + {8, day_of_year - (212 + extra_day)} + end + + defp year_day_to_year_date(extra_day, day_of_year) when day_of_year < 273 + extra_day do + {9, day_of_year - (243 + extra_day)} + end + + defp year_day_to_year_date(extra_day, day_of_year) when day_of_year < 304 + extra_day do + {10, day_of_year - (273 + extra_day)} + end + + defp year_day_to_year_date(extra_day, day_of_year) when day_of_year < 334 + extra_day do + {11, day_of_year - (304 + extra_day)} + end + + defp year_day_to_year_date(extra_day, day_of_year) do + {12, day_of_year - (334 + extra_day)} + end + + defp iso_seconds_to_datetime(seconds) do + {days, rest_seconds} = div_rem(seconds, @seconds_per_day) + + date = date_from_iso_days(days) + time = seconds_to_time(rest_seconds) + {date, time} + end + + defp seconds_to_time(seconds) when seconds in 0..@last_second_of_the_day do + {hour, rest_seconds} = div_rem(seconds, @seconds_per_hour) + {minute, second} = div_rem(rest_seconds, @seconds_per_minute) + + {hour, minute, second} + end + + defp ensure_day_in_month!(year, month, day) when is_integer(day) do + if day < 1 or day > days_in_month(year, month) do + raise ArgumentError, "invalid date: #{date_to_string(year, month, day)}" + end + end +end diff --git a/lib/elixir/lib/calendar/naive_datetime.ex b/lib/elixir/lib/calendar/naive_datetime.ex new file mode 100644 index 00000000000..44f257ada7d --- /dev/null +++ b/lib/elixir/lib/calendar/naive_datetime.ex @@ -0,0 +1,1145 @@ +defmodule NaiveDateTime do + @moduledoc """ + A NaiveDateTime struct (without a time zone) and functions. + + The NaiveDateTime struct contains the fields year, month, day, hour, + minute, second, microsecond and calendar. New naive datetimes can be + built with the `new/2` and `new/8` functions or using the + `~N` (see `sigil_N/2`) sigil: + + iex> ~N[2000-01-01 23:00:07] + ~N[2000-01-01 23:00:07] + + The date and time fields in the struct can be accessed directly: + + iex> naive = ~N[2000-01-01 23:00:07] + iex> naive.year + 2000 + iex> naive.second + 7 + + We call them "naive" because this datetime representation does not + have a time zone. This means the datetime may not actually exist in + certain areas in the world even though it is valid. + + For example, when daylight saving changes are applied by a region, + the clock typically moves forward or backward by one hour. This means + certain datetimes never occur or may occur more than once. Since + `NaiveDateTime` is not validated against a time zone, such errors + would go unnoticed. + + Developers should avoid creating the NaiveDateTime structs directly + and instead, rely on the functions provided by this module as well + as the ones in third-party calendar libraries. + + ## Comparing naive date times + + Comparisons in Elixir using `==/2`, `>/2`, ` Enum.min([~N[2020-01-01 23:00:07], ~N[2000-01-01 23:00:07]], NaiveDateTime) + ~N[2000-01-01 23:00:07] + + ## Using epochs + + The `add/3` and `diff/3` functions can be used for computing with + date times or retrieving the number of seconds between instants. + For example, if there is an interest in computing the number of + seconds from the Unix epoch (1970-01-01 00:00:00): + + iex> NaiveDateTime.diff(~N[2010-04-17 14:00:00], ~N[1970-01-01 00:00:00]) + 1271512800 + + iex> NaiveDateTime.add(~N[1970-01-01 00:00:00], 1_271_512_800) + ~N[2010-04-17 14:00:00] + + Those functions are optimized to deal with common epochs, such + as the Unix Epoch above or the Gregorian Epoch (0000-01-01 00:00:00). + """ + + @enforce_keys [:year, :month, :day, :hour, :minute, :second] + defstruct [ + :year, + :month, + :day, + :hour, + :minute, + :second, + microsecond: {0, 0}, + calendar: Calendar.ISO + ] + + @type t :: %__MODULE__{ + year: Calendar.year(), + month: Calendar.month(), + day: Calendar.day(), + calendar: Calendar.calendar(), + hour: Calendar.hour(), + minute: Calendar.minute(), + second: Calendar.second(), + microsecond: Calendar.microsecond() + } + + @seconds_per_day 24 * 60 * 60 + + @doc """ + Returns the current naive datetime in UTC. + + Prefer using `DateTime.utc_now/0` when possible as, opposite + to `NaiveDateTime`, it will keep the time zone information. + + ## Examples + + iex> naive_datetime = NaiveDateTime.utc_now() + iex> naive_datetime.year >= 2016 + true + + """ + @doc since: "1.4.0" + @spec utc_now(Calendar.calendar()) :: t + def utc_now(calendar \\ Calendar.ISO) + + def utc_now(Calendar.ISO) do + {:ok, {year, month, day}, {hour, minute, second}, microsecond} = + Calendar.ISO.from_unix(:os.system_time(), :native) + + %NaiveDateTime{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: Calendar.ISO + } + end + + def utc_now(calendar) do + calendar + |> DateTime.utc_now() + |> DateTime.to_naive() + end + + @doc """ + Returns the "local time" for the machine the Elixir program is running on. + + WARNING: This function can cause insidious bugs. It depends on the time zone + configuration at run time. This can changed and be set to a time zone that has + daylight saving jumps (spring forward or fall back). + + This function can be used to display what the time is right now for the time + zone configuration that the machine happens to have. An example would be a + desktop program displaying a clock to the user. For any other uses it is + probably a bad idea to use this function. + + For most cases, use `DateTime.now/2` or `DateTime.utc_now/1` instead. + + Does not include fractional seconds. + + ## Examples + + iex> naive_datetime = NaiveDateTime.local_now() + iex> naive_datetime.year >= 2019 + true + + """ + @doc since: "1.10.0" + @spec local_now(Calendar.calendar()) :: t + def local_now(calendar \\ Calendar.ISO) + + def local_now(Calendar.ISO) do + {{year, month, day}, {hour, minute, second}} = :erlang.localtime() + {:ok, ndt} = NaiveDateTime.new(year, month, day, hour, minute, second) + ndt + end + + def local_now(calendar) do + naive_datetime = local_now() + + case convert(naive_datetime, calendar) do + {:ok, value} -> + value + + {:error, :incompatible_calendars} -> + raise ArgumentError, + ~s(cannot get "local now" in target calendar #{inspect(calendar)}, ) <> + "reason: cannot convert from Calendar.ISO to #{inspect(calendar)}." + end + end + + @doc """ + Builds a new ISO naive datetime. + + Expects all values to be integers. Returns `{:ok, naive_datetime}` + if each entry fits its appropriate range, returns `{:error, reason}` + otherwise. + + ## Examples + + iex> NaiveDateTime.new(2000, 1, 1, 0, 0, 0) + {:ok, ~N[2000-01-01 00:00:00]} + iex> NaiveDateTime.new(2000, 13, 1, 0, 0, 0) + {:error, :invalid_date} + iex> NaiveDateTime.new(2000, 2, 29, 0, 0, 0) + {:ok, ~N[2000-02-29 00:00:00]} + iex> NaiveDateTime.new(2000, 2, 30, 0, 0, 0) + {:error, :invalid_date} + iex> NaiveDateTime.new(2001, 2, 29, 0, 0, 0) + {:error, :invalid_date} + + iex> NaiveDateTime.new(2000, 1, 1, 23, 59, 59, {0, 1}) + {:ok, ~N[2000-01-01 23:59:59.0]} + iex> NaiveDateTime.new(2000, 1, 1, 23, 59, 59, 999_999) + {:ok, ~N[2000-01-01 23:59:59.999999]} + iex> NaiveDateTime.new(2000, 1, 1, 24, 59, 59, 999_999) + {:error, :invalid_time} + iex> NaiveDateTime.new(2000, 1, 1, 23, 60, 59, 999_999) + {:error, :invalid_time} + iex> NaiveDateTime.new(2000, 1, 1, 23, 59, 60, 999_999) + {:error, :invalid_time} + iex> NaiveDateTime.new(2000, 1, 1, 23, 59, 59, 1_000_000) + {:error, :invalid_time} + + iex> NaiveDateTime.new(2000, 1, 1, 23, 59, 59, {0, 1}, Calendar.ISO) + {:ok, ~N[2000-01-01 23:59:59.0]} + + """ + @spec new( + Calendar.year(), + Calendar.month(), + Calendar.day(), + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() | non_neg_integer, + Calendar.calendar() + ) :: {:ok, t} | {:error, atom} + def new(year, month, day, hour, minute, second, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) + + def new(year, month, day, hour, minute, second, microsecond, calendar) + when is_integer(microsecond) do + new(year, month, day, hour, minute, second, {microsecond, 6}, calendar) + end + + def new(year, month, day, hour, minute, second, microsecond, calendar) do + cond do + not calendar.valid_date?(year, month, day) -> + {:error, :invalid_date} + + not calendar.valid_time?(hour, minute, second, microsecond) -> + {:error, :invalid_time} + + true -> + naive_datetime = %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + + {:ok, naive_datetime} + end + end + + @doc """ + Builds a new ISO naive datetime. + + Expects all values to be integers. Returns `naive_datetime` + if each entry fits its appropriate range, raises if + time or date is invalid. + + ## Examples + + iex> NaiveDateTime.new!(2000, 1, 1, 0, 0, 0) + ~N[2000-01-01 00:00:00] + iex> NaiveDateTime.new!(2000, 2, 29, 0, 0, 0) + ~N[2000-02-29 00:00:00] + iex> NaiveDateTime.new!(2000, 1, 1, 23, 59, 59, {0, 1}) + ~N[2000-01-01 23:59:59.0] + iex> NaiveDateTime.new!(2000, 1, 1, 23, 59, 59, 999_999) + ~N[2000-01-01 23:59:59.999999] + iex> NaiveDateTime.new!(2000, 1, 1, 23, 59, 59, {0, 1}, Calendar.ISO) + ~N[2000-01-01 23:59:59.0] + iex> NaiveDateTime.new!(2000, 1, 1, 24, 59, 59, 999_999) + ** (ArgumentError) cannot build naive datetime, reason: :invalid_time + + """ + @doc since: "1.11.0" + @spec new!( + Calendar.year(), + Calendar.month(), + Calendar.day(), + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() | non_neg_integer, + Calendar.calendar() + ) :: t + def new!( + year, + month, + day, + hour, + minute, + second, + microsecond \\ {0, 0}, + calendar \\ Calendar.ISO + ) + + def new!(year, month, day, hour, minute, second, microsecond, calendar) do + case new(year, month, day, hour, minute, second, microsecond, calendar) do + {:ok, naive_datetime} -> + naive_datetime + + {:error, reason} -> + raise ArgumentError, "cannot build naive datetime, reason: #{inspect(reason)}" + end + end + + @doc """ + Builds a naive datetime from date and time structs. + + ## Examples + + iex> NaiveDateTime.new(~D[2010-01-13], ~T[23:00:07.005]) + {:ok, ~N[2010-01-13 23:00:07.005]} + + """ + @spec new(Date.t(), Time.t()) :: {:ok, t} + def new(date, time) + + def new(%Date{calendar: calendar} = date, %Time{calendar: calendar} = time) do + %{year: year, month: month, day: day} = date + %{hour: hour, minute: minute, second: second, microsecond: microsecond} = time + + naive_datetime = %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + + {:ok, naive_datetime} + end + + @doc """ + Builds a naive datetime from date and time structs. + + ## Examples + + iex> NaiveDateTime.new!(~D[2010-01-13], ~T[23:00:07.005]) + ~N[2010-01-13 23:00:07.005] + + """ + @doc since: "1.11.0" + @spec new!(Date.t(), Time.t()) :: t + def new!(date, time) + + def new!(%Date{calendar: calendar} = date, %Time{calendar: calendar} = time) do + {:ok, naive_datetime} = new(date, time) + naive_datetime + end + + @doc """ + Adds a specified amount of time to a `NaiveDateTime`. + + Accepts an `amount_to_add` in any `unit` available from `t:System.time_unit/0`. + Negative values will move backwards in time. + + ## Examples + + # adds seconds by default + iex> NaiveDateTime.add(~N[2014-10-02 00:29:10], 2) + ~N[2014-10-02 00:29:12] + + # accepts negative offsets + iex> NaiveDateTime.add(~N[2014-10-02 00:29:10], -2) + ~N[2014-10-02 00:29:08] + + # can work with other units + iex> NaiveDateTime.add(~N[2014-10-02 00:29:10], 2_000, :millisecond) + ~N[2014-10-02 00:29:12] + + # keeps the same precision + iex> NaiveDateTime.add(~N[2014-10-02 00:29:10.021], 21, :second) + ~N[2014-10-02 00:29:31.021] + + # changes below the precision will not be visible + iex> hidden = NaiveDateTime.add(~N[2014-10-02 00:29:10], 21, :millisecond) + iex> hidden.microsecond # ~N[2014-10-02 00:29:10] + {21000, 0} + + # from Gregorian seconds + iex> NaiveDateTime.add(~N[0000-01-01 00:00:00], 63_579_428_950) + ~N[2014-10-02 00:29:10] + + Passing a `DateTime` automatically converts it to `NaiveDateTime`, + discarding the time zone information: + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> NaiveDateTime.add(dt, 21, :second) + ~N[2000-02-29 23:00:28] + + """ + @doc since: "1.4.0" + @spec add(Calendar.naive_datetime(), integer, System.time_unit()) :: t + def add( + %{microsecond: {_, precision}, calendar: calendar} = naive_datetime, + amount_to_add, + unit \\ :second + ) + when is_integer(amount_to_add) do + ppd = System.convert_time_unit(86400, :second, unit) + + naive_datetime + |> to_iso_days() + |> Calendar.ISO.add_day_fraction_to_iso_days(amount_to_add, ppd) + |> from_iso_days(calendar, precision) + end + + @doc """ + Subtracts `naive_datetime2` from `naive_datetime1`. + + The answer can be returned in any `unit` available from `t:System.time_unit/0`. + + This function returns the difference in seconds where seconds are measured + according to `Calendar.ISO`. + + ## Examples + + iex> NaiveDateTime.diff(~N[2014-10-02 00:29:12], ~N[2014-10-02 00:29:10]) + 2 + iex> NaiveDateTime.diff(~N[2014-10-02 00:29:12], ~N[2014-10-02 00:29:10], :microsecond) + 2_000_000 + iex> NaiveDateTime.diff(~N[2014-10-02 00:29:10.042], ~N[2014-10-02 00:29:10.021], :millisecond) + 21 + iex> NaiveDateTime.diff(~N[2014-10-02 00:29:10], ~N[2014-10-02 00:29:12]) + -2 + iex> NaiveDateTime.diff(~N[-0001-10-02 00:29:10], ~N[-0001-10-02 00:29:12]) + -2 + + # to Gregorian seconds + iex> NaiveDateTime.diff(~N[2014-10-02 00:29:10], ~N[0000-01-01 00:00:00]) + 63579428950 + + """ + @doc since: "1.4.0" + @spec diff(Calendar.naive_datetime(), Calendar.naive_datetime(), System.time_unit()) :: integer + def diff( + %{calendar: calendar1} = naive_datetime1, + %{calendar: calendar2} = naive_datetime2, + unit \\ :second + ) do + if not Calendar.compatible_calendars?(calendar1, calendar2) do + raise ArgumentError, + "cannot calculate the difference between #{inspect(naive_datetime1)} and " <> + "#{inspect(naive_datetime2)} because their calendars are not compatible " <> + "and thus the result would be ambiguous" + end + + units1 = naive_datetime1 |> to_iso_days() |> Calendar.ISO.iso_days_to_unit(unit) + units2 = naive_datetime2 |> to_iso_days() |> Calendar.ISO.iso_days_to_unit(unit) + units1 - units2 + end + + @doc """ + Returns the given naive datetime with the microsecond field truncated to the + given precision (`:microsecond`, `:millisecond` or `:second`). + + The given naive datetime is returned unchanged if it already has lower precision + than the given precision. + + ## Examples + + iex> NaiveDateTime.truncate(~N[2017-11-06 00:23:51.123456], :microsecond) + ~N[2017-11-06 00:23:51.123456] + + iex> NaiveDateTime.truncate(~N[2017-11-06 00:23:51.123456], :millisecond) + ~N[2017-11-06 00:23:51.123] + + iex> NaiveDateTime.truncate(~N[2017-11-06 00:23:51.123456], :second) + ~N[2017-11-06 00:23:51] + + """ + @doc since: "1.6.0" + @spec truncate(t(), :microsecond | :millisecond | :second) :: t() + def truncate(%NaiveDateTime{microsecond: microsecond} = naive_datetime, precision) do + %{naive_datetime | microsecond: Calendar.truncate(microsecond, precision)} + end + + def truncate( + %{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }, + precision + ) do + %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: Calendar.truncate(microsecond, precision) + } + end + + @doc """ + Converts a `NaiveDateTime` into a `Date`. + + Because `Date` does not hold time information, + data will be lost during the conversion. + + ## Examples + + iex> NaiveDateTime.to_date(~N[2002-01-13 23:00:07]) + ~D[2002-01-13] + + """ + @spec to_date(Calendar.naive_datetime()) :: Date.t() + def to_date(%{ + year: year, + month: month, + day: day, + calendar: calendar, + hour: _, + minute: _, + second: _, + microsecond: _ + }) do + %Date{year: year, month: month, day: day, calendar: calendar} + end + + @doc """ + Converts a `NaiveDateTime` into `Time`. + + Because `Time` does not hold date information, + data will be lost during the conversion. + + ## Examples + + iex> NaiveDateTime.to_time(~N[2002-01-13 23:00:07]) + ~T[23:00:07] + + """ + @spec to_time(Calendar.naive_datetime()) :: Time.t() + def to_time(%{ + year: _, + month: _, + day: _, + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }) do + %Time{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + } + end + + @doc """ + Converts the given naive datetime to a string according to its calendar. + + ### Examples + + iex> NaiveDateTime.to_string(~N[2000-02-28 23:00:13]) + "2000-02-28 23:00:13" + iex> NaiveDateTime.to_string(~N[2000-02-28 23:00:13.001]) + "2000-02-28 23:00:13.001" + iex> NaiveDateTime.to_string(~N[-0100-12-15 03:20:31]) + "-0100-12-15 03:20:31" + + This function can also be used to convert a DateTime to a string without + the time zone information: + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> NaiveDateTime.to_string(dt) + "2000-02-29 23:00:07" + + """ + @spec to_string(Calendar.naive_datetime()) :: String.t() + def to_string(%{calendar: calendar} = naive_datetime) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } = naive_datetime + + calendar.naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) + end + + @doc """ + Parses the extended "Date and time of day" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + Time zone offset may be included in the string but they will be + simply discarded as such information is not included in naive date + times. + + As specified in the standard, the separator "T" may be omitted if + desired as there is no ambiguity within this function. + + Note leap seconds are not supported by the built-in Calendar.ISO. + + ## Examples + + iex> NaiveDateTime.from_iso8601("2015-01-23 23:50:07") + {:ok, ~N[2015-01-23 23:50:07]} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07") + {:ok, ~N[2015-01-23 23:50:07]} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07Z") + {:ok, ~N[2015-01-23 23:50:07]} + + iex> NaiveDateTime.from_iso8601("2015-01-23 23:50:07.0") + {:ok, ~N[2015-01-23 23:50:07.0]} + iex> NaiveDateTime.from_iso8601("2015-01-23 23:50:07,0123456") + {:ok, ~N[2015-01-23 23:50:07.012345]} + iex> NaiveDateTime.from_iso8601("2015-01-23 23:50:07.0123456") + {:ok, ~N[2015-01-23 23:50:07.012345]} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123Z") + {:ok, ~N[2015-01-23 23:50:07.123]} + + iex> NaiveDateTime.from_iso8601("2015-01-23P23:50:07") + {:error, :invalid_format} + iex> NaiveDateTime.from_iso8601("2015:01:23 23-50-07") + {:error, :invalid_format} + iex> NaiveDateTime.from_iso8601("2015-01-23 23:50:07A") + {:error, :invalid_format} + iex> NaiveDateTime.from_iso8601("2015-01-23 23:50:61") + {:error, :invalid_time} + iex> NaiveDateTime.from_iso8601("2015-01-32 23:50:07") + {:error, :invalid_date} + + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123+02:30") + {:ok, ~N[2015-01-23 23:50:07.123]} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123+00:00") + {:ok, ~N[2015-01-23 23:50:07.123]} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123-02:30") + {:ok, ~N[2015-01-23 23:50:07.123]} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123-00:00") + {:error, :invalid_format} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123-00:60") + {:error, :invalid_format} + iex> NaiveDateTime.from_iso8601("2015-01-23T23:50:07.123-24:00") + {:error, :invalid_format} + + """ + @spec from_iso8601(String.t(), Calendar.calendar()) :: {:ok, t} | {:error, atom} + def from_iso8601(string, calendar \\ Calendar.ISO) do + with {:ok, {year, month, day, hour, minute, second, microsecond}} <- + Calendar.ISO.parse_naive_datetime(string) do + convert( + %NaiveDateTime{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }, + calendar + ) + end + end + + @doc """ + Parses the extended "Date and time of day" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + Raises if the format is invalid. + + ## Examples + + iex> NaiveDateTime.from_iso8601!("2015-01-23T23:50:07.123Z") + ~N[2015-01-23 23:50:07.123] + iex> NaiveDateTime.from_iso8601!("2015-01-23T23:50:07,123Z") + ~N[2015-01-23 23:50:07.123] + iex> NaiveDateTime.from_iso8601!("2015-01-23P23:50:07") + ** (ArgumentError) cannot parse "2015-01-23P23:50:07" as naive datetime, reason: :invalid_format + + """ + @spec from_iso8601!(String.t(), Calendar.calendar()) :: t + def from_iso8601!(string, calendar \\ Calendar.ISO) do + case from_iso8601(string, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, + "cannot parse #{inspect(string)} as naive datetime, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts the given naive datetime to + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + By default, `NaiveDateTime.to_iso8601/2` returns naive datetimes formatted in the "extended" + format, for human readability. It also supports the "basic" format through passing the `:basic` option. + + Only supports converting naive datetimes which are in the ISO calendar, + attempting to convert naive datetimes from other calendars will raise. + + ### Examples + + iex> NaiveDateTime.to_iso8601(~N[2000-02-28 23:00:13]) + "2000-02-28T23:00:13" + + iex> NaiveDateTime.to_iso8601(~N[2000-02-28 23:00:13.001]) + "2000-02-28T23:00:13.001" + + iex> NaiveDateTime.to_iso8601(~N[2000-02-28 23:00:13.001], :basic) + "20000228T230013.001" + + This function can also be used to convert a DateTime to ISO 8601 without + the time zone information: + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> NaiveDateTime.to_iso8601(dt) + "2000-02-29T23:00:07" + + """ + @spec to_iso8601(Calendar.naive_datetime(), :basic | :extended) :: String.t() + def to_iso8601(naive_datetime, format \\ :extended) + + def to_iso8601(%{calendar: Calendar.ISO} = naive_datetime, format) + when format in [:basic, :extended] do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } = naive_datetime + + Calendar.ISO.date_to_string(year, month, day, format) <> + "T" <> Calendar.ISO.time_to_string(hour, minute, second, microsecond, format) + end + + def to_iso8601(%{calendar: _} = naive_datetime, format) when format in [:basic, :extended] do + naive_datetime + |> convert!(Calendar.ISO) + |> to_iso8601(format) + end + + @doc """ + Converts a `NaiveDateTime` struct to an Erlang datetime tuple. + + Only supports converting naive datetimes which are in the ISO calendar, + attempting to convert naive datetimes from other calendars will raise. + + WARNING: Loss of precision may occur, as Erlang time tuples only store + hour/minute/second. + + ## Examples + + iex> NaiveDateTime.to_erl(~N[2000-01-01 13:30:15]) + {{2000, 1, 1}, {13, 30, 15}} + + This function can also be used to convert a DateTime to an Erlang + datetime tuple without the time zone information: + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> NaiveDateTime.to_erl(dt) + {{2000, 2, 29}, {23, 00, 07}} + + """ + @spec to_erl(Calendar.naive_datetime()) :: :calendar.datetime() + def to_erl(%{calendar: _} = naive_datetime) do + %{year: year, month: month, day: day, hour: hour, minute: minute, second: second} = + convert!(naive_datetime, Calendar.ISO) + + {{year, month, day}, {hour, minute, second}} + end + + @doc """ + Converts an Erlang datetime tuple to a `NaiveDateTime` struct. + + Attempting to convert an invalid ISO calendar date will produce an error tuple. + + ## Examples + + iex> NaiveDateTime.from_erl({{2000, 1, 1}, {13, 30, 15}}) + {:ok, ~N[2000-01-01 13:30:15]} + iex> NaiveDateTime.from_erl({{2000, 1, 1}, {13, 30, 15}}, {5000, 3}) + {:ok, ~N[2000-01-01 13:30:15.005]} + iex> NaiveDateTime.from_erl({{2000, 13, 1}, {13, 30, 15}}) + {:error, :invalid_date} + iex> NaiveDateTime.from_erl({{2000, 13, 1}, {13, 30, 15}}) + {:error, :invalid_date} + + """ + @spec from_erl(:calendar.datetime(), Calendar.microsecond(), Calendar.calendar()) :: + {:ok, t} | {:error, atom} + def from_erl(tuple, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) + + def from_erl({{year, month, day}, {hour, minute, second}}, microsecond, calendar) do + with {:ok, iso_naive_dt} <- new(year, month, day, hour, minute, second, microsecond), + do: convert(iso_naive_dt, calendar) + end + + @doc """ + Converts an Erlang datetime tuple to a `NaiveDateTime` struct. + + Raises if the datetime is invalid. + Attempting to convert an invalid ISO calendar date will produce an error tuple. + + ## Examples + + iex> NaiveDateTime.from_erl!({{2000, 1, 1}, {13, 30, 15}}) + ~N[2000-01-01 13:30:15] + iex> NaiveDateTime.from_erl!({{2000, 1, 1}, {13, 30, 15}}, {5000, 3}) + ~N[2000-01-01 13:30:15.005] + iex> NaiveDateTime.from_erl!({{2000, 13, 1}, {13, 30, 15}}) + ** (ArgumentError) cannot convert {{2000, 13, 1}, {13, 30, 15}} to naive datetime, reason: :invalid_date + + """ + @spec from_erl!(:calendar.datetime(), Calendar.microsecond(), Calendar.calendar()) :: t + def from_erl!(tuple, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) do + case from_erl(tuple, microsecond, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, + "cannot convert #{inspect(tuple)} to naive datetime, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts a number of gregorian seconds to a `NaiveDateTime` struct. + + ## Examples + + iex> NaiveDateTime.from_gregorian_seconds(1) + ~N[0000-01-01 00:00:01] + iex> NaiveDateTime.from_gregorian_seconds(63_755_511_991, {5000, 3}) + ~N[2020-05-01 00:26:31.005] + iex> NaiveDateTime.from_gregorian_seconds(-1) + ~N[-0001-12-31 23:59:59] + + """ + @doc since: "1.11.0" + @spec from_gregorian_seconds(integer(), Calendar.microsecond(), Calendar.calendar()) :: t + def from_gregorian_seconds( + seconds, + {microsecond, precision} \\ {0, 0}, + calendar \\ Calendar.ISO + ) + when is_integer(seconds) do + iso_days = Calendar.ISO.gregorian_seconds_to_iso_days(seconds, microsecond) + + {year, month, day, hour, minute, second, {microsecond, _}} = + calendar.naive_datetime_from_iso_days(iso_days) + + %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision} + } + end + + @doc """ + Converts a `NaiveDateTime` struct to a number of gregorian seconds and microseconds. + + ## Examples + + iex> NaiveDateTime.to_gregorian_seconds(~N[0000-01-01 00:00:01]) + {1, 0} + iex> NaiveDateTime.to_gregorian_seconds(~N[2020-05-01 00:26:31.005]) + {63_755_511_991, 5000} + + """ + @doc since: "1.11.0" + @spec to_gregorian_seconds(Calendar.naive_datetime()) :: {integer(), non_neg_integer()} + def to_gregorian_seconds(%{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision} + }) do + {days, day_fraction} = + calendar.naive_datetime_to_iso_days( + year, + month, + day, + hour, + minute, + second, + {microsecond, precision} + ) + + seconds_in_day = seconds_from_day_fraction(day_fraction) + {days * @seconds_per_day + seconds_in_day, microsecond} + end + + @doc """ + Compares two `NaiveDateTime` structs. + + Returns `:gt` if first is later than the second + and `:lt` for vice versa. If the two NaiveDateTime + are equal `:eq` is returned. + + ## Examples + + iex> NaiveDateTime.compare(~N[2016-04-16 13:30:15], ~N[2016-04-28 16:19:25]) + :lt + iex> NaiveDateTime.compare(~N[2016-04-16 13:30:15.1], ~N[2016-04-16 13:30:15.01]) + :gt + + This function can also be used to compare a DateTime without + the time zone information: + + iex> dt = %DateTime{year: 2000, month: 2, day: 29, zone_abbr: "CET", + ...> hour: 23, minute: 0, second: 7, microsecond: {0, 0}, + ...> utc_offset: 3600, std_offset: 0, time_zone: "Europe/Warsaw"} + iex> NaiveDateTime.compare(dt, ~N[2000-02-29 23:00:07]) + :eq + iex> NaiveDateTime.compare(dt, ~N[2000-01-29 23:00:07]) + :gt + iex> NaiveDateTime.compare(dt, ~N[2000-03-29 23:00:07]) + :lt + + """ + @doc since: "1.4.0" + @spec compare(Calendar.naive_datetime(), Calendar.naive_datetime()) :: :lt | :eq | :gt + def compare(%{calendar: calendar1} = naive_datetime1, %{calendar: calendar2} = naive_datetime2) do + if Calendar.compatible_calendars?(calendar1, calendar2) do + case {to_iso_days(naive_datetime1), to_iso_days(naive_datetime2)} do + {first, second} when first > second -> :gt + {first, second} when first < second -> :lt + _ -> :eq + end + else + raise ArgumentError, """ + cannot compare #{inspect(naive_datetime1)} with #{inspect(naive_datetime2)}. + + This comparison would be ambiguous as their calendars have incompatible day rollover moments. + Specify an exact time of day (using `DateTime`s) to resolve this ambiguity + """ + end + end + + @doc """ + Converts the given `naive_datetime` from one calendar to another. + + If it is not possible to convert unambiguously between the calendars + (see `Calendar.compatible_calendars?/2`), an `{:error, :incompatible_calendars}` tuple + is returned. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10,000 years to the current Gregorian + year: + + iex> NaiveDateTime.convert(~N[2000-01-01 13:30:15], Calendar.Holocene) + {:ok, %NaiveDateTime{calendar: Calendar.Holocene, year: 12000, month: 1, day: 1, + hour: 13, minute: 30, second: 15, microsecond: {0, 0}}} + + """ + @doc since: "1.5.0" + @spec convert(Calendar.naive_datetime(), Calendar.calendar()) :: + {:ok, t} | {:error, :incompatible_calendars} + + # Keep it multiline for proper function clause errors. + def convert( + %{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }, + calendar + ) do + naive_datetime = %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + + {:ok, naive_datetime} + end + + def convert(%{calendar: ndt_calendar, microsecond: {_, precision}} = naive_datetime, calendar) do + if Calendar.compatible_calendars?(ndt_calendar, calendar) do + result_naive_datetime = + naive_datetime + |> to_iso_days + |> from_iso_days(calendar, precision) + + {:ok, result_naive_datetime} + else + {:error, :incompatible_calendars} + end + end + + @doc """ + Converts the given `naive_datetime` from one calendar to another. + + If it is not possible to convert unambiguously between the calendars + (see `Calendar.compatible_calendars?/2`), an ArgumentError is raised. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10,000 years to the current Gregorian + year: + + iex> NaiveDateTime.convert!(~N[2000-01-01 13:30:15], Calendar.Holocene) + %NaiveDateTime{calendar: Calendar.Holocene, year: 12000, month: 1, day: 1, + hour: 13, minute: 30, second: 15, microsecond: {0, 0}} + + """ + @doc since: "1.5.0" + @spec convert!(Calendar.naive_datetime(), Calendar.calendar()) :: t + def convert!(naive_datetime, calendar) do + case convert(naive_datetime, calendar) do + {:ok, value} -> + value + + {:error, :incompatible_calendars} -> + raise ArgumentError, + "cannot convert #{inspect(naive_datetime)} to target calendar #{inspect(calendar)}, " <> + "reason: #{inspect(naive_datetime.calendar)} and #{inspect(calendar)} " <> + "have different day rollover moments, making this conversion ambiguous" + end + end + + ## Helpers + + defp seconds_from_day_fraction({parts_in_day, @seconds_per_day}), + do: parts_in_day + + defp seconds_from_day_fraction({parts_in_day, parts_per_day}), + do: div(parts_in_day * @seconds_per_day, parts_per_day) + + # Keep it multiline for proper function clause errors. + defp to_iso_days(%{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }) do + calendar.naive_datetime_to_iso_days(year, month, day, hour, minute, second, microsecond) + end + + defp from_iso_days(iso_days, calendar, precision) do + {year, month, day, hour, minute, second, {microsecond, _}} = + calendar.naive_datetime_from_iso_days(iso_days) + + %NaiveDateTime{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision} + } + end + + defimpl String.Chars do + def to_string(naive_datetime) do + %{ + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } = naive_datetime + + calendar.naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) + end + end + + defimpl Inspect do + def inspect(naive_datetime, _) do + %{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + } = naive_datetime + + formatted = + calendar.naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) + + "~N[" <> formatted <> suffix(calendar) <> "]" + end + + defp suffix(Calendar.ISO), do: "" + defp suffix(calendar), do: " " <> inspect(calendar) + end +end diff --git a/lib/elixir/lib/calendar/time.ex b/lib/elixir/lib/calendar/time.ex new file mode 100644 index 00000000000..c88b5716c84 --- /dev/null +++ b/lib/elixir/lib/calendar/time.ex @@ -0,0 +1,789 @@ +defmodule Time do + @moduledoc """ + A Time struct and functions. + + The Time struct contains the fields hour, minute, second and microseconds. + New times can be built with the `new/4` function or using the + `~T` (see `sigil_T/2`) sigil: + + iex> ~T[23:00:07.001] + ~T[23:00:07.001] + + Both `new/4` and sigil return a struct where the time fields can + be accessed directly: + + iex> time = ~T[23:00:07.001] + iex> time.hour + 23 + iex> time.microsecond + {1000, 3} + + The functions on this module work with the `Time` struct as well + as any struct that contains the same fields as the `Time` struct, + such as `NaiveDateTime` and `DateTime`. Such functions expect + `t:Calendar.time/0` in their typespecs (instead of `t:t/0`). + + Developers should avoid creating the Time structs directly + and instead rely on the functions provided by this module as well + as the ones in third-party calendar libraries. + + ## Comparing times + + Comparisons in Elixir using `==/2`, `>/2`, ` Enum.min([~T[23:00:07.001], ~T[10:00:07.001]], Time) + ~T[10:00:07.001] + """ + + @enforce_keys [:hour, :minute, :second] + defstruct [:hour, :minute, :second, microsecond: {0, 0}, calendar: Calendar.ISO] + + @type t :: %__MODULE__{ + hour: Calendar.hour(), + minute: Calendar.minute(), + second: Calendar.second(), + microsecond: Calendar.microsecond(), + calendar: Calendar.calendar() + } + + @parts_per_day 86_400_000_000 + @seconds_per_day 24 * 60 * 60 + + @doc """ + Returns the current time in UTC. + + ## Examples + + iex> time = Time.utc_now() + iex> time.hour >= 0 + true + + """ + @doc since: "1.4.0" + @spec utc_now(Calendar.calendar()) :: t + def utc_now(calendar \\ Calendar.ISO) do + {:ok, _, time, microsecond} = Calendar.ISO.from_unix(:os.system_time(), :native) + {hour, minute, second} = time + + iso_time = %Time{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: Calendar.ISO + } + + convert!(iso_time, calendar) + end + + @doc """ + Builds a new time. + + Expects all values to be integers. Returns `{:ok, time}` if each + entry fits its appropriate range, returns `{:error, reason}` otherwise. + + Microseconds can also be given with a precision, which must be an + integer between 0 and 6. + + The built-in calendar does not support leap seconds. + + ## Examples + + iex> Time.new(0, 0, 0, 0) + {:ok, ~T[00:00:00.000000]} + iex> Time.new(23, 59, 59, 999_999) + {:ok, ~T[23:59:59.999999]} + + iex> Time.new(24, 59, 59, 999_999) + {:error, :invalid_time} + iex> Time.new(23, 60, 59, 999_999) + {:error, :invalid_time} + iex> Time.new(23, 59, 60, 999_999) + {:error, :invalid_time} + iex> Time.new(23, 59, 59, 1_000_000) + {:error, :invalid_time} + + # Invalid precision + Time.new(23, 59, 59, {999_999, 10}) + {:error, :invalid_time} + + """ + @spec new( + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() | non_neg_integer, + Calendar.calendar() + ) :: {:ok, t} | {:error, atom} + def new(hour, minute, second, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) + + def new(hour, minute, second, microsecond, calendar) when is_integer(microsecond) do + new(hour, minute, second, {microsecond, 6}, calendar) + end + + def new(hour, minute, second, {microsecond, precision}, calendar) + when is_integer(hour) and is_integer(minute) and is_integer(second) and + is_integer(microsecond) and is_integer(precision) do + case calendar.valid_time?(hour, minute, second, {microsecond, precision}) do + true -> + time = %Time{ + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision}, + calendar: calendar + } + + {:ok, time} + + false -> + {:error, :invalid_time} + end + end + + @doc """ + Builds a new time. + + Expects all values to be integers. Returns `time` if each + entry fits its appropriate range, raises if the time is invalid. + + Microseconds can also be given with a precision, which must be an + integer between 0 and 6. + + The built-in calendar does not support leap seconds. + + ## Examples + + iex> Time.new!(0, 0, 0, 0) + ~T[00:00:00.000000] + iex> Time.new!(23, 59, 59, 999_999) + ~T[23:59:59.999999] + iex> Time.new!(24, 59, 59, 999_999) + ** (ArgumentError) cannot build time, reason: :invalid_time + """ + @doc since: "1.11.0" + @spec new!( + Calendar.hour(), + Calendar.minute(), + Calendar.second(), + Calendar.microsecond() | non_neg_integer, + Calendar.calendar() + ) :: t + def new!(hour, minute, second, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) do + case new(hour, minute, second, microsecond, calendar) do + {:ok, time} -> + time + + {:error, reason} -> + raise ArgumentError, "cannot build time, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts the given `time` to a string. + + ### Examples + + iex> Time.to_string(~T[23:00:00]) + "23:00:00" + iex> Time.to_string(~T[23:00:00.001]) + "23:00:00.001" + iex> Time.to_string(~T[23:00:00.123456]) + "23:00:00.123456" + + iex> Time.to_string(~N[2015-01-01 23:00:00.001]) + "23:00:00.001" + iex> Time.to_string(~N[2015-01-01 23:00:00.123456]) + "23:00:00.123456" + + """ + @spec to_string(Calendar.time()) :: String.t() + def to_string(time) + + def to_string(%{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + }) do + calendar.time_to_string(hour, minute, second, microsecond) + end + + @doc """ + Parses the extended "Local time" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + Time zone offset may be included in the string but they will be + simply discarded as such information is not included in times. + + As specified in the standard, the separator "T" may be omitted if + desired as there is no ambiguity within this function. + + ## Examples + + iex> Time.from_iso8601("23:50:07") + {:ok, ~T[23:50:07]} + iex> Time.from_iso8601("23:50:07Z") + {:ok, ~T[23:50:07]} + iex> Time.from_iso8601("T23:50:07Z") + {:ok, ~T[23:50:07]} + + iex> Time.from_iso8601("23:50:07,0123456") + {:ok, ~T[23:50:07.012345]} + iex> Time.from_iso8601("23:50:07.0123456") + {:ok, ~T[23:50:07.012345]} + iex> Time.from_iso8601("23:50:07.123Z") + {:ok, ~T[23:50:07.123]} + + iex> Time.from_iso8601("2015:01:23 23-50-07") + {:error, :invalid_format} + iex> Time.from_iso8601("23:50:07A") + {:error, :invalid_format} + iex> Time.from_iso8601("23:50:07.") + {:error, :invalid_format} + iex> Time.from_iso8601("23:50:61") + {:error, :invalid_time} + + """ + @spec from_iso8601(String.t(), Calendar.calendar()) :: {:ok, t} | {:error, atom} + def from_iso8601(string, calendar \\ Calendar.ISO) do + with {:ok, {hour, minute, second, microsecond}} <- Calendar.ISO.parse_time(string) do + convert( + %Time{hour: hour, minute: minute, second: second, microsecond: microsecond}, + calendar + ) + end + end + + @doc """ + Parses the extended "Local time" format described by + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + Raises if the format is invalid. + + ## Examples + + iex> Time.from_iso8601!("23:50:07,123Z") + ~T[23:50:07.123] + iex> Time.from_iso8601!("23:50:07.123Z") + ~T[23:50:07.123] + iex> Time.from_iso8601!("2015:01:23 23-50-07") + ** (ArgumentError) cannot parse "2015:01:23 23-50-07" as time, reason: :invalid_format + + """ + @spec from_iso8601!(String.t(), Calendar.calendar()) :: t + def from_iso8601!(string, calendar \\ Calendar.ISO) do + case from_iso8601(string, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, "cannot parse #{inspect(string)} as time, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts the given time to + [ISO 8601:2019](https://en.wikipedia.org/wiki/ISO_8601). + + By default, `Time.to_iso8601/2` returns times formatted in the "extended" + format, for human readability. It also supports the "basic" format through + passing the `:basic` option. + + ### Examples + + iex> Time.to_iso8601(~T[23:00:13]) + "23:00:13" + + iex> Time.to_iso8601(~T[23:00:13.001]) + "23:00:13.001" + + iex> Time.to_iso8601(~T[23:00:13.001], :basic) + "230013.001" + + iex> Time.to_iso8601(~N[2010-04-17 23:00:13]) + "23:00:13" + + """ + @spec to_iso8601(Calendar.time(), :extended | :basic) :: String.t() + def to_iso8601(time, format \\ :extended) + + def to_iso8601(%{calendar: Calendar.ISO} = time, format) when format in [:extended, :basic] do + %{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } = time + + Calendar.ISO.time_to_string(hour, minute, second, microsecond, format) + end + + def to_iso8601(%{calendar: _} = time, format) when format in [:extended, :basic] do + time + |> convert!(Calendar.ISO) + |> to_iso8601(format) + end + + @doc """ + Converts given `time` to an Erlang time tuple. + + WARNING: Loss of precision may occur, as Erlang time tuples + only contain hours/minutes/seconds. + + ## Examples + + iex> Time.to_erl(~T[23:30:15.999]) + {23, 30, 15} + + iex> Time.to_erl(~N[2010-04-17 23:30:15.999]) + {23, 30, 15} + + """ + @spec to_erl(Calendar.time()) :: :calendar.time() + def to_erl(time) do + %{hour: hour, minute: minute, second: second} = convert!(time, Calendar.ISO) + {hour, minute, second} + end + + @doc """ + Converts an Erlang time tuple to a `Time` struct. + + ## Examples + + iex> Time.from_erl({23, 30, 15}, {5000, 3}) + {:ok, ~T[23:30:15.005]} + iex> Time.from_erl({24, 30, 15}) + {:error, :invalid_time} + + """ + @spec from_erl(:calendar.time(), Calendar.microsecond(), Calendar.calendar()) :: + {:ok, t} | {:error, atom} + def from_erl(tuple, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) + + def from_erl({hour, minute, second}, microsecond, calendar) do + with {:ok, time} <- new(hour, minute, second, microsecond, Calendar.ISO), + do: convert(time, calendar) + end + + @doc """ + Converts an Erlang time tuple to a `Time` struct. + + ## Examples + + iex> Time.from_erl!({23, 30, 15}) + ~T[23:30:15] + iex> Time.from_erl!({23, 30, 15}, {5000, 3}) + ~T[23:30:15.005] + iex> Time.from_erl!({24, 30, 15}) + ** (ArgumentError) cannot convert {24, 30, 15} to time, reason: :invalid_time + + """ + @spec from_erl!(:calendar.time(), Calendar.microsecond(), Calendar.calendar()) :: t + def from_erl!(tuple, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) do + case from_erl(tuple, microsecond, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, + "cannot convert #{inspect(tuple)} to time, reason: #{inspect(reason)}" + end + end + + @doc """ + Converts a number of seconds after midnight to a `Time` struct. + + ## Examples + + iex> Time.from_seconds_after_midnight(10_000) + ~T[02:46:40] + iex> Time.from_seconds_after_midnight(30_000, {5000, 3}) + ~T[08:20:00.005] + iex> Time.from_seconds_after_midnight(-1) + ~T[23:59:59] + iex> Time.from_seconds_after_midnight(100_000) + ~T[03:46:40] + + """ + @doc since: "1.11.0" + @spec from_seconds_after_midnight( + integer(), + Calendar.microsecond(), + Calendar.calendar() + ) :: t + def from_seconds_after_midnight(seconds, microsecond \\ {0, 0}, calendar \\ Calendar.ISO) + when is_integer(seconds) do + seconds_in_day = Integer.mod(seconds, @seconds_per_day) + + {hour, minute, second, {_, _}} = + calendar.time_from_day_fraction({seconds_in_day, @seconds_per_day}) + + %Time{ + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + end + + @doc """ + Converts a `Time` struct to a number of seconds after midnight. + + The returned value is a two-element tuple with the number of seconds and microseconds. + + ## Examples + + iex> Time.to_seconds_after_midnight(~T[23:30:15]) + {84615, 0} + iex> Time.to_seconds_after_midnight(~N[2010-04-17 23:30:15.999]) + {84615, 999000} + + """ + @doc since: "1.11.0" + @spec to_seconds_after_midnight(Calendar.time()) :: {integer(), non_neg_integer()} + def to_seconds_after_midnight(%{microsecond: {microsecond, _precision}} = time) do + iso_days = {0, to_day_fraction(time)} + {Calendar.ISO.iso_days_to_unit(iso_days, :second), microsecond} + end + + @doc """ + Adds the `number` of `unit`s to the given `time`. + + This function accepts the `number` measured according to `Calendar.ISO`. + The time is returned in the same calendar as it was given in. + + Note the result value represents the time of day, meaning that it is cyclic, + for instance, it will never go over 24 hours for the ISO calendar. + + ## Examples + + iex> Time.add(~T[10:00:00], 27000) + ~T[17:30:00.000000] + iex> Time.add(~T[11:00:00.005], 2400) + ~T[11:40:00.005000] + iex> Time.add(~T[00:00:00], 86_399_999, :millisecond) + ~T[23:59:59.999000] + iex> Time.add(~T[17:10:05], 86400) + ~T[17:10:05.000000] + iex> Time.add(~T[23:00:00], -60) + ~T[22:59:00.000000] + + """ + @doc since: "1.6.0" + @spec add(Calendar.time(), integer, System.time_unit()) :: t + def add(%{calendar: calendar} = time, number, unit \\ :second) when is_integer(number) do + number = System.convert_time_unit(number, unit, :microsecond) + total = time_to_microseconds(time) + number + parts = Integer.mod(total, @parts_per_day) + {hour, minute, second, microsecond} = calendar.time_from_day_fraction({parts, @parts_per_day}) + + %Time{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + } + end + + defp time_to_microseconds(%{ + calendar: Calendar.ISO, + hour: 0, + minute: 0, + second: 0, + microsecond: {0, _} + }) do + 0 + end + + defp time_to_microseconds(time) do + iso_days = {0, to_day_fraction(time)} + Calendar.ISO.iso_days_to_unit(iso_days, :microsecond) + end + + @doc """ + Compares two time structs. + + Returns `:gt` if first time is later than the second + and `:lt` for vice versa. If the two times are equal + `:eq` is returned. + + ## Examples + + iex> Time.compare(~T[16:04:16], ~T[16:04:28]) + :lt + iex> Time.compare(~T[16:04:16], ~T[16:04:16]) + :eq + iex> Time.compare(~T[16:04:16.01], ~T[16:04:16.001]) + :gt + + This function can also be used to compare across more + complex calendar types by considering only the time fields: + + iex> Time.compare(~N[1900-01-01 16:04:16], ~N[2015-01-01 16:04:16]) + :eq + iex> Time.compare(~N[2015-01-01 16:04:16], ~N[2015-01-01 16:04:28]) + :lt + iex> Time.compare(~N[2015-01-01 16:04:16.01], ~N[2000-01-01 16:04:16.001]) + :gt + + """ + @doc since: "1.4.0" + @spec compare(Calendar.time(), Calendar.time()) :: :lt | :eq | :gt + def compare(%{calendar: calendar} = time1, %{calendar: calendar} = time2) do + %{hour: hour1, minute: minute1, second: second1, microsecond: {microsecond1, _}} = time1 + %{hour: hour2, minute: minute2, second: second2, microsecond: {microsecond2, _}} = time2 + + case {{hour1, minute1, second1, microsecond1}, {hour2, minute2, second2, microsecond2}} do + {first, second} when first > second -> :gt + {first, second} when first < second -> :lt + _ -> :eq + end + end + + def compare(time1, time2) do + {parts1, ppd1} = to_day_fraction(time1) + {parts2, ppd2} = to_day_fraction(time2) + + case {parts1 * ppd2, parts2 * ppd1} do + {first, second} when first > second -> :gt + {first, second} when first < second -> :lt + _ -> :eq + end + end + + @doc """ + Converts given `time` to a different calendar. + + Returns `{:ok, time}` if the conversion was successful, + or `{:error, reason}` if it was not, for some reason. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10,000 years to the current Gregorian + year: + + iex> Time.convert(~T[13:30:15], Calendar.Holocene) + {:ok, %Time{calendar: Calendar.Holocene, hour: 13, minute: 30, second: 15, microsecond: {0, 0}}} + + """ + @doc since: "1.5.0" + @spec convert(Calendar.time(), Calendar.calendar()) :: {:ok, t} | {:error, atom} + + # Keep it multiline for proper function clause errors. + def convert( + %{ + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + }, + calendar + ) do + time = %Time{ + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + } + + {:ok, time} + end + + def convert(%{microsecond: {_, precision}} = time, calendar) do + {hour, minute, second, {microsecond, _}} = + time + |> to_day_fraction() + |> calendar.time_from_day_fraction() + + time = %Time{ + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: {microsecond, precision} + } + + {:ok, time} + end + + @doc """ + Similar to `Time.convert/2`, but raises an `ArgumentError` + if the conversion between the two calendars is not possible. + + ## Examples + + Imagine someone implements `Calendar.Holocene`, a calendar based on the + Gregorian calendar that adds exactly 10,000 years to the current Gregorian + year: + + iex> Time.convert!(~T[13:30:15], Calendar.Holocene) + %Time{calendar: Calendar.Holocene, hour: 13, minute: 30, second: 15, microsecond: {0, 0}} + + """ + @doc since: "1.5.0" + @spec convert!(Calendar.time(), Calendar.calendar()) :: t + def convert!(time, calendar) do + case convert(time, calendar) do + {:ok, value} -> + value + + {:error, reason} -> + raise ArgumentError, + "cannot convert #{inspect(time)} to target calendar #{inspect(calendar)}, " <> + "reason: #{inspect(reason)}" + end + end + + @doc """ + Returns the difference between two times, considering only the hour, minute, + second and microsecond. + + As with the `compare/2` function both `Time` structs and other structures + containing time can be used. If for instance a `NaiveDateTime` or `DateTime` + is passed, only the hour, minute, second, and microsecond is considered. Any + additional information about a date or time zone is ignored when calculating + the difference. + + The answer can be returned in any `unit` available from + `t:System.time_unit/0`. If the first time value is earlier than + the second, a negative number is returned. + + This function returns the difference in seconds where seconds + are measured according to `Calendar.ISO`. + + ## Examples + + iex> Time.diff(~T[00:29:12], ~T[00:29:10]) + 2 + + # When passing a `NaiveDateTime` the date part is ignored. + iex> Time.diff(~N[2017-01-01 00:29:12], ~T[00:29:10]) + 2 + + # Two `NaiveDateTime` structs could have big differences in the date + # but only the time part is considered. + iex> Time.diff(~N[2017-01-01 00:29:12], ~N[1900-02-03 00:29:10]) + 2 + + iex> Time.diff(~T[00:29:12], ~T[00:29:10], :microsecond) + 2_000_000 + iex> Time.diff(~T[00:29:10], ~T[00:29:12], :microsecond) + -2_000_000 + + """ + @doc since: "1.5.0" + @spec diff(Calendar.time(), Calendar.time(), System.time_unit()) :: integer + def diff(time1, time2, unit \\ :second) + + def diff( + %{ + calendar: Calendar.ISO, + hour: hour1, + minute: minute1, + second: second1, + microsecond: {microsecond1, @parts_per_day} + }, + %{ + calendar: Calendar.ISO, + hour: hour2, + minute: minute2, + second: second2, + microsecond: {microsecond2, @parts_per_day} + }, + unit + ) do + total = + (hour1 - hour2) * 3_600_000_000 + (minute1 - minute2) * 60_000_000 + + (second1 - second2) * 1_000_000 + (microsecond1 - microsecond2) + + System.convert_time_unit(total, :microsecond, unit) + end + + def diff(time1, time2, unit) do + fraction1 = to_day_fraction(time1) + fraction2 = to_day_fraction(time2) + + Calendar.ISO.iso_days_to_unit({0, fraction1}, unit) - + Calendar.ISO.iso_days_to_unit({0, fraction2}, unit) + end + + @doc """ + Returns the given time with the microsecond field truncated to the given + precision (`:microsecond`, `millisecond` or `:second`). + + The given time is returned unchanged if it already has lower precision than + the given precision. + + ## Examples + + iex> Time.truncate(~T[01:01:01.123456], :microsecond) + ~T[01:01:01.123456] + + iex> Time.truncate(~T[01:01:01.123456], :millisecond) + ~T[01:01:01.123] + + iex> Time.truncate(~T[01:01:01.123456], :second) + ~T[01:01:01] + + """ + @doc since: "1.6.0" + @spec truncate(t(), :microsecond | :millisecond | :second) :: t() + def truncate(%Time{microsecond: microsecond} = time, precision) do + %{time | microsecond: Calendar.truncate(microsecond, precision)} + end + + ## Helpers + + defp to_day_fraction(%{ + hour: hour, + minute: minute, + second: second, + microsecond: {_, _} = microsecond, + calendar: calendar + }) do + calendar.time_to_day_fraction(hour, minute, second, microsecond) + end + + defimpl String.Chars do + def to_string(time) do + %{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + } = time + + calendar.time_to_string(hour, minute, second, microsecond) + end + end + + defimpl Inspect do + def inspect(time, _) do + %{ + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: calendar + } = time + + "~T[" <> + calendar.time_to_string(hour, minute, second, microsecond) <> suffix(calendar) <> "]" + end + + defp suffix(Calendar.ISO), do: "" + defp suffix(calendar), do: " " <> inspect(calendar) + end +end diff --git a/lib/elixir/lib/calendar/time_zone_database.ex b/lib/elixir/lib/calendar/time_zone_database.ex new file mode 100644 index 00000000000..ac597f9ed3f --- /dev/null +++ b/lib/elixir/lib/calendar/time_zone_database.ex @@ -0,0 +1,96 @@ +defmodule Calendar.TimeZoneDatabase do + @moduledoc """ + This module defines a behaviour for providing time zone data. + + IANA provides time zone data that includes data about different + UTC offsets and standard offsets for time zones. + """ + + @typedoc """ + A period where a certain combination of UTC offset, standard offset and zone + abbreviation is in effect. + + For instance one period could be the summer of 2018 in "Europe/London" where summer time / + daylight saving time is in effect and lasts from spring to autumn. At autumn the `std_offset` + changes along with the `zone_abbr` so a different period is needed during winter. + """ + @type time_zone_period :: %{ + optional(any) => any, + utc_offset: Calendar.utc_offset(), + std_offset: Calendar.std_offset(), + zone_abbr: Calendar.zone_abbr() + } + + @typedoc """ + Limit for when a certain time zone period begins or ends. + + A beginning is inclusive. An ending is exclusive. Eg. if a period is from + 2015-03-29 01:00:00 and until 2015-10-25 01:00:00, the period includes and + begins from the beginning of 2015-03-29 01:00:00 and lasts until just before + 2015-10-25 01:00:00. + + A beginning or end for certain periods are infinite. For instance the latest + period for time zones without DST or plans to change. However for the purpose + of this behaviour they are only used for gaps in wall time where the needed + period limits are at a certain time. + """ + @type time_zone_period_limit :: Calendar.naive_datetime() + + @doc """ + Time zone period for a point in time in UTC for a specific time zone. + + Takes a time zone name and a point in time for UTC and returns a + `time_zone_period` for that point in time. + """ + @doc since: "1.8.0" + @callback time_zone_period_from_utc_iso_days(Calendar.iso_days(), Calendar.time_zone()) :: + {:ok, time_zone_period} + | {:error, :time_zone_not_found | :utc_only_time_zone_database} + + @doc """ + Possible time zone periods for a certain time zone and wall clock date and time. + + When the provided `datetime` is ambiguous a tuple with `:ambiguous` and two possible + periods. The periods in the list are sorted with the first element being the one that begins first. + + When the provided `datetime` is in a gap - for instance during the "spring forward" when going + from winter time to summer time, a tuple with `:gap` and two periods with limits are returned + in a nested tuple. The first nested two-tuple is the period before the gap and a naive datetime + with a limit for when the period ends (wall time). The second nested two-tuple is the period + just after the gap and a datetime (wall time) for when the period begins just after the gap. + + If there is only a single possible period for the provided `datetime`, then a tuple with `:ok` + and the `time_zone_period` is returned. + """ + @doc since: "1.8.0" + @callback time_zone_periods_from_wall_datetime(Calendar.naive_datetime(), Calendar.time_zone()) :: + {:ok, time_zone_period} + | {:ambiguous, time_zone_period, time_zone_period} + | {:gap, {time_zone_period, time_zone_period_limit}, + {time_zone_period, time_zone_period_limit}} + | {:error, :time_zone_not_found | :utc_only_time_zone_database} +end + +defmodule Calendar.UTCOnlyTimeZoneDatabase do + @moduledoc """ + Built-in time zone database that works only in Etc/UTC. + + For all other time zones, it returns `{:error, :utc_only_time_zone_database}`. + """ + + @behaviour Calendar.TimeZoneDatabase + + @impl true + def time_zone_period_from_utc_iso_days(_, "Etc/UTC"), + do: {:ok, %{std_offset: 0, utc_offset: 0, zone_abbr: "UTC"}} + + def time_zone_period_from_utc_iso_days(_, _), + do: {:error, :utc_only_time_zone_database} + + @impl true + def time_zone_periods_from_wall_datetime(_, "Etc/UTC"), + do: {:ok, %{std_offset: 0, utc_offset: 0, zone_abbr: "UTC"}} + + def time_zone_periods_from_wall_datetime(_, _), + do: {:error, :utc_only_time_zone_database} +end diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index 21ec4ac78d6..8da2a7ba412 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -1,274 +1,1204 @@ defmodule Code do - @moduledoc """ - Utilities for managing code compilation, code evaluation and code loading. + @moduledoc ~S""" + Utilities for managing code compilation, code evaluation, and code loading. - This module complements [Erlang's code module](http://www.erlang.org/doc/man/code.html) - to add behaviour which is specific to Elixir. + This module complements Erlang's [`:code` module](`:code`) + to add behaviour which is specific to Elixir. For functions to + manipulate Elixir's AST (rather than evaluating it), see the + `Macro` module. + + ## Working with files + + This module contains three functions for compiling and evaluating files. + Here is a summary of them and their behaviour: + + * `require_file/2` - compiles a file and tracks its name. It does not + compile the file again if it has been previously required. + + * `compile_file/2` - compiles a file without tracking its name. Compiles the + file multiple times when invoked multiple times. + + * `eval_file/2` - evaluates the file contents without tracking its name. It + returns the result of the last expression in the file, instead of the modules + defined in it. Evaluated files do not trigger the compilation tracers described + in the next section. + + In a nutshell, the first must be used when you want to keep track of the files + handled by the system, to avoid the same file from being compiled multiple + times. This is common in scripts. + + `compile_file/2` must be used when you are interested in the modules defined in a + file, without tracking. `eval_file/2` should be used when you are interested in + the result of evaluating the file rather than the modules it defines. + + The functions above work with Elixir source. If you want to work + with modules compiled to bytecode, which have the `.beam` extension + and are typically found below the _build directory of a Mix project, + see the functions in Erlang's [`:code`](`:code`) module. + + ## Code loading on the Erlang VM + + Erlang has two modes to load code: interactive and embedded. + + By default, the Erlang VM runs in interactive mode, where modules + are loaded as needed. In embedded mode the opposite happens, as all + modules need to be loaded upfront or explicitly. + + You can use `ensure_loaded/1` (as well as `ensure_loaded?/1` and + `ensure_loaded!/1`) to check if a module is loaded before using it and + act. + + ## `ensure_compiled/1` and `ensure_compiled!/1` + + Elixir also includes `ensure_compiled/1` and `ensure_compiled!/1` + functions that are a superset of `ensure_loaded/1`. + + Since Elixir's compilation happens in parallel, in some situations + you may need to use a module that was not yet compiled, therefore + it can't even be loaded. + + When invoked, `ensure_compiled/1` and `ensure_compiled!/1` halt the + compilation of the caller until the module becomes available. Note + the distinction between `ensure_compiled/1` and `ensure_compiled!/1` + is important: if you are using `ensure_compiled!/1`, you are + indicating to the compiler that you can only continue if said module + is available. + + If you are using `Code.ensure_compiled/1`, you are implying you may + continue without the module and therefore Elixir may return + `{:error, :unavailable}` for cases where the module is not yet available + (but may be available later on). + + For those reasons, developers must typically use `Code.ensure_compiled!/1`. + In particular, do not do this: + + case Code.ensure_compiled(module) do + {:module, _} -> module + {:error, _} -> raise ... + end + + Finally, note you only need `ensure_compiled!/1` to check for modules + being defined within the same project. It does not apply to modules from + dependencies as dependencies are always compiled upfront. + + In most cases, `ensure_loaded/1` is enough. `ensure_compiled!/1` + must be used in rare cases, usually involving macros that need to + invoke a module for callback information. The use of `ensure_compiled/1` + is even less likely. + + ## Compilation tracers + + Elixir supports compilation tracers, which allows modules to observe constructs + handled by the Elixir compiler when compiling files. A tracer is a module + that implements the `trace/2` function. The function receives the event name + as first argument and `Macro.Env` as second and it must return `:ok`. It is + very important for a tracer to do as little work as possible synchronously + and dispatch the bulk of the work to a separate process. **Slow tracers will + slow down compilation**. + + You can configure your list of tracers via `put_compiler_option/2`. The + following events are available to tracers: + + * `:start` - (since v1.11.0) invoked whenever the compiler starts to trace + a new lexical context, such as a new file. Keep in mind the compiler runs + in parallel, so multiple files may invoke `:start` and run at the same + time. The value of the `lexical_tracker` of the macro environment, albeit + opaque, can be used to uniquely identify the environment. + + * `:stop` - (since v1.11.0) invoked whenever the compiler stops tracing a + new lexical context, such as a new file. + + * `{:import, meta, module, opts}` - traced whenever `module` is imported. + `meta` is the import AST metadata and `opts` are the import options. + + * `{:imported_function, meta, module, name, arity}` and + `{:imported_macro, meta, module, name, arity}` - traced whenever an + imported function or macro is invoked. `meta` is the call AST metadata, + `module` is the module the import is from, followed by the `name` and `arity` + of the imported function/macro. + + * `{:alias, meta, alias, as, opts}` - traced whenever `alias` is aliased + to `as`. `meta` is the alias AST metadata and `opts` are the alias options. + + * `{:alias_expansion, meta, as, alias}` traced whenever there is an alias + expansion for a previously defined `alias`, i.e. when the user writes `as` + which is expanded to `alias`. `meta` is the alias expansion AST metadata. + + * `{:alias_reference, meta, module}` - traced whenever there is an alias + in the code, i.e. whenever the user writes `MyModule.Foo.Bar` in the code, + regardless if it was expanded or not. + + * `{:require, meta, module, opts}` - traced whenever `module` is required. + `meta` is the require AST metadata and `opts` are the require options. + + * `{:struct_expansion, meta, module, keys}` - traced whenever `module`'s struct + is expanded. `meta` is the struct AST metadata and `keys` are the keys being + used by expansion + + * `{:remote_function, meta, module, name, arity}` and + `{:remote_macro, meta, module, name, arity}` - traced whenever a remote + function or macro is referenced. `meta` is the call AST metadata, `module` + is the invoked module, followed by the `name` and `arity`. + + * `{:local_function, meta, name, arity}` and + `{:local_macro, meta, name, arity}` - traced whenever a local + function or macro is referenced. `meta` is the call AST metadata, followed by + the `name` and `arity`. + + * `{:compile_env, app, path, return}` - traced whenever `Application.compile_env/3` + or `Application.compile_env!/2` are called. `app` is an atom, `path` is a list + of keys to traverse in the application environment and `return` is either + `{:ok, value}` or `:error`. + + * `{:on_module, bytecode, :none}` - (since v1.11.0) traced whenever a module + is defined. This is equivalent to the `@after_compile` callback and invoked + after any `@after_compile` in the given module. The third element is currently + `:none` but it may provide more metadata in the future. It is best to ignore + it at the moment. Note that `Module` functions expecting not yet compiled modules + (such as `Module.definitions_in/1`) are still available at the time this event + is emitted. + + The `:tracers` compiler option can be combined with the `:parser_options` + compiler option to enrich the metadata of the traced events above. + + New events may be added at any time in the future, therefore it is advised + for the `trace/2` function to have a "catch-all" clause. + + Below is an example tracer that prints all remote function invocations: + + defmodule MyTracer do + def trace({:remote_function, _meta, module, name, arity}, env) do + IO.puts "#{env.file}:#{env.line} #{inspect(module)}.#{name}/#{arity}" + :ok + end + + def trace(_event, _env) do + :ok + end + end + """ + + @typedoc """ + A list with all variable bindings. + + The binding keys are usually atoms, but they may be a tuple for variables + defined in a different context. """ + @type binding :: [{atom() | tuple(), any}] + + @boolean_compiler_options [ + :docs, + :debug_info, + :ignore_module_conflict, + :relative_paths, + :warnings_as_errors + ] + + @list_compiler_options [:no_warn_undefined, :tracers, :parser_options] + + @available_compiler_options @boolean_compiler_options ++ @list_compiler_options @doc """ - List all loaded files. + Lists all required files. + + ## Examples + + Code.require_file("../eex/test/eex_test.exs") + List.first(Code.required_files()) =~ "eex_test.exs" + #=> true + """ + @doc since: "1.7.0" + @spec required_files() :: [binary] + def required_files do + :elixir_code_server.call(:required) + end + + @deprecated "Use Code.required_files/0 instead" + @doc false def loaded_files do - :elixir_code_server.call :loaded + required_files() + end + + @doc false + @deprecated "Use Code.Fragment.cursor_context/2 instead" + def cursor_context(code, options \\ []) do + Code.Fragment.cursor_context(code, options) end @doc """ - Remove files from the loaded files list. + Removes files from the required files list. The modules defined in the file are not removed; calling this function only removes them from the list, allowing them to be required again. + + The list of files is managed per Erlang VM node. + + ## Examples + + # Require EEx test code + Code.require_file("../eex/test/eex_test.exs") + + # Now unrequire all files + Code.unrequire_files(Code.required_files()) + + # Note that modules are still available + function_exported?(EExTest.Compiled, :before_compile, 0) + #=> true + """ + @doc since: "1.7.0" + @spec unrequire_files([binary]) :: :ok + def unrequire_files(files) when is_list(files) do + :elixir_code_server.cast({:unrequire_files, files}) + end + + @deprecated "Use Code.unrequire_files/1 instead" + @doc false def unload_files(files) do - :elixir_code_server.cast {:unload_files, files} + unrequire_files(files) end @doc """ - Append a path to the Erlang VM code path. + Appends a path to the end of the Erlang VM code path list. + + This is the list of directories the Erlang VM uses for + finding module code. The list of files is managed per Erlang + VM node. The path is expanded with `Path.expand/1` before being appended. + If this path does not exist, an error is returned. + + ## Examples + + Code.append_path(".") + #=> true + + Code.append_path("/does_not_exist") + #=> {:error, :bad_directory} + """ + @spec append_path(Path.t()) :: true | {:error, :bad_directory} def append_path(path) do - :code.add_pathz(to_char_list(Path.expand path)) + :code.add_pathz(to_charlist(Path.expand(path))) end @doc """ - Prepend a path to the Erlang VM code path. + Prepends a path to the beginning of the Erlang VM code path list. + + This is the list of directories the Erlang VM uses for finding + module code. The list of files is managed per Erlang VM node. The path is expanded with `Path.expand/1` before being prepended. + If this path does not exist, an error is returned. + + ## Examples + + Code.prepend_path(".") + #=> true + + Code.prepend_path("/does_not_exist") + #=> {:error, :bad_directory} + """ + @spec prepend_path(Path.t()) :: true | {:error, :bad_directory} def prepend_path(path) do - :code.add_patha(to_char_list(Path.expand path)) + :code.add_patha(to_charlist(Path.expand(path))) end @doc """ - Delete a path from the Erlang VM code path. + Deletes a path from the Erlang VM code path list. + + This is the list of directories the Erlang VM uses for finding + module code. The list of files is managed per Erlang VM node. + + The path is expanded with `Path.expand/1` before being deleted. If the + path does not exist, this function returns `false`. + + ## Examples + + Code.prepend_path(".") + Code.delete_path(".") + #=> true + + Code.delete_path("/does_not_exist") + #=> false - The path is expanded with `Path.expand/1` before being deleted. """ + @spec delete_path(Path.t()) :: boolean def delete_path(path) do - :code.del_path(to_char_list(Path.expand path)) + :code.del_path(to_charlist(Path.expand(path))) end @doc """ - Evaluate the contents given by `string`. + Evaluates the contents given by `string`. - The `binding` argument is a keyword list of variable bindings. + The `binding` argument is a list of variable bindings. The `opts` argument is a keyword list of environment options. - Those options can be: + **Warning**: `string` can be any Elixir code and will be executed with + the same privileges as the Erlang VM: this means that such code could + compromise the machine (for example by executing system commands). + Don't use `eval_string/3` with untrusted input (such as strings coming + from the network). - * `:file` - the file to be considered in the evaluation - * `:line` - the line on which the script starts - * `:delegate_locals_to` - delegate local calls to the given module, - the default is to not delegate - - Additionally, the following scope values can be configured: - - * `:aliases` - a list of tuples with the alias and its target + ## Options - * `:requires` - a list of modules required + Options can be: - * `:functions` - a list of tuples where the first element is a module - and the second a list of imported function names and arity; the list - of function names and arity must be sorted + * `:file` - the file to be considered in the evaluation - * `:macros` - a list of tuples where the first element is a module - and the second a list of imported macro names and arity; the list - of function names and arity must be sorted + * `:line` - the line on which the script starts - Notice that setting any of the values above overrides Elixir's default - values. For example, setting `:requires` to `[]`, will no longer - automatically require the `Kernel` module; in the same way setting - `:macros` will no longer auto-import `Kernel` macros like `if`, `case`, - etc. + Additionally, you may also pass an environment as second argument, + so the evaluation happens within that environment. However, if the evaluated + code requires or compiles another file, the environment given to this function + will not apply to said files. - Returns a tuple of the form `{value, binding}`, - where `value` is the value returned from evaluating `string`. - If an error occurs while evaluating `string` an exception will be raised. + Returns a tuple of the form `{value, binding}`, where `value` is the value + returned from evaluating `string`. If an error occurs while evaluating + `string` an exception will be raised. - `binding` is a keyword list with the value of all variable bindings - after evaluating `string`. The binding key is usually an atom, but it - may be a tuple for variables defined in a different context. + `binding` is a list with all variable bindings after evaluating `string`. + The binding keys are usually atoms, but they may be a tuple for variables + defined in a different context. ## Examples - iex> Code.eval_string("a + b", [a: 1, b: 2], file: __ENV__.file, line: __ENV__.line) - {3, [a: 1, b: 2]} - - iex> Code.eval_string("c = a + b", [a: 1, b: 2], __ENV__) - {3, [a: 1, b: 2, c: 3]} - - iex> Code.eval_string("a = a + b", [a: 1, b: 2]) - {3, [a: 3, b: 2]} - - For convenience, you can pass `__ENV__` as the `opts` argument and + iex> {result, binding} = Code.eval_string("a + b", [a: 1, b: 2], file: __ENV__.file, line: __ENV__.line) + iex> result + 3 + iex> Enum.sort(binding) + [a: 1, b: 2] + + iex> {result, binding} = Code.eval_string("c = a + b", [a: 1, b: 2], __ENV__) + iex> result + 3 + iex> Enum.sort(binding) + [a: 1, b: 2, c: 3] + + iex> {result, binding} = Code.eval_string("a = a + b", [a: 1, b: 2]) + iex> result + 3 + iex> Enum.sort(binding) + [a: 3, b: 2] + + For convenience, you can pass `__ENV__/0` as the `opts` argument and all imports, requires and aliases defined in the current environment will be automatically carried over: - iex> Code.eval_string("a + b", [a: 1, b: 2], __ENV__) - {3, [a: 1, b: 2]} + iex> {result, binding} = Code.eval_string("a + b", [a: 1, b: 2], __ENV__) + iex> result + 3 + iex> Enum.sort(binding) + [a: 1, b: 2] """ + @spec eval_string(List.Chars.t(), binding, Macro.Env.t() | keyword) :: {term, binding} def eval_string(string, binding \\ [], opts \\ []) def eval_string(string, binding, %Macro.Env{} = env) do - {value, binding, _env, _scope} = :elixir.eval to_char_list(string), binding, Map.to_list(env) - {value, binding} + validated_eval_string(string, binding, env) end def eval_string(string, binding, opts) when is_list(opts) do - validate_eval_opts(opts) - {value, binding, _env, _scope} = :elixir.eval to_char_list(string), binding, opts + validated_eval_string(string, binding, opts) + end + + defp validated_eval_string(string, binding, opts_or_env) do + %{line: line, file: file} = env = env_for_eval(opts_or_env) + forms = :elixir.string_to_quoted!(to_charlist(string), line, 1, file, []) + {value, binding, _env} = eval_verify(:eval_forms, forms, binding, env) {value, binding} end + defp eval_verify(fun, forms, binding, env) do + Module.ParallelChecker.verify(fn -> + previous = :erlang.get(:elixir_module_binaries) + + try do + Process.put(:elixir_module_binaries, []) + result = apply(:elixir, fun, [forms, binding, env]) + {result, Enum.map(Process.get(:elixir_module_binaries), &elem(&1, 1))} + after + if previous == :undefined do + :erlang.erase(:elixir_module_binaries) + else + :erlang.put(:elixir_module_binaries, previous) + end + end + end) + end + + @doc ~S""" + Formats the given code `string`. + + The formatter receives a string representing Elixir code and + returns iodata representing the formatted code according to + pre-defined rules. + + ## Options + + * `:file` - the file which contains the string, used for error + reporting + + * `:line` - the line the string starts, used for error reporting + + * `:line_length` - the line length to aim for when formatting + the document. Defaults to 98. Note this value is used as + guideline but there are situations where it is not enforced. + See the "Line length" section below for more information + + * `:locals_without_parens` - a keyword list of name and arity + pairs that should be kept without parens whenever possible. + The arity may be the atom `:*`, which implies all arities of + that name. The formatter already includes a list of functions + and this option augments this list. + + * `:force_do_end_blocks` (since v1.9.0) - when `true`, converts all + inline usages of `do: ...`, `else: ...` and friends into `do`-`end` + blocks. Defaults to `false`. Note that this option is convergent: + once you set it to `true`, **all keywords** will be converted. + If you set it to `false` later on, `do`-`end` blocks won't be + converted back to keywords. + + * `:normalize_bitstring_modifiers` (since v1.14.0) - when `true`, + removes unnecessary parentheses in known bitstring + [modifiers](`<<>>/1`), for example `<>` + becomes `<>`, or adds parentheses for custom + modifiers, where `<>` becomes `<>`. + Defaults to `true`. This option changes the AST. + + ## Design principles + + The formatter was designed under three principles. + + First, the formatter never changes the semantics of the code. + This means the input AST and the output AST are almost always equivalent. + The only cases where the formatter will change the AST is when the input AST + would cause *compiler warnings* and the output AST won't. The cases where + the formatter changes the AST can be disabled through formatting options + if desired. + + The second principle is to provide as little configuration as possible. + This eases the formatter adoption by removing contention points while + making sure a single style is followed consistently by the community as + a whole. + + The formatter does not hard code names. The formatter will not behave + specially because a function is named `defmodule`, `def`, or the like. This + principle mirrors Elixir's goal of being an extensible language where + developers can extend the language with new constructs as if they were + part of the language. When it is absolutely necessary to change behaviour + based on the name, this behaviour should be configurable, such as the + `:locals_without_parens` option. + + ## Running the formatter + + The formatter attempts to fit the most it can on a single line and + introduces line breaks wherever possible when it cannot. + + In some cases, this may lead to undesired formatting. Therefore, **some + code generated by the formatter may not be aesthetically pleasing and + may require explicit intervention from the developer**. That's why we + do not recommend to run the formatter blindly in an existing codebase. + Instead you should format and sanity check each formatted file. + + For example, the formatter may break a long function definition over + multiple clauses: + + def my_function( + %User{name: name, age: age, ...}, + arg1, + arg2 + ) do + ... + end + + While the code above is completely valid, you may prefer to match on + the struct variables inside the function body in order to keep the + definition on a single line: + + def my_function(%User{} = user, arg1, arg2) do + %{name: name, age: age, ...} = user + ... + end + + In some situations, you can use the fact the formatter does not generate + elegant code as a hint for refactoring. Take this code: + + def board?(board_id, %User{} = user, available_permissions, required_permissions) do + Tracker.OrganizationMembers.user_in_organization?(user.id, board.organization_id) and + required_permissions == Enum.to_list(MapSet.intersection(MapSet.new(required_permissions), MapSet.new(available_permissions))) + end + + The code above has very long lines and running the formatter is not going + to address this issue. In fact, the formatter may make it more obvious that + you have complex expressions: + + def board?(board_id, %User{} = user, available_permissions, required_permissions) do + Tracker.OrganizationMembers.user_in_organization?(user.id, board.organization_id) and + required_permissions == + Enum.to_list( + MapSet.intersection( + MapSet.new(required_permissions), + MapSet.new(available_permissions) + ) + ) + end + + Take such cases as a suggestion that your code should be refactored: + + def board?(board_id, %User{} = user, available_permissions, required_permissions) do + Tracker.OrganizationMembers.user_in_organization?(user.id, board.organization_id) and + matching_permissions?(required_permissions, available_permissions) + end + + defp matching_permissions?(required_permissions, available_permissions) do + intersection = + required_permissions + |> MapSet.new() + |> MapSet.intersection(MapSet.new(available_permissions)) + |> Enum.to_list() + + required_permissions == intersection + end + + To sum it up: since the formatter cannot change the semantics of your + code, sometimes it is necessary to tweak or refactor the code to get + optimal formatting. To help better understand how to control the formatter, + we describe in the next sections the cases where the formatter keeps the + user encoding and how to control multiline expressions. + + ## Line length + + Another point about the formatter is that the `:line_length` configuration + is a guideline. In many cases, it is not possible for the formatter to break + your code apart, which means it will go over the line length. For example, + if you have a long string: + + "this is a very long string that will go over the line length" + + The formatter doesn't know how to break it apart without changing the + code underlying syntax representation, so it is up to you to step in: + + "this is a very long string " <> + "that will go over the line length" + + The string concatenation makes the code fit on a single line and also + gives more options to the formatter. + + This may also appear in do/end blocks, where the `do` keyword (or `->`) + may go over the line length because there is no opportunity for the + formatter to introduce a line break in a readable way. For example, + if you do: + + case very_long_expression() do + end + + And only the `do` keyword is above the line length, Elixir **will not** + emit this: + + case very_long_expression() + do + end + + So it prefers to not touch the line at all and leave `do` above the + line limit. + + ## Keeping user's formatting + + The formatter respects the input format in some cases. Those are + listed below: + + * Insignificant digits in numbers are kept as is. The formatter + however always inserts underscores for decimal numbers with more + than 5 digits and converts hexadecimal digits to uppercase + + * Strings, charlists, atoms and sigils are kept as is. No character + is automatically escaped or unescaped. The choice of delimiter is + also respected from the input + + * Newlines inside blocks are kept as in the input except for: + 1) expressions that take multiple lines will always have an empty + line before and after and 2) empty lines are always squeezed + together into a single empty line + + * The choice between `:do` keyword and `do`-`end` blocks is left + to the user + + * Lists, tuples, bitstrings, maps, structs and function calls will be + broken into multiple lines if they are followed by a newline in the + opening bracket and preceded by a new line in the closing bracket + + * Newlines before certain operators (such as the pipeline operators) + and before other operators (such as comparison operators) + + The behaviours above are not guaranteed. We may remove or add new + rules in the future. The goal of documenting them is to provide better + understanding on what to expect from the formatter. + + ### Multi-line lists, maps, tuples, and the like + + You can force lists, tuples, bitstrings, maps, structs and function + calls to have one entry per line by adding a newline after the opening + bracket and a new line before the closing bracket lines. For example: + + [ + foo, + bar + ] + + If there are no newlines around the brackets, then the formatter will + try to fit everything on a single line, such that the snippet below + + [foo, + bar] + + will be formatted as + + [foo, bar] + + You can also force function calls and keywords to be rendered on multiple + lines by having each entry on its own line: + + defstruct name: nil, + age: 0 + + The code above will be kept with one keyword entry per line by the + formatter. To avoid that, just squash everything into a single line. + + ### Parens and no parens in function calls + + Elixir has two syntaxes for function calls. With parens and no parens. + By default, Elixir will add parens to all calls except for: + + 1. calls that have `do`-`end` blocks + 2. local calls without parens where the name and arity of the local + call is also listed under `:locals_without_parens` (except for + calls with arity 0, where the compiler always require parens) + + The choice of parens and no parens also affects indentation. When a + function call with parens doesn't fit on the same line, the formatter + introduces a newline around parens and indents the arguments with two + spaces: + + some_call( + arg1, + arg2, + arg3 + ) + + On the other hand, function calls without parens are always indented + by the function call length itself, like this: + + some_call arg1, + arg2, + arg3 + + If the last argument is a data structure, such as maps and lists, and + the beginning of the data structure fits on the same line as the function + call, then no indentation happens, this allows code like this: + + Enum.reduce(some_collection, initial_value, fn element, acc -> + # code + end) + + some_function_without_parens %{ + foo: :bar, + baz: :bat + } + + ## Code comments + + The formatter also handles code comments in a way to guarantee a space + is always added between the beginning of the comment (#) and the next + character. + + The formatter also extracts all trailing comments to their previous line. + For example, the code below + + hello #world + + will be rewritten to + + # world + hello + + Because code comments are handled apart from the code representation (AST), + there are some situations where code comments are seen as ambiguous by the + code formatter. For example, the comment in the anonymous function below + + fn + arg1 -> + body1 + # comment + + arg2 -> + body2 + end + + and in this one + + fn + arg1 -> + body1 + + # comment + arg2 -> + body2 + end + + are considered equivalent (the nesting is discarded alongside most of + user formatting). In such cases, the code formatter will always format to + the latter. + + ## Newlines + + The formatter converts all newlines in code from `\r\n` to `\n`. + """ + @doc since: "1.6.0" + @spec format_string!(binary, keyword) :: iodata + def format_string!(string, opts \\ []) when is_binary(string) and is_list(opts) do + line_length = Keyword.get(opts, :line_length, 98) + + to_quoted_opts = + [ + unescape: false, + warn_on_unnecessary_quotes: false, + literal_encoder: &{:ok, {:__block__, &2, [&1]}}, + token_metadata: true, + emit_warnings: false + ] ++ opts + + {forms, comments} = string_to_quoted_with_comments!(string, to_quoted_opts) + to_algebra_opts = [comments: comments] ++ opts + doc = Code.Formatter.to_algebra(forms, to_algebra_opts) + Inspect.Algebra.format(doc, line_length) + end + + @doc """ + Formats a file. + + See `format_string!/2` for more information on code formatting and + available options. + """ + @doc since: "1.6.0" + @spec format_file!(binary, keyword) :: iodata + def format_file!(file, opts \\ []) when is_binary(file) and is_list(opts) do + string = File.read!(file) + formatted = format_string!(string, [file: file, line: 1] ++ opts) + [formatted, ?\n] + end + @doc """ - Evaluate the quoted contents. + Evaluates the quoted contents. + + **Warning**: Calling this function inside a macro is considered bad + practice as it will attempt to evaluate runtime values at compile time. + Macro arguments are typically transformed by unquoting them into the + returned quoted expressions (instead of evaluated). - See `eval_string/3` for a description of arguments and return values. + See `eval_string/3` for a description of `binding` and `opts`. ## Examples iex> contents = quote(do: var!(a) + var!(b)) - iex> Code.eval_quoted(contents, [a: 1, b: 2], file: __ENV__.file, line: __ENV__.line) - {3, [a: 1, b: 2]} + iex> {result, binding} = Code.eval_quoted(contents, [a: 1, b: 2], file: __ENV__.file, line: __ENV__.line) + iex> result + 3 + iex> Enum.sort(binding) + [a: 1, b: 2] - For convenience, you can pass `__ENV__` as the `opts` argument and + For convenience, you can pass `__ENV__/0` as the `opts` argument and all options will be automatically extracted from the current environment: iex> contents = quote(do: var!(a) + var!(b)) - iex> Code.eval_quoted(contents, [a: 1, b: 2], __ENV__) - {3, [a: 1, b: 2]} + iex> {result, binding} = Code.eval_quoted(contents, [a: 1, b: 2], __ENV__) + iex> result + 3 + iex> Enum.sort(binding) + [a: 1, b: 2] """ - def eval_quoted(quoted, binding \\ [], opts \\ []) - - def eval_quoted(quoted, binding, %Macro.Env{} = env) do - {value, binding, _env, _scope} = :elixir.eval_quoted quoted, binding, Map.to_list(env) + @spec eval_quoted(Macro.t(), binding, Macro.Env.t() | keyword) :: {term, binding} + def eval_quoted(quoted, binding \\ [], env_or_opts \\ []) do + {value, binding, _env} = eval_verify(:eval_quoted, quoted, binding, env_for_eval(env_or_opts)) {value, binding} end - def eval_quoted(quoted, binding, opts) when is_list(opts) do - validate_eval_opts(opts) - {value, binding, _env, _scope} = :elixir.eval_quoted quoted, binding, opts - {value, binding} - end + @doc """ + Returns an environment for evaluation. - defp validate_eval_opts(opts) do - if f = opts[:functions], do: validate_imports(:functions, f) - if m = opts[:macros], do: validate_imports(:macros, m) - if a = opts[:aliases], do: validate_aliases(:aliases, a) - if r = opts[:requires], do: validate_requires(:requires, r) - end + It accepts either a `Macro.Env`, that is then pruned and prepared, + or a list of options. It returns an environment that is ready for + evaluation. - defp validate_requires(kind, requires) do - valid = is_list(requires) and Enum.all?(requires, &is_atom(&1)) + Most functions in this module will automatically prepare the given + environment for evaluation, so you don't need to explicitly call + this function, with the exception of `eval_quoted_with_env/3`, + which was designed precisely to be called in a loop, to implement + features such as interactive shells or anything else with multiple + evaluations. - unless valid do - raise ArgumentError, "expected :#{kind} option given to eval in the format: [module]" - end - end + ## Options - defp validate_aliases(kind, aliases) do - valid = is_list(aliases) and Enum.all?(aliases, fn {k, v} -> - is_atom(k) and is_atom(v) - end) + If an env is not given, the options can be: - unless valid do - raise ArgumentError, "expected :#{kind} option given to eval in the format: [{module, module}]" - end - end + * `:file` - the file to be considered in the evaluation - defp validate_imports(kind, imports) do - valid = is_list(imports) and Enum.all?(imports, fn {k, v} -> - is_atom(k) and is_list(v) and Enum.all?(v, fn {name, arity} -> - is_atom(name) and is_integer(arity) - end) - end) + * `:line` - the line on which the script starts + """ + @doc since: "1.14.0" + def env_for_eval(env_or_opts), do: :elixir.env_for_eval(env_or_opts) - unless valid do - raise ArgumentError, "expected :#{kind} option given to eval in the format: [{module, [{name, arity}]}]" - end + @doc """ + Evaluates the given `quoted` contents with `binding` and `env`. + + This function is meant to be called in a loop, to implement features + such as interactive shells or anything else with multiple evaluations. + Therefore, the first time you call this function, you must compute + the initial environment with `env_for_eval/1`. The remaining calls + must pass the environment that was returned by this function. + """ + @doc since: "1.14.0" + def eval_quoted_with_env(quoted, binding, %Macro.Env{} = env) when is_list(binding) do + eval_verify(:eval_forms, quoted, binding, env) end - @doc """ - Convert the given string to its quoted form. + @doc ~S""" + Converts the given string to its quoted form. - Returns `{:ok, quoted_form}` - if it succeeds, `{:error, {line, error, token}}` otherwise. + Returns `{:ok, quoted_form}` if it succeeds, + `{:error, {meta, message_info, token}}` otherwise. ## Options - * `:file` - the filename to be used in stacktraces - and the file reported in the `__ENV__` variable + * `:file` - the filename to be reported in case of parsing errors. + Defaults to `"nofile"`. - * `:line` - the line reported in the `__ENV__` variable + * `:line` - the starting line of the string being parsed. + Defaults to 1. + + * `:column` - (since v1.11.0) the starting column of the string being parsed. + Defaults to 1. + + * `:columns` - when `true`, attach a `:column` key to the quoted + metadata. Defaults to `false`. + + * `:unescape` (since v1.10.0) - when `false`, preserves escaped sequences. + For example, `"null byte\\t\\x00"` will be kept as is instead of being + converted to a bitstring literal. Note if you set this option to false, the + resulting AST is no longer valid, but it can be useful to analyze/transform + source code, typically in in combination with `quoted_to_algebra/2`. + Defaults to `true`. * `:existing_atoms_only` - when `true`, raises an error - when non-existing atoms are found by the tokenizer + when non-existing atoms are found by the tokenizer. + Defaults to `false`. + + * `:token_metadata` (since v1.10.0) - when `true`, includes token-related + metadata in the expression AST, such as metadata for `do` and `end` + tokens, for closing tokens, end of expressions, as well as delimiters + for sigils. See `t:Macro.metadata/0`. Defaults to `false`. + + * `:literal_encoder` (since v1.10.0) - how to encode literals in the AST. + It must be a function that receives two arguments, the literal and its + metadata, and it must return `{:ok, ast :: Macro.t}` or + `{:error, reason :: binary}`. If you return anything than the literal + itself as the `term`, then the AST is no longer valid. This option + may still useful for textual analysis of the source code. + + * `:static_atoms_encoder` - the static atom encoder function, see + "The `:static_atoms_encoder` function" section below. Note this + option overrides the `:existing_atoms_only` behaviour for static + atoms but `:existing_atoms_only` is still used for dynamic atoms, + such as atoms with interpolations. - ## Macro.to_string/2 + * `:warn_on_unnecessary_quotes` - when `false`, does not warn + when atoms, keywords or calls have unnecessary quotes on + them. Defaults to `true`. + + ## `Macro.to_string/2` The opposite of converting a string to its quoted form is `Macro.to_string/2`, which converts a quoted form to a string/binary representation. + + ## The `:static_atoms_encoder` function + + When `static_atoms_encoder: &my_encoder/2` is passed as an argument, + `my_encoder/2` is called every time the tokenizer needs to create a + "static" atom. Static atoms are atoms in the AST that function as + aliases, remote calls, local calls, variable names, regular atoms + and keyword lists. + + The encoder function will receive the atom name (as a binary) and a + keyword list with the current file, line and column. It must return + `{:ok, token :: term} | {:error, reason :: binary}`. + + The encoder function is supposed to create an atom from the given + string. To produce a valid AST, it is required to return `{:ok, term}`, + where `term` is an atom. It is possible to return something other than an atom, + however, in that case the AST is no longer "valid" in that it cannot + be used to compile or evaluate Elixir code. A use case for this is + if you want to use the Elixir parser in a user-facing situation, but + you don't want to exhaust the atom table. + + The atom encoder is not called for *all* atoms that are present in + the AST. It won't be invoked for the following atoms: + + * operators (`:+`, `:-`, and so on) + + * syntax keywords (`fn`, `do`, `else`, and so on) + + * atoms containing interpolation (`:"#{1 + 1} is two"`), as these + atoms are constructed at runtime. + """ + @spec string_to_quoted(List.Chars.t(), keyword) :: + {:ok, Macro.t()} | {:error, {location :: keyword, binary | {binary, binary}, binary}} def string_to_quoted(string, opts \\ []) when is_list(opts) do - file = Keyword.get opts, :file, "nofile" - line = Keyword.get opts, :line, 1 - :elixir.string_to_quoted(to_char_list(string), line, file, opts) + file = Keyword.get(opts, :file, "nofile") + line = Keyword.get(opts, :line, 1) + column = Keyword.get(opts, :column, 1) + + case :elixir.string_to_tokens(to_charlist(string), line, column, file, opts) do + {:ok, tokens} -> + :elixir.tokens_to_quoted(tokens, file, opts) + + {:error, _error_msg} = error -> + error + end end @doc """ - Convert the given string to its quoted form. + Converts the given string to its quoted form. - It returns the ast if it succeeds, + It returns the AST if it succeeds, raises an exception otherwise. The exception is a `TokenMissingError` in case a token is missing (usually because the expression is incomplete), `SyntaxError` otherwise. Check `string_to_quoted/2` for options information. """ + @spec string_to_quoted!(List.Chars.t(), keyword) :: Macro.t() def string_to_quoted!(string, opts \\ []) when is_list(opts) do - file = Keyword.get opts, :file, "nofile" - line = Keyword.get opts, :line, 1 - :elixir.string_to_quoted!(to_char_list(string), line, file, opts) + file = Keyword.get(opts, :file, "nofile") + line = Keyword.get(opts, :line, 1) + column = Keyword.get(opts, :column, 1) + :elixir.string_to_quoted!(to_charlist(string), line, column, file, opts) end @doc """ - Evals the given file. + Converts the given string to its quoted form and a list of comments. - Accepts `relative_to` as an argument to tell where the file is located. + This function is useful when performing textual changes to the source code, + while preserving information like comments and literals position. + + Returns `{:ok, quoted_form, comments}` if it succeeds, + `{:error, {line, error, token}}` otherwise. + + Comments are maps with the following fields: + + * `:line` - The line number the source code + + * `:text` - The full text of the comment, including the leading `#` + + * `:previous_eol_count` - How many end of lines there are between the comment and the previous AST node or comment + + * `:next_eol_count` - How many end of lines there are between the comment and the next AST node or comment + + Check `string_to_quoted/2` for options information. + + ## Examples + + iex> Code.string_to_quoted_with_comments("\"" + ...> :foo + ...> + ...> # Hello, world! + ...> + ...> + ...> # Some more comments! + ...> "\"") + {:ok, :foo, [ + %{line: 3, column: 1, previous_eol_count: 2, next_eol_count: 3, text: "\# Hello, world!"}, + %{line: 6, column: 1, previous_eol_count: 3, next_eol_count: 1, text: "\# Some more comments!"}, + ]} + + iex> Code.string_to_quoted_with_comments(":foo # :bar") + {:ok, :foo, [ + %{line: 1, column: 6, previous_eol_count: 0, next_eol_count: 0, text: "\# :bar"} + ]} - While `load_file` loads a file and returns the loaded modules and their - byte code, `eval_file` simply evalutes the file contents and returns the - evaluation result and its bindings. """ - def eval_file(file, relative_to \\ nil) do - file = find_file(file, relative_to) - eval_string File.read!(file), [], [file: file, line: 1] + @doc since: "1.13.0" + @spec string_to_quoted_with_comments(List.Chars.t(), keyword) :: + {:ok, Macro.t(), list(map())} | {:error, {location :: keyword, term, term}} + def string_to_quoted_with_comments(string, opts \\ []) when is_list(opts) do + charlist = to_charlist(string) + file = Keyword.get(opts, :file, "nofile") + line = Keyword.get(opts, :line, 1) + column = Keyword.get(opts, :column, 1) + + Process.put(:code_formatter_comments, []) + opts = [preserve_comments: &preserve_comments/5] ++ opts + + with {:ok, tokens} <- :elixir.string_to_tokens(charlist, line, column, file, opts), + {:ok, forms} <- :elixir.tokens_to_quoted(tokens, file, opts) do + comments = Enum.reverse(Process.get(:code_formatter_comments)) + {:ok, forms, comments} + end + after + Process.delete(:code_formatter_comments) end @doc """ - Load the given file. + Converts the given string to its quoted form and a list of comments. - Accepts `relative_to` as an argument to tell where the file is located. - If the file was already required/loaded, loads it again. + Returns the AST and a list of comments if it succeeds, raises an exception + otherwise. The exception is a `TokenMissingError` in case a token is missing + (usually because the expression is incomplete), `SyntaxError` otherwise. + + Check `string_to_quoted/2` for options information. + """ + @doc since: "1.13.0" + @spec string_to_quoted_with_comments!(List.Chars.t(), keyword) :: {Macro.t(), list(map())} + def string_to_quoted_with_comments!(string, opts \\ []) do + charlist = to_charlist(string) + + case string_to_quoted_with_comments(charlist, opts) do + {:ok, forms, comments} -> + {forms, comments} + + {:error, {location, error, token}} -> + :elixir_errors.parse_error( + location, + Keyword.get(opts, :file, "nofile"), + error, + token, + {charlist, Keyword.get(opts, :line, 1), Keyword.get(opts, :column, 1)} + ) + end + end + + defp preserve_comments(line, column, tokens, comment, rest) do + comments = Process.get(:code_formatter_comments) + + comment = %{ + line: line, + column: column, + previous_eol_count: previous_eol_count(tokens), + next_eol_count: next_eol_count(rest, 0), + text: List.to_string(comment) + } + + Process.put(:code_formatter_comments, [comment | comments]) + end - It returns a list of tuples `{ModuleName, <>}`, one tuple for - each module defined in the file. + defp next_eol_count('\s' ++ rest, count), do: next_eol_count(rest, count) + defp next_eol_count('\t' ++ rest, count), do: next_eol_count(rest, count) + defp next_eol_count('\n' ++ rest, count), do: next_eol_count(rest, count + 1) + defp next_eol_count('\r\n' ++ rest, count), do: next_eol_count(rest, count + 1) + defp next_eol_count(_, count), do: count - Notice that if `load_file` is invoked by different processes concurrently, - the target file will be loaded concurrently many times. Check `require_file/2` - if you don't want a file to be loaded concurrently. + defp previous_eol_count([{token, {_, _, count}} | _]) + when token in [:eol, :",", :";"] and count > 0 do + count + end + + defp previous_eol_count([]), do: 1 + defp previous_eol_count(_), do: 0 + + @doc ~S""" + Converts a quoted expression to an algebra document using Elixir's formatter rules. + + The algebra document can be converted into a string by calling: + + doc + |> Inspect.Algebra.format(:infinity) + |> IO.iodata_to_binary() + + For a high-level function that does the same, see `Macro.to_string/1`. + + ## Formatting considerations + + The Elixir AST does not contain metadata for literals like strings, lists, or + tuples with two elements, which means that the produced algebra document will + not respect all of the user preferences and comments may be misplaced. + To get better results, you can use the `:token_metadata`, `:unescape` and + `:literal_encoder` options to `string_to_quoted/2` to provide additional + information to the formatter: + + [ + literal_encoder: &{:ok, {:__block__, &2, [&1]}}, + token_metadata: true, + unescape: false + ] + + This will produce an AST that contains information such as `do` blocks start + and end lines or sigil delimiters, and by wrapping literals in blocks they can + now hold metadata like line number, string delimiter and escaped sequences, or + integer formatting (such as `0x2a` instead of `47`). However, **note this AST is + not valid**. If you evaluate it, it won't have the same semantics as the regular + Elixir AST due to the `:unescape` and `:literal_encoder` options. However, + those options are useful if you're doing source code manipulation, where it's + important to preserve user choices and comments placing. + + ## Options + + * `:comments` - the list of comments associated with the quoted expression. + Defaults to `[]`. It is recommended that both `:token_metadata` and + `:literal_encoder` options are given to `string_to_quoted_with_comments/2` + in order to get proper placement for comments + + * `:escape` - when `true`, escaped sequences like `\n` will be escaped into + `\\n`. If the `:unescape` option was set to `false` when using + `string_to_quoted/2`, setting this option to `false` will prevent it from + escaping the sequences twice. Defaults to `true`. + + * `:locals_without_parens` - a keyword list of name and arity + pairs that should be kept without parens whenever possible. + The arity may be the atom `:*`, which implies all arities of + that name. The formatter already includes a list of functions + and this option augments this list. + """ + @doc since: "1.13.0" + @spec quoted_to_algebra(Macro.t(), keyword) :: Inspect.Algebra.t() + def quoted_to_algebra(quoted, opts \\ []) do + quoted + |> Code.Normalizer.normalize(opts) + |> Code.Formatter.to_algebra(opts) + end + + @doc """ + Evaluates the given file. + + Accepts `relative_to` as an argument to tell where the file is located. + + While `require_file/2` and `compile_file/2` return the loaded modules and their + bytecode, `eval_file/2` simply evaluates the file contents and returns the + evaluation result and its binding (exactly the same return value as `eval_string/3`). """ + @spec eval_file(binary, nil | binary) :: {term, binding} + def eval_file(file, relative_to \\ nil) when is_binary(file) do + file = find_file(file, relative_to) + eval_string(File.read!(file), [], file: file, line: 1) + end + + @deprecated "Use Code.require_file/2 or Code.compile_file/2 instead" + @doc false def load_file(file, relative_to \\ nil) when is_binary(file) do file = find_file(file, relative_to) - :elixir_code_server.call {:acquire, file} - loaded = :elixir_compiler.file file - :elixir_code_server.cast {:loaded, file} + :elixir_code_server.call({:acquire, file}) + + loaded = + Module.ParallelChecker.verify(fn -> + :elixir_compiler.file(file, fn _, _ -> :ok end) + end) + + :elixir_code_server.cast({:required, file}) loaded end @@ -276,140 +1206,323 @@ defmodule Code do Requires the given `file`. Accepts `relative_to` as an argument to tell where the file is located. - The return value is the same as that of `load_file/2`. If the file was already - required/loaded, doesn't do anything and returns `nil`. + If the file was already required, `require_file/2` doesn't do anything and + returns `nil`. + + Note that if `require_file/2` is invoked by different processes concurrently, + the first process to invoke `require_file/2` acquires a lock and the remaining + ones will block until the file is available. This means that if `require_file/2` + is called more than once with a given file, that file will be compiled only once. + The first process to call `require_file/2` will get the list of loaded modules, + others will get `nil`. The list of required files is managed per Erlang VM node. + + See `compile_file/2` if you would like to compile a file without tracking its + filenames. Finally, if you would like to get the result of evaluating a file rather + than the modules defined in it, see `eval_file/2`. + + ## Examples + + If the file has not been required, it returns the list of modules: - Notice that if `require_file` is invoked by different processes concurrently, - the first process to invoke `require_file` acquires a lock and the remaining - ones will block until the file is available. I.e. if `require_file` is called - N times with a given file, it will be loaded only once. The first process to - call `require_file` will get the list of loaded modules, others will get `nil`. + modules = Code.require_file("eex_test.exs", "../eex/test") + List.first(modules) + #=> {EExTest.Compiled, <<70, 79, 82, 49, ...>>} + + If the file has been required, it returns `nil`: + + Code.require_file("eex_test.exs", "../eex/test") + #=> nil - Check `load_file/2` if you want a file to be loaded multiple times. """ + @spec require_file(binary, nil | binary) :: [{module, binary}] | nil def require_file(file, relative_to \\ nil) when is_binary(file) do file = find_file(file, relative_to) case :elixir_code_server.call({:acquire, file}) do - :loaded -> + :required -> nil - {:queued, ref} -> - receive do {:elixir_code_server, ^ref, :loaded} -> nil end + :proceed -> - loaded = :elixir_compiler.file file - :elixir_code_server.cast {:loaded, file} + loaded = + Module.ParallelChecker.verify(fn -> + :elixir_compiler.file(file, fn _, _ -> :ok end) + end) + + :elixir_code_server.cast({:required, file}) loaded end end @doc """ - Gets the compilation options from the code server. + Gets all compilation options from the code server. + + To get individual options, see `get_compiler_option/1`. + For a description of all options, see `put_compiler_option/2`. + + ## Examples + + Code.compiler_options() + #=> %{debug_info: true, docs: true, ...} - Check `compiler_options/1` for more information. """ + @spec compiler_options :: map def compiler_options do - :elixir_code_server.call :compiler_options + for key <- @available_compiler_options, into: %{} do + {key, :elixir_config.get(key)} + end end @doc """ - Returns a list with the available compiler options. + Stores all given compilation options. + + Changing the compilation options affect all processes + running in a given Erlang VM node. To store individual + options and for a description of all options, see + `put_compiler_option/2`. + + ## Examples + + Code.compiler_options() + #=> %{debug_info: true, docs: true, ...} - See `Code.compiler_options/1` for more info. """ + @spec compiler_options(Enumerable.t()) :: %{optional(atom) => boolean} + def compiler_options(opts) do + for {key, value} <- opts, into: %{} do + previous = get_compiler_option(key) + put_compiler_option(key, value) + {key, previous} + end + end + + @doc """ + Returns the value of a given compiler option. + + For a description of all options, see `put_compiler_option/2`. + + ## Examples + + Code.get_compiler_option(:debug_info) + #=> true + + """ + @doc since: "1.10.0" + @spec get_compiler_option(atom) :: term + def get_compiler_option(key) when key in @available_compiler_options do + :elixir_config.get(key) + end + + @doc """ + Returns a list with all available compiler options. + + For a description of all options, see `put_compiler_option/2`. + + ## Examples + + Code.available_compiler_options() + #=> [:docs, :debug_info, ...] + + """ + @spec available_compiler_options() :: [atom] def available_compiler_options do - [:docs, :debug_info, :ignore_module_conflict, :warnings_as_errors] + @available_compiler_options end @doc """ - Sets compilation options. + Stores a compilation option. - These options are global since they are stored by Elixir's Code Server. + Changing the compilation options affect all processes running in a + given Erlang VM node. Available options are: - * `:docs` - when `true`, retain documentation in the compiled module, - `true` by default + * `:docs` - when `true`, retain documentation in the compiled module. + Defaults to `true`. * `:debug_info` - when `true`, retain debug information in the compiled - module; this allows a developer to reconstruct the original source - code, `false` by default + module. This allows a developer to reconstruct the original source + code. Defaults to `true`. * `:ignore_module_conflict` - when `true`, override modules that were - already defined without raising errors, `false` by default + already defined without raising errors. Defaults to `false`. + + * `:relative_paths` - when `true`, use relative paths in quoted nodes, + warnings and errors generated by the compiler. Note disabling this option + won't affect runtime warnings and errors. Defaults to `true`. + + * `:warnings_as_errors` - causes compilation to fail when warnings are + generated. Defaults to `false`. + + * `:no_warn_undefined` (since v1.10.0) - list of modules and `{Mod, fun, arity}` + tuples that will not emit warnings that the module or function does not exist + at compilation time. Pass atom `:all` to skip warning for all undefined + functions. This can be useful when doing dynamic compilation. Defaults to `[]`. - * `:warnings_as_errors` - cause compilation to fail when warnings are - generated + * `:tracers` (since v1.10.0) - a list of tracers (modules) to be used during + compilation. See the module docs for more information. Defaults to `[]`. + + * `:parser_options` (since v1.10.0) - a keyword list of options to be given + to the parser when compiling files. It accepts the same options as + `string_to_quoted/2` (except by the options that change the AST itself). + This can be used in combination with the tracer to retrieve localized + information about events happening during compilation. Defaults to `[]`. + + It always returns `:ok`. Raises an error for invalid options. + + ## Examples + + Code.put_compiler_option(:debug_info, true) + #=> :ok """ - def compiler_options(opts) do - {opts, bad} = Keyword.split(opts, available_compiler_options) - if bad != [] do - bad = bad |> Keyword.keys |> Enum.join(", ") - raise ArgumentError, message: "unknown compiler options: #{bad}" + @doc since: "1.10.0" + @spec put_compiler_option(atom, term) :: :ok + def put_compiler_option(key, value) when key in @boolean_compiler_options do + if not is_boolean(value) do + raise "compiler option #{inspect(key)} should be a boolean, got: #{inspect(value)}" end - :elixir_code_server.cast {:compiler_options, opts} + + :elixir_config.put(key, value) + :ok + end + + def put_compiler_option(:no_warn_undefined, value) do + if value != :all and not is_list(value) do + raise "compiler option :no_warn_undefined should be a list or the atom :all, " <> + "got: #{inspect(value)}" + end + + :elixir_config.put(:no_warn_undefined, value) + :ok + end + + def put_compiler_option(key, value) when key in @list_compiler_options do + if not is_list(value) do + raise "compiler option #{inspect(key)} should be a list, got: #{inspect(value)}" + end + + if key == :parser_options and not Keyword.keyword?(value) do + raise "compiler option #{inspect(key)} should be a keyword list, " <> + "got: #{inspect(value)}" + end + + if key == :tracers and not Enum.all?(value, &is_atom/1) do + raise "compiler option #{inspect(key)} should be a list of modules, " <> + "got: #{inspect(value)}" + end + + :elixir_config.put(key, value) + :ok + end + + def put_compiler_option(key, _value) do + raise "unknown compiler option: #{inspect(key)}" + end + + @doc """ + Purge compiler modules. + + The compiler utilizes temporary modules to compile code. For example, + `elixir_compiler_1`, `elixir_compiler_2`, and so on. In case the compiled code + stores references to anonymous functions or similar, the Elixir compiler + may be unable to reclaim those modules, keeping an unnecessary amount of + code in memory and eventually leading to modules such as `elixir_compiler_12345`. + + This function purges all modules currently kept by the compiler, allowing + old compiler module names to be reused. If there are any processes running + any code from such modules, they will be terminated too. + + This function is only meant to be called if you have a long running node + that is constantly evaluating code. + + It returns `{:ok, number_of_modules_purged}`. + """ + @doc since: "1.7.0" + @spec purge_compiler_modules() :: {:ok, non_neg_integer()} + def purge_compiler_modules() do + :elixir_code_server.call(:purge_compiler_modules) end @doc """ Compiles the given string. Returns a list of tuples where the first element is the module name - and the second one is its byte code (as a binary). - - For compiling many files at once, check `Kernel.ParallelCompiler.files/2`. + and the second one is its bytecode (as a binary). A `file` can be + given as second argument which will be used for reporting warnings + and errors. + + **Warning**: `string` can be any Elixir code and code can be executed with + the same privileges as the Erlang VM: this means that such code could + compromise the machine (for example by executing system commands). + Don't use `compile_string/2` with untrusted input (such as strings coming + from the network). """ + @spec compile_string(List.Chars.t(), binary) :: [{module, binary}] def compile_string(string, file \\ "nofile") when is_binary(file) do - :elixir_compiler.string to_char_list(string), file + Module.ParallelChecker.verify(fn -> + :elixir_compiler.string(to_charlist(string), file, fn _, _ -> :ok end) + end) end @doc """ Compiles the quoted expression. Returns a list of tuples where the first element is the module name and - the second one is its byte code (as a binary). + the second one is its bytecode (as a binary). A `file` can be + given as second argument which will be used for reporting warnings + and errors. """ + @spec compile_quoted(Macro.t(), binary) :: [{module, binary}] def compile_quoted(quoted, file \\ "nofile") when is_binary(file) do - :elixir_compiler.quoted quoted, file + Module.ParallelChecker.verify(fn -> + :elixir_compiler.quoted(quoted, file, fn _, _ -> :ok end) + end) end @doc """ - Ensures the given module is loaded. + Compiles the given file. - If the module is already loaded, this works as no-op. If the module - was not yet loaded, it tries to load it. + Accepts `relative_to` as an argument to tell where the file is located. - If it succeeds loading the module, it returns `{:module, module}`. - If not, returns `{:error, reason}` with the error reason. + Returns a list of tuples where the first element is the module name and + the second one is its bytecode (as a binary). Opposite to `require_file/2`, + it does not track the filename of the compiled file. - ## Code loading on the Erlang VM + If you would like to get the result of evaluating file rather than the + modules defined in it, see `eval_file/2`. - Erlang has two modes to load code: interactive and embedded. + For compiling many files concurrently, see `Kernel.ParallelCompiler.compile/2`. + """ + @doc since: "1.7.0" + @spec compile_file(binary, nil | binary) :: [{module, binary}] + def compile_file(file, relative_to \\ nil) when is_binary(file) do + Module.ParallelChecker.verify(fn -> + :elixir_compiler.file(find_file(file, relative_to), fn _, _ -> :ok end) + end) + end - By default, the Erlang VM runs in interactive mode, where modules - are loaded as needed. In embedded mode the opposite happens, as all - modules need to be loaded upfront or explicitly. + @doc """ + Ensures the given module is loaded. - Therefore, this function is used to check if a module is loaded - before using it and allows one to react accordingly. For example, the `URI` - module uses this function to check if a specific parser exists for a given - URI scheme. + If the module is already loaded, this works as no-op. If the module + was not yet loaded, it tries to load it. - ## `Code.ensure_compiled/1` + If it succeeds in loading the module, it returns `{:module, module}`. + If not, returns `{:error, reason}` with the error reason. - Elixir also contains an `ensure_compiled/1` function that is a - superset of `ensure_loaded/1`. + See the module documentation for more information on code loading. - Since Elixir's compilation happens in parallel, in some situations - you may need to use a module that was not yet compiled, therefore - it can't even be loaded. + ## Examples - `ensure_compiled/1` halts the current process until the - module we are depending on is available. + iex> Code.ensure_loaded(Atom) + {:module, Atom} + + iex> Code.ensure_loaded(DoesNotExist) + {:error, :nofile} - In most cases, `ensure_loaded/1` is enough. `ensure_compiled/1` - must be used in rare cases, usually involving macros that need to - invoke a module for callback information. """ + @spec ensure_loaded(module) :: + {:module, module} | {:error, :embedded | :badfile | :nofile | :on_load_failure} def ensure_loaded(module) when is_atom(module) do :code.ensure_loaded(module) end @@ -420,105 +1533,258 @@ defmodule Code do Similar to `ensure_loaded/1`, but returns `true` if the module is already loaded or was successfully loaded. Returns `false` otherwise. + + ## Examples + + iex> Code.ensure_loaded?(Atom) + true + """ - def ensure_loaded?(module) do + @spec ensure_loaded?(module) :: boolean + def ensure_loaded?(module) when is_atom(module) do match?({:module, ^module}, ensure_loaded(module)) end @doc """ - Ensures the given module is compiled and loaded. + Same as `ensure_loaded/1` but raises if the module cannot be loaded. + """ + @doc since: "1.12.0" + @spec ensure_loaded!(module) :: module + def ensure_loaded!(module) do + case ensure_loaded(module) do + {:module, module} -> + module + + {:error, reason} -> + raise ArgumentError, + "could not load module #{inspect(module)} due to reason #{inspect(reason)}" + end + end - If the module is already loaded, it works as no-op. If the module was - not loaded yet, it checks if it needs to be compiled first and then - tries to load it. + @doc """ + Similar to `ensure_compiled!/1` but indicates you can continue without said module. - If it succeeds loading the module, it returns `{:module, module}`. + While `ensure_compiled!/1` indicates to the Elixir compiler you can + only continue when said module is available, this function indicates + you may continue compilation without said module. + + If it succeeds in loading the module, it returns `{:module, module}`. If not, returns `{:error, reason}` with the error reason. + If the module being checked is currently in a compiler deadlock, + this function returns `{:error, :unavailable}`. Unavailable doesn't + necessarily mean the module doesn't exist, just that it is not currently + available, but it (or may not) become available in the future. + + Therefore, if you can only continue if the module is available, use + `ensure_compiled!/1` instead. In particular, do not do this: + + case Code.ensure_compiled(module) do + {:module, _} -> module + {:error, _} -> raise ... + end - Check `ensure_loaded/1` for more information on module loading - and when to use `ensure_loaded/1` or `ensure_compiled/1`. + See the module documentation for more information on code loading. """ + @spec ensure_compiled(module) :: + {:module, module} + | {:error, :embedded | :badfile | :nofile | :on_load_failure | :unavailable} def ensure_compiled(module) when is_atom(module) do + ensure_compiled(module, :soft) + end + + @doc """ + Ensures the given module is compiled and loaded. + + If the module is already loaded, it works as no-op. If the module was + not compiled yet, `ensure_compiled!/1` halts the compilation of the caller + until the module given to `ensure_compiled!/1` becomes available or + all files for the current project have been compiled. If compilation + finishes and the module is not available or is in a deadlock, an error + is raised. + + Given this function halts compilation, use it carefully. In particular, + avoid using it to guess which modules are in the system. Overuse of this + function can also lead to deadlocks, where two modules check at the same time + if the other is compiled. This returns a specific unavailable error code, + where we cannot successfully verify a module is available or not. + + See the module documentation for more information on code loading. + """ + @doc since: "1.12.0" + @spec ensure_compiled!(module) :: module + def ensure_compiled!(module) do + case ensure_compiled(module, :hard) do + {:module, module} -> + module + + {:error, reason} -> + raise ArgumentError, + "could not load module #{inspect(module)} due to reason #{inspect(reason)}" + end + end + + defp ensure_compiled(module, mode) do case :code.ensure_loaded(module) do {:error, :nofile} = error -> - case :erlang.get(:elixir_ensure_compiled) do - :undefined -> error - _ -> - try do - module.__info__(:module) - {:module, module} - rescue - UndefinedFunctionError -> error - end + if can_await_module_compilation?() do + case Kernel.ErrorHandler.ensure_compiled(module, :module, mode) do + :found -> {:module, module} + :deadlock -> {:error, :unavailable} + :not_found -> {:error, :nofile} + end + else + error end - other -> other + + other -> + other end end @doc """ - Ensures the given module is compiled and loaded. + Returns true if the current process can await for module compilation. - Similar to `ensure_compiled/1`, but returns `true` if the module - is already loaded or was successfully loaded and compiled. - Returns `false` otherwise. + When compiling Elixir code via `Kernel.ParallelCompiler`, which is + used by Mix and `elixirc`, calling a module that has not yet been + compiled will block the caller until the module becomes available. + Executing Elixir scripts, such as passing a filename to `elixir`, + does not await. """ - def ensure_compiled?(module) do + @doc since: "1.11.0" + @spec can_await_module_compilation? :: boolean + def can_await_module_compilation? do + :erlang.process_info(self(), :error_handler) == {:error_handler, Kernel.ErrorHandler} + end + + @doc false + @deprecated "Use Code.ensure_compiled/1 instead (see the proper disclaimers in its docs)" + def ensure_compiled?(module) when is_atom(module) do match?({:module, ^module}, ensure_compiled(module)) end - @doc """ - Returns the docs for the given module. + @doc ~S""" + Returns the docs for the given module or path to `.beam` file. When given a module name, it finds its BEAM code and reads the docs from it. - When given a path to a .beam file, it will load the docs directly from that + When given a path to a `.beam` file, it will load the docs directly from that file. - The return value depends on the `kind` value: + It returns the term stored in the documentation chunk in the format defined by + [EEP 48](https://www.erlang.org/eeps/eep-0048.html) or `{:error, reason}` if + the chunk is not available. - * `:docs` - list of all docstrings attached to functions and macros - using the `@doc` attribute + ## Examples - * `:moduledoc` - tuple `{, }` where `line` is the line on - which module definition starts and `doc` is the string - attached to the module using the `@moduledoc` attribute + # Module documentation of an existing module + iex> {:docs_v1, _, :elixir, _, %{"en" => module_doc}, _, _} = Code.fetch_docs(Atom) + iex> module_doc |> String.split("\n") |> Enum.at(0) + "Atoms are constants whose values are their own name." - * `:all` - a keyword list with both `:docs` and `:moduledoc` + # A module that doesn't exist + iex> Code.fetch_docs(ModuleNotGood) + {:error, :module_not_found} """ - def get_docs(module, kind) when is_atom(module) do + @doc since: "1.7.0" + @spec fetch_docs(module | String.t()) :: + {:docs_v1, annotation, beam_language, format, module_doc :: doc_content, metadata, + docs :: [doc_element]} + | {:error, :module_not_found | :chunk_not_found | {:invalid_chunk, binary}} + when annotation: :erl_anno.anno(), + beam_language: :elixir | :erlang | atom(), + doc_content: %{optional(binary) => binary} | :none | :hidden, + doc_element: + {{kind :: atom, function_name :: atom, arity}, annotation, signature, doc_content, + metadata}, + format: binary, + signature: [binary], + metadata: map + def fetch_docs(module_or_path) + + def fetch_docs(module) when is_atom(module) do case :code.get_object_code(module) do - {_module, bin, _beam_path} -> - do_get_docs(bin, kind) + {_module, bin, beam_path} -> + case fetch_docs_from_beam(bin) do + {:error, :chunk_not_found} -> + app_root = Path.expand(Path.join(["..", ".."]), beam_path) + path = Path.join([app_root, "doc", "chunks", "#{module}.chunk"]) + fetch_docs_from_chunk(path) + + other -> + other + end + + :error -> + case :code.which(module) do + :preloaded -> + # The ERTS directory is not necessarily included in releases + # unless it is listed as an extra application. + case :code.lib_dir(:erts) do + path when is_list(path) -> + path = Path.join([path, "doc", "chunks", "#{module}.chunk"]) + fetch_docs_from_chunk(path) + + {:error, _} -> + {:error, :chunk_not_found} + end - :error -> nil + _ -> + {:error, :module_not_found} + end end end - def get_docs(binpath, kind) when is_binary(binpath) do - do_get_docs(String.to_char_list(binpath), kind) + def fetch_docs(path) when is_binary(path) do + fetch_docs_from_beam(String.to_charlist(path)) end - @docs_chunk 'ExDc' + @docs_chunk 'Docs' - defp do_get_docs(bin_or_path, kind) do + defp fetch_docs_from_beam(bin_or_path) do case :beam_lib.chunks(bin_or_path, [@docs_chunk]) do {:ok, {_module, [{@docs_chunk, bin}]}} -> - lookup_docs(:erlang.binary_to_term(bin), kind) + load_docs_chunk(bin) + + {:error, :beam_lib, {:missing_chunk, _, @docs_chunk}} -> + {:error, :chunk_not_found} + + {:error, :beam_lib, {:file_error, _, :enoent}} -> + {:error, :module_not_found} + end + end - {:error, :beam_lib, {:missing_chunk, _, @docs_chunk}} -> nil + defp fetch_docs_from_chunk(path) do + case File.read(path) do + {:ok, bin} -> + load_docs_chunk(bin) + + {:error, _} -> + {:error, :chunk_not_found} end end - defp lookup_docs({:elixir_docs_v1, docs}, kind), - do: do_lookup_docs(docs, kind) + defp load_docs_chunk(bin) do + :erlang.binary_to_term(bin) + rescue + _ -> + {:error, {:invalid_chunk, bin}} + end - # unsupported chunk version - defp lookup_docs(_, _), do: nil + @doc ~S""" + Deprecated function to retrieve old documentation format. - defp do_lookup_docs(docs, :all), do: docs - defp do_lookup_docs(docs, kind) when kind in [:docs, :moduledoc], - do: Keyword.get(docs, kind) + Elixir v1.7 adopts [EEP 48](https://www.erlang.org/eeps/eep-0048.html) + which is a new documentation format meant to be shared across all + BEAM languages. The old format, used by `Code.get_docs/2`, is no + longer available, and therefore this function always returns `nil`. + Use `Code.fetch_docs/1` instead. + """ + @deprecated "Code.get_docs/2 always returns nil as its outdated documentation is no longer stored on BEAM files. Use Code.fetch_docs/1 instead" + @spec get_docs(module, :moduledoc | :docs | :callback_docs | :type_docs | :all) :: nil + def get_docs(_module, _kind) do + nil + end ## Helpers @@ -526,11 +1792,12 @@ defmodule Code do # # If the file is found, returns its path in binary, fails otherwise. defp find_file(file, relative_to) do - file = if relative_to do - Path.expand(file, relative_to) - else - Path.expand(file) - end + file = + if relative_to do + Path.expand(file, relative_to) + else + Path.expand(file) + end if File.regular?(file) do file diff --git a/lib/elixir/lib/code/formatter.ex b/lib/elixir/lib/code/formatter.ex new file mode 100644 index 00000000000..e1bcdf1f411 --- /dev/null +++ b/lib/elixir/lib/code/formatter.ex @@ -0,0 +1,2349 @@ +defmodule Code.Formatter do + @moduledoc false + import Inspect.Algebra, except: [format: 2, surround: 3, surround: 4] + + @double_quote "\"" + @double_heredoc "\"\"\"" + @single_quote "'" + @single_heredoc "'''" + @newlines 2 + @min_line 0 + @max_line 9_999_999 + @empty empty() + @ampersand_prec Code.Identifier.unary_op(:&) |> elem(1) + + # Operators that are composed of multiple binary operators + @multi_binary_operators [:"..//"] + + # Operators that do not have space between operands + @no_space_binary_operators [:.., :"//"] + + # Operators that do not have newline between operands (as well as => and keywords) + @no_newline_binary_operators [:\\, :in] + + # Left associative operators that start on the next line in case of breaks (always pipes) + @pipeline_operators [:|>, :~>>, :<<~, :~>, :<~, :<~>, :"<|>"] + + # Right associative operators that start on the next line in case of breaks + @right_new_line_before_binary_operators [:|, :when] + + # Operators that are logical cannot be mixed without parens + @required_parens_logical_binary_operands [:||, :|||, :or, :&&, :&&&, :and] + + # Operators with next break fits. = and :: do not consider new lines though + @next_break_fits_operators [:<-, :==, :!=, :=~, :===, :!==, :<, :>, :<=, :>=, :=, :"::"] + + # Operators that always require parens on operands when they are the parent + @required_parens_on_binary_operands [ + :|>, + :<<<, + :>>>, + :<~, + :~>, + :<<~, + :~>>, + :<~>, + :"<|>", + :"^^^", + :+++, + :---, + :in, + :++, + :--, + :.., + :<> + ] + + @locals_without_parens [ + # Special forms + alias: 1, + alias: 2, + case: 2, + cond: 1, + for: :*, + import: 1, + import: 2, + quote: 1, + quote: 2, + receive: 1, + require: 1, + require: 2, + try: 1, + with: :*, + + # Kernel + def: 1, + def: 2, + defp: 1, + defp: 2, + defguard: 1, + defguardp: 1, + defmacro: 1, + defmacro: 2, + defmacrop: 1, + defmacrop: 2, + defmodule: 2, + defdelegate: 2, + defexception: 1, + defoverridable: 1, + defstruct: 1, + destructure: 2, + raise: 1, + raise: 2, + reraise: 2, + reraise: 3, + if: 2, + unless: 2, + use: 1, + use: 2, + + # Stdlib, + defrecord: 2, + defrecord: 3, + defrecordp: 2, + defrecordp: 3, + + # Testing + assert: 1, + assert: 2, + assert_in_delta: 3, + assert_in_delta: 4, + assert_raise: 2, + assert_raise: 3, + assert_receive: 1, + assert_receive: 2, + assert_receive: 3, + assert_received: 1, + assert_received: 2, + doctest: 1, + doctest: 2, + refute: 1, + refute: 2, + refute_in_delta: 3, + refute_in_delta: 4, + refute_receive: 1, + refute_receive: 2, + refute_receive: 3, + refute_received: 1, + refute_received: 2, + setup: 1, + setup: 2, + setup_all: 1, + setup_all: 2, + test: 1, + test: 2, + + # Mix config + config: 2, + config: 3, + import_config: 1 + ] + + @do_end_keywords [:rescue, :catch, :else, :after] + + @bitstring_modifiers [ + :integer, + :float, + :bits, + :bitstring, + :binary, + :bytes, + :utf8, + :utf16, + :utf32, + :signed, + :unsigned, + :little, + :big, + :native, + :_ + ] + + @doc """ + Converts the quoted expression into an algebra document. + """ + def to_algebra(quoted, opts \\ []) do + comments = Keyword.get(opts, :comments, []) + + state = + comments + |> Enum.map(&format_comment/1) + |> gather_comments() + |> state(opts) + + {doc, _} = block_to_algebra(quoted, @min_line, @max_line, state) + doc + end + + @doc """ + Lists all default locals without parens. + """ + def locals_without_parens do + @locals_without_parens + end + + @doc """ + Checks if a function is a local without parens. + """ + def local_without_parens?(fun, arity, locals_without_parens) do + arity > 0 and + Enum.any?(locals_without_parens, fn {key, val} -> + key == fun and (val == :* or val == arity) + end) + end + + defp state(comments, opts) do + force_do_end_blocks = Keyword.get(opts, :force_do_end_blocks, false) + locals_without_parens = Keyword.get(opts, :locals_without_parens, []) + file = Keyword.get(opts, :file, nil) + sigils = Keyword.get(opts, :sigils, []) + normalize_bitstring_modifiers = Keyword.get(opts, :normalize_bitstring_modifiers, true) + + sigils = + Map.new(sigils, fn {key, value} -> + with true <- is_atom(key) and is_function(value, 2), + [char] <- Atom.to_charlist(key), + true <- char in ?A..?Z do + {char, value} + else + _ -> + raise ArgumentError, + ":sigils must be a keyword list with a single uppercased letter as key and an " <> + "anonymous function expecting two arguments as value, got: #{inspect(sigils)}" + end + end) + + %{ + force_do_end_blocks: force_do_end_blocks, + locals_without_parens: locals_without_parens ++ locals_without_parens(), + operand_nesting: 2, + skip_eol: false, + comments: comments, + sigils: sigils, + file: file, + normalize_bitstring_modifiers: normalize_bitstring_modifiers + } + end + + defp format_comment(%{text: text} = comment) do + %{comment | text: format_comment_text(text)} + end + + defp format_comment_text("#"), do: "#" + defp format_comment_text("#!" <> rest), do: "#!" <> rest + defp format_comment_text("##" <> rest), do: "#" <> format_comment_text("#" <> rest) + defp format_comment_text("# " <> rest), do: "# " <> rest + defp format_comment_text("#" <> rest), do: "# " <> rest + + # If there is a no new line before, we can't gather all followup comments. + defp gather_comments([%{previous_eol_count: 0} = comment | comments]) do + comment = %{comment | previous_eol_count: @newlines} + [comment | gather_comments(comments)] + end + + defp gather_comments([comment | comments]) do + %{line: line, next_eol_count: next_eol_count, text: doc} = comment + + {next_eol_count, comments, doc} = + gather_followup_comments(line + 1, next_eol_count, comments, doc) + + comment = %{comment | next_eol_count: next_eol_count, text: doc} + [comment | gather_comments(comments)] + end + + defp gather_comments([]) do + [] + end + + defp gather_followup_comments(line, _, [%{line: line} = comment | comments], doc) + when comment.previous_eol_count != 0 do + %{next_eol_count: next_eol_count, text: text} = comment + gather_followup_comments(line + 1, next_eol_count, comments, line(doc, text)) + end + + defp gather_followup_comments(_line, next_eol_count, comments, doc) do + {next_eol_count, comments, doc} + end + + # Special AST nodes from compiler feedback + + defp quoted_to_algebra({{:special, :clause_args}, _meta, [args]}, _context, state) do + {doc, state} = clause_args_to_algebra(args, state) + {group(doc), state} + end + + defp quoted_to_algebra({{:special, :bitstring_segment}, _meta, [arg, last]}, _context, state) do + bitstring_segment_to_algebra({arg, -1}, state, last) + end + + defp quoted_to_algebra({var, _meta, var_context}, _context, state) when is_atom(var_context) do + {var |> Atom.to_string() |> string(), state} + end + + defp quoted_to_algebra({:<<>>, meta, entries}, _context, state) do + cond do + entries == [] -> + {"<<>>", state} + + not interpolated?(entries) -> + bitstring_to_algebra(meta, entries, state) + + meta[:delimiter] == ~s["""] -> + {doc, state} = + entries + |> prepend_heredoc_line() + |> interpolation_to_algebra(~s["""], state, @double_heredoc, @double_heredoc) + + {force_unfit(doc), state} + + true -> + interpolation_to_algebra(entries, @double_quote, state, @double_quote, @double_quote) + end + end + + defp quoted_to_algebra( + {{:., _, [List, :to_charlist]}, meta, [entries]} = quoted, + context, + state + ) do + cond do + not list_interpolated?(entries) -> + remote_to_algebra(quoted, context, state) + + meta[:delimiter] == ~s['''] -> + {doc, state} = + entries + |> prepend_heredoc_line() + |> list_interpolation_to_algebra(~s['''], state, @single_heredoc, @single_heredoc) + + {force_unfit(doc), state} + + true -> + list_interpolation_to_algebra(entries, @single_quote, state, @single_quote, @single_quote) + end + end + + defp quoted_to_algebra( + {{:., _, [:erlang, :binary_to_atom]}, _, [{:<<>>, _, entries}, :utf8]} = quoted, + context, + state + ) do + if interpolated?(entries) do + interpolation_to_algebra(entries, @double_quote, state, ":\"", @double_quote) + else + remote_to_algebra(quoted, context, state) + end + end + + # foo[bar] + defp quoted_to_algebra({{:., _, [Access, :get]}, meta, [target, arg]}, _context, state) do + {target_doc, state} = remote_target_to_algebra(target, state) + + {access_doc, state} = + if keyword?(arg) do + list_to_algebra(meta, arg, state) + else + list_to_algebra(meta, [arg], state) + end + + {concat(target_doc, access_doc), state} + end + + # %Foo{} + # %name{foo: 1} + # %name{bar | foo: 1} + defp quoted_to_algebra({:%, _, [name, {:%{}, meta, args}]}, _context, state) do + {name_doc, state} = quoted_to_algebra(name, :parens_arg, state) + map_to_algebra(meta, name_doc, args, state) + end + + # %{foo: 1} + # %{foo => bar} + # %{name | foo => bar} + defp quoted_to_algebra({:%{}, meta, args}, _context, state) do + map_to_algebra(meta, @empty, args, state) + end + + # {} + # {1, 2} + defp quoted_to_algebra({:{}, meta, args}, _context, state) do + tuple_to_algebra(meta, args, :flex_break, state) + end + + defp quoted_to_algebra({:__block__, meta, [{left, right}]}, _context, state) do + tuple_to_algebra(meta, [left, right], :flex_break, state) + end + + defp quoted_to_algebra({:__block__, meta, [list]}, _context, state) when is_list(list) do + case meta[:delimiter] do + ~s['''] -> + string = list |> List.to_string() |> escape_heredoc(~s[''']) + {@single_heredoc |> concat(string) |> concat(@single_heredoc) |> force_unfit(), state} + + ~s['] -> + string = list |> List.to_string() |> escape_string(@single_quote) + {@single_quote |> concat(string) |> concat(@single_quote), state} + + _other -> + list_to_algebra(meta, list, state) + end + end + + defp quoted_to_algebra({:__block__, meta, [string]}, _context, state) when is_binary(string) do + if meta[:delimiter] == ~s["""] do + string = escape_heredoc(string, ~s["""]) + {@double_heredoc |> concat(string) |> concat(@double_heredoc) |> force_unfit(), state} + else + string = escape_string(string, @double_quote) + {@double_quote |> concat(string) |> concat(@double_quote), state} + end + end + + defp quoted_to_algebra({:__block__, meta, [atom]}, _context, state) when is_atom(atom) do + {atom_to_algebra(atom, meta), state} + end + + defp quoted_to_algebra({:__block__, meta, [integer]}, _context, state) + when is_integer(integer) do + {integer_to_algebra(Keyword.fetch!(meta, :token)), state} + end + + defp quoted_to_algebra({:__block__, meta, [float]}, _context, state) when is_float(float) do + {float_to_algebra(Keyword.fetch!(meta, :token)), state} + end + + defp quoted_to_algebra( + {:__block__, _meta, [{:unquote_splicing, meta, [_] = args}]}, + context, + state + ) do + {doc, state} = local_to_algebra(:unquote_splicing, meta, args, context, state) + {wrap_in_parens(doc), state} + end + + defp quoted_to_algebra({:__block__, _meta, [arg]}, context, state) do + quoted_to_algebra(arg, context, state) + end + + defp quoted_to_algebra({:__block__, _meta, []}, _context, state) do + {"nil", state} + end + + defp quoted_to_algebra({:__block__, meta, _} = block, _context, state) do + {block, state} = block_to_algebra(block, line(meta), closing_line(meta), state) + {surround("(", block, ")"), state} + end + + defp quoted_to_algebra({:__aliases__, _meta, [head | tail]}, context, state) do + {doc, state} = + if is_atom(head) do + {Atom.to_string(head), state} + else + quoted_to_algebra_with_parens_if_operator(head, context, state) + end + + {Enum.reduce(tail, doc, &concat(&2, "." <> Atom.to_string(&1))), state} + end + + # &1 + # &local(&1) + # &local/1 + # &Mod.remote/1 + # & &1 + # & &1 + &2 + defp quoted_to_algebra({:&, _, [arg]}, context, state) do + capture_to_algebra(arg, context, state) + end + + defp quoted_to_algebra({:@, meta, [arg]}, context, state) do + module_attribute_to_algebra(meta, arg, context, state) + end + + # not(left in right) + # left not in right + defp quoted_to_algebra({:not, meta, [{:in, _, [left, right]}]}, context, state) do + binary_op_to_algebra(:in, "not in", meta, left, right, context, state) + end + + # .. + defp quoted_to_algebra({:.., _meta, []}, context, state) do + if context in [:no_parens_arg, :no_parens_one_arg] do + {"(..)", state} + else + {"..", state} + end + end + + # 1..2//3 + defp quoted_to_algebra({:"..//", meta, [left, middle, right]}, context, state) do + quoted_to_algebra({:"//", meta, [{:.., meta, [left, middle]}, right]}, context, state) + end + + defp quoted_to_algebra({:fn, meta, [_ | _] = clauses}, _context, state) do + anon_fun_to_algebra(clauses, line(meta), closing_line(meta), state, eol?(meta, state)) + end + + defp quoted_to_algebra({fun, meta, args}, context, state) when is_atom(fun) and is_list(args) do + with :error <- maybe_sigil_to_algebra(fun, meta, args, state), + :error <- maybe_unary_op_to_algebra(fun, meta, args, context, state), + :error <- maybe_binary_op_to_algebra(fun, meta, args, context, state), + do: local_to_algebra(fun, meta, args, context, state) + end + + defp quoted_to_algebra({_, _, args} = quoted, context, state) when is_list(args) do + remote_to_algebra(quoted, context, state) + end + + # (left -> right) + defp quoted_to_algebra([{:->, _, _} | _] = clauses, _context, state) do + type_fun_to_algebra(clauses, @max_line, @min_line, state) + end + + # [keyword: :list] (inner part) + # %{:foo => :bar} (inner part) + defp quoted_to_algebra(list, context, state) when is_list(list) do + many_args_to_algebra(list, state, "ed_to_algebra(&1, context, &2)) + end + + # keyword: :list + # key => value + defp quoted_to_algebra({left_arg, right_arg}, context, state) do + {left, op, right, state} = + if keyword_key?(left_arg) do + {left, state} = + case left_arg do + # TODO: Remove this clause in v1.16 when we no longer quote operator :..// + {:__block__, _, [:"..//"]} -> + {string(~S{"..//":}), state} + + {:__block__, _, [atom]} when is_atom(atom) -> + key = + if Macro.classify_atom(atom) in [:identifier, :unquoted] do + IO.iodata_to_binary([Atom.to_string(atom), ?:]) + else + IO.iodata_to_binary([?", Atom.to_string(atom), ?", ?:]) + end + + {string(key), state} + + {{:., _, [:erlang, :binary_to_atom]}, _, [{:<<>>, _, entries}, :utf8]} -> + interpolation_to_algebra(entries, @double_quote, state, "\"", "\":") + end + + {right, state} = quoted_to_algebra(right_arg, context, state) + {left, "", right, state} + else + {left, state} = quoted_to_algebra(left_arg, context, state) + {right, state} = quoted_to_algebra(right_arg, context, state) + left = wrap_in_parens_if_binary_operator(left, left_arg) + {left, " =>", right, state} + end + + doc = + with_next_break_fits(next_break_fits?(right_arg, state), right, fn right -> + concat(group(left), group(nest(glue(op, group(right)), 2, :break))) + end) + + {doc, state} + end + + # #PID's and #Ref's may appear on regular AST + defp quoted_to_algebra(unknown, _context, state) do + {inspect(unknown), state} + end + + ## Blocks + + defp block_to_algebra([{:->, _, _} | _] = type_fun, min_line, max_line, state) do + type_fun_to_algebra(type_fun, min_line, max_line, state) + end + + defp block_to_algebra({:__block__, _, []}, min_line, max_line, state) do + block_args_to_algebra([], min_line, max_line, state) + end + + defp block_to_algebra({:__block__, _, [_, _ | _] = args}, min_line, max_line, state) do + block_args_to_algebra(args, min_line, max_line, state) + end + + defp block_to_algebra(block, min_line, max_line, state) do + block_args_to_algebra([block], min_line, max_line, state) + end + + defp block_args_to_algebra(args, min_line, max_line, state) do + quoted_to_algebra = fn {kind, meta, _} = arg, _args, state -> + newlines = meta[:end_of_expression][:newlines] || 1 + {doc, state} = quoted_to_algebra(arg, :block, state) + {{doc, block_next_line(kind), newlines}, state} + end + + {args_docs, _comments?, state} = + quoted_to_algebra_with_comments(args, [], min_line, max_line, state, quoted_to_algebra) + + case args_docs do + [] -> {@empty, state} + [line] -> {line, state} + lines -> {lines |> Enum.reduce(&line(&2, &1)) |> force_unfit(), state} + end + end + + defp block_next_line(:@), do: @empty + defp block_next_line(_), do: break("") + + ## Operators + + defp maybe_unary_op_to_algebra(fun, meta, args, context, state) do + with [arg] <- args, + {_, _} <- Code.Identifier.unary_op(fun) do + unary_op_to_algebra(fun, meta, arg, context, state) + else + _ -> :error + end + end + + defp unary_op_to_algebra(op, _meta, arg, context, state) do + {doc, state} = quoted_to_algebra(arg, force_many_args_or_operand(context, :operand), state) + + # not and ! are nestable, all others are not. + doc = + case arg do + {^op, _, [_]} when op in [:!, :not] -> doc + _ -> wrap_in_parens_if_operator(doc, arg) + end + + # not requires a space unless the doc was wrapped in parens. + op_string = + if op == :not do + "not " + else + Atom.to_string(op) + end + + {concat(op_string, doc), state} + end + + defp maybe_binary_op_to_algebra(fun, meta, args, context, state) do + with [left, right] <- args, + {_, _} <- Code.Identifier.binary_op(fun) do + binary_op_to_algebra(fun, Atom.to_string(fun), meta, left, right, context, state) + else + _ -> :error + end + end + + # There are five kinds of operators. + # + # 1. no space binary operators, for example, 1..2 + # 2. no newline binary operators, for example, left in right + # 3. strict newlines before a left precedent operator, for example, foo |> bar |> baz + # 4. strict newlines before a right precedent operator, for example, foo when bar when baz + # 5. flex newlines after the operator, for example, foo ++ bar ++ baz + # + # Cases 1, 2 and 5 are handled fairly easily by relying on the + # operator precedence and making sure nesting is applied only once. + # + # Cases 3 and 4 are the complex ones, as it requires passing the + # strict or flex mode around. + defp binary_op_to_algebra(op, op_string, meta, left_arg, right_arg, context, state) do + %{operand_nesting: nesting} = state + binary_op_to_algebra(op, op_string, meta, left_arg, right_arg, context, state, nesting) + end + + defp binary_op_to_algebra(op, op_string, meta, left_arg, right_arg, context, state, _nesting) + when op in @right_new_line_before_binary_operators do + op_info = Code.Identifier.binary_op(op) + op_string = op_string <> " " + left_context = left_op_context(context) + right_context = right_op_context(context) + + min_line = + case left_arg do + {_, left_meta, _} -> line(left_meta) + _ -> line(meta) + end + + {operands, max_line} = + unwrap_right(right_arg, op, meta, right_context, [{{:root, left_context}, left_arg}]) + + fun = fn + {{:root, context}, arg}, _args, state -> + {doc, state} = binary_operand_to_algebra(arg, context, state, op, op_info, :left, 2) + {{doc, @empty, 1}, state} + + {{kind, context}, arg}, _args, state -> + {doc, state} = binary_operand_to_algebra(arg, context, state, op, op_info, kind, 0) + doc = doc |> nest_by_length(op_string) |> force_keyword(arg) + {{concat(op_string, doc), @empty, 1}, state} + end + + {doc, state} = + operand_to_algebra_with_comments(operands, meta, min_line, max_line, context, state, fun) + + if keyword?(right_arg) and context in [:parens_arg, :no_parens_arg] do + {wrap_in_parens(doc), state} + else + {doc, state} + end + end + + defp binary_op_to_algebra(op, _, meta, left_arg, right_arg, context, state, _nesting) + when op in @pipeline_operators do + op_info = Code.Identifier.binary_op(op) + left_context = left_op_context(context) + right_context = right_op_context(context) + max_line = line(meta) + + {pipes, min_line} = + unwrap_pipes(left_arg, meta, left_context, [{{op, right_context}, right_arg}]) + + fun = fn + {{:root, context}, arg}, _args, state -> + {doc, state} = binary_operand_to_algebra(arg, context, state, op, op_info, :left, 2) + {{doc, @empty, 1}, state} + + {{op, context}, arg}, _args, state -> + op_info = Code.Identifier.binary_op(op) + op_string = Atom.to_string(op) <> " " + {doc, state} = binary_operand_to_algebra(arg, context, state, op, op_info, :right, 0) + {{concat(op_string, doc), @empty, 1}, state} + end + + operand_to_algebra_with_comments(pipes, meta, min_line, max_line, context, state, fun) + end + + defp binary_op_to_algebra(op, op_string, meta, left_arg, right_arg, context, state, nesting) do + op_info = Code.Identifier.binary_op(op) + left_context = left_op_context(context) + right_context = right_op_context(context) + + {left, state} = + binary_operand_to_algebra(left_arg, left_context, state, op, op_info, :left, 2) + + {right, state} = + binary_operand_to_algebra(right_arg, right_context, state, op, op_info, :right, 0) + + doc = + cond do + op in @no_space_binary_operators -> + concat(concat(group(left), op_string), group(right)) + + op in @no_newline_binary_operators -> + op_string = " " <> op_string <> " " + concat(concat(group(left), op_string), group(right)) + + true -> + eol? = eol?(meta, state) + + next_break_fits? = + op in @next_break_fits_operators and next_break_fits?(right_arg, state) and not eol? + + with_next_break_fits(next_break_fits?, right, fn right -> + op_string = " " <> op_string + right = nest(glue(op_string, group(right)), nesting, :break) + right = if eol?, do: force_unfit(right), else: right + concat(group(left), group(right)) + end) + end + + {doc, state} + end + + # TODO: We can remove this workaround once we remove + # ?rearrange_uop from the parser on v2.0. + # (! left) in right + # (not left) in right + defp binary_operand_to_algebra( + {:__block__, _, [{op, meta, [arg]}]}, + context, + state, + :in, + _parent_info, + :left, + _nesting + ) + when op in [:not, :!] do + {doc, state} = unary_op_to_algebra(op, meta, arg, context, state) + {wrap_in_parens(doc), state} + end + + defp binary_operand_to_algebra(operand, context, state, parent_op, parent_info, side, nesting) do + {parent_assoc, parent_prec} = parent_info + + with {op, meta, [left, right]} <- operand, + op_info = Code.Identifier.binary_op(op), + {_assoc, prec} <- op_info do + op_string = Atom.to_string(op) + + cond do + # If the operator has the same precedence as the parent and is on + # the correct side, we respect the nesting rule to avoid multiple + # nestings. This only applies for left associativity or same operator. + parent_prec == prec and parent_assoc == side and (side == :left or op == parent_op) -> + binary_op_to_algebra(op, op_string, meta, left, right, context, state, nesting) + + # If the parent requires parens or the precedence is inverted or + # it is in the wrong side, then we *need* parenthesis. + (parent_op in @required_parens_on_binary_operands and op not in @no_space_binary_operators) or + (op in @required_parens_logical_binary_operands and + parent_op in @required_parens_logical_binary_operands) or parent_prec > prec or + (parent_prec == prec and parent_assoc != side) -> + {operand, state} = + binary_op_to_algebra(op, op_string, meta, left, right, context, state, 2) + + {wrap_in_parens(operand), state} + + # Otherwise, we rely on precedence but also nest. + true -> + binary_op_to_algebra(op, op_string, meta, left, right, context, state, 2) + end + else + {:&, _, [arg]} + when not is_integer(arg) and side == :left + when not is_integer(arg) and parent_assoc == :left and parent_prec > @ampersand_prec -> + {doc, state} = quoted_to_algebra(operand, context, state) + {wrap_in_parens(doc), state} + + _ -> + quoted_to_algebra(operand, context, state) + end + end + + defp unwrap_pipes({op, meta, [left, right]}, _meta, context, acc) + when op in @pipeline_operators do + left_context = left_op_context(context) + right_context = right_op_context(context) + unwrap_pipes(left, meta, left_context, [{{op, right_context}, right} | acc]) + end + + defp unwrap_pipes(left, meta, context, acc) do + min_line = + case left do + {_, meta, _} -> line(meta) + _ -> line(meta) + end + + {[{{:root, context}, left} | acc], min_line} + end + + defp unwrap_right({op, meta, [left, right]}, op, _meta, context, acc) do + left_context = left_op_context(context) + right_context = right_op_context(context) + unwrap_right(right, op, meta, right_context, [{{:left, left_context}, left} | acc]) + end + + defp unwrap_right(right, _op, meta, context, acc) do + acc = [{{:right, context}, right} | acc] + {Enum.reverse(acc), line(meta)} + end + + defp operand_to_algebra_with_comments(operands, meta, min_line, max_line, context, state, fun) do + # If we are in a no_parens_one_arg expression, we actually cannot + # extract comments from the first operand, because it would rewrite: + # + # @spec function(x) :: + # # Comment + # any + # when x: any + # + # to: + # + # @spec # Comment + # function(x) :: + # any + # when x: any + # + # Instead we get: + # + # @spec function(x) :: + # any + # # Comment + # when x: any + # + # Which may look counter-intuitive but it actually makes sense, + # as the closest possible location for the comment is the when + # operator. + {operands, acc, state} = + if context == :no_parens_one_arg do + [operand | operands] = operands + {doc_triplet, state} = fun.(operand, :unused, state) + {operands, [doc_triplet], state} + else + {operands, [], state} + end + + {docs, comments?, state} = + quoted_to_algebra_with_comments(operands, acc, min_line, max_line, state, fun) + + if comments? or eol?(meta, state) do + {docs |> Enum.reduce(&line(&2, &1)) |> force_unfit(), state} + else + {docs |> Enum.reduce(&glue(&2, &1)), state} + end + end + + ## Module attributes + + # @Foo + # @Foo.Bar + defp module_attribute_to_algebra(_meta, {:__aliases__, _, [_, _ | _]} = quoted, _context, state) do + {doc, state} = quoted_to_algebra(quoted, :parens_arg, state) + {concat(concat("@(", doc), ")"), state} + end + + # @foo bar + # @foo(bar) + defp module_attribute_to_algebra(meta, {name, call_meta, [_] = args} = expr, context, state) + when is_atom(name) and name not in [:__block__, :__aliases__] do + if Macro.classify_atom(name) == :identifier do + {{call_doc, state}, wrap_in_parens?} = + call_args_to_algebra(args, call_meta, context, :skip_unless_many_args, false, state) + + doc = + "@#{name}" + |> string() + |> concat(call_doc) + + doc = if wrap_in_parens?, do: wrap_in_parens(doc), else: doc + {doc, state} + else + unary_op_to_algebra(:@, meta, expr, context, state) + end + end + + # @foo + # @(foo.bar()) + defp module_attribute_to_algebra(meta, quoted, context, state) do + unary_op_to_algebra(:@, meta, quoted, context, state) + end + + ## Capture operator + + defp capture_to_algebra(integer, _context, state) when is_integer(integer) do + {"&" <> Integer.to_string(integer), state} + end + + defp capture_to_algebra(arg, context, state) do + {doc, state} = capture_target_to_algebra(arg, context, state) + + if doc |> format_to_string() |> String.starts_with?("&") do + {concat("& ", doc), state} + else + {concat("&", doc), state} + end + end + + defp capture_target_to_algebra( + {:/, _, [{{:., _, [target, fun]}, _, []}, {:__block__, _, [arity]}]}, + _context, + state + ) + when is_atom(fun) and is_integer(arity) do + {target_doc, state} = remote_target_to_algebra(target, state) + fun = Macro.inspect_atom(:remote_call, fun) + {target_doc |> nest(1) |> concat(string(".#{fun}/#{arity}")), state} + end + + defp capture_target_to_algebra( + {:/, _, [{name, _, var_context}, {:__block__, _, [arity]}]}, + _context, + state + ) + when is_atom(name) and is_atom(var_context) and is_integer(arity) do + {string("#{name}/#{arity}"), state} + end + + defp capture_target_to_algebra(arg, context, state) do + {doc, state} = quoted_to_algebra(arg, context, state) + {wrap_in_parens_if_operator(doc, arg), state} + end + + ## Calls (local, remote and anonymous) + + # expression.{arguments} + defp remote_to_algebra({{:., _, [target, :{}]}, meta, args}, _context, state) do + {target_doc, state} = remote_target_to_algebra(target, state) + {call_doc, state} = tuple_to_algebra(meta, args, :break, state) + {concat(concat(target_doc, "."), call_doc), state} + end + + # expression.(arguments) + defp remote_to_algebra({{:., _, [target]}, meta, args}, context, state) do + {target_doc, state} = remote_target_to_algebra(target, state) + + {{call_doc, state}, wrap_in_parens?} = + call_args_to_algebra(args, meta, context, :skip_if_do_end, true, state) + + doc = concat(concat(target_doc, "."), call_doc) + doc = if wrap_in_parens?, do: wrap_in_parens(doc), else: doc + {doc, state} + end + + # Mod.function() + # var.function + # expression.function(arguments) + defp remote_to_algebra({{:., _, [target, fun]}, meta, args}, context, state) + when is_atom(fun) do + {target_doc, state} = remote_target_to_algebra(target, state) + fun = Macro.inspect_atom(:remote_call, fun) + remote_doc = target_doc |> concat(".") |> concat(string(fun)) + + if args == [] and not remote_target_is_a_module?(target) and not meta?(meta, :closing) do + {remote_doc, state} + else + {{call_doc, state}, wrap_in_parens?} = + call_args_to_algebra(args, meta, context, :skip_if_do_end, true, state) + + doc = concat(remote_doc, call_doc) + doc = if wrap_in_parens?, do: wrap_in_parens(doc), else: doc + {doc, state} + end + end + + # call(call)(arguments) + defp remote_to_algebra({target, meta, args}, context, state) do + {target_doc, state} = quoted_to_algebra(target, :no_parens_arg, state) + + {{call_doc, state}, wrap_in_parens?} = + call_args_to_algebra(args, meta, context, :required, true, state) + + doc = concat(target_doc, call_doc) + doc = if wrap_in_parens?, do: wrap_in_parens(doc), else: doc + {doc, state} + end + + defp remote_target_is_a_module?(target) do + case target do + {:__MODULE__, _, context} when is_atom(context) -> true + {:__block__, _, [atom]} when is_atom(atom) -> true + {:__aliases__, _, _} -> true + _ -> false + end + end + + defp remote_target_to_algebra({:fn, _, [_ | _]} = quoted, state) do + # This change is not semantically required but for beautification. + {doc, state} = quoted_to_algebra(quoted, :no_parens_arg, state) + {wrap_in_parens(doc), state} + end + + defp remote_target_to_algebra(quoted, state) do + quoted_to_algebra_with_parens_if_operator(quoted, :no_parens_arg, state) + end + + # function(arguments) + defp local_to_algebra(fun, meta, args, context, state) when is_atom(fun) do + skip_parens = + cond do + meta?(meta, :closing) -> + :skip_if_only_do_end + + local_without_parens?(fun, length(args), state.locals_without_parens) -> + :skip_unless_many_args + + true -> + :skip_if_do_end + end + + {{call_doc, state}, wrap_in_parens?} = + call_args_to_algebra(args, meta, context, skip_parens, true, state) + + doc = + fun + |> Atom.to_string() + |> string() + |> concat(call_doc) + + doc = if wrap_in_parens?, do: wrap_in_parens(doc), else: doc + {doc, state} + end + + # parens may be one of: + # + # * :skip_unless_many_args - skips parens unless we are the argument context + # * :skip_if_only_do_end - skip parens if we are do-end and the only arg + # * :skip_if_do_end - skip parens if we are do-end + # * :required - never skip parens + # + defp call_args_to_algebra([], meta, _context, _parens, _list_to_keyword?, state) do + {args_doc, _join, state} = + args_to_algebra_with_comments([], meta, false, :none, :break, state, &{&1, &2}) + + {{surround("(", args_doc, ")"), state}, false} + end + + defp call_args_to_algebra(args, meta, context, parens, list_to_keyword?, state) do + {rest, last} = split_last(args) + + if blocks = do_end_blocks(meta, last, state) do + {call_doc, state} = + case rest do + [] when parens == :required -> + {"() do", state} + + [] -> + {" do", state} + + _ -> + no_parens? = parens not in [:required, :skip_if_only_do_end] + call_args_to_algebra_no_blocks(meta, rest, no_parens?, list_to_keyword?, " do", state) + end + + {blocks_doc, state} = do_end_blocks_to_algebra(blocks, state) + call_doc = call_doc |> concat(blocks_doc) |> line("end") |> force_unfit() + {{call_doc, state}, context in [:no_parens_arg, :no_parens_one_arg]} + else + no_parens? = + parens == :skip_unless_many_args and + context in [:block, :operand, :no_parens_one_arg, :parens_one_arg] + + res = + call_args_to_algebra_no_blocks(meta, args, no_parens?, list_to_keyword?, @empty, state) + + {res, false} + end + end + + defp call_args_to_algebra_no_blocks(meta, args, skip_parens?, list_to_keyword?, extra, state) do + {left, right} = split_last(args) + {keyword?, right} = last_arg_to_keyword(right, list_to_keyword?, skip_parens?, state.comments) + + context = + if left == [] and not keyword? do + if skip_parens?, do: :no_parens_one_arg, else: :parens_one_arg + else + if skip_parens?, do: :no_parens_arg, else: :parens_arg + end + + args = if keyword?, do: left ++ right, else: left ++ [right] + many_eol? = match?([_, _ | _], args) and eol?(meta, state) + no_generators? = no_generators?(args) + to_algebra_fun = "ed_to_algebra(&1, context, &2) + + {args_doc, next_break_fits?, state} = + if left != [] and keyword? and no_generators? do + join = if force_args?(left) or many_eol?, do: :line, else: :break + + {left_doc, _join, state} = + args_to_algebra_with_comments( + left, + Keyword.delete(meta, :closing), + skip_parens?, + :force_comma, + join, + state, + to_algebra_fun + ) + + join = if force_args?(right) or force_args?(args) or many_eol?, do: :line, else: :break + + {right_doc, _join, state} = + args_to_algebra_with_comments(right, meta, false, :none, join, state, to_algebra_fun) + + right_doc = apply(Inspect.Algebra, join, []) |> concat(right_doc) + + args_doc = + if skip_parens? do + left_doc + |> concat(next_break_fits(group(right_doc, :inherit), :enabled)) + |> nest(:cursor, :break) + else + right_doc = + right_doc + |> nest(2, :break) + |> concat(break("")) + |> group(:inherit) + |> next_break_fits(:enabled) + + concat(nest(left_doc, 2, :break), right_doc) + end + + {args_doc, true, state} + else + join = if force_args?(args) or many_eol?, do: :line, else: :break + next_break_fits? = join == :break and next_break_fits?(right, state) + last_arg_mode = if next_break_fits?, do: :next_break_fits, else: :none + + {args_doc, _join, state} = + args_to_algebra_with_comments( + args, + meta, + skip_parens?, + last_arg_mode, + join, + state, + to_algebra_fun + ) + + # If we have a single argument, then we won't have an option to break + # before the "extra" part, so we ungroup it and build it later. + args_doc = ungroup_if_group(args_doc) + + args_doc = + if skip_parens? do + nest(args_doc, :cursor, :break) + else + nest(args_doc, 2, :break) |> concat(break("")) + end + + {args_doc, next_break_fits?, state} + end + + doc = + cond do + left != [] and keyword? and skip_parens? and no_generators? -> + " " + |> concat(args_doc) + |> nest(2) + |> concat(extra) + |> group() + + skip_parens? -> + " " + |> concat(args_doc) + |> concat(extra) + |> group() + + true -> + "(" + |> concat(break("")) + |> nest(2, :break) + |> concat(args_doc) + |> concat(")") + |> concat(extra) + |> group() + end + + if next_break_fits? do + {next_break_fits(doc, :disabled), state} + else + {doc, state} + end + end + + defp no_generators?(args) do + not Enum.any?(args, &match?({:<-, _, [_, _]}, &1)) + end + + defp do_end_blocks(meta, [{{:__block__, _, [:do]}, _} | rest] = blocks, state) do + if meta?(meta, :do) or can_force_do_end_blocks?(rest, state) do + blocks + |> Enum.map(fn {{:__block__, meta, [key]}, value} -> {key, line(meta), value} end) + |> do_end_blocks_with_range(end_line(meta)) + end + end + + defp do_end_blocks(_, _, _), do: nil + + defp can_force_do_end_blocks?(rest, state) do + state.force_do_end_blocks and + Enum.all?(rest, fn {{:__block__, _, [key]}, _} -> key in @do_end_keywords end) + end + + defp do_end_blocks_with_range([{key1, line1, value1}, {_, line2, _} = h | t], end_line) do + [{key1, line1, line2, value1} | do_end_blocks_with_range([h | t], end_line)] + end + + defp do_end_blocks_with_range([{key, line, value}], end_line) do + [{key, line, end_line, value}] + end + + defp do_end_blocks_to_algebra([{:do, line, end_line, value} | blocks], state) do + {acc, state} = do_end_block_to_algebra(@empty, line, end_line, value, state) + + Enum.reduce(blocks, {acc, state}, fn {key, line, end_line, value}, {acc, state} -> + {doc, state} = do_end_block_to_algebra(Atom.to_string(key), line, end_line, value, state) + {line(acc, doc), state} + end) + end + + defp do_end_block_to_algebra(key_doc, line, end_line, value, state) do + case clauses_to_algebra(value, line, end_line, state) do + {@empty, state} -> {key_doc, state} + {value_doc, state} -> {key_doc |> line(value_doc) |> nest(2), state} + end + end + + ## Interpolation + + defp list_interpolated?(entries) do + Enum.all?(entries, fn + {{:., _, [Kernel, :to_string]}, _, [_]} -> true + entry when is_binary(entry) -> true + _ -> false + end) + end + + defp interpolated?(entries) do + Enum.all?(entries, fn + {:"::", _, [{{:., _, [Kernel, :to_string]}, _, [_]}, {:binary, _, _}]} -> true + entry when is_binary(entry) -> true + _ -> false + end) + end + + defp prepend_heredoc_line([entry | entries]) when is_binary(entry) do + ["\n" <> entry | entries] + end + + defp prepend_heredoc_line(entries) do + ["\n" | entries] + end + + defp list_interpolation_to_algebra([entry | entries], escape, state, acc, last) + when is_binary(entry) do + acc = concat(acc, escape_string(entry, escape)) + list_interpolation_to_algebra(entries, escape, state, acc, last) + end + + defp list_interpolation_to_algebra([entry | entries], escape, state, acc, last) do + {{:., _, [Kernel, :to_string]}, _meta, [quoted]} = entry + {doc, state} = interpolation_to_algebra(quoted, state) + list_interpolation_to_algebra(entries, escape, state, concat(acc, doc), last) + end + + defp list_interpolation_to_algebra([], _escape, state, acc, last) do + {concat(acc, last), state} + end + + defp interpolation_to_algebra([entry | entries], escape, state, acc, last) + when is_binary(entry) do + acc = concat(acc, escape_string(entry, escape)) + interpolation_to_algebra(entries, escape, state, acc, last) + end + + defp interpolation_to_algebra([entry | entries], escape, state, acc, last) do + {:"::", _, [{{:., _, [Kernel, :to_string]}, _meta, [quoted]}, {:binary, _, _}]} = entry + {doc, state} = interpolation_to_algebra(quoted, state) + interpolation_to_algebra(entries, escape, state, concat(acc, doc), last) + end + + defp interpolation_to_algebra([], _escape, state, acc, last) do + {concat(acc, last), state} + end + + defp interpolation_to_algebra(quoted, %{skip_eol: skip_eol} = state) do + {doc, state} = block_to_algebra(quoted, @max_line, @min_line, %{state | skip_eol: true}) + {no_limit(surround("\#{", doc, "}")), %{state | skip_eol: skip_eol}} + end + + ## Sigils + + defp maybe_sigil_to_algebra(fun, meta, args, state) do + with <<"sigil_", name>> <- Atom.to_string(fun), + [{:<<>>, _, entries}, modifiers] when is_list(modifiers) <- args, + opening_delimiter when not is_nil(opening_delimiter) <- meta[:delimiter] do + doc = <> + + entries = + case state.sigils do + %{^name => callback} -> + metadata = [ + file: state.file, + line: meta[:line], + sigil: List.to_atom([name]), + modifiers: modifiers, + opening_delimiter: opening_delimiter + ] + + case callback.(hd(entries), metadata) do + iodata when is_binary(iodata) or is_list(iodata) -> + [IO.iodata_to_binary(iodata)] + + other -> + raise ArgumentError, + "expected sigil callback to return iodata, got: #{inspect(other)}" + end + + %{} -> + entries + end + + if opening_delimiter in [@double_heredoc, @single_heredoc] do + closing_delimiter = concat(opening_delimiter, List.to_string(modifiers)) + + {doc, state} = + entries + |> prepend_heredoc_line() + |> interpolation_to_algebra(opening_delimiter, state, doc, closing_delimiter) + + {force_unfit(doc), state} + else + escape = closing_sigil_delimiter(opening_delimiter) + closing_delimiter = concat(escape, List.to_string(modifiers)) + interpolation_to_algebra(entries, escape, state, doc, closing_delimiter) + end + else + _ -> + :error + end + end + + defp closing_sigil_delimiter("("), do: ")" + defp closing_sigil_delimiter("["), do: "]" + defp closing_sigil_delimiter("{"), do: "}" + defp closing_sigil_delimiter("<"), do: ">" + defp closing_sigil_delimiter(other) when other in ["\"", "'", "|", "/"], do: other + + ## Bitstrings + + defp bitstring_to_algebra(meta, args, state) do + last = length(args) - 1 + join = if eol?(meta, state), do: :line, else: :flex_break + to_algebra_fun = &bitstring_segment_to_algebra(&1, &2, last) + + {args_doc, join, state} = + args + |> Enum.with_index() + |> args_to_algebra_with_comments(meta, false, :none, join, state, to_algebra_fun) + + if join == :flex_break do + {"<<" |> concat(args_doc) |> nest(2) |> concat(">>") |> group(), state} + else + {surround("<<", args_doc, ">>"), state} + end + end + + defp bitstring_segment_to_algebra({{:<-, meta, [left, right]}, i}, state, last) do + left = {{:special, :bitstring_segment}, meta, [left, last]} + {doc, state} = quoted_to_algebra({:<-, meta, [left, right]}, :parens_arg, state) + {bitstring_wrap_parens(doc, i, last), state} + end + + defp bitstring_segment_to_algebra({{:"::", _, [segment, spec]}, i}, state, last) do + {doc, state} = quoted_to_algebra(segment, :parens_arg, state) + {spec, state} = bitstring_spec_to_algebra(spec, state) + + spec = wrap_in_parens_if_inspected_atom(spec) + spec = if i == last, do: bitstring_wrap_parens(spec, i, last), else: spec + + doc = + doc + |> bitstring_wrap_parens(i, -1) + |> concat("::") + |> concat(spec) + + {doc, state} + end + + defp bitstring_segment_to_algebra({segment, i}, state, last) do + {doc, state} = quoted_to_algebra(segment, :parens_arg, state) + {bitstring_wrap_parens(doc, i, last), state} + end + + defp bitstring_spec_to_algebra({op, _, [left, right]}, state) when op in [:-, :*] do + {left, state} = bitstring_spec_to_algebra(left, state) + {right, state} = bitstring_spec_element_to_algebra(right, state) + {concat(concat(left, Atom.to_string(op)), right), state} + end + + defp bitstring_spec_to_algebra(spec, state) do + bitstring_spec_element_to_algebra(spec, state) + end + + defp bitstring_spec_element_to_algebra( + {atom, meta, empty_args}, + state = %{normalize_bitstring_modifiers: true} + ) + when is_atom(atom) and empty_args in [nil, []] do + empty_args = bitstring_spec_normalize_empty_args(atom) + quoted_to_algebra_with_parens_if_operator({atom, meta, empty_args}, :parens_arg, state) + end + + defp bitstring_spec_element_to_algebra(spec_element, state) do + quoted_to_algebra_with_parens_if_operator(spec_element, :parens_arg, state) + end + + defp bitstring_spec_normalize_empty_args(atom) when atom in @bitstring_modifiers, do: nil + defp bitstring_spec_normalize_empty_args(_atom), do: [] + + defp bitstring_wrap_parens(doc, i, last) when i == 0 or i == last do + string = format_to_string(doc) + + if (i == 0 and String.starts_with?(string, ["~", "<<"])) or + (i == last and String.ends_with?(string, [">>"])) do + wrap_in_parens(doc) + else + doc + end + end + + defp bitstring_wrap_parens(doc, _, _), do: doc + + ## Literals + + defp list_to_algebra(meta, args, state) do + join = if eol?(meta, state), do: :line, else: :break + fun = "ed_to_algebra(&1, :parens_arg, &2) + + {args_doc, _join, state} = + args_to_algebra_with_comments(args, meta, false, :none, join, state, fun) + + {surround("[", args_doc, "]"), state} + end + + defp map_to_algebra(meta, name_doc, [{:|, _, [left, right]}], state) do + join = if eol?(meta, state), do: :line, else: :break + fun = "ed_to_algebra(&1, :parens_arg, &2) + {left_doc, state} = fun.(left, state) + + {right_doc, _join, state} = + args_to_algebra_with_comments(right, meta, false, :none, join, state, fun) + + args_doc = + left_doc + |> wrap_in_parens_if_binary_operator(left) + |> glue(concat("| ", nest(right_doc, 2))) + + name_doc = "%" |> concat(name_doc) |> concat("{") + {surround(name_doc, args_doc, "}"), state} + end + + defp map_to_algebra(meta, name_doc, args, state) do + join = if eol?(meta, state), do: :line, else: :break + fun = "ed_to_algebra(&1, :parens_arg, &2) + + {args_doc, _join, state} = + args_to_algebra_with_comments(args, meta, false, :none, join, state, fun) + + name_doc = "%" |> concat(name_doc) |> concat("{") + {surround(name_doc, args_doc, "}"), state} + end + + defp tuple_to_algebra(meta, args, join, state) do + join = if eol?(meta, state), do: :line, else: join + fun = "ed_to_algebra(&1, :parens_arg, &2) + + {args_doc, join, state} = + args_to_algebra_with_comments(args, meta, false, :none, join, state, fun) + + if join == :flex_break do + {"{" |> concat(args_doc) |> nest(1) |> concat("}") |> group(), state} + else + {surround("{", args_doc, "}"), state} + end + end + + defp atom_to_algebra(atom, _) when atom in [nil, true, false] do + Atom.to_string(atom) + end + + # TODO: Remove this clause in v1.16 when we no longer quote operator :..// + defp atom_to_algebra(:"..//", _) do + string(":\"..//\"") + end + + defp atom_to_algebra(:\\, meta) do + # Since we parse strings without unescaping, the atoms + # :\\ and :"\\" have the same representation, so we need + # to check the delimiter and handle them accordingly. + string = + case Keyword.get(meta, :delimiter) do + "\"" -> ":\"\\\\\"" + _ -> ":\\\\" + end + + string(string) + end + + defp atom_to_algebra(atom, _) do + string = Atom.to_string(atom) + + iodata = + if Macro.classify_atom(atom) in [:unquoted, :identifier] do + [?:, string] + else + [?:, ?", String.replace(string, "\"", "\\\""), ?"] + end + + iodata |> IO.iodata_to_binary() |> string() + end + + defp integer_to_algebra(text) do + case text do + <> -> + "0x" <> String.upcase(rest) + + <> = digits when base in [?b, ?o] -> + digits + + <> = char -> + char + + decimal -> + insert_underscores(decimal) + end + end + + defp float_to_algebra(text) do + [int_part, decimal_part] = :binary.split(text, ".") + decimal_part = String.downcase(decimal_part) + insert_underscores(int_part) <> "." <> decimal_part + end + + defp insert_underscores(digits) do + cond do + digits =~ "_" -> + digits + + byte_size(digits) >= 6 -> + digits + |> String.to_charlist() + |> Enum.reverse() + |> Enum.chunk_every(3) + |> Enum.intersperse('_') + |> List.flatten() + |> Enum.reverse() + |> List.to_string() + + true -> + digits + end + end + + defp escape_heredoc(string, escape) do + string = String.replace(string, escape, "\\" <> escape) + heredoc_to_algebra(["" | String.split(string, "\n")]) + end + + defp escape_string(string, <<_, _, _>> = escape) do + string = String.replace(string, escape, "\\" <> escape) + heredoc_to_algebra(String.split(string, "\n")) + end + + defp escape_string(string, escape) when is_binary(escape) do + string + |> String.replace(escape, "\\" <> escape) + |> String.split("\n") + |> Enum.reverse() + |> Enum.map(&string/1) + |> Enum.reduce(&concat(&1, concat(nest(line(), :reset), &2))) + end + + defp heredoc_to_algebra([string]) do + string(string) + end + + defp heredoc_to_algebra(["" | rest]) do + rest + |> heredoc_line() + |> concat(heredoc_to_algebra(rest)) + end + + defp heredoc_to_algebra([string | rest]) do + string + |> string() + |> concat(heredoc_line(rest)) + |> concat(heredoc_to_algebra(rest)) + end + + defp heredoc_line(["", _ | _]), do: nest(line(), :reset) + defp heredoc_line(_), do: line() + + defp args_to_algebra_with_comments(args, meta, skip_parens?, last_arg_mode, join, state, fun) do + min_line = line(meta) + max_line = closing_line(meta) + + arg_to_algebra = fn arg, args, state -> + {doc, state} = fun.(arg, state) + + doc = + case args do + [_ | _] -> concat_to_last_group(doc, ",") + [] when last_arg_mode == :force_comma -> concat_to_last_group(doc, ",") + [] when last_arg_mode == :next_break_fits -> next_break_fits(doc, :enabled) + [] when last_arg_mode == :none -> doc + end + + {{doc, @empty, 1}, state} + end + + # If skipping parens, we cannot extract the comments of the first + # argument as there is no place to move them to, so we handle it now. + {args, acc, state} = + case args do + [head | tail] when skip_parens? -> + {doc_triplet, state} = arg_to_algebra.(head, tail, state) + {tail, [doc_triplet], state} + + _ -> + {args, [], state} + end + + {args_docs, comments?, state} = + quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, arg_to_algebra) + + cond do + args_docs == [] -> + {@empty, :empty, state} + + join == :line or comments? -> + {args_docs |> Enum.reduce(&line(&2, &1)) |> force_unfit(), :line, state} + + join == :break -> + {args_docs |> Enum.reduce(&glue(&2, &1)), :break, state} + + join == :flex_break -> + {args_docs |> Enum.reduce(&flex_glue(&2, &1)), :flex_break, state} + end + end + + ## Anonymous functions + + # fn -> block end + defp anon_fun_to_algebra( + [{:->, meta, [[], body]}] = clauses, + _min_line, + max_line, + state, + _multi_clauses_style + ) do + min_line = line(meta) + {body_doc, state} = block_to_algebra(body, min_line, max_line, state) + break_or_line = clause_break_or_line(clauses, state) + + doc = + "fn ->" + |> concat(break_or_line) + |> concat(body_doc) + |> nest(2) + |> concat(break_or_line) + |> concat("end") + |> maybe_force_clauses(clauses, state) + |> group() + + {doc, state} + end + + # fn x -> y end + # fn x -> + # y + # end + defp anon_fun_to_algebra( + [{:->, meta, [args, body]}] = clauses, + _min_line, + max_line, + state, + false = _multi_clauses_style + ) do + min_line = line(meta) + {args_doc, state} = clause_args_to_algebra(args, min_line, state) + {body_doc, state} = block_to_algebra(body, min_line, max_line, state) + + head = + args_doc + |> ungroup_if_group() + |> concat(" ->") + |> nest(:cursor) + |> group() + + break_or_line = clause_break_or_line(clauses, state) + + doc = + "fn " + |> concat(head) + |> concat(break_or_line) + |> concat(body_doc) + |> nest(2) + |> concat(break_or_line) + |> concat("end") + |> maybe_force_clauses(clauses, state) + |> group() + + {doc, state} + end + + # fn + # args1 -> + # block1 + # args2 -> + # block2 + # end + defp anon_fun_to_algebra(clauses, min_line, max_line, state, _multi_clauses_style) do + {clauses_doc, state} = clauses_to_algebra(clauses, min_line, max_line, state) + {"fn" |> line(clauses_doc) |> nest(2) |> line("end") |> force_unfit(), state} + end + + ## Type functions + + # (() -> block) + defp type_fun_to_algebra([{:->, meta, [[], body]}] = clauses, _min_line, max_line, state) do + min_line = line(meta) + {body_doc, state} = block_to_algebra(body, min_line, max_line, state) + + doc = + "(() -> " + |> concat(nest(body_doc, :cursor)) + |> concat(")") + |> maybe_force_clauses(clauses, state) + |> group() + + {doc, state} + end + + # (x -> y) + # (x -> + # y) + defp type_fun_to_algebra([{:->, meta, [args, body]}] = clauses, _min_line, max_line, state) do + min_line = line(meta) + {args_doc, state} = clause_args_to_algebra(args, min_line, state) + {body_doc, state} = block_to_algebra(body, min_line, max_line, state) + break_or_line = clause_break_or_line(clauses, state) + + doc = + args_doc + |> ungroup_if_group() + |> concat(" ->") + |> group() + |> concat(break_or_line |> concat(body_doc) |> nest(2)) + |> wrap_in_parens() + |> maybe_force_clauses(clauses, state) + |> group() + + {doc, state} + end + + # ( + # args1 -> + # block1 + # args2 -> + # block2 + # ) + defp type_fun_to_algebra(clauses, min_line, max_line, state) do + {clauses_doc, state} = clauses_to_algebra(clauses, min_line, max_line, state) + {"(" |> line(clauses_doc) |> nest(2) |> line(")") |> force_unfit(), state} + end + + ## Clauses + + defp multi_line_clauses?(clauses, state) do + Enum.any?(clauses, fn {:->, meta, [_, block]} -> + eol?(meta, state) or multi_line_block?(block) + end) + end + + defp multi_line_block?({:__block__, _, [_, _ | _]}), do: true + defp multi_line_block?(_), do: false + + defp clause_break_or_line(clauses, state) do + if multi_line_clauses?(clauses, state), do: line(), else: break() + end + + defp maybe_force_clauses(doc, clauses, state) do + if multi_line_clauses?(clauses, state), do: force_unfit(doc), else: doc + end + + defp clauses_to_algebra([{:->, _, _} | _] = clauses, min_line, max_line, state) do + [clause | clauses] = add_max_line_to_last_clause(clauses, max_line) + {clause_doc, state} = clause_to_algebra(clause, min_line, state) + + {clauses_doc, state} = + Enum.reduce(clauses, {clause_doc, state}, fn clause, {doc_acc, state_acc} -> + {clause_doc, state_acc} = clause_to_algebra(clause, min_line, state_acc) + + doc_acc = + doc_acc + |> concat(maybe_empty_line()) + |> line(clause_doc) + + {doc_acc, state_acc} + end) + + {clauses_doc |> maybe_force_clauses([clause | clauses], state) |> group(), state} + end + + defp clauses_to_algebra(other, min_line, max_line, state) do + case block_to_algebra(other, min_line, max_line, state) do + {@empty, state} -> {@empty, state} + {doc, state} -> {group(doc), state} + end + end + + defp clause_to_algebra({:->, meta, [[], body]}, _min_line, state) do + {body_doc, state} = block_to_algebra(body, line(meta), closing_line(meta), state) + {"() ->" |> glue(body_doc) |> nest(2), state} + end + + defp clause_to_algebra({:->, meta, [args, body]}, min_line, state) do + %{operand_nesting: nesting} = state + + state = %{state | operand_nesting: nesting + 2} + {args_doc, state} = clause_args_to_algebra(args, min_line, state) + + state = %{state | operand_nesting: nesting} + {body_doc, state} = block_to_algebra(body, min_line, closing_line(meta), state) + + doc = + args_doc + |> ungroup_if_group() + |> concat(" ->") + |> group() + |> concat(break() |> concat(body_doc) |> nest(2)) + + {doc, state} + end + + defp add_max_line_to_last_clause([{op, meta, args}], max_line) do + [{op, [closing: [line: max_line]] ++ meta, args}] + end + + defp add_max_line_to_last_clause([clause | clauses], max_line) do + [clause | add_max_line_to_last_clause(clauses, max_line)] + end + + defp clause_args_to_algebra(args, min_line, state) do + arg_to_algebra = fn arg, _args, state -> + {doc, state} = clause_args_to_algebra(arg, state) + {{doc, @empty, 1}, state} + end + + {args_docs, comments?, state} = + quoted_to_algebra_with_comments([args], [], min_line, @min_line, state, arg_to_algebra) + + if comments? do + {Enum.reduce(args_docs, &line(&2, &1)), state} + else + {Enum.reduce(args_docs, &glue(&2, &1)), state} + end + end + + # fn a, b, c when d -> e end + defp clause_args_to_algebra([{:when, meta, args}], state) do + {args, right} = split_last(args) + left = {{:special, :clause_args}, meta, [args]} + binary_op_to_algebra(:when, "when", meta, left, right, :no_parens_arg, state) + end + + # fn () -> e end + defp clause_args_to_algebra([], state) do + {"()", state} + end + + # fn a, b, c -> e end + defp clause_args_to_algebra(args, state) do + many_args_to_algebra(args, state, "ed_to_algebra(&1, :no_parens_arg, &2)) + end + + ## Quoted helpers for comments + + defp quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, fun) do + {pre_comments, state} = + get_and_update_in(state.comments, fn comments -> + Enum.split_while(comments, fn %{line: line} -> line <= min_line end) + end) + + {docs, comments?, state} = + each_quoted_to_algebra_with_comments(args, acc, max_line, state, false, fun) + + {docs, comments?, update_in(state.comments, &(pre_comments ++ &1))} + end + + defp each_quoted_to_algebra_with_comments([], acc, max_line, state, comments?, _fun) do + {acc, comments, comments?} = extract_comments_before(max_line, acc, state.comments, comments?) + args_docs = merge_algebra_with_comments(Enum.reverse(acc), @empty) + {args_docs, comments?, %{state | comments: comments}} + end + + defp each_quoted_to_algebra_with_comments([arg | args], acc, max_line, state, comments?, fun) do + case traverse_line(arg, {@max_line, @min_line}) do + {@max_line, @min_line} -> + {doc_triplet, state} = fun.(arg, args, state) + acc = [doc_triplet | acc] + each_quoted_to_algebra_with_comments(args, acc, max_line, state, comments?, fun) + + {doc_start, doc_end} -> + {acc, comments, comments?} = + extract_comments_before(doc_start, acc, state.comments, comments?) + + {doc_triplet, state} = fun.(arg, args, %{state | comments: comments}) + + {acc, comments, comments?} = + extract_comments_trailing(doc_start, doc_end, acc, state.comments, comments?) + + acc = [adjust_trailing_newlines(doc_triplet, doc_end, comments) | acc] + state = %{state | comments: comments} + each_quoted_to_algebra_with_comments(args, acc, max_line, state, comments?, fun) + end + end + + defp extract_comments_before(max, acc, [%{line: line} = comment | rest], _) when line < max do + %{previous_eol_count: previous, next_eol_count: next, text: doc} = comment + acc = [{doc, @empty, next} | add_previous_to_acc(acc, previous)] + extract_comments_before(max, acc, rest, true) + end + + defp extract_comments_before(_max, acc, rest, comments?) do + {acc, rest, comments?} + end + + defp add_previous_to_acc([{doc, next_line, newlines} | acc], previous) when newlines < previous, + do: [{doc, next_line, previous} | acc] + + defp add_previous_to_acc(acc, _previous), + do: acc + + defp extract_comments_trailing(min, max, acc, [%{line: line, text: doc_comment} | rest], _) + when line >= min and line <= max do + acc = [{doc_comment, @empty, 1} | acc] + extract_comments_trailing(min, max, acc, rest, true) + end + + defp extract_comments_trailing(_min, _max, acc, rest, comments?) do + {acc, rest, comments?} + end + + # If the document is immediately followed by comment which is followed by newlines, + # its newlines wouldn't have considered the comment, so we need to adjust it. + defp adjust_trailing_newlines({doc, next_line, newlines}, doc_end, [%{line: line} | _]) + when newlines > 1 and line == doc_end + 1 do + {doc, next_line, 1} + end + + defp adjust_trailing_newlines(doc_triplet, _, _), do: doc_triplet + + defp traverse_line({expr, meta, args}, {min, max}) do + acc = + case Keyword.fetch(meta, :line) do + {:ok, line} -> {min(line, min), max(line, max)} + :error -> {min, max} + end + + traverse_line(args, traverse_line(expr, acc)) + end + + defp traverse_line({left, right}, acc) do + traverse_line(right, traverse_line(left, acc)) + end + + defp traverse_line(args, acc) when is_list(args) do + Enum.reduce(args, acc, &traverse_line/2) + end + + defp traverse_line(_, acc) do + acc + end + + # Below are the rules for line rendering in the formatter: + # + # 1. respect the user's choice + # 2. and add empty lines around expressions that take multiple lines + # (except for module attributes) + # 3. empty lines are collapsed as to not exceed more than one + # + defp merge_algebra_with_comments([{doc, next_line, newlines} | docs], left) do + right = if newlines >= @newlines, do: line(), else: next_line + + doc = + if left != @empty do + concat(left, doc) + else + doc + end + + doc = + if docs != [] and right != @empty do + concat(doc, concat(collapse_lines(2), right)) + else + doc + end + + [group(doc) | merge_algebra_with_comments(docs, right)] + end + + defp merge_algebra_with_comments([], _) do + [] + end + + ## Quoted helpers + + defp left_op_context(context), do: force_many_args_or_operand(context, :parens_arg) + defp right_op_context(context), do: force_many_args_or_operand(context, :operand) + + defp force_many_args_or_operand(:no_parens_one_arg, _choice), do: :no_parens_arg + defp force_many_args_or_operand(:parens_one_arg, _choice), do: :parens_arg + defp force_many_args_or_operand(:no_parens_arg, _choice), do: :no_parens_arg + defp force_many_args_or_operand(:parens_arg, _choice), do: :parens_arg + defp force_many_args_or_operand(:operand, choice), do: choice + defp force_many_args_or_operand(:block, choice), do: choice + + defp quoted_to_algebra_with_parens_if_operator(ast, context, state) do + {doc, state} = quoted_to_algebra(ast, context, state) + {wrap_in_parens_if_operator(doc, ast), state} + end + + defp wrap_in_parens_if_operator(doc, {:__block__, _, [expr]}) do + wrap_in_parens_if_operator(doc, expr) + end + + defp wrap_in_parens_if_operator(doc, quoted) do + if operator?(quoted) and not module_attribute_read?(quoted) and not integer_capture?(quoted) do + wrap_in_parens(doc) + else + doc + end + end + + defp wrap_in_parens_if_binary_operator(doc, quoted) do + if binary_operator?(quoted) do + wrap_in_parens(doc) + else + doc + end + end + + defp wrap_in_parens_if_inspected_atom(":" <> _ = doc) do + "(" <> doc <> ")" + end + + defp wrap_in_parens_if_inspected_atom(doc) do + doc + end + + defp wrap_in_parens(doc) do + concat(concat("(", nest(doc, :cursor)), ")") + end + + defp many_args_to_algebra([arg | args], state, fun) do + Enum.reduce(args, fun.(arg, state), fn arg, {doc_acc, state_acc} -> + {arg_doc, state_acc} = fun.(arg, state_acc) + {glue(concat(doc_acc, ","), arg_doc), state_acc} + end) + end + + defp module_attribute_read?({:@, _, [{var, _, var_context}]}) + when is_atom(var) and is_atom(var_context) do + Macro.classify_atom(var) == :identifier + end + + defp module_attribute_read?(_), do: false + + defp integer_capture?({:&, _, [integer]}) when is_integer(integer), do: true + defp integer_capture?(_), do: false + + defp operator?(quoted) do + unary_operator?(quoted) or binary_operator?(quoted) + end + + defp binary_operator?(quoted) do + case quoted do + {op, _, [_, _, _]} when op in @multi_binary_operators -> true + {op, _, [_, _]} when is_atom(op) -> Code.Identifier.binary_op(op) != :error + _ -> false + end + end + + defp unary_operator?(quoted) do + case quoted do + {op, _, [_]} when is_atom(op) -> Code.Identifier.unary_op(op) != :error + _ -> false + end + end + + defp with_next_break_fits(condition, doc, fun) do + if condition do + doc + |> next_break_fits(:enabled) + |> fun.() + |> next_break_fits(:disabled) + else + fun.(doc) + end + end + + defp next_break_fits?({:{}, meta, _args}, state) do + eol_or_comments?(meta, state) + end + + defp next_break_fits?({:__block__, meta, [{_, _}]}, state) do + eol_or_comments?(meta, state) + end + + defp next_break_fits?({:<<>>, meta, [_ | _] = entries}, state) do + meta[:delimiter] == ~s["""] or + (not interpolated?(entries) and eol_or_comments?(meta, state)) + end + + defp next_break_fits?({{:., _, [List, :to_charlist]}, meta, [[_ | _]]}, _state) do + meta[:delimiter] == ~s['''] + end + + defp next_break_fits?({{:., _, [_left, :{}]}, _, _}, _state) do + true + end + + defp next_break_fits?({:__block__, meta, [string]}, _state) when is_binary(string) do + meta[:delimiter] == ~s["""] + end + + defp next_break_fits?({:__block__, meta, [list]}, _state) when is_list(list) do + meta[:delimiter] != ~s['] + end + + defp next_break_fits?({form, _, [_ | _]}, _state) when form in [:fn, :%{}, :%] do + true + end + + defp next_break_fits?({fun, meta, args}, _state) when is_atom(fun) and is_list(args) do + meta[:delimiter] in [@double_heredoc, @single_heredoc] and + fun |> Atom.to_string() |> String.starts_with?("sigil_") + end + + defp next_break_fits?({{:__block__, _, [atom]}, expr}, state) when is_atom(atom) do + next_break_fits?(expr, state) + end + + defp next_break_fits?(_, _state) do + false + end + + defp eol_or_comments?(meta, %{comments: comments} = state) do + eol?(meta, state) or + ( + min_line = line(meta) + max_line = closing_line(meta) + Enum.any?(comments, fn %{line: line} -> line > min_line and line < max_line end) + ) + end + + # A literal list is a keyword or (... -> ...) + defp last_arg_to_keyword([_ | _] = arg, _list_to_keyword?, _skip_parens?, _comments) do + {keyword?(arg), arg} + end + + # This is a list of tuples, it can be converted to keywords. + defp last_arg_to_keyword( + {:__block__, meta, [[_ | _] = arg]} = block, + true, + skip_parens?, + comments + ) do + cond do + not keyword?(arg) -> + {false, block} + + skip_parens? -> + block_line = line(meta) + {{_, arg_meta, _}, _} = hd(arg) + first_line = line(arg_meta) + + case Enum.drop_while(comments, fn %{line: line} -> line <= block_line end) do + [%{line: line} | _] when line <= first_line -> + {false, block} + + _ -> + {true, arg} + end + + true -> + {true, arg} + end + end + + # Otherwise we don't have a keyword. + defp last_arg_to_keyword(arg, _list_to_keyword?, _skip_parens?, _comments) do + {false, arg} + end + + defp force_args?(args) do + match?([_ | _], args) and force_args?(args, %{}) + end + + defp force_args?([[arg | _] | args], lines) do + force_args?([arg | args], lines) + end + + defp force_args?([arg | args], lines) do + line = + case arg do + {{_, meta, _}, _} -> meta[:line] + {_, meta, _} -> meta[:line] + end + + cond do + # Line may be missing from non-formatter AST + is_nil(line) -> force_args?(args, lines) + Map.has_key?(lines, line) -> false + true -> force_args?(args, Map.put(lines, line, true)) + end + end + + defp force_args?([], lines), do: map_size(lines) >= 2 + + defp force_keyword(doc, arg) do + if force_args?(arg), do: force_unfit(doc), else: doc + end + + defp keyword?([{_, _} | list]), do: keyword?(list) + defp keyword?(rest), do: rest == [] + + defp keyword_key?({:__block__, meta, [atom]}) when is_atom(atom), + do: meta[:format] == :keyword + + defp keyword_key?({{:., _, [:erlang, :binary_to_atom]}, meta, [{:<<>>, _, _}, :utf8]}), + do: meta[:format] == :keyword + + defp keyword_key?(_), + do: false + + defp eol?(_meta, %{skip_eol: true}), do: false + defp eol?(meta, _state), do: Keyword.get(meta, :newlines, 0) > 0 + + defp meta?(meta, key) do + is_list(meta[key]) + end + + defp line(meta) do + meta[:line] || @max_line + end + + defp end_line(meta) do + meta[:end][:line] || @min_line + end + + defp closing_line(meta) do + meta[:closing][:line] || @min_line + end + + ## Algebra helpers + + # Relying on the inner document is brittle and error prone. + # It would be best if we had a mechanism to apply this. + defp concat_to_last_group({:doc_cons, left, right}, concat) do + {:doc_cons, left, concat_to_last_group(right, concat)} + end + + defp concat_to_last_group({:doc_group, group, mode}, concat) do + {:doc_group, {:doc_cons, group, concat}, mode} + end + + defp concat_to_last_group(other, concat) do + {:doc_cons, other, concat} + end + + defp ungroup_if_group({:doc_group, group, _mode}), do: group + defp ungroup_if_group(other), do: other + + defp format_to_string(doc) do + doc |> Inspect.Algebra.format(:infinity) |> IO.iodata_to_binary() + end + + defp maybe_empty_line() do + nest(break(""), :reset) + end + + defp surround(left, doc, right) do + if doc == @empty do + concat(left, right) + else + group(glue(nest(glue(left, "", doc), 2, :break), "", right)) + end + end + + defp nest_by_length(doc, string) do + nest(doc, String.length(string)) + end + + defp split_last(list) do + {left, [right]} = Enum.split(list, -1) + {left, right} + end +end diff --git a/lib/elixir/lib/code/fragment.ex b/lib/elixir/lib/code/fragment.ex new file mode 100644 index 00000000000..a497b7bd2f3 --- /dev/null +++ b/lib/elixir/lib/code/fragment.ex @@ -0,0 +1,844 @@ +defmodule Code.Fragment do + @moduledoc """ + This module provides conveniences for analyzing fragments of + textual code and extract available information whenever possible. + + Most of the functions in this module provide a best-effort + and may not be accurate under all circumstances. Read each + documentation for more information. + + This module should be considered experimental. + """ + + @type position :: {line :: pos_integer(), column :: pos_integer()} + + @doc """ + Receives a string and returns the cursor context. + + This function receives a string with an Elixir code fragment, + representing a cursor position, and based on the string, it + provides contextual information about said position. The + return of this function can then be used to provide tips, + suggestions, and autocompletion functionality. + + This function provides a best-effort detection and may not be + accurate under all circumstances. See the "Limitations" + section below. + + Consider adding a catch-all clause when handling the return + type of this function as new cursor information may be added + in future releases. + + ## Examples + + iex> Code.Fragment.cursor_context("") + :expr + + iex> Code.Fragment.cursor_context("hello_wor") + {:local_or_var, 'hello_wor'} + + ## Return values + + * `{:alias, charlist}` - the context is an alias, potentially + a nested one, such as `Hello.Wor` or `HelloWor` + + * `{:dot, inside_dot, charlist}` - the context is a dot + where `inside_dot` is either a `{:var, charlist}`, `{:alias, charlist}`, + `{:module_attribute, charlist}`, `{:unquoted_atom, charlist}` or a `dot` + itself. If a var is given, this may either be a remote call or a map + field access. Examples are `Hello.wor`, `:hello.wor`, `hello.wor`, + `Hello.nested.wor`, `hello.nested.wor`, and `@hello.world`. If `charlist` + is empty and `inside_dot` is an alias, then the autocompletion may either + be an alias or a remote call. + + * `{:dot_arity, inside_dot, charlist}` - the context is a dot arity + where `inside_dot` is either a `{:var, charlist}`, `{:alias, charlist}`, + `{:module_attribute, charlist}`, `{:unquoted_atom, charlist}` or a `dot` + itself. If a var is given, it must be a remote arity. Examples are + `Hello.world/`, `:hello.world/`, `hello.world/2`, and `@hello.world/2` + + * `{:dot_call, inside_dot, charlist}` - the context is a dot + call. This means parentheses or space have been added after the expression. + where `inside_dot` is either a `{:var, charlist}`, `{:alias, charlist}`, + `{:module_attribute, charlist}`, `{:unquoted_atom, charlist}` or a `dot` + itself. If a var is given, it must be a remote call. Examples are + `Hello.world(`, `:hello.world(`, `Hello.world `, `hello.world(`, `hello.world `, + and `@hello.world(` + + * `:expr` - may be any expression. Autocompletion may suggest an alias, + local or var + + * `{:local_or_var, charlist}` - the context is a variable or a local + (import or local) call, such as `hello_wor` + + * `{:local_arity, charlist}` - the context is a local (import or local) + arity, such as `hello_world/` + + * `{:local_call, charlist}` - the context is a local (import or local) + call, such as `hello_world(` and `hello_world ` + + * `{:module_attribute, charlist}` - the context is a module attribute, + such as `@hello_wor` + + * `{:operator, charlist}` - the context is an operator, such as `+` or + `==`. Note textual operators, such as `when` do not appear as operators + but rather as `:local_or_var`. `@` is never an `:operator` and always a + `:module_attribute` + + * `{:operator_arity, charlist}` - the context is an operator arity, which + is an operator followed by /, such as `+/`, `not/` or `when/` + + * `{:operator_call, charlist}` - the context is an operator call, which is + an operator followed by space, such as `left + `, `not ` or `x when ` + + * `:none` - no context possible + + * `{:sigil, charlist}` - the context is a sigil. It may be either the beginning + of a sigil, such as `~` or `~s`, or an operator starting with `~`, such as + `~>` and `~>>` + + * `{:struct, charlist}` - the context is a struct, such as `%`, `%UR` or `%URI` + + * `{:unquoted_atom, charlist}` - the context is an unquoted atom. This + can be any atom or an atom representing a module + + ## Limitations + + The current algorithm only considers the last line of the input. This means + it will also show suggestions inside strings, heredocs, etc, which is + intentional as it helps with doctests, references, and more. + """ + @doc since: "1.13.0" + @spec cursor_context(List.Chars.t(), keyword()) :: + {:alias, charlist} + | {:dot, inside_dot, charlist} + | {:dot_arity, inside_dot, charlist} + | {:dot_call, inside_dot, charlist} + | :expr + | {:local_or_var, charlist} + | {:local_arity, charlist} + | {:local_call, charlist} + | {:module_attribute, charlist} + | {:operator, charlist} + | {:operator_arity, charlist} + | {:operator_call, charlist} + | :none + | {:sigil, charlist} + | {:struct, charlist} + | {:unquoted_atom, charlist} + when inside_dot: + {:alias, charlist} + | {:dot, inside_dot, charlist} + | {:module_attribute, charlist} + | {:unquoted_atom, charlist} + | {:var, charlist} + def cursor_context(fragment, opts \\ []) + + def cursor_context(binary, opts) when is_binary(binary) and is_list(opts) do + # CRLF not relevant here - we discard everything before last `\n` + binary = + case :binary.matches(binary, "\n") do + [] -> + binary + + matches -> + {position, _} = List.last(matches) + binary_part(binary, position + 1, byte_size(binary) - position - 1) + end + + binary + |> String.to_charlist() + |> :lists.reverse() + |> codepoint_cursor_context(opts) + |> elem(0) + end + + def cursor_context(charlist, opts) when is_list(charlist) and is_list(opts) do + # CRLF not relevant here - we discard everything before last `\n` + charlist = + case charlist |> Enum.chunk_by(&(&1 == ?\n)) |> List.last([]) do + [?\n | _] -> [] + rest -> rest + end + + charlist + |> :lists.reverse() + |> codepoint_cursor_context(opts) + |> elem(0) + end + + def cursor_context(other, opts) when is_list(opts) do + cursor_context(to_charlist(other), opts) + end + + @operators '\\<>+-*/:=|&~^%!' + @starter_punctuation ',([{;' + @non_starter_punctuation ')]}"\'.$' + @space '\t\s' + @trailing_identifier '?!' + @tilde_op_prefix '<=~' + + @non_identifier @trailing_identifier ++ + @operators ++ @starter_punctuation ++ @non_starter_punctuation ++ @space + + @textual_operators ~w(when not and or in)c + @keywords ~w(do end after else catch rescue fn true false nil)c + + defp codepoint_cursor_context(reverse, _opts) do + {stripped, spaces} = strip_spaces(reverse, 0) + + case stripped do + # It is empty + [] -> {:expr, 0} + # Structs + [?%, ?:, ?: | _] -> {{:struct, ''}, 1} + [?%, ?: | _] -> {{:unquoted_atom, '%'}, 2} + [?% | _] -> {{:struct, ''}, 1} + # Token/AST only operators + [?>, ?= | rest] when rest == [] or hd(rest) != ?: -> {:expr, 0} + [?>, ?- | rest] when rest == [] or hd(rest) != ?: -> {:expr, 0} + # Two-digit containers + [?<, ?< | rest] when rest == [] or hd(rest) != ?< -> {:expr, 0} + # Ambiguity around : + [?: | rest] when rest == [] or hd(rest) != ?: -> unquoted_atom_or_expr(spaces) + # Dots + [?.] -> {:none, 0} + [?. | rest] when hd(rest) not in '.:' -> dot(rest, spaces + 1, '') + # It is a local or remote call with parens + [?( | rest] -> call_to_cursor_context(strip_spaces(rest, spaces + 1)) + # A local arity definition + [?/ | rest] -> arity_to_cursor_context(strip_spaces(rest, spaces + 1)) + # Starting a new expression + [h | _] when h in @starter_punctuation -> {:expr, 0} + # It is a local or remote call without parens + rest when spaces > 0 -> call_to_cursor_context({rest, spaces}) + # It is an identifier + _ -> identifier_to_cursor_context(reverse, 0, false) + end + end + + defp strip_spaces([h | rest], count) when h in @space, do: strip_spaces(rest, count + 1) + defp strip_spaces(rest, count), do: {rest, count} + + defp unquoted_atom_or_expr(0), do: {{:unquoted_atom, ''}, 1} + defp unquoted_atom_or_expr(_), do: {:expr, 0} + + defp arity_to_cursor_context({reverse, spaces}) do + case identifier_to_cursor_context(reverse, spaces, true) do + {{:local_or_var, acc}, count} -> {{:local_arity, acc}, count} + {{:dot, base, acc}, count} -> {{:dot_arity, base, acc}, count} + {{:operator, acc}, count} -> {{:operator_arity, acc}, count} + {_, _} -> {:none, 0} + end + end + + defp call_to_cursor_context({reverse, spaces}) do + case identifier_to_cursor_context(reverse, spaces, true) do + {{:local_or_var, acc}, count} -> {{:local_call, acc}, count} + {{:dot, base, acc}, count} -> {{:dot_call, base, acc}, count} + {{:operator, acc}, count} -> {{:operator_call, acc}, count} + {_, _} -> {:none, 0} + end + end + + defp identifier_to_cursor_context([?., ?., ?: | _], n, _), do: {{:unquoted_atom, '..'}, n + 3} + defp identifier_to_cursor_context([?., ?., ?. | _], n, _), do: {{:local_or_var, '...'}, n + 3} + defp identifier_to_cursor_context([?., ?: | _], n, _), do: {{:unquoted_atom, '.'}, n + 2} + defp identifier_to_cursor_context([?., ?. | _], n, _), do: {{:operator, '..'}, n + 2} + + defp identifier_to_cursor_context(reverse, count, call_op?) do + case identifier(reverse, count) do + :none -> + {:none, 0} + + :operator -> + operator(reverse, count, [], call_op?) + + {:module_attribute, acc, count} -> + {{:module_attribute, acc}, count} + + {:sigil, acc, count} -> + {{:sigil, acc}, count} + + {:unquoted_atom, acc, count} -> + {{:unquoted_atom, acc}, count} + + {:alias, rest, acc, count} -> + case strip_spaces(rest, count) do + {'.' ++ rest, count} when rest == [] or hd(rest) != ?. -> + nested_alias(rest, count + 1, acc) + + {'%' ++ _, count} -> + {{:struct, acc}, count + 1} + + _ -> + {{:alias, acc}, count} + end + + {:identifier, _, acc, count} when call_op? and acc in @textual_operators -> + {{:operator, acc}, count} + + {:identifier, rest, acc, count} -> + case strip_spaces(rest, count) do + {'.' ++ rest, count} when rest == [] or hd(rest) != ?. -> + dot(rest, count + 1, acc) + + _ -> + {{:local_or_var, acc}, count} + end + end + end + + defp identifier([?? | rest], count), do: check_identifier(rest, count + 1, [??]) + defp identifier([?! | rest], count), do: check_identifier(rest, count + 1, [?!]) + defp identifier(rest, count), do: check_identifier(rest, count, []) + + defp check_identifier([h | t], count, acc) when h not in @non_identifier, + do: rest_identifier(t, count + 1, [h | acc]) + + defp check_identifier(_, _, _), do: :operator + + defp rest_identifier([h | rest], count, acc) when h not in @non_identifier do + rest_identifier(rest, count + 1, [h | acc]) + end + + defp rest_identifier(rest, count, [?@ | acc]) do + case tokenize_identifier(rest, count, acc) do + {:identifier, _rest, acc, count} -> {:module_attribute, acc, count} + :none when acc == [] -> {:module_attribute, '', count} + _ -> :none + end + end + + defp rest_identifier([?~ | rest], count, [letter]) + when (letter in ?A..?Z or letter in ?a..?z) and + (rest == [] or hd(rest) not in @tilde_op_prefix) do + {:sigil, [letter], count + 1} + end + + defp rest_identifier([?: | rest], count, acc) when rest == [] or hd(rest) != ?: do + case String.Tokenizer.tokenize(acc) do + {_, _, [], _, _, _} -> {:unquoted_atom, acc, count + 1} + _ -> :none + end + end + + defp rest_identifier([?? | _], _count, _acc) do + :none + end + + defp rest_identifier(rest, count, acc) do + tokenize_identifier(rest, count, acc) + end + + defp tokenize_identifier(rest, count, acc) do + case String.Tokenizer.tokenize(acc) do + # Not actually an atom cause rest is not a : + {:atom, _, _, _, _, _} -> + :none + + # Aliases must be ascii only + {:alias, _, _, _, false, _} -> + :none + + {kind, _, [], _, _, extra} -> + if :at in extra do + :none + else + {kind, rest, acc, count} + end + + _ -> + :none + end + end + + defp nested_alias(rest, count, acc) do + {rest, count} = strip_spaces(rest, count) + + case identifier_to_cursor_context(rest, count, true) do + {{:struct, prev}, count} -> {{:struct, prev ++ '.' ++ acc}, count} + {{:alias, prev}, count} -> {{:alias, prev ++ '.' ++ acc}, count} + _ -> {:none, 0} + end + end + + defp dot(rest, count, acc) do + {rest, count} = strip_spaces(rest, count) + + case identifier_to_cursor_context(rest, count, true) do + {{:local_or_var, var}, count} -> {{:dot, {:var, var}, acc}, count} + {{:unquoted_atom, _} = prev, count} -> {{:dot, prev, acc}, count} + {{:alias, _} = prev, count} -> {{:dot, prev, acc}, count} + {{:dot, _, _} = prev, count} -> {{:dot, prev, acc}, count} + {{:module_attribute, _} = prev, count} -> {{:dot, prev, acc}, count} + {{:struct, acc}, count} -> {{:struct, acc ++ '.'}, count} + {_, _} -> {:none, 0} + end + end + + defp operator([h | rest], count, acc, call_op?) when h in @operators do + operator(rest, count + 1, [h | acc], call_op?) + end + + # If we are opening a sigil, ignore the operator. + defp operator([letter, ?~ | rest], _count, [op], _call_op?) + when op in '<|/' and (letter in ?A..?Z or letter in ?a..?z) and + (rest == [] or hd(rest) not in @tilde_op_prefix) do + {:none, 0} + end + + defp operator(rest, count, '~', call_op?) do + {rest, _} = strip_spaces(rest, count) + + if call_op? or match?([?. | rest] when rest == [] or hd(rest) != ?., rest) do + {:none, 0} + else + {{:sigil, ''}, count} + end + end + + defp operator(rest, count, acc, _call_op?) do + case :elixir_tokenizer.tokenize(acc, 1, 1, []) do + {:ok, _, _, _, [{:atom, _, _}]} -> + {{:unquoted_atom, tl(acc)}, count} + + {:ok, _, _, _, [{_, _, op}]} -> + {rest, dot_count} = strip_spaces(rest, count) + + cond do + Code.Identifier.unary_op(op) == :error and Code.Identifier.binary_op(op) == :error -> + :none + + match?([?. | rest] when rest == [] or hd(rest) != ?., rest) -> + dot(tl(rest), dot_count + 1, acc) + + true -> + {{:operator, acc}, count} + end + + _ -> + {:none, 0} + end + end + + @doc """ + Receives a string and returns the surround context. + + This function receives a string with an Elixir code fragment + and a `position`. It returns a map containing the beginning + and ending of the identifier alongside its context, or `:none` + if there is nothing with a known context. + + The difference between `cursor_context/2` and `surround_context/3` + is that the former assumes the expression in the code fragment + is incomplete. For example, `do` in `cursor_context/2` may be + a keyword or a variable or a local call, while `surround_context/3` + assumes the expression in the code fragment is complete, therefore + `do` would always be a keyword. + + The `position` contains both the `line` and `column`, both starting + with the index of 1. The column must precede the surrounding expression. + For example, the expression `foo`, will return something for the columns + 1, 2, and 3, but not 4: + + foo + ^ column 1 + + foo + ^ column 2 + + foo + ^ column 3 + + foo + ^ column 4 + + The returned map contains the column the expression starts and the + first column after the expression ends. + + Similar to `cursor_context/2`, this function also provides a best-effort + detection and may not be accurate under all circumstances. See the + "Return values" and "Limitations" section under `cursor_context/2` for + more information. + + ## Examples + + iex> Code.Fragment.surround_context("foo", {1, 1}) + %{begin: {1, 1}, context: {:local_or_var, 'foo'}, end: {1, 4}} + + ## Differences to `cursor_context/2` + + Because `surround_context/3` deals with complete code, it has some + difference to `cursor_context/2`: + + * `dot_call`/`dot_arity` and `operator_call`/`operator_arity` + are collapsed into `dot` and `operator` contexts respectively + as there aren't any meaningful distinctions between them + + * On the other hand, this function still makes a distinction between + `local_call`/`local_arity` and `local_or_var`, since the latter can + be a local or variable + + * `@` when not followed by any identifier is returned as `{:operator, '@'}` + (in contrast to `{:module_attribute, ''}` in `cursor_context/2` + + * This function never returns empty sigils `{:sigil, ''}` or empty structs + `{:struct, ''}` as context + + * This function returns keywords as `{:keyword, 'do'}` + + * This function never returns `:expr` + + """ + @doc since: "1.13.0" + @spec surround_context(List.Chars.t(), position(), keyword()) :: + %{begin: position, end: position, context: context} | :none + when context: + {:alias, charlist} + | {:dot, inside_dot, charlist} + | {:local_or_var, charlist} + | {:local_arity, charlist} + | {:local_call, charlist} + | {:module_attribute, charlist} + | {:operator, charlist} + | {:sigil, charlist} + | {:struct, charlist} + | {:unquoted_atom, charlist} + | {:keyword, charlist}, + inside_dot: + {:alias, charlist} + | {:dot, inside_dot, charlist} + | {:module_attribute, charlist} + | {:unquoted_atom, charlist} + | {:var, charlist} + def surround_context(fragment, position, options \\ []) + + def surround_context(binary, {line, column}, opts) when is_binary(binary) do + binary + |> String.split(["\r\n", "\n"]) + |> Enum.at(line - 1, '') + |> String.to_charlist() + |> position_surround_context(line, column, opts) + end + + def surround_context(charlist, {line, column}, opts) when is_list(charlist) do + charlist + |> :string.replace('\r\n', '\n', :all) + |> :string.join('') + |> :string.split('\n', :all) + |> Enum.at(line - 1, '') + |> position_surround_context(line, column, opts) + end + + def surround_context(other, {_, _} = position, opts) do + surround_context(to_charlist(other), position, opts) + end + + defp position_surround_context(charlist, line, column, opts) + when is_integer(line) and line >= 1 and is_integer(column) and column >= 1 do + {reversed_pre, post} = string_reverse_at(charlist, column - 1, []) + {reversed_pre, post} = adjust_position(reversed_pre, post) + + case take_identifier(post, []) do + {_, [], _} -> + maybe_operator(reversed_pre, post, line, opts) + + {:identifier, reversed_post, rest} -> + {rest, _} = strip_spaces(rest, 0) + reversed = reversed_post ++ reversed_pre + + case codepoint_cursor_context(reversed, opts) do + {{:struct, acc}, offset} -> + build_surround({:struct, acc}, reversed, line, offset) + + {{:alias, acc}, offset} -> + build_surround({:alias, acc}, reversed, line, offset) + + {{:dot, _, [_ | _]} = dot, offset} -> + build_surround(dot, reversed, line, offset) + + {{:local_or_var, acc}, offset} when hd(rest) == ?( -> + build_surround({:local_call, acc}, reversed, line, offset) + + {{:local_or_var, acc}, offset} when hd(rest) == ?/ -> + build_surround({:local_arity, acc}, reversed, line, offset) + + {{:local_or_var, acc}, offset} when acc in @textual_operators -> + build_surround({:operator, acc}, reversed, line, offset) + + {{:local_or_var, acc}, offset} when acc in @keywords -> + build_surround({:keyword, acc}, reversed, line, offset) + + {{:local_or_var, acc}, offset} -> + build_surround({:local_or_var, acc}, reversed, line, offset) + + {{:module_attribute, ''}, offset} -> + build_surround({:operator, '@'}, reversed, line, offset) + + {{:module_attribute, acc}, offset} -> + build_surround({:module_attribute, acc}, reversed, line, offset) + + {{:sigil, acc}, offset} -> + build_surround({:sigil, acc}, reversed, line, offset) + + {{:unquoted_atom, acc}, offset} -> + build_surround({:unquoted_atom, acc}, reversed, line, offset) + + _ -> + maybe_operator(reversed_pre, post, line, opts) + end + + {:alias, reversed_post, _rest} -> + reversed = reversed_post ++ reversed_pre + + case codepoint_cursor_context(reversed, opts) do + {{:alias, acc}, offset} -> + build_surround({:alias, acc}, reversed, line, offset) + + {{:struct, acc}, offset} -> + build_surround({:struct, acc}, reversed, line, offset) + + _ -> + :none + end + end + end + + defp maybe_operator(reversed_pre, post, line, opts) do + case take_operator(post, []) do + {[], _rest} -> + :none + + {reversed_post, rest} -> + reversed = reversed_post ++ reversed_pre + + case codepoint_cursor_context(reversed, opts) do + {{:operator, acc}, offset} -> + build_surround({:operator, acc}, reversed, line, offset) + + {{:sigil, ''}, offset} when hd(rest) in ?A..?Z or hd(rest) in ?a..?z -> + build_surround({:sigil, [hd(rest)]}, [hd(rest) | reversed], line, offset + 1) + + {{:dot, _, [_ | _]} = dot, offset} -> + build_surround(dot, reversed, line, offset) + + _ -> + :none + end + end + end + + defp build_surround(context, reversed, line, offset) do + {post, reversed_pre} = enum_reverse_at(reversed, offset, []) + pre = :lists.reverse(reversed_pre) + pre_length = :string.length(pre) + 1 + + %{ + context: context, + begin: {line, pre_length}, + end: {line, pre_length + :string.length(post)} + } + end + + defp take_identifier([h | t], acc) when h in @trailing_identifier, + do: {:identifier, [h | acc], t} + + defp take_identifier([h | t], acc) when h not in @non_identifier, + do: take_identifier(t, [h | acc]) + + defp take_identifier(rest, acc) do + with {[?. | t], _} <- strip_spaces(rest, 0), + {[h | _], _} when h in ?A..?Z <- strip_spaces(t, 0) do + take_alias(rest, acc) + else + _ -> {:identifier, acc, rest} + end + end + + defp take_alias([h | t], acc) when h not in @non_identifier, + do: take_alias(t, [h | acc]) + + defp take_alias(rest, acc) do + with {[?. | t], acc} <- move_spaces(rest, acc), + {[h | t], acc} when h in ?A..?Z <- move_spaces(t, [?. | acc]) do + take_alias(t, [h | acc]) + else + _ -> {:alias, acc, rest} + end + end + + defp take_operator([h | t], acc) when h in @operators, do: take_operator(t, [h | acc]) + defp take_operator([h | t], acc) when h == ?., do: take_operator(t, [h | acc]) + defp take_operator(rest, acc), do: {acc, rest} + + # Unquoted atom handling + defp adjust_position(reversed_pre, [?: | post]) + when hd(post) != ?: and (reversed_pre == [] or hd(reversed_pre) != ?:) do + {[?: | reversed_pre], post} + end + + defp adjust_position(reversed_pre, [?% | post]) do + adjust_position([?% | reversed_pre], post) + end + + # Dot/struct handling + defp adjust_position(reversed_pre, post) do + case move_spaces(post, reversed_pre) do + # If we are between spaces and a dot, move past the dot + {[?. | post], reversed_pre} when hd(post) != ?. and hd(reversed_pre) != ?. -> + {post, reversed_pre} = move_spaces(post, [?. | reversed_pre]) + {reversed_pre, post} + + _ -> + case strip_spaces(reversed_pre, 0) do + # If there is a dot to our left, make sure to move to the first character + {[?. | rest], _} when rest == [] or hd(rest) not in '.:' -> + {post, reversed_pre} = move_spaces(post, reversed_pre) + {reversed_pre, post} + + # If there is a % to our left, make sure to move to the first character + {[?% | _], _} -> + case move_spaces(post, reversed_pre) do + {[h | _] = post, reversed_pre} when h in ?A..?Z -> + {reversed_pre, post} + + _ -> + {reversed_pre, post} + end + + _ -> + {reversed_pre, post} + end + end + end + + defp move_spaces([h | t], acc) when h in @space, do: move_spaces(t, [h | acc]) + defp move_spaces(t, acc), do: {t, acc} + + defp string_reverse_at(charlist, 0, acc), do: {acc, charlist} + + defp string_reverse_at(charlist, n, acc) do + case :unicode_util.gc(charlist) do + [gc | cont] when is_integer(gc) -> string_reverse_at(cont, n - 1, [gc | acc]) + [gc | cont] when is_list(gc) -> string_reverse_at(cont, n - 1, :lists.reverse(gc, acc)) + [] -> {acc, []} + end + end + + defp enum_reverse_at([h | t], n, acc) when n > 0, do: enum_reverse_at(t, n - 1, [h | acc]) + defp enum_reverse_at(rest, _, acc), do: {acc, rest} + + @doc """ + Receives a code fragment and returns a quoted expression + with a cursor at the nearest argument position. + + A container is any Elixir expression starting with `(`, + `{`, and `[`. This includes function calls, tuples, lists, + maps, and so on. For example, take this code, which would + be given as input: + + max(some_value, + + This function will return the AST equivalent to: + + max(some_value, __cursor__()) + + In other words, this function is capable of closing any open + brackets and insert the cursor position. Any content at the + cursor position that is after a comma or an opening bracket + is discarded. For example, if this is given as input: + + max(some_value, another_val + + It will return the same AST: + + max(some_value, __cursor__()) + + Similarly, if only this is given: + + max(some_va + + Then it returns: + + max(__cursor__()) + + Calls without parenthesis are also supported, as we assume the + brackets are implicit. + + Operators and anonymous functions are not containers, and therefore + will be discarded. The following will all return the same AST: + + max(some_value, + max(some_value, fn x -> x end + max(some_value, 1 + another_val + max(some_value, 1 |> some_fun() |> another_fun + + On the other hand, tuples, lists, maps, etc all retain the + cursor position: + + max(some_value, [1, 2, + + Returns the following AST: + + max(some_value, [1, 2, __cursor__()]) + + Keyword lists (and do-end blocks) are also retained. The following: + + if(some_value, do: + if(some_value, do: :token + if(some_value, do: 1 + val + + all return: + + if(some_value, do: __cursor__()) + + The AST returned by this function is not safe to evaluate but + it can be analyzed and expanded. + + ## Examples + + iex> Code.Fragment.container_cursor_to_quoted("max(some_value, ") + {:ok, {:max, [line: 1], [{:some_value, [line: 1], nil}, {:__cursor__, [line: 1], []}]}} + + ## Options + + * `:file` - the filename to be reported in case of parsing errors. + Defaults to `"nofile"`. + + * `:line` - the starting line of the string being parsed. + Defaults to 1. + + * `:column` - the starting column of the string being parsed. + Defaults to 1. + + * `:columns` - when `true`, attach a `:column` key to the quoted + metadata. Defaults to `false`. + + * `:token_metadata` - when `true`, includes token-related + metadata in the expression AST, such as metadata for `do` and `end` + tokens, for closing tokens, end of expressions, as well as delimiters + for sigils. See `t:Macro.metadata/0`. Defaults to `false`. + + """ + @doc since: "1.13.0" + @spec container_cursor_to_quoted(List.Chars.t(), keyword()) :: + {:ok, Macro.t()} | {:error, {location :: keyword, binary | {binary, binary}, binary}} + def container_cursor_to_quoted(fragment, opts \\ []) do + file = Keyword.get(opts, :file, "nofile") + line = Keyword.get(opts, :line, 1) + column = Keyword.get(opts, :column, 1) + columns = Keyword.get(opts, :columns, false) + token_metadata = Keyword.get(opts, :token_metadata, false) + + Code.string_to_quoted(fragment, + file: file, + line: line, + column: column, + columns: columns, + token_metadata: token_metadata, + cursor_completion: true, + emit_warnings: false + ) + end +end diff --git a/lib/elixir/lib/code/identifier.ex b/lib/elixir/lib/code/identifier.ex new file mode 100644 index 00000000000..e1d7b374ee4 --- /dev/null +++ b/lib/elixir/lib/code/identifier.ex @@ -0,0 +1,152 @@ +defmodule Code.Identifier do + @moduledoc false + + @doc """ + Checks if the given identifier is an unary op. + + ## Examples + + iex> Code.Identifier.unary_op(:+) + {:non_associative, 300} + + """ + @spec unary_op(atom) :: {:non_associative, precedence :: pos_integer} | :error + def unary_op(op) do + cond do + op in [:&] -> {:non_associative, 90} + op in [:!, :^, :not, :+, :-, :"~~~"] -> {:non_associative, 300} + op in [:@] -> {:non_associative, 320} + true -> :error + end + end + + @doc """ + Checks if the given identifier is a binary op. + + ## Examples + + iex> Code.Identifier.binary_op(:+) + {:left, 210} + + """ + @spec binary_op(atom) :: {:left | :right, precedence :: pos_integer} | :error + def binary_op(op) do + cond do + op in [:<-, :\\] -> {:left, 40} + op in [:when] -> {:right, 50} + op in [:"::"] -> {:right, 60} + op in [:|] -> {:right, 70} + op in [:=] -> {:right, 100} + op in [:||, :|||, :or] -> {:left, 120} + op in [:&&, :&&&, :and] -> {:left, 130} + op in [:==, :!=, :=~, :===, :!==] -> {:left, 140} + op in [:<, :<=, :>=, :>] -> {:left, 150} + op in [:|>, :<<<, :>>>, :<~, :~>, :<<~, :~>>, :<~>, :"<|>"] -> {:left, 160} + op in [:in] -> {:left, 170} + op in [:"^^^"] -> {:left, 180} + op in [:"//"] -> {:right, 190} + op in [:++, :--, :.., :<>, :+++, :---] -> {:right, 200} + op in [:+, :-] -> {:left, 210} + op in [:*, :/] -> {:left, 220} + op in [:**] -> {:left, 230} + op in [:.] -> {:left, 310} + true -> :error + end + end + + @doc """ + Extracts the name and arity of the parent from the anonymous function identifier. + """ + # Example of this format: -NAME/ARITY-fun-COUNT- + def extract_anonymous_fun_parent(atom) when is_atom(atom) do + with "-" <> rest <- Atom.to_string(atom), + [trailing | reversed] = rest |> String.split("/") |> Enum.reverse(), + [arity, _inner, _count, ""] <- String.split(trailing, "-") do + {reversed |> Enum.reverse() |> Enum.join("/") |> String.to_atom(), arity} + else + _ -> :error + end + end + + @doc """ + Escapes the given identifier. + """ + @spec escape(binary(), char() | nil, :infinity | non_neg_integer, (char() -> iolist() | false)) :: + {escaped :: iolist(), remaining :: binary()} + def escape(binary, char, limit \\ :infinity, fun \\ &escape_map/1) + when ((char in 0..0x10FFFF or is_nil(char)) and limit == :infinity) or + (is_integer(limit) and limit >= 0) do + escape(binary, char, limit, [], fun) + end + + defp escape(<<_, _::binary>> = binary, _char, 0, acc, _fun) do + {acc, binary} + end + + defp escape(<>, char, count, acc, fun) do + escape(t, char, decrement(count), [acc | [?\\, char]], fun) + end + + defp escape(<>, char, count, acc, fun) do + escape(t, char, decrement(count), [acc | '\\\#{'], fun) + end + + defp escape(<>, char, count, acc, fun) do + escaped = if value = fun.(h), do: value, else: escape_char(h) + escape(t, char, decrement(count), [acc | escaped], fun) + end + + defp escape(<>, char, count, acc, fun) do + escape(t, char, decrement(count), [acc | ['\\x', to_hex(a), to_hex(b)]], fun) + end + + defp escape(<<>>, _char, _count, acc, _fun) do + {acc, <<>>} + end + + defp escape_char(0), do: '\\0' + + defp escape_char(65279), do: '\\uFEFF' + + defp escape_char(char) + when char in 0x20..0x7E + when char in 0xA0..0xD7FF + when char in 0xE000..0xFFFD + when char in 0x10000..0x10FFFF do + <> + end + + defp escape_char(char) when char < 0x100 do + <> = <> + ['\\x', to_hex(a), to_hex(b)] + end + + defp escape_char(char) when char < 0x10000 do + <> = <> + ['\\x{', to_hex(a), to_hex(b), to_hex(c), to_hex(d), ?}] + end + + defp escape_char(char) when char < 0x1000000 do + <> = <> + ['\\x{', to_hex(a), to_hex(b), to_hex(c), to_hex(d), to_hex(e), to_hex(f), ?}] + end + + defp escape_map(?\a), do: '\\a' + defp escape_map(?\b), do: '\\b' + defp escape_map(?\d), do: '\\d' + defp escape_map(?\e), do: '\\e' + defp escape_map(?\f), do: '\\f' + defp escape_map(?\n), do: '\\n' + defp escape_map(?\r), do: '\\r' + defp escape_map(?\t), do: '\\t' + defp escape_map(?\v), do: '\\v' + defp escape_map(?\\), do: '\\\\' + defp escape_map(_), do: false + + @compile {:inline, to_hex: 1, decrement: 1} + defp to_hex(c) when c in 0..9, do: ?0 + c + defp to_hex(c) when c in 10..15, do: ?A + c - 10 + + defp decrement(:infinity), do: :infinity + defp decrement(counter), do: counter - 1 +end diff --git a/lib/elixir/lib/code/normalizer.ex b/lib/elixir/lib/code/normalizer.ex new file mode 100644 index 00000000000..a3018c6bff2 --- /dev/null +++ b/lib/elixir/lib/code/normalizer.ex @@ -0,0 +1,585 @@ +defmodule Code.Normalizer do + @moduledoc false + + defguard is_literal(x) + when is_integer(x) or + is_float(x) or + is_binary(x) or + is_atom(x) + + @doc """ + Wraps literals in the quoted expression to conform to the AST format expected + by the formatter. + """ + def normalize(quoted, opts \\ []) do + line = Keyword.get(opts, :line, nil) + escape = Keyword.get(opts, :escape, true) + locals_without_parens = Keyword.get(opts, :locals_without_parens, []) + + state = %{ + escape: escape, + parent_meta: [line: line], + locals_without_parens: locals_without_parens ++ Code.Formatter.locals_without_parens() + } + + do_normalize(quoted, state) + end + + # Wrapped literals should receive the block meta + defp do_normalize({:__block__, meta, [literal]}, state) + when not is_tuple(literal) or tuple_size(literal) == 2 do + normalize_literal(literal, meta, state) + end + + # Only normalize the first argument of an alias if it's not an atom + defp do_normalize({:__aliases__, meta, [first | rest]}, state) when not is_atom(first) do + meta = patch_meta_line(meta, state.parent_meta) + first = do_normalize(first, %{state | parent_meta: meta}) + {:__aliases__, meta, [first | rest]} + end + + defp do_normalize({:__aliases__, _, _} = quoted, _state) do + quoted + end + + # Skip captured arguments like &1 + defp do_normalize({:&, meta, [term]}, state) when is_integer(term) do + meta = patch_meta_line(meta, state.parent_meta) + {:&, meta, [term]} + end + + # Ranges + defp do_normalize(left..right//step, state) do + left = do_normalize(left, state) + right = do_normalize(right, state) + meta = meta_line(state) + + if step == 1 do + {:.., meta, [left, right]} + else + step = do_normalize(step, state) + {:"..//", meta, [left, right, step]} + end + end + + # Bit containers + defp do_normalize({:<<>>, _, args} = quoted, state) when is_list(args) do + normalize_bitstring(quoted, state) + end + + # Atoms with interpolations + defp do_normalize( + {{:., dot_meta, [:erlang, :binary_to_atom]}, call_meta, + [{:<<>>, _, args} = string, :utf8]}, + state + ) + when is_list(args) do + dot_meta = patch_meta_line(dot_meta, state.parent_meta) + call_meta = patch_meta_line(call_meta, dot_meta) + + string = + if state.escape do + normalize_bitstring(string, state, true) + else + normalize_bitstring(string, state) + end + + {{:., dot_meta, [:erlang, :binary_to_atom]}, call_meta, [string, :utf8]} + end + + # Charlists with interpolations + defp do_normalize({{:., dot_meta, [List, :to_charlist]}, call_meta, [parts]}, state) do + parts = + Enum.map(parts, fn + {{:., part_dot_meta, [Kernel, :to_string]}, part_call_meta, args} -> + args = normalize_args(args, state) + + {{:., part_dot_meta, [Kernel, :to_string]}, part_call_meta, args} + + part -> + if state.escape do + maybe_escape_literal(part, state) + else + part + end + end) + + {{:., dot_meta, [List, :to_charlist]}, call_meta, [parts]} + end + + # Don't normalize the `Access` atom in access syntax + defp do_normalize({:., meta, [Access, :get]}, state) do + meta = patch_meta_line(meta, state.parent_meta) + {:., meta, [Access, :get]} + end + + # Only normalize the left side of the dot operator + # The right hand side is an atom in the AST but it's not an atom literal, so + # it should not be wrapped + defp do_normalize({:., meta, [left, right]}, state) do + meta = patch_meta_line(meta, state.parent_meta) + + left = do_normalize(left, %{state | parent_meta: meta}) + + {:., meta, [left, right]} + end + + # A list of left to right arrows is not considered as a list literal, so it's not wrapped + defp do_normalize([{:->, _, [_ | _]} | _] = quoted, state) do + normalize_args(quoted, state) + end + + # left -> right + defp do_normalize({:->, meta, [left, right]}, state) do + meta = patch_meta_line(meta, state.parent_meta) + + left = normalize_args(left, %{state | parent_meta: meta}) + right = do_normalize(right, %{state | parent_meta: meta}) + {:->, meta, [left, right]} + end + + # Maps + defp do_normalize({:%{}, meta, args}, state) when is_list(args) do + meta = + if meta == [] do + line = state.parent_meta[:line] + [line: line, closing: [line: line]] + else + meta + end + + state = %{state | parent_meta: meta} + + args = + case args do + [{:|, pipe_meta, [left, right]}] -> + left = do_normalize(left, state) + right = normalize_map_args(right, state) + [{:|, pipe_meta, [left, right]}] + + [{_, _, _} = call] -> + [do_normalize(call, state)] + + args -> + normalize_map_args(args, state) + end + + {:%{}, meta, args} + end + + # Sigils + defp do_normalize({sigil, meta, [{:<<>>, _, args} = string, modifiers]} = quoted, state) + when is_list(args) and is_atom(sigil) do + case Atom.to_string(sigil) do + <<"sigil_", _name>> -> + meta = + meta + |> patch_meta_line(state.parent_meta) + |> Keyword.put_new(:delimiter, "\"") + + {sigil, meta, [do_normalize(string, %{state | parent_meta: meta}), modifiers]} + + _ -> + normalize_call(quoted, state) + end + end + + # Tuples + defp do_normalize({:{}, meta, args} = quoted, state) when is_list(args) do + {last_arg, args} = List.pop_at(args, -1) + + if args != [] and match?([_ | _], last_arg) and keyword?(last_arg) do + args = normalize_args(args, state) + kw_list = normalize_kw_args(last_arg, state, true) + {:{}, meta, args ++ kw_list} + else + normalize_call(quoted, state) + end + end + + # Module attributes + defp do_normalize({:@, meta, [{name, name_meta, [value]}]}, state) do + value = + cond do + keyword?(value) -> + normalize_kw_args(value, state, true) + + is_list(value) -> + normalize_literal(value, meta, state) + + true -> + do_normalize(value, state) + end + + {:@, meta, [{name, name_meta, [value]}]} + end + + # Regular blocks + defp do_normalize({:__block__, meta, args}, state) when is_list(args) do + {:__block__, meta, normalize_args(args, state)} + end + + # Calls + defp do_normalize({_, _, args} = quoted, state) when is_list(args) do + normalize_call(quoted, state) + end + + # Vars + defp do_normalize({_, _, context} = quoted, _state) when is_atom(context) do + quoted + end + + # Literals + defp do_normalize(quoted, state) do + normalize_literal(quoted, [], state) + end + + # Numbers + defp normalize_literal(number, meta, state) when is_number(number) do + meta = + meta + |> Keyword.put_new(:token, inspect(number)) + |> patch_meta_line(state.parent_meta) + + {:__block__, meta, [number]} + end + + # Atom, Strings + defp normalize_literal(literal, meta, state) when is_atom(literal) or is_binary(literal) do + meta = patch_meta_line(meta, state.parent_meta) + literal = maybe_escape_literal(literal, state) + + if is_atom(literal) and Macro.classify_atom(literal) == :alias and + is_nil(meta[:delimiter]) do + segments = + case Atom.to_string(literal) do + "Elixir" -> + [:"Elixir"] + + "Elixir." <> segments -> + segments + |> String.split(".") + |> Enum.map(&String.to_atom/1) + end + + {:__aliases__, meta, segments} + else + {:__block__, meta, [literal]} + end + end + + # 2-tuples + defp normalize_literal({left, right}, meta, state) do + meta = patch_meta_line(meta, state.parent_meta) + state = %{state | parent_meta: meta} + + if match?([_ | _], right) and keyword?(right) do + {:__block__, meta, [{do_normalize(left, state), normalize_kw_args(right, state, true)}]} + else + {:__block__, meta, [{do_normalize(left, state), do_normalize(right, state)}]} + end + end + + # Lists + defp normalize_literal(list, meta, state) when is_list(list) do + if list != [] and List.ascii_printable?(list) do + # It's a charlist + list = + if state.escape do + {string, _} = Code.Identifier.escape(IO.chardata_to_string(list), nil) + IO.iodata_to_binary(string) |> to_charlist() + else + list + end + + meta = + meta + |> Keyword.put_new(:delimiter, "'") + |> patch_meta_line(state.parent_meta) + + {:__block__, meta, [list]} + else + meta = + if line = state.parent_meta[:line] do + meta + |> Keyword.put_new(:closing, line: line) + |> patch_meta_line(state.parent_meta) + else + meta + end + + {:__block__, meta, [normalize_kw_args(list, state, false)]} + end + end + + # Probably an invalid value, wrap it and send it upstream + defp normalize_literal(quoted, meta, _state) do + {:__block__, meta, [quoted]} + end + + defp normalize_call({form, meta, args}, state) do + meta = patch_meta_line(meta, state.parent_meta) + arity = length(args) + + # Only normalize the form if it's a qualified call + form = + if is_atom(form) do + form + else + do_normalize(form, %{state | parent_meta: meta}) + end + + meta = + if is_nil(meta[:no_parens]) and is_nil(meta[:closing]) and is_nil(meta[:do]) and + not Code.Formatter.local_without_parens?(form, arity, state.locals_without_parens) do + [closing: [line: meta[:line]]] ++ meta + else + meta + end + + cond do + Keyword.has_key?(meta, :do) or match?([{{:__block__, _, [:do]}, _} | _], List.last(args)) -> + # def foo do :ok end + # def foo, do: :ok + normalize_kw_blocks(form, meta, args, state) + + match?([{:do, _} | _], List.last(args)) -> + # Non normalized kw blocks + line = state.parent_meta[:line] + meta = meta ++ [do: [line: line], end: [line: line]] + normalize_kw_blocks(form, meta, args, state) + + allow_keyword?(form, arity) -> + args = normalize_args(args, %{state | parent_meta: state.parent_meta}) + {last_arg, leading_args} = List.pop_at(args, -1, []) + + last_args = + case last_arg do + {:__block__, _, [[{{:__block__, key_meta, _}, _} | _]] = last_args} -> + if key_meta[:format] == :keyword do + last_args + else + [last_arg] + end + + [] -> + [] + + _ -> + [last_arg] + end + + {form, meta, leading_args ++ last_args} + + true -> + args = normalize_args(args, %{state | parent_meta: state.parent_meta}) + {form, meta, args} + end + end + + defp allow_keyword?(:when, 2), do: true + defp allow_keyword?(:{}, _), do: false + defp allow_keyword?(op, arity), do: not is_atom(op) or not Macro.operator?(op, arity) + + defp normalize_bitstring({:<<>>, meta, parts} = quoted, state, escape_interpolation \\ false) do + meta = patch_meta_line(meta, state.parent_meta) + + parts = + if interpolated?(quoted) do + normalize_interpolation_parts(parts, %{state | parent_meta: meta}, escape_interpolation) + else + state = %{state | parent_meta: meta} + + Enum.map(parts, fn part -> + with {:"::", meta, [left, _]} <- part, + true <- meta[:inferred_bitstring_spec] do + do_normalize(left, state) + else + _ -> do_normalize(part, state) + end + end) + end + + {:<<>>, meta, parts} + end + + defp normalize_interpolation_parts(parts, state, escape_interpolation) do + Enum.map(parts, fn + {:"::", interpolation_meta, + [ + {{:., dot_meta, [Kernel, :to_string]}, middle_meta, [middle]}, + {:binary, binary_meta, context} + ]} -> + middle = do_normalize(middle, %{state | parent_meta: dot_meta}) + + {:"::", interpolation_meta, + [ + {{:., dot_meta, [Kernel, :to_string]}, middle_meta, [middle]}, + {:binary, binary_meta, context} + ]} + + part -> + if escape_interpolation do + maybe_escape_literal(part, state) + else + part + end + end) + end + + defp normalize_map_args(args, state) do + Enum.map(normalize_kw_args(args, state, false), fn + {:__block__, _, [{_, _} = pair]} -> pair + pair -> pair + end) + end + + defp normalize_kw_blocks(form, meta, args, state) do + {kw_blocks, leading_args} = List.pop_at(args, -1) + + kw_blocks = + Enum.map(kw_blocks, fn {tag, block} -> + block = do_normalize(block, %{state | parent_meta: meta}) + + block = + case block do + {_, _, [[{:->, _, _} | _] = block]} -> block + block -> block + end + + # Only wrap the tag if it isn't already wrapped + tag = + case tag do + {:__block__, _, _} -> tag + _ -> {:__block__, [line: meta[:line]], [tag]} + end + + {tag, block} + end) + + leading_args = normalize_args(leading_args, %{state | parent_meta: meta}) + {form, meta, leading_args ++ [kw_blocks]} + end + + defp normalize_kw_args(elems, state, keyword?) + + defp normalize_kw_args( + [{{:__block__, key_meta, [key]}, value} = first | rest] = current, + state, + keyword? + ) + when is_atom(key) do + keyword? = keyword? or keyword?(current) + + first = + if key_meta[:format] == :keyword and not keyword? do + key_meta = Keyword.delete(key_meta, :format) + line = key_meta[:line] || meta_line(state) + {:__block__, [line: line], [{{:__block__, key_meta, [key]}, value}]} + else + first + end + + [first | normalize_kw_args(rest, state, keyword?)] + end + + defp normalize_kw_args([{left, right} | rest] = current, state, keyword?) do + keyword? = keyword? or keyword?(current) + + left = + if keyword? do + meta = [format: :keyword] ++ meta_line(state) + {:__block__, meta, [maybe_escape_literal(left, state)]} + else + do_normalize(left, state) + end + + right = do_normalize(right, state) + + pair = + with {:__block__, meta, _} <- left, + :keyword <- meta[:format] do + {left, right} + else + _ -> {:__block__, meta_line(state), [{left, right}]} + end + + [pair | normalize_kw_args(rest, state, keyword?)] + end + + defp normalize_kw_args([first | rest], state, keyword?) do + [do_normalize(first, state) | normalize_kw_args(rest, state, keyword?)] + end + + defp normalize_kw_args([], _state, _keyword?) do + [] + end + + defp normalize_args(args, state) do + Enum.map(args, &do_normalize(&1, state)) + end + + defp maybe_escape_literal(string, %{escape: true}) when is_binary(string) do + {string, _} = Code.Identifier.escape(string, nil) + IO.iodata_to_binary(string) + end + + defp maybe_escape_literal(atom, %{escape: true} = state) when is_atom(atom) do + atom + |> Atom.to_string() + |> maybe_escape_literal(state) + |> String.to_atom() + end + + defp maybe_escape_literal(term, _) do + term + end + + # Check if we have an interpolated string. + defp interpolated?({:<<>>, _, [_ | _] = parts}) do + Enum.all?(parts, fn + {:"::", _, [{{:., _, [Kernel, :to_string]}, _, [_]}, {:binary, _, _}]} -> true + binary when is_binary(binary) -> true + _ -> false + end) + end + + defp interpolated?(_) do + false + end + + defp patch_meta_line(meta, parent_meta) do + with nil <- meta[:line], + line when is_integer(line) <- parent_meta[:line] do + [line: line] ++ meta + else + _ -> meta + end + end + + defp meta_line(state) do + if line = state.parent_meta[:line] do + [line: line] + else + [] + end + end + + defp keyword?([{{:__block__, key_meta, [key]}, _} | rest]) when is_atom(key) do + if key_meta[:format] == :keyword do + keyword?(rest) + else + false + end + end + + defp keyword?([{key, _value} | rest]) when is_atom(key) do + case Atom.to_charlist(key) do + 'Elixir.' ++ _ -> false + _ -> keyword?(rest) + end + end + + defp keyword?([]), do: true + defp keyword?(_other), do: false +end diff --git a/lib/elixir/lib/code/typespec.ex b/lib/elixir/lib/code/typespec.ex new file mode 100644 index 00000000000..ff4f8061bf3 --- /dev/null +++ b/lib/elixir/lib/code/typespec.ex @@ -0,0 +1,421 @@ +defmodule Code.Typespec do + @moduledoc false + + @doc """ + Converts a spec clause back to Elixir quoted expression. + """ + @spec spec_to_quoted(atom, tuple) :: {atom, keyword, [Macro.t()]} + def spec_to_quoted(name, spec) + + def spec_to_quoted(name, {:type, anno, :fun, [{:type, _, :product, args}, result]}) + when is_atom(name) do + meta = meta(anno) + body = {name, meta, Enum.map(args, &typespec_to_quoted/1)} + + vars = + for type_expr <- args ++ [result], + var <- collect_vars(type_expr), + uniq: true, + do: {var, {:var, meta, nil}} + + spec = {:"::", meta, [body, typespec_to_quoted(result)]} + + if vars == [] do + spec + else + {:when, meta, [spec, vars]} + end + end + + def spec_to_quoted(name, {:type, anno, :fun, []}) when is_atom(name) do + meta = meta(anno) + {:"::", meta, [{name, meta, []}, quote(do: term)]} + end + + def spec_to_quoted(name, {:type, anno, :bounded_fun, [type, constrs]}) when is_atom(name) do + meta = meta(anno) + {:type, _, :fun, [{:type, _, :product, args}, result]} = type + + guards = + for {:type, _, :constraint, [{:atom, _, :is_subtype}, [{:var, _, var}, type]]} <- constrs do + {erl_to_ex_var(var), typespec_to_quoted(type)} + end + + ignore_vars = Keyword.keys(guards) + + vars = + for type_expr <- args ++ [result], + var <- collect_vars(type_expr), + var not in ignore_vars, + uniq: true, + do: {var, {:var, meta, nil}} + + args = for arg <- args, do: typespec_to_quoted(arg) + + when_args = [ + {:"::", meta, [{name, meta, args}, typespec_to_quoted(result)]}, + guards ++ vars + ] + + {:when, meta, when_args} + end + + @doc """ + Converts a type clause back to Elixir AST. + """ + def type_to_quoted(type) + + def type_to_quoted({{:record, record}, fields, args}) when is_atom(record) do + fields = for field <- fields, do: typespec_to_quoted(field) + args = for arg <- args, do: typespec_to_quoted(arg) + type = {:{}, [], [record | fields]} + quote(do: unquote(record)(unquote_splicing(args)) :: unquote(type)) + end + + def type_to_quoted({name, type, args}) when is_atom(name) do + args = for arg <- args, do: typespec_to_quoted(arg) + quote(do: unquote(name)(unquote_splicing(args)) :: unquote(typespec_to_quoted(type))) + end + + @doc """ + Returns all types available from the module's BEAM code. + + The result is returned as a list of tuples where the first + element is the type (`:typep`, `:type` and `:opaque`). + + The module must have a corresponding BEAM file which can be + located by the runtime system. The types will be in the Erlang + Abstract Format. + """ + @spec fetch_types(module | binary) :: {:ok, [tuple]} | :error + def fetch_types(module) when is_atom(module) or is_binary(module) do + case typespecs_abstract_code(module) do + {:ok, abstract_code} -> + exported_types = for {:attribute, _, :export_type, types} <- abstract_code, do: types + exported_types = List.flatten(exported_types) + + types = + for {:attribute, _, kind, {name, _, args} = type} <- abstract_code, + kind in [:opaque, :type] do + cond do + kind == :opaque -> {:opaque, type} + {name, length(args)} in exported_types -> {:type, type} + true -> {:typep, type} + end + end + + {:ok, types} + + _ -> + :error + end + end + + @doc """ + Returns all specs available from the module's BEAM code. + + The result is returned as a list of tuples where the first + element is spec name and arity and the second is the spec. + + The module must have a corresponding BEAM file which can be + located by the runtime system. The types will be in the Erlang + Abstract Format. + """ + @spec fetch_specs(module) :: {:ok, [tuple]} | :error + def fetch_specs(module) when is_atom(module) or is_binary(module) do + case typespecs_abstract_code(module) do + {:ok, abstract_code} -> + {:ok, for({:attribute, _, :spec, value} <- abstract_code, do: value)} + + :error -> + :error + end + end + + @doc """ + Returns all callbacks available from the module's BEAM code. + + The result is returned as a list of tuples where the first + element is spec name and arity and the second is the spec. + + The module must have a corresponding BEAM file + which can be located by the runtime system. The types will be + in the Erlang Abstract Format. + """ + @spec fetch_callbacks(module) :: {:ok, [tuple]} | :error + def fetch_callbacks(module) when is_atom(module) or is_binary(module) do + case typespecs_abstract_code(module) do + {:ok, abstract_code} -> + {:ok, for({:attribute, _, :callback, value} <- abstract_code, do: value)} + + :error -> + :error + end + end + + defp typespecs_abstract_code(module) do + with {module, binary} <- get_module_and_beam(module), + {:ok, {_, [debug_info: {:debug_info_v1, backend, data}]}} <- + :beam_lib.chunks(binary, [:debug_info]) do + case data do + {:elixir_v1, %{}, specs} -> + # Fast path to avoid translation to Erlang from Elixir. + {:ok, specs} + + _ -> + case backend.debug_info(:erlang_v1, module, data, []) do + {:ok, abstract_code} -> {:ok, abstract_code} + _ -> :error + end + end + else + _ -> :error + end + end + + defp get_module_and_beam(module) when is_atom(module) do + case :code.get_object_code(module) do + {^module, beam, _filename} -> {module, beam} + :error -> :error + end + end + + defp get_module_and_beam(beam) when is_binary(beam) do + case :beam_lib.info(beam) do + [_ | _] = info -> {info[:module], beam} + _ -> :error + end + end + + ## To AST conversion + + defp collect_vars({:ann_type, _anno, args}) when is_list(args) do + [] + end + + defp collect_vars({:type, _anno, _kind, args}) when is_list(args) do + Enum.flat_map(args, &collect_vars/1) + end + + defp collect_vars({:remote_type, _anno, args}) when is_list(args) do + Enum.flat_map(args, &collect_vars/1) + end + + defp collect_vars({:typed_record_field, _anno, type}) do + collect_vars(type) + end + + defp collect_vars({:paren_type, _anno, [type]}) do + collect_vars(type) + end + + defp collect_vars({:var, _anno, var}) do + [erl_to_ex_var(var)] + end + + defp collect_vars(_) do + [] + end + + defp typespec_to_quoted({:user_type, anno, name, args}) do + args = for arg <- args, do: typespec_to_quoted(arg) + {name, meta(anno), args} + end + + defp typespec_to_quoted({:type, anno, :tuple, :any}) do + {:tuple, meta(anno), []} + end + + defp typespec_to_quoted({:type, anno, :tuple, args}) do + args = for arg <- args, do: typespec_to_quoted(arg) + {:{}, meta(anno), args} + end + + defp typespec_to_quoted({:type, _anno, :list, [{:type, _, :union, unions} = arg]}) do + case unpack_typespec_kw(unions, []) do + {:ok, ast} -> ast + :error -> [typespec_to_quoted(arg)] + end + end + + defp typespec_to_quoted({:type, anno, :list, []}) do + {:list, meta(anno), []} + end + + defp typespec_to_quoted({:type, _anno, :list, [arg]}) do + [typespec_to_quoted(arg)] + end + + defp typespec_to_quoted({:type, anno, :nonempty_list, []}) do + [{:..., meta(anno), nil}] + end + + defp typespec_to_quoted({:type, anno, :nonempty_list, [arg]}) do + [typespec_to_quoted(arg), {:..., meta(anno), nil}] + end + + defp typespec_to_quoted({:type, anno, :map, :any}) do + {:map, meta(anno), []} + end + + defp typespec_to_quoted({:type, anno, :map, fields}) do + fields = + Enum.map(fields, fn + {:type, _, :map_field_assoc, :any} -> + {{:optional, [], [{:any, [], []}]}, {:any, [], []}} + + {:type, _, :map_field_exact, [{:atom, _, k}, v]} -> + {k, typespec_to_quoted(v)} + + {:type, _, :map_field_exact, [k, v]} -> + {{:required, [], [typespec_to_quoted(k)]}, typespec_to_quoted(v)} + + {:type, _, :map_field_assoc, [k, v]} -> + {{:optional, [], [typespec_to_quoted(k)]}, typespec_to_quoted(v)} + end) + + case List.keytake(fields, :__struct__, 0) do + {{:__struct__, struct}, fields_pruned} when is_atom(struct) and struct != nil -> + map_pruned = {:%{}, meta(anno), fields_pruned} + {:%, meta(anno), [struct, map_pruned]} + + _ -> + {:%{}, meta(anno), fields} + end + end + + defp typespec_to_quoted({:type, anno, :binary, [arg1, arg2]}) do + [arg1, arg2] = for arg <- [arg1, arg2], do: typespec_to_quoted(arg) + line = meta(anno)[:line] + + case {typespec_to_quoted(arg1), typespec_to_quoted(arg2)} do + {arg1, 0} -> + quote(line: line, do: <<_::unquote(arg1)>>) + + {0, arg2} -> + quote(line: line, do: <<_::_*unquote(arg2)>>) + + {arg1, arg2} -> + quote(line: line, do: <<_::unquote(arg1), _::_*unquote(arg2)>>) + end + end + + defp typespec_to_quoted({:type, anno, :union, args}) do + args = for arg <- args, do: typespec_to_quoted(arg) + Enum.reduce(Enum.reverse(args), fn arg, expr -> {:|, meta(anno), [arg, expr]} end) + end + + defp typespec_to_quoted({:type, anno, :fun, [{:type, _, :product, args}, result]}) do + args = for arg <- args, do: typespec_to_quoted(arg) + [{:->, meta(anno), [args, typespec_to_quoted(result)]}] + end + + defp typespec_to_quoted({:type, anno, :fun, [args, result]}) do + [{:->, meta(anno), [[typespec_to_quoted(args)], typespec_to_quoted(result)]}] + end + + defp typespec_to_quoted({:type, anno, :fun, []}) do + typespec_to_quoted({:type, anno, :fun, [{:type, anno, :any}, {:type, anno, :any, []}]}) + end + + defp typespec_to_quoted({:type, anno, :range, [left, right]}) do + {:.., meta(anno), [typespec_to_quoted(left), typespec_to_quoted(right)]} + end + + defp typespec_to_quoted({:type, _anno, nil, []}) do + [] + end + + defp typespec_to_quoted({:type, anno, name, args}) do + args = for arg <- args, do: typespec_to_quoted(arg) + {name, meta(anno), args} + end + + defp typespec_to_quoted({:var, anno, var}) do + {erl_to_ex_var(var), meta(anno), nil} + end + + defp typespec_to_quoted({:op, anno, op, arg}) do + {op, meta(anno), [typespec_to_quoted(arg)]} + end + + defp typespec_to_quoted({:remote_type, anno, [mod, name, args]}) do + remote_type(anno, mod, name, args) + end + + defp typespec_to_quoted({:ann_type, anno, [var, type]}) do + {:"::", meta(anno), [typespec_to_quoted(var), typespec_to_quoted(type)]} + end + + defp typespec_to_quoted( + {:typed_record_field, {:record_field, anno1, {:atom, anno2, name}}, type} + ) do + typespec_to_quoted({:ann_type, anno1, [{:var, anno2, name}, type]}) + end + + defp typespec_to_quoted({:type, _, :any}) do + quote(do: ...) + end + + defp typespec_to_quoted({:paren_type, _, [type]}) do + typespec_to_quoted(type) + end + + defp typespec_to_quoted({type, _anno, atom}) when is_atom(type) do + atom + end + + defp typespec_to_quoted(other), do: other + + ## Helpers + + defp remote_type(anno, {:atom, _, :elixir}, {:atom, _, :charlist}, []) do + typespec_to_quoted({:type, anno, :charlist, []}) + end + + defp remote_type(anno, {:atom, _, :elixir}, {:atom, _, :nonempty_charlist}, []) do + typespec_to_quoted({:type, anno, :nonempty_charlist, []}) + end + + defp remote_type(anno, {:atom, _, :elixir}, {:atom, _, :struct}, []) do + typespec_to_quoted({:type, anno, :struct, []}) + end + + defp remote_type(anno, {:atom, _, :elixir}, {:atom, _, :as_boolean}, [arg]) do + typespec_to_quoted({:type, anno, :as_boolean, [arg]}) + end + + defp remote_type(anno, {:atom, _, :elixir}, {:atom, _, :keyword}, args) do + typespec_to_quoted({:type, anno, :keyword, args}) + end + + defp remote_type(anno, mod, name, args) do + args = for arg <- args, do: typespec_to_quoted(arg) + dot = {:., meta(anno), [typespec_to_quoted(mod), typespec_to_quoted(name)]} + {dot, meta(anno), args} + end + + defp erl_to_ex_var(var) do + case Atom.to_string(var) do + <<"_", c::utf8, rest::binary>> -> + String.to_atom("_#{String.downcase(<>)}#{rest}") + + <> -> + String.to_atom("#{String.downcase(<>)}#{rest}") + end + end + + defp unpack_typespec_kw([{:type, _, :tuple, [{:atom, _, atom}, type]} | t], acc) do + unpack_typespec_kw(t, [{atom, typespec_to_quoted(type)} | acc]) + end + + defp unpack_typespec_kw([], acc) do + {:ok, Enum.reverse(acc)} + end + + defp unpack_typespec_kw(_, _acc) do + :error + end + + defp meta(anno), do: [line: :erl_anno.line(anno)] +end diff --git a/lib/elixir/lib/collectable.ex b/lib/elixir/lib/collectable.ex index 426f290a4e7..2ee45f14eb1 100644 --- a/lib/elixir/lib/collectable.ex +++ b/lib/elixir/lib/collectable.ex @@ -8,110 +8,172 @@ defprotocol Collectable do iex> Enum.into([a: 1, b: 2], %{}) %{a: 1, b: 2} - If a collection implements both `Enumerable` and `Collectable`, both - operations can be combined with `Enum.traverse/2`: - - iex> Enum.traverse(%{a: 1, b: 2}, fn {k, v} -> {k, v * 2} end) - %{a: 2, b: 4} - ## Why Collectable? The `Enumerable` protocol is useful to take values out of a collection. In order to support a wide range of values, the functions provided by the `Enumerable` protocol do not keep shape. For example, passing a - dictionary to `Enum.map/2` always returns a list. + map to `Enum.map/2` always returns a list. This design is intentional. `Enumerable` was designed to support infinite collections, resources and other structures with fixed shape. For example, - it doesn't make sense to insert values into a range, as it has a fixed - shape where just the range limits are stored. + it doesn't make sense to insert values into a `Range`, as it has a + fixed shape where only the range limits and step are stored. The `Collectable` module was designed to fill the gap left by the - `Enumerable` protocol. It provides two functions: `into/1` and `empty/1`. + `Enumerable` protocol. `Collectable.into/1` can be seen as the opposite of + `Enumerable.reduce/3`. If the functions in `Enumerable` are about taking values out, + then `Collectable.into/1` is about collecting those values into a structure. + + ## Examples + + To show how to manually use the `Collectable` protocol, let's play with a + simplified implementation for `MapSet`. + + iex> {initial_acc, collector_fun} = Collectable.into(MapSet.new()) + iex> updated_acc = Enum.reduce([1, 2, 3], initial_acc, fn elem, acc -> + ...> collector_fun.(acc, {:cont, elem}) + ...> end) + iex> collector_fun.(updated_acc, :done) + MapSet.new([1, 2, 3]) + + To show how the protocol can be implemented, we can again look at the + simplified implementation for `MapSet`. In this implementation "collecting" elements + simply means inserting them in the set through `MapSet.put/2`. + + defimpl Collectable, for: MapSet do + def into(map_set) do + collector_fun = fn + map_set_acc, {:cont, elem} -> + MapSet.put(map_set_acc, elem) + + map_set_acc, :done -> + map_set_acc - `into/1` can be seen as the opposite of `Enumerable.reduce/3`. If - `Enumerable` is about taking values out, `Collectable.into/1` is about - collecting those values into a structure. + _map_set_acc, :halt -> + :ok + end + + initial_acc = map_set + + {initial_acc, collector_fun} + end + end + + So now we can call `Enum.into/2`: + + iex> Enum.into([1, 2, 3], MapSet.new()) + MapSet.new([1, 2, 3]) - `empty/1` receives a collectable and returns an empty version of the - same collectable. By combining the enumerable functionality with `into/1` - and `empty/1`, one can, for example, implement a traversal mechanism. """ @type command :: {:cont, term} | :done | :halt @doc """ - Receives a collectable structure and returns an empty one. - """ - @spec empty(t) :: t - def empty(collectable) + Returns an initial accumulator and a "collector" function. - @doc """ - Returns a function that collects values alongside - the initial accumulation value. + Receives a `collectable` which can be used as the initial accumulator that will + be passed to the function. + + The collector function receives a term and a command and injects the term into + the collectable accumulator on every `{:cont, term}` command. - The returned function receives a collectable and injects a given - value into it for every `{:cont, term}` instruction. + `:done` is passed as a command when no further values will be injected. This + is useful when there's a need to close resources or normalizing values. A + collectable must be returned when the command is `:done`. - `:done` is passed when no further values will be injected, useful - for closing resources and normalizing values. A collectable must - be returned on `:done`. + If injection is suddenly interrupted, `:halt` is passed and the function + can return any value as it won't be used. - If injection is suddenly interrupted, `:halt` is passed and it can - return any value, as it won't be used. + For examples on how to use the `Collectable` protocol and `into/1` see the + module documentation. """ - @spec into(t) :: {term, (term, command -> t | term)} + @spec into(t) :: {initial_acc :: term, collector :: (term, command -> t | term)} def into(collectable) end defimpl Collectable, for: List do - def empty(_list) do - [] - end - - def into(original) do - {[], fn - list, {:cont, x} -> [x|list] - list, :done -> original ++ :lists.reverse(list) - _, :halt -> :ok - end} + def into(list) do + # TODO: Change the behaviour so the into always comes last on Elixir v2.0 + if list != [] do + IO.warn( + "the Collectable protocol is deprecated for non-empty lists. The behaviour of " <> + "Enum.into/2 and \"for\" comprehensions with an :into option is incorrect " <> + "when collecting into non-empty lists. If you're collecting into a non-empty keyword " <> + "list, consider using Keyword.merge/2 instead. If you're collecting into a non-empty " <> + "list, consider concatenating the two lists with the ++ operator." + ) + end + + fun = fn + list_acc, {:cont, elem} -> + [elem | list_acc] + + list_acc, :done -> + list ++ :lists.reverse(list_acc) + + _list_acc, :halt -> + :ok + end + + {[], fun} end end defimpl Collectable, for: BitString do - def empty(_bitstring) do - "" - end + def into(binary) when is_binary(binary) do + fun = fn + acc, {:cont, x} when is_binary(x) and is_list(acc) -> + [acc | x] - def into(original) do - {original, fn - bitstring, {:cont, x} -> <> - bitstring, :done -> bitstring - _, :halt -> :ok - end} - end -end + acc, {:cont, x} when is_bitstring(x) and is_bitstring(acc) -> + <> + + acc, {:cont, x} when is_bitstring(x) -> + <> + + acc, :done when is_bitstring(acc) -> + acc + + acc, :done -> + IO.iodata_to_binary(acc) -defimpl Collectable, for: Function do - def empty(function) do - function + __acc, :halt -> + :ok + end + + {[binary], fun} end - def into(function) do - {function, function} + def into(bitstring) do + fun = fn + acc, {:cont, x} when is_bitstring(x) -> + <> + + acc, :done -> + acc + + _acc, :halt -> + :ok + end + + {bitstring, fun} end end defimpl Collectable, for: Map do - def empty(_map) do - %{} - end + def into(map) do + fun = fn + map_acc, {:cont, {key, value}} -> + Map.put(map_acc, key, value) + + map_acc, :done -> + map_acc + + _map_acc, :halt -> + :ok + end - def into(original) do - {original, fn - map, {:cont, {k, v}} -> :maps.put(k, v, map) - map, :done -> map - _, :halt -> :ok - end} + {map, fun} end end diff --git a/lib/elixir/lib/config.ex b/lib/elixir/lib/config.ex new file mode 100644 index 00000000000..078d51a1116 --- /dev/null +++ b/lib/elixir/lib/config.ex @@ -0,0 +1,361 @@ +defmodule Config do + @moduledoc ~S""" + A simple keyword-based configuration API. + + ## Example + + This module is most commonly used to define application configuration, + typically in `config/config.exs`: + + import Config + + config :some_app, + key1: "value1", + key2: "value2" + + import_config "#{config_env()}.exs" + + `import Config` will import the functions `config/2`, `config/3` + `config_env/0`, `config_target/0`, and `import_config/1` + to help you manage your configuration. + + `config/2` and `config/3` are used to define key-value configuration + for a given application. Once Mix starts, it will automatically + evaluate the configuration file and persist the configuration above + into `:some_app`'s application environment, which can be accessed in + as follows: + + "value1" = Application.fetch_env!(:some_app, :key1) + + Finally, the line `import_config "#{config_env()}.exs"` will import + other config files based on the current configuration environment, + such as `config/dev.exs` and `config/test.exs`. + + `Config` also provides a low-level API for evaluating and reading + configuration, under the `Config.Reader` module. + + > **Important:** if you are writing a library to be used by other developers, + > it is generally recommended to avoid the application environment, as the + > application environment is effectively a global storage. Also note that + > the `config/config.exs` of a library is not evaluated when the library is + > used as a dependency, as configuration is always meant to configure the + > current project. For more information, read our [library guidelines](library-guidelines.md). + + ## Migrating from `use Mix.Config` + + The `Config` module in Elixir was introduced in v1.9 as a replacement to + `Mix.Config`, which was specific to Mix and has been deprecated. + + You can leverage `Config` instead of `Mix.Config` in three steps. The first + step is to replace `use Mix.Config` at the top of your config files by + `import Config`. + + The second is to make sure your `import_config/1` calls do not have a + wildcard character. If so, you need to perform the wildcard lookup + manually. For example, if you did: + + import_config "../apps/*/config/config.exs" + + It has to be replaced by: + + for config <- "../apps/*/config/config.exs" |> Path.expand(__DIR__) |> Path.wildcard() do + import_config config + end + + The last step is to replace all `Mix.env()` calls in the config files with `config_env()`. + + Keep in mind you must also avoid using `Mix.env()` inside your project files. + To check the environment at _runtime_, you may add a configuration key: + + # config.exs + ... + config :my_app, env: config_env() + + Then, in other scripts and modules, you may get the environment with + `Application.fetch_env!/2`: + + # router.exs + ... + if Application.fetch_env!(:my_app, :env) == :prod do + ... + end + + The only files where you may access functions from the `Mix` module are + the `mix.exs` file and inside custom Mix tasks, which always within the + `Mix.Tasks` namespace. + + ## config/runtime.exs + + For runtime configuration, you can use the `config/runtime.exs` file. + It is executed right before applications start in both Mix and releases + (assembled with `mix release`). + """ + + @opts_key {__MODULE__, :opts} + @config_key {__MODULE__, :config} + @imports_key {__MODULE__, :imports} + + defp get_opts!(), do: Process.get(@opts_key) || raise_improper_use!() + defp put_opts(value), do: Process.put(@opts_key, value) + defp delete_opts(), do: Process.delete(@opts_key) + + defp get_config!(), do: Process.get(@config_key) || raise_improper_use!() + defp put_config(value), do: Process.put(@config_key, value) + defp delete_config(), do: Process.delete(@config_key) + + defp get_imports!(), do: Process.get(@imports_key) || raise_improper_use!() + defp put_imports(value), do: Process.put(@imports_key, value) + defp delete_imports(), do: Process.delete(@imports_key) + + defp raise_improper_use!() do + raise "could not set configuration via Config. " <> + "This usually means you are trying to execute a configuration file " <> + "directly, instead of reading it with Config.Reader" + end + + @doc """ + Configures the given `root_key`. + + Keyword lists are always deep-merged. + + ## Examples + + The given `opts` are merged into the existing configuration + for the given `root_key`. Conflicting keys are overridden by the + ones specified in `opts`, unless they are keywords, which are + deep merged recursively. For example, the application configuration + below + + config :logger, + level: :warn, + backends: [:console] + + config :logger, + level: :info, + truncate: 1024 + + will have a final configuration for `:logger` of: + + [level: :info, backends: [:console], truncate: 1024] + + """ + @doc since: "1.9.0" + def config(root_key, opts) when is_atom(root_key) and is_list(opts) do + unless Keyword.keyword?(opts) do + raise ArgumentError, "config/2 expected a keyword list, got: #{inspect(opts)}" + end + + get_config!() + |> __merge__([{root_key, opts}]) + |> put_config() + end + + @doc """ + Configures the given `key` for the given `root_key`. + + Keyword lists are always deep merged. + + ## Examples + + The given `opts` are merged into the existing values for `key` + in the given `root_key`. Conflicting keys are overridden by the + ones specified in `opts`, unless they are keywords, which are + deep merged recursively. For example, the application configuration + below + + config :ecto, Repo, + log_level: :warn, + adapter: Ecto.Adapters.Postgres, + metadata: [read_only: true] + + config :ecto, Repo, + log_level: :info, + pool_size: 10, + metadata: [replica: true] + + will have a final value of the configuration for the `Repo` + key in the `:ecto` application of: + + Application.get_env(:ecto, Repo) + #=> [ + #=> log_level: :info, + #=> pool_size: 10, + #=> adapter: Ecto.Adapters.Postgres, + #=> metadata: [read_only: true, replica: true] + #=> ] + + """ + @doc since: "1.9.0" + def config(root_key, key, opts) when is_atom(root_key) and is_atom(key) do + get_config!() + |> __merge__([{root_key, [{key, opts}]}]) + |> put_config() + end + + @doc """ + Returns the environment this configuration file is executed on. + + In Mix projects this function returns the environment this configuration + file is executed on. In releases, the environment when `mix release` ran. + + This is most often used to execute conditional code: + + if config_env() == :prod do + config :my_app, :debug, false + end + + """ + @doc since: "1.11.0" + defmacro config_env() do + quote do + Config.__env__!() + end + end + + @doc false + @spec __env__!() :: atom() + def __env__!() do + elem(get_opts!(), 0) || raise "no :env key was given to this configuration file" + end + + @doc """ + Returns the target this configuration file is executed on. + + This is most often used to execute conditional code: + + if config_target() == :host do + config :my_app, :debug, false + end + + """ + @doc since: "1.11.0" + defmacro config_target() do + quote do + Config.__target__!() + end + end + + @doc false + @spec __target__!() :: atom() + def __target__!() do + elem(get_opts!(), 1) || raise "no :target key was given to this configuration file" + end + + @doc ~S""" + Imports configuration from the given file. + + In case the file doesn't exist, an error is raised. + + If file is a relative, it will be expanded relatively to the + directory the current configuration file is in. + + ## Examples + + This is often used to emulate configuration across environments: + + import_config "#{config_env()}.exs" + + Note, however, some configuration files, such as `config/runtime.exs` + does not support imports, as they are meant to be copied across + systems. + """ + @doc since: "1.9.0" + defmacro import_config(file) do + quote do + Config.__import__!(Path.expand(unquote(file), __DIR__)) + :ok + end + end + + @doc false + @spec __import__!(Path.t()) :: {term, Code.binding()} + def __import__!(file) when is_binary(file) do + import_config!(file, File.read!(file), true) + end + + @doc false + @spec __eval__!(Path.t(), binary(), keyword) :: {keyword, [Path.t()] | :disabled} + def __eval__!(file, content, opts \\ []) when is_binary(file) and is_list(opts) do + env = Keyword.get(opts, :env) + target = Keyword.get(opts, :target) + imports = Keyword.get(opts, :imports, []) + + previous_opts = put_opts({env, target}) + previous_config = put_config([]) + previous_imports = put_imports(imports) + + try do + {eval_config, _} = import_config!(file, content, false) + + case get_config!() do + [] when is_list(eval_config) -> + {validate!(eval_config, file), get_imports!()} + + pdict_config -> + {pdict_config, get_imports!()} + end + after + if previous_opts, do: put_opts(previous_opts), else: delete_opts() + if previous_config, do: put_config(previous_config), else: delete_config() + if previous_imports, do: put_imports(previous_imports), else: delete_imports() + end + end + + defp import_config!(file, contents, raise_when_disabled?) do + current_imports = get_imports!() + + cond do + current_imports == :disabled -> + if raise_when_disabled? do + raise "import_config/1 is not enabled for this configuration file. " <> + "Some configuration files do not allow importing other files " <> + "as they are often copied to external systems" + end + + file in current_imports -> + raise ArgumentError, + "attempting to load configuration #{Path.relative_to_cwd(file)} recursively" + + true -> + put_imports([file | current_imports]) + :ok + end + + # TODO: Emit a warning if Mix.env() is found in said files in Elixir v1.15. + # Note this won't be a deprecation warning as it will always be emitted. + Code.eval_string(contents, [], file: file) + end + + @doc false + def __merge__(config1, config2) when is_list(config1) and is_list(config2) do + Keyword.merge(config1, config2, fn _, app1, app2 -> + Keyword.merge(app1, app2, &deep_merge/3) + end) + end + + defp deep_merge(_key, value1, value2) do + if Keyword.keyword?(value1) and Keyword.keyword?(value2) do + Keyword.merge(value1, value2, &deep_merge/3) + else + value2 + end + end + + defp validate!(config, file) do + Enum.all?(config, fn + {app, value} when is_atom(app) -> + if Keyword.keyword?(value) do + true + else + raise ArgumentError, + "expected config for app #{inspect(app)} in #{Path.relative_to_cwd(file)} " <> + "to return keyword list, got: #{inspect(value)}" + end + + _ -> + false + end) + + config + end +end diff --git a/lib/elixir/lib/config/provider.ex b/lib/elixir/lib/config/provider.ex new file mode 100644 index 00000000000..af835375d87 --- /dev/null +++ b/lib/elixir/lib/config/provider.ex @@ -0,0 +1,415 @@ +defmodule Config.Provider do + @moduledoc """ + Specifies a provider API that loads configuration during boot. + + Config providers are typically used during releases to load + external configuration while the system boots. This is done + by starting the VM with the minimum amount of applications + running, then invoking all of the providers, and then + restarting the system. This requires a mutable configuration + file on disk, as the results of the providers are written to + the file system. For more information on runtime configuration, + see `mix release`. + + ## Multiple config files + + One common use of config providers is to specify multiple + configuration files in a release. Elixir ships with one provider, + called `Config.Reader`, which is capable of handling Elixir's + built-in config files. + + For example, imagine you want to list some basic configuration + on Mix's built-in `config/runtime.exs` file, but you also want + to support additional configuration files. To do so, you can add + this inside the `def project` portion of your `mix.exs`: + + releases: [ + demo: [ + config_providers: [ + {Config.Reader, {:system, "RELEASE_ROOT", "/extra_config.exs"}} + ] + ] + ] + + You can place this `extra_config.exs` file in your release in + multiple ways: + + 1. If it is available on the host when assembling the release, + you can place it on "rel/overlays/extra_config.exs" and it + will be automatically copied to the release root + + 2. If it is available on the target during deployment, you can + simply copy it to the release root as a step in your deployment + + Now once the system boots, it will load both `config/runtime.exs` + and `extra_config.exs` early in the boot process. You can learn + more options on `Config.Reader`. + + ## Custom config provider + + You can also implement custom config providers, similar to how + `Config.Reader` works. For example, imagine you need to load + some configuration from a JSON file and load that into the system. + Said configuration provider would look like: + + defmodule JSONConfigProvider do + @behaviour Config.Provider + + # Let's pass the path to the JSON file as config + @impl true + def init(path) when is_binary(path), do: path + + @impl true + def load(config, path) do + # We need to start any app we may depend on. + {:ok, _} = Application.ensure_all_started(:jason) + + json = path |> File.read!() |> Jason.decode!() + + Config.Reader.merge( + config, + my_app: [ + some_value: json["my_app_some_value"], + another_value: json["my_app_another_value"], + ] + ) + end + end + + Then, when specifying your release, you can specify the provider in + the release configuration: + + releases: [ + demo: [ + config_providers: [ + {JSONConfigProvider, "/etc/config.json"} + ] + ] + ] + + """ + + @type config :: keyword + @type state :: term + + @typedoc """ + A path pointing to a configuration file. + + Since configuration files are often accessed on target machines, + it can be expressed either as: + + * a binary representing an absolute path + + * a `{:system, system_var, path}` tuple where the config is the + concatenation of the environment variable `system_var` with + the given `path` + + """ + @type config_path :: {:system, binary(), binary()} | binary() + + @doc """ + Invoked when initializing a config provider. + + A config provider is typically initialized on the machine + where the system is assembled and not on the target machine. + The `c:init/1` callback is useful to verify the arguments + given to the provider and prepare the state that will be + given to `c:load/2`. + + Furthermore, because the state returned by `c:init/1` can + be written to text-based config files, it should be + restricted only to simple data types, such as integers, + strings, atoms, tuples, maps, and lists. Entries such as + PIDs, references, and functions cannot be serialized. + """ + @callback init(term) :: state + + @doc """ + Loads configuration (typically during system boot). + + It receives the current `config` and the `state` returned by + `c:init/1`. Then, you typically read the extra configuration + from an external source and merge it into the received `config`. + Merging should be done with `Config.Reader.merge/2`, as it + performs deep merge. It should return the updated config. + + Note that `c:load/2` is typically invoked very early in the + boot process, therefore if you need to use an application + in the provider, it is your responsibility to start it. + """ + @callback load(config, state) :: config + + @doc false + defstruct [ + :providers, + :config_path, + extra_config: [], + prune_runtime_sys_config_after_boot: false, + reboot_system_after_config: false, + validate_compile_env: false + ] + + @reserved_apps [:kernel, :stdlib] + + @doc """ + Validates a `t:config_path/0`. + """ + @doc since: "1.9.0" + @spec validate_config_path!(config_path) :: :ok + def validate_config_path!({:system, name, path}) + when is_binary(name) and is_binary(path), + do: :ok + + def validate_config_path!(path) do + if is_binary(path) and Path.type(path) != :relative do + :ok + else + raise ArgumentError, """ + expected configuration path to be: + + * a binary representing an absolute path + * a tuple {:system, system_var, path} where the config is the \ + concatenation of the `system_var` with the given `path` + + Got: #{inspect(path)} + """ + end + end + + @doc """ + Resolves a `t:config_path/0` to an actual path. + """ + @doc since: "1.9.0" + @spec resolve_config_path!(config_path) :: binary + def resolve_config_path!(path) when is_binary(path), do: path + def resolve_config_path!({:system, name, path}), do: System.fetch_env!(name) <> path + + # Private keys + @init_key :config_provider_init + @booted_key :config_provider_booted + + # Public keys + @reboot_mode_key :config_provider_reboot_mode + + @doc false + def init(providers, config_path, opts \\ []) when is_list(providers) and is_list(opts) do + validate_config_path!(config_path) + providers = for {provider, init} <- providers, do: {provider, provider.init(init)} + init = struct!(%Config.Provider{config_path: config_path, providers: providers}, opts) + [elixir: [{@init_key, init}]] + end + + @doc false + def boot(reboot_fun \\ &restart_and_sleep/0) do + # The config provider typically runs very early in the + # release process, so we need to make sure Elixir is started + # before we go around running Elixir code. + {:ok, _} = :application.ensure_all_started(:elixir) + + case Application.fetch_env(:elixir, @booted_key) do + {:ok, {:booted, path}} -> + path && File.rm(path) + + with {:ok, %Config.Provider{} = provider} <- Application.fetch_env(:elixir, @init_key) do + maybe_validate_compile_env(provider) + end + + :booted + + _ -> + case Application.fetch_env(:elixir, @init_key) do + {:ok, %Config.Provider{} = provider} -> + path = resolve_config_path!(provider.config_path) + reboot_config = [elixir: [{@booted_key, booted_value(provider, path)}]] + boot_providers(path, provider, reboot_config, reboot_fun) + + _ -> + :skip + end + end + end + + defp boot_providers(path, provider, reboot_config, reboot_fun) do + original_config = read_config!(path) + + config = + original_config + |> Config.__merge__(provider.extra_config) + |> run_providers(provider) + + if provider.reboot_system_after_config do + config + |> Config.__merge__(reboot_config) + |> write_config!(path) + + reboot_fun.() + else + for app <- @reserved_apps, config[app] != original_config[app] do + abort(""" + Cannot configure #{inspect(app)} because :reboot_system_after_config has been set \ + to false and #{inspect(app)} has already been loaded, meaning any further \ + configuration won't have an effect. + + The configuration for #{inspect(app)} before config providers was: + + #{inspect(original_config[app])} + + The configuration for #{inspect(app)} after config providers was: + + #{inspect(config[app])} + """) + end + + _ = Application.put_all_env(config, persistent: true) + maybe_validate_compile_env(provider) + :ok + end + end + + defp maybe_validate_compile_env(provider) do + with [_ | _] = compile_env <- provider.validate_compile_env do + validate_compile_env(compile_env) + end + end + + @doc false + def validate_compile_env(compile_env, ensure_loaded? \\ true) do + for {app, [key | path], compile_return} <- compile_env, + ensure_app_loaded?(app, ensure_loaded?) do + try do + traverse_env(Application.fetch_env(app, key), path) + rescue + e -> + abort(""" + application #{inspect(app)} failed reading its compile environment #{path(key, path)}: + + #{Exception.format(:error, e, __STACKTRACE__)} + + Expected it to match the compile time value of #{return_to_text(compile_return)}. + + #{compile_env_tips(app)} + """) + else + ^compile_return -> + :ok + + runtime_return -> + abort(""" + the application #{inspect(app)} has a different value set #{path(key, path)} \ + during runtime compared to compile time. Since this application environment entry was \ + marked as compile time, this difference can lead to different behaviour than expected: + + * Compile time value #{return_to_text(compile_return)} + * Runtime value #{return_to_text(runtime_return)} + + #{compile_env_tips(app)} + """) + end + end + + :ok + end + + defp ensure_app_loaded?(app, true), do: Application.ensure_loaded(app) == :ok + defp ensure_app_loaded?(app, false), do: Application.spec(app, :vsn) != nil + + defp path(key, []), do: "for key #{inspect(key)}" + defp path(key, path), do: "for path #{inspect(path)} inside key #{inspect(key)}" + + defp compile_env_tips(app), + do: """ + To fix this error, you might: + + * Make the runtime value match the compile time one + + * Recompile your project. If the misconfigured application is a dependency, \ + you may need to run "mix deps.compile #{app} --force" + + * Alternatively, you can disable this check. If you are using releases, you can \ + set :validate_compile_env to false in your release configuration. If you are \ + using Mix to start your system, you can pass the --no-validate-compile-env flag + """ + + defp return_to_text({:ok, value}), do: "was set to: #{inspect(value)}" + defp return_to_text(:error), do: "was not set" + + defp traverse_env(return, []), do: return + defp traverse_env(:error, _paths), do: :error + defp traverse_env({:ok, value}, [key | keys]), do: traverse_env(Access.fetch(value, key), keys) + + @compile {:no_warn_undefined, {:init, :restart, 1}} + defp restart_and_sleep() do + mode = Application.get_env(:elixir, @reboot_mode_key) + + if mode in [:embedded, :interactive] do + :init.restart(mode: mode) + else + :init.restart() + end + + Process.sleep(:infinity) + end + + defp booted_value(%{prune_runtime_sys_config_after_boot: true}, path), do: {:booted, path} + defp booted_value(%{prune_runtime_sys_config_after_boot: false}, _path), do: {:booted, nil} + + defp read_config!(path) do + case :file.consult(path) do + {:ok, [inner]} -> + inner + + {:error, reason} -> + bad_path_abort( + "Could not read runtime configuration due to reason: #{inspect(reason)}", + path + ) + end + end + + defp run_providers(config, %{providers: providers}) do + Enum.reduce(providers, config, fn {provider, state}, acc -> + try do + provider.load(acc, state) + catch + kind, error -> + IO.puts(:stderr, "ERROR! Config provider #{inspect(provider)} failed with:") + IO.puts(:stderr, Exception.format(kind, error, __STACKTRACE__)) + :erlang.raise(kind, error, __STACKTRACE__) + else + term when is_list(term) -> + term + + term -> + abort("Expected provider #{inspect(provider)} to return a list, got: #{inspect(term)}") + end + end) + end + + defp write_config!(config, path) do + contents = :io_lib.format("%% coding: utf-8~n~tw.~n", [config]) + + case File.write(path, IO.chardata_to_string(contents)) do + :ok -> + :ok + + {:error, reason} -> + bad_path_abort( + "Could not write runtime configuration due to reason: #{inspect(reason)}", + path + ) + end + end + + defp bad_path_abort(msg, path) do + abort( + msg <> + ". Please make sure #{inspect(path)} is writable and accessible " <> + "or choose a different path" + ) + end + + defp abort(msg) do + IO.puts("ERROR! " <> msg) + :erlang.raise(:error, "aborting boot", [{Config.Provider, :boot, 2, []}]) + end +end diff --git a/lib/elixir/lib/config/reader.ex b/lib/elixir/lib/config/reader.ex new file mode 100644 index 00000000000..a9df052d0e7 --- /dev/null +++ b/lib/elixir/lib/config/reader.ex @@ -0,0 +1,138 @@ +defmodule Config.Reader do + @moduledoc """ + API for reading config files defined with `Config`. + + ## As a provider + + `Config.Reader` can also be used as a `Config.Provider`. A config + provider is used during releases to customize how applications are + configured. When used as a provider, it expects a single argument: + the configuration path (as outlined in `t:Config.Provider.config_path/0`) + for the file to be read and loaded during the system boot. + + For example, if you expect the target system to have a config file + in an absolute path, you can add this inside the `def project` portion + of your `mix.exs`: + + releases: [ + demo: [ + config_providers: [ + {Config.Reader, "/etc/config.exs"} + ] + ] + ] + + Or if you want to read a custom path inside the release: + + config_providers: [{Config.Reader, {:system, "RELEASE_ROOT", "/config.exs"}}] + + You can also pass a keyword list of options to the reader, + where the `:path` is a required key: + + config_providers: [ + {Config.Reader, + path: "/etc/config.exs", + env: :prod, + imports: :disabled} + ] + + Remember Mix already loads `config/runtime.exs` by default. + For more examples and scenarios, see the `Config.Provider` module. + """ + + @behaviour Config.Provider + + @impl true + def init(opts) when is_list(opts) do + {path, opts} = Keyword.pop!(opts, :path) + Config.Provider.validate_config_path!(path) + {path, opts} + end + + def init(path) do + init(path: path) + end + + @impl true + def load(config, {path, opts}) do + merge(config, path |> Config.Provider.resolve_config_path!() |> read!(opts)) + end + + @doc """ + Evaluates the configuration `contents` for the given `file`. + + Accepts the same options as `read!/2`. + """ + @doc since: "1.11.0" + @spec eval!(Path.t(), binary, keyword) :: keyword + def eval!(file, contents, opts \\ []) + when is_binary(file) and is_binary(contents) and is_list(opts) do + Config.__eval__!(Path.expand(file), contents, opts) |> elem(0) + end + + @doc """ + Reads the configuration file. + + ## Options + + * `:imports` - a list of already imported paths or `:disabled` + to disable imports + + * `:env` - the environment the configuration file runs on. + See `Config.config_env/0` for sample usage + + * `:target` - the target the configuration file runs on. + See `Config.config_target/0` for sample usage + + """ + @doc since: "1.9.0" + @spec read!(Path.t(), keyword) :: keyword + def read!(file, opts \\ []) when is_binary(file) and is_list(opts) do + file = Path.expand(file) + Config.__eval__!(file, File.read!(file), opts) |> elem(0) + end + + @doc """ + Reads the given configuration file and returns the configuration + with its imports. + + Accepts the same options as `read!/2`. Although note the `:imports` + option cannot be disabled in `read_imports!/2`. + """ + @doc since: "1.9.0" + @spec read_imports!(Path.t(), keyword) :: {keyword, [Path.t()]} + def read_imports!(file, opts \\ []) when is_binary(file) and is_list(opts) do + if opts[:imports] == :disabled do + raise ArgumentError, ":imports must be a list of paths" + end + + file = Path.expand(file) + Config.__eval__!(file, File.read!(file), opts) + end + + @doc """ + Merges two configurations. + + The configurations are merged together with the values in + the second one having higher preference than the first in + case of conflicts. In case both values are set to keyword + lists, it deep merges them. + + ## Examples + + iex> Config.Reader.merge([app: [k: :v1]], [app: [k: :v2]]) + [app: [k: :v2]] + + iex> Config.Reader.merge([app: [k: [v1: 1, v2: 2]]], [app: [k: [v2: :a, v3: :b]]]) + [app: [k: [v1: 1, v2: :a, v3: :b]]] + + iex> Config.Reader.merge([app1: []], [app2: []]) + [app1: [], app2: []] + + """ + @doc since: "1.9.0" + @spec merge(keyword, keyword) :: keyword + def merge(config1, config2) when is_list(config1) and is_list(config2) do + Config.__merge__(config1, config2) + end +end diff --git a/lib/elixir/lib/dict.ex b/lib/elixir/lib/dict.ex index 3f3b5d3e444..a393f26089f 100644 --- a/lib/elixir/lib/dict.ex +++ b/lib/elixir/lib/dict.ex @@ -1,139 +1,35 @@ defmodule Dict do @moduledoc ~S""" - This module specifies the Dict API expected to be - implemented by different dictionaries. It also provides - functions that redirect to the underlying Dict, allowing - a developer to work with different Dict implementations - using one API. + Generic API for dictionaries. - To create a new dict, use the `new` functions defined - by each dict type: - - HashDict.new #=> creates an empty HashDict - - In the examples below, `dict_impl` means a specific - `Dict` implementation, for example `HashDict` or `Map`. - - ## Protocols - - Besides implementing the functions in this module, all - dictionaries are required to implement the `Access` - protocol: - - iex> dict = dict_impl.new - iex> dict = Dict.put(dict, :hello, :world) - iex> dict[:hello] - :world - - As well as the `Enumerable` and `Collectable` protocols. - - ## Match - - Dictionaries are required to implement all operations - using the match (`===`) operator. - - ## Default implementation - - Default implementations for some functions in the `Dict` module - are provided via `use Dict`. - - For example: - - defmodule MyDict do - use Dict - - # implement required functions (see below) - # override default implementations if optimization - # is needed - end - - The client module must contain the following functions: - - * `delete/2` - * `fetch/2` - * `put/3` - * `reduce/3` - * `size/1` - - All functions, except `reduce/3`, are required by the Dict behaviour. - `reduce/3` must be implemtented as per the Enumerable protocol. - - Based on these functions, `Dict` generates default implementations - for the following functions: - - * `drop/2` - * `equal?/2` - * `fetch!/2` - * `get/2` - * `get/3` - * `has_key?/2` - * `keys/1` - * `merge/2` - * `merge/3` - * `pop/2` - * `pop/3` - * `put_new/3` - * `split/2` - * `take/2` - * `to_list/1` - * `update/4` - * `update!/3` - * `values/1` - - All of these functions are defined as overridable, so you can provide - your own implementation if needed. - - Note you can also test your custom module via `Dict`'s doctests: - - defmodule MyDict do - # ... - end - - defmodule MyTests do - use ExUnit.Case - doctest Dict - defp dict_impl, do: MyDict - end + If you need a general dictionary, use the `Map` module. + If you need to manipulate keyword lists, use `Keyword`. + To convert maps into keywords and vice-versa, use the + `new` function in the respective modules. """ - use Behaviour + @moduledoc deprecated: "Use Map or Keyword modules instead" @type key :: any @type value :: any @type t :: list | map - defcallback new :: t - defcallback delete(t, key) :: t - defcallback drop(t, Enum.t) :: t - defcallback equal?(t, t) :: boolean - defcallback get(t, key) :: value - defcallback get(t, key, value) :: value - defcallback fetch(t, key) :: {:ok, value} | :error - defcallback fetch!(t, key) :: value | no_return - defcallback has_key?(t, key) :: boolean - defcallback keys(t) :: [key] - defcallback merge(t, t) :: t - defcallback merge(t, t, (key, value, value -> value)) :: t - defcallback pop(t, key) :: {value, t} - defcallback pop(t, key, value) :: {value, t} - defcallback put(t, key, value) :: t - defcallback put_new(t, key, value) :: t - defcallback size(t) :: non_neg_integer() - defcallback split(t, Enum.t) :: {t, t} - defcallback take(t, Enum.t) :: t - defcallback to_list(t) :: list() - defcallback update(t, key, value, (value -> value)) :: t - defcallback update!(t, key, (value -> value)) :: t | no_return - defcallback values(t) :: list(value) + message = + "Use the Map module for working with maps or the Keyword module for working with keyword lists" defmacro __using__(_) do # Use this import to guarantee proper code expansion import Kernel, except: [size: 1] + if __CALLER__.module != HashDict do + IO.warn("use Dict is deprecated. " <> unquote(message), __CALLER__) + end + quote do - @behaviour Dict + message = "Use maps and the Map module instead" + @deprecated message def get(dict, key, default \\ nil) do case fetch(dict, key) do {:ok, value} -> value @@ -141,6 +37,22 @@ defmodule Dict do end end + @deprecated message + def get_lazy(dict, key, fun) when is_function(fun, 0) do + case fetch(dict, key) do + {:ok, value} -> value + :error -> fun.() + end + end + + @deprecated message + def get_and_update(dict, key, fun) do + current_value = get(dict, key) + {get, new_value} = fun.(current_value) + {get, put(dict, key, new_value)} + end + + @deprecated message def fetch!(dict, key) do case fetch(dict, key) do {:ok, value} -> value @@ -148,23 +60,35 @@ defmodule Dict do end end + @deprecated message def has_key?(dict, key) do - match? {:ok, _}, fetch(dict, key) + match?({:ok, _}, fetch(dict, key)) end + @deprecated message def put_new(dict, key, value) do case has_key?(dict, key) do - true -> dict + true -> dict false -> put(dict, key, value) end end + @deprecated message + def put_new_lazy(dict, key, fun) when is_function(fun, 0) do + case has_key?(dict, key) do + true -> dict + false -> put(dict, key, fun.()) + end + end + + @deprecated message def drop(dict, keys) do Enum.reduce(keys, dict, &delete(&2, &1)) end + @deprecated message def take(dict, keys) do - Enum.reduce(keys, new, fn key, acc -> + Enum.reduce(keys, new(), fn key, acc -> case fetch(dict, key) do {:ok, value} -> put(acc, key, value) :error -> acc @@ -172,41 +96,49 @@ defmodule Dict do end) end + @deprecated message def to_list(dict) do - reduce(dict, {:cont, []}, fn - kv, acc -> {:cont, [kv|acc]} - end) |> elem(1) |> :lists.reverse + reduce(dict, {:cont, []}, fn kv, acc -> {:cont, [kv | acc]} end) + |> elem(1) + |> :lists.reverse() end + @deprecated message def keys(dict) do - reduce(dict, {:cont, []}, fn - {k, _}, acc -> {:cont, [k|acc]} - end) |> elem(1) |> :lists.reverse + reduce(dict, {:cont, []}, fn {k, _}, acc -> {:cont, [k | acc]} end) + |> elem(1) + |> :lists.reverse() end + @deprecated message def values(dict) do - reduce(dict, {:cont, []}, fn - {_, v}, acc -> {:cont, [v|acc]} - end) |> elem(1) |> :lists.reverse + reduce(dict, {:cont, []}, fn {_, v}, acc -> {:cont, [v | acc]} end) + |> elem(1) + |> :lists.reverse() end + @deprecated message def equal?(dict1, dict2) do # Use this import to avoid conflicts in the user code import Kernel, except: [size: 1] case size(dict1) == size(dict2) do - false -> false - true -> - reduce(dict1, {:cont, true}, fn({k, v}, _acc) -> + false -> + false + + true -> + reduce(dict1, {:cont, true}, fn {k, v}, _acc -> case fetch(dict2, k) do {:ok, ^v} -> {:cont, true} _ -> {:halt, false} end - end) |> elem(1) + end) + |> elem(1) end end - def merge(dict1, dict2, fun \\ fn(_k, _v1, v2) -> v2 end) do + @deprecated message + def merge(dict1, dict2, fun \\ fn _k, _v1, v2 -> v2 end) do # Use this import to avoid conflicts in the user code import Kernel, except: [size: 1] @@ -218,444 +150,263 @@ defmodule Dict do reduce(dict2, {:cont, dict1}, fn {k, v2}, acc -> {:cont, update(acc, k, v2, &fun.(k, &1, v2))} end) - end |> elem(1) + end + |> elem(1) end - def update(dict, key, initial, fun) do + @deprecated message + def update(dict, key, default, fun) do case fetch(dict, key) do {:ok, value} -> put(dict, key, fun.(value)) + :error -> - put(dict, key, initial) + put(dict, key, default) end end + @deprecated message def update!(dict, key, fun) do case fetch(dict, key) do {:ok, value} -> put(dict, key, fun.(value)) + :error -> raise KeyError, key: key, term: dict end end + @deprecated message def pop(dict, key, default \\ nil) do case fetch(dict, key) do {:ok, value} -> {value, delete(dict, key)} + :error -> {default, dict} end end + @deprecated message + def pop_lazy(dict, key, fun) when is_function(fun, 0) do + case fetch(dict, key) do + {:ok, value} -> + {value, delete(dict, key)} + + :error -> + {fun.(), dict} + end + end + + @deprecated message def split(dict, keys) do - Enum.reduce(keys, {new, dict}, fn key, {inc, exc} = acc -> + Enum.reduce(keys, {new(), dict}, fn key, {inc, exc} = acc -> case fetch(exc, key) do {:ok, value} -> {put(inc, key, value), delete(exc, key)} + :error -> acc end end) end - defoverridable merge: 2, merge: 3, equal?: 2, to_list: 1, keys: 1, - values: 1, take: 2, drop: 2, get: 2, get: 3, fetch!: 2, - has_key?: 2, put_new: 3, pop: 2, pop: 3, split: 2, - update: 4, update!: 3 + defoverridable merge: 2, + merge: 3, + equal?: 2, + to_list: 1, + keys: 1, + values: 1, + take: 2, + drop: 2, + get: 2, + get: 3, + fetch!: 2, + has_key?: 2, + put_new: 3, + pop: 2, + pop: 3, + split: 2, + update: 4, + update!: 3, + get_and_update: 3, + get_lazy: 3, + pop_lazy: 3, + put_new_lazy: 3 end end - defmacrop target(dict) do quote do case unquote(dict) do - %{__struct__: x} when is_atom(x) -> - x - %{} -> - Map - x when is_list(x) -> - Keyword - x -> - unsupported_dict(x) + %module{} -> module + %{} -> Map + dict when is_list(dict) -> Keyword + dict -> unsupported_dict(dict) end end end - @doc """ - Returns a list of all keys in `dict`. - The keys are not guaranteed to be in any order. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> Enum.sort(Dict.keys(d)) - [:a,:b] - - """ + @deprecated message @spec keys(t) :: [key] def keys(dict) do target(dict).keys(dict) end - @doc """ - Returns a list of all values in `dict`. - The values are not guaranteed to be in any order. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> Enum.sort(Dict.values(d)) - [1,2] - - """ + @deprecated message @spec values(t) :: [value] def values(dict) do target(dict).values(dict) end - @doc """ - Returns the number of elements in `dict`. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> Dict.size(d) - 2 - - """ + @deprecated message @spec size(t) :: non_neg_integer def size(dict) do target(dict).size(dict) end - @doc """ - Returns whether the given `key` exists in the given `dict`. - - ## Examples - - iex> d = Enum.into([a: 1], dict_impl.new) - iex> Dict.has_key?(d, :a) - true - iex> Dict.has_key?(d, :b) - false - - """ + @deprecated message @spec has_key?(t, key) :: boolean def has_key?(dict, key) do target(dict).has_key?(dict, key) end - @doc """ - Returns the value associated with `key` in `dict`. If `dict` does not - contain `key`, returns `default` (or `nil` if not provided). - - ## Examples - - iex> d = Enum.into([a: 1], dict_impl.new) - iex> Dict.get(d, :a) - 1 - iex> Dict.get(d, :b) - nil - iex> Dict.get(d, :b, 3) - 3 - """ + @deprecated message @spec get(t, key, value) :: value def get(dict, key, default \\ nil) do target(dict).get(dict, key, default) end - @doc """ - Returns `{:ok, value}` associated with `key` in `dict`. - If `dict` does not contain `key`, returns `:error`. - - ## Examples + @deprecated message + @spec get_lazy(t, key, (() -> value)) :: value + def get_lazy(dict, key, fun) do + target(dict).get_lazy(dict, key, fun) + end - iex> d = Enum.into([a: 1], dict_impl.new) - iex> Dict.fetch(d, :a) - {:ok, 1} - iex> Dict.fetch(d, :b) - :error + @deprecated message + @spec get_and_update(t, key, (value -> {value, value})) :: {value, t} + def get_and_update(dict, key, fun) do + target(dict).get_and_update(dict, key, fun) + end - """ + @deprecated message @spec fetch(t, key) :: value def fetch(dict, key) do target(dict).fetch(dict, key) end - @doc """ - Returns the value associated with `key` in `dict`. If `dict` does not - contain `key`, it raises `KeyError`. - - ## Examples - - iex> d = Enum.into([a: 1], dict_impl.new) - iex> Dict.fetch!(d, :a) - 1 - - """ - @spec fetch!(t, key) :: value | no_return + @deprecated message + @spec fetch!(t, key) :: value def fetch!(dict, key) do target(dict).fetch!(dict, key) end - @doc """ - Stores the given `value` under `key` in `dict`. - If `dict` already has `key`, the stored value is replaced by the new one. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.put(d, :a, 3) - iex> Dict.get(d, :a) - 3 - - """ + @deprecated message @spec put(t, key, value) :: t def put(dict, key, val) do target(dict).put(dict, key, val) end - @doc """ - Puts the given `value` under `key` in `dict` unless `key` already exists. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.put_new(d, :a, 3) - iex> Dict.get(d, :a) - 1 - - """ + @deprecated message @spec put_new(t, key, value) :: t def put_new(dict, key, val) do target(dict).put_new(dict, key, val) end - @doc """ - Removes the entry stored under the given `key` from `dict`. - If `dict` does not contain `key`, returns the dictionary unchanged. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.delete(d, :a) - iex> Dict.get(d, :a) - nil - - iex> d = Enum.into([b: 2], dict_impl.new) - iex> Dict.delete(d, :a) == d - true + @deprecated message + @spec put_new_lazy(t, key, (() -> value)) :: t + def put_new_lazy(dict, key, fun) do + target(dict).put_new_lazy(dict, key, fun) + end - """ + @deprecated message @spec delete(t, key) :: t def delete(dict, key) do target(dict).delete(dict, key) end - @doc """ - Merges the dict `b` into dict `a`. - - If one of the dict `b` entries already exists in the `dict`, - the functions in entries in `b` have higher precedence unless a - function is given to resolve conflicts. - - Notice this function is polymorphic as it merges dicts of any - type. Each dict implementation also provides a `merge` function, - but they can only merge dicts of the same type. - - ## Examples - - iex> d1 = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d2 = Enum.into([a: 3, d: 4], dict_impl.new) - iex> d = Dict.merge(d1, d2) - iex> [a: Dict.get(d, :a), b: Dict.get(d, :b), d: Dict.get(d, :d)] - [a: 3, b: 2, d: 4] + @deprecated message + @spec merge(t, t) :: t + def merge(dict1, dict2) do + target1 = target(dict1) + target2 = target(dict2) - iex> d1 = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d2 = Enum.into([a: 3, d: 4], dict_impl.new) - iex> d = Dict.merge(d1, d2, fn(_k, v1, v2) -> - ...> v1 + v2 - ...> end) - iex> [a: Dict.get(d, :a), b: Dict.get(d, :b), d: Dict.get(d, :d)] - [a: 4, b: 2, d: 4] + if target1 == target2 do + target1.merge(dict1, dict2) + else + do_merge(target1, dict1, dict2, fn _k, _v1, v2 -> v2 end) + end + end - """ + @deprecated message @spec merge(t, t, (key, value, value -> value)) :: t - def merge(dict1, dict2, fun \\ fn(_k, _v1, v2) -> v2 end) do + def merge(dict1, dict2, fun) do target1 = target(dict1) target2 = target(dict2) if target1 == target2 do target1.merge(dict1, dict2, fun) else - Enumerable.reduce(dict2, {:cont, dict1}, fn({k, v}, acc) -> - {:cont, target1.update(acc, k, v, fn(other) -> fun.(k, other, v) end)} - end) |> elem(1) + do_merge(target1, dict1, dict2, fun) end end - @doc """ - Returns the value associated with `key` in `dict` as - well as the `dict` without `key`. - - ## Examples - - iex> dict = Enum.into([a: 1], dict_impl.new) - iex> {v, d} = Dict.pop dict, :a - iex> {v, Enum.sort(d)} - {1,[]} - - iex> dict = Enum.into([a: 1], dict_impl.new) - iex> {v, d} = Dict.pop dict, :b - iex> {v, Enum.sort(d)} - {nil,[a: 1]} - - iex> dict = Enum.into([a: 1], dict_impl.new) - iex> {v, d} = Dict.pop dict, :b, 3 - iex> {v, Enum.sort(d)} - {3,[a: 1]} + defp do_merge(target1, dict1, dict2, fun) do + Enumerable.reduce(dict2, {:cont, dict1}, fn {k, v}, acc -> + {:cont, target1.update(acc, k, v, fn other -> fun.(k, other, v) end)} + end) + |> elem(1) + end - """ + @deprecated message @spec pop(t, key, value) :: {value, t} def pop(dict, key, default \\ nil) do target(dict).pop(dict, key, default) end - @doc """ - Update a value in `dict` by calling `fun` on the value to get a new - value. An exception is generated if `key` is not present in the dict. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.update!(d, :a, fn(val) -> -val end) - iex> Dict.get(d, :a) - -1 + @deprecated message + @spec pop_lazy(t, key, (() -> value)) :: {value, t} + def pop_lazy(dict, key, fun) do + target(dict).pop_lazy(dict, key, fun) + end - """ + @deprecated message @spec update!(t, key, (value -> value)) :: t def update!(dict, key, fun) do target(dict).update!(dict, key, fun) end - @doc """ - Update a value in `dict` by calling `fun` on the value to get a new value. If - `key` is not present in `dict` then `initial` will be stored as the first - value. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.update(d, :c, 3, fn(val) -> -val end) - iex> Dict.get(d, :c) - 3 - - """ + @deprecated message @spec update(t, key, value, (value -> value)) :: t - def update(dict, key, initial, fun) do - target(dict).update(dict, key, initial, fun) + def update(dict, key, default, fun) do + target(dict).update(dict, key, default, fun) end - @doc """ - Returns a tuple of two dicts, where the first dict contains only - entries from `dict` with keys in `keys`, and the second dict - contains only entries from `dict` with keys not in `keys` - - Any non-member keys are ignored. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2, c: 3, d: 4], dict_impl.new) - iex> {d1, d2} = Dict.split(d, [:a, :c, :e]) - iex> {Dict.to_list(d1) |> Enum.sort, Dict.to_list(d2) |> Enum.sort} - {[a: 1, c: 3], [b: 2, d: 4]} - - iex> d = Enum.into([], dict_impl.new) - iex> {d1, d2} = Dict.split(d, [:a, :c]) - iex> {Dict.to_list(d1), Dict.to_list(d2)} - {[], []} - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> {d1, d2} = Dict.split(d, [:a, :b, :c]) - iex> {Dict.to_list(d1) |> Enum.sort, Dict.to_list(d2)} - {[a: 1, b: 2], []} - - """ + @deprecated message @spec split(t, [key]) :: {t, t} def split(dict, keys) do target(dict).split(dict, keys) end - @doc """ - Returns a new dict where the given `keys` are removed from `dict`. - Any non-member keys are ignored. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.drop(d, [:a, :c, :d]) - iex> Dict.to_list(d) - [b: 2] - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.drop(d, [:c, :d]) - iex> Dict.to_list(d) |> Enum.sort - [a: 1, b: 2] - - """ + @deprecated message @spec drop(t, [key]) :: t def drop(dict, keys) do target(dict).drop(dict, keys) end - @doc """ - Returns a new dict where only the keys in `keys` from `dict` are included. - - Any non-member keys are ignored. - - ## Examples - - iex> d = Enum.into([a: 1, b: 2], dict_impl.new) - iex> d = Dict.take(d, [:a, :c, :d]) - iex> Dict.to_list(d) - [a: 1] - iex> d = Dict.take(d, [:c, :d]) - iex> Dict.to_list(d) - [] - - """ + @deprecated message @spec take(t, [key]) :: t def take(dict, keys) do target(dict).take(dict, keys) end - @doc false + @deprecated message @spec empty(t) :: t def empty(dict) do target(dict).empty(dict) end - @doc """ - Check if two dicts are equal using `===`. - - Notice this function is polymorphic as it compares dicts of any - type. Each dict implementation also provides an `equal?` function, - but they can only compare dicts of the same type. - - ## Examples - - iex> a = Enum.into([a: 2, b: 3, f: 5, c: 123], dict_impl.new) - iex> b = [a: 2, b: 3, f: 5, c: 123] - iex> Dict.equal?(a, b) - true - - iex> a = Enum.into([a: 2, b: 3, f: 5, c: 123], dict_impl.new) - iex> b = [] - iex> Dict.equal?(a, b) - false - - """ + @deprecated message @spec equal?(t, t) :: boolean def equal?(dict1, dict2) do target1 = target(dict1) @@ -666,28 +417,27 @@ defmodule Dict do target1.equal?(dict1, dict2) target1.size(dict1) == target2.size(dict2) -> - Enumerable.reduce(dict2, {:cont, true}, fn({k, v}, _acc) -> + Enumerable.reduce(dict2, {:cont, true}, fn {k, v}, _acc -> case target1.fetch(dict1, k) do {:ok, ^v} -> {:cont, true} - _ -> {:halt, false} + _ -> {:halt, false} end - end) |> elem(1) + end) + |> elem(1) true -> false end end - @doc """ - Returns a list of key-value pairs stored in `dict`. - No particular order is enforced. - """ + @deprecated message @spec to_list(t) :: list def to_list(dict) do target(dict).to_list(dict) end + @spec unsupported_dict(t) :: no_return defp unsupported_dict(dict) do - raise ArgumentError, "unsupported dict: #{inspect dict}" + raise ArgumentError, "unsupported dict: #{inspect(dict)}" end end diff --git a/lib/elixir/lib/dynamic_supervisor.ex b/lib/elixir/lib/dynamic_supervisor.ex new file mode 100644 index 00000000000..8f2d72a4fe1 --- /dev/null +++ b/lib/elixir/lib/dynamic_supervisor.ex @@ -0,0 +1,1112 @@ +defmodule DynamicSupervisor do + @moduledoc ~S""" + A supervisor that starts children dynamically. + + The `Supervisor` module was designed to handle mostly static children + that are started in the given order when the supervisor starts. A + `DynamicSupervisor` starts with no children. Instead, children are + started on demand via `start_child/2`. When a dynamic supervisor + terminates, all children are shut down at the same time, with no guarantee + of ordering. + + ## Examples + + A dynamic supervisor is started with no children and often a name: + + children = [ + {DynamicSupervisor, name: MyApp.DynamicSupervisor} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + + The options given in the child specification are documented in `start_link/1`. + + Once the dynamic supervisor is running, we can start children + with `start_child/2`, which receives a child specification: + + {:ok, agent1} = DynamicSupervisor.start_child(MyApp.DynamicSupervisor, {Agent, fn -> %{} end}) + Agent.update(agent1, &Map.put(&1, :key, "value")) + Agent.get(agent1, & &1) + #=> %{key: "value"} + + {:ok, agent2} = DynamicSupervisor.start_child(MyApp.DynamicSupervisor, {Agent, fn -> %{} end}) + Agent.get(agent2, & &1) + #=> %{} + + DynamicSupervisor.count_children(MyApp.DynamicSupervisor) + #=> %{active: 2, specs: 2, supervisors: 0, workers: 2} + + ## Scalability and partitioning + + The `DynamicSupervisor` is a single process responsible for starting + other processes. In some applications, the `DynamicSupervisor` may + become a bottleneck. To address this, you can start multiple instances + of the `DynamicSupervisor` and then pick a "random" instance to start + the child on. + + Instead of: + + children = [ + {DynamicSupervisor, name: MyApp.DynamicSupervisor} + ] + + and: + + DynamicSupervisor.start_child(MyApp.DynamicSupervisor, {Agent, fn -> %{} end}) + + You can do this: + + children = [ + {PartitionSupervisor, + child_spec: DynamicSupervisor, + name: MyApp.DynamicSupervisors} + ] + + and then: + + DynamicSupervisor.start_child( + {:via, PartitionSupervisor, {MyApp.DynamicSupervisors, self()}}, + {Agent, fn -> %{} end} + ) + + In the code above, we start a partition supervisor that will by default + start a dynamic supervisor for each core in your machine. Then, instead + of calling the `DynamicSupervisor` by name, you call it through the + partition supervisor, using `self()` as the routing key. This means each + process will be assigned one of the existing dynamic supervisors. + Read the `PartitionSupervisor` docs for more information. + + ## Module-based supervisors + + Similar to `Supervisor`, dynamic supervisors also support module-based + supervisors. + + defmodule MyApp.DynamicSupervisor do + # Automatically defines child_spec/1 + use DynamicSupervisor + + def start_link(init_arg) do + DynamicSupervisor.start_link(__MODULE__, init_arg, name: __MODULE__) + end + + @impl true + def init(_init_arg) do + DynamicSupervisor.init(strategy: :one_for_one) + end + end + + See the `Supervisor` docs for a discussion of when you may want to use + module-based supervisors. A `@doc` annotation immediately preceding + `use DynamicSupervisor` will be attached to the generated `child_spec/1` + function. + + ## Name registration + + A supervisor is bound to the same name registration rules as a `GenServer`. + Read more about these rules in the documentation for `GenServer`. + + ## Migrating from Supervisor's :simple_one_for_one + + In case you were using the deprecated `:simple_one_for_one` strategy from + the `Supervisor` module, you can migrate to the `DynamicSupervisor` in + few steps. + + Imagine the given "old" code: + + defmodule MySupervisor do + use Supervisor + + def start_link(init_arg) do + Supervisor.start_link(__MODULE__, init_arg, name: __MODULE__) + end + + def start_child(foo, bar, baz) do + # This will start child by calling MyWorker.start_link(init_arg, foo, bar, baz) + Supervisor.start_child(__MODULE__, [foo, bar, baz]) + end + + @impl true + def init(init_arg) do + children = [ + # Or the deprecated: worker(MyWorker, [init_arg]) + %{id: MyWorker, start: {MyWorker, :start_link, [init_arg]}} + ] + + Supervisor.init(children, strategy: :simple_one_for_one) + end + end + + It can be upgraded to the DynamicSupervisor like this: + + defmodule MySupervisor do + use DynamicSupervisor + + def start_link(init_arg) do + DynamicSupervisor.start_link(__MODULE__, init_arg, name: __MODULE__) + end + + def start_child(foo, bar, baz) do + # If MyWorker is not using the new child specs, we need to pass a map: + # spec = %{id: MyWorker, start: {MyWorker, :start_link, [foo, bar, baz]}} + spec = {MyWorker, foo: foo, bar: bar, baz: baz} + DynamicSupervisor.start_child(__MODULE__, spec) + end + + @impl true + def init(init_arg) do + DynamicSupervisor.init( + strategy: :one_for_one, + extra_arguments: [init_arg] + ) + end + end + + The difference is that the `DynamicSupervisor` expects the child specification + at the moment `start_child/2` is called, and no longer on the init callback. + If there are any initial arguments given on initialization, such as `[initial_arg]`, + it can be given in the `:extra_arguments` flag on `DynamicSupervisor.init/1`. + """ + + @behaviour GenServer + + @doc """ + Callback invoked to start the supervisor and during hot code upgrades. + + Developers typically invoke `DynamicSupervisor.init/1` at the end of + their init callback to return the proper supervision flags. + """ + @callback init(init_arg :: term) :: {:ok, sup_flags()} | :ignore + + @typedoc "The supervisor flags returned on init" + @type sup_flags() :: %{ + strategy: strategy(), + intensity: non_neg_integer(), + period: pos_integer(), + max_children: non_neg_integer() | :infinity, + extra_arguments: [term()] + } + + @typedoc "Options given to `start_link` functions" + @type option :: GenServer.option() + + @typedoc "Options given to `start_link` and `init/1` functions" + @type init_option :: + {:strategy, strategy()} + | {:max_restarts, non_neg_integer()} + | {:max_seconds, pos_integer()} + | {:max_children, non_neg_integer() | :infinity} + | {:extra_arguments, [term()]} + + @typedoc "Supported strategies" + @type strategy :: :one_for_one + + @typedoc "Return values of `start_child` functions" + @type on_start_child :: + {:ok, pid} + | {:ok, pid, info :: term} + | :ignore + | {:error, {:already_started, pid} | :max_children | term} + + # In this struct, `args` refers to the arguments passed to init/1 (the `init_arg`). + defstruct [ + :args, + :extra_arguments, + :mod, + :name, + :strategy, + :max_children, + :max_restarts, + :max_seconds, + children: %{}, + restarts: [] + ] + + @doc """ + Returns a specification to start a dynamic supervisor under a supervisor. + + See `Supervisor`. + """ + @doc since: "1.6.1" + def child_spec(opts) when is_list(opts) do + id = + case Keyword.get(opts, :name, DynamicSupervisor) do + name when is_atom(name) -> name + {:global, name} -> name + {:via, _module, name} -> name + end + + %{ + id: id, + start: {DynamicSupervisor, :start_link, [opts]}, + type: :supervisor + } + end + + @doc false + defmacro __using__(opts) do + quote location: :keep, bind_quoted: [opts: opts] do + @behaviour DynamicSupervisor + unless Module.has_attribute?(__MODULE__, :doc) do + @doc """ + Returns a specification to start this module under a supervisor. + + See `Supervisor`. + """ + end + + def child_spec(arg) do + default = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [arg]}, + type: :supervisor + } + + Supervisor.child_spec(default, unquote(Macro.escape(opts))) + end + + defoverridable child_spec: 1 + end + end + + @doc """ + Starts a supervisor with the given options. + + This function is typically not invoked directly, instead it is invoked + when using a `DynamicSupervisor` as a child of another supervisor: + + children = [ + {DynamicSupervisor, name: MySupervisor} + ] + + If the supervisor is successfully spawned, this function returns + `{:ok, pid}`, where `pid` is the PID of the supervisor. If the supervisor + is given a name and a process with the specified name already exists, + the function returns `{:error, {:already_started, pid}}`, where `pid` + is the PID of that process. + + Note that a supervisor started with this function is linked to the parent + process and exits not only on crashes but also if the parent process exits + with `:normal` reason. + + ## Options + + * `:name` - registers the supervisor under the given name. + The supported values are described under the "Name registration" + section in the `GenServer` module docs. + + * `:strategy` - the restart strategy option. The only supported + value is `:one_for_one` which means that no other child is + terminated if a child process terminates. You can learn more + about strategies in the `Supervisor` module docs. + + * `:max_restarts` - the maximum number of restarts allowed in + a time frame. Defaults to `3`. + + * `:max_seconds` - the time frame in which `:max_restarts` applies. + Defaults to `5`. + + * `:max_children` - the maximum amount of children to be running + under this supervisor at the same time. When `:max_children` is + exceeded, `start_child/2` returns `{:error, :max_children}`. Defaults + to `:infinity`. + + * `:extra_arguments` - arguments that are prepended to the arguments + specified in the child spec given to `start_child/2`. Defaults to + an empty list. + + """ + @doc since: "1.6.0" + @spec start_link([option | init_option]) :: Supervisor.on_start() + def start_link(options) when is_list(options) do + keys = [:extra_arguments, :max_children, :max_seconds, :max_restarts, :strategy] + {sup_opts, start_opts} = Keyword.split(options, keys) + start_link(Supervisor.Default, init(sup_opts), start_opts) + end + + @doc """ + Starts a module-based supervisor process with the given `module` and `init_arg`. + + To start the supervisor, the `c:init/1` callback will be invoked in the given + `module`, with `init_arg` as its argument. The `c:init/1` callback must return a + supervisor specification which can be created with the help of the `init/1` + function. + + If the `c:init/1` callback returns `:ignore`, this function returns + `:ignore` as well and the supervisor terminates with reason `:normal`. + If it fails or returns an incorrect value, this function returns + `{:error, term}` where `term` is a term with information about the + error, and the supervisor terminates with reason `term`. + + The `:name` option can also be given in order to register a supervisor + name, the supported values are described in the "Name registration" + section in the `GenServer` module docs. + + If the supervisor is successfully spawned, this function returns + `{:ok, pid}`, where `pid` is the PID of the supervisor. If the supervisor + is given a name and a process with the specified name already exists, + the function returns `{:error, {:already_started, pid}}`, where `pid` + is the PID of that process. + + Note that a supervisor started with this function is linked to the parent + process and exits not only on crashes but also if the parent process exits + with `:normal` reason. + """ + @doc since: "1.6.0" + @spec start_link(module, term, [option]) :: Supervisor.on_start() + def start_link(module, init_arg, opts \\ []) do + GenServer.start_link(__MODULE__, {module, init_arg, opts[:name]}, opts) + end + + @doc """ + Dynamically adds a child specification to `supervisor` and starts that child. + + `child_spec` should be a valid child specification as detailed in the + "Child specification" section of the documentation for `Supervisor`. The child + process will be started as defined in the child specification. Note that while + the `:id` field is still required in the spec, the value is ignored and + therefore does not need to be unique. + + If the child process start function returns `{:ok, child}` or `{:ok, child, + info}`, then child specification and PID are added to the supervisor and + this function returns the same value. + + If the child process start function returns `:ignore`, then no child is added + to the supervision tree and this function returns `:ignore` too. + + If the child process start function returns an error tuple or an erroneous + value, or if it fails, the child specification is discarded and this function + returns `{:error, error}` where `error` is the error or erroneous value + returned from child process start function, or failure reason if it fails. + + If the supervisor already has N children in a way that N exceeds the amount + of `:max_children` set on the supervisor initialization (see `init/1`), then + this function returns `{:error, :max_children}`. + """ + @doc since: "1.6.0" + @spec start_child( + Supervisor.supervisor(), + Supervisor.child_spec() + | {module, term} + | module + | (old_erlang_child_spec :: :supervisor.child_spec()) + ) :: + on_start_child() + def start_child(supervisor, {_, _, _, _, _, _} = child_spec) do + validate_and_start_child(supervisor, child_spec) + end + + def start_child(supervisor, child_spec) do + validate_and_start_child(supervisor, Supervisor.child_spec(child_spec, [])) + end + + defp validate_and_start_child(supervisor, child_spec) do + case validate_child(child_spec) do + {:ok, child} -> call(supervisor, {:start_child, child}) + error -> {:error, error} + end + end + + defp validate_child(%{id: _, start: {mod, _, _} = start} = child) do + restart = Map.get(child, :restart, :permanent) + type = Map.get(child, :type, :worker) + modules = Map.get(child, :modules, [mod]) + + shutdown = + case type do + :worker -> Map.get(child, :shutdown, 5_000) + :supervisor -> Map.get(child, :shutdown, :infinity) + end + + validate_child(start, restart, shutdown, type, modules) + end + + defp validate_child({_, start, restart, shutdown, type, modules}) do + validate_child(start, restart, shutdown, type, modules) + end + + defp validate_child(other) do + {:invalid_child_spec, other} + end + + defp validate_child(start, restart, shutdown, type, modules) do + with :ok <- validate_start(start), + :ok <- validate_restart(restart), + :ok <- validate_shutdown(shutdown), + :ok <- validate_type(type), + :ok <- validate_modules(modules) do + {:ok, {start, restart, shutdown, type, modules}} + end + end + + defp validate_start({m, f, args}) when is_atom(m) and is_atom(f) and is_list(args), do: :ok + defp validate_start(mfa), do: {:invalid_mfa, mfa} + + defp validate_type(type) when type in [:supervisor, :worker], do: :ok + defp validate_type(type), do: {:invalid_child_type, type} + + defp validate_restart(restart) when restart in [:permanent, :temporary, :transient], do: :ok + defp validate_restart(restart), do: {:invalid_restart_type, restart} + + defp validate_shutdown(shutdown) when is_integer(shutdown) and shutdown > 0, do: :ok + defp validate_shutdown(shutdown) when shutdown in [:infinity, :brutal_kill], do: :ok + defp validate_shutdown(shutdown), do: {:invalid_shutdown, shutdown} + + defp validate_modules(:dynamic), do: :ok + + defp validate_modules(mods) do + if is_list(mods) and Enum.all?(mods, &is_atom/1) do + :ok + else + {:invalid_modules, mods} + end + end + + @doc """ + Terminates the given child identified by `pid`. + + If successful, this function returns `:ok`. If there is no process with + the given PID, this function returns `{:error, :not_found}`. + """ + @doc since: "1.6.0" + @spec terminate_child(Supervisor.supervisor(), pid) :: :ok | {:error, :not_found} + def terminate_child(supervisor, pid) when is_pid(pid) do + call(supervisor, {:terminate_child, pid}) + end + + @doc """ + Returns a list with information about all children. + + Note that calling this function when supervising a large number + of children under low memory conditions can cause an out of memory + exception. + + This function returns a list of tuples containing: + + * `id` - it is always `:undefined` for dynamic supervisors + + * `child` - the PID of the corresponding child process or the + atom `:restarting` if the process is about to be restarted + + * `type` - `:worker` or `:supervisor` as defined in the child + specification + + * `modules` - as defined in the child specification + + """ + @doc since: "1.6.0" + @spec which_children(Supervisor.supervisor()) :: [ + # module() | :dynamic here because :supervisor.modules() is not exported + {:undefined, pid | :restarting, :worker | :supervisor, [module()] | :dynamic} + ] + def which_children(supervisor) do + call(supervisor, :which_children) + end + + @doc """ + Returns a map containing count values for the supervisor. + + The map contains the following keys: + + * `:specs` - the number of children processes + + * `:active` - the count of all actively running child processes managed by + this supervisor + + * `:supervisors` - the count of all supervisors whether or not the child + process is still alive + + * `:workers` - the count of all workers, whether or not the child process + is still alive + + """ + @doc since: "1.6.0" + @spec count_children(Supervisor.supervisor()) :: %{ + specs: non_neg_integer, + active: non_neg_integer, + supervisors: non_neg_integer, + workers: non_neg_integer + } + def count_children(supervisor) do + call(supervisor, :count_children) |> :maps.from_list() + end + + @doc """ + Synchronously stops the given supervisor with the given `reason`. + + It returns `:ok` if the supervisor terminates with the given + reason. If it terminates with another reason, the call exits. + + This function keeps OTP semantics regarding error reporting. + If the reason is any other than `:normal`, `:shutdown` or + `{:shutdown, _}`, an error report is logged. + """ + @doc since: "1.7.0" + @spec stop(Supervisor.supervisor(), reason :: term, timeout) :: :ok + def stop(supervisor, reason \\ :normal, timeout \\ :infinity) do + GenServer.stop(supervisor, reason, timeout) + end + + @doc """ + Receives a set of `options` that initializes a dynamic supervisor. + + This is typically invoked at the end of the `c:init/1` callback of + module-based supervisors. See the "Module-based supervisors" section + in the module documentation for more information. + + It accepts the same `options` as `start_link/1` (except for `:name`) + and it returns a tuple containing the supervisor options. + + ## Examples + + def init(_arg) do + DynamicSupervisor.init(max_children: 1000) + end + + """ + @doc since: "1.6.0" + @spec init([init_option]) :: {:ok, sup_flags()} + def init(options) when is_list(options) do + strategy = Keyword.get(options, :strategy, :one_for_one) + intensity = Keyword.get(options, :max_restarts, 3) + period = Keyword.get(options, :max_seconds, 5) + max_children = Keyword.get(options, :max_children, :infinity) + extra_arguments = Keyword.get(options, :extra_arguments, []) + + flags = %{ + strategy: strategy, + intensity: intensity, + period: period, + max_children: max_children, + extra_arguments: extra_arguments + } + + {:ok, flags} + end + + ## Callbacks + + @impl true + def init({mod, init_arg, name}) do + Process.put(:"$initial_call", {:supervisor, mod, 1}) + Process.flag(:trap_exit, true) + + case mod.init(init_arg) do + {:ok, flags} when is_map(flags) -> + name = + cond do + is_nil(name) -> {self(), mod} + is_atom(name) -> {:local, name} + is_tuple(name) -> name + end + + state = %DynamicSupervisor{mod: mod, args: init_arg, name: name} + + case init(state, flags) do + {:ok, state} -> {:ok, state} + {:error, reason} -> {:stop, {:supervisor_data, reason}} + end + + :ignore -> + :ignore + + other -> + {:stop, {:bad_return, {mod, :init, other}}} + end + end + + defp init(state, flags) do + extra_arguments = Map.get(flags, :extra_arguments, []) + max_children = Map.get(flags, :max_children, :infinity) + max_restarts = Map.get(flags, :intensity, 1) + max_seconds = Map.get(flags, :period, 5) + strategy = Map.get(flags, :strategy, :one_for_one) + + with :ok <- validate_strategy(strategy), + :ok <- validate_restarts(max_restarts), + :ok <- validate_seconds(max_seconds), + :ok <- validate_dynamic(max_children), + :ok <- validate_extra_arguments(extra_arguments) do + {:ok, + %{ + state + | extra_arguments: extra_arguments, + max_children: max_children, + max_restarts: max_restarts, + max_seconds: max_seconds, + strategy: strategy + }} + end + end + + defp validate_strategy(strategy) when strategy in [:one_for_one], do: :ok + defp validate_strategy(strategy), do: {:error, {:invalid_strategy, strategy}} + + defp validate_restarts(restart) when is_integer(restart) and restart >= 0, do: :ok + defp validate_restarts(restart), do: {:error, {:invalid_intensity, restart}} + + defp validate_seconds(seconds) when is_integer(seconds) and seconds > 0, do: :ok + defp validate_seconds(seconds), do: {:error, {:invalid_period, seconds}} + + defp validate_dynamic(:infinity), do: :ok + defp validate_dynamic(dynamic) when is_integer(dynamic) and dynamic >= 0, do: :ok + defp validate_dynamic(dynamic), do: {:error, {:invalid_max_children, dynamic}} + + defp validate_extra_arguments(list) when is_list(list), do: :ok + defp validate_extra_arguments(extra), do: {:error, {:invalid_extra_arguments, extra}} + + @impl true + def handle_call(:which_children, _from, state) do + %{children: children} = state + + reply = + for {pid, args} <- children do + case args do + {:restarting, {_, _, _, type, modules}} -> + {:undefined, :restarting, type, modules} + + {_, _, _, type, modules} -> + {:undefined, pid, type, modules} + end + end + + {:reply, reply, state} + end + + def handle_call(:count_children, _from, state) do + %{children: children} = state + specs = map_size(children) + + {active, workers, supervisors} = + Enum.reduce(children, {0, 0, 0}, fn + {_pid, {:restarting, {_, _, _, :worker, _}}}, {active, worker, supervisor} -> + {active, worker + 1, supervisor} + + {_pid, {:restarting, {_, _, _, :supervisor, _}}}, {active, worker, supervisor} -> + {active, worker, supervisor + 1} + + {_pid, {_, _, _, :worker, _}}, {active, worker, supervisor} -> + {active + 1, worker + 1, supervisor} + + {_pid, {_, _, _, :supervisor, _}}, {active, worker, supervisor} -> + {active + 1, worker, supervisor + 1} + end) + + reply = [specs: specs, active: active, supervisors: supervisors, workers: workers] + {:reply, reply, state} + end + + def handle_call({:terminate_child, pid}, _from, %{children: children} = state) do + case children do + %{^pid => info} -> + :ok = terminate_children(%{pid => info}, state) + {:reply, :ok, delete_child(pid, state)} + + %{} -> + {:reply, {:error, :not_found}, state} + end + end + + def handle_call({:start_task, args, restart, shutdown}, from, state) do + {init_restart, init_shutdown} = Process.get(Task.Supervisor) + restart = restart || init_restart + shutdown = shutdown || init_shutdown + child = {{Task.Supervised, :start_link, args}, restart, shutdown, :worker, [Task.Supervised]} + handle_call({:start_child, child}, from, state) + end + + def handle_call({:start_child, child}, _from, state) do + %{children: children, max_children: max_children} = state + + if map_size(children) < max_children do + handle_start_child(child, state) + else + {:reply, {:error, :max_children}, state} + end + end + + defp handle_start_child({{m, f, args} = mfa, restart, shutdown, type, modules}, state) do + %{extra_arguments: extra} = state + + case reply = start_child(m, f, extra ++ args) do + {:ok, pid, _} -> + {:reply, reply, save_child(pid, mfa, restart, shutdown, type, modules, state)} + + {:ok, pid} -> + {:reply, reply, save_child(pid, mfa, restart, shutdown, type, modules, state)} + + _ -> + {:reply, reply, state} + end + end + + defp start_child(m, f, a) do + try do + apply(m, f, a) + catch + kind, reason -> + {:error, exit_reason(kind, reason, __STACKTRACE__)} + else + {:ok, pid, extra} when is_pid(pid) -> {:ok, pid, extra} + {:ok, pid} when is_pid(pid) -> {:ok, pid} + :ignore -> :ignore + {:error, _} = error -> error + other -> {:error, other} + end + end + + defp save_child(pid, mfa, restart, shutdown, type, modules, state) do + mfa = mfa_for_restart(mfa, restart) + put_in(state.children[pid], {mfa, restart, shutdown, type, modules}) + end + + defp mfa_for_restart({m, f, _}, :temporary), do: {m, f, :undefined} + defp mfa_for_restart(mfa, _), do: mfa + + defp exit_reason(:exit, reason, _), do: reason + defp exit_reason(:error, reason, stack), do: {reason, stack} + defp exit_reason(:throw, value, stack), do: {{:nocatch, value}, stack} + + @impl true + def handle_cast(_msg, state) do + {:noreply, state} + end + + @impl true + def handle_info({:EXIT, pid, reason}, state) do + case maybe_restart_child(pid, reason, state) do + {:ok, state} -> {:noreply, state} + {:shutdown, state} -> {:stop, :shutdown, state} + end + end + + def handle_info({:"$gen_restart", pid}, state) do + %{children: children} = state + + case children do + %{^pid => restarting_args} -> + {:restarting, child} = restarting_args + + case restart_child(pid, child, state) do + {:ok, state} -> {:noreply, state} + {:shutdown, state} -> {:stop, :shutdown, state} + end + + # We may hit clause if we send $gen_restart and then + # someone calls terminate_child, removing the child. + %{} -> + {:noreply, state} + end + end + + def handle_info(msg, state) do + :logger.error( + %{ + label: {DynamicSupervisor, :unexpected_msg}, + report: %{ + msg: msg + } + }, + %{ + domain: [:otp, :elixir], + error_logger: %{tag: :error_msg}, + report_cb: &__MODULE__.format_report/1 + } + ) + + {:noreply, state} + end + + @impl true + def code_change(_, %{mod: mod, args: init_arg} = state, _) do + case mod.init(init_arg) do + {:ok, flags} when is_map(flags) -> + case init(state, flags) do + {:ok, state} -> {:ok, state} + {:error, reason} -> {:error, {:supervisor_data, reason}} + end + + :ignore -> + {:ok, state} + + error -> + error + end + end + + @impl true + def terminate(_, %{children: children} = state) do + :ok = terminate_children(children, state) + end + + defp terminate_children(children, state) do + {pids, times, stacks} = monitor_children(children) + size = map_size(pids) + + timers = + Enum.reduce(times, %{}, fn {time, pids}, acc -> + Map.put(acc, :erlang.start_timer(time, self(), :kill), pids) + end) + + stacks = wait_children(pids, size, timers, stacks) + + for {pid, {child, reason}} <- stacks do + report_error(:shutdown_error, reason, pid, child, state) + end + + :ok + end + + defp monitor_children(children) do + Enum.reduce(children, {%{}, %{}, %{}}, fn + {_, {:restarting, _}}, acc -> + acc + + {pid, {_, restart, _, _, _} = child}, {pids, times, stacks} -> + case monitor_child(pid) do + :ok -> + times = exit_child(pid, child, times) + {Map.put(pids, pid, child), times, stacks} + + {:error, :normal} when restart != :permanent -> + {pids, times, stacks} + + {:error, reason} -> + {pids, times, Map.put(stacks, pid, {child, reason})} + end + end) + end + + defp monitor_child(pid) do + ref = Process.monitor(pid) + Process.unlink(pid) + + receive do + {:EXIT, ^pid, reason} -> + receive do + {:DOWN, ^ref, :process, ^pid, _} -> {:error, reason} + end + after + 0 -> :ok + end + end + + defp exit_child(pid, {_, _, shutdown, _, _}, times) do + case shutdown do + :brutal_kill -> + Process.exit(pid, :kill) + times + + :infinity -> + Process.exit(pid, :shutdown) + times + + time -> + Process.exit(pid, :shutdown) + Map.update(times, time, [pid], &[pid | &1]) + end + end + + defp wait_children(_pids, 0, timers, stacks) do + for {timer, _} <- timers do + _ = :erlang.cancel_timer(timer) + + receive do + {:timeout, ^timer, :kill} -> :ok + after + 0 -> :ok + end + end + + stacks + end + + defp wait_children(pids, size, timers, stacks) do + receive do + {:DOWN, _ref, :process, pid, reason} -> + case pids do + %{^pid => child} -> + stacks = wait_child(pid, child, reason, stacks) + wait_children(pids, size - 1, timers, stacks) + + %{} -> + wait_children(pids, size, timers, stacks) + end + + {:timeout, timer, :kill} -> + for pid <- Map.fetch!(timers, timer), do: Process.exit(pid, :kill) + wait_children(pids, size, Map.delete(timers, timer), stacks) + end + end + + defp wait_child(pid, {_, _, :brutal_kill, _, _} = child, reason, stacks) do + case reason do + :killed -> stacks + _ -> Map.put(stacks, pid, {child, reason}) + end + end + + defp wait_child(pid, {_, restart, _, _, _} = child, reason, stacks) do + case reason do + {:shutdown, _} -> stacks + :shutdown -> stacks + :normal when restart != :permanent -> stacks + reason -> Map.put(stacks, pid, {child, reason}) + end + end + + defp maybe_restart_child(pid, reason, %{children: children} = state) do + case children do + %{^pid => {_, restart, _, _, _} = child} -> + maybe_restart_child(restart, reason, pid, child, state) + + %{} -> + {:ok, state} + end + end + + defp maybe_restart_child(:permanent, reason, pid, child, state) do + report_error(:child_terminated, reason, pid, child, state) + restart_child(pid, child, state) + end + + defp maybe_restart_child(_, :normal, pid, _child, state) do + {:ok, delete_child(pid, state)} + end + + defp maybe_restart_child(_, :shutdown, pid, _child, state) do + {:ok, delete_child(pid, state)} + end + + defp maybe_restart_child(_, {:shutdown, _}, pid, _child, state) do + {:ok, delete_child(pid, state)} + end + + defp maybe_restart_child(:transient, reason, pid, child, state) do + report_error(:child_terminated, reason, pid, child, state) + restart_child(pid, child, state) + end + + defp maybe_restart_child(:temporary, reason, pid, child, state) do + report_error(:child_terminated, reason, pid, child, state) + {:ok, delete_child(pid, state)} + end + + defp delete_child(pid, %{children: children} = state) do + %{state | children: Map.delete(children, pid)} + end + + defp restart_child(pid, child, state) do + case add_restart(state) do + {:ok, %{strategy: strategy} = state} -> + case restart_child(strategy, pid, child, state) do + {:ok, state} -> + {:ok, state} + + {:try_again, state} -> + send(self(), {:"$gen_restart", pid}) + {:ok, state} + end + + {:shutdown, state} -> + report_error(:shutdown, :reached_max_restart_intensity, pid, child, state) + {:shutdown, delete_child(pid, state)} + end + end + + defp add_restart(state) do + %{max_seconds: max_seconds, max_restarts: max_restarts, restarts: restarts} = state + + now = :erlang.monotonic_time(1) + restarts = add_restart([now | restarts], now, max_seconds) + state = %{state | restarts: restarts} + + if length(restarts) <= max_restarts do + {:ok, state} + else + {:shutdown, state} + end + end + + defp add_restart(restarts, now, period) do + for then <- restarts, now <= then + period, do: then + end + + defp restart_child(:one_for_one, current_pid, child, state) do + {{m, f, args} = mfa, restart, shutdown, type, modules} = child + %{extra_arguments: extra} = state + + case start_child(m, f, extra ++ args) do + {:ok, pid, _} -> + state = delete_child(current_pid, state) + {:ok, save_child(pid, mfa, restart, shutdown, type, modules, state)} + + {:ok, pid} -> + state = delete_child(current_pid, state) + {:ok, save_child(pid, mfa, restart, shutdown, type, modules, state)} + + :ignore -> + {:ok, delete_child(current_pid, state)} + + {:error, reason} -> + report_error(:start_error, reason, {:restarting, current_pid}, child, state) + state = put_in(state.children[current_pid], {:restarting, child}) + {:try_again, state} + end + end + + defp report_error(error, reason, pid, child, %{name: name, extra_arguments: extra}) do + :logger.error( + %{ + label: {:supervisor, error}, + report: [ + {:supervisor, name}, + {:errorContext, error}, + {:reason, reason}, + {:offender, extract_child(pid, child, extra)} + ] + }, + %{ + domain: [:otp, :sasl], + report_cb: &:logger.format_otp_report/1, + logger_formatter: %{title: "SUPERVISOR REPORT"}, + error_logger: %{tag: :error_report, type: :supervisor_report} + } + ) + end + + defp extract_child(pid, {{m, f, args}, restart, shutdown, type, _modules}, extra) do + [ + pid: pid, + id: :undefined, + mfargs: {m, f, extra ++ args}, + restart_type: restart, + shutdown: shutdown, + child_type: type + ] + end + + @impl true + def format_status(:terminate, [_pdict, state]) do + state + end + + def format_status(_, [_pdict, %{mod: mod} = state]) do + [data: [{~c"State", state}], supervisor: [{~c"Callback", mod}]] + end + + ## Helpers + + @compile {:inline, call: 2} + + defp call(supervisor, req) do + GenServer.call(supervisor, req, :infinity) + end + + @doc false + def format_report(%{ + label: {__MODULE__, :unexpected_msg}, + report: %{msg: msg} + }) do + {'DynamicSupervisor received unexpected message: ~p~n', [msg]} + end +end diff --git a/lib/elixir/lib/enum.ex b/lib/elixir/lib/enum.ex index 46c97a9bc5a..1d2ec44c85f 100644 --- a/lib/elixir/lib/enum.ex +++ b/lib/elixir/lib/enum.ex @@ -3,36 +3,56 @@ defprotocol Enumerable do Enumerable protocol used by `Enum` and `Stream` modules. When you invoke a function in the `Enum` module, the first argument - is usually a collection that must implement this protocol. For example, - the expression - - Enum.map([1, 2, 3], &(&1 * 2)) - - invokes underneath `Enumerable.reduce/3` to perform the reducing - operation that builds a mapped list by calling the mapping function - `&(&1 * 2)` on every element in the collection and cons'ing the - element with an accumulated list. + is usually a collection that must implement this protocol. + For example, the expression `Enum.map([1, 2, 3], &(&1 * 2))` + invokes `Enumerable.reduce/3` to perform the reducing operation that + builds a mapped list by calling the mapping function `&(&1 * 2)` on + every element in the collection and consuming the element with an + accumulated list. Internally, `Enum.map/2` is implemented as follows: - def map(enum, fun) do - reducer = fn x, acc -> {:cont, [fun.(x)|acc]} end - Enumerable.reduce(enum, {:cont, []}, reducer) |> elem(1) |> :lists.reverse() + def map(enumerable, fun) do + reducer = fn x, acc -> {:cont, [fun.(x) | acc]} end + Enumerable.reduce(enumerable, {:cont, []}, reducer) |> elem(1) |> :lists.reverse() end - Notice the user given function is wrapped into a `reducer` function. - The `reducer` function must return a tagged tuple after each step, - as described in the `acc/0` type. + Note that the user-supplied function is wrapped into a `t:reducer/0` function. + The `t:reducer/0` function must return a tagged tuple after each step, + as described in the `t:acc/0` type. At the end, `Enumerable.reduce/3` + returns `t:result/0`. + + This protocol uses tagged tuples to exchange information between the + reducer function and the data type that implements the protocol. This + allows enumeration of resources, such as files, to be done efficiently + while also guaranteeing the resource will be closed at the end of the + enumeration. This protocol also allows suspension of the enumeration, + which is useful when interleaving between many enumerables is required + (as in the `zip/1` and `zip/2` functions). + + This protocol requires four functions to be implemented, `reduce/3`, + `count/1`, `member?/2`, and `slice/1`. The core of the protocol is the + `reduce/3` function. All other functions exist as optimizations paths + for data structures that can implement certain properties in better + than linear time. + """ + + @typedoc """ + An enumerable of elements of type `element`. + + This type is equivalent to `t:t/0` but is especially useful for documentation. - The reason the accumulator requires a tagged tuple is to allow the - reducer function to communicate to the underlying enumerable the end - of enumeration, allowing any open resource to be properly closed. It - also allows suspension of the enumeration, which is useful when - interleaving between many enumerables is required (as in zip). + For example, imagine you define a function that expects an enumerable of + integers and returns an enumerable of strings: + + @spec integers_to_strings(Enumerable.t(integer())) :: Enumerable.t(String.t()) + def integers_to_strings(integers) do + Stream.map(integers, &Integer.to_string/1) + end - Finally, `Enumerable.reduce/3` will return another tagged tuple, - as represented by the `result/0` type. """ + @typedoc since: "1.14.0" + @type t(_element) :: t() @typedoc """ The accumulator value for each step. @@ -44,10 +64,10 @@ defprotocol Enumerable do * `:suspend` - the enumeration should be suspended immediately Depending on the accumulator value, the result returned by - `Enumerable.reduce/3` will change. Please check the `result` - type docs for more information. + `Enumerable.reduce/3` will change. Please check the `t:result/0` + type documentation for more information. - In case a reducer function returns a `:suspend` accumulator, + In case a `t:reducer/0` function returns a `:suspend` accumulator, it must be explicitly handled by the caller and never leak. """ @type acc :: {:cont, term} | {:halt, term} | {:suspend, term} @@ -55,28 +75,37 @@ defprotocol Enumerable do @typedoc """ The reducer function. - Should be called with the collection element and the - accumulator contents. Returns the accumulator for - the next enumeration step. + Should be called with the `enumerable` element and the + accumulator contents. + + Returns the accumulator for the next enumeration step. """ - @type reducer :: (term, term -> acc) + @type reducer :: (element :: term, current_acc :: acc -> updated_acc :: acc) @typedoc """ The result of the reduce operation. It may be *done* when the enumeration is finished by reaching its end, or *halted*/*suspended* when the enumeration was halted - or suspended by the reducer function. - - In case a reducer function returns the `:suspend` accumulator, the - `:suspended` tuple must be explicitly handled by the caller and - never leak. In practice, this means regular enumeration functions - just need to be concerned about `:done` and `:halted` results. - - Furthermore, a `:suspend` call must always be followed by another call, - eventually halting or continuing until the end. + or suspended by the tagged accumulator. + + In case the tagged `:halt` accumulator is given, the `:halted` tuple + with the accumulator must be returned. Functions like `Enum.take_while/2` + use `:halt` underneath and can be used to test halting enumerables. + + In case the tagged `:suspend` accumulator is given, the caller must + return the `:suspended` tuple with the accumulator and a continuation. + The caller is then responsible of managing the continuation and the + caller must always call the continuation, eventually halting or continuing + until the end. `Enum.zip/2` uses suspension, so it can be used to test + whether your implementation handles suspension correctly. You can also use + `Stream.zip/2` with `Enum.take_while/2` to test the combination of + `:suspend` with `:halt`. """ - @type result :: {:done, term} | {:halted, term} | {:suspended, term, continuation} + @type result :: + {:done, term} + | {:halted, term} + | {:suspended, term, continuation} @typedoc """ A partially applied reduce function. @@ -85,129 +114,211 @@ defprotocol Enumerable do the enumeration is suspended. When invoked, it expects a new accumulator and it returns the result. - A continuation is easily implemented as long as the reduce + A continuation can be trivially implemented as long as the reduce function is defined in a tail recursive fashion. If the function is tail recursive, all the state is passed as arguments, so - the continuation would simply be the reducing function partially - applied. + the continuation is the reducing function partially applied. """ @type continuation :: (acc -> result) + @typedoc """ + A slicing function that receives the initial position, + the number of elements in the slice, and the step. + + The `start` position is a number `>= 0` and guaranteed to + exist in the `enumerable`. The length is a number `>= 1` + in a way that `start + length * step <= count`, where + `count` is the maximum amount of elements in the enumerable. + + The function should return a non empty list where + the amount of elements is equal to `length`. + """ + @type slicing_fun :: + (start :: non_neg_integer, length :: pos_integer, step :: pos_integer -> [term()]) + + @typedoc """ + Receives an enumerable and returns a list. + """ + @type to_list_fun :: (t -> [term()]) + @doc """ - Reduces the collection into a value. + Reduces the `enumerable` into an element. Most of the operations in `Enum` are implemented in terms of reduce. - This function should apply the given `reducer` function to each - item in the collection and proceed as expected by the returned accumulator. + This function should apply the given `t:reducer/0` function to each + element in the `enumerable` and proceed as expected by the returned + accumulator. + + See the documentation of the types `t:result/0` and `t:acc/0` for + more information. + + ## Examples As an example, here is the implementation of `reduce` for lists: - def reduce(_, {:halt, acc}, _fun), do: {:halted, acc} - def reduce(list, {:suspend, acc}, fun), do: {:suspended, acc, &reduce(list, &1, fun)} - def reduce([], {:cont, acc}, _fun), do: {:done, acc} - def reduce([h|t], {:cont, acc}, fun), do: reduce(t, fun.(h, acc), fun) + def reduce(_list, {:halt, acc}, _fun), do: {:halted, acc} + def reduce(list, {:suspend, acc}, fun), do: {:suspended, acc, &reduce(list, &1, fun)} + def reduce([], {:cont, acc}, _fun), do: {:done, acc} + def reduce([head | tail], {:cont, acc}, fun), do: reduce(tail, fun.(head, acc), fun) """ @spec reduce(t, acc, reducer) :: result - def reduce(collection, acc, fun) + def reduce(enumerable, acc, fun) + + @doc """ + Retrieves the number of elements in the `enumerable`. + + It should return `{:ok, count}` if you can count the number of elements + in `enumerable` without traversing it. + + Otherwise it should return `{:error, __MODULE__}` and a default algorithm + built on top of `reduce/3` that runs in linear time will be used. + """ + @spec count(t) :: {:ok, non_neg_integer} | {:error, module} + def count(enumerable) @doc """ - Checks if a value exists within the collection. + Checks if an `element` exists within the `enumerable`. - It should return `{:ok, boolean}`. + It should return `{:ok, boolean}` if you can check the membership of a + given element in `enumerable` with `===/2` without traversing the whole + of it. - If `{:error, __MODULE__}` is returned a default algorithm using `reduce` and - the match (`===`) operator is used. This algorithm runs in linear time. + Otherwise it should return `{:error, __MODULE__}` and a default algorithm + built on top of `reduce/3` that runs in linear time will be used. - Please force use of the default algorithm unless you can implement an - algorithm that is significantly faster. + When called outside guards, the [`in`](`in/2`) and [`not in`](`in/2`) + operators work by using this function. """ @spec member?(t, term) :: {:ok, boolean} | {:error, module} - def member?(collection, value) + def member?(enumerable, element) @doc """ - Retrieves the collection's size. + Returns a function that slices the data structure contiguously. + + It should return either: + + * `{:ok, size, slicing_fun}` - if the `enumerable` has a known + bound and can access a position in the `enumerable` without + traversing all previous elements. The `slicing_fun` will receive + a `start` position, the `amount` of elements to fetch, and a + `step`. + + * `{:ok, size, to_list_fun}` - if the `enumerable` has a known bound + and can access a position in the `enumerable` by first converting + it to a list via `to_list_fun`. + + * `{:error, __MODULE__}` - the enumerable cannot be sliced efficiently + and a default algorithm built on top of `reduce/3` that runs in + linear time will be used. - It should return `{:ok, size}`. + ## Differences to `count/1` - If `{:error, __MODULE__}` is returned a default algorithm using `reduce` and - the match (`===`) operator is used. This algorithm runs in linear time. + The `size` value returned by this function is used for boundary checks, + therefore it is extremely important that this function only returns `:ok` + if retrieving the `size` of the `enumerable` is cheap, fast, and takes + constant time. Otherwise the simplest of operations, such as + `Enum.at(enumerable, 0)`, will become too expensive. - Please force use of the default algorithm unless you can implement an - algorithm that is significantly faster. + On the other hand, the `count/1` function in this protocol should be + implemented whenever you can count the number of elements in the collection + without traversing it. """ - @spec count(t) :: {:ok, non_neg_integer} | {:error, module} - def count(collection) + @spec slice(t) :: + {:ok, size :: non_neg_integer(), slicing_fun() | to_list_fun()} + | {:error, module()} + def slice(enumerable) end defmodule Enum do import Kernel, except: [max: 2, min: 2] @moduledoc """ - Provides a set of algorithms that enumerate over collections according to the - `Enumerable` protocol: - - iex> Enum.map([1, 2, 3], fn(x) -> x * 2 end) - [2,4,6] + Provides a set of algorithms to work with enumerables. - Some particular types, like dictionaries, yield a specific format on - enumeration. For dicts, the argument is always a `{key, value}` tuple: + In Elixir, an enumerable is any data type that implements the + `Enumerable` protocol. `List`s (`[1, 2, 3]`), `Map`s (`%{foo: 1, bar: 2}`) + and `Range`s (`1..3`) are common data types used as enumerables: - iex> dict = %{a: 1, b: 2} - iex> Enum.map(dict, fn {k, v} -> {k, v * 2} end) - [a: 2, b: 4] + iex> Enum.map([1, 2, 3], fn x -> x * 2 end) + [2, 4, 6] - Note that the functions in the `Enum` module are eager: they always start - the enumeration of the given collection. The `Stream` module allows - lazy enumeration of collections and provides infinite streams. + iex> Enum.sum([1, 2, 3]) + 6 - Since the majority of the functions in `Enum` enumerate the whole - collection and return a list as result, infinite streams need to - be carefully used with such functions, as they can potentially run - forever. For example: + iex> Enum.map(1..3, fn x -> x * 2 end) + [2, 4, 6] - Enum.each Stream.cycle([1,2,3]), &IO.puts(&1) + iex> Enum.sum(1..3) + 6 + iex> map = %{"a" => 1, "b" => 2} + iex> Enum.map(map, fn {k, v} -> {k, v * 2} end) + [{"a", 2}, {"b", 4}] + + However, many other enumerables exist in the language, such as `MapSet`s + and the data type returned by `File.stream!/3` which allows a file to be + traversed as if it was an enumerable. + + The functions in this module work in linear time. This means that, the + time it takes to perform an operation grows at the same rate as the length + of the enumerable. This is expected on operations such as `Enum.map/2`. + After all, if we want to traverse every element on a list, the longer the + list, the more elements we need to traverse, and the longer it will take. + + This linear behaviour should also be expected on operations like `count/1`, + `member?/2`, `at/2` and similar. While Elixir does allow data types to + provide performant variants for such operations, you should not expect it + to always be available, since the `Enum` module is meant to work with a + large variety of data types and not all data types can provide optimized + behaviour. + + Finally, note the functions in the `Enum` module are eager: they will + traverse the enumerable as soon as they are invoked. This is particularly + dangerous when working with infinite enumerables. In such cases, you should + use the `Stream` module, which allows you to lazily express computations, + without traversing collections, and work with possibly infinite collections. + See the `Stream` module for examples and documentation. """ @compile :inline_list_funcs - @type t :: Enumerable.t + @type t :: Enumerable.t() + @type acc :: any @type element :: any - @type index :: non_neg_integer + + @typedoc "Zero-based index. It can also be a negative integer." + @type index :: integer + @type default :: any - # Require Stream.Reducers and its callbacks require Stream.Reducers, as: R - defmacrop cont(_, entry, acc) do - quote do: {:cont, [unquote(entry)|unquote(acc)]} + defmacrop skip(acc) do + acc + end + + defmacrop next(_, entry, acc) do + quote(do: [unquote(entry) | unquote(acc)]) end - defmacrop acc(h, n, _) do - quote do: {unquote(h), unquote(n)} + defmacrop acc(head, state, _) do + quote(do: {unquote(head), unquote(state)}) end - defmacrop cont_with_acc(f, entry, h, n, _) do + defmacrop next_with_acc(_, entry, head, state, _) do quote do - {:cont, {[unquote(entry)|unquote(h)], unquote(n)}} + {[unquote(entry) | unquote(head)], unquote(state)} end end @doc """ - Invokes the given `fun` for each item in the `collection` and returns `false` - if at least one invocation returns `false`. Otherwise returns `true`. - - ## Examples - - iex> Enum.all?([2, 4, 6], fn(x) -> rem(x, 2) == 0 end) - true + Returns `true` if all elements in `enumerable` are truthy. - iex> Enum.all?([2, 3, 4], fn(x) -> rem(x, 2) == 0 end) - false + When an element has a falsy value (`false` or `nil`) iteration stops immediately + and `false` is returned. In all other cases `true` is returned. - If no function is given, it defaults to checking if - all items in the collection evaluate to `true`. + ## Examples iex> Enum.all?([1, 2, 3]) true @@ -215,36 +326,65 @@ defmodule Enum do iex> Enum.all?([1, nil, 3]) false + iex> Enum.all?([]) + true + """ @spec all?(t) :: boolean - @spec all?(t, (element -> as_boolean(term))) :: boolean - - def all?(collection, fun \\ fn(x) -> x end) - - def all?(collection, fun) when is_list(collection) do - do_all?(collection, fun) + def all?(enumerable) when is_list(enumerable) do + all_list(enumerable) end - def all?(collection, fun) do - Enumerable.reduce(collection, {:cont, true}, fn(entry, _) -> - if fun.(entry), do: {:cont, true}, else: {:halt, false} - end) |> elem(1) + def all?(enumerable) do + Enumerable.reduce(enumerable, {:cont, true}, fn entry, _ -> + if entry, do: {:cont, true}, else: {:halt, false} + end) + |> elem(1) end @doc """ - Invokes the given `fun` for each item in the `collection` and returns `true` if - at least one invocation returns `true`. Returns `false` otherwise. + Returns `true` if `fun.(element)` is truthy for all elements in `enumerable`. + + Iterates over `enumerable` and invokes `fun` on each element. If `fun` ever + returns a falsy value (`false` or `nil`), iteration stops immediately and + `false` is returned. Otherwise, `true` is returned. ## Examples - iex> Enum.any?([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) + iex> Enum.all?([2, 4, 6], fn x -> rem(x, 2) == 0 end) + true + + iex> Enum.all?([2, 3, 4], fn x -> rem(x, 2) == 0 end) false - iex> Enum.any?([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) + iex> Enum.all?([], fn _ -> nil end) true - If no function is given, it defaults to checking if - at least one item in the collection evaluates to `true`. + As the last example shows, `Enum.all?/2` returns `true` if `enumerable` is + empty, regardless of `fun`. In an empty enumerable there is no element for + which `fun` returns a falsy value, so the result must be `true`. This is a + well-defined logical argument for empty collections. + + """ + @spec all?(t, (element -> as_boolean(term))) :: boolean + def all?(enumerable, fun) when is_list(enumerable) do + all_list(enumerable, fun) + end + + def all?(enumerable, fun) do + Enumerable.reduce(enumerable, {:cont, true}, fn entry, _ -> + if fun.(entry), do: {:cont, true}, else: {:halt, false} + end) + |> elem(1) + end + + @doc """ + Returns `true` if at least one element in `enumerable` is truthy. + + When an element has a truthy value (neither `false` nor `nil`) iteration stops + immediately and `true` is returned. In all other cases `false` is returned. + + ## Examples iex> Enum.any?([false, false, false]) false @@ -252,25 +392,61 @@ defmodule Enum do iex> Enum.any?([false, true, false]) true + iex> Enum.any?([]) + false + """ @spec any?(t) :: boolean - @spec any?(t, (element -> as_boolean(term))) :: boolean + def any?(enumerable) when is_list(enumerable) do + any_list(enumerable) + end + + def any?(enumerable) do + Enumerable.reduce(enumerable, {:cont, false}, fn entry, _ -> + if entry, do: {:halt, true}, else: {:cont, false} + end) + |> elem(1) + end - def any?(collection, fun \\ fn(x) -> x end) + @doc """ + Returns `true` if `fun.(element)` is truthy for at least one element in `enumerable`. + + Iterates over the `enumerable` and invokes `fun` on each element. When an invocation + of `fun` returns a truthy value (neither `false` nor `nil`) iteration stops + immediately and `true` is returned. In all other cases `false` is returned. + + ## Examples + + iex> Enum.any?([2, 4, 6], fn x -> rem(x, 2) == 1 end) + false + + iex> Enum.any?([2, 3, 4], fn x -> rem(x, 2) == 1 end) + true - def any?(collection, fun) when is_list(collection) do - do_any?(collection, fun) + iex> Enum.any?([], fn x -> x > 0 end) + false + + """ + @spec any?(t, (element -> as_boolean(term))) :: boolean + def any?(enumerable, fun) when is_list(enumerable) do + any_list(enumerable, fun) end - def any?(collection, fun) do - Enumerable.reduce(collection, {:cont, false}, fn(entry, _) -> + def any?(enumerable, fun) do + Enumerable.reduce(enumerable, {:cont, false}, fn entry, _ -> if fun.(entry), do: {:halt, true}, else: {:cont, false} - end) |> elem(1) + end) + |> elem(1) end @doc """ - Finds the element at the given index (zero-based). - Returns `default` if index is out of bounds. + Finds the element at the given `index` (zero-based). + + Returns `default` if `index` is out of bounds. + + A negative `index` can be passed, which means the `enumerable` is + enumerated once and the `index` is counted from the end (for example, + `-1` finds the last element). ## Examples @@ -287,67 +463,151 @@ defmodule Enum do :none """ - @spec at(t, integer) :: element | nil - @spec at(t, integer, default) :: element | default - def at(collection, n, default \\ nil) do - case fetch(collection, n) do - {:ok, h} -> h - :error -> default + @spec at(t, index, default) :: element | default + def at(enumerable, index, default \\ nil) when is_integer(index) do + case slice_forward(enumerable, index, 1, 1) do + [value] -> value + [] -> default end end + @doc false + @deprecated "Use Enum.chunk_every/2 instead" + def chunk(enumerable, count), do: chunk(enumerable, count, count, nil) + + @doc false + @deprecated "Use Enum.chunk_every/3 instead" + def chunk(enum, n, step) do + chunk_every(enum, n, step, :discard) + end + + @doc false + @deprecated "Use Enum.chunk_every/4 instead" + def chunk(enumerable, count, step, leftover) do + chunk_every(enumerable, count, step, leftover || :discard) + end + @doc """ - Shortcut to `chunk(coll, n, n)`. + Shortcut to `chunk_every(enumerable, count, count)`. """ - @spec chunk(t, non_neg_integer) :: [list] - def chunk(coll, n), do: chunk(coll, n, n, nil) + @doc since: "1.5.0" + @spec chunk_every(t, pos_integer) :: [list] + def chunk_every(enumerable, count), do: chunk_every(enumerable, count, count, []) @doc """ - Returns a collection of lists containing `n` items each, where - each new chunk starts `step` elements into the collection. + Returns list of lists containing `count` elements each, where + each new chunk starts `step` elements into the `enumerable`. - `step` is optional and, if not passed, defaults to `n`, i.e. - chunks do not overlap. If the final chunk does not have `n` - elements to fill the chunk, elements are taken as necessary - from `pad` if it was passed. If `pad` is passed and does not - have enough elements to fill the chunk, then the chunk is - returned anyway with less than `n` elements. If `pad` is not - passed at all or is `nil`, then the partial chunk is discarded - from the result. + `step` is optional and, if not passed, defaults to `count`, i.e. + chunks do not overlap. + + If the last chunk does not have `count` elements to fill the chunk, + elements are taken from `leftover` to fill in the chunk. If `leftover` + does not have enough elements to fill the chunk, then a partial chunk + is returned with less than `count` elements. + + If `:discard` is given in `leftover`, the last chunk is discarded + unless it has exactly `count` elements. ## Examples - iex> Enum.chunk([1, 2, 3, 4, 5, 6], 2) + iex> Enum.chunk_every([1, 2, 3, 4, 5, 6], 2) [[1, 2], [3, 4], [5, 6]] - iex> Enum.chunk([1, 2, 3, 4, 5, 6], 3, 2) + iex> Enum.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, :discard) [[1, 2, 3], [3, 4, 5]] - iex> Enum.chunk([1, 2, 3, 4, 5, 6], 3, 2, [7]) + iex> Enum.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, [7]) [[1, 2, 3], [3, 4, 5], [5, 6, 7]] - iex> Enum.chunk([1, 2, 3, 4, 5, 6], 3, 3, []) - [[1, 2, 3], [4, 5, 6]] + iex> Enum.chunk_every([1, 2, 3, 4], 3, 3, []) + [[1, 2, 3], [4]] + + iex> Enum.chunk_every([1, 2, 3, 4], 10) + [[1, 2, 3, 4]] + + iex> Enum.chunk_every([1, 2, 3, 4, 5], 2, 3, []) + [[1, 2], [4, 5]] """ - @spec chunk(t, non_neg_integer, non_neg_integer) :: [list] - @spec chunk(t, non_neg_integer, non_neg_integer, t | nil) :: [list] - def chunk(coll, n, step, pad \\ nil) when n > 0 and step > 0 do - limit = :erlang.max(n, step) + @doc since: "1.5.0" + @spec chunk_every(t, pos_integer, pos_integer, t | :discard) :: [list] + def chunk_every(enumerable, count, step, leftover \\ []) + when is_integer(count) and count > 0 and is_integer(step) and step > 0 do + R.chunk_every(&chunk_while/4, enumerable, count, step, leftover) + end + + @doc """ + Chunks the `enumerable` with fine grained control when every chunk is emitted. - {_, {acc, {buffer, i}}} = - Enumerable.reduce(coll, {:cont, {[], {[], 0}}}, R.chunk(n, step, limit)) + `chunk_fun` receives the current element and the accumulator and must return: - if nil?(pad) || i == 0 do - :lists.reverse(acc) - else - buffer = :lists.reverse(buffer) ++ take(pad, n - i) - :lists.reverse([buffer|acc]) + * `{:cont, chunk, acc}` to emit a chunk and continue with the accumulator + * `{:cont, acc}` to not emit any chunk and continue with the accumulator + * `{:halt, acc}` to halt chunking over the `enumerable`. + + `after_fun` is invoked with the final accumulator when iteration is + finished (or `halt`ed) to handle any trailing elements that were returned + as part of an accumulator, but were not emitted as a chunk by `chunk_fun`. + It must return: + + * `{:cont, chunk, acc}` to emit a chunk. The chunk will be appended to the + list of already emitted chunks. + * `{:cont, acc}` to not emit a chunk + + The `acc` in `after_fun` is required in order to mirror the tuple format + from `chunk_fun` but it will be discarded since the traversal is complete. + + Returns a list of emitted chunks. + + ## Examples + + iex> chunk_fun = fn element, acc -> + ...> if rem(element, 2) == 0 do + ...> {:cont, Enum.reverse([element | acc]), []} + ...> else + ...> {:cont, [element | acc]} + ...> end + ...> end + iex> after_fun = fn + ...> [] -> {:cont, []} + ...> acc -> {:cont, Enum.reverse(acc), []} + ...> end + iex> Enum.chunk_while(1..10, [], chunk_fun, after_fun) + [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + iex> Enum.chunk_while([1, 2, 3, 5, 7], [], chunk_fun, after_fun) + [[1, 2], [3, 5, 7]] + + """ + @doc since: "1.5.0" + @spec chunk_while( + t, + acc, + (element, acc -> {:cont, chunk, acc} | {:cont, acc} | {:halt, acc}), + (acc -> {:cont, chunk, acc} | {:cont, acc}) + ) :: Enumerable.t() + when chunk: any + def chunk_while(enumerable, acc, chunk_fun, after_fun) do + {_, {res, acc}} = + Enumerable.reduce(enumerable, {:cont, {[], acc}}, fn entry, {buffer, acc} -> + case chunk_fun.(entry, acc) do + {:cont, chunk, acc} -> {:cont, {[chunk | buffer], acc}} + {:cont, acc} -> {:cont, {buffer, acc}} + {:halt, acc} -> {:halt, {buffer, acc}} + end + end) + + case after_fun.(acc) do + {:cont, _acc} -> :lists.reverse(res) + {:cont, chunk, _acc} -> :lists.reverse([chunk | res]) end end @doc """ - Splits `coll` on every element for which `fun` returns a new value. + Splits enumerable on every element for which `fun` returns a new + value. + + Returns a list of lists. ## Examples @@ -356,47 +616,48 @@ defmodule Enum do """ @spec chunk_by(t, (element -> any)) :: [list] - def chunk_by(coll, fun) do - {_, {acc, res}} = - Enumerable.reduce(coll, {:cont, {[], nil}}, R.chunk_by(fun)) - - case res do - {buffer, _} -> - :lists.reverse([:lists.reverse(buffer) | acc]) - nil -> - [] - end + def chunk_by(enumerable, fun) do + R.chunk_by(&chunk_while/4, enumerable, fun) end @doc """ - Given an enumerable of enumerables, concatenate the enumerables into a single list. + Given an enumerable of enumerables, concatenates the `enumerables` into + a single list. ## Examples iex> Enum.concat([1..3, 4..6, 7..9]) - [1,2,3,4,5,6,7,8,9] + [1, 2, 3, 4, 5, 6, 7, 8, 9] iex> Enum.concat([[1, [2], 3], [4], [5, 6]]) - [1,[2],3,4,5,6] + [1, [2], 3, 4, 5, 6] """ @spec concat(t) :: t - def concat(enumerables) do - do_concat(enumerables) + def concat(enumerables) + + def concat(list) when is_list(list) do + concat_list(list) + end + + def concat(enums) do + concat_enum(enums) end @doc """ - Concatenates the enumerable on the right with the enumerable on the left. + Concatenates the enumerable on the `right` with the enumerable on the + `left`. - This function produces the same result as the `Kernel.++/2` operator for lists. + This function produces the same result as the `++/2` operator + for lists. ## Examples iex> Enum.concat(1..3, 4..6) - [1,2,3,4,5,6] + [1, 2, 3, 4, 5, 6] iex> Enum.concat([1, 2, 3], [4, 5, 6]) - [1,2,3,4,5,6] + [1, 2, 3, 4, 5, 6] """ @spec concat(t, t) :: t @@ -405,16 +666,11 @@ defmodule Enum do end def concat(left, right) do - do_concat([left, right]) - end - - defp do_concat(enumerable) do - fun = &[&1|&2] - reduce(enumerable, [], &reduce(&1, &2, fun)) |> :lists.reverse + concat_enum([left, right]) end @doc """ - Returns the collection's size. + Returns the size of the `enumerable`. ## Examples @@ -423,45 +679,186 @@ defmodule Enum do """ @spec count(t) :: non_neg_integer - def count(collection) when is_list(collection) do - :erlang.length(collection) + def count(enumerable) when is_list(enumerable) do + length(enumerable) end - def count(collection) do - case Enumerable.count(collection) do + def count(enumerable) do + case Enumerable.count(enumerable) do {:ok, value} when is_integer(value) -> value + {:error, module} -> - module.reduce(collection, {:cont, 0}, fn - _, acc -> {:cont, acc + 1} - end) |> elem(1) + enumerable |> module.reduce({:cont, 0}, fn _, acc -> {:cont, acc + 1} end) |> elem(1) end end @doc """ - Returns the count of items in the collection for which - `fun` returns `true`. + Returns the count of elements in the `enumerable` for which `fun` returns + a truthy value. ## Examples - iex> Enum.count([1, 2, 3, 4, 5], fn(x) -> rem(x, 2) == 0 end) + iex> Enum.count([1, 2, 3, 4, 5], fn x -> rem(x, 2) == 0 end) 2 """ @spec count(t, (element -> as_boolean(term))) :: non_neg_integer - def count(collection, fun) do - Enumerable.reduce(collection, {:cont, 0}, fn(entry, acc) -> - {:cont, if(fun.(entry), do: acc + 1, else: acc)} - end) |> elem(1) + def count(enumerable, fun) do + reduce(enumerable, 0, fn entry, acc -> + if(fun.(entry), do: acc + 1, else: acc) + end) + end + + @doc """ + Counts the enumerable stopping at `limit`. + + This is useful for checking certain properties of the count of an enumerable + without having to actually count the entire enumerable. For example, if you + wanted to check that the count was exactly, at least, or more than a value. + + If the enumerable implements `c:Enumerable.count/1`, the enumerable is + not traversed and we return the lower of the two numbers. To force + enumeration, use `count_until/3` with `fn _ -> true end` as the second + argument. + + ## Examples + + iex> Enum.count_until(1..20, 5) + 5 + iex> Enum.count_until(1..20, 50) + 20 + iex> Enum.count_until(1..10, 10) == 10 # At least 10 + true + iex> Enum.count_until(1..11, 10 + 1) > 10 # More than 10 + true + iex> Enum.count_until(1..5, 10) < 10 # Less than 10 + true + iex> Enum.count_until(1..10, 10 + 1) == 10 # Exactly ten + true + + """ + @doc since: "1.12.0" + @spec count_until(t, pos_integer) :: non_neg_integer + def count_until(enumerable, limit) when is_integer(limit) and limit > 0 do + stop_at = limit - 1 + + case Enumerable.count(enumerable) do + {:ok, value} -> + Kernel.min(value, limit) + + {:error, module} -> + enumerable + |> module.reduce( + {:cont, 0}, + fn + _, ^stop_at -> + {:halt, limit} + + _, acc -> + {:cont, acc + 1} + end + ) + |> elem(1) + end + end + + @doc """ + Counts the elements in the enumerable for which `fun` returns a truthy value, stopping at `limit`. + + See `count/2` and `count_until/3` for more information. + + ## Examples + + iex> Enum.count_until(1..20, fn x -> rem(x, 2) == 0 end, 7) + 7 + iex> Enum.count_until(1..20, fn x -> rem(x, 2) == 0 end, 11) + 10 + """ + @doc since: "1.12.0" + @spec count_until(t, (element -> as_boolean(term)), pos_integer) :: non_neg_integer + def count_until(enumerable, fun, limit) when is_integer(limit) and limit > 0 do + stop_at = limit - 1 + + Enumerable.reduce(enumerable, {:cont, 0}, fn + entry, ^stop_at -> + if fun.(entry) do + {:halt, limit} + else + {:cont, stop_at} + end + + entry, acc -> + if fun.(entry) do + {:cont, acc + 1} + else + {:cont, acc} + end + end) + |> elem(1) + end + + @doc """ + Enumerates the `enumerable`, returning a list where all consecutive + duplicated elements are collapsed to a single element. + + Elements are compared using `===/2`. + + If you want to remove all duplicated elements, regardless of order, + see `uniq/1`. + + ## Examples + + iex> Enum.dedup([1, 2, 3, 3, 2, 1]) + [1, 2, 3, 2, 1] + + iex> Enum.dedup([1, 1, 2, 2.0, :three, :three]) + [1, 2, 2.0, :three] + + """ + @spec dedup(t) :: list + def dedup(enumerable) when is_list(enumerable) do + dedup_list(enumerable, []) |> :lists.reverse() + end + + def dedup(enumerable) do + Enum.reduce(enumerable, [], fn x, acc -> + case acc do + [^x | _] -> acc + _ -> [x | acc] + end + end) + |> :lists.reverse() + end + + @doc """ + Enumerates the `enumerable`, returning a list where all consecutive + duplicated elements are collapsed to a single element. + + The function `fun` maps every element to a term which is used to + determine if two elements are duplicates. + + ## Examples + + iex> Enum.dedup_by([{1, :a}, {2, :b}, {2, :c}, {1, :a}], fn {x, _} -> x end) + [{1, :a}, {2, :b}, {1, :a}] + + iex> Enum.dedup_by([5, 1, 2, 3, 2, 1], fn x -> x > 2 end) + [5, 1, 3, 2] + + """ + @spec dedup_by(t, (element -> term)) :: list + def dedup_by(enumerable, fun) do + {list, _} = reduce(enumerable, {[], []}, R.dedup(fun)) + :lists.reverse(list) end @doc """ - Drops the first `count` items from `collection`. + Drops the `amount` of elements from the `enumerable`. - If a negative value `count` is given, the last `count` - values will be dropped. The collection is enumerated - once to retrieve the proper index and the remaining - calculation is performed from the end. + If a negative `amount` is given, the `amount` of last values will be dropped. + The `enumerable` will be enumerated once to retrieve the proper index and + the remaining calculation is performed from the end. ## Examples @@ -472,79 +869,122 @@ defmodule Enum do [] iex> Enum.drop([1, 2, 3], 0) - [1,2,3] + [1, 2, 3] iex> Enum.drop([1, 2, 3], -1) - [1,2] + [1, 2] """ @spec drop(t, integer) :: list - def drop(collection, count) when is_list(collection) and count >= 0 do - do_drop(collection, count) + def drop(enumerable, amount) + when is_list(enumerable) and is_integer(amount) and amount >= 0 do + drop_list(enumerable, amount) end - def drop(collection, count) when count >= 0 do - res = - reduce(collection, count, fn - x, acc when is_list(acc) -> [x|acc] - x, 0 -> [x] - _, acc when acc > 0 -> acc - 1 - end) - if is_list(res), do: :lists.reverse(res), else: [] + def drop(enumerable, 0) do + to_list(enumerable) + end + + def drop(enumerable, amount) when is_integer(amount) and amount > 0 do + {result, _} = reduce(enumerable, {[], amount}, R.drop()) + if is_list(result), do: :lists.reverse(result), else: [] + end + + def drop(enumerable, amount) when is_integer(amount) and amount < 0 do + {count, fun} = slice_count_and_fun(enumerable, 1) + amount = Kernel.min(amount + count, count) + + if amount > 0 do + fun.(0, amount, 1) + else + [] + end end - def drop(collection, count) when count < 0 do - do_drop(reverse(collection), abs(count)) |> :lists.reverse + @doc """ + Returns a list of every `nth` element in the `enumerable` dropped, + starting with the first element. + + The first element is always dropped, unless `nth` is 0. + + The second argument specifying every `nth` element must be a non-negative + integer. + + ## Examples + + iex> Enum.drop_every(1..10, 2) + [2, 4, 6, 8, 10] + + iex> Enum.drop_every(1..10, 0) + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + iex> Enum.drop_every([1, 2, 3], 1) + [] + + """ + @spec drop_every(t, non_neg_integer) :: list + def drop_every(enumerable, nth) + + def drop_every(_enumerable, 1), do: [] + def drop_every(enumerable, 0), do: to_list(enumerable) + def drop_every([], nth) when is_integer(nth), do: [] + + def drop_every(enumerable, nth) when is_integer(nth) and nth > 1 do + {res, _} = reduce(enumerable, {[], :first}, R.drop_every(nth)) + :lists.reverse(res) end @doc """ - Drops items at the beginning of `collection` while `fun` returns `true`. + Drops elements at the beginning of the `enumerable` while `fun` returns a + truthy value. ## Examples - iex> Enum.drop_while([1, 2, 3, 4, 5], fn(x) -> x < 3 end) - [3,4,5] + iex> Enum.drop_while([1, 2, 3, 2, 1], fn x -> x < 3 end) + [3, 2, 1] """ @spec drop_while(t, (element -> as_boolean(term))) :: list - def drop_while(collection, fun) when is_list(collection) do - do_drop_while(collection, fun) + def drop_while(enumerable, fun) when is_list(enumerable) do + drop_while_list(enumerable, fun) end - def drop_while(collection, fun) do - {_, {res, _}} = - Enumerable.reduce(collection, {:cont, {[], true}}, R.drop_while(fun)) + def drop_while(enumerable, fun) do + {res, _} = reduce(enumerable, {[], true}, R.drop_while(fun)) :lists.reverse(res) end @doc """ - Invokes the given `fun` for each item in the `collection`. + Invokes the given `fun` for each element in the `enumerable`. + Returns `:ok`. ## Examples - Enum.each(["some", "example"], fn(x) -> IO.puts x end) + Enum.each(["some", "example"], fn x -> IO.puts(x) end) "some" "example" #=> :ok """ @spec each(t, (element -> any)) :: :ok - def each(collection, fun) when is_list(collection) do - :lists.foreach(fun, collection) - :ok + def each(enumerable, fun) when is_list(enumerable) do + :lists.foreach(fun, enumerable) end - def each(collection, fun) do - reduce(collection, nil, fn(entry, _) -> + def each(enumerable, fun) do + reduce(enumerable, nil, fn entry, _ -> fun.(entry) nil end) + :ok end @doc """ - Returns `true` if the collection is empty, otherwise `false`. + Determines if the `enumerable` is empty. + + Returns `true` if `enumerable` is empty, otherwise `false`. ## Examples @@ -556,20 +996,29 @@ defmodule Enum do """ @spec empty?(t) :: boolean - def empty?(collection) when is_list(collection) do - collection == [] + def empty?(enumerable) when is_list(enumerable) do + enumerable == [] end - def empty?(collection) do - Enumerable.reduce(collection, {:cont, true}, fn(_, _) -> {:halt, false} end) |> elem(1) + def empty?(enumerable) do + case Enumerable.slice(enumerable) do + {:ok, value, _} -> + value == 0 + + {:error, module} -> + enumerable + |> module.reduce({:cont, true}, fn _, _ -> {:halt, false} end) + |> elem(1) + end end @doc """ - Finds the element at the given index (zero-based). + Finds the element at the given `index` (zero-based). + Returns `{:ok, element}` if found, otherwise `:error`. - A negative index can be passed, which means the collection is - enumerated once and the index is counted from the end (i.e. + A negative `index` can be passed, which means the `enumerable` is + enumerated once and the `index` is counted from the end (for example, `-1` fetches the last element). ## Examples @@ -577,6 +1026,9 @@ defmodule Enum do iex> Enum.fetch([2, 4, 6], 0) {:ok, 2} + iex> Enum.fetch([2, 4, 6], -3) + {:ok, 2} + iex> Enum.fetch([2, 4, 6], 2) {:ok, 6} @@ -584,35 +1036,19 @@ defmodule Enum do :error """ - @spec fetch(t, integer) :: {:ok, element} | :error - def fetch(collection, n) when is_list(collection) and n >= 0 do - do_fetch(collection, n) - end - - def fetch(collection, n) when n >= 0 do - res = - Enumerable.reduce(collection, {:cont, 0}, fn(entry, acc) -> - if acc == n do - {:halt, entry} - else - {:cont, acc + 1} - end - end) - - case res do - {:halted, entry} -> {:ok, entry} - {:done, _} -> :error + @spec fetch(t, index) :: {:ok, element} | :error + def fetch(enumerable, index) when is_integer(index) do + case slice_forward(enumerable, index, 1, 1) do + [value] -> {:ok, value} + [] -> :error end end - def fetch(collection, n) when n < 0 do - do_fetch(reverse(collection), abs(n + 1)) - end - @doc """ - Finds the element at the given index (zero-based). - Raises `OutOfBoundsError` if the given position - is outside the range of the collection. + Finds the element at the given `index` (zero-based). + + Raises `OutOfBoundsError` if the given `index` is outside the range of + the `enumerable`. ## Examples @@ -626,192 +1062,238 @@ defmodule Enum do ** (Enum.OutOfBoundsError) out of bounds error """ - @spec fetch!(t, integer) :: element | no_return - def fetch!(collection, n) do - case fetch(collection, n) do - {:ok, h} -> h - :error -> raise Enum.OutOfBoundsError + @spec fetch!(t, index) :: element + def fetch!(enumerable, index) when is_integer(index) do + case slice_forward(enumerable, index, 1, 1) do + [value] -> value + [] -> raise Enum.OutOfBoundsError end end @doc """ - Filters the collection, i.e. returns only those elements - for which `fun` returns `true`. + Filters the `enumerable`, i.e. returns only those elements + for which `fun` returns a truthy value. + + See also `reject/2` which discards all elements where the + function returns a truthy value. ## Examples - iex> Enum.filter([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) + iex> Enum.filter([1, 2, 3], fn x -> rem(x, 2) == 0 end) [2] + Keep in mind that `filter` is not capable of filtering and + transforming an element at the same time. If you would like + to do so, consider using `flat_map/2`. For example, if you + want to convert all strings that represent an integer and + discard the invalid one in one pass: + + strings = ["1234", "abc", "12ab"] + + Enum.flat_map(strings, fn string -> + case Integer.parse(string) do + # transform to integer + {int, _rest} -> [int] + # skip the value + :error -> [] + end + end) + """ @spec filter(t, (element -> as_boolean(term))) :: list - def filter(collection, fun) when is_list(collection) do - for item <- collection, fun.(item), do: item + def filter(enumerable, fun) when is_list(enumerable) do + filter_list(enumerable, fun) end - def filter(collection, fun) do - Enumerable.reduce(collection, {:cont, []}, R.filter(fun)) - |> elem(1) |> :lists.reverse + def filter(enumerable, fun) do + reduce(enumerable, [], R.filter(fun)) |> :lists.reverse() end - @doc """ - Filters the collection and maps its values in one pass. - - ## Examples - - iex> Enum.filter_map([1, 2, 3], fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) - [4] - - """ - @spec filter_map(t, (element -> as_boolean(term)), (element -> element)) :: list - def filter_map(collection, filter, mapper) when is_list(collection) do - for item <- collection, filter.(item), do: mapper.(item) + @doc false + @deprecated "Use Enum.filter/2 + Enum.map/2 or for comprehensions instead" + def filter_map(enumerable, filter, mapper) when is_list(enumerable) do + for element <- enumerable, filter.(element), do: mapper.(element) end - def filter_map(collection, filter, mapper) do - Enumerable.reduce(collection, {:cont, []}, R.filter_map(filter, mapper)) - |> elem(1) |> :lists.reverse + def filter_map(enumerable, filter, mapper) do + enumerable + |> reduce([], R.filter_map(filter, mapper)) + |> :lists.reverse() end @doc """ - Returns the first item for which `fun` returns a truthy value. If no such - item is found, returns `ifnone`. + Returns the first element for which `fun` returns a truthy value. + If no such element is found, returns `default`. ## Examples - iex> Enum.find([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) - nil + iex> Enum.find([2, 3, 4], fn x -> rem(x, 2) == 1 end) + 3 - iex> Enum.find([2, 4, 6], 0, fn(x) -> rem(x, 2) == 1 end) + iex> Enum.find([2, 4, 6], fn x -> rem(x, 2) == 1 end) + nil + iex> Enum.find([2, 4, 6], 0, fn x -> rem(x, 2) == 1 end) 0 - iex> Enum.find([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) - 3 - """ - @spec find(t, (element -> any)) :: element | nil @spec find(t, default, (element -> any)) :: element | default - def find(collection, ifnone \\ nil, fun) + def find(enumerable, default \\ nil, fun) - def find(collection, ifnone, fun) when is_list(collection) do - do_find(collection, ifnone, fun) + def find(enumerable, default, fun) when is_list(enumerable) do + find_list(enumerable, default, fun) end - def find(collection, ifnone, fun) do - Enumerable.reduce(collection, {:cont, ifnone}, fn(entry, ifnone) -> - if fun.(entry), do: {:halt, entry}, else: {:cont, ifnone} - end) |> elem(1) + def find(enumerable, default, fun) do + Enumerable.reduce(enumerable, {:cont, default}, fn entry, default -> + if fun.(entry), do: {:halt, entry}, else: {:cont, default} + end) + |> elem(1) end @doc """ - Similar to `find/3`, but returns the value of the function - invocation instead of the element itself. + Similar to `find/3`, but returns the index (zero-based) + of the element instead of the element itself. ## Examples - iex> Enum.find_value([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) + iex> Enum.find_index([2, 4, 6], fn x -> rem(x, 2) == 1 end) nil - iex> Enum.find_value([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) - true + iex> Enum.find_index([2, 3, 4], fn x -> rem(x, 2) == 1 end) + 1 """ - @spec find_value(t, (element -> any)) :: any | :nil - @spec find_value(t, any, (element -> any)) :: any | :nil - def find_value(collection, ifnone \\ nil, fun) - - def find_value(collection, ifnone, fun) when is_list(collection) do - do_find_value(collection, ifnone, fun) + @spec find_index(t, (element -> any)) :: non_neg_integer | nil + def find_index(enumerable, fun) when is_list(enumerable) do + find_index_list(enumerable, 0, fun) end - def find_value(collection, ifnone, fun) do - Enumerable.reduce(collection, {:cont, ifnone}, fn(entry, ifnone) -> - fun_entry = fun.(entry) - if fun_entry, do: {:halt, fun_entry}, else: {:cont, ifnone} - end) |> elem(1) + def find_index(enumerable, fun) do + result = + Enumerable.reduce(enumerable, {:cont, {:not_found, 0}}, fn entry, {_, index} -> + if fun.(entry), do: {:halt, {:found, index}}, else: {:cont, {:not_found, index + 1}} + end) + + case elem(result, 1) do + {:found, index} -> index + {:not_found, _} -> nil + end end @doc """ - Similar to `find/3`, but returns the index (zero-based) - of the element instead of the element itself. + Similar to `find/3`, but returns the value of the function + invocation instead of the element itself. + + The return value is considered to be found when the result is truthy + (neither `nil` nor `false`). ## Examples - iex> Enum.find_index([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) + iex> Enum.find_value([2, 3, 4], fn x -> + ...> if x > 2, do: x * x + ...> end) + 9 + + iex> Enum.find_value([2, 4, 6], fn x -> rem(x, 2) == 1 end) nil - iex> Enum.find_index([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) - 1 + iex> Enum.find_value([2, 3, 4], fn x -> rem(x, 2) == 1 end) + true + + iex> Enum.find_value([1, 2, 3], "no bools!", &is_boolean/1) + "no bools!" """ - @spec find_index(t, (element -> any)) :: index | :nil - def find_index(collection, fun) when is_list(collection) do - do_find_index(collection, 0, fun) - end + @spec find_value(t, any, (element -> any)) :: any | nil + def find_value(enumerable, default \\ nil, fun) - def find_index(collection, fun) do - res = - Enumerable.reduce(collection, {:cont, 0}, fn(entry, acc) -> - if fun.(entry), do: {:halt, acc}, else: {:cont, acc + 1} - end) + def find_value(enumerable, default, fun) when is_list(enumerable) do + find_value_list(enumerable, default, fun) + end - case res do - {:halted, entry} -> entry - {:done, _} -> nil - end + def find_value(enumerable, default, fun) do + Enumerable.reduce(enumerable, {:cont, default}, fn entry, default -> + fun_entry = fun.(entry) + if fun_entry, do: {:halt, fun_entry}, else: {:cont, default} + end) + |> elem(1) end @doc """ - Returns a new collection appending the result of invoking `fun` - on each corresponding item of `collection`. + Maps the given `fun` over `enumerable` and flattens the result. - The given function should return an enumerable. + This function returns a new enumerable built by appending the result of invoking `fun` + on each element of `enumerable` together; conceptually, this is similar to a + combination of `map/2` and `concat/1`. ## Examples - iex> Enum.flat_map([:a, :b, :c], fn(x) -> [x, x] end) + iex> Enum.flat_map([:a, :b, :c], fn x -> [x, x] end) [:a, :a, :b, :b, :c, :c] - iex> Enum.flat_map([{1,3}, {4,6}], fn({x,y}) -> x..y end) + iex> Enum.flat_map([{1, 3}, {4, 6}], fn {x, y} -> x..y end) [1, 2, 3, 4, 5, 6] + iex> Enum.flat_map([:a, :b, :c], fn x -> [[x]] end) + [[:a], [:b], [:c]] + """ @spec flat_map(t, (element -> t)) :: list - def flat_map(collection, fun) do - reduce(collection, [], fn(entry, acc) -> - reduce(fun.(entry), acc, &[&1|&2]) - end) |> :lists.reverse + def flat_map(enumerable, fun) when is_list(enumerable) do + flat_map_list(enumerable, fun) + end + + def flat_map(enumerable, fun) do + reduce(enumerable, [], fn entry, acc -> + case fun.(entry) do + list when is_list(list) -> [list | acc] + other -> [to_list(other) | acc] + end + end) + |> flat_reverse([]) end + defp flat_reverse([h | t], acc), do: flat_reverse(t, h ++ acc) + defp flat_reverse([], acc), do: acc + @doc """ - Maps and reduces a collection, flattening the given results. + Maps and reduces an `enumerable`, flattening the given results (only one level deep). - It expects an accumulator and a function that receives each stream item - and an accumulator, and must return a tuple containing a new stream - (often a list) with the new accumulator or a tuple with `:halt` as first - element and the accumulator as second. + It expects an accumulator and a function that receives each enumerable + element, and must return a tuple containing a new enumerable (often a list) + with the new accumulator or a tuple with `:halt` as first element and + the accumulator as second. ## Examples - iex> enum = 1..100 + iex> enumerable = 1..100 iex> n = 3 - iex> Enum.flat_map_reduce(enum, 0, fn i, acc -> - ...> if acc < n, do: {[i], acc + 1}, else: {:halt, acc} + iex> Enum.flat_map_reduce(enumerable, 0, fn x, acc -> + ...> if acc < n, do: {[x], acc + 1}, else: {:halt, acc} ...> end) - {[1,2,3], 3} + {[1, 2, 3], 3} + + iex> Enum.flat_map_reduce(1..5, 0, fn x, acc -> {[[x]], acc + x} end) + {[[1], [2], [3], [4], [5]], 15} """ - @spec flat_map_reduce(t, acc, fun) :: {[any], any} when - fun: (element, acc -> {t, acc} | {:halt, acc}), - acc: any - def flat_map_reduce(collection, acc, fun) do + @spec flat_map_reduce(t, acc, fun) :: {[any], acc} + when fun: (element, acc -> {t, acc} | {:halt, acc}) + def flat_map_reduce(enumerable, acc, fun) do {_, {list, acc}} = - Enumerable.reduce(collection, {:cont, {[], acc}}, fn(entry, {list, acc}) -> + Enumerable.reduce(enumerable, {:cont, {[], acc}}, fn entry, {list, acc} -> case fun.(entry, acc) do {:halt, acc} -> {:halt, {list, acc}} + + {[], acc} -> + {:cont, {list, acc}} + + {[entry], acc} -> + {:cont, {[entry | list], acc}} + {entries, acc} -> - {:cont, {reduce(entries, list, &[&1|&2]), acc}} + {:cont, {reduce(entries, list, &[&1 | &2]), acc}} end end) @@ -819,9 +1301,100 @@ defmodule Enum do end @doc """ - Intersperses `element` between each element of the enumeration. + Returns a map with keys as unique elements of `enumerable` and values + as the count of every element. + + ## Examples + + iex> Enum.frequencies(~w{ant buffalo ant ant buffalo dingo}) + %{"ant" => 3, "buffalo" => 2, "dingo" => 1} + + """ + @doc since: "1.10.0" + @spec frequencies(t) :: map + def frequencies(enumerable) do + reduce(enumerable, %{}, fn key, acc -> + case acc do + %{^key => value} -> %{acc | key => value + 1} + %{} -> Map.put(acc, key, 1) + end + end) + end + + @doc """ + Returns a map with keys as unique elements given by `key_fun` and values + as the count of every element. + + ## Examples + + iex> Enum.frequencies_by(~w{aa aA bb cc}, &String.downcase/1) + %{"aa" => 2, "bb" => 1, "cc" => 1} + + iex> Enum.frequencies_by(~w{aaa aA bbb cc c}, &String.length/1) + %{3 => 2, 2 => 2, 1 => 1} + + """ + @doc since: "1.10.0" + @spec frequencies_by(t, (element -> any)) :: map + def frequencies_by(enumerable, key_fun) when is_function(key_fun) do + reduce(enumerable, %{}, fn entry, acc -> + key = key_fun.(entry) + + case acc do + %{^key => value} -> %{acc | key => value + 1} + %{} -> Map.put(acc, key, 1) + end + end) + end + + @doc """ + Splits the `enumerable` into groups based on `key_fun`. + + The result is a map where each key is given by `key_fun` + and each value is a list of elements given by `value_fun`. + The order of elements within each list is preserved from the `enumerable`. + However, like all maps, the resulting map is unordered. - Complexity: O(n) + ## Examples + + iex> Enum.group_by(~w{ant buffalo cat dingo}, &String.length/1) + %{3 => ["ant", "cat"], 5 => ["dingo"], 7 => ["buffalo"]} + + iex> Enum.group_by(~w{ant buffalo cat dingo}, &String.length/1, &String.first/1) + %{3 => ["a", "c"], 5 => ["d"], 7 => ["b"]} + + """ + @spec group_by(t, (element -> any), (element -> any)) :: map + def group_by(enumerable, key_fun, value_fun \\ fn x -> x end) + + def group_by(enumerable, key_fun, value_fun) when is_function(key_fun) do + reduce(reverse(enumerable), %{}, fn entry, acc -> + key = key_fun.(entry) + value = value_fun.(entry) + + case acc do + %{^key => existing} -> %{acc | key => [value | existing]} + %{} -> Map.put(acc, key, [value]) + end + end) + end + + def group_by(enumerable, dict, fun) do + IO.warn( + "Enum.group_by/3 with a map/dictionary as second element is deprecated. " <> + "A map is used by default and it is no longer required to pass one to this function" + ) + + # Avoid warnings about Dict + dict_module = Dict + + reduce(reverse(enumerable), dict, fn entry, categories -> + dict_module.update(categories, fun.(entry), [entry], &[entry | &1]) + end) + end + + @doc """ + Intersperses `separator` between each element of the enumeration. ## Examples @@ -836,91 +1409,190 @@ defmodule Enum do """ @spec intersperse(t, element) :: list - def intersperse(collection, element) do + def intersperse(enumerable, separator) when is_list(enumerable) do + case enumerable do + [] -> [] + list -> intersperse_non_empty_list(list, separator) + end + end + + def intersperse(enumerable, separator) do list = - reduce(collection, [], fn(x, acc) -> - [x, element | acc] - end) |> :lists.reverse() + enumerable + |> reduce([], fn x, acc -> [x, separator | acc] end) + |> :lists.reverse() + # Head is a superfluous separator case list do - [] -> [] - [_|t] -> t # Head is a superfluous intersperser element + [] -> [] + [_ | t] -> t end end @doc """ - Inserts the given enumerable into a collectable. + Inserts the given `enumerable` into a `collectable`. + + Note that passing a non-empty list as the `collectable` is deprecated. + If you're collecting into a non-empty keyword list, consider using + `Keyword.merge(collectable, Enum.to_list(enumerable))`. If you're collecting + into a non-empty list, consider something like `Enum.to_list(enumerable) ++ collectable`. ## Examples - iex> Enum.into([1, 2], [0]) - [0, 1, 2] + iex> Enum.into([1, 2], []) + [1, 2] iex> Enum.into([a: 1, b: 2], %{}) %{a: 1, b: 2} + iex> Enum.into(%{a: 1}, %{b: 2}) + %{a: 1, b: 2} + + iex> Enum.into([a: 1, a: 2], %{}) + %{a: 2} + """ - @spec into(Enumerable.t, Collectable.t) :: Collectable.t - def into(collection, list) when is_list(list) do - list ++ to_list(collection) + @spec into(Enumerable.t(), Collectable.t()) :: Collectable.t() + def into(enumerable, collectable) + + def into(enumerable, []) do + to_list(enumerable) + end + + def into(%_{} = enumerable, collectable) do + into_protocol(enumerable, collectable) + end + + def into(enumerable, %_{} = collectable) do + into_protocol(enumerable, collectable) + end + + def into(enumerable, %{} = collectable) do + if map_size(collectable) == 0 do + into_map(enumerable) + else + into_map(enumerable, collectable) + end end - def into(collection, %{} = map) when is_list(collection) and map_size(map) == 0 do - :maps.from_list(collection) + def into(enumerable, collectable) do + into_protocol(enumerable, collectable) end - def into(collection, collectable) do + defp into_map(%{} = enumerable), do: enumerable + defp into_map(enumerable) when is_list(enumerable), do: :maps.from_list(enumerable) + defp into_map(enumerable), do: enumerable |> Enum.to_list() |> :maps.from_list() + + defp into_map(%{} = enumerable, collectable), + do: Map.merge(collectable, enumerable) + + defp into_map(enumerable, collectable) when is_list(enumerable), + do: Map.merge(collectable, :maps.from_list(enumerable)) + + defp into_map(enumerable, collectable), + do: Enum.reduce(enumerable, collectable, fn {key, val}, acc -> Map.put(acc, key, val) end) + + defp into_protocol(enumerable, collectable) do {initial, fun} = Collectable.into(collectable) - into(collection, initial, fun, fn x, acc -> - fun.(acc, {:cont, x}) + + try do + reduce_into_protocol(enumerable, initial, fun) + catch + kind, reason -> + fun.(initial, :halt) + :erlang.raise(kind, reason, __STACKTRACE__) + else + acc -> fun.(acc, :done) + end + end + + defp reduce_into_protocol(enumerable, initial, fun) when is_list(enumerable) do + :lists.foldl(fn x, acc -> fun.(acc, {:cont, x}) end, initial, enumerable) + end + + defp reduce_into_protocol(enumerable, initial, fun) do + enumerable + |> Enumerable.reduce({:cont, initial}, fn x, acc -> + {:cont, fun.(acc, {:cont, x})} end) + |> elem(1) end @doc """ - Inserts the given enumerable into a collectable - according to the transformation function. + Inserts the given `enumerable` into a `collectable` according to the + transformation function. ## Examples - iex> Enum.into([2, 3], [3], fn x -> x * 3 end) + iex> Enum.into([1, 2, 3], [], fn x -> x * 3 end) [3, 6, 9] + iex> Enum.into(%{a: 1, b: 2}, %{c: 3}, fn {k, v} -> {k, v * 2} end) + %{a: 2, b: 4, c: 3} + """ - @spec into(Enumerable.t, Collectable.t, (term -> term)) :: Collectable.t + @spec into(Enumerable.t(), Collectable.t(), (term -> term)) :: Collectable.t() + def into(enumerable, [], transform) do + Enum.map(enumerable, transform) + end - def into(collection, list, transform) when is_list(list) and is_function(transform, 1) do - list ++ map(collection, transform) + def into(%_{} = enumerable, collectable, transform) do + into_protocol(enumerable, collectable, transform) end - def into(collection, collectable, transform) when is_function(transform, 1) do - {initial, fun} = Collectable.into(collectable) - into(collection, initial, fun, fn x, acc -> - fun.(acc, {:cont, transform.(x)}) - end) + def into(enumerable, %_{} = collectable, transform) do + into_protocol(enumerable, collectable, transform) + end + + def into(enumerable, %{} = collectable, transform) do + if map_size(collectable) == 0 do + enumerable |> Enum.map(transform) |> :maps.from_list() + else + Enum.reduce(enumerable, collectable, fn entry, acc -> + {key, val} = transform.(entry) + Map.put(acc, key, val) + end) + end end - defp into(collection, initial, fun, callback) do + def into(enumerable, collectable, transform) do + into_protocol(enumerable, collectable, transform) + end + + defp into_protocol(enumerable, collectable, transform) do + {initial, fun} = Collectable.into(collectable) + try do - reduce(collection, initial, callback) + reduce_into_protocol(enumerable, initial, transform, fun) catch kind, reason -> - stacktrace = System.stacktrace fun.(initial, :halt) - :erlang.raise(kind, reason, stacktrace) + :erlang.raise(kind, reason, __STACKTRACE__) else acc -> fun.(acc, :done) end end + defp reduce_into_protocol(enumerable, initial, transform, fun) when is_list(enumerable) do + :lists.foldl(fn x, acc -> fun.(acc, {:cont, transform.(x)}) end, initial, enumerable) + end + + defp reduce_into_protocol(enumerable, initial, transform, fun) do + enumerable + |> Enumerable.reduce({:cont, initial}, fn x, acc -> + {:cont, fun.(acc, {:cont, transform.(x)})} + end) + |> elem(1) + end + @doc """ - Joins the given `collection` according to `joiner`. - `joiner` can be either a binary or a list and the - result will be of the same type as `joiner`. If - `joiner` is not passed at all, it defaults to an - empty binary. + Joins the given `enumerable` into a string using `joiner` as a + separator. + + If `joiner` is not passed at all, it defaults to an empty string. - All items in the collection must be convertible - to a binary, otherwise an error is raised. + All elements in the `enumerable` must be convertible to a string, + otherwise an error is raised. ## Examples @@ -931,346 +1603,804 @@ defmodule Enum do "1 = 2 = 3" """ - @spec join(t) :: String.t - @spec join(t, String.t) :: String.t - def join(collection, joiner \\ "") + @spec join(t, String.t()) :: String.t() + def join(enumerable, joiner \\ "") + + def join(enumerable, "") do + enumerable + |> map(&entry_to_string(&1)) + |> IO.iodata_to_binary() + end + + def join(enumerable, joiner) when is_list(enumerable) and is_binary(joiner) do + join_list(enumerable, joiner) + end + + def join(enumerable, joiner) when is_binary(joiner) do + reduced = + reduce(enumerable, :first, fn + entry, :first -> entry_to_string(entry) + entry, acc -> [acc, joiner | entry_to_string(entry)] + end) - def join(collection, joiner) when is_binary(joiner) do - reduced = reduce(collection, :first, fn - entry, :first -> enum_to_string(entry) - entry, acc -> [acc, joiner|enum_to_string(entry)] - end) if reduced == :first do "" else - IO.iodata_to_binary reduced + IO.iodata_to_binary(reduced) end end @doc """ - Returns a new collection, where each item is the result - of invoking `fun` on each corresponding item of `collection`. + Returns a list where each element is the result of invoking + `fun` on each corresponding element of `enumerable`. - For dicts, the function expects a key-value tuple. + For maps, the function expects a key-value tuple. ## Examples - iex> Enum.map([1, 2, 3], fn(x) -> x * 2 end) + iex> Enum.map([1, 2, 3], fn x -> x * 2 end) [2, 4, 6] - iex> Enum.map([a: 1, b: 2], fn({k, v}) -> {k, -v} end) + iex> Enum.map([a: 1, b: 2], fn {k, v} -> {k, -v} end) [a: -1, b: -2] """ @spec map(t, (element -> any)) :: list - def map(collection, fun) when is_list(collection) do - for item <- collection, do: fun.(item) + def map(enumerable, fun) + + def map(enumerable, fun) when is_list(enumerable) do + :lists.map(fun, enumerable) end - def map(collection, fun) do - Enumerable.reduce(collection, {:cont, []}, R.map(fun)) |> elem(1) |> :lists.reverse + def map(enumerable, fun) do + reduce(enumerable, [], R.map(fun)) |> :lists.reverse() end @doc """ - Maps and joins the given `collection` in one pass. - `joiner` can be either a binary or a list and the - result will be of the same type as `joiner`. If - `joiner` is not passed at all, it defaults to an - empty binary. + Returns a list of results of invoking `fun` on every `nth` + element of `enumerable`, starting with the first element. + + The first element is always passed to the given function, unless `nth` is `0`. + + The second argument specifying every `nth` element must be a non-negative + integer. - All items in the collection must be convertible - to a binary, otherwise an error is raised. + If `nth` is `0`, then `enumerable` is directly converted to a list, + without `fun` being ever applied. ## Examples - iex> Enum.map_join([1, 2, 3], &(&1 * 2)) - "246" + iex> Enum.map_every(1..10, 2, fn x -> x + 1000 end) + [1001, 2, 1003, 4, 1005, 6, 1007, 8, 1009, 10] - iex> Enum.map_join([1, 2, 3], " = ", &(&1 * 2)) - "2 = 4 = 6" + iex> Enum.map_every(1..10, 3, fn x -> x + 1000 end) + [1001, 2, 3, 1004, 5, 6, 1007, 8, 9, 1010] + + iex> Enum.map_every(1..5, 0, fn x -> x + 1000 end) + [1, 2, 3, 4, 5] + + iex> Enum.map_every([1, 2, 3], 1, fn x -> x + 1000 end) + [1001, 1002, 1003] """ - @spec map_join(t, (element -> any)) :: String.t - @spec map_join(t, String.t, (element -> any)) :: String.t - def map_join(collection, joiner \\ "", mapper) + @doc since: "1.4.0" + @spec map_every(t, non_neg_integer, (element -> any)) :: list + def map_every(enumerable, nth, fun) - def map_join(collection, joiner, mapper) when is_binary(joiner) do - reduced = reduce(collection, :first, fn - entry, :first -> enum_to_string(mapper.(entry)) - entry, acc -> [acc, joiner|enum_to_string(mapper.(entry))] - end) + def map_every(enumerable, 1, fun), do: map(enumerable, fun) + def map_every(enumerable, 0, _fun), do: to_list(enumerable) + def map_every([], nth, _fun) when is_integer(nth) and nth > 1, do: [] + + def map_every(enumerable, nth, fun) when is_integer(nth) and nth > 1 do + {res, _} = reduce(enumerable, {[], :first}, R.map_every(nth, fun)) + :lists.reverse(res) + end + + @doc """ + Maps and intersperses the given enumerable in one pass. + + ## Examples + + iex> Enum.map_intersperse([1, 2, 3], :a, &(&1 * 2)) + [2, :a, 4, :a, 6] + """ + @doc since: "1.10.0" + @spec map_intersperse(t, element(), (element -> any())) :: list() + def map_intersperse(enumerable, separator, mapper) + + def map_intersperse(enumerable, separator, mapper) when is_list(enumerable) do + map_intersperse_list(enumerable, separator, mapper) + end + + def map_intersperse(enumerable, separator, mapper) do + reduced = + reduce(enumerable, :first, fn + entry, :first -> [mapper.(entry)] + entry, acc -> [mapper.(entry), separator | acc] + end) if reduced == :first do - "" + [] else - IO.iodata_to_binary reduced + :lists.reverse(reduced) end end @doc """ - Invokes the given `fun` for each item in the `collection` - while also keeping an accumulator. Returns a tuple where - the first element is the mapped collection and the second - one is the final accumulator. + Maps and joins the given `enumerable` in one pass. + + If `joiner` is not passed at all, it defaults to an empty string. + + All elements returned from invoking the `mapper` must be convertible to + a string, otherwise an error is raised. + + ## Examples + + iex> Enum.map_join([1, 2, 3], &(&1 * 2)) + "246" + + iex> Enum.map_join([1, 2, 3], " = ", &(&1 * 2)) + "2 = 4 = 6" + + """ + @spec map_join(t, String.t(), (element -> String.Chars.t())) :: String.t() + def map_join(enumerable, joiner \\ "", mapper) when is_binary(joiner) do + enumerable + |> map_intersperse(joiner, &entry_to_string(mapper.(&1))) + |> IO.iodata_to_binary() + end + + @doc """ + Invokes the given function to each element in the `enumerable` to reduce + it to a single element, while keeping an accumulator. + + Returns a tuple where the first element is the mapped enumerable and + the second one is the final accumulator. - For dicts, the first tuple element must be a `{key, value}` - tuple. + The function, `fun`, receives two arguments: the first one is the + element, and the second one is the accumulator. `fun` must return + a tuple with two elements in the form of `{result, accumulator}`. + + For maps, the first tuple element must be a `{key, value}` tuple. ## Examples - iex> Enum.map_reduce([1, 2, 3], 0, fn(x, acc) -> {x * 2, x + acc} end) + iex> Enum.map_reduce([1, 2, 3], 0, fn x, acc -> {x * 2, x + acc} end) {[2, 4, 6], 6} """ - @spec map_reduce(t, any, (element, any -> any)) :: any - def map_reduce(collection, acc, fun) when is_list(collection) do - :lists.mapfoldl(fun, acc, collection) + @spec map_reduce(t, acc, (element, acc -> {element, acc})) :: {list, acc} + def map_reduce(enumerable, acc, fun) when is_list(enumerable) do + :lists.mapfoldl(fun, acc, enumerable) end - def map_reduce(collection, acc, fun) do - {list, acc} = reduce(collection, {[], acc}, fn(entry, {list, acc}) -> - {new_entry, acc} = fun.(entry, acc) - {[new_entry|list], acc} - end) + def map_reduce(enumerable, acc, fun) do + {list, acc} = + reduce(enumerable, {[], acc}, fn entry, {list, acc} -> + {new_entry, acc} = fun.(entry, acc) + {[new_entry | list], acc} + end) + {:lists.reverse(list), acc} end + @doc false + def max(list = [_ | _]), do: :lists.max(list) + + @doc false + def max(list = [_ | _], empty_fallback) when is_function(empty_fallback, 0) do + :lists.max(list) + end + + @doc false + @spec max(t, (() -> empty_result)) :: element | empty_result when empty_result: any + def max(enumerable, empty_fallback) when is_function(empty_fallback, 0) do + max(enumerable, &>=/2, empty_fallback) + end + @doc """ - Returns the maximum value. - Raises `EmptyError` if the collection is empty. + Returns the maximal element in the `enumerable` according + to Erlang's term ordering. + + By default, the comparison is done with the `>=` sorter function. + If multiple elements are considered maximal, the first one that + was found is returned. If you want the last element considered + maximal to be returned, the sorter function should not return true + for equal elements. + + If the enumerable is empty, the provided `empty_fallback` is called. + The default `empty_fallback` raises `Enum.EmptyError`. ## Examples iex> Enum.max([1, 2, 3]) 3 + The fact this function uses Erlang's term ordering means that the comparison + is structural and not semantic. For example: + + iex> Enum.max([~D[2017-03-31], ~D[2017-04-01]]) + ~D[2017-03-31] + + In the example above, `max/2` returned March 31st instead of April 1st + because the structural comparison compares the day before the year. + For this reason, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: + + iex> Enum.max([~D[2017-03-31], ~D[2017-04-01]], Date) + ~D[2017-04-01] + + Finally, if you don't want to raise on empty enumerables, you can pass + the empty fallback: + + iex> Enum.max([], &>=/2, fn -> 0 end) + 0 + """ - @spec max(t) :: element | no_return - def max(collection) do - reduce(collection, &Kernel.max(&1, &2)) + @spec max(t, (element, element -> boolean) | module()) :: + element | empty_result + when empty_result: any + @spec max(t, (element, element -> boolean) | module(), (() -> empty_result)) :: + element | empty_result + when empty_result: any + def max(enumerable, sorter \\ &>=/2, empty_fallback \\ fn -> raise Enum.EmptyError end) do + aggregate(enumerable, max_sort_fun(sorter), empty_fallback) + end + + defp max_sort_fun(sorter) when is_function(sorter, 2), do: sorter + defp max_sort_fun(module) when is_atom(module), do: &(module.compare(&1, &2) != :lt) + + @doc false + @spec max_by( + t, + (element -> any), + (() -> empty_result) | (element, element -> boolean) | module() + ) :: element | empty_result + when empty_result: any + def max_by(enumerable, fun, empty_fallback) + when is_function(fun, 1) and is_function(empty_fallback, 0) do + max_by(enumerable, fun, &>=/2, empty_fallback) end @doc """ - Returns the maximum value as calculated by the given function. - Raises `EmptyError` if the collection is empty. + Returns the maximal element in the `enumerable` as calculated + by the given `fun`. + + By default, the comparison is done with the `>=` sorter function. + If multiple elements are considered maximal, the first one that + was found is returned. If you want the last element considered + maximal to be returned, the sorter function should not return true + for equal elements. + + Calls the provided `empty_fallback` function and returns its value if + `enumerable` is empty. The default `empty_fallback` raises `Enum.EmptyError`. ## Examples - iex> Enum.max_by(["a", "aa", "aaa"], fn(x) -> String.length(x) end) + iex> Enum.max_by(["a", "aa", "aaa"], fn x -> String.length(x) end) "aaa" - """ - @spec max_by(t, (element -> any)) :: element | no_return - def max_by([h|t], fun) do - reduce(t, {h, fun.(h)}, fn(entry, {_, fun_max} = old) -> - fun_entry = fun.(entry) - if(fun_entry > fun_max, do: {entry, fun_entry}, else: old) - end) |> elem(0) - end - - def max_by([], _fun) do - raise Enum.EmptyError - end + iex> Enum.max_by(["a", "aa", "aaa", "b", "bbb"], &String.length/1) + "aaa" - def max_by(collection, fun) do - result = - reduce(collection, :first, fn - entry, {_, fun_max} = old -> - fun_entry = fun.(entry) - if(fun_entry > fun_max, do: {entry, fun_entry}, else: old) - entry, :first -> - {entry, fun.(entry)} - end) + The fact this function uses Erlang's term ordering means that the + comparison is structural and not semantic. Therefore, if you want + to compare structs, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: + + iex> users = [ + ...> %{name: "Ellis", birthday: ~D[1943-05-11]}, + ...> %{name: "Lovelace", birthday: ~D[1815-12-10]}, + ...> %{name: "Turing", birthday: ~D[1912-06-23]} + ...> ] + iex> Enum.max_by(users, &(&1.birthday), Date) + %{name: "Ellis", birthday: ~D[1943-05-11]} + + Finally, if you don't want to raise on empty enumerables, you can pass + the empty fallback: + + iex> Enum.max_by([], &String.length/1, fn -> nil end) + nil - case result do - :first -> raise Enum.EmptyError - {entry, _} -> entry - end + """ + @spec max_by( + t, + (element -> any), + (element, element -> boolean) | module(), + (() -> empty_result) + ) :: element | empty_result + when empty_result: any + def max_by(enumerable, fun, sorter \\ &>=/2, empty_fallback \\ fn -> raise Enum.EmptyError end) + when is_function(fun, 1) do + aggregate_by(enumerable, fun, max_sort_fun(sorter), empty_fallback) end @doc """ - Checks if `value` exists within the `collection`. + Checks if `element` exists within the `enumerable`. - Membership is tested with the match (`===`) operator, although - enumerables like ranges may include floats inside the given - range. + Membership is tested with the match (`===/2`) operator. ## Examples iex> Enum.member?(1..10, 5) true + iex> Enum.member?(1..10, 5.0) + false + + iex> Enum.member?([1.0, 2.0, 3.0], 2) + false + iex> Enum.member?([1.0, 2.0, 3.0], 2.000) + true iex> Enum.member?([:a, :b, :c], :d) false + + When called outside guards, the [`in`](`in/2`) and [`not in`](`in/2`) + operators work by using this function. """ @spec member?(t, element) :: boolean - def member?(collection, value) when is_list(collection) do - :lists.member(value, collection) + def member?(enumerable, element) when is_list(enumerable) do + :lists.member(element, enumerable) end - def member?(collection, value) do - case Enumerable.member?(collection, value) do - {:ok, value} when is_boolean(value) -> - value + def member?(enumerable, element) do + case Enumerable.member?(enumerable, element) do + {:ok, element} when is_boolean(element) -> + element + {:error, module} -> - module.reduce(collection, {:cont, false}, fn - v, _ when v === value -> {:halt, true} - _, _ -> {:cont, false} - end) |> elem(1) + module.reduce(enumerable, {:cont, false}, fn + v, _ when v === element -> {:halt, true} + _, _ -> {:cont, false} + end) + |> elem(1) end end + @doc false + def min(list = [_ | _]), do: :lists.min(list) + + @doc false + def min(list = [_ | _], empty_fallback) when is_function(empty_fallback, 0) do + :lists.min(list) + end + + @doc false + @spec min(t, (() -> empty_result)) :: element | empty_result when empty_result: any + def min(enumerable, empty_fallback) when is_function(empty_fallback, 0) do + min(enumerable, &<=/2, empty_fallback) + end + @doc """ - Returns the minimum value. - Raises `EmptyError` if the collection is empty. + Returns the minimal element in the `enumerable` according + to Erlang's term ordering. + + By default, the comparison is done with the `<=` sorter function. + If multiple elements are considered minimal, the first one that + was found is returned. If you want the last element considered + minimal to be returned, the sorter function should not return true + for equal elements. + + If the enumerable is empty, the provided `empty_fallback` is called. + The default `empty_fallback` raises `Enum.EmptyError`. ## Examples iex> Enum.min([1, 2, 3]) 1 - """ - @spec min(t) :: element | no_return - def min(collection) do - reduce(collection, &Kernel.min(&1, &2)) - end - - @doc """ - Returns the minimum value as calculated by the given function. - Raises `EmptyError` if the collection is empty. + The fact this function uses Erlang's term ordering means that the comparison + is structural and not semantic. For example: - ## Examples + iex> Enum.min([~D[2017-03-31], ~D[2017-04-01]]) + ~D[2017-04-01] - iex> Enum.min_by(["a", "aa", "aaa"], fn(x) -> String.length(x) end) - "a" + In the example above, `min/2` returned April 1st instead of March 31st + because the structural comparison compares the day before the year. + For this reason, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: - """ - @spec min_by(t, (element -> any)) :: element | no_return - def min_by([h|t], fun) do - reduce(t, {h, fun.(h)}, fn(entry, {_, fun_min} = old) -> - fun_entry = fun.(entry) - if(fun_entry < fun_min, do: {entry, fun_entry}, else: old) - end) |> elem(0) - end + iex> Enum.min([~D[2017-03-31], ~D[2017-04-01]], Date) + ~D[2017-03-31] - def min_by([], _fun) do - raise Enum.EmptyError - end + Finally, if you don't want to raise on empty enumerables, you can pass + the empty fallback: - def min_by(collection, fun) do - result = - reduce(collection, :first, fn - entry, {_, fun_min} = old -> - fun_entry = fun.(entry) - if(fun_entry < fun_min, do: {entry, fun_entry}, else: old) - entry, :first -> - {entry, fun.(entry)} - end) + iex> Enum.min([], fn -> 0 end) + 0 - case result do - :first -> raise Enum.EmptyError - {entry, _} -> entry - end + """ + @spec min(t, (element, element -> boolean) | module()) :: + element | empty_result + when empty_result: any + @spec min(t, (element, element -> boolean) | module(), (() -> empty_result)) :: + element | empty_result + when empty_result: any + def min(enumerable, sorter \\ &<=/2, empty_fallback \\ fn -> raise Enum.EmptyError end) do + aggregate(enumerable, min_sort_fun(sorter), empty_fallback) + end + + defp min_sort_fun(sorter) when is_function(sorter, 2), do: sorter + defp min_sort_fun(module) when is_atom(module), do: &(module.compare(&1, &2) != :gt) + + @doc false + @spec min_by( + t, + (element -> any), + (() -> empty_result) | (element, element -> boolean) | module() + ) :: element | empty_result + when empty_result: any + def min_by(enumerable, fun, empty_fallback) + when is_function(fun, 1) and is_function(empty_fallback, 0) do + min_by(enumerable, fun, &<=/2, empty_fallback) end @doc """ - Returns the sum of all values. + Returns the minimal element in the `enumerable` as calculated + by the given `fun`. - Raises `ArithmeticError` if collection contains a non-numeric value. + By default, the comparison is done with the `<=` sorter function. + If multiple elements are considered minimal, the first one that + was found is returned. If you want the last element considered + minimal to be returned, the sorter function should not return true + for equal elements. + + Calls the provided `empty_fallback` function and returns its value if + `enumerable` is empty. The default `empty_fallback` raises `Enum.EmptyError`. ## Examples - iex> Enum.sum([1, 2, 3]) - 6 + iex> Enum.min_by(["a", "aa", "aaa"], fn x -> String.length(x) end) + "a" + + iex> Enum.min_by(["a", "aa", "aaa", "b", "bbb"], &String.length/1) + "a" + + The fact this function uses Erlang's term ordering means that the + comparison is structural and not semantic. Therefore, if you want + to compare structs, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: + + iex> users = [ + ...> %{name: "Ellis", birthday: ~D[1943-05-11]}, + ...> %{name: "Lovelace", birthday: ~D[1815-12-10]}, + ...> %{name: "Turing", birthday: ~D[1912-06-23]} + ...> ] + iex> Enum.min_by(users, &(&1.birthday), Date) + %{name: "Lovelace", birthday: ~D[1815-12-10]} + + Finally, if you don't want to raise on empty enumerables, you can pass + the empty fallback: + + iex> Enum.min_by([], &String.length/1, fn -> nil end) + nil """ - @spec sum(t) :: number - def sum(collection) do - reduce(collection, 0, &+/2) + @spec min_by( + t, + (element -> any), + (element, element -> boolean) | module(), + (() -> empty_result) + ) :: element | empty_result + when empty_result: any + def min_by(enumerable, fun, sorter \\ &<=/2, empty_fallback \\ fn -> raise Enum.EmptyError end) + when is_function(fun, 1) do + aggregate_by(enumerable, fun, min_sort_fun(sorter), empty_fallback) end @doc """ - Partitions `collection` into two collections, where the first one contains elements - for which `fun` returns a truthy value, and the second one -- for which `fun` - returns `false` or `nil`. + Returns a tuple with the minimal and the maximal elements in the + enumerable according to Erlang's term ordering. - ## Examples + If multiple elements are considered maximal or minimal, the first one + that was found is returned. - iex> Enum.partition([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) - {[2], [1,3]} + Calls the provided `empty_fallback` function and returns its value if + `enumerable` is empty. The default `empty_fallback` raises `Enum.EmptyError`. + + ## Examples + + iex> Enum.min_max([2, 3, 1]) + {1, 3} + + iex> Enum.min_max([], fn -> {nil, nil} end) + {nil, nil} + + """ + @spec min_max(t, (() -> empty_result)) :: {element, element} | empty_result + when empty_result: any + def min_max(enumerable, empty_fallback \\ fn -> raise Enum.EmptyError end) + + def min_max(first..last//step = range, empty_fallback) when is_function(empty_fallback, 0) do + case Range.size(range) do + 0 -> + empty_fallback.() + + _ -> + last = last - rem(last - first, step) + {Kernel.min(first, last), Kernel.max(first, last)} + end + end + + def min_max(enumerable, empty_fallback) when is_function(empty_fallback, 0) do + first_fun = &[&1 | &1] + + reduce_fun = fn entry, [min | max] -> + [Kernel.min(min, entry) | Kernel.max(max, entry)] + end + + case reduce_by(enumerable, first_fun, reduce_fun) do + :empty -> empty_fallback.() + [min | max] -> {min, max} + end + end + + @doc false + @spec min_max_by(t, (element -> any), (() -> empty_result)) :: {element, element} | empty_result + when empty_result: any + def min_max_by(enumerable, fun, empty_fallback) + when is_function(fun, 1) and is_function(empty_fallback, 0) do + min_max_by(enumerable, fun, & Enum.min_max_by(["aaa", "bb", "c"], fn x -> String.length(x) end) + {"c", "aaa"} + + iex> Enum.min_max_by(["aaa", "a", "bb", "c", "ccc"], &String.length/1) + {"a", "aaa"} + + iex> Enum.min_max_by([], &String.length/1, fn -> {nil, nil} end) + {nil, nil} + + The fact this function uses Erlang's term ordering means that the + comparison is structural and not semantic. Therefore, if you want + to compare structs, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: + + iex> users = [ + ...> %{name: "Ellis", birthday: ~D[1943-05-11]}, + ...> %{name: "Lovelace", birthday: ~D[1815-12-10]}, + ...> %{name: "Turing", birthday: ~D[1912-06-23]} + ...> ] + iex> Enum.min_max_by(users, &(&1.birthday), Date) + { + %{name: "Lovelace", birthday: ~D[1815-12-10]}, + %{name: "Ellis", birthday: ~D[1943-05-11]} + } + + Finally, if you don't want to raise on empty enumerables, you can pass + the empty fallback: + + iex> Enum.min_max_by([], &String.length/1, fn -> nil end) + nil + + """ + @spec min_max_by(t, (element -> any), (element, element -> boolean) | module()) :: + {element, element} | empty_result + when empty_result: any + @spec min_max_by( + t, + (element -> any), + (element, element -> boolean) | module(), + (() -> empty_result) + ) :: {element, element} | empty_result + when empty_result: any + def min_max_by( + enumerable, + fun, + sorter_or_empty_fallback \\ & raise Enum.EmptyError end + ) + + def min_max_by(enumerable, fun, sorter, empty_fallback) + when is_function(fun, 1) and is_atom(sorter) and is_function(empty_fallback, 0) do + min_max_by(enumerable, fun, min_max_by_sort_fun(sorter), empty_fallback) + end + + def min_max_by(enumerable, fun, sorter, empty_fallback) + when is_function(fun, 1) and is_function(sorter, 2) and is_function(empty_fallback, 0) do + first_fun = fn entry -> + fun_entry = fun.(entry) + {entry, entry, fun_entry, fun_entry} + end + + reduce_fun = fn entry, {prev_min, prev_max, fun_min, fun_max} = acc -> + fun_entry = fun.(entry) + + cond do + sorter.(fun_entry, fun_min) -> + {entry, prev_max, fun_entry, fun_max} + + sorter.(fun_max, fun_entry) -> + {prev_min, entry, fun_min, fun_entry} + + true -> + acc + end + end + + case reduce_by(enumerable, first_fun, reduce_fun) do + :empty -> empty_fallback.() + {min, max, _, _} -> {min, max} + end + end + + defp min_max_by_sort_fun(module) when is_atom(module), do: &(module.compare(&1, &2) == :lt) + + @doc """ + Splits the `enumerable` in two lists according to the given function `fun`. + + Splits the given `enumerable` in two lists by calling `fun` with each element + in the `enumerable` as its only argument. Returns a tuple with the first list + containing all the elements in `enumerable` for which applying `fun` returned + a truthy value, and a second list with all the elements for which applying + `fun` returned a falsy value (`false` or `nil`). + + The elements in both the returned lists are in the same relative order as they + were in the original enumerable (if such enumerable was ordered, like a + list). See the examples below. + + ## Examples + + iex> Enum.split_with([5, 4, 3, 2, 1, 0], fn x -> rem(x, 2) == 0 end) + {[4, 2, 0], [5, 3, 1]} + + iex> Enum.split_with(%{a: 1, b: -2, c: 1, d: -3}, fn {_k, v} -> v < 0 end) + {[b: -2, d: -3], [a: 1, c: 1]} + + iex> Enum.split_with(%{a: 1, b: -2, c: 1, d: -3}, fn {_k, v} -> v > 50 end) + {[], [a: 1, b: -2, c: 1, d: -3]} + + iex> Enum.split_with(%{}, fn {_k, v} -> v > 50 end) + {[], []} """ - @spec partition(t, (element -> any)) :: {list, list} - def partition(collection, fun) do + @doc since: "1.4.0" + @spec split_with(t, (element -> as_boolean(term))) :: {list, list} + def split_with(enumerable, fun) do {acc1, acc2} = - reduce(collection, {[], []}, fn(entry, {acc1, acc2}) -> + reduce(enumerable, {[], []}, fn entry, {acc1, acc2} -> if fun.(entry) do - {[entry|acc1], acc2} + {[entry | acc1], acc2} else - {acc1, [entry|acc2]} + {acc1, [entry | acc2]} end end) {:lists.reverse(acc1), :lists.reverse(acc2)} end + @doc false + @deprecated "Use Enum.split_with/2 instead" + def partition(enumerable, fun) do + split_with(enumerable, fun) + end + @doc """ - Splits `collection` into groups based on `fun`. + Returns a random element of an `enumerable`. + + Raises `Enum.EmptyError` if `enumerable` is empty. + + This function uses Erlang's [`:rand` module](`:rand`) to calculate + the random value. Check its documentation for setting a + different random algorithm or a different seed. - The result is a dict (by default a map) where each key is - a group and each value is a list of elements from `collection` - for which `fun` returned that group. Ordering is not necessarily - preserved. + The implementation is based on the + [reservoir sampling](https://en.wikipedia.org/wiki/Reservoir_sampling#Relation_to_Fisher-Yates_shuffle) + algorithm. + It assumes that the sample being returned can fit into memory; + the input `enumerable` doesn't have to, as it is traversed just once. + + If a range is passed into the function, this function will pick a + random value between the range limits, without traversing the whole + range (thus executing in constant time and constant memory). ## Examples - iex> Enum.group_by(~w{ant buffalo cat dingo}, &String.length/1) - %{3 => ["cat", "ant"], 7 => ["buffalo"], 5 => ["dingo"]} + The examples below use the `:exsss` pseudorandom algorithm since it's + the default from Erlang/OTP 22: + + # Although not necessary, let's seed the random algorithm + iex> :rand.seed(:exsss, {100, 101, 102}) + iex> Enum.random([1, 2, 3]) + 2 + iex> Enum.random([1, 2, 3]) + 1 + iex> Enum.random(1..1_000) + 309 """ - @spec group_by(t, dict, (element -> any)) :: dict when dict: Dict.t - def group_by(collection, dict \\ %{}, fun) do - reduce(collection, dict, fn(entry, categories) -> - Dict.update(categories, fun.(entry), [entry], &[entry|&1]) - end) + @spec random(t) :: element + def random(enumerable) + + def random(enumerable) when is_list(enumerable) do + case length(enumerable) do + 0 -> raise Enum.EmptyError + length -> enumerable |> drop_list(random_integer(0, length - 1)) |> hd() + end end - @doc """ - Invokes `fun` for each element in the collection passing that element and the - accumulator `acc` as arguments. `fun`'s return value is stored in `acc`. - Returns the accumulator. + def random(enumerable) do + result = + case Enumerable.slice(enumerable) do + {:ok, 0, _} -> + [] - ## Examples + {:ok, count, fun} when is_function(fun, 1) -> + slice_list(fun.(enumerable), random_integer(0, count - 1), 1, 1) - iex> Enum.reduce([1, 2, 3], 0, fn(x, acc) -> x + acc end) - 6 + # TODO: Deprecate me in Elixir v1.18. + {:ok, count, fun} when is_function(fun, 2) -> + fun.(random_integer(0, count - 1), 1) - """ - @spec reduce(t, any, (element, any -> any)) :: any - def reduce(collection, acc, fun) when is_list(collection) do - :lists.foldl(fun, acc, collection) - end + {:ok, count, fun} when is_function(fun, 3) -> + fun.(random_integer(0, count - 1), 1, 1) + + {:error, _} -> + take_random(enumerable, 1) + end - def reduce(collection, acc, fun) do - Enumerable.reduce(collection, {:cont, acc}, - fn x, acc -> {:cont, fun.(x, acc)} end) |> elem(1) + case result do + [] -> raise Enum.EmptyError + [elem] -> elem + end end @doc """ - Invokes `fun` for each element in the collection passing that element and the - accumulator `acc` as arguments. `fun`'s return value is stored in `acc`. - The first element of the collection is used as the initial value of `acc`. - Returns the accumulator. + Invokes `fun` for each element in the `enumerable` with the + accumulator. + + Raises `Enum.EmptyError` if `enumerable` is empty. + + The first element of the `enumerable` is used as the initial value + of the accumulator. Then, the function is invoked with the next + element and the accumulator. The result returned by the function + is used as the accumulator for the next iteration, recursively. + When the `enumerable` is done, the last accumulator is returned. + + Since the first element of the enumerable is used as the initial + value of the accumulator, `fun` will only be executed `n - 1` times + where `n` is the length of the enumerable. This function won't call + the specified function for enumerables that are one-element long. + + If you wish to use another value for the accumulator, use + `Enum.reduce/3`. ## Examples - iex> Enum.reduce([1, 2, 3, 4], fn(x, acc) -> x * acc end) + iex> Enum.reduce([1, 2, 3, 4], fn x, acc -> x * acc end) 24 """ - @spec reduce(t, (element, any -> any)) :: any - def reduce([h|t], fun) do + @spec reduce(t, (element, acc -> acc)) :: acc + def reduce(enumerable, fun) + + def reduce([h | t], fun) do reduce(t, h, fun) end @@ -1278,41 +2408,129 @@ defmodule Enum do raise Enum.EmptyError end - def reduce(collection, fun) do - result = - Enumerable.reduce(collection, {:cont, :first}, fn - x, :first -> - {:cont, {:acc, x}} - x, {:acc, acc} -> - {:cont, {:acc, fun.(x, acc)}} - end) |> elem(1) - - case result do - :first -> raise Enum.EmptyError + def reduce(enumerable, fun) do + Enumerable.reduce(enumerable, {:cont, :first}, fn + x, {:acc, acc} -> {:cont, {:acc, fun.(x, acc)}} + x, :first -> {:cont, {:acc, x}} + end) + |> elem(1) + |> case do + :first -> raise Enum.EmptyError {:acc, acc} -> acc end end @doc """ - Returns elements of collection for which `fun` returns `false`. + Invokes `fun` for each element in the `enumerable` with the accumulator. + + The initial value of the accumulator is `acc`. The function is invoked for + each element in the enumerable with the accumulator. The result returned + by the function is used as the accumulator for the next iteration. + The function returns the last accumulator. ## Examples - iex> Enum.reject([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) + iex> Enum.reduce([1, 2, 3], 0, fn x, acc -> x + acc end) + 6 + + ## Reduce as a building block + + Reduce (sometimes called `fold`) is a basic building block in functional + programming. Almost all of the functions in the `Enum` module can be + implemented on top of reduce. Those functions often rely on other operations, + such as `Enum.reverse/1`, which are optimized by the runtime. + + For example, we could implement `map/2` in terms of `reduce/3` as follows: + + def my_map(enumerable, fun) do + enumerable + |> Enum.reduce([], fn x, acc -> [fun.(x) | acc] end) + |> Enum.reverse() + end + + In the example above, `Enum.reduce/3` accumulates the result of each call + to `fun` into a list in reverse order, which is correctly ordered at the + end by calling `Enum.reverse/1`. + + Implementing functions like `map/2`, `filter/2` and others are a good + exercise for understanding the power behind `Enum.reduce/3`. When an + operation cannot be expressed by any of the functions in the `Enum` + module, developers will most likely resort to `reduce/3`. + """ + @spec reduce(t, acc, (element, acc -> acc)) :: acc + def reduce(enumerable, acc, fun) when is_list(enumerable) do + :lists.foldl(fun, acc, enumerable) + end + + def reduce(first..last//step, acc, fun) do + reduce_range(first, last, step, acc, fun) + end + + def reduce(%_{} = enumerable, acc, fun) do + reduce_enumerable(enumerable, acc, fun) + end + + def reduce(%{} = enumerable, acc, fun) do + :maps.fold(fn k, v, acc -> fun.({k, v}, acc) end, acc, enumerable) + end + + def reduce(enumerable, acc, fun) do + reduce_enumerable(enumerable, acc, fun) + end + + @doc """ + Reduces `enumerable` until `fun` returns `{:halt, term}`. + + The return value for `fun` is expected to be + + * `{:cont, acc}` to continue the reduction with `acc` as the new + accumulator or + * `{:halt, acc}` to halt the reduction + + If `fun` returns `{:halt, acc}` the reduction is halted and the function + returns `acc`. Otherwise, if the enumerable is exhausted, the function returns + the accumulator of the last `{:cont, acc}`. + + ## Examples + + iex> Enum.reduce_while(1..100, 0, fn x, acc -> + ...> if x < 5, do: {:cont, acc + x}, else: {:halt, acc} + ...> end) + 10 + iex> Enum.reduce_while(1..100, 0, fn x, acc -> + ...> if x > 0, do: {:cont, acc + x}, else: {:halt, acc} + ...> end) + 5050 + + """ + @spec reduce_while(t, any, (element, any -> {:cont, any} | {:halt, any})) :: any + def reduce_while(enumerable, acc, fun) do + Enumerable.reduce(enumerable, {:cont, acc}, fun) |> elem(1) + end + + @doc """ + Returns a list of elements in `enumerable` excluding those for which the function `fun` returns + a truthy value. + + See also `filter/2`. + + ## Examples + + iex> Enum.reject([1, 2, 3], fn x -> rem(x, 2) == 0 end) [1, 3] """ @spec reject(t, (element -> as_boolean(term))) :: list - def reject(collection, fun) when is_list(collection) do - for item <- collection, !fun.(item), do: item + def reject(enumerable, fun) when is_list(enumerable) do + reject_list(enumerable, fun) end - def reject(collection, fun) do - Enumerable.reduce(collection, {:cont, []}, R.reject(fun)) |> elem(1) |> :lists.reverse + def reject(enumerable, fun) do + reduce(enumerable, [], R.reject(fun)) |> :lists.reverse() end @doc """ - Reverses the collection. + Returns a list of elements in `enumerable` in reverse order. ## Examples @@ -1321,18 +2539,20 @@ defmodule Enum do """ @spec reverse(t) :: list - def reverse(collection) when is_list(collection) do - :lists.reverse(collection) - end + def reverse(enumerable) - def reverse(collection) do - reverse(collection, []) - end + def reverse([]), do: [] + def reverse([_] = list), do: list + def reverse([element1, element2]), do: [element2, element1] + def reverse([element1, element2 | rest]), do: :lists.reverse(rest, [element2, element1]) + def reverse(enumerable), do: reduce(enumerable, [], &[&1 | &2]) @doc """ - Reverses the collection and appends the tail. + Reverses the elements in `enumerable`, appends the `tail`, and returns + it as a list. + This is an optimization for - `Enum.concat(Enum.reverse(collection), tail)`. + `enumerable |> Enum.reverse() |> Enum.concat(tail)`. ## Examples @@ -1341,184 +2561,466 @@ defmodule Enum do """ @spec reverse(t, t) :: list - def reverse(collection, tail) when is_list(collection) and is_list(tail) do - :lists.reverse(collection, tail) + def reverse(enumerable, tail) when is_list(enumerable) do + :lists.reverse(enumerable, to_list(tail)) end - def reverse(collection, tail) do - reduce(collection, to_list(tail), fn(entry, acc) -> - [entry|acc] + def reverse(enumerable, tail) do + reduce(enumerable, to_list(tail), fn entry, acc -> + [entry | acc] end) end @doc """ - Applies the given function to each element in the collection, + Reverses the `enumerable` in the range from initial `start_index` + through `count` elements. + + If `count` is greater than the size of the rest of the `enumerable`, + then this function will reverse the rest of the enumerable. + + ## Examples + + iex> Enum.reverse_slice([1, 2, 3, 4, 5, 6], 2, 4) + [1, 2, 6, 5, 4, 3] + + """ + @spec reverse_slice(t, non_neg_integer, non_neg_integer) :: list + def reverse_slice(enumerable, start_index, count) + when is_integer(start_index) and start_index >= 0 and is_integer(count) and count >= 0 do + list = reverse(enumerable) + length = length(list) + count = Kernel.min(count, length - start_index) + + if count > 0 do + reverse_slice(list, length, start_index + count, count, []) + else + :lists.reverse(list) + end + end + + @doc """ + Slides a single or multiple elements given by `range_or_single_index` from `enumerable` + to `insertion_index`. + + The semantics of the range to be moved match the semantics of `Enum.slice/2`. + Specifically, that means: + + * Indices are normalized, meaning that negative indexes will be counted from the end + (for example, -1 means the last element of the enumerable). This will result in *two* + traversals of your enumerable on types like lists that don't provide a constant-time count. + + * If the normalized index range's `last` is out of bounds, the range is truncated to the last element. + + * If the normalized index range's `first` is out of bounds, the selected range for sliding + will be empty, so you'll get back your input list. + + * Decreasing ranges (such as `5..0//1`) also select an empty range to be moved, + so you'll get back your input list. + + * Ranges with any step but 1 will raise an error. + + ## Examples + + # Slide a single element + iex> Enum.slide([:a, :b, :c, :d, :e, :f, :g], 5, 1) + [:a, :f, :b, :c, :d, :e, :g] + + # Slide a range of elements backward + iex> Enum.slide([:a, :b, :c, :d, :e, :f, :g], 3..5, 1) + [:a, :d, :e, :f, :b, :c, :g] + + # Slide a range of elements forward + iex> Enum.slide([:a, :b, :c, :d, :e, :f, :g], 1..3, 5) + [:a, :e, :f, :b, :c, :d, :g] + + # Slide with negative indices (counting from the end) + iex> Enum.slide([:a, :b, :c, :d, :e, :f, :g], 3..-1//1, 2) + [:a, :b, :d, :e, :f, :g, :c] + iex> Enum.slide([:a, :b, :c, :d, :e, :f, :g], -4..-2, 1) + [:a, :d, :e, :f, :b, :c, :g] + + # Insert at negative indices (counting from the end) + iex> Enum.slide([:a, :b, :c, :d, :e, :f, :g], 3, -1) + [:a, :b, :c, :e, :f, :g, :d] + + """ + @doc since: "1.13.0" + def slide(enumerable, range_or_single_index, insertion_index) + + def slide(enumerable, single_index, insertion_index) when is_integer(single_index) do + slide(enumerable, single_index..single_index, insertion_index) + end + + # This matches the behavior of Enum.slice/2 + def slide(_, _.._//step = index_range, _insertion_index) when step != 1 do + raise ArgumentError, + "Enum.slide/3 does not accept ranges with custom steps, got: #{inspect(index_range)}" + end + + # Normalize negative input ranges like Enum.slice/2 + def slide(enumerable, first..last, insertion_index) + when first < 0 or last < 0 or insertion_index < 0 do + count = Enum.count(enumerable) + normalized_first = if first >= 0, do: first, else: Kernel.max(first + count, 0) + normalized_last = if last >= 0, do: last, else: last + count + + normalized_insertion_index = + if insertion_index >= 0, do: insertion_index, else: insertion_index + count + + if normalized_first < count and normalized_first != normalized_insertion_index do + normalized_range = normalized_first..normalized_last//1 + slide(enumerable, normalized_range, normalized_insertion_index) + else + Enum.to_list(enumerable) + end + end + + def slide(enumerable, insertion_index.._, insertion_index) do + Enum.to_list(enumerable) + end + + def slide(_, first..last, insertion_index) + when insertion_index > first and insertion_index <= last do + raise ArgumentError, + "insertion index for slide must be outside the range being moved " <> + "(tried to insert #{first}..#{last} at #{insertion_index})" + end + + # Guarantees at this point: step size == 1 and first <= last and (insertion_index < first or insertion_index > last) + def slide(enumerable, first..last, insertion_index) do + impl = if is_list(enumerable), do: &slide_list_start/4, else: &slide_any/4 + + cond do + insertion_index <= first -> impl.(enumerable, insertion_index, first, last) + insertion_index > last -> impl.(enumerable, first, last + 1, insertion_index) + end + end + + # Takes the range from middle..last and moves it to be in front of index start + defp slide_any(enumerable, start, middle, last) do + # We're going to deal with 4 "chunks" of the enumerable: + # 0. "Head," before the start index + # 1. "Slide back," between start (inclusive) and middle (exclusive) + # 2. "Slide front," between middle (inclusive) and last (inclusive) + # 3. "Tail," after last + # + # But, we're going to accumulate these into only two lists: pre and post. + # We'll reverse-accumulate the head into our pre list, then "slide back" into post, + # then "slide front" into pre, then "tail" into post. + # + # Then at the end, we're going to reassemble and reverse them, and end up with the + # chunks in the correct order. + {_size, pre, post} = + Enum.reduce(enumerable, {0, [], []}, fn item, {index, pre, post} -> + {pre, post} = + cond do + index < start -> {[item | pre], post} + index >= start and index < middle -> {pre, [item | post]} + index >= middle and index <= last -> {[item | pre], post} + true -> {pre, [item | post]} + end + + {index + 1, pre, post} + end) + + :lists.reverse(pre, :lists.reverse(post)) + end + + # Like slide_any/4 above, this optimized implementation of slide for lists depends + # on the indices being sorted such that we're moving middle..last to be in front of start. + defp slide_list_start([h | t], start, middle, last) + when start > 0 and start <= middle and middle <= last do + [h | slide_list_start(t, start - 1, middle - 1, last - 1)] + end + + defp slide_list_start(list, 0, middle, last), do: slide_list_middle(list, middle, last, []) + + defp slide_list_middle([h | t], middle, last, acc) when middle > 0 do + slide_list_middle(t, middle - 1, last - 1, [h | acc]) + end + + defp slide_list_middle(list, 0, last, start_to_middle) do + {slid_range, tail} = slide_list_last(list, last + 1, []) + slid_range ++ :lists.reverse(start_to_middle, tail) + end + + # You asked for a middle index off the end of the list... you get what we've got + defp slide_list_middle([], _, _, acc) do + :lists.reverse(acc) + end + + defp slide_list_last([h | t], last, acc) when last > 0 do + slide_list_last(t, last - 1, [h | acc]) + end + + defp slide_list_last(rest, 0, acc) do + {:lists.reverse(acc), rest} + end + + defp slide_list_last([], _, acc) do + {:lists.reverse(acc), []} + end + + @doc """ + Applies the given function to each element in the `enumerable`, storing the result in a list and passing it as the accumulator - for the next computation. + for the next computation. Uses the first element in the `enumerable` + as the starting value. ## Examples iex> Enum.scan(1..5, &(&1 + &2)) - [1,3,6,10,15] + [1, 3, 6, 10, 15] """ @spec scan(t, (element, any -> any)) :: list - def scan(enum, fun) do - {_, {res, _}} = - Enumerable.reduce(enum, {:cont, {[], :first}}, R.scan_2(fun)) + def scan(enumerable, fun) + + def scan([], _fun), do: [] + + def scan([elem | rest], fun) do + scanned = scan_list(rest, elem, fun) + [elem | scanned] + end + + def scan(enumerable, fun) do + {res, _} = reduce(enumerable, {[], :first}, R.scan2(fun)) :lists.reverse(res) end @doc """ - Applies the given function to each element in the collection, + Applies the given function to each element in the `enumerable`, storing the result in a list and passing it as the accumulator for the next computation. Uses the given `acc` as the starting value. ## Examples iex> Enum.scan(1..5, 0, &(&1 + &2)) - [1,3,6,10,15] + [1, 3, 6, 10, 15] """ @spec scan(t, any, (element, any -> any)) :: list - def scan(enum, acc, fun) do - {_, {res, _}} = - Enumerable.reduce(enum, {:cont, {[], acc}}, R.scan_3(fun)) + def scan(enumerable, acc, fun) when is_list(enumerable) do + scan_list(enumerable, acc, fun) + end + + def scan(enumerable, acc, fun) do + {res, _} = reduce(enumerable, {[], acc}, R.scan3(fun)) :lists.reverse(res) end @doc """ - Returns a list of collection elements shuffled. - - Notice that you need to explicitly call `:random.seed/1` and - set a seed value for the random algorithm. Otherwise, the - default seed will be set which will always return the same - result. For example, one could do the following to set a seed - dynamically: + Returns a list with the elements of `enumerable` shuffled. - :random.seed(:erlang.now) + This function uses Erlang's [`:rand` module](`:rand`) to calculate + the random value. Check its documentation for setting a + different random algorithm or a different seed. ## Examples + The examples below use the `:exsss` pseudorandom algorithm since it's + the default from Erlang/OTP 22: + + # Although not necessary, let's seed the random algorithm + iex> :rand.seed(:exsss, {1, 2, 3}) iex> Enum.shuffle([1, 2, 3]) [3, 2, 1] iex> Enum.shuffle([1, 2, 3]) - [3, 1, 2] + [2, 1, 3] """ @spec shuffle(t) :: list - def shuffle(collection) do - randomized = reduce(collection, [], fn x, acc -> - [{:random.uniform, x}|acc] - end) - unwrap(:lists.keysort(1, randomized), []) + def shuffle(enumerable) do + randomized = + reduce(enumerable, [], fn x, acc -> + [{:rand.uniform(), x} | acc] + end) + + shuffle_unwrap(:lists.keysort(1, randomized), []) end @doc """ - Returns a subset list of the given collection. Drops elements - until element position `start`, then takes `count` elements. - - If the count is greater than collection length, it returns as - much as possible. If zero, then it returns `[]`. + Returns a subset list of the given `enumerable` by `index_range`. + + `index_range` must be a `Range`. Given an `enumerable`, it drops + elements before `index_range.first` (zero-base), then it takes elements + until element `index_range.last` (inclusively). + + Indexes are normalized, meaning that negative indexes will be counted + from the end (for example, `-1` means the last element of the `enumerable`). + + If `index_range.last` is out of bounds, then it is assigned as the index + of the last element. + + If the normalized `index_range.first` is out of bounds of the given + `enumerable`, or this one is greater than the normalized `index_range.last`, + then `[]` is returned. + + If a step `n` (other than `1`) is used in `index_range`, then it takes + every `n`th element from `index_range.first` to `index_range.last` + (according to the same rules described above). ## Examples - iex> Enum.slice(1..100, 5, 10) - [6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + iex> Enum.slice([1, 2, 3, 4, 5], 1..3) + [2, 3, 4] - iex> Enum.slice(1..10, 5, 100) - [6, 7, 8, 9, 10] + iex> Enum.slice([1, 2, 3, 4, 5], 3..10) + [4, 5] - iex> Enum.slice(1..10, 5, 0) + # Last three elements (negative indexes) + iex> Enum.slice([1, 2, 3, 4, 5], -3..-1) + [3, 4, 5] + + For ranges where `start > stop`, you need to explicit + mark them as increasing: + + iex> Enum.slice([1, 2, 3, 4, 5], 1..-2//1) + [2, 3, 4] + + The step can be any positive number. For example, to + get every 2 elements of the collection: + + iex> Enum.slice([1, 2, 3, 4, 5], 0..-1//2) + [1, 3, 5] + + To get every third element of the first ten elements: + + iex> integers = Enum.to_list(1..20) + iex> Enum.slice(integers, 0..9//3) + [1, 4, 7, 10] + + If the first position is after the end of the enumerable + or after the last position of the range, it returns an + empty list: + + iex> Enum.slice([1, 2, 3, 4, 5], 6..10) + [] + + # first is greater than last + iex> Enum.slice([1, 2, 3, 4, 5], 6..5) [] """ - @spec slice(t, integer, non_neg_integer) :: list - - def slice(_coll, _start, 0), do: [] + @doc since: "1.6.0" + @spec slice(t, Range.t()) :: list + def slice(enumerable, first..last//step = index_range) do + # TODO: Deprecate negative steps on Elixir v1.16 + # TODO: There are two features we can add to slicing ranges: + # 1. We can allow the step to be any positive number + # 2. We can allow slice and reverse at the same time. However, we can't + # implement so right now. First we will have to raise if a decreasing + # range is given on Elixir v2.0. + cond do + step > 0 -> + slice_range(enumerable, first, last, step) + + step == -1 and first > last -> + slice_range(enumerable, first, last, 1) - def slice(coll, start, count) when start < 0 do - {list, new_start} = enumerate_and_count(coll, start) - if new_start >= 0 do - slice(list, new_start, count) + true -> + raise ArgumentError, + "Enum.slice/2 does not accept ranges with negative steps, got: #{inspect(index_range)}" + end + end + + # TODO: Remove me on v2.0 + def slice(enumerable, %{__struct__: Range, first: first, last: last} = index_range) do + step = if first <= last, do: 1, else: -1 + slice(enumerable, Map.put(index_range, :step, step)) + end + + defp slice_range(enumerable, first, -1, step) when first >= 0 do + if step == 1 do + drop(enumerable, first) else - [] + enumerable |> drop(first) |> take_every_list(step - 1) end end - def slice(coll, start, count) when is_list(coll) and start >= 0 and count > 0 do - do_slice(coll, start, count) + defp slice_range(enumerable, first, last, step) + when last >= first and last >= 0 and first >= 0 do + slice_forward(enumerable, first, last - first + 1, step) end - def slice(coll, start, count) when start >= 0 and count > 0 do - {_, _, list} = Enumerable.reduce(coll, {:cont, {start, count, []}}, fn - _entry, {start, count, _list} when start > 0 -> - {:cont, {start-1, count, []}} - entry, {start, count, list} when count > 1 -> - {:cont, {start, count-1, [entry|list]}} - entry, {start, count, list} -> - {:halt, {start, count, [entry|list]}} - end) |> elem(1) + defp slice_range(enumerable, first, last, step) do + {count, fun} = slice_count_and_fun(enumerable, step) + first = if first >= 0, do: first, else: Kernel.max(first + count, 0) + last = if last >= 0, do: last, else: last + count + amount = last - first + 1 - :lists.reverse(list) + if first < count and amount > 0 do + amount = Kernel.min(amount, count - first) + amount = if step == 1, do: amount, else: div(amount - 1, step) + 1 + fun.(first, amount, step) + else + [] + end end @doc """ - Returns a subset list of the given collection. Drops elements - until element position `range.first`, then takes elements until element - position `range.last` (inclusive). + Returns a subset list of the given `enumerable`, from `start_index` (zero-based) + with `amount` number of elements if available. - Positions are calculated by adding the number of items in the collection to - negative positions (so position -3 in a collection with count 5 becomes - position 2). + Given an `enumerable`, it drops elements right before element `start_index`; + then, it takes `amount` of elements, returning as many elements as possible if + there are not enough elements. - The first position (after adding count to negative positions) must be smaller - or equal to the last position. + A negative `start_index` can be passed, which means the `enumerable` is + enumerated once and the index is counted from the end (for example, + `-1` starts slicing from the last element). - If the start of the range is not a valid offset for the given - collection or if the range is in reverse order, returns `[]`. + It returns `[]` if `amount` is `0` or if `start_index` is out of bounds. ## Examples - iex> Enum.slice(1..100, 5..10) - [6, 7, 8, 9, 10, 11] + iex> Enum.slice(1..100, 5, 10) + [6, 7, 8, 9, 10, 11, 12, 13, 14, 15] - iex> Enum.slice(1..10, 5..20) + # amount to take is greater than the number of elements + iex> Enum.slice(1..10, 5, 100) [6, 7, 8, 9, 10] - iex> Enum.slice(1..10, 11..20) + iex> Enum.slice(1..10, 5, 0) [] - iex> Enum.slice(1..10, 6..5) + # using a negative start index + iex> Enum.slice(1..10, -6, 3) + [5, 6, 7] + iex> Enum.slice(1..10, -11, 5) + [1, 2, 3, 4, 5] + + # out of bound start index + iex> Enum.slice(1..10, 10, 5) [] """ - @spec slice(t, Range.t) :: list - def slice(coll, first..last) when first >= 0 and last >= 0 do - # Simple case, which works on infinite collections - if last - first >= 0 do - slice(coll, first, last - first + 1) + @spec slice(t, index, non_neg_integer) :: list + def slice(_enumerable, start_index, 0) when is_integer(start_index), do: [] + + def slice(enumerable, start_index, amount) + when is_integer(start_index) and start_index < 0 and is_integer(amount) and amount >= 0 do + {count, fun} = slice_count_and_fun(enumerable, 1) + start_index = Kernel.max(count + start_index, 0) + amount = Kernel.min(amount, count - start_index) + + if amount > 0 do + fun.(start_index, amount, 1) else [] end end - def slice(coll, first..last) do - {list, count} = enumerate_and_count(coll, 0) - corr_first = if first >= 0, do: first, else: first + count - corr_last = if last >= 0, do: last, else: last + count - length = corr_last - corr_first + 1 - if corr_first >= 0 and length > 0 do - slice(list, corr_first, length) - else - [] - end + def slice(enumerable, start_index, amount) + when is_integer(start_index) and is_integer(amount) and amount >= 0 do + slice_forward(enumerable, start_index, amount, 1) end @doc """ - Sorts the collection according to Elixir's term ordering. + Sorts the `enumerable` according to Erlang's term ordering. - Uses the merge sort algorithm. + This function uses the merge sort algorithm. Do not use this + function to sort structs, see `sort/2` for more information. ## Examples @@ -1527,138 +3029,394 @@ defmodule Enum do """ @spec sort(t) :: list - def sort(collection) when is_list(collection) do - :lists.sort(collection) + def sort(enumerable) when is_list(enumerable) do + :lists.sort(enumerable) end - def sort(collection) do - sort(collection, &(&1 <= &2)) + def sort(enumerable) do + sort(enumerable, &(&1 <= &2)) end @doc """ - Sorts the collection by the given function. + Sorts the `enumerable` by the given function. - This function uses the merge sort algorithm. The given function - must return false if the first argument is less than right one. + This function uses the merge sort algorithm. The given function should compare + two arguments, and return `true` if the first argument precedes or is in the + same place as the second one. ## Examples - iex> Enum.sort([1, 2, 3], &(&1 > &2)) + iex> Enum.sort([1, 2, 3], &(&1 >= &2)) [3, 2, 1] The sorting algorithm will be stable as long as the given function - returns true for values considered equal: + returns `true` for values considered equal: - iex> Enum.sort ["some", "kind", "of", "monster"], &(byte_size(&1) <= byte_size(&2)) + iex> Enum.sort(["some", "kind", "of", "monster"], &(byte_size(&1) <= byte_size(&2))) ["of", "some", "kind", "monster"] - If the function does not return true, the sorting is not stable and - the order of equal terms may be shuffled: + If the function does not return `true` for equal values, the sorting + is not stable and the order of equal terms may be shuffled. + For example: - iex> Enum.sort ["some", "kind", "of", "monster"], &(byte_size(&1) < byte_size(&2)) + iex> Enum.sort(["some", "kind", "of", "monster"], &(byte_size(&1) < byte_size(&2))) ["of", "kind", "some", "monster"] + ## Ascending and descending (since v1.10.0) + + `sort/2` allows a developer to pass `:asc` or `:desc` as the sorter, which is a convenience for + [`&<=/2`](`<=/2`) and [`&>=/2`](`>=/2`) respectively. + + iex> Enum.sort([2, 3, 1], :asc) + [1, 2, 3] + iex> Enum.sort([2, 3, 1], :desc) + [3, 2, 1] + + ## Sorting structs + + Do not use `/2`, `>=/2` and friends when sorting structs. + That's because the built-in operators above perform structural comparison + and not a semantic one. Imagine we sort the following list of dates: + + iex> dates = [~D[2019-01-01], ~D[2020-03-02], ~D[2019-06-06]] + iex> Enum.sort(dates) + [~D[2019-01-01], ~D[2020-03-02], ~D[2019-06-06]] + + Note that the returned result is incorrect, because `sort/1` by default uses + `<=/2`, which will compare their structure. When comparing structures, the + fields are compared in alphabetical order, which means the dates above will + be compared by `day`, `month` and then `year`, which is the opposite of what + we want. + + For this reason, most structs provide a "compare" function, such as + `Date.compare/2`, which receives two structs and returns `:lt` (less-than), + `:eq` (equal to), and `:gt` (greater-than). If you pass a module as the + sorting function, Elixir will automatically use the `compare/2` function + of said module: + + iex> dates = [~D[2019-01-01], ~D[2020-03-02], ~D[2019-06-06]] + iex> Enum.sort(dates, Date) + [~D[2019-01-01], ~D[2019-06-06], ~D[2020-03-02]] + + To retrieve all dates in descending order, you can wrap the module in + a tuple with `:asc` or `:desc` as first element: + + iex> dates = [~D[2019-01-01], ~D[2020-03-02], ~D[2019-06-06]] + iex> Enum.sort(dates, {:asc, Date}) + [~D[2019-01-01], ~D[2019-06-06], ~D[2020-03-02]] + iex> dates = [~D[2019-01-01], ~D[2020-03-02], ~D[2019-06-06]] + iex> Enum.sort(dates, {:desc, Date}) + [~D[2020-03-02], ~D[2019-06-06], ~D[2019-01-01]] + + """ + @spec sort( + t, + (element, element -> boolean) | :asc | :desc | module() | {:asc | :desc, module()} + ) :: list + def sort(enumerable, sorter) when is_list(enumerable) do + :lists.sort(to_sort_fun(sorter), enumerable) + end + + def sort(enumerable, sorter) do + fun = to_sort_fun(sorter) + + reduce(enumerable, [], &sort_reducer(&1, &2, fun)) + |> sort_terminator(fun) + end + + defp to_sort_fun(sorter) when is_function(sorter, 2), do: sorter + defp to_sort_fun(:asc), do: &<=/2 + defp to_sort_fun(:desc), do: &>=/2 + defp to_sort_fun(module) when is_atom(module), do: &(module.compare(&1, &2) != :gt) + defp to_sort_fun({:asc, module}) when is_atom(module), do: &(module.compare(&1, &2) != :gt) + defp to_sort_fun({:desc, module}) when is_atom(module), do: &(module.compare(&1, &2) != :lt) + + @doc """ + Sorts the mapped results of the `enumerable` according to the provided `sorter` + function. + + This function maps each element of the `enumerable` using the + provided `mapper` function. The enumerable is then sorted by + the mapped elements using the `sorter`, which defaults to `:asc` + and sorts the elements ascendingly. + + `sort_by/3` differs from `sort/2` in that it only calculates the + comparison value for each element in the enumerable once instead of + once for each element in each comparison. If the same function is + being called on both elements, it's more efficient to use `sort_by/3`. + + ## Ascending and descending (since v1.10.0) + + `sort_by/3` allows a developer to pass `:asc` or `:desc` as the sorter, + which is a convenience for [`&<=/2`](`<=/2`) and [`&>=/2`](`>=/2`) respectively: + iex> Enum.sort_by([2, 3, 1], &(&1), :asc) + [1, 2, 3] + + iex> Enum.sort_by([2, 3, 1], &(&1), :desc) + [3, 2, 1] + + ## Examples + + Using the default `sorter` of `:asc` : + + iex> Enum.sort_by(["some", "kind", "of", "monster"], &byte_size/1) + ["of", "some", "kind", "monster"] + + Sorting by multiple properties - first by size, then by first letter + (this takes advantage of the fact that tuples are compared element-by-element): + + iex> Enum.sort_by(["some", "kind", "of", "monster"], &{byte_size(&1), String.first(&1)}) + ["of", "kind", "some", "monster"] + + Similar to `sort/2`, you can pass a custom sorter: + + iex> Enum.sort_by(["some", "kind", "of", "monster"], &byte_size/1, :desc) + ["monster", "some", "kind", "of"] + + As in `sort/2`, avoid using the default sorting function to sort + structs, as by default it performs structural comparison instead of + a semantic one. In such cases, you shall pass a sorting function as + third element or any module that implements a `compare/2` function. + For example, to sort users by their birthday in both ascending and + descending order respectively: + + iex> users = [ + ...> %{name: "Ellis", birthday: ~D[1943-05-11]}, + ...> %{name: "Lovelace", birthday: ~D[1815-12-10]}, + ...> %{name: "Turing", birthday: ~D[1912-06-23]} + ...> ] + iex> Enum.sort_by(users, &(&1.birthday), Date) + [ + %{name: "Lovelace", birthday: ~D[1815-12-10]}, + %{name: "Turing", birthday: ~D[1912-06-23]}, + %{name: "Ellis", birthday: ~D[1943-05-11]} + ] + iex> Enum.sort_by(users, &(&1.birthday), {:desc, Date}) + [ + %{name: "Ellis", birthday: ~D[1943-05-11]}, + %{name: "Turing", birthday: ~D[1912-06-23]}, + %{name: "Lovelace", birthday: ~D[1815-12-10]} + ] + + ## Performance characteristics + + As detailed in the initial section, `sort_by/3` calculates the comparison + value for each element in the enumerable once instead of once for each + element in each comparison. This implies `sort_by/3` must do an initial + pass on the data to compute those values. + + However, if those values are cheap to compute, for example, you have + already extracted the field you want to sort by into a tuple, then those + extra passes become overhead. In such cases, consider using `List.keysort/3` + instead. + + Let's see an example. Imagine you have a list of products and you have a + list of IDs. You want to keep all products that are in the given IDs and + return their names sorted by their price. You could write it like this: + + for( + product <- products, + product.id in ids, + do: product + ) + |> Enum.sort_by(& &1.price) + |> Enum.map(& &1.name) + + However, you could also write it like this: + + for( + product <- products, + product.id in ids, + do: {product.name, product.price} + ) + |> List.keysort(1) + |> Enum.map(&elem(&1, 0)) + + Using `List.keysort/3` will be a better choice for performance sensitive + code as it avoids additional traversals. """ - @spec sort(t, (element, element -> boolean)) :: list - def sort(collection, fun) when is_list(collection) do - :lists.sort(fun, collection) + @spec sort_by( + t, + (element -> mapped_element), + (element, element -> boolean) | :asc | :desc | module() | {:asc | :desc, module()} + ) :: + list + when mapped_element: element + def sort_by(enumerable, mapper, sorter \\ :asc) + + def sort_by(enumerable, mapper, :desc) when is_function(mapper, 1) do + enumerable + |> Enum.reduce([], &[{&1, mapper.(&1)} | &2]) + |> List.keysort(1, :asc) + |> List.foldl([], &[elem(&1, 0) | &2]) end - def sort(collection, fun) do - reduce(collection, [], &sort_reducer(&1, &2, fun)) |> sort_terminator(fun) + def sort_by(enumerable, mapper, sorter) when is_function(mapper, 1) do + enumerable + |> map(&{&1, mapper.(&1)}) + |> List.keysort(1, sorter) + |> map(&elem(&1, 0)) end @doc """ - Splits the enumerable into two collections, leaving `count` - elements in the first one. If `count` is a negative number, - it starts counting from the back to the beginning of the - collection. + Splits the `enumerable` into two enumerables, leaving `count` + elements in the first one. - Be aware that a negative `count` implies the collection + If `count` is a negative number, it starts counting from the + back to the beginning of the `enumerable`. + + Be aware that a negative `count` implies the `enumerable` will be enumerated twice: once to calculate the position, and a second time to do the actual splitting. ## Examples iex> Enum.split([1, 2, 3], 2) - {[1,2], [3]} + {[1, 2], [3]} iex> Enum.split([1, 2, 3], 10) - {[1,2,3], []} + {[1, 2, 3], []} iex> Enum.split([1, 2, 3], 0) - {[], [1,2,3]} + {[], [1, 2, 3]} iex> Enum.split([1, 2, 3], -1) - {[1,2], [3]} + {[1, 2], [3]} iex> Enum.split([1, 2, 3], -5) - {[], [1,2,3]} + {[], [1, 2, 3]} """ @spec split(t, integer) :: {list, list} - def split(collection, count) when is_list(collection) and count >= 0 do - do_split(collection, count, []) + def split(enumerable, count) when is_list(enumerable) and is_integer(count) and count >= 0 do + split_list(enumerable, count, []) end - def split(collection, count) when count >= 0 do + def split(enumerable, count) when is_integer(count) and count >= 0 do {_, list1, list2} = - reduce(collection, {count, [], []}, fn(entry, {counter, acc1, acc2}) -> + reduce(enumerable, {count, [], []}, fn entry, {counter, acc1, acc2} -> if counter > 0 do - {counter - 1, [entry|acc1], acc2} + {counter - 1, [entry | acc1], acc2} else - {counter, acc1, [entry|acc2]} + {counter, acc1, [entry | acc2]} end end) {:lists.reverse(list1), :lists.reverse(list2)} end - def split(collection, count) when count < 0 do - do_split_reverse(reverse(collection), abs(count), []) + def split(enumerable, count) when is_integer(count) and count < 0 do + split_reverse_list(reverse(enumerable), -count, []) end @doc """ - Splits `collection` in two while `fun` returns `true`. + Splits enumerable in two at the position of the element for which + `fun` returns a falsy value (`false` or `nil`) for the first time. + + It returns a two-element tuple with two lists of elements. + The element that triggered the split is part of the second list. ## Examples - iex> Enum.split_while([1, 2, 3, 4], fn(x) -> x < 3 end) + iex> Enum.split_while([1, 2, 3, 4], fn x -> x < 3 end) {[1, 2], [3, 4]} + iex> Enum.split_while([1, 2, 3, 4], fn x -> x < 0 end) + {[], [1, 2, 3, 4]} + + iex> Enum.split_while([1, 2, 3, 4], fn x -> x > 0 end) + {[1, 2, 3, 4], []} + """ @spec split_while(t, (element -> as_boolean(term))) :: {list, list} - def split_while(collection, fun) when is_list(collection) do - do_split_while(collection, fun, []) + def split_while(enumerable, fun) when is_list(enumerable) do + split_while_list(enumerable, fun, []) end - def split_while(collection, fun) do + def split_while(enumerable, fun) do {list1, list2} = - reduce(collection, {[], []}, fn + reduce(enumerable, {[], []}, fn entry, {acc1, []} -> - if(fun.(entry), do: {[entry|acc1], []}, else: {acc1, [entry]}) + if(fun.(entry), do: {[entry | acc1], []}, else: {acc1, [entry]}) + entry, {acc1, acc2} -> - {acc1, [entry|acc2]} + {acc1, [entry | acc2]} end) {:lists.reverse(list1), :lists.reverse(list2)} end @doc """ - Takes the first `count` items from the collection. + Returns the sum of all elements. + + Raises `ArithmeticError` if `enumerable` contains a non-numeric value. + + ## Examples + + iex> Enum.sum([1, 2, 3]) + 6 + + iex> Enum.sum(1..10) + 55 + + iex> Enum.sum(1..10//2) + 25 + + """ + @spec sum(t) :: number + def sum(enumerable) + + def sum(first..last//step = range) do + range + |> Range.size() + |> Kernel.*(first + last - rem(last - first, step)) + |> div(2) + end + + def sum(enumerable) do + reduce(enumerable, 0, &+/2) + end + + @doc """ + Returns the product of all elements. + + Raises `ArithmeticError` if `enumerable` contains a non-numeric value. + + ## Examples + + iex> Enum.product([]) + 1 + iex> Enum.product([2, 3, 4]) + 24 + iex> Enum.product([2.0, 3.0, 4.0]) + 24.0 + + """ + @doc since: "1.12.0" + @spec product(t) :: number + def product(enumerable) do + reduce(enumerable, 1, &*/2) + end + + @doc """ + Takes an `amount` of elements from the beginning or the end of the `enumerable`. + + If a positive `amount` is given, it takes the `amount` elements from the + beginning of the `enumerable`. + + If a negative `amount` is given, the `amount` of elements will be taken from the end. + The `enumerable` will be enumerated once to retrieve the proper index and + the remaining calculation is performed from the end. - If a negative `count` is given, the last `count` values will - be taken. For such, the collection is fully enumerated keeping up - to `2 * count` elements in memory. Once the end of the collection is - reached, the last `count` elements are returned. + If amount is `0`, it returns `[]`. ## Examples iex> Enum.take([1, 2, 3], 2) - [1,2] + [1, 2] iex> Enum.take([1, 2, 3], 10) - [1,2,3] + [1, 2, 3] iex> Enum.take([1, 2, 3], 0) [] @@ -1668,310 +3426,1105 @@ defmodule Enum do """ @spec take(t, integer) :: list + def take(enumerable, amount) - def take(_collection, 0) do - [] - end + def take(_enumerable, 0), do: [] - def take(collection, count) when is_list(collection) and count > 0 do - do_take(collection, count) + def take(enumerable, amount) + when is_list(enumerable) and is_integer(amount) and amount > 0 do + take_list(enumerable, amount) end - def take(collection, count) when count > 0 do + def take(enumerable, amount) when is_integer(amount) and amount > 0 do {_, {res, _}} = - Enumerable.reduce(collection, {:cont, {[], count}}, fn(entry, {list, count}) -> - if count > 1 do - {:cont, {[entry|list], count - 1}} + Enumerable.reduce(enumerable, {:cont, {[], amount}}, fn entry, {list, n} -> + case n do + 1 -> {:halt, {[entry | list], n - 1}} + _ -> {:cont, {[entry | list], n - 1}} + end + end) + + :lists.reverse(res) + end + + def take(enumerable, amount) when is_integer(amount) and amount < 0 do + {count, fun} = slice_count_and_fun(enumerable, 1) + first = Kernel.max(amount + count, 0) + fun.(first, count - first, 1) + end + + @doc """ + Returns a list of every `nth` element in the `enumerable`, + starting with the first element. + + The first element is always included, unless `nth` is 0. + + The second argument specifying every `nth` element must be a non-negative + integer. + + ## Examples + + iex> Enum.take_every(1..10, 2) + [1, 3, 5, 7, 9] + + iex> Enum.take_every(1..10, 0) + [] + + iex> Enum.take_every([1, 2, 3], 1) + [1, 2, 3] + + """ + @spec take_every(t, non_neg_integer) :: list + def take_every(enumerable, nth) + + def take_every(_enumerable, 0), do: [] + def take_every(enumerable, 1), do: to_list(enumerable) + + def take_every(list, nth) when is_list(list) and is_integer(nth) and nth > 1 do + take_every_list(list, nth - 1) + end + + def take_every(enumerable, nth) when is_integer(nth) and nth > 1 do + {res, _} = reduce(enumerable, {[], :first}, R.take_every(nth)) + :lists.reverse(res) + end + + @doc """ + Takes `count` random elements from `enumerable`. + + Note that this function will traverse the whole `enumerable` to + get the random sublist. + + See `random/1` for notes on implementation and random seed. + + ## Examples + + # Although not necessary, let's seed the random algorithm + iex> :rand.seed(:exsss, {1, 2, 3}) + iex> Enum.take_random(1..10, 2) + [3, 1] + iex> Enum.take_random(?a..?z, 5) + 'mikel' + + """ + @spec take_random(t, non_neg_integer) :: list + def take_random(enumerable, count) + def take_random(_enumerable, 0), do: [] + + def take_random([], _), do: [] + def take_random([h | t], 1), do: take_random_list_one(t, h, 1) + + def take_random(enumerable, 1) do + enumerable + |> reduce([], fn + x, [current | index] -> + if :rand.uniform(index + 1) == 1 do + [x | index + 1] + else + [current | index + 1] + end + + x, [] -> + [x | 1] + end) + |> case do + [] -> [] + [current | _index] -> [current] + end + end + + def take_random(enumerable, count) when is_integer(count) and count in 0..128 do + sample = Tuple.duplicate(nil, count) + + reducer = fn elem, {idx, sample} -> + jdx = random_integer(0, idx) + + cond do + idx < count -> + value = elem(sample, jdx) + {idx + 1, put_elem(sample, idx, value) |> put_elem(jdx, elem)} + + jdx < count -> + {idx + 1, put_elem(sample, jdx, elem)} + + true -> + {idx + 1, sample} + end + end + + {size, sample} = reduce(enumerable, {0, sample}, reducer) + sample |> Tuple.to_list() |> take(Kernel.min(count, size)) + end + + def take_random(enumerable, count) when is_integer(count) and count >= 0 do + reducer = fn elem, {idx, sample} -> + jdx = random_integer(0, idx) + + cond do + idx < count -> + value = Map.get(sample, jdx) + {idx + 1, Map.put(sample, idx, value) |> Map.put(jdx, elem)} + + jdx < count -> + {idx + 1, Map.put(sample, jdx, elem)} + + true -> + {idx + 1, sample} + end + end + + {size, sample} = reduce(enumerable, {0, %{}}, reducer) + take_random(sample, Kernel.min(count, size), []) + end + + defp take_random(_sample, 0, acc), do: acc + + defp take_random(sample, position, acc) do + position = position - 1 + take_random(sample, position, [Map.get(sample, position) | acc]) + end + + defp take_random_list_one([h | t], current, index) do + if :rand.uniform(index + 1) == 1 do + take_random_list_one(t, h, index + 1) + else + take_random_list_one(t, current, index + 1) + end + end + + defp take_random_list_one([], current, _), do: [current] + + @doc """ + Takes the elements from the beginning of the `enumerable` while `fun` returns + a truthy value. + + ## Examples + + iex> Enum.take_while([1, 2, 3], fn x -> x < 3 end) + [1, 2] + + """ + @spec take_while(t, (element -> as_boolean(term))) :: list + def take_while(enumerable, fun) when is_list(enumerable) do + take_while_list(enumerable, fun) + end + + def take_while(enumerable, fun) do + {_, res} = + Enumerable.reduce(enumerable, {:cont, []}, fn entry, acc -> + if fun.(entry) do + {:cont, [entry | acc]} else - {:halt, {[entry|list], count}} + {:halt, acc} end end) + :lists.reverse(res) end - def take(collection, count) when count < 0 do - Stream.take(collection, count).({:cont, []}, &{:cont, [&1|&2]}) - |> elem(1) |> :lists.reverse + @doc """ + Converts `enumerable` to a list. + + ## Examples + + iex> Enum.to_list(1..3) + [1, 2, 3] + + """ + @spec to_list(t) :: [element] + def to_list(enumerable) when is_list(enumerable), do: enumerable + def to_list(%_{} = enumerable), do: reverse(enumerable) |> :lists.reverse() + def to_list(%{} = enumerable), do: Map.to_list(enumerable) + def to_list(enumerable), do: reverse(enumerable) |> :lists.reverse() + + @doc """ + Enumerates the `enumerable`, removing all duplicated elements. + + ## Examples + + iex> Enum.uniq([1, 2, 3, 3, 2, 1]) + [1, 2, 3] + + """ + @spec uniq(t) :: list + def uniq(enumerable) do + uniq_by(enumerable, fn x -> x end) + end + + @doc false + @deprecated "Use Enum.uniq_by/2 instead" + def uniq(enumerable, fun) do + uniq_by(enumerable, fun) + end + + @doc """ + Enumerates the `enumerable`, by removing the elements for which + function `fun` returned duplicate elements. + + The function `fun` maps every element to a term. Two elements are + considered duplicates if the return value of `fun` is equal for + both of them. + + The first occurrence of each element is kept. + + ## Example + + iex> Enum.uniq_by([{1, :x}, {2, :y}, {1, :z}], fn {x, _} -> x end) + [{1, :x}, {2, :y}] + + iex> Enum.uniq_by([a: {:tea, 2}, b: {:tea, 2}, c: {:coffee, 1}], fn {_, y} -> y end) + [a: {:tea, 2}, c: {:coffee, 1}] + + """ + @spec uniq_by(t, (element -> term)) :: list + + def uniq_by(enumerable, fun) when is_list(enumerable) do + uniq_list(enumerable, %{}, fun) + end + + def uniq_by(enumerable, fun) do + {list, _} = reduce(enumerable, {[], %{}}, R.uniq_by(fun)) + :lists.reverse(list) + end + + @doc """ + Opposite of `zip/2`. Extracts two-element tuples from the + given `enumerable` and groups them together. + + It takes an `enumerable` with elements being two-element tuples and returns + a tuple with two lists, each of which is formed by the first and + second element of each tuple, respectively. + + This function fails unless `enumerable` is or can be converted into a + list of tuples with *exactly* two elements in each tuple. + + ## Examples + + iex> Enum.unzip([{:a, 1}, {:b, 2}, {:c, 3}]) + {[:a, :b, :c], [1, 2, 3]} + + iex> Enum.unzip(%{a: 1, b: 2}) + {[:a, :b], [1, 2]} + + """ + @spec unzip(t) :: {[element], [element]} + + def unzip([_ | _] = list) do + :lists.reverse(list) |> unzip([], []) + end + + def unzip([]) do + {[], []} + end + + def unzip(enumerable) do + {list1, list2} = + reduce(enumerable, {[], []}, fn {el1, el2}, {list1, list2} -> + {[el1 | list1], [el2 | list2]} + end) + + {:lists.reverse(list1), :lists.reverse(list2)} + end + + defp unzip([{el1, el2} | reversed_list], list1, list2) do + unzip(reversed_list, [el1 | list1], [el2 | list2]) + end + + defp unzip([], list1, list2) do + {list1, list2} + end + + @doc """ + Returns the `enumerable` with each element wrapped in a tuple + alongside its index. + + May receive a function or an integer offset. + + If an `offset` is given, it will index from the given offset instead of from + zero. + + If a `function` is given, it will index by invoking the function for each + element and index (zero-based) of the enumerable. + + ## Examples + + iex> Enum.with_index([:a, :b, :c]) + [a: 0, b: 1, c: 2] + + iex> Enum.with_index([:a, :b, :c], 3) + [a: 3, b: 4, c: 5] + + iex> Enum.with_index([:a, :b, :c], fn element, index -> {index, element} end) + [{0, :a}, {1, :b}, {2, :c}] + + """ + @spec with_index(t, integer) :: [{term, integer}] + @spec with_index(t, (element, index -> value)) :: [value] when value: any + def with_index(enumerable, fun_or_offset \\ 0) + + def with_index(enumerable, offset) when is_list(enumerable) and is_integer(offset) do + with_index_list(enumerable, offset) + end + + def with_index(enumerable, fun) when is_list(enumerable) and is_function(fun, 2) do + with_index_list(enumerable, 0, fun) + end + + def with_index(enumerable, offset) when is_integer(offset) do + enumerable + |> map_reduce(offset, fn x, i -> {{x, i}, i + 1} end) + |> elem(0) + end + + def with_index(enumerable, fun) when is_function(fun, 2) do + enumerable + |> map_reduce(0, fn x, i -> {fun.(x, i), i + 1} end) + |> elem(0) + end + + @doc """ + Zips corresponding elements from two enumerables into a list + of tuples. + + The zipping finishes as soon as either enumerable completes. + + ## Examples + + iex> Enum.zip([1, 2, 3], [:a, :b, :c]) + [{1, :a}, {2, :b}, {3, :c}] + + iex> Enum.zip([1, 2, 3, 4, 5], [:a, :b, :c]) + [{1, :a}, {2, :b}, {3, :c}] + + """ + @spec zip(t, t) :: [{any, any}] + def zip(enumerable1, enumerable2) when is_list(enumerable1) and is_list(enumerable2) do + zip_list(enumerable1, enumerable2, []) + end + + def zip(enumerable1, enumerable2) do + zip([enumerable1, enumerable2]) + end + + @doc """ + Zips corresponding elements from a finite collection of enumerables + into a list of tuples. + + The zipping finishes as soon as any enumerable in the given collection completes. + + ## Examples + + iex> Enum.zip([[1, 2, 3], [:a, :b, :c], ["foo", "bar", "baz"]]) + [{1, :a, "foo"}, {2, :b, "bar"}, {3, :c, "baz"}] + + iex> Enum.zip([[1, 2, 3, 4, 5], [:a, :b, :c]]) + [{1, :a}, {2, :b}, {3, :c}] + + """ + @doc since: "1.4.0" + @spec zip(enumerables) :: [tuple()] when enumerables: [t()] | t() + def zip([]), do: [] + + def zip(enumerables) do + zip_reduce(enumerables, [], &[List.to_tuple(&1) | &2]) + |> :lists.reverse() end @doc """ - Returns a collection of every `nth` item in the collection, - starting with the first element. + Zips corresponding elements from two enumerables into a list, transforming them with + the `zip_fun` function as it goes. - ## Examples + The corresponding elements from each collection are passed to the provided two-arity `zip_fun` + function in turn. Returns a list that contains the result of calling `zip_fun` for each pair of + elements. - iex> Enum.take_every(1..10, 2) - [1, 3, 5, 7, 9] + The zipping finishes as soon as either enumerable runs out of elements. - """ - @spec take_every(t, integer) :: list - def take_every(_collection, 0), do: [] - def take_every(collection, nth) do - {_, {res, _}} = - Enumerable.reduce(collection, {:cont, {[], :first}}, R.take_every(nth)) - :lists.reverse(res) - end + ## Zipping Maps - @doc """ - Takes the items at the beginning of `collection` while `fun` returns `true`. + It's important to remember that zipping inherently relies on order. + If you zip two lists you get the element at the index from each list in turn. + If we zip two maps together it's tempting to think that you will get the given + key in the left map and the matching key in the right map, but there is no such + guarantee because map keys are not ordered! Consider the following: + + left = %{:a => 1, 1 => 3} + right = %{:a => 1, :b => :c} + Enum.zip(left, right) + # [{{1, 3}, {:a, 1}}, {{:a, 1}, {:b, :c}}] + + As you can see `:a` does not get paired with `:a`. If this is what you want, + you should use `Map.merge/3`. ## Examples - iex> Enum.take_while([1, 2, 3], fn(x) -> x < 3 end) - [1, 2] + iex> Enum.zip_with([1, 2], [3, 4], fn x, y -> x + y end) + [4, 6] + + iex> Enum.zip_with([1, 2], [3, 4, 5, 6], fn x, y -> x + y end) + [4, 6] + + iex> Enum.zip_with([1, 2, 5, 6], [3, 4], fn x, y -> x + y end) + [4, 6] """ - @spec take_while(t, (element -> as_boolean(term))) :: list - def take_while(collection, fun) when is_list(collection) do - do_take_while(collection, fun) + @doc since: "1.12.0" + @spec zip_with(t, t, (enum1_elem :: term, enum2_elem :: term -> term)) :: [term] + def zip_with(enumerable1, enumerable2, zip_fun) + when is_list(enumerable1) and is_list(enumerable2) and is_function(zip_fun, 2) do + zip_with_list(enumerable1, enumerable2, zip_fun) end - def take_while(collection, fun) do - Enumerable.reduce(collection, {:cont, []}, R.take_while(fun)) - |> elem(1) |> :lists.reverse + def zip_with(enumerable1, enumerable2, zip_fun) when is_function(zip_fun, 2) do + zip_reduce(enumerable1, enumerable2, [], fn l, r, acc -> [zip_fun.(l, r) | acc] end) + |> :lists.reverse() end @doc """ - Convert `collection` to a list. + Zips corresponding elements from a finite collection of enumerables + into list, transforming them with the `zip_fun` function as it goes. + + The first element from each of the enums in `enumerables` will be put + into a list which is then passed to the one-arity `zip_fun` function. + Then, the second elements from each of the enums are put into a list + and passed to `zip_fun`, and so on until any one of the enums in + `enumerables` runs out of elements. + + Returns a list with all the results of calling `zip_fun`. ## Examples - iex> Enum.to_list(1 .. 3) - [1, 2, 3] + iex> Enum.zip_with([[1, 2], [3, 4], [5, 6]], fn [x, y, z] -> x + y + z end) + [9, 12] + + iex> Enum.zip_with([[1, 2], [3, 4]], fn [x, y] -> x + y end) + [4, 6] """ - @spec to_list(t) :: [term] - def to_list(collection) when is_list(collection) do - collection - end + @doc since: "1.12.0" + @spec zip_with(t, ([term] -> term)) :: [term] + def zip_with([], _fun), do: [] - def to_list(collection) do - reverse(collection) |> :lists.reverse + def zip_with(enumerables, zip_fun) do + zip_reduce(enumerables, [], fn values, acc -> [zip_fun.(values) | acc] end) + |> :lists.reverse() end - @doc """ - Traverses the given enumerable keeping its shape. + Reduces over two enumerables halting as soon as either enumerable is empty. + + In practice, the behaviour provided by this function can be achieved with: - It also expects the enumerable to implement the `Collectable` protocol. + Enum.reduce(Stream.zip(left, right), acc, reducer) + + But `zip_reduce/4` exists for convenience purposes. ## Examples - iex> Enum.traverse(%{a: 1, b: 2}, fn {k, v} -> {k, v * 2} end) - %{a: 2, b: 4} + iex> Enum.zip_reduce([1, 2], [3, 4], 0, fn x, y, acc -> x + y + acc end) + 10 + iex> Enum.zip_reduce([1, 2], [3, 4], [], fn x, y, acc -> [x + y | acc] end) + [6, 4] """ - @spec traverse(Enumerable.t, (term -> term)) :: Collectable.t - def traverse(collection, transform) when is_list(collection) do - :lists.map(transform, collection) + @doc since: "1.12.0" + @spec zip_reduce(t, t, acc, (enum1_elem :: term, enum2_elem :: term, acc -> acc)) :: acc + when acc: term + def zip_reduce(left, right, acc, reducer) + when is_list(left) and is_list(right) and is_function(reducer, 3) do + zip_reduce_list(left, right, acc, reducer) end - def traverse(collection, transform) do - into(collection, Collectable.empty(collection), transform) + def zip_reduce(left, right, acc, reducer) when is_function(reducer, 3) do + reduce = fn [l, r], acc -> {:cont, reducer.(l, r, acc)} end + Stream.zip_with([left, right], & &1).({:cont, acc}, reduce) |> elem(1) end @doc """ - Enumerates the collection, removing all duplicated items. + Reduces over all of the given enumerables, halting as soon as any enumerable is + empty. - ## Examples + The reducer will receive 2 args: a list of elements (one from each enum) and the + accumulator. - iex> Enum.uniq([1, 2, 3, 2, 1]) - [1, 2, 3] + In practice, the behaviour provided by this function can be achieved with: + + Enum.reduce(Stream.zip(enums), acc, reducer) + + But `zip_reduce/3` exists for convenience purposes. + + ## Examples - iex> Enum.uniq([{1, :x}, {2, :y}, {1, :z}], fn {x, _} -> x end) - [{1,:x}, {2,:y}] + iex> enums = [[1, 1], [2, 2], [3, 3]] + ...> Enum.zip_reduce(enums, [], fn elements, acc -> + ...> [List.to_tuple(elements) | acc] + ...> end) + [{1, 2, 3}, {1, 2, 3}] + iex> enums = [[1, 2], %{a: 3, b: 4}, [5, 6]] + ...> Enum.zip_reduce(enums, [], fn elements, acc -> + ...> [List.to_tuple(elements) | acc] + ...> end) + [{2, {:b, 4}, 6}, {1, {:a, 3}, 5}] """ - @spec uniq(t) :: list - @spec uniq(t, (element -> term)) :: list - def uniq(collection, fun \\ fn x -> x end) + @doc since: "1.12.0" + @spec zip_reduce(t, acc, ([term], acc -> acc)) :: acc when acc: term + def zip_reduce([], acc, reducer) when is_function(reducer, 2), do: acc - def uniq(collection, fun) when is_list(collection) do - do_uniq(collection, [], fun) + def zip_reduce(enums, acc, reducer) when is_function(reducer, 2) do + Stream.zip_with(enums, & &1).({:cont, acc}, &{:cont, reducer.(&1, &2)}) |> elem(1) end - def uniq(collection, fun) do - {_, {list, _}} = - Enumerable.reduce(collection, {:cont, {[], []}}, R.uniq(fun)) - :lists.reverse(list) + ## Helpers + + @compile {:inline, entry_to_string: 1, reduce: 3, reduce_by: 3, reduce_enumerable: 3} + + defp entry_to_string(entry) when is_binary(entry), do: entry + defp entry_to_string(entry), do: String.Chars.to_string(entry) + + defp aggregate([head | tail], fun, _empty) do + aggregate_list(tail, head, fun) end - @doc """ - Zips corresponding elements from two collections into one list - of tuples. + defp aggregate([], _fun, empty) do + empty.() + end - The zipping finishes as soon as any enumerable completes. + defp aggregate(first..last//step = range, fun, empty) do + case Range.size(range) do + 0 -> + empty.() - ## Examples + _ -> + last = last - rem(last - first, step) - iex> Enum.zip([1, 2, 3], [:a, :b, :c]) - [{1,:a},{2,:b},{3,:c}] + case fun.(first, last) do + true -> first + false -> last + end + end + end - iex> Enum.zip([1,2,3,4,5], [:a, :b, :c]) - [{1,:a},{2,:b},{3,:c}] + defp aggregate(enumerable, fun, empty) do + ref = make_ref() - """ - @spec zip(t, t) :: [{any, any}] - def zip(coll1, coll2) when is_list(coll1) and is_list(coll2) do - do_zip(coll1, coll2) + enumerable + |> reduce(ref, fn + element, ^ref -> + element + + element, acc -> + case fun.(acc, element) do + true -> acc + false -> element + end + end) + |> case do + ^ref -> empty.() + result -> result + end end - def zip(coll1, coll2) do - Stream.zip(coll1, coll2).({:cont, []}, &{:cont, [&1|&2]}) |> elem(1) |> :lists.reverse + defp aggregate_list([head | tail], acc, fun) do + acc = + case fun.(acc, head) do + true -> acc + false -> head + end + + aggregate_list(tail, acc, fun) end - @doc """ - Returns the collection with each element wrapped in a tuple - alongside its index. + defp aggregate_list([], acc, _fun), do: acc - ## Examples + defp aggregate_by(enumerable, fun, sorter, empty_fallback) do + first_fun = &[&1 | fun.(&1)] - iex> Enum.with_index [1,2,3] - [{1,0},{2,1},{3,2}] + reduce_fun = fn entry, [_ | fun_ref] = old -> + fun_entry = fun.(entry) - """ - @spec with_index(t) :: list({element, non_neg_integer}) - def with_index(collection) do - map_reduce(collection, 0, fn x, acc -> - {{x, acc}, acc + 1} - end) |> elem(0) + case sorter.(fun_ref, fun_entry) do + true -> old + false -> [entry | fun_entry] + end + end + + case reduce_by(enumerable, first_fun, reduce_fun) do + :empty -> empty_fallback.() + [entry | _] -> entry + end end - ## Helpers + defp reduce_by([head | tail], first, fun) do + :lists.foldl(fun, first.(head), tail) + end - @compile {:inline, enum_to_string: 1} + defp reduce_by([], _first, _fun) do + :empty + end + + defp reduce_by(enumerable, first, fun) do + reduce(enumerable, :empty, fn + element, :empty -> first.(element) + element, acc -> fun.(element, acc) + end) + end - defp enumerate_and_count(collection, count) when is_list(collection) do - {collection, length(collection) - abs(count)} + defp random_integer(limit, limit) when is_integer(limit) do + limit end - defp enumerate_and_count(collection, count) do - map_reduce(collection, -abs(count), fn(x, acc) -> {x, acc + 1} end) + defp random_integer(lower_limit, upper_limit) when upper_limit < lower_limit do + random_integer(upper_limit, lower_limit) end - defp enum_to_string(entry) when is_binary(entry), do: entry - defp enum_to_string(entry), do: String.Chars.to_string(entry) + defp random_integer(lower_limit, upper_limit) do + lower_limit + :rand.uniform(upper_limit - lower_limit + 1) - 1 + end ## Implementations ## all? - defp do_all?([h|t], fun) do + defp all_list([h | t]) do + if h do + all_list(t) + else + false + end + end + + defp all_list([]) do + true + end + + defp all_list([h | t], fun) do if fun.(h) do - do_all?(t, fun) + all_list(t, fun) else false end end - defp do_all?([], _) do + defp all_list([], _) do true end ## any? - defp do_any?([h|t], fun) do + defp any_list([h | t]) do + if h do + true + else + any_list(t) + end + end + + defp any_list([]) do + false + end + + defp any_list([h | t], fun) do if fun.(h) do true else - do_any?(t, fun) + any_list(t, fun) end end - defp do_any?([], _) do + defp any_list([], _) do false end - ## fetch + ## concat - defp do_fetch([h|_], 0), do: {:ok, h} - defp do_fetch([_|t], n), do: do_fetch(t, n - 1) - defp do_fetch([], _), do: :error + defp concat_list([h | t]) when is_list(h), do: h ++ concat_list(t) + defp concat_list([h | t]), do: concat_enum([h | t]) + defp concat_list([]), do: [] - ## drop + defp concat_enum(enum) do + fun = &[&1 | &2] + enum |> reduce([], &reduce(&1, &2, fun)) |> :lists.reverse() + end + + # dedup - defp do_drop([_|t], counter) when counter > 0 do - do_drop(t, counter - 1) + defp dedup_list([value | tail], acc) do + acc = + case acc do + [^value | _] -> acc + _ -> [value | acc] + end + + dedup_list(tail, acc) + end + + defp dedup_list([], acc) do + acc end - defp do_drop(list, 0) do - list + ## drop + + defp drop_list(list, 0), do: list + defp drop_list([_ | tail], counter), do: drop_list(tail, counter - 1) + defp drop_list([], _), do: [] + + ## drop_while + + defp drop_while_list([head | tail], fun) do + if fun.(head) do + drop_while_list(tail, fun) + else + [head | tail] + end end - defp do_drop([], _) do + defp drop_while_list([], _) do [] end - ## drop_while + ## filter - defp do_drop_while([h|t], fun) do - if fun.(h) do - do_drop_while(t, fun) + defp filter_list([head | tail], fun) do + if fun.(head) do + [head | filter_list(tail, fun)] else - [h|t] + filter_list(tail, fun) end end - defp do_drop_while([], _) do + defp filter_list([], _fun) do [] end ## find - defp do_find([h|t], ifnone, fun) do - if fun.(h) do - h + defp find_list([head | tail], default, fun) do + if fun.(head) do + head else - do_find(t, ifnone, fun) + find_list(tail, default, fun) end end - defp do_find([], ifnone, _) do - ifnone + defp find_list([], default, _) do + default end ## find_index - defp do_find_index([h|t], counter, fun) do - if fun.(h) do + defp find_index_list([head | tail], counter, fun) do + if fun.(head) do counter else - do_find_index(t, counter + 1, fun) + find_index_list(tail, counter + 1, fun) end end - defp do_find_index([], _, _) do + defp find_index_list([], _, _) do nil end ## find_value - defp do_find_value([h|t], ifnone, fun) do - fun.(h) || do_find_value(t, ifnone, fun) + defp find_value_list([head | tail], default, fun) do + fun.(head) || find_value_list(tail, default, fun) + end + + defp find_value_list([], default, _) do + default + end + + ## flat_map + + defp flat_map_list([head | tail], fun) do + case fun.(head) do + list when is_list(list) -> list ++ flat_map_list(tail, fun) + other -> to_list(other) ++ flat_map_list(tail, fun) + end + end + + defp flat_map_list([], _fun) do + [] + end + + ## intersperse + + defp intersperse_non_empty_list([head], _separator), do: [head] + + defp intersperse_non_empty_list([head | rest], separator) do + [head, separator | intersperse_non_empty_list(rest, separator)] + end + + ## join + + defp join_list([], _joiner), do: "" + + defp join_list(list, joiner) do + join_non_empty_list(list, joiner, []) + |> :lists.reverse() + |> IO.iodata_to_binary() + end + + defp join_non_empty_list([first], _joiner, acc), do: [entry_to_string(first) | acc] + + defp join_non_empty_list([first | rest], joiner, acc) do + join_non_empty_list(rest, joiner, [joiner, entry_to_string(first) | acc]) + end + + ## map_intersperse + + defp map_intersperse_list([], _, _), + do: [] + + defp map_intersperse_list([last], _, mapper), + do: [mapper.(last)] + + defp map_intersperse_list([head | rest], separator, mapper), + do: [mapper.(head), separator | map_intersperse_list(rest, separator, mapper)] + + ## reduce + + defp reduce_range(first, last, step, acc, fun) + when step > 0 and first <= last + when step < 0 and first >= last do + reduce_range(first + step, last, step, fun.(first, acc), fun) + end + + defp reduce_range(_first, _last, _step, acc, _fun) do + acc + end + + defp reduce_enumerable(enumerable, acc, fun) do + Enumerable.reduce(enumerable, {:cont, acc}, fn x, acc -> {:cont, fun.(x, acc)} end) |> elem(1) + end + + ## reject + + defp reject_list([head | tail], fun) do + if fun.(head) do + reject_list(tail, fun) + else + [head | reject_list(tail, fun)] + end + end + + defp reject_list([], _fun) do + [] end - defp do_find_value([], ifnone, _) do - ifnone + ## reverse_slice + + defp reverse_slice(rest, idx, idx, count, acc) do + {slice, rest} = head_slice(rest, count, []) + :lists.reverse(rest, :lists.reverse(slice, acc)) + end + + defp reverse_slice([elem | rest], idx, start, count, acc) do + reverse_slice(rest, idx - 1, start, count, [elem | acc]) + end + + defp head_slice(rest, 0, acc), do: {acc, rest} + + defp head_slice([elem | rest], count, acc) do + head_slice(rest, count - 1, [elem | acc]) + end + + ## scan + + defp scan_list([], _acc, _fun), do: [] + + defp scan_list([elem | rest], acc, fun) do + acc = fun.(elem, acc) + [acc | scan_list(rest, acc, fun)] end ## shuffle - defp unwrap([{_, h} | collection], t) do - unwrap(collection, [h|t]) + defp shuffle_unwrap([{_, h} | enumerable], t) do + shuffle_unwrap(enumerable, [h | t]) + end + + defp shuffle_unwrap([], t), do: t + + ## slice + + defp slice_forward(enumerable, start, amount, step) when start < 0 do + {count, fun} = slice_count_and_fun(enumerable, step) + start = count + start + + if start >= 0 do + amount = Kernel.min(amount, count - start) + amount = if step == 1, do: amount, else: div(amount - 1, step) + 1 + fun.(start, amount, step) + else + [] + end + end + + defp slice_forward(list, start, amount, step) when is_list(list) do + amount = if step == 1, do: amount, else: div(amount - 1, step) + 1 + slice_list(list, start, amount, step) + end + + defp slice_forward(enumerable, start, amount, step) do + case Enumerable.slice(enumerable) do + {:ok, count, _} when start >= count -> + [] + + {:ok, count, fun} when is_function(fun, 1) -> + amount = Kernel.min(amount, count - start) + enumerable |> fun.() |> slice_exact(start, amount, step, count) + + # TODO: Deprecate me in Elixir v1.18. + {:ok, count, fun} when is_function(fun, 2) -> + amount = Kernel.min(amount, count - start) + + if step == 1 do + fun.(start, amount) + else + fun.(start, Kernel.min(amount * step, count - start)) + |> take_every_list(amount, step - 1) + end + + {:ok, count, fun} when is_function(fun, 3) -> + amount = Kernel.min(amount, count - start) + amount = if step == 1, do: amount, else: div(amount - 1, step) + 1 + fun.(start, amount, step) + + {:error, module} -> + slice_enum(enumerable, module, start, amount, step) + end + end + + defp slice_list(list, start, amount, step) do + if step == 1 do + list |> drop_list(start) |> take_list(amount) + else + list |> drop_list(start) |> take_every_list(amount, step - 1) + end + end + + defp slice_enum(enumerable, module, start, amount, 1) do + {_, {_, _, slice}} = + module.reduce(enumerable, {:cont, {start, amount, []}}, fn + _entry, {start, amount, _list} when start > 0 -> + {:cont, {start - 1, amount, []}} + + entry, {start, amount, list} when amount > 1 -> + {:cont, {start, amount - 1, [entry | list]}} + + entry, {start, amount, list} -> + {:halt, {start, amount, [entry | list]}} + end) + + :lists.reverse(slice) + end + + defp slice_enum(enumerable, module, start, amount, step) do + {_, {_, _, _, slice}} = + module.reduce(enumerable, {:cont, {start, amount, 1, []}}, fn + _entry, {start, amount, to_drop, _list} when start > 0 -> + {:cont, {start - 1, amount, to_drop, []}} + + entry, {start, amount, to_drop, list} when amount > 1 -> + case to_drop do + 1 -> {:cont, {start, amount - 1, step, [entry | list]}} + _ -> {:cont, {start, amount - 1, to_drop - 1, list}} + end + + entry, {start, amount, to_drop, list} -> + {:halt, {start, amount, to_drop, [entry | list]}} + end) + + :lists.reverse(slice) + end + + defp slice_count_and_fun(list, _step) when is_list(list) do + length = length(list) + {length, &slice_exact(list, &1, &2, &3, length)} + end + + defp slice_count_and_fun(enumerable, step) do + case Enumerable.slice(enumerable) do + {:ok, count, fun} when is_function(fun, 3) -> + {count, fun} + + # TODO: Deprecate me in Elixir v1.18. + {:ok, count, fun} when is_function(fun, 2) -> + if step == 1 do + {count, fn start, amount, 1 -> fun.(start, amount) end} + else + {count, + fn start, amount, step -> + fun.(start, Kernel.min(amount * step, count - start)) + |> take_every_list(amount, step - 1) + end} + end + + {:ok, count, fun} when is_function(fun, 1) -> + {count, &slice_exact(fun.(enumerable), &1, &2, &3, count)} + + {:error, module} -> + {list, count} = + if step == 1 do + enumerable + |> module.reduce({:cont, {[], 0}}, fn elem, {acc, count} -> + {:cont, {[elem | acc], count + 1}} + end) + |> elem(1) + else + # We want to count all elements but we only keep the ones we need + {_, {list, _, count}} = + module.reduce(enumerable, {:cont, {[], 1, 0}}, fn + elem, {acc, 1, count} -> + {:cont, {[elem | acc], step, count + 1}} + + _elem, {acc, to_drop, count} -> + {:cont, {acc, to_drop - 1, count + 1}} + end) + + {list, count} + end + + {count, + fn start, amount, _step -> + list |> :lists.reverse() |> slice_exact(start, amount, 1, 1) + end} + end end - defp unwrap([], t), do: t + # Slice a list when we know the bounds + defp slice_exact(_list, _start, 0, _step, _), do: [] + + defp slice_exact(list, start, amount, 1, size) when start + amount == size, + do: list |> drop_exact(start) + + defp slice_exact(list, start, amount, 1, _), + do: list |> drop_exact(start) |> take_exact(amount) + + defp slice_exact(list, start, amount, step, _), + do: list |> drop_exact(start) |> take_every_list(amount, step - 1) + + defp drop_exact(list, 0), do: list + defp drop_exact([_ | tail], amount), do: drop_exact(tail, amount - 1) + + defp take_exact(_list, 0), do: [] + defp take_exact([head | tail], amount), do: [head | take_exact(tail, amount - 1)] ## sort defp sort_reducer(entry, {:split, y, x, r, rs, bool}, fun) do cond do fun.(y, entry) == bool -> - {:split, entry, y, [x|r], rs, bool} + {:split, entry, y, [x | r], rs, bool} + fun.(x, entry) == bool -> - {:split, y, entry, [x|r], rs, bool} + {:split, y, entry, [x | r], rs, bool} + r == [] -> {:split, y, x, [entry], rs, bool} + true -> {:pivot, y, x, r, rs, entry, bool} end @@ -1981,10 +4534,13 @@ defmodule Enum do cond do fun.(y, entry) == bool -> {:pivot, entry, y, [x | r], rs, s, bool} + fun.(x, entry) == bool -> {:pivot, y, entry, [x | r], rs, s, bool} + fun.(s, entry) == bool -> {:split, entry, s, [], [[y, x | r] | rs], bool} + true -> {:split, s, entry, [], [[y, x | r] | rs], bool} end @@ -1995,7 +4551,7 @@ defmodule Enum do end defp sort_reducer(entry, acc, _fun) do - [entry|acc] + [entry | acc] end defp sort_terminator({:split, y, x, r, rs, bool}, fun) do @@ -2010,214 +4566,229 @@ defmodule Enum do acc end - defp sort_merge(list, fun, true), do: - reverse_sort_merge(list, [], fun, true) - - defp sort_merge(list, fun, false), do: - sort_merge(list, [], fun, false) + defp sort_merge(list, fun, true), do: reverse_sort_merge(list, [], fun, true) + defp sort_merge(list, fun, false), do: sort_merge(list, [], fun, false) - defp sort_merge([t1, [h2 | t2] | l], acc, fun, true), do: - sort_merge(l, [sort_merge_1(t1, h2, t2, [], fun, false) | acc], fun, true) + defp sort_merge([t1, [h2 | t2] | l], acc, fun, true), + do: sort_merge(l, [sort_merge1(t1, h2, t2, [], fun, false) | acc], fun, true) - defp sort_merge([[h2 | t2], t1 | l], acc, fun, false), do: - sort_merge(l, [sort_merge_1(t1, h2, t2, [], fun, false) | acc], fun, false) + defp sort_merge([[h2 | t2], t1 | l], acc, fun, false), + do: sort_merge(l, [sort_merge1(t1, h2, t2, [], fun, false) | acc], fun, false) defp sort_merge([l], [], _fun, _bool), do: l - defp sort_merge([l], acc, fun, bool), do: - reverse_sort_merge([:lists.reverse(l, []) | acc], [], fun, bool) + defp sort_merge([l], acc, fun, bool), + do: reverse_sort_merge([:lists.reverse(l, []) | acc], [], fun, bool) - defp sort_merge([], acc, fun, bool), do: - reverse_sort_merge(acc, [], fun, bool) + defp sort_merge([], acc, fun, bool), do: reverse_sort_merge(acc, [], fun, bool) + defp reverse_sort_merge([[h2 | t2], t1 | l], acc, fun, true), + do: reverse_sort_merge(l, [sort_merge1(t1, h2, t2, [], fun, true) | acc], fun, true) - defp reverse_sort_merge([[h2 | t2], t1 | l], acc, fun, true), do: - reverse_sort_merge(l, [sort_merge_1(t1, h2, t2, [], fun, true) | acc], fun, true) + defp reverse_sort_merge([t1, [h2 | t2] | l], acc, fun, false), + do: reverse_sort_merge(l, [sort_merge1(t1, h2, t2, [], fun, true) | acc], fun, false) - defp reverse_sort_merge([t1, [h2 | t2] | l], acc, fun, false), do: - reverse_sort_merge(l, [sort_merge_1(t1, h2, t2, [], fun, true) | acc], fun, false) + defp reverse_sort_merge([l], acc, fun, bool), + do: sort_merge([:lists.reverse(l, []) | acc], [], fun, bool) - defp reverse_sort_merge([l], acc, fun, bool), do: - sort_merge([:lists.reverse(l, []) | acc], [], fun, bool) + defp reverse_sort_merge([], acc, fun, bool), do: sort_merge(acc, [], fun, bool) - defp reverse_sort_merge([], acc, fun, bool), do: - sort_merge(acc, [], fun, bool) - - - defp sort_merge_1([h1 | t1], h2, t2, m, fun, bool) do + defp sort_merge1([h1 | t1], h2, t2, m, fun, bool) do if fun.(h1, h2) == bool do - sort_merge_2(h1, t1, t2, [h2 | m], fun, bool) + sort_merge2(h1, t1, t2, [h2 | m], fun, bool) else - sort_merge_1(t1, h2, t2, [h1 | m], fun, bool) + sort_merge1(t1, h2, t2, [h1 | m], fun, bool) end end - defp sort_merge_1([], h2, t2, m, _fun, _bool), do: - :lists.reverse(t2, [h2 | m]) + defp sort_merge1([], h2, t2, m, _fun, _bool), do: :lists.reverse(t2, [h2 | m]) - - defp sort_merge_2(h1, t1, [h2 | t2], m, fun, bool) do + defp sort_merge2(h1, t1, [h2 | t2], m, fun, bool) do if fun.(h1, h2) == bool do - sort_merge_2(h1, t1, t2, [h2 | m], fun, bool) + sort_merge2(h1, t1, t2, [h2 | m], fun, bool) else - sort_merge_1(t1, h2, t2, [h1 | m], fun, bool) + sort_merge1(t1, h2, t2, [h1 | m], fun, bool) end end - defp sort_merge_2(h1, t1, [], m, _fun, _bool), do: - :lists.reverse(t1, [h1 | m]) + defp sort_merge2(h1, t1, [], m, _fun, _bool), do: :lists.reverse(t1, [h1 | m]) ## split - defp do_split([h|t], counter, acc) when counter > 0 do - do_split(t, counter - 1, [h|acc]) + defp split_list([head | tail], counter, acc) when counter > 0 do + split_list(tail, counter - 1, [head | acc]) end - defp do_split(list, 0, acc) do + defp split_list(list, 0, acc) do {:lists.reverse(acc), list} end - defp do_split([], _, acc) do + defp split_list([], _, acc) do {:lists.reverse(acc), []} end - defp do_split_reverse([h|t], counter, acc) when counter > 0 do - do_split_reverse(t, counter - 1, [h|acc]) + defp split_reverse_list([head | tail], counter, acc) when counter > 0 do + split_reverse_list(tail, counter - 1, [head | acc]) end - defp do_split_reverse(list, 0, acc) do + defp split_reverse_list(list, 0, acc) do {:lists.reverse(list), acc} end - defp do_split_reverse([], _, acc) do + defp split_reverse_list([], _, acc) do {[], acc} end ## split_while - defp do_split_while([h|t], fun, acc) do - if fun.(h) do - do_split_while(t, fun, [h|acc]) + defp split_while_list([head | tail], fun, acc) do + if fun.(head) do + split_while_list(tail, fun, [head | acc]) else - {:lists.reverse(acc), [h|t]} + {:lists.reverse(acc), [head | tail]} end end - defp do_split_while([], _, acc) do + defp split_while_list([], _, acc) do {:lists.reverse(acc), []} end ## take - defp do_take([h|t], counter) when counter > 0 do - [h|do_take(t, counter - 1)] - end + defp take_list(_list, 0), do: [] + defp take_list([head | tail], counter), do: [head | take_list(tail, counter - 1)] + defp take_list([], _counter), do: [] - defp do_take(_list, 0) do - [] - end + defp take_every_list([head | tail], to_drop), + do: [head | tail |> drop_list(to_drop) |> take_every_list(to_drop)] - defp do_take([], _) do - [] - end + defp take_every_list([], _to_drop), do: [] + + defp take_every_list(_list, 0, _to_drop), do: [] + + defp take_every_list([head | tail], counter, to_drop), + do: [head | tail |> drop_list(to_drop) |> take_every_list(counter - 1, to_drop)] + + defp take_every_list([], _counter, _to_drop), do: [] ## take_while - defp do_take_while([h|t], fun) do - if fun.(h) do - [h|do_take_while(t, fun)] + defp take_while_list([head | tail], fun) do + if fun.(head) do + [head | take_while_list(tail, fun)] else [] end end - defp do_take_while([], _) do + defp take_while_list([], _) do [] end ## uniq - defp do_uniq([h|t], acc, fun) do - fun_h = fun.(h) - case :lists.member(fun_h, acc) do - true -> do_uniq(t, acc, fun) - false -> [h|do_uniq(t, [fun_h|acc], fun)] + defp uniq_list([head | tail], set, fun) do + value = fun.(head) + + case set do + %{^value => true} -> uniq_list(tail, set, fun) + %{} -> [head | uniq_list(tail, Map.put(set, value, true), fun)] end end - defp do_uniq([], _acc, _fun) do + defp uniq_list([], _set, _fun) do [] end - ## zip + ## with_index - defp do_zip([h1|next1], [h2|next2]) do - [{h1, h2}|do_zip(next1, next2)] + defp with_index_list([head | tail], offset) do + [{head, offset} | with_index_list(tail, offset + 1)] end - defp do_zip(_, []), do: [] - defp do_zip([], _), do: [] - - ## slice + defp with_index_list([], _offset), do: [] - defp do_slice([], _start, _count) do - [] + defp with_index_list([head | tail], offset, fun) do + [fun.(head, offset) | with_index_list(tail, offset + 1, fun)] end - defp do_slice(_list, _start, 0) do - [] + defp with_index_list([], _offset, _fun), do: [] + + ## zip + + defp zip_list([head1 | next1], [head2 | next2], acc) do + zip_list(next1, next2, [{head1, head2} | acc]) end - defp do_slice([h|t], 0, count) do - [h|do_slice(t, 0, count-1)] + defp zip_list([], _, acc), do: :lists.reverse(acc) + defp zip_list(_, [], acc), do: :lists.reverse(acc) + + defp zip_with_list([head1 | next1], [head2 | next2], fun) do + [fun.(head1, head2) | zip_with_list(next1, next2, fun)] end - defp do_slice([_|t], start, count) do - do_slice(t, start-1, count) + defp zip_with_list(_, [], _fun), do: [] + defp zip_with_list([], _, _fun), do: [] + + defp zip_reduce_list([head1 | next1], [head2 | next2], acc, fun) do + zip_reduce_list(next1, next2, fun.(head1, head2, acc), fun) end + + defp zip_reduce_list(_, [], acc, _fun), do: acc + defp zip_reduce_list([], _, acc, _fun), do: acc end defimpl Enumerable, for: List do - def reduce(_, {:halt, acc}, _fun), do: {:halted, acc} - def reduce(list, {:suspend, acc}, fun), do: {:suspended, acc, &reduce(list, &1, fun)} - def reduce([], {:cont, acc}, _fun), do: {:done, acc} - def reduce([h|t], {:cont, acc}, fun), do: reduce(t, fun.(h, acc), fun) - - def member?(_list, _value), - do: {:error, __MODULE__} - def count(_list), - do: {:error, __MODULE__} + def count([]), do: {:ok, 0} + def count(_list), do: {:error, __MODULE__} + + def member?([], _value), do: {:ok, false} + def member?(_list, _value), do: {:error, __MODULE__} + + def slice([]), do: {:ok, 0, fn _, _, _ -> [] end} + def slice(_list), do: {:error, __MODULE__} + + def reduce(_list, {:halt, acc}, _fun), do: {:halted, acc} + def reduce(list, {:suspend, acc}, fun), do: {:suspended, acc, &reduce(list, &1, fun)} + def reduce([], {:cont, acc}, _fun), do: {:done, acc} + def reduce([head | tail], {:cont, acc}, fun), do: reduce(tail, fun.(head, acc), fun) end defimpl Enumerable, for: Map do - def reduce(map, acc, fun) do - do_reduce(:maps.to_list(map), acc, fun) + def count(map) do + {:ok, map_size(map)} end - defp do_reduce(_, {:halt, acc}, _fun), do: {:halted, acc} - defp do_reduce(list, {:suspend, acc}, fun), do: {:suspended, acc, &do_reduce(list, &1, fun)} - defp do_reduce([], {:cont, acc}, _fun), do: {:done, acc} - defp do_reduce([h|t], {:cont, acc}, fun), do: do_reduce(t, fun.(h, acc), fun) - def member?(map, {key, value}) do - {:ok, match?({:ok, ^value}, :maps.find(key, map))} + {:ok, match?(%{^key => ^value}, map)} end def member?(_map, _other) do {:ok, false} end - def count(map) do - {:ok, map_size(map)} + def slice(map) do + size = map_size(map) + {:ok, size, &:maps.to_list/1} + end + + def reduce(map, acc, fun) do + Enumerable.List.reduce(:maps.to_list(map), acc, fun) end end defimpl Enumerable, for: Function do - def reduce(function, acc, fun) when is_function(function, 2), - do: function.(acc, fun) - def member?(_function, _value), - do: {:error, __MODULE__} - def count(_function), - do: {:error, __MODULE__} + def count(_function), do: {:error, __MODULE__} + def member?(_function, _value), do: {:error, __MODULE__} + def slice(_function), do: {:error, __MODULE__} + + def reduce(function, acc, fun) when is_function(function, 2), do: function.(acc, fun) + + def reduce(function, _acc, _fun) do + raise Protocol.UndefinedError, + protocol: @protocol, + value: function, + description: "only anonymous functions of arity 2 are enumerable" + end end diff --git a/lib/elixir/lib/exception.ex b/lib/elixir/lib/exception.ex index 398f0f1f5a6..0c1cc3c31b5 100644 --- a/lib/elixir/lib/exception.ex +++ b/lib/elixir/lib/exception.ex @@ -2,52 +2,81 @@ defmodule Exception do @moduledoc """ Functions to format throw/catch/exit and exceptions. - Note that stacktraces in Elixir are updated on throw, - errors and exits. For example, at any given moment, - `System.stacktrace` will return the stacktrace for the - last throw/error/exit that ocurred in the current process. + Note that stacktraces in Elixir are only available inside + catch and rescue by using the `__STACKTRACE__/0` variable. - Do not rely on the particular format returned by the `format` + Do not rely on the particular format returned by the `format*` functions in this module. They may be changed in future releases in order to better suit Elixir's tool chain. In other words, - by using the functions in this module it is guarantee you will + by using the functions in this module it is guaranteed you will format exceptions as in the current Elixir version being used. """ - @typedoc "The exception type (as generated by defexception)" - @type t :: %{__struct__: module, __exception__: true} + @typedoc "The exception type" + @type t :: %{ + required(:__struct__) => module, + required(:__exception__) => true, + optional(atom) => any + } @typedoc "The kind handled by formatting functions" - @type kind :: :error | :exit | :throw | {:EXIT, pid} + @type kind :: :error | non_error_kind + @type non_error_kind :: :exit | :throw | {:EXIT, pid} @type stacktrace :: [stacktrace_entry] @type stacktrace_entry :: - {module, function, arity_or_args, location} | - {function, arity_or_args, location} + {module, atom, arity_or_args, location} + | {(... -> any), arity_or_args, location} - @typep arity_or_args :: non_neg_integer | list - @typep location :: Keyword.t + @type arity_or_args :: non_neg_integer | list + @type location :: keyword @callback exception(term) :: t - @callback message(t) :: String.t + @callback message(t) :: String.t() @doc """ - Returns true if the given argument is an exception. + Called from `Exception.blame/3` to augment the exception struct. + + Can be used to collect additional information about the exception + or do some additional expensive computation. + """ + @callback blame(t, stacktrace) :: {t, stacktrace} + @optional_callbacks [blame: 2] + + @doc false + # Callback for formatting Erlang exceptions + def format_error(%struct{} = exception, _stacktrace) do + %{general: message(exception), reason: "#" <> Atom.to_string(struct)} + end + + @doc """ + Returns `true` if the given `term` is an exception. """ - def exception?(%{__struct__: struct, __exception__: true}) when is_atom(struct), do: true + # TODO: Deprecate this on Elixir v1.15 + @doc deprecated: "Use Kernel.is_exception/1 instead" + def exception?(term) + def exception?(%_{__exception__: true}), do: true def exception?(_), do: false @doc """ - Gets the message for an exception. + Gets the message for an `exception`. """ - def message(%{__struct__: module, __exception__: true} = exception) when is_atom(module) do + def message(%module{__exception__: true} = exception) do try do module.message(exception) rescue - e -> - raise ArgumentError, - "Got #{inspect e.__struct__} with message " <> - "\"#{message(e)}\" while retrieving message for #{inspect(exception)}" + caught_exception -> + "got #{inspect(caught_exception.__struct__)} with message " <> + "#{inspect(message(caught_exception))} while retrieving Exception.message/1 " <> + "for #{inspect(exception)}. Stacktrace:\n#{format_stacktrace(__STACKTRACE__)}" + else + result when is_binary(result) -> + result + + result -> + "got #{inspect(result)} " <> + "while retrieving Exception.message/1 for #{inspect(exception)} " <> + "(expected a string)" end end @@ -59,47 +88,29 @@ defmodule Exception do normalizes only `:error`, returning the untouched payload for others. - The third argument, a stacktrace, is optional. If it is - not supplied `System.stacktrace/0` will sometimes be used - to get additional information for the `kind` `:error`. If - the stacktrace is unknown and `System.stacktrace/0` would - not return the stacktrace corresponding to the exception - an empty stacktrace, `[]`, must be used. + The third argument is the stacktrace which is used to enrich + a normalized error with more information. It is only used when + the kind is an error. """ @spec normalize(:error, any, stacktrace) :: t - @spec normalize(kind, payload, stacktrace) :: payload when payload: var - - # Generating a stacktrace is expensive, default to nil - # to only fetch it when needed. - def normalize(kind, payload, stacktrace \\ nil) - - def normalize(:error, exception, stacktrace) do - if exception?(exception) do - exception - else - ErlangError.normalize(exception, stacktrace) - end - end - - def normalize(_kind, payload, _stacktrace) do - payload - end + @spec normalize(non_error_kind, payload, stacktrace) :: payload when payload: var + def normalize(kind, payload, stacktrace \\ []) + def normalize(:error, %_{__exception__: true} = payload, _stacktrace), do: payload + def normalize(:error, payload, stacktrace), do: ErlangError.normalize(payload, stacktrace) + def normalize(_kind, payload, _stacktrace), do: payload @doc """ - Normalizes and formats any throw, error and exit. + Normalizes and formats any throw/error/exit. The message is formatted and displayed in the same format as used by Elixir's CLI. - The third argument, a stacktrace, is optional. If it is - not supplied `System.stacktrace/0` will sometimes be used - to get additional information for the `kind` `:error`. If - the stacktrace is unknown and `System.stacktrace/0` would - not return the stacktrace corresponding to the exception - an empty stacktrace, `[]`, must be used. + The third argument is the stacktrace which is used to enrich + a normalized error with more information. It is only used when + the kind is an error. """ - @spec format_banner(kind, any, stacktrace | nil) :: String.t - def format_banner(kind, exception, stacktrace \\ nil) + @spec format_banner(kind, any, stacktrace) :: String.t() + def format_banner(kind, exception, stacktrace \\ []) def format_banner(:error, exception, stacktrace) do exception = normalize(:error, exception, stacktrace) @@ -107,7 +118,7 @@ defmodule Exception do end def format_banner(:throw, reason, _stacktrace) do - "** (throw) " <> inspect(reason) + "** (throw) " <> inspect(reason) end def format_banner(:exit, reason, _stacktrace) do @@ -115,98 +126,339 @@ defmodule Exception do end def format_banner({:EXIT, pid}, reason, _stacktrace) do - "** (EXIT from #{inspect pid}) " <> format_exit(reason, <<"\n ">>) + "** (EXIT from #{inspect(pid)}) " <> format_exit(reason, <<"\n ">>) end @doc """ - Normalizes and formats throw/errors/exits and stacktrace. + Normalizes and formats throw/errors/exits and stacktraces. It relies on `format_banner/3` and `format_stacktrace/1` to generate the final format. - Note that `{:EXIT, pid}` do not generate a stacktrace though - (as they are retrieved as messages without stacktraces). + If `kind` is `{:EXIT, pid}`, it does not generate a stacktrace, + as such exits are retrieved as messages without stacktraces. """ - - @spec format(kind, any, stacktrace | nil) :: String.t - - def format(kind, payload, stacktrace \\ nil) + @spec format(kind, any, stacktrace) :: String.t() + def format(kind, payload, stacktrace \\ []) def format({:EXIT, _} = kind, any, _) do format_banner(kind, any) end def format(kind, payload, stacktrace) do - stacktrace = stacktrace || System.stacktrace message = format_banner(kind, payload, stacktrace) + case stacktrace do [] -> message - _ -> message <> "\n" <> format_stacktrace(stacktrace) + _ -> message <> "\n" <> format_stacktrace(stacktrace) end end @doc """ - Formats an exit, returns a string. + Attaches information to exceptions for extra debugging. + + This operation is potentially expensive, as it reads data + from the file system, parses beam files, evaluates code and + so on. + + If the exception module implements the optional `c:blame/2` + callback, it will be invoked to perform the computation. + """ + @doc since: "1.5.0" + @spec blame(:error, any, stacktrace) :: {t, stacktrace} + @spec blame(non_error_kind, payload, stacktrace) :: {payload, stacktrace} when payload: var + def blame(kind, error, stacktrace) + + def blame(:error, error, stacktrace) do + %module{} = struct = normalize(:error, error, stacktrace) + + if Code.ensure_loaded?(module) and function_exported?(module, :blame, 2) do + module.blame(struct, stacktrace) + else + {struct, stacktrace} + end + end + + def blame(_kind, reason, stacktrace) do + {reason, stacktrace} + end + + @doc """ + Blames the invocation of the given module, function and arguments. + + This function will retrieve the available clauses from bytecode + and evaluate them against the given arguments. The clauses are + returned as a list of `{args, guards}` pairs where each argument + and each top-level condition in a guard separated by `and`/`or` + is wrapped in a tuple with blame metadata. + + This function returns either `{:ok, definition, clauses}` or `:error`. + Where `definition` is `:def`, `:defp`, `:defmacro` or `:defmacrop`. + """ + @doc since: "1.5.0" + @spec blame_mfa(module, function :: atom, args :: [term]) :: + {:ok, :def | :defp | :defmacro | :defmacrop, [{args :: [term], guards :: [term]}]} + | :error + def blame_mfa(module, function, args) + when is_atom(module) and is_atom(function) and is_list(args) do + try do + blame_mfa(module, function, length(args), args) + rescue + _ -> :error + end + end + + defp blame_mfa(module, function, arity, call_args) do + with [_ | _] = path <- :code.which(module), + {:ok, {_, [debug_info: debug_info]}} <- :beam_lib.chunks(path, [:debug_info]), + {:debug_info_v1, backend, data} <- debug_info, + {:ok, %{definitions: defs}} <- backend.debug_info(:elixir_v1, module, data, []), + {_, kind, _, clauses} <- List.keyfind(defs, {function, arity}, 0) do + clauses = + for {meta, ex_args, guards, _block} <- clauses do + scope = :elixir_erl.scope(meta, true) + ann = :elixir_erl.get_ann(meta) + + {erl_args, scope} = + :elixir_erl_clauses.match(ann, &:elixir_erl_pass.translate_args/3, ex_args, scope) + + {args, binding} = + [call_args, ex_args, erl_args] + |> Enum.zip() + |> Enum.map_reduce([], &blame_arg/2) + + guards = + guards + |> Enum.map(&blame_guard(&1, ann, scope, binding)) + |> Enum.map(&Macro.prewalk(&1, fn guard -> translate_guard(guard) end)) + + {args, guards} + end + + {:ok, kind, clauses} + else + _ -> :error + end + end + + defp is_map_node?({:is_map, _, [_]}), do: true + defp is_map_node?(_), do: false + defp is_map_key_node?({:is_map_key, _, [_, _]}), do: true + defp is_map_key_node?(_), do: false + + defp struct_validation_node?( + {:is_atom, _, [{{:., [], [:erlang, :map_get]}, _, [:__struct__, _]}]} + ), + do: true + + defp struct_validation_node?( + {:==, _, [{{:., [], [:erlang, :map_get]}, _, [:__struct__, _]}, _module]} + ), + do: true + + defp struct_validation_node?(_), do: false + + defp is_struct_macro?( + {:and, _, + [ + {:and, _, [%{node: node_1 = {_, _, [arg]}}, %{node: node_2 = {_, _, [arg, _]}}]}, + %{node: node_3 = {_, _, [{_, _, [_, arg]}]}} + ]} + ), + do: is_map_node?(node_1) and is_map_key_node?(node_2) and struct_validation_node?(node_3) + + defp is_struct_macro?( + {:and, _, + [ + {:and, _, + [ + {:and, _, + [ + %{node: node_1 = {_, _, [arg]}}, + {:or, _, [%{node: {:is_atom, _, [_]}}, %{node: :fail}]} + ]}, + %{node: node_2 = {_, _, [arg, _]}} + ]}, + %{node: node_3 = {_, _, [{_, _, [_, arg]}, _]}} + ]} + ), + do: is_map_node?(node_1) and is_map_key_node?(node_2) and struct_validation_node?(node_3) + + defp is_struct_macro?(_), do: false + + defp translate_guard(guard) do + if is_struct_macro?(guard) do + undo_is_struct_guard(guard) + else + guard + end + end + + defp undo_is_struct_guard( + {:and, meta, [_, %{node: {_, _, [{_, _, [_, {struct, _, _}]} | optional]}}]} + ) do + args = + case optional do + [] -> [{struct, meta, nil}] + [module] -> [{struct, meta, nil}, module] + end + + %{match?: meta[:value], node: {:is_struct, meta, args}} + end + + defp blame_arg({call_arg, ex_arg, erl_arg}, binding) do + {match?, binding} = blame_arg(erl_arg, call_arg, binding) + {blame_wrap(match?, rewrite_arg(ex_arg)), binding} + end + + defp blame_arg(erl_arg, call_arg, binding) do + binding = :orddict.store(:VAR, call_arg, binding) + + try do + ann = :erl_anno.new(0) + + {:value, _, binding} = + :erl_eval.expr({:match, ann, erl_arg, {:var, ann, :VAR}}, binding, :none) + + {true, binding} + rescue + _ -> {false, binding} + end + end + + defp rewrite_arg(arg) do + Macro.prewalk(arg, fn + {:%{}, meta, [__struct__: Range, first: first, last: last, step: step]} -> + {:"..//", meta, [first, last, step]} + + other -> + other + end) + end + + defp blame_guard({{:., _, [:erlang, op]}, meta, [left, right]}, ann, scope, binding) + when op == :andalso or op == :orelse do + guards = [ + blame_guard(left, ann, scope, binding), + blame_guard(right, ann, scope, binding) + ] + + kernel_op = + case op do + :orelse -> :or + :andalso -> :and + end + + evaluate_guard(kernel_op, meta, guards) + end + + defp blame_guard(ex_guard, ann, scope, binding) do + ex_guard + |> blame_guard?(binding, ann, scope) + |> blame_wrap(rewrite_guard(ex_guard)) + end + + defp blame_guard?(ex_guard, binding, ann, scope) do + {erl_guard, _} = :elixir_erl_pass.translate(ex_guard, ann, scope) + {:value, true, _} = :erl_eval.expr(erl_guard, binding, :none) + true + rescue + _ -> false + end + + defp evaluate_guard(kernel_op, meta, guards = [_, _]) do + [x, y] = Enum.map(guards, &evaluate_guard/1) + + logic_value = + case kernel_op do + :or -> x or y + :and -> x and y + end + + {kernel_op, Keyword.put(meta, :value, logic_value), guards} + end + + defp evaluate_guard(%{match?: value}), do: value + defp evaluate_guard({_, meta, _}) when is_list(meta), do: meta[:value] + + defp rewrite_guard(guard) do + Macro.prewalk(guard, fn + {{:., _, [mod, fun]}, meta, args} -> erl_to_ex(mod, fun, args, meta) + other -> other + end) + end + + defp erl_to_ex(mod, fun, args, meta) do + case :elixir_rewrite.erl_to_ex(mod, fun, args) do + {Kernel, fun, args} -> {fun, meta, args} + {mod, fun, args} -> {{:., [], [mod, fun]}, meta, args} + end + end + + defp blame_wrap(match?, ast), do: %{match?: match?, node: ast} + + @doc """ + Formats an exit. It returns a string. Often there are errors/exceptions inside exits. Exits are often wrapped by the caller and provide stacktraces too. This function formats exits in a way to nicely show the exit reason, caller and stacktrace. """ - @spec format_exit(any) :: String.t + @spec format_exit(any) :: String.t() def format_exit(reason) do format_exit(reason, <<"\n ">>) end # 2-Tuple could be caused by an error if the second element is a stacktrace. defp format_exit({exception, maybe_stacktrace} = reason, joiner) - when is_list(maybe_stacktrace) and maybe_stacktrace !== [] do + when is_list(maybe_stacktrace) and maybe_stacktrace !== [] do try do Enum.map(maybe_stacktrace, &format_stacktrace_entry/1) + catch + :error, _ -> + # Not a stacktrace, was an exit. + format_exit_reason(reason) else formatted_stacktrace -> # Assume a non-empty list formattable as stacktrace is a # stacktrace, so exit was caused by an error. - message = "an exception was raised:" <> joiner <> - format_banner(:error, exception, maybe_stacktrace) + message = + "an exception was raised:" <> + joiner <> format_banner(:error, exception, maybe_stacktrace) + Enum.join([message | formatted_stacktrace], joiner <> <<" ">>) - catch - :error, _ -> - # Not a stacktrace, was an exit. - format_exit_reason(reason) end end # :supervisor.start_link returns this error reason when it fails to init # because a child's start_link raises. - defp format_exit({:shutdown, - {:failed_to_start_child, child, {:EXIT, reason}}}, joiner) do + defp format_exit({:shutdown, {:failed_to_start_child, child, {:EXIT, reason}}}, joiner) do format_start_child(child, reason, joiner) end # :supervisor.start_link returns this error reason when it fails to init # because a child's start_link returns {:error, reason}. - defp format_exit({:shutdown, {:failed_to_start_child, child, reason}}, - joiner) do + defp format_exit({:shutdown, {:failed_to_start_child, child, reason}}, joiner) do format_start_child(child, reason, joiner) end # 2-Tuple could be an exit caused by mfa if second element is mfa, args # must be a list of arguments - max length 255 due to max arity. defp format_exit({reason2, {mod, fun, args}} = reason, joiner) - when length(args) < 256 do + when length(args) < 256 do try do format_mfa(mod, fun, args) - else - mfa -> - # Assume tuple formattable as an mfa is an mfa, so exit was caused by - # failed mfa. - "exited in: " <> mfa <> joiner <> - "** (EXIT) " <> format_exit(reason2, joiner <> <<" ">>) catch :error, _ -> # Not an mfa, was an exit. format_exit_reason(reason) + else + mfa -> + # Assume tuple formattable as an mfa is an mfa, + # so exit was caused by failed mfa. + "exited in: " <> + mfa <> joiner <> "** (EXIT) " <> format_exit(reason2, joiner <> <<" ">>) end end @@ -221,12 +473,13 @@ defmodule Exception do "shutdown: #{inspect(reason)}" end + defp format_exit_reason(:calling_self), do: "process attempted to call itself" defp format_exit_reason(:timeout), do: "time out" defp format_exit_reason(:killed), do: "killed" defp format_exit_reason(:noconnection), do: "no connection" defp format_exit_reason(:noproc) do - "no process" + "no process: the process is not alive or there's no process currently associated with the given name, possibly because its application isn't started" end defp format_exit_reason({:nodedown, node_name}) when is_atom(node_name) do @@ -253,29 +506,29 @@ defmodule Exception do # :supervisor.start_link error reasons - # If value is a list will be be formatted by mfa exit in format_exit/1 + # If value is a list will be formatted by mfa exit in format_exit/1 defp format_exit_reason({:bad_return, {mod, :init, value}}) - when is_atom(mod) do + when is_atom(mod) do format_mfa(mod, :init, 1) <> " returned a bad value: " <> inspect(value) end defp format_exit_reason({:bad_start_spec, start_spec}) do - "bad start spec: invalid children: " <> inspect(start_spec) + "bad child specification, invalid children: " <> inspect(start_spec) end defp format_exit_reason({:start_spec, start_spec}) do - "bad start spec: " <> format_sup_spec(start_spec) + "bad child specification, " <> format_sup_spec(start_spec) end defp format_exit_reason({:supervisor_data, data}) do - "bad supervisor data: " <> format_sup_data(data) + "bad supervisor configuration, " <> format_sup_data(data) end defp format_exit_reason(reason), do: inspect(reason) defp format_start_child(child, reason, joiner) do - "shutdown: failed to start child: " <> inspect(child) <> joiner <> - "** (EXIT) " <> format_exit(reason, joiner <> <<" ">>) + "shutdown: failed to start child: " <> + inspect(child) <> joiner <> "** (EXIT) " <> format_exit(reason, joiner <> <<" ">>) end defp format_sup_data({:invalid_type, type}) do @@ -287,49 +540,70 @@ defmodule Exception do end defp format_sup_data({:invalid_intensity, intensity}) do - "invalid intensity: " <> inspect(intensity) + "invalid max_restarts (intensity): " <> inspect(intensity) end defp format_sup_data({:invalid_period, period}) do - "invalid period: " <> inspect(period) + "invalid max_seconds (period): " <> inspect(period) end - defp format_sup_data(other), do: inspect(other) + defp format_sup_data({:invalid_max_children, max_children}) do + "invalid max_children: " <> inspect(max_children) + end + + defp format_sup_data({:invalid_extra_arguments, extra}) do + "invalid extra_arguments: " <> inspect(extra) + end + + defp format_sup_data(other), do: "got: #{inspect(other)}" + + defp format_sup_spec({:duplicate_child_name, id}) do + """ + more than one child specification has the id: #{inspect(id)}. + If using maps as child specifications, make sure the :id keys are unique. + If using a module or {module, arg} as child, use Supervisor.child_spec/2 to change the :id, for example: + + children = [ + Supervisor.child_spec({MyWorker, arg}, id: :my_worker_1), + Supervisor.child_spec({MyWorker, arg}, id: :my_worker_2) + ] + """ + end defp format_sup_spec({:invalid_child_spec, child_spec}) do - "invalid child spec: " <> inspect(child_spec) + "invalid child specification: #{inspect(child_spec)}" end defp format_sup_spec({:invalid_child_type, type}) do - "invalid child type: " <> inspect(type) + "invalid child type: #{inspect(type)}. Must be :worker or :supervisor." end defp format_sup_spec({:invalid_mfa, mfa}) do - "invalid mfa: " <> inspect(mfa) + "invalid mfa: #{inspect(mfa)}" end defp format_sup_spec({:invalid_restart_type, restart}) do - "invalid restart type: " <> inspect(restart) + "invalid restart type: #{inspect(restart)}. Must be :permanent, :transient or :temporary." end defp format_sup_spec({:invalid_shutdown, shutdown}) do - "invalid shutdown: " <> inspect(shutdown) + "invalid shutdown: #{inspect(shutdown)}. Must be an integer >= 0, :infinity or :brutal_kill." end defp format_sup_spec({:invalid_module, mod}) do - "invalid module: " <> inspect(mod) + "invalid module: #{inspect(mod)}. Must be an atom." end defp format_sup_spec({:invalid_modules, modules}) do - "invalid modules: " <> inspect(modules) + "invalid modules: #{inspect(modules)}. Must be a list of atoms or :dynamic." end - defp format_sup_spec(other), do: inspect(other) + defp format_sup_spec(other), do: "got: #{inspect(other)}" @doc """ Receives a stacktrace entry and formats it into a string. """ - @spec format_stacktrace_entry(stacktrace_entry) :: String.t + @spec format_stacktrace_entry(stacktrace_entry) :: String.t() def format_stacktrace_entry(entry) # From Macro.Env.stacktrace @@ -356,9 +630,19 @@ defmodule Exception do end defp format_application(module) do + # We cannot use Application due to bootstrap issues case :application.get_application(module) do - {:ok, app} -> "(" <> Atom.to_string(app) <> ") " - :undefined -> "" + {:ok, app} -> + case :application.get_key(app, :vsn) do + {:ok, vsn} when is_list(vsn) -> + "(" <> Atom.to_string(app) <> " " <> List.to_string(vsn) <> ") " + + _ -> + "(" <> Atom.to_string(app) <> ") " + end + + :undefined -> + "" end end @@ -369,13 +653,18 @@ defmodule Exception do is retrieved from `Process.info/2`. """ def format_stacktrace(trace \\ nil) do - trace = trace || case Process.info(self, :current_stacktrace) do - {:current_stacktrace, t} -> Enum.drop(t, 3) - end + trace = + if trace do + trace + else + case Process.info(self(), :current_stacktrace) do + {:current_stacktrace, t} -> Enum.drop(t, 3) + end + end case trace do [] -> "\n" - s -> " " <> Enum.map_join(s, "\n ", &format_stacktrace_entry(&1)) <> "\n" + _ -> " " <> Enum.map_join(trace, "\n ", &format_stacktrace_entry(&1)) <> "\n" end end @@ -385,12 +674,12 @@ defmodule Exception do ## Examples - Exception.format_fa(fn -> end, 1) + Exception.format_fa(fn -> nil end, 1) #=> "#Function<...>/1" """ def format_fa(fun, arity) when is_function(fun) do - "#{inspect fun}#{format_arity(arity)}" + "#{inspect(fun)}#{format_arity(arity)}" end @doc """ @@ -400,13 +689,13 @@ defmodule Exception do ## Examples - iex> Exception.format_mfa Foo, :bar, 1 + iex> Exception.format_mfa(Foo, :bar, 1) "Foo.bar/1" - iex> Exception.format_mfa Foo, :bar, [] + iex> Exception.format_mfa(Foo, :bar, []) "Foo.bar()" - iex> Exception.format_mfa nil, :bar, [] + iex> Exception.format_mfa(nil, :bar, []) "nil.bar()" Anonymous functions are reported as -func/arity-anonfn-count-, @@ -414,17 +703,15 @@ defmodule Exception do "anonymous fn in func/arity" """ def format_mfa(module, fun, arity) when is_atom(module) and is_atom(fun) do - fun = - case inspect(fun) do - ":" <> fun -> fun - fun -> fun - end - - case match?("\"-" <> _, fun) and String.split(fun, "-") do - [ "\"", outer_fun, "fun", _count, "\"" ] -> - "anonymous fn#{format_arity(arity)} in #{inspect module}.#{outer_fun}" - _ -> - "#{inspect module}.#{fun}#{format_arity(arity)}" + case Code.Identifier.extract_anonymous_fun_parent(fun) do + {outer_name, outer_arity} -> + "anonymous fn#{format_arity(arity)} in " <> + "#{Macro.inspect_atom(:literal, module)}." <> + "#{Macro.inspect_atom(:remote_call, outer_name)}/#{outer_arity}" + + :error -> + "#{Macro.inspect_atom(:literal, module)}." <> + "#{Macro.inspect_atom(:remote_call, fun)}#{format_arity(arity)}" end end @@ -438,8 +725,9 @@ defmodule Exception do end @doc """ - Formats the given file and line as shown in stacktraces. - If any of the values are nil, they are omitted. + Formats the given `file` and `line` as shown in stacktraces. + + If any of the values are `nil`, they are omitted. ## Examples @@ -453,101 +741,229 @@ defmodule Exception do "" """ - def format_file_line(file, line) do - format_file_line(file, line, "") + def format_file_line(file, line, suffix \\ "") do + cond do + is_nil(file) -> "" + is_nil(line) or line == 0 -> "#{file}:#{suffix}" + true -> "#{file}:#{line}:#{suffix}" + end end - defp format_file_line(file, line, suffix) do - if file do - if line && line != 0 do - "#{file}:#{line}:#{suffix}" - else - "#{file}:#{suffix}" - end - else + @doc """ + Formats the given `file`, `line`, and `column` as shown in stacktraces. + + If any of the values are `nil`, they are omitted. + + ## Examples + + iex> Exception.format_file_line_column("foo", 1, 2) + "foo:1:2:" + + iex> Exception.format_file_line_column("foo", 1, nil) + "foo:1:" + + iex> Exception.format_file_line_column("foo", nil, nil) + "foo:" + + iex> Exception.format_file_line_column("foo", nil, 2) + "foo:" + + iex> Exception.format_file_line_column(nil, nil, nil) "" + + """ + def format_file_line_column(file, line, column, suffix \\ "") do + cond do + is_nil(file) -> "" + is_nil(line) or line == 0 -> "#{file}:#{suffix}" + is_nil(column) or column == 0 -> "#{file}:#{line}:#{suffix}" + true -> "#{file}:#{line}:#{column}:#{suffix}" end end + @doc false + def format_snippet(snippet, error_line) do + line_digits = error_line |> Integer.to_string() |> byte_size() + placeholder = String.duplicate(" ", max(line_digits, 2)) + padding = if line_digits < 2, do: " " + + " #{placeholder} |\n" <> + " #{padding}#{error_line} | #{snippet.content}\n" <> + " #{placeholder} | #{String.duplicate(" ", snippet.offset)}^" + end + defp format_location(opts) when is_list(opts) do - format_file_line Keyword.get(opts, :file), Keyword.get(opts, :line), " " + format_file_line(Keyword.get(opts, :file), Keyword.get(opts, :line), " ") end end -# Some exceptions implement `message/1` instead of `exception/1` mostly +# Some exceptions implement "message/1" instead of "exception/1" mostly # for bootstrap reasons. It is recommended for applications to implement -# `exception/1` instead of `message/1` as described in `defexception/1` +# "exception/1" instead of "message/1" as described in "defexception/1" # docs. defmodule RuntimeError do defexception message: "runtime error" - - def exception(msg) when is_binary(msg) do - %RuntimeError{message: msg} - end - - def exception(arg) do - super(arg) - end end defmodule ArgumentError do defexception message: "argument error" - def exception(msg) when is_binary(msg) do - %ArgumentError{message: msg} + @impl true + def blame( + exception, + [{:erlang, :apply, [module, function, args], _} | _] = stacktrace + ) do + message = + cond do + not proper_list?(args) -> + "you attempted to apply a function named #{inspect(function)} on module #{inspect(module)} " <> + "with arguments #{inspect(args)}. Arguments (the third argument of apply) must always be a proper list" + + # Note that args may be an empty list even if they were supplied + not is_atom(module) and is_atom(function) and args == [] -> + "you attempted to apply a function named #{inspect(function)} on #{inspect(module)}. " <> + "If you are using Kernel.apply/3, make sure the module is an atom. " <> + "If you are using the dot syntax, such as map.field or module.function(), " <> + "make sure the left side of the dot is an atom or a map" + + not is_atom(module) -> + "you attempted to apply a function on #{inspect(module)}. " <> + "Modules (the first argument of apply) must always be an atom" + + not is_atom(function) -> + "you attempted to apply a function named #{inspect(function)} on module #{inspect(module)}. " <> + "However #{inspect(function)} is not a valid function name. Function names (the second argument " <> + "of apply) must always be an atom" + end + + {%{exception | message: message}, stacktrace} end - def exception(arg) do - super(arg) + def blame(exception, stacktrace) do + {exception, stacktrace} end + + defp proper_list?(list) when length(list) >= 0, do: true + defp proper_list?(_), do: false end defmodule ArithmeticError do - defexception [] + defexception message: "bad argument in arithmetic expression" + + @unary_ops [:+, :-] + @binary_ops [:+, :-, :*, :/] + @binary_funs [:div, :rem] + @bitwise_binary_funs [:band, :bor, :bxor, :bsl, :bsr] + + @impl true + def blame(%{message: message} = exception, [{:erlang, fun, args, _} | _] = stacktrace) do + message = + message <> + case {fun, args} do + {op, [a]} when op in @unary_ops -> + ": #{op}(#{inspect(a)})" - def message(_) do - "bad argument in arithmetic expression" + {op, [a, b]} when op in @binary_ops -> + ": #{inspect(a)} #{op} #{inspect(b)}" + + {fun, [a, b]} when fun in @binary_funs -> + ": #{fun}(#{inspect(a)}, #{inspect(b)})" + + {fun, [a, b]} when fun in @bitwise_binary_funs -> + ": Bitwise.#{fun}(#{inspect(a)}, #{inspect(b)})" + + {:bnot, [a]} -> + ": Bitwise.bnot(#{inspect(a)})" + + _ -> + "" + end + + {%{exception | message: message}, stacktrace} + end + + def blame(exception, stacktrace) do + {exception, stacktrace} end end defmodule SystemLimitError do - defexception [] - - def message(_) do - "a system limit has been reached" - end + defexception message: "a system limit has been reached" end defmodule SyntaxError do - defexception [file: nil, line: nil, description: "syntax error"] - - def message(exception) do - Exception.format_file_line(Path.relative_to_cwd(exception.file), exception.line) <> - " " <> exception.description + defexception [:file, :line, :column, :snippet, description: "syntax error"] + + @impl true + def message(%{ + file: file, + line: line, + column: column, + description: description, + snippet: snippet + }) + when not is_nil(snippet) and not is_nil(column) do + Exception.format_file_line_column(Path.relative_to_cwd(file), line, column) <> + " " <> description <> "\n" <> Exception.format_snippet(snippet, line) + end + + @impl true + def message(%{ + file: file, + line: line, + column: column, + description: description + }) do + Exception.format_file_line_column(Path.relative_to_cwd(file), line, column) <> + " " <> description end end defmodule TokenMissingError do - defexception [file: nil, line: nil, description: "expression is incomplete"] - - def message(exception) do - Exception.format_file_line(Path.relative_to_cwd(exception.file), exception.line) <> - " " <> exception.description + defexception [:file, :line, :snippet, :column, description: "expression is incomplete"] + + @impl true + def message(%{ + file: file, + line: line, + column: column, + description: description, + snippet: snippet + }) + when not is_nil(snippet) and not is_nil(column) do + Exception.format_file_line_column(Path.relative_to_cwd(file), line, column) <> + " " <> description <> "\n" <> Exception.format_snippet(snippet, line) + end + + @impl true + def message(%{ + file: file, + line: line, + column: column, + description: description + }) do + Exception.format_file_line_column(file && Path.relative_to_cwd(file), line, column) <> + " " <> description end end defmodule CompileError do - defexception [file: nil, line: nil, description: "compile error"] + defexception [:file, :line, description: "compile error"] - def message(exception) do - Exception.format_file_line(Path.relative_to_cwd(exception.file), exception.line) <> - " " <> exception.description + @impl true + def message(%{file: file, line: line, description: description}) do + Exception.format_file_line(file && Path.relative_to_cwd(file), line) <> " " <> description end end defmodule BadFunctionError do - defexception [term: nil] + defexception [:term] + + @impl true + def message(%{term: term}) when is_function(term) do + "function #{inspect(term)} is invalid, likely because it points to an old version of the code" + end def message(exception) do "expected a function, got: #{inspect(exception.term)}" @@ -555,84 +971,369 @@ defmodule BadFunctionError do end defmodule BadStructError do - defexception [struct: nil, term: nil] + defexception [:struct, :term] + @impl true def message(exception) do "expected a struct named #{inspect(exception.struct)}, got: #{inspect(exception.term)}" end end +defmodule BadMapError do + defexception [:term] + + @impl true + def message(exception) do + "expected a map, got: #{inspect(exception.term)}" + end +end + +defmodule BadBooleanError do + defexception [:term, :operator] + + @impl true + def message(exception) do + "expected a boolean on left-side of \"#{exception.operator}\", got: #{inspect(exception.term)}" + end +end + defmodule MatchError do - defexception [term: nil] + defexception [:term] + @impl true def message(exception) do "no match of right hand side value: #{inspect(exception.term)}" end end defmodule CaseClauseError do - defexception [term: nil] + defexception [:term] + @impl true def message(exception) do "no case clause matching: #{inspect(exception.term)}" end end +defmodule WithClauseError do + defexception [:term] + + @impl true + def message(exception) do + "no with clause matching: #{inspect(exception.term)}" + end +end + defmodule CondClauseError do defexception [] + @impl true def message(_exception) do - "no cond clause evaluated to a true value" + "no cond clause evaluated to a truthy value" end end defmodule TryClauseError do - defexception [term: nil] + defexception [:term] + @impl true def message(exception) do "no try clause matching: #{inspect(exception.term)}" end end defmodule BadArityError do - defexception [function: nil, args: nil] + defexception [:function, :args] + @impl true def message(exception) do - fun = exception.function + fun = exception.function args = exception.args insp = Enum.map_join(args, ", ", &inspect/1) - {:arity, arity} = :erlang.fun_info(fun, :arity) + {:arity, arity} = Function.info(fun, :arity) "#{inspect(fun)} with arity #{arity} called with #{count(length(args), insp)}" end defp count(0, _insp), do: "no arguments" - defp count(1, insp), do: "1 argument (#{insp})" - defp count(x, insp), do: "#{x} arguments (#{insp})" + defp count(1, insp), do: "1 argument (#{insp})" + defp count(x, insp), do: "#{x} arguments (#{insp})" end defmodule UndefinedFunctionError do - defexception [module: nil, function: nil, arity: nil] + defexception [:module, :function, :arity, :reason, :message] - def message(exception) do - if exception.function do - formatted = Exception.format_mfa exception.module, exception.function, exception.arity - "undefined function: #{formatted}" + @impl true + def message(%{message: nil} = exception) do + %{reason: reason, module: module, function: function, arity: arity} = exception + {message, _loaded?} = message(reason, module, function, arity) + message + end + + def message(%{message: message}) do + message + end + + defp message(nil, module, function, arity) do + cond do + is_nil(function) or is_nil(arity) -> + {"undefined function", false} + + is_nil(module) -> + formatted_fun = Exception.format_mfa(module, function, arity) + {"function #{formatted_fun} is undefined", false} + + function_exported?(module, :module_info, 0) -> + message(:"function not exported", module, function, arity) + + true -> + message(:"module could not be loaded", module, function, arity) + end + end + + defp message(:"module could not be loaded", module, function, arity) do + formatted_fun = Exception.format_mfa(module, function, arity) + {"function #{formatted_fun} is undefined (module #{inspect(module)} is not available)", false} + end + + defp message(:"function not exported", module, function, arity) do + formatted_fun = Exception.format_mfa(module, function, arity) + {"function #{formatted_fun} is undefined or private", true} + end + + defp message(reason, module, function, arity) do + formatted_fun = Exception.format_mfa(module, function, arity) + {"function #{formatted_fun} is undefined (#{reason})", false} + end + + @impl true + def blame(exception, stacktrace) do + %{reason: reason, module: module, function: function, arity: arity} = exception + {message, loaded?} = message(reason, module, function, arity) + message = message <> hint(module, function, arity, loaded?) + {%{exception | message: message}, stacktrace} + end + + defp hint(nil, _function, 0, _loaded?) do + ". If you are using the dot syntax, such as map.field or module.function(), " <> + "make sure the left side of the dot is an atom or a map" + end + + defp hint(module, function, arity, true) do + behaviour_hint(module, function, arity) <> + hint_for_loaded_module(module, function, arity, nil) + end + + defp hint(_module, _function, _arity, _loaded?) do + "" + end + + @doc false + def hint_for_loaded_module(module, function, arity, exports) do + cond do + macro_exported?(module, function, arity) -> + ". However there is a macro with the same name and arity. " <> + "Be sure to require #{inspect(module)} if you intend to invoke this macro" + + message = otp_obsolete(module, function, arity) -> + ", #{message}" + + true -> + IO.iodata_to_binary(did_you_mean(module, function, exports)) + end + end + + defp otp_obsolete(module, function, arity) do + case :otp_internal.obsolete(module, function, arity) do + {:removed, [_ | _] = string} -> string + _ -> nil + end + end + + @function_threshold 0.77 + @max_suggestions 5 + + defp did_you_mean(module, function, exports) do + exports = exports || exports_for(module) + + result = + case Keyword.take(exports, [function]) do + [] -> + candidates = exports -- deprecated_functions_for(module) + base = Atom.to_string(function) + + for {key, val} <- candidates, + dist = String.jaro_distance(base, Atom.to_string(key)), + dist >= @function_threshold, + do: {dist, key, val} + + arities -> + for {key, val} <- arities, do: {1.0, key, val} + end + |> Enum.sort(&(elem(&1, 0) >= elem(&2, 0))) + |> Enum.take(@max_suggestions) + |> Enum.sort(&(elem(&1, 1) <= elem(&2, 1))) + + case result do + [] -> [] + suggestions -> [". Did you mean:\n\n" | Enum.map(suggestions, &format_fa/1)] + end + end + + defp format_fa({_dist, fun, arity}) do + [" * ", Macro.inspect_atom(:remote_call, fun), ?/, Integer.to_string(arity), ?\n] + end + + defp behaviour_hint(module, function, arity) do + case behaviours_for(module) do + [] -> + "" + + behaviours -> + case Enum.find(behaviours, &expects_callback?(&1, function, arity)) do + nil -> "" + behaviour -> ", but the behaviour #{inspect(behaviour)} expects it to be present" + end + end + rescue + # In case the module was removed while we are computing this + UndefinedFunctionError -> "" + end + + defp behaviours_for(module) do + :attributes + |> module.module_info() + |> Keyword.get(:behaviour, []) + end + + defp expects_callback?(behaviour, function, arity) do + callbacks = + behaviour.behaviour_info(:callbacks) -- behaviour.behaviour_info(:optional_callbacks) + + Enum.member?(callbacks, {function, arity}) + end + + defp exports_for(module) do + if function_exported?(module, :__info__, 1) do + module.__info__(:macros) ++ module.__info__(:functions) + else + module.module_info(:exports) + end + rescue + # In case the module was removed while we are computing this + UndefinedFunctionError -> + [] + end + + defp deprecated_functions_for(module) do + if function_exported?(module, :__info__, 1) do + for {name_arity, _message} <- module.__info__(:deprecated), do: name_arity else - "undefined function" + [] end + rescue + # In case the module was removed while we are computing this + UndefinedFunctionError -> + [] end end defmodule FunctionClauseError do - defexception [module: nil, function: nil, arity: nil] + defexception [:module, :function, :arity, :kind, :args, :clauses] + @clause_limit 10 + + @impl true def message(exception) do - if exception.function do - formatted = Exception.format_mfa exception.module, exception.function, exception.arity - "no function clause matching in #{formatted}" - else - "no function clause matches" + case exception do + %{function: nil} -> + "no function clause matches" + + %{module: module, function: function, arity: arity} -> + formatted = Exception.format_mfa(module, function, arity) + blamed = blame(exception, &inspect/1, &blame_match/1) + "no function clause matching in #{formatted}" <> blamed + end + end + + @impl true + def blame(%{module: module, function: function, arity: arity} = exception, stacktrace) do + case stacktrace do + [{^module, ^function, args, meta} | rest] when length(args) == arity -> + exception = + case Exception.blame_mfa(module, function, args) do + {:ok, kind, clauses} -> %{exception | args: args, kind: kind, clauses: clauses} + :error -> %{exception | args: args} + end + + {exception, [{module, function, arity, meta} | rest]} + + stacktrace -> + {exception, stacktrace} + end + end + + defp blame_match(%{match?: true, node: node}), do: Macro.to_string(node) + defp blame_match(%{match?: false, node: node}), do: "-" <> Macro.to_string(node) <> "-" + + @doc false + def blame(%{args: nil}, _, _) do + "" + end + + def blame(exception, inspect_fun, fun) do + %{module: module, function: function, arity: arity, kind: kind, args: args, clauses: clauses} = + exception + + mfa = Exception.format_mfa(module, function, arity) + + format_clause_fun = fn {args, guards} -> + args = Enum.map_join(args, ", ", fun) + base = " #{kind} #{function}(#{args})" + Enum.reduce(guards, base, &"#{&2} when #{clause_to_string(&1, fun)}") <> "\n" end + + "\n\nThe following arguments were given to #{mfa}:\n" <> + "#{format_args(args, inspect_fun)}" <> + "#{format_clauses(clauses, format_clause_fun, @clause_limit)}" + end + + defp clause_to_string({op, _, [left, right]}, fun), + do: clause_to_string(left, fun) <> " #{op} " <> clause_to_string(right, fun) + + defp clause_to_string(node, fun), do: fun.(node) + + defp format_args(args, inspect_fun) do + args + |> Enum.with_index(1) + |> Enum.map(fn {arg, i} -> + [pad("\n# "), Integer.to_string(i), pad("\n"), pad(inspect_fun.(arg)), "\n"] + end) + end + + defp format_clauses(clauses, format_clause_fun, limit) + defp format_clauses(nil, _, _), do: "" + defp format_clauses([], _, _), do: "" + + defp format_clauses(clauses, format_clause_fun, limit) do + top_clauses = + clauses + |> Enum.take(limit) + |> Enum.map(format_clause_fun) + + [ + "\nAttempted function clauses (showing #{length(top_clauses)} out of #{length(clauses)}):", + "\n\n", + top_clauses, + non_visible_clauses(length(clauses) - limit) + ] + end + + defp non_visible_clauses(n) when n <= 0, do: [] + defp non_visible_clauses(1), do: [" ...\n (1 clause not shown)\n"] + defp non_visible_clauses(n), do: [" ...\n (#{n} clauses not shown)\n"] + + defp pad(string) do + String.replace(string, "\n", "\n ") end end @@ -646,23 +1347,115 @@ defmodule Code.LoadError do end defmodule Protocol.UndefinedError do - defexception [protocol: nil, value: nil, description: nil] - - def message(exception) do - msg = "protocol #{inspect exception.protocol} not implemented for #{inspect exception.value}" - if exception.description do - msg <> ", " <> exception.description - else - msg + defexception [:protocol, :value, description: ""] + + @impl true + def message(%{protocol: protocol, value: value, description: description}) do + "protocol #{inspect(protocol)} not implemented for #{inspect(value)} of type " <> + value_type(value) <> maybe_description(description) <> maybe_available(protocol) + end + + defp value_type(%{__struct__: struct}), do: "#{inspect(struct)} (a struct)" + defp value_type(value) when is_atom(value), do: "Atom" + defp value_type(value) when is_bitstring(value), do: "BitString" + defp value_type(value) when is_float(value), do: "Float" + defp value_type(value) when is_function(value), do: "Function" + defp value_type(value) when is_integer(value), do: "Integer" + defp value_type(value) when is_list(value), do: "List" + defp value_type(value) when is_map(value), do: "Map" + defp value_type(value) when is_pid(value), do: "PID" + defp value_type(value) when is_port(value), do: "Port" + defp value_type(value) when is_reference(value), do: "Reference" + defp value_type(value) when is_tuple(value), do: "Tuple" + + defp maybe_description(""), do: "" + defp maybe_description(description), do: ", " <> description + + defp maybe_available(protocol) do + case protocol.__protocol__(:impls) do + {:consolidated, []} -> + ". There are no implementations for this protocol." + + {:consolidated, types} -> + ". This protocol is implemented for the following type(s): " <> + Enum.map_join(types, ", ", &inspect/1) + + :not_consolidated -> + "" end end end defmodule KeyError do - defexception key: nil, term: nil + defexception [:key, :term, :message] - def message(exception) do - "key #{inspect exception.key} not found in: #{inspect exception.term}" + @impl true + def message(exception = %{message: nil}), do: message(exception.key, exception.term) + def message(%{message: message}), do: message + + defp message(key, term) do + message = "key #{inspect(key)} not found" + + if term != nil do + message <> " in: #{inspect(term)}" + else + message + end + end + + @impl true + def blame(exception = %{message: message}, stacktrace) when is_binary(message) do + {exception, stacktrace} + end + + def blame(exception = %{term: nil}, stacktrace) do + message = message(exception.key, exception.term) + {%{exception | message: message}, stacktrace} + end + + def blame(exception, stacktrace) do + %{term: term, key: key} = exception + message = message(key, term) + + if is_atom(key) and (map_with_atom_keys_only?(term) or Keyword.keyword?(term)) do + hint = did_you_mean(key, available_keys(term)) + message = message <> IO.iodata_to_binary(hint) + {%{exception | message: message}, stacktrace} + else + {%{exception | message: message}, stacktrace} + end + end + + defp map_with_atom_keys_only?(term) do + is_map(term) and Enum.all?(Map.to_list(term), fn {k, _} -> is_atom(k) end) + end + + defp available_keys(term) when is_map(term), do: Map.keys(term) + defp available_keys(term) when is_list(term), do: Keyword.keys(term) + + @threshold 0.77 + @max_suggestions 5 + defp did_you_mean(missing_key, available_keys) do + stringified_key = Atom.to_string(missing_key) + + suggestions = + for key <- available_keys, + distance = String.jaro_distance(stringified_key, Atom.to_string(key)), + distance >= @threshold, + do: {distance, key} + + case suggestions do + [] -> [] + suggestions -> [". Did you mean:\n\n" | format_suggestions(suggestions)] + end + end + + defp format_suggestions(suggestions) do + suggestions + |> Enum.sort(&(elem(&1, 0) >= elem(&2, 0))) + |> Enum.take(@max_suggestions) + |> Enum.sort(&(elem(&1, 1) <= elem(&2, 1))) + |> Enum.map(fn {_, key} -> [" * ", inspect(key), ?\n] end) end end @@ -672,73 +1465,130 @@ defmodule UnicodeConversionError do def exception(opts) do %UnicodeConversionError{ encoded: Keyword.fetch!(opts, :encoded), - message: "#{Keyword.fetch!(opts, :kind)} #{detail Keyword.fetch!(opts, :rest)}" + message: "#{Keyword.fetch!(opts, :kind)} #{detail(Keyword.fetch!(opts, :rest))}" } end defp detail(rest) when is_binary(rest) do - "encoding starting at #{inspect rest}" + "encoding starting at #{inspect(rest)}" end - defp detail([h|_]) do + defp detail([h | _]) when is_integer(h) do "code point #{h}" end + + defp detail([h | _]) do + detail(h) + end end defmodule Enum.OutOfBoundsError do - defexception [] + defexception message: "out of bounds error" +end - def message(_) do - "out of bounds error" +defmodule Enum.EmptyError do + defexception message: "empty error" +end + +defmodule File.Error do + defexception [:reason, :path, action: ""] + + @impl true + def message(%{action: action, reason: reason, path: path}) do + formatted = + case {action, reason} do + {"remove directory", :eexist} -> + "directory is not empty" + + _ -> + IO.iodata_to_binary(:file.format_error(reason)) + end + + "could not #{action} #{inspect(path)}: #{formatted}" end end -defmodule Enum.EmptyError do - defexception [] +defmodule File.CopyError do + defexception [:reason, :source, :destination, on: "", action: ""] - def message(_) do - "empty error" + @impl true + def message(exception) do + formatted = IO.iodata_to_binary(:file.format_error(exception.reason)) + + location = + case exception.on do + "" -> "" + on -> ". #{on}" + end + + "could not #{exception.action} from #{inspect(exception.source)} to " <> + "#{inspect(exception.destination)}#{location}: #{formatted}" end end -defmodule File.Error do - defexception [reason: nil, action: "", path: nil] +defmodule File.RenameError do + defexception [:reason, :source, :destination, on: "", action: ""] + @impl true def message(exception) do formatted = IO.iodata_to_binary(:file.format_error(exception.reason)) - "could not #{exception.action} #{exception.path}: #{formatted}" + + location = + case exception.on do + "" -> "" + on -> ". #{on}" + end + + "could not #{exception.action} from #{inspect(exception.source)} to " <> + "#{inspect(exception.destination)}#{location}: #{formatted}" end end -defmodule File.CopyError do - defexception [reason: nil, action: "", source: nil, destination: nil, on: nil] +defmodule File.LinkError do + defexception [:reason, :existing, :new, action: ""] + @impl true def message(exception) do formatted = IO.iodata_to_binary(:file.format_error(exception.reason)) - location = if on = exception.on, do: ". #{on}", else: "" - "could not #{exception.action} from #{exception.source} to " <> - "#{exception.destination}#{location}: #{formatted}" + + "could not #{exception.action} from #{inspect(exception.existing)} to " <> + "#{inspect(exception.new)}: #{formatted}" end end defmodule ErlangError do - defexception [original: nil] + defexception [:original, :reason] - def message(exception) do - "erlang error: #{inspect(exception.original)}" + @impl true + def message(exception) + + def message(%__MODULE__{original: original, reason: nil}) do + "Erlang error: #{inspect(original)}" + end + + def message(%__MODULE__{original: original, reason: reason}) do + IO.iodata_to_binary(["Erlang error: ", inspect(original), reason]) end @doc false - def normalize(:badarg, _stacktrace) do - %ArgumentError{} + def normalize(:badarg, stacktrace) do + case error_info(:badarg, stacktrace, "errors were found at the given arguments") do + {:ok, reason, details} -> %ArgumentError{message: reason <> details} + :error -> %ArgumentError{} + end end def normalize(:badarith, _stacktrace) do %ArithmeticError{} end - def normalize(:system_limit, _stacktrace) do - %SystemLimitError{} + def normalize(:system_limit, stacktrace) do + default_reason = "a system limit has been reached due to errors at the given arguments" + + case error_info(:system_limit, stacktrace, default_reason) do + {:ok, reason, details} -> %SystemLimitError{message: reason <> details} + :error -> %SystemLimitError{} + end end def normalize(:cond_clause, _stacktrace) do @@ -761,21 +1611,51 @@ defmodule ErlangError do %MatchError{term: term} end + def normalize({:badmap, term}, _stacktrace) do + %BadMapError{term: term} + end + + def normalize({:badbool, op, term}, _stacktrace) do + %BadBooleanError{operator: op, term: term} + end + + def normalize({:badkey, key}, stacktrace) do + term = + case stacktrace do + [{Map, :get_and_update!, [map, _, _], _} | _] -> map + [{Map, :update!, [map, _, _], _} | _] -> map + [{:maps, :update, [_, _, map], _} | _] -> map + [{:maps, :get, [_, map], _} | _] -> map + [{:erlang, :map_get, [_, map], _} | _] -> map + _ -> nil + end + + %KeyError{key: key, term: term} + end + + def normalize({:badkey, key, map}, _stacktrace) do + %KeyError{key: key, term: map} + end + def normalize({:case_clause, term}, _stacktrace) do %CaseClauseError{term: term} end + def normalize({:with_clause, term}, _stacktrace) do + %WithClauseError{term: term} + end + def normalize({:try_clause, term}, _stacktrace) do %TryClauseError{term: term} end def normalize(:undef, stacktrace) do - {mod, fun, arity} = from_stacktrace(stacktrace || :erlang.get_stacktrace) + {mod, fun, arity} = from_stacktrace(stacktrace) %UndefinedFunctionError{module: mod, function: fun, arity: arity} end def normalize(:function_clause, stacktrace) do - {mod, fun, arity} = from_stacktrace(stacktrace || :erlang.get_stacktrace) + {mod, fun, arity} = from_stacktrace(stacktrace) %FunctionClauseError{module: mod, function: fun, arity: arity} end @@ -783,19 +1663,122 @@ defmodule ErlangError do %ArgumentError{message: "argument error: #{inspect(payload)}"} end - def normalize(other, _stacktrace) do - %ErlangError{original: other} + def normalize(other, stacktrace) do + case error_info(other, stacktrace, "") do + {:ok, _reason, details} -> %ErlangError{original: other, reason: details} + :error -> %ErlangError{original: other} + end end - defp from_stacktrace([{module, function, args, _}|_]) when is_list(args) do + defp from_stacktrace([{module, function, args, _} | _]) when is_list(args) do {module, function, length(args)} end - defp from_stacktrace([{module, function, arity, _}|_]) do + defp from_stacktrace([{module, function, arity, _} | _]) do {module, function, arity} end defp from_stacktrace(_) do {nil, nil, nil} end + + defp error_info(erl_exception, stacktrace, default_reason) do + with [{module, fun, args_or_arity, opts} | tail] <- stacktrace, + %{} = error_info <- opts[:error_info] do + error_module = Map.get(error_info, :module, module) + error_fun = Map.get(error_info, :function, :format_error) + + error_info = Map.put(error_info, :pretty_printer, &inspect/1) + head = {module, fun, args_or_arity, Keyword.put(opts, :error_info, error_info)} + + extra = + try do + apply(error_module, error_fun, [erl_exception, [head | tail]]) + rescue + _ -> %{} + end + + arity = if is_integer(args_or_arity), do: args_or_arity, else: length(args_or_arity) + args_errors = Map.take(extra, Enum.to_list(1..arity//1)) + reason = Map.get(extra, :reason, default_reason) + + cond do + map_size(args_errors) > 0 -> + {:ok, reason, IO.iodata_to_binary([":\n\n" | Enum.map(args_errors, &arg_error/1)])} + + general = extra[:general] -> + {:ok, reason, ": " <> general} + + true -> + :error + end + else + _ -> :error + end + end + + defp arg_error({n, message}), do: " * #{nth(n)} argument: #{message}\n" + + defp nth(1), do: "1st" + defp nth(2), do: "2nd" + defp nth(3), do: "3rd" + defp nth(n), do: "#{n}th" +end + +defmodule Inspect.Error do + @moduledoc """ + Raised when a struct cannot be inspected. + """ + @enforce_keys [:exception_module, :exception_message, :stacktrace, :inspected_struct] + defexception @enforce_keys + + @impl true + def exception(arguments) when is_list(arguments) do + exception = Keyword.fetch!(arguments, :exception) + exception_module = exception.__struct__ + exception_message = Exception.message(exception) |> String.trim_trailing("\n") + stacktrace = Keyword.fetch!(arguments, :stacktrace) + inspected_struct = Keyword.fetch!(arguments, :inspected_struct) + + %Inspect.Error{ + exception_module: exception_module, + exception_message: exception_message, + stacktrace: stacktrace, + inspected_struct: inspected_struct + } + end + + @impl true + def message(%__MODULE__{ + exception_module: exception_module, + exception_message: exception_message, + inspected_struct: inspected_struct + }) do + ~s''' + got #{inspect(exception_module)} with message: + + """ + #{pad(exception_message, 4)} + """ + + while inspecting: + + #{pad(inspected_struct, 4)} + ''' + end + + @doc false + def pad(message, padding_length) + when is_binary(message) and is_integer(padding_length) and padding_length >= 0 do + padding = String.duplicate(" ", padding_length) + + message + |> String.split("\n") + |> Enum.map(fn + "" -> "\n" + line -> [padding, line, ?\n] + end) + |> IO.iodata_to_binary() + |> String.trim_trailing("\n") + end end diff --git a/lib/elixir/lib/file.ex b/lib/elixir/lib/file.ex index 23cc854db4c..0f761ab34a4 100644 --- a/lib/elixir/lib/file.ex +++ b/lib/elixir/lib/file.ex @@ -3,35 +3,41 @@ defmodule File do This module contains functions to manipulate files. Some of those functions are low-level, allowing the user - to interact with the file or IO devices, like `open/2`, + to interact with files or IO devices, like `open/2`, `copy/3` and others. This module also provides higher level functions that work with filenames and have their naming - based on UNIX variants. For example, one can copy a file + based on Unix variants. For example, one can copy a file via `cp/3` and remove files and directories recursively - via `rm_rf/1` + via `rm_rf/1`. + + Paths given to functions in this module can be either relative to the + current working directory (as returned by `File.cwd/0`), or absolute + paths. Shell conventions like `~` are not expanded automatically. + To use paths like `~/Downloads`, you can use `Path.expand/1` or + `Path.expand/2` to expand your path to an absolute path. ## Encoding In order to write and read files, one must use the functions - in the `IO` module. By default, a file is opened in binary mode + in the `IO` module. By default, a file is opened in binary mode, which requires the functions `IO.binread/2` and `IO.binwrite/2` to interact with the file. A developer may pass `:utf8` as an option when opening the file, then the slower `IO.read/2` and `IO.write/2` functions must be used as they are responsible for - doing the proper conversions and data guarantees. + doing the proper conversions and providing the proper data guarantees. - Note that filenames when given as char lists in Elixir are + Note that filenames when given as charlists in Elixir are always treated as UTF-8. In particular, we expect that the - shell and the operating system are configured to use UTF8 - encoding. Binary filenames are considering raw and passed - to the OS as is. + shell and the operating system are configured to use UTF-8 + encoding. Binary filenames are considered raw and passed + to the operating system as is. ## API Most of the functions in this module return `:ok` or `{:ok, result}` in case of success, `{:error, reason}` - otherwise. Those function are also followed by a variant - that ends with `!` which returns the result (without the + otherwise. Those functions also have a variant + that ends with `!` which returns the result (instead of the `{:ok, result}` tuple) in case of success or raises an exception in case it fails. For example: @@ -47,15 +53,15 @@ defmodule File do File.read!("invalid.txt") #=> raises File.Error - In general, a developer should use the former in case he wants + In general, a developer should use the former in case they want to react if the file does not exist. The latter should be used - when the developer expects his software to fail in case the + when the developer expects their software to fail in case the file cannot be read (i.e. it is literally an exception). ## Processes and raw files Every time a file is opened, Elixir spawns a new process. Writing - to a file is equivalent to sending messages to that process that + to a file is equivalent to sending messages to the process that writes to the file descriptor. This means files can be passed between nodes and message passing @@ -63,44 +69,134 @@ defmodule File do However, you may not always want to pay the price for this abstraction. In such cases, a file can be opened in `:raw` mode. The options `:read_ahead` - and `:delayed_write` are also useful when operating large files or + and `:delayed_write` are also useful when operating on large files or working with files in tight loops. - Check http://www.erlang.org/doc/man/file.html#open-2 for more information - about such options and other performance considerations. + Check `:file.open/2` for more information about such options and + other performance considerations. """ - alias :file, as: F - @type posix :: :file.posix() @type io_device :: :file.io_device() @type stat_options :: [time: :local | :universal | :posix] + @type mode :: + :append + | :binary + | :charlist + | :compressed + | :delayed_write + | :exclusive + | :raw + | :read + | :read_ahead + | :sync + | :write + | {:read_ahead, pos_integer} + | {:delayed_write, non_neg_integer, non_neg_integer} + | encoding_mode() + + @type encoding_mode :: + :utf8 + | { + :encoding, + :latin1 + | :unicode + | :utf8 + | :utf16 + | :utf32 + | {:utf16, :big | :little} + | {:utf32, :big | :little} + } + + @type stream_mode :: + encoding_mode() + | :append + | :compressed + | :delayed_write + | :trim_bom + | {:read_ahead, pos_integer | false} + | {:delayed_write, non_neg_integer, non_neg_integer} + + @type erlang_time :: + {{year :: non_neg_integer(), month :: 1..12, day :: 1..31}, + {hour :: 0..23, minute :: 0..59, second :: 0..59}} + + @type posix_time :: integer() @doc """ Returns `true` if the path is a regular file. + This function follows symbolic links, so if a symbolic link points to a + regular file, `true` is returned. + + ## Options + + The supported options are: + + * `:raw` - a single atom to bypass the file server and only check + for the file locally + ## Examples - File.regular? __ENV__.file #=> true + File.regular?(__ENV__.file) + #=> true """ - @spec regular?(Path.t) :: boolean - def regular?(path) do - :elixir_utils.read_file_type(IO.chardata_to_string(path)) == {:ok, :regular} + @spec regular?(Path.t(), [regular_option]) :: boolean + when regular_option: :raw + def regular?(path, opts \\ []) do + :elixir_utils.read_file_type(IO.chardata_to_string(path), opts) == {:ok, :regular} end @doc """ - Returns `true` if the path is a directory. + Returns `true` if the given path is a directory. + + This function follows symbolic links, so if a symbolic link points to a + directory, `true` is returned. + + ## Options + + The supported options are: + + * `:raw` - a single atom to bypass the file server and only check + for the file locally + + ## Examples + + File.dir?("./test") + #=> true + + File.dir?("test") + #=> true + + File.dir?("/usr/bin") + #=> true + + File.dir?("~/Downloads") + #=> false + + "~/Downloads" |> Path.expand() |> File.dir?() + #=> true + """ - @spec dir?(Path.t) :: boolean - def dir?(path) do - :elixir_utils.read_file_type(IO.chardata_to_string(path)) == {:ok, :directory} + @spec dir?(Path.t(), [dir_option]) :: boolean + when dir_option: :raw + def dir?(path, opts \\ []) do + :elixir_utils.read_file_type(IO.chardata_to_string(path), opts) == {:ok, :directory} end @doc """ Returns `true` if the given path exists. - It can be regular file, directory, socket, - symbolic link, named pipe or device file. + + It can be a regular file, directory, socket, symbolic link, named pipe, or device file. + Returns `false` for symbolic links pointing to non-existing targets. + + ## Options + + The supported options are: + + * `:raw` - a single atom to bypass the file server and only check + for the file locally ## Examples @@ -114,55 +210,68 @@ defmodule File do #=> true """ - @spec exists?(Path.t) :: boolean - def exists?(path) do - match?({:ok, _}, F.read_file_info(IO.chardata_to_string(path))) + @spec exists?(Path.t(), [exists_option]) :: boolean + when exists_option: :raw + def exists?(path, opts \\ []) do + opts = [{:time, :posix}] ++ opts + match?({:ok, _}, :file.read_file_info(IO.chardata_to_string(path), opts)) end @doc """ - Tries to create the directory `path`. Missing parent directories are not created. + Tries to create the directory `path`. + + Missing parent directories are not created. Returns `:ok` if successful, or `{:error, reason}` if an error occurs. Typical error reasons are: * `:eacces` - missing search or write permissions for the parent - directories of `path` + directories of `path` * `:eexist` - there is already a file or directory named `path` * `:enoent` - a component of `path` does not exist - * `:enospc` - there is a no space left on the device + * `:enospc` - there is no space left on the device * `:enotdir` - a component of `path` is not a directory; - on some platforms, `:enoent` is returned instead + on some platforms, `:enoent` is returned instead + """ - @spec mkdir(Path.t) :: :ok | {:error, posix} + @spec mkdir(Path.t()) :: :ok | {:error, posix} def mkdir(path) do - F.make_dir(IO.chardata_to_string(path)) + :file.make_dir(IO.chardata_to_string(path)) end @doc """ - Same as `mkdir/1`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `mkdir/1`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. """ - @spec mkdir!(Path.t) :: :ok | no_return + @spec mkdir!(Path.t()) :: :ok def mkdir!(path) do - path = IO.chardata_to_string(path) case mkdir(path) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "make directory", path: path + raise File.Error, + reason: reason, + action: "make directory", + path: IO.chardata_to_string(path) end end @doc """ - Tries to create the directory `path`. Missing parent directories are created. - Returns `:ok` if successful, or `{:error, reason}` if an error occurs. + Tries to create the directory `path`. + + Missing parent directories are created. Returns `:ok` if successful, or + `{:error, reason}` if an error occurs. Typical error reasons are: * `:eacces` - missing search or write permissions for the parent - directories of `path` - * `:enospc` - there is a no space left on the device + directories of `path` + * `:enospc` - there is no space left on the device * `:enotdir` - a component of `path` is not a directory + """ - @spec mkdir_p(Path.t) :: :ok | {:error, posix} + @spec mkdir_p(Path.t()) :: :ok | {:error, posix} def mkdir_p(path) do do_mkdir_p(IO.chardata_to_string(path)) end @@ -176,14 +285,17 @@ defmodule File do :ok else parent = Path.dirname(path) + if parent == path do # Protect against infinite loop {:error, :einval} else _ = do_mkdir_p(parent) + case :file.make_dir(path) do {:error, :eexist} = error -> if dir?(path), do: :ok, else: error + other -> other end @@ -192,15 +304,20 @@ defmodule File do end @doc """ - Same as `mkdir_p/1`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `mkdir_p/1`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. """ - @spec mkdir_p!(Path.t) :: :ok | no_return + @spec mkdir_p!(Path.t()) :: :ok def mkdir_p!(path) do - path = IO.chardata_to_string(path) case mkdir_p(path) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "make directory (with -p)", path: path + raise File.Error, + reason: reason, + action: "make directory (with -p)", + path: IO.chardata_to_string(path) end end @@ -212,31 +329,31 @@ defmodule File do * `:enoent` - the file does not exist * `:eacces` - missing permission for reading the file, - or for searching one of the parent directories + or for searching one of the parent directories * `:eisdir` - the named file is a directory * `:enotdir` - a component of the file name is not a directory; - on some platforms, `:enoent` is returned instead + on some platforms, `:enoent` is returned instead * `:enomem` - there is not enough memory for the contents of the file You can use `:file.format_error/1` to get a descriptive string of the error. """ - @spec read(Path.t) :: {:ok, binary} | {:error, posix} + @spec read(Path.t()) :: {:ok, binary} | {:error, posix} def read(path) do - F.read_file(IO.chardata_to_string(path)) + :file.read_file(IO.chardata_to_string(path)) end @doc """ - Returns binary with the contents of the given filename or raises - `File.Error` if an error occurs. + Returns a binary with the contents of the given filename, + or raises a `File.Error` exception if an error occurs. """ - @spec read!(Path.t) :: binary | no_return + @spec read!(Path.t()) :: binary def read!(path) do - path = IO.chardata_to_string(path) case read(path) do {:ok, binary} -> binary + {:error, reason} -> - raise File.Error, reason: reason, action: "read file", path: path + raise File.Error, reason: reason, action: "read file", path: IO.chardata_to_string(path) end end @@ -250,83 +367,273 @@ defmodule File do The accepted options are: - * `:time` - `:local | :universal | :posix`; default: `:local` + * `:time` - configures how the file timestamps are returned + + The values for `:time` can be: + * `:universal` - returns a `{date, time}` tuple in UTC (default) + * `:local` - returns a `{date, time}` tuple using the same time zone as the + machine + * `:posix` - returns the time as integer seconds since epoch + + Note: Since file times are stored in POSIX time format on most operating systems, + it is faster to retrieve file information with the `time: :posix` option. """ - @spec stat(Path.t, stat_options) :: {:ok, File.Stat.t} | {:error, posix} + @spec stat(Path.t(), stat_options) :: {:ok, File.Stat.t()} | {:error, posix} def stat(path, opts \\ []) do - case F.read_file_info(IO.chardata_to_string(path), opts) do + opts = Keyword.put_new(opts, :time, :universal) + + case :file.read_file_info(IO.chardata_to_string(path), opts) do {:ok, fileinfo} -> {:ok, File.Stat.from_record(fileinfo)} + error -> error end end @doc """ - Same as `stat/2` but returns the `File.Stat` directly and - throws `File.Error` if an error is returned. + Same as `stat/2` but returns the `File.Stat` directly, + or raises a `File.Error` exception if an error is returned. """ - @spec stat!(Path.t, stat_options) :: File.Stat.t | no_return + @spec stat!(Path.t(), stat_options) :: File.Stat.t() def stat!(path, opts \\ []) do - path = IO.chardata_to_string(path) case stat(path, opts) do - {:ok, info} -> info + {:ok, info} -> + info + + {:error, reason} -> + raise File.Error, + reason: reason, + action: "read file stats", + path: IO.chardata_to_string(path) + end + end + + @doc """ + Returns information about the `path`. If the file is a symlink, sets + the `type` to `:symlink` and returns a `File.Stat` struct for the link. For any + other file, returns exactly the same values as `stat/2`. + + For more details, see `:file.read_link_info/2`. + + ## Options + + The accepted options are: + + * `:time` - configures how the file timestamps are returned + + The values for `:time` can be: + + * `:universal` - returns a `{date, time}` tuple in UTC (default) + * `:local` - returns a `{date, time}` tuple using the machine time + * `:posix` - returns the time as integer seconds since epoch + + Note: Since file times are stored in POSIX time format on most operating systems, + it is faster to retrieve file information with the `time: :posix` option. + """ + @spec lstat(Path.t(), stat_options) :: {:ok, File.Stat.t()} | {:error, posix} + def lstat(path, opts \\ []) do + opts = Keyword.put_new(opts, :time, :universal) + + case :file.read_link_info(IO.chardata_to_string(path), opts) do + {:ok, fileinfo} -> + {:ok, File.Stat.from_record(fileinfo)} + + error -> + error + end + end + + @doc """ + Same as `lstat/2` but returns the `File.Stat` struct directly, + or raises a `File.Error` exception if an error is returned. + """ + @spec lstat!(Path.t(), stat_options) :: File.Stat.t() + def lstat!(path, opts \\ []) do + case lstat(path, opts) do + {:ok, info} -> + info + {:error, reason} -> - raise File.Error, reason: reason, action: "read file stats", path: path + raise File.Error, + reason: reason, + action: "read file stats", + path: IO.chardata_to_string(path) end end @doc """ - Writes the given `File.Stat` back to the filesystem at the given + Reads the symbolic link at `path`. + + If `path` exists and is a symlink, returns `{:ok, target}`, otherwise returns + `{:error, reason}`. + + For more details, see `:file.read_link/1`. + + Typical error reasons are: + + * `:einval` - path is not a symbolic link + * `:enoent` - path does not exist + * `:enotsup` - symbolic links are not supported on the current platform + + """ + @doc since: "1.5.0" + @spec read_link(Path.t()) :: {:ok, binary} | {:error, posix} + def read_link(path) do + case path |> IO.chardata_to_string() |> :file.read_link() do + {:ok, target} -> {:ok, IO.chardata_to_string(target)} + error -> error + end + end + + @doc """ + Same as `read_link/1` but returns the target directly, + or raises a `File.Error` exception if an error is returned. + """ + @doc since: "1.5.0" + @spec read_link!(Path.t()) :: binary + def read_link!(path) do + case read_link(path) do + {:ok, resolved} -> + resolved + + {:error, reason} -> + raise File.Error, reason: reason, action: "read link", path: IO.chardata_to_string(path) + end + end + + @doc """ + Writes the given `File.Stat` back to the file system at the given path. Returns `:ok` or `{:error, reason}`. """ - @spec write_stat(Path.t, File.Stat.t, stat_options) :: :ok | {:error, posix} + @spec write_stat(Path.t(), File.Stat.t(), stat_options) :: :ok | {:error, posix} def write_stat(path, stat, opts \\ []) do - F.write_file_info(IO.chardata_to_string(path), File.Stat.to_record(stat), opts) + opts = Keyword.put_new(opts, :time, :universal) + :file.write_file_info(IO.chardata_to_string(path), File.Stat.to_record(stat), opts) end @doc """ - Same as `write_stat/3` but raises an exception if it fails. + Same as `write_stat/3` but raises a `File.Error` exception if it fails. Returns `:ok` otherwise. """ - @spec write_stat!(Path.t, File.Stat.t, stat_options) :: :ok | no_return + @spec write_stat!(Path.t(), File.Stat.t(), stat_options) :: :ok def write_stat!(path, stat, opts \\ []) do - path = IO.chardata_to_string(path) case write_stat(path, stat, opts) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "write file stats", path: path + raise File.Error, + reason: reason, + action: "write file stats", + path: IO.chardata_to_string(path) end end @doc """ Updates modification time (mtime) and access time (atime) of - the given file. File is created if it doesn’t exist. + the given file. + + The file is created if it doesn't exist. Requires datetime in UTC + (as returned by `:erlang.universaltime()`) or an integer + representing the POSIX timestamp (as returned by `System.os_time(:second)`). + + In Unix-like systems, changing the modification time may require + you to be either `root` or the owner of the file. Having write + access may not be enough. In those cases, touching the file the + first time (to create it) will succeed, but touching an existing + file with fail with `{:error, :eperm}`. + + ## Examples + + File.touch("/tmp/a.txt", {{2018, 1, 30}, {13, 59, 59}}) + #=> :ok + File.touch("/fakedir/b.txt", {{2018, 1, 30}, {13, 59, 59}}) + {:error, :enoent} + + File.touch("/tmp/a.txt", 1544519753) + #=> :ok + """ - @spec touch(Path.t, :calendar.datetime) :: :ok | {:error, posix} - def touch(path, time \\ :calendar.local_time) do + @spec touch(Path.t(), erlang_time() | posix_time()) :: :ok | {:error, posix} + def touch(path, time \\ System.os_time(:second)) + + def touch(path, time) when is_tuple(time) do path = IO.chardata_to_string(path) - case F.change_time(path, time) do - {:error, :enoent} -> - write(path, "") - F.change_time(path, time) - other -> - other - end + + with {:error, :enoent} <- :elixir_utils.change_universal_time(path, time), + :ok <- write(path, "", [:append]), + do: :elixir_utils.change_universal_time(path, time) + end + + def touch(path, time) when is_integer(time) do + path = IO.chardata_to_string(path) + + with {:error, :enoent} <- :elixir_utils.change_posix_time(path, time), + :ok <- write(path, "", [:append]), + do: :elixir_utils.change_posix_time(path, time) end @doc """ - Same as `touch/2` but raises an exception if it fails. + Same as `touch/2` but raises a `File.Error` exception if it fails. Returns `:ok` otherwise. + + The file is created if it doesn't exist. Requires datetime in UTC + (as returned by `:erlang.universaltime()`) or an integer + representing the POSIX timestamp (as returned by `System.os_time(:second)`). + + ## Examples + + File.touch!("/tmp/a.txt", {{2018, 1, 30}, {13, 59, 59}}) + #=> :ok + File.touch!("/fakedir/b.txt", {{2018, 1, 30}, {13, 59, 59}}) + ** (File.Error) could not touch "/fakedir/b.txt": no such file or directory + + File.touch!("/tmp/a.txt", 1544519753) + """ - @spec touch!(Path.t, :calendar.datetime) :: :ok | no_return - def touch!(path, time \\ :calendar.local_time) do - path = IO.chardata_to_string(path) + @spec touch!(Path.t(), erlang_time() | posix_time()) :: :ok + def touch!(path, time \\ System.os_time(:second)) do case touch(path, time) do - :ok -> :ok + :ok -> + :ok + + {:error, reason} -> + raise File.Error, reason: reason, action: "touch", path: IO.chardata_to_string(path) + end + end + + @doc """ + Creates a hard link `new` to the file `existing`. + + Returns `:ok` if successful, `{:error, reason}` otherwise. + If the operating system does not support hard links, returns + `{:error, :enotsup}`. + """ + @doc since: "1.5.0" + @spec ln(Path.t(), Path.t()) :: :ok | {:error, posix} + def ln(existing, new) do + :file.make_link(IO.chardata_to_string(existing), IO.chardata_to_string(new)) + end + + @doc """ + Same as `ln/2` but raises a `File.LinkError` exception if it fails. + Returns `:ok` otherwise. + """ + @doc since: "1.5.0" + @spec ln!(Path.t(), Path.t()) :: :ok + def ln!(existing, new) do + case ln(existing, new) do + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "touch", path: path + raise File.LinkError, + reason: reason, + action: "create hard link", + existing: IO.chardata_to_string(existing), + new: IO.chardata_to_string(new) end end @@ -337,14 +644,35 @@ defmodule File do If the operating system does not support symlinks, returns `{:error, :enotsup}`. """ + @doc since: "1.5.0" + @spec ln_s(Path.t(), Path.t()) :: :ok | {:error, posix} def ln_s(existing, new) do - F.make_symlink(existing, new) + :file.make_symlink(IO.chardata_to_string(existing), IO.chardata_to_string(new)) + end + + @doc """ + Same as `ln_s/2` but raises a `File.LinkError` exception if it fails. + Returns `:ok` otherwise. + """ + @spec ln_s!(Path.t(), Path.t()) :: :ok + def ln_s!(existing, new) do + case ln_s(existing, new) do + :ok -> + :ok + + {:error, reason} -> + raise File.LinkError, + reason: reason, + action: "create symlink", + existing: IO.chardata_to_string(existing), + new: IO.chardata_to_string(new) + end end @doc """ Copies the contents of `source` to `destination`. - Both parameters can be a filename or an io device opened + Both parameters can be a filename or an IO device opened with `open/2`. `bytes_count` specifies the number of bytes to copy, the default being `:infinity`. @@ -363,121 +691,197 @@ defmodule File do Typical error reasons are the same as in `open/2`, `read/1` and `write/3`. """ - @spec copy(Path.t, Path.t, pos_integer | :infinity) :: {:ok, non_neg_integer} | {:error, posix} + @spec copy(Path.t() | io_device, Path.t() | io_device, pos_integer | :infinity) :: + {:ok, non_neg_integer} | {:error, posix} def copy(source, destination, bytes_count \\ :infinity) do - F.copy(IO.chardata_to_string(source), IO.chardata_to_string(destination), bytes_count) + :file.copy(maybe_to_string(source), maybe_to_string(destination), bytes_count) end @doc """ - The same as `copy/3` but raises an `File.CopyError` if it fails. + The same as `copy/3` but raises a `File.CopyError` exception if it fails. Returns the `bytes_copied` otherwise. """ - @spec copy!(Path.t, Path.t, pos_integer | :infinity) :: non_neg_integer | no_return + @spec copy!(Path.t() | io_device, Path.t() | io_device, pos_integer | :infinity) :: + non_neg_integer def copy!(source, destination, bytes_count \\ :infinity) do - source = IO.chardata_to_string(source) - destination = IO.chardata_to_string(destination) case copy(source, destination, bytes_count) do - {:ok, bytes_count} -> bytes_count + {:ok, bytes_count} -> + bytes_count + {:error, reason} -> - raise File.CopyError, reason: reason, action: "copy", - source: source, destination: destination + raise File.CopyError, + reason: reason, + action: "copy", + source: maybe_to_string(source), + destination: maybe_to_string(destination) end end @doc """ - Copies the contents in `source` to `destination` preserving its mode. + Renames the `source` file to `destination` file. It can be used to move files + (and directories) between directories. If moving a file, you must fully + specify the `destination` filename, it is not sufficient to simply specify + its directory. - If a file already exists in the destination, it invokes a - callback which should return `true` if the existing file - should be overwritten, `false` otherwise. It defaults to return `true`. + Returns `:ok` in case of success, `{:error, reason}` otherwise. - It returns `:ok` in case of success, returns - `{:error, reason}` otherwise. + Note: The command `mv` in Unix-like systems behaves differently depending on + whether `source` is a file and the `destination` is an existing directory. + We have chosen to explicitly disallow this behaviour. - If you want to copy contents from an io device to another device + ## Examples + + # Rename file "a.txt" to "b.txt" + File.rename("a.txt", "b.txt") + + # Rename directory "samples" to "tmp" + File.rename("samples", "tmp") + + """ + @doc since: "1.1.0" + @spec rename(Path.t(), Path.t()) :: :ok | {:error, posix} + def rename(source, destination) do + :file.rename(source, destination) + end + + @doc """ + The same as `rename/2` but raises a `File.RenameError` exception if it fails. + Returns `:ok` otherwise. + """ + @doc since: "1.9.0" + @spec rename!(Path.t(), Path.t()) :: :ok + def rename!(source, destination) do + case rename(source, destination) do + :ok -> + :ok + + {:error, reason} -> + raise File.RenameError, + reason: reason, + action: "rename", + source: IO.chardata_to_string(source), + destination: IO.chardata_to_string(destination) + end + end + + @doc """ + Copies the contents of `source_file` to `destination_file` preserving its modes. + + `source_file` must be a file or a symbolic link to one. `destination_file` must + be a path to a non-existent file. If either is a directory, `{:error, :eisdir}` + will be returned. + + The `callback` function is invoked if the `destination_file` already exists. + The function receives arguments for `source_file` and `destination_file`; + it should return `true` if the existing file should be overwritten, `false` if + otherwise. The default callback returns `true`. + + The function returns `:ok` in case of success. Otherwise, it returns + `{:error, reason}`. + + If you want to copy contents from an IO device to another device or do a straight copy from a source to a destination without preserving modes, check `copy/3` instead. - Note: The command `cp` in Unix systems behaves differently depending - if `destination` is an existing directory or not. We have chosen to - explicitly disallow this behaviour. If destination is a directory, an - error will be returned. + Note: The command `cp` in Unix-like systems behaves differently depending on + whether the destination is an existing directory or not. We have chosen to + explicitly disallow copying to a destination which is a directory, + and an error will be returned if tried. """ - @spec cp(Path.t, Path.t, (Path.t, Path.t -> boolean)) :: :ok | {:error, posix} - def cp(source, destination, callback \\ fn(_, _) -> true end) do - source = IO.chardata_to_string(source) - destination = IO.chardata_to_string(destination) + @spec cp(Path.t(), Path.t(), (Path.t(), Path.t() -> boolean)) :: :ok | {:error, posix} + def cp(source_file, destination_file, callback \\ fn _, _ -> true end) do + source_file = IO.chardata_to_string(source_file) + destination_file = IO.chardata_to_string(destination_file) - case do_cp_file(source, destination, callback, []) do + case do_cp_file(source_file, destination_file, callback, []) do {:error, reason, _} -> {:error, reason} _ -> :ok end end + defp path_differs?(path, path), do: false + + defp path_differs?(p1, p2) do + Path.expand(p1) !== Path.expand(p2) + end + @doc """ - The same as `cp/3`, but raises `File.CopyError` if it fails. - Returns the list of copied files otherwise. + The same as `cp/3`, but raises a `File.CopyError` exception if it fails. + Returns `:ok` otherwise. """ - @spec cp!(Path.t, Path.t, (Path.t, Path.t -> boolean)) :: :ok | no_return - def cp!(source, destination, callback \\ fn(_, _) -> true end) do - source = IO.chardata_to_string(source) - destination = IO.chardata_to_string(destination) + @spec cp!(Path.t(), Path.t(), (Path.t(), Path.t() -> boolean)) :: :ok + def cp!(source_file, destination_file, callback \\ fn _, _ -> true end) do + case cp(source_file, destination_file, callback) do + :ok -> + :ok - case cp(source, destination, callback) do - :ok -> :ok {:error, reason} -> - raise File.CopyError, reason: reason, action: "copy recursively", - source: source, destination: destination + raise File.CopyError, + reason: reason, + action: "copy", + source: IO.chardata_to_string(source_file), + destination: IO.chardata_to_string(destination_file) end end @doc ~S""" - Copies the contents in source to destination. + Copies the contents in `source` to `destination` recursively, maintaining the + source directory structure and modes. - If the source is a file, it copies `source` to - `destination`. If the source is a directory, it copies - the contents inside source into the destination. + If `source` is a file or a symbolic link to it, `destination` must be a path + to an existent file, a symbolic link to one, or a path to a non-existent file. + + If `source` is a directory, or a symbolic link to it, then `destination` must + be an existent `directory` or a symbolic link to one, or a path to a non-existent directory. - If a file already exists in the destination, - it invokes a callback which should return - `true` if the existing file should be overwritten, - `false` otherwise. It defaults to return `true`. + If the source is a file, it copies `source` to + `destination`. If the `source` is a directory, it copies + the contents inside source into the `destination` directory. - If a directory already exists in the destination - where a file is meant to be (or otherwise), this - function will fail. + If a file already exists in the destination, it invokes `callback`. + `callback` must be a function that takes two arguments: `source` and `destination`. + The callback should return `true` if the existing file should be overwritten and `false` otherwise. This function may fail while copying files, in such cases, it will leave the destination - directory in a dirty state, where already - copied files won't be removed. + directory in a dirty state, where file which have already been copied + won't be removed. - It returns `{:ok, files_and_directories}` in case of - success with all files and directories copied in no - specific order, `{:error, reason, file}` otherwise. + The function returns `{:ok, files_and_directories}` in case of + success, `files_and_directories` lists all files and directories copied in no + specific order. It returns `{:error, reason, file}` otherwise. - Note: The command `cp` in Unix systems behaves differently - depending if `destination` is an existing directory or not. - We have chosen to explicitly disallow this behaviour. + Note: The command `cp` in Unix-like systems behaves differently depending on + whether `destination` is an existing directory or not. We have chosen to + explicitly disallow this behaviour. If `source` is a `file` and `destination` + is a directory, `{:error, :eisdir}` will be returned. ## Examples - # Copies "a.txt" to "tmp" - File.cp_r "a.txt", "tmp.txt" + # Copies file "a.txt" to "b.txt" + File.cp_r("a.txt", "b.txt") # Copies all files in "samples" to "tmp" - File.cp_r "samples", "tmp" + File.cp_r("samples", "tmp") # Same as before, but asks the user how to proceed in case of conflicts - File.cp_r "samples", "tmp", fn(source, destination) -> - IO.gets("Overwriting #{destination} by #{source}. Type y to confirm.") == "y" - end + File.cp_r("samples", "tmp", fn source, destination -> + IO.gets("Overwriting #{destination} by #{source}. Type y to confirm. ") == "y\n" + end) """ - @spec cp_r(Path.t, Path.t, (Path.t, Path.t -> boolean)) :: {:ok, [binary]} | {:error, posix, binary} - def cp_r(source, destination, callback \\ fn(_, _) -> true end) when is_function(callback) do - source = IO.chardata_to_string(source) - destination = IO.chardata_to_string(destination) + @spec cp_r(Path.t(), Path.t(), (Path.t(), Path.t() -> boolean)) :: + {:ok, [binary]} | {:error, posix, binary} + def cp_r(source, destination, callback \\ fn _, _ -> true end) when is_function(callback, 2) do + source = + source + |> IO.chardata_to_string() + |> assert_no_null_byte!("File.cp_r/3") + + destination = + destination + |> IO.chardata_to_string() + |> assert_no_null_byte!("File.cp_r/3") case do_cp_r(source, destination, callback, []) do {:error, _, _} = error -> error @@ -486,47 +890,58 @@ defmodule File do end @doc """ - The same as `cp_r/3`, but raises `File.CopyError` if it fails. + The same as `cp_r/3`, but raises a `File.CopyError` exception if it fails. Returns the list of copied files otherwise. """ - @spec cp_r!(Path.t, Path.t, (Path.t, Path.t -> boolean)) :: [binary] | no_return - def cp_r!(source, destination, callback \\ fn(_, _) -> true end) do - source = IO.chardata_to_string(source) - destination = IO.chardata_to_string(destination) - + @spec cp_r!(Path.t(), Path.t(), (Path.t(), Path.t() -> boolean)) :: [binary] + def cp_r!(source, destination, callback \\ fn _, _ -> true end) do case cp_r(source, destination, callback) do - {:ok, files} -> files + {:ok, files} -> + files + {:error, reason, file} -> - raise File.CopyError, reason: reason, action: "copy recursively", - source: source, destination: destination, on: file + raise File.CopyError, + reason: reason, + action: "copy recursively", + on: file, + source: IO.chardata_to_string(source), + destination: IO.chardata_to_string(destination) end end - # src may be a file or a directory, dest is definitely - # a directory. Returns nil unless an error is found. defp do_cp_r(src, dest, callback, acc) when is_list(acc) do case :elixir_utils.read_link_type(src) do {:ok, :regular} -> do_cp_file(src, dest, callback, acc) + {:ok, :symlink} -> - case F.read_link(src) do + case :file.read_link(src) do {:ok, link} -> do_cp_link(link, src, dest, callback, acc) {:error, reason} -> {:error, reason, src} end + {:ok, :directory} -> - case F.list_dir(src) do + case :file.list_dir(src) do {:ok, files} -> case mkdir(dest) do success when success in [:ok, {:error, :eexist}] -> - Enum.reduce(files, [dest|acc], fn(x, acc) -> + Enum.reduce(files, [dest | acc], fn x, acc -> do_cp_r(Path.join(src, x), Path.join(dest, x), callback, acc) end) - {:error, reason} -> {:error, reason, dest} + + {:error, reason} -> + {:error, reason, dest} end - {:error, reason} -> {:error, reason, src} + + {:error, reason} -> + {:error, reason, src} end - {:ok, _} -> {:error, :eio, src} - {:error, reason} -> {:error, reason, src} + + {:ok, _} -> + {:error, :eio, src} + + {:error, reason} -> + {:error, reason, src} end end @@ -542,42 +957,51 @@ defmodule File do # Both src and dest are files. defp do_cp_file(src, dest, callback, acc) do - case F.copy(src, {dest, [:exclusive]}) do + case :file.copy(src, {dest, [:exclusive]}) do {:ok, _} -> copy_file_mode!(src, dest) - [dest|acc] + [dest | acc] + {:error, :eexist} -> - if callback.(src, dest) do - rm(dest) + if path_differs?(src, dest) and callback.(src, dest) do case copy(src, dest) do {:ok, _} -> copy_file_mode!(src, dest) - [dest|acc] - {:error, reason} -> {:error, reason, src} + [dest | acc] + + {:error, reason} -> + {:error, reason, src} end else acc end - {:error, reason} -> {:error, reason, src} + + {:error, reason} -> + {:error, reason, src} end end # Both src and dest are files. defp do_cp_link(link, src, dest, callback, acc) do - case F.make_symlink(link, dest) do + case :file.make_symlink(link, dest) do :ok -> - [dest|acc] + [dest | acc] + {:error, :eexist} -> - if callback.(src, dest) do - rm(dest) - case F.make_symlink(link, dest) do - :ok -> [dest|acc] + if path_differs?(src, dest) and callback.(src, dest) do + # If rm/1 fails, :file.make_symlink/2 will fail + _ = rm(dest) + + case :file.make_symlink(link, dest) do + :ok -> [dest | acc] {:error, reason} -> {:error, reason, src} end else acc end - {:error, reason} -> {:error, reason, src} + + {:error, reason} -> + {:error, reason, src} end end @@ -588,39 +1012,48 @@ defmodule File do contents are overwritten. Returns `:ok` if successful, or `{:error, reason}` if an error occurs. + `content` must be `iodata` (a list of bytes or a binary). Setting the + encoding for this function has no effect. + **Warning:** Every time this function is invoked, a file descriptor is opened and a new process is spawned to write to the file. For this reason, if you are doing multiple writes in a loop, opening the file via `File.open/2` and using the functions in `IO` to write to the file will yield much better performance - then calling this function multiple times. + than calling this function multiple times. Typical error reasons are: * `:enoent` - a component of the file name does not exist * `:enotdir` - a component of the file name is not a directory; - on some platforms, enoent is returned instead - * `:enospc` - there is a no space left on the device + on some platforms, `:enoent` is returned instead + * `:enospc` - there is no space left on the device * `:eacces` - missing permission for writing the file or searching one of - the parent directories + the parent directories * `:eisdir` - the named file is a directory Check `File.open/2` for other available options. """ - @spec write(Path.t, iodata, list) :: :ok | {:error, posix} + @spec write(Path.t(), iodata, [mode]) :: :ok | {:error, posix} def write(path, content, modes \\ []) do - F.write_file(IO.chardata_to_string(path), content, modes) + modes = normalize_modes(modes, false) + :file.write_file(IO.chardata_to_string(path), content, modes) end @doc """ - Same as `write/3` but raises an exception if it fails, returns `:ok` otherwise. + Same as `write/3` but raises a `File.Error` exception if it fails. + Returns `:ok` otherwise. """ - @spec write!(Path.t, iodata, list) :: :ok | no_return + @spec write!(Path.t(), iodata, [mode]) :: :ok def write!(path, content, modes \\ []) do - path = IO.chardata_to_string(path) - case F.write_file(path, content, modes) do - :ok -> :ok + case write(path, content, modes) do + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "write to file", path: path + raise File.Error, + reason: reason, + action: "write to file", + path: IO.chardata_to_string(path) end end @@ -628,6 +1061,7 @@ defmodule File do Tries to delete the file `path`. Returns `:ok` if successful, or `{:error, reason}` if an error occurs. + Note the file is deleted even if in read-only mode. Typical error reasons are: @@ -636,89 +1070,111 @@ defmodule File do * `:eacces` - missing permission for the file or one of its parents * `:eperm` - the file is a directory and user is not super-user * `:enotdir` - a component of the file name is not a directory; - on some platforms, enoent is returned instead + on some platforms, `:enoent` is returned instead * `:einval` - filename had an improper type, such as tuple ## Examples - File.rm('file.txt') + File.rm("file.txt") #=> :ok - File.rm('tmp_dir/') + File.rm("tmp_dir/") #=> {:error, :eperm} """ - @spec rm(Path.t) :: :ok | {:error, posix} + @spec rm(Path.t()) :: :ok | {:error, posix} def rm(path) do path = IO.chardata_to_string(path) - case F.delete(path) do + + case :file.delete(path) do :ok -> :ok + {:error, :eacces} = e -> change_mode_windows(path) || e + {:error, _} = e -> e end end defp change_mode_windows(path) do - if match? {:win32, _}, :os.type do - case F.read_file_info(IO.chardata_to_string(path)) do + if match?({:win32, _}, :os.type()) do + case :file.read_file_info(path) do {:ok, file_info} when elem(file_info, 3) in [:read, :none] -> - File.chmod(path, (elem(file_info, 7) + 0200)) - F.delete(path) + change_mode_windows(path, file_info) + _ -> nil end end end + defp change_mode_windows(path, file_info) do + case chmod(path, elem(file_info, 7) + 0o200) do + :ok -> :file.delete(path) + {:error, _reason} = error -> error + end + end + @doc """ - Same as `rm/1`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `rm/1`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. """ - @spec rm!(Path.t) :: :ok | no_return + @spec rm!(Path.t()) :: :ok def rm!(path) do - path = IO.chardata_to_string(path) case rm(path) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "remove file", path: path + raise File.Error, reason: reason, action: "remove file", path: IO.chardata_to_string(path) end end @doc """ Tries to delete the dir at `path`. + Returns `:ok` if successful, or `{:error, reason}` if an error occurs. + It returns `{:error, :eexist}` if the directory is not empty. ## Examples - File.rmdir('tmp_dir') + File.rmdir("tmp_dir") #=> :ok - File.rmdir('file.txt') + File.rmdir("non_empty_dir") + #=> {:error, :eexist} + + File.rmdir("file.txt") #=> {:error, :enotdir} """ - @spec rmdir(Path.t) :: :ok | {:error, posix} + @spec rmdir(Path.t()) :: :ok | {:error, posix} def rmdir(path) do - F.del_dir(IO.chardata_to_string(path)) + :file.del_dir(IO.chardata_to_string(path)) end @doc """ - Same as `rmdir/1`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `rmdir/1`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. """ - @spec rmdir!(Path.t) :: :ok | {:error, posix} + @spec rmdir!(Path.t()) :: :ok | {:error, posix} def rmdir!(path) do - path = IO.chardata_to_string(path) case rmdir(path) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "remove directory", path: path + raise File.Error, + reason: reason, + action: "remove directory", + path: IO.chardata_to_string(path) end end @doc """ - Remove files and directories recursively at the given `path`. + Removes files and directories recursively at the given `path`. Symlinks are not followed but simply removed, non-existing files are simply ignored (i.e. doesn't make this function fail). @@ -728,111 +1184,135 @@ defmodule File do ## Examples - File.rm_rf "samples" + File.rm_rf("samples") #=> {:ok, ["samples", "samples/1.txt"]} - File.rm_rf "unknown" + File.rm_rf("unknown") #=> {:ok, []} """ - @spec rm_rf(Path.t) :: {:ok, [binary]} | {:error, posix, binary} + @spec rm_rf(Path.t()) :: {:ok, [binary]} | {:error, posix, binary} def rm_rf(path) do - do_rm_rf(IO.chardata_to_string(path), {:ok, []}) + {major, _} = :os.type() + + path + |> IO.chardata_to_string() + |> assert_no_null_byte!("File.rm_rf/1") + |> do_rm_rf([], major) end - defp do_rm_rf(path, {:ok, _} = entry) do - case safe_list_dir(path) do + defp do_rm_rf(path, acc, major) do + case safe_list_dir(path, major) do {:ok, files} when is_list(files) -> - res = - Enum.reduce files, entry, fn(file, tuple) -> - do_rm_rf(Path.join(path, file), tuple) - end - - case res do - {:ok, acc} -> - case rmdir(path) do - :ok -> {:ok, [path|acc]} - {:error, :enoent} -> res - {:error, reason} -> {:error, reason, path} + acc = + Enum.reduce(files, acc, fn file, acc -> + # In case we can't delete, continue anyway, we might succeed + # to delete it on Windows due to how they handle symlinks. + case do_rm_rf(Path.join(path, file), acc, major) do + {:ok, acc} -> acc + {:error, _, _} -> acc end - reason -> - reason + end) + + case rmdir(path) do + :ok -> {:ok, [path | acc]} + {:error, :enoent} -> {:ok, acc} + {:error, reason} -> {:error, reason, path} end - {:ok, :directory} -> do_rm_directory(path, entry) - {:ok, :regular} -> do_rm_regular(path, entry) - {:error, reason} when reason in [:enoent, :enotdir] -> entry - {:error, reason} -> {:error, reason, path} - end - end - defp do_rm_rf(_, reason) do - reason + {:ok, :directory} -> + do_rm_directory(path, acc) + + {:ok, :regular} -> + do_rm_regular(path, acc) + + {:error, reason} when reason in [:enoent, :enotdir] -> + {:ok, acc} + + {:error, reason} -> + {:error, reason, path} + end end - defp do_rm_regular(path, {:ok, acc} = entry) do + defp do_rm_regular(path, acc) do case rm(path) do - :ok -> {:ok, [path|acc]} - {:error, :enoent} -> entry + :ok -> {:ok, [path | acc]} + {:error, :enoent} -> {:ok, acc} {:error, reason} -> {:error, reason, path} end end - # On windows, symlinks are treated as directory and must be removed - # with rmdir/1. But on Unix, we remove them via rm/1. So we first try - # to remove it as a directory and, if we get :enotdir, we fallback to - # a file removal. - defp do_rm_directory(path, {:ok, acc} = entry) do + # On Windows, symlinks are treated as directory and must be removed + # with rmdir/1. But on Unix-like systems, we remove them via rm/1. + # So we first try to remove it as a directory and, if we get :enotdir, + # we fall back to a file removal. + defp do_rm_directory(path, acc) do case rmdir(path) do - :ok -> {:ok, [path|acc]} - {:error, :enotdir} -> do_rm_regular(path, entry) - {:error, :enoent} -> entry + :ok -> {:ok, [path | acc]} + {:error, :enotdir} -> do_rm_regular(path, acc) + {:error, :enoent} -> {:ok, acc} {:error, reason} -> {:error, reason, path} end end - defp safe_list_dir(path) do + defp safe_list_dir(path, major) do case :elixir_utils.read_link_type(path) do - {:ok, :symlink} -> + {:ok, :directory} -> + :file.list_dir_all(path) + + {:ok, :symlink} when major == :win32 -> case :elixir_utils.read_file_type(path) do {:ok, :directory} -> {:ok, :directory} _ -> {:ok, :regular} end - {:ok, :directory} -> - F.list_dir(path) + {:ok, _} -> {:ok, :regular} + {:error, reason} -> {:error, reason} end end @doc """ - Same as `rm_rf/1` but raises `File.Error` in case of failures, + Same as `rm_rf/1` but raises a `File.Error` exception in case of failures, otherwise the list of files or directories removed. """ - @spec rm_rf!(Path.t) :: [binary] | no_return + @spec rm_rf!(Path.t()) :: [binary] def rm_rf!(path) do - path = IO.chardata_to_string(path) case rm_rf(path) do - {:ok, files} -> files + {:ok, files} -> + files + {:error, reason, _} -> - raise File.Error, reason: reason, path: path, + raise File.Error, + reason: reason, + path: IO.chardata_to_string(path), action: "remove files and directories recursively from" end end @doc ~S""" - Opens the given `path` according to the given list of modes. + Opens the given `path`. In order to write and read files, one must use the functions - in the `IO` module. By default, a file is opened in binary mode + in the `IO` module. By default, a file is opened in `:binary` mode, which requires the functions `IO.binread/2` and `IO.binwrite/2` to interact with the file. A developer may pass `:utf8` as an option when opening the file and then all other functions from `IO` are available, since they work directly with Unicode data. + `modes_or_function` can either be a list of modes or a function. If it's a + list, it's considered to be a list of modes (that are documented below). If + it's a function, then it's equivalent to calling `open(path, [], + modes_or_function)`. See the documentation for `open/3` for more information + on this function. + The allowed modes: + * `:binary` - opens the file in binary mode, disabling special handling of Unicode sequences + (default mode). + * `:read` - the file, which must exist, is opened for reading. * `:write` - the file is opened for writing. It is created if it does not @@ -848,8 +1328,8 @@ defmodule File do * `:exclusive` - the file, when opened for writing, is created if it does not exist. If the file exists, open will return `{:error, :eexist}`. - * `:char_list` - when this term is given, read operations on the file will - return char lists rather than binaries. + * `:charlist` - when this term is given, read operations on the file will + return charlists rather than binaries. * `:compressed` - makes it possible to read or write gzip compressed files. @@ -859,24 +1339,27 @@ defmodule File do * `:utf8` - this option denotes how data is actually stored in the disk file and makes the file perform automatic translation of characters to - and from utf-8. + and from UTF-8. If data is sent to a file in a format that cannot be converted to the - utf-8 or if data is read by a function that returns data in a format that + UTF-8 or if data is read by a function that returns data in a format that cannot cope with the character range of the data, an error occurs and the file will be closed. - Check http://www.erlang.org/doc/man/file.html#open-2 for more information about - other options like `:read_ahead` and `:delayed_write`. + * `:delayed_write`, `:raw`, `:ram`, `:read_ahead`, `:sync`, `{:encoding, ...}`, + `{:read_ahead, pos_integer}`, `{:delayed_write, non_neg_integer, non_neg_integer}` - + for more information about these options see `:file.open/2`. This function returns: * `{:ok, io_device}` - the file has been opened in the requested mode. - `io_device` is actually the pid of the process which handles the file. - This process is linked to the process which originally opened the file. - If any process to which the `io_device` is linked terminates, the file - will be closed and the process itself will be terminated. + `io_device` is actually the PID of the process which handles the file. + This process monitors the process that originally opened the file (the + owner process). If the owner process terminates, the file is closed and + the process itself terminates too. If any process to which the `io_device` + is linked terminates, the file will be closed and the process itself will + be terminated. An `io_device` returned from this call can be used as an argument to the `IO` module functions. @@ -890,25 +1373,26 @@ defmodule File do File.close(file) """ - @spec open(Path.t, list) :: {:ok, io_device} | {:error, posix} - def open(path, modes \\ []) + @spec open(Path.t(), [mode | :ram]) :: {:ok, io_device} | {:error, posix} + @spec open(Path.t(), (io_device -> res)) :: {:ok, res} | {:error, posix} when res: var + def open(path, modes_or_function \\ []) def open(path, modes) when is_list(modes) do - F.open(IO.chardata_to_string(path), open_defaults(modes, true)) + :file.open(IO.chardata_to_string(path), normalize_modes(modes, true)) end - def open(path, function) when is_function(function) do + def open(path, function) when is_function(function, 1) do open(path, [], function) end @doc """ - Similar to `open/2` but expects a function as last argument. + Similar to `open/2` but expects a function as its last argument. - The file is opened, given to the function as argument and + The file is opened, given to the function as an argument and automatically closed after the function returns, regardless if there was an error when executing the function. - It returns `{:ok, function_result}` in case of success, + Returns `{:ok, function_result}` in case of success, `{:error, reason}` otherwise. This function expects the file to be closed with success, @@ -918,117 +1402,156 @@ defmodule File do ## Examples - File.open("file.txt", [:read, :write], fn(file) -> + File.open("file.txt", [:read, :write], fn file -> IO.read(file, :line) end) + See `open/2` for the list of available `modes`. """ - @spec open(Path.t, list, (io_device -> res)) :: {:ok, res} | {:error, posix} when res: var - def open(path, modes, function) do + @spec open(Path.t(), [mode | :ram], (io_device -> res)) :: {:ok, res} | {:error, posix} + when res: var + def open(path, modes, function) when is_list(modes) and is_function(function, 1) do case open(path, modes) do - {:ok, device} -> + {:ok, io_device} -> try do - {:ok, function.(device)} + {:ok, function.(io_device)} after - :ok = close(device) + :ok = close(io_device) end - other -> other + + other -> + other end end @doc """ - Same as `open/2` but raises an error if file could not be opened. + Similar to `open/2` but raises a `File.Error` exception if the file + could not be opened. Returns the IO device otherwise. - Returns the `io_device` otherwise. + See `open/2` for the list of available modes. """ - @spec open!(Path.t, list) :: io_device | no_return - def open!(path, modes \\ []) do - path = IO.chardata_to_string(path) - case open(path, modes) do - {:ok, device} -> device + @spec open!(Path.t(), [mode | :ram]) :: io_device + @spec open!(Path.t(), (io_device -> res)) :: res when res: var + def open!(path, modes_or_function \\ []) do + case open(path, modes_or_function) do + {:ok, io_device_or_function_result} -> + io_device_or_function_result + {:error, reason} -> - raise File.Error, reason: reason, action: "open", path: path + raise File.Error, reason: reason, action: "open", path: IO.chardata_to_string(path) end end @doc """ - Same as `open/3` but raises an error if file could not be opened. + Similar to `open/3` but raises a `File.Error` exception if the file + could not be opened. + + If it succeeds opening the file, it returns the `function` result on the IO device. - Returns the function result otherwise. + See `open/2` for the list of available `modes`. """ - @spec open!(Path.t, list, (io_device -> res)) :: res | no_return when res: var + @spec open!(Path.t(), [mode | :ram], (io_device -> res)) :: res when res: var def open!(path, modes, function) do - path = IO.chardata_to_string(path) case open(path, modes, function) do - {:ok, device} -> device + {:ok, function_result} -> + function_result + {:error, reason} -> - raise File.Error, reason: reason, action: "open", path: path + raise File.Error, reason: reason, action: "open", path: IO.chardata_to_string(path) end end @doc """ Gets the current working directory. - In rare circumstances, this function can fail on Unix. It may happen - if read permission does not exist for the parent directories of the + In rare circumstances, this function can fail on Unix-like systems. It may happen + if read permissions do not exist for the parent directories of the current directory. For this reason, returns `{:ok, cwd}` in case of success, `{:error, reason}` otherwise. """ @spec cwd() :: {:ok, binary} | {:error, posix} def cwd() do - case F.get_cwd do - {:ok, base} -> {:ok, IO.chardata_to_string(base)} + case :file.get_cwd() do + {:ok, base} -> {:ok, IO.chardata_to_string(fix_drive_letter(base))} {:error, _} = error -> error end end + defp fix_drive_letter([l, ?:, ?/ | rest] = original) when l in ?A..?Z do + case :os.type() do + {:win32, _} -> [l + ?a - ?A, ?:, ?/ | rest] + _ -> original + end + end + + defp fix_drive_letter(original), do: original + @doc """ - The same as `cwd/0`, but raises an exception if it fails. + The same as `cwd/0`, but raises a `File.Error` exception if it fails. """ - @spec cwd!() :: binary | no_return + @spec cwd!() :: binary def cwd!() do - case F.get_cwd do - {:ok, cwd} -> IO.chardata_to_string(cwd) + case cwd() do + {:ok, cwd} -> + cwd + {:error, reason} -> - raise File.Error, reason: reason, action: "get current working directory" + raise File.Error, reason: reason, action: "get current working directory" end end @doc """ Sets the current working directory. + The current working directory is set for the BEAM globally. This can lead to + race conditions if multiple processes are changing the current working + directory concurrently. To run an external command in a given directory + without changing the global current working directory, use the `:cd` option + of `System.cmd/3` and `Port.open/2`. + Returns `:ok` if successful, `{:error, reason}` otherwise. """ - @spec cd(Path.t) :: :ok | {:error, posix} + @spec cd(Path.t()) :: :ok | {:error, posix} def cd(path) do - F.set_cwd(IO.chardata_to_string(path)) + :file.set_cwd(IO.chardata_to_string(path)) end @doc """ - The same as `cd/1`, but raises an exception if it fails. + The same as `cd/1`, but raises a `File.Error` exception if it fails. """ - @spec cd!(Path.t) :: :ok | no_return + @spec cd!(Path.t()) :: :ok def cd!(path) do - path = IO.chardata_to_string(path) - case F.set_cwd(path) do - :ok -> :ok + case cd(path) do + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "set current working directory to", path: path + raise File.Error, + reason: reason, + action: "set current working directory to", + path: IO.chardata_to_string(path) end end @doc """ Changes the current directory to the given `path`, - executes the given function and then revert back - to the previous path regardless if there is an exception. + executes the given function and then reverts back + to the previous path regardless of whether there is an exception. + + The current working directory is temporarily set for the BEAM globally. This + can lead to race conditions if multiple processes are changing the current + working directory concurrently. To run an external command in a given + directory without changing the global current working directory, use the + `:cd` option of `System.cmd/3` and `Port.open/2`. Raises an error if retrieving or changing the current directory fails. """ - @spec cd!(Path.t, (() -> res)) :: res | no_return when res: var + @spec cd!(Path.t(), (() -> res)) :: res when res: var def cd!(path, function) do - old = cwd! + old = cwd!() cd!(path) + try do function.() after @@ -1037,30 +1560,33 @@ defmodule File do end @doc """ - Returns list of files in the given directory. + Returns the list of files in the given directory. - It returns `{:ok, [files]}` in case of success, + Returns `{:ok, files}` in case of success, `{:error, reason}` otherwise. """ - @spec ls(Path.t) :: {:ok, [binary]} | {:error, posix} + @spec ls(Path.t()) :: {:ok, [binary]} | {:error, posix} def ls(path \\ ".") do - case F.list_dir(IO.chardata_to_string(path)) do + case :file.list_dir(IO.chardata_to_string(path)) do {:ok, file_list} -> {:ok, Enum.map(file_list, &IO.chardata_to_string/1)} {:error, _} = error -> error end end @doc """ - The same as `ls/1` but raises `File.Error` - in case of an error. + The same as `ls/1` but raises a `File.Error` exception in case of an error. """ - @spec ls!(Path.t) :: [binary] | no_return + @spec ls!(Path.t()) :: [binary] def ls!(path \\ ".") do - path = IO.chardata_to_string(path) case ls(path) do - {:ok, value} -> value + {:ok, value} -> + value + {:error, reason} -> - raise File.Error, reason: reason, action: "list directory", path: path + raise File.Error, + reason: reason, + action: "list directory", + path: IO.chardata_to_string(path) end end @@ -1070,26 +1596,28 @@ defmodule File do Note that if the option `:delayed_write` was used when opening the file, `close/1` might return an old write error and not even try to close the file. - See `open/2`. + See `open/2` for more information. """ @spec close(io_device) :: :ok | {:error, posix | :badarg | :terminated} def close(io_device) do - F.close(io_device) + :file.close(io_device) end - @doc """ + @doc ~S""" Returns a `File.Stream` for the given `path` with the given `modes`. The stream implements both `Enumerable` and `Collectable` protocols, which means it can be used both for read and write. - The `line_or_byte` argument configures how the file is read when - streaming, by `:line` (default) or by a given number of bytes. + The `line_or_bytes` argument configures how the file is read when + streaming, by `:line` (default) or by a given number of bytes. When + using the `:line` option, CRLF line breaks (`"\r\n"`) are normalized + to LF (`"\n"`). Operating the stream can fail on open for the same reasons as - `File.open!/2`. Note that the file is automatically opened only and - every time streaming begins. There is no need to pass `:read` and - `:write` modes, as those are automatically set by Elixir. + `File.open!/2`. Note that the file is automatically opened each time streaming + begins. There is no need to pass `:read` and `:write` modes, as those are + automatically set by Elixir. ## Raw files @@ -1097,105 +1625,183 @@ defmodule File do device cannot be shared and as such it is convenient to open the file in raw mode for performance reasons. Therefore, Elixir **will** open streams in `:raw` mode with the `:read_ahead` option unless an encoding - is specified. + is specified. This means any data streamed into the file must be + converted to `t:iodata/0` type. If you pass, for example, `[encoding: :utf8]` + or `[encoding: {:utf16, :little}]` in the modes parameter, + the underlying stream will use `IO.write/2` and the `String.Chars` protocol + to convert the data. See `IO.binwrite/2` and `IO.write/2` . One may also consider passing the `:delayed_write` option if the stream is meant to be written to under a tight loop. + + ## Byte order marks + + If you pass `:trim_bom` in the modes parameter, the stream will + trim UTF-8, UTF-16 and UTF-32 byte order marks when reading from file. + + Note that this function does not try to discover the file encoding basing + on BOM. + + ## Examples + + # Read in 2048 byte chunks rather than lines + File.stream!("./test/test.data", [], 2048) + #=> %File.Stream{line_or_bytes: 2048, modes: [:raw, :read_ahead, :binary], + #=> path: "./test/test.data", raw: true} + + See `Stream.run/1` for an example of streaming into a file. """ + @spec stream!(Path.t(), [stream_mode], :line | pos_integer) :: File.Stream.t() def stream!(path, modes \\ [], line_or_bytes \\ :line) do - modes = open_defaults(modes, true) + modes = normalize_modes(modes, true) File.Stream.__build__(IO.chardata_to_string(path), modes, line_or_bytes) end @doc """ - Changes the unix file `mode` for a given `file`. - Returns `:ok` on success, or `{:error, reason}` - on failure. + Changes the `mode` for a given `file`. + + Returns `:ok` on success, or `{:error, reason}` on failure. + + ## Permissions + + File permissions are specified by adding together the following octal modes: + + * `0o400` - read permission: owner + * `0o200` - write permission: owner + * `0o100` - execute permission: owner + + * `0o040` - read permission: group + * `0o020` - write permission: group + * `0o010` - execute permission: group + + * `0o004` - read permission: other + * `0o002` - write permission: other + * `0o001` - execute permission: other + + For example, setting the mode `0o755` gives it + write, read and execute permission to the owner + and both read and execute permission to group + and others. """ - @spec chmod(Path.t, integer) :: :ok | {:error, posix} + @spec chmod(Path.t(), non_neg_integer) :: :ok | {:error, posix} def chmod(path, mode) do - F.change_mode(IO.chardata_to_string(path), mode) + :file.change_mode(IO.chardata_to_string(path), mode) end @doc """ - Same as `chmod/2`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `chmod/2`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. """ - @spec chmod!(Path.t, integer) :: :ok | no_return + @spec chmod!(Path.t(), non_neg_integer) :: :ok def chmod!(path, mode) do - path = IO.chardata_to_string(path) case chmod(path, mode) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "change mode for", path: path + raise File.Error, + reason: reason, + action: "change mode for", + path: IO.chardata_to_string(path) end end @doc """ - Changes the user group given by the group id `gid` + Changes the group given by the group ID `gid` for a given `file`. Returns `:ok` on success, or `{:error, reason}` on failure. """ - @spec chgrp(Path.t, integer) :: :ok | {:error, posix} + @spec chgrp(Path.t(), non_neg_integer) :: :ok | {:error, posix} def chgrp(path, gid) do - F.change_group(IO.chardata_to_string(path), gid) + :file.change_group(IO.chardata_to_string(path), gid) end @doc """ - Same as `chgrp/2`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `chgrp/2`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. """ - @spec chgrp!(Path.t, integer) :: :ok | no_return + @spec chgrp!(Path.t(), non_neg_integer) :: :ok def chgrp!(path, gid) do - path = IO.chardata_to_string(path) case chgrp(path, gid) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "change group for", path: path + raise File.Error, + reason: reason, + action: "change group for", + path: IO.chardata_to_string(path) end end @doc """ - Changes the owner given by the user id `uid` + Changes the owner given by the user ID `uid` for a given `file`. Returns `:ok` on success, or `{:error, reason}` on failure. """ - @spec chown(Path.t, integer) :: :ok | {:error, posix} + @spec chown(Path.t(), non_neg_integer) :: :ok | {:error, posix} def chown(path, uid) do - F.change_owner(IO.chardata_to_string(path), uid) + :file.change_owner(IO.chardata_to_string(path), uid) end @doc """ - Same as `chown/2`, but raises an exception in case of failure. Otherwise `:ok`. + Same as `chown/2`, but raises a `File.Error` exception in case of failure. + Otherwise `:ok`. """ - @spec chown!(Path.t, integer) :: :ok | no_return + @spec chown!(Path.t(), non_neg_integer) :: :ok def chown!(path, uid) do - path = IO.chardata_to_string(path) case chown(path, uid) do - :ok -> :ok + :ok -> + :ok + {:error, reason} -> - raise File.Error, reason: reason, action: "change owner for", path: path + raise File.Error, + reason: reason, + action: "change owner for", + path: IO.chardata_to_string(path) end end ## Helpers - @read_ahead 64*1024 + @read_ahead_size 64 * 1024 + + defp assert_no_null_byte!(binary, operation) do + case :binary.match(binary, "\0") do + {_, _} -> + raise ArgumentError, + "cannot execute #{operation} for path with null byte, got: #{inspect(binary)}" + + :nomatch -> + binary + end + end - defp open_defaults([:char_list|t], _add_binary) do - open_defaults(t, false) + defp normalize_modes([:utf8 | rest], binary?) do + [encoding: :utf8] ++ normalize_modes(rest, binary?) end - defp open_defaults([:utf8|t], add_binary) do - open_defaults([{:encoding, :utf8}|t], add_binary) + defp normalize_modes([:read_ahead | rest], binary?) do + [read_ahead: @read_ahead_size] ++ normalize_modes(rest, binary?) end - defp open_defaults([:read_ahead|t], add_binary) do - open_defaults([{:read_ahead, @read_ahead}|t], add_binary) + # TODO: Remove :char_list mode on v2.0 + defp normalize_modes([mode | rest], _binary?) when mode in [:charlist, :char_list] do + if mode == :char_list do + IO.warn("the :char_list mode is deprecated, use :charlist") + end + + normalize_modes(rest, false) end - defp open_defaults([h|t], add_binary) do - [h|open_defaults(t, add_binary)] + defp normalize_modes([mode | rest], binary?) do + [mode | normalize_modes(rest, binary?)] end - defp open_defaults([], true), do: [:binary] - defp open_defaults([], false), do: [] + defp normalize_modes([], true), do: [:binary] + defp normalize_modes([], false), do: [] + + defp maybe_to_string(path) when is_list(path), do: IO.chardata_to_string(path) + defp maybe_to_string(path) when is_binary(path), do: path + defp maybe_to_string(path), do: path end diff --git a/lib/elixir/lib/file/stat.ex b/lib/elixir/lib/file/stat.ex index 5a684ff58ca..bc683249171 100644 --- a/lib/elixir/lib/file/stat.ex +++ b/lib/elixir/lib/file/stat.ex @@ -2,17 +2,17 @@ require Record defmodule File.Stat do @moduledoc """ - A struct responsible to hold file information. + A struct that holds file information. In Erlang, this struct is represented by a `:file_info` record. Therefore this module also provides functions for converting - in between the Erlang record and the Elixir struct. + between the Erlang record and the Elixir struct. Its fields are: * `size` - size of file in bytes. - * `type` - `:device | :directory | :regular | :other`; the type of the + * `type` - `:device | :directory | :regular | :other | :symlink`; the type of the file. * `access` - `:read | :write | :read_write | :none`; the current system @@ -23,7 +23,7 @@ defmodule File.Stat do * `mtime` - the last time the file was written. * `ctime` - the interpretation of this time field depends on the operating - system. On Unix, it is the last time the file or the inode was changed. + system. On Unix-like operating systems, it is the last time the file or the inode was changed. In Windows, it is the time of creation. * `mode` - the file permissions. @@ -32,35 +32,53 @@ defmodule File.Stat do systems which have no concept of links. * `major_device` - identifies the file system where the file is located. - In windows, the number indicates a drive as follows: 0 means A:, 1 means + In Windows, the number indicates a drive as follows: 0 means A:, 1 means B:, and so on. - * `minor_device` - only valid for character devices on Unix. In all other + * `minor_device` - only valid for character devices on Unix-like systems. In all other cases, this field is zero. - * `inode` - gives the inode number. On non-Unix file systems, this field + * `inode` - gives the inode number. On non-Unix-like file systems, this field will be zero. - * `uid` - indicates the owner of the file. + * `uid` - indicates the owner of the file. Will be zero for non-Unix-like file + systems. - * `gid` - gives the group that the owner of the file belongs to. Will be - zero for non-Unix file systems. + * `gid` - indicates the group that owns the file. Will be zero for + non-Unix-like file systems. The time type returned in `atime`, `mtime`, and `ctime` is dependent on the time type set in options. `{:time, type}` where type can be `:local`, - `:universal`, or `:posix`. Default is `:local`. + `:universal`, or `:posix`. Default is `:universal`. """ record = Record.extract(:file_info, from_lib: "kernel/include/file.hrl") - keys = :lists.map(&elem(&1, 0), record) - vals = :lists.map(&{&1, [], nil}, keys) - pairs = :lists.zip(keys, vals) + keys = :lists.map(&elem(&1, 0), record) + vals = :lists.map(&{&1, [], nil}, keys) + pairs = :lists.zip(keys, vals) defstruct keys + @type t :: %__MODULE__{ + size: non_neg_integer(), + type: :device | :directory | :regular | :other | :symlink, + access: :read | :write | :read_write | :none, + atime: :calendar.datetime() | integer(), + mtime: :calendar.datetime() | integer(), + ctime: :calendar.datetime() | integer(), + mode: non_neg_integer(), + links: non_neg_integer(), + major_device: non_neg_integer(), + minor_device: non_neg_integer(), + inode: non_neg_integer(), + uid: non_neg_integer(), + gid: non_neg_integer() + } + @doc """ Converts a `File.Stat` struct to a `:file_info` record. """ + @spec to_record(t()) :: :file.file_info() def to_record(%File.Stat{unquote_splicing(pairs)}) do {:file_info, unquote_splicing(vals)} end @@ -68,6 +86,9 @@ defmodule File.Stat do @doc """ Converts a `:file_info` record into a `File.Stat`. """ + @spec from_record(:file.file_info()) :: t() + def from_record(file_info) + def from_record({:file_info, unquote_splicing(vals)}) do %File.Stat{unquote_splicing(pairs)} end diff --git a/lib/elixir/lib/file/stream.ex b/lib/elixir/lib/file/stream.ex index 3c7ffed9190..2cfd4c05fab 100644 --- a/lib/elixir/lib/file/stream.ex +++ b/lib/elixir/lib/file/stream.ex @@ -7,41 +7,42 @@ defmodule File.Stream do * `path` - the file path * `modes` - the file modes * `raw` - a boolean indicating if bin functions should be used - * `line_or_bytes` - if reading should read lines or a given amount of bytes + * `line_or_bytes` - if reading should read lines or a given number of bytes """ defstruct path: nil, modes: [], line_or_bytes: :line, raw: true + @type t :: %__MODULE__{} + @doc false def __build__(path, modes, line_or_bytes) do raw = :lists.keyfind(:encoding, 1, modes) == false modes = - if raw do - if :lists.keyfind(:read_ahead, 1, modes) == {:read_ahead, false} do - [:raw|modes] - else - [:raw, :read_ahead|modes] - end - else - modes + case raw do + true -> + case :lists.keyfind(:read_ahead, 1, modes) do + {:read_ahead, false} -> [:raw | :lists.keydelete(:read_ahead, 1, modes)] + {:read_ahead, _} -> [:raw | modes] + false -> [:raw, :read_ahead | modes] + end + + false -> + modes end %File.Stream{path: path, modes: modes, raw: raw, line_or_bytes: line_or_bytes} end defimpl Collectable do - def empty(stream) do - stream - end - def into(%{path: path, modes: modes, raw: raw} = stream) do - modes = for mode <- modes, not mode in [:read], do: mode + modes = for mode <- modes, mode not in [:read], do: mode - case :file.open(path, [:write|modes]) do + case :file.open(path, [:write | modes]) do {:ok, device} -> {:ok, into(device, stream, raw)} + {:error, reason} -> raise File.Error, reason: reason, action: "stream", path: path end @@ -51,40 +52,74 @@ defmodule File.Stream do fn :ok, {:cont, x} -> case raw do - true -> IO.binwrite(device, x) + true -> IO.binwrite(device, x) false -> IO.write(device, x) end + :ok, :done -> - :file.close(device) + # If delayed_write option is used and the last write failed will + # MatchError here as {:error, _} is returned. + :ok = :file.close(device) stream + :ok, :halt -> - :file.close(device) + # If delayed_write option is used and the last write failed will + # MatchError here as {:error, _} is returned. + :ok = :file.close(device) end end end defimpl Enumerable do + @read_ahead_size 64 * 1024 + def reduce(%{path: path, modes: modes, line_or_bytes: line_or_bytes, raw: raw}, acc, fun) do - modes = for mode <- modes, not mode in [:write, :append], do: mode - - start_fun = - fn -> - case :file.open(path, modes) do - {:ok, device} -> device - {:error, reason} -> - raise File.Error, reason: reason, action: "stream", path: path - end + start_fun = fn -> + case :file.open(path, read_modes(modes)) do + {:ok, device} -> + if :trim_bom in modes, do: trim_bom(device, raw) |> elem(0), else: device + + {:error, reason} -> + raise File.Error, reason: reason, action: "stream", path: path end + end next_fun = case raw do - true -> &IO.each_binstream(&1, line_or_bytes) + true -> &IO.each_binstream(&1, line_or_bytes) false -> &IO.each_stream(&1, line_or_bytes) end Stream.resource(start_fun, next_fun, &:file.close/1).(acc, fun) end + def count(%{path: path, modes: modes, line_or_bytes: :line} = stream) do + pattern = :binary.compile_pattern("\n") + counter = &count_lines(&1, path, pattern, read_function(stream), 0) + + case File.open(path, read_modes(modes), counter) do + {:ok, count} -> + {:ok, count} + + {:error, reason} -> + raise File.Error, reason: reason, action: "stream", path: path + end + end + + def count(%{path: path, line_or_bytes: bytes, raw: true, modes: modes}) do + case File.stat(path) do + {:ok, %{size: 0}} -> + {:error, __MODULE__} + + {:ok, %{size: size}} -> + remainder = if rem(size, bytes) == 0, do: 0, else: 1 + {:ok, div(size, bytes) + remainder - count_raw_bom(path, modes)} + + {:error, reason} -> + raise File.Error, reason: reason, action: "stream", path: path + end + end + def count(_stream) do {:error, __MODULE__} end @@ -92,5 +127,64 @@ defmodule File.Stream do def member?(_stream, _term) do {:error, __MODULE__} end + + def slice(_stream) do + {:error, __MODULE__} + end + + defp count_raw_bom(path, modes) do + if :trim_bom in modes do + File.open!(path, read_modes(modes), &(&1 |> trim_bom(true) |> elem(1))) + else + 0 + end + end + + defp trim_bom(device, true) do + bom_length = device |> IO.binread(4) |> bom_length() + {:ok, new_pos} = :file.position(device, bom_length) + {device, new_pos} + end + + defp trim_bom(device, false) do + # Or we read the bom in the correct amount or it isn't there + case bom_length(IO.read(device, 1)) do + 0 -> + {:ok, _} = :file.position(device, 0) + {device, 0} + + _ -> + {device, 1} + end + end + + defp bom_length(<<239, 187, 191, _rest::binary>>), do: 3 + defp bom_length(<<254, 255, _rest::binary>>), do: 2 + defp bom_length(<<255, 254, _rest::binary>>), do: 2 + defp bom_length(<<0, 0, 254, 255, _rest::binary>>), do: 4 + defp bom_length(<<254, 255, 0, 0, _rest::binary>>), do: 4 + defp bom_length(_binary), do: 0 + + defp read_modes(modes) do + for mode <- modes, mode not in [:write, :append, :trim_bom], do: mode + end + + defp count_lines(device, path, pattern, read, count) do + case read.(device) do + data when is_binary(data) -> + count_lines(device, path, pattern, read, count + count_lines(data, pattern)) + + :eof -> + count + + {:error, reason} -> + raise File.Error, reason: reason, action: "stream", path: path + end + end + + defp count_lines(data, pattern), do: length(:binary.matches(data, pattern)) + + defp read_function(%{raw: true}), do: &IO.binread(&1, @read_ahead_size) + defp read_function(%{raw: false}), do: &IO.read(&1, @read_ahead_size) end end diff --git a/lib/elixir/lib/float.ex b/lib/elixir/lib/float.ex index 720d043958f..8472ba01753 100644 --- a/lib/elixir/lib/float.ex +++ b/lib/elixir/lib/float.ex @@ -1,24 +1,137 @@ +import Kernel, except: [round: 1] + defmodule Float do @moduledoc """ - Functions for working with floating point numbers. + Functions for working with floating-point numbers. + + ## Kernel functions + + There are functions related to floating-point numbers on the `Kernel` module + too. Here is a list of them: + + * `Kernel.round/1`: rounds a number to the nearest integer. + * `Kernel.trunc/1`: returns the integer part of a number. + + ## Known issues + + There are some very well known problems with floating-point numbers + and arithmetic due to the fact most decimal fractions cannot be + represented by a floating-point binary and most operations are not exact, + but operate on approximations. Those issues are not specific + to Elixir, they are a property of floating point representation itself. + + For example, the numbers 0.1 and 0.01 are two of them, what means the result + of squaring 0.1 does not give 0.01 neither the closest representable. Here is + what happens in this case: + + * The closest representable number to 0.1 is 0.1000000014 + * The closest representable number to 0.01 is 0.0099999997 + * Doing 0.1 * 0.1 should return 0.01, but because 0.1 is actually 0.1000000014, + the result is 0.010000000000000002, and because this is not the closest + representable number to 0.01, you'll get the wrong result for this operation + + There are also other known problems like flooring or rounding numbers. See + `round/2` and `floor/2` for more details about them. + + To learn more about floating-point arithmetic visit: + + * [0.30000000000000004.com](http://0.30000000000000004.com/) + * [What Every Programmer Should Know About Floating-Point Arithmetic](https://floating-point-gui.de/) + + """ + + import Bitwise + + @power_of_2_to_52 4_503_599_627_370_496 + @precision_range 0..15 + @type precision_range :: 0..15 + + @min_finite then(<<0xFFEFFFFFFFFFFFFF::64>>, fn <> -> num end) + @max_finite then(<<0x7FEFFFFFFFFFFFFF::64>>, fn <> -> num end) + + @doc """ + Returns the maximum finite value for a float. + + ## Examples + + iex> Float.max_finite() + 1.7976931348623157e308 + + """ + def max_finite, do: @max_finite + + @doc """ + Returns the minimum finite value for a float. + + ## Examples + + iex> Float.min_finite() + -1.7976931348623157e308 + """ + def min_finite, do: @min_finite + + @doc """ + Computes `base` raised to power of `exponent`. + + `base` must be a float and `exponent` can be any number. + However, if a negative base and a fractional exponent + are given, it raises `ArithmeticError`. + + It always returns a float. See `Integer.pow/2` for + exponentiation that returns integers. + + ## Examples + + iex> Float.pow(2.0, 0) + 1.0 + iex> Float.pow(2.0, 1) + 2.0 + iex> Float.pow(2.0, 10) + 1024.0 + iex> Float.pow(2.0, -1) + 0.5 + iex> Float.pow(2.0, -3) + 0.125 + + iex> Float.pow(3.0, 1.5) + 5.196152422706632 + + iex> Float.pow(-2.0, 3) + -8.0 + iex> Float.pow(-2.0, 4) + 16.0 + + iex> Float.pow(-1.0, 0.5) + ** (ArithmeticError) bad argument in arithmetic expression + + """ + @doc since: "1.12.0" + @spec pow(float, number) :: float + def pow(base, exponent) when is_float(base) and is_number(exponent), + do: :math.pow(base, exponent) @doc """ Parses a binary into a float. - If successful, returns a tuple of the form `{float, remainder_of_binary}`. - Otherwise `:error`. + If successful, returns a tuple in the form of `{float, remainder_of_binary}`; + when the binary cannot be coerced into a valid float, the atom `:error` is + returned. + + If the size of float exceeds the maximum size of `1.7976931348623157e+308`, + the `ArgumentError` exception is raised. + + If you want to convert a string-formatted float directly to a float, + `String.to_float/1` can be used instead. ## Examples iex> Float.parse("34") - {34.0,""} - + {34.0, ""} iex> Float.parse("34.25") - {34.25,""} - + {34.25, ""} iex> Float.parse("56.5xyz") - {56.5,"xyz"} + {56.5, "xyz"} iex> Float.parse("pi") :error @@ -26,229 +139,476 @@ defmodule Float do """ @spec parse(binary) :: {float, binary} | :error def parse("-" <> binary) do - case parse_unsign(binary) do + case parse_unsigned(binary) do :error -> :error {number, remainder} -> {-number, remainder} end end - def parse(binary) do - parse_unsign(binary) + def parse("+" <> binary) do + parse_unsigned(binary) end - defp parse_unsign("-" <> _), do: :error - defp parse_unsign(binary) when is_binary(binary) do - case Integer.parse binary do - :error -> :error - {integer_part, after_integer} -> parse_unsign after_integer, integer_part - end + def parse(binary) do + parse_unsigned(binary) end - # Dot followed by digit is required afterwards or we are done - defp parse_unsign(<< ?., char, rest :: binary >>, int) when char in ?0..?9 do - parse_unsign(rest, char - ?0, 1, int) - end + defp parse_unsigned(<>) when digit in ?0..?9, + do: parse_unsigned(rest, false, false, <>) - defp parse_unsign(rest, int) do - {:erlang.float(int), rest} - end + defp parse_unsigned(binary) when is_binary(binary), do: :error - # Handle decimal points - defp parse_unsign(<< char, rest :: binary >>, float, decimal, int) when char in ?0..?9 do - parse_unsign rest, 10 * float + (char - ?0), decimal + 1, int - end + defp parse_unsigned(<>, dot?, e?, acc) when digit in ?0..?9, + do: parse_unsigned(rest, dot?, e?, <>) - defp parse_unsign(<< ?e, after_e :: binary >>, float, decimal, int) do - case Integer.parse after_e do - :error -> - # Note we rebuild the binary here instead of breaking it apart at - # the function clause because the current approach copies a binary - # just on this branch. If we broke it apart in the function clause, - # the copy would happen when calling Integer.parse/1. - {floatify(int, float, decimal), << ?e, after_e :: binary >>} - {exponential, after_exponential} -> - {floatify(int, float, decimal, exponential), after_exponential} - end - end + defp parse_unsigned(<>, false, false, acc) when digit in ?0..?9, + do: parse_unsigned(rest, true, false, <>) - defp parse_unsign(bitstring, float, decimal, int) do - {floatify(int, float, decimal), bitstring} - end + defp parse_unsigned(<>, dot?, false, acc) + when exp_marker in 'eE' and digit in ?0..?9, + do: parse_unsigned(rest, true, true, <>) - defp floatify(int, float, decimal, exponential \\ 0) do - multiplier = if int < 0, do: -1.0, else: 1.0 + defp parse_unsigned(<>, dot?, false, acc) + when exp_marker in 'eE' and sign in '-+' and digit in ?0..?9, + do: parse_unsigned(rest, true, true, <>) - # Try to ensure the minimum amount of rounding errors - result = multiplier * (abs(int) * :math.pow(10, decimal) + float) * :math.pow(10, exponential - decimal) + defp parse_unsigned(rest, dot?, _e?, acc), + do: {:erlang.binary_to_float(add_dot(acc, dot?)), rest} - # Try avoiding stuff like this: - # iex(1)> 0.0001 * 75 - # 0.007500000000000001 - # Due to IEEE 754 floating point standard - # http://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html - - final_decimal_places = decimal - exponential - if final_decimal_places > 0 do - decimal_power_round = :math.pow(10, final_decimal_places) - trunc(result * decimal_power_round) / decimal_power_round - else - result - end - end + defp add_dot(acc, true), do: acc + defp add_dot(acc, false), do: acc <> ".0" @doc """ - Rounds a float to the largest integer less than or equal to `num`. + Rounds a float to the largest number less than or equal to `num`. - ## Examples + `floor/2` also accepts a precision to round a floating-point value down + to an arbitrary number of fractional digits (between 0 and 15). + The operation is performed on the binary floating point, without a + conversion to decimal. - iex> Float.floor(34) - 34 + This function always returns a float. `Kernel.trunc/1` may be used instead to + truncate the result to an integer afterwards. - iex> Float.floor(34.25) - 34 + ## Known issues + + The behaviour of `floor/2` for floats can be surprising. For example: + + iex> Float.floor(12.52, 2) + 12.51 + + One may have expected it to floor to 12.52. This is not a bug. + Most decimal fractions cannot be represented as a binary floating point + and therefore the number above is internally represented as 12.51999999, + which explains the behaviour above. + + ## Examples + iex> Float.floor(34.25) + 34.0 iex> Float.floor(-56.5) - -57 + -57.0 + iex> Float.floor(34.259, 2) + 34.25 """ - @spec floor(float | integer) :: integer - def floor(num) when is_integer(num), do: num - def floor(num) when is_float(num) do - truncated = :erlang.trunc(num) - case :erlang.abs(num - truncated) do - x when x > 0 and num < 0 -> truncated - 1 - _ -> truncated - end + @spec floor(float, precision_range) :: float + def floor(number, precision \\ 0) + + def floor(number, 0) when is_float(number) do + :math.floor(number) + end + + def floor(number, precision) when is_float(number) and precision in @precision_range do + round(number, precision, :floor) + end + + def floor(number, precision) when is_float(number) do + raise ArgumentError, invalid_precision_message(precision) end @doc """ - Rounds a float to the largest integer greater than or equal to `num`. + Rounds a float to the smallest integer greater than or equal to `num`. - ## Examples + `ceil/2` also accepts a precision to round a floating-point value down + to an arbitrary number of fractional digits (between 0 and 15). - iex> Float.ceil(34) - 34 + The operation is performed on the binary floating point, without a + conversion to decimal. - iex> Float.ceil(34.25) - 35 + The behaviour of `ceil/2` for floats can be surprising. For example: + iex> Float.ceil(-12.52, 2) + -12.51 + + One may have expected it to ceil to -12.52. This is not a bug. + Most decimal fractions cannot be represented as a binary floating point + and therefore the number above is internally represented as -12.51999999, + which explains the behaviour above. + + This function always returns floats. `Kernel.trunc/1` may be used instead to + truncate the result to an integer afterwards. + + ## Examples + + iex> Float.ceil(34.25) + 35.0 iex> Float.ceil(-56.5) - -56 + -56.0 + iex> Float.ceil(34.251, 2) + 34.26 """ - @spec ceil(float | integer) :: integer - def ceil(num) when is_integer(num), do: num - def ceil(num) when is_float(num) do - truncated = :erlang.trunc(num) - case :erlang.abs(num - truncated) do - x when x > 0 and num > 0 -> truncated + 1 - _ -> truncated - end + @spec ceil(float, precision_range) :: float + def ceil(number, precision \\ 0) + + def ceil(number, 0) when is_float(number) do + :math.ceil(number) + end + + def ceil(number, precision) when is_float(number) and precision in @precision_range do + round(number, precision, :ceil) + end + + def ceil(number, precision) when is_float(number) do + raise ArgumentError, invalid_precision_message(precision) end @doc """ - Rounds a floating point value to an arbitrary number of fractional digits - (between 0 and 15). + Rounds a floating-point value to an arbitrary number of fractional + digits (between 0 and 15). + + The rounding direction always ties to half up. The operation is + performed on the binary floating point, without a conversion to decimal. + + This function only accepts floats and always returns a float. Use + `Kernel.round/1` if you want a function that accepts both floats + and integers and always returns an integer. + + ## Known issues + + The behaviour of `round/2` for floats can be surprising. For example: + + iex> Float.round(5.5675, 3) + 5.567 + + One may have expected it to round to the half up 5.568. This is not a bug. + Most decimal fractions cannot be represented as a binary floating point + and therefore the number above is internally represented as 5.567499999, + which explains the behaviour above. If you want exact rounding for decimals, + you must use a decimal library. The behaviour above is also in accordance + to reference implementations, such as "Correctly Rounded Binary-Decimal and + Decimal-Binary Conversions" by David M. Gay. ## Examples + iex> Float.round(12.5) + 13.0 iex> Float.round(5.5674, 3) 5.567 - iex> Float.round(5.5675, 3) - 5.568 - + 5.567 iex> Float.round(-5.5674, 3) -5.567 - - iex> Float.round(-5.5675, 3) - -5.568 + iex> Float.round(-5.5675) + -6.0 + iex> Float.round(12.341444444444441, 15) + 12.341444444444441 """ - @spec round(float, integer) :: float - def round(number, precision) when is_float(number) and is_integer(precision) and precision in 0..15 do - Kernel.round(number * :math.pow(10, precision)) / :math.pow(10, precision) + @spec round(float, precision_range) :: float + # This implementation is slow since it relies on big integers. + # Faster implementations are available on more recent papers + # and could be implemented in the future. + def round(float, precision \\ 0) + + def round(float, 0) when is_float(float) do + float |> :erlang.round() |> :erlang.float() end - @doc """ - Returns a char list which corresponds to the text representation of the given float. + def round(float, precision) when is_float(float) and precision in @precision_range do + round(float, precision, :half_up) + end - Inlined by the compiler. + def round(float, precision) when is_float(float) do + raise ArgumentError, invalid_precision_message(precision) + end - ## Examples + defp round(0.0 = num, _precision, _rounding), do: num + + defp round(float, precision, rounding) do + <> = <> + {num, count} = decompose(significant, 1) + count = count - exp + 1023 + + cond do + # Precision beyond 15 digits + count >= 104 -> + case rounding do + :ceil when sign === 0 -> 1 / power_of_10(precision) + :floor when sign === 1 -> -1 / power_of_10(precision) + _ -> 0.0 + end + + # We are asking more precision than we have + count <= precision -> + float + + true -> + # Difference in precision between float and asked precision + # We subtract 1 because we need to calculate the remainder too + diff = count - precision - 1 + + # Get up to latest so we calculate the remainder + power_of_10 = power_of_10(diff) + + # Convert the numerand to decimal base + num = num * power_of_5(count) + + # Move to the given precision - 1 + num = div(num, power_of_10) + div = div(num, 10) + num = rounding(rounding, sign, num, div) + + # Convert back to float without loss + # https://www.exploringbinary.com/correct-decimal-to-floating-point-using-big-integers/ + den = power_of_10(precision) + boundary = den <<< 52 + + cond do + num == 0 -> + 0.0 + + num >= boundary -> + {den, exp} = scale_down(num, boundary, 52) + decimal_to_float(sign, num, den, exp) + + true -> + {num, exp} = scale_up(num, boundary, 52) + decimal_to_float(sign, num, den, exp) + end + end + end - iex> Float.to_char_list(7.0) - '7.00000000000000000000e+00' + defp decompose(significant, initial) do + decompose(significant, 1, 0, initial) + end - """ - @spec to_char_list(float) :: char_list - def to_char_list(number) do - :erlang.float_to_list(number) + defp decompose(<<1::1, bits::bitstring>>, count, last_count, acc) do + decompose(bits, count + 1, count, (acc <<< (count - last_count)) + 1) end - @doc """ - Returns a list which corresponds to the text representation - of `float`. + defp decompose(<<0::1, bits::bitstring>>, count, last_count, acc) do + decompose(bits, count + 1, last_count, acc) + end + + defp decompose(<<>>, _count, last_count, acc) do + {acc, last_count} + end + + defp scale_up(num, boundary, exp) when num >= boundary, do: {num, exp} + defp scale_up(num, boundary, exp), do: scale_up(num <<< 1, boundary, exp - 1) - ## Options + defp scale_down(num, den, exp) do + new_den = den <<< 1 - * `:decimals` — number of decimal points to show - * `:scientific` — number of decimal points to show, in scientific format - * `:compact` — when true, use the most compact representation (ignored - with the `scientific` option) + if num < new_den do + {den >>> 52, exp} + else + scale_down(num, new_den, exp + 1) + end + end + + defp decimal_to_float(sign, num, den, exp) do + quo = div(num, den) + rem = num - quo * den + + tmp = + case den >>> 1 do + den when rem > den -> quo + 1 + den when rem < den -> quo + _ when (quo &&& 1) === 1 -> quo + 1 + _ -> quo + end + + tmp = tmp - @power_of_2_to_52 + <> = <> + tmp + end + + defp rounding(:floor, 1, _num, div), do: div + 1 + defp rounding(:ceil, 0, _num, div), do: div + 1 + + defp rounding(:half_up, _sign, num, div) do + case rem(num, 10) do + rem when rem < 5 -> div + rem when rem >= 5 -> div + 1 + end + end + + defp rounding(_, _, _, div), do: div + + Enum.reduce(0..104, 1, fn x, acc -> + defp power_of_10(unquote(x)), do: unquote(acc) + acc * 10 + end) + + Enum.reduce(0..104, 1, fn x, acc -> + defp power_of_5(unquote(x)), do: unquote(acc) + acc * 5 + end) + + @doc """ + Returns a pair of integers whose ratio is exactly equal + to the original float and with a positive denominator. ## Examples - iex> Float.to_char_list 7.1, [decimals: 2, compact: true] - '7.1' + iex> Float.ratio(0.0) + {0, 1} + iex> Float.ratio(3.14) + {7070651414971679, 2251799813685248} + iex> Float.ratio(-3.14) + {-7070651414971679, 2251799813685248} + iex> Float.ratio(1.5) + {3, 2} + iex> Float.ratio(-1.5) + {-3, 2} + iex> Float.ratio(16.0) + {16, 1} + iex> Float.ratio(-16.0) + {-16, 1} """ - @spec to_char_list(float, list) :: char_list - def to_char_list(float, options) do - :erlang.float_to_list(float, expand_compact(options)) + @doc since: "1.4.0" + @spec ratio(float) :: {integer, pos_integer} + def ratio(0.0), do: {0, 1} + + def ratio(float) when is_float(float) do + <> = <> + + {num, den_exp} = + if exp != 0 do + # Floats are expressed like this: + # (2**52 + mantissa) * 2**(-52 + exp - 1023) + # + # We compute the root factors of the mantissa so we have this: + # (2**52 + mantissa * 2**count) * 2**(-52 + exp - 1023) + {mantissa, count} = root_factors(mantissa, 0) + + # Now we can move the count around so we have this: + # (2**(52-count) + mantissa) * 2**(count + -52 + exp - 1023) + if mantissa == 0 do + {1, exp - 1023} + else + num = (1 <<< (52 - count)) + mantissa + den_exp = count - 52 + exp - 1023 + {num, den_exp} + end + else + # Subnormals are expressed like this: + # (mantissa) * 2**(-52 + 1 - 1023) + # + # So we compute it to this: + # (mantissa * 2**(count)) * 2**(-52 + 1 - 1023) + # + # Which becomes: + # mantissa * 2**(count-1074) + root_factors(mantissa, -1074) + end + + if den_exp > 0 do + {sign(sign, num <<< den_exp), 1} + else + {sign(sign, num), 1 <<< -den_exp} + end end + defp root_factors(mantissa, count) when mantissa != 0 and (mantissa &&& 1) == 0, + do: root_factors(mantissa >>> 1, count + 1) + + defp root_factors(mantissa, count), + do: {mantissa, count} + + @compile {:inline, sign: 2} + defp sign(0, num), do: num + defp sign(1, num), do: -num + @doc """ - Returns a binary which corresponds to the text representation - of `some_float`. + Returns a charlist which corresponds to the shortest text representation + of the given float. + + The underlying algorithm changes depending on the Erlang/OTP version: + + * For OTP >= 24, it uses the algorithm presented in "Ryū: fast + float-to-string conversion" in Proceedings of the SIGPLAN '2018 + Conference on Programming Language Design and Implementation. + + * For OTP < 24, it uses the algorithm presented in "Printing Floating-Point + Numbers Quickly and Accurately" in Proceedings of the SIGPLAN '1996 + Conference on Programming Language Design and Implementation. - Inlined by the compiler. + For a configurable representation, use `:erlang.float_to_list/2`. ## Examples - iex> Float.to_string(7.0) - "7.00000000000000000000e+00" + iex> Float.to_charlist(7.0) + '7.0' """ - @spec to_string(float) :: String.t - def to_string(some_float) do - :erlang.float_to_binary(some_float) + @spec to_charlist(float) :: charlist + def to_charlist(float) when is_float(float) do + :io_lib_format.fwrite_g(float) end @doc """ - Returns a binary which corresponds to the text representation - of `float`. + Returns a binary which corresponds to the shortest text representation + of the given float. + + The underlying algorithm changes depending on the Erlang/OTP version: - ## Options + * For OTP >= 24, it uses the algorithm presented in "Ryū: fast + float-to-string conversion" in Proceedings of the SIGPLAN '2018 + Conference on Programming Language Design and Implementation. - * `:decimals` — number of decimal points to show - * `:scientific` — number of decimal points to show, in scientific format - * `:compact` — when true, use the most compact representation (ignored - with the `scientific` option) + * For OTP < 24, it uses the algorithm presented in "Printing Floating-Point + Numbers Quickly and Accurately" in Proceedings of the SIGPLAN '1996 + Conference on Programming Language Design and Implementation. + + For a configurable representation, use `:erlang.float_to_binary/2`. ## Examples - iex> Float.to_string 7.1, [decimals: 2, compact: true] - "7.1" + iex> Float.to_string(7.0) + "7.0" """ - @spec to_string(float, list) :: String.t + @spec to_string(float) :: String.t() + def to_string(float) when is_float(float) do + IO.iodata_to_binary(:io_lib_format.fwrite_g(float)) + end + + @doc false + @deprecated "Use Float.to_charlist/1 instead" + def to_char_list(float), do: Float.to_charlist(float) + + @doc false + @deprecated "Use :erlang.float_to_list/2 instead" + def to_char_list(float, options) do + :erlang.float_to_list(float, expand_compact(options)) + end + + @doc false + @deprecated "Use :erlang.float_to_binary/2 instead" def to_string(float, options) do :erlang.float_to_binary(float, expand_compact(options)) end - defp expand_compact([{:compact, false}|t]), do: expand_compact(t) - defp expand_compact([{:compact, true}|t]), do: [:compact|expand_compact(t)] - defp expand_compact([h|t]), do: [h|expand_compact(t)] - defp expand_compact([]), do: [] + defp invalid_precision_message(precision) do + "precision #{precision} is out of valid range of #{inspect(@precision_range)}" + end + + defp expand_compact([{:compact, false} | t]), do: expand_compact(t) + defp expand_compact([{:compact, true} | t]), do: [:compact | expand_compact(t)] + defp expand_compact([h | t]), do: [h | expand_compact(t)] + defp expand_compact([]), do: [] end diff --git a/lib/elixir/lib/function.ex b/lib/elixir/lib/function.ex new file mode 100644 index 00000000000..72589dd5787 --- /dev/null +++ b/lib/elixir/lib/function.ex @@ -0,0 +1,208 @@ +defmodule Function do + @moduledoc """ + A set of functions for working with functions. + + Anonymous functions are typically created by using `fn`: + + iex> add = fn a, b -> a + b end + iex> add.(1, 2) + 3 + + Anonymous functions can also have multiple clauses. All clauses + should expect the same number of arguments: + + iex> negate = fn + ...> true -> false + ...> false -> true + ...> end + iex> negate.(false) + true + + ## The capture operator + + It is also possible to capture public module functions and pass them + around as if they were anonymous functions by using the capture + operator `&/1`: + + iex> add = &Kernel.+/2 + iex> add.(1, 2) + 3 + + iex> length = &String.length/1 + iex> length.("hello") + 5 + + To capture a definition within the current module, you can skip the + module prefix, such as `&my_fun/2`. In those cases, the captured + function can be public (`def`) or private (`defp`). + + The capture operator can also be used to create anonymous functions + that expect at least one argument: + + iex> add = &(&1 + &2) + iex> add.(1, 2) + 3 + + In such cases, using the capture operator is no different than using `fn`. + + ## Internal and external functions + + We say that functions that point to definitions residing in modules, such + as `&String.length/1`, are **external** functions. All other functions are + **local** and they are always bound to the file or module that defined them. + + Besides the functions in this module to work with functions, `Kernel` also + has an `apply/2` function that invokes a function with a dynamic number of + arguments, as well as `is_function/1` and `is_function/2`, to check + respectively if a given value is a function or a function of a given arity. + """ + + @type information :: + :arity + | :env + | :index + | :module + | :name + | :new_index + | :new_uniq + | :pid + | :type + | :uniq + + @doc """ + Captures the given function. + + Inlined by the compiler. + + ## Examples + + iex> Function.capture(String, :length, 1) + &String.length/1 + + """ + @doc since: "1.7.0" + @spec capture(module, atom, arity) :: fun + def capture(module, function_name, arity) do + :erlang.make_fun(module, function_name, arity) + end + + @doc """ + Returns a keyword list with information about a function. + + The returned keys (with the corresponding possible values) for + all types of functions (local and external) are the following: + + * `:type` - `:local` (for anonymous functions) or `:external` (for + named functions). + + * `:module` - an atom which is the module where the function is defined when + anonymous or the module which the function refers to when it's a named function. + + * `:arity` - (integer) the number of arguments the function is to be called with. + + * `:name` - (atom) the name of the function. + + * `:env` - a list of the environment or free variables. For named + functions, the returned list is always empty. + + When `fun` is an anonymous function (that is, the type is `:local`), the following + additional keys are returned: + + * `:pid` - PID of the process that originally created the function. + + * `:index` - (integer) an index into the module function table. + + * `:new_index` - (integer) an index into the module function table. + + * `:new_uniq` - (binary) a unique value for this function. It's + calculated from the compiled code for the entire module. + + * `:uniq` - (integer) a unique value for this function. This integer is + calculated from the compiled code for the entire module. + + **Note**: this function must be used only for debugging purposes. + + Inlined by the compiler. + + ## Examples + + iex> fun = fn x -> x end + iex> info = Function.info(fun) + iex> Keyword.get(info, :arity) + 1 + iex> Keyword.get(info, :type) + :local + + iex> fun = &String.length/1 + iex> info = Function.info(fun) + iex> Keyword.get(info, :type) + :external + iex> Keyword.get(info, :name) + :length + + """ + @doc since: "1.7.0" + @spec info(fun) :: [{information, term}] + def info(fun), do: :erlang.fun_info(fun) + + @doc """ + Returns a specific information about the function. + + The returned information is a two-element tuple in the shape of + `{info, value}`. + + For any function, the information asked for can be any of the atoms + `:module`, `:name`, `:arity`, `:env`, or `:type`. + + For anonymous functions, there is also information about any of the + atoms `:index`, `:new_index`, `:new_uniq`, `:uniq`, and `:pid`. + For a named function, the value of any of these items is always the + atom `:undefined`. + + For more information on each of the possible returned values, see + `info/1`. + + Inlined by the compiler. + + ## Examples + + iex> f = fn x -> x end + iex> Function.info(f, :arity) + {:arity, 1} + iex> Function.info(f, :type) + {:type, :local} + + iex> fun = &String.length/1 + iex> Function.info(fun, :name) + {:name, :length} + iex> Function.info(fun, :pid) + {:pid, :undefined} + + """ + @doc since: "1.7.0" + @spec info(fun, item) :: {item, term} when item: information + def info(fun, item), do: :erlang.fun_info(fun, item) + + @doc """ + Returns its input `value`. This function can be passed as an anonymous function + to transformation functions. + + ## Examples + + iex> Function.identity("Hello world!") + "Hello world!" + + iex> 'abcdaabccc' |> Enum.sort() |> Enum.chunk_by(&Function.identity/1) + ['aaa', 'bb', 'cccc', 'd'] + + iex> Enum.group_by('abracadabra', &Function.identity/1) + %{97 => 'aaaaa', 98 => 'bb', 99 => 'c', 100 => 'd', 114 => 'rr'} + + iex> Enum.map([1, 2, 3, 4], &Function.identity/1) + [1, 2, 3, 4] + + """ + @doc since: "1.10.0" + @spec identity(value) :: value when value: var + def identity(value), do: value +end diff --git a/lib/elixir/lib/gen_event.ex b/lib/elixir/lib/gen_event.ex index 99ab1c48388..e05577bfb2c 100644 --- a/lib/elixir/lib/gen_event.ex +++ b/lib/elixir/lib/gen_event.ex @@ -1,185 +1,98 @@ defmodule GenEvent do - @moduledoc """ - A behaviour module for implementing event handling functionality. - - The event handling model consists of a generic event manager - process with an arbitrary number of event handlers which are - added and deleted dynamically. - - An event manager implemented using this module will have a standard - set of interface functions and include functionality for tracing and - error reporting. It will also fit into an supervision tree. - - ## Example - - There are many use cases for event handlers. For example, a logging - system can be built using event handlers where which log message is - an event and different event handlers can be plugged to handle the - log messages. One handler may print error messages on the terminal, - another can write it to a file, while a third one can keep the - messages in memory (like a buffer) until they are read. - - As an example, let's have a GenEvent that accumulates messages until - they are collected by an explicit call. - - defmodule LoggerHandler do - use GenEvent - - # Callbacks - - def handle_event({:log, x}, messages) do - {:ok, [x|messages]} - end - - def handle_call(:messages, messages) do - {:ok, Enum.reverse(messages), []} - end - end - - {:ok, pid} = GenEvent.start_link() - - GenEvent.add_handler(pid, LoggerHandler, []) - #=> :ok - - GenEvent.notify(pid, {:log, 1}) - #=> :ok - - GenEvent.notify(pid, {:log, 2}) - #=> :ok - - GenEvent.call(pid, LoggerHandler, :messages) - #=> [1, 2] - - GenEvent.call(pid, LoggerHandler, :messages) - #=> [] - - We start a new event manager by calling `GenEvent.start_link/0`. - Notifications can be sent to the event manager which will then - invoke `handle_event/0` for each registered handler. - - We can add new handlers with `add_handler/4`. Calls can also - be made to specific handlers by using `call/3`. - - ## Callbacks - - There are 6 callbacks required to be implemented in a `GenEvent`. By - adding `use GenEvent` to your module, Elixir will automatically define - all 6 callbacks for you, leaving it up to you to implement the ones - you want to customize. The callbacks are: - - * `init(args)` - invoked when the event handler is added. - - It must return: - - - `{:ok, state}` - - `{:ok, state, :hibernate}` - - `{:error, reason}` - - * `handle_event(msg, state)` - invoked whenever an event is sent via - `notify/2` or `sync_notify/2`. - - It must return: - - - `{:ok, new_state}` - - `{:ok, new_state, :hibernate}` - - `{:swap_handler, args1, new_state, handler2, args2}` - - `:remove_handler` - - * `handle_call(msg, state)` - invoked when a `call/3` is done to a specific - handler. - - It must return: - - - `{:ok, reply, new_state}` - - `{:ok, reply, new_state, :hibernate}` - - `{:swap_handler, reply, args1, new_state, handler2, args2}` - - `{:remove_handler, reply}` - - * `handle_info(msg, state)` - invoked to handle all other messages which - are received by the process. Must return the same values as - `handle_event/2`. + # Functions from this module are deprecated in elixir_dispatch. - It must return: + @moduledoc """ + An event manager with event handlers behaviour. - - `{:noreply, state}` - - `{:noreply, state, timeout}` - - `{:stop, reason, state}` + If you are interested in implementing an event manager, please read the + "Alternatives" section below. If you have to implement an event handler to + integrate with an existing system, such as Elixir's Logger, please use + [`:gen_event`](`:gen_event`) instead. - * `terminate(reason, state)` - called when the event handler is removed or - the event manager is terminating. It can return any term. + ## Alternatives - * `code_change(old_vsn, state, extra)` - called when the application - code is being upgraded live (hot code swapping). + There are a few suitable alternatives to replace GenEvent. Each of them can be + the most beneficial based on the use case. - It must return: + ### Supervisor and GenServers - - `{:ok, new_state}` + One alternative to GenEvent is a very minimal solution consisting of using a + supervisor and multiple GenServers started under it. The supervisor acts as + the "event manager" and the children GenServers act as the "event handlers". + This approach has some shortcomings (it provides no backpressure for example) + but can still replace GenEvent for low-profile usages of it. [This blog post + by José + Valim](http://blog.plataformatec.com.br/2016/11/replacing-genevent-by-a-supervisor-genserver/) + has more detailed information on this approach. - ## Name Registration + ### GenStage - A GenEvent is bound to the same name registration rules as a `GenServer`. - Read more about it in the `GenServer` docs. + If the use case where you were using GenEvent requires more complex logic, + [GenStage](https://github.com/elixir-lang/gen_stage) provides a great + alternative. GenStage is an external Elixir library maintained by the Elixir + team; it provides a tool to implement systems that exchange events in a + demand-driven way with built-in support for backpressure. See the [GenStage + documentation](https://hexdocs.pm/gen_stage) for more information. - ## Streaming + ### `:gen_event` - `GenEvent`s can be streamed from and streamed with the help of `stream/2`. - Here are some examples: + If your use case requires exactly what GenEvent provided, or you have to + integrate with an existing `:gen_event`-based system, you can still use the + [`:gen_event`](`:gen_event`) Erlang module. + """ - stream = GenEvent.stream(pid) + @moduledoc deprecated: "Use Erlang/OTP's :gen_event module instead" - # Take the next 10 events - Enum.take(stream, 10) + @callback init(args :: term) :: + {:ok, state} + | {:ok, state, :hibernate} + | {:error, reason :: any} + when state: any - # Print all remaining events - for event <- stream do - IO.inspect event - end + @callback handle_event(event :: term, state :: term) :: + {:ok, new_state} + | {:ok, new_state, :hibernate} + | :remove_handler + when new_state: term - A stream may also be given an id, which allows all streams with the given - id to be cancelled at any moment via `cancel_streams/1`. + @callback handle_call(request :: term, state :: term) :: + {:ok, reply, new_state} + | {:ok, reply, new_state, :hibernate} + | {:remove_handler, reply} + when reply: term, new_state: term - ## Learn more + @callback handle_info(msg :: term, state :: term) :: + {:ok, new_state} + | {:ok, new_state, :hibernate} + | :remove_handler + when new_state: term - If you wish to find out more about gen events, Elixir getting started - guides provide a tutorial-like introduction. The documentation and links - in Erlang can also provide extra insight. + @callback terminate(reason, state :: term) :: term + when reason: :stop | {:stop, term} | :remove_handler | {:error, term} | term - * http://elixir-lang.org/getting_started/mix/1.html - * http://www.erlang.org/doc/man/gen_event.html - * http://learnyousomeerlang.com/event-handlers - """ + @callback code_change(old_vsn, state :: term, extra :: term) :: {:ok, new_state :: term} + when old_vsn: term | {:down, term} - @typedoc "Return values of `start*` functions" @type on_start :: {:ok, pid} | {:error, {:already_started, pid}} - @typedoc "The GenEvent manager name" @type name :: atom | {:global, term} | {:via, module, term} - @typedoc "Options used by the `start*` functions" @type options :: [name: name] - @typedoc "The event manager reference" @type manager :: pid | name | {atom, node} - @typedoc "Supported values for new handlers" - @type handler :: module | {module, term} + @type handler :: atom | {atom, term} - @doc """ - Defines a `GenEvent` stream. - - This is a struct returned by `stream/2`. The struct is public and - contains the following fields: - - * `:manager` - the manager reference given to `GenEvent.stream/2` - * `:id` - the event stream id for cancellation - * `:timeout` - the timeout in between events, defaults to `:infinity` - * `:duration` - the duration of the subscription, defaults to `:infinity` - * `:mode` - if the subscription mode is sync or async, defaults to `:sync` - """ - defstruct manager: nil, id: nil, timeout: :infinity, duration: :infinity, mode: :sync + message = "Use one of the alternatives described in the documentation for the GenEvent module" + @deprecated message @doc false defmacro __using__(_) do + deprecation_message = + "the GenEvent module is deprecated, see its documentation for alternatives" + + IO.warn(deprecation_message, __CALLER__) + quote location: :keep do @behaviour :gen_event @@ -194,8 +107,21 @@ defmodule GenEvent do end @doc false - def handle_call(_request, state) do - {:ok, {:error, :bad_call}, state} + def handle_call(msg, state) do + proc = + case Process.info(self(), :registered_name) do + {_, []} -> self() + {_, name} -> name + end + + # We do this to trick Dialyzer to not complain about non-local returns. + case :erlang.phash2(1, 1) do + 0 -> + raise "attempted to call GenEvent #{inspect(proc)} but no handle_call/2 clause was provided" + + 1 -> + {:remove_handler, {:bad_call, msg}} + end end @doc false @@ -204,7 +130,7 @@ defmodule GenEvent do end @doc false - def terminate(reason, state) do + def terminate(_reason, _state) do :ok end @@ -213,452 +139,765 @@ defmodule GenEvent do {:ok, state} end - defoverridable [init: 1, handle_event: 2, handle_call: 2, - handle_info: 2, terminate: 2, code_change: 3] + defoverridable init: 1, + handle_event: 2, + handle_call: 2, + handle_info: 2, + terminate: 2, + code_change: 3 end end - @doc """ - Starts an event manager linked to the current process. - - This is often used to start the `GenEvent` as part of a supervision tree. - - It accepts the `:name` option which is described under the `Name Registration` - section in the `GenServer` module docs. - - If the event manager is successfully created and initialized, the function - returns `{:ok, pid}`, where pid is the pid of the server. If there already - exists a process with the specified server name, the function returns - `{:error, {:already_started, pid}}` with the pid of that process. - - Note that a `GenEvent` started with `start_link/1` is linked to the - parent process and will exit not only on crashes but also if the parent - process exits with `:normal` reason. - """ + @doc false + @deprecated message @spec start_link(options) :: on_start def start_link(options \\ []) when is_list(options) do do_start(:link, options) end - @doc """ - Starts an event manager process without links (outside of a supervision tree). - - See `start_link/1` for more information. - """ + @doc false + @deprecated message @spec start(options) :: on_start def start(options \\ []) when is_list(options) do do_start(:nolink, options) end + @no_callback :"no callback module" + defp do_start(mode, options) do case Keyword.get(options, :name) do nil -> - :gen.start(:gen_event, mode, :"no callback module", [], []) + :gen.start(GenEvent, mode, @no_callback, [], []) + atom when is_atom(atom) -> - :gen.start(:gen_event, mode, {:local, atom}, :"no callback module", [], []) - other when is_tuple(other) -> - :gen.start(:gen_event, mode, other, :"no callback module", [], []) + :gen.start(GenEvent, mode, {:local, atom}, @no_callback, [], []) + + {:global, _term} = tuple -> + :gen.start(GenEvent, mode, tuple, @no_callback, [], []) + + {:via, via_module, _term} = tuple when is_atom(via_module) -> + :gen.start(GenEvent, mode, tuple, @no_callback, [], []) + + other -> + raise ArgumentError, """ + expected :name option to be one of the following: + + * nil + * atom + * {:global, term} + * {:via, module, term} + + Got: #{inspect(other)} + """ + end + end + + @doc false + @deprecated message + @spec stream(manager, keyword) :: GenEvent.Stream.t() + def stream(manager, options \\ []) do + %GenEvent.Stream{manager: manager, timeout: Keyword.get(options, :timeout, :infinity)} + end + + @doc false + @deprecated message + @spec add_handler(manager, handler, term) :: :ok | {:error, term} + def add_handler(manager, handler, args) do + rpc(manager, {:add_handler, handler, args}) + end + + @doc false + @deprecated message + @spec add_mon_handler(manager, handler, term) :: :ok | {:error, term} + def add_mon_handler(manager, handler, args) do + rpc(manager, {:add_mon_handler, handler, args, self()}) + end + + @doc false + @deprecated message + @spec notify(manager, term) :: :ok + def notify(manager, event) + + def notify({:global, name}, msg) do + try do + :global.send(name, {:notify, msg}) + :ok + catch + _, _ -> :ok end end - @doc """ - Returns a stream that consumes and notifies events to the `manager`. + def notify({:via, mod, name}, msg) when is_atom(mod) do + try do + mod.send(name, {:notify, msg}) + :ok + catch + _, _ -> :ok + end + end - The stream is a `GenEvent` struct that implements the `Enumerable` - protocol. The supported options are: + def notify(manager, msg) + when is_pid(manager) + when is_atom(manager) + when tuple_size(manager) == 2 and is_atom(elem(manager, 0)) and is_atom(elem(manager, 1)) do + send(manager, {:notify, msg}) + :ok + end - * `:id` - an id to identify all live stream instances; when an `:id` is - given, existing streams can be called with via `cancel_streams`. + @doc false + @deprecated message + @spec sync_notify(manager, term) :: :ok + def sync_notify(manager, event) do + rpc(manager, {:sync_notify, event}) + end - * `:timeout` (Enumerable) - raises if no event arrives in X milliseconds. + @doc false + @deprecated message + @spec ack_notify(manager, term) :: :ok + def ack_notify(manager, event) do + rpc(manager, {:ack_notify, event}) + end - * `:duration` (Enumerable) - only consume events during the X milliseconds - from the streaming start. + @doc false + @deprecated message + @spec call(manager, handler, term, timeout) :: term | {:error, term} + def call(manager, handler, request, timeout \\ 5000) do + try do + :gen.call(manager, self(), {:call, handler, request}, timeout) + catch + :exit, reason -> + exit({reason, {__MODULE__, :call, [manager, handler, request, timeout]}}) + else + {:ok, res} -> res + end + end - * `:mode` - the mode to consume events, can be `:sync` (default) or - `:async`. On sync, the event manager waits for the event to be consumed - before moving on to the next event handler. + @doc false + @deprecated message + @spec remove_handler(manager, handler, term) :: term | {:error, term} + def remove_handler(manager, handler, args) do + rpc(manager, {:delete_handler, handler, args}) + end - """ - def stream(manager, options \\ []) do - %GenEvent{manager: manager, - id: Keyword.get(options, :id), - timeout: Keyword.get(options, :timeout, :infinity), - duration: Keyword.get(options, :duration, :infinity), - mode: Keyword.get(options, :mode, :sync)} + @doc false + @deprecated message + @spec swap_handler(manager, handler, term, handler, term) :: :ok | {:error, term} + def swap_handler(manager, handler1, args1, handler2, args2) do + rpc(manager, {:swap_handler, handler1, args1, handler2, args2}) end - @doc """ - Adds a new event handler to the event `manager`. + @doc false + @deprecated message + @spec swap_mon_handler(manager, handler, term, handler, term) :: :ok | {:error, term} + def swap_mon_handler(manager, handler1, args1, handler2, args2) do + rpc(manager, {:swap_mon_handler, handler1, args1, handler2, args2, self()}) + end + + @doc false + @deprecated message + @spec which_handlers(manager) :: [handler] + def which_handlers(manager) do + rpc(manager, :which_handlers) + end + + @doc false + @deprecated message + @spec stop(manager, reason :: term, timeout) :: :ok + def stop(manager, reason \\ :normal, timeout \\ :infinity) do + :gen.stop(manager, reason, timeout) + end - The event manager will call the `init/1` callback with `args` to - initiate the event handler and its internal state. + defp rpc(module, cmd) do + {:ok, reply} = :gen.call(module, self(), cmd, :infinity) + reply + end - If `init/1` returns a correct value indicating successful completion, - the event manager adds the event handler and this function returns - `:ok`. If the callback fails with `reason` or returns `{:error, reason}`, - the event handler is ignored and this function returns `{:EXIT, reason}` - or `{:error, reason}`, respectively. + ## Init callbacks - ## Linked handlers + require Record + Record.defrecordp(:handler, [:module, :id, :state, :pid, :ref]) - When adding a handler, a `:link` option with value `true` can be given. - This means the event handler and the calling process are now linked. + @doc false + def init_it(starter, :self, name, mod, args, options) do + init_it(starter, self(), name, mod, args, options) + end - If the calling process later terminates with `reason`, the event manager - will delete the event handler by calling the `terminate/2` callback with - `{:stop, reason}` as argument. If the event handler later is deleted, - the event manager sends a message `{:gen_event_EXIT, handler, reason}` - to the calling process. Reason is one of the following: + def init_it(starter, parent, name, _mod, _args, options) do + Process.put(:"$initial_call", {__MODULE__, :init_it, 6}) + debug = :gen.debug_options(name, options) + :proc_lib.init_ack(starter, {:ok, self()}) + loop(parent, name(name), [], debug, false) + end - * `:normal` - if the event handler has been removed due to a call to - `remove_handler/3`, or `:remove_handler` has been returned by a callback - function + @doc false + def init_hib(parent, name, handlers, debug) do + fetch_msg(parent, name, handlers, debug, true) + end - * `:shutdown` - if the event handler has been removed because the event - manager is terminating + defp name({:local, name}), do: name + defp name({:global, name}), do: name + defp name({:via, _, name}), do: name + defp name(pid) when is_pid(pid), do: pid - * `{:swapped, new_handler, pid}` - if the process pid has replaced the - event handler by another + ## Loop - * a term - if the event handler is removed due to an error. Which term - depends on the error + defp loop(parent, name, handlers, debug, true) do + :proc_lib.hibernate(__MODULE__, :init_hib, [parent, name, handlers, debug]) + end - """ - @spec add_handler(manager, handler, term, [link: boolean]) :: :ok | {:EXIT, term} | {:error, term} - def add_handler(manager, handler, args, options \\ []) do - case Keyword.get(options, :link, false) do - true -> :gen_event.add_sup_handler(manager, handler, args) - false -> :gen_event.add_handler(manager, handler, args) + defp loop(parent, name, handlers, debug, false) do + fetch_msg(parent, name, handlers, debug, false) + end + + defp fetch_msg(parent, name, handlers, debug, hib) do + receive do + {:system, from, req} -> + :sys.handle_system_msg(req, from, parent, __MODULE__, debug, [name, handlers, hib], hib) + + {:EXIT, ^parent, reason} -> + server_terminate(reason, parent, handlers, name) + + msg when debug == [] -> + handle_msg(msg, parent, name, handlers, []) + + msg -> + debug = :sys.handle_debug(debug, &print_event/3, name, {:in, msg}) + handle_msg(msg, parent, name, handlers, debug) end end - @doc """ - Sends an event notification to the event `manager`. + defp handle_msg(msg, parent, name, handlers, debug) do + case msg do + {:notify, event} -> + {hib, handlers} = server_event(:async, event, handlers, name) + loop(parent, name, handlers, debug, hib) + + {_from, _tag, {:notify, event}} -> + {hib, handlers} = server_event(:async, event, handlers, name) + loop(parent, name, handlers, debug, hib) + + {_from, tag, {:ack_notify, event}} -> + reply(tag, :ok) + {hib, handlers} = server_event(:ack, event, handlers, name) + loop(parent, name, handlers, debug, hib) + + {_from, tag, {:sync_notify, event}} -> + {hib, handlers} = server_event(:sync, event, handlers, name) + reply(tag, :ok) + loop(parent, name, handlers, debug, hib) + + {:DOWN, ref, :process, _pid, reason} = other -> + case handle_down(ref, reason, handlers, name) do + {:ok, handlers} -> + loop(parent, name, handlers, debug, false) + + :error -> + {hib, handlers} = server_info(other, handlers, name) + loop(parent, name, handlers, debug, hib) + end - The event manager will call `handle_event/2` for each installed event handler. + {_from, tag, {:call, handler, query}} -> + {hib, reply, handlers} = server_call(handler, query, handlers, name) + reply(tag, reply) + loop(parent, name, handlers, debug, hib) - `notify` is asynchronous and will return immediately after the notification is - sent. `notify` will not fail even if the specified event manager does not exist, - unless it is specified as `name` (atom). - """ - @spec notify(manager, term) :: :ok - defdelegate notify(manager, event), to: :gen_event + {_from, tag, {:add_handler, handler, args}} -> + {hib, reply, handlers} = server_add_handler(handler, args, handlers) + reply(tag, reply) + loop(parent, name, handlers, debug, hib) - @doc """ - Sends a sync event notification to the event `manager`. + {_from, tag, {:add_mon_handler, handler, args, notify}} -> + {hib, reply, handlers} = server_add_mon_handler(handler, args, handlers, notify) + reply(tag, reply) + loop(parent, name, handlers, debug, hib) - In other words, this function only returns `:ok` after the event manager - invokes the `handle_event/2` on each installed event handler. + {_from, tag, {:add_process_handler, pid, notify}} -> + {hib, reply, handlers} = server_add_process_handler(pid, handlers, notify) + reply(tag, reply) + loop(parent, name, handlers, debug, hib) - See `notify/2` for more info. - """ - @spec sync_notify(manager, term) :: :ok - defdelegate sync_notify(manager, event), to: :gen_event + {_from, tag, {:delete_handler, handler, args}} -> + {reply, handlers} = server_remove_handler(handler, args, handlers, name) + reply(tag, reply) + loop(parent, name, handlers, debug, false) - @doc """ - Makes a synchronous call to the event `handler` installed in `manager`. + {_from, tag, {:swap_handler, handler1, args1, handler2, args2}} -> + {hib, reply, handlers} = + server_swap_handler(handler1, args1, handler2, args2, handlers, nil, name) - The given `request` is sent and the caller waits until a reply arrives or - a timeout occurs. The event manager will call `handle_call/2` to handle - the request. + reply(tag, reply) + loop(parent, name, handlers, debug, hib) - The return value `reply` is defined in the return value of `handle_call/2`. - If the specified event handler is not installed, the function returns - `{:error, :bad_module}`. - """ - @spec call(manager, handler, term, timeout) :: term | {:error, term} - def call(manager, handler, request, timeout \\ 5000) do - :gen_event.call(manager, handler, request, timeout) + {_from, tag, {:swap_mon_handler, handler1, args1, handler2, args2, mon}} -> + {hib, reply, handlers} = + server_swap_handler(handler1, args1, handler2, args2, handlers, mon, name) + + reply(tag, reply) + loop(parent, name, handlers, debug, hib) + + {_from, tag, :which_handlers} -> + reply(tag, server_which_handlers(handlers)) + loop(parent, name, handlers, debug, false) + + {_from, tag, :get_modules} -> + reply(tag, server_get_modules(handlers)) + loop(parent, name, handlers, debug, false) + + other -> + {hib, handlers} = server_info(other, handlers, name) + loop(parent, name, handlers, debug, hib) + end end - @doc """ - Cancels all streams currently running with the given `:id`. + ## System callbacks - In order for a stream to be cancelled, an `:id` must be passed - when the stream is created via `stream/2`. Passing a stream without - an id leads to an argument error. - """ - @spec cancel_streams(t) :: :ok - def cancel_streams(%GenEvent{id: nil}) do - raise ArgumentError, "cannot cancel streams without an id" + @doc false + def system_continue(parent, debug, [name, handlers, hib]) do + loop(parent, name, handlers, debug, hib) end - def cancel_streams(%GenEvent{manager: manager, id: id}) do - handlers = :gen_event.which_handlers(manager) + @doc false + def system_terminate(reason, parent, _debug, [name, handlers, _hib]) do + server_terminate(reason, parent, handlers, name) + end - for {Enumerable.GenEvent, {handler_id, _}} = ref <- handlers, - handler_id === id do - :gen_event.delete_handler(manager, ref, :remove_handler) - end + @doc false + def system_code_change([name, handlers, hib], module, old_vsn, extra) do + handlers = + for handler <- handlers do + if handler(handler, :module) == module do + {:ok, state} = module.code_change(old_vsn, handler(handler, :state), extra) + handler(handler, state: state) + else + handler + end + end - :ok + {:ok, [name, handlers, hib]} end - @doc """ - Removes an event handler from the event `manager`. + @doc false + def system_get_state([_name, handlers, _hib]) do + tuples = + for handler(module: mod, id: id, state: state) <- handlers do + {mod, id, state} + end - The event manager will call `terminate/2` to terminate the event handler - and return the callback value. If the specified event handler is not - installed, the function returns `{:error, :module_not_found}`. - """ - @spec remove_handler(manager, handler, term) :: term | {:error, term} - def remove_handler(manager, handler, args) do - :gen_event.delete_handler(manager, handler, args) + {:ok, tuples} end - @doc """ - Replaces an old event handler with a new one in the event `manager`. + @doc false + def system_replace_state(fun, [name, handlers, hib]) do + {handlers, states} = + :lists.unzip( + for handler <- handlers do + handler(module: mod, id: id, state: state) = handler + cur = {mod, id, state} + + try do + new = {^mod, ^id, new_state} = fun.(cur) + {handler(handler, state: new_state), new} + catch + _, _ -> + {handler, cur} + end + end + ) + + {:ok, states, [name, handlers, hib]} + end + + @doc false + def format_status(opt, status_data) do + [pdict, sys_state, parent, _debug, [name, handlers, _hib]] = status_data + header = :gen.format_status_header('Status for event handler', name) + + formatted = + for handler <- handlers do + handler(module: module, state: state) = handler + + if function_exported?(module, :format_status, 2) do + try do + state = module.format_status(opt, [pdict, state]) + handler(handler, state: state) + catch + _, _ -> handler + end + else + handler + end + end + + [ + header: header, + data: [{'Status', sys_state}, {'Parent', parent}], + items: {'Installed handlers', formatted} + ] + end - First, the old event handler is deleted by calling `terminate/2` with - the given `args1` and collects the return value. Then the new event handler - is added and initiated by calling `init({args2, term}), where term is the - return value of calling `terminate/2` in the old handler. This makes it - possible to transfer information from one handler to another. + ## Loop helpers - The new handler will be added even if the specified old event handler - is not installed in which case `term = :error` or if the handler fails to - terminate with a given reason. + defp print_event(dev, {:in, msg}, name) do + case msg do + {:notify, event} -> + IO.puts(dev, "*DBG* #{inspect(name)} got event #{inspect(event)}") - If there was a linked connection between handler1 and a process pid, there - will be a link connection between handler2 and pid instead. A new link in - between the caller process and the new handler can also be set with by - giving `link: true` as option. See `add_handler/4` for more information. + {_, _, {:call, handler, query}} -> + IO.puts( + dev, + "*DBG* #{inspect(name)} (handler #{inspect(handler)}) got call #{inspect(query)}" + ) - If `init/1` in the second handler returns a correct value, this function - returns `:ok`. - """ - @spec swap_handler(manager, handler, term, handler, term, [link: boolean]) :: :ok | {:error, term} - def swap_handler(manager, handler1, args1, handler2, args2, options \\ []) do - case Keyword.get(options, :link, false) do - true -> :gen_event.swap_sup_handler(manager, {handler1, args1}, {handler2, args2}) - false -> :gen_event.swap_handler(manager, {handler1, args1}, {handler2, args2}) + _ -> + IO.puts(dev, "*DBG* #{inspect(name)} got #{inspect(msg)}") end end - @doc """ - Returns a list of all event handlers installed in the `manager`. - """ - @spec which_handlers(manager) :: [handler] - defdelegate which_handlers(manager), to: :gen_event + defp print_event(dev, dbg, name) do + IO.puts(dev, "*DBG* #{inspect(name)}: #{inspect(dbg)}") + end - @doc """ - Terminates the event `manager`. + defp server_add_handler({module, id}, args, handlers) do + handler = handler(module: module, id: {module, id}) + do_add_handler(module, handler, args, handlers, :ok) + end - Before terminating, the event manager will call `terminate(:stop, ...)` - for each installed event handler. - """ - @spec stop(manager) :: :ok - defdelegate stop(manager), to: :gen_event -end + defp server_add_handler(module, args, handlers) do + handler = handler(module: module, id: module) + do_add_handler(module, handler, args, handlers, :ok) + end -defimpl Enumerable, for: GenEvent do - use GenEvent + defp server_add_mon_handler({module, id}, args, handlers, notify) do + ref = Process.monitor(notify) + handler = handler(module: module, id: {module, id}, pid: notify, ref: ref) + do_add_handler(module, handler, args, handlers, :ok) + end - @doc false - def init({_mode, mon_pid, _pid, ref} = state) do - # Tell the mon_pid we are good to go, and send self() so that this handler - # can be removed later without using the managers name. - send(mon_pid, {:UP, ref, self()}) - {:ok, state} + defp server_add_mon_handler(module, args, handlers, notify) do + ref = Process.monitor(notify) + handler = handler(module: module, id: module, pid: notify, ref: ref) + do_add_handler(module, handler, args, handlers, :ok) end - @doc false - def handle_event(event, {:sync, mon_pid, pid, ref} = state) do - sync = Process.monitor(mon_pid) - send pid, {ref, sync, event} - receive do - {^sync, :done} -> - Process.demonitor(sync, [:flush]) - :remove_handler - {^sync, :next} -> - Process.demonitor(sync, [:flush]) - {:ok, state} - {:DOWN, ^sync, _, _, _} -> - {:ok, state} + defp server_add_process_handler(pid, handlers, notify) do + ref = Process.monitor(pid) + handler = handler(module: GenEvent.Stream, id: {self(), ref}, pid: notify, ref: ref) + do_add_handler(GenEvent.Stream, handler, {pid, ref}, handlers, {self(), ref}) + end + + defp server_remove_handler(module, args, handlers, name) do + do_take_handler(module, args, handlers, name, :remove, :normal) + end + + defp server_swap_handler(module1, args1, module2, args2, handlers, sup, name) do + {state, handlers} = + do_take_handler(module1, args1, handlers, name, :swapped, {:swapped, module2, sup}) + + if sup do + server_add_mon_handler(module2, {args2, state}, handlers, sup) + else + server_add_handler(module2, {args2, state}, handlers) + end + end + + defp server_info(event, handlers, name) do + handlers = :lists.reverse(handlers) + server_notify(event, :handle_info, handlers, name, handlers, [], false) + end + + defp server_event(mode, event, handlers, name) do + {handlers, streams} = server_split_process_handlers(mode, event, handlers, [], []) + {hib, handlers} = server_notify(event, :handle_event, handlers, name, handlers, [], false) + {hib, server_collect_process_handlers(mode, event, streams, handlers, name)} + end + + defp server_split_process_handlers(mode, event, [handler | t], handlers, streams) do + case handler(handler, :id) do + {pid, _ref} when is_pid(pid) -> + server_process_notify(mode, event, handler) + server_split_process_handlers(mode, event, t, handlers, [handler | streams]) + + _ -> + server_split_process_handlers(mode, event, t, [handler | handlers], streams) end end - def handle_event(event, {:async, _mon_pid, pid, ref} = state) do - send pid, {ref, nil, event} - {:ok, state} + defp server_split_process_handlers(_mode, _event, [], handlers, streams) do + {handlers, streams} end - def reduce(stream, acc, fun) do - start_fun = fn() -> start(stream) end - next_fun = &next(stream, &1) - stop_fun = &stop(stream, &1) - Stream.resource(start_fun, next_fun, stop_fun).(acc, wrap_reducer(fun)) + defp server_process_notify(mode, event, handler(state: {pid, ref})) do + send(pid, {self(), {self(), ref}, {mode_to_tag(mode), event}}) end - def count(_stream) do - {:error, __MODULE__} + defp mode_to_tag(:ack), do: :ack_notify + defp mode_to_tag(:sync), do: :sync_notify + defp mode_to_tag(:async), do: :notify + + defp server_notify(event, fun, [handler | t], name, handlers, acc, hib) do + case server_update(handler, fun, event, name, handlers) do + {new_hib, handler} -> + server_notify(event, fun, t, name, handlers, [handler | acc], hib or new_hib) + + :error -> + server_notify(event, fun, t, name, handlers, acc, hib) + end end - def member?(_stream, _item) do - {:error, __MODULE__} + defp server_notify(_, _, [], _, _, acc, hib) do + {hib, acc} end - defp wrap_reducer(fun) do - fn - {nil, _manager, event}, acc -> - fun.(event, acc) - {ref, manager, event}, acc -> - try do - fun.(event, acc) - after - send manager, {ref, :next} + defp server_update(handler, fun, event, name, _handlers) do + handler(module: module, state: state) = handler + + case do_handler(module, fun, [event, state]) do + {:ok, res} -> + case res do + {:ok, state} -> + {false, handler(handler, state: state)} + + {:ok, state, :hibernate} -> + {true, handler(handler, state: state)} + + :remove_handler -> + do_terminate(handler, :remove_handler, event, name, :normal) + :error + + other -> + reason = {:bad_return_value, other} + do_terminate(handler, {:error, reason}, event, name, reason) + :error end + + {:error, reason} -> + do_terminate(handler, {:error, reason}, event, name, reason) + :error end end - defp start(%{manager: manager, id: id, duration: duration, mode: mode} = stream) do - {mon_pid, mon_ref} = add_handler(mode, manager, id, duration) - send mon_pid, {:UP, mon_ref, self()} + defp server_collect_process_handlers(:async, event, [handler | t], handlers, name) do + server_collect_process_handlers(:async, event, t, [handler | handlers], name) + end + + defp server_collect_process_handlers(mode, event, [handler | t], handlers, name) + when mode in [:sync, :ack] do + handler(ref: ref, id: id) = handler receive do - # The subscription process gave us a go. - {:UP, ^mon_ref, manager_pid} -> - {mon_ref, mon_pid, manager_pid} - # The subscription process died due to an abnormal reason. - {:DOWN, ^mon_ref, _, _, reason} -> - exit({reason, {__MODULE__, :start, [stream]}}) + {^ref, :ok} -> + server_collect_process_handlers(mode, event, t, [handler | handlers], name) + + {_from, tag, {:delete_handler, ^id, args}} -> + do_terminate(handler, args, :remove, name, :normal) + reply(tag, :ok) + server_collect_process_handlers(mode, event, t, handlers, name) + + {:DOWN, ^ref, _, _, reason} -> + do_terminate(handler, {:stop, reason}, :DOWN, name, :shutdown) + server_collect_process_handlers(mode, event, t, handlers, name) end end - defp next(%{timeout: timeout} = stream, {mon_ref, mon_pid, manager_pid} = acc) do - # If :DOWN is received must resend it to self so that stop/2 can receive it - # and know that the handler has been removed. - receive do - {:DOWN, ^mon_ref, _, _, :normal} -> - send(self(), {:DOWN, mon_ref, :process, mon_pid, :normal}) - nil - {:DOWN, ^mon_ref, _, _, reason} -> - send(self(), {:DOWN, mon_ref, :process, mon_pid, :normal}) - exit({reason, {__MODULE__, :next, [stream, acc]}}) - {^mon_ref, sync_ref, event} -> - {{sync_ref, manager_pid, event}, acc} - after - timeout -> - exit({:timeout, {__MODULE__, :next, [stream, acc]}}) + defp server_collect_process_handlers(_mode, _event, [], handlers, _name) do + handlers + end + + defp server_call(module, query, handlers, name) do + case :lists.keyfind(module, handler(:id) + 1, handlers) do + false -> + {false, {:error, :not_found}, handlers} + + handler -> + case server_call_update(handler, query, name, handlers) do + {{hib, handler}, reply} -> + {hib, reply, :lists.keyreplace(module, handler(:id) + 1, handlers, handler)} + + {:error, reply} -> + {false, reply, :lists.keydelete(module, handler(:id) + 1, handlers)} + end end end - defp stop(%{mode: mode} = stream, {mon_ref, mon_pid, manager_pid} = acc) do - case remove_handler(mon_ref, mon_pid, manager_pid) do - :ok when mode == :async -> - flush_events(mon_ref) - :ok -> - :ok + defp server_call_update(handler, query, name, _handlers) do + handler(module: module, state: state) = handler + + case do_handler(module, :handle_call, [query, state]) do + {:ok, res} -> + case res do + {:ok, reply, state} -> + {{false, handler(handler, state: state)}, reply} + + {:ok, reply, state, :hibernate} -> + {{true, handler(handler, state: state)}, reply} + + {:remove_handler, reply} -> + do_terminate(handler, :remove_handler, query, name, :normal) + {:error, reply} + + other -> + reason = {:bad_return_value, other} + do_terminate(handler, {:error, reason}, query, name, reason) + {:error, {:error, reason}} + end + {:error, reason} -> - exit({reason, {__MODULE__, :stop, [stream, acc]}}) + do_terminate(handler, {:error, reason}, query, name, reason) + {:error, {:error, reason}} end end - defp add_handler(mode, manager, id, duration) do - parent = self() - - # The subscription is managed by another process, that dies if - # the handler dies, and is killed when there is a need to remove - # the subscription. - spawn_monitor(fn -> - # It is possible that the handler could be removed, and then the GenEvent - # could exit before this process has exited normally. Because the removal - # does not cause an unlinking this process would exit with the same - # reason. Trapping exits ensures that no errors is raised in this case. - Process.flag(:trap_exit, true) - parent_ref = Process.monitor(parent) - - # Receive the notification from the parent, unless it died. - mon_ref = receive do - {:UP, ref, ^parent} -> ref - {:DOWN, ^parent_ref, _, _, _} -> exit(:normal) - end + defp server_get_modules(handlers) do + for(handler(module: module) <- handlers, do: module) + |> :ordsets.from_list() + |> :ordsets.to_list() + end - cancel = cancel_ref(id, mon_ref) - :ok = :gen_event.add_sup_handler(manager, {__MODULE__, cancel}, - {mode, self(), parent, mon_ref}) - - receive do - # This message is already in the mailbox if we got this far. - {:UP, ^mon_ref, manager_pid} -> - send(parent, {:UP, mon_ref, manager_pid}) - receive do - # The stream has finished, remove the handler. - {:DONE, ^mon_ref} -> - exit_handler(manager_pid, parent_ref, cancel) - - # If the parent died, we can exit normally. - {:DOWN, ^parent_ref, _, _, _} -> - exit(:normal) - - # reason should be normal unless the handler is swapped. - {:gen_event_EXIT, {__MODULE__, ^cancel}, reason} -> - exit(reason) - - # Exit if the manager dies, so the streamer is notified. - {:EXIT, ^manager_pid, :noconnection} -> - exit({:nodedown, node(manager_pid)}) - - {:EXIT, ^manager_pid, reason} -> - exit(reason) - after - # Our time is over, notify the parent. - duration -> exit(:normal) - end + defp server_which_handlers(handlers) do + for handler(id: id) <- handlers, do: id + end + + defp server_terminate(reason, _parent, handlers, name) do + _ = + for handler <- handlers do + do_terminate(handler, :stop, :stop, name, :shutdown) end - end) + + exit(reason) end - defp cancel_ref(nil, mon_ref), do: mon_ref - defp cancel_ref(id, mon_ref), do: {id, mon_ref} + defp reply({from, ref}, msg) do + send(from, {ref, msg}) + end - defp exit_handler(manager_pid, parent_ref, cancel) do - # Send exit signal so manager removes handler. - Process.exit(manager_pid, :shutdown) - receive do - # If the parent died, we can exit normally. - {:DOWN, ^parent_ref, _, _, _} -> - exit(:normal) - - # Probably the reason is :shutdown, which occurs when the manager receives - # an exit signal from a handler supervising process. However whatever the - # reason the handler has been removed so it is ok. - {:gen_event_EXIT, {__MODULE__, ^cancel}, _} -> - exit(:normal) - - # The connection broke, perhaps the handler might try to forward events - # before it removes the handler, so must exit abnormally. - {:EXIT, ^manager_pid, :noconnection} -> - exit({:nodedown, node(manager_pid)}) - - # The manager has exited but don't exit abnormally as the handler has died - # with the manager and all expected events have been handled. This is ok. - {:EXIT, ^manager_pid, _} -> - exit(:normal) + defp handle_down(ref, reason, handlers, name) do + case :lists.keyfind(ref, handler(:ref) + 1, handlers) do + false -> + :error + + handler -> + do_terminate(handler, {:stop, reason}, :DOWN, name, :shutdown) + {:ok, :lists.keydelete(ref, handler(:ref) + 1, handlers)} end end - defp remove_handler(mon_ref, mon_pid, manager_pid) do - send(mon_pid, {:DONE, mon_ref}) - receive do - {^mon_ref, sync, _} when sync != nil -> - send(manager_pid, {sync, :done}) - Process.demonitor(mon_ref, [:flush]) - :ok - {:DOWN, ^mon_ref, _, _, :normal} -> - :ok - {:DOWN, ^mon_ref, _, _, reason} -> - {:error, reason} + defp do_add_handler(module, handler, arg, handlers, succ) do + case :lists.keyfind(handler(handler, :id), handler(:id) + 1, handlers) do + false -> + case do_handler(module, :init, [arg]) do + {:ok, res} -> + case res do + {:ok, state} -> + {false, succ, [handler(handler, state: state) | handlers]} + + {:ok, state, :hibernate} -> + {true, succ, [handler(handler, state: state) | handlers]} + + {:error, _} = error -> + {false, error, handlers} + + other -> + {false, {:error, {:bad_return_value, other}}, handlers} + end + + {:error, _} = error -> + {false, error, handlers} + end + + _ -> + {false, {:error, :already_present}, handlers} end end - defp flush_events(mon_ref) do - receive do - {^mon_ref, _, _} -> - flush_events(mon_ref) - after - 0 -> :ok + defp do_take_handler(module, args, handlers, name, last_in, reason) do + case :lists.keytake(module, handler(:id) + 1, handlers) do + {:value, handler, handlers} -> + {do_terminate(handler, args, last_in, name, reason), handlers} + + false -> + {{:error, :not_found}, handlers} + end + end + + defp do_terminate(handler, arg, last_in, name, reason) do + handler(module: module, state: state) = handler + + res = + case do_handler(module, :terminate, [arg, state]) do + {:ok, res} -> res + {:error, _} = error -> error + end + + report_terminate(handler, reason, state, last_in, name) + res + end + + defp do_handler(mod, fun, args) do + try do + apply(mod, fun, args) + catch + :throw, val -> {:ok, val} + :error, val -> {:error, {val, __STACKTRACE__}} + :exit, val -> {:error, val} + else + res -> {:ok, res} + end + end + + defp report_terminate(handler, reason, state, last_in, name) do + report_error(handler, reason, state, last_in, name) + + if ref = handler(handler, :ref) do + Process.demonitor(ref, [:flush]) + end + + if pid = handler(handler, :pid) do + send(pid, {:gen_event_EXIT, handler(handler, :id), reason}) + end + end + + defp report_error(_handler, :normal, _, _, _), do: :ok + defp report_error(_handler, :shutdown, _, _, _), do: :ok + defp report_error(_handler, {:swapped, _, _}, _, _, _), do: :ok + + defp report_error(handler, reason, state, last_in, name) do + reason = + case reason do + {:undef, [{m, f, a, _} | _] = mfas} -> + cond do + :code.is_loaded(m) == false -> + {:"module could not be loaded", mfas} + + function_exported?(m, f, length(a)) -> + reason + + true -> + {:"function not exported", mfas} + end + + _ -> + reason + end + + formatted = report_status(handler, state) + + :error_logger.error_msg( + '** gen_event handler ~p crashed.~n' ++ + '** Was installed in ~p~n' ++ + '** Last event was: ~p~n' ++ '** When handler state == ~p~n' ++ '** Reason == ~p~n', + [handler(handler, :id), name, last_in, formatted, reason] + ) + end + + defp report_status(handler(module: module), state) do + if function_exported?(module, :format_status, 2) do + try do + module.format_status(:terminate, [Process.get(), state]) + catch + _, _ -> state + end + else + state end end end diff --git a/lib/elixir/lib/gen_event/stream.ex b/lib/elixir/lib/gen_event/stream.ex new file mode 100644 index 00000000000..0ff94bd5249 --- /dev/null +++ b/lib/elixir/lib/gen_event/stream.ex @@ -0,0 +1,171 @@ +defmodule GenEvent.Stream do + @moduledoc false + defstruct manager: nil, timeout: :infinity + + @type t :: %__MODULE__{manager: GenEvent.manager(), timeout: timeout} + + @doc false + def init({_pid, _ref} = state) do + {:ok, state} + end + + @doc false + def handle_event(event, _state) do + # We do this to trick Dialyzer to not complain about non-local returns. + case :erlang.phash2(1, 1) do + 0 -> exit({:bad_event, event}) + 1 -> :remove_handler + end + end + + @doc false + def handle_call(msg, _state) do + # We do this to trick Dialyzer to not complain about non-local returns. + reason = {:bad_call, msg} + + case :erlang.phash2(1, 1) do + 0 -> exit(reason) + 1 -> {:remove_handler, reason} + end + end + + @doc false + def handle_info(_msg, state) do + {:ok, state} + end + + @doc false + def terminate(_reason, _state) do + :ok + end + + @doc false + def code_change(_old, state, _extra) do + {:ok, state} + end +end + +defimpl Enumerable, for: GenEvent.Stream do + def reduce(stream, acc, fun) do + start_fun = fn -> start(stream) end + next_fun = &next(stream, &1) + stop_fun = &stop(stream, &1) + Stream.resource(start_fun, next_fun, stop_fun).(acc, wrap_reducer(fun)) + end + + def count(_stream) do + {:error, __MODULE__} + end + + def member?(_stream, _item) do + {:error, __MODULE__} + end + + def slice(_stream) do + {:error, __MODULE__} + end + + defp wrap_reducer(fun) do + fn + {:ack, manager, ref, event}, acc -> + send(manager, {ref, :ok}) + fun.(event, acc) + + {:async, _manager, _ref, event}, acc -> + fun.(event, acc) + + {:sync, manager, ref, event}, acc -> + try do + fun.(event, acc) + after + send(manager, {ref, :ok}) + end + end + end + + defp start(%{manager: manager} = stream) do + try do + {:ok, {pid, ref}} = + :gen.call(manager, self(), {:add_process_handler, self(), self()}, :infinity) + + mon_ref = Process.monitor(pid) + {pid, ref, mon_ref} + catch + :exit, reason -> exit({reason, {__MODULE__, :start, [stream]}}) + end + end + + defp next(%{timeout: timeout} = stream, {pid, ref, mon_ref} = acc) do + self = self() + + receive do + # Got an async event. + {_from, {^pid, ^ref}, {:notify, event}} -> + {[{:async, pid, ref, event}], acc} + + # Got a sync event. + {_from, {^pid, ^ref}, {:sync_notify, event}} -> + {[{:sync, pid, ref, event}], acc} + + # Got an ack event. + {_from, {^pid, ^ref}, {:ack_notify, event}} -> + {[{:ack, pid, ref, event}], acc} + + # The handler was removed. Stop iteration, resolve the + # event later. We need to demonitor now, otherwise DOWN + # appears with higher priority in the shutdown process. + {:gen_event_EXIT, {^pid, ^ref}, _reason} = event -> + Process.demonitor(mon_ref, [:flush]) + send(self, event) + {:halt, {:removed, acc}} + + # The manager died. Stop iteration, resolve the event later. + {:DOWN, ^mon_ref, _, _, _} = event -> + send(self, event) + {:halt, {:removed, acc}} + after + timeout -> + exit({:timeout, {__MODULE__, :next, [stream, acc]}}) + end + end + + # If we reach this branch, we know the handler was already + # removed, so we don't trigger a request for doing so. + defp stop(stream, {:removed, {pid, ref, mon_ref} = acc}) do + case wait_for_handler_removal(pid, ref, mon_ref) do + :ok -> + flush_events(ref) + + {:error, reason} -> + exit({reason, {__MODULE__, :stop, [stream, acc]}}) + end + end + + # If we reach this branch, the handler was not removed yet, + # so we trigger a request for doing so. + defp stop(stream, {pid, ref, _} = acc) do + _ = :gen_event.delete_handler(pid, {pid, ref}, :shutdown) + stop(stream, {:removed, acc}) + end + + defp wait_for_handler_removal(pid, ref, mon_ref) do + receive do + {:gen_event_EXIT, {^pid, ^ref}, _reason} -> + Process.demonitor(mon_ref, [:flush]) + :ok + + {:DOWN, ^mon_ref, _, _, reason} -> + {:error, reason} + end + end + + defp flush_events(ref) do + receive do + {_from, {_pid, ^ref}, {notify, _event}} + when notify in [:notify, :ack_notify, :sync_notify] -> + flush_events(ref) + after + 0 -> :ok + end + end +end diff --git a/lib/elixir/lib/gen_server.ex b/lib/elixir/lib/gen_server.ex index faeb3344aab..7fbd6660493 100644 --- a/lib/elixir/lib/gen_server.ex +++ b/lib/elixir/lib/gen_server.ex @@ -2,7 +2,7 @@ defmodule GenServer do @moduledoc """ A behaviour module for implementing the server of a client-server relation. - A GenServer is a process as any other Elixir process and it can be used + A GenServer is a process like any other Elixir process and it can be used to keep state, execute code asynchronously and so on. The advantage of using a generic server process (GenServer) implemented using this module is that it will have a standard set of interface functions and include functionality for @@ -11,24 +11,31 @@ defmodule GenServer do ## Example The GenServer behaviour abstracts the common client-server interaction. - Developer are only required to implement the callbacks and functionality they are - interested in. + Developers are only required to implement the callbacks and functionality + they are interested in. Let's start with a code example and then explore the available callbacks. Imagine we want a GenServer that works like a stack, allowing us to push - and pop items: + and pop elements: defmodule Stack do use GenServer # Callbacks - def handle_call(:pop, _from, [h|t]) do - {:reply, h, t} + @impl true + def init(stack) do + {:ok, stack} end - def handle_cast({:push, item}, state) do - {:noreply, [item|state]} + @impl true + def handle_call(:pop, _from, [head | tail]) do + {:reply, head, tail} + end + + @impl true + def handle_cast({:push, element}, state) do + {:noreply, [element | state]} end end @@ -45,163 +52,663 @@ defmodule GenServer do GenServer.call(pid, :pop) #=> :world - We start our `Stack` by calling `start_link/3`, passing the module + We start our `Stack` by calling `start_link/2`, passing the module with the server implementation and its initial argument (a list - representing the stack containing the item `:hello`). We can primarily + representing the stack containing the element `:hello`). We can primarily interact with the server by sending two types of messages. **call** messages expect a reply from the server (and are therefore synchronous) while **cast** messages do not. Every time you do a `GenServer.call/3`, the client will send a message - that must be handled by the `handle_call/3` callback in the GenServer. - A `cast/2` message must be handled by `handle_cast/2`. + that must be handled by the `c:handle_call/3` callback in the GenServer. + A `cast/2` message must be handled by `c:handle_cast/2`. There are 8 possible + callbacks to be implemented when you use a `GenServer`. The only required + callback is `c:init/1`. + + ## Client / Server APIs + + Although in the example above we have used `GenServer.start_link/3` and + friends to directly start and communicate with the server, most of the + time we don't call the `GenServer` functions directly. Instead, we wrap + the calls in new functions representing the public API of the server. + + Here is a better implementation of our Stack module: + + defmodule Stack do + use GenServer + + # Client + + def start_link(default) when is_list(default) do + GenServer.start_link(__MODULE__, default) + end + + def push(pid, element) do + GenServer.cast(pid, {:push, element}) + end + + def pop(pid) do + GenServer.call(pid, :pop) + end + + # Server (callbacks) + + @impl true + def init(stack) do + {:ok, stack} + end + + @impl true + def handle_call(:pop, _from, [head | tail]) do + {:reply, head, tail} + end + + @impl true + def handle_cast({:push, element}, state) do + {:noreply, [element | state]} + end + end - ## Callbacks + In practice, it is common to have both server and client functions in + the same module. If the server and/or client implementations are growing + complex, you may want to have them in different modules. - There are 6 callbacks required to be implemented in a `GenServer`. By - adding `use GenServer` to your module, Elixir will automatically define - all 6 callbacks for you, leaving it up to you to implement the ones - you want to customize. The callbacks are: + ## How to supervise - * `init(args)` - invoked when the server is started. + A `GenServer` is most commonly started under a supervision tree. + When we invoke `use GenServer`, it automatically defines a `child_spec/1` + function that allows us to start the `Stack` directly under a supervisor. + To start a default stack of `[:hello]` under a supervisor, one may do: - It must return: + children = [ + {Stack, [:hello]} + ] - - `{:ok, state}` - - `{:ok, state, timeout}` - - `:ignore` - - `{:stop, reason}` + Supervisor.start_link(children, strategy: :one_for_all) - * `handle_call(msg, {from, ref}, state)` and `handle_cast(msg, state)` - - invoked to handle call (sync) and cast (async) messages. + Note you can also start it simply as `Stack`, which is the same as + `{Stack, []}`: - It must return: + children = [ + Stack # The same as {Stack, []} + ] - - `{:reply, reply, new_state}` - - `{:reply, reply, new_state, timeout}` - - `{:reply, reply, new_state, :hibernate}` - - `{:noreply, new_state}` - - `{:noreply, new_state, timeout}` - - `{:noreply, new_state, :hibernate}` - - `{:stop, reason, new_state}` - - `{:stop, reason, reply, new_state}` + Supervisor.start_link(children, strategy: :one_for_all) - * `handle_info(msg, state)` - invoked to handle all other messages which - are received by the process. + In both cases, `Stack.start_link/1` is always invoked. - It must return: + `use GenServer` also accepts a list of options which configures the + child specification and therefore how it runs under a supervisor. + The generated `child_spec/1` can be customized with the following options: - - `{:noreply, state}` - - `{:noreply, state, timeout}` - - `{:stop, reason, state}` + * `:id` - the child specification identifier, defaults to the current module + * `:restart` - when the child should be restarted, defaults to `:permanent` + * `:shutdown` - how to shut down the child, either immediately or by giving it time to shut down - * `terminate(reason, state)` - called when the server is about to - terminate, useful for cleaning up. It must return `:ok`. + For example: - * `code_change(old_vsn, state, extra)` - called when the application - code is being upgraded live (hot code swapping). + use GenServer, restart: :transient, shutdown: 10_000 - It must return: + See the "Child specification" section in the `Supervisor` module for more + detailed information. The `@doc` annotation immediately preceding + `use GenServer` will be attached to the generated `child_spec/1` function. - - `{:ok, new_state}` - - `{:error, reason}` + When stopping the GenServer, for example by returning a `{:stop, reason, new_state}` + tuple from a callback, the exit reason is used by the supervisor to determine + whether the GenServer needs to be restarted. See the "Exit reasons and restarts" + section in the `Supervisor` module. - ## Name Registration + ## Name registration Both `start_link/3` and `start/3` support the `GenServer` to register a name on start via the `:name` option. Registered names are also automatically cleaned up on termination. The supported values are: - * an atom - the GenServer is registered locally with the given name - using `Process.register/2`. + * an atom - the GenServer is registered locally (to the current node) + with the given name using `Process.register/2`. - * `{:global, term}`- the GenServer is registered globally with the given - term using the functions in the `:global` module. + * `{:global, term}` - the GenServer is registered globally with the given + term using the functions in the [`:global` module](`:global`). * `{:via, module, term}` - the GenServer is registered with the given - mechanism and name. The `:via` option expects a module name to control - the registration mechanism alongside a name which can be any term. + mechanism and name. The `:via` option expects a module that exports + `register_name/2`, `unregister_name/1`, `whereis_name/1` and `send/2`. + One such example is the [`:global` module](`:global`) which uses these functions + for keeping the list of names of processes and their associated PIDs + that are available globally for a network of Elixir nodes. Elixir also + ships with a local, decentralized and scalable registry called `Registry` + for locally storing names that are generated dynamically. - For example, we could start and register our Stack server locally as follows: + For example, we could start and register our `Stack` server locally as follows: # Start the server and register it locally with name MyStack {:ok, _} = GenServer.start_link(Stack, [:hello], name: MyStack) # Now messages can be sent directly to MyStack - GenServer.call(MyStack, :pop) #=> :hello + GenServer.call(MyStack, :pop) + #=> :hello Once the server is started, the remaining functions in this module (`call/3`, - `cast/2`, and friends) will also accept an atom, or any `:global` or `:via` - tuples. In general, the following formats are supported: + `cast/2`, and friends) will also accept an atom, or any `{:global, ...}` or + `{:via, ...}` tuples. In general, the following formats are supported: - * a `pid` - * an `atom` if the server is locally registered + * a PID + * an atom if the server is locally registered * `{atom, node}` if the server is locally registered at another node * `{:global, term}` if the server is globally registered * `{:via, module, name}` if the server is registered through an alternative registry - ## Client / Server APIs + If there is an interest to register dynamic names locally, do not use + atoms, as atoms are never garbage-collected and therefore dynamically + generated atoms won't be garbage-collected. For such cases, you can + set up your own local registry by using the `Registry` module. - Although in the example above we have used `GenServer.start_link/3` and - friends to directly start and communicate with the server, most of the - time we don't call the `GenServer` functions directly. Instead, we wrap - the calls in new functions representing the public API of the server. + ## Receiving "regular" messages - Here is a better implementation of our Stack module: + The goal of a `GenServer` is to abstract the "receive" loop for developers, + automatically handling system messages, supporting code change, synchronous + calls and more. Therefore, you should never call your own "receive" inside + the GenServer callbacks as doing so will cause the GenServer to misbehave. - defmodule Stack do - use GenServer + Besides the synchronous and asynchronous communication provided by `call/3` + and `cast/2`, "regular" messages sent by functions such as `send/2`, + `Process.send_after/4` and similar, can be handled inside the `c:handle_info/2` + callback. - # Client + `c:handle_info/2` can be used in many situations, such as handling monitor + DOWN messages sent by `Process.monitor/1`. Another use case for `c:handle_info/2` + is to perform periodic work, with the help of `Process.send_after/4`: - def start_link(default) do - GenServer.start_link(__MODULE__, default) - end + defmodule MyApp.Periodically do + use GenServer - def push(pid, item) do - GenServer.cast(pid, {:push, item}) + def start_link(_) do + GenServer.start_link(__MODULE__, %{}) end - def pop(pid) do - GenServer.call(pid, :pop) + @impl true + def init(state) do + # Schedule work to be performed on start + schedule_work() + + {:ok, state} end - # Server (callbacks) + @impl true + def handle_info(:work, state) do + # Do the desired work here + # ... - def handle_call(:pop, _from, [h|t]) do - {:reply, h, t} - end + # Reschedule once more + schedule_work() - def handle_call(request, from, state) do - # Call the default implementation from GenServer - super(request, from, state) + {:noreply, state} end - def handle_cast({:push, item}, state) do - {:noreply, [item|state]} + defp schedule_work do + # We schedule the work to happen in 2 hours (written in milliseconds). + # Alternatively, one might write :timer.hours(2) + Process.send_after(self(), :work, 2 * 60 * 60 * 1000) end + end - def handle_cast(request, state) do - super(request, state) - end + ## Timeouts + + The return value of `c:init/1` or any of the `handle_*` callbacks may include + a timeout value in milliseconds; if not, `:infinity` is assumed. + The timeout can be used to detect a lull in incoming messages. + + The `timeout()` value is used as follows: + + * If the process has any message already waiting when the `timeout()` value + is returned, the timeout is ignored and the waiting message is handled as + usual. This means that even a timeout of `0` milliseconds is not guaranteed + to execute (if you want to take another action immediately and unconditionally, + use a `:continue` instruction instead). + + * If any message arrives before the specified number of milliseconds + elapse, the timeout is cleared and that message is handled as usual. + + * Otherwise, when the specified number of milliseconds have elapsed with no + message arriving, `handle_info/2` is called with `:timeout` as the first + argument. + + ## When (not) to use a GenServer + + So far, we have learned that a `GenServer` can be used as a supervised process + that handles sync and async calls. It can also handle system messages, such as + periodic messages and monitoring events. GenServer processes may also be named. + + A GenServer, or a process in general, must be used to model runtime characteristics + of your system. A GenServer must never be used for code organization purposes. + + In Elixir, code organization is done by modules and functions, processes are not + necessary. For example, imagine you are implementing a calculator and you decide + to put all the calculator operations behind a GenServer: + + def add(a, b) do + GenServer.call(__MODULE__, {:add, a, b}) end - In practice, it is common to have both server and client functions in - the same module. If the server and/or client implementations are growing - complex, you may want to have them in different modules. + def subtract(a, b) do + GenServer.call(__MODULE__, {:subtract, a, b}) + end + + def handle_call({:add, a, b}, _from, state) do + {:reply, a + b, state} + end + + def handle_call({:subtract, a, b}, _from, state) do + {:reply, a - b, state} + end + + This is an anti-pattern not only because it convolutes the calculator logic but + also because you put the calculator logic behind a single process that will + potentially become a bottleneck in your system, especially as the number of + calls grow. Instead just define the functions directly: + + def add(a, b) do + a + b + end + + def subtract(a, b) do + a - b + end + + If you don't need a process, then you don't need a process. Use processes only to + model runtime properties, such as mutable state, concurrency and failures, never + for code organization. + + ## Debugging with the :sys module + + GenServers, as [special processes](https://www.erlang.org/doc/design_principles/spec_proc.html), + can be debugged using the [`:sys` module](`:sys`). + Through various hooks, this module allows developers to introspect the state of + the process and trace system events that happen during its execution, such as + received messages, sent replies and state changes. + + Let's explore the basic functions from the + [`:sys` module](`:sys`) used for debugging: + + * `:sys.get_state/2` - allows retrieval of the state of the process. + In the case of a GenServer process, it will be the callback module state, + as passed into the callback functions as last argument. + * `:sys.get_status/2` - allows retrieval of the status of the process. + This status includes the process dictionary, if the process is running + or is suspended, the parent PID, the debugger state, and the state of + the behaviour module, which includes the callback module state + (as returned by `:sys.get_state/2`). It's possible to change how this + status is represented by defining the optional `c:GenServer.format_status/2` + callback. + * `:sys.trace/3` - prints all the system events to `:stdio`. + * `:sys.statistics/3` - manages collection of process statistics. + * `:sys.no_debug/2` - turns off all debug handlers for the given process. + It is very important to switch off debugging once we're done. Excessive + debug handlers or those that should be turned off, but weren't, can + seriously damage the performance of the system. + * `:sys.suspend/2` - allows to suspend a process so that it only + replies to system messages but no other messages. A suspended process + can be reactivated via `:sys.resume/2`. + + Let's see how we could use those functions for debugging the stack server + we defined earlier. + + iex> {:ok, pid} = Stack.start_link([]) + iex> :sys.statistics(pid, true) # turn on collecting process statistics + iex> :sys.trace(pid, true) # turn on event printing + iex> Stack.push(pid, 1) + *DBG* <0.122.0> got cast {push,1} + *DBG* <0.122.0> new state [1] + :ok + + iex> :sys.get_state(pid) + [1] + + iex> Stack.pop(pid) + *DBG* <0.122.0> got call pop from <0.80.0> + *DBG* <0.122.0> sent 1 to <0.80.0>, new state [] + 1 + + iex> :sys.statistics(pid, :get) + {:ok, + [ + start_time: {{2016, 7, 16}, {12, 29, 41}}, + current_time: {{2016, 7, 16}, {12, 29, 50}}, + reductions: 117, + messages_in: 2, + messages_out: 0 + ]} + + iex> :sys.no_debug(pid) # turn off all debug handlers + :ok + + iex> :sys.get_status(pid) + {:status, #PID<0.122.0>, {:module, :gen_server}, + [ + [ + "$initial_call": {Stack, :init, 1}, # process dictionary + "$ancestors": [#PID<0.80.0>, #PID<0.51.0>] + ], + :running, # :running | :suspended + #PID<0.80.0>, # parent + [], # debugger state + [ + header: 'Status for generic server <0.122.0>', # module status + data: [ + {'Status', :running}, + {'Parent', #PID<0.80.0>}, + {'Logged events', []} + ], + data: [{'State', [1]}] + ] + ]} ## Learn more - If you wish to find out more about gen servers, Elixir getting started - guides provide a tutorial-like introduction. The documentation and links + If you wish to find out more about GenServers, the Elixir Getting Started + guide provides a tutorial-like introduction. The documentation and links in Erlang can also provide extra insight. - * http://elixir-lang.org/getting_started/mix/1.html - * http://www.erlang.org/doc/man/gen_server.html - * http://www.erlang.org/doc/design_principles/gen_server_concepts.html - * http://learnyousomeerlang.com/clients-and-servers + * [GenServer - Elixir's Getting Started Guide](https://elixir-lang.org/getting-started/mix-otp/genserver.html) + * [`:gen_server` module documentation](`:gen_server`) + * [gen_server Behaviour - OTP Design Principles](https://www.erlang.org/doc/design_principles/gen_server_concepts.html) + * [Clients and Servers - Learn You Some Erlang for Great Good!](http://learnyousomeerlang.com/clients-and-servers) + + """ + + @doc """ + Invoked when the server is started. `start_link/3` or `start/3` will + block until it returns. + + `init_arg` is the argument term (second argument) passed to `start_link/3`. + + Returning `{:ok, state}` will cause `start_link/3` to return + `{:ok, pid}` and the process to enter its loop. + + Returning `{:ok, state, timeout}` is similar to `{:ok, state}`, + except that it also sets a timeout. See the "Timeouts" section + in the module documentation for more information. + + Returning `{:ok, state, :hibernate}` is similar to `{:ok, state}` + except the process is hibernated before entering the loop. See + `c:handle_call/3` for more information on hibernation. + + Returning `{:ok, state, {:continue, continue_arg}}` is similar to + `{:ok, state}` except that immediately after entering the loop, + the `c:handle_continue/2` callback will be invoked with `continue_arg` + as the first argument and `state` as the second one. + + Returning `:ignore` will cause `start_link/3` to return `:ignore` and + the process will exit normally without entering the loop or calling + `c:terminate/2`. If used when part of a supervision tree the parent + supervisor will not fail to start nor immediately try to restart the + `GenServer`. The remainder of the supervision tree will be started + and so the `GenServer` should not be required by other processes. + It can be started later with `Supervisor.restart_child/2` as the child + specification is saved in the parent supervisor. The main use cases for + this are: + + * The `GenServer` is disabled by configuration but might be enabled later. + * An error occurred and it will be handled by a different mechanism than the + `Supervisor`. Likely this approach involves calling `Supervisor.restart_child/2` + after a delay to attempt a restart. + + Returning `{:stop, reason}` will cause `start_link/3` to return + `{:error, reason}` and the process to exit with reason `reason` without + entering the loop or calling `c:terminate/2`. + """ + @callback init(init_arg :: term) :: + {:ok, state} + | {:ok, state, timeout | :hibernate | {:continue, continue_arg :: term}} + | :ignore + | {:stop, reason :: any} + when state: any + + @doc """ + Invoked to handle synchronous `call/3` messages. `call/3` will block until a + reply is received (unless the call times out or nodes are disconnected). + + `request` is the request message sent by a `call/3`, `from` is a 2-tuple + containing the caller's PID and a term that uniquely identifies the call, and + `state` is the current state of the `GenServer`. + + Returning `{:reply, reply, new_state}` sends the response `reply` to the + caller and continues the loop with new state `new_state`. + + Returning `{:reply, reply, new_state, timeout}` is similar to + `{:reply, reply, new_state}` except that it also sets a timeout. + See the "Timeouts" section in the module documentation for more information. + + Returning `{:reply, reply, new_state, :hibernate}` is similar to + `{:reply, reply, new_state}` except the process is hibernated and will + continue the loop once a message is in its message queue. However, if a message is + already in the message queue, the process will continue the loop immediately. Hibernating a + `GenServer` causes garbage collection and leaves a continuous heap that + minimises the memory used by the process. + + Returning `{:reply, reply, new_state, {:continue, continue_arg}}` is similar to + `{:reply, reply, new_state}` except that `c:handle_continue/2` will be invoked + immediately after with `continue_arg` as the first argument and + `state` as the second one. + + Hibernating should not be used aggressively as too much time could be spent + garbage collecting. Normally it should only be used when a message is not + expected soon and minimising the memory of the process is shown to be + beneficial. + + Returning `{:noreply, new_state}` does not send a response to the caller and + continues the loop with new state `new_state`. The response must be sent with + `reply/2`. + + There are three main use cases for not replying using the return value: + + * To reply before returning from the callback because the response is known + before calling a slow function. + * To reply after returning from the callback because the response is not yet + available. + * To reply from another process, such as a task. + + When replying from another process the `GenServer` should exit if the other + process exits without replying as the caller will be blocking awaiting a + reply. + + Returning `{:noreply, new_state, timeout | :hibernate | {:continue, continue_arg}}` + is similar to `{:noreply, new_state}` except a timeout, hibernation or continue + occurs as with a `:reply` tuple. + + Returning `{:stop, reason, reply, new_state}` stops the loop and `c:terminate/2` + is called with reason `reason` and state `new_state`. Then, the `reply` is sent + as the response to call and the process exits with reason `reason`. + + Returning `{:stop, reason, new_state}` is similar to + `{:stop, reason, reply, new_state}` except a reply is not sent. + + This callback is optional. If one is not implemented, the server will fail + if a call is performed against it. + """ + @callback handle_call(request :: term, from, state :: term) :: + {:reply, reply, new_state} + | {:reply, reply, new_state, + timeout | :hibernate | {:continue, continue_arg :: term}} + | {:noreply, new_state} + | {:noreply, new_state, timeout | :hibernate | {:continue, continue_arg :: term}} + | {:stop, reason, reply, new_state} + | {:stop, reason, new_state} + when reply: term, new_state: term, reason: term + + @doc """ + Invoked to handle asynchronous `cast/2` messages. + + `request` is the request message sent by a `cast/2` and `state` is the current + state of the `GenServer`. + + Returning `{:noreply, new_state}` continues the loop with new state `new_state`. + + Returning `{:noreply, new_state, timeout}` is similar to `{:noreply, new_state}` + except that it also sets a timeout. See the "Timeouts" section in the module + documentation for more information. + + Returning `{:noreply, new_state, :hibernate}` is similar to + `{:noreply, new_state}` except the process is hibernated before continuing the + loop. See `c:handle_call/3` for more information. + + Returning `{:noreply, new_state, {:continue, continue_arg}}` is similar to + `{:noreply, new_state}` except `c:handle_continue/2` will be invoked + immediately after with `continue_arg` as the first argument and + `state` as the second one. + + Returning `{:stop, reason, new_state}` stops the loop and `c:terminate/2` is + called with the reason `reason` and state `new_state`. The process exits with + reason `reason`. + + This callback is optional. If one is not implemented, the server will fail + if a cast is performed against it. + """ + @callback handle_cast(request :: term, state :: term) :: + {:noreply, new_state} + | {:noreply, new_state, timeout | :hibernate | {:continue, continue_arg :: term}} + | {:stop, reason :: term, new_state} + when new_state: term + + @doc """ + Invoked to handle all other messages. + + `msg` is the message and `state` is the current state of the `GenServer`. When + a timeout occurs the message is `:timeout`. + + Return values are the same as `c:handle_cast/2`. + + This callback is optional. If one is not implemented, the received message + will be logged. + """ + @callback handle_info(msg :: :timeout | term, state :: term) :: + {:noreply, new_state} + | {:noreply, new_state, timeout | :hibernate | {:continue, continue_arg :: term}} + | {:stop, reason :: term, new_state} + when new_state: term + + @doc """ + Invoked to handle continue instructions. + + It is useful for performing work after initialization or for splitting the work + in a callback in multiple steps, updating the process state along the way. + + Return values are the same as `c:handle_cast/2`. + + This callback is optional. If one is not implemented, the server will fail + if a continue instruction is used. + """ + @callback handle_continue(continue_arg, state :: term) :: + {:noreply, new_state} + | {:noreply, new_state, timeout | :hibernate | {:continue, continue_arg}} + | {:stop, reason :: term, new_state} + when new_state: term, continue_arg: term + + @doc """ + Invoked when the server is about to exit. It should do any cleanup required. + + `reason` is exit reason and `state` is the current state of the `GenServer`. + The return value is ignored. + + `c:terminate/2` is called if the `GenServer` traps exits (using `Process.flag/2`) + *and* the parent process sends an exit signal, or a callback (except `c:init/1`) + does one of the following: + + * returns a `:stop` tuple + * raises (via `raise/2`) or exits (via `exit/1`) + * returns an invalid value + + If part of a supervision tree, a `GenServer` will receive an exit + signal when the tree is shutting down. The exit signal is based on + the shutdown strategy in the child's specification, where this + value can be: + + * `:brutal_kill`: the `GenServer` is killed and so `c:terminate/2` is not called. + + * a timeout value, where the supervisor will send the exit signal `:shutdown` and + the `GenServer` will have the duration of the timeout to terminate. + If after duration of this timeout the process is still alive, it will be killed + immediately. + + For a more in-depth explanation, please read the "Shutdown values (:shutdown)" + section in the `Supervisor` module. + + If the `GenServer` receives an exit signal (that is not `:normal`) from any + process when it is not trapping exits it will exit abruptly with the same + reason and so not call `c:terminate/2`. Note that a process does *NOT* trap + exits by default and an exit signal is sent when a linked process exits or its + node is disconnected. + + Therefore it is not guaranteed that `c:terminate/2` is called when a `GenServer` + exits. For such reasons, we usually recommend important clean-up rules to + happen in separated processes either by use of monitoring or by links + themselves. There is no cleanup needed when the `GenServer` controls a `port` (for example, + `:gen_tcp.socket`) or `t:File.io_device/0`, because these will be closed on + receiving a `GenServer`'s exit signal and do not need to be closed manually + in `c:terminate/2`. + + If `reason` is neither `:normal`, `:shutdown`, nor `{:shutdown, term}` an error is + logged. + + This callback is optional. + """ + @callback terminate(reason, state :: term) :: term + when reason: :normal | :shutdown | {:shutdown, term} | term + + @doc """ + Invoked to change the state of the `GenServer` when a different version of a + module is loaded (hot code swapping) and the state's term structure should be + changed. + + `old_vsn` is the previous version of the module (defined by the `@vsn` + attribute) when upgrading. When downgrading the previous version is wrapped in + a 2-tuple with first element `:down`. `state` is the current state of the + `GenServer` and `extra` is any extra data required to change the state. + + Returning `{:ok, new_state}` changes the state to `new_state` and the code + change is successful. + + Returning `{:error, reason}` fails the code change with reason `reason` and + the state remains as the previous state. + + If `c:code_change/3` raises the code change fails and the loop will continue + with its previous state. Therefore this callback does not usually contain side effects. + + This callback is optional. + """ + @callback code_change(old_vsn, state :: term, extra :: term) :: + {:ok, new_state :: term} + | {:error, reason :: term} + when old_vsn: term | {:down, term} + + @doc """ + Invoked in some cases to retrieve a formatted version of the `GenServer` status. + + This callback can be useful to control the *appearance* of the status of the + `GenServer`. For example, it can be used to return a compact representation of + the `GenServer`'s state to avoid having large state terms printed. + + * one of `:sys.get_status/1` or `:sys.get_status/2` is invoked to get the + status of the `GenServer`; in such cases, `reason` is `:normal` + + * the `GenServer` terminates abnormally and logs an error; in such cases, + `reason` is `:terminate` + + `pdict_and_state` is a two-elements list `[pdict, state]` where `pdict` is a + list of `{key, value}` tuples representing the current process dictionary of + the `GenServer` and `state` is the current state of the `GenServer`. """ + @callback format_status(reason, pdict_and_state :: list) :: term + when reason: :normal | :terminate + + @optional_callbacks code_change: 3, + terminate: 2, + handle_info: 2, + handle_cast: 2, + handle_call: 3, + format_status: 2, + handle_continue: 2 @typedoc "Return values of `start*` functions" @type on_start :: {:ok, pid} | :ignore | {:error, {:already_started, pid} | term} @@ -210,40 +717,123 @@ defmodule GenServer do @type name :: atom | {:global, term} | {:via, module, term} @typedoc "Options used by the `start*` functions" - @type options :: [debug: debug, - name: name, - timeout: timeout, - spawn_opt: Process.spawn_opt] + @type options :: [option] + + @typedoc "Option values used by the `start*` functions" + @type option :: + {:debug, debug} + | {:name, name} + | {:timeout, timeout} + | {:spawn_opt, [Process.spawn_opt()]} + | {:hibernate_after, timeout} + + @typedoc "Debug options supported by the `start*` functions" + @type debug :: [:trace | :log | :statistics | {:log_to_file, Path.t()}] - @typedoc "debug options supported by the `start*` functions" - @type debug :: [:trace | :log | :statistics | {:log_to_file, Path.t}] + @typedoc """ + The server reference. - @typedoc "The server reference" + This is either a plain PID or a value representing a registered name. + See the "Name registration" section of this document for more information. + """ @type server :: pid | name | {atom, node} + @typedoc """ + Tuple describing the client of a call request. + + `pid` is the PID of the caller and `tag` is a unique term used to identify the + call. + """ + @type from :: {pid, tag :: term} + @doc false - defmacro __using__(_) do - quote location: :keep do - @behaviour :gen_server + defmacro __using__(opts) do + quote location: :keep, bind_quoted: [opts: opts] do + @behaviour GenServer - @doc false - def init(args) do - {:ok, args} + unless Module.has_attribute?(__MODULE__, :doc) do + @doc """ + Returns a specification to start this module under a supervisor. + + See `Supervisor`. + """ + end + + def child_spec(init_arg) do + default = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [init_arg]} + } + + Supervisor.child_spec(default, unquote(Macro.escape(opts))) end + defoverridable child_spec: 1 + + # TODO: Remove this on v2.0 + @before_compile GenServer + @doc false def handle_call(msg, _from, state) do - {:stop, {:bad_call, msg}, state} + proc = + case Process.info(self(), :registered_name) do + {_, []} -> self() + {_, name} -> name + end + + # We do this to trick Dialyzer to not complain about non-local returns. + case :erlang.phash2(1, 1) do + 0 -> + raise "attempted to call GenServer #{inspect(proc)} but no handle_call/3 clause was provided" + + 1 -> + {:stop, {:bad_call, msg}, state} + end end @doc false - def handle_info(_msg, state) do + def handle_info(msg, state) do + proc = + case Process.info(self(), :registered_name) do + {_, []} -> self() + {_, name} -> name + end + + :logger.error( + %{ + label: {GenServer, :no_handle_info}, + report: %{ + module: __MODULE__, + message: msg, + name: proc + } + }, + %{ + domain: [:otp, :elixir], + error_logger: %{tag: :error_msg}, + report_cb: &GenServer.format_report/1 + } + ) + {:noreply, state} end @doc false def handle_cast(msg, state) do - {:stop, {:bad_cast, msg}, state} + proc = + case Process.info(self(), :registered_name) do + {_, []} -> self() + {_, name} -> name + end + + # We do this to trick Dialyzer to not complain about non-local returns. + case :erlang.phash2(1, 1) do + 0 -> + raise "attempted to cast GenServer #{inspect(proc)} but no handle_cast/2 clause was provided" + + 1 -> + {:stop, {:bad_cast, msg}, state} + end end @doc false @@ -256,8 +846,36 @@ defmodule GenServer do {:ok, state} end - defoverridable [init: 1, handle_call: 3, handle_info: 2, - handle_cast: 2, terminate: 2, code_change: 3] + defoverridable code_change: 3, terminate: 2, handle_info: 2, handle_cast: 2, handle_call: 3 + end + end + + defmacro __before_compile__(env) do + unless Module.defines?(env.module, {:init, 1}) do + message = """ + function init/1 required by behaviour GenServer is not implemented \ + (in module #{inspect(env.module)}). + + We will inject a default implementation for now: + + def init(init_arg) do + {:ok, init_arg} + end + + You can copy the implementation above or define your own that converts \ + the arguments given to GenServer.start_link/3 to the server state. + """ + + IO.warn(message, env) + + quote do + @doc false + def init(init_arg) do + {:ok, init_arg} + end + + defoverridable init: 1 + end end end @@ -266,43 +884,49 @@ defmodule GenServer do This is often used to start the `GenServer` as part of a supervision tree. - Once the server is started, it calls the `init/1` function in the given `module` - passing the given `args` to initialize it. To ensure a synchronized start-up - procedure, this function does not return until `init/1` has returned. + Once the server is started, the `c:init/1` function of the given `module` is + called with `init_arg` as its argument to initialize the server. To ensure a + synchronized start-up procedure, this function does not return until `c:init/1` + has returned. Note that a `GenServer` started with `start_link/3` is linked to the - parent process and will exit in case of crashes. The GenServer will also - exit due to the `:normal` reasons in case it is configured to trap exits - in the `init/1` callback. + parent process and will exit in case of crashes from the parent. The GenServer + will also exit due to the `:normal` reasons in case it is configured to trap + exits in the `c:init/1` callback. ## Options - The `:name` option is used for name registration as described in the module - documentation. If the option `:timeout` option is present, the server is - allowed to spend the given milliseconds initializing or it will be - terminated and the start function will return `{:error, :timeout}`. + * `:name` - used for name registration as described in the "Name + registration" section in the documentation for `GenServer` + + * `:timeout` - if present, the server is allowed to spend the given number of + milliseconds initializing or it will be terminated and the start function + will return `{:error, :timeout}` - If the `:debug` option is present, the corresponding function in the - [`:sys` module](http://www.erlang.org/doc/man/sys.html) will be invoked. + * `:debug` - if present, the corresponding function in the [`:sys` module](`:sys`) is invoked - If the `:spawn_opt` option is present, its value will be passed as options - to the underlying process as in `Process.spawn/4`. + * `:spawn_opt` - if present, its value is passed as options to the + underlying process as in `Process.spawn/4` + + * `:hibernate_after` - if present, the GenServer process awaits any message for + the given number of milliseconds and if no message is received, the process goes + into hibernation automatically (by calling `:proc_lib.hibernate/3`). ## Return values - If the server is successfully created and initialized, the function returns - `{:ok, pid}`, where pid is the pid of the server. If there already exists a - process with the specified server name, the function returns - `{:error, {:already_started, pid}}` with the pid of that process. + If the server is successfully created and initialized, this function returns + `{:ok, pid}`, where `pid` is the PID of the server. If a process with the + specified server name already exists, this function returns + `{:error, {:already_started, pid}}` with the PID of that process. - If the `init/1` callback fails with `reason`, the function returns + If the `c:init/1` callback fails with `reason`, this function returns `{:error, reason}`. Otherwise, if it returns `{:stop, reason}` - or `:ignore`, the process is terminated and the function returns + or `:ignore`, the process is terminated and this function returns `{:error, reason}` or `:ignore`, respectively. """ @spec start_link(module, any, options) :: on_start - def start_link(module, args, options \\ []) when is_atom(module) and is_list(options) do - do_start(:link, module, args, options) + def start_link(module, init_arg, options \\ []) when is_atom(module) and is_list(options) do + do_start(:link, module, init_arg, options) end @doc """ @@ -311,18 +935,65 @@ defmodule GenServer do See `start_link/3` for more information. """ @spec start(module, any, options) :: on_start - def start(module, args, options \\ []) when is_atom(module) and is_list(options) do - do_start(:nolink, module, args, options) + def start(module, init_arg, options \\ []) when is_atom(module) and is_list(options) do + do_start(:nolink, module, init_arg, options) end - defp do_start(link, module, args, options) do + defp do_start(link, module, init_arg, options) do case Keyword.pop(options, :name) do {nil, opts} -> - :gen.start(:gen_server, link, module, args, opts) + :gen.start(:gen_server, link, module, init_arg, opts) + {atom, opts} when is_atom(atom) -> - :gen.start(:gen_server, link, {:local, atom}, module, args, opts) - {other, opts} when is_tuple(other) -> - :gen.start(:gen_server, link, other, module, args, opts) + :gen.start(:gen_server, link, {:local, atom}, module, init_arg, opts) + + {{:global, _term} = tuple, opts} -> + :gen.start(:gen_server, link, tuple, module, init_arg, opts) + + {{:via, via_module, _term} = tuple, opts} when is_atom(via_module) -> + :gen.start(:gen_server, link, tuple, module, init_arg, opts) + + {other, _} -> + raise ArgumentError, """ + expected :name option to be one of the following: + + * nil + * atom + * {:global, term} + * {:via, module, term} + + Got: #{inspect(other)} + """ + end + end + + @doc """ + Synchronously stops the server with the given `reason`. + + The `c:terminate/2` callback of the given `server` will be invoked before + exiting. This function returns `:ok` if the server terminates with the + given reason; if it terminates with another reason, the call exits. + + This function keeps OTP semantics regarding error reporting. + If the reason is any other than `:normal`, `:shutdown` or + `{:shutdown, _}`, an error report is logged. + """ + @spec stop(server, reason :: term, timeout) :: :ok + def stop(server, reason \\ :normal, timeout \\ :infinity) do + case whereis(server) do + nil -> + exit({:noproc, {__MODULE__, :stop, [server, reason, timeout]}}) + + pid when pid == self() -> + exit({:calling_self, {__MODULE__, :stop, [server, reason, timeout]}}) + + pid -> + try do + :proc_lib.stop(pid, reason, timeout) + catch + :exit, err -> + exit({err, {__MODULE__, :stop, [server, reason, timeout]}}) + end end end @@ -330,101 +1001,238 @@ defmodule GenServer do Makes a synchronous call to the `server` and waits for its reply. The client sends the given `request` to the server and waits until a reply - arrives or a timeout occurs. `handle_call/3` will be called on the server + arrives or a timeout occurs. `c:handle_call/3` will be called on the server to handle the request. - The server can be any of the values described in the `Name Registration` - section of the module documentation. + `server` can be any of the values described in the "Name registration" + section of the documentation for this module. ## Timeouts - The `timeout` is an integer greater than zero which specifies how many + `timeout` is an integer greater than zero which specifies how many milliseconds to wait for a reply, or the atom `:infinity` to wait - indefinitely. The default value is 5000. If no reply is received within - the specified time, the function call fails. If the caller catches the - failure and continues running, and the server is just late with the reply, - it may arrive at any time later into the caller's message queue. The caller - must in this case be prepared for this and discard any such garbage messages - that are two element tuples with a reference as the first element. + indefinitely. The default value is `5000`. If no reply is received within + the specified time, the function call fails and the caller exits. If the + caller catches the failure and continues running, and the server is just late + with the reply, it may arrive at any time later into the caller's message + queue. The caller must in this case be prepared for this and discard any such + garbage messages that are two-element tuples with a reference as the first + element. """ @spec call(server, term, timeout) :: term - def call(server, request, timeout \\ 5000) do - :gen_server.call(server, request, timeout) + def call(server, request, timeout \\ 5000) + when (is_integer(timeout) and timeout >= 0) or timeout == :infinity do + case whereis(server) do + nil -> + exit({:noproc, {__MODULE__, :call, [server, request, timeout]}}) + + # TODO: remove this clause when we require Erlang/OTP 25+ + pid when pid == self() -> + exit({:calling_self, {__MODULE__, :call, [server, request, timeout]}}) + + pid -> + try do + :gen.call(pid, :"$gen_call", request, timeout) + catch + :exit, reason -> + exit({reason, {__MODULE__, :call, [server, request, timeout]}}) + else + {:ok, res} -> res + end + end end @doc """ Sends an asynchronous request to the `server`. - This function returns `:ok` immediately, regardless of whether the - destination node or server does exists. `handle_cast/2` will be called on the - server to handle the request. + This function always returns `:ok` regardless of whether + the destination `server` (or node) exists. Therefore it + is unknown whether the destination `server` successfully + handled the message. + + `server` can be any of the values described in the "Name registration" + section of the documentation for this module. """ @spec cast(server, term) :: :ok - defdelegate cast(server, request), to: :gen_server + def cast(server, request) + + def cast({:global, name}, request) do + try do + :global.send(name, cast_msg(request)) + :ok + catch + _, _ -> :ok + end + end + + def cast({:via, mod, name}, request) do + try do + mod.send(name, cast_msg(request)) + :ok + catch + _, _ -> :ok + end + end + + def cast({name, node}, request) when is_atom(name) and is_atom(node), + do: do_send({name, node}, cast_msg(request)) + + def cast(dest, request) when is_atom(dest) or is_pid(dest), do: do_send(dest, cast_msg(request)) @doc """ Casts all servers locally registered as `name` at the specified nodes. - The function returns immediately and ignores nodes that do not exist, or where the + This function returns immediately and ignores nodes that do not exist, or where the server name does not exist. See `multi_call/4` for more information. """ @spec abcast([node], name :: atom, term) :: :abcast - def abcast(nodes \\ nodes(), name, request) do - :gen_server.abcast(nodes, name, request) + def abcast(nodes \\ [node() | Node.list()], name, request) + when is_list(nodes) and is_atom(name) do + msg = cast_msg(request) + _ = for node <- nodes, do: do_send({name, node}, msg) + :abcast + end + + defp cast_msg(req) do + {:"$gen_cast", req} + end + + defp do_send(dest, msg) do + try do + send(dest, msg) + :ok + catch + _, _ -> :ok + end end @doc """ Calls all servers locally registered as `name` at the specified `nodes`. - The `request` is first sent to every node and then we wait for the - replies. This function returns a tuple containing the node and its reply - as first element and all bad nodes as second element. The bad nodes is a - list of nodes that either did not exist, or where a server with the given - `name` did not exist or did not reply. + First, the `request` is sent to every node in `nodes`; then, the caller waits + for the replies. This function returns a two-element tuple `{replies, + bad_nodes}` where: + + * `replies` - is a list of `{node, reply}` tuples where `node` is the node + that replied and `reply` is its reply + * `bad_nodes` - is a list of nodes that either did not exist or where a + server with the given `name` did not exist or did not reply - Nodes is a list of node names to which the request is sent. The default - value is the list of all known nodes. + `nodes` is a list of node names to which the request is sent. The default + value is the list of all known nodes (including this node). To avoid that late answers (after the timeout) pollute the caller's message queue, a middleman process is used to do the actual calls. Late answers will then be discarded when they arrive to a terminated process. + + ## Examples + + Assuming the `Stack` GenServer mentioned in the docs for the `GenServer` + module is registered as `Stack` in the `:"foo@my-machine"` and + `:"bar@my-machine"` nodes: + + GenServer.multi_call(Stack, :pop) + #=> {[{:"foo@my-machine", :hello}, {:"bar@my-machine", :world}], []} + """ @spec multi_call([node], name :: atom, term, timeout) :: - {replies :: [{node, term}], bad_nodes :: [node]} - def multi_call(nodes \\ nodes(), name, request, timeout \\ :infinity) do + {replies :: [{node, term}], bad_nodes :: [node]} + def multi_call(nodes \\ [node() | Node.list()], name, request, timeout \\ :infinity) do :gen_server.multi_call(nodes, name, request, timeout) end @doc """ Replies to a client. - This function can be used by a server to explicitly send a reply to a - client that called `call/3` or `multi_call/4`. When the reply cannot be - defined in the return value of `handle_call/3`. + This function can be used to explicitly send a reply to a client that called + `call/3` or `multi_call/4` when the reply cannot be specified in the return + value of `c:handle_call/3`. - The `client` must be the `from` argument (the second argument) received - in `handle_call/3` callbacks. Reply is an arbitrary term which will be - given back to the client as the return value of the call. + `client` must be the `from` argument (the second argument) accepted by + `c:handle_call/3` callbacks. `reply` is an arbitrary term which will be given + back to the client as the return value of the call. + + Note that `reply/2` can be called from any process, not just the GenServer + that originally received the call (as long as that GenServer communicated the + `from` argument somehow). This function always returns `:ok`. + + ## Examples + + def handle_call(:reply_in_one_second, from, state) do + Process.send_after(self(), {:reply, from}, 1_000) + {:noreply, state} + end + + def handle_info({:reply, from}, state) do + GenServer.reply(from, :one_second_has_passed) + {:noreply, state} + end + """ - @spec reply({pid, reference}, term) :: :ok + @spec reply(from, term) :: :ok def reply(client, reply) - def reply({to, tag}, reply) do - try do - send(to, {tag, reply}) - :ok - catch - _, _ -> :ok + def reply({to, tag}, reply) when is_pid(to) do + send(to, {tag, reply}) + :ok + end + + @doc """ + Returns the `pid` or `{name, node}` of a GenServer process, `nil` otherwise. + + To be precise, `nil` is returned whenever a `pid` or `{name, node}` cannot + be returned. Note there is no guarantee the returned `pid` or `{name, node}` + is alive, as a process could terminate immediately after it is looked up. + + ## Examples + + For example, to lookup a server process, monitor it and send a cast to it: + + process = GenServer.whereis(server) + monitor = Process.monitor(process) + GenServer.cast(process, :hello) + + """ + @spec whereis(server) :: pid | {atom, node} | nil + def whereis(server) + + def whereis(pid) when is_pid(pid), do: pid + + def whereis(name) when is_atom(name) do + Process.whereis(name) + end + + def whereis({:global, name}) do + case :global.whereis_name(name) do + pid when is_pid(pid) -> pid + :undefined -> nil end end - @compile {:inline, [nodes: 0]} + def whereis({:via, mod, name}) do + case apply(mod, :whereis_name, [name]) do + pid when is_pid(pid) -> pid + :undefined -> nil + end + end + + def whereis({name, local}) when is_atom(name) and local == node() do + Process.whereis(name) + end - defp nodes do - [node()|:erlang.nodes()] + def whereis({name, node} = server) when is_atom(name) and is_atom(node) do + server + end + + @doc false + def format_report(%{ + label: {GenServer, :no_handle_info}, + report: %{module: mod, message: msg, name: proc} + }) do + {'~p ~p received unexpected message in handle_info/2: ~p~n', [mod, proc, msg]} end end diff --git a/lib/elixir/lib/hash_dict.ex b/lib/elixir/lib/hash_dict.ex index edcb8e86329..ae145c798d0 100644 --- a/lib/elixir/lib/hash_dict.ex +++ b/lib/elixir/lib/hash_dict.ex @@ -1,19 +1,12 @@ defmodule HashDict do @moduledoc """ - A key-value store. - - The `HashDict` is represented internally as a struct, therefore - `%HashDict{}` can be used whenever there is a need to match - on any `HashDict`. Note though the struct fields are private and - must not be accessed directly. Instead, use the functions on this - or in the `Dict` module. - - Implementation-wise, `HashDict` is implemented using tries, which - grows in space as the number of keys grows, working well with both - small and large set of keys. For more information about the - functions and their APIs, please consult the `Dict` module. + Tuple-based HashDict implementation. + + This module is deprecated. Use the `Map` module instead. """ + @moduledoc deprecated: "Use Map instead" + use Dict @node_bitmap 0b111 @@ -21,7 +14,7 @@ defmodule HashDict do @node_size 8 @node_template :erlang.make_tuple(@node_size, []) - @opaque t :: map + @opaque t :: %__MODULE__{size: non_neg_integer, root: term} @doc false defstruct size: 0, root: @node_template @@ -29,58 +22,70 @@ defmodule HashDict do @compile :inline_list_funcs @compile {:inline, key_hash: 1, key_mask: 1, key_shift: 1} + message = "Use maps and the Map module instead" + @doc """ Creates a new empty dict. """ - @spec new :: Dict.t + @spec new :: Dict.t() + @deprecated message def new do %HashDict{} end + @deprecated message def put(%HashDict{root: root, size: size}, key, value) do {root, counter} = do_put(root, key, value, key_hash(key)) %HashDict{root: root, size: size + counter} end + @deprecated message def update!(%HashDict{root: root, size: size} = dict, key, fun) when is_function(fun, 1) do - {root, counter} = do_update(root, key, fn -> raise KeyError, key: key, term: dict end, - fun, key_hash(key)) + {root, counter} = + do_update(root, key, fn -> raise KeyError, key: key, term: dict end, fun, key_hash(key)) + %HashDict{root: root, size: size + counter} end - def update(%HashDict{root: root, size: size}, key, initial, fun) when is_function(fun, 1) do - {root, counter} = do_update(root, key, fn -> initial end, fun, key_hash(key)) + @deprecated message + def update(%HashDict{root: root, size: size}, key, default, fun) when is_function(fun, 1) do + {root, counter} = do_update(root, key, fn -> default end, fun, key_hash(key)) %HashDict{root: root, size: size + counter} end + @deprecated message def fetch(%HashDict{root: root}, key) do do_fetch(root, key, key_hash(key)) end + @deprecated message def delete(dict, key) do case dict_delete(dict, key) do {dict, _value} -> dict - :error -> dict + :error -> dict end end + @deprecated message def pop(dict, key, default \\ nil) do case dict_delete(dict, key) do {dict, value} -> {value, dict} - :error -> {default, dict} + :error -> {default, dict} end end + @deprecated message def size(%HashDict{size: size}) do size end @doc false + @deprecated message def reduce(%HashDict{root: root}, acc, fun) do do_reduce(root, acc, fun, @node_size, fn {:suspend, acc} -> {:suspended, acc, &{:done, elem(&1, 1)}} - {:halt, acc} -> {:halted, acc} - {:cont, acc} -> {:done, acc} + {:halt, acc} -> {:halted, acc} + {:cont, acc} -> {:done, acc} end) end @@ -90,7 +95,7 @@ defmodule HashDict do def dict_delete(%HashDict{root: root, size: size}, key) do case do_delete(root, key, key_hash(key)) do {root, value} -> {%HashDict{root: root, size: size - 1}, value} - :error -> :error + :error -> :error end end @@ -98,86 +103,105 @@ defmodule HashDict do defp do_fetch(node, key, hash) do index = key_mask(hash) + case elem(node, index) do - [^key|v] -> {:ok, v} + [^key | v] -> {:ok, v} {^key, v, _} -> {:ok, v} - {_, _, n} -> do_fetch(n, key, key_shift(hash)) - _ -> :error + {_, _, n} -> do_fetch(n, key, key_shift(hash)) + _ -> :error end end defp do_put(node, key, value, hash) do index = key_mask(hash) + case elem(node, index) do [] -> - {put_elem(node, index, [key|value]), 1} - [^key|_] -> - {put_elem(node, index, [key|value]), 0} - [k|v] -> - n = put_elem(@node_template, key_mask(key_shift(hash)), [key|value]) + {put_elem(node, index, [key | value]), 1} + + [^key | _] -> + {put_elem(node, index, [key | value]), 0} + + [k | v] -> + n = put_elem(@node_template, key_mask(key_shift(hash)), [key | value]) {put_elem(node, index, {k, v, n}), 1} + {^key, _, n} -> {put_elem(node, index, {key, value, n}), 0} + {k, v, n} -> {n, counter} = do_put(n, key, value, key_shift(hash)) {put_elem(node, index, {k, v, n}), counter} end end - defp do_update(node, key, initial, fun, hash) do + defp do_update(node, key, default, fun, hash) do index = key_mask(hash) + case elem(node, index) do [] -> - {put_elem(node, index, [key|initial.()]), 1} - [^key|value] -> - {put_elem(node, index, [key|fun.(value)]), 0} - [k|v] -> - n = put_elem(@node_template, key_mask(key_shift(hash)), [key|initial.()]) + {put_elem(node, index, [key | default.()]), 1} + + [^key | value] -> + {put_elem(node, index, [key | fun.(value)]), 0} + + [k | v] -> + n = put_elem(@node_template, key_mask(key_shift(hash)), [key | default.()]) {put_elem(node, index, {k, v, n}), 1} + {^key, value, n} -> {put_elem(node, index, {key, fun.(value), n}), 0} + {k, v, n} -> - {n, counter} = do_update(n, key, initial, fun, key_shift(hash)) + {n, counter} = do_update(n, key, default, fun, key_shift(hash)) {put_elem(node, index, {k, v, n}), counter} end end defp do_delete(node, key, hash) do index = key_mask(hash) + case elem(node, index) do [] -> :error - [^key|value] -> + + [^key | value] -> {put_elem(node, index, []), value} - [_|_] -> + + [_ | _] -> :error + {^key, value, n} -> {put_elem(node, index, do_compact_node(n)), value} + {k, v, n} -> case do_delete(n, key, key_shift(hash)) do {@node_template, value} -> - {put_elem(node, index, [k|v]), value} + {put_elem(node, index, [k | v]), value} + {n, value} -> {put_elem(node, index, {k, v, n}), value} + :error -> :error end end end - Enum.each 0..(@node_size - 1), fn index -> + Enum.each(0..(@node_size - 1), fn index -> defp do_compact_node(node) when elem(node, unquote(index)) != [] do case elem(node, unquote(index)) do - [k|v] -> + [k | v] -> case put_elem(node, unquote(index), []) do - @node_template -> [k|v] + @node_template -> [k | v] n -> {k, v, n} end + {k, v, n} -> {k, v, put_elem(node, unquote(index), do_compact_node(n))} end end - end + end) ## Dict reduce @@ -193,8 +217,8 @@ defmodule HashDict do next.(acc) end - defp do_reduce_each([k|v], {:cont, acc}, fun, next) do - next.(fun.({k,v}, acc)) + defp do_reduce_each([k | v], {:cont, acc}, fun, next) do + next.(fun.({k, v}, acc)) end defp do_reduce_each({k, v, n}, {:cont, acc}, fun, next) do @@ -202,7 +226,12 @@ defmodule HashDict do end defp do_reduce(node, acc, fun, count, next) when count > 0 do - do_reduce_each(:erlang.element(count, node), acc, fun, &do_reduce(node, &1, fun, count - 1, next)) + do_reduce_each( + :erlang.element(count, node), + acc, + fun, + &do_reduce(node, &1, fun, count - 1, next) + ) end defp do_reduce(_node, acc, _fun, 0, next) do @@ -227,34 +256,45 @@ defmodule HashDict do end defimpl Enumerable, for: HashDict do - def reduce(dict, acc, fun), do: HashDict.reduce(dict, acc, fun) - def member?(dict, {k, v}), do: {:ok, match?({:ok, ^v}, HashDict.fetch(dict, k))} - def member?(_dict, _), do: {:ok, false} - def count(dict), do: {:ok, HashDict.size(dict)} -end + def reduce(dict, acc, fun) do + # Avoid warnings about HashDict being deprecated. + module = HashDict + module.reduce(dict, acc, fun) + end -defimpl Access, for: HashDict do - def get(dict, key) do - HashDict.get(dict, key, nil) + def member?(dict, {key, value}) do + # Avoid warnings about HashDict being deprecated. + module = HashDict + {:ok, match?({:ok, ^value}, module.fetch(dict, key))} end - def get_and_update(dict, key, fun) do - {get, update} = fun.(HashDict.get(dict, key, nil)) - {get, HashDict.put(dict, key, update)} + def member?(_dict, _) do + {:ok, false} end -end -defimpl Collectable, for: HashDict do - def empty(_dict) do - HashDict.new + def count(dict) do + # Avoid warnings about HashDict being deprecated. + module = HashDict + {:ok, module.size(dict)} + end + + def slice(_dict) do + {:error, __MODULE__} end +end +defimpl Collectable, for: HashDict do def into(original) do - {original, fn - dict, {:cont, {k, v}} -> Dict.put(dict, k, v) + # Avoid warnings about HashDict being deprecated. + module = HashDict + + collector_fun = fn + dict, {:cont, {key, value}} -> module.put(dict, key, value) dict, :done -> dict _, :halt -> :ok - end} + end + + {original, collector_fun} end end @@ -262,6 +302,8 @@ defimpl Inspect, for: HashDict do import Inspect.Algebra def inspect(dict, opts) do - concat ["#HashDict<", Inspect.List.inspect(HashDict.to_list(dict), opts), ">"] + # Avoid warnings about HashDict being deprecated. + module = HashDict + concat(["#HashDict<", Inspect.List.inspect(module.to_list(dict), opts), ">"]) end end diff --git a/lib/elixir/lib/hash_set.ex b/lib/elixir/lib/hash_set.ex index acbc21ae943..a9f4fc0026d 100644 --- a/lib/elixir/lib/hash_set.ex +++ b/lib/elixir/lib/hash_set.ex @@ -1,27 +1,20 @@ defmodule HashSet do @moduledoc """ - A set store. - - The `HashSet` is represented internally as a struct, therefore - `%HashSet{}` can be used whenever there is a need to match - on any `HashSet`. Note though the struct fields are private and - must not be accessed directly. Instead, use the functions on this - or in the `Set` module. - - The `HashSet` is implemented using tries, which grows in - space as the number of keys grows, working well with both - small and large set of keys. For more information about the - functions and their APIs, please consult the `Set` module. + Tuple-based HashSet implementation. + + This module is deprecated. Use the `MapSet` module instead. """ - @behaviour Set + @moduledoc deprecated: "Use MapSet instead" @node_bitmap 0b111 @node_shift 3 @node_size 8 @node_template :erlang.make_tuple(@node_size, []) - @opaque t :: map + message = "Use the MapSet module instead" + + @opaque t :: %__MODULE__{size: non_neg_integer, root: term} @doc false defstruct size: 0, root: @node_template @@ -29,74 +22,85 @@ defmodule HashSet do @compile :inline_list_funcs @compile {:inline, key_hash: 1, key_mask: 1, key_shift: 1} - @doc """ - Creates a new empty set. - """ - @spec new :: Set.t + @deprecated message + @spec new :: Set.t() def new do %HashSet{} end + @deprecated message def union(%HashSet{size: size1} = set1, %HashSet{size: size2} = set2) when size1 <= size2 do - set_fold set1, set2, fn v, acc -> put(acc, v) end + set_fold(set1, set2, fn v, acc -> put(acc, v) end) end + @deprecated message def union(%HashSet{} = set1, %HashSet{} = set2) do - set_fold set2, set1, fn v, acc -> put(acc, v) end + set_fold(set2, set1, fn v, acc -> put(acc, v) end) end + @deprecated message def intersection(%HashSet{} = set1, %HashSet{} = set2) do - set_fold set1, %HashSet{}, fn v, acc -> + set_fold(set1, %HashSet{}, fn v, acc -> if member?(set2, v), do: put(acc, v), else: acc - end + end) end + @deprecated message def difference(%HashSet{} = set1, %HashSet{} = set2) do - set_fold set2, set1, fn v, acc -> delete(acc, v) end + set_fold(set2, set1, fn v, acc -> delete(acc, v) end) end + @deprecated message def to_list(set) do - set_fold(set, [], &[&1|&2]) |> :lists.reverse + set_fold(set, [], &[&1 | &2]) |> :lists.reverse() end + @deprecated message def equal?(%HashSet{size: size1} = set1, %HashSet{size: size2} = set2) do case size1 do ^size2 -> subset?(set1, set2) - _ -> false + _ -> false end end + @deprecated message def subset?(%HashSet{} = set1, %HashSet{} = set2) do reduce(set1, {:cont, true}, fn member, acc -> case member?(set2, member) do true -> {:cont, acc} - _ -> {:halt, false} + _ -> {:halt, false} end - end) |> elem(1) + end) + |> elem(1) end + @deprecated message def disjoint?(%HashSet{} = set1, %HashSet{} = set2) do reduce(set2, {:cont, true}, fn member, acc -> case member?(set1, member) do false -> {:cont, acc} - _ -> {:halt, false} + _ -> {:halt, false} end - end) |> elem(1) + end) + |> elem(1) end + @deprecated message def member?(%HashSet{root: root}, term) do do_member?(root, term, key_hash(term)) end + @deprecated message def put(%HashSet{root: root, size: size}, term) do {root, counter} = do_put(root, term, key_hash(term)) %HashSet{root: root, size: size + counter} end + @deprecated message def delete(%HashSet{root: root, size: size} = set, term) do case do_delete(root, term, key_hash(term)) do {:ok, root} -> %HashSet{root: root, size: size - 1} - :error -> set + :error -> set end end @@ -104,11 +108,12 @@ defmodule HashSet do def reduce(%HashSet{root: root}, acc, fun) do do_reduce(root, acc, fun, @node_size, fn {:suspend, acc} -> {:suspended, acc, &{:done, elem(&1, 1)}} - {:halt, acc} -> {:halted, acc} - {:cont, acc} -> {:done, acc} + {:halt, acc} -> {:halted, acc} + {:cont, acc} -> {:done, acc} end) end + @deprecated message def size(%HashSet{size: size}) do size end @@ -123,72 +128,85 @@ defmodule HashSet do defp do_member?(node, term, hash) do index = key_mask(hash) + case elem(node, index) do - [] -> false - [^term|_] -> true - [_] -> false - [_|n] -> do_member?(n, term, key_shift(hash)) + [] -> false + [^term | _] -> true + [_] -> false + [_ | n] -> do_member?(n, term, key_shift(hash)) end end defp do_put(node, term, hash) do index = key_mask(hash) + case elem(node, index) do [] -> {put_elem(node, index, [term]), 1} - [^term|_] -> + + [^term | _] -> {node, 0} + [t] -> n = put_elem(@node_template, key_mask(key_shift(hash)), [term]) - {put_elem(node, index, [t|n]), 1} - [t|n] -> + {put_elem(node, index, [t | n]), 1} + + [t | n] -> {n, counter} = do_put(n, term, key_shift(hash)) - {put_elem(node, index, [t|n]), counter} + {put_elem(node, index, [t | n]), counter} end end defp do_delete(node, term, hash) do index = key_mask(hash) + case elem(node, index) do [] -> :error + [^term] -> {:ok, put_elem(node, index, [])} + [_] -> :error - [^term|n] -> + + [^term | n] -> {:ok, put_elem(node, index, do_compact_node(n))} - [t|n] -> + + [t | n] -> case do_delete(n, term, key_shift(hash)) do {:ok, @node_template} -> {:ok, put_elem(node, index, [t])} + {:ok, n} -> - {:ok, put_elem(node, index, [t|n])} + {:ok, put_elem(node, index, [t | n])} + :error -> :error end end end - Enum.each 0..(@node_size - 1), fn index -> + Enum.each(0..(@node_size - 1), fn index -> defp do_compact_node(node) when elem(node, unquote(index)) != [] do case elem(node, unquote(index)) do [t] -> case put_elem(node, unquote(index), []) do @node_template -> [t] - n -> [t|n] + n -> [t | n] end - [t|n] -> - [t|put_elem(node, unquote(index), do_compact_node(n))] + + [t | n] -> + [t | put_elem(node, unquote(index), do_compact_node(n))] end end - end + end) ## Set fold - defp do_fold_each([], acc, _fun), do: acc - defp do_fold_each([t], acc, fun), do: fun.(t, acc) - defp do_fold_each([t|n], acc, fun), do: do_fold(n, fun.(t, acc), fun, @node_size) + defp do_fold_each([], acc, _fun), do: acc + defp do_fold_each([t], acc, fun), do: fun.(t, acc) + defp do_fold_each([t | n], acc, fun), do: do_fold(n, fun.(t, acc), fun, @node_size) defp do_fold(node, acc, fun, count) when count > 0 do acc = do_fold_each(:erlang.element(count, node), acc, fun) @@ -217,12 +235,17 @@ defmodule HashSet do next.(fun.(t, acc)) end - defp do_reduce_each([t|n], {:cont, acc}, fun, next) do + defp do_reduce_each([t | n], {:cont, acc}, fun, next) do do_reduce(n, fun.(t, acc), fun, @node_size, next) end defp do_reduce(node, acc, fun, count, next) when count > 0 do - do_reduce_each(:erlang.element(count, node), acc, fun, &do_reduce(node, &1, fun, count - 1, next)) + do_reduce_each( + :erlang.element(count, node), + acc, + fun, + &do_reduce(node, &1, fun, count - 1, next) + ) end defp do_reduce(_node, acc, _fun, 0, next) do @@ -247,22 +270,41 @@ defmodule HashSet do end defimpl Enumerable, for: HashSet do - def reduce(set, acc, fun), do: HashSet.reduce(set, acc, fun) - def member?(set, v), do: {:ok, HashSet.member?(set, v)} - def count(set), do: {:ok, HashSet.size(set)} -end + def reduce(set, acc, fun) do + # Avoid warnings about HashSet being deprecated. + module = HashSet + module.reduce(set, acc, fun) + end -defimpl Collectable, for: HashSet do - def empty(_dict) do - HashSet.new + def member?(set, term) do + # Avoid warnings about HashSet being deprecated. + module = HashSet + {:ok, module.member?(set, term)} + end + + def count(set) do + # Avoid warnings about HashSet being deprecated. + module = HashSet + {:ok, module.size(set)} + end + + def slice(_set) do + {:error, __MODULE__} end +end +defimpl Collectable, for: HashSet do def into(original) do - {original, fn - set, {:cont, x} -> HashSet.put(set, x) + # Avoid warnings about HashSet being deprecated. + module = HashSet + + collector_fun = fn + set, {:cont, term} -> module.put(set, term) set, :done -> set _, :halt -> :ok - end} + end + + {original, collector_fun} end end @@ -270,6 +312,8 @@ defimpl Inspect, for: HashSet do import Inspect.Algebra def inspect(set, opts) do - concat ["#HashSet<", Inspect.List.inspect(HashSet.to_list(set), opts), ">"] + # Avoid warnings about HashSet being deprecated. + module = HashSet + concat(["#HashSet<", Inspect.List.inspect(module.to_list(set), opts), ">"]) end end diff --git a/lib/elixir/lib/inspect.ex b/lib/elixir/lib/inspect.ex index 26f1dbd5977..29923748c5a 100644 --- a/lib/elixir/lib/inspect.ex +++ b/lib/elixir/lib/inspect.ex @@ -1,471 +1,441 @@ import Kernel, except: [inspect: 1] import Inspect.Algebra +alias Code.Identifier + defprotocol Inspect do @moduledoc """ - The `Inspect` protocol is responsible for converting any Elixir - data structure into an algebra document. This document is then - formatted, either in pretty printing format or a regular one. - - The `inspect/2` function receives the entity to be inspected - followed by the inspecting options, represented by the struct - `Inspect.Opts`. + The `Inspect` protocol converts an Elixir data structure into an + algebra document. - Inspection is done using the functions available in `Inspect.Algebra`. + This is typically done when you want to customize how your own + structs are inspected in logs and the terminal. - ## Examples + This documentation refers to implementing the `Inspect` protocol + for your own data structures. To learn more about using inspect, + see `Kernel.inspect/2` and `IO.inspect/2`. - Many times, inspecting a structure can be implemented in function - of existing entities. For example, here is `HashSet`'s `inspect` - implementation: + ## Inspect representation - defimpl Inspect, for: HashSet do - import Inspect.Algebra + There are typically three choices of inspect representation. In order + to understand them, let's imagine we have the following `User` struct: - def inspect(dict, opts) do - concat ["#HashSet<", to_doc(HashSet.to_list(dict), opts), ">"] - end + defmodule User do + defstruct [:id, :name, :address] end - The `concat` function comes from `Inspect.Algebra` and it - concatenates algebra documents together. In the example above, - it is concatenating the string `"HashSet<"` (all strings are - valid algebra documents that keep their formatting when pretty - printed), the document returned by `Inspect.Algebra.to_doc/2` and the - other string `">"`. + Our choices are: - Since regular strings are valid entities in an algebra document, - an implementation of inspect may simply return a string, - although that will devoid it of any pretty-printing. + 1. Print the struct using Elixir's struct syntax, for example: + `%User{address: "Earth", id: 13, name: "Jane"}`. This is the + default representation and best choice if all struct fields + are public. - ## Error handling + 2. Print using the `#User<...>` notation, for example: `#User`. + This notation does not emit valid Elixir code and is typically + used when the struct has private fields (for example, you may want + to hide the field `:address` to redact person identifiable information). - In case there is an error while your structure is being inspected, - Elixir will automatically fall back to a raw representation. + 3. Print the struct using the expression syntax, for example: + `User.new(13, "Jane", "Earth")`. This assumes there is a `User.new/3` + function. This option is mostly used as an alternative to option 2 + for representing custom data structures, such as `MapSet`, `Date.Range`, + and others. - You can however access the underlying error by invoking the Inspect - implementation directly. For example, to test Inspect.HashSet above, - you can invoke it as: + You can implement the Inspect protocol for your own structs while + adhering to the conventions above. Option 1 is the default representation + and you can quickly achieve option 2 by deriving the `Inspect` protocol. + For option 3, you need your custom implementation. - Inspect.HashSet.inspect(HashSet.new, Inspect.Opts.new) + ## Deriving - """ + The `Inspect` protocol can be derived to customize the order of fields + (the default is alphabetical) and hide certain fields from structs, + so they don't show up in logs, inspects and similar. The latter is + especially useful for fields containing private information. - # Handle structs in Any - @fallback_to_any true + The supported options are: - def inspect(thing, opts) -end + * `:only` - only include the given fields when inspecting. -defimpl Inspect, for: Atom do - require Macro + * `:except` - remove the given fields when inspecting. - def inspect(atom, _opts) do - inspect(atom) - end + * `:optional` - (since v1.14.0) do not include a field if it + matches its default value. This can be used to simplify the + struct representation at the cost of hiding information. - def inspect(false), do: "false" - def inspect(true), do: "true" - def inspect(nil), do: "nil" - def inspect(:""), do: ":\"\"" + Whenever `:only` or `:except` are used to restrict fields, + the struct will be printed using the `#User<...>` notation, + as the struct can no longer be copy and pasted as valid Elixir + code. Let's see an example: - def inspect(atom) do - binary = Atom.to_string(atom) + defmodule User do + @derive {Inspect, only: [:id, :name]} + defstruct [:id, :name, :address] + end - cond do - valid_ref_identifier?(binary) -> - if only_elixir?(binary) do - binary - else - "Elixir." <> rest = binary - rest - end - valid_atom_identifier?(binary) -> - ":" <> binary - atom in [:%{}, :{}, :<<>>, :..., :%] -> - ":" <> binary - atom in Macro.binary_ops or atom in Macro.unary_ops -> - ":" <> binary - true -> - << ?:, ?", Inspect.BitString.escape(binary, ?") :: binary, ?" >> - end - end + inspect(%User{id: 1, name: "Jane", address: "Earth"}) + #=> #User - defp only_elixir?("Elixir." <> rest), do: only_elixir?(rest) - defp only_elixir?("Elixir"), do: true - defp only_elixir?(_), do: false + If you use only the `:optional` option, the struct will still be + printed as `%User{...}`. - # Detect if atom is an atom alias (Elixir.Foo.Bar.Baz) + ## Custom implementation - defp valid_ref_identifier?("Elixir" <> rest) do - valid_ref_piece?(rest) - end + You can also define your custom protocol implementation by + defining the `inspect/2` function. The function receives the + entity to be inspected followed by the inspecting options, + represented by the struct `Inspect.Opts`. Building of the + algebra document is done with `Inspect.Algebra`. - defp valid_ref_identifier?(_), do: false + Many times, inspecting a structure can be implemented in function + of existing entities. For example, here is `MapSet`'s `inspect/2` + implementation: - defp valid_ref_piece?(<>) when h in ?A..?Z do - valid_ref_piece? valid_identifier?(t) - end + defimpl Inspect, for: MapSet do + import Inspect.Algebra - defp valid_ref_piece?(<<>>), do: true - defp valid_ref_piece?(_), do: false + def inspect(map_set, opts) do + concat(["MapSet.new(", Inspect.List.inspect(MapSet.to_list(map_set), opts), ")"]) + end + end - # Detect if atom + The [`concat/1`](`Inspect.Algebra.concat/1`) function comes from + `Inspect.Algebra` and it concatenates algebra documents together. + In the example above it is concatenating the string `"MapSet.new("`, + the document returned by `Inspect.Algebra.to_doc/2`, and the final + string `")"`. Therefore, the MapSet with the numbers 1, 2, and 3 + will be printed as: - defp valid_atom_identifier?(<>) when h in ?a..?z or h in ?A..?Z or h == ?_ do - valid_atom_piece?(t) - end + iex> MapSet.new([1, 2, 3], fn x -> x * 2 end) + MapSet.new([2, 4, 6]) - defp valid_atom_identifier?(_), do: false + In other words, `MapSet`'s inspect representation returns an expression + that, when evaluated, builds the `MapSet` itself. - defp valid_atom_piece?(t) do - case valid_identifier?(t) do - <<>> -> true - <> -> true - <> -> true - <> -> valid_atom_piece?(t) - _ -> false - end - end + ### Error handling - defp valid_identifier?(<>) - when h in ?a..?z - when h in ?A..?Z - when h in ?0..?9 - when h == ?_ do - valid_identifier? t - end + In case there is an error while your structure is being inspected, + Elixir will raise an `ArgumentError` error and will automatically fall back + to a raw representation for printing the structure. - defp valid_identifier?(other), do: other -end + You can however access the underlying error by invoking the `Inspect` + implementation directly. For example, to test `Inspect.MapSet` above, + you can invoke it as: -defimpl Inspect, for: BitString do - def inspect(thing, %Inspect.Opts{binaries: bins} = opts) when is_binary(thing) do - if bins == :as_strings or (bins == :infer and String.printable?(thing)) do - <> - else - inspect_bitstring(thing, opts) - end - end + Inspect.MapSet.inspect(MapSet.new(), %Inspect.Opts{}) - def inspect(thing, opts) do - inspect_bitstring(thing, opts) - end + """ - ## Escaping + # Handle structs in Any + @fallback_to_any true - @doc false - def escape(other, char) do - escape(other, char, <<>>) - end + @doc """ + Converts `term` into an algebra document. - defp escape(<< char, t :: binary >>, char, binary) do - escape(t, char, << binary :: binary, ?\\, char >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?#, ?{>>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?a >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?b >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?d >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?e >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?f >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?n >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?r >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?\\ >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?t >>) - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, ?\\, ?v >>) - end - defp escape(<>, char, binary) do - head = << h :: utf8 >> - if String.printable?(head) do - escape(t, char, append(head, binary)) - else - << byte :: size(8), h :: binary >> = head - t = << h :: binary, t :: binary >> - escape(t, char, << binary :: binary, escape_char(byte) :: binary >>) - end - end - defp escape(<>, char, binary) do - escape(t, char, << binary :: binary, escape_char(h) :: binary >>) + This function shouldn't be invoked directly, unless when implementing + a custom `inspect_fun` to be given to `Inspect.Opts`. Everywhere else, + `Inspect.Algebra.to_doc/2` should be preferred as it handles structs + and exceptions. + """ + @spec inspect(t, Inspect.Opts.t()) :: Inspect.Algebra.t() + def inspect(term, opts) +end + +defimpl Inspect, for: Atom do + require Macro + + def inspect(atom, opts) do + color(Macro.inspect_atom(:literal, atom), color_key(atom), opts) end - defp escape(<<>>, _char, binary), do: binary - @doc false - # Also used by Regex - def escape_char(char) when char in ?\000..?\377, - do: octify(char) + defp color_key(atom) when is_boolean(atom), do: :boolean + defp color_key(nil), do: nil + defp color_key(_), do: :atom +end - def escape_char(char), do: hexify(char) +defimpl Inspect, for: BitString do + def inspect(term, opts) when is_binary(term) do + %Inspect.Opts{binaries: bins, base: base, printable_limit: printable_limit} = opts + + if base == :decimal and + (bins == :as_strings or (bins == :infer and String.printable?(term, printable_limit))) do + inspected = + case Identifier.escape(term, ?", printable_limit) do + {escaped, ""} -> [?", escaped, ?"] + {escaped, _} -> [?", escaped, ?", " <> ..."] + end - defp octify(byte) do - << hi :: size(2), mi :: size(3), lo :: size(3) >> = << byte >> - << ?\\, ?0 + hi, ?0 + mi, ?0 + lo >> + color(IO.iodata_to_binary(inspected), :string, opts) + else + inspect_bitstring(term, opts) + end end - defp hexify(char) when char < 0x10000 do - <> = <> - <> + def inspect(term, opts) do + inspect_bitstring(term, opts) end - defp hexify(char) when char < 0x1000000 do - <> = <> - <> + defp inspect_bitstring("", opts) do + color("<<>>", :binary, opts) end - defp to_hex(c) when c in 0..9, do: ?0+c - defp to_hex(c) when c in 10..15, do: ?a+c-10 - - defp append(<>, binary), do: append(t, << binary :: binary, h >>) - defp append(<<>>, binary), do: binary - - ## Bitstrings - defp inspect_bitstring(bitstring, opts) do - each_bit(bitstring, opts.limit, "<<") <> ">>" + left = color("<<", :binary, opts) + right = color(">>", :binary, opts) + inner = each_bit(bitstring, opts.limit, opts) + group(concat(concat(left, nest(inner, 2)), right)) end - defp each_bit(_, 0, acc) do - acc <> "..." + defp each_bit(_, 0, _) do + "..." end - defp each_bit(<>, counter, acc) when t != <<>> do - each_bit(t, decrement(counter), acc <> Integer.to_string(h) <> ", ") + defp each_bit(<<>>, _counter, _opts) do + :doc_nil end - defp each_bit(<>, _counter, acc) do - acc <> Integer.to_string(h) + defp each_bit(<>, _counter, opts) do + Inspect.Integer.inspect(h, opts) end - defp each_bit(<<>>, _counter, acc) do - acc + defp each_bit(<>, counter, opts) do + flex_glue( + concat(Inspect.Integer.inspect(h, opts), ","), + each_bit(t, decrement(counter), opts) + ) end - defp each_bit(bitstring, _counter, acc) do + defp each_bit(bitstring, _counter, opts) do size = bit_size(bitstring) - <> = bitstring - acc <> Integer.to_string(h) <> "::size(" <> Integer.to_string(size) <> ")" + <> = bitstring + concat(Inspect.Integer.inspect(h, opts), "::size(" <> Integer.to_string(size) <> ")") end + @compile {:inline, decrement: 1} defp decrement(:infinity), do: :infinity - defp decrement(counter), do: counter - 1 + defp decrement(counter), do: counter - 1 end defimpl Inspect, for: List do - @doc ~S""" - Represents a list, checking if it can be printed or not. - If so, a single-quoted representation is returned, - otherwise the brackets syntax is used. Keywords are - printed in keywords syntax. - - ## Examples - - iex> inspect('bar') - "'bar'" + def inspect([], opts) do + color("[]", :list, opts) + end + + # TODO: Remove :char_list and :as_char_lists handling on v2.0 + def inspect(term, opts) do + %Inspect.Opts{ + charlists: lists, + char_lists: lists_deprecated, + printable_limit: printable_limit + } = opts + + lists = + if lists == :infer and lists_deprecated != :infer do + case lists_deprecated do + :as_char_lists -> + IO.warn( + "the :char_lists inspect option and its :as_char_lists " <> + "value are deprecated, use the :charlists option and its " <> + ":as_charlists value instead" + ) + + :as_charlists + + _ -> + IO.warn("the :char_lists inspect option is deprecated, use :charlists instead") + lists_deprecated + end + else + lists + end - iex> inspect([0|'bar']) - "[0, 98, 97, 114]" + open = color("[", :list, opts) + sep = color(",", :list, opts) + close = color("]", :list, opts) - iex> inspect([:foo,:bar]) - "[:foo, :bar]" + cond do + lists == :as_charlists or (lists == :infer and List.ascii_printable?(term, printable_limit)) -> + inspected = + case Identifier.escape(IO.chardata_to_string(term), ?', printable_limit) do + {escaped, ""} -> [?', escaped, ?'] + {escaped, _} -> [?', escaped, ?', " ++ ..."] + end - """ + IO.iodata_to_binary(inspected) - def inspect([], _opts), do: "[]" + keyword?(term) -> + container_doc(open, term, close, opts, &keyword/2, separator: sep, break: :strict) - def inspect(thing, %Inspect.Opts{char_lists: lists} = opts) do - cond do - lists == :as_char_lists or (lists == :infer and printable?(thing)) -> - << ?', Inspect.BitString.escape(IO.chardata_to_string(thing), ?') :: binary, ?' >> - keyword?(thing) -> - surround_many("[", thing, "]", opts.limit, &keyword(&1, opts)) true -> - surround_many("[", thing, "]", opts.limit, &to_doc(&1, opts)) + container_doc(open, term, close, opts, &to_doc/2, separator: sep) end end + @doc false def keyword({key, value}, opts) do - concat( - key_to_binary(key) <> ": ", - to_doc(value, opts) - ) + key = color(Macro.inspect_atom(:key, key), :atom, opts) + concat(key, concat(" ", to_doc(value, opts))) end + @doc false def keyword?([{key, _value} | rest]) when is_atom(key) do - case Atom.to_char_list(key) do + case Atom.to_charlist(key) do 'Elixir.' ++ _ -> false _ -> keyword?(rest) end end - def keyword?([]), do: true + def keyword?([]), do: true def keyword?(_other), do: false - - ## Private - - defp key_to_binary(key) do - case Inspect.Atom.inspect(key) do - ":" <> right -> right - other -> other - end - end - - defp printable?([c|cs]) when is_integer(c) and c in 32..126, do: printable?(cs) - defp printable?([?\n|cs]), do: printable?(cs) - defp printable?([?\r|cs]), do: printable?(cs) - defp printable?([?\t|cs]), do: printable?(cs) - defp printable?([?\v|cs]), do: printable?(cs) - defp printable?([?\b|cs]), do: printable?(cs) - defp printable?([?\f|cs]), do: printable?(cs) - defp printable?([?\e|cs]), do: printable?(cs) - defp printable?([?\a|cs]), do: printable?(cs) - defp printable?([]), do: true - defp printable?(_), do: false end defimpl Inspect, for: Tuple do - def inspect({}, _opts), do: "{}" - def inspect(tuple, opts) do - surround_many("{", Tuple.to_list(tuple), "}", opts.limit, &to_doc(&1, opts)) + open = color("{", :tuple, opts) + sep = color(",", :tuple, opts) + close = color("}", :tuple, opts) + container_opts = [separator: sep, break: :flex] + container_doc(open, Tuple.to_list(tuple), close, opts, &to_doc/2, container_opts) end end defimpl Inspect, for: Map do def inspect(map, opts) do - nest inspect(map, "", opts), 1 - end + list = Map.to_list(map) + + fun = + if Inspect.List.keyword?(list) do + &Inspect.List.keyword/2 + else + sep = color(" => ", :map, opts) + &to_assoc(&1, &2, sep) + end - def inspect(map, name, opts) do - map = :maps.to_list(map) - surround_many("%" <> name <> "{", map, "}", opts.limit, traverse_fun(map, opts)) + map_container_doc(list, "", opts, fun) end - defp traverse_fun(list, opts) do - if Inspect.List.keyword?(list) do - &Inspect.List.keyword(&1, opts) - else - &to_map(&1, opts) - end + def inspect(map, name, infos, opts) do + fun = fn %{field: field}, opts -> Inspect.List.keyword({field, Map.get(map, field)}, opts) end + map_container_doc(infos, name, opts, fun) end - defp to_map({key, value}, opts) do - concat( - concat(to_doc(key, opts), " => "), - to_doc(value, opts) - ) + defp to_assoc({key, value}, opts, sep) do + concat(concat(to_doc(key, opts), sep), to_doc(value, opts)) end -end -defimpl Inspect, for: Integer do - def inspect(thing, _opts) do - Integer.to_string(thing) + defp map_container_doc(list, name, opts, fun) do + open = color("%" <> name <> "{", :map, opts) + sep = color(",", :map, opts) + close = color("}", :map, opts) + container_doc(open, list, close, opts, fun, separator: sep, break: :strict) end end -defimpl Inspect, for: Float do - def inspect(thing, _opts) do - IO.iodata_to_binary(:io_lib_format.fwrite_g(thing)) +defimpl Inspect, for: Integer do + def inspect(term, %Inspect.Opts{base: base} = opts) do + inspected = Integer.to_string(term, base_to_value(base)) |> prepend_prefix(base) + color(inspected, :number, opts) end -end -defimpl Inspect, for: Regex do - def inspect(regex, _opts) do - delim = ?/ - concat ["~r", - <>, - regex.opts] + defp base_to_value(base) do + case base do + :binary -> 2 + :decimal -> 10 + :octal -> 8 + :hex -> 16 + end end - defp escape(bin, term), - do: escape(bin, <<>>, term) - - defp escape(<> <> rest, buf, term), - do: escape(rest, buf <> <>, term) - - defp escape(<> <> rest, buf, term), - do: escape(rest, buf <> <>, term) - - # the list of characters is from `String.printable?` impl - # minus characters treated specially by regex: \s, \d, \b, \e - - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) - - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) + defp prepend_prefix(value, :decimal), do: value - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) + defp prepend_prefix(<>, base) do + "-" <> prepend_prefix(value, base) + end - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) + defp prepend_prefix(value, base) do + prefix = + case base do + :binary -> "0b" + :octal -> "0o" + :hex -> "0x" + end - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) + prefix <> value + end +end - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) +defimpl Inspect, for: Float do + def inspect(float, opts) do + abs = abs(float) + + formatted = + if abs >= 1.0 and abs < 1.0e16 and trunc(float) == float do + [Integer.to_string(trunc(float)), ?., ?0] + else + :io_lib_format.fwrite_g(float) + end - defp escape(<> <> rest, buf, term) do - charstr = <> - if String.printable?(charstr) and not c in [?\d, ?\b, ?\e] do - escape(rest, buf <> charstr, term) - else - escape(rest, buf <> Inspect.BitString.escape_char(c), term) - end + color(IO.iodata_to_binary(formatted), :number, opts) end +end - defp escape(<> <> rest, buf, term), - do: escape(rest, <>, term) - - defp escape(<<>>, buf, _), do: buf +defimpl Inspect, for: Regex do + def inspect(regex, opts) do + {escaped, _} = + regex.source + |> normalize(<<>>) + |> Identifier.escape(?/, :infinity, &escape_map/1) + + source = IO.iodata_to_binary(['~r/', escaped, ?/, regex.opts]) + color(source, :regex, opts) + end + + defp normalize(<>, acc), do: normalize(rest, <>) + defp normalize(<>, acc), do: normalize(rest, <>) + defp normalize(<>, acc), do: normalize(rest, <>) + defp normalize(<<>>, acc), do: acc + + defp escape_map(?\a), do: '\\a' + defp escape_map(?\f), do: '\\f' + defp escape_map(?\n), do: '\\n' + defp escape_map(?\r), do: '\\r' + defp escape_map(?\t), do: '\\t' + defp escape_map(?\v), do: '\\v' + defp escape_map(_), do: false end defimpl Inspect, for: Function do def inspect(function, _opts) do - fun_info = :erlang.fun_info(function) + fun_info = Function.info(function) mod = fun_info[:module] + name = fun_info[:name] - if fun_info[:type] == :external and fun_info[:env] == [] do - "&#{Inspect.Atom.inspect(mod)}.#{fun_info[:name]}/#{fun_info[:arity]}" - else - case Atom.to_char_list(mod) do - 'elixir_compiler_' ++ _ -> - if function_exported?(mod, :__RELATIVE__, 0) do - "#Function<#{uniq(fun_info)} in file:#{mod.__RELATIVE__}>" - else - default_inspect(mod, fun_info) - end - _ -> + cond do + not is_atom(mod) -> + "#Function<#{uniq(fun_info)}/#{fun_info[:arity]}>" + + fun_info[:type] == :external and fun_info[:env] == [] -> + inspected_as_atom = Macro.inspect_atom(:literal, mod) + inspected_as_function = Macro.inspect_atom(:remote_call, name) + "&#{inspected_as_atom}.#{inspected_as_function}/#{fun_info[:arity]}" + + match?('elixir_compiler_' ++ _, Atom.to_charlist(mod)) -> + if function_exported?(mod, :__RELATIVE__, 0) do + "#Function<#{uniq(fun_info)} in file:#{mod.__RELATIVE__}>" + else default_inspect(mod, fun_info) - end + end + + true -> + default_inspect(mod, fun_info) end end defp default_inspect(mod, fun_info) do - "#Function<#{uniq(fun_info)}/#{fun_info[:arity]} in " <> - "#{Inspect.Atom.inspect(mod)}#{extract_name(fun_info[:name])}>" + inspected_as_atom = Macro.inspect_atom(:literal, mod) + extracted_name = extract_name(fun_info[:name]) + "#Function<#{uniq(fun_info)}/#{fun_info[:arity]} in #{inspected_as_atom}#{extracted_name}>" end defp extract_name([]) do @@ -473,16 +443,47 @@ defimpl Inspect, for: Function do end defp extract_name(name) do - name = Atom.to_string(name) - case :binary.split(name, "-", [:global]) do - ["", name | _] -> "." <> name - _ -> "." <> name + case Identifier.extract_anonymous_fun_parent(name) do + {name, arity} -> + "." <> Macro.inspect_atom(:remote_call, name) <> "/" <> arity + + :error -> + "." <> Macro.inspect_atom(:remote_call, name) end end defp uniq(fun_info) do - Integer.to_string(fun_info[:new_index]) <> "." <> - Integer.to_string(fun_info[:uniq]) + Integer.to_string(fun_info[:new_index]) <> "." <> Integer.to_string(fun_info[:uniq]) + end +end + +defimpl Inspect, for: Inspect.Error do + @impl true + def inspect(%{stacktrace: stacktrace} = inspect_error, _opts) do + message = Exception.message(inspect_error) + format_output(message, stacktrace) + end + + defp format_output(message, [_ | _] = stacktrace) do + stacktrace = Exception.format_stacktrace(stacktrace) + + """ + #Inspect.Error< + #{Inspect.Error.pad(message, 2)} + + Stacktrace: + + #{stacktrace} + >\ + """ + end + + defp format_output(message, []) do + """ + #Inspect.Error< + #{Inspect.Error.pad(message, 2)} + >\ + """ end end @@ -494,7 +495,7 @@ end defimpl Inspect, for: Port do def inspect(port, _opts) do - IO.iodata_to_binary :erlang.port_to_list(port) + IO.iodata_to_binary(:erlang.port_to_list(port)) end end @@ -506,19 +507,123 @@ defimpl Inspect, for: Reference do end defimpl Inspect, for: Any do - def inspect(%{__struct__: struct} = map, opts) do + defmacro __deriving__(module, struct, options) do + fields = Map.keys(struct) -- [:__exception__, :__struct__] + only = Keyword.get(options, :only, fields) + except = Keyword.get(options, :except, []) + optional = Keyword.get(options, :optional, []) + + :ok = validate_option(:only, only, fields, module) + :ok = validate_option(:except, except, fields, module) + :ok = validate_option(:optional, optional, fields, module) + + inspect_module = + if fields == only and except == [] do + Inspect.Map + else + Inspect.Any + end + + filtered_fields = + fields + |> Enum.reject(&(&1 in except)) + |> Enum.filter(&(&1 in only)) + + optional? = + if optional == [] do + false + else + optional_map = for field <- optional, into: %{}, do: {field, Map.fetch!(struct, field)} + + quote do + case unquote(Macro.escape(optional_map)) do + %{^var!(field) => var!(default)} -> + var!(default) == Map.get(var!(struct), var!(field)) + + %{} -> + false + end + end + end + + quote do + defimpl Inspect, for: unquote(module) do + def inspect(var!(struct), var!(opts)) do + var!(infos) = + for %{field: var!(field)} = var!(info) <- unquote(module).__info__(:struct), + var!(field) in unquote(filtered_fields) and not unquote(optional?), + do: var!(info) + + var!(name) = Macro.inspect_atom(:literal, unquote(module)) + unquote(inspect_module).inspect(var!(struct), var!(name), var!(infos), var!(opts)) + end + end + end + end + + defp validate_option(option, option_list, fields, module) do + case option_list -- fields do + [] -> + :ok + + unknown_fields -> + raise ArgumentError, + "unknown fields #{Kernel.inspect(unknown_fields)} in #{Kernel.inspect(option)} " <> + "when deriving the Inspect protocol for #{Kernel.inspect(module)}" + end + end + + def inspect(%module{} = struct, opts) do try do - struct.__struct__ + {module.__struct__(), module.__info__(:struct)} rescue - _ -> Inspect.Map.inspect(map, opts) + _ -> Inspect.Map.inspect(struct, opts) else - dunder -> - if :maps.keys(dunder) == :maps.keys(map) do - pruned = :maps.remove(:__exception__, :maps.remove(:__struct__, map)) - Inspect.Map.inspect(pruned, Inspect.Atom.inspect(struct, opts), opts) + {dunder, fields} -> + if Map.keys(dunder) == Map.keys(struct) do + infos = + for %{field: field} = info <- fields, + field not in [:__struct__, :__exception__], + do: info + + Inspect.Map.inspect(struct, Macro.inspect_atom(:literal, module), infos, opts) else - Inspect.Map.inspect(map, opts) + Inspect.Map.inspect(struct, opts) end end end + + def inspect(map, name, infos, opts) do + open = color("#" <> name <> "<", :map, opts) + sep = color(",", :map, opts) + close = color(">", :map, opts) + + fun = fn + %{field: field}, opts -> Inspect.List.keyword({field, Map.get(map, field)}, opts) + :..., _opts -> "..." + end + + container_doc(open, infos ++ [:...], close, opts, fun, separator: sep, break: :strict) + end end + +require Protocol + +Protocol.derive( + Inspect, + Macro.Env, + only: [ + :module, + :file, + :line, + :function, + :context, + :aliases, + :requires, + :functions, + :macros, + :macro_aliases, + :context_modules, + :lexical_tracker + ] +) diff --git a/lib/elixir/lib/inspect/algebra.ex b/lib/elixir/lib/inspect/algebra.ex index 3a3b915d786..f298d251372 100644 --- a/lib/elixir/lib/inspect/algebra.ex +++ b/lib/elixir/lib/inspect/algebra.ex @@ -1,54 +1,179 @@ defmodule Inspect.Opts do @moduledoc """ - Defines the Inspect.Opts used by the Inspect protocol. + Defines the options used by the `Inspect` protocol. The following fields are available: - * `:structs` - when false, structs are not formatted by the inspect - protocol, they are instead printed as maps, defaults to true. + * `:base` - prints integers as `:binary`, `:octal`, `:decimal`, or `:hex`, + defaults to `:decimal`. When inspecting binaries any `:base` other than + `:decimal` implies `binaries: :as_binaries`. - * `:binaries` - when `:as_strings` all binaries will be printed as strings, - non-printable bytes will be escaped. + * `:binaries` - when `:as_binaries` all binaries will be printed in bit + syntax. - When `:as_binaries` all binaries will be printed in bit syntax. + When `:as_strings` all binaries will be printed as strings, non-printable + bytes will be escaped. When the default `:infer`, the binary will be printed as a string if it - is printable, otherwise in bit syntax. + is printable, otherwise in bit syntax. See `String.printable?/1` to learn + when a string is printable. - * `:char_lists` - when `:as_char_lists` all lists will be printed as char - lists, non-printable elements will be escaped. + * `:charlists` - when `:as_charlists` all lists will be printed as charlists, + non-printable elements will be escaped. When `:as_lists` all lists will be printed as lists. - When the default `:infer`, the list will be printed as a char list if it - is printable, otherwise as list. + When the default `:infer`, the list will be printed as a charlist if it + is printable, otherwise as list. See `List.ascii_printable?/1` to learn + when a charlist is printable. + + * `:custom_options` (since v1.9.0) - a keyword list storing custom user-defined + options. Useful when implementing the `Inspect` protocol for nested structs + to pass the custom options through. + + * `:inspect_fun` (since v1.9.0) - a function to build algebra documents. + Defaults to `Inspect.Opts.default_inspect_fun/0`. + + * `:limit` - limits the number of items that are inspected for tuples, + bitstrings, maps, lists and any other collection of items, with the exception of + printable strings and printable charlists which use the `:printable_limit` option. + If you don't want to limit the number of items to a particular number, + use `:infinity`. It accepts a positive integer or `:infinity`. + Defaults to `50`. + + * `:pretty` - if set to `true` enables pretty printing. Defaults to `false`. + + * `:printable_limit` - limits the number of characters that are inspected + on printable strings and printable charlists. You can use `String.printable?/1` + and `List.ascii_printable?/1` to check if a given string or charlist is + printable. If you don't want to limit the number of characters to a particular + number, use `:infinity`. It accepts a positive integer or `:infinity`. + Defaults to `4096`. + + * `:safe` - when `false`, failures while inspecting structs will be raised + as errors instead of being wrapped in the `Inspect.Error` exception. This + is useful when debugging failures and crashes for custom inspect + implementations. Defaults to `true`. + + * `:structs` - when `false`, structs are not formatted by the inspect + protocol, they are instead printed as maps. Defaults to `true`. + + * `:syntax_colors` - when set to a keyword list of colors the output is + colorized. The keys are types and the values are the colors to use for + each type (for example, `[number: :red, atom: :blue]`). Types can include + `:atom`, `:binary`, `:boolean`, `:list`, `:map`, `:number`, `:regex`, + `:string`, and `:tuple`. Custom data types may provide their own options. + Colors can be any `t:IO.ANSI.ansidata/0` as accepted by `IO.ANSI.format/1`. + + * `:width` - number of characters per line used when pretty is `true` or when + printing to IO devices. Set to `0` to force each item to be printed on its + own line. If you don't want to limit the number of items to a particular + number, use `:infinity`. Defaults to `80`. - * `:limit` - limits the number of items that are printed for tuples, - bitstrings, and lists, does not apply to strings nor char lists, defaults - to 50. + """ - * `:pretty` - if set to true enables pretty printing, defaults to false. + # TODO: Remove :char_lists key on v2.0 + defstruct base: :decimal, + binaries: :infer, + char_lists: :infer, + charlists: :infer, + custom_options: [], + inspect_fun: &Inspect.inspect/2, + limit: 50, + pretty: false, + printable_limit: 4096, + safe: true, + structs: true, + syntax_colors: [], + width: 80 + + @type color_key :: atom + + # TODO: Remove :char_lists key and :as_char_lists value on v2.0 + @type t :: %__MODULE__{ + base: :decimal | :binary | :hex | :octal, + binaries: :infer | :as_binaries | :as_strings, + char_lists: :infer | :as_lists | :as_char_lists, + charlists: :infer | :as_lists | :as_charlists, + custom_options: keyword, + inspect_fun: (any, t -> Inspect.Algebra.t()), + limit: non_neg_integer | :infinity, + pretty: boolean, + printable_limit: non_neg_integer | :infinity, + safe: boolean, + structs: boolean, + syntax_colors: [{color_key, IO.ANSI.ansidata()}], + width: non_neg_integer | :infinity + } - * `:width` - defaults to the 80 characters. + @doc """ + Builds an `Inspect.Opts` struct. """ + @doc since: "1.13.0" + @spec new(keyword()) :: t + def new(opts) do + struct(%Inspect.Opts{inspect_fun: default_inspect_fun()}, opts) + end + + @doc """ + Returns the default inspect function. + """ + @doc since: "1.13.0" + @spec default_inspect_fun() :: (term, t -> Inspect.Algebra.t()) + def default_inspect_fun do + :persistent_term.get({__MODULE__, :inspect_fun}, &Inspect.inspect/2) + end + + @doc """ + Sets the default inspect function. + + Set this option with care as it will change how all values + in the system are inspected. The main use of this functionality + is to provide an entry point to filter inspected values, + in order for entities to comply with rules and legislations + on data security and data privacy. + + It is **extremely discouraged** for libraries to set their own + function as this must be controlled by applications. Libraries + should instead define their own structs with custom inspect + implementations. If a library must change the default inspect + function, then it is best to define to ask users of your library + to explicitly call `default_inspect_fun/1` with your function of + choice. + + The default is `Inspect.inspect/2`. + + ## Examples - defstruct structs: true :: boolean, - binaries: :infer :: :infer | :as_binaries | :as_strings, - char_lists: :infer :: :infer | :as_lists | :as_char_lists, - limit: 50 :: pos_integer, - width: 80 :: pos_integer | :infinity, - pretty: false :: boolean + previous_fun = Inspect.Opts.default_inspect_fun() + + Inspect.Opts.default_inspect_fun(fn + %{address: _} = map, opts -> + previous_fun.(%{map | address: "[REDACTED]"}, opts) + + value, opts -> + previous_fun.(value, opts) + end) + + """ + @doc since: "1.13.0" + @spec default_inspect_fun((term, t -> Inspect.Algebra.t())) :: :ok + def default_inspect_fun(fun) when is_function(fun, 2) do + :persistent_term.put({__MODULE__, :inspect_fun}, fun) + end end defmodule Inspect.Algebra do @moduledoc ~S""" A set of functions for creating and manipulating algebra - documents, as described in ["Strictly Pretty" (2000) by Christian Lindig][0]. + documents. - An algebra document is represented by an `Inspect.Algebra` node - or a regular string. + This module implements the functionality described in + ["Strictly Pretty" (2000) by Christian Lindig][0] with small + additions, like support for binary nodes and a break mode that + maximises use of horizontal space. - iex> Inspect.Algebra.empty + iex> Inspect.Algebra.empty() :doc_nil iex> "foo" @@ -57,445 +182,959 @@ defmodule Inspect.Algebra do With the functions in this module, we can concatenate different elements together and render them: - iex> doc = Inspect.Algebra.concat(Inspect.Algebra.empty, "foo") - iex> Inspect.Algebra.pretty(doc, 80) - "foo" + iex> doc = Inspect.Algebra.concat(Inspect.Algebra.empty(), "foo") + iex> Inspect.Algebra.format(doc, 80) + ["foo"] The functions `nest/2`, `space/2` and `line/2` help you put the document together into a rigid structure. However, the document - algebra gets interesting when using functions like `break/2`, which - converts the given string into a line break depending on how much space - there is to print. Let's glue two docs together with a break and then - render it: + algebra gets interesting when using functions like `glue/3` and + `group/1`. A glue inserts a break between two documents. A group + indicates a document that must fit the current line, otherwise + breaks are rendered as new lines. Let's glue two docs together + with a break, group it and then render it: iex> doc = Inspect.Algebra.glue("a", " ", "b") - iex> Inspect.Algebra.pretty(doc, 80) - "a b" + iex> doc = Inspect.Algebra.group(doc) + iex> Inspect.Algebra.format(doc, 80) + ["a", " ", "b"] - Notice the break was represented as is, because we haven't reached + Note that the break was represented as is, because we haven't reached a line limit. Once we do, it is replaced by a newline: iex> doc = Inspect.Algebra.glue(String.duplicate("a", 20), " ", "b") - iex> Inspect.Algebra.pretty(doc, 10) - "aaaaaaaaaaaaaaaaaaaa\nb" + iex> doc = Inspect.Algebra.group(doc) + iex> Inspect.Algebra.format(doc, 10) + ["aaaaaaaaaaaaaaaaaaaa", "\n", "b"] + + This module uses the byte size to compute how much space there is + left. If your document contains strings, then those need to be + wrapped in `string/1`, which then relies on `String.length/1` to + precompute the document size. Finally, this module also contains Elixir related functions, a bit - tied to Elixir formatting, namely `surround/3` and `surround_many/5`. + tied to Elixir formatting, such as `to_doc/2`. ## Implementation details - The original Haskell implementation of the algorithm by [Wadler][1] - relies on lazy evaluation to unfold document groups on two alternatives: - `:flat` (breaks as spaces) and `:break` (breaks as newlines). - Implementing the same logic in a strict language such as Elixir leads - to an exponential growth of possible documents, unless document groups - are encoded explictly as `:flat` or `:break`. Those groups are then reduced - to a simple document, where the layout is already decided, per [Lindig][0]. - - This implementation slightly changes the semantic of Lindig's algorithm - to allow elements that belong to the same group to be printed together - in the same line, even if they do not fit the line fully. This was achieved - by changing `:break` to mean a possible break and `:flat` to force a flat - structure. Then deciding if a break works as a newline is just a matter - of checking if we have enough space until the next break that is not - inside a group (which is still flat). - - Custom pretty printers can be implemented using the documents returned - by this module and by providing their own rendering functions. + The implementation of `Inspect.Algebra` is based on the Strictly Pretty + paper by [Lindig][0] which builds on top of previous pretty printing + algorithms but is tailored to strict languages, such as Elixir. + The core idea in the paper is the use of explicit document groups which + are rendered as flat (breaks as spaces) or as break (breaks as newlines). + + This implementation provides two types of breaks: `:strict` and `:flex`. + When a group does not fit, all strict breaks are treated as newlines. + Flex breaks however are re-evaluated on every occurrence and may still + be rendered flat. See `break/1` and `flex_break/1` for more information. + + This implementation also adds `force_unfit/1` and `next_break_fits/2` which + give more control over the document fitting. [0]: http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.34.2200 - [1]: http://homepages.inf.ed.ac.uk/wadler/papers/prettier/prettier.pdf """ - @surround_separator "," + @container_separator "," @tail_separator " |" @newline "\n" - @nesting 1 - @break " " - - # Functional interface to `doc` records + @next_break_fits :enabled + + # Functional interface to "doc" records + + @type t :: + binary + | :doc_line + | :doc_nil + | doc_break + | doc_collapse + | doc_color + | doc_cons + | doc_fits + | doc_force + | doc_group + | doc_nest + | doc_string + | doc_limit + + @typep doc_string :: {:doc_string, t, non_neg_integer} + defmacrop doc_string(string, length) do + quote do: {:doc_string, unquote(string), unquote(length)} + end - @type t :: :doc_nil | :doc_line | doc_cons | doc_nest | doc_break | doc_group | binary + @typep doc_limit :: {:doc_limit, t, pos_integer | :infinity} + defmacrop doc_limit(doc, limit) do + quote do: {:doc_limit, unquote(doc), unquote(limit)} + end @typep doc_cons :: {:doc_cons, t, t} defmacrop doc_cons(left, right) do quote do: {:doc_cons, unquote(left), unquote(right)} end - @typep doc_nest :: {:doc_nest, t, non_neg_integer} - defmacrop doc_nest(doc, indent) do - quote do: {:doc_nest, unquote(doc), unquote(indent) } + @typep doc_nest :: {:doc_nest, t, :cursor | :reset | non_neg_integer, :always | :break} + defmacrop doc_nest(doc, indent, always_or_break) do + quote do: {:doc_nest, unquote(doc), unquote(indent), unquote(always_or_break)} end - @typep doc_break :: {:doc_break, binary} - defmacrop doc_break(break) do - quote do: {:doc_break, unquote(break)} + @typep doc_break :: {:doc_break, binary, :flex | :strict} + defmacrop doc_break(break, mode) do + quote do: {:doc_break, unquote(break), unquote(mode)} end - @typep doc_group :: {:doc_group, t} - defmacrop doc_group(group) do - quote do: {:doc_group, unquote(group)} + @typep doc_group :: {:doc_group, t, :inherit | :self} + defmacrop doc_group(group, mode) do + quote do: {:doc_group, unquote(group), unquote(mode)} end - defmacrop is_doc(doc) do - if Macro.Env.in_guard?(__CALLER__) do - do_is_doc(doc) - else - var = quote do: doc - quote do - unquote(var) = unquote(doc) - unquote(do_is_doc(var)) - end - end + @typep doc_fits :: {:doc_fits, t, :enabled | :disabled} + defmacrop doc_fits(group, mode) do + quote do: {:doc_fits, unquote(group), unquote(mode)} end - defp do_is_doc(doc) do - quote do - is_binary(unquote(doc)) or - unquote(doc) in [:doc_nil, :doc_line] or - (is_tuple(unquote(doc)) and - elem(unquote(doc), 0) in [:doc_cons, :doc_nest, :doc_break, :doc_group]) - end + @typep doc_force :: {:doc_force, t} + defmacrop doc_force(group) do + quote do: {:doc_force, unquote(group)} + end + + @typep doc_collapse :: {:doc_collapse, pos_integer()} + defmacrop doc_collapse(count) do + quote do: {:doc_collapse, unquote(count)} end + @typep doc_color :: {:doc_color, t, IO.ANSI.ansidata()} + defmacrop doc_color(doc, color) do + quote do: {:doc_color, unquote(doc), unquote(color)} + end + + @docs [ + :doc_break, + :doc_collapse, + :doc_color, + :doc_cons, + :doc_fits, + :doc_force, + :doc_group, + :doc_nest, + :doc_string, + :doc_limit + ] + + defguard is_doc(doc) + when is_binary(doc) or doc in [:doc_nil, :doc_line] or + (is_tuple(doc) and elem(doc, 0) in @docs) + + defguardp is_limit(limit) when limit == :infinity or (is_integer(limit) and limit >= 0) + defguardp is_width(limit) when limit == :infinity or (is_integer(limit) and limit >= 0) + + # Elixir + Inspect.Opts conveniences + @doc """ - Converts an Elixir structure to an algebra document - according to the inspect protocol. + Converts an Elixir term to an algebra document + according to the `Inspect` protocol. """ - @spec to_doc(any, Inspect.Opts.t) :: t - def to_doc(%{__struct__: struct} = map, %Inspect.Opts{} = opts) when is_atom(struct) do + @spec to_doc(any, Inspect.Opts.t()) :: t + def to_doc(term, opts) + + def to_doc(%_{} = struct, %Inspect.Opts{inspect_fun: fun} = opts) do if opts.structs do try do - Inspect.inspect(map, opts) + fun.(struct, opts) rescue - e -> - res = Inspect.Map.inspect(map, opts) - raise ArgumentError, - "Got #{inspect e.__struct__} with message " <> - "\"#{Exception.message(e)}\" while inspecting #{pretty(res, opts.width)}" + caught_exception -> + # Because we try to raise a nice error message in case + # we can't inspect a struct, there is a chance the error + # message itself relies on the struct being printed, so + # we need to trap the inspected messages to guarantee + # we won't try to render any failed instruct when building + # the error message. + if Process.get(:inspect_trap) do + Inspect.Map.inspect(struct, opts) + else + try do + Process.put(:inspect_trap, true) + + inspected_struct = + struct + |> Inspect.Map.inspect(%{ + opts + | syntax_colors: [], + inspect_fun: Inspect.Opts.default_inspect_fun() + }) + |> format(opts.width) + |> IO.iodata_to_binary() + + inspect_error = + Inspect.Error.exception( + exception: caught_exception, + stacktrace: __STACKTRACE__, + inspected_struct: inspected_struct + ) + + if opts.safe do + opts = %{opts | inspect_fun: Inspect.Opts.default_inspect_fun()} + Inspect.inspect(inspect_error, opts) + else + reraise(inspect_error, __STACKTRACE__) + end + after + Process.delete(:inspect_trap) + end + end end else - Inspect.Map.inspect(map, opts) + Inspect.Map.inspect(struct, opts) + end + end + + def to_doc(arg, %Inspect.Opts{inspect_fun: fun} = opts) do + fun.(arg, opts) + end + + @doc ~S""" + Wraps `collection` in `left` and `right` according to limit and contents. + + It uses the given `left` and `right` documents as surrounding and the + separator document `separator` to separate items in `docs`. If all entries + in the collection are simple documents (texts or strings), then this function + attempts to put as much as possible on the same line. If they are not simple, + only one entry is shown per line if they do not fit. + + The limit in the given `inspect_opts` is respected and when reached this + function stops processing and outputs `"..."` instead. + + ## Options + + * `:separator` - the separator used between each doc + * `:break` - If `:strict`, always break between each element. If `:flex`, + breaks only when necessary. If `:maybe`, chooses `:flex` only if all + elements are text-based, otherwise is `:strict` + + ## Examples + + iex> inspect_opts = %Inspect.Opts{limit: :infinity} + iex> fun = fn i, _opts -> to_string(i) end + iex> doc = Inspect.Algebra.container_doc("[", Enum.to_list(1..5), "]", inspect_opts, fun) + iex> Inspect.Algebra.format(doc, 5) |> IO.iodata_to_binary() + "[1,\n 2,\n 3,\n 4,\n 5]" + + iex> inspect_opts = %Inspect.Opts{limit: 3} + iex> fun = fn i, _opts -> to_string(i) end + iex> doc = Inspect.Algebra.container_doc("[", Enum.to_list(1..5), "]", inspect_opts, fun) + iex> Inspect.Algebra.format(doc, 20) |> IO.iodata_to_binary() + "[1, 2, 3, ...]" + + iex> inspect_opts = %Inspect.Opts{limit: 3} + iex> fun = fn i, _opts -> to_string(i) end + iex> opts = [separator: "!"] + iex> doc = Inspect.Algebra.container_doc("[", Enum.to_list(1..5), "]", inspect_opts, fun, opts) + iex> Inspect.Algebra.format(doc, 20) |> IO.iodata_to_binary() + "[1! 2! 3! ...]" + + """ + @doc since: "1.6.0" + @spec container_doc(t, [any], t, Inspect.Opts.t(), (term, Inspect.Opts.t() -> t), keyword()) :: + t + def container_doc(left, collection, right, inspect_opts, fun, opts \\ []) + when is_doc(left) and is_list(collection) and is_doc(right) and is_function(fun, 2) and + is_list(opts) do + case collection do + [] -> + concat(left, right) + + _ -> + break = Keyword.get(opts, :break, :maybe) + separator = Keyword.get(opts, :separator, @container_separator) + + {docs, simple?} = + container_each(collection, inspect_opts.limit, inspect_opts, fun, [], break == :maybe) + + flex? = simple? or break == :flex + docs = fold_doc(docs, &join(&1, &2, flex?, separator)) + + case flex? do + true -> group(concat(concat(left, nest(docs, 1)), right)) + false -> group(glue(nest(glue(left, "", docs), 2), "", right)) + end end end - def to_doc(arg, %Inspect.Opts{} = opts) do - Inspect.inspect(arg, opts) + defp container_each([], _limit, _opts, _fun, acc, simple?) do + {:lists.reverse(acc), simple?} + end + + defp container_each(_, 0, _opts, _fun, acc, simple?) do + {:lists.reverse(["..." | acc]), simple?} + end + + defp container_each([term | terms], limit, opts, fun, acc, simple?) + when is_list(terms) and is_limit(limit) do + new_limit = decrement(limit) + doc = fun.(term, %{opts | limit: new_limit}) + limit = if doc == :doc_nil, do: limit, else: new_limit + container_each(terms, limit, opts, fun, [doc | acc], simple? and simple?(doc)) + end + + defp container_each([left | right], limit, opts, fun, acc, simple?) when is_limit(limit) do + limit = decrement(limit) + left = fun.(left, %{opts | limit: limit}) + right = fun.(right, %{opts | limit: limit}) + simple? = simple? and simple?(left) and simple?(right) + + doc = join(left, right, simple?, @tail_separator) + {:lists.reverse([doc | acc]), simple?} + end + + defp decrement(:infinity), do: :infinity + defp decrement(counter), do: counter - 1 + + defp join(:doc_nil, :doc_nil, _, _), do: :doc_nil + defp join(left, :doc_nil, _, _), do: left + defp join(:doc_nil, right, _, _), do: right + defp join(left, right, true, sep), do: flex_glue(concat(left, sep), right) + defp join(left, right, false, sep), do: glue(concat(left, sep), right) + + defp simple?(doc_cons(left, right)), do: simple?(left) and simple?(right) + defp simple?(doc_color(doc, _)), do: simple?(doc) + defp simple?(doc_string(_, _)), do: true + defp simple?(:doc_nil), do: true + defp simple?(other), do: is_binary(other) + + @doc false + @deprecated "Use a combination of concat/2 and nest/2 instead" + def surround(left, doc, right) when is_doc(left) and is_doc(doc) and is_doc(right) do + concat(concat(left, nest(doc, 1)), right) + end + + @doc false + @deprecated "Use Inspect.Algebra.container_doc/6 instead" + def surround_many( + left, + docs, + right, + %Inspect.Opts{} = inspect, + fun, + separator \\ @container_separator + ) + when is_doc(left) and is_list(docs) and is_doc(right) and is_function(fun, 2) do + container_doc(left, docs, right, inspect, fun, separator: separator) end + # Algebra API + @doc """ Returns a document entity used to represent nothingness. ## Examples - iex> Inspect.Algebra.empty + iex> Inspect.Algebra.empty() :doc_nil """ @spec empty() :: :doc_nil def empty, do: :doc_nil - @doc """ - Concatenates two document entities. + @doc ~S""" + Creates a document represented by string. + + While `Inspect.Algebra` accepts binaries as documents, + those are counted by binary size. On the other hand, + `string` documents are measured in terms of graphemes + towards the document size. ## Examples - iex> doc = Inspect.Algebra.concat "Tasteless", "Artosis" - iex> Inspect.Algebra.pretty(doc, 80) - "TastelessArtosis" + The following document has 10 bytes and therefore it + does not format to width 9 without breaks: + + iex> doc = Inspect.Algebra.glue("olá", " ", "mundo") + iex> doc = Inspect.Algebra.group(doc) + iex> Inspect.Algebra.format(doc, 9) + ["olá", "\n", "mundo"] + + However, if we use `string`, then the string length is + used, instead of byte size, correctly fitting: + + iex> string = Inspect.Algebra.string("olá") + iex> doc = Inspect.Algebra.glue(string, " ", "mundo") + iex> doc = Inspect.Algebra.group(doc) + iex> Inspect.Algebra.format(doc, 9) + ["olá", " ", "mundo"] """ - @spec concat(t, t) :: doc_cons - def concat(x, y) when is_doc(x) and is_doc(y) do - doc_cons(x, y) + @doc since: "1.6.0" + @spec string(String.t()) :: doc_string + def string(string) when is_binary(string) do + doc_string(string, String.length(string)) end - @doc """ - Concatenates a list of documents. + @doc ~S""" + Concatenates two document entities returning a new document. + + ## Examples + + iex> doc = Inspect.Algebra.concat("hello", "world") + iex> Inspect.Algebra.format(doc, 80) + ["hello", "world"] + """ - @spec concat([t]) :: doc_cons - def concat(docs) do - folddoc(docs, &concat(&1, &2)) + @spec concat(t, t) :: t + def concat(doc1, doc2) when is_doc(doc1) and is_doc(doc2) do + doc_cons(doc1, doc2) + end + + def no_limit(doc) do + doc_limit(doc, :infinity) end @doc ~S""" - Nests document entity `x` positions deep. + Concatenates a list of documents returning a new document. + + ## Examples - Nesting will be appended to the line breaks. + iex> doc = Inspect.Algebra.concat(["a", "b", "c"]) + iex> Inspect.Algebra.format(doc, 80) + ["a", "b", "c"] + + """ + @spec concat([t]) :: t + def concat(docs) when is_list(docs) do + fold_doc(docs, &concat(&1, &2)) + end + + @doc ~S""" + Colors a document if the `color_key` has a color in the options. + """ + @doc since: "1.4.0" + @spec color(t, Inspect.Opts.color_key(), Inspect.Opts.t()) :: t + def color(doc, color_key, %Inspect.Opts{syntax_colors: syntax_colors}) when is_doc(doc) do + if precolor = Keyword.get(syntax_colors, color_key) do + postcolor = Keyword.get(syntax_colors, :reset, :reset) + concat(doc_color(doc, precolor), doc_color(empty(), postcolor)) + else + doc + end + end + + @doc ~S""" + Nests the given document at the given `level`. + + If `level` is an integer, that's the indentation appended + to line breaks whenever they occur. If the level is `:cursor`, + the current position of the "cursor" in the document becomes + the nesting. If the level is `:reset`, it is set back to 0. + + `mode` can be `:always`, which means nesting always happen, + or `:break`, which means nesting only happens inside a group + that has been broken. ## Examples iex> doc = Inspect.Algebra.nest(Inspect.Algebra.glue("hello", "world"), 5) - iex> Inspect.Algebra.pretty(doc, 5) - "hello\n world" + iex> doc = Inspect.Algebra.group(doc) + iex> Inspect.Algebra.format(doc, 5) + ["hello", "\n ", "world"] """ - @spec nest(t, non_neg_integer) :: doc_nest - def nest(x, 0) when is_doc(x) do - x + @spec nest(t, non_neg_integer | :cursor | :reset, :always | :break) :: doc_nest + def nest(doc, level, mode \\ :always) + + def nest(doc, :cursor, mode) when is_doc(doc) and mode in [:always, :break] do + doc_nest(doc, :cursor, mode) + end + + def nest(doc, :reset, mode) when is_doc(doc) and mode in [:always, :break] do + doc_nest(doc, :reset, mode) + end + + def nest(doc, 0, _mode) when is_doc(doc) do + doc end - def nest(x, i) when is_doc(x) and is_integer(i) do - doc_nest(x, i) + def nest(doc, level, mode) + when is_doc(doc) and is_integer(level) and level > 0 and mode in [:always, :break] do + doc_nest(doc, level, mode) end @doc ~S""" - Document entity representing a break. + Returns a break document based on the given `string`. - This break can be rendered as a linebreak or as spaces, - depending on the `mode` of the chosen layout or the provided - separator. + This break can be rendered as a linebreak or as the given `string`, + depending on the `mode` of the chosen layout. ## Examples - Let's glue two docs together with a break and then render it: + Let's create a document by concatenating two strings with a break between + them: - iex> doc = Inspect.Algebra.glue("a", " ", "b") - iex> Inspect.Algebra.pretty(doc, 80) - "a b" + iex> doc = Inspect.Algebra.concat(["a", Inspect.Algebra.break("\t"), "b"]) + iex> Inspect.Algebra.format(doc, 80) + ["a", "\t", "b"] - Notice the break was represented as is, because we haven't reached - a line limit. Once we do, it is replaced by a newline: + Note that the break was represented with the given string, because we didn't + reach a line limit. Once we do, it is replaced by a newline: - iex> doc = Inspect.Algebra.glue(String.duplicate("a", 20), " ", "b") - iex> Inspect.Algebra.pretty(doc, 10) - "aaaaaaaaaaaaaaaaaaaa\nb" + iex> break = Inspect.Algebra.break("\t") + iex> doc = Inspect.Algebra.concat([String.duplicate("a", 20), break, "b"]) + iex> doc = Inspect.Algebra.group(doc) + iex> Inspect.Algebra.format(doc, 10) + ["aaaaaaaaaaaaaaaaaaaa", "\n", "b"] """ @spec break(binary) :: doc_break - def break(s) when is_binary(s), do: doc_break(s) + def break(string \\ " ") when is_binary(string) do + doc_break(string, :strict) + end - @spec break() :: doc_break - def break(), do: doc_break(@break) + @doc """ + Collapse any new lines and whitespace following this + node, emitting up to `max` new lines. + """ + @doc since: "1.6.0" + @spec collapse_lines(pos_integer) :: doc_collapse + def collapse_lines(max) when is_integer(max) and max > 0 do + doc_collapse(max) + end @doc """ - Inserts a break between two docs. See `break/1` for more info. + Considers the next break as fit. + + `mode` can be `:enabled` or `:disabled`. When `:enabled`, + it will consider the document as fit as soon as it finds + the next break, effectively cancelling the break. It will + also ignore any `force_unfit/1` in search of the next break. + + When disabled, it behaves as usual and it will ignore + any further `next_break_fits/2` instruction. + + ## Examples + + This is used by Elixir's code formatter to avoid breaking + code at some specific locations. For example, consider this + code: + + some_function_call(%{..., key: value, ...}) + + Now imagine that this code does not fit its line. The code + formatter introduces breaks inside `(` and `)` and inside + `%{` and `}`. Therefore the document would break as: + + some_function_call( + %{ + ..., + key: value, + ... + } + ) + + The formatter wraps the algebra document representing the + map in `next_break_fits/1` so the code is formatted as: + + some_function_call(%{ + ..., + key: value, + ... + }) + """ - @spec glue(t, t) :: doc_cons - def glue(x, y), do: concat(x, concat(break, y)) + @doc since: "1.6.0" + @spec next_break_fits(t, :enabled | :disabled) :: doc_fits + def next_break_fits(doc, mode \\ @next_break_fits) + when is_doc(doc) and mode in [:enabled, :disabled] do + doc_fits(doc, mode) + end @doc """ - Inserts a break, passed as the second argument, between two docs, - the first and the third arguments. + Forces the current group to be unfit. """ - @spec glue(t, binary, t) :: doc_cons - def glue(x, g, y) when is_binary(g), do: concat(x, concat(break(g), y)) + @doc since: "1.6.0" + @spec force_unfit(t) :: doc_force + def force_unfit(doc) when is_doc(doc) do + doc_force(doc) + end + + @doc """ + Returns a flex break document based on the given `string`. + + A flex break still causes a group to break, like `break/1`, + but it is re-evaluated when the documented is rendered. + + For example, take a group document represented as `[1, 2, 3]` + where the space after every comma is a break. When the document + above does not fit a single line, all breaks are enabled, + causing the document to be rendered as: + + [1, + 2, + 3] + + However, if flex breaks are used, then each break is re-evaluated + when rendered, so the document could be possible rendered as: + + [1, 2, + 3] + + Hence the name "flex". they are more flexible when it comes + to the document fitting. On the other hand, they are more expensive + since each break needs to be re-evaluated. + + This function is used by `container_doc/6` and friends to the + maximum number of entries on the same line. + """ + @doc since: "1.6.0" + @spec flex_break(binary) :: doc_break + def flex_break(string \\ " ") when is_binary(string) do + doc_break(string, :flex) + end + + @doc """ + Glues two documents (`doc1` and `doc2`) inserting a + `flex_break/1` given by `break_string` between them. + + This function is used by `container_doc/6` and friends + to the maximum number of entries on the same line. + """ + @doc since: "1.6.0" + @spec flex_glue(t, binary, t) :: t + def flex_glue(doc1, break_string \\ " ", doc2) when is_binary(break_string) do + concat(doc1, concat(flex_break(break_string), doc2)) + end @doc ~S""" - Returns a group containing the specified document. + Glues two documents (`doc1` and `doc2`) inserting the given + break `break_string` between them. + + For more information on how the break is inserted, see `break/1`. ## Examples - iex> doc = Inspect.Algebra.group( - ...> Inspect.Algebra.concat( - ...> Inspect.Algebra.group( - ...> Inspect.Algebra.concat( - ...> "Hello,", + iex> doc = Inspect.Algebra.glue("hello", "world") + iex> Inspect.Algebra.format(doc, 80) + ["hello", " ", "world"] + + iex> doc = Inspect.Algebra.glue("hello", "\t", "world") + iex> Inspect.Algebra.format(doc, 80) + ["hello", "\t", "world"] + + """ + @spec glue(t, binary, t) :: t + def glue(doc1, break_string \\ " ", doc2) when is_binary(break_string) do + concat(doc1, concat(break(break_string), doc2)) + end + + @doc ~S""" + Returns a group containing the specified document `doc`. + + Documents in a group are attempted to be rendered together + to the best of the renderer ability. + + The group mode can also be set to `:inherit`, which means it + automatically breaks if the parent group has broken too. + + ## Examples + + iex> doc = + ...> Inspect.Algebra.group( + ...> Inspect.Algebra.concat( + ...> Inspect.Algebra.group( ...> Inspect.Algebra.concat( - ...> Inspect.Algebra.break, - ...> "A" + ...> "Hello,", + ...> Inspect.Algebra.concat( + ...> Inspect.Algebra.break(), + ...> "A" + ...> ) ...> ) + ...> ), + ...> Inspect.Algebra.concat( + ...> Inspect.Algebra.break(), + ...> "B" ...> ) - ...> ), - ...> Inspect.Algebra.concat( - ...> Inspect.Algebra.break, - ...> "B" ...> ) - ...> )) - iex> Inspect.Algebra.pretty(doc, 80) - "Hello, A B" - iex> Inspect.Algebra.pretty(doc, 6) - "Hello,\nA B" + ...> ) + iex> Inspect.Algebra.format(doc, 80) + ["Hello,", " ", "A", " ", "B"] + iex> Inspect.Algebra.format(doc, 6) + ["Hello,", "\n", "A", "\n", "B"] """ - @spec group(t) :: doc_group - def group(d) when is_doc(d) do - doc_group(d) + @spec group(t, :self | :inherit) :: doc_group + def group(doc, mode \\ :self) when is_doc(doc) do + doc_group(doc, mode) end - @doc """ - Inserts a mandatory single space between two document entities. + @doc ~S""" + Inserts a mandatory single space between two documents. ## Examples - iex> doc = Inspect.Algebra.space "Hughes", "Wadler" - iex> Inspect.Algebra.pretty(doc, 80) - "Hughes Wadler" + iex> doc = Inspect.Algebra.space("Hughes", "Wadler") + iex> Inspect.Algebra.format(doc, 5) + ["Hughes", " ", "Wadler"] """ - @spec space(t, t) :: doc_cons - def space(x, y), do: concat(x, concat(" ", y)) + @spec space(t, t) :: t + def space(doc1, doc2), do: concat(doc1, concat(" ", doc2)) @doc ~S""" - Inserts a mandatory linebreak between two document entities. + A mandatory linebreak. + + A group with linebreaks will fit if all lines in the group fit. ## Examples - iex> doc = Inspect.Algebra.line "Hughes", "Wadler" - iex> Inspect.Algebra.pretty(doc, 80) - "Hughes\nWadler" + iex> doc = + ...> Inspect.Algebra.concat( + ...> Inspect.Algebra.concat( + ...> "Hughes", + ...> Inspect.Algebra.line() + ...> ), + ...> "Wadler" + ...> ) + iex> Inspect.Algebra.format(doc, 80) + ["Hughes", "\n", "Wadler"] """ - @spec line(t, t) :: doc_cons - def line(x, y), do: concat(x, concat(:doc_line, y)) + @doc since: "1.6.0" + @spec line() :: t + def line(), do: :doc_line - @doc """ - Folds a list of document entities into a document entity - using a function that is passed as the first argument. + @doc ~S""" + Inserts a mandatory linebreak between two documents. + + See `line/0`. ## Examples - iex> doc = ["A", "B"] - iex> doc = Inspect.Algebra.folddoc(doc, fn(x,y) -> - ...> Inspect.Algebra.concat [x, "!", y] - ...> end) - iex> Inspect.Algebra.pretty(doc, 80) - "A!B" + iex> doc = Inspect.Algebra.line("Hughes", "Wadler") + iex> Inspect.Algebra.format(doc, 80) + ["Hughes", "\n", "Wadler"] """ - @spec folddoc([t], ((t, t) -> t)) :: t - def folddoc([], _), do: empty - def folddoc([doc], _), do: doc - def folddoc([d|ds], f), do: f.(d, folddoc(ds, f)) - - # Elixir conveniences + @spec line(t, t) :: t + def line(doc1, doc2), do: concat(doc1, concat(line(), doc2)) @doc ~S""" - Surrounds a document with characters. + Folds a list of documents into a document using the given folder function. - Puts the document between left and right enclosing and nesting it. - The document is marked as a group, to show the maximum as possible - concisely together. + The list of documents is folded "from the right"; in that, this function is + similar to `List.foldr/3`, except that it doesn't expect an initial + accumulator and uses the last element of `docs` as the initial accumulator. ## Examples - iex> doc = Inspect.Algebra.surround "[", Inspect.Algebra.glue("a", "b"), "]" - iex> Inspect.Algebra.pretty(doc, 3) - "[a\n b]" + iex> docs = ["A", "B", "C"] + iex> docs = + ...> Inspect.Algebra.fold_doc(docs, fn doc, acc -> + ...> Inspect.Algebra.concat([doc, "!", acc]) + ...> end) + iex> Inspect.Algebra.format(docs, 80) + ["A", "!", "B", "!", "C"] """ - @spec surround(binary, t, binary) :: t - def surround(left, doc, right) do - group concat left, concat(nest(doc, @nesting), right) - end + @spec fold_doc([t], (t, t -> t)) :: t + def fold_doc(docs, folder_fun) + + def fold_doc([], _folder_fun), do: empty() + def fold_doc([doc], _folder_fun), do: doc + + def fold_doc([doc | docs], folder_fun) when is_function(folder_fun, 2), + do: folder_fun.(doc, fold_doc(docs, folder_fun)) @doc ~S""" - Maps and glues a collection of items together using the given separator - and surrounds them. A limit can be passed which, once reached, stops - gluing and outputs "..." instead. + Formats a given document for a given width. - ## Examples + Takes the maximum width and a document to print as its arguments + and returns an IO data representation of the best layout for the + document to fit in the given width. - iex> doc = Inspect.Algebra.surround_many("[", Enum.to_list(1..5), "]", :infinity, &Integer.to_string(&1)) - iex> Inspect.Algebra.pretty(doc, 5) - "[1,\n 2,\n 3,\n 4,\n 5]" + The document starts flat (without breaks) until a group is found. - iex> doc = Inspect.Algebra.surround_many("[", Enum.to_list(1..5), "]", 3, &Integer.to_string(&1)) - iex> Inspect.Algebra.pretty(doc, 20) - "[1, 2, 3, ...]" + ## Examples - iex> doc = Inspect.Algebra.surround_many("[", Enum.to_list(1..5), "]", 3, &Integer.to_string(&1), "!") - iex> Inspect.Algebra.pretty(doc, 20) - "[1! 2! 3! ...]" + iex> doc = Inspect.Algebra.glue("hello", " ", "world") + iex> doc = Inspect.Algebra.group(doc) + iex> doc |> Inspect.Algebra.format(30) |> IO.iodata_to_binary() + "hello world" + iex> doc |> Inspect.Algebra.format(10) |> IO.iodata_to_binary() + "hello\nworld" """ - @spec surround_many(binary, [any], binary, integer | :infinity, (term -> t), binary) :: t - def surround_many(left, docs, right, limit, fun, separator \\ @surround_separator) - - def surround_many(left, [], right, _, _fun, _) do - concat(left, right) + @spec format(t, non_neg_integer | :infinity) :: iodata + def format(doc, width) when is_doc(doc) and is_width(width) do + format(width, 0, [{0, :flat, doc}]) end - def surround_many(left, docs, right, limit, fun, sep) do - surround(left, surround_many(docs, limit, fun, sep), right) - end + # Type representing the document mode to be rendered: + # + # * flat - represents a document with breaks as flats (a break may fit, as it may break) + # * break - represents a document with breaks as breaks (a break always fits, since it breaks) + # + # The following modes are exclusive to fitting: + # + # * flat_no_break - represents a document with breaks as flat not allowed to enter in break mode + # * break_no_flat - represents a document with breaks as breaks not allowed to enter in flat mode + # + @typep mode :: :flat | :flat_no_break | :break | :break_no_flat + + @spec fits?( + width :: non_neg_integer() | :infinity, + column :: non_neg_integer(), + break? :: boolean(), + entries + ) :: boolean() + when entries: + maybe_improper_list({integer(), mode(), t()}, {:tail, boolean(), entries} | []) + + # We need at least a break to consider the document does not fit since a + # large document without breaks has no option but fitting its current line. + # + # In case we have groups and the group fits, we need to consider the group + # parent without the child breaks, hence {:tail, b?, t} below. + defp fits?(w, k, b?, _) when k > w and b?, do: false + defp fits?(_, _, _, []), do: true + defp fits?(w, k, _, {:tail, b?, t}), do: fits?(w, k, b?, t) + + ## Flat no break + + defp fits?(w, k, b?, [{i, _, doc_fits(x, :disabled)} | t]), + do: fits?(w, k, b?, [{i, :flat_no_break, x} | t]) + + defp fits?(w, k, b?, [{i, :flat_no_break, doc_fits(x, _)} | t]), + do: fits?(w, k, b?, [{i, :flat_no_break, x} | t]) + + ## Breaks no flat + + defp fits?(w, k, b?, [{i, _, doc_fits(x, :enabled)} | t]), + do: fits?(w, k, b?, [{i, :break_no_flat, x} | t]) + + defp fits?(w, k, b?, [{i, :break_no_flat, doc_force(x)} | t]), + do: fits?(w, k, b?, [{i, :break_no_flat, x} | t]) + + defp fits?(_, _, _, [{_, :break_no_flat, doc_break(_, _)} | _]), do: true + defp fits?(_, _, _, [{_, :break_no_flat, :doc_line} | _]), do: true + + ## Breaks + + defp fits?(_, _, _, [{_, :break, doc_break(_, _)} | _]), do: true + defp fits?(_, _, _, [{_, :break, :doc_line} | _]), do: true + + defp fits?(w, k, b?, [{i, :break, doc_group(x, _)} | t]), + do: fits?(w, k, b?, [{i, :flat, x} | {:tail, b?, t}]) + + ## Catch all + + defp fits?(w, _, _, [{i, _, :doc_line} | t]), do: fits?(w, i, false, t) + defp fits?(w, k, b?, [{_, _, :doc_nil} | t]), do: fits?(w, k, b?, t) + defp fits?(w, _, b?, [{i, _, doc_collapse(_)} | t]), do: fits?(w, i, b?, t) + defp fits?(w, k, b?, [{i, m, doc_color(x, _)} | t]), do: fits?(w, k, b?, [{i, m, x} | t]) + defp fits?(w, k, b?, [{_, _, doc_string(_, l)} | t]), do: fits?(w, k + l, b?, t) + defp fits?(w, k, b?, [{_, _, s} | t]) when is_binary(s), do: fits?(w, k + byte_size(s), b?, t) + defp fits?(_, _, _, [{_, _, doc_force(_)} | _]), do: false + defp fits?(w, k, _, [{_, _, doc_break(s, _)} | t]), do: fits?(w, k + byte_size(s), true, t) + defp fits?(w, k, b?, [{i, m, doc_nest(x, _, :break)} | t]), do: fits?(w, k, b?, [{i, m, x} | t]) + + defp fits?(w, k, b?, [{i, m, doc_nest(x, j, _)} | t]), + do: fits?(w, k, b?, [{apply_nesting(i, k, j), m, x} | t]) + + defp fits?(w, k, b?, [{i, m, doc_cons(x, y)} | t]), + do: fits?(w, k, b?, [{i, m, x}, {i, m, y} | t]) + + defp fits?(w, k, b?, [{i, m, doc_group(x, _)} | t]), + do: fits?(w, k, b?, [{i, m, x} | {:tail, b?, t}]) + + defp fits?(w, k, b?, [{i, m, doc_limit(x, :infinity)} | t]) when w != :infinity, + do: fits?(:infinity, k, b?, [{i, :flat, x}, {i, m, doc_limit(empty(), w)} | t]) + + defp fits?(_w, k, b?, [{i, m, doc_limit(x, w)} | t]), + do: fits?(w, k, b?, [{i, m, x} | t]) + + @spec format( + width :: non_neg_integer() | :infinity, + column :: non_neg_integer(), + [{integer, mode, t}] + ) :: [binary] + defp format(_, _, []), do: [] + defp format(w, k, [{_, _, :doc_nil} | t]), do: format(w, k, t) + defp format(w, _, [{i, _, :doc_line} | t]), do: [indent(i) | format(w, i, t)] + defp format(w, k, [{i, m, doc_cons(x, y)} | t]), do: format(w, k, [{i, m, x}, {i, m, y} | t]) + defp format(w, k, [{i, m, doc_color(x, c)} | t]), do: [ansi(c) | format(w, k, [{i, m, x} | t])] + defp format(w, k, [{_, _, doc_string(s, l)} | t]), do: [s | format(w, k + l, t)] + defp format(w, k, [{_, _, s} | t]) when is_binary(s), do: [s | format(w, k + byte_size(s), t)] + defp format(w, k, [{i, m, doc_force(x)} | t]), do: format(w, k, [{i, m, x} | t]) + defp format(w, k, [{i, m, doc_fits(x, _)} | t]), do: format(w, k, [{i, m, x} | t]) + defp format(w, _, [{i, _, doc_collapse(max)} | t]), do: collapse(format(w, i, t), max, 0, i) + + # Flex breaks are not conditional to the mode + defp format(w, k, [{i, m, doc_break(s, :flex)} | t]) do + k = k + byte_size(s) - defp surround_many(_, 0, _fun, _sep) do - "..." + if w == :infinity or m == :flat or fits?(w, k, true, t) do + [s | format(w, k, t)] + else + [indent(i) | format(w, i, t)] + end end - defp surround_many([h], _limit, fun, _sep) do - fun.(h) + # Strict breaks are conditional to the mode + defp format(w, k, [{i, mode, doc_break(s, :strict)} | t]) do + if mode == :break do + [indent(i) | format(w, i, t)] + else + [s | format(w, k + byte_size(s), t)] + end end - defp surround_many([h|t], limit, fun, sep) when is_list(t) do - glue( - concat(fun.(h), sep), - surround_many(t, decrement(limit), fun, sep) - ) + # Nesting is conditional to the mode. + defp format(w, k, [{i, mode, doc_nest(x, j, nest)} | t]) do + if nest == :always or (nest == :break and mode == :break) do + format(w, k, [{apply_nesting(i, k, j), mode, x} | t]) + else + format(w, k, [{i, mode, x} | t]) + end end - defp surround_many([h|t], _limit, fun, _sep) do - glue( - concat(fun.(h), @tail_separator), - fun.(t) - ) + # Groups must do the fitting decision. + defp format(w, k, [{i, :break, doc_group(x, :inherit)} | t]) do + format(w, k, [{i, :break, x} | t]) end - defp decrement(:infinity), do: :infinity - defp decrement(counter), do: counter - 1 - - @doc """ - The pretty printing function. + defp format(w, k, [{i, _, doc_group(x, _)} | t]) do + if w == :infinity or fits?(w, k, false, [{i, :flat, x}]) do + format(w, k, [{i, :flat, x} | t]) + else + format(w, k, [{i, :break, x} | t]) + end + end - Takes the maximum width and a document to print as its arguments - and returns the string representation of the best layout for the - document to fit in the given width. - """ - @spec pretty(t, non_neg_integer | :infinity) :: binary - def pretty(d, w) do - sdoc = format w, 0, [{0, default_mode(w), doc_group(d)}] - render(sdoc) + # Limit is set to infinity and then reverts + defp format(w, k, [{i, m, doc_limit(x, :infinity)} | t]) when w != :infinity do + format(:infinity, k, [{i, :flat, x}, {i, m, doc_limit(empty(), w)} | t]) end - defp default_mode(:infinity), do: :flat - defp default_mode(_), do: :break + defp format(_w, k, [{i, m, doc_limit(x, w)} | t]) do + format(w, k, [{i, m, x} | t]) + end - # Rendering and internal helpers + defp collapse(["\n" <> _ | t], max, count, i) do + collapse(t, max, count + 1, i) + end - # Record representing the document mode to be rendered: flat or broken - @typep mode :: :flat | :break + defp collapse(["" | t], max, count, i) do + collapse(t, max, count, i) + end - @doc false - @spec fits?(integer, [{integer, mode, t}]) :: boolean - def fits?(w, _) when w < 0, do: false - def fits?(_, []), do: true - def fits?(_, [{_, _, :doc_line} | _]), do: true - def fits?(w, [{_, _, :doc_nil} | t]), do: fits?(w, t) - def fits?(w, [{i, m, doc_cons(x, y)} | t]), do: fits?(w, [{i, m, x} | [{i, m, y} | t]]) - def fits?(w, [{i, m, doc_nest(x, j)} | t]), do: fits?(w, [{i + j, m, x} | t]) - def fits?(w, [{i, _, doc_group(x)} | t]), do: fits?(w, [{i, :flat, x} | t]) - def fits?(w, [{_, _, s} | t]) when is_binary(s), do: fits?((w - byte_size s), t) - def fits?(w, [{_, :flat, doc_break(s)} | t]), do: fits?((w - byte_size s), t) - def fits?(_, [{_, :break, doc_break(_)} | _]), do: true + defp collapse(t, max, count, i) do + [:binary.copy("\n", min(max, count)) <> :binary.copy(" ", i) | t] + end - @doc false - @spec format(integer | :infinity, integer, [{integer, mode, t}]) :: [binary] - def format(_, _, []), do: [] - def format(w, _, [{i, _, :doc_line} | t]), do: [indent(i) | format(w, i, t)] - def format(w, k, [{_, _, :doc_nil} | t]), do: format(w, k, t) - def format(w, k, [{i, m, doc_cons(x, y)} | t]), do: format(w, k, [{i, m, x} | [{i, m, y} | t]]) - def format(w, k, [{i, m, doc_nest(x, j)} | t]), do: format(w, k, [{i + j, m, x} | t]) - def format(w, k, [{i, m, doc_group(x)} | t]), do: format(w, k, [{i, m, x} | t]) - def format(w, k, [{_, _, s} | t]) when is_binary(s), do: [s | format(w, (k + byte_size s), t)] - def format(w, k, [{_, :flat, doc_break(s)} | t]), do: [s | format(w, (k + byte_size s), t)] - def format(w, k, [{i, :break, doc_break(s)} | t]) do - k = k + byte_size(s) + defp apply_nesting(_, k, :cursor), do: k + defp apply_nesting(_, _, :reset), do: 0 + defp apply_nesting(i, _, j), do: i + j - if w == :infinity or fits?(w - k, t) do - [s | format(w, k, t)] - else - [indent(i) | format(w, i, t)] - end + defp ansi(color) do + IO.ANSI.format_fragment(color, true) end defp indent(0), do: @newline defp indent(i), do: @newline <> :binary.copy(" ", i) - - @doc false - @spec render([binary]) :: binary - def render(sdoc) do - IO.iodata_to_binary sdoc - end end diff --git a/lib/elixir/lib/integer.ex b/lib/elixir/lib/integer.ex index 85177480fae..bd60a8ca62e 100644 --- a/lib/elixir/lib/integer.ex +++ b/lib/elixir/lib/integer.ex @@ -1,78 +1,323 @@ defmodule Integer do @moduledoc """ Functions for working with integers. + + Some functions that work on integers are found in `Kernel`: + + * `Kernel.abs/1` + * `Kernel.div/2` + * `Kernel.max/2` + * `Kernel.min/2` + * `Kernel.rem/2` + """ import Bitwise @doc """ - Determines if an integer is odd. + Determines if `integer` is odd. + + Returns `true` if the given `integer` is an odd number, + otherwise it returns `false`. + + Allowed in guard clauses. + + ## Examples + + iex> Integer.is_odd(5) + true + + iex> Integer.is_odd(6) + false + + iex> Integer.is_odd(-5) + true + + iex> Integer.is_odd(0) + false + + """ + defguard is_odd(integer) when is_integer(integer) and (integer &&& 1) == 1 + + @doc """ + Determines if an `integer` is even. + + Returns `true` if the given `integer` is an even number, + otherwise it returns `false`. + + Allowed in guard clauses. + + ## Examples + + iex> Integer.is_even(10) + true + + iex> Integer.is_even(5) + false + + iex> Integer.is_even(-10) + true + + iex> Integer.is_even(0) + true + + """ + defguard is_even(integer) when is_integer(integer) and (integer &&& 1) == 0 + + @doc """ + Computes `base` raised to power of `exponent`. + + Both `base` and `exponent` must be integers. + The exponent must be zero or positive. + + See `Float.pow/2` for exponentiation of negative + exponents as well as floats. + + ## Examples + + iex> Integer.pow(2, 0) + 1 + iex> Integer.pow(2, 1) + 2 + iex> Integer.pow(2, 10) + 1024 + iex> Integer.pow(2, 11) + 2048 + iex> Integer.pow(2, 64) + 0x10000000000000000 + + iex> Integer.pow(3, 4) + 81 + iex> Integer.pow(4, 3) + 64 + + iex> Integer.pow(-2, 3) + -8 + iex> Integer.pow(-2, 4) + 16 + + iex> Integer.pow(2, -2) + ** (ArithmeticError) bad argument in arithmetic expression + + """ + @doc since: "1.12.0" + @spec pow(integer, non_neg_integer) :: integer + def pow(base, exponent) when is_integer(base) and is_integer(exponent) do + if exponent < 0, do: :erlang.error(:badarith, [base, exponent]) + base ** exponent + end + + @doc """ + Computes the modulo remainder of an integer division. + + This function performs a [floored division](`floor_div/2`), which means that + the result will always have the sign of the `divisor`. + + Raises an `ArithmeticError` exception if one of the arguments is not an + integer, or when the `divisor` is `0`. + + ## Examples + + iex> Integer.mod(5, 2) + 1 + iex> Integer.mod(6, -4) + -2 - Returns `true` if `n` is an odd number, otherwise `false`. - Implemented as a macro so it is allowed in guard clauses. """ - defmacro odd?(n) do - quote do: (unquote(n) &&& 1) == 1 + @doc since: "1.4.0" + @spec mod(integer, neg_integer | pos_integer) :: integer + def mod(dividend, divisor) do + remainder = rem(dividend, divisor) + + if remainder * divisor < 0 do + remainder + divisor + else + remainder + end end @doc """ - Determines if an integer is even. + Performs a floored integer division. + + Raises an `ArithmeticError` exception if one of the arguments is not an + integer, or when the `divisor` is `0`. + + This function performs a *floored* integer division, which means that + the result will always be rounded towards negative infinity. + + If you want to perform truncated integer division (rounding towards zero), + use `Kernel.div/2` instead. + + ## Examples + + iex> Integer.floor_div(5, 2) + 2 + iex> Integer.floor_div(6, -4) + -2 + iex> Integer.floor_div(-99, 2) + -50 - Returns `true` if `n` is an even number, otherwise `false`. - Implemented as a macro so it is allowed in guard clauses. """ - defmacro even?(n) do - quote do: (unquote(n) &&& 1) == 0 + @doc since: "1.4.0" + @spec floor_div(integer, neg_integer | pos_integer) :: integer + def floor_div(dividend, divisor) do + if dividend * divisor < 0 and rem(dividend, divisor) != 0 do + div(dividend, divisor) - 1 + else + div(dividend, divisor) + end end @doc """ - Converts a binary to an integer. + Returns the ordered digits for the given `integer`. + + An optional `base` value may be provided representing the radix for the returned + digits. This one must be an integer >= 2. + + ## Examples + + iex> Integer.digits(123) + [1, 2, 3] + + iex> Integer.digits(170, 2) + [1, 0, 1, 0, 1, 0, 1, 0] + + iex> Integer.digits(-170, 2) + [-1, 0, -1, 0, -1, 0, -1, 0] + + """ + @spec digits(integer, pos_integer) :: [integer, ...] + def digits(integer, base \\ 10) + when is_integer(integer) and is_integer(base) and base >= 2 do + do_digits(integer, base, []) + end + + defp do_digits(integer, base, acc) when abs(integer) < base, do: [integer | acc] + + defp do_digits(integer, base, acc), + do: do_digits(div(integer, base), base, [rem(integer, base) | acc]) + + @doc """ + Returns the integer represented by the ordered `digits`. + + An optional `base` value may be provided representing the radix for the `digits`. + Base has to be an integer greater than or equal to `2`. + + ## Examples - If successful, returns a tuple of the form `{integer, remainder_of_binary}`. + iex> Integer.undigits([1, 2, 3]) + 123 + + iex> Integer.undigits([1, 4], 16) + 20 + + iex> Integer.undigits([]) + 0 + + """ + @spec undigits([integer], pos_integer) :: integer + def undigits(digits, base \\ 10) when is_list(digits) and is_integer(base) and base >= 2 do + do_undigits(digits, base, 0) + end + + defp do_undigits([], _base, acc), do: acc + + defp do_undigits([digit | _], base, _) when is_integer(digit) and digit >= base, + do: raise(ArgumentError, "invalid digit #{digit} in base #{base}") + + defp do_undigits([digit | tail], base, acc) when is_integer(digit), + do: do_undigits(tail, base, acc * base + digit) + + @doc """ + Parses a text representation of an integer. + + An optional `base` to the corresponding integer can be provided. + If `base` is not given, 10 will be used. + + If successful, returns a tuple in the form of `{integer, remainder_of_binary}`. Otherwise `:error`. + Raises an error if `base` is less than 2 or more than 36. + + If you want to convert a string-formatted integer directly to an integer, + `String.to_integer/1` or `String.to_integer/2` can be used instead. + ## Examples iex> Integer.parse("34") - {34,""} + {34, ""} iex> Integer.parse("34.5") - {34,".5"} + {34, ".5"} iex> Integer.parse("three") :error + iex> Integer.parse("34", 10) + {34, ""} + + iex> Integer.parse("f4", 16) + {244, ""} + + iex> Integer.parse("Awww++", 36) + {509216, "++"} + + iex> Integer.parse("fab", 10) + :error + + iex> Integer.parse("a2", 38) + ** (ArgumentError) invalid base 38 + """ - @spec parse(binary) :: {integer, binary} | :error - def parse(<< ?-, bin :: binary >>) do - case do_parse(bin) do - :error -> :error - {number, remainder} -> {-number, remainder} + @spec parse(binary, 2..36) :: {integer, remainder_of_binary :: binary} | :error + def parse(binary, base \\ 10) + + def parse(_binary, base) when base not in 2..36 do + raise ArgumentError, "invalid base #{inspect(base)}" + end + + def parse(binary, base) when is_binary(binary) do + case count_digits(binary, base) do + 0 -> + :error + + count -> + {digits, rem} = :erlang.split_binary(binary, count) + {:erlang.binary_to_integer(digits, base), rem} end end - def parse(<< ?+, bin :: binary >>) do - do_parse(bin) + defp count_digits(<>, base) when sign in '+-' do + case count_digits_nosign(rest, base, 1) do + 1 -> 0 + count -> count + end end - def parse(bin) when is_binary(bin) do - do_parse(bin) + defp count_digits(<>, base) do + count_digits_nosign(rest, base, 0) end - defp do_parse(<< char, bin :: binary >>) when char in ?0..?9, do: do_parse(bin, char - ?0) - defp do_parse(_), do: :error + digits = [{?0..?9, -?0}, {?A..?Z, 10 - ?A}, {?a..?z, 10 - ?a}] - defp do_parse(<< char, rest :: binary >>, acc) when char in ?0..?9 do - do_parse rest, 10 * acc + (char - ?0) - end + for {chars, diff} <- digits, + char <- chars do + digit = char + diff - defp do_parse(bitstring, acc) do - {acc, bitstring} + defp count_digits_nosign(<>, base, count) + when base > unquote(digit) do + count_digits_nosign(rest, base, count + 1) + end end + defp count_digits_nosign(<<_::bits>>, _, count), do: count + @doc """ Returns a binary which corresponds to the text representation - of `some_integer`. + of `integer` in the given `base`. + + `base` can be an integer between 2 and 36. If no `base` is given, + it defaults to `10`. Inlined by the compiler. @@ -81,59 +326,164 @@ defmodule Integer do iex> Integer.to_string(123) "123" + iex> Integer.to_string(+456) + "456" + + iex> Integer.to_string(-789) + "-789" + + iex> Integer.to_string(0123) + "123" + + iex> Integer.to_string(100, 16) + "64" + + iex> Integer.to_string(-100, 16) + "-64" + + iex> Integer.to_string(882_681_651, 36) + "ELIXIR" + """ - @spec to_string(integer) :: String.t - def to_string(some_integer) do - :erlang.integer_to_binary(some_integer) + @spec to_string(integer, 2..36) :: String.t() + def to_string(integer, base \\ 10) do + :erlang.integer_to_binary(integer, base) end @doc """ - Returns a binary which corresponds to the text representation - of `some_integer` in base `base`. + Returns a charlist which corresponds to the text representation + of `integer` in the given `base`. + + `base` can be an integer between 2 and 36. If no `base` is given, + it defaults to `10`. Inlined by the compiler. ## Examples - iex> Integer.to_string(100, 16) - "64" + iex> Integer.to_charlist(123) + '123' + + iex> Integer.to_charlist(+456) + '456' + + iex> Integer.to_charlist(-789) + '-789' + + iex> Integer.to_charlist(0123) + '123' + + iex> Integer.to_charlist(100, 16) + '64' + + iex> Integer.to_charlist(-100, 16) + '-64' + + iex> Integer.to_charlist(882_681_651, 36) + 'ELIXIR' """ - @spec to_string(integer, pos_integer) :: String.t - def to_string(some_integer, base) do - :erlang.integer_to_binary(some_integer, base) + @spec to_charlist(integer, 2..36) :: charlist + def to_charlist(integer, base \\ 10) do + :erlang.integer_to_list(integer, base) end @doc """ - Returns a char list which corresponds to the text representation of the given integer. + Returns the greatest common divisor of the two given integers. - Inlined by the compiler. + The greatest common divisor (GCD) of `integer1` and `integer2` is the largest positive + integer that divides both `integer1` and `integer2` without leaving a remainder. + + By convention, `gcd(0, 0)` returns `0`. ## Examples - iex> Integer.to_char_list(7) - '7' + iex> Integer.gcd(2, 3) + 1 + + iex> Integer.gcd(8, 12) + 4 + + iex> Integer.gcd(8, -12) + 4 + + iex> Integer.gcd(10, 0) + 10 + + iex> Integer.gcd(7, 7) + 7 + + iex> Integer.gcd(0, 0) + 0 """ - @spec to_char_list(integer) :: list - def to_char_list(number) do - :erlang.integer_to_list(number) + @doc since: "1.5.0" + @spec gcd(integer, integer) :: non_neg_integer + def gcd(integer1, integer2) when is_integer(integer1) and is_integer(integer2) do + gcd_positive(abs(integer1), abs(integer2)) end + defp gcd_positive(0, integer2), do: integer2 + defp gcd_positive(integer1, 0), do: integer1 + defp gcd_positive(integer1, integer2), do: gcd_positive(integer2, rem(integer1, integer2)) + @doc """ - Returns a char list which corresponds to the text representation of the - given integer in the given case. + Returns the extended greatest common divisor of the two given integers. - Inlined by the compiler. + This function uses the extended Euclidean algorithm to return a three-element tuple with the `gcd` + and the coefficients `m` and `n` of Bézout's identity such that: + + gcd(a, b) = m*a + n*b + + By convention, `extended_gcd(0, 0)` returns `{0, 0, 0}`. ## Examples - iex> Integer.to_char_list(1023, 16) - '3FF' + iex> Integer.extended_gcd(240, 46) + {2, -9, 47} + iex> Integer.extended_gcd(46, 240) + {2, 47, -9} + iex> Integer.extended_gcd(-46, 240) + {2, -47, -9} + iex> Integer.extended_gcd(-46, -240) + {2, -47, 9} + + iex> Integer.extended_gcd(14, 21) + {7, -1, 1} + + iex> Integer.extended_gcd(10, 0) + {10, 1, 0} + iex> Integer.extended_gcd(0, 10) + {10, 0, 1} + iex> Integer.extended_gcd(0, 0) + {0, 0, 0} """ - @spec to_char_list(integer, pos_integer) :: list - def to_char_list(number, base) do - :erlang.integer_to_list(number, base) + @doc since: "1.12.0" + @spec extended_gcd(integer, integer) :: {non_neg_integer, integer, integer} + def extended_gcd(0, 0), do: {0, 0, 0} + def extended_gcd(0, b), do: {b, 0, 1} + def extended_gcd(a, 0), do: {a, 1, 0} + + def extended_gcd(integer1, integer2) when is_integer(integer1) and is_integer(integer2) do + extended_gcd(integer2, integer1, 0, 1, 1, 0) + end + + defp extended_gcd(r1, r0, s1, s0, t1, t0) do + div = div(r0, r1) + + case r0 - div * r1 do + 0 when r1 > 0 -> {r1, s1, t1} + 0 when r1 < 0 -> {-r1, -s1, -t1} + r2 -> extended_gcd(r2, r1, s0 - div * s1, s1, t0 - div * t1, t1) + end end + + @doc false + @deprecated "Use Integer.to_charlist/1 instead" + def to_char_list(integer), do: Integer.to_charlist(integer) + + @doc false + @deprecated "Use Integer.to_charlist/2 instead" + def to_char_list(integer, base), do: Integer.to_charlist(integer, base) end diff --git a/lib/elixir/lib/io.ex b/lib/elixir/lib/io.ex index 947b6ffa61f..e89e3057771 100644 --- a/lib/elixir/lib/io.ex +++ b/lib/elixir/lib/io.ex @@ -1,84 +1,204 @@ defmodule IO do - @moduledoc """ - Functions handling IO. + @moduledoc ~S""" + Functions handling input/output (IO). - Many functions in this module expects an IO device as argument. - An IO device must be a pid or an atom representing a process. + Many functions in this module expect an IO device as an argument. + An IO device must be a PID or an atom representing a process. For convenience, Elixir provides `:stdio` and `:stderr` as shortcuts to Erlang's `:standard_io` and `:standard_error`. - The majority of the functions expect char data, i.e. strings or - lists of characters and strings. In case another type is given, - it will do a conversion to string via the `String.Chars` protocol - (as shown in typespecs). - - The functions starting with `bin*` expects iodata as argument, - i.e. binaries or lists of bytes and binaries. + The majority of the functions expect chardata. In case another type is given, + functions will convert those types to string via the `String.Chars` protocol + (as shown in typespecs). For more information on chardata, see the + "IO data" section below. ## IO devices - An IO device may be an atom or a pid. In case it is an atom, - the atom must be the name of a registered process. However, - there are three exceptions for this rule: + An IO device may be an atom or a PID. In case it is an atom, + the atom must be the name of a registered process. In addition, + Elixir provides two shortcuts: + + * `:stdio` - a shortcut for `:standard_io`, which maps to + the current `Process.group_leader/0` in Erlang + + * `:stderr` - a shortcut for the named process `:standard_error` + provided in Erlang + + IO devices maintain their position, which means subsequent calls to any + reading or writing functions will start from the place where the device + was last accessed. The position of files can be changed using the + `:file.position/2` function. + + ## IO data + + IO data is a data type that can be used as a more efficient alternative to binaries + in certain situations. + + A term of type **IO data** is a binary or a list containing bytes (integers within the `0..255` range) + or nested IO data. The type is recursive. Let's see an example of one of + the possible IO data representing the binary `"hello"`: + + [?h, "el", ["l", [?o]]] + + The built-in `t:iodata/0` type is defined in terms of `t:iolist/0`. An IO list is + the same as IO data but it doesn't allow for a binary at the top level (but binaries + are still allowed in the list itself). + + ### Use cases for IO data + + IO data exists because often you need to do many append operations + on smaller chunks of binaries in order to create a bigger binary. However, in + Erlang and Elixir concatenating binaries will copy the concatenated binaries + into a new binary. + + def email(username, domain) do + username <> "@" <> domain + end + + In this function, creating the email address will copy the `username` and `domain` + binaries. Now imagine you want to use the resulting email inside another binary: + + def welcome_message(name, username, domain) do + "Welcome #{name}, your email is: #{email(username, domain)}" + end + + IO.puts(welcome_message("Meg", "meg", "example.com")) + #=> "Welcome Meg, your email is: meg@example.com" + + Every time you concatenate binaries or use interpolation (`#{}`) you are making + copies of those binaries. However, in many cases you don't need the complete + binary while you create it, but only at the end to print it out or send it + somewhere. In such cases, you can construct the binary by creating IO data: + + def email(username, domain) do + [username, ?@, domain] + end + + def welcome_message(name, username, domain) do + ["Welcome ", name, ", your email is: ", email(username, domain)] + end + + IO.puts(welcome_message("Meg", "meg", "example.com")) + #=> "Welcome Meg, your email is: meg@example.com" + + Building IO data is cheaper than concatenating binaries. Concatenating multiple + pieces of IO data just means putting them together inside a list since IO data + can be arbitrarily nested, and that's a cheap and efficient operation. Most of + the IO-based APIs, such as `:gen_tcp` and `IO`, receive IO data and write it + to the socket directly without converting it to binary. + + One drawback of IO data is that you can't do things like pattern match on the + first part of a piece of IO data like you can with a binary, because you usually + don't know the shape of the IO data. In those cases, you may need to convert it + to a binary by calling `iodata_to_binary/1`, which is reasonably efficient + since it's implemented natively in C. Other functionality, like computing the + length of IO data, can be computed directly on the iodata by calling `iodata_length/1`. + + ### Chardata - * `:standard_io` - when the `:standard_io` atom is given, - it is treated as a shortcut for `Process.group_leader` + Erlang and Elixir also have the idea of `t:chardata/0`. Chardata is very + similar to IO data: the only difference is that integers in IO data represent + bytes while integers in chardata represent Unicode code points. Bytes + (`t:byte/0`) are integers within the `0..255` range, while Unicode code points + (`t:char/0`) are integers within the `0..0x10FFFF` range. The `IO` module provides + the `chardata_to_string/1` function for chardata as the "counter-part" of the + `iodata_to_binary/1` function for IO data. - * `:stdio` - is a shortcut for `:standard_io` + If you try to use `iodata_to_binary/1` on chardata, it will result in an + argument error. For example, let's try to put a code point that is not + representable with one byte, like `?π`, inside IO data: - * `:stderr` - is a shortcut for `:standard_error` + IO.iodata_to_binary(["The symbol for pi is: ", ?π]) + #=> ** (ArgumentError) argument error + + If we use chardata instead, it will work as expected: + + iex> IO.chardata_to_string(["The symbol for pi is: ", ?π]) + "The symbol for pi is: π" """ @type device :: atom | pid @type nodata :: {:error, term} | :eof - @type chardata() :: :unicode.chardata() - - import :erlang, only: [group_leader: 0] + @type chardata :: String.t() | maybe_improper_list(char | chardata, String.t() | []) - defmacrop is_iodata(data) do - quote do - is_list(unquote(data)) or is_binary(unquote(data)) - end - end + defguardp is_device(term) when is_atom(term) or is_pid(term) + defguardp is_iodata(data) when is_list(data) or is_binary(data) @doc """ - Reads `count` characters from the IO device or until - the end of the line if `:line` is given. It returns: + Reads from the IO `device`. - * `data` - the input characters + The `device` is iterated by the given number of characters, line by line if + `:line` is given, or until `:eof`. + + It returns: + + * `data` - the output characters * `:eof` - end of file was encountered * `{:error, reason}` - other (rare) error condition; for instance, `{:error, :estale}` if reading from an NFS volume + """ - @spec read(device, :line | non_neg_integer) :: chardata | nodata - def read(device \\ group_leader, chars_or_line) + @spec read(device, :eof | :line | non_neg_integer) :: chardata | nodata + def read(device \\ :stdio, line_or_chars) + + # TODO: Deprecate me on v1.17 + def read(device, :all) do + with :eof <- read(device, :eof) do + with [_ | _] = opts <- :io.getopts(device), + false <- Keyword.get(opts, :binary, true) do + '' + else + _ -> "" + end + end + end + + def read(device, :eof) do + getn(device, '', :eof) + end def read(device, :line) do :io.get_line(map_dev(device), '') end - def read(device, count) when count >= 0 do + def read(device, count) when is_integer(count) and count >= 0 do :io.get_chars(map_dev(device), '', count) end @doc """ - Reads `count` bytes from the IO device or until - the end of the line if `:line` is given. It returns: + Reads from the IO `device`. The operation is Unicode unsafe. - * `data` - the input characters + The `device` is iterated by the given number of bytes, line by line if + `:line` is given, or until `:eof`. + + It returns: + + * `data` - the output bytes * `:eof` - end of file was encountered * `{:error, reason}` - other (rare) error condition; for instance, `{:error, :estale}` if reading from an NFS volume + + Note: do not use this function on IO devices in Unicode mode + as it will return the wrong result. """ - @spec binread(device, :line | non_neg_integer) :: iodata | nodata - def binread(device \\ group_leader, chars_or_line) + @spec binread(device, :eof | :line | non_neg_integer) :: iodata | nodata + def binread(device \\ :stdio, line_or_chars) + + # TODO: Deprecate me on v1.17 + def binread(device, :all) do + with :eof <- binread(device, :eof), do: "" + end + + def binread(device, :eof) do + binread_eof(map_dev(device), "") + end def binread(device, :line) do case :file.read_line(map_dev(device)) do @@ -87,91 +207,293 @@ defmodule IO do end end - def binread(device, count) when count >= 0 do + def binread(device, count) when is_integer(count) and count >= 0 do case :file.read(map_dev(device), count) do {:ok, data} -> data other -> other end end + @read_all_size 4096 + defp binread_eof(mapped_dev, acc) do + case :file.read(mapped_dev, @read_all_size) do + {:ok, data} -> binread_eof(mapped_dev, acc <> data) + :eof -> if acc == "", do: :eof, else: acc + other -> other + end + end + @doc """ - Writes the given argument to the given device. + Writes `chardata` to the given `device`. - By default the device is the standard output. - It returns `:ok` if it succeeds. + By default, the `device` is the standard output. ## Examples - IO.write "sample" - #=> "sample" + IO.write("sample") + #=> sample - IO.write :stderr, "error" - #=> "error" + IO.write(:stderr, "error") + #=> error """ - @spec write(device, chardata | String.Chars.t) :: :ok - def write(device \\ group_leader(), item) do - :io.put_chars map_dev(device), to_chardata(item) + @spec write(device, chardata | String.Chars.t()) :: :ok + def write(device \\ :stdio, chardata) do + :io.put_chars(map_dev(device), to_chardata(chardata)) end @doc """ - Writes the given argument to the given device - as a binary, no unicode conversion happens. + Writes `iodata` to the given `device`. + + This operation is meant to be used with "raw" devices + that are started without an encoding. The given `iodata` + is written as is to the device, without conversion. For + more information on IO data, see the "IO data" section in + the module documentation. + + Use `write/2` for devices with encoding. - Check `write/2` for more information. + Important: do **not** use this function on IO devices in + Unicode mode as it will write the wrong data. In particular, + the standard IO device is set to Unicode by default, so writing + to stdio with this function will likely result in the wrong data + being sent down the wire. """ @spec binwrite(device, iodata) :: :ok | {:error, term} - def binwrite(device \\ group_leader(), item) when is_iodata(item) do - :file.write map_dev(device), item + def binwrite(device \\ :stdio, iodata) when is_iodata(iodata) do + :file.write(map_dev(device), iodata) + end + + @doc """ + Writes `item` to the given `device`, similar to `write/2`, + but adds a newline at the end. + + By default, the `device` is the standard output. It returns `:ok` + if it succeeds. + + ## Examples + + IO.puts("Hello World!") + #=> Hello World! + + IO.puts(:stderr, "error") + #=> error + + """ + @spec puts(device, chardata | String.Chars.t()) :: :ok + def puts(device \\ :stdio, item) when is_device(device) do + :io.put_chars(map_dev(device), [to_chardata(item), ?\n]) + end + + @doc """ + Writes a `message` to stderr, along with the given `stacktrace_info`. + + The `stacktrace_info` must be one of: + + * a `__STACKTRACE__`, where all entries in the stacktrace will be + included in the error message + + * a `Macro.Env` structure (since v1.14.0), where a single stacktrace + entry from the compilation environment will be used + + * a keyword list with at least the `:file` option representing + a single stacktrace entry (since v1.14.0). The `:line`, `:module`, + `:function` options are also supported + + This function also notifies the compiler a warning was printed + (in case --warnings-as-errors was enabled). It returns `:ok` + if it succeeds. + + ## Examples + + stacktrace = [{MyApp, :main, 1, [file: 'my_app.ex', line: 4]}] + IO.warn("variable bar is unused", stacktrace) + #=> warning: variable bar is unused + #=> my_app.ex:4: MyApp.main/1 + + """ + @spec warn(chardata | String.Chars.t(), Exception.stacktrace() | keyword() | Macro.Env.t()) :: + :ok + def warn(message, stacktrace_info) + + def warn(message, []) do + message = [to_chardata(message), ?\n] + :elixir_errors.log_and_print_warning(0, nil, message, message) + end + + def warn(message, %Macro.Env{} = env) do + warn(message, Macro.Env.stacktrace(env)) + end + + def warn(message, [{_, _} | _] = keyword) do + if file = keyword[:file] do + warn( + message, + %{ + __ENV__ + | module: keyword[:module], + function: keyword[:function], + line: keyword[:line], + file: file + } + ) + else + warn(message, []) + end + end + + def warn(message, [{_, _, _, opts} | _] = stacktrace) do + message = to_chardata(message) + formatted_trace = Enum.map_join(stacktrace, "\n ", &Exception.format_stacktrace_entry(&1)) + line = opts[:line] + file = opts[:file] + + :elixir_errors.log_and_print_warning( + line || 0, + file && List.to_string(file), + message, + [message, ?\n, " ", formatted_trace, ?\n] + ) + end + + @doc false + def warn_once(key, message, stacktrace_drop_levels) do + {:current_stacktrace, stacktrace} = Process.info(self(), :current_stacktrace) + stacktrace = Enum.drop(stacktrace, stacktrace_drop_levels) + + if :elixir_config.warn(key, stacktrace) do + warn(message, stacktrace) + else + :ok + end end @doc """ - Writes the argument to the device, similar to `write/2`, - but adds a newline at the end. The argument is expected - to be a chardata. + Writes a `message` to stderr, along with the current stacktrace. + + It returns `:ok` if it succeeds. + + Do not call this function at the tail of another function. Due to tail + call optimization, a stacktrace entry would not be added and the + stacktrace would be incorrectly trimmed. Therefore make sure at least + one expression (or an atom such as `:ok`) follows the `IO.warn/1` call. + + ## Examples + + IO.warn("variable bar is unused") + #=> warning: variable bar is unused + #=> (iex) evaluator.ex:108: IEx.Evaluator.eval/4 + """ - @spec puts(device, chardata | String.Chars.t) :: :ok - def puts(device \\ group_leader(), item) do - erl_dev = map_dev(device) - :io.put_chars erl_dev, [to_chardata(item), ?\n] + @spec warn(chardata | String.Chars.t()) :: :ok + def warn(message) do + {:current_stacktrace, stacktrace} = Process.info(self(), :current_stacktrace) + warn(message, Enum.drop(stacktrace, 2)) end @doc """ - Inspects and writes the given argument to the device. + Inspects and writes the given `item` to the device. + + It's important to note that it returns the given `item` unchanged. + This makes it possible to "spy" on values by inserting an + `IO.inspect/2` call almost anywhere in your code, for example, + in the middle of a pipeline. + + It enables pretty printing by default with width of + 80 characters. The width can be changed by explicitly + passing the `:width` option. - It sets by default pretty printing to true and returns - the item itself. + The output can be decorated with a label, by providing the `:label` + option to easily distinguish it from other `IO.inspect/2` calls. + The label will be printed before the inspected `item`. - Note this function does not use the IO device width - because some IO devices does not implement the - appropriate functions. Setting the width must be done - explicitly by passing the `:width` option. + See `Inspect.Opts` for a full list of remaining formatting options. ## Examples - IO.inspect Process.list + IO.inspect(<<0, 1, 2>>, width: 40) + + Prints: + + <<0, 1, 2>> + + We can use the `:label` option to decorate the output: + + IO.inspect(1..100, label: "a wonderful range") + + Prints: + + a wonderful range: 1..100 + + The `:label` option is especially useful with pipelines: + + [1, 2, 3] + |> IO.inspect(label: "before") + |> Enum.map(&(&1 * 2)) + |> IO.inspect(label: "after") + |> Enum.sum() + + Prints: + + before: [1, 2, 3] + after: [2, 4, 6] """ - @spec inspect(term, Keyword.t) :: term + @spec inspect(item, keyword) :: item when item: var def inspect(item, opts \\ []) do - inspect group_leader(), item, opts + inspect(:stdio, item, opts) end @doc """ - Inspects the item with options using the given device. + Inspects `item` according to the given options using the IO `device`. + + See `inspect/2` for a full list of options. """ - @spec inspect(device, term, Keyword.t) :: term - def inspect(device, item, opts) when is_list(opts) do - opts = Keyword.put_new(opts, :pretty, true) - puts device, Kernel.inspect(item, opts) + @spec inspect(device, item, keyword) :: item when item: var + def inspect(device, item, opts) when is_device(device) and is_list(opts) do + label = if label = opts[:label], do: [to_chardata(label), ": "], else: [] + opts = Inspect.Opts.new(opts) + doc = Inspect.Algebra.group(Inspect.Algebra.to_doc(item, opts)) + chardata = Inspect.Algebra.format(doc, opts.width) + puts(device, [label, chardata]) item end @doc """ - Gets a number of bytes from the io device. If the - io device is a unicode device, `count` implies - the number of unicode codepoints to be retrieved. + Gets a number of bytes from IO device `:stdio`. + + If `:stdio` is a Unicode device, `count` implies + the number of Unicode code points to be retrieved. Otherwise, `count` is the number of raw bytes to be retrieved. + + See `IO.getn/3` for a description of return values. + """ + @spec getn( + device | chardata | String.Chars.t(), + pos_integer | :eof | chardata | String.Chars.t() + ) :: + chardata | nodata + def getn(prompt, count \\ 1) + + def getn(prompt, :eof) do + getn(:stdio, prompt, :eof) + end + + def getn(prompt, count) when is_integer(count) and count > 0 do + getn(:stdio, prompt, count) + end + + def getn(device, prompt) when not is_integer(prompt) do + getn(device, prompt, 1) + end + + @doc """ + Gets a number of bytes from the IO `device`. + + If the IO `device` is a Unicode device, `count` implies + the number of Unicode code points to be retrieved. + Otherwise, `count` is the number of raw bytes to be retrieved. + It returns: * `data` - the input characters @@ -181,100 +503,145 @@ defmodule IO do * `{:error, reason}` - other (rare) error condition; for instance, `{:error, :estale}` if reading from an NFS volume - """ - @spec getn(chardata | String.Chars.t, pos_integer) :: chardata | nodata - @spec getn(device, chardata | String.Chars.t) :: chardata | nodata - def getn(prompt, count \\ 1) - def getn(prompt, count) when is_integer(count) do - getn(group_leader, prompt, count) + """ + @spec getn(device, chardata | String.Chars.t(), pos_integer | :eof) :: chardata | nodata + def getn(device, prompt, :eof) do + getn_eof(map_dev(device), to_chardata(prompt), []) end - def getn(device, prompt) do - getn(device, prompt, 1) + def getn(device, prompt, count) when is_integer(count) and count > 0 do + :io.get_chars(map_dev(device), to_chardata(prompt), count) end - @doc """ - Gets a number of bytes from the io device. If the - io device is a unicode device, `count` implies - the number of unicode codepoints to be retrieved. - Otherwise, `count` is the number of raw bytes to be retrieved. - """ - @spec getn(device, chardata | String.Chars.t, pos_integer) :: chardata | nodata - def getn(device, prompt, count) do - :io.get_chars(map_dev(device), to_chardata(prompt), count) + defp getn_eof(device, prompt, acc) do + case :io.get_line(device, prompt) do + line when is_binary(line) or is_list(line) -> getn_eof(device, '', [line | acc]) + :eof -> wrap_eof(:lists.reverse(acc)) + other -> other + end end - @doc """ - Reads a line from the IO device. It returns: + defp wrap_eof([h | _] = acc) when is_binary(h), do: IO.iodata_to_binary(acc) + defp wrap_eof([h | _] = acc) when is_list(h), do: :lists.flatten(acc) + defp wrap_eof([]), do: :eof + + @doc ~S""" + Reads a line from the IO `device`. + + It returns: * `data` - the characters in the line terminated - by a LF (or end of file) + by a line-feed (LF) or end of file (EOF) * `:eof` - end of file was encountered * `{:error, reason}` - other (rare) error condition; for instance, `{:error, :estale}` if reading from an NFS volume + + ## Examples + + To display "What is your name?" as a prompt and await user input: + + IO.gets("What is your name?\n") + """ - @spec gets(device, chardata | String.Chars.t) :: chardata | nodata - def gets(device \\ group_leader(), prompt) do + @spec gets(device, chardata | String.Chars.t()) :: chardata | nodata + def gets(device \\ :stdio, prompt) do :io.get_line(map_dev(device), to_chardata(prompt)) end @doc """ - Converts the io device into a `IO.Stream`. + Returns a line-based `IO.Stream` on `:stdio`. + + This is equivalent to: + + IO.stream(:stdio, :line) + + """ + @doc since: "1.12.0" + def stream, do: stream(:stdio, :line) + + @doc """ + Converts the IO `device` into an `IO.Stream`. An `IO.Stream` implements both `Enumerable` and `Collectable`, allowing it to be used for both read and write. - The device is iterated line by line if `:line` is given or - by a given number of codepoints. + The `device` is iterated by the given number of characters or line by line if + `:line` is given. - This reads the IO as utf-8. Check out + This reads from the IO as UTF-8. Check out `IO.binstream/2` to handle the IO as a raw binary. Note that an IO stream has side effects and every time you go over the stream you may get different results. + `stream/1` has been introduced in Elixir v1.12.0, + while `stream/2` has been available since v1.0.0. + ## Examples Here is an example on how we mimic an echo server from the command line: - Enum.each IO.stream(:stdio, :line), &IO.write(&1) + Enum.each(IO.stream(:stdio, :line), &IO.write(&1)) """ - @spec stream(device, :line | pos_integer) :: Enumerable.t - def stream(device, line_or_codepoints) do + @spec stream(device, :line | pos_integer) :: Enumerable.t() + def stream(device \\ :stdio, line_or_codepoints) + when line_or_codepoints == :line + when is_integer(line_or_codepoints) and line_or_codepoints > 0 do IO.Stream.__build__(map_dev(device), false, line_or_codepoints) end @doc """ - Converts the IO device into a `IO.Stream`. + Returns a raw, line-based `IO.Stream` on `:stdio`. The operation is Unicode unsafe. + + This is equivalent to: + + IO.binstream(:stdio, :line) + + """ + @doc since: "1.12.0" + def binstream, do: binstream(:stdio, :line) + + @doc """ + Converts the IO `device` into an `IO.Stream`. The operation is Unicode unsafe. An `IO.Stream` implements both `Enumerable` and `Collectable`, allowing it to be used for both read and write. - The device is iterated line by line or by a number of bytes. - This reads the IO device as a raw binary. + The `device` is iterated by the given number of bytes or line by line if + `:line` is given. This reads from the IO device as a raw binary. Note that an IO stream has side effects and every time you go over the stream you may get different results. + + Finally, do not use this function on IO devices in Unicode + mode as it will return the wrong result. + + `binstream/1` has been introduced in Elixir v1.12.0, + while `binstream/2` has been available since v1.0.0. """ - @spec binstream(device, :line | pos_integer) :: Enumerable.t - def binstream(device, line_or_bytes) do + @spec binstream(device, :line | pos_integer) :: Enumerable.t() + def binstream(device \\ :stdio, line_or_bytes) + when line_or_bytes == :line + when is_integer(line_or_bytes) and line_or_bytes > 0 do IO.Stream.__build__(map_dev(device), true, line_or_bytes) end @doc """ - Converts chardata (a list of integers representing codepoints, - lists and strings) into a string. + Converts chardata into a string. - In case the conversion fails, it raises a `UnicodeConversionError`. - If a string is given, returns the string itself. + For more information about chardata, see the ["Chardata"](#module-chardata) + section in the module documentation. + + In case the conversion fails, it raises an `UnicodeConversionError`. + If a string is given, it returns the string itself. ## Examples @@ -284,33 +651,32 @@ defmodule IO do iex> IO.chardata_to_string([0x0061, "bc"]) "abc" + iex> IO.chardata_to_string("string") + "string" + """ - @spec chardata_to_string(chardata) :: String.t | no_return + @spec chardata_to_string(chardata) :: String.t() + def chardata_to_string(chardata) + def chardata_to_string(string) when is_binary(string) do string end def chardata_to_string(list) when is_list(list) do - case :unicode.characters_to_binary(list) do - result when is_binary(result) -> - result - - {:error, encoded, rest} -> - raise UnicodeConversionError, encoded: encoded, rest: rest, kind: :invalid - - {:incomplete, encoded, rest} -> - raise UnicodeConversionError, encoded: encoded, rest: rest, kind: :incomplete - end + List.to_string(list) end @doc """ - Converts iodata (a list of integers representing bytes, lists - and binaries) into a binary. + Converts IO data into a binary + + The operation is Unicode unsafe. - Notice that this function treats lists of integers as raw bytes - and does not perform any kind of encoding conversion. If you want - to convert from a char list to a string (UTF-8 encoded), please - use `chardata_to_string/1` instead. + Note that this function treats integers in the given IO data as + raw bytes and does not perform any kind of encoding conversion. + If you want to convert from a charlist to a UTF-8-encoded string, + use `chardata_to_string/1` instead. For more information about + IO data and chardata, see the ["IO data"](#module-io-data) section in the + module documentation. If this function receives a binary, the same binary is returned. @@ -321,63 +687,70 @@ defmodule IO do iex> bin1 = <<1, 2, 3>> iex> bin2 = <<4, 5>> iex> bin3 = <<6>> - iex> IO.iodata_to_binary([bin1, 1, [2, 3, bin2], 4|bin3]) - <<1,2,3,1,2,3,4,5,4,6>> + iex> IO.iodata_to_binary([bin1, 1, [2, 3, bin2], 4 | bin3]) + <<1, 2, 3, 1, 2, 3, 4, 5, 4, 6>> iex> bin = <<1, 2, 3>> iex> IO.iodata_to_binary(bin) - <<1,2,3>> + <<1, 2, 3>> """ @spec iodata_to_binary(iodata) :: binary - def iodata_to_binary(item) do - :erlang.iolist_to_binary(item) + def iodata_to_binary(iodata) do + :erlang.iolist_to_binary(iodata) end @doc """ - Returns the size of an iodata. + Returns the size of an IO data. + + For more information about IO data, see the ["IO data"](#module-io-data) + section in the module documentation. Inlined by the compiler. ## Examples - iex> IO.iodata_length([1, 2|<<3, 4>>]) + iex> IO.iodata_length([1, 2 | <<3, 4>>]) 4 """ @spec iodata_length(iodata) :: non_neg_integer - def iodata_length(item) do - :erlang.iolist_size(item) + def iodata_length(iodata) do + :erlang.iolist_size(iodata) end @doc false - def each_stream(device, what) do - case read(device, what) do + def each_stream(device, line_or_codepoints) do + case read(device, line_or_codepoints) do :eof -> - nil + {:halt, device} + {:error, reason} -> raise IO.StreamError, reason: reason + data -> - {data, device} + {[data], device} end end @doc false - def each_binstream(device, what) do - case binread(device, what) do + def each_binstream(device, line_or_chars) do + case binread(device, line_or_chars) do :eof -> - nil + {:halt, device} + {:error, reason} -> raise IO.StreamError, reason: reason + data -> - {data, device} + {[data], device} end end @compile {:inline, map_dev: 1, to_chardata: 1} - # Map the Elixir names for standard io and error to Erlang names - defp map_dev(:stdio), do: :standard_io + # Map the Elixir names for standard IO and error to Erlang names + defp map_dev(:stdio), do: :standard_io defp map_dev(:stderr), do: :standard_error defp map_dev(other) when is_atom(other) or is_pid(other) or is_tuple(other), do: other diff --git a/lib/elixir/lib/io/ansi.ex b/lib/elixir/lib/io/ansi.ex index bb52036357d..0be13dafbc1 100644 --- a/lib/elixir/lib/io/ansi.ex +++ b/lib/elixir/lib/io/ansi.ex @@ -1,13 +1,13 @@ defmodule IO.ANSI.Sequence do @moduledoc false - defmacro defsequence(name, code \\ "", terminator \\ "m") do + defmacro defsequence(name, code, terminator \\ "m") do quote bind_quoted: [name: name, code: code, terminator: terminator] do def unquote(name)() do "\e[#{unquote(code)}#{unquote(terminator)}" end - defp escape_sequence(unquote(Atom.to_char_list(name))) do + defp format_sequence(unquote(name)) do unquote(name)() end end @@ -16,213 +16,290 @@ end defmodule IO.ANSI do @moduledoc """ - Functionality to render ANSI escape sequences - (http://en.wikipedia.org/wiki/ANSI_escape_code) — characters embedded - in text used to control formatting, color, and other output options - on video text terminals. + Functionality to render ANSI escape sequences. + + [ANSI escape sequences](https://en.wikipedia.org/wiki/ANSI_escape_code) + are characters embedded in text used to control formatting, color, and + other output options on video text terminals. + + ANSI escapes are typically enabled on all Unix terminals. They are also + available on Windows consoles from Windows 10, although it must be + explicitly enabled for the current user in the registry by running the + following command: + + reg add HKCU\\Console /v VirtualTerminalLevel /t REG_DWORD /d 1 + + After running the command above, you must restart your current console. + + ## Examples + + Because the ANSI escape sequences are embedded in text, the normal usage of + these functions is to concatenate their output with text. + + formatted_text = IO.ANSI.blue_background() <> "Example" <> IO.ANSI.reset() + IO.puts(formatted_text) + + A higher level and more convenient API is also available via `IO.ANSI.format/1`, + where you use atoms to represent each ANSI escape sequence and by default + checks if ANSI is enabled: + + IO.puts(IO.ANSI.format([:blue_background, "Example"])) + + In case ANSI is disabled, the ANSI escape sequences are simply discarded. """ import IO.ANSI.Sequence + @type ansicode :: atom + @type ansilist :: + maybe_improper_list(char | ansicode | binary | ansilist, binary | ansicode | []) + @type ansidata :: ansilist | ansicode | binary + @doc """ - Checks whether the default I/O device is a terminal or a file. + Checks if ANSI coloring is supported and enabled on this machine. - Used to identify whether printing ANSI escape sequences will likely - be displayed as intended. This is checked by sending a message to - the group leader. In case the group leader does not support the message, - it will likely lead to a timeout (and a slow down on execution time). + This function simply reads the configuration value for + `:ansi_enabled` in the `:elixir` application. The value is by + default `false` unless Elixir can detect during startup that + both `stdout` and `stderr` are terminals. """ - @spec terminal? :: boolean - @spec terminal?(:io.device) :: boolean - def terminal?(device \\ :erlang.group_leader) do - !match?({:win32, _}, :os.type()) and - match?({:ok, _}, :io.columns(device)) + @spec enabled? :: boolean + def enabled? do + Application.get_env(:elixir, :ansi_enabled, false) end - @doc "Resets all attributes" - defsequence :reset, 0 + @doc "Sets foreground color." + @spec color(0..255) :: String.t() + def color(code) when code in 0..255, do: "\e[38;5;#{code}m" + + @doc ~S""" + Sets the foreground color from individual RGB values. + + Valid values for each color are in the range 0 to 5. + """ + @spec color(0..5, 0..5, 0..5) :: String.t() + def color(r, g, b) when r in 0..5 and g in 0..5 and b in 0..5 do + color(16 + 36 * r + 6 * g + b) + end + + @doc "Sets background color." + @spec color_background(0..255) :: String.t() + def color_background(code) when code in 0..255, do: "\e[48;5;#{code}m" + + @doc ~S""" + Sets the background color from individual RGB values. + + Valid values for each color are in the range 0 to 5. + """ + @spec color_background(0..5, 0..5, 0..5) :: String.t() + def color_background(r, g, b) when r in 0..5 and g in 0..5 and b in 0..5 do + color_background(16 + 36 * r + 6 * g + b) + end - @doc "Bright (increased intensity) or Bold" - defsequence :bright, 1 + @doc "Resets all attributes." + defsequence(:reset, 0) - @doc "Faint (decreased intensity), not widely supported" - defsequence :faint, 2 + @doc "Bright (increased intensity) or bold." + defsequence(:bright, 1) + + @doc "Faint (decreased intensity). Not widely supported." + defsequence(:faint, 2) @doc "Italic: on. Not widely supported. Sometimes treated as inverse." - defsequence :italic, 3 + defsequence(:italic, 3) - @doc "Underline: Single" - defsequence :underline, 4 + @doc "Underline: single." + defsequence(:underline, 4) - @doc "Blink: Slow. Less than 150 per minute" - defsequence :blink_slow, 5 + @doc "Blink: slow. Less than 150 per minute." + defsequence(:blink_slow, 5) - @doc "Blink: Rapid. MS-DOS ANSI.SYS; 150 per minute or more; not widely supported" - defsequence :blink_rapid, 6 + @doc "Blink: rapid. MS-DOS ANSI.SYS; 150 per minute or more; not widely supported." + defsequence(:blink_rapid, 6) - @doc "Image: Negative. Swap foreground and background" - defsequence :inverse, 7 + @doc "Image: negative. Swap foreground and background." + defsequence(:inverse, 7) - @doc "Image: Negative. Swap foreground and background" - defsequence :reverse, 7 + @doc "Image: negative. Swap foreground and background." + defsequence(:reverse, 7) - @doc "Conceal. Not widely supported" - defsequence :conceal, 8 + @doc "Conceal. Not widely supported." + defsequence(:conceal, 8) @doc "Crossed-out. Characters legible, but marked for deletion. Not widely supported." - defsequence :crossed_out, 9 + defsequence(:crossed_out, 9) - @doc "Sets primary (default) font" - defsequence :primary_font, 10 + @doc "Sets primary (default) font." + defsequence(:primary_font, 10) for font_n <- [1, 2, 3, 4, 5, 6, 7, 8, 9] do - @doc "Sets alternative font #{font_n}" - defsequence :"font_#{font_n}", font_n + 10 + @doc "Sets alternative font #{font_n}." + defsequence(:"font_#{font_n}", font_n + 10) end - @doc "Normal color or intensity" - defsequence :normal, 22 + @doc "Normal color or intensity." + defsequence(:normal, 22) + + @doc "Not italic." + defsequence(:not_italic, 23) + + @doc "Underline: none." + defsequence(:no_underline, 24) - @doc "Not italic" - defsequence :not_italic, 23 + @doc "Blink: off." + defsequence(:blink_off, 25) - @doc "Underline: None" - defsequence :no_underline, 24 + @doc "Image: positive. Normal foreground and background." + defsequence(:inverse_off, 27) - @doc "Blink: off" - defsequence :blink_off, 25 + @doc "Image: positive. Normal foreground and background." + defsequence(:reverse_off, 27) colors = [:black, :red, :green, :yellow, :blue, :magenta, :cyan, :white] - colors = Enum.zip(0..(length(colors)-1), colors) - for {code, color} <- colors do - @doc "Sets foreground color to #{color}" - defsequence color, code + 30 + for {color, code} <- Enum.with_index(colors) do + @doc "Sets foreground color to #{color}." + defsequence(color, code + 30) - @doc "Sets background color to #{color}" - defsequence :"#{color}_background", code + 40 + @doc "Sets foreground color to light #{color}." + defsequence(:"light_#{color}", code + 90) + + @doc "Sets background color to #{color}." + defsequence(:"#{color}_background", code + 40) + + @doc "Sets background color to light #{color}." + defsequence(:"light_#{color}_background", code + 100) end - @doc "Default text color" - defsequence :default_color, 39 + @doc "Default text color." + defsequence(:default_color, 39) + + @doc "Default background color." + defsequence(:default_background, 49) + + @doc "Framed." + defsequence(:framed, 51) + + @doc "Encircled." + defsequence(:encircled, 52) + + @doc "Overlined." + defsequence(:overlined, 53) + + @doc "Not framed or encircled." + defsequence(:not_framed_encircled, 54) + + @doc "Not overlined." + defsequence(:not_overlined, 55) - @doc "Default background color" - defsequence :default_background, 49 + @doc "Sends cursor home." + defsequence(:home, "", "H") - @doc "Framed" - defsequence :framed, 51 + @doc """ + Sends cursor to the absolute position specified by `line` and `column`. + + Line `0` and column `0` would mean the top left corner. + """ + @spec cursor(non_neg_integer, non_neg_integer) :: String.t() + def cursor(line, column) + when is_integer(line) and line >= 0 and is_integer(column) and column >= 0 do + "\e[#{line};#{column}H" + end - @doc "Encircled" - defsequence :encircled, 52 + @doc "Sends cursor `lines` up." + @spec cursor_up(pos_integer) :: String.t() + def cursor_up(lines \\ 1) when is_integer(lines) and lines >= 1, do: "\e[#{lines}A" - @doc "Overlined" - defsequence :overlined, 53 + @doc "Sends cursor `lines` down." + @spec cursor_down(pos_integer) :: String.t() + def cursor_down(lines \\ 1) when is_integer(lines) and lines >= 1, do: "\e[#{lines}B" - @doc "Not framed or encircled" - defsequence :not_framed_encircled, 54 + @doc "Sends cursor `columns` to the right." + @spec cursor_right(pos_integer) :: String.t() + def cursor_right(columns \\ 1) when is_integer(columns) and columns >= 1, do: "\e[#{columns}C" - @doc "Not overlined" - defsequence :not_overlined, 55 + @doc "Sends cursor `columns` to the left." + @spec cursor_left(pos_integer) :: String.t() + def cursor_left(columns \\ 1) when is_integer(columns) and columns >= 1, do: "\e[#{columns}D" - @doc "Send cursor home" - defsequence :home, "", "H" + @doc "Clears screen." + defsequence(:clear, "2", "J") - @doc "Clear screen" - defsequence :clear, "2", "J" + @doc "Clears line." + defsequence(:clear_line, "2", "K") - defp escape_sequence(other) do - raise ArgumentError, "invalid ANSI sequence specification: #{other}" + defp format_sequence(other) do + raise ArgumentError, "invalid ANSI sequence specification: #{inspect(other)}" end @doc ~S""" - Escapes a string by converting named ANSI sequences into actual ANSI codes. + Formats a chardata-like argument by converting named ANSI sequences into actual + ANSI codes. - The format for referring to sequences is `%{red}` and `%{red,bright}` (for - multiple sequences). + The named sequences are represented by atoms. - It will also append a `%{reset}` to the string. If you don't want this - behaviour, use `escape_fragment/2`. + It will also append an `IO.ANSI.reset/0` to the chardata when a conversion is + performed. If you don't want this behaviour, use `format_fragment/2`. An optional boolean parameter can be passed to enable or disable - emitting actual ANSI codes. When `false`, no ANSI codes will emitted. - By default, standard output will be checked if it is a terminal capable - of handling these sequences (using `terminal?/1` function) + emitting actual ANSI codes. When `false`, no ANSI codes will be emitted. + By default checks if ANSI is enabled using the `enabled?/0` function. ## Examples - iex> IO.ANSI.escape("Hello %{red,bright,green}yes", true) - "Hello \e[31m\e[1m\e[32myes\e[0m" + iex> IO.ANSI.format(["Hello, ", :red, :bright, "world!"], true) + [[[[[[], "Hello, "] | "\e[31m"] | "\e[1m"], "world!"] | "\e[0m"] """ - @spec escape(String.t, emit :: boolean) :: String.t - def escape(string, emit \\ terminal?) when is_binary(string) and is_boolean(emit) do - {rendered, emitted} = do_escape(string, emit, false, nil, []) - if emitted do - rendered <> reset - else - rendered - end + def format(chardata, emit? \\ enabled?()) when is_boolean(emit?) do + do_format(chardata, [], [], emit?, :maybe) end @doc ~S""" - Escapes a string by converting named ANSI sequences into actual ANSI codes. + Formats a chardata-like argument by converting named ANSI sequences into actual + ANSI codes. - The format for referring to sequences is `%{red}` and `%{red,bright}` (for - multiple sequences). + The named sequences are represented by atoms. An optional boolean parameter can be passed to enable or disable - emitting actual ANSI codes. When `false`, no ANSI codes will emitted. - By default, standard output will be checked if it is a terminal capable - of handling these sequences (using `terminal?/1` function) + emitting actual ANSI codes. When `false`, no ANSI codes will be emitted. + By default checks if ANSI is enabled using the `enabled?/0` function. ## Examples - iex> IO.ANSI.escape_fragment("Hello %{red,bright,green}yes", true) - "Hello \e[31m\e[1m\e[32myes" - - iex> IO.ANSI.escape_fragment("%{reset}bye", true) - "\e[0mbye" + iex> IO.ANSI.format_fragment([:bright, 'Word'], true) + [[[[[[] | "\e[1m"], 87], 111], 114], 100] """ - @spec escape_fragment(String.t, emit :: boolean) :: String.t - def escape_fragment(string, emit \\ terminal?) when is_binary(string) and is_boolean(emit) do - {escaped, _emitted} = do_escape(string, emit, false, nil, []) - escaped - end - - defp do_escape(<>, emit, emitted, buffer, acc) when is_list(buffer) do - sequences = - buffer - |> Enum.reverse() - |> :string.tokens(',') - |> Enum.map(&(&1 |> :string.strip |> escape_sequence)) - |> Enum.reverse() - - if emit and sequences != [] do - do_escape(t, emit, true, nil, sequences ++ acc) - else - do_escape(t, emit, emitted, nil, acc) - end + def format_fragment(chardata, emit? \\ enabled?()) when is_boolean(emit?) do + do_format(chardata, [], [], emit?, false) + end + + defp do_format([term | rest], rem, acc, emit?, append_reset) do + do_format(term, [rest | rem], acc, emit?, append_reset) + end + + defp do_format(term, rem, acc, true, append_reset) when is_atom(term) do + do_format([], rem, [acc | format_sequence(term)], true, !!append_reset) end - defp do_escape(<>, emit, emitted, buffer, acc) when is_list(buffer) do - do_escape(t, emit, emitted, [h|buffer], acc) + defp do_format(term, rem, acc, false, append_reset) when is_atom(term) do + do_format([], rem, acc, false, append_reset) end - defp do_escape(<<>>, _emit, _emitted, buffer, _acc) when is_list(buffer) do - buffer = IO.iodata_to_binary Enum.reverse(buffer) - raise ArgumentError, "missing } for escape fragment #{buffer}" + defp do_format(term, rem, acc, emit?, append_reset) when not is_list(term) do + do_format([], rem, [acc, term], emit?, append_reset) end - defp do_escape(<>, emit, emitted, nil, acc) do - do_escape(t, emit, emitted, [], acc) + defp do_format([], [next | rest], acc, emit?, append_reset) do + do_format(next, rest, acc, emit?, append_reset) end - defp do_escape(<>, emit, emitted, nil, acc) do - do_escape(t, emit, emitted, nil, [h|acc]) + defp do_format([], [], acc, true, true) do + [acc | IO.ANSI.reset()] end - defp do_escape(<<>>, _emit, emitted, nil, acc) do - {IO.iodata_to_binary(Enum.reverse(acc)), emitted} + defp do_format([], [], acc, _emit?, _append_reset) do + acc end end diff --git a/lib/elixir/lib/io/ansi/docs.ex b/lib/elixir/lib/io/ansi/docs.ex index 4dfcfb21ac0..b1aeb22d998 100644 --- a/lib/elixir/lib/io/ansi/docs.ex +++ b/lib/elixir/lib/io/ansi/docs.ex @@ -1,34 +1,46 @@ defmodule IO.ANSI.Docs do @moduledoc false + @bullet_text_unicode "• " + @bullet_text_ascii "* " @bullets [?*, ?-, ?+] + @spaces [" ", "\n", "\t"] @doc """ The default options used by this module. - The supported values are: + The supported keys are: - * `:enabled` - toggles coloring on and off (true) - * `:doc_code` - code blocks (cyan, bright) - * `:doc_inline_code` - inline code (cyan) - * `:doc_headings` - h1 and h2 headings (yellow, bright) - * `:doc_title` - top level heading (reverse, yellow, bright) - * `:doc_bold` - bold text (bright) - * `:doc_underline` - underlined text (underline) - * `:width` - the width to format the text (80) + * `:enabled` - toggles coloring on and off (true) + * `:doc_bold` - bold text (bright) + * `:doc_code` - code blocks (cyan) + * `:doc_headings` - h1, h2, h3, h4, h5, h6 headings (yellow) + * `:doc_metadata` - documentation metadata keys (yellow) + * `:doc_quote` - leading quote character `> ` (light black) + * `:doc_inline_code` - inline code (cyan) + * `:doc_table_heading` - the style for table headings + * `:doc_title` - top level heading (reverse, yellow) + * `:doc_underline` - underlined text (underline) + * `:width` - the width to format the text (80) Values for the color settings are strings with comma-separated ANSI values. """ + @spec default_options() :: keyword def default_options do - [enabled: true, - doc_code: "cyan,bright", - doc_inline_code: "cyan", - doc_headings: "yellow,bright", - doc_title: "reverse,yellow,bright", - doc_bold: "bright", - doc_underline: "underline", - width: 80] + [ + enabled: true, + doc_bold: [:bright], + doc_code: [:cyan], + doc_headings: [:yellow], + doc_metadata: [:yellow], + doc_quote: [:light_black], + doc_inline_code: [:cyan], + doc_table_heading: [:reverse], + doc_title: [:reverse, :yellow], + doc_underline: [:underline], + width: 80 + ] end @doc """ @@ -36,215 +48,715 @@ defmodule IO.ANSI.Docs do See `default_options/0` for docs on the supported options. """ - def print_heading(heading, options \\ []) do - IO.puts IO.ANSI.reset - options = Keyword.merge(default_options, options) - width = options[:width] - padding = div(width + String.length(heading), 2) - heading = heading |> String.rjust(padding) |> String.ljust(width) - write(:doc_title, heading, options) + @spec print_headings([String.t()], keyword) :: :ok + def print_headings(headings, options \\ []) do + options = Keyword.merge(default_options(), options) + newline_after_block(options) + width = options[:width] + + for heading <- headings do + padding = div(width + String.length(heading), 2) + heading = String.pad_leading(heading, padding) + heading = if options[:enabled], do: String.pad_trailing(heading, width), else: heading + write(:doc_title, heading, options) + end + + newline_after_block(options) end @doc """ - Prints the documentation body. + Prints documentation metadata (only `delegate_to`, `deprecated`, `guard`, and `since` for now). - In addition to the priting string, takes a set of options - defined in `default_options/1`. + See `default_options/0` for docs on the supported options. """ - def print(doc, options \\ []) do - options = Keyword.merge(default_options, options) - doc - |> String.split(["\r\n","\n"], trim: false) - |> Enum.map(&String.rstrip/1) - |> process("", options) + @spec print_metadata(map, keyword) :: :ok + def print_metadata(metadata, options \\ []) when is_map(metadata) do + options = Keyword.merge(default_options(), options) + print_each_metadata(metadata, options) && IO.write("\n") end - defp process([], _indent, _options), do: nil + @metadata_filter [:deprecated, :guard, :since] - defp process(["# " <> heading | rest], _indent, options) do - write_h1(String.strip(heading), options) - process(rest, "", options) + defp print_each_metadata(metadata, options) do + Enum.reduce(metadata, false, fn + {key, value}, _printed when is_binary(value) and key in @metadata_filter -> + label = metadata_label(key, options) + indent = String.duplicate(" ", length_without_escape(label, 0) + 1) + write_with_wrap([label | String.split(value, @spaces)], options[:width], indent, true, "") + + {key, value}, _printed when is_boolean(value) and key in @metadata_filter -> + IO.puts([metadata_label(key, options), ?\s, to_string(value)]) + + {:delegate_to, {m, f, a}}, _printed -> + label = metadata_label(:delegate_to, options) + IO.puts([label, ?\s, Exception.format_mfa(m, f, a)]) + + _metadata, printed -> + printed + end) end - defp process(["## " <> heading | rest], _indent, options) do - write_h2(String.strip(heading), options) - process(rest, "", options) + defp metadata_label(key, options) do + "#{color(:doc_metadata, options)}#{key}:#{maybe_reset(options)}" end - defp process(["### " <> heading | rest], indent, options) do - write_h3(String.strip(heading), indent, options) - process(rest, indent, options) + @doc """ + Prints the documentation body `doc` according to `format`. + + It takes a set of `options` defined in `default_options/0`. + """ + @spec print(term(), String.t(), keyword) :: :ok + def print(doc, format, options \\ []) + + def print(doc, "text/markdown", options) when is_binary(doc) and is_list(options) do + print_markdown(doc, options) end - defp process(["" | rest], indent, options) do - process(rest, indent, options) + def print(doc, "application/erlang+html", options) when is_list(options) do + print_erlang_html(doc, options) end - defp process([" " <> line | rest], indent, options) do - process_code(rest, [line], indent, options) + def print(_doc, format, options) when is_binary(format) and is_list(options) do + IO.puts("\nUnknown documentation format #{inspect(format)}\n") end - defp process([line | rest], indent, options) do - {stripped, count} = strip_spaces(line, 0) - case stripped do - <> when bullet in @bullets -> - process_list(item, rest, count, indent, options) - _ -> - process_text(rest, [line], indent, false, options) + ## Erlang+html + + def print_erlang_html(doc, options) do + options = Keyword.merge(default_options(), options) + IO.write(traverse_erlang_html(doc, "", options)) + end + + defp traverse_erlang_html(text, _indent, _options) when is_binary(text) do + text + end + + defp traverse_erlang_html(nodes, indent, options) when is_list(nodes) do + for node <- nodes do + traverse_erlang_html(node, indent, options) end end - defp strip_spaces(" " <> line, acc) do - strip_spaces(line, acc + 1) + defp traverse_erlang_html({:div, [class: class] ++ _, entries}, indent, options) do + prefix = indent <> quote_prefix(options) + + content = + entries + |> traverse_erlang_html(indent, options) + |> IO.iodata_to_binary() + |> String.trim_trailing() + + [ + prefix, + class |> to_string() |> String.upcase(), + "\n#{prefix}\n#{prefix}" | String.replace(content, "\n", "\n#{prefix}") + ] + |> newline_cons() end - defp strip_spaces(rest, acc) do - {rest, acc} + defp traverse_erlang_html({:p, _, entries}, indent, options) do + [indent | handle_erlang_html_text(entries, indent, options)] end - ## Headings + defp traverse_erlang_html({:h1, _, entries}, indent, options) do + entries |> traverse_erlang_html(indent, options) |> heading(1, options) |> newline_cons() + end - defp write_h1(heading, options) do - write_h2(String.upcase(heading), options) + defp traverse_erlang_html({:h2, _, entries}, indent, options) do + entries |> traverse_erlang_html(indent, options) |> heading(2, options) |> newline_cons() end - defp write_h2(heading, options) do - write(:doc_headings, heading, options) + defp traverse_erlang_html({:h3, _, entries}, indent, options) do + entries |> traverse_erlang_html(indent, options) |> heading(3, options) |> newline_cons() end - defp write_h3(heading, indent, options) do - IO.write(indent) - write(:doc_headings, heading, options) + defp traverse_erlang_html({:h4, _, entries}, indent, options) do + entries |> traverse_erlang_html(indent, options) |> heading(4, options) |> newline_cons() end - ## Lists + defp traverse_erlang_html({:h5, _, entries}, indent, options) do + entries |> traverse_erlang_html(indent, options) |> heading(5, options) |> newline_cons() + end - defp process_list(line, rest, count, indent, options) do - IO.write indent <> "• " - {contents, rest, done} = process_list_next(rest, count, false, []) - process_text(contents, [line], indent <> " ", true, options) - if done, do: IO.puts(IO.ANSI.reset) - process(rest, indent, options) + defp traverse_erlang_html({:h6, _, entries}, indent, options) do + entries |> traverse_erlang_html(indent, options) |> heading(6, options) |> newline_cons() end - # Process the thing after a list item entry. It can be either: - # - # * Continuation of the list - # * A nested list - # * The end of the list - # - defp process_list_next([" " <> _ = line | rest], count, _done, acc) do - case list_next(line, count) do - :done -> {Enum.reverse(acc), [line|rest], false} - chopped -> process_list_next(rest, count, false, [chopped|acc]) + defp traverse_erlang_html({:br, _, []}, _indent, _options) do + [] + end + + defp traverse_erlang_html({:i, _, entries}, indent, options) do + inline_text("_", traverse_erlang_html(entries, indent, options), options) + end + + defp traverse_erlang_html({:em, _, entries}, indent, options) do + inline_text("*", traverse_erlang_html(entries, indent, options), options) + end + + defp traverse_erlang_html({tag, _, entries}, indent, options) when tag in [:strong, :b] do + inline_text("**", traverse_erlang_html(entries, indent, options), options) + end + + defp traverse_erlang_html({:code, _, entries}, indent, options) do + inline_text("`", traverse_erlang_html(entries, indent, options), options) + end + + defp traverse_erlang_html({:pre, _, [{:code, _, entries}]}, indent, options) do + string = + entries + |> traverse_erlang_html(indent, options) + |> IO.iodata_to_binary() + + ["#{indent} ", String.replace(string, "\n", "\n#{indent} ")] |> newline_cons() + end + + defp traverse_erlang_html({:a, attributes, entries}, indent, options) do + if href = attributes[:href] do + [traverse_erlang_html(entries, indent, options), ?\s, ?(, href, ?)] + else + traverse_erlang_html(entries, indent, options) end end - defp process_list_next([<> | _] = rest, _count, _done, acc) when bullet in @bullets do - {Enum.reverse(acc), rest, false} + defp traverse_erlang_html({:dl, _, entries}, indent, options) do + traverse_erlang_html(entries, indent, options) end - defp process_list_next(["" | rest], count, _done, acc) do - process_list_next(rest, count, true, [""|acc]) + defp traverse_erlang_html({:dt, _, entries}, indent, options) do + [ + "#{indent} ", + bullet_text(options) | handle_erlang_html_text(entries, indent <> " ", options) + ] end - defp process_list_next(rest, _count, done, acc) do - {Enum.reverse(acc), rest, done} + defp traverse_erlang_html({:dd, _, entries}, indent, options) do + ["#{indent} " | handle_erlang_html_text(entries, indent <> " ", options)] end - defp list_next(<>, 0) when bullet in @bullets, do: :done - defp list_next(line, 0), do: chop(line, 2) - defp list_next(" " <> line, acc), do: list_next(line, acc - 1) - defp list_next(line, _acc), do: line + defp traverse_erlang_html({:ul, attributes, entries}, indent, options) do + if attributes[:class] == "types" do + types = + for {:li, _, lines} <- entries, + line <- lines, + do: ["#{indent} ", traverse_erlang_html(line, indent <> " ", options), ?\n] - defp chop(" " <> line, acc) when acc > 0, do: chop(line, acc - 1) - defp chop(line, _acc), do: line + if types != [] do + ["#{indent}Typespecs:\n\n", types, ?\n] + else + [] + end + else + for {:li, _, lines} <- entries do + [ + "#{indent} ", + bullet_text(options) | handle_erlang_html_text(lines, indent <> " ", options) + ] + end + end + end - ## Text (paragraphs / lists) + defp traverse_erlang_html({:ol, _, entries}, indent, options) do + for {{:li, _, lines}, i} <- Enum.with_index(entries, 1) do + [ + "#{indent} ", + Integer.to_string(i), + ". " | handle_erlang_html_text(lines, indent <> " ", options) + ] + end + end + + defp traverse_erlang_html({tag, _, entries}, indent, options) do + [ + indent <> "<#{tag}>\n", + traverse_erlang_html(entries, indent <> " ", options) + |> IO.iodata_to_binary() + |> String.trim_trailing(), + "\n" <> indent <> "" + ] + |> newline_cons() + end + + defp newline_cons(text) do + [text | "\n\n"] + end + + defp handle_erlang_html_text(entries, indent, options) do + if Enum.all?(entries, &inline_html?/1) do + entries + |> traverse_erlang_html(indent, options) + |> IO.iodata_to_binary() + |> String.split(@spaces) + |> wrap_text(options[:width], indent, true, "", []) + |> tl() + |> newline_cons() + else + entries + |> traverse_erlang_html(indent, options) + |> IO.iodata_to_binary() + |> String.trim_leading() + end + end + + defp inline_html?(binary) when is_binary(binary), do: true + defp inline_html?({tag, _, _}) when tag in [:a, :code, :em, :i, :strong, :b, :br], do: true + defp inline_html?(_), do: false + + ## Markdown + + def print_markdown(doc, options) do + options = Keyword.merge(default_options(), options) + + doc + |> String.split(["\r\n", "\n"], trim: false) + |> Enum.map(&String.trim_trailing/1) + |> process([], "", options) + end + + defp process([], text, indent, options) do + write_text(text, indent, options) + end + + defp process(["# " <> _ = heading | rest], text, indent, options) do + write_heading(heading, rest, text, indent, options) + end - defp process_text(doc=["" | _], para, indent, from_list, options) do - write_text(Enum.reverse(para), indent, from_list, options) - process(doc, indent, options) + defp process(["## " <> _ = heading | rest], text, indent, options) do + write_heading(heading, rest, text, indent, options) end - defp process_text([], para, indent, from_list, options) do - write_text(Enum.reverse(para), indent, from_list, options) + defp process(["### " <> _ = heading | rest], text, indent, options) do + write_heading(heading, rest, text, indent, options) end - defp process_text([line | rest], para, indent, true, options) do - {stripped, count} = strip_spaces(line, 0) + defp process(["#### " <> _ = heading | rest], text, indent, options) do + write_heading(heading, rest, text, indent, options) + end + + defp process(["##### " <> _ = heading | rest], text, indent, options) do + write_heading(heading, rest, text, indent, options) + end + + defp process(["###### " <> _ = heading | rest], text, indent, options) do + write_heading(heading, rest, text, indent, options) + end + + defp process([">" <> line | rest], text, indent, options) do + write_text(text, indent, options) + process_quote(rest, [line], indent, options) + end + + defp process(["" | rest], text, indent, options) do + write_text(text, indent, options) + process(rest, [], indent, options) + end + + defp process([" " <> line | rest], text, indent, options) do + write_text(text, indent, options) + process_code(rest, [line], indent, options) + end + + defp process(["```" <> _line | rest], text, indent, options) do + process_fenced_code_block(rest, text, indent, options, _delimiter = "```") + end + + defp process(["~~~" <> _line | rest], text, indent, options) do + process_fenced_code_block(rest, text, indent, options, _delimiter = "~~~") + end + + defp process(all = [line | rest], text, indent, options) do + {stripped, count} = strip_spaces(line, 0, :infinity) + + cond do + link_label?(stripped, count) -> + write_text([line], indent, options, true) + process(rest, text, indent, options) + + table_line?(stripped) and rest != [] and table_line?(hd(rest)) -> + write_text(text, indent, options) + process_table(all, indent, options) + + true -> + process_rest(stripped, rest, count, text, indent, options) + end + end + + ### Headings + + defp write_heading(heading, rest, text, indent, options) do + write_text(text, indent, options) + write(:doc_headings, heading, options) + newline_after_block(options) + process(rest, [], "", options) + end + + ### Quotes + + defp process_quote([], lines, indent, options) do + write_quote(lines, indent, options, false) + end + + defp process_quote([">", ">" <> line | rest], lines, indent, options) do + write_quote(lines, indent, options, true) + write_empty_quote_line(options) + process_quote(rest, [line], indent, options) + end + + defp process_quote([">" <> line | rest], lines, indent, options) do + process_quote(rest, [line | lines], indent, options) + end + + defp process_quote(rest, lines, indent, options) do + write_quote(lines, indent, options, false) + process(rest, [], indent, options) + end + + defp write_quote(lines, indent, options, no_wrap) do + lines + |> Enum.map(&String.trim/1) + |> Enum.reverse() + |> write_lines( + indent, + options, + no_wrap, + quote_prefix(options) + ) + end + + defp write_empty_quote_line(options) do + options + |> quote_prefix() + |> IO.puts() + end + + ### Lists + + defp process_rest(stripped, rest, count, text, indent, options) do case stripped do - <> when bullet in @bullets -> - write_text(Enum.reverse(para), indent, true, options) - process_list(item, rest, count, indent, options) + <> when bullet in @bullets -> + write_text(text, indent, options) + process_list(bullet_text(options), item, rest, count, indent, options) + + <> when d1 in ?0..?9 -> + write_text(text, indent, options) + process_list(<>, item, rest, count, indent, options) + + <> when d1 in ?0..?9 and d2 in ?0..?9 -> + write_text(text, indent, options) + process_list(<>, item, rest, count, indent, options) + _ -> - process_text(rest, [line | para], indent, true, options) + process(rest, [stripped | text], indent, options) + end + end + + defp process_list(entry, line, rest, count, indent, options) do + # The first list always win some extra padding + entry = if indent == "", do: " " <> entry, else: entry + new_indent = indent <> String.duplicate(" ", String.length(entry)) + + {contents, rest, done} = + process_list_next(rest, count, byte_size(new_indent) - byte_size(indent), []) + + process(contents, [indent <> entry <> line, :no_wrap], new_indent, options) + + if done, do: newline_after_block(options) + process(rest, [], indent, options) + end + + defp process_list_next([line | rest], count, max, acc) do + {stripped, next_count} = strip_spaces(line, 0, max) + + case process_list_next_kind(stripped, rest, count, next_count) do + :next -> process_list_next(rest, count, max, [stripped | acc]) + :done -> {Enum.reverse(acc), [line | rest], true} + :list -> {Enum.reverse(acc), [line | rest], false} end end - defp process_text([line | rest], para, indent, from_list, options) do - process_text(rest, [line | para], indent, from_list, options) + defp process_list_next([], _count, _max, acc) do + {Enum.reverse(acc), [], true} end - defp write_text(lines, indent, from_list, options) do + defp process_list_next_kind(stripped, rest, count, next_count) do + case {stripped, rest} do + {<>, _} when bullet in @bullets and next_count <= count -> + :list + + {<>, _} when d1 in ?0..?9 and next_count <= count -> + :list + + {<>, _} + when d1 in ?0..?9 and d2 in ?0..?9 and next_count <= count -> + :list + + {"", [" " <> _ | _]} -> + :next + + {"", _} -> + :done + + _ -> + :next + end + end + + ### Text + + defp write_text(text, indent, options) do + case Enum.reverse(text) do + [:no_wrap | rest] -> write_text(rest, indent, options, true) + rest -> write_text(rest, indent, options, false) + end + end + + defp write_text([], _indent, _options, _no_wrap) do + :ok + end + + defp write_text(lines, indent, options, no_wrap) do + write_lines(lines, indent, options, no_wrap, "") + end + + defp write_lines(lines, indent, options, no_wrap, prefix) do lines |> Enum.join(" ") - |> handle_links - |> handle_inline(nil, [], [], options) - |> String.split(~r{\s}) - |> write_with_wrap(options[:width] - byte_size(indent), indent, from_list) + |> format_text(options) + |> String.split(@spaces) + |> write_with_wrap(options[:width] - byte_size(indent), indent, no_wrap, prefix) - unless from_list, do: IO.puts(IO.ANSI.reset) + unless no_wrap, do: newline_after_block(options) end - ## Code blocks + defp format_text(text, options) do + text + |> handle_links() + |> handle_inline(options) + end + + ### Code blocks defp process_code([], code, indent, options) do write_code(code, indent, options) end # Blank line between code blocks - defp process_code([ "", " " <> line | rest ], code, indent, options) do + defp process_code(["", " " <> line | rest], code, indent, options) do process_code(rest, [line, "" | code], indent, options) end - defp process_code([ " " <> line | rest ], code, indent, options) do - process_code(rest, [line|code], indent, options) + defp process_code([" " <> line | rest], code, indent, options) do + process_code(rest, [line | code], indent, options) end defp process_code(rest, code, indent, options) do write_code(code, indent, options) - process(rest, indent, options) + process(rest, [], indent, options) + end + + defp process_fenced_code_block(rest, text, indent, options, delimiter) do + write_text(text, indent, options) + process_fenced_code(rest, [], indent, options, delimiter) + end + + defp process_fenced_code([], code, indent, options, _delimiter) do + write_code(code, indent, options) + end + + defp process_fenced_code([line | rest], code, indent, options, delimiter) do + if line === delimiter do + process_code(rest, code, indent, options) + else + process_fenced_code(rest, [line | code], indent, options, delimiter) + end end defp write_code(code, indent, options) do - write(:doc_code, "#{indent}┃ #{Enum.join(Enum.reverse(code), "\n#{indent}┃ ")}", options) + write(:doc_code, "#{indent} #{Enum.join(Enum.reverse(code), "\n#{indent} ")}", options) + newline_after_block(options) + end + + ### Tables + + defp process_table(lines, indent, options) do + {table, rest} = Enum.split_while(lines, &table_line?/1) + table_lines(table, options) + newline_after_block(options) + process(rest, [], indent, options) + end + + defp table_lines(lines, options) do + lines = Enum.map(lines, &split_into_columns(&1, options)) + count = Enum.map(lines, &length/1) |> Enum.max() + lines = Enum.map(lines, &pad_to_number_of_columns(&1, count)) + + widths = + for line <- lines do + if table_header?(line) do + for _ <- line, do: 0 + else + for {_col, length} <- line, do: length + end + end + + col_widths = Enum.reduce(widths, List.duplicate(0, count), &max_column_widths/2) + render_table(lines, col_widths, options) + end + + defp split_into_columns(line, options) do + line + |> String.trim(" ") + |> String.trim("|") + |> String.split(~r{(? Enum.map(&render_column(&1, options)) + end + + defp render_column(col, options) do + col = + col + |> String.trim() + |> String.replace("\\\|", "|") + |> handle_links + |> handle_inline(options) + + {col, length_without_escape(col, 0)} + end + + defp pad_to_number_of_columns(cols, col_count), + do: cols ++ List.duplicate({"", 0}, col_count - length(cols)) + + defp max_column_widths(cols, widths), + do: Enum.zip(cols, widths) |> Enum.map(fn {a, b} -> max(a, b) end) + + # If second line is heading separator, use the heading style on the first + defp render_table([first, second | rest], widths, options) do + combined = Enum.zip(first, widths) + + if table_header?(second) do + alignments = Enum.map(second, &column_alignment/1) + options = Keyword.put_new(options, :alignments, alignments) + draw_table_row(combined, options, :heading) + render_table(rest, widths, options) + else + draw_table_row(combined, options) + render_table([second | rest], widths, options) + end + end + + defp render_table([first | rest], widths, options) do + combined = Enum.zip(first, widths) + draw_table_row(combined, options) + render_table(rest, widths, options) + end + + defp render_table([], _, _), do: nil + + defp column_alignment({line, _}) do + cond do + String.starts_with?(line, ":") and String.ends_with?(line, ":") -> :center + String.ends_with?(line, ":") -> :right + true -> :left + end + end + + defp table_header?(line) do + Enum.all?(line, fn {col, _} -> table_header_column?(col) end) + end + + defp table_header_column?(":" <> rest), do: table_header_contents?(rest) + defp table_header_column?(col), do: table_header_contents?(col) + + defp table_header_contents?("-" <> rest), do: table_header_contents?(rest) + defp table_header_contents?(":"), do: true + defp table_header_contents?(""), do: true + defp table_header_contents?(_), do: false + + defp draw_table_row(cols_and_widths, options, heading \\ false) do + default_alignments = List.duplicate(:left, length(cols_and_widths)) + alignments = Keyword.get(options, :alignments, default_alignments) + + columns = + cols_and_widths + |> Enum.zip(alignments) + |> Enum.map_join(" | ", &generate_table_cell/1) + + if heading do + write(:doc_table_heading, columns, options) + else + IO.puts(columns) + end + end + + defp generate_table_cell({{{col, length}, width}, :center}) do + ansi_diff = byte_size(col) - length + width = width + ansi_diff + + col + |> String.pad_leading(div(width, 2) - div(length, 2) + length) + |> String.pad_trailing(width + 1 - rem(width, 2)) + end + + defp generate_table_cell({{{col, length}, width}, :right}) do + ansi_diff = byte_size(col) - length + String.pad_leading(col, width + ansi_diff) + end + + defp generate_table_cell({{{col, length}, width}, :left}) do + ansi_diff = byte_size(col) - length + String.pad_trailing(col, width + ansi_diff) + end + + defp table_line?(line) do + line =~ ~r/[:\ -]\|[:\ -]/ end ## Helpers + defp link_label?("[" <> rest, count) when count <= 3, do: link_label?(rest) + defp link_label?(_, _), do: false + + defp link_label?("]: " <> _), do: true + defp link_label?("]" <> _), do: false + defp link_label?(""), do: false + defp link_label?(<<_>> <> rest), do: link_label?(rest) + + defp strip_spaces(" " <> line, acc, max) when acc < max, do: strip_spaces(line, acc + 1, max) + defp strip_spaces(rest, acc, _max), do: {rest, acc} + defp write(style, string, options) do - IO.puts color(style, options) <> string <> IO.ANSI.reset - IO.puts IO.ANSI.reset + IO.puts([color(style, options), string, maybe_reset(options)]) end - defp write_with_wrap([], _available, _indent, _first) do + defp write_with_wrap([], _available, _indent, _first, _prefix) do :ok end - defp write_with_wrap(words, available, indent, first) do - {words, rest} = take_words(words, available, []) - IO.puts (if first, do: "", else: indent) <> Enum.join(words, " ") - write_with_wrap(rest, available, indent, false) + defp write_with_wrap(words, available, indent, first, prefix) do + words + |> wrap_text(available, indent, first, prefix, []) + |> tl() + |> IO.puts() + end + + defp wrap_text([], _available, _indent, _first, _prefix, wrapped_lines) do + Enum.reverse(wrapped_lines) + end + + defp wrap_text(words, available, indent, first, prefix, wrapped_lines) do + prefix_length = length_without_escape(prefix, 0) + {words, rest} = take_words(words, available - prefix_length, []) + line = [if(first, do: "", else: indent), prefix, Enum.join(words, " ")] + + wrap_text(rest, available, indent, false, prefix, [line, ?\n | wrapped_lines]) end - defp take_words([word|words], available, acc) do + defp take_words([word | words], available, acc) do available = available - length_without_escape(word, 0) cond do # It fits, take one for space and continue decreasing available > 0 -> - take_words(words, available - 1, [word|acc]) + take_words(words, available - 1, [word | acc]) # No space but we got no words acc == [] -> @@ -252,7 +764,7 @@ defmodule IO.ANSI.Docs do # Otherwise true -> - {Enum.reverse(acc), [word|words]} + {Enum.reverse(acc), [word | words]} end end @@ -260,17 +772,17 @@ defmodule IO.ANSI.Docs do {Enum.reverse(acc), []} end - defp length_without_escape(<< ?\e, ?[, _, _, ?m, rest :: binary >>, count) do + defp length_without_escape(<> <> rest, count) do length_without_escape(rest, count) end - defp length_without_escape(<< ?\e, ?[, _, ?m, rest :: binary >>, count) do + defp length_without_escape(<> <> rest, count) do length_without_escape(rest, count) end defp length_without_escape(rest, count) do case String.next_grapheme(rest) do - {_, rest} -> length_without_escape(rest, count + 1) + {_, rest} -> length_without_escape(rest, count + 1) nil -> count end end @@ -282,95 +794,169 @@ defmodule IO.ANSI.Docs do end defp escape_underlines_in_link(text) do - case Regex.match?(~r{.*(https?\S*)}, text) do - true -> - Regex.replace(~r{_}, text, "\\\\_") - _ -> - text - end + # Regular expression adapted from https://tools.ietf.org/html/rfc3986#appendix-B + Regex.replace(~r{[a-z][a-z0-9\+\-\.]*://\S*}i, text, &String.replace(&1, "_", "\\_")) end defp remove_square_brackets_in_link(text) do - Regex.replace(~r{\[(.*?)\]\((.*?)\)}, text, "\\1 (\\2)") + Regex.replace(~r{\[([^\]]*?)\]\((.*?)\)}, text, "\\1 (\\2)") end - # Single inline quotes. - @single [?`, ?_, ?*] + # We have four entries: **, __, *, _ and `. + # + # The first four behave the same while the last one is simpler + # when it comes to delimiters as it ignores spaces and escape + # characters. But, since the first two has two characters, + # we need to handle 3 cases: + # + # 1. __ and ** + # 2. _ and * + # 3. ` + # + # Where the first two should have the same code but match differently. + @single [?_, ?*] + + # Characters that can mark the beginning or the end of a word. + # Only support the most common ones at this moment. + @delimiters [?\s, ?', ?", ?!, ?@, ?#, ?$, ?%, ?^, ?&] ++ + [?-, ?+, ?(, ?), ?[, ?], ?{, ?}, ?<, ?>, ?.] - # ` does not require space in between - @spaced [?_, ?*] + ### Inline start - # Clauses for handling spaces - defp handle_inline(<>, nil, buffer, acc, options) do - handle_inline(rest, nil, [?\s, ?*, ?*|buffer], acc, options) + defp handle_inline(<>, options) when mark in @single do + handle_inline(rest, [mark | mark], [<>], [], options) end - defp handle_inline(<>, nil, buffer, acc, options) when mark in @spaced do - handle_inline(rest, nil, [?\s, mark|buffer], acc, options) + defp handle_inline(<>, options) when mark in @single do + handle_inline(rest, mark, [<>], [], options) end - defp handle_inline(<>, limit, buffer, acc, options) do - handle_inline(rest, limit, [?*, ?*, ?\s|buffer], acc, options) + defp handle_inline(rest, options) do + handle_inline(rest, nil, [], [], options) end - defp handle_inline(<>, limit, buffer, acc, options) when mark in @spaced do - handle_inline(rest, limit, [mark, ?\s|buffer], acc, options) + ### Inline delimiters + + defp handle_inline(<>, nil, buffer, acc, options) + when rest != "" and delimiter in @delimiters and mark in @single do + acc = [delimiter, Enum.reverse(buffer) | acc] + handle_inline(rest, [mark | mark], [<>], acc, options) + end + + defp handle_inline(<>, nil, buffer, acc, options) + when rest != "" and delimiter in @delimiters and mark in @single do + handle_inline(rest, mark, [<>], [delimiter, Enum.reverse(buffer) | acc], options) + end + + defp handle_inline(<>, nil, buffer, acc, options) + when rest != "" do + handle_inline(rest, ?`, ["`"], [Enum.reverse(buffer) | acc], options) + end + + ### Clauses for handling escape + + defp handle_inline(<>, nil, buffer, acc, options) + when rest != "" and mark in @single do + acc = [?\\, Enum.reverse(buffer) | acc] + handle_inline(rest, [mark | mark], [<>], acc, options) end - # Clauses for handling escape - defp handle_inline(<>, limit, buffer, acc, options) do - handle_inline(rest, limit, [?\\|buffer], acc, options) + defp handle_inline(<>, nil, buffer, acc, options) + when rest != "" and mark in @single do + handle_inline(rest, mark, [<>], [?\\, Enum.reverse(buffer) | acc], options) end - defp handle_inline(<>, limit, buffer, acc, options) do - handle_inline(rest, limit, [?*, ?*|buffer], acc, options) + defp handle_inline(<>, limit, buffer, acc, options) do + handle_inline(rest, limit, [?\\ | buffer], acc, options) end - # A escape is not valid inside ` - defp handle_inline(<>, limit, buffer, acc, options) - when mark in [?_, ?*, ?`] and not(mark == limit and mark == ?`) do - handle_inline(rest, limit, [mark|buffer], acc, options) + # An escape is not valid inside ` + defp handle_inline(<>, limit, buffer, acc, options) when limit != ?` do + handle_inline(rest, limit, [mark | buffer], acc, options) end - # Inline start - defp handle_inline(<>, nil, buffer, acc, options) when rest != "" do - handle_inline(rest, ?d, ["**"], [Enum.reverse(buffer)|acc], options) + ### Inline end + + defp handle_inline(<>, [mark | mark], buffer, acc, options) + when delimiter in @delimiters and mark in @single do + inline_buffer = inline_buffer(buffer, options) + handle_inline(<>, nil, [], [inline_buffer | acc], options) end - defp handle_inline(<>, nil, buffer, acc, options) when rest != "" and mark in @single do - handle_inline(rest, mark, [<>], [Enum.reverse(buffer)|acc], options) + defp handle_inline(<>, mark, buffer, acc, options) + when delimiter in @delimiters and mark in @single do + inline_buffer = inline_buffer(buffer, options) + handle_inline(<>, nil, [], [inline_buffer | acc], options) end - # Inline end - defp handle_inline(<>, ?d, buffer, acc, options) do - handle_inline(rest, nil, [], [inline_buffer(buffer, options)|acc], options) + defp handle_inline(<>, [mark | mark], buffer, acc, options) + when rest == "" and mark in @single do + handle_inline(<<>>, nil, [], [inline_buffer(buffer, options) | acc], options) end - defp handle_inline(<>, mark, buffer, acc, options) when mark in @single do - handle_inline(rest, nil, [], [inline_buffer(buffer, options)|acc], options) + defp handle_inline(<>, mark, buffer, acc, options) + when rest == "" and mark in @single do + handle_inline(<<>>, nil, [], [inline_buffer(buffer, options) | acc], options) end - defp handle_inline(<>, mark, buffer, acc, options) do - handle_inline(rest, mark, [char|buffer], acc, options) + defp handle_inline(<>, ?`, buffer, acc, options) do + handle_inline(rest, nil, [], [inline_buffer(buffer, options) | acc], options) + end + + ### Catch all + + defp handle_inline(<>, mark, buffer, acc, options) do + handle_inline(rest, mark, [char | buffer], acc, options) end defp handle_inline(<<>>, _mark, buffer, acc, _options) do - IO.iodata_to_binary Enum.reverse([Enum.reverse(buffer)|acc]) + IO.iodata_to_binary(Enum.reverse([Enum.reverse(buffer) | acc])) end defp inline_buffer(buffer, options) do - [h|t] = Enum.reverse([IO.ANSI.reset|buffer]) - [color_for(h, options)|t] + [mark | t] = Enum.reverse(buffer) + inline_text(mark, t, options) + end + + ## Helpers + + defp quote_prefix(options), do: "#{color(:doc_quote, options)}> #{maybe_reset(options)}" + + defp heading(text, n, options) do + [color(:doc_headings, options), String.duplicate("#", n), " ", text, maybe_reset(options)] + end + + defp inline_text(mark, text, options) do + if options[:enabled] do + [[color_for(mark, options) | text] | IO.ANSI.reset()] + else + [[mark | text] | mark] + end + end + + defp color_for(mark, colors) do + case mark do + "__" -> color(:doc_bold, colors) + "**" -> color(:doc_bold, colors) + "_" -> color(:doc_underline, colors) + "*" -> color(:doc_underline, colors) + "`" -> color(:doc_inline_code, colors) + end end - defp color_for("`", colors), do: color(:doc_inline_code, colors) - defp color_for("_", colors), do: color(:doc_underline, colors) - defp color_for("*", colors), do: color(:doc_bold, colors) - defp color_for("**", colors), do: color(:doc_bold, colors) + defp bullet_text(options) do + if options[:enabled], do: @bullet_text_unicode, else: @bullet_text_ascii + end defp color(style, colors) do - color = colors[style] - enabled = colors[:enabled] - IO.ANSI.escape_fragment("%{#{color}}", enabled) + IO.ANSI.format_fragment(colors[style], colors[:enabled]) + end + + defp newline_after_block(options) do + IO.puts(maybe_reset(options)) + end + + defp maybe_reset(options) do + if options[:enabled], do: IO.ANSI.reset(), else: "" end end diff --git a/lib/elixir/lib/io/stream.ex b/lib/elixir/lib/io/stream.ex index d275626226e..02c16a13972 100644 --- a/lib/elixir/lib/io/stream.ex +++ b/lib/elixir/lib/io/stream.ex @@ -1,8 +1,9 @@ defmodule IO.StreamError do defexception [:reason, :message] + @impl true def exception(opts) do - reason = opts[:reason] + reason = opts[:reason] formatted = IO.iodata_to_binary(:file.format_error(reason)) %IO.StreamError{message: "error during streaming: #{formatted}", reason: reason} end @@ -10,28 +11,29 @@ end defmodule IO.Stream do @moduledoc """ - Defines a `IO.Stream` struct returned by `IO.stream/2` and `IO.binstream/2`. + Defines an `IO.Stream` struct returned by `IO.stream/2` and `IO.binstream/2`. The following fields are public: * `device` - the IO device * `raw` - a boolean indicating if bin functions should be used - * `line_or_bytes` - if reading should read lines or a given amount of bytes + * `line_or_bytes` - if reading should read lines or a given number of bytes + + It is worth noting that an IO stream has side effects and every time you go + over the stream you may get different results. """ defstruct device: nil, raw: true, line_or_bytes: :line + @type t :: %__MODULE__{} + @doc false def __build__(device, raw, line_or_bytes) do %IO.Stream{device: device, raw: raw, line_or_bytes: line_or_bytes} end defimpl Collectable do - def empty(stream) do - stream - end - def into(%{device: device, raw: raw} = stream) do {:ok, into(stream, device, raw)} end @@ -40,10 +42,12 @@ defmodule IO.Stream do fn :ok, {:cont, x} -> case raw do - true -> IO.binwrite(device, x) + true -> IO.binwrite(device, x) false -> IO.write(device, x) end - :ok, _ -> stream + + :ok, _ -> + stream end end end @@ -52,10 +56,11 @@ defmodule IO.Stream do def reduce(%{device: device, raw: raw, line_or_bytes: line_or_bytes}, acc, fun) do next_fun = case raw do - true -> &IO.each_binstream(&1, line_or_bytes) + true -> &IO.each_binstream(&1, line_or_bytes) false -> &IO.each_stream(&1, line_or_bytes) end - Stream.unfold(device, next_fun).(acc, fun) + + Stream.resource(fn -> device end, next_fun, & &1).(acc, fun) end def count(_stream) do @@ -65,5 +70,9 @@ defmodule IO.Stream do def member?(_stream, _term) do {:error, __MODULE__} end + + def slice(_stream) do + {:error, __MODULE__} + end end end diff --git a/lib/elixir/lib/kernel.ex b/lib/elixir/lib/kernel.ex index a0778bbd4d1..bc6f04114bc 100644 --- a/lib/elixir/lib/kernel.ex +++ b/lib/elixir/lib/kernel.ex @@ -1,32 +1,236 @@ # Use elixir_bootstrap module to be able to bootstrap Kernel. # The bootstrap module provides simpler implementations of the # functions removed, simple enough to bootstrap. -import Kernel, except: [@: 1, defmodule: 2, def: 1, def: 2, defp: 2, - defmacro: 1, defmacro: 2, defmacrop: 2] +import Kernel, + except: [@: 1, defmodule: 2, def: 1, def: 2, defp: 2, defmacro: 1, defmacro: 2, defmacrop: 2] + import :elixir_bootstrap defmodule Kernel do @moduledoc """ - `Kernel` provides the default macros and functions - Elixir imports into your environment. These macros and functions - can be skipped or cherry-picked via the `import` macro. For - instance, if you want to tell Elixir not to import the `if` - macro, you can do: + `Kernel` is Elixir's default environment. + + It mainly consists of: + + * basic language primitives, such as arithmetic operators, spawning of processes, + data type handling, and others + * macros for control-flow and defining new functionality (modules, functions, and the like) + * guard checks for augmenting pattern matching + + You can invoke `Kernel` functions and macros anywhere in Elixir code + without the use of the `Kernel.` prefix since they have all been + automatically imported. For example, in IEx, you can call: + + iex> is_number(13) + true + + If you don't want to import a function or macro from `Kernel`, use the `:except` + option and then list the function/macro by arity: + + import Kernel, except: [if: 2, unless: 2] - import Kernel, except: [if: 2] + See `Kernel.SpecialForms.import/2` for more information on importing. Elixir also has special forms that are always imported and cannot be skipped. These are described in `Kernel.SpecialForms`. + ## The standard library + + `Kernel` provides the basic capabilities the Elixir standard library + is built on top of. It is recommended to explore the standard library + for advanced functionality. Here are the main groups of modules in the + standard library (this list is not a complete reference, see the + documentation sidebar for all entries). + + ### Built-in types + + The following modules handle Elixir built-in data types: + + * `Atom` - literal constants with a name (`true`, `false`, and `nil` are atoms) + * `Float` - numbers with floating point precision + * `Function` - a reference to code chunk, created with the `fn/1` special form + * `Integer` - whole numbers (not fractions) + * `List` - collections of a variable number of elements (linked lists) + * `Map` - collections of key-value pairs + * `Process` - light-weight threads of execution + * `Port` - mechanisms to interact with the external world + * `Tuple` - collections of a fixed number of elements + + There are two data types without an accompanying module: + + * Bitstring - a sequence of bits, created with `Kernel.SpecialForms.<<>>/1`. + When the number of bits is divisible by 8, they are called binaries and can + be manipulated with Erlang's `:binary` module + * Reference - a unique value in the runtime system, created with `make_ref/0` + + ### Data types + + Elixir also provides other data types that are built on top of the types + listed above. Some of them are: + + * `Date` - `year-month-day` structs in a given calendar + * `DateTime` - date and time with time zone in a given calendar + * `Exception` - data raised from errors and unexpected scenarios + * `MapSet` - unordered collections of unique elements + * `NaiveDateTime` - date and time without time zone in a given calendar + * `Keyword` - lists of two-element tuples, often representing optional values + * `Range` - inclusive ranges between two integers + * `Regex` - regular expressions + * `String` - UTF-8 encoded binaries representing characters + * `Time` - `hour:minute:second` structs in a given calendar + * `URI` - representation of URIs that identify resources + * `Version` - representation of versions and requirements + + ### System modules + + Modules that interface with the underlying system, such as: + + * `IO` - handles input and output + * `File` - interacts with the underlying file system + * `Path` - manipulates file system paths + * `System` - reads and writes system information + + ### Protocols + + Protocols add polymorphic dispatch to Elixir. They are contracts + implementable by data types. See `Protocol` for more information on + protocols. Elixir provides the following protocols in the standard library: + + * `Collectable` - collects data into a data type + * `Enumerable` - handles collections in Elixir. The `Enum` module + provides eager functions for working with collections, the `Stream` + module provides lazy functions + * `Inspect` - converts data types into their programming language + representation + * `List.Chars` - converts data types to their outside world + representation as charlists (non-programming based) + * `String.Chars` - converts data types to their outside world + representation as strings (non-programming based) + + ### Process-based and application-centric functionality + + The following modules build on top of processes to provide concurrency, + fault-tolerance, and more. + + * `Agent` - a process that encapsulates mutable state + * `Application` - functions for starting, stopping and configuring + applications + * `GenServer` - a generic client-server API + * `Registry` - a key-value process-based storage + * `Supervisor` - a process that is responsible for starting, + supervising and shutting down other processes + * `Task` - a process that performs computations + * `Task.Supervisor` - a supervisor for managing tasks exclusively + + ### Supporting documents + + Elixir documentation also includes supporting documents under the + "Pages" section. Those are: + + * [Compatibility and deprecations](compatibility-and-deprecations.md) - lists + compatibility between every Elixir version and Erlang/OTP, release schema; + lists all deprecated functions, when they were deprecated and alternatives + * [Library guidelines](library-guidelines.md) - general guidelines, anti-patterns, + and rules for those writing libraries + * [Naming conventions](naming-conventions.md) - naming conventions for Elixir code + * [Operators](operators.md) - lists all Elixir operators and their precedences + * [Patterns and guards](patterns-and-guards.md) - an introduction to patterns, + guards, and extensions + * [Syntax reference](syntax-reference.md) - the language syntax reference + * [Typespecs](typespecs.md)- types and function specifications, including list of types + * [Unicode syntax](unicode-syntax.md) - outlines Elixir support for Unicode + * [Writing documentation](writing-documentation.md) - guidelines for writing + documentation in Elixir + + ## Guards + + This module includes the built-in guards used by Elixir developers. + They are a predefined set of functions and macros that augment pattern + matching, typically invoked after the `when` operator. For example: + + def drive(%User{age: age}) when age >= 16 do + ... + end + + The clause above will only be invoked if the user's age is more than + or equal to 16. Guards also support joining multiple conditions with + `and` and `or`. The whole guard is true if all guard expressions will + evaluate to `true`. A more complete introduction to guards is available + in the [Patterns and guards](patterns-and-guards.md) page. + + ## Structural comparison + + The comparison functions in this module perform structural comparison. + This means structures are compared based on their representation and + not on their semantic value. This is specially important for functions + that are meant to provide ordering, such as `>/2`, `=/2`, + `<=/2`, `min/2`, and `max/2`. For example: + + ~D[2017-03-31] > ~D[2017-04-01] + + will return `true` because structural comparison compares the `:day` + field before `:month` or `:year`. Therefore, when comparing structs, + you often use the `compare/2` function made available by the structs + modules themselves: + + iex> Date.compare(~D[2017-03-31], ~D[2017-04-01]) + :lt + + Alternatively, you can use the functions in the `Enum` module to + sort or compute a maximum/minimum: + + iex> Enum.sort([~D[2017-03-31], ~D[2017-04-01]], Date) + [~D[2017-03-31], ~D[2017-04-01]] + iex> Enum.max([~D[2017-03-31], ~D[2017-04-01]], Date) + ~D[2017-04-01] + + ## Truthy and falsy values + + Besides the booleans `true` and `false`, Elixir has the + concept of a "truthy" or "falsy" value. + + * a value is truthy when it is neither `false` nor `nil` + * a value is falsy when it is either `false` or `nil` + + Elixir has functions, like `and/2`, that *only* work with + booleans, but also functions that work with these + truthy/falsy values, like `&&/2` and `!/1`. + + ### Examples + + We can check the truthiness of a value by using the `!/1` + function twice. + + Truthy values: + + iex> !!true + true + iex> !!5 + true + iex> !![1,2] + true + iex> !!"foo" + true + + Falsy values (of which there are exactly two): + + iex> !!false + false + iex> !!nil + false + + ## Inlining + Some of the functions described in this module are inlined by - the Elixir compiler into their Erlang counterparts in the `:erlang` - module. Those functions are called BIFs (builtin internal functions) - in Erlang-land and they exhibit interesting properties, as some of - them are allowed in guards and others are used for compiler + the Elixir compiler into their Erlang counterparts in the + [`:erlang`](`:erlang`) module. + Those functions are called BIFs (built-in internal functions) + in Erlang-land and they exhibit interesting properties, as some + of them are allowed in guards and others are used for compiler optimizations. - Most of the inlined functions can be seen in effect when capturing - the function: + Most of the inlined functions can be seen in effect when + capturing the function: iex> &Kernel.is_atom/1 &:erlang.is_atom/1 @@ -35,6 +239,17 @@ defmodule Kernel do "inlined by the compiler". """ + # We need this check only for bootstrap purposes. + # Once Kernel is loaded and we recompile, it is a no-op. + @compile {:inline, bootstrapped?: 1} + case :code.ensure_loaded(Kernel) do + {:module, _} -> + defp bootstrapped?(_), do: true + + {:error, _} -> + defp bootstrapped?(module), do: :code.ensure_loaded(module) == {:module, module} + end + ## Delegations to Erlang with inlining (macros) @doc """ @@ -51,13 +266,19 @@ defmodule Kernel do 3 """ + @doc guard: true @spec abs(number) :: number def abs(number) do :erlang.abs(number) end @doc """ - Invokes the given `fun` with the array of arguments `args`. + Invokes the given anonymous function `fun` with the list of + arguments `args`. + + If the number of arguments is known at compile time, prefer + `fun.(arg_1, arg_2, ..., arg_n)` as it is clearer than + `apply(fun, [arg_1, arg_2, ..., arg_n])`. Inlined by the compiler. @@ -73,26 +294,37 @@ defmodule Kernel do end @doc """ - Invokes the given `fun` from `module` with the array of arguments `args`. + Invokes the given function from `module` with the list of + arguments `args`. + + `apply/3` is used to invoke functions where the module, function + name or arguments are defined dynamically at runtime. For this + reason, you can't invoke macros using `apply/3`, only functions. + + If the number of arguments and the function name are known at compile time, + prefer `module.function(arg_1, arg_2, ..., arg_n)` as it is clearer than + `apply(module, :function, [arg_1, arg_2, ..., arg_n])`. + + `apply/3` cannot be used to call private functions. Inlined by the compiler. ## Examples iex> apply(Enum, :reverse, [[1, 2, 3]]) - [3,2,1] + [3, 2, 1] """ - @spec apply(module, atom, [any]) :: any - def apply(module, fun, args) do - :erlang.apply(module, fun, args) + @spec apply(module, function_name :: atom, [any]) :: any + def apply(module, function_name, args) do + :erlang.apply(module, function_name, args) end @doc """ Extracts the part of the binary starting at `start` with length `length`. Binaries are zero-indexed. - If start or length references in any way outside the binary, an + If `start` or `length` reference in any way outside the binary, an `ArgumentError` exception is raised. Allowed in guard tests. Inlined by the compiler. @@ -102,13 +334,20 @@ defmodule Kernel do iex> binary_part("foo", 1, 2) "oo" - A negative length can be used to extract bytes at the end of a binary: + A negative `length` can be used to extract bytes that come *before* the byte + at `start`: + + iex> binary_part("Hello", 5, -3) + "llo" - iex> binary_part("foo", 3, -1) - "o" + An `ArgumentError` is raised when the length is outside of the binary: + + binary_part("Hello", 0, 10) + ** (ArgumentError) argument error """ - @spec binary_part(binary, pos_integer, integer) :: binary + @doc guard: true + @spec binary_part(binary, non_neg_integer, integer) :: binary def binary_part(binary, start, length) do :erlang.binary_part(binary, start, length) end @@ -127,6 +366,7 @@ defmodule Kernel do 24 """ + @doc guard: true @spec bit_size(bitstring) :: non_neg_integer def bit_size(bitstring) do :erlang.bit_size(bitstring) @@ -135,8 +375,8 @@ defmodule Kernel do @doc """ Returns the number of bytes needed to contain `bitstring`. - That is, if the number of bits in `bitstring` is not divisible by 8, - the resulting number of bytes will be rounded up. This operation + That is, if the number of bits in `bitstring` is not divisible by 8, the + resulting number of bytes will be rounded up (by excess). This operation happens in constant time. Allowed in guard tests. Inlined by the compiler. @@ -150,27 +390,59 @@ defmodule Kernel do 3 """ + @doc guard: true @spec byte_size(bitstring) :: non_neg_integer def byte_size(bitstring) do :erlang.byte_size(bitstring) end + @doc """ + Returns the smallest integer greater than or equal to `number`. + + If you want to perform ceil operation on other decimal places, + use `Float.ceil/2` instead. + + Allowed in guard tests. Inlined by the compiler. + """ + @doc since: "1.8.0", guard: true + @spec ceil(number) :: integer + def ceil(number) do + :erlang.ceil(number) + end + @doc """ Performs an integer division. - Raises an error if one of the arguments is not an integer. + Raises an `ArithmeticError` exception if one of the arguments is not an + integer, or when the `divisor` is `0`. + + `div/2` performs *truncated* integer division. This means that + the result is always rounded towards zero. + + If you want to perform floored integer division (rounding towards negative infinity), + use `Integer.floor_div/2` instead. Allowed in guard tests. Inlined by the compiler. ## Examples - iex> div(5, 2) - 2 + div(5, 2) + #=> 2 + + div(6, -4) + #=> -1 + + div(-99, 2) + #=> -49 + + div(100, 0) + ** (ArithmeticError) bad argument in arithmetic expression """ - @spec div(integer, integer) :: integer - def div(left, right) do - :erlang.div(left, right) + @doc guard: true + @spec div(integer, neg_integer | pos_integer) :: integer + def div(dividend, divisor) do + :erlang.div(dividend, divisor) end @doc """ @@ -183,9 +455,56 @@ defmodule Kernel do ## Examples + When a process reaches its end, by default it exits with + reason `:normal`. You can also call `exit/1` explicitly if you + want to terminate a process but not signal any failure: + exit(:normal) + + In case something goes wrong, you can also use `exit/1` with + a different reason: + exit(:seems_bad) + If the exit reason is not `:normal`, all the processes linked to the process + that exited will crash (unless they are trapping exits). + + ## OTP exits + + Exits are used by the OTP to determine if a process exited abnormally + or not. The following exits are considered "normal": + + * `exit(:normal)` + * `exit(:shutdown)` + * `exit({:shutdown, term})` + + Exiting with any other reason is considered abnormal and treated + as a crash. This means the default supervisor behaviour kicks in, + error reports are emitted, and so forth. + + This behaviour is relied on in many different places. For example, + `ExUnit` uses `exit(:shutdown)` when exiting the test process to + signal linked processes, supervision trees and so on to politely + shut down too. + + ## CLI exits + + Building on top of the exit signals mentioned above, if the + process started by the command line exits with any of the three + reasons above, its exit is considered normal and the Operating + System process will exit with status 0. + + It is, however, possible to customize the operating system exit + signal by invoking: + + exit({:shutdown, integer}) + + This will cause the operating system process to exit with the status given by + `integer` while signaling all linked Erlang processes to politely + shut down. + + Any other exit reason will cause the operating system process to exit with + status `1` and linked Erlang processes to crash. """ @spec exit(term) :: no_return def exit(reason) do @@ -193,11 +512,44 @@ defmodule Kernel do end @doc """ - Returns the head of a list, raises `badarg` if the list is empty. + Returns the largest integer smaller than or equal to `number`. + + If you want to perform floor operation on other decimal places, + use `Float.floor/2` instead. + + Allowed in guard tests. Inlined by the compiler. + """ + @doc since: "1.8.0", guard: true + @spec floor(number) :: integer + def floor(number) do + :erlang.floor(number) + end + + @doc """ + Returns the head of a list. Raises `ArgumentError` if the list is empty. + + The head of a list is its first element. + + It works with improper lists. + + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + hd([1, 2, 3, 4]) + #=> 1 + + hd([1 | 2]) + #=> 1 + + Giving it an empty list raises: + + hd([]) + #=> ** (ArgumentError) argument error - Inlined by the compiler. """ - @spec hd(list) :: term + @doc guard: true + @spec hd(nonempty_maybe_improper_list(elem, any)) :: elem when elem: term def hd(list) do :erlang.hd(list) end @@ -206,7 +558,23 @@ defmodule Kernel do Returns `true` if `term` is an atom; otherwise returns `false`. Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> is_atom(false) + true + + iex> is_atom(:name) + true + + iex> is_atom(AnAtom) + true + + iex> is_atom("true") + false + """ + @doc guard: true @spec is_atom(term) :: boolean def is_atom(term) do :erlang.is_atom(term) @@ -218,7 +586,16 @@ defmodule Kernel do A binary always contains a complete number of bytes. Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> is_binary("foo") + true + iex> is_binary(<<1::3>>) + false + """ + @doc guard: true @spec is_binary(term) :: boolean def is_binary(term) do :erlang.is_binary(term) @@ -228,28 +605,51 @@ defmodule Kernel do Returns `true` if `term` is a bitstring (including a binary); otherwise returns `false`. Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> is_bitstring("foo") + true + iex> is_bitstring(<<1::3>>) + true + """ + @doc guard: true @spec is_bitstring(term) :: boolean def is_bitstring(term) do :erlang.is_bitstring(term) end @doc """ - Returns `true` if `term` is either the atom `true` or the atom `false` (i.e. a boolean); - otherwise returns false. + Returns `true` if `term` is either the atom `true` or the atom `false` (i.e., + a boolean); otherwise returns `false`. Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> is_boolean(false) + true + + iex> is_boolean(true) + true + + iex> is_boolean(:test) + false + """ + @doc guard: true @spec is_boolean(term) :: boolean def is_boolean(term) do :erlang.is_boolean(term) end @doc """ - Returns `true` if `term` is a floating point number; otherwise returns `false`. + Returns `true` if `term` is a floating-point number; otherwise returns `false`. Allowed in guard tests. Inlined by the compiler. """ + @doc guard: true @spec is_float(term) :: boolean def is_float(term) do :erlang.is_float(term) @@ -259,7 +659,17 @@ defmodule Kernel do Returns `true` if `term` is a function; otherwise returns `false`. Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> is_function(fn x -> x + x end) + true + + iex> is_function("not a function") + false + """ + @doc guard: true @spec is_function(term) :: boolean def is_function(term) do :erlang.is_function(term) @@ -270,7 +680,16 @@ defmodule Kernel do otherwise returns `false`. Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> is_function(fn x -> x * 2 end, 1) + true + iex> is_function(fn x -> x * 2 end, 2) + false + """ + @doc guard: true @spec is_function(term, non_neg_integer) :: boolean def is_function(term, arity) do :erlang.is_function(term, arity) @@ -281,6 +700,7 @@ defmodule Kernel do Allowed in guard tests. Inlined by the compiler. """ + @doc guard: true @spec is_integer(term) :: boolean def is_integer(term) do :erlang.is_integer(term) @@ -291,27 +711,30 @@ defmodule Kernel do Allowed in guard tests. Inlined by the compiler. """ + @doc guard: true @spec is_list(term) :: boolean def is_list(term) do :erlang.is_list(term) end @doc """ - Returns `true` if `term` is either an integer or a floating point number; + Returns `true` if `term` is either an integer or a floating-point number; otherwise returns `false`. Allowed in guard tests. Inlined by the compiler. """ + @doc guard: true @spec is_number(term) :: boolean def is_number(term) do :erlang.is_number(term) end @doc """ - Returns `true` if `term` is a pid (process identifier); otherwise returns `false`. + Returns `true` if `term` is a PID (process identifier); otherwise returns `false`. Allowed in guard tests. Inlined by the compiler. """ + @doc guard: true @spec is_pid(term) :: boolean def is_pid(term) do :erlang.is_pid(term) @@ -322,6 +745,7 @@ defmodule Kernel do Allowed in guard tests. Inlined by the compiler. """ + @doc guard: true @spec is_port(term) :: boolean def is_port(term) do :erlang.is_port(term) @@ -332,6 +756,7 @@ defmodule Kernel do Allowed in guard tests. Inlined by the compiler. """ + @doc guard: true @spec is_reference(term) :: boolean def is_reference(term) do :erlang.is_reference(term) @@ -342,6 +767,7 @@ defmodule Kernel do Allowed in guard tests. Inlined by the compiler. """ + @doc guard: true @spec is_tuple(term) :: boolean def is_tuple(term) do :erlang.is_tuple(term) @@ -352,11 +778,25 @@ defmodule Kernel do Allowed in guard tests. Inlined by the compiler. """ + @doc guard: true @spec is_map(term) :: boolean def is_map(term) do :erlang.is_map(term) end + @doc """ + Returns `true` if `key` is a key in `map`; otherwise returns `false`. + + It raises `BadMapError` if the first element is not a map. + + Allowed in guard tests. Inlined by the compiler. + """ + @doc guard: true, since: "1.10.0" + @spec is_map_key(map, term) :: boolean + def is_map_key(map, key) do + :erlang.is_map_key(key, map) + end + @doc """ Returns the length of `list`. @@ -368,6 +808,7 @@ defmodule Kernel do 9 """ + @doc guard: true @spec length(list) :: non_neg_integer def length(list) do :erlang.length(list) @@ -383,7 +824,8 @@ defmodule Kernel do ## Examples - make_ref() #=> #Reference<0.0.0.135> + make_ref() + #=> #Reference<0.0.0.135> """ @spec make_ref() :: reference @@ -394,19 +836,34 @@ defmodule Kernel do @doc """ Returns the size of a map. + The size of a map is the number of key-value pairs that the map contains. + This operation happens in constant time. Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> map_size(%{a: "foo", b: "bar"}) + 2 + """ + @doc guard: true @spec map_size(map) :: non_neg_integer def map_size(map) do :erlang.map_size(map) end @doc """ - Return the biggest of the two given terms according to - Erlang's term ordering. If the terms compare equal, the - first one is returned. + Returns the biggest of the two given terms according to + their structural comparison. + + If the terms compare equal, the first one is returned. + + This performs a structural comparison where all Elixir + terms can be compared with each other. See the ["Structural + comparison" section](#module-structural-comparison) section + for more information. Inlined by the compiler. @@ -414,17 +871,25 @@ defmodule Kernel do iex> max(1, 2) 2 + iex> max(:a, :b) + :b """ - @spec max(term, term) :: term + @spec max(first, second) :: first | second when first: term, second: term def max(first, second) do :erlang.max(first, second) end @doc """ - Return the smallest of the two given terms according to - Erlang's term ordering. If the terms compare equal, the - first one is returned. + Returns the smallest of the two given terms according to + their structural comparison. + + If the terms compare equal, the first one is returned. + + This performs a structural comparison where all Elixir + terms can be compared with each other. See the ["Structural + comparison" section](#module-structural-comparison) section + for more information. Inlined by the compiler. @@ -432,9 +897,11 @@ defmodule Kernel do iex> min(1, 2) 1 + iex> min("foo", "bar") + "bar" """ - @spec min(term, term) :: term + @spec min(first, second) :: first | second when first: term, second: term def min(first, second) do :erlang.min(first, second) end @@ -445,27 +912,33 @@ defmodule Kernel do Allowed in guard tests. Inlined by the compiler. """ + @doc guard: true @spec node() :: node def node do - :erlang.node + :erlang.node() end @doc """ Returns the node where the given argument is located. - The argument can be a pid, a reference, or a port. - If the local node is not alive, `nonode@nohost` is returned. + The argument can be a PID, a reference, or a port. + If the local node is not alive, `:nonode@nohost` is returned. Allowed in guard tests. Inlined by the compiler. """ - @spec node(pid|reference|port) :: node + @doc guard: true + @spec node(pid | reference | port) :: node def node(arg) do :erlang.node(arg) end @doc """ - Calculates the remainder of an integer division. + Computes the remainder of an integer division. + + `rem/2` uses truncated division, which means that + the result will always have the sign of the `dividend`. - Raises an error if one of the arguments is not an integer. + Raises an `ArithmeticError` exception if one of the arguments is not an + integer, or when the `divisor` is `0`. Allowed in guard tests. Inlined by the compiler. @@ -473,24 +946,45 @@ defmodule Kernel do iex> rem(5, 2) 1 + iex> rem(6, -4) + 2 """ - @spec rem(integer, integer) :: integer - def rem(left, right) do - :erlang.rem(left, right) + @doc guard: true + @spec rem(integer, neg_integer | pos_integer) :: integer + def rem(dividend, divisor) do + :erlang.rem(dividend, divisor) end @doc """ - Returns an integer by rounding the given number. + Rounds a number to the nearest integer. + + If the number is equidistant to the two nearest integers, rounds away from zero. Allowed in guard tests. Inlined by the compiler. ## Examples - iex> round(5.5) + iex> round(5.6) 6 + iex> round(5.2) + 5 + + iex> round(-9.9) + -10 + + iex> round(-9) + -9 + + iex> round(2.5) + 3 + + iex> round(-2.5) + -3 + """ + @doc guard: true @spec round(number) :: integer def round(number) do :erlang.round(number) @@ -499,48 +993,55 @@ defmodule Kernel do @doc """ Sends a message to the given `dest` and returns the message. - `dest` may be a remote or local pid, a (local) port, a locally - registered name, or a tuple `{registered_name, node}` for a registered - name at another node. + `dest` may be a remote or local PID, a local port, a locally + registered name, or a tuple in the form of `{registered_name, node}` for a + registered name at another node. Inlined by the compiler. ## Examples - iex> send self(), :hello + iex> send(self(), :hello) :hello """ - @spec send(dest :: pid | port | atom | {atom, node}, msg) :: msg when msg: any - def send(dest, msg) do - :erlang.send(dest, msg) + @spec send(dest :: Process.dest(), message) :: message when message: any + def send(dest, message) do + :erlang.send(dest, message) end @doc """ - Returns the pid (process identifier) of the calling process. + Returns the PID (process identifier) of the calling process. Allowed in guard clauses. Inlined by the compiler. """ + @doc guard: true @spec self() :: pid def self() do :erlang.self() end @doc """ - Spawns the given function and returns its pid. + Spawns the given function and returns its PID. + + Typically developers do not use the `spawn` functions, instead they use + abstractions such as `Task`, `GenServer` and `Agent`, built on top of + `spawn`, that spawns processes with more conveniences in terms of + introspection and debugging. - Check the modules `Process` and `Node` for other functions - to handle processes, including spawning functions in nodes. + Check the `Process` module for more process-related functions. + + The anonymous function receives 0 arguments, and may return any value. Inlined by the compiler. ## Examples - current = Kernel.self - child = spawn(fn -> send current, {Kernel.self, 1 + 2} end) + current = self() + child = spawn(fn -> send(current, {self(), 1 + 2}) end) receive do - {^child, 3} -> IO.puts "Received 3 back" + {^child, 3} -> IO.puts("Received 3 back") end """ @@ -550,11 +1051,15 @@ defmodule Kernel do end @doc """ - Spawns the given module and function passing the given args - and returns its pid. + Spawns the given function `fun` from the given `module` passing it the given + `args` and returns its PID. + + Typically developers do not use the `spawn` functions, instead they use + abstractions such as `Task`, `GenServer` and `Agent`, built on top of + `spawn`, that spawns processes with more conveniences in terms of + introspection and debugging. - Check the modules `Process` and `Node` for other functions - to handle processes, including spawning functions in nodes. + Check the `Process` module for more process-related functions. Inlined by the compiler. @@ -569,20 +1074,27 @@ defmodule Kernel do end @doc """ - Spawns the given function, links it to the current process and returns its pid. + Spawns the given function, links it to the current process, and returns its PID. + + Typically developers do not use the `spawn` functions, instead they use + abstractions such as `Task`, `GenServer` and `Agent`, built on top of + `spawn`, that spawns processes with more conveniences in terms of + introspection and debugging. + + Check the `Process` module for more process-related functions. For more + information on linking, check `Process.link/1`. - Check the modules `Process` and `Node` for other functions - to handle processes, including spawning functions in nodes. + The anonymous function receives 0 arguments, and may return any value. Inlined by the compiler. ## Examples - current = Kernel.self - child = spawn_link(fn -> send current, {Kernel.self, 1 + 2} end) + current = self() + child = spawn_link(fn -> send(current, {self(), 1 + 2}) end) receive do - {^child, 3} -> IO.puts "Received 3 back" + {^child, 3} -> IO.puts("Received 3 back") end """ @@ -592,11 +1104,16 @@ defmodule Kernel do end @doc """ - Spawns the given module and function passing the given args, - links it to the current process and returns its pid. + Spawns the given function `fun` from the given `module` passing it the given + `args`, links it to the current process, and returns its PID. + + Typically developers do not use the `spawn` functions, instead they use + abstractions such as `Task`, `GenServer` and `Agent`, built on top of + `spawn`, that spawns processes with more conveniences in terms of + introspection and debugging. - Check the modules `Process` and `Node` for other functions - to handle processes, including spawning functions in nodes. + Check the `Process` module for more process-related functions. For more + information on linking, check `Process.link/1`. Inlined by the compiler. @@ -611,18 +1128,24 @@ defmodule Kernel do end @doc """ - Spawns the given function, monitors it and returns its pid + Spawns the given function, monitors it and returns its PID and monitoring reference. - Check the modules `Process` and `Node` for other functions - to handle processes, including spawning functions in nodes. + Typically developers do not use the `spawn` functions, instead they use + abstractions such as `Task`, `GenServer` and `Agent`, built on top of + `spawn`, that spawns processes with more conveniences in terms of + introspection and debugging. + + Check the `Process` module for more process-related functions. + + The anonymous function receives 0 arguments, and may return any value. Inlined by the compiler. ## Examples - current = Kernel.self - spawn_monitor(fn -> send current, {Kernel.self, 1 + 2} end) + current = self() + spawn_monitor(fn -> send(current, {self(), 1 + 2}) end) """ @spec spawn_monitor((() -> any)) :: {pid, reference} @@ -632,10 +1155,14 @@ defmodule Kernel do @doc """ Spawns the given module and function passing the given args, - monitors it and returns its pid and monitoring reference. + monitors it and returns its PID and monitoring reference. - Check the modules `Process` and `Node` for other functions - to handle processes, including spawning functions in nodes. + Typically developers do not use the `spawn` functions, instead they use + abstractions such as `Task`, `GenServer` and `Agent`, built on top of + `spawn`, that spawns processes with more conveniences in terms of + introspection and debugging. + + Check the `Process` module for more process-related functions. Inlined by the compiler. @@ -650,7 +1177,41 @@ defmodule Kernel do end @doc """ - A non-local return from a function. Check `Kernel.SpecialForms.try/1` for more information. + Pipes the first argument, `value`, into the second argument, a function `fun`, + and returns `value` itself. + + Useful for running synchronous side effects in a pipeline, using the `|>/2` operator. + + ## Examples + + iex> tap(1, fn x -> x + 1 end) + 1 + + Most commonly, this is used in pipelines, using the `|>/2` operator. + For example, let's suppose you want to inspect part of a data structure. + You could write: + + %{a: 1} + |> Map.update!(:a, & &1 + 2) + |> tap(&IO.inspect(&1.a)) + |> Map.update!(:a, & &1 * 2) + + """ + @doc since: "1.12.0" + defmacro tap(value, fun) do + quote bind_quoted: [fun: fun, value: value] do + _ = fun.(value) + value + end + end + + @doc """ + A non-local return from a function. + + Using `throw/1` is generally discouraged, as it allows a function + to escape from its regular execution flow, which can make the code + harder to read. Furthermore, all thrown values must be caught by + `try/catch`. See `try/1` for more information. Inlined by the compiler. """ @@ -662,24 +1223,57 @@ defmodule Kernel do @doc """ Returns the tail of a list. Raises `ArgumentError` if the list is empty. + The tail of a list is the list without its first element. + + It works with improper lists. + Allowed in guard tests. Inlined by the compiler. + + ## Examples + + tl([1, 2, 3, :go]) + #=> [2, 3, :go] + + tl([:one]) + #=> [] + + tl([:a, :b | :improper_end]) + #=> [:b | :improper_end] + + tl([:a | %{b: 1}]) + #=> %{b: 1} + + Giving it an empty list raises: + + tl([]) + #=> ** (ArgumentError) argument error + """ - @spec tl(maybe_improper_list) :: maybe_improper_list + @doc guard: true + @spec tl(nonempty_maybe_improper_list(elem, last)) :: maybe_improper_list(elem, last) | last + when elem: term, last: term def tl(list) do :erlang.tl(list) end @doc """ - Returns an integer by truncating the given number. + Returns the integer part of `number`. Allowed in guard tests. Inlined by the compiler. ## Examples - iex> trunc(5.5) + iex> trunc(5.4) 5 + iex> trunc(-5.99) + -5 + + iex> trunc(-5) + -5 + """ + @doc guard: true @spec trunc(number) :: integer def trunc(number) do :erlang.trunc(number) @@ -691,14 +1285,21 @@ defmodule Kernel do This operation happens in constant time. Allowed in guard tests. Inlined by the compiler. + + ## Examples + + iex> tuple_size({:a, :b, :c}) + 3 + """ + @doc guard: true @spec tuple_size(tuple) :: non_neg_integer def tuple_size(tuple) do :erlang.tuple_size(tuple) end @doc """ - Arithmetic plus. + Arithmetic addition operator. Allowed in guard tests. Inlined by the compiler. @@ -708,13 +1309,17 @@ defmodule Kernel do 3 """ - @spec (number + number) :: number + @doc guard: true + @spec integer + integer :: integer + @spec float + float :: float + @spec integer + float :: float + @spec float + integer :: float def left + right do :erlang.+(left, right) end @doc """ - Arithmetic minus. + Arithmetic subtraction operator. Allowed in guard tests. Inlined by the compiler. @@ -724,13 +1329,17 @@ defmodule Kernel do -1 """ - @spec (number - number) :: number + @doc guard: true + @spec integer - integer :: integer + @spec float - float :: float + @spec integer - float :: float + @spec float - integer :: float def left - right do :erlang.-(left, right) end @doc """ - Arithmetic unary plus. + Arithmetic positive unary operator. Allowed in guard tests. Inlined by the compiler. @@ -740,13 +1349,15 @@ defmodule Kernel do 1 """ - @spec (+number) :: number - def (+value) do + @doc guard: true + @spec +integer :: integer + @spec +float :: float + def +value do :erlang.+(value) end @doc """ - Arithmetic unary minus. + Arithmetic negative unary operator. Allowed in guard tests. Inlined by the compiler. @@ -756,13 +1367,17 @@ defmodule Kernel do -2 """ - @spec (-number) :: number - def (-value) do + @doc guard: true + @spec -0 :: 0 + @spec -pos_integer :: neg_integer + @spec -neg_integer :: pos_integer + @spec -float :: float + def -value do :erlang.-(value) end @doc """ - Arithmetic multiplication. + Arithmetic multiplication operator. Allowed in guard tests. Inlined by the compiler. @@ -772,57 +1387,93 @@ defmodule Kernel do 2 """ - @spec (number * number) :: number + @doc guard: true + @spec integer * integer :: integer + @spec float * float :: float + @spec integer * float :: float + @spec float * integer :: float def left * right do :erlang.*(left, right) end @doc """ - Arithmetic division. + Arithmetic division operator. + + The result is always a float. Use `div/2` and `rem/2` if you want + an integer division or the remainder. - The result is always a float. Use `div` and `rem` if you want - a natural division or the remainder. + Raises `ArithmeticError` if `right` is 0 or 0.0. Allowed in guard tests. Inlined by the compiler. ## Examples - iex> 1 / 2 - 0.5 + 1 / 2 + #=> 0.5 + + -3.0 / 2.0 + #=> -1.5 + + 5 / 1 + #=> 5.0 - iex> 2 / 1 - 2.0 + 7 / 0 + ** (ArithmeticError) bad argument in arithmetic expression """ - @spec (number / number) :: float + @doc guard: true + @spec number / number :: float def left / right do :erlang./(left, right) end @doc """ - Concatenates two lists. + List concatenation operator. Concatenates a proper list and a term, returning a list. - Allowed in guard tests. Inlined by the compiler. + The complexity of `a ++ b` is proportional to `length(a)`, so avoid repeatedly + appending to lists of arbitrary length, for example, `list ++ [element]`. + Instead, consider prepending via `[element | rest]` and then reversing. + + If the `right` operand is not a proper list, it returns an improper list. + If the `left` operand is not a proper list, it raises `ArgumentError`. + + Inlined by the compiler. ## Examples iex> [1] ++ [2, 3] - [1,2,3] + [1, 2, 3] iex> 'foo' ++ 'bar' 'foobar' + # returns an improper list + iex> [1] ++ 2 + [1 | 2] + + # returns a proper list + iex> [1] ++ [2] + [1, 2] + + # improper list on the right will return an improper list + iex> [1] ++ [2 | 3] + [1, 2 | 3] + """ - @spec (list ++ term) :: maybe_improper_list + @spec list ++ term :: maybe_improper_list def left ++ right do :erlang.++(left, right) end @doc """ - Removes the first occurrence of an item on the left - for each item on the right. + List subtraction operator. Removes the first occurrence of an element + on the left list for each element on the right. - Allowed in guard tests. Inlined by the compiler. + This function is optimized so the complexity of `a -- b` is proportional + to `length(a) * log(length(b))`. See also the [Erlang efficiency + guide](https://www.erlang.org/doc/efficiency_guide/retired_myths.html). + + Inlined by the compiler. ## Examples @@ -830,21 +1481,28 @@ defmodule Kernel do [3] iex> [1, 2, 3, 2, 1] -- [1, 2, 2] - [3,1] + [3, 1] + + The `--/2` operator is right associative, meaning: + + iex> [1, 2, 3] -- [2] -- [3] + [1, 3] + + As it is equivalent to: + + iex> [1, 2, 3] -- ([2] -- [3]) + [1, 3] """ - @spec (list -- list) :: list + @spec list -- list :: list def left -- right do :erlang.--(left, right) end - @doc false - def left xor right do - :erlang.xor(left, right) - end - @doc """ - Boolean not. Argument must be a boolean. + Strictly boolean "not" operator. + + `value` must be a boolean; if it's not, an `ArgumentError` exception is raised. Allowed in guard tests. Inlined by the compiler. @@ -854,15 +1512,22 @@ defmodule Kernel do true """ - @spec not(boolean) :: boolean - def not(arg) do - :erlang.not(arg) + @doc guard: true + @spec not true :: false + @spec not false :: true + def not value do + :erlang.not(value) end @doc """ - Returns `true` if left is less than right. + Less-than operator. - All terms in Elixir can be compared with each other. + Returns `true` if `left` is less than `right`. + + This performs a structural comparison where all Elixir + terms can be compared with each other. See the ["Structural + comparison" section](#module-structural-comparison) section + for more information. Allowed in guard tests. Inlined by the compiler. @@ -872,15 +1537,21 @@ defmodule Kernel do true """ - @spec (term < term) :: boolean + @doc guard: true + @spec term < term :: boolean def left < right do :erlang.<(left, right) end @doc """ - Returns `true` if left is more than right. + Greater-than operator. - All terms in Elixir can be compared with each other. + Returns `true` if `left` is more than `right`. + + This performs a structural comparison where all Elixir + terms can be compared with each other. See the ["Structural + comparison" section](#module-structural-comparison) section + for more information. Allowed in guard tests. Inlined by the compiler. @@ -890,15 +1561,21 @@ defmodule Kernel do false """ - @spec (term > term) :: boolean + @doc guard: true + @spec term > term :: boolean def left > right do :erlang.>(left, right) end @doc """ - Returns `true` if left is less than or equal to right. + Less-than or equal to operator. - All terms in Elixir can be compared with each other. + Returns `true` if `left` is less than or equal to `right`. + + This performs a structural comparison where all Elixir + terms can be compared with each other. See the ["Structural + comparison" section](#module-structural-comparison) section + for more information. Allowed in guard tests. Inlined by the compiler. @@ -908,15 +1585,21 @@ defmodule Kernel do true """ - @spec (term <= term) :: boolean + @doc guard: true + @spec term <= term :: boolean def left <= right do :erlang."=<"(left, right) end @doc """ - Returns `true` if left is more than or equal to right. + Greater-than or equal to operator. - All terms in Elixir can be compared with each other. + Returns `true` if `left` is more than or equal to `right`. + + This performs a structural comparison where all Elixir + terms can be compared with each other. See the ["Structural + comparison" section](#module-structural-comparison) section + for more information. Allowed in guard tests. Inlined by the compiler. @@ -926,16 +1609,17 @@ defmodule Kernel do false """ - @spec (term >= term) :: boolean + @doc guard: true + @spec term >= term :: boolean def left >= right do :erlang.>=(left, right) end @doc """ - Returns `true` if the two items are equal. + Equal to operator. Returns `true` if the two terms are equal. - This operator considers 1 and 1.0 to be equal. For match - semantics, use `===` instead. + This operator considers 1 and 1.0 to be equal. For stricter + semantics, use `===/2` instead. All terms in Elixir can be compared with each other. @@ -950,16 +1634,19 @@ defmodule Kernel do true """ - @spec (term == term) :: boolean + @doc guard: true + @spec term == term :: boolean def left == right do :erlang.==(left, right) end @doc """ - Returns `true` if the two items are not equal. + Not equal to operator. + + Returns `true` if the two terms are not equal. This operator considers 1 and 1.0 to be equal. For match - comparison, use `!==` instead. + comparison, use `!==/2` instead. All terms in Elixir can be compared with each other. @@ -974,17 +1661,21 @@ defmodule Kernel do false """ - @spec (term != term) :: boolean + @doc guard: true + @spec term != term :: boolean def left != right do :erlang."/="(left, right) end @doc """ - Returns `true` if the two items are match. + Strictly equal to operator. + + Returns `true` if the two terms are exactly equal. - This operator gives the same semantics as the one existing in - pattern matching, i.e., `1` and `1.0` are equal, but they do - not match. + The terms are only considered to be exactly equal if they + have the same value and are of the same type. For example, + `1 == 1.0` returns `true`, but since they are of different + types, `1 === 1.0` returns `false`. All terms in Elixir can be compared with each other. @@ -999,13 +1690,17 @@ defmodule Kernel do false """ - @spec (term === term) :: boolean + @doc guard: true + @spec term === term :: boolean def left === right do :erlang."=:="(left, right) end @doc """ - Returns `true` if the two items do not match. + Strictly not equal to operator. + + Returns `true` if the two terms are not exactly equal. + See `===/2` for a definition of what is considered "exactly equal". All terms in Elixir can be compared with each other. @@ -1020,34 +1715,44 @@ defmodule Kernel do true """ - @spec (term !== term) :: boolean + @doc guard: true + @spec term !== term :: boolean def left !== right do :erlang."=/="(left, right) end @doc """ - Get the element at the zero-based `index` in `tuple`. + Gets the element at the zero-based `index` in `tuple`. + + It raises `ArgumentError` when index is negative or it is out of range of the tuple elements. Allowed in guard tests. Inlined by the compiler. - ## Example + ## Examples - iex> tuple = {:foo, :bar, 3} - iex> elem(tuple, 1) - :bar + tuple = {:foo, :bar, 3} + elem(tuple, 1) + #=> :bar + + elem({}, 0) + ** (ArgumentError) argument error + + elem({:foo, :bar}, 2) + ** (ArgumentError) argument error """ + @doc guard: true @spec elem(tuple, non_neg_integer) :: term def elem(tuple, index) do :erlang.element(index + 1, tuple) end @doc """ - Puts the element in `tuple` at the zero-based `index` to the given `value`. + Puts `value` at the given zero-based `index` in `tuple`. Inlined by the compiler. - ## Example + ## Examples iex> tuple = {:foo, :bar, 3} iex> put_elem(tuple, 0, :baz) @@ -1061,9 +1766,18 @@ defmodule Kernel do ## Implemented in Elixir - @doc """ - Boolean or. Requires only the first argument to be a - boolean since it short-circuits. + defp optimize_boolean({:case, meta, args}) do + {:case, [{:optimize_boolean, true} | meta], args} + end + + @doc """ + Strictly boolean "or" operator. + + If `left` is `true`, returns `true`; otherwise returns `right`. + + Requires only the `left` operand to be a boolean since it short-circuits. + If the `left` operand is not a boolean, a `BadBooleanError` exception is + raised. Allowed in guard tests. @@ -1072,14 +1786,29 @@ defmodule Kernel do iex> true or false true + iex> false or 42 + 42 + + iex> 42 or false + ** (BadBooleanError) expected a boolean on left-side of "or", got: 42 + """ + @doc guard: true defmacro left or right do - quote do: __op__(:orelse, unquote(left), unquote(right)) + case __CALLER__.context do + nil -> build_boolean_check(:or, left, true, right) + :match -> invalid_match!(:or) + :guard -> quote(do: :erlang.orelse(unquote(left), unquote(right))) + end end @doc """ - Boolean and. Requires only the first argument to be a - boolean since it short-circuits. + Strictly boolean "and" operator. + + If `left` is `false`, returns `false`; otherwise returns `right`. + + Requires only the `left` operand to be a boolean since it short-circuits. If + the `left` operand is not a boolean, a `BadBooleanError` exception is raised. Allowed in guard tests. @@ -1088,15 +1817,41 @@ defmodule Kernel do iex> true and false false + iex> true and "yay!" + "yay!" + + iex> "yay!" and true + ** (BadBooleanError) expected a boolean on left-side of "and", got: "yay!" + """ + @doc guard: true defmacro left and right do - quote do: __op__(:andalso, unquote(left), unquote(right)) + case __CALLER__.context do + nil -> build_boolean_check(:and, left, right, false) + :match -> invalid_match!(:and) + :guard -> quote(do: :erlang.andalso(unquote(left), unquote(right))) + end + end + + defp build_boolean_check(operator, check, true_clause, false_clause) do + optimize_boolean( + quote do + case unquote(check) do + false -> unquote(false_clause) + true -> unquote(true_clause) + other -> :erlang.error({:badbool, unquote(operator), other}) + end + end + ) end @doc """ - Receives any argument and returns `true` if it is `false` - or `nil`. Returns `false` otherwise. Not allowed in guard - clauses. + Boolean "not" operator. + + Receives any value (not just booleans) and returns `true` if `value` + is `false` or `nil`; returns `false` otherwise. + + Not allowed in guard clauses. ## Examples @@ -1107,142 +1862,207 @@ defmodule Kernel do true """ - defmacro !(arg) + defmacro !value - defmacro !({:!, _, [arg]}) do - optimize_boolean(quote do - case unquote(arg) do - x when x in [false, nil] -> false - _ -> true + defmacro !{:!, _, [value]} do + assert_no_match_or_guard_scope(__CALLER__.context, "!") + + optimize_boolean( + quote do + case unquote(value) do + x when :"Elixir.Kernel".in(x, [false, nil]) -> false + _ -> true + end end - end) + ) end - defmacro !(arg) do - optimize_boolean(quote do - case unquote(arg) do - x when x in [false, nil] -> true - _ -> false + defmacro !value do + assert_no_match_or_guard_scope(__CALLER__.context, "!") + + optimize_boolean( + quote do + case unquote(value) do + x when :"Elixir.Kernel".in(x, [false, nil]) -> true + _ -> false + end end - end) + ) end @doc """ - Concatenates two binaries. + Binary concatenation operator. Concatenates two binaries. + + Raises an `ArgumentError` if one of the sides aren't binaries. ## Examples iex> "foo" <> "bar" "foobar" - The `<>` operator can also be used in guard clauses as - long as the first part is a literal binary: + The `<>/2` operator can also be used in pattern matching (and guard clauses) as + long as the left argument is a literal binary: iex> "foo" <> x = "foobar" iex> x "bar" + `x <> "bar" = "foobar"` would result in an `ArgumentError` exception. + """ defmacro left <> right do - concats = extract_concatenations({:<>, [], [left, right]}) - quote do: << unquote_splicing(concats) >> + concats = extract_concatenations({:<>, [], [left, right]}, __CALLER__) + quote(do: <>) end # Extracts concatenations in order to optimize many # concatenations into one single clause. - defp extract_concatenations({:<>, _, [left, right]}) do - [wrap_concatenation(left)|extract_concatenations(right)] + defp extract_concatenations({:<>, _, [left, right]}, caller) do + [wrap_concatenation(left, :left, caller) | extract_concatenations(right, caller)] end - defp extract_concatenations(other) do - [wrap_concatenation(other)] + defp extract_concatenations(other, caller) do + [wrap_concatenation(other, :right, caller)] end - defp wrap_concatenation(binary) when is_binary(binary) do + defp wrap_concatenation(binary, _side, _caller) when is_binary(binary) do binary end - defp wrap_concatenation(other) do - {:::, [], [other, {:binary, [], nil}]} + defp wrap_concatenation(literal, _side, _caller) + when is_list(literal) or is_atom(literal) or is_integer(literal) or is_float(literal) do + :erlang.error( + ArgumentError.exception( + "expected binary argument in <> operator but got: #{Macro.to_string(literal)}" + ) + ) + end + + defp wrap_concatenation(other, side, caller) do + expanded = expand_concat_argument(other, side, caller) + {:"::", [], [expanded, {:binary, [], nil}]} + end + + defp expand_concat_argument(arg, :left, %{context: :match} = caller) do + expanded_arg = + case bootstrapped?(Macro) do + true -> Macro.expand(arg, caller) + false -> arg + end + + case expanded_arg do + {var, _, nil} when is_atom(var) -> + invalid_concat_left_argument_error(Atom.to_string(var)) + + {:^, _, [{var, _, nil}]} when is_atom(var) -> + invalid_concat_left_argument_error("^#{Atom.to_string(var)}") + + _ -> + expanded_arg + end + end + + defp expand_concat_argument(arg, _, _) do + arg + end + + defp invalid_concat_left_argument_error(arg) do + :erlang.error( + ArgumentError.exception( + "the left argument of <> operator inside a match should always be a literal " <> + "binary because its size can't be verified. Got: #{arg}" + ) + ) end @doc """ Raises an exception. - If the argument is a binary, it raises `RuntimeError` - using the given argument as message. + If `message` is a string, it raises a `RuntimeError` exception with it. + + If `message` is an atom, it just calls `raise/2` with the atom as the first + argument and `[]` as the second one. - If an atom, it will become a call to `raise(atom, [])`. + If `message` is an exception struct, it is raised as is. - If anything else, it will just raise the given exception. + If `message` is anything else, `raise` will fail with an `ArgumentError` + exception. ## Examples - raise "Given values do not match" + iex> raise "oops" + ** (RuntimeError) oops try do 1 + :foo rescue x in [ArithmeticError] -> - IO.puts "that was expected" + IO.puts("that was expected") raise x end """ - defmacro raise(msg) do + defmacro raise(message) do # Try to figure out the type at compilation time - # to avoid dead code and make dialyzer happy. - msg = case not is_binary(msg) and bootstraped?(Macro) do - true -> Macro.expand(msg, __CALLER__) - false -> msg - end + # to avoid dead code and make Dialyzer happy. + message = + case not is_binary(message) and bootstrapped?(Macro) do + true -> Macro.expand(message, __CALLER__) + false -> message + end + + erlang_error = + case :erlang.system_info(:otp_release) >= '24' do + true -> + fn x -> + quote do + :erlang.error(unquote(x), :none, error_info: %{module: Exception}) + end + end + + false -> + fn x -> + quote do + :erlang.error(unquote(x)) + end + end + end + + case message do + message when is_binary(message) -> + erlang_error.(quote do: RuntimeError.exception(unquote(message))) + + {:<<>>, _, _} = message -> + erlang_error.(quote do: RuntimeError.exception(unquote(message))) - case msg do - msg when is_binary(msg) -> - quote do - :erlang.error RuntimeError.exception(unquote(msg)) - end - {:<<>>, _, _} = msg -> - quote do - :erlang.error RuntimeError.exception(unquote(msg)) - end alias when is_atom(alias) -> - quote do - :erlang.error unquote(alias).exception([]) - end + erlang_error.(quote do: unquote(alias).exception([])) + _ -> - quote do - case unquote(msg) do - msg when is_binary(msg) -> - :erlang.error RuntimeError.exception(msg) - atom when is_atom(atom) -> - :erlang.error atom.exception([]) - %{__struct__: struct, __exception__: true} = other when is_atom(struct) -> - :erlang.error other - end - end + erlang_error.(quote do: Kernel.Utils.raise(unquote(message))) end end @doc """ Raises an exception. - Calls `.exception` on the given argument passing - the attributes in order to retrieve the appropriate exception - structure. + Calls the `exception/1` function on the given argument (which has to be a + module name like `ArgumentError` or `RuntimeError`) passing `attributes` + in order to retrieve the exception struct. - Any module defined via `defexception/1` automatically - implements `exception(attrs)` callback expected by `raise/2`. + Any module that contains a call to the `defexception/1` macro automatically + implements the `c:Exception.exception/1` callback expected by `raise/2`. + For more information, see `defexception/1`. ## Examples - iex> raise(ArgumentError, message: "Sample") + iex> raise(ArgumentError, "Sample") ** (ArgumentError) Sample """ - defmacro raise(exception, attrs) do + defmacro raise(exception, attributes) do quote do - :erlang.error unquote(exception).exception(unquote(attrs)) + :erlang.error(unquote(exception).exception(unquote(attributes))) end end @@ -1251,53 +2071,47 @@ defmodule Kernel do Works like `raise/1` but does not generate a new stacktrace. - Notice that `System.stacktrace` returns the stacktrace - of the last exception. That said, it is common to assign - the stacktrace as the first expression inside a `rescue` - clause as any other exception potentially raised (and - rescued) in between the rescue clause and the raise call - may change the `System.stacktrace` value. + Note that `__STACKTRACE__` can be used inside catch/rescue + to retrieve the current stacktrace. ## Examples try do - raise "Oops" + raise "oops" rescue exception -> - stacktrace = System.stacktrace - if Exception.message(exception) == "Oops" do - reraise exception, stacktrace - end + reraise exception, __STACKTRACE__ end + """ - defmacro reraise(msg, stacktrace) do + defmacro reraise(message, stacktrace) do # Try to figure out the type at compilation time - # to avoid dead code and make dialyzer happy. - - case Macro.expand(msg, __CALLER__) do - msg when is_binary(msg) -> + # to avoid dead code and make Dialyzer happy. + case Macro.expand(message, __CALLER__) do + message when is_binary(message) -> quote do - :erlang.raise :error, RuntimeError.exception(unquote(msg)), unquote(stacktrace) + :erlang.error( + :erlang.raise(:error, RuntimeError.exception(unquote(message)), unquote(stacktrace)) + ) end - {:<<>>, _, _} = msg -> + + {:<<>>, _, _} = message -> quote do - :erlang.raise :error, RuntimeError.exception(unquote(msg)), unquote(stacktrace) + :erlang.error( + :erlang.raise(:error, RuntimeError.exception(unquote(message)), unquote(stacktrace)) + ) end + alias when is_atom(alias) -> quote do - :erlang.raise :error, unquote(alias).exception([]), unquote(stacktrace) + :erlang.error(:erlang.raise(:error, unquote(alias).exception([]), unquote(stacktrace))) end - msg -> + + message -> quote do - stacktrace = unquote(stacktrace) - case unquote(msg) do - msg when is_binary(msg) -> - :erlang.raise :error, RuntimeError.exception(msg), stacktrace - atom when is_atom(atom) -> - :erlang.raise :error, atom.exception([]), stacktrace - %{__struct__: struct, __exception__: true} = other when is_atom(struct) -> - :erlang.raise :error, other, stacktrace - end + :erlang.error( + :erlang.raise(:error, Kernel.Utils.raise(unquote(message)), unquote(stacktrace)) + ) end end end @@ -1305,30 +2119,36 @@ defmodule Kernel do @doc """ Raises an exception preserving a previous stacktrace. - Works like `raise/2` but does not generate a new stacktrace. - - See `reraise/2` for more details. + `reraise/3` works like `reraise/2`, except it passes arguments to the + `exception/1` function as explained in `raise/2`. ## Examples try do - raise "Oops" + raise "oops" rescue exception -> - stacktrace = System.stacktrace - reraise WrapperError, [exception: exception], stacktrace + reraise WrapperError, [exception: exception], __STACKTRACE__ end + """ - defmacro reraise(exception, attrs, stacktrace) do + defmacro reraise(exception, attributes, stacktrace) do quote do - :erlang.raise :error, unquote(exception).exception(unquote(attrs)), unquote(stacktrace) + :erlang.raise( + :error, + unquote(exception).exception(unquote(attributes)), + unquote(stacktrace) + ) end end @doc """ - Matches the term on the left against the regular expression or string on the - right. Returns true if `left` matches `right` (if it's a regular expression) - or contains `right` (if it's a string). + Text-based match operator. Matches the term on the `left` + against the regular expression or string on the `right`. + + If `right` is a regular expression, returns `true` if `left` matches right. + + If `right` is a string, returns `true` if `left` contains `right`. ## Examples @@ -1338,13 +2158,26 @@ defmodule Kernel do iex> "abcd" =~ ~r/e/ false + iex> "abcd" =~ ~r// + true + iex> "abcd" =~ "bc" true iex> "abcd" =~ "ad" false + iex> "abcd" =~ "abcd" + true + + iex> "abcd" =~ "" + true + + For more information about regular expressions, please check the `Regex` module. """ + @spec String.t() =~ (String.t() | Regex.t()) :: boolean + def left =~ "" when is_binary(left), do: true + def left =~ right when is_binary(left) and is_binary(right) do :binary.match(left, right) != :nomatch end @@ -1354,8 +2187,8 @@ defmodule Kernel do end @doc ~S""" - Inspect the given argument according to the `Inspect` protocol. - The second argument is a keywords list with options to control + Inspects the given argument according to the `Inspect` protocol. + The second argument is a keyword list with options to control inspection. ## Options @@ -1369,374 +2202,971 @@ defmodule Kernel do iex> inspect(:foo) ":foo" - iex> inspect [1, 2, 3, 4, 5], limit: 3 + iex> inspect([1, 2, 3, 4, 5], limit: 3) "[1, 2, 3, ...]" - iex> inspect("josé" <> <<0>>) - "<<106, 111, 115, 195, 169, 0>>" + iex> inspect([1, 2, 3], pretty: true, width: 0) + "[1,\n 2,\n 3]" - iex> inspect("josé" <> <<0>>, binaries: :as_strings) - "\"josé\\000\"" + iex> inspect("olá" <> <<0>>) + "<<111, 108, 195, 161, 0>>" - iex> inspect("josé", binaries: :as_binaries) - "<<106, 111, 115, 195, 169>>" + iex> inspect("olá" <> <<0>>, binaries: :as_strings) + "\"olá\\0\"" - Note that the inspect protocol does not necessarily return a valid + iex> inspect("olá", binaries: :as_binaries) + "<<111, 108, 195, 161>>" + + iex> inspect('bar') + "'bar'" + + iex> inspect([0 | 'bar']) + "[0, 98, 97, 114]" + + iex> inspect(100, base: :octal) + "0o144" + + iex> inspect(100, base: :hex) + "0x64" + + Note that the `Inspect` protocol does not necessarily return a valid representation of an Elixir term. In such cases, the inspected result must start with `#`. For example, inspecting a function will return: - inspect fn a, b -> a + b end + inspect(fn a, b -> a + b end) #=> #Function<...> + The `Inspect` protocol can be derived to hide certain fields + from structs, so they don't show up in logs, inspects and similar. + See the "Deriving" section of the documentation of the `Inspect` + protocol for more information. """ - @spec inspect(Inspect.t, Keyword.t) :: String.t - def inspect(arg, opts \\ []) when is_list(opts) do - opts = struct(Inspect.Opts, opts) - limit = case opts.pretty do - true -> opts.width - false -> :infinity - end - Inspect.Algebra.pretty(Inspect.Algebra.to_doc(arg, opts), limit) + @spec inspect(Inspect.t(), keyword) :: String.t() + def inspect(term, opts \\ []) when is_list(opts) do + opts = Inspect.Opts.new(opts) + + limit = + case opts.pretty do + true -> opts.width + false -> :infinity + end + + doc = Inspect.Algebra.group(Inspect.Algebra.to_doc(term, opts)) + IO.iodata_to_binary(Inspect.Algebra.format(doc, limit)) end @doc """ - Creates and updates structs. + Creates and updates a struct. - The struct argument may be an atom (which defines `defstruct`) - or a struct itself. The second argument is any Enumerable that - emits two-item tuples (key-value) during enumeration. + The `struct` argument may be an atom (which defines `defstruct`) + or a `struct` itself. The second argument is any `Enumerable` that + emits two-element tuples (key-value pairs) during enumeration. - If one of the keys in the Enumerable does not exist in the struct, - they are automatically discarded. + Keys in the `Enumerable` that don't exist in the struct are automatically + discarded. Note that keys must be atoms, as only atoms are allowed when + defining a struct. If keys in the `Enumerable` are duplicated, the last + entry will be taken (same behaviour as `Map.new/1`). - This function is useful for dynamically creating and updating - structs. + This function is useful for dynamically creating and updating structs, as + well as for converting maps to structs; in the latter case, just inserting + the appropriate `:__struct__` field into the map may not be enough and + `struct/2` should be used instead. - ## Example + ## Examples defmodule User do - defstruct name: "jose" + defstruct name: "john" end struct(User) - #=> %User{name: "jose"} + #=> %User{name: "john"} - opts = [name: "eric"] + opts = [name: "meg"] user = struct(User, opts) - #=> %User{name: "eric"} + #=> %User{name: "meg"} struct(user, unknown: "value") - #=> %User{name: "eric"} + #=> %User{name: "meg"} - """ - @spec struct(module | map, Enum.t) :: map - def struct(struct, kv \\ []) + struct(User, %{name: "meg"}) + #=> %User{name: "meg"} + + # String keys are ignored + struct(User, %{"name" => "meg"}) + #=> %User{name: "john"} - def struct(struct, []) when is_atom(struct) or is_tuple(struct) do - apply(struct, :__struct__, []) + """ + @spec struct(module | struct, Enumerable.t()) :: struct + def struct(struct, fields \\ []) do + struct(struct, fields, fn + {:__struct__, _val}, acc -> + acc + + {key, val}, acc -> + case acc do + %{^key => _} -> %{acc | key => val} + _ -> acc + end + end) end - def struct(struct, kv) when is_atom(struct) or is_tuple(struct) do - struct(apply(struct, :__struct__, []), kv) + @doc """ + Similar to `struct/2` but checks for key validity. + + The function `struct!/2` emulates the compile time behaviour + of structs. This means that: + + * when building a struct, as in `struct!(SomeStruct, key: :value)`, + it is equivalent to `%SomeStruct{key: :value}` and therefore this + function will check if every given key-value belongs to the struct. + If the struct is enforcing any key via `@enforce_keys`, those will + be enforced as well; + + * when updating a struct, as in `struct!(%SomeStruct{}, key: :value)`, + it is equivalent to `%SomeStruct{struct | key: :value}` and therefore this + function will check if every given key-value belongs to the struct. + However, updating structs does not enforce keys, as keys are enforced + only when building; + + """ + @spec struct!(module | struct, Enumerable.t()) :: struct + def struct!(struct, fields \\ []) + + def struct!(struct, fields) when is_atom(struct) do + validate_struct!(struct.__struct__(fields), struct, 1) end - def struct(%{__struct__: _} = struct, kv) do - Enum.reduce(kv, struct, fn {k, v}, acc -> - case :maps.is_key(k, acc) and k != :__struct__ do - true -> :maps.put(k, v, acc) - false -> acc - end + def struct!(struct, fields) when is_map(struct) do + struct(struct, fields, fn + {:__struct__, _}, acc -> + acc + + {key, val}, acc -> + Map.replace!(acc, key, val) end) end - @doc """ - Gets a value from a nested structure. + defp struct(struct, [], _fun) when is_atom(struct) do + validate_struct!(struct.__struct__(), struct, 0) + end - Uses the `Access` protocol to traverse the structures - according to the given `keys`. + defp struct(struct, fields, fun) when is_atom(struct) do + struct(validate_struct!(struct.__struct__(), struct, 0), fields, fun) + end - ## Examples + defp struct(%_{} = struct, [], _fun) do + struct + end - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> get_in(users, ["josé", :age]) - 27 + defp struct(%_{} = struct, fields, fun) do + Enum.reduce(fields, struct, fun) + end - In case any of entries in the middle returns `nil`, `nil` will be returned - as per the Access protocol: + defp validate_struct!(%{__struct__: module} = struct, module, _arity) do + struct + end - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> get_in(users, ["unknown", :age]) - nil + defp validate_struct!(%{__struct__: struct_name}, module, arity) when is_atom(struct_name) do + error_message = + "expected struct name returned by #{inspect(module)}.__struct__/#{arity} to be " <> + "#{inspect(module)}, got: #{inspect(struct_name)}" - """ - @spec get_in(Access.t, nonempty_list(term)) :: term - def get_in(data, keys) - def get_in(nil, list) when is_list(list), do: nil - def get_in(data, [h]), do: Access.get(data, h) - def get_in(data, [h|t]), do: get_in(Access.get(data, h), t) + :erlang.error(ArgumentError.exception(error_message)) + end + + defp validate_struct!(expr, module, arity) do + error_message = + "expected #{inspect(module)}.__struct__/#{arity} to return a map with a :__struct__ " <> + "key that holds the name of the struct (atom), got: #{inspect(expr)}" + + :erlang.error(ArgumentError.exception(error_message)) + end @doc """ - Puts a value in a nested structure. + Returns true if `term` is a struct; otherwise returns `false`. - Uses the `Access` protocol to traverse the structures - according to the given `keys`. + Allowed in guard tests. ## Examples - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> put_in(users, ["josé", :age], 28) - %{"josé" => %{age: 28}, "eric" => %{age: 23}} - - In case any of entries in the middle returns `nil`, a map is dynamically - created: + iex> is_struct(URI.parse("/")) + true - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> put_in(users, ["dave", :age], 13) - %{"josé" => %{age: 27}, "eric" => %{age: 23}, "dave" => %{age: 13}} + iex> is_struct(%{}) + false """ - @spec put_in(Access.t, nonempty_list(term), term) :: Access.t - def put_in(data, keys, value) do - elem(get_and_update_in(data, keys, fn _ -> {nil, value} end), 1) + @doc since: "1.10.0", guard: true + defmacro is_struct(term) do + case __CALLER__.context do + nil -> + quote do + case unquote(term) do + %_{} -> true + _ -> false + end + end + + :match -> + invalid_match!(:is_struct) + + :guard -> + quote do + is_map(unquote(term)) and :erlang.is_map_key(:__struct__, unquote(term)) and + is_atom(:erlang.map_get(:__struct__, unquote(term))) + end + end end @doc """ - Updates a key in a nested structure. + Returns true if `term` is a struct of `name`; otherwise returns `false`. - Uses the `Access` protocol to traverse the structures - according to the given `keys`. + Allowed in guard tests. ## Examples - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> update_in(users, ["josé", :age], &(&1 + 1)) - %{"josé" => %{age: 28}, "eric" => %{age: 23}} - - In case any of entries in the middle returns `nil`, a map is dynamically - created: + iex> is_struct(URI.parse("/"), URI) + true - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> update_in(users, ["dave", :age], &((&1 || 0) + 1)) - %{"josé" => %{age: 27}, "eric" => %{age: 23}, "dave" => %{age: 1}} + iex> is_struct(URI.parse("/"), Macro.Env) + false """ - @spec update_in(Access.t, nonempty_list(term), (term -> term)) :: Access.t - def update_in(data, keys, fun) do - elem(get_and_update_in(data, keys, fn x -> {nil, fun.(x)} end), 1) + @doc since: "1.11.0", guard: true + defmacro is_struct(term, name) do + case __CALLER__.context do + nil -> + quote generated: true do + case unquote(name) do + name when is_atom(name) -> + case unquote(term) do + %{__struct__: ^name} -> true + _ -> false + end + + _ -> + raise ArgumentError + end + end + + :match -> + invalid_match!(:is_struct) + + :guard -> + quote do + is_map(unquote(term)) and + (is_atom(unquote(name)) or :fail) and + :erlang.is_map_key(:__struct__, unquote(term)) and + :erlang.map_get(:__struct__, unquote(term)) == unquote(name) + end + end end @doc """ - Gets a value and updates a nested structure. + Returns true if `term` is an exception; otherwise returns `false`. - It expects a tuple to be returned, containing the value retrieved - and the update one. Uses the `Access` protocol to traverse the - structures according to the given `keys`. + Allowed in guard tests. ## Examples - This function is useful when there is a need to retrieve the current - value (or something calculated in function of the current value) and - update it at the same time. For example, it could be used to increase - the age of a user by one and return the previous age in one pass: - - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> get_and_update_in(users, ["josé", :age], &{&1, &1 + 1}) - {27, %{"josé" => %{age: 28}, "eric" => %{age: 23}}} - - In case any of entries in the middle returns `nil`, a map is dynamically - created: + iex> is_exception(%RuntimeError{}) + true - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> get_and_update_in(users, ["dave", :age], &{&1, 13}) - {nil, %{"josé" => %{age: 27}, "eric" => %{age: 23}, "dave" => %{age: 13}}} + iex> is_exception(%{}) + false """ - @spec get_and_update_in(Access.t, nonempty_list(term), - (term -> {get, term})) :: {get, Access.t} when get: var - def get_and_update_in(data, keys, fun) + @doc since: "1.11.0", guard: true + defmacro is_exception(term) do + case __CALLER__.context do + nil -> + quote do + case unquote(term) do + %_{__exception__: true} -> true + _ -> false + end + end + + :match -> + invalid_match!(:is_exception) - def get_and_update_in(nil, list, fun), do: get_and_update_in(%{}, list, fun) - def get_and_update_in(data, [h], fun), do: Access.get_and_update(data, h, fun) - def get_and_update_in(data, [h|t], fun) do - Access.get_and_update(data, h, &get_and_update_in(&1, t, fun)) + :guard -> + quote do + is_map(unquote(term)) and :erlang.is_map_key(:__struct__, unquote(term)) and + is_atom(:erlang.map_get(:__struct__, unquote(term))) and + :erlang.is_map_key(:__exception__, unquote(term)) and + :erlang.map_get(:__exception__, unquote(term)) == true + end + end end @doc """ - Puts a value in a nested structure via the given `path`. + Returns true if `term` is an exception of `name`; otherwise returns `false`. - This is similar to `put_in/3`, except the path is extracted via - a macro rather than passing a list. For example: + Allowed in guard tests. - put_in(opts[:foo][:bar], :baz) + ## Examples - Is equivalent to: + iex> is_exception(%RuntimeError{}, RuntimeError) + true - put_in(opts, [:foo, :bar], :baz) + iex> is_exception(%RuntimeError{}, Macro.Env) + false - Note that in order for this macro to work, the complete path must always - be visible by this macro. For more information about the supported path - expressions, please check `get_and_update_in/2` docs. + """ + @doc since: "1.11.0", guard: true + defmacro is_exception(term, name) do + case __CALLER__.context do + nil -> + quote do + case unquote(name) do + name when is_atom(name) -> + case unquote(term) do + %{__struct__: ^name, __exception__: true} -> true + _ -> false + end + + _ -> + raise ArgumentError + end + end - ## Examples + :match -> + invalid_match!(:is_exception) + + :guard -> + quote do + is_map(unquote(term)) and + (is_atom(unquote(name)) or :fail) and + :erlang.is_map_key(:__struct__, unquote(term)) and + :erlang.map_get(:__struct__, unquote(term)) == unquote(name) and + :erlang.is_map_key(:__exception__, unquote(term)) and + :erlang.map_get(:__exception__, unquote(term)) == true + end + end + end + + @doc """ + Pipes the first argument, `value`, into the second argument, a function `fun`, + and returns the result of calling `fun`. - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> put_in(users["josé"][:age], 28) - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + In other words, it invokes the function `fun` with `value` as argument, + and returns its result. - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> put_in(users["josé"].age, 28) - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + This is most commonly used in pipelines, using the `|>/2` operator, allowing you + to pipe a value to a function outside of its first argument. + ### Examples + + iex> 1 |> then(fn x -> x * 2 end) + 2 + + iex> 1 |> then(fn x -> Enum.drop(["a", "b", "c"], x) end) + ["b", "c"] """ - defmacro put_in(path, value) do - [h|t] = unnest(path, [], "put_in/2") - expr = nest_get_and_update_in(h, t, quote(do: fn _ -> {nil, unquote(value)} end)) - quote do: :erlang.element(2, unquote(expr)) + @doc since: "1.12.0" + defmacro then(value, fun) do + quote do + unquote(fun).(unquote(value)) + end end @doc """ - Updates a nested structure via the given `path`. + Gets a value from a nested structure. - This is similar to `update_in/3`, except the path is extracted via - a macro rather than passing a list. For example: + Uses the `Access` module to traverse the structures + according to the given `keys`, unless the `key` is a + function, which is detailed in a later section. - update_in(opts[:foo][:bar], &(&1 + 1)) + Note that if none of the given keys are functions, + there is rarely a reason to use `get_in` over + writing "regular" Elixir code using `[]`. - Is equivalent to: + ## Examples - update_in(opts, [:foo, :bar], &(&1 + 1)) + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> get_in(users, ["john", :age]) + 27 + iex> # Equivalent to: + iex> users["john"][:age] + 27 - Note that in order for this macro to work, the complete path must always - be visible by this macro. For more information about the supported path - expressions, please check `get_and_update_in/2` docs. + `get_in/2` can also use the accessors in the `Access` module + to traverse more complex data structures. For example, here we + use `Access.all/0` to traverse a list: - ## Examples + iex> users = [%{name: "john", age: 27}, %{name: "meg", age: 23}] + iex> get_in(users, [Access.all(), :age]) + [27, 23] - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> update_in(users["josé"][:age], &(&1 + 1)) - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + In case any of the components returns `nil`, `nil` will be returned + and `get_in/2` won't traverse any further: - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> update_in(users["josé"].age, &(&1 + 1)) - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> get_in(users, ["unknown", :age]) + nil + iex> # Equivalent to: + iex> users["unknown"][:age] + nil - """ - defmacro update_in(path, fun) do - [h|t] = unnest(path, [], "update_in/2") - expr = nest_get_and_update_in(h, t, quote(do: fn x -> {nil, unquote(fun).(x)} end)) - quote do: :erlang.element(2, unquote(expr)) - end + iex> users = nil + iex> get_in(users, [Access.all(), :age]) + nil - @doc """ - Gets a value and updates a nested data structure via the given `path`. + Alternatively, if you need to access complex data-structures, you can + use pattern matching: - This is similar to `get_and_update_in/3`, except the path is extracted - via a macro rather than passing a list. For example: + case users do + %{"john" => %{age: age}} -> age + _ -> default_value + end - get_and_update_in(opts[:foo][:bar], &{&1, &1 + 1}) + ## Functions as keys - Is equivalent to: + If a key given to `get_in/2` is a function, the function will be invoked + passing three arguments: - get_and_update_in(opts, [:foo, :bar], &{&1, &1 + 1}) + * the operation (`:get`) + * the data to be accessed + * a function to be invoked next - Note that in order for this macro to work, the complete path must always - be visible by this macro. See the Paths section below. + This means `get_in/2` can be extended to provide custom lookups. + That's precisely how the `Access.all/0` key in the previous section + behaves. For example, we can manually implement such traversal as + follows: - ## Examples + iex> users = [%{name: "john", age: 27}, %{name: "meg", age: 23}] + iex> all = fn :get, data, next -> Enum.map(data, next) end + iex> get_in(users, [all, :age]) + [27, 23] - iex> users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - iex> get_and_update_in(users["josé"][:age], &{&1, &1 + 1}) - {27, %{"josé" => %{age: 28}, "eric" => %{age: 23}}} + The `Access` module ships with many convenience accessor functions. + See `Access.all/0`, `Access.key/2`, and others as examples. - ## Paths + ## Working with structs - A path may start with a variable, local or remote call, and must be - followed by one or more: + By default, structs do not implement the `Access` behaviour required + by this function. Therefore, you can't do this: - * `foo[bar]` - access a field; in case an intermediate field is not - present or returns nil, an empty map is used + get_in(some_struct, [:some_key, :nested_key]) - * `foo.bar` - access a map/struct field; in case the field is not - present, an error is raised + The good news is that structs have predefined shape. Therefore, + you can write instead: - Here are some valid paths: + some_struct.some_key.nested_key - users["josé"][:age] - users["josé"].age - User.all["josé"].age - all_users()["josé"].age + If, by any chance, `some_key` can return nil, you can always + fallback to pattern matching to provide nested struct handling: - Here are some invalid ones: + case some_struct do + %{some_key: %{nested_key: value}} -> value + %{} -> nil + end - # Does a remote call after the initial value - users["josé"].do_something(arg1, arg2) + """ + @spec get_in(Access.t(), nonempty_list(term)) :: term + def get_in(data, keys) - # Does not access any field - users + def get_in(nil, [_ | _]), do: nil - """ - defmacro get_and_update_in(path, fun) do - [h|t] = unnest(path, [], "get_and_update_in/2") - nest_get_and_update_in(h, t, fun) - end + def get_in(data, [h]) when is_function(h), do: h.(:get, data, & &1) + def get_in(data, [h | t]) when is_function(h), do: h.(:get, data, &get_in(&1, t)) - defp nest_get_and_update_in([], fun), do: fun - defp nest_get_and_update_in(list, fun) do - quote do - fn x -> unquote(nest_get_and_update_in(quote(do: x), list, fun)) end - end + def get_in(data, [h]), do: Access.get(data, h) + def get_in(data, [h | t]), do: get_in(Access.get(data, h), t) + + @doc """ + Puts a value in a nested structure. + + Uses the `Access` module to traverse the structures + according to the given `keys`, unless the `key` is a + function. If the key is a function, it will be invoked + as specified in `get_and_update_in/3`. + + ## Examples + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> put_in(users, ["john", :age], 28) + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + If any of the intermediate values are nil, it will raise: + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> put_in(users, ["jane", :age], "oops") + ** (ArgumentError) could not put/update key :age on a nil value + + """ + @spec put_in(Access.t(), nonempty_list(term), term) :: Access.t() + def put_in(data, [_ | _] = keys, value) do + elem(get_and_update_in(data, keys, fn _ -> {nil, value} end), 1) + end + + @doc """ + Updates a key in a nested structure. + + Uses the `Access` module to traverse the structures + according to the given `keys`, unless the `key` is a + function. If the key is a function, it will be invoked + as specified in `get_and_update_in/3`. + + `data` is a nested structure (that is, a map, keyword + list, or struct that implements the `Access` behaviour). + The `fun` argument receives the value of `key` (or `nil` + if `key` is not present) and the result replaces the value + in the structure. + + ## Examples + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> update_in(users, ["john", :age], &(&1 + 1)) + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + Note the current value given to the anonymous function may be `nil`. + If any of the intermediate values are nil, it will raise: + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> update_in(users, ["jane", :age], & &1 + 1) + ** (ArgumentError) could not put/update key :age on a nil value + + """ + @spec update_in(Access.t(), nonempty_list(term), (term -> term)) :: Access.t() + def update_in(data, [_ | _] = keys, fun) when is_function(fun) do + elem(get_and_update_in(data, keys, fn x -> {nil, fun.(x)} end), 1) + end + + @doc """ + Gets a value and updates a nested structure. + + `data` is a nested structure (that is, a map, keyword + list, or struct that implements the `Access` behaviour). + + The `fun` argument receives the value of `key` (or `nil` if `key` + is not present) and must return one of the following values: + + * a two-element tuple `{current_value, new_value}`. In this case, + `current_value` is the retrieved value which can possibly be operated on before + being returned. `new_value` is the new value to be stored under `key`. + + * `:pop`, which implies that the current value under `key` + should be removed from the structure and returned. + + This function uses the `Access` module to traverse the structures + according to the given `keys`, unless the `key` is a function, + which is detailed in a later section. + + ## Examples + + This function is useful when there is a need to retrieve the current + value (or something calculated in function of the current value) and + update it at the same time. For example, it could be used to read the + current age of a user while increasing it by one in one pass: + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> get_and_update_in(users, ["john", :age], &{&1, &1 + 1}) + {27, %{"john" => %{age: 28}, "meg" => %{age: 23}}} + + Note the current value given to the anonymous function may be `nil`. + If any of the intermediate values are nil, it will raise: + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> get_and_update_in(users, ["jane", :age], &{&1, &1 + 1}) + ** (ArgumentError) could not put/update key :age on a nil value + + ## Functions as keys + + If a key is a function, the function will be invoked passing three + arguments: + + * the operation (`:get_and_update`) + * the data to be accessed + * a function to be invoked next + + This means `get_and_update_in/3` can be extended to provide custom + lookups. The downside is that functions cannot be stored as keys + in the accessed data structures. + + When one of the keys is a function, the function is invoked. + In the example below, we use a function to get and increment all + ages inside a list: + + iex> users = [%{name: "john", age: 27}, %{name: "meg", age: 23}] + iex> all = fn :get_and_update, data, next -> + ...> data |> Enum.map(next) |> Enum.unzip() + ...> end + iex> get_and_update_in(users, [all, :age], &{&1, &1 + 1}) + {[27, 23], [%{name: "john", age: 28}, %{name: "meg", age: 24}]} + + If the previous value before invoking the function is `nil`, + the function *will* receive `nil` as a value and must handle it + accordingly (be it by failing or providing a sane default). + + The `Access` module ships with many convenience accessor functions, + like the `all` anonymous function defined above. See `Access.all/0`, + `Access.key/2`, and others as examples. + """ + @spec get_and_update_in( + structure, + keys, + (term | nil -> {current_value, new_value} | :pop) + ) :: {current_value, new_structure :: structure} + when structure: Access.t(), + keys: nonempty_list(any), + current_value: Access.value(), + new_value: Access.value() + def get_and_update_in(data, keys, fun) + + def get_and_update_in(data, [head], fun) when is_function(head, 3), + do: head.(:get_and_update, data, fun) + + def get_and_update_in(data, [head | tail], fun) when is_function(head, 3), + do: head.(:get_and_update, data, &get_and_update_in(&1, tail, fun)) + + def get_and_update_in(data, [head], fun) when is_function(fun, 1), + do: Access.get_and_update(data, head, fun) + + def get_and_update_in(data, [head | tail], fun) when is_function(fun, 1), + do: Access.get_and_update(data, head, &get_and_update_in(&1, tail, fun)) + + @doc """ + Pops a key from the given nested structure. + + Uses the `Access` protocol to traverse the structures + according to the given `keys`, unless the `key` is a + function. If the key is a function, it will be invoked + as specified in `get_and_update_in/3`. + + ## Examples + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> pop_in(users, ["john", :age]) + {27, %{"john" => %{}, "meg" => %{age: 23}}} + + In case any entry returns `nil`, its key will be removed + and the deletion will be considered a success. + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> pop_in(users, ["jane", :age]) + {nil, %{"john" => %{age: 27}, "meg" => %{age: 23}}} + + """ + @spec pop_in(data, nonempty_list(Access.get_and_update_fun(term, data) | term)) :: {term, data} + when data: Access.container() + def pop_in(data, keys) + + def pop_in(nil, [key | _]) do + raise ArgumentError, "could not pop key #{inspect(key)} on a nil value" + end + + def pop_in(data, [_ | _] = keys) do + pop_in_data(data, keys) + end + + defp pop_in_data(nil, [_ | _]), do: :pop + + defp pop_in_data(data, [fun]) when is_function(fun), + do: fun.(:get_and_update, data, fn _ -> :pop end) + + defp pop_in_data(data, [fun | tail]) when is_function(fun), + do: fun.(:get_and_update, data, &pop_in_data(&1, tail)) + + defp pop_in_data(data, [key]), do: Access.pop(data, key) + + defp pop_in_data(data, [key | tail]), + do: Access.get_and_update(data, key, &pop_in_data(&1, tail)) + + @doc """ + Puts a value in a nested structure via the given `path`. + + This is similar to `put_in/3`, except the path is extracted via + a macro rather than passing a list. For example: + + put_in(opts[:foo][:bar], :baz) + + Is equivalent to: + + put_in(opts, [:foo, :bar], :baz) + + This also works with nested structs and the `struct.path.to.value` way to specify + paths: + + put_in(struct.foo.bar, :baz) + + Note that in order for this macro to work, the complete path must always + be visible by this macro. For more information about the supported path + expressions, please check `get_and_update_in/2` docs. + + ## Examples + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> put_in(users["john"][:age], 28) + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> put_in(users["john"].age, 28) + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + """ + defmacro put_in(path, value) do + case unnest(path, [], true, "put_in/2") do + {[h | t], true} -> + nest_update_in(h, t, quote(do: fn _ -> unquote(value) end)) + + {[h | t], false} -> + expr = nest_get_and_update_in(h, t, quote(do: fn _ -> {nil, unquote(value)} end)) + quote(do: :erlang.element(2, unquote(expr))) + end + end + + @doc """ + Pops a key from the nested structure via the given `path`. + + This is similar to `pop_in/2`, except the path is extracted via + a macro rather than passing a list. For example: + + pop_in(opts[:foo][:bar]) + + Is equivalent to: + + pop_in(opts, [:foo, :bar]) + + Note that in order for this macro to work, the complete path must always + be visible by this macro. For more information about the supported path + expressions, please check `get_and_update_in/2` docs. + + ## Examples + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> pop_in(users["john"][:age]) + {27, %{"john" => %{}, "meg" => %{age: 23}}} + + iex> users = %{john: %{age: 27}, meg: %{age: 23}} + iex> pop_in(users.john[:age]) + {27, %{john: %{}, meg: %{age: 23}}} + + In case any entry returns `nil`, its key will be removed + and the deletion will be considered a success. + """ + defmacro pop_in(path) do + {[h | t], _} = unnest(path, [], true, "pop_in/1") + nest_pop_in(:map, h, t) + end + + @doc """ + Updates a nested structure via the given `path`. + + This is similar to `update_in/3`, except the path is extracted via + a macro rather than passing a list. For example: + + update_in(opts[:foo][:bar], &(&1 + 1)) + + Is equivalent to: + + update_in(opts, [:foo, :bar], &(&1 + 1)) + + This also works with nested structs and the `struct.path.to.value` way to specify + paths: + + update_in(struct.foo.bar, &(&1 + 1)) + + Note that in order for this macro to work, the complete path must always + be visible by this macro. For more information about the supported path + expressions, please check `get_and_update_in/2` docs. + + ## Examples + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> update_in(users["john"][:age], &(&1 + 1)) + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> update_in(users["john"].age, &(&1 + 1)) + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + """ + defmacro update_in(path, fun) do + case unnest(path, [], true, "update_in/2") do + {[h | t], true} -> + nest_update_in(h, t, fun) + + {[h | t], false} -> + expr = nest_get_and_update_in(h, t, quote(do: fn x -> {nil, unquote(fun).(x)} end)) + quote(do: :erlang.element(2, unquote(expr))) + end + end + + @doc """ + Gets a value and updates a nested data structure via the given `path`. + + This is similar to `get_and_update_in/3`, except the path is extracted + via a macro rather than passing a list. For example: + + get_and_update_in(opts[:foo][:bar], &{&1, &1 + 1}) + + Is equivalent to: + + get_and_update_in(opts, [:foo, :bar], &{&1, &1 + 1}) + + This also works with nested structs and the `struct.path.to.value` way to specify + paths: + + get_and_update_in(struct.foo.bar, &{&1, &1 + 1}) + + Note that in order for this macro to work, the complete path must always + be visible by this macro. See the "Paths" section below. + + ## Examples + + iex> users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + iex> get_and_update_in(users["john"].age, &{&1, &1 + 1}) + {27, %{"john" => %{age: 28}, "meg" => %{age: 23}}} + + ## Paths + + A path may start with a variable, local or remote call, and must be + followed by one or more: + + * `foo[bar]` - accesses the key `bar` in `foo`; in case `foo` is nil, + `nil` is returned + + * `foo.bar` - accesses a map/struct field; in case the field is not + present, an error is raised + + Here are some valid paths: + + users["john"][:age] + users["john"].age + User.all()["john"].age + all_users()["john"].age + + Here are some invalid ones: + + # Does a remote call after the initial value + users["john"].do_something(arg1, arg2) + + # Does not access any key or field + users + + """ + defmacro get_and_update_in(path, fun) do + {[h | t], _} = unnest(path, [], true, "get_and_update_in/2") + nest_get_and_update_in(h, t, fun) end - defp nest_get_and_update_in(h, [{:access, key}|t], fun) do + defp nest_update_in([], fun), do: fun + + defp nest_update_in(list, fun) do quote do - Access.get_and_update( - case(unquote(h), do: (nil -> %{}; o -> o)), - unquote(key), - unquote(nest_get_and_update_in(t, fun)) - ) + fn x -> unquote(nest_update_in(quote(do: x), list, fun)) end + end + end + + defp nest_update_in(h, [{:map, key} | t], fun) do + quote do + Map.update!(unquote(h), unquote(key), unquote(nest_update_in(t, fun))) + end + end + + defp nest_get_and_update_in([], fun), do: fun + + defp nest_get_and_update_in(list, fun) do + quote do + fn x -> unquote(nest_get_and_update_in(quote(do: x), list, fun)) end + end + end + + defp nest_get_and_update_in(h, [{:access, key} | t], fun) do + quote do + Access.get_and_update(unquote(h), unquote(key), unquote(nest_get_and_update_in(t, fun))) end end - defp nest_get_and_update_in(h, [{:map, key}|t], fun) do + defp nest_get_and_update_in(h, [{:map, key} | t], fun) do quote do - Access.Map.get_and_update!(unquote(h), unquote(key), unquote(nest_get_and_update_in(t, fun))) + Map.get_and_update!(unquote(h), unquote(key), unquote(nest_get_and_update_in(t, fun))) end end - defp unnest({{:., _, [Access, :get]}, _, [expr, key]}, acc, kind) do - unnest(expr, [{:access, key}|acc], kind) + defp nest_pop_in(kind, list) do + quote do + fn x -> unquote(nest_pop_in(kind, quote(do: x), list)) end + end end - defp unnest({{:., _, [expr, key]}, _, []}, acc, kind) - when is_tuple(expr) and elem(expr, 0) != :__aliases__ and elem(expr, 0) != :__MODULE__ do - unnest(expr, [{:map, key}|acc], kind) + defp nest_pop_in(:map, h, [{:access, key}]) do + quote do + case unquote(h) do + nil -> {nil, nil} + h -> Access.pop(h, unquote(key)) + end + end end - defp unnest(other, [], kind) do + defp nest_pop_in(_, _, [{:map, key}]) do raise ArgumentError, - "expected expression given to #{kind} to access at least one field, got: #{Macro.to_string other}" + "cannot use pop_in when the last segment is a map/struct field. " <> + "This would effectively remove the field #{inspect(key)} from the map/struct" + end + + defp nest_pop_in(_, h, [{:map, key} | t]) do + quote do + Map.get_and_update!(unquote(h), unquote(key), unquote(nest_pop_in(:map, t))) + end + end + + defp nest_pop_in(_, h, [{:access, key}]) do + quote do + case unquote(h) do + nil -> :pop + h -> Access.pop(h, unquote(key)) + end + end end - defp unnest(other, acc, kind) do + defp nest_pop_in(_, h, [{:access, key} | t]) do + quote do + Access.get_and_update(unquote(h), unquote(key), unquote(nest_pop_in(:access, t))) + end + end + + defp unnest({{:., _, [Access, :get]}, _, [expr, key]}, acc, _all_map?, kind) do + unnest(expr, [{:access, key} | acc], false, kind) + end + + defp unnest({{:., _, [expr, key]}, _, []}, acc, all_map?, kind) + when is_tuple(expr) and :erlang.element(1, expr) != :__aliases__ and + :erlang.element(1, expr) != :__MODULE__ do + unnest(expr, [{:map, key} | acc], all_map?, kind) + end + + defp unnest(other, [], _all_map?, kind) do + raise ArgumentError, + "expected expression given to #{kind} to access at least one element, " <> + "got: #{Macro.to_string(other)}" + end + + defp unnest(other, acc, all_map?, kind) do case proper_start?(other) do - true -> [other|acc] + true -> + {[other | acc], all_map?} + false -> raise ArgumentError, - "expression given to #{kind} must start with a variable, local or remote call " <> - "and be followed by field access, got: #{Macro.to_string other}" + "expression given to #{kind} must start with a variable, local or remote call " <> + "and be followed by an element access, got: #{Macro.to_string(other)}" end end defp proper_start?({{:., _, [expr, _]}, _, _args}) - when is_atom(expr) - when elem(expr, 0) == :__aliases__ - when elem(expr, 0) == :__MODULE__, do: true + when is_atom(expr) + when :erlang.element(1, expr) == :__aliases__ + when :erlang.element(1, expr) == :__MODULE__, + do: true defp proper_start?({atom, _, _args}) - when is_atom(atom), do: true + when is_atom(atom), + do: true - defp proper_start?(other), - do: not is_tuple(other) + defp proper_start?(other), do: not is_tuple(other) @doc """ Converts the argument to a string according to the @@ -1750,121 +3180,148 @@ defmodule Kernel do "foo" """ - # If it is a binary at compilation time, simply return it. - defmacro to_string(arg) when is_binary(arg), do: arg - - defmacro to_string(arg) do - quote do: String.Chars.to_string(unquote(arg)) + defmacro to_string(term) do + quote(do: :"Elixir.String.Chars".to_string(unquote(term))) end @doc """ - Convert the argument to a list according to the List.Chars protocol. + Converts the given term to a charlist according to the `List.Chars` protocol. ## Examples - iex> to_char_list(:foo) + iex> to_charlist(:foo) 'foo' """ - defmacro to_char_list(arg) do - quote do: List.Chars.to_char_list(unquote(arg)) + defmacro to_charlist(term) do + quote(do: List.Chars.to_charlist(unquote(term))) end @doc """ - Checks if the given argument is nil or not. + Returns `true` if `term` is `nil`, `false` otherwise. + Allowed in guard clauses. ## Examples - iex> nil?(1) + iex> is_nil(1) false - iex> nil?(nil) + iex> is_nil(nil) true """ - defmacro nil?(x) do - quote do: unquote(x) == nil + @doc guard: true + defmacro is_nil(term) do + quote(do: unquote(term) == nil) end @doc """ - A convenient macro that checks if the right side matches - the left side. The left side is allowed to be a match pattern. + A convenience macro that checks if the right side (an expression) matches the + left side (a pattern). ## Examples iex> match?(1, 1) true - iex> match?(1, 2) - false - iex> match?({1, _}, {1, 2}) true - Match can also be used to filter or find a value in an enumerable: + iex> map = %{a: 1, b: 2} + iex> match?(%{a: _}, map) + true + + iex> a = 1 + iex> match?(^a, 1) + true + + `match?/2` is very useful when filtering or finding a value in an enumerable: - list = [{:a, 1}, {:b, 2}, {:a, 3}] - Enum.filter list, &match?({:a, _}, &1) + iex> list = [a: 1, b: 2, a: 3] + iex> Enum.filter(list, &match?({:a, _}, &1)) + [a: 1, a: 3] Guard clauses can also be given to the match: - list = [{:a, 1}, {:b, 2}, {:a, 3}] - Enum.filter list, &match?({:a, x} when x < 2, &1) + iex> list = [a: 1, b: 2, a: 3] + iex> Enum.filter(list, &match?({:a, x} when x < 2, &1)) + [a: 1] However, variables assigned in the match will not be available - outside of the function call: + outside of the function call (unlike regular pattern matching with the `=` + operator): - iex> match?(x, 1) + iex> match?(_x, 1) true + iex> binding() + [] - iex> binding([:x]) == [] - true + Furthermore, remember the pin operator matches _values_, not _patterns_: - """ - defmacro match?(pattern, expr) + match?(%{x: 1}, %{x: 1, y: 2}) + #=> true - # Special case underscore since it always matches - defmacro match?({:_, _, atom}, _right) when is_atom(atom) do - true - end + attrs = %{x: 1} + match?(^attrs, %{x: 1, y: 2}) + #=> false - defmacro match?(left, right) do - quote do - case unquote(right) do - unquote(left) -> - true - _ -> - false + The pin operator will check if the values are equal, using `===/2`, while + patterns have their own rules when matching maps, lists, and so forth. + Such behaviour is not specific to `match?/2`. The following code also + throws an exception: + + attrs = %{x: 1} + ^attrs = %{x: 1, y: 2} + #=> (MatchError) no match of right hand side value: %{x: 1, y: 2} + + """ + defmacro match?(pattern, expr) do + success = + quote do + unquote(pattern) -> true end - end + + failure = + quote generated: true do + _ -> false + end + + {:case, [], [expr, [do: success ++ failure]]} end @doc """ - Read and write attributes of th current module. + Module attribute unary operator. + + Reads and writes attributes in the current module. The canonical example for attributes is annotating that a module - implements the OTP behaviour called `gen_server`: + implements an OTP behaviour, such as `GenServer`: defmodule MyServer do - @behaviour :gen_server + @behaviour GenServer # ... callbacks ... end - By default Elixir supports all Erlang module attributes, but any developer - can also add custom attributes: + By default Elixir supports all the module attributes supported by Erlang, but + custom attributes can be used as well: defmodule MyServer do @my_data 13 - IO.inspect @my_data #=> 13 + IO.inspect(@my_data) + #=> 13 end - Unlike Erlang, such attributes are not stored in the module by - default since it is common in Elixir to use such attributes to store - temporary data. A developer can configure an attribute to behave closer - to Erlang by calling `Module.register_attribute/3`. + Unlike Erlang, such attributes are not stored in the module by default since + it is common in Elixir to use custom attributes to store temporary data that + will be available at compile-time. Custom attributes may be configured to + behave closer to Erlang by using `Module.register_attribute/3`. - Finally, notice that attributes can also be read inside functions: + > **Important:** Libraries and frameworks should consider prefixing any + > module attributes that are private by underscore, such as `@_my_data` + > so code completion tools do not show them on suggestions and prompts. + + Finally, note that attributes can also be read inside functions: defmodule MyServer do @my_data 11 @@ -1873,99 +3330,268 @@ defmodule Kernel do def second_data, do: @my_data end - MyServer.first_data #=> 11 - MyServer.second_data #=> 13 + MyServer.first_data() + #=> 11 + + MyServer.second_data() + #=> 13 It is important to note that reading an attribute takes a snapshot of its current value. In other words, the value is read at compilation - time and not at runtime. Check the module `Module` for other functions + time and not at runtime. Check the `Module` module for other functions to manipulate module attributes. + + ## Attention! Multiple references of the same attribute + + As mentioned above, every time you read a module attribute, a snapshot + of its current value is taken. Therefore, if you are storing large + values inside module attributes (for example, embedding external files + in module attributes), you should avoid referencing the same attribute + multiple times. For example, don't do this: + + @files %{ + example1: File.read!("lib/example1.data"), + example2: File.read!("lib/example2.data") + } + + def example1, do: @files[:example1] + def example2, do: @files[:example2] + + In the above, each reference to `@files` may end-up with a complete + and individual copy of the whole `@files` module attribute. Instead, + reference the module attribute once in a private function: + + @files %{ + example1: File.read!("lib/example1.data"), + example2: File.read!("lib/example2.data") + } + + defp files(), do: @files + def example1, do: files()[:example1] + def example2, do: files()[:example2] + + ## Attention! Compile-time dependencies + + Keep in mind references to other modules, even in module attributes, + generate compile-time dependencies to said modules. + + For example, take this common pattern: + + @values [:foo, :bar, :baz] + + def handle_arg(arg) when arg in @values do + ... + end + + While the above is fine, imagine if instead you have actual + module names in the module attribute, like this: + + @values [Foo, Bar, Baz] + + def handle_arg(arg) when arg in @values do + ... + end + + The code above will define a compile-time dependency on the modules + `Foo`, `Bar`, and `Baz`, in a way that, if any of them change, the + current module will have to recompile. In such cases, it may be + preferred to avoid the module attribute altogether: + + def handle_arg(arg) when arg in [Foo, Bar, Baz] do + ... + end + """ - defmacro @(expr) + defmacro @expr + + defmacro @{:__aliases__, _meta, _args} do + raise ArgumentError, "module attributes set via @ cannot start with an uppercase letter" + end + + defmacro @{name, meta, args} do + assert_module_scope(__CALLER__, :@, 1) + function? = __CALLER__.function != nil + + cond do + # Check for Macro as it is compiled later than Kernel + not bootstrapped?(Macro) -> + nil + + not function? and __CALLER__.context == :match -> + raise ArgumentError, + """ + invalid write attribute syntax. If you want to define an attribute, don't do this: + + @foo = :value + + Instead, do this: - # Typespecs attributes are special cased by the compiler so far - defmacro @({name, _, args}) do - # Check for Macro as it is compiled later than Module - case bootstraped?(Module) do - false -> nil - true -> - assert_module_scope(__CALLER__, :@, 1) - function? = __CALLER__.function != nil + @foo :value + """ - case is_list(args) and length(args) == 1 and typespec(name) do + # Typespecs attributes are currently special cased by the compiler + is_list(args) and typespec?(name) -> + case bootstrapped?(Kernel.Typespec) do false -> - case name == :typedoc and not bootstraped?(Kernel.Typespec) do - true -> nil - false -> do_at(args, name, function?, __CALLER__) - end - macro -> - case bootstraped?(Kernel.Typespec) do - false -> nil - true -> quote do: Kernel.Typespec.unquote(macro)(unquote(hd(args))) + :ok + + true -> + pos = :elixir_locals.cache_env(__CALLER__) + %{line: line, file: file, module: module} = __CALLER__ + + quote do + Kernel.Typespec.deftypespec( + unquote(name), + unquote(Macro.escape(hd(args), unquote: true)), + unquote(line), + unquote(file), + unquote(module), + unquote(pos) + ) end end + + true -> + do_at(args, meta, name, function?, __CALLER__) end end - # @attribute value - defp do_at([arg], name, function?, env) do - case function? do - true -> - raise ArgumentError, "cannot dynamically set attribute @#{name} inside function" - false -> - case name do - :behavior -> - :elixir_errors.warn warn_info(env_stacktrace(env)), - "@behavior attribute is not supported, please use @behaviour instead" - _ -> - :ok + # @attribute(value) + defp do_at([arg], meta, name, function?, env) do + line = + case :lists.keymember(:context, 1, meta) do + true -> nil + false -> env.line + end + + cond do + function? -> + raise ArgumentError, "cannot set attribute @#{name} inside function/macro" + + name == :behavior -> + warn_message = "@behavior attribute is not supported, please use @behaviour instead" + IO.warn(warn_message, env) + + :lists.member(name, [:moduledoc, :typedoc, :doc]) -> + arg = {env.line, arg} + + quote do + Module.__put_attribute__(__MODULE__, unquote(name), unquote(arg), unquote(line)) end - quote do: Module.put_attribute(__MODULE__, unquote(name), unquote(arg)) + true -> + arg = expand_attribute(name, arg, env) + + quote do + Module.__put_attribute__(__MODULE__, unquote(name), unquote(arg), unquote(line)) + end end end - # @attribute or @attribute() - defp do_at(args, name, function?, env) when is_atom(args) or args == [] do - stack = env_stacktrace(env) + # @attribute() + defp do_at([], meta, name, function?, env) do + IO.warn( + "the @#{name}() notation (with parenthesis) is deprecated, please use @#{name} (without parenthesis) instead", + Macro.Env.stacktrace(env) + ) + + do_at(nil, meta, name, function?, env) + end + + # @attribute + defp do_at(args, _meta, name, function?, env) when is_atom(args) do + line = env.line + doc_attr? = :lists.member(name, [:moduledoc, :typedoc, :doc]) case function? do true -> - attr = Module.get_attribute(env.module, name, stack) - :erlang.element(1, :elixir_quote.escape(attr, false)) + value = + case Module.__get_attribute__(env.module, name, line) do + {_, doc} when doc_attr? -> doc + other -> other + end + + try do + :elixir_quote.escape(value, :none, false) + rescue + ex in [ArgumentError] -> + raise ArgumentError, + "cannot inject attribute @#{name} into function/macro because " <> + Exception.message(ex) + end + + false when doc_attr? -> + quote do + case Module.__get_attribute__(__MODULE__, unquote(name), unquote(line)) do + {_, doc} -> doc + other -> other + end + end + false -> - escaped = case stack do - [] -> [] - _ -> Macro.escape(stack) + quote do + Module.__get_attribute__(__MODULE__, unquote(name), unquote(line)) end - quote do: Module.get_attribute(__MODULE__, unquote(name), unquote(escaped)) end end - # All other cases - defp do_at(args, name, _function?, _env) do - raise ArgumentError, "expected 0 or 1 argument for @#{name}, got: #{length(args)}" + # Error cases + defp do_at([{call, meta, ctx_or_args}, [{:do, _} | _] = kw], _meta, name, _function?, _env) do + args = + case is_atom(ctx_or_args) do + true -> [] + false -> ctx_or_args + end + + code = "\n@#{name} (#{Macro.to_string({call, meta, args ++ [kw]})})" + + raise ArgumentError, """ + expected 0 or 1 argument for @#{name}, got 2. + + It seems you are trying to use the do-syntax with @module attributes \ + but the do-block is binding to the attribute name. You probably want \ + to wrap the argument value in parentheses, like this: + #{String.replace(code, "\n", "\n ")} + """ end - defp warn_info([entry|_]) do - opts = elem(entry, tuple_size(entry) - 1) - Exception.format_file_line(Keyword.get(opts, :file), Keyword.get(opts, :line)) <> " " + defp do_at(args, _meta, name, _function?, _env) do + raise ArgumentError, "expected 0 or 1 argument for @#{name}, got: #{length(args)}" end - defp warn_info([]) do - "" + defp expand_attribute(:compile, arg, env) do + Macro.prewalk(arg, fn + # {:no_warn_undefined, alias} + {elem, {:__aliases__, _, _} = alias} -> + {elem, Macro.expand(alias, %{env | function: {:__info__, 1}})} + + # {alias, fun, arity} + {:{}, meta, [{:__aliases__, _, _} = alias, fun, arity]} -> + {:{}, meta, [Macro.expand(alias, %{env | function: {:__info__, 1}}), fun, arity]} + + node -> + node + end) end - defp typespec(:type), do: :deftype - defp typespec(:typep), do: :deftypep - defp typespec(:opaque), do: :defopaque - defp typespec(:spec), do: :defspec - defp typespec(:callback), do: :defcallback - defp typespec(_), do: false + defp expand_attribute(_, arg, _), do: arg + + defp typespec?(:type), do: true + defp typespec?(:typep), do: true + defp typespec?(:opaque), do: true + defp typespec?(:spec), do: true + defp typespec?(:callback), do: true + defp typespec?(:macrocallback), do: true + defp typespec?(_), do: false @doc """ - Returns the binding as a keyword list where the variable name - is the key and the variable value is the value. + Returns the binding for the given context as a keyword list. + + In the returned result, keys are variable names and values are the + corresponding variable values. + + If the given `context` is `nil` (by default it is), the binding for the + current context is returned. ## Examples @@ -1976,216 +3602,338 @@ defmodule Kernel do iex> binding() [x: 2] + iex> binding(:foo) + [] + iex> var!(x, :foo) = 1 + 1 + iex> binding(:foo) + [x: 1] + """ - defmacro binding() do - do_binding(nil, nil, __CALLER__.vars, Macro.Env.in_match?(__CALLER__)) + defmacro binding(context \\ nil) do + in_match? = Macro.Env.in_match?(__CALLER__) + + bindings = + for {v, c} <- Macro.Env.vars(__CALLER__), c == context do + {v, wrap_binding(in_match?, {v, [generated: true], c})} + end + + :lists.sort(bindings) + end + + defp wrap_binding(true, var) do + quote(do: ^unquote(var)) + end + + defp wrap_binding(_, var) do + var end @doc """ - Receives a list of atoms at compilation time and returns the - binding of the given variables as a keyword list where the - variable name is the key and the variable value is the value. + Provides an `if/2` macro. - In case a variable in the list does not exist in the binding, - it is not included in the returned result. + This macro expects the first argument to be a condition and the second + argument to be a keyword list. - ## Examples + ## One-liner examples - iex> x = 1 - iex> binding([:x, :y]) - [x: 1] + if(foo, do: bar) + + In the example above, `bar` will be returned if `foo` evaluates to + a truthy value (neither `false` nor `nil`). Otherwise, `nil` will be + returned. + + An `else` option can be given to specify the opposite: + + if(foo, do: bar, else: baz) + + ## Blocks examples + + It's also possible to pass a block to the `if/2` macro. The first + example above would be translated to: + + if foo do + bar + end + + Note that `do`-`end` become delimiters. The second example would + translate to: + + if foo do + bar + else + baz + end + In order to compare more than two clauses, the `cond/1` macro has to be used. """ - defmacro binding(list) when is_list(list) do - do_binding(list, nil, __CALLER__.vars, Macro.Env.in_match?(__CALLER__)) + defmacro if(condition, clauses) do + build_if(condition, clauses) + end + + defp build_if(condition, do: do_clause) do + build_if(condition, do: do_clause, else: nil) + end + + defp build_if(condition, do: do_clause, else: else_clause) do + optimize_boolean( + quote do + case unquote(condition) do + x when :"Elixir.Kernel".in(x, [false, nil]) -> unquote(else_clause) + _ -> unquote(do_clause) + end + end + ) end - defmacro binding(context) when is_atom(context) do - do_binding(nil, context, __CALLER__.vars, Macro.Env.in_match?(__CALLER__)) + defp build_if(_condition, _arguments) do + raise ArgumentError, + "invalid or duplicate keys for if, only \"do\" and an optional \"else\" are permitted" end @doc """ - Receives a list of atoms at compilation time and returns the - binding of the given variables in the given context as a keyword - list where the variable name is the key and the variable value - is the value. + Provides an `unless` macro. + + This macro evaluates and returns the `do` block passed in as the second + argument if `condition` evaluates to a falsy value (`false` or `nil`). + Otherwise, it returns the value of the `else` block if present or `nil` if not. - In case a variable in the list does not exist in the binding, - it is not included in the returned result. + See also `if/2`. ## Examples - iex> var!(x, :foo) = 1 - iex> binding([:x, :y]) - [] - iex> binding([:x, :y], :foo) - [x: 1] + iex> unless(Enum.empty?([]), do: "Hello") + nil + + iex> unless(Enum.empty?([1, 2, 3]), do: "Hello") + "Hello" + + iex> unless Enum.sum([2, 2]) == 5 do + ...> "Math still works" + ...> else + ...> "Math is broken" + ...> end + "Math still works" """ - defmacro binding(list, context) when is_list(list) and is_atom(context) do - do_binding(list, context, __CALLER__.vars, Macro.Env.in_match?(__CALLER__)) + defmacro unless(condition, clauses) do + build_unless(condition, clauses) + end + + defp build_unless(condition, do: do_clause) do + build_unless(condition, do: do_clause, else: nil) end - defp do_binding(list, context, vars, in_match) do - for {v, c} <- vars, c == context, list == nil or :lists.member(v, list) do - {v, wrap_binding(in_match, {v, [], c})} + defp build_unless(condition, do: do_clause, else: else_clause) do + quote do + if(unquote(condition), do: unquote(else_clause), else: unquote(do_clause)) end end - defp wrap_binding(true, var) do - quote do: ^(unquote(var)) + defp build_unless(_condition, _arguments) do + raise ArgumentError, + "invalid or duplicate keys for unless, " <> + "only \"do\" and an optional \"else\" are permitted" end - defp wrap_binding(_, var) do - var + @doc """ + Destructures two lists, assigning each term in the + right one to the matching term in the left one. + + Unlike pattern matching via `=`, if the sizes of the left + and right lists don't match, destructuring simply stops + instead of raising an error. + + ## Examples + + iex> destructure([x, y, z], [1, 2, 3, 4, 5]) + iex> {x, y, z} + {1, 2, 3} + + In the example above, even though the right list has more entries than the + left one, destructuring works fine. If the right list is smaller, the + remaining elements are simply set to `nil`: + + iex> destructure([x, y, z], [1]) + iex> {x, y, z} + {1, nil, nil} + + The left-hand side supports any expression you would use + on the left-hand side of a match: + + x = 1 + destructure([^x, y, z], [1, 2, 3]) + + The example above will only work if `x` matches the first value in the right + list. Otherwise, it will raise a `MatchError` (like the `=` operator would + do). + """ + defmacro destructure(left, right) when is_list(left) do + quote do + unquote(left) = Kernel.Utils.destructure(unquote(right), unquote(length(left))) + end end @doc """ - Provides an `if` macro. This macro expects the first argument to - be a condition and the rest are keyword arguments. + Creates a range from `first` to `last`. - ## One-liner examples + If first is less than last, the range will be increasing from + first to last. If first is equal to last, the range will contain + one element, which is the number itself. - if(foo, do: bar) + If first is more than last, the range will be decreasing from first + to last, albeit this behaviour is deprecated. Instead prefer to + explicitly list the step with `first..last//-1`. - In the example above, `bar` will be returned if `foo` evaluates to - `true` (i.e. it is neither `false` nor `nil`). Otherwise, `nil` will be returned. + See the `Range` module for more information. - An `else` option can be given to specify the opposite: + ## Examples - if(foo, do: bar, else: baz) + iex> 0 in 1..3 + false + iex> 2 in 1..3 + true - ## Blocks examples + iex> Enum.to_list(1..3) + [1, 2, 3] - Elixir also allows you to pass a block to the `if` macro. The first - example above would be translated to: + """ + defmacro first..last do + case bootstrapped?(Macro) do + true -> + first = Macro.expand(first, __CALLER__) + last = Macro.expand(last, __CALLER__) + validate_range!(first, last) + range(__CALLER__.context, first, last) - if foo do - bar - end + false -> + range(__CALLER__.context, first, last) + end + end - Notice that `do/end` becomes delimiters. The second example would - then translate to: + defp range(_context, first, last) when is_integer(first) and is_integer(last) do + # TODO: Deprecate inferring a range with a step of -1 on Elixir v1.17 + step = if first <= last, do: 1, else: -1 + {:%{}, [], [__struct__: Elixir.Range, first: first, last: last, step: step]} + end - if foo do - bar - else - baz - end + defp range(nil, first, last) do + quote(do: Elixir.Range.new(unquote(first), unquote(last))) + end - If you want to compare more than two clauses, you can use the `cond/1` - macro. - """ - defmacro if(condition, clauses) do - do_clause = Keyword.get(clauses, :do, nil) - else_clause = Keyword.get(clauses, :else, nil) + defp range(:guard, first, last) do + # TODO: Deprecate me inside guard when sides are not integers on Elixir v1.17 + {:%{}, [], [__struct__: Elixir.Range, first: first, last: last, step: nil]} + end - optimize_boolean(quote do - case unquote(condition) do - x when x in [false, nil] -> unquote(else_clause) - _ -> unquote(do_clause) - end - end) + defp range(:match, first, last) do + # TODO: Deprecate me inside match in all occasions (including literals) on Elixir v1.17 + {:%{}, [], [__struct__: Elixir.Range, first: first, last: last]} end @doc """ - Evaluates and returns the do-block passed in as a second argument - unless clause evaluates to true. - Returns nil otherwise. - See also `if`. + Creates a range from `first` to `last` with `step`. + + See the `Range` module for more information. ## Examples - iex> unless(Enum.empty?([]), do: "Hello") - nil + iex> 0 in 1..3//1 + false + iex> 2 in 1..3//1 + true + iex> 2 in 1..3//2 + false - iex> unless(Enum.empty?([1,2,3]), do: "Hello") - "Hello" + iex> Enum.to_list(1..3//1) + [1, 2, 3] + iex> Enum.to_list(1..3//2) + [1, 3] + iex> Enum.to_list(3..1//-1) + [3, 2, 1] + iex> Enum.to_list(1..0//1) + [] """ - defmacro unless(clause, options) do - do_clause = Keyword.get(options, :do, nil) - else_clause = Keyword.get(options, :else, nil) - quote do - if(unquote(clause), do: unquote(else_clause), else: unquote(do_clause)) + @doc since: "1.12.0" + defmacro first..last//step do + case bootstrapped?(Macro) do + true -> + first = Macro.expand(first, __CALLER__) + last = Macro.expand(last, __CALLER__) + step = Macro.expand(step, __CALLER__) + validate_range!(first, last) + validate_step!(step) + range(__CALLER__.context, first, last, step) + + false -> + range(__CALLER__.context, first, last, step) end end - @doc """ - Allows you to destructure two lists, assigning each term in the right to the - matching term in the left. Unlike pattern matching via `=`, if the sizes of - the left and right lists don't match, destructuring simply stops instead of - raising an error. + defp range(context, first, last, step) + when is_integer(first) and is_integer(last) and is_integer(step) + when context != nil do + {:%{}, [], [__struct__: Elixir.Range, first: first, last: last, step: step]} + end - ## Examples + defp range(nil, first, last, step) do + quote(do: Elixir.Range.new(unquote(first), unquote(last), unquote(step))) + end - iex> destructure([x, y, z], [1, 2, 3, 4, 5]) - iex> {x, y, z} - {1, 2, 3} + defp validate_range!(first, last) + when is_float(first) or is_float(last) or is_atom(first) or is_atom(last) or + is_binary(first) or is_binary(last) or is_list(first) or is_list(last) do + raise ArgumentError, + "ranges (first..last//step) expect both sides to be integers, " <> + "got: #{Macro.to_string({:.., [], [first, last]})}" + end - Notice in the example above, even though the right - size has more entries than the left, destructuring works - fine. If the right size is smaller, the remaining items - are simply assigned to nil: + defp validate_range!(_, _), do: :ok - iex> destructure([x, y, z], [1]) - iex> {x, y, z} - {1, nil, nil} + defp validate_step!(step) + when is_float(step) or is_atom(step) or is_binary(step) or is_list(step) or step == 0 do + raise ArgumentError, + "ranges (first..last//step) expect the step to be a non-zero integer, " <> + "got: #{Macro.to_string(step)}" + end - The left side supports any expression you would use - on the left side of a match: + defp validate_step!(_), do: :ok - x = 1 - destructure([^x, y, z], [1, 2, 3]) + @doc """ + Creates the full-slice range `0..-1//1`. - The example above will only work if x matches - the first value from the right side. Otherwise, - it will raise a CaseClauseError. - """ - defmacro destructure(left, right) when is_list(left) do - Enum.reduce left, right, fn item, acc -> - {:case, meta, args} = - quote do - case unquote(acc) do - [unquote(item)|t] -> - t - other when other == [] or other == nil -> - unquote(item) = nil - end - end - {:case, [{:export_head,true}|meta], args} - end - end + This macro returns a range with the following properties: - @doc """ - Returns a range with the specified start and end. - Includes both ends. + * When enumerated, it is empty - ## Examples + * When used as a `slice`, it returns the sliced element as is - iex> 0 in 1..3 - false + See `..///3` and the `Range` module for more information. - iex> 1 in 1..3 - true + ## Examples - iex> 2 in 1..3 - true + iex> Enum.to_list(..) + [] - iex> 3 in 1..3 - true + iex> String.slice("Hello world!", ..) + "Hello world!" """ - defmacro first .. last do - {:%{}, [], [__struct__: Elixir.Range, first: first, last: last]} + defmacro (..) do + range(__CALLER__.context, 0, -1, 1) end @doc """ + Boolean "and" operator. + Provides a short-circuit operator that evaluates and returns - the second expression only if the first one evaluates to true - (i.e. it is not nil nor false). Returns the first expression + the second expression only if the first one evaluates to a truthy value + (neither `false` nor `nil`). Returns the first expression otherwise. + Not allowed in guard clauses. + ## Examples iex> Enum.empty?([]) && Enum.empty?([]) @@ -2200,15 +3948,17 @@ defmodule Kernel do iex> false && throw(:bad) false - Notice that, unlike Erlang's `and` operator, - this operator accepts any expression as an argument, - not only booleans, however it is not allowed in guards. + Note that, unlike `and/2`, this operator accepts any expression + as the first argument, not only booleans. """ defmacro left && right do + assert_no_match_or_guard_scope(__CALLER__.context, "&&") + quote do case unquote(left) do - x when x in [false, nil] -> + x when :"Elixir.Kernel".in(x, [false, nil]) -> x + _ -> unquote(right) end @@ -2216,9 +3966,13 @@ defmodule Kernel do end @doc """ + Boolean "or" operator. + Provides a short-circuit operator that evaluates and returns the second - expression only if the first one does not evaluate to true (i.e. it - is either nil or false). Returns the first expression otherwise. + expression only if the first one does not evaluate to a truthy value (that is, + it is either `nil` or `false`). Returns the first expression otherwise. + + Not allowed in guard clauses. ## Examples @@ -2234,15 +3988,17 @@ defmodule Kernel do iex> Enum.empty?([]) || throw(:bad) true - Notice that, unlike Erlang's `or` operator, - this operator accepts any expression as an argument, - not only booleans, however it is not allowed in guards. + Note that, unlike `or/2`, this operator accepts any expression + as the first argument, not only booleans. """ defmacro left || right do + assert_no_match_or_guard_scope(__CALLER__.context, "||") + quote do case unquote(left) do - x when x in [false, nil] -> + x when :"Elixir.Kernel".in(x, [false, nil]) -> unquote(right) + x -> x end @@ -2250,104 +4006,188 @@ defmodule Kernel do end @doc """ - `|>` is the pipe operator. + Pipe operator. - This operator introduces the expression on the left as - the first argument to the function call on the right. + This operator introduces the expression on the left-hand side as + the first argument to the function call on the right-hand side. ## Examples iex> [1, [2], 3] |> List.flatten() [1, 2, 3] - The example above is the same as calling `List.flatten([1, [2], 3])`, - i.e. the argument on the left side of `|>` is introduced as the first - argument of the function call on the right side. + The example above is the same as calling `List.flatten([1, [2], 3])`. - This pattern is mostly useful when there is a desire to execute - a bunch of operations, resembling a pipeline: + The `|>/2` operator is mostly useful when there is a desire to execute a series + of operations resembling a pipeline: - iex> [1, [2], 3] |> List.flatten |> Enum.map(fn x -> x * 2 end) + iex> [1, [2], 3] |> List.flatten() |> Enum.map(fn x -> x * 2 end) [2, 4, 6] - The example above will pass the list to `List.flatten/1`, then get - the flattened list and pass to `Enum.map/2`, which will multiply - each entry in the list per two. + In the example above, the list `[1, [2], 3]` is passed as the first argument + to the `List.flatten/1` function, then the flattened list is passed as the + first argument to the `Enum.map/2` function which doubles each element of the + list. In other words, the expression above simply translates to: Enum.map(List.flatten([1, [2], 3]), fn x -> x * 2 end) - Beware of operator precedence when using the pipe operator. - For example, the following expression: + ## Pitfalls + + There are two common pitfalls when using the pipe operator. + + The first one is related to operator precedence. For example, + the following expression: String.graphemes "Hello" |> Enum.reverse Translates to: - String.graphemes("Hello" |> Enum.reverse) + String.graphemes("Hello" |> Enum.reverse()) - Which will result in an error as Enumerable protocol is not defined - for binaries. Adding explicit parenthesis resolves the ambiguity: + which results in an error as the `Enumerable` protocol is not defined + for binaries. Adding explicit parentheses resolves the ambiguity: - String.graphemes("Hello") |> Enum.reverse + String.graphemes("Hello") |> Enum.reverse() Or, even better: - "Hello" |> String.graphemes |> Enum.reverse + "Hello" |> String.graphemes() |> Enum.reverse() + + The second limitation is that Elixir always pipes to a function + call. Therefore, to pipe into an anonymous function, you need to + invoke it: + + some_fun = &Regex.replace(~r/l/, &1, "L") + "Hello" |> some_fun.() + + Alternatively, you can use `then/2` for the same effect: + + some_fun = &Regex.replace(~r/l/, &1, "L") + "Hello" |> then(some_fun) + + `then/2` is most commonly used when you want to pipe to a function + but the value is expected outside of the first argument, such as + above. By replacing `some_fun` by its value, we get: + + "Hello" |> then(&Regex.replace(~r/l/, &1, "L")) """ defmacro left |> right do - [{h, _}|t] = Macro.unpipe({:|>, [], [left, right]}) - :lists.foldl fn {x, pos}, acc -> Macro.pipe(acc, x, pos) end, h, t + [{h, _} | t] = Macro.unpipe({:|>, [], [left, right]}) + + fun = fn {x, pos}, acc -> + Macro.pipe(acc, x, pos) + end + + :lists.foldl(fun, h, t) end @doc """ - Returns true if the `module` is loaded and contains a - public `function` with the given `arity`, otherwise false. + Returns `true` if `module` is loaded and contains a + public `function` with the given `arity`, otherwise `false`. - Notice that this function does not load the module in case + Note that this function does not load the module in case it is not loaded. Check `Code.ensure_loaded/1` for more information. + + Inlined by the compiler. + + ## Examples + + iex> function_exported?(Enum, :map, 2) + true + + iex> function_exported?(Enum, :map, 10) + false + + iex> function_exported?(List, :to_string, 1) + true """ - @spec function_exported?(atom | tuple, atom, integer) :: boolean + @spec function_exported?(module, atom, arity) :: boolean def function_exported?(module, function, arity) do :erlang.function_exported(module, function, arity) end @doc """ - Returns true if the `module` is loaded and contains a - public `macro` with the given `arity`, otherwise false. + Returns `true` if `module` is loaded and contains a + public `macro` with the given `arity`, otherwise `false`. - Notice that this function does not load the module in case + Note that this function does not load the module in case it is not loaded. Check `Code.ensure_loaded/1` for more information. + + If `module` is an Erlang module (as opposed to an Elixir module), this + function always returns `false`. + + ## Examples + + iex> macro_exported?(Kernel, :use, 2) + true + + iex> macro_exported?(:erlang, :abs, 1) + false + """ - @spec macro_exported?(atom, atom, integer) :: boolean - def macro_exported?(module, macro, arity) do - case :code.is_loaded(module) do - {:file, _} -> :lists.member({macro, arity}, module.__info__(:macros)) - _ -> false - end + @spec macro_exported?(module, atom, arity) :: boolean + def macro_exported?(module, macro, arity) + when is_atom(module) and is_atom(macro) and is_integer(arity) and + (arity >= 0 and arity <= 255) do + function_exported?(module, :__info__, 1) and + :lists.member({macro, arity}, module.__info__(:macros)) end @doc """ - Access the given element using the qualifier according - to the `Access` protocol. All calls in the form `foo[bar]` - are translated to `access(foo, bar)`. + Power operator. - The usage of this protocol is to access a raw value in a - keyword list. + It expects two numbers are input. If the left-hand side is an integer + and the right-hand side is more than or equal to 0, then the result is + integer. Otherwise it returns a float. - iex> sample = [a: 1, b: 2, c: 3] - iex> sample[:b] - 2 + ## Examples + + iex> 2 ** 2 + 4 + iex> 2 ** -4 + 0.0625 + + iex> 2.0 ** 2 + 4.0 + iex> 2 ** 2.0 + 4.0 """ + @doc since: "1.13.0" + @spec integer ** non_neg_integer :: integer + @spec integer ** neg_integer :: float + @spec float ** float :: float + def base ** exponent when is_integer(base) and is_integer(exponent) and exponent >= 0 do + integer_pow(base, 1, exponent) + end + + def base ** exponent when is_number(base) and is_number(exponent) do + :math.pow(base, exponent) + end + + # https://en.wikipedia.org/wiki/Exponentiation_by_squaring + defp integer_pow(_, _, 0), + do: 1 + + defp integer_pow(b, a, 1), + do: b * a + + defp integer_pow(b, a, e) when :erlang.band(e, 1) == 0, + do: integer_pow(b * b, a, :erlang.bsr(e, 1)) + + defp integer_pow(b, a, e), + do: integer_pow(b * b, a * b, :erlang.bsr(e, 1)) @doc """ - Checks if the element on the left side is member of the - collection on the right side. + Membership operator. + + Checks if the element on the left-hand side is a member of the + collection on the right-hand side. ## Examples @@ -2355,144 +4195,451 @@ defmodule Kernel do iex> x in [1, 2, 3] true - This macro simply translates the expression above to: + This operator (which is a macro) simply translates to a call to + `Enum.member?/2`. The example above would translate to: + + Enum.member?([1, 2, 3], x) - Enum.member?([1,2,3], x) + Elixir also supports `left not in right`, which evaluates to + `not(left in right)`: + + iex> x = 1 + iex> x not in [1, 2, 3] + false ## Guards - The `in` operator can be used on guard clauses as long as the - right side is a range or a list. Elixir will then expand the - operator to a valid guard expression. For example: + The `in/2` operator (as well as `not in`) can be used in guard clauses as + long as the right-hand side is a range or a list. - when x in [1,2,3] + If the right-hand side is a list, Elixir will expand the operator to a valid + guard expression which needs to check each value. For example: - Translates to: + when x in [1, 2, 3] + + translates to: when x === 1 or x === 2 or x === 3 - When using ranges: + However, this construct will be inneficient for large lists. In such cases, it + is best to stop using guards and use a more appropriate data structure, such + as `MapSet`. - when x in 1..3 + If the right-hand side is a range, a more efficient comparison check will be + done. For example: - Translates to: + when x in 1..1000 - when x >= 1 and x <= 3 + translates roughly to: + when x >= 1 and x <= 1000 + + ### AST considerations + + `left not in right` is parsed by the compiler into the AST: + + {:not, _, [{:in, _, [left, right]}]} + + This is the same AST as `not(left in right)`. + + Additionally, `Macro.to_string/2` and `Code.format_string!/2` + will translate all occurrences of this AST to `left not in right`. """ + @doc guard: true defmacro left in right do - cache = (__CALLER__.context == nil) + in_body? = __CALLER__.context == nil - right = case bootstraped?(Macro) do - true -> Macro.expand(right, __CALLER__) - false -> right - end + expand = + case bootstrapped?(Macro) do + true -> &Macro.expand(&1, __CALLER__) + false -> & &1 + end - case right do - _ when cache -> - quote do: Elixir.Enum.member?(unquote(right), unquote(left)) - [] -> + case expand.(right) do + [] when not in_body? -> false - [h|t] -> - :lists.foldr(fn x, acc -> - quote do - unquote(comp(left, x)) or unquote(acc) - end - end, comp(left, h), t) - {:%{}, [], [__struct__: Elixir.Range, first: first, last: last]} -> - in_range(left, Macro.expand(first, __CALLER__), Macro.expand(last, __CALLER__)) - _ -> - raise ArgumentError, <<"invalid args for operator in, it expects a compile time list ", - "or range on the right side when used in guard expressions, got: ", - Macro.to_string(right) :: binary>> + + [] -> + quote do + _ = unquote(left) + false + end + + [head | tail] = list -> + # We only expand lists in the body if they are relatively + # short and it is made only of literal expressions. + case not in_body? or small_literal_list?(right) do + true -> in_var(in_body?, left, &in_list(&1, head, tail, expand, list, in_body?)) + false -> quote(do: :lists.member(unquote(left), unquote(right))) + end + + {:%{}, _meta, [__struct__: Elixir.Range, first: first, last: last, step: step]} -> + in_var(in_body?, left, &in_range(&1, expand.(first), expand.(last), expand.(step))) + + right when in_body? -> + quote(do: Elixir.Enum.member?(unquote(right), unquote(left))) + + %{__struct__: Elixir.Range, first: _, last: _, step: _} -> + raise ArgumentError, "non-literal range in guard should be escaped with Macro.escape/2" + + right -> + raise_on_invalid_args_in_2(right) + end + end + + defp raise_on_invalid_args_in_2(right) do + raise ArgumentError, << + "invalid right argument for operator \"in\", it expects a compile-time proper list ", + "or compile-time range on the right side when used in guard expressions, got: ", + Macro.to_string(right)::binary + >> + end + + defp in_var(false, ast, fun), do: fun.(ast) + + defp in_var(true, {atom, _, context} = var, fun) when is_atom(atom) and is_atom(context), + do: fun.(var) + + defp in_var(true, ast, fun) do + quote do + var = unquote(ast) + unquote(fun.(quote(do: var))) + end + end + + defp small_literal_list?(list) when is_list(list) and length(list) <= 32 do + :lists.all(fn x -> is_binary(x) or is_atom(x) or is_number(x) end, list) + end + + defp small_literal_list?(_list), do: false + + defp in_range(left, first, last, nil) do + # TODO: nil steps are only supported due to x..y in guards. Remove me on Elixir 2.0. + quote do + :erlang.is_integer(unquote(left)) and :erlang.is_integer(unquote(first)) and + :erlang.is_integer(unquote(last)) and + ((:erlang."=<"(unquote(first), unquote(last)) and + unquote(increasing_compare(left, first, last))) or + (:erlang.<(unquote(last), unquote(first)) and + unquote(decreasing_compare(left, first, last)))) + end + end + + defp in_range(left, first, last, step) when is_integer(step) do + in_range_literal(left, first, last, step) + end + + defp in_range(left, first, last, step) do + quoted = + quote do + :erlang.is_integer(unquote(left)) and :erlang.is_integer(unquote(first)) and + :erlang.is_integer(unquote(last)) and + ((:erlang.>(unquote(step), 0) and + unquote(increasing_compare(left, first, last))) or + (:erlang.<(unquote(step), 0) and + unquote(decreasing_compare(left, first, last)))) + end + + in_range_step(quoted, left, first, step) + end + + defp in_range_literal(left, first, first, _step) when is_integer(first) do + quote do: :erlang."=:="(unquote(left), unquote(first)) + end + + defp in_range_literal(left, first, last, step) when step > 0 do + quoted = + quote do + :erlang.andalso( + :erlang.is_integer(unquote(left)), + unquote(increasing_compare(left, first, last)) + ) + end + + in_range_step(quoted, left, first, step) + end + + defp in_range_literal(left, first, last, step) when step < 0 do + quoted = + quote do + :erlang.andalso( + :erlang.is_integer(unquote(left)), + unquote(decreasing_compare(left, first, last)) + ) + end + + in_range_step(quoted, left, first, step) + end + + defp in_range_step(quoted, _left, _first, step) when step == 1 or step == -1 do + quoted + end + + defp in_range_step(quoted, left, first, step) do + quote do + :erlang.andalso( + unquote(quoted), + :erlang."=:="(:erlang.rem(unquote(left) - unquote(first), unquote(step)), 0) + ) end end - defp in_range(left, first, last) do - case opt_in?(first) and opt_in?(last) do - true -> - case first <= last do - true -> increasing_compare(left, first, last) - false -> decreasing_compare(left, first, last) + defp in_list(left, head, tail, expand, right, in_body?) do + [head | tail] = :lists.map(&comp(left, &1, expand, right, in_body?), [head | tail]) + :lists.foldl("e(do: :erlang.orelse(unquote(&2), unquote(&1))), head, tail) + end + + defp comp(left, {:|, _, [head, tail]}, expand, right, in_body?) do + case expand.(tail) do + [] -> + quote(do: :erlang."=:="(unquote(left), unquote(head))) + + [tail_head | tail] -> + quote do + :erlang.orelse( + :erlang."=:="(unquote(left), unquote(head)), + unquote(in_list(left, tail_head, tail, expand, right, in_body?)) + ) end - false -> + + tail when in_body? -> quote do - (:erlang."=<"(unquote(first), unquote(last)) and - unquote(increasing_compare(left, first, last))) - or - (:erlang."<"(unquote(last), unquote(first)) and - unquote(decreasing_compare(left, first, last))) + :erlang.orelse( + :erlang."=:="(unquote(left), unquote(head)), + :lists.member(unquote(left), unquote(tail)) + ) end + + _ -> + raise_on_invalid_args_in_2(right) end end - defp opt_in?(x), do: is_integer(x) or is_float(x) or is_atom(x) - - defp comp(left, right) do + defp comp(left, right, _expand, _right, _in_body?) do quote(do: :erlang."=:="(unquote(left), unquote(right))) end defp increasing_compare(var, first, last) do quote do - :erlang.">="(unquote(var), unquote(first)) and - :erlang."=<"(unquote(var), unquote(last)) + :erlang.andalso( + :erlang.>=(unquote(var), unquote(first)), + :erlang."=<"(unquote(var), unquote(last)) + ) end end defp decreasing_compare(var, first, last) do quote do - :erlang."=<"(unquote(var), unquote(first)) and - :erlang.">="(unquote(var), unquote(last)) + :erlang.andalso( + :erlang."=<"(unquote(var), unquote(first)), + :erlang.>=(unquote(var), unquote(last)) + ) end end @doc """ - When used inside quoting, marks that the variable should - not be hygienized. The argument can be either a variable - unquoted or in standard tuple form `{name, meta, context}`. + Marks that the given variable should not be hygienized. + + This macro expects a variable and it is typically invoked + inside `Kernel.SpecialForms.quote/2` to mark that a variable + should not be hygienized. See `Kernel.SpecialForms.quote/2` + for more information. + + ## Examples + + iex> Kernel.var!(example) = 1 + 1 + iex> Kernel.var!(example) + 1 - Check `Kernel.SpecialForms.quote/2` for more information. """ defmacro var!(var, context \\ nil) defmacro var!({name, meta, atom}, context) when is_atom(name) and is_atom(atom) do - do_var!(name, meta, context, __CALLER__) + # Remove counter and force them to be vars + meta = :lists.keydelete(:counter, 1, meta) + meta = :lists.keystore(:if_undefined, 1, meta, {:if_undefined, :raise}) + + case Macro.expand(context, __CALLER__) do + context when is_atom(context) -> + {name, meta, context} + + other -> + raise ArgumentError, + "expected var! context to expand to an atom, got: #{Macro.to_string(other)}" + end end - defmacro var!(x, _context) do - raise ArgumentError, "expected a var to be given to var!, got: #{Macro.to_string(x)}" + defmacro var!(other, _context) do + raise ArgumentError, "expected a variable to be given to var!, got: #{Macro.to_string(other)}" end - defp do_var!(name, meta, context, env) do - # Remove counter and force them to be vars - meta = :lists.keydelete(:counter, 1, meta) - meta = :lists.keystore(:var, 1, meta, {:var, true}) + @doc """ + When used inside quoting, marks that the given alias should not + be hygienized. This means the alias will be expanded when + the macro is expanded. + + Check `Kernel.SpecialForms.quote/2` for more information. + """ + defmacro alias!(alias) when is_atom(alias) do + alias + end + + defmacro alias!({:__aliases__, meta, args}) do + # Simply remove the alias metadata from the node + # so it does not affect expansion. + {:__aliases__, :lists.keydelete(:alias, 1, meta), args} + end + + @doc """ + Returns a binary starting at the offset `start` and of the given `size`. + + This is similar to `binary_part/3` except that if `start + size` + is greater than the binary size, it automatically clips it to + the binary size instead of raising. Opposite to `binary_part/3`, + this function is not allowed in guards. - case Macro.expand(context, env) do - x when is_atom(x) -> - {name, meta, x} - x -> - raise ArgumentError, "expected var! context to expand to an atom, got: #{Macro.to_string(x)}" + This function works with bytes. For a slicing operation that + considers characters, see `String.slice/3`. + + ## Examples + + iex> binary_slice("elixir", 0, 6) + "elixir" + iex> binary_slice("elixir", 0, 5) + "elixi" + iex> binary_slice("elixir", 1, 4) + "lixi" + iex> binary_slice("elixir", 0, 10) + "elixir" + + If `start` is negative, it is normalized against the binary + size and clamped to 0: + + iex> binary_slice("elixir", -3, 10) + "xir" + iex> binary_slice("elixir", -10, 10) + "elixir" + + If the `size` is zero, an empty binary is returned: + + iex> binary_slice("elixir", 1, 0) + "" + + If `start` is greater than or equal to the binary size, + an empty binary is returned: + + iex> binary_slice("elixir", 10, 10) + "" + + """ + @doc since: "1.14.0" + def binary_slice(binary, start, size) + when is_binary(binary) and is_integer(start) and is_integer(size) and size >= 0 do + total = byte_size(binary) + start = if start < 0, do: max(total + start, 0), else: start + + case start < total do + true -> :erlang.binary_part(binary, start, min(size, total - start)) + false -> "" end end - @doc """ - When used inside quoting, marks that the alias should not - be hygienezed. This means the alias will be expanded when - the macro is expanded. + @doc """ + Returns a binary from the offset given by the start of the + range to the offset given by the end of the range. + + If the start or end of the range are negative, they are converted + into positive indices based on the binary size. For example, + `-1` means the last byte of the binary. + + This is similar to `binary_part/3` except that it works with ranges + and it is not allowed in guards. + + This function works with bytes. For a slicing operation that + considers characters, see `String.slice/2`. + + ## Examples + + iex> binary_slice("elixir", 0..5) + "elixir" + iex> binary_slice("elixir", 1..3) + "lix" + iex> binary_slice("elixir", 1..10) + "lixir" + + iex> binary_slice("elixir", -4..-1) + "ixir" + iex> binary_slice("elixir", -4..6) + "ixir" + iex> binary_slice("elixir", -10..10) + "elixir" + + For ranges where `start > stop`, you need to explicitly + mark them as increasing: + + iex> binary_slice("elixir", 2..-1//1) + "ixir" + iex> binary_slice("elixir", 1..-2//1) + "lixi" + + You can use `../0` as a shortcut for `0..-1//1`, which returns + the whole binary as is: + + iex> binary_slice("elixir", ..) + "elixir" + + The step can be any positive number. For example, to + get every 2 characters of the binary: + + iex> binary_slice("elixir", 0..-1//2) + "eii" + + If the first position is after the string ends or after + the last position of the range, it returns an empty string: + + iex> binary_slice("elixir", 10..3//1) + "" + iex> binary_slice("elixir", -10..-7) + "" + iex> binary_slice("a", 1..1500) + "" + + """ + @doc since: "1.14.0" + def binary_slice(binary, first..last//step) + when is_binary(binary) and step > 0 do + total = byte_size(binary) + + first = + case first < 0 do + true -> max(first + total, 0) + false -> first + end + + last = + case last < 0 do + true -> last + total + false -> last + end + + case first < total do + true -> + part = binary_part(binary, first, min(total - first, last - first + 1)) - Check `Kernel.SpecialForms.quote/2` for more information. - """ - defmacro alias!(alias) + case step do + 1 -> part + _ -> for <>, into: "", do: <> + end - defmacro alias!(alias) when is_atom(alias) do - alias + false -> + "" + end end - defmacro alias!({:__aliases__, meta, args}) do - # Simply remove the alias metadata from the node - # so it does not affect expansion. - {:__aliases__, :lists.keydelete(:alias, 1, meta), args} + def binary_slice(binary, _.._//_ = range) when is_binary(binary) do + raise ArgumentError, + "binary_slice/2 does not accept ranges with negative steps, got: #{inspect(range)}" end ## Definitions implemented in Elixir @@ -2500,166 +4647,185 @@ defmodule Kernel do @doc ~S""" Defines a module given by name with the given contents. - It returns the module name, the module binary and the - block contents result. + This macro defines a module with the given `alias` as its name and with the + given contents. It returns a tuple with four elements: + + * `:module` + * the module name + * the binary contents of the module + * the result of evaluating the contents block ## Examples - iex> defmodule Foo do - ...> def bar, do: :baz - ...> end - iex> Foo.bar - :baz + defmodule Number do + def one, do: 1 + def two, do: 2 + end + #=> {:module, Number, <<70, 79, 82, ...>>, {:two, 0}} + + Number.one() + #=> 1 + + Number.two() + #=> 2 ## Nesting - Nesting a module inside another module affects its name: + Nesting a module inside another module affects the name of the nested module: defmodule Foo do defmodule Bar do end end - In the example above, two modules `Foo` and `Foo.Bar` are created. - When nesting, Elixir automatically creates an alias, allowing the - second module `Foo.Bar` to be accessed as `Bar` in the same lexical - scope. + In the example above, two modules - `Foo` and `Foo.Bar` - are created. + When nesting, Elixir automatically creates an alias to the inner module, + allowing the second module `Foo.Bar` to be accessed as `Bar` in the same + lexical scope where it's defined (the `Foo` module). This only happens + if the nested module is defined via an alias. + + If the `Foo.Bar` module is moved somewhere else, the references to `Bar` in + the `Foo` module need to be updated to the fully-qualified name (`Foo.Bar`) or + an alias has to be explicitly set in the `Foo` module with the help of + `Kernel.SpecialForms.alias/2`. + + defmodule Foo.Bar do + # code + end - This means that, if the module `Bar` is moved to another file, - the references to `Bar` needs to be updated or an alias needs to - be explicitly set with the help of `Kernel.SpecialForms.alias/2`. + defmodule Foo do + alias Foo.Bar + # code here can refer to "Foo.Bar" as just "Bar" + end ## Dynamic names Elixir module names can be dynamically generated. This is very - useful for macros. For instance, one could write: + useful when working with macros. For instance, one could write: defmodule String.to_atom("Foo#{1}") do # contents ... end - Elixir will accept any module name as long as the expression - returns an atom. Note that, when a dynamic name is used, Elixir - won't nest the name under the current module nor automatically - set up an alias. - """ - defmacro defmodule(alias, do: block) do - env = __CALLER__ - boot? = bootstraped?(Macro) + Elixir will accept any module name as long as the expression passed as the + first argument to `defmodule/2` evaluates to an atom. + Note that, when a dynamic name is used, Elixir won't nest the name under + the current module nor automatically set up an alias. - expanded = - case boot? do - true -> Macro.expand(alias, env) - false -> alias + ## Reserved module names + + If you attempt to define a module that already exists, you will get a + warning saying that a module has been redefined. + + There are some modules that Elixir does not currently implement but it + may be implement in the future. Those modules are reserved and defining + them will result in a compilation error: + + defmodule Any do + # code end + ** (CompileError) iex:1: module Any is reserved and cannot be defined + + Elixir reserves the following module names: `Elixir`, `Any`, `BitString`, + `PID`, and `Reference`. + """ + defmacro defmodule(alias, do_block) + + defmacro defmodule(alias, do: block) do + env = __CALLER__ + expanded = expand_module_alias(alias, env) {expanded, with_alias} = - case boot? and is_atom(expanded) do + case is_atom(expanded) do true -> # Expand the module considering the current environment/nesting - full = expand_module(alias, expanded, env) - - # Generate the alias for this module definition - {new, old} = module_nesting(env.module, full) - meta = [defined: full, context: true] ++ alias_meta(alias) - + {full, old, new} = alias_defmodule(alias, expanded, env) + meta = [defined: full, context: env.module] ++ alias_meta(alias) {full, {:alias, meta, [old, [as: new, warn: false]]}} + false -> {expanded, nil} end - {escaped, _} = :elixir_quote.escape(block, false) - module_vars = module_vars(env.vars, 0) + # We do this so that the block is not tail-call optimized and stacktraces + # are not messed up. Basically, we just insert something between the return + # value of the block and what is returned by defmodule. Using just ":ok" or + # similar doesn't work because it's likely optimized away by the compiler. + block = + quote do + result = unquote(block) + :elixir_utils.noop() + result + end + + escaped = + case env do + %{function: nil, lexical_tracker: pid} when is_pid(pid) -> + integer = Kernel.LexicalTracker.write_cache(pid, block) + quote(do: Kernel.LexicalTracker.read_cache(unquote(pid), unquote(integer))) + + %{} -> + :elixir_quote.escape(block, :none, false) + end + + module_vars = :lists.map(&module_var/1, :maps.keys(env.versioned_vars)) quote do unquote(with_alias) - :elixir_module.compile(unquote(expanded), unquote(escaped), - unquote(module_vars), __ENV__) + :elixir_module.compile(unquote(expanded), unquote(escaped), unquote(module_vars), __ENV__) end end defp alias_meta({:__aliases__, meta, _}), do: meta defp alias_meta(_), do: [] - # defmodule :foo - defp expand_module(raw, _module, _env) when is_atom(raw), - do: raw - - # defmodule Elixir.Alias - defp expand_module({:__aliases__, _, [:Elixir|t]}, module, _env) when t != [], - do: module - - # defmodule Alias in root - defp expand_module({:__aliases__, _, _}, module, %{module: nil}), - do: module + # We don't want to trace :alias_reference since we are defining the alias + defp expand_module_alias({:__aliases__, _, _} = original, env) do + case :elixir_aliases.expand_or_concat(original, env) do + receiver when is_atom(receiver) -> + receiver - # defmodule Alias nested - defp expand_module({:__aliases__, _, t}, _module, env), - do: :elixir_aliases.concat([env.module|t]) + aliases -> + aliases = :lists.map(&Macro.expand(&1, env), aliases) - # defmodule _ - defp expand_module(_raw, module, env), - do: :elixir_aliases.concat([env.module, module]) + case :lists.all(&is_atom/1, aliases) do + true -> :elixir_aliases.concat(aliases) + false -> original + end + end + end - # quote vars to be injected into the module definition - defp module_vars([{key, kind}|vars], counter) do - var = - case is_atom(kind) do - true -> {key, [], kind} - false -> {key, [counter: kind], nil} - end + defp expand_module_alias(other, env), do: Macro.expand(other, env) - under = String.to_atom(<<"_@", :erlang.integer_to_binary(counter)::binary>>) - args = [key, kind, under, var] - [{:{}, [], args}|module_vars(vars, counter+1)] - end + # defmodule Elixir.Alias + defp alias_defmodule({:__aliases__, _, [:"Elixir", _ | _]}, module, _env), + do: {module, module, nil} - defp module_vars([], _counter) do - [] - end + # defmodule Alias in root + defp alias_defmodule({:__aliases__, _, _}, module, %{module: nil}), + do: {module, module, nil} - # Gets two modules names and return an alias - # which can be passed down to the alias directive - # and it will create a proper shortcut representing - # the given nesting. - # - # Examples: - # - # module_nesting('Elixir.Foo.Bar', 'Elixir.Foo.Bar.Baz.Bat') - # {'Elixir.Baz', 'Elixir.Foo.Bar.Baz'} - # - # In case there is no nesting/no module: - # - # module_nesting(nil, 'Elixir.Foo.Bar.Baz.Bat') - # {false, 'Elixir.Foo.Bar.Baz.Bat'} - # - defp module_nesting(nil, full), - do: {false, full} + # defmodule Alias nested + defp alias_defmodule({:__aliases__, _, [h | t]}, _module, env) when is_atom(h) do + module = :elixir_aliases.concat([env.module, h]) + alias = String.to_atom("Elixir." <> Atom.to_string(h)) - defp module_nesting(prefix, full) do - case split_module(prefix) do - [] -> {false, full} - prefix -> module_nesting(prefix, split_module(full), [], full) + case t do + [] -> {module, module, alias} + _ -> {String.to_atom(Enum.join([module | t], ".")), module, alias} end end - defp module_nesting([x|t1], [x|t2], acc, full), - do: module_nesting(t1, t2, [x|acc], full) - defp module_nesting([], [h|_], acc, _full), - do: {String.to_atom(<<"Elixir.", h::binary>>), - :elixir_aliases.concat(:lists.reverse([h|acc]))} - defp module_nesting(_, _, _acc, full), - do: {false, full} - - defp split_module(atom) do - case :binary.split(Atom.to_string(atom), ".", [:global]) do - ["Elixir"|t] -> t - _ -> [] - end + # defmodule _ + defp alias_defmodule(_raw, module, _env) do + {module, module, nil} end - @doc """ - Defines a function with the given name and contents. + defp module_var({name, kind}) when is_atom(kind), do: {name, [generated: true], kind} + defp module_var({name, kind}), do: {name, [counter: kind, generated: true], nil} + + @doc ~S""" + Defines a public function with the given name and body. ## Examples @@ -2667,9 +4833,10 @@ defmodule Kernel do def bar, do: :baz end - Foo.bar #=> :baz + Foo.bar() + #=> :baz - A function that expects arguments can be defined as follow: + A function that expects arguments can be defined as follows: defmodule Foo do def sum(a, b) do @@ -2677,8 +4844,114 @@ defmodule Kernel do end end - In the example above, we defined a function `sum` that receives - two arguments and sums them. + In the example above, a `sum/2` function is defined; this function receives + two arguments and returns their sum. + + ## Default arguments + + `\\` is used to specify a default value for a parameter of a function. For + example: + + defmodule MyMath do + def multiply_by(number, factor \\ 2) do + number * factor + end + end + + MyMath.multiply_by(4, 3) + #=> 12 + + MyMath.multiply_by(4) + #=> 8 + + The compiler translates this into multiple functions with different arities, + here `MyMath.multiply_by/1` and `MyMath.multiply_by/2`, that represent cases when + arguments for parameters with default values are passed or not passed. + + When defining a function with default arguments as well as multiple + explicitly declared clauses, you must write a function head that declares the + defaults. For example: + + defmodule MyString do + def join(string1, string2 \\ nil, separator \\ " ") + + def join(string1, nil, _separator) do + string1 + end + + def join(string1, string2, separator) do + string1 <> separator <> string2 + end + end + + Note that `\\` can't be used with anonymous functions because they + can only have a sole arity. + + ### Keyword lists with default arguments + + Functions containing many arguments can benefit from using `Keyword` + lists to group and pass attributes as a single value. + + defmodule MyConfiguration do + @default_opts [storage: "local"] + + def configure(resource, opts \\ []) do + opts = Keyword.merge(@default_opts, opts) + storage = opts[:storage] + # ... + end + end + + The difference between using `Map` and `Keyword` to store many + arguments is `Keyword`'s keys: + + * must be atoms + * can be given more than once + * ordered, as specified by the developer + + ## Function and variable names + + Function and variable names have the following syntax: + A _lowercase ASCII letter_ or an _underscore_, followed by any number of + _lowercase or uppercase ASCII letters_, _numbers_, or _underscores_. + Optionally they can end in either an _exclamation mark_ or a _question mark_. + + For variables, any identifier starting with an underscore should indicate an + unused variable. For example: + + def foo(bar) do + [] + end + #=> warning: variable bar is unused + + def foo(_bar) do + [] + end + #=> no warning + + def foo(_bar) do + _bar + end + #=> warning: the underscored variable "_bar" is used after being set + + ## `rescue`/`catch`/`after`/`else` + + Function bodies support `rescue`, `catch`, `after`, and `else` as `Kernel.SpecialForms.try/1` + does (known as "implicit try"). For example, the following two functions are equivalent: + + def convert(number) do + try do + String.to_integer(number) + rescue + e in ArgumentError -> {:error, e.message} + end + end + + def convert(number) do + String.to_integer(number) + rescue + e in ArgumentError -> {:error, e.message} + end """ defmacro def(call, expr \\ nil) do @@ -2686,10 +4959,13 @@ defmodule Kernel do end @doc """ - Defines a function that is private. Private functions are - only accessible from within the module in which they are defined. + Defines a private function with the given name and body. + + Private functions are only accessible from within the module in which they are + defined. Trying to access a private function from outside the module it's + defined in results in an `UndefinedFunctionError` exception. - Check `def/2` for more information + Check `def/2` for more information. ## Examples @@ -2701,15 +4977,23 @@ defmodule Kernel do defp sum(a, b), do: a + b end - In the example above, `sum` is private and accessing it - through `Foo.sum` will raise an error. + Foo.bar() + #=> 3 + + Foo.sum(1, 2) + ** (UndefinedFunctionError) undefined function Foo.sum/2 + """ defmacro defp(call, expr \\ nil) do define(:defp, call, expr, __CALLER__) end @doc """ - Defines a macro with the given name and contents. + Defines a public macro with the given name and body. + + Macros must be defined before its usage. + + Check `def/2` for rules on naming and default arguments. ## Examples @@ -2722,8 +5006,9 @@ defmodule Kernel do end require MyLogic + MyLogic.unless false do - IO.puts "It works" + IO.puts("It works") end """ @@ -2732,48 +5017,67 @@ defmodule Kernel do end @doc """ - Defines a macro that is private. Private macros are - only accessible from the same module in which they are defined. + Defines a private macro with the given name and body. + + Private macros are only accessible from the same module in which they are + defined. + + Private macros must be defined before its usage. + + Check `defmacro/2` for more information, and check `def/2` for rules on + naming and default arguments. - Check `defmacro/2` for more information """ defmacro defmacrop(call, expr \\ nil) do define(:defmacrop, call, expr, __CALLER__) end defp define(kind, call, expr, env) do - assert_module_scope(env, kind, 2) + module = assert_module_scope(env, kind, 2) assert_no_function_scope(env, kind, 2) - line = env.line - {call, uc} = :elixir_quote.escape(call, true) - {expr, ue} = :elixir_quote.escape(expr, true) + unquoted_call = :elixir_quote.has_unquotes(call) + unquoted_expr = :elixir_quote.has_unquotes(expr) + escaped_call = :elixir_quote.escape(call, :none, true) + + escaped_expr = + case unquoted_expr do + true -> + :elixir_quote.escape(expr, :none, true) + + false -> + key = :erlang.unique_integer() + :elixir_module.write_cache(module, key, expr) + quote(do: :elixir_module.read_cache(unquote(module), unquote(key))) + end # Do not check clauses if any expression was unquoted - check_clauses = not(ue or uc) + check_clauses = not (unquoted_expr or unquoted_call) pos = :elixir_locals.cache_env(env) quote do - :elixir_def.store_definition(unquote(line), unquote(kind), unquote(check_clauses), - unquote(call), unquote(expr), unquote(pos)) + :elixir_def.store_definition( + unquote(kind), + unquote(check_clauses), + unquote(escaped_call), + unquote(escaped_expr), + unquote(pos) + ) end end @doc """ - Defines a struct for the current module. + Defines a struct. A struct is a tagged map that allows developers to provide default values for keys, tags to be used in polymorphic - dispatches and compile time assertions. - - To define a struct, a developer needs to only define - a function named `__struct__/0` that returns a map with the - structs field. This macro is a convenience for defining such - function, with the addition of a type `t` and deriving - conveniences. + dispatches and compile time assertions. For more information + about structs, please check `Kernel.SpecialForms.%/2`. - For more information about structs, please check - `Kernel.SpecialForms.%/2`. + It is only possible to define a struct per module, as the + struct it tied to the module itself. Calling `defstruct/1` + also defines a `__struct__/0` function that returns the + struct itself. ## Examples @@ -2781,421 +5085,485 @@ defmodule Kernel do defstruct name: nil, age: nil end - Struct fields are evaluated at definition time, which allows - them to be dynamic. In the example below, `10 + 11` will be - evaluated at compilation time and the age field will be stored + Struct fields are evaluated at compile-time, which allows + them to be dynamic. In the example below, `10 + 11` is + evaluated at compile-time and the age field is stored with value `21`: defmodule User do defstruct name: nil, age: 10 + 11 end + The `fields` argument is usually a keyword list with field names + as atom keys and default values as corresponding values. `defstruct/1` + also supports a list of atoms as its argument: in that case, the atoms + in the list will be used as the struct's field names and they will all + default to `nil`. + + defmodule Post do + defstruct [:title, :content, :author] + end + ## Deriving Although structs are maps, by default structs do not implement - any of the protocols implemented for maps. For example, if you - attempt to use the access protocol with the User struct, it - will lead to an error: + any of the protocols implemented for maps. For example, attempting + to use a protocol with the `User` struct leads to an error: - %User{}[:age] - ** (Protocol.UndefinedError) protocol Access not implemented for %User{...} + john = %User{name: "John"} + MyProtocol.call(john) + ** (Protocol.UndefinedError) protocol MyProtocol not implemented for %User{...} - However, `defstruct/2` allows implementation for protocols to - derived by defining a `@derive` attribute as a list before `defstruct/2` - is invoked: + `defstruct/1`, however, allows protocol implementations to be + *derived*. This can be done by defining a `@derive` attribute as a + list before invoking `defstruct/1`: defmodule User do - @derive [Access] - defstruct name: nil, age: 10 + 11 + @derive MyProtocol + defstruct name: nil, age: nil end - %User{}[:age] #=> 21 - - For each protocol given to `@derive`, Elixir will assert there is an - implementation of that protocol for maps and check if the map - implementation defines a `__deriving__/3` callback. If so, the callback - is invoked, otherwise an implementation that simply points to the map - one is automatically derived. + MyProtocol.call(john) # it works! - ## Types - - `defstruct` automatically generates a type `t` unless one exists. - The following definition: + A common example is to `@derive` the `Inspect` protocol to hide certain fields + when the struct is printed: defmodule User do - defstruct name: "José" :: String.t, - age: 25 :: integer + @derive {Inspect, only: :name} + defstruct name: nil, age: nil end - Generates a type as follows: - - @type t :: %User{name: String.t, age: integer} + For each protocol in `@derive`, Elixir will assert the protocol has + been implemented for `Any`. If the `Any` implementation defines a + `__deriving__/3` callback, the callback will be invoked and it should define + the implementation module. Otherwise an implementation that simply points to + the `Any` implementation is automatically derived. For more information on + the `__deriving__/3` callback, see `Protocol.derive/3`. - In case a struct does not declare a field type, it defaults to `term`. - """ - defmacro defstruct(kv) do - {fields, types} = split_fields_and_types(kv) + ## Enforcing keys - fields = - quote bind_quoted: [fields: fields] do - fields = :lists.map(fn - { key, _ } = pair when is_atom(key) -> pair - key when is_atom(key) -> { key, nil } - other -> raise ArgumentError, "struct field names must be atoms, got: #{inspect other}" - end, fields) + When building a struct, Elixir will automatically guarantee all keys + belongs to the struct: - @struct :maps.put(:__struct__, __MODULE__, :maps.from_list(fields)) + %User{name: "john", unknown: :key} + ** (KeyError) key :unknown not found in: %User{age: 21, name: nil} - case Module.get_attribute(__MODULE__, :derive) do - [] -> - :ok - derive -> - Protocol.__derive__(derive, __MODULE__, __ENV__) - end + Elixir also allows developers to enforce certain keys must always be + given when building the struct: - @spec __struct__() :: t - def __struct__() do - @struct - end + defmodule User do + @enforce_keys [:name] + defstruct name: nil, age: 10 + 11 end - types = - case bootstraped?(Kernel.Typespec) do - true when types == [] -> - quote unquote: false do - unless Kernel.Typespec.defines_type?(__MODULE__, :t, 0) do - types = :lists.map(fn {key, _} -> - {key, quote(do: term)} - end, fields) - @type t :: %{unquote_splicing(types), __struct__: __MODULE__} - end - end - true -> - quote do - unless Kernel.Typespec.defines_type?(__MODULE__, :t, 0) do - @type t :: %{unquote_splicing(types), __struct__: __MODULE__} - end - end - false -> - nil + Now trying to build a struct without the name key will fail: + + %User{age: 21} + ** (ArgumentError) the following keys must also be given when building struct User: [:name] + + Keep in mind `@enforce_keys` is a simple compile-time guarantee + to aid developers when building structs. It is not enforced on + updates and it does not provide any sort of value-validation. + + ## Types + + It is recommended to define types for structs. By convention such type + is called `t`. To define a struct inside a type, the struct literal syntax + is used: + + defmodule User do + defstruct name: "John", age: 25 + @type t :: %__MODULE__{name: String.t(), age: non_neg_integer} end - quote do - unquote(fields) - unquote(types) - fields - end - end + It is recommended to only use the struct syntax when defining the struct's + type. When referring to another struct it's better to use `User.t()` instead of + `%User{}`. - defp split_fields_and_types(kv) do - case Keyword.keyword?(kv) do - true -> split_fields_and_types(kv, [], []) - false -> {kv, []} - end - end + The types of the struct fields that are not included in `%User{}` default to + `term()` (see `t:term/0`). - defp split_fields_and_types([{field, {:::, _, [default, type]}}|t], fields, types) do - split_fields_and_types(t, [{field, default}|fields], [{field, type}|types]) - end + Structs whose internal structure is private to the local module (pattern + matching them or directly accessing their fields should not be allowed) should + use the `@opaque` attribute. Structs whose internal structure is public should + use `@type`. + """ + defmacro defstruct(fields) do + quote bind_quoted: [fields: fields, bootstrapped?: bootstrapped?(Enum)] do + {struct, derive, body} = Kernel.Utils.defstruct(__MODULE__, fields, bootstrapped?) - defp split_fields_and_types([{field, default}|t], fields, types) do - split_fields_and_types(t, [{field, default}|fields], [{field, quote(do: term)}|types]) - end + case derive do + [] -> :ok + _ -> Protocol.__derive__(derive, __MODULE__, __ENV__) + end - defp split_fields_and_types([field|t], fields, types) do - split_fields_and_types(t, [field|fields], [{field, quote(do: term)}|types]) - end + def __struct__(), do: @__struct__ + def __struct__(var!(kv)), do: unquote(body) - defp split_fields_and_types([], fields, types) do - {:lists.reverse(fields), :lists.reverse(types)} + Kernel.Utils.announce_struct(__MODULE__) + struct + end end @doc ~S""" Defines an exception. Exceptions are structs backed by a module that implements - the Exception behaviour. The Exception behaviour requires + the `Exception` behaviour. The `Exception` behaviour requires two functions to be implemented: - * `exception/1` - that receives the arguments given to `raise/2` - and returns the exception struct. The default implementation - accepts a set of keyword arguments that is merged into the - struct. + * [`exception/1`](`c:Exception.exception/1`) - receives the arguments given to `raise/2` + and returns the exception struct. The default implementation + accepts either a set of keyword arguments that is merged into + the struct or a string to be used as the exception's message. - * `message/1` - receives the exception struct and must return its + * [`message/1`](`c:Exception.message/1`) - receives the exception struct and must return its message. Most commonly exceptions have a message field which - by default is accessed by this function. However, if your exception + by default is accessed by this function. However, if an exception does not have a message field, this function must be explicitly implemented. - Since exceptions are structs, all the API supported by `defstruct/1` + Since exceptions are structs, the API supported by `defstruct/1` is also available in `defexception/1`. ## Raising exceptions - The most common way to raise an exception is via the `raise/2` - function: + The most common way to raise an exception is via `raise/2`: defmodule MyAppError do defexception [:message] end + value = [:hello] + raise MyAppError, - message: "did not get what was expected, got: #{inspect value}" + message: "did not get what was expected, got: #{inspect(value)}" In many cases it is more convenient to pass the expected value to - `raise` and generate the message in the `exception/1` callback: + `raise/2` and generate the message in the `c:Exception.exception/1` callback: defmodule MyAppError do defexception [:message] + @impl true def exception(value) do - msg = "did not get what was expected, got: #{inspect value}" + msg = "did not get what was expected, got: #{inspect(value)}" %MyAppError{message: msg} end end raise MyAppError, value - The example above is the preferred mechanism for customizing + The example above shows the preferred strategy for customizing exception messages. """ defmacro defexception(fields) do - fields = case is_list(fields) do - true -> [{:__exception__, true}|fields] - false -> quote(do: [{:__exception__, true}] ++ unquote(fields)) - end - - quote do + quote bind_quoted: [fields: fields] do @behaviour Exception - fields = defstruct unquote(fields) + struct = defstruct([__exception__: true] ++ fields) - @spec exception(term) :: t - def exception(args) when is_list(args) do - Kernel.struct(__struct__, args) - end - - defoverridable exception: 1 - - if Keyword.has_key?(fields, :message) do - @spec message(t) :: String.t + if Map.has_key?(struct, :message) do + @impl true def message(exception) do exception.message end defoverridable message: 1 + + @impl true + def exception(msg) when Kernel.is_binary(msg) do + exception(message: msg) + end + end + + # Calls to Kernel functions must be fully-qualified to ensure + # reproducible builds; otherwise, this macro will generate ASTs + # with different metadata (:import, :context) depending on if + # it is the bootstrapped version or not. + # TODO: Change the implementation on v2.0 to simply call Kernel.struct!/2 + @impl true + def exception(args) when Kernel.is_list(args) do + struct = __struct__() + {valid, invalid} = Enum.split_with(args, fn {k, _} -> Map.has_key?(struct, k) end) + + case invalid do + [] -> + :ok + + _ -> + IO.warn( + "the following fields are unknown when raising " <> + "#{Kernel.inspect(__MODULE__)}: #{Kernel.inspect(invalid)}. " <> + "Please make sure to only give known fields when raising " <> + "or redefine #{Kernel.inspect(__MODULE__)}.exception/1 to " <> + "discard unknown fields. Future Elixir versions will raise on " <> + "unknown fields given to raise/2" + ) + end + + Kernel.struct!(struct, valid) end + + defoverridable exception: 1 end end @doc """ Defines a protocol. - A protocol specifies an API that should be defined by its - implementations. - - ## Examples + See the `Protocol` module for more information. + """ + defmacro defprotocol(name, do_block) - In Elixir, only `false` and `nil` are considered falsy values. - Everything else evaluates to true in `if` clauses. Depending - on the application, it may be important to specify a `blank?` - protocol that returns a boolean for other data types that should - be considered `blank?`. For instance, an empty list or an empty - binary could be considered blanks. + defmacro defprotocol(name, do: block) do + Protocol.__protocol__(name, do: block) + end - We could implement this protocol as follow: + @doc """ + Defines an implementation for the given protocol. - defprotocol Blank do - @doc "Returns true if data is considered blank/empty" - def blank?(data) - end + See the `Protocol` module for more information. + """ + defmacro defimpl(name, opts, do_block \\ []) do + merged = Keyword.merge(opts, do_block) + merged = Keyword.put_new(merged, :for, __CALLER__.module) - Now that the protocol is defined, we can implement it. We need - to implement the protocol for each Elixir type. For example: + if Keyword.fetch!(merged, :for) == nil do + raise ArgumentError, "defimpl/3 expects a :for option when declared outside a module" + end - # Integers are never blank - defimpl Blank, for: Integer do - def blank?(number), do: false - end + Protocol.__impl__(name, merged) + end - # Just empty list is blank - defimpl Blank, for: List do - def blank?([]), do: true - def blank?(_), do: false - end + @doc """ + Makes the given definitions in the current module overridable. - # Just the atoms false and nil are blank - defimpl Blank, for: Atom do - def blank?(false), do: true - def blank?(nil), do: true - def blank?(_), do: false - end + If the user defines a new function or macro with the same name + and arity, then the overridable ones are discarded. Otherwise, the + original definitions are used. - And we would have to define the implementation for all types. - The supported types available are: + It is possible for the overridden definition to have a different visibility + than the original: a public function can be overridden by a private + function and vice-versa. - * Structs (see below) - * `Tuple` - * `Atom` - * `List` - * `BitString` - * `Integer` - * `Float` - * `Function` - * `PID` - * `Map` - * `Port` - * `Reference` - * `Any` (see below) + Macros cannot be overridden as functions and vice-versa. - ## Protocols + Structs + ## Example - The real benefit of protocols comes when mixed with structs. - For instance, Elixir ships with many data types implemented as - structs, like `HashDict` and `HashSet`. We can implement the - `Blank` protocol for those types as well: + defmodule DefaultMod do + defmacro __using__(_opts) do + quote do + def test(x, y) do + x + y + end - defimpl Blank, for: [HashDict, HashSet] do - def blank?(enum_like), do: Enum.empty?(enum_like) + defoverridable test: 2 + end + end end - If a protocol is not found for a given type, it will fallback to - `Any`. - - ## Fallback to any - - In some cases, it may be convenient to provide a default - implementation for all types. This can be achieved by - setting `@fallback_to_any` to `true` in the protocol - definition: + defmodule ChildMod do + use DefaultMod - defprotocol Blank do - @fallback_to_any true - def blank?(data) + def test(x, y) do + x * y + super(x, y) + end end - Which can now be implemented as: + As seen as in the example above, `super` can be used to call the default + implementation. - defimpl Blank, for: Any do - def blank?(_), do: true + > Note: use `defoverridable` with care. If you need to define multiple modules + > with the same behaviour, it may be best to move the default implementation + > to the caller, and check if a callback exists via `Code.ensure_loaded?/1` and + > `function_exported?/3`. + > + > For example, in the example above, imagine there is a module that calls the + > `test/2` function. This module could be defined as such: + > + > defmodule CallsTest do + > def receives_module_and_calls_test(module, x, y) do + > if Code.ensure_loaded?(module) and function_exported?(module, :test, 2) do + > module.test(x, y) + > else + > x + y + > end + > end + > end + + ## Example with behaviour + + You can also pass a behaviour to `defoverridable` and it will mark all of the + callbacks in the behaviour as overridable: + + + defmodule Behaviour do + @callback test(number(), number()) :: number() end - One may wonder why such fallback is not true by default. - - It is two-fold: first, the majority of protocols cannot - implement an action in a generic way for all types. In fact, - providing a default implementation may be harmful, because users - may rely on the default implementation instead of providing a - specialized one. + defmodule DefaultMod do + defmacro __using__(_opts) do + quote do + @behaviour Behaviour - Second, falling back to `Any` adds an extra lookup to all types, - which is unnecessary overhead unless an implementation for Any is - required. + def test(x, y) do + x + y + end - ## Types + defoverridable Behaviour + end + end + end - Defining a protocol automatically defines a type named `t`, which - can be used as: + defmodule ChildMod do + use DefaultMod - @spec present?(Blank.t) :: boolean - def present?(blank) do - not Blank.blank?(blank) + def test(x, y) do + x * y + super(x, y) + end end - The `@spec` above expresses that all types allowed to implement the - given protocol are valid argument types for the given function. - - ## Reflection + """ + defmacro defoverridable(keywords_or_behaviour) do + quote do + Module.make_overridable(__MODULE__, unquote(keywords_or_behaviour)) + end + end - Any protocol module contains three extra functions: + @doc """ + Generates a macro suitable for use in guard expressions. + It raises at compile time if the definition uses expressions that aren't + allowed in guards, and otherwise creates a macro that can be used both inside + or outside guards. - * `__protocol__/1` - returns the protocol name when `:name` is given, and a - keyword list with the protocol functions when `:functions` is given + Note the convention in Elixir is to name functions/macros allowed in + guards with the `is_` prefix, such as `is_list/1`. If, however, the + function/macro returns a boolean and is not allowed in guards, it should + have no prefix and end with a question mark, such as `Keyword.keyword?/1`. - * `impl_for/1` - receives a structure and returns the module that - implements the protocol for the structure, `nil` otherwise + ## Example - * `impl_for!/1` - same as above but raises an error if an implementation is - not found + defmodule Integer.Guards do + defguard is_even(value) when is_integer(value) and rem(value, 2) == 0 + end - ## Consolidation + defmodule Collatz do + @moduledoc "Tools for working with the Collatz sequence." + import Integer.Guards - In order to cope with code loading in development, protocols in - Elixir provide a slow implementation of protocol dispatching specific - to development. + @doc "Determines the number of steps `n` takes to reach `1`." + # If this function never converges, please let me know what `n` you used. + def converge(n) when n > 0, do: step(n, 0) - In order to speed up dispatching in production environments, where - all implementations are known up-front, Elixir provides a feature - called protocol consolidation. For this reason, all protocols are - compiled with `debug_info` set to true, regardless of the option - set by `elixirc` compiler. The debug info though may be removed - after consolidation. + defp step(1, step_count) do + step_count + end - For more information on how to apply protocol consolidation to - a given project, please check the functions in the `Protocol` - module or the `mix compile.protocols` task. - """ - defmacro defprotocol(name, do: block) do - Protocol.__protocol__(name, do: block) - end + defp step(n, step_count) when is_even(n) do + step(div(n, 2), step_count + 1) + end - @doc """ - Defines an implementation for the given protocol. See - `defprotocol/2` for examples. + defp step(n, step_count) do + step(3 * n + 1, step_count + 1) + end + end - Inside an implementation, the name of the protocol can be accessed - via `@protocol` and the current target as `@for`. - """ - defmacro defimpl(name, opts, do_block \\ []) do - merged = Keyword.merge(opts, do_block) - merged = Keyword.put_new(merged, :for, __CALLER__.module) - Protocol.__impl__(name, merged) + """ + @doc since: "1.6.0" + @spec defguard(Macro.t()) :: Macro.t() + defmacro defguard(guard) do + define_guard(:defmacro, guard, __CALLER__) end @doc """ - Makes the given functions in the current module overridable. An overridable - function is lazily defined, allowing a developer to customize it. + Generates a private macro suitable for use in guard expressions. - ## Example + It raises at compile time if the definition uses expressions that aren't + allowed in guards, and otherwise creates a private macro that can be used + both inside or outside guards in the current module. - defmodule DefaultMod do - defmacro __using__(_opts) do - quote do - def test(x, y) do - x + y + Similar to `defmacrop/2`, `defguardp/1` must be defined before its use + in the current module. + """ + @doc since: "1.6.0" + @spec defguardp(Macro.t()) :: Macro.t() + defmacro defguardp(guard) do + define_guard(:defmacrop, guard, __CALLER__) + end + + defp define_guard(kind, guard, env) do + case :elixir_utils.extract_guards(guard) do + {call, [_, _ | _]} -> + raise ArgumentError, + "invalid syntax in defguard #{Macro.to_string(call)}, " <> + "only a single when clause is allowed" + + {call, impls} -> + case Macro.decompose_call(call) do + {_name, args} -> + validate_variable_only_args!(call, args) + + macro_definition = + case impls do + [] -> + define(kind, call, nil, env) + + [guard] -> + quoted = + quote do + require Kernel.Utils + Kernel.Utils.defguard(unquote(args), unquote(guard)) + end + + define(kind, call, [do: quoted], env) + end + + quote do + @doc guard: true + unquote(macro_definition) end - defoverridable [test: 2] - end + _invalid_definition -> + raise ArgumentError, "invalid syntax in defguard #{Macro.to_string(call)}" end - end + end + end - defmodule InheritMod do - use DefaultMod + defp validate_variable_only_args!(call, args) do + Enum.each(args, fn + {ref, _meta, context} when is_atom(ref) and is_atom(context) -> + :ok - def test(x, y) do - x * y + super(x, y) - end - end + {:\\, _m1, [{ref, _m2, context}, _default]} when is_atom(ref) and is_atom(context) -> + :ok - As seen as in the example `super` can be used to call the default - implementation. - """ - defmacro defoverridable(tuples) do - quote do - Module.make_overridable(__MODULE__, unquote(tuples)) - end + _match -> + raise ArgumentError, "invalid syntax in defguard #{Macro.to_string(call)}" + end) end @doc """ - `use` is a simple mechanism for using a given module into - the current context. + Uses the given module in the current context. + + When calling: + + use MyModule, some: :options + + Elixir will invoke `MyModule.__using__/1` passing the second argument of + `use` as its argument. Since `__using__/1` is typically a macro, all + the usual macro rules apply, and its return value should be quoted code + that is then inserted where `use/2` is called. + + > Note: `use MyModule` works as a code injection point in the caller. + > Given the caller of `use MyModule` has little control over how the + > code is injected, `use/2` should be used with care. If you can, + > avoid use in favor of `import/2` or `alias/2` whenever possible. ## Examples - For example, in order to write tests using the ExUnit framework, - a developer should use the `ExUnit.Case` module: + For example, to write test cases using the `ExUnit` framework provided + with Elixir, a developer should `use` the `ExUnit.Case` module: defmodule AssertionTest do use ExUnit.Case, async: true @@ -3205,103 +5573,174 @@ defmodule Kernel do end end - By calling `use`, a hook called `__using__` will be invoked in - `ExUnit.Case` which will then do the proper setup. + In this example, Elixir will call the `__using__/1` macro in the + `ExUnit.Case` module with the keyword list `[async: true]` as its + argument. - Simply put, `use` is simply a translation to: + In other words, `use/2` translates to: defmodule AssertionTest do require ExUnit.Case - ExUnit.Case.__using__([async: true]) + ExUnit.Case.__using__(async: true) test "always pass" do assert true end end + where `ExUnit.Case` defines the `__using__/1` macro: + + defmodule ExUnit.Case do + defmacro __using__(opts) do + # do something with opts + quote do + # return some code to inject in the caller + end + end + end + + ## Best practices + + `__using__/1` is typically used when there is a need to set some state + (via module attributes) or callbacks (like `@before_compile`, see the + documentation for `Module` for more information) into the caller. + + `__using__/1` may also be used to alias, require, or import functionality + from different modules: + + defmodule MyModule do + defmacro __using__(_opts) do + quote do + import MyModule.Foo + import MyModule.Bar + import MyModule.Baz + + alias MyModule.Repo + end + end + end + + However, do not provide `__using__/1` if all it does is to import, + alias or require the module itself. For example, avoid this: + + defmodule MyModule do + defmacro __using__(_opts) do + quote do + import MyModule + end + end + end + + In such cases, developers should instead import or alias the module + directly, so that they can customize those as they wish, + without the indirection behind `use/2`. + + Finally, developers should also avoid defining functions inside + the `__using__/1` callback, unless those functions are the default + implementation of a previously defined `@callback` or are functions + meant to be overridden (see `defoverridable/1`). Even in these cases, + defining functions should be seen as a "last resort". """ defmacro use(module, opts \\ []) do - expanded = Macro.expand(module, __CALLER__) + calls = + Enum.map(expand_aliases(module, __CALLER__), fn + expanded when is_atom(expanded) -> + quote do + require unquote(expanded) + unquote(expanded).__using__(unquote(opts)) + end - case is_atom(expanded) do - false -> - raise ArgumentError, "invalid arguments for use, expected an atom or alias as argument" - true -> - quote do - require unquote(expanded) - unquote(expanded).__using__(unquote(opts)) - end - end + _otherwise -> + raise ArgumentError, + "invalid arguments for use, " <> + "expected a compile time atom or alias, got: #{Macro.to_string(module)}" + end) + + quote(do: (unquote_splicing(calls))) + end + + defp expand_aliases({{:., _, [base, :{}]}, _, refs}, env) do + base = Macro.expand(base, env) + + Enum.map(refs, fn + {:__aliases__, _, ref} -> + Module.concat([base | ref]) + + ref when is_atom(ref) -> + Module.concat(base, ref) + + other -> + other + end) + end + + defp expand_aliases(module, env) do + [Macro.expand(module, env)] end @doc """ - Defines the given functions in the current module that will - delegate to the given `target`. Functions defined with - `defdelegate` are public and are allowed to be invoked - from external. If you find yourself wishing to define a - delegation as private, you should likely use import - instead. + Defines a function that delegates to another module. + + Functions defined with `defdelegate/2` are public and can be invoked from + outside the module they're defined in, as if they were defined using `def/2`. + Therefore, `defdelegate/2` is about extending the current module's public API. + If what you want is to invoke a function defined in another module without + using its full module name, then use `alias/2` to shorten the module name or use + `import/2` to be able to invoke the function without the module name altogether. - Delegation only works with functions, delegating to macros - is not supported. + Delegation only works with functions; delegating macros is not supported. + + Check `def/2` for rules on naming and default arguments. ## Options - * `:to` - the expression to delegate to. Any expression - is allowed and its results will be calculated on runtime. + * `:to` - the module to dispatch to. * `:as` - the function to call on the target given in `:to`. This parameter is optional and defaults to the name being - delegated. - - * `:append_first` - if true, when delegated, first argument - passed to the delegate will be relocated to the end of the - arguments when dispatched to the target. - - The motivation behind this is because Elixir normalizes - the "handle" as a first argument and some Erlang modules - expect it as last argument. + delegated (`funs`). ## Examples defmodule MyList do - defdelegate reverse(list), to: :lists - defdelegate [reverse(list), map(callback, list)], to: :lists - defdelegate other_reverse(list), to: :lists, as: :reverse + defdelegate reverse(list), to: Enum + defdelegate other_reverse(list), to: Enum, as: :reverse end MyList.reverse([1, 2, 3]) - #=> [3,2,1] + #=> [3, 2, 1] MyList.other_reverse([1, 2, 3]) - #=> [3,2,1] + #=> [3, 2, 1] """ defmacro defdelegate(funs, opts) do funs = Macro.escape(funs, unquote: true) - quote bind_quoted: [funs: funs, opts: opts] do - target = Keyword.get(opts, :to) || - raise ArgumentError, "Expected to: to be given as argument" - append_first = Keyword.get(opts, :append_first, false) - - for fun <- List.wrap(funs) do - {name, args} = - case Macro.decompose_call(fun) do - {_, _} = pair -> pair - _ -> raise ArgumentError, "invalid syntax in defdelegate #{Macro.to_string(fun)}" - end + # don't add compile-time dependency on :to + opts = + with true <- is_list(opts), + {:ok, target} <- Keyword.fetch(opts, :to), + {:__aliases__, _, _} <- target do + target = Macro.expand(target, %{__CALLER__ | function: {:__info__, 1}}) + Keyword.replace!(opts, :to, target) + else + _ -> + opts + end - actual_args = - case append_first and args != [] do - true -> tl(args) ++ [hd(args)] - false -> args - end + quote bind_quoted: [funs: funs, opts: opts] do + target = Kernel.Utils.defdelegate_all(funs, opts, __ENV__) - fun = Keyword.get(opts, :as, name) + # TODO: Remove List.wrap when multiple funs are no longer supported + for fun <- List.wrap(funs) do + {name, args, as, as_args} = Kernel.Utils.defdelegate_each(fun, opts) + @doc delegate_to: {target, as, :erlang.length(as_args)} - def unquote(name)(unquote_splicing(args)) do - unquote(target).unquote(fun)(unquote_splicing(actual_args)) + # Build the call AST by hand so it doesn't get a + # context and it warns on things like missing @impl + def unquote({name, [line: __ENV__.line], args}) do + unquote(target).unquote(as)(unquote_splicing(as_args)) end end end @@ -3309,129 +5748,425 @@ defmodule Kernel do ## Sigils - @doc """ - Handles the sigil ~S. It simply returns a string - without escaping characters and without interpolations. + @doc ~S""" + Handles the sigil `~S` for strings. + + It returns a string without interpolations and without escape + characters, except for the escaping of the closing sigil character + itself. ## Examples iex> ~S(foo) "foo" + iex> ~S(f#{o}o) + "f\#{o}o" + iex> ~S(\o/) + "\\o/" - iex> ~S(f\#{o}o) - "f\\\#{o}o" + However, if you want to re-use the sigil character itself on + the string, you need to escape it: + + iex> ~S((\)) + "()" """ - defmacro sigil_S(string, []) do - string - end + defmacro sigil_S(term, modifiers) + defmacro sigil_S({:<<>>, _, [binary]}, []) when is_binary(binary), do: binary - @doc """ - Handles the sigil ~s. It returns a string as if it was double quoted - string, unescaping characters and replacing interpolations. + @doc ~S""" + Handles the sigil `~s` for strings. + + It returns a string as if it was a double quoted string, unescaping characters + and replacing interpolations. ## Examples iex> ~s(foo) "foo" - iex> ~s(f\#{:o}o) + iex> ~s(f#{:o}o) "foo" + iex> ~s(f\#{:o}o) + "f\#{:o}o" + """ + defmacro sigil_s(term, modifiers) + + defmacro sigil_s({:<<>>, _, [piece]}, []) when is_binary(piece) do + :elixir_interpolation.unescape_string(piece) + end + defmacro sigil_s({:<<>>, line, pieces}, []) do - {:<<>>, line, Macro.unescape_tokens(pieces)} + {:<<>>, line, unescape_tokens(pieces)} end - @doc """ - Handles the sigil ~C. It simply returns a char list - without escaping characters and without interpolations. + @doc ~S""" + Handles the sigil `~C` for charlists. + + It returns a charlist without interpolations and without escape + characters, except for the escaping of the closing sigil character + itself. ## Examples iex> ~C(foo) 'foo' - iex> ~C(f\#{o}o) - 'f\\\#{o}o' + iex> ~C(f#{o}o) + 'f\#{o}o' """ - defmacro sigil_C({:<<>>, _line, [string]}, []) when is_binary(string) do - String.to_char_list(string) + defmacro sigil_C(term, modifiers) + + defmacro sigil_C({:<<>>, _meta, [string]}, []) when is_binary(string) do + String.to_charlist(string) end - @doc """ - Handles the sigil ~c. It returns a char list as if it were a single - quoted string, unescaping characters and replacing interpolations. + @doc ~S""" + Handles the sigil `~c` for charlists. + + It returns a charlist as if it was a single quoted string, unescaping + characters and replacing interpolations. ## Examples iex> ~c(foo) 'foo' - iex> ~c(f\#{:o}o) + iex> ~c(f#{:o}o) 'foo' + iex> ~c(f\#{:o}o) + 'f\#{:o}o' + """ + defmacro sigil_c(term, modifiers) # We can skip the runtime conversion if we are # creating a binary made solely of series of chars. - defmacro sigil_c({:<<>>, _line, [string]}, []) when is_binary(string) do - String.to_char_list(Macro.unescape_string(string)) + defmacro sigil_c({:<<>>, _meta, [string]}, []) when is_binary(string) do + String.to_charlist(:elixir_interpolation.unescape_string(string)) end - defmacro sigil_c({:<<>>, line, pieces}, []) do - binary = {:<<>>, line, Macro.unescape_tokens(pieces)} - quote do: String.to_char_list(unquote(binary)) + defmacro sigil_c({:<<>>, _meta, pieces}, []) do + quote(do: List.to_charlist(unquote(unescape_list_tokens(pieces)))) end - @doc """ - Handles the sigil ~r. It returns a Regex pattern. + @doc ~S""" + Handles the sigil `~r` for regular expressions. + + It returns a regular expression pattern, unescaping characters and replacing + interpolations. + + More information on regular expressions can be found in the `Regex` module. ## Examples - iex> Regex.match?(~r(foo), "foo") + iex> Regex.match?(~r/foo/, "foo") + true + + iex> Regex.match?(~r/a#{:b}c/, "abc") true + While the `~r` sigil allows parens and brackets to be used as delimiters, + it is preferred to use `"` or `/` to avoid escaping conflicts with reserved + regex characters. """ - defmacro sigil_r({:<<>>, _line, [string]}, options) when is_binary(string) do - binary = Macro.unescape_string(string, fn(x) -> Regex.unescape_map(x) end) - regex = Regex.compile!(binary, :binary.list_to_bin(options)) + defmacro sigil_r(term, modifiers) + + defmacro sigil_r({:<<>>, _meta, [string]}, options) when is_binary(string) do + binary = :elixir_interpolation.unescape_string(string, &Regex.unescape_map/1) + regex = Regex.compile!(binary, :binary.list_to_bin(options)) Macro.escape(regex) end - defmacro sigil_r({:<<>>, line, pieces}, options) do - binary = {:<<>>, line, Macro.unescape_tokens(pieces, fn(x) -> Regex.unescape_map(x) end)} - quote do: Regex.compile!(unquote(binary), unquote(:binary.list_to_bin(options))) + defmacro sigil_r({:<<>>, meta, pieces}, options) do + binary = {:<<>>, meta, unescape_tokens(pieces, &Regex.unescape_map/1)} + quote(do: Regex.compile!(unquote(binary), unquote(:binary.list_to_bin(options)))) end - @doc """ - Handles the sigil ~R. It returns a Regex pattern without escaping - nor interpreting interpolations. + @doc ~S""" + Handles the sigil `~R` for regular expressions. + + It returns a regular expression pattern without interpolations and + without escape characters. Note it still supports escape of Regex + tokens (such as escaping `+` or `?`) and it also requires you to + escape the closing sigil character itself if it appears on the Regex. + + More information on regexes can be found in the `Regex` module. ## Examples - iex> Regex.match?(~R(f\#{1,3}o), "f\#o") + iex> Regex.match?(~R(f#{1,3}o), "f#o") true """ - defmacro sigil_R({:<<>>, _line, [string]}, options) when is_binary(string) do + defmacro sigil_R(term, modifiers) + + defmacro sigil_R({:<<>>, _meta, [string]}, options) when is_binary(string) do regex = Regex.compile!(string, :binary.list_to_bin(options)) Macro.escape(regex) end - @doc """ - Handles the sigil ~w. It returns a list of "words" split by whitespace. + @doc ~S""" + Handles the sigil `~D` for dates. + + By default, this sigil uses the built-in `Calendar.ISO`, which + requires dates to be written in the ISO8601 format: + + ~D[yyyy-mm-dd] + + such as: + + ~D[2015-01-13] + + If you are using alternative calendars, any representation can + be used as long as you follow the representation by a single space + and the calendar name: + + ~D[SOME-REPRESENTATION My.Alternative.Calendar] + + The lower case `~d` variant does not exist as interpolation + and escape characters are not useful for date sigils. + + More information on dates can be found in the `Date` module. + + ## Examples + + iex> ~D[2015-01-13] + ~D[2015-01-13] + + """ + defmacro sigil_D(date_string, modifiers) + + defmacro sigil_D({:<<>>, _, [string]}, []) do + {{:ok, {year, month, day}}, calendar} = parse_with_calendar!(string, :parse_date, "Date") + to_calendar_struct(Date, calendar: calendar, year: year, month: month, day: day) + end + + @doc ~S""" + Handles the sigil `~T` for times. + + By default, this sigil uses the built-in `Calendar.ISO`, which + requires times to be written in the ISO8601 format: + + ~T[hh:mm:ss] + ~T[hh:mm:ss.ssssss] + + such as: + + ~T[13:00:07] + ~T[13:00:07.123] + + If you are using alternative calendars, any representation can + be used as long as you follow the representation by a single space + and the calendar name: + + ~T[SOME-REPRESENTATION My.Alternative.Calendar] + + The lower case `~t` variant does not exist as interpolation + and escape characters are not useful for time sigils. + + More information on times can be found in the `Time` module. + + ## Examples + + iex> ~T[13:00:07] + ~T[13:00:07] + iex> ~T[13:00:07.001] + ~T[13:00:07.001] + + """ + defmacro sigil_T(time_string, modifiers) + + defmacro sigil_T({:<<>>, _, [string]}, []) do + {{:ok, {hour, minute, second, microsecond}}, calendar} = + parse_with_calendar!(string, :parse_time, "Time") + + to_calendar_struct(Time, + calendar: calendar, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + ) + end + + @doc ~S""" + Handles the sigil `~N` for naive date times. + + By default, this sigil uses the built-in `Calendar.ISO`, which + requires naive date times to be written in the ISO8601 format: + + ~N[yyyy-mm-dd hh:mm:ss] + ~N[yyyy-mm-dd hh:mm:ss.ssssss] + ~N[yyyy-mm-ddThh:mm:ss.ssssss] + + such as: + + ~N[2015-01-13 13:00:07] + ~N[2015-01-13T13:00:07.123] + + If you are using alternative calendars, any representation can + be used as long as you follow the representation by a single space + and the calendar name: + + ~N[SOME-REPRESENTATION My.Alternative.Calendar] + + The lower case `~n` variant does not exist as interpolation + and escape characters are not useful for date time sigils. + + More information on naive date times can be found in the + `NaiveDateTime` module. + + ## Examples + + iex> ~N[2015-01-13 13:00:07] + ~N[2015-01-13 13:00:07] + iex> ~N[2015-01-13T13:00:07.001] + ~N[2015-01-13 13:00:07.001] + + """ + defmacro sigil_N(naive_datetime_string, modifiers) + + defmacro sigil_N({:<<>>, _, [string]}, []) do + {{:ok, {year, month, day, hour, minute, second, microsecond}}, calendar} = + parse_with_calendar!(string, :parse_naive_datetime, "NaiveDateTime") + + to_calendar_struct(NaiveDateTime, + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond + ) + end + + @doc ~S""" + Handles the sigil `~U` to create a UTC `DateTime`. + + By default, this sigil uses the built-in `Calendar.ISO`, which + requires UTC date times to be written in the ISO8601 format: + + ~U[yyyy-mm-dd hh:mm:ssZ] + ~U[yyyy-mm-dd hh:mm:ss.ssssssZ] + ~U[yyyy-mm-ddThh:mm:ss.ssssss+00:00] + + such as: + + ~U[2015-01-13 13:00:07Z] + ~U[2015-01-13T13:00:07.123+00:00] + + If you are using alternative calendars, any representation can + be used as long as you follow the representation by a single space + and the calendar name: + + ~U[SOME-REPRESENTATION My.Alternative.Calendar] + + The given `datetime_string` must include "Z" or "00:00" offset + which marks it as UTC, otherwise an error is raised. + + The lower case `~u` variant does not exist as interpolation + and escape characters are not useful for date time sigils. + + More information on date times can be found in the `DateTime` module. + + ## Examples + + iex> ~U[2015-01-13 13:00:07Z] + ~U[2015-01-13 13:00:07Z] + iex> ~U[2015-01-13T13:00:07.001+00:00] + ~U[2015-01-13 13:00:07.001Z] + + """ + @doc since: "1.9.0" + defmacro sigil_U(datetime_string, modifiers) + + defmacro sigil_U({:<<>>, _, [string]}, []) do + {{:ok, {year, month, day, hour, minute, second, microsecond}, offset}, calendar} = + parse_with_calendar!(string, :parse_utc_datetime, "UTC DateTime") + + if offset != 0 do + raise ArgumentError, + "cannot parse #{inspect(string)} as UTC DateTime for #{inspect(calendar)}, reason: :non_utc_offset" + end + + to_calendar_struct(DateTime, + calendar: calendar, + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + time_zone: "Etc/UTC", + zone_abbr: "UTC", + utc_offset: 0, + std_offset: 0 + ) + end + + defp parse_with_calendar!(string, fun, context) do + {calendar, string} = extract_calendar(string) + result = apply(calendar, fun, [string]) + {maybe_raise!(result, calendar, context, string), calendar} + end + + defp extract_calendar(string) do + case :binary.split(string, " ", [:global]) do + [_] -> {Calendar.ISO, string} + parts -> maybe_atomize_calendar(List.last(parts), string) + end + end + + defp maybe_atomize_calendar(<> = last_part, string) + when alias >= ?A and alias <= ?Z do + string = binary_part(string, 0, byte_size(string) - byte_size(last_part) - 1) + {String.to_atom("Elixir." <> last_part), string} + end + + defp maybe_atomize_calendar(_last_part, string) do + {Calendar.ISO, string} + end + + defp maybe_raise!({:error, reason}, calendar, type, string) do + raise ArgumentError, + "cannot parse #{inspect(string)} as #{type} for #{inspect(calendar)}, " <> + "reason: #{inspect(reason)}" + end + + defp maybe_raise!(other, _calendar, _type, _string), do: other + + defp to_calendar_struct(type, fields) do + quote do + %{unquote_splicing([__struct__: type] ++ fields)} + end + end + + @doc ~S""" + Handles the sigil `~w` for list of words. + + It returns a list of "words" split by whitespace. Character unescaping and + interpolation happens for each word. ## Modifiers - * `s`: strings (default) - * `a`: atoms - * `c`: char lists + * `s`: words in the list are strings (default) + * `a`: words in the list are atoms + * `c`: words in the list are charlists ## Examples - iex> ~w(foo \#{:bar} baz) + iex> ~w(foo #{:bar} baz) + ["foo", "bar", "baz"] + + iex> ~w(foo #{" bar baz "}) ["foo", "bar", "baz"] iex> ~w(--source test/enum_test.exs) @@ -3440,96 +6175,169 @@ defmodule Kernel do iex> ~w(foo bar baz)a [:foo, :bar, :baz] + iex> ~w(foo bar baz)c + ['foo', 'bar', 'baz'] + """ + defmacro sigil_w(term, modifiers) - defmacro sigil_w({:<<>>, _line, [string]}, modifiers) when is_binary(string) do - split_words(Macro.unescape_string(string), modifiers) + defmacro sigil_w({:<<>>, _meta, [string]}, modifiers) when is_binary(string) do + split_words(:elixir_interpolation.unescape_string(string), modifiers, __CALLER__) end - defmacro sigil_w({:<<>>, line, pieces}, modifiers) do - binary = {:<<>>, line, Macro.unescape_tokens(pieces)} - split_words(binary, modifiers) + defmacro sigil_w({:<<>>, meta, pieces}, modifiers) do + binary = {:<<>>, meta, unescape_tokens(pieces)} + split_words(binary, modifiers, __CALLER__) end - @doc """ - Handles the sigil ~W. It returns a list of "words" split by whitespace - without escaping nor interpreting interpolations. + @doc ~S""" + Handles the sigil `~W` for list of words. + + It returns a list of "words" split by whitespace without interpolations + and without escape characters, except for the escaping of the closing + sigil character itself. ## Modifiers - * `s`: strings (default) - * `a`: atoms - * `c`: char lists + * `s`: words in the list are strings (default) + * `a`: words in the list are atoms + * `c`: words in the list are charlists ## Examples - iex> ~W(foo \#{bar} baz) - ["foo", "\\\#{bar}", "baz"] + iex> ~W(foo #{bar} baz) + ["foo", "\#{bar}", "baz"] """ - defmacro sigil_W({:<<>>, _line, [string]}, modifiers) when is_binary(string) do - split_words(string, modifiers) - end + defmacro sigil_W(term, modifiers) - defp split_words("", _modifiers), do: [] + defmacro sigil_W({:<<>>, _meta, [string]}, modifiers) when is_binary(string) do + split_words(string, modifiers, __CALLER__) + end - defp split_words(string, modifiers) do - mod = - case modifiers do - [] -> ?s - [mod] when mod == ?s or mod == ?a or mod == ?c -> mod - _else -> raise ArgumentError, "modifier must be one of: s, a, c" - end + defp split_words(string, [], caller) do + split_words(string, [?s], caller) + end + defp split_words(string, [mod], caller) + when mod == ?s or mod == ?a or mod == ?c do case is_binary(string) do true -> + parts = String.split(string) + + parts_with_trailing_comma = + :lists.filter(&(byte_size(&1) > 1 and :binary.last(&1) == ?,), parts) + + if parts_with_trailing_comma != [] do + stacktrace = Macro.Env.stacktrace(caller) + + IO.warn( + "the sigils ~w/~W do not allow trailing commas at the end of each word. " <> + "If the comma is necessary, define a regular list with [...], otherwise remove the comma.", + stacktrace + ) + end + case mod do - ?s -> String.split(string) - ?a -> for p <- String.split(string), do: String.to_atom(p) - ?c -> for p <- String.split(string), do: String.to_char_list(p) + ?s -> parts + ?a -> :lists.map(&String.to_atom/1, parts) + ?c -> :lists.map(&String.to_charlist/1, parts) end + false -> + parts = quote(do: String.split(unquote(string))) + case mod do - ?s -> quote do: String.split(unquote(string)) - ?a -> quote do: for(p <- String.split(unquote(string)), do: String.to_atom(p)) - ?c -> quote do: for(p <- String.split(unquote(string)), do: String.to_char_list(p)) + ?s -> parts + ?a -> quote(do: :lists.map(&String.to_atom/1, unquote(parts))) + ?c -> quote(do: :lists.map(&String.to_charlist/1, unquote(parts))) end end end - ## Shared functions - - defp optimize_boolean({:case, meta, args}) do - {:case, [{:optimize_boolean, true}|meta], args} + defp split_words(_string, _mods, _caller) do + raise ArgumentError, "modifier must be one of: s, a, c" end - # We need this check only for bootstrap purposes. - # Once Kernel is loaded and we recompile, it is a no-op. - case :code.ensure_loaded(Kernel) do - {:module, _} -> - defp bootstraped?(_), do: true - {:error, _} -> - defp bootstraped?(module), do: :code.ensure_loaded(module) == {:module, module} - end + ## Shared functions defp assert_module_scope(env, fun, arity) do case env.module do nil -> raise ArgumentError, "cannot invoke #{fun}/#{arity} outside module" - _ -> :ok + mod -> mod end end defp assert_no_function_scope(env, fun, arity) do case env.function do nil -> :ok - _ -> raise ArgumentError, "cannot invoke #{fun}/#{arity} inside function/macro" + _ -> raise ArgumentError, "cannot invoke #{fun}/#{arity} inside function/macro" + end + end + + defp assert_no_match_or_guard_scope(context, exp) do + case context do + :match -> + invalid_match!(exp) + + :guard -> + raise ArgumentError, + "invalid expression in guard, #{exp} is not allowed in guards. " <> + "To learn more about guards, visit: https://hexdocs.pm/elixir/patterns-and-guards.html" + + _ -> + :ok end end - defp env_stacktrace(env) do - case bootstraped?(Path) do - true -> Macro.Env.stacktrace(env) - false -> [] + defp invalid_match!(exp) do + raise ArgumentError, + "invalid expression in match, #{exp} is not allowed in patterns " <> + "such as function clauses, case clauses or on the left side of the = operator" + end + + # Helper to handle the :ok | :error tuple returned from :elixir_interpolation.unescape_tokens + # We need to do this for bootstrapping purposes, actual code can use Macro.unescape_string. + defp unescape_tokens(tokens) do + :lists.map( + fn token -> + case is_binary(token) do + true -> :elixir_interpolation.unescape_string(token) + false -> token + end + end, + tokens + ) + end + + defp unescape_tokens(tokens, unescape_map) do + :lists.map( + fn token -> + case is_binary(token) do + true -> :elixir_interpolation.unescape_string(token, unescape_map) + false -> token + end + end, + tokens + ) + end + + defp unescape_list_tokens(tokens) do + escape = fn + {:"::", _, [expr, _]} -> expr + binary when is_binary(binary) -> :elixir_interpolation.unescape_string(binary) end + + :lists.map(escape, tokens) + end + + @doc false + defmacro to_char_list(arg) do + IO.warn( + "Kernel.to_char_list/1 is deprecated, use Kernel.to_charlist/1 instead", + Macro.Env.stacktrace(__CALLER__) + ) + + quote(do: Kernel.to_charlist(unquote(arg))) end end diff --git a/lib/elixir/lib/kernel/cli.ex b/lib/elixir/lib/kernel/cli.ex index 1e36a9a176c..5c6b9b3769e 100644 --- a/lib/elixir/lib/kernel/cli.ex +++ b/lib/elixir/lib/kernel/cli.ex @@ -1,9 +1,22 @@ defmodule Kernel.CLI do @moduledoc false - @blank_config %{commands: [], output: ".", compile: [], - halt: true, compiler_options: [], errors: [], - verbose_compile: false} + @compile {:no_warn_undefined, [Logger, IEx]} + + @blank_config %{ + commands: [], + output: ".", + compile: [], + no_halt: false, + compiler_options: [], + errors: [], + pa: [], + pz: [], + verbose_compile: false, + profile: nil + } + + @standalone_opts ["-h", "--help", "--short-version"] @doc """ This is the API invoked by Elixir boot process. @@ -13,15 +26,18 @@ defmodule Kernel.CLI do {config, argv} = parse_argv(argv) System.argv(argv) + System.no_halt(config.no_halt) - run fn -> + fun = fn _ -> errors = process_commands(config) if errors != [] do Enum.each(errors, &IO.puts(:stderr, &1)) System.halt(1) end - end, config.halt + end + + run(fun) end @doc """ @@ -32,89 +48,168 @@ defmodule Kernel.CLI do This function is used by Elixir's CLI and also by escripts generated by Elixir. """ - def run(fun, halt \\ true) do - try do - fun.() - if halt do - at_exit(0) - System.halt(0) + def run(fun) do + {ok_or_shutdown, status} = exec_fun(fun, {:ok, 0}) + + if ok_or_shutdown == :shutdown or not System.no_halt() do + {_, status} = at_exit({ok_or_shutdown, status}) + + # Ensure Logger messages are flushed before halting + case :erlang.whereis(Logger) do + pid when is_pid(pid) -> Logger.flush() + _ -> :ok end - catch - :exit, reason when is_integer(reason) -> - at_exit(reason) - System.halt(reason) - :exit, :normal -> - at_exit(0) - System.halt(0) - kind, reason -> - at_exit(1) - print_error(kind, reason, System.stacktrace) - System.halt(1) + + System.halt(status) end end @doc """ - Parses ARGV returning the CLI config and trailing args. + Parses the CLI arguments. Made public for testing. """ def parse_argv(argv) do parse_argv(argv, @blank_config) end @doc """ - Process commands according to the parsed config from `parse_argv/1`. - Returns all errors. + Process CLI commands. Made public for testing. """ def process_commands(config) do results = Enum.map(Enum.reverse(config.commands), &process_command(&1, config)) - errors = for {:error, msg} <- results, do: msg + errors = for {:error, msg} <- results, do: msg Enum.reverse(config.errors, errors) end - ## Helpers + @doc """ + Shared helper for error formatting on CLI tools. + """ + def format_error(kind, reason, stacktrace) do + {blamed, stacktrace} = Exception.blame(kind, reason, stacktrace) - defp at_exit(status) do - hooks = :elixir_code_server.call(:flush_at_exit) + iodata = + case blamed do + %FunctionClauseError{} -> + formatted = Exception.format_banner(kind, reason, stacktrace) + padded_blame = pad(FunctionClauseError.blame(blamed, &inspect/1, &blame_match/1)) + [formatted, padded_blame] - for hook <- hooks do - try do - hook.(status) - catch - kind, reason -> - print_error(kind, reason, System.stacktrace) + _ -> + Exception.format_banner(kind, blamed, stacktrace) end - end - # If an at_exit callback adds a - # new hook we need to invoke it. - unless hooks == [], do: at_exit(status) + [iodata, ?\n, Exception.format_stacktrace(prune_stacktrace(stacktrace))] + end + + @doc """ + Function invoked across nodes for `--rpc-eval`. + """ + def rpc_eval(expr) do + wrapper(fn -> Code.eval_string(expr) end) + catch + kind, reason -> {kind, reason, __STACKTRACE__} end + ## Helpers + + defp at_exit(res) do + hooks = :elixir_config.get_and_put(:at_exit, []) + res = Enum.reduce(hooks, res, &exec_fun/2) + if hooks == [], do: res, else: at_exit(res) + end + + defp exec_fun(fun, res) when is_function(fun, 1) and is_tuple(res) do + parent = self() + + {pid, ref} = + spawn_monitor(fn -> + try do + fun.(elem(res, 1)) + catch + :exit, {:shutdown, int} when is_integer(int) -> + send(parent, {self(), {:shutdown, int}}) + exit({:shutdown, int}) + + :exit, reason + when reason == :normal + when reason == :shutdown + when tuple_size(reason) == 2 and elem(reason, 0) == :shutdown -> + send(parent, {self(), {:shutdown, 0}}) + exit(reason) + + kind, reason -> + print_error(kind, reason, __STACKTRACE__) + send(parent, {self(), {:shutdown, 1}}) + exit(to_exit(kind, reason, __STACKTRACE__)) + else + _ -> + send(parent, {self(), res}) + end + end) + + receive do + {^pid, res} -> + :erlang.demonitor(ref, [:flush]) + res + + {:DOWN, ^ref, _, _, other} -> + print_error({:EXIT, pid}, other, []) + {:shutdown, 1} + end + end + + defp to_exit(:throw, reason, stack), do: {{:nocatch, reason}, stack} + defp to_exit(:error, reason, stack), do: {reason, stack} + defp to_exit(:exit, reason, _stack), do: reason + defp shared_option?(list, config, callback) do case parse_shared(list, config) do - {[h|hs], _} when h == hd(list) -> + {[h | hs], _} when h == hd(list) -> new_config = %{config | errors: ["#{h} : Unknown option" | config.errors]} callback.(hs, new_config) + {new_list, new_config} -> callback.(new_list, new_config) end end - defp print_error(kind, reason, trace) do - IO.puts :stderr, Exception.format(kind, reason, prune_stacktrace(trace)) + ## Error handling + + defp print_error(kind, reason, stacktrace) do + IO.write(:stderr, format_error(kind, reason, stacktrace)) + end + + defp blame_match(%{match?: true, node: node}), do: blame_ansi(:normal, "+", node) + defp blame_match(%{match?: false, node: node}), do: blame_ansi(:red, "-", node) + + defp blame_ansi(color, no_ansi, node) do + if IO.ANSI.enabled?() do + [color | Macro.to_string(node)] + |> IO.ANSI.format(true) + |> IO.iodata_to_binary() + else + no_ansi <> Macro.to_string(node) <> no_ansi + end + end + + defp pad(string) do + " " <> String.replace(string, "\n", "\n ") end - @elixir_internals [:elixir_compiler, :elixir_module, :elixir_translator, :elixir_expand] + @elixir_internals [:elixir, :elixir_aliases, :elixir_expand, :elixir_compiler, :elixir_module] ++ + [:elixir_clauses, :elixir_lexical, :elixir_def, :elixir_map, :elixir_locals] ++ + [:elixir_erl, :elixir_erl_clauses, :elixir_erl_compiler, :elixir_erl_pass] ++ + [Kernel.ErrorHandler, Module.ParallelChecker] - defp prune_stacktrace([{mod, _, _, _}|t]) when mod in @elixir_internals do + defp prune_stacktrace([{mod, _, _, _} | t]) when mod in @elixir_internals do prune_stacktrace(t) end - defp prune_stacktrace([{__MODULE__, :wrapper, 1, _}|_]) do + defp prune_stacktrace([{__MODULE__, :wrapper, 1, _} | _]) do [] end - defp prune_stacktrace([h|t]) do - [h|prune_stacktrace(t)] + defp prune_stacktrace([h | t]) do + [h | prune_stacktrace(t)] end defp prune_stacktrace([]) do @@ -123,86 +218,121 @@ defmodule Kernel.CLI do # Parse shared options - defp parse_shared([opt|_t], _config) when opt in ["-v", "--version"] do - IO.puts "Elixir #{System.version}" - System.halt 0 + defp halt_standalone(opt) do + IO.puts(:stderr, "#{opt} : Standalone options can't be combined with other options") + System.halt(1) + end + + defp parse_shared([opt | _], _config) when opt in @standalone_opts do + halt_standalone(opt) + end + + defp parse_shared([opt | t], _config) when opt in ["-v", "--version"] do + if function_exported?(IEx, :started?, 0) and IEx.started?() do + IO.puts("IEx " <> System.build_info()[:build]) + else + IO.puts(:erlang.system_info(:system_version)) + IO.puts("Elixir " <> System.build_info()[:build]) + end + + if t != [] do + halt_standalone(opt) + else + System.halt(0) + end + end + + defp parse_shared(["-pa", h | t], config) do + paths = expand_code_path(h) + Enum.each(paths, &:code.add_patha/1) + parse_shared(t, %{config | pa: config.pa ++ paths}) end - defp parse_shared(["-pa", h|t], config) do - add_code_path(h, &Code.prepend_path/1) - parse_shared t, config + defp parse_shared(["-pz", h | t], config) do + paths = expand_code_path(h) + Enum.each(paths, &:code.add_pathz/1) + parse_shared(t, %{config | pz: config.pz ++ paths}) end - defp parse_shared(["-pz", h|t], config) do - add_code_path(h, &Code.append_path/1) - parse_shared t, config + defp parse_shared(["--app", h | t], config) do + parse_shared(t, %{config | commands: [{:app, h} | config.commands]}) end - defp parse_shared(["--app", h|t], config) do - parse_shared t, %{config | commands: [{:app, h} | config.commands]} + defp parse_shared(["--no-halt" | t], config) do + parse_shared(t, %{config | no_halt: true}) end - defp parse_shared(["--no-halt"|t], config) do - parse_shared t, %{config | halt: false} + defp parse_shared(["-e", h | t], config) do + parse_shared(t, %{config | commands: [{:eval, h} | config.commands]}) end - defp parse_shared(["-e", h|t], config) do - parse_shared t, %{config | commands: [{:eval, h} | config.commands]} + defp parse_shared(["--eval", h | t], config) do + parse_shared(t, %{config | commands: [{:eval, h} | config.commands]}) end - defp parse_shared(["-r", h|t], config) do - parse_shared t, %{config | commands: [{:require, h} | config.commands]} + defp parse_shared(["--rpc-eval", node, h | t], config) do + node = append_hostname(node) + parse_shared(t, %{config | commands: [{:rpc_eval, node, h} | config.commands]}) end - defp parse_shared(["-pr", h|t], config) do - parse_shared t, %{config | commands: [{:parallel_require, h} | config.commands]} + defp parse_shared(["--rpc-eval" | _], config) do + new_config = %{config | errors: ["--rpc-eval : wrong number of arguments" | config.errors]} + {[], new_config} end - defp parse_shared([erl, _|t], config) when erl in ["--erl", "--sname", "--name", "--cookie"] do - parse_shared t, config + defp parse_shared(["-r", h | t], config) do + parse_shared(t, %{config | commands: [{:require, h} | config.commands]}) end - defp parse_shared([erl|t], config) when erl in ["--detached", "--hidden"] do - parse_shared t, config + defp parse_shared(["-pr", h | t], config) do + parse_shared(t, %{config | commands: [{:parallel_require, h} | config.commands]}) end defp parse_shared(list, config) do {list, config} end + defp append_hostname(node) do + case :string.find(node, "@") do + :nomatch -> node <> :string.find(Atom.to_string(node()), "@") + _ -> node + end + end - defp add_code_path(path, fun) do + defp expand_code_path(path) do path = Path.expand(path) + case Path.wildcard(path) do - [] -> fun.(path) - list -> Enum.each(list, fun) + [] -> [to_charlist(path)] + list -> Enum.map(list, &to_charlist/1) end end # Process init options - defp parse_argv(["--"|t], config) do + defp parse_argv(["--" | t], config) do {config, t} end - defp parse_argv(["+elixirc"|t], config) do - parse_compiler t, config + defp parse_argv(["+elixirc" | t], config) do + parse_compiler(t, config) end - defp parse_argv(["+iex"|t], config) do - parse_iex t, config + defp parse_argv(["+iex" | t], config) do + parse_iex(t, config) end - defp parse_argv(["-S", h|t], config) do + defp parse_argv(["-S", h | t], config) do {%{config | commands: [{:script, h} | config.commands]}, t} end - defp parse_argv([h|t] = list, config) do + defp parse_argv([h | t] = list, config) do case h do "-" <> _ -> - shared_option? list, config, &parse_argv(&1, &2) + shared_option?(list, config, &parse_argv(&1, &2)) + _ -> - if Keyword.has_key?(config.commands, :eval) do + if List.keymember?(config.commands, :eval, 0) do {config, list} else {%{config | commands: [{:file, h} | config.commands]}, t} @@ -216,74 +346,76 @@ defmodule Kernel.CLI do # Parse compiler options - defp parse_compiler(["--"|t], config) do + defp parse_compiler(["--" | t], config) do {config, t} end - defp parse_compiler(["-o", h|t], config) do - parse_compiler t, %{config | output: h} + defp parse_compiler(["-o", h | t], config) do + parse_compiler(t, %{config | output: h}) end - defp parse_compiler(["--no-docs"|t], config) do - parse_compiler t, %{config | compiler_options: [{:docs, false} | config.compiler_options]} + defp parse_compiler(["--no-docs" | t], config) do + parse_compiler(t, %{config | compiler_options: [{:docs, false} | config.compiler_options]}) end - defp parse_compiler(["--no-debug-info"|t], config) do - parse_compiler t, %{config | compiler_options: [{:debug_info, false} | config.compiler_options]} + defp parse_compiler(["--no-debug-info" | t], config) do + compiler_options = [{:debug_info, false} | config.compiler_options] + parse_compiler(t, %{config | compiler_options: compiler_options}) end - defp parse_compiler(["--ignore-module-conflict"|t], config) do - parse_compiler t, %{config | compiler_options: [{:ignore_module_conflict, true} | config.compiler_options]} + defp parse_compiler(["--ignore-module-conflict" | t], config) do + compiler_options = [{:ignore_module_conflict, true} | config.compiler_options] + parse_compiler(t, %{config | compiler_options: compiler_options}) end - defp parse_compiler(["--warnings-as-errors"|t], config) do - parse_compiler t, %{config | compiler_options: [{:warnings_as_errors, true} | config.compiler_options]} + defp parse_compiler(["--warnings-as-errors" | t], config) do + compiler_options = [{:warnings_as_errors, true} | config.compiler_options] + parse_compiler(t, %{config | compiler_options: compiler_options}) end - defp parse_compiler(["--verbose"|t], config) do - parse_compiler t, %{config | verbose_compile: true} + defp parse_compiler(["--verbose" | t], config) do + parse_compiler(t, %{config | verbose_compile: true}) + end + + # Private compiler options + + defp parse_compiler(["--profile", "time" | t], config) do + parse_compiler(t, %{config | profile: :time}) end - defp parse_compiler([h|t] = list, config) do + defp parse_compiler([h | t] = list, config) do case h do "-" <> _ -> - shared_option? list, config, &parse_compiler(&1, &2) + shared_option?(list, config, &parse_compiler(&1, &2)) + _ -> pattern = if File.dir?(h), do: "#{h}/**/*.ex", else: h - parse_compiler t, %{config | compile: [pattern | config.compile]} + parse_compiler(t, %{config | compile: [pattern | config.compile]}) end end defp parse_compiler([], config) do - {%{config | commands: [{:compile, config.compile}|config.commands]}, []} + {%{config | commands: [{:compile, config.compile} | config.commands]}, []} end - # Parse iex options + # Parse IEx options - defp parse_iex(["--"|t], config) do + defp parse_iex(["--" | t], config) do {config, t} end - # This clause is here so that Kernel.CLI does not - # error out with "unknown option" - defp parse_iex(["--dot-iex", _|t], config) do - parse_iex t, config - end - - defp parse_iex([opt, _|t], config) when opt in ["--remsh"] do - parse_iex t, config - end + # These clauses are here so that Kernel.CLI does not error out with "unknown option" + defp parse_iex(["--dot-iex", _ | t], config), do: parse_iex(t, config) + defp parse_iex(["--remsh", _ | t], config), do: parse_iex(t, config) - defp parse_iex(["-S", h|t], config) do + defp parse_iex(["-S", h | t], config) do {%{config | commands: [{:script, h} | config.commands]}, t} end - defp parse_iex([h|t] = list, config) do + defp parse_iex([h | t] = list, config) do case h do - "-" <> _ -> - shared_option? list, config, &parse_iex(&1, &2) - _ -> - {%{config | commands: [{:file, h} | config.commands]}, t} + "-" <> _ -> shared_option?(list, config, &parse_iex(&1, &2)) + _ -> {%{config | commands: [{:file, h} | config.commands]}, t} end end @@ -294,22 +426,32 @@ defmodule Kernel.CLI do # Process commands defp process_command({:cookie, h}, _config) do - if Node.alive? do - wrapper fn -> Node.set_cookie(String.to_atom(h)) end + if Node.alive?() do + wrapper(fn -> Node.set_cookie(String.to_atom(h)) end) else {:error, "--cookie : Cannot set cookie if the node is not alive (set --name or --sname)"} end end defp process_command({:eval, expr}, _config) when is_binary(expr) do - wrapper fn -> Code.eval_string(expr, []) end + wrapper(fn -> Code.eval_string(expr, []) end) + end + + defp process_command({:rpc_eval, node, expr}, _config) when is_binary(expr) do + case :rpc.call(String.to_atom(node), __MODULE__, :rpc_eval, [expr]) do + :ok -> :ok + {:badrpc, {:EXIT, exit}} -> Process.exit(self(), exit) + {:badrpc, reason} -> {:error, "--rpc-eval : RPC failed with reason #{inspect(reason)}"} + {kind, error, stack} -> :erlang.raise(kind, error, stack) + end end defp process_command({:app, app}, _config) when is_binary(app) do case Application.ensure_all_started(String.to_atom(app)) do {:error, {app, reason}} -> - {:error, "--app : Could not start application #{app}: " <> - Application.format_error(reason)} + msg = "--app : Could not start application #{app}: " <> Application.format_error(reason) + {:error, msg} + {:ok, _} -> :ok end @@ -317,7 +459,7 @@ defmodule Kernel.CLI do defp process_command({:script, file}, _config) when is_binary(file) do if exec = find_elixir_executable(file) do - wrapper fn -> Code.require_file(exec) end + wrapper(fn -> Code.require_file(exec) end) else {:error, "-S : Could not find executable #{file}"} end @@ -325,7 +467,7 @@ defmodule Kernel.CLI do defp process_command({:file, file}, _config) when is_binary(file) do if File.regular?(file) do - wrapper fn -> Code.require_file(file) end + wrapper(fn -> Code.require_file(file) end) else {:error, "No file named #{file}"} end @@ -335,7 +477,7 @@ defmodule Kernel.CLI do files = filter_patterns(pattern) if files != [] do - wrapper fn -> Enum.map files, &Code.require_file(&1) end + wrapper(fn -> Enum.map(files, &Code.require_file(&1)) end) else {:error, "-r : No files matched pattern #{pattern}"} end @@ -345,58 +487,84 @@ defmodule Kernel.CLI do files = filter_patterns(pattern) if files != [] do - wrapper fn -> Kernel.ParallelRequire.files(files) end + wrapper(fn -> + case Kernel.ParallelCompiler.require(files) do + {:ok, _, _} -> :ok + {:error, _, _} -> exit({:shutdown, 1}) + end + end) else {:error, "-pr : No files matched pattern #{pattern}"} end end defp process_command({:compile, patterns}, config) do - :filelib.ensure_dir(:filename.join(config.output, ".")) + # If ensuring the dir returns an error no files will be found. + _ = :filelib.ensure_dir(:filename.join(config.output, ".")) case filter_multiple_patterns(patterns) do {:ok, []} -> {:error, "No files matched provided patterns"} + {:ok, files} -> - wrapper fn -> + wrapper(fn -> Code.compiler_options(config.compiler_options) - Kernel.ParallelCompiler.files_to_path(files, config.output, - each_file: fn file -> if config.verbose_compile do IO.puts "Compiled #{file}" end end) - end + + verbose_opts = + if config.verbose_compile do + [each_file: &IO.puts("Compiling #{Path.relative_to_cwd(&1)}")] + else + [ + each_long_compilation: + &IO.puts("Compiling #{Path.relative_to_cwd(&1)} (it's taking more than 10s)") + ] + end + + profile_opts = + if config.profile do + [profile: config.profile] + else + [] + end + + opts = verbose_opts ++ profile_opts + + case Kernel.ParallelCompiler.compile_to_path(files, config.output, opts) do + {:ok, _, _} -> :ok + {:error, _, _} -> exit({:shutdown, 1}) + end + end) + {:missing, missing} -> {:error, "No files matched pattern(s) #{Enum.join(missing, ",")}"} end end defp filter_patterns(pattern) do - Enum.filter(Enum.uniq(Path.wildcard(pattern)), &File.regular?(&1)) + pattern + |> Path.expand() + |> Path.wildcard() + |> :lists.usort() + |> Enum.filter(&File.regular?/1) end defp filter_multiple_patterns(patterns) do - matched_files = Enum.map patterns, fn(pattern) -> - case filter_patterns(pattern) do - [] -> {:missing, pattern} - files -> {:ok, files} - end - end - - files = Enum.filter_map matched_files, - fn(match) -> elem(match, 0) == :ok end, - &elem(&1, 1) - - missing_patterns = Enum.filter_map matched_files, - fn(match) -> elem(match, 0) == :missing end, - &elem(&1, 1) + {files, missing} = + Enum.reduce(patterns, {[], []}, fn pattern, {files, missing} -> + case filter_patterns(pattern) do + [] -> {files, [pattern | missing]} + match -> {match ++ files, missing} + end + end) - if missing_patterns == [] do - {:ok, Enum.uniq(Enum.concat(files))} - else - {:missing, Enum.uniq(missing_patterns)} + case missing do + [] -> {:ok, :lists.usort(files)} + _ -> {:missing, :lists.usort(missing)} end end defp wrapper(fun) do - fun.() + _ = fun.() :ok end @@ -407,8 +575,9 @@ defmodule Kernel.CLI do # the actual Elixir executable. case :os.type() do {:win32, _} -> - exec = Path.rootname(exec) - if File.regular?(exec), do: exec + base = Path.rootname(exec) + if File.regular?(base), do: base, else: exec + _ -> exec end diff --git a/lib/elixir/lib/kernel/error_handler.ex b/lib/elixir/lib/kernel/error_handler.ex index dfc1dbb58e8..8fe1a2058a9 100644 --- a/lib/elixir/lib/kernel/error_handler.ex +++ b/lib/elixir/lib/kernel/error_handler.ex @@ -3,38 +3,41 @@ defmodule Kernel.ErrorHandler do @moduledoc false + @spec undefined_function(module, atom, list) :: term def undefined_function(module, fun, args) do - ensure_loaded(module) + ensure_loaded(module) or ensure_compiled(module, :module, :raise) :error_handler.undefined_function(module, fun, args) end + @spec undefined_lambda(module, fun, list) :: term def undefined_lambda(module, fun, args) do - ensure_loaded(module) + ensure_loaded(module) or ensure_compiled(module, :module, :raise) :error_handler.undefined_lambda(module, fun, args) end - def release() do - # On release, no further allow elixir_ensure_compiled - # directives and revert to the original error handler. - # Note we should not delete the elixir_compiler_pid though, - # as we still want to send notifications to the compiler. - :erlang.erase(:elixir_ensure_compiled) - :erlang.process_flag(:error_handler, :error_handler) - :ok + @spec ensure_loaded(module) :: boolean + def ensure_loaded(module) do + case :code.ensure_loaded(module) do + {:module, _} -> true + {:error, _} -> false + end + end + + @spec ensure_compiled(module, atom, atom) :: :found | :not_found | :deadlock + # Never wait on nil because it should never be defined. + def ensure_compiled(nil, _kind, _deadlock) do + :not_found end - defp ensure_loaded(module) do - case Code.ensure_loaded(module) do - {:module, _} -> [] - {:error, _} -> - parent = :erlang.get(:elixir_compiler_pid) - ref = :erlang.make_ref - send parent, {:waiting, module, self(), ref, module} - :erlang.garbage_collect(self) - receive do - {^ref, :ready} -> :ok - {^ref, :release} -> release() - end + def ensure_compiled(module, kind, deadlock) do + {compiler_pid, file_pid} = :erlang.get(:elixir_compiler_info) + ref = :erlang.make_ref() + modules = :elixir_module.compiler_modules() + send(compiler_pid, {:waiting, kind, self(), ref, file_pid, module, modules, deadlock}) + :erlang.garbage_collect(self()) + + receive do + {^ref, value} -> value end end end diff --git a/lib/elixir/lib/kernel/lexical_tracker.ex b/lib/elixir/lib/kernel/lexical_tracker.ex index 09e205b5214..b27cf193512 100644 --- a/lib/elixir/lib/kernel/lexical_tracker.ex +++ b/lib/elixir/lib/kernel/lexical_tracker.ex @@ -1,176 +1,251 @@ -# This is a module Elixir responsible for tracking -# the usage of aliases, imports and requires in the Elixir scope. -# -# The implementation simply stores dispatch information in an -# ETS table and then consults this table once compilation is done. +# This is an Elixir module responsible for tracking references +# to modules, remote dispatches, and the usage of +# aliases/imports/requires in the Elixir scope. # # Note that since this is required for bootstrap, we can't use # any of the `GenServer.Behaviour` conveniences. defmodule Kernel.LexicalTracker do @moduledoc false - @timeout 30_000 + @timeout :infinity @behaviour :gen_server - @import 2 - @alias 3 - @doc """ - Returns all remotes linked to in this lexical scope. + Returns all references in this lexical scope. """ - def remotes(arg) do - # If the module is compiled from a function, its lexical - # scope may be long gone, so it has no associated PID. - if pid = to_pid(arg) do - ets = :gen_server.call(pid, :ets, @timeout) - :ets.match(ets, {:"$1", :_, :_}) |> List.flatten - else - [] - end - end - - defp to_pid(pid) when is_pid(pid), do: pid - defp to_pid(mod) when is_atom(mod) do - table = :elixir_module.data_table(mod) - [{_, val}] = :ets.lookup(table, :__lexical_tracker) - val + def references(pid) do + :gen_server.call(pid, :references, @timeout) end # Internal API - # Starts the tracker and returns its pid. + # Starts the tracker and returns its PID. @doc false - def start_link do - {:ok, pid} = :gen_server.start_link(__MODULE__, [], []) - pid + def start_link() do + :gen_server.start_link(__MODULE__, :ok, []) end @doc false def stop(pid) do - :gen_server.cast(pid, :stop) + :gen_server.call(pid, :stop) end @doc false - def add_import(pid, module, line, warn) do - :gen_server.cast(pid, {:add_import, module, line, warn}) + def add_require(pid, module) when is_atom(module) do + :gen_server.cast(pid, {:add_require, module}) end @doc false - def add_alias(pid, module, line, warn) do + def add_import(pid, module, fas, line, warn) when is_atom(module) do + :gen_server.cast(pid, {:add_import, module, fas, line, warn}) + end + + @doc false + def add_alias(pid, module, line, warn) when is_atom(module) do :gen_server.cast(pid, {:add_alias, module, line, warn}) end @doc false - def remote_dispatch(pid, module) do - :gen_server.cast(pid, {:remote_dispatch, module}) + def remote_dispatch(pid, module, mode) when is_atom(module) do + :gen_server.cast(pid, {:remote_dispatch, module, mode}) end @doc false - def import_dispatch(pid, module) do - :gen_server.cast(pid, {:import_dispatch, module}) + def import_dispatch(pid, module, fa, mode) when is_atom(module) do + :gen_server.cast(pid, {:import_dispatch, module, fa, mode}) end @doc false - def alias_dispatch(pid, module) do + def alias_dispatch(pid, module) when is_atom(module) do :gen_server.cast(pid, {:alias_dispatch, module}) end + @doc false + def add_compile_env(pid, app, path, return) do + :gen_server.cast(pid, {:compile_env, app, path, return}) + end + + @doc false + def set_file(pid, file) do + :gen_server.cast(pid, {:set_file, file}) + end + + @doc false + def reset_file(pid) do + :gen_server.cast(pid, :reset_file) + end + + @doc false + def write_cache(pid, value) do + key = :erlang.unique_integer() + :gen_server.cast(pid, {:write_cache, key, value}) + key + end + + @doc false + def read_cache(pid, key) do + :gen_server.call(pid, {:read_cache, key}, @timeout) + end + @doc false def collect_unused_imports(pid) do - unused(pid, @import) + unused(pid, :import) end @doc false def collect_unused_aliases(pid) do - unused(pid, @alias) + unused(pid, :alias) end - defp unused(pid, pos) do - ets = :gen_server.call(pid, :ets, @timeout) - :ets.foldl(fn - {module, _, _} = tuple, acc when is_integer(:erlang.element(pos, tuple)) -> - [{module, :erlang.element(pos, tuple)}|acc] - _, acc -> - acc - end, [], ets) |> Enum.sort + defp unused(pid, tag) do + :gen_server.call(pid, {:unused, tag}, @timeout) end # Callbacks + def init(:ok) do + state = %{ + directives: %{}, + references: %{}, + exports: %{}, + cache: %{}, + compile_env: :ordsets.new(), + file: nil + } - def init([]) do - {:ok, :ets.new(:lexical, [:protected])} + {:ok, state} end - def handle_call(:ets, _from, d) do - {:reply, d, d} + @doc false + def handle_call({:unused, tag}, _from, state) do + directives = + for {{^tag, module_or_mfa}, marker} <- state.directives, is_integer(marker) do + {module_or_mfa, marker} + end + + {:reply, Enum.sort(directives), state} end - def handle_call(request, _from, d) do - {:stop, {:bad_call, request}, d} + def handle_call(:references, _from, state) do + {compile, runtime} = partition(Map.to_list(state.references), [], []) + {:reply, {compile, Map.keys(state.exports), runtime, state.compile_env}, state} end - def handle_cast({:remote_dispatch, module}, d) do - add_module(d, module) - {:noreply, d} + def handle_call({:read_cache, key}, _from, %{cache: cache} = state) do + {:reply, Map.get(cache, key), state} end - def handle_cast({:import_dispatch, module}, d) do - add_dispatch(d, module, @import) - {:noreply, d} + def handle_call(:stop, _from, state) do + {:stop, :normal, :ok, state} end - def handle_cast({:alias_dispatch, module}, d) do - add_dispatch(d, module, @alias) - {:noreply, d} + def handle_cast({:write_cache, key, value}, %{cache: cache} = state) do + {:noreply, %{state | cache: Map.put(cache, key, value)}} end - def handle_cast({:add_import, module, line, warn}, d) do - add_directive(d, module, line, warn, @import) - {:noreply, d} + def handle_cast({:remote_dispatch, module, mode}, state) do + references = add_reference(state.references, module, mode) + {:noreply, %{state | references: references}} end - def handle_cast({:add_alias, module, line, warn}, d) do - add_directive(d, module, line, warn, @alias) - {:noreply, d} + def handle_cast({:import_dispatch, module, {function, arity}, mode}, state) do + state = add_import_dispatch(state, module, function, arity, mode) + {:noreply, state} end - def handle_cast(:stop, d) do - {:stop, :normal, d} + def handle_cast({:alias_dispatch, module}, state) do + {:noreply, %{state | directives: add_dispatch(state.directives, module, :alias)}} end - def handle_cast(msg, d) do - {:stop, {:bad_cast, msg}, d} + def handle_cast({:set_file, file}, state) do + {:noreply, %{state | file: file}} end - def handle_info(_msg, d) do - {:noreply, d} + def handle_cast(:reset_file, state) do + {:noreply, %{state | file: nil}} end - def terminate(_reason, _d) do + def handle_cast({:compile_env, app, path, return}, state) do + {:noreply, update_in(state.compile_env, &:ordsets.add_element({app, path, return}, &1))} + end + + def handle_cast({:add_require, module}, state) do + {:noreply, put_in(state.exports[module], true)} + end + + def handle_cast({:add_import, module, fas, line, warn}, state) do + to_remove = for {{:import, {^module, _, _}} = key, _} <- state.directives, do: key + + directives = + state.directives + |> Map.drop(to_remove) + |> add_directive(module, line, warn, :import) + + directives = + Enum.reduce(fas, directives, fn {function, arity}, directives -> + add_directive(directives, {module, function, arity}, line, warn, :import) + end) + + {:noreply, %{state | directives: directives}} + end + + def handle_cast({:add_alias, module, line, warn}, state) do + {:noreply, %{state | directives: add_directive(state.directives, module, line, warn, :alias)}} + end + + @doc false + def handle_info(_msg, state) do + {:noreply, state} + end + + @doc false + def terminate(_reason, _state) do :ok end - def code_change(_old, d, _extra) do - {:ok, d} + @doc false + def code_change(_old, state, _extra) do + {:ok, state} end + defp partition([{remote, :compile} | t], compile, runtime), + do: partition(t, [remote | compile], runtime) + + defp partition([{remote, :runtime} | t], compile, runtime), + do: partition(t, compile, [remote | runtime]) + + defp partition([], compile, runtime), do: {compile, runtime} + # Callbacks helpers - # In the table we keep imports and aliases. - # If the value is false, it was not imported/aliased - # If the value is true, it was imported/aliased - # If the value is a line, it was imported/aliased and has a pending warning - defp add_module(d, module) do - :ets.insert_new(d, {module, false, false}) + defp add_reference(references, module, :compile) when is_atom(module), + do: Map.put(references, module, :compile) + + defp add_reference(references, module, :runtime) when is_atom(module) do + case Map.fetch(references, module) do + {:ok, _} -> references + :error -> Map.put(references, module, :runtime) + end end - defp add_dispatch(d, module, pos) do - :ets.update_element(d, module, {pos, true}) + defp add_import_dispatch(state, module, function, arity, mode) do + directives = + state.directives + |> add_dispatch(module, :import) + |> add_dispatch({module, function, arity}, :import) + + references = add_reference(state.references, module, mode) + %{state | directives: directives, references: references} end - defp add_directive(d, module, line, warn, pos) do - add_module(d, module) + # In the map we keep imports and aliases. + # If the value is a line, it was imported/aliased and has a pending warning + # If the value is true, it was imported/aliased and used + defp add_directive(directives, module_or_mfa, line, warn, tag) do marker = if warn, do: line, else: true - :ets.update_element(d, module, {pos, marker}) + Map.put(directives, {tag, module_or_mfa}, marker) + end + + defp add_dispatch(directives, module_or_mfa, tag) do + Map.put(directives, {tag, module_or_mfa}, true) end end diff --git a/lib/elixir/lib/kernel/parallel_compiler.ex b/lib/elixir/lib/kernel/parallel_compiler.ex index ee1e58f48df..bf3bcdd267b 100644 --- a/lib/elixir/lib/kernel/parallel_compiler.ex +++ b/lib/elixir/lib/kernel/parallel_compiler.ex @@ -1,8 +1,52 @@ defmodule Kernel.ParallelCompiler do @moduledoc """ - A module responsible for compiling files in parallel. + A module responsible for compiling and requiring files in parallel. """ + @typedoc "The line. 0 indicates no line." + @type line() :: non_neg_integer() + @type location() :: line() | {line(), column :: non_neg_integer} + @type warning() :: {file :: Path.t(), location(), message :: String.t()} + @type error() :: {file :: Path.t(), line(), message :: String.t()} + + @doc """ + Starts a task for parallel compilation. + + If you have a file that needs to compile other modules in parallel, + the spawned processes need to be aware of the compiler environment. + This function allows a developer to create a task that is aware of + those environments. + + See `Task.async/1` for more information. The task spawned must be + always awaited on by calling `Task.await/1` + """ + @doc since: "1.6.0" + def async(fun) when is_function(fun, 0) do + case :erlang.get(:elixir_compiler_info) do + {compiler, _} -> + file = :erlang.get(:elixir_compiler_file) + dest = :erlang.get(:elixir_compiler_dest) + + {:error_handler, error_handler} = :erlang.process_info(self(), :error_handler) + checker = Module.ParallelChecker.get() + + Task.async(fn -> + send(compiler, {:async, self()}) + Module.ParallelChecker.put(compiler, checker) + :erlang.put(:elixir_compiler_info, {compiler, self()}) + :erlang.put(:elixir_compiler_file, file) + dest != :undefined and :erlang.put(:elixir_compiler_dest, dest) + :erlang.process_flag(:error_handler, error_handler) + fun.() + end) + + :undefined -> + raise ArgumentError, + "cannot spawn parallel compiler task because " <> + "the current file is not being compiled/required" + end + end + @doc """ Compiles the given files. @@ -11,239 +55,735 @@ defmodule Kernel.ParallelCompiler do the current file stops being compiled until the dependency is resolved. - If there is an error during compilation or if `warnings_as_errors` - is set to `true` and there is a warning, this function will fail - with an exception. + It returns `{:ok, modules, warnings}` or `{:error, errors, warnings}`. - This function receives a set of callbacks as options: + Both errors and warnings are a list of three-element tuples containing + the file, line and the formatted error/warning. + + ## Options * `:each_file` - for each file compiled, invokes the callback passing the file + * `:each_long_compilation` - for each file that takes more than a given + timeout (see the `:long_compilation_threshold` option) to compile, invoke + this callback passing the file as its argument + * `:each_module` - for each module compiled, invokes the callback passing the file, module and the module bytecode - The compiler doesn't care about the return values of the callbacks. - Returns the modules generated by each compiled file. + * `:each_cycle` - after the given files are compiled, invokes this function + that should return the following values: + * `{:compile, modules, warnings}` - to continue compilation with a list of + further modules to compile + * `{:runtime, modules, warnings}` - to stop compilation and verify the list + of modules because dependent modules have changed + + * `:long_compilation_threshold` - the timeout (in seconds) to check for modules + taking too long to compile. For each file that exceeds the threshold, the + `:each_long_compilation` callback is invoked. From Elixir v1.11, only the time + spent compiling the actual module is taken into account by the threshold, the + time spent waiting is not considered. Defaults to `10` seconds. + + * `:profile` - if set to `:time` measure the compilation time of each compilation cycle + and group pass checker + + * `:dest` - the destination directory for the BEAM files. When using `compile/2`, + this information is only used to properly annotate the BEAM files before + they are loaded into memory. If you want a file to actually be written to + `dest`, use `compile_to_path/3` instead. + + * `:beam_timestamp` - the modification timestamp to give all BEAM files + """ - def files(files, callbacks \\ []) + @doc since: "1.6.0" + @spec compile([Path.t()], keyword()) :: {:ok, [atom], [warning]} | {:error, [error], [warning]} + def compile(files, options \\ []) when is_list(options) do + spawn_workers(files, :compile, options) + end - def files(files, callbacks) when is_list(callbacks) do - spawn_compilers(files, nil, callbacks) + @doc """ + Compiles the given files and writes resulting BEAM files into path. + + See `compile/2` for more information. + """ + @doc since: "1.6.0" + @spec compile_to_path([Path.t()], Path.t(), keyword()) :: + {:ok, [atom], [warning]} | {:error, [error], [warning]} + def compile_to_path(files, path, options \\ []) when is_binary(path) and is_list(options) do + spawn_workers(files, {:compile, path}, options) + end + + @doc """ + Requires the given files in parallel. + + Opposite to compile, dependencies are not attempted to be + automatically solved between files. + + It returns `{:ok, modules, warnings}` or `{:error, errors, warnings}`. + + Both errors and warnings are a list of three-element tuples containing + the file, line and the formatted error/warning. + + ## Options + + * `:each_file` - for each file compiled, invokes the callback passing the + file + + * `:each_module` - for each module compiled, invokes the callback passing + the file, module and the module bytecode + + """ + @doc since: "1.6.0" + @spec require([Path.t()], keyword()) :: + {:ok, [atom], [warning]} | {:error, [error], [warning]} + def require(files, options \\ []) when is_list(options) do + spawn_workers(files, :require, options) end @doc """ - Compiles the given files to the given path. - Read `files/2` for more information. + Prints a warning returned by the compiler. """ - def files_to_path(files, path, callbacks \\ []) + @doc since: "1.13.0" + @spec print_warning(warning) :: :ok + def print_warning({file, location, warning}) do + :elixir_errors.print_warning(location, file, warning) + end - def files_to_path(files, path, callbacks) when is_binary(path) and is_list(callbacks) do - spawn_compilers(files, path, callbacks) + @doc false + @deprecated "Use Kernel.ParallelCompiler.compile/2 instead" + def files(files, options \\ []) when is_list(options) do + case spawn_workers(files, :compile, options) do + {:ok, modules, _} -> modules + {:error, _, _} -> exit({:shutdown, 1}) + end end - defp spawn_compilers(files, path, callbacks) do - Code.ensure_loaded(Kernel.ErrorHandler) - compiler_pid = self() - :elixir_code_server.cast({:reset_warnings, compiler_pid}) + @doc false + @deprecated "Use Kernel.ParallelCompiler.compile_to_path/2 instead" + def files_to_path(files, path, options \\ []) when is_binary(path) and is_list(options) do + case spawn_workers(files, {:compile, path}, options) do + {:ok, modules, _} -> modules + {:error, _, _} -> exit({:shutdown, 1}) + end + end + + defp spawn_workers(files, output, options) do + {:module, _} = :code.ensure_loaded(Kernel.ErrorHandler) schedulers = max(:erlang.system_info(:schedulers_online), 2) + {:ok, checker} = Module.ParallelChecker.start_link(schedulers) + + try do + outcome = spawn_workers(schedulers, checker, files, output, options) + {outcome, Code.get_compiler_option(:warnings_as_errors)} + else + {{:ok, _, [_ | _] = warnings}, true} -> + message = "Compilation failed due to warnings while using the --warnings-as-errors option" + IO.puts(:stderr, message) + {:error, warnings, []} + + {{:ok, outcome, warnings}, _} -> + beam_timestamp = Keyword.get(options, :beam_timestamp) + {:ok, write_module_binaries(outcome, output, beam_timestamp), warnings} + + {{:error, errors, warnings}, true} -> + {:error, errors ++ warnings, []} + + {{:error, errors, warnings}, _} -> + {:error, errors, warnings} + after + Module.ParallelChecker.stop(checker) + end + end - result = spawn_compilers(files, files, path, callbacks, [], [], schedulers, []) + defp spawn_workers(schedulers, checker, files, output, options) do + threshold = Keyword.get(options, :long_compilation_threshold, 10) * 1000 + timer_ref = Process.send_after(self(), :threshold_check, threshold) + + {outcome, state} = + spawn_workers(files, 0, [], [], %{}, [], %{ + dest: Keyword.get(options, :dest), + each_cycle: Keyword.get(options, :each_cycle, fn -> {:runtime, [], []} end), + each_file: Keyword.get(options, :each_file, fn _, _ -> :ok end) |> each_file(), + each_long_compilation: Keyword.get(options, :each_long_compilation, fn _file -> :ok end), + each_module: Keyword.get(options, :each_module, fn _file, _module, _binary -> :ok end), + profile: profile_init(Keyword.get(options, :profile)), + output: output, + timer_ref: timer_ref, + long_compilation_threshold: threshold, + schedulers: schedulers, + checker: checker + }) + + Process.cancel_timer(state.timer_ref) - # In case --warning-as-errors is enabled and there was a warning, - # compilation status will be set to error and we fail with CompileError - case :elixir_code_server.call({:compilation_status, compiler_pid}) do - :ok -> result - :error -> exit(1) + receive do + :threshold_check -> :ok + after + 0 -> :ok end + + outcome end - # We already have 4 currently running, don't spawn new ones - defp spawn_compilers(entries, original, output, callbacks, waiting, queued, schedulers, result) when - length(queued) - length(waiting) >= schedulers do - wait_for_messages(entries, original, output, callbacks, waiting, queued, schedulers, result) + defp each_file(fun) when is_function(fun, 1), do: fn file, _ -> fun.(file) end + defp each_file(fun) when is_function(fun, 2), do: fun + + defp each_file(file, lexical, parent) do + ref = Process.monitor(parent) + send(parent, {:file_ok, self(), ref, file, lexical}) + + receive do + ^ref -> :ok + {:DOWN, ^ref, _, _, _} -> :ok + end + end + + defp write_module_binaries(result, {:compile, path}, timestamp) do + Enum.flat_map(result, fn + {{:module, module}, {binary, _map}} -> + full_path = Path.join(path, Atom.to_string(module) <> ".beam") + File.write!(full_path, binary) + if timestamp, do: File.touch!(full_path, timestamp) + [module] + + _ -> + [] + end) + end + + defp write_module_binaries(result, _output, _timestamp) do + for {{:module, module}, _} <- result, do: module + end + + ## Verification + + defp verify_modules(result, warnings, dependent_modules, state) do + checker_warnings = maybe_check_modules(result, dependent_modules, state) + warnings = Enum.reverse(warnings, checker_warnings) + {{:ok, result, warnings}, state} + end + + defp maybe_check_modules(result, runtime_modules, state) do + %{profile: profile, checker: checker} = state + + compiled_modules = + for {{:module, _module}, {_binary, info}} <- result, + do: info + + runtime_modules = + for module <- runtime_modules, + path = :code.which(module), + is_list(path) and path != [], + do: {module, path} + + profile_checker(profile, compiled_modules, runtime_modules, fn -> + Module.ParallelChecker.verify(checker, compiled_modules, runtime_modules) + end) + end + + defp profile_init(:time), do: {:time, System.monotonic_time(), 0} + defp profile_init(nil), do: :none + + defp profile_checker({:time, _, _}, compiled_modules, runtime_modules, fun) do + {time, result} = :timer.tc(fun) + time = div(time, 1000) + num_modules = length(compiled_modules) + length(runtime_modules) + IO.puts(:stderr, "[profile] Finished group pass check of #{num_modules} modules in #{time}ms") + result + end + + defp profile_checker(:none, _compiled_modules, _runtime_modules, fun) do + fun.() + end + + ## Compiler worker spawning + + # We already have n=schedulers currently running, don't spawn new ones + defp spawn_workers( + queue, + spawned, + waiting, + files, + result, + warnings, + %{schedulers: schedulers} = state + ) + when spawned - length(waiting) >= schedulers do + wait_for_messages(queue, spawned, waiting, files, result, warnings, state) end # Release waiting processes - defp spawn_compilers([h|t], original, output, callbacks, waiting, queued, schedulers, result) when is_pid(h) do - {_kind, ^h, ref, _module} = List.keyfind(waiting, h, 1) - send h, {ref, :ready} - waiting = List.keydelete(waiting, h, 1) - spawn_compilers(t, original, output, callbacks, waiting, queued, schedulers, result) + defp spawn_workers([{ref, found} | t], spawned, waiting, files, result, warnings, state) do + {files, waiting} = + case List.keytake(waiting, ref, 2) do + {{_kind, pid, ^ref, file_pid, _on, _defining, _deadlock}, waiting} -> + send(pid, {ref, found}) + {update_timing(files, file_pid, :waiting), waiting} + + nil -> + # In case the waiting process died (for example, it was an async process), + # it will no longer be on the list. So we need to take it into account here. + {files, waiting} + end + + spawn_workers(t, spawned, waiting, files, result, warnings, state) end - # Spawn a compiler for each file in the list until we reach the limit - defp spawn_compilers([h|t], original, output, callbacks, waiting, queued, schedulers, result) do + defp spawn_workers([file | queue], spawned, waiting, files, result, warnings, state) do + %{output: output, dest: dest, checker: checker} = state parent = self() + file = Path.expand(file) {pid, ref} = - :erlang.spawn_monitor fn -> - # Notify Code.ensure_compiled/2 that we should - # attempt to compile the module by doing a dispatch. - :erlang.put(:elixir_ensure_compiled, true) - - # Set the elixir_compiler_pid used by our custom Kernel.ErrorHandler. - :erlang.put(:elixir_compiler_pid, parent) - :erlang.process_flag(:error_handler, Kernel.ErrorHandler) - - exit(try do - if output do - :elixir_compiler.file_to_path(h, output) - else - :elixir_compiler.file(h) + :erlang.spawn_monitor(fn -> + Module.ParallelChecker.put(parent, checker) + :erlang.put(:elixir_compiler_info, {parent, self()}) + :erlang.put(:elixir_compiler_file, file) + + try do + case output do + {:compile, path} -> compile_file(file, path, parent) + :compile -> compile_file(file, dest, parent) + :require -> require_file(file, parent) end - {:compiled, h} catch kind, reason -> - {:failure, kind, reason, System.stacktrace} - end) - end + send(parent, {:file_error, self(), file, {kind, reason, __STACKTRACE__}}) + end + + exit(:shutdown) + end) + + file_data = %{ + pid: pid, + ref: ref, + file: file, + timestamp: System.monotonic_time(), + compiling: 0, + waiting: 0, + warned: false + } + + files = [file_data | files] + spawn_workers(queue, spawned + 1, waiting, files, result, warnings, state) + end + + # No more queue, nothing waiting, this cycle is done + defp spawn_workers([], 0, [], [], result, warnings, state) do + cycle_return = each_cycle_return(state.each_cycle.()) + state = cycle_timing(result, state) - spawn_compilers(t, original, output, callbacks, waiting, - [{pid, ref, h}|queued], schedulers, result) + case cycle_return do + {:runtime, dependent_modules, extra_warnings} -> + verify_modules(result, extra_warnings ++ warnings, dependent_modules, state) + + {:compile, [], extra_warnings} -> + verify_modules(result, extra_warnings ++ warnings, [], state) + + {:compile, more, extra_warnings} -> + spawn_workers(more, 0, [], [], result, extra_warnings ++ warnings, state) + end end - # No more files, nothing waiting, queue is empty, we are done - defp spawn_compilers([], _original, _output, _callbacks, [], [], _schedulers, result) do - for {:module, mod} <- result, do: mod + # files x, waiting for x: POSSIBLE ERROR! Release processes so we get the failures + + # Single entry, just release it. + defp spawn_workers( + [], + 1, + [{_, pid, ref, _, _, _, _}] = waiting, + [%{pid: pid}] = files, + result, + warnings, + state + ) do + spawn_workers([{ref, :not_found}], 1, waiting, files, result, warnings, state) end - # Queued x, waiting for x: POSSIBLE ERROR! Release processes so we get the failures - defp spawn_compilers([], original, output, callbacks, waiting, queued, schedulers, result) when length(waiting) == length(queued) do - Enum.each queued, fn {child, _, _} -> - {_kind, ^child, ref, _module} = List.keyfind(waiting, child, 1) - send child, {ref, :release} + # Multiple entries, try to release modules. + defp spawn_workers([], spawned, waiting, files, result, warnings, state) + when length(waiting) == spawned do + # There is potentially a deadlock. We will release modules with + # the following order: + # + # 1. Code.ensure_compiled/1 checks without a known definition (deadlock = soft) + # 2. Code.ensure_compiled/1 checks with a known definition (deadlock = soft) + # 3. Struct/import/require/ensure_compiled! checks without a known definition (deadlock = hard) + # 4. Modules without a known definition + # 5. Code invocation (deadlock = raise) + # + # The reason for step 3 and 4 is to not treat typos as deadlocks and + # help developers handle those sooner. However, this can have false + # positives in case multiple modules are defined in the same file + # and the module we are waiting for is defined later on. + # + # Finally, note there is no difference between hard and raise, the + # difference is where the raise is happening, inside the compiler + # or in the caller. + + deadlocked = + deadlocked(waiting, :soft, false) || + deadlocked(waiting, :soft, true) || deadlocked(waiting, :hard, false) || + without_definition(waiting, files) + + if deadlocked do + spawn_workers(deadlocked, spawned, waiting, files, result, warnings, state) + else + errors = handle_deadlock(waiting, files) + {{:error, errors, warnings}, state} end - wait_for_messages([], original, output, callbacks, waiting, queued, schedulers, result) end - # No more files, but queue and waiting are not full or do not match - defp spawn_compilers([], original, output, callbacks, waiting, queued, schedulers, result) do - wait_for_messages([], original, output, callbacks, waiting, queued, schedulers, result) + # No more queue, but spawned and length(waiting) do not match + defp spawn_workers([], spawned, waiting, files, result, warnings, state) do + wait_for_messages([], spawned, waiting, files, result, warnings, state) + end + + defp compile_file(file, path, parent) do + :erlang.process_flag(:error_handler, Kernel.ErrorHandler) + :erlang.put(:elixir_compiler_dest, path) + :elixir_compiler.file(file, &each_file(&1, &2, parent)) end + defp require_file(file, parent) do + case :elixir_code_server.call({:acquire, file}) do + :required -> + send(parent, {:file_cancel, self()}) + + :proceed -> + :elixir_compiler.file(file, &each_file(&1, &2, parent)) + :elixir_code_server.cast({:required, file}) + end + end + + defp cycle_timing(_result, %{profile: :none} = state) do + state + end + + defp cycle_timing(result, %{profile: {:time, cycle_start, module_counter}} = state) do + num_modules = count_modules(result) + diff_modules = num_modules - module_counter + now = System.monotonic_time() + time = System.convert_time_unit(now - cycle_start, :native, :millisecond) + + IO.puts( + :stderr, + "[profile] Finished compilation cycle of #{diff_modules} modules in #{time}ms" + ) + + %{state | profile: {:time, now, num_modules}} + end + + defp count_modules(result) do + Enum.count(result, &match?({{:module, _}, _}, &1)) + end + + defp each_cycle_return({kind, modules, warnings}), do: {kind, modules, warnings} + + defp each_cycle_return(other) do + IO.warn( + "the :each_cycle callback must return a tuple of format {:compile | :runtime, modules, warnings}" + ) + + case other do + {kind, modules} -> {kind, modules, []} + modules when is_list(modules) -> {:compile, modules, []} + end + end + + # The goal of this function is to find leaves in the dependency graph, + # i.e. to find code that depends on code that we know is not being defined. + # Note that not all files have been compiled yet, so they may not be in waiting. + defp without_definition(waiting, files) do + nilify_empty( + for %{pid: pid} <- files, + {_, _, ref, ^pid, on, _, _} <- waiting, + not defining?(on, waiting), + do: {ref, :not_found} + ) + end + + defp deadlocked(waiting, type, defining?) do + nilify_empty( + for {_, _, ref, _, on, _, ^type} <- waiting, + defining?(on, waiting) == defining?, + do: {ref, :deadlock} + ) + end + + defp defining?(on, waiting) do + Enum.any?(waiting, fn {_, _, _, _, _, defining, _} -> on in defining end) + end + + defp nilify_empty([]), do: nil + defp nilify_empty([_ | _] = list), do: list + # Wait for messages from child processes - defp wait_for_messages(entries, original, output, callbacks, waiting, queued, schedulers, result) do + defp wait_for_messages(queue, spawned, waiting, files, result, warnings, state) do + %{output: output} = state + receive do - {:struct_available, module} -> - available = for {:struct, pid, _, waiting_module} <- waiting, - module == waiting_module, - not pid in entries, - do: pid - - spawn_compilers(available ++ entries, original, output, callbacks, - waiting, queued, schedulers, [{:struct, module}|result]) - - {:module_available, child, ref, file, module, binary} -> - if callback = Keyword.get(callbacks, :each_module) do - callback.(file, module, binary) - end + {:async, process} -> + Process.monitor(process) + wait_for_messages(queue, spawned + 1, waiting, files, result, warnings, state) - # Release the module loader which is waiting for an ack - send child, {ref, :ack} + {:available, kind, module} -> + available = + for {^kind, _, ref, _, ^module, _defining, _deadlock} <- waiting, + do: {ref, :found} - available = for {_kind, pid, _, waiting_module} <- waiting, - module == waiting_module, - not pid in entries, - do: pid + result = Map.put(result, {kind, module}, true) + spawn_workers(available ++ queue, spawned, waiting, files, result, warnings, state) - spawn_compilers(available ++ entries, original, output, callbacks, - waiting, queued, schedulers, [{:module, module}|result]) + {:module_available, child, ref, file, module, binary, checker_info} -> + state.each_module.(file, module, binary) - {:waiting, kind, child, ref, on} -> - defined = fn {k, m} -> on == m and k in [kind, :module] end + # Release the module loader which is waiting for an ack + send(child, {ref, :ack}) + + available = + for {:module, _, ref, _, ^module, _defining, _deadlock} <- waiting, + do: {ref, :found} + + result = Map.put(result, {:module, module}, {binary, checker_info}) + spawn_workers(available ++ queue, spawned, waiting, files, result, warnings, state) + + # If we are simply requiring files, we do not add to waiting. + {:waiting, _kind, child, ref, _file_pid, _on, _defining, _deadlock} when output == :require -> + send(child, {ref, :not_found}) + spawn_workers(queue, spawned, waiting, files, result, warnings, state) + + {:waiting, kind, child_pid, ref, file_pid, on, defining, deadlock?} -> + # If we already got what we were waiting for, do not put it on waiting. + # If we're waiting on ourselves, send :found so that we can crash with + # a better error. + {files, waiting} = + if Map.has_key?(result, {kind, on}) or on in defining do + send(child_pid, {ref, :found}) + {files, waiting} + else + files = update_timing(files, file_pid, :compiling) + {files, [{kind, child_pid, ref, file_pid, on, defining, deadlock?} | waiting]} + end + + spawn_workers(queue, spawned, waiting, files, result, warnings, state) + + :threshold_check -> + files = + for data <- files do + if data.warned or List.keymember?(waiting, data.pid, 1) do + data + else + data = update_timing(data, :compiling) + data = maybe_warn_long_compilation(data, state) + data + end + end - # Oops, we already got it, do not put it on waiting. - if :lists.any(defined, result) do - send child, {ref, :ready} - else - waiting = [{kind, child, ref, on}|waiting] + timer_ref = Process.send_after(self(), :threshold_check, state.long_compilation_threshold) + state = %{state | timer_ref: timer_ref} + spawn_workers(queue, spawned, waiting, files, result, warnings, state) + + {:warning, file, location, message} -> + file = file && Path.absname(file) + message = :unicode.characters_to_binary(message) + warning = {file, location, message} + wait_for_messages(queue, spawned, waiting, files, result, [warning | warnings], state) + + {:file_ok, child_pid, ref, file, lexical} -> + state.each_file.(file, lexical) + send(child_pid, ref) + + discard_down(child_pid) + new_files = discard_and_maybe_log_file(files, child_pid, state) + + # Sometimes we may have spurious entries in the waiting list + # because someone invoked try/rescue UndefinedFunctionError + new_waiting = List.keydelete(waiting, child_pid, 1) + spawn_workers(queue, spawned - 1, new_waiting, new_files, result, warnings, state) + + {:file_cancel, child_pid} -> + discard_down(child_pid) + new_files = Enum.reject(files, &(&1.pid == child_pid)) + spawn_workers(queue, spawned - 1, waiting, new_files, result, warnings, state) + + {:file_error, child_pid, file, {kind, reason, stack}} -> + print_error(file, kind, reason, stack) + discard_down(child_pid) + files |> Enum.reject(&(&1.pid == child_pid)) |> terminate() + {{:error, [to_error(file, kind, reason, stack)], warnings}, state} + + {:DOWN, ref, :process, pid, reason} -> + waiting = List.keydelete(waiting, pid, 1) + + case handle_down(files, ref, reason) do + :ok -> wait_for_messages(queue, spawned - 1, waiting, files, result, warnings, state) + {:error, errors} -> {{:error, errors, warnings}, state} end + end + end - spawn_compilers(entries, original, output, callbacks, waiting, queued, schedulers, result) + defp update_timing(files, pid, key) do + Enum.map(files, fn data -> + if data.pid == pid do + time = System.monotonic_time() + %{data | key => data[key] + time - data.timestamp, timestamp: time} + else + data + end + end) + end - {:DOWN, _down_ref, :process, down_pid, {:compiled, file}} -> - if callback = Keyword.get(callbacks, :each_file) do - callback.(file) - end + defp update_timing(data, key) do + time = System.monotonic_time() + %{data | key => data[key] + time - data.timestamp, timestamp: time} + end - # Sometimes we may have spurious entries in the waiting - # list because someone invoked try/rescue UndefinedFunctionError - new_entries = List.delete(entries, down_pid) - new_queued = List.keydelete(queued, down_pid, 0) - new_waiting = List.keydelete(waiting, down_pid, 1) - spawn_compilers(new_entries, original, output, callbacks, new_waiting, new_queued, schedulers, result) + defp maybe_warn_long_compilation(data, state) do + compiling = System.convert_time_unit(data.compiling, :native, :millisecond) - {:DOWN, down_ref, :process, _down_pid, reason} -> - handle_failure(down_ref, reason, entries, waiting, queued) - wait_for_messages(entries, original, output, callbacks, waiting, queued, schedulers, result) + if not data.warned and compiling >= state.long_compilation_threshold do + state.each_long_compilation.(data.file) + %{data | warned: true} + else + data end end - defp handle_failure(ref, reason, entries, waiting, queued) do - if file = find_failure(ref, queued) do - print_failure(file, reason) - if all_missing?(entries, waiting, queued) do - collect_failures(queued, length(queued) - 1) + defp discard_and_maybe_log_file(files, pid, state) do + Enum.reject(files, fn data -> + if data.pid == pid do + data = update_timing(data, :compiling) + data = maybe_warn_long_compilation(data, state) + + if state.profile != :none do + compiling = to_padded_ms(data.compiling) + waiting = to_padded_ms(data.waiting) + relative = Path.relative_to_cwd(data.file) + + IO.puts( + :stderr, + "[profile] #{compiling}ms compiling + #{waiting}ms waiting for #{relative}" + ) + end + + true + else + false end - exit(1) - end + end) end - defp find_failure(ref, queued) do - case List.keyfind(queued, ref, 1) do - {_child, ^ref, file} -> file - _ -> nil + defp to_padded_ms(time) do + time + |> System.convert_time_unit(:native, :millisecond) + |> Integer.to_string() + |> String.pad_leading(6, " ") + end + + defp discard_down(pid) do + receive do + {:DOWN, _, :process, ^pid, _} -> :ok end end - defp print_failure(_file, {:compiled, _}) do + defp handle_down(_files, _ref, :normal) do :ok end - defp print_failure(file, {:failure, kind, reason, stacktrace}) do - IO.puts "\n== Compilation error on file #{Path.relative_to_cwd(file)} ==" - IO.puts Exception.format(kind, reason, prune_stacktrace(stacktrace)) + defp handle_down(files, ref, reason) do + case Enum.find(files, &(&1.ref == ref)) do + %{pid: pid, file: file} -> + print_error(file, :exit, reason, []) + files |> Enum.reject(&(&1.pid == pid)) |> terminate() + {:error, [to_error(file, :exit, reason, [])]} + + nil -> + :ok + end end - defp print_failure(file, reason) do - IO.puts "\n== Compilation error on file #{Path.relative_to_cwd(file)} ==" - IO.puts Exception.format(:exit, reason, []) + defp handle_deadlock(waiting, files) do + deadlock = + for %{pid: pid, file: file} <- files do + {:current_stacktrace, stacktrace} = Process.info(pid, :current_stacktrace) + Process.exit(pid, :kill) + + {kind, ^pid, _, _, on, _, _} = List.keyfind(waiting, pid, 1) + description = "deadlocked waiting on #{kind} #{inspect(on)}" + error = CompileError.exception(description: description, file: nil, line: nil) + print_error(file, :error, error, stacktrace) + {Path.relative_to_cwd(file), on, description} + end + + IO.puts(""" + + Compilation failed because of a deadlock between files. + The following files depended on the following modules: + """) + + max = + deadlock + |> Enum.map(&(&1 |> elem(0) |> String.length())) + |> Enum.max() + + for {file, mod, _} <- deadlock do + IO.puts([" ", String.pad_leading(file, max), " => " | inspect(mod)]) + end + + IO.puts( + "\nEnsure there are no compile-time dependencies between those files " <> + "and that the modules they reference exist and are correctly named\n" + ) + + for {file, _, description} <- deadlock, do: {Path.absname(file), nil, description} end - @elixir_internals [:elixir_compiler, :elixir_module, :elixir_translator, :elixir_expand] + defp terminate(files) do + for %{pid: pid} <- files, do: Process.exit(pid, :kill) + for %{pid: pid} <- files, do: discard_down(pid) + :ok + end - defp prune_stacktrace([{mod, _, _, _}|t]) when mod in @elixir_internals do - prune_stacktrace(t) + defp print_error(file, kind, reason, stack) do + IO.write([ + "\n== Compilation error in file #{Path.relative_to_cwd(file)} ==\n", + Kernel.CLI.format_error(kind, reason, stack) + ]) end - defp prune_stacktrace([h|t]) do - [h|prune_stacktrace(t)] + defp to_error(file, kind, reason, stack) do + line = get_line(file, reason, stack) + file = Path.absname(file) + message = :unicode.characters_to_binary(Kernel.CLI.format_error(kind, reason, stack)) + {file, line || 0, message} end - defp prune_stacktrace([]) do - [] + defp get_line(_file, %{line: line}, _stack) when is_integer(line) and line > 0 do + line end - defp all_missing?(entries, waiting, queued) do - entries == [] and waiting != [] and - length(waiting) == length(queued) + defp get_line(file, :undef, [{_, _, _, []}, {_, _, _, info} | _]) do + if Keyword.get(info, :file) == to_charlist(Path.relative_to_cwd(file)) do + Keyword.get(info, :line) + end end - defp collect_failures(_queued, 0), do: :ok + defp get_line(file, _reason, [{_, _, _, [file: expanding]}, {_, _, _, info} | _]) + when expanding in ['expanding macro', 'expanding struct'] do + if Keyword.get(info, :file) == to_charlist(Path.relative_to_cwd(file)) do + Keyword.get(info, :line) + end + end - defp collect_failures(queued, remaining) do - receive do - {:DOWN, down_ref, :process, _down_pid, reason} -> - if file = find_failure(down_ref, queued) do - print_failure(file, reason) - collect_failures(queued, remaining - 1) - else - collect_failures(queued, remaining) - end - after - # Give up if no failure appears in 5 seconds - 5000 -> :ok + defp get_line(file, _reason, [{_, _, _, info} | _]) do + if Keyword.get(info, :file) == to_charlist(Path.relative_to_cwd(file)) do + Keyword.get(info, :line) end end + + defp get_line(_, _, _) do + nil + end end diff --git a/lib/elixir/lib/kernel/parallel_require.ex b/lib/elixir/lib/kernel/parallel_require.ex index a9132618b33..869d18d9b09 100644 --- a/lib/elixir/lib/kernel/parallel_require.ex +++ b/lib/elixir/lib/kernel/parallel_require.ex @@ -1,80 +1,17 @@ defmodule Kernel.ParallelRequire do - @moduledoc """ - A module responsible for requiring files in parallel. - """ + @moduledoc false - defmacrop default_callback, do: quote(do: fn x -> x end) + @deprecated "Use Kernel.ParallelCompiler.require/2 instead" + def files(files, callbacks \\ []) - @doc """ - Requires the given files. - - A callback that is invoked every time a file is required - can be optionally given as argument. - - Returns the modules generated by each required file. - """ - def files(files, callback \\ default_callback) do - schedulers = max(:erlang.system_info(:schedulers_online), 2) - spawn_requires(files, [], callback, schedulers, []) - end - - defp spawn_requires([], [], _callback, _schedulers, result), do: result - - defp spawn_requires([], waiting, callback, schedulers, result) do - wait_for_messages([], waiting, callback, schedulers, result) - end - - defp spawn_requires(files, waiting, callback, schedulers, result) when length(waiting) >= schedulers do - wait_for_messages(files, waiting, callback, schedulers, result) - end - - defp spawn_requires([h|t], waiting, callback, schedulers, result) do - parent = self - - compiler_pid = :erlang.get(:elixir_compiler_pid) - ensure_compiled = :erlang.get(:elixir_ensure_compiled) - {:error_handler, handler} = :erlang.process_info(parent, :error_handler) - - {pid, ref} = :erlang.spawn_monitor fn -> - if compiler_pid != :undefined do - :erlang.put(:elixir_compiler_pid, compiler_pid) - end - - if ensure_compiled != :undefined do - :erlang.put(:elixir_ensure_compiled, ensure_compiled) - end - - :erlang.process_flag(:error_handler, handler) - - exit(try do - new = Code.require_file(h) || [] - {:required, Enum.map(new, &elem(&1, 0)), h} - catch - kind, reason -> - {:failure, kind, reason, System.stacktrace} - end) - end - - spawn_requires(t, [{pid, ref}|waiting], callback, schedulers, result) + def files(files, callback) when is_function(callback, 1) do + files(files, each_file: callback) end - defp wait_for_messages(files, waiting, callback, schedulers, result) do - receive do - {:DOWN, ref, :process, pid, status} -> - tuple = {pid, ref} - if tuple in waiting do - case status do - {:required, mods, file} -> - callback.(file) - result = mods ++ result - waiting = List.delete(waiting, tuple) - {:failure, kind, reason, stacktrace} -> - :erlang.raise(kind, reason, stacktrace) - other -> - :erlang.raise(:exit, other, []) - end - end - spawn_requires(files, waiting, callback, schedulers, result) + def files(files, options) when is_list(options) do + case Kernel.ParallelCompiler.require(files, options) do + {:ok, modules, _} -> modules + {:error, _, _} -> exit({:shutdown, 1}) end end end diff --git a/lib/elixir/lib/kernel/special_forms.ex b/lib/elixir/lib/kernel/special_forms.ex index b3b97bcf8b1..47bc2eb77ba 100644 --- a/lib/elixir/lib/kernel/special_forms.ex +++ b/lib/elixir/lib/kernel/special_forms.ex @@ -1,156 +1,127 @@ defmodule Kernel.SpecialForms do @moduledoc """ - In this module we define Elixir special forms. Special forms - cannot be overridden by the developer and are the basic - building blocks of Elixir code. + Special forms are the basic building blocks of Elixir, and therefore + cannot be overridden by the developer. - Some of those forms are lexical (like `alias`, `case`, etc). - The macros `{}` and `<<>>` are also special forms used to define - tuple and binary data structures respectively. + The `Kernel.SpecialForms` module consists solely of macros that can be + invoked anywhere in Elixir code without the use of the + `Kernel.SpecialForms.` prefix. This is possible because they all have + been automatically imported, in the same fashion as the functions and + macros from the `Kernel` module. - This module also documents Elixir's pseudo variables (`__ENV__`, - `__MODULE__`, `__DIR__` and `__CALLER__`). Pseudo variables return - information about Elixir's compilation environment and can only - be read, never assigned to. + These building blocks are defined in this module. Some of these special forms are lexical (such as + `alias/2` and `case/2`). The macros `{}/1` and `<<>>/1` are also special + forms used to define tuple and binary data structures respectively. - Finally, it also documents 2 special forms, `__block__` and - `__aliases__`, which are not intended to be called directly by the + This module also documents macros that return information about Elixir's + compilation environment, such as (`__ENV__/0`, `__MODULE__/0`, `__DIR__/0`, + `__STACKTRACE__/0`, and `__CALLER__/0`). + + Additionally, it documents two special forms, `__block__/1` and + `__aliases__/1`, which are not intended to be called directly by the developer but they appear in quoted contents since they are essential in Elixir's constructs. """ + defmacrop error!(args) do + quote do + _ = unquote(args) + + message = + "Elixir's special forms are expanded by the compiler and must not be invoked directly" + + :erlang.error(RuntimeError.exception(message)) + end + end + @doc """ Creates a tuple. - Only two item tuples are considered literals in Elixir. - Therefore all other tuples are represented in the AST - as a call to the special form `:{}`. + More information about the tuple data type and about functions to manipulate + tuples can be found in the `Tuple` module; some functions for working with + tuples are also available in `Kernel` (such as `Kernel.elem/2` or + `Kernel.tuple_size/1`). - Conveniences for manipulating tuples can be found in the - `Tuple` module. Some functions for working with tuples are - also available in `Kernel`, namely `Kernel.elem/2`, - `Kernel.put_elem/3` and `Kernel.tuple_size/1`. + ## AST representation - ## Examples + Only two-element tuples are considered literals in Elixir and return themselves + when quoted. Therefore, all other tuples are represented in the AST as calls to + the `:{}` special form. - iex> {1, 2, 3} - {1, 2, 3} + iex> quote do + ...> {1, 2} + ...> end + {1, 2} - iex> quote do: {1, 2, 3} - {:{}, [], [1,2,3]} + iex> quote do + ...> {1, 2, 3} + ...> end + {:{}, [], [1, 2, 3]} """ - defmacro unquote(:{})(args) + defmacro unquote(:{})(args), do: error!([args]) @doc """ Creates a map. - Maps are key-value stores where keys are compared - using the match operator (`===`). Maps can be created with - the `%{}` special form where keys are associated via `=>`: - - %{1 => 2} - - Maps also support the keyword notation, as other special forms, - as long as they are at the end of the argument list: - - %{hello: :world, with: :keywords} - %{:hello => :world, with: :keywords} - - If a map has duplicated keys, the last key will always have - higher precedence: - - iex> %{a: :b, a: :c} - %{a: :c} - - Conveniences for manipulating maps can be found in the - `Map` module. - - ## Access syntax - - Besides the access functions available in the `Map` module, - like `Map.get/3` and `Map.fetch/2`, a map can be accessed using the - `.` operator: - - iex> map = %{a: :b} - iex> map.a - :b - - Note that the `.` operator expects the field to exist in the map. - If not, an `ArgumentError` is raised. - - ## Update syntax - - Maps also support an update syntax: - - iex> map = %{:a => :b} - iex> %{map | :a => :c} - %{:a => :c} - - Notice the update syntax requires the given keys to exist. - Trying to update a key that does not exist will raise an `ArgumentError`. + See the `Map` module for more information about maps, their syntax, and ways to + access and manipulate them. ## AST representation - Regardless if `=>` or the keywords syntax is used, Maps are - always represented internally as a list of two-items tuples - for simplicity: + Regardless of whether `=>` or the keyword syntax is used, key-value pairs in + maps are always represented internally as a list of two-element tuples for + simplicity: - iex> quote do: %{:a => :b, c: :d} - {:%{}, [], [{:a, :b}, {:c, :d}]} + iex> quote do + ...> %{"a" => :b, c: :d} + ...> end + {:%{}, [], [{"a", :b}, {:c, :d}]} """ - defmacro unquote(:%{})(args) + defmacro unquote(:%{})(args), do: error!([args]) @doc """ - Creates a struct. + Matches on or builds a struct. A struct is a tagged map that allows developers to provide default values for keys, tags to be used in polymorphic dispatches and compile time assertions. - To define a struct, you just need to implement the `__struct__/0` - function in a module: - - defmodule User do - def __struct__ do - %{name: "josé", age: 27} - end - end - - In practice though, structs are usually defined with the - `Kernel.defstruct/1` macro: + Structs are usually defined with the `Kernel.defstruct/1` macro: defmodule User do - defstruct name: "josé", age: 27 + defstruct name: "john", age: 27 end Now a struct can be created as follows: %User{} - Underneath a struct is just a map with a `__struct__` field + Underneath a struct is just a map with a `:__struct__` key pointing to the `User` module: - %User{} == %{__struct__: User, name: "josé", age: 27} + %User{} == %{__struct__: User, name: "john", age: 27} - A struct also validates that the given keys are part of the defined - struct. The example below will fail because there is no key - `:full_name` in the `User` struct: + The struct fields can be given when building the struct: - %User{full_name: "José Valim"} + %User{age: 31} + #=> %{__struct__: User, name: "john", age: 31} - Note that a struct specifies a minimum set of keys required - for operations. Other keys can be added to structs via the - regular map operations: + Or also on pattern matching to extract values out: - user = %User{} - Map.put(user, :a_non_struct_key, :value) + %User{age: age} = user An update operation specific for structs is also available: %User{user | age: 28} + The advantage of structs is that they validate that the given + keys are part of the defined struct. The example below will fail + because there is no key `:full_name` in the `User` struct: + + %User{full_name: "john doe"} + The syntax above will guarantee the given keys are valid at compilation time and it will guarantee at runtime the given argument is a struct, failing with `BadStructError` otherwise. @@ -159,147 +130,248 @@ defmodule Kernel.SpecialForms do any of the protocols implemented for maps. Check `Kernel.defprotocol/2` for more information on how structs can be used with protocols for polymorphic dispatch. Also - see `Kernel.struct/2` for examples on how to create and update - structs dynamically. + see `Kernel.struct/2` and `Kernel.struct!/2` for examples on + how to create and update structs dynamically. + + ## Pattern matching on struct names + + Besides allowing pattern matching on struct fields, such as: + + %User{age: age} = user + + Structs also allow pattern matching on the struct name: + + %struct_name{} = user + struct_name #=> User + + You can also assign the struct name to `_` when you want to + check if something is a struct but you are not interested in + its name: + + %_{} = user + """ - defmacro unquote(:%)(struct, map) + defmacro unquote(:%)(struct, map), do: error!([struct, map]) @doc """ Defines a new bitstring. ## Examples - iex> << 1, 2, 3 >> - << 1, 2, 3 >> + iex> <<1, 2, 3>> + <<1, 2, 3>> + + ## Types + + A bitstring is made of many segments and each segment has a + type. There are 9 types used in bitstrings: - ## Bitstring types + - `integer` + - `float` + - `bits` (alias for `bitstring`) + - `bitstring` + - `binary` + - `bytes` (alias for `binary`) + - `utf8` + - `utf16` + - `utf32` - A bitstring is made of many segments. Each segment has a - type, which defaults to integer: + When no type is specified, the default is `integer`: iex> <<1, 2, 3>> <<1, 2, 3>> Elixir also accepts by default the segment to be a literal - string or a literal char list, which are by expanded to integers: + string which expands to integers: iex> <<0, "foo">> <<0, 102, 111, 111>> - Any other type needs to be explicitly tagged. For example, - in order to store a float type in the binary, one has to do: + Binaries need to be explicitly tagged as `binary`: + + iex> rest = "oo" + iex> <<102, rest::binary>> + "foo" + + The `utf8`, `utf16`, and `utf32` types are for Unicode code points. They + can also be applied to literal strings and charlists: - iex> <<3.14 :: float>> - <<64, 9, 30, 184, 81, 235, 133, 31>> + iex> <<"foo"::utf16>> + <<0, 102, 0, 111, 0, 111>> + iex> <<"foo"::utf32>> + <<0, 0, 0, 102, 0, 0, 0, 111, 0, 0, 0, 111>> - This also means that variables need to be explicitly tagged, - otherwise Elixir defaults to integer: + Otherwise we get an `ArgumentError` when constructing the binary: - iex> rest = "oo" - iex> <<102, rest>> + rest = "oo" + <<102, rest>> ** (ArgumentError) argument error - We can solve this by explicitly tagging it as a binary: + ## Options - <<102, rest :: binary>> + Many options can be given by using `-` as separator. Order is + arbitrary, so the following are all equivalent: - The type can be integer, float, bitstring/bits, binary/bytes, - utf8, utf16 or utf32, e.g.: + <<102::integer-native, rest::binary>> + <<102::native-integer, rest::binary>> + <<102::unsigned-big-integer, rest::binary>> + <<102::unsigned-big-integer-size(8), rest::binary>> + <<102::unsigned-big-integer-8, rest::binary>> + <<102::8-integer-big-unsigned, rest::binary>> + <<102, rest::binary>> - <<102 :: float, rest :: binary>> + ### Unit and Size - An integer can be any arbitrary precision integer. A float is an - IEEE 754 binary32 or binary64 floating point number. A bitstring - is an arbitrary series of bits. A binary is a special case of - bitstring that has a total size divisible by 8. + The length of the match is equal to the `unit` (a number of bits) times the + `size` (the number of repeated segments of length `unit`). - The utf8, utf16, and utf32 types are for unicode codepoints. They - can also be applied to literal strings and char lists: + Type | Default Unit + --------- | ------------ + `integer` | 1 bit + `float` | 1 bit + `binary` | 8 bits - iex> <<"foo" :: utf16>> - <<0,102,0,111,0,111>> + Sizes for types are a bit more nuanced. The default size for integers is 8. - The bits type is an alias for bitstring. The bytes type is an - alias for binary. + For floats, it is 64. For floats, `size * unit` must result in 16, 32, or 64, + corresponding to [IEEE 754](https://en.wikipedia.org/wiki/IEEE_floating_point) + binary16, binary32, and binary64, respectively. - The signedness can also be given as signed or unsigned. The - signedness only matters for matching and relevant only for - integers. If unspecified, it defaults to unsigned. Example: + For binaries, the default is the size of the binary. Only the last binary in a + match can use the default size. All others must have their size specified + explicitly, even if the match is unambiguous. For example: - iex> <<-100 :: signed, _rest :: binary>> = <<-100, "foo">> - <<156,102,111,111>> + iex> <> = <<"Frank the Walrus">> + "Frank the Walrus" + iex> {name, species} + {"Frank", "Walrus"} - This match would have failed if we did not specify that the - value -100 is signed. If we're matching into a variable instead - of a value, the signedness won't be checked; rather, the number - will simply be interpreted as having the given (or implied) - signedness, e.g.: + The size can be a variable or any valid guard expression: - iex> <> = <<-100, "foo">> - iex> val - 156 + iex> name_size = 5 + iex> <> = <<"Frank the Walrus">> + iex> {name, species} + {"Frank", "Walrus"} - Here, `val` is interpreted as unsigned. + The size can access prior variables defined in the binary itself: - The endianness of a segment can be big, little or native (the - latter meaning it will be resolved at VM load time). Passing - many options can be done by giving a list: + iex> <> = <<5, "Frank the Walrus">> + iex> {name, species} + {"Frank", "Walrus"} - <<102 :: [integer, native], rest :: binary>> + However, it cannot access variables defined in the match outside of the binary/bitstring: - Or: + {name_size, <>} = {5, <<"Frank the Walrus">>} + ** (CompileError): undefined variable "name_size" in bitstring segment - <<102 :: [unsigned, big, integer], rest :: binary>> + Failing to specify the size for the non-last causes compilation to fail: - And so on. + <> = <<"Frank the Walrus">> + ** (CompileError): a binary field without size is only allowed at the end of a binary pattern - Endianness only makes sense for integers and some UTF code - point types (utf16 and utf32). + #### Shortcut Syntax - Finally, we can also specify size and unit for each segment. The - unit is multiplied by the size to give the effective size of - the segment in bits. The default unit for integers, floats, - and bitstrings is 1. For binaries, it is 8. + Size and unit can also be specified using a syntax shortcut + when passing integer values: - Since integers are default, the default unit is 1. The example below - matches because the string "foo" takes 24 bits and we match it - against a segment of 24 bits, 8 of which are taken by the integer - 102 and the remaining 16 bits are specified on the rest. + iex> x = 1 + iex> <> == <> + true + iex> <> == <> + true - iex> <<102, _rest :: size(16)>> = "foo" - "foo" + This syntax reflects the fact the effective size is given by + multiplying the size by the unit. - We can also match by specifying size and unit explicitly: + ### Modifiers - iex> <<102, _rest :: [size(2), unit(8)]>> = "foo" - "foo" + Some types have associated modifiers to clear up ambiguity in byte + representation. - However, if we expect a size of 32, it won't match: + Modifier | Relevant Type(s) + -------------------- | ---------------- + `signed` | `integer` + `unsigned` (default) | `integer` + `little` | `integer`, `float`, `utf16`, `utf32` + `big` (default) | `integer`, `float`, `utf16`, `utf32` + `native` | `integer`, `utf16`, `utf32` - iex> <<102, _rest :: size(32)>> = "foo" - ** (MatchError) no match of right hand side value: "foo" + ### Sign - Size and unit are not applicable to utf8, utf16, and utf32. + Integers can be `signed` or `unsigned`, defaulting to `unsigned`. - The default size for integers is 8. For floats, it is 64. For - binaries, it is the size of the binary. Only the last binary - in a binary match can use the default size (all others must - have their size specified explicitly). + iex> <> = <<-100>> + <<156>> + iex> int + 156 + iex> <> = <<-100>> + <<156>> + iex> int + -100 - Size can also be specified using a syntax shortcut. Instead of - writing `size(8)`, one can write just `8` and it will be interpreted - as `size(8)` + `signed` and `unsigned` are only used for matching binaries (see below) and + are only used for integers. - iex> << 1 :: 3 >> == << 1 :: size(3) >> - true + iex> <<-100::signed, _rest::binary>> = <<-100, "foo">> + <<156, 102, 111, 111>> + + ### Endianness + + Elixir has three options for endianness: `big`, `little`, and `native`. + The default is `big`: + + iex> <> = <<0, 1>> + <<0, 1>> + iex> number + 256 + iex> <> = <<0, 1>> + <<0, 1>> + iex> number + 1 + + `native` is determined by the VM at startup and will depend on the + host operating system. + + ## Binary/Bitstring Matching + + Binary matching is a powerful feature in Elixir that is useful for extracting + information from binaries as well as pattern matching. + + Binary matching can be used by itself to extract information from binaries: + + iex> <<"Hello, ", place::binary>> = "Hello, World" + "Hello, World" + iex> place + "World" + + Or as a part of function definitions to pattern match: + + defmodule ImageTyper do + @png_signature <<137::size(8), 80::size(8), 78::size(8), 71::size(8), + 13::size(8), 10::size(8), 26::size(8), 10::size(8)>> + @jpg_signature <<255::size(8), 216::size(8)>> + + def type(<<@png_signature, _rest::binary>>), do: :png + def type(<<@jpg_signature, _rest::binary>>), do: :jpg + def type(_), do: :unknown + end - For floats, `size * unit` must result in 32 or 64, corresponding - to binary32 and binary64, respectively. + ### Performance & Optimizations + + The Erlang compiler can provide a number of optimizations on binary creation + and matching. To see optimization output, set the `bin_opt_info` compiler + option: + + ERL_COMPILER_OPTIONS=bin_opt_info mix compile + + To learn more about specific optimizations and performance considerations, + check out the + ["Constructing and matching binaries" chapter of the Erlang's Efficiency Guide](https://www.erlang.org/doc/efficiency_guide/binaryhandling.html). """ - defmacro unquote(:<<>>)(args) + defmacro unquote(:<<>>)(args), do: error!([args]) @doc """ - Defines a remote call or an alias. + Dot operator. Defines a remote call, a call to an anonymous function, or an alias. The dot (`.`) in Elixir can be used for remote calls: @@ -307,8 +379,16 @@ defmodule Kernel.SpecialForms do "foo" In this example above, we have used `.` to invoke `downcase` in the - `String` alias, passing "FOO" as argument. We can also use the dot - for creating aliases: + `String` module, passing `"FOO"` as argument. + + The dot may be used to invoke anonymous functions too: + + iex> (fn n -> n end).(7) + 7 + + in which case there is a function on the left hand side. + + We can also use the dot for creating aliases: iex> Hello.World Hello.World @@ -318,7 +398,7 @@ defmodule Kernel.SpecialForms do ## Syntax - The right side of `.` may be a word starting in upcase, which represents + The right side of `.` may be a word starting with an uppercase letter, which represents an alias, a word starting with lowercase or underscore, any valid language operator or any name wrapped in single- or double-quotes. Those are all valid examples: @@ -326,41 +406,26 @@ defmodule Kernel.SpecialForms do iex> Kernel.Sample Kernel.Sample - iex> Kernel.length([1,2,3]) + iex> Kernel.length([1, 2, 3]) 3 iex> Kernel.+(1, 2) 3 - iex> Kernel."length"([1,2,3]) + iex> Kernel."+"(1, 2) 3 - iex> Kernel.'+'(1, 2) - 3 - - Note that `Kernel."HELLO"` will be treated as a remote call and not an alias. - This choice was done so every time single- or double-quotes are used, we have - a remote call irregardless of the quote contents. This decision is also reflected - in the quoted expressions discussed below. - - ## Runtime (dynamic) behaviour + Wrapping the function name in single- or double-quotes is always a + remote call. Therefore `Kernel."Foo"` will attempt to call the function "Foo" + and not return the alias `Kernel.Foo`. This is done by design as module names + are more strict than function names. - The result returned by `.` is always specified by the right-side: + When the dot is used to invoke an anonymous function there is only one + operand, but it is still written using a postfix notation: - iex> x = String - iex> x.downcase("FOO") - "foo" - iex> x.Sample - String.Sample - - In case the right-side is also dynamic, `.`'s behaviour can be reproduced - at runtime via `apply/3` and `Module.concat/2`: - - iex> apply(:erlang, :+, [1,2]) - 3 - - iex> Module.concat(Kernel, Sample) - Kernel.Sample + iex> negate = fn n -> -n end + iex> negate.(7) + -7 ## Quoted expression @@ -368,32 +433,39 @@ defmodule Kernel.SpecialForms do forms. When the right side starts with a lowercase letter (or underscore): - iex> quote do: String.downcase("FOO") + iex> quote do + ...> String.downcase("FOO") + ...> end {{:., [], [{:__aliases__, [alias: false], [:String]}, :downcase]}, [], ["FOO"]} - Notice we have an inner tuple, containing the atom `:.` representing + Note that we have an inner tuple, containing the atom `:.` representing the dot as first element: {:., [], [{:__aliases__, [alias: false], [:String]}, :downcase]} This tuple follows the general quoted expression structure in Elixir, with the name as first argument, some keyword list as metadata as second, - and the number of arguments as third. In this case, the arguments is the - alias `String` and the atom `:downcase`. The second argument is **always** - an atom: + and the list of arguments as third. In this case, the arguments are the + alias `String` and the atom `:downcase`. The second argument in a remote call + is **always** an atom. - iex> quote do: String."downcase"("FOO") - {{:., [], [{:__aliases__, [alias: false], [:String]}, :downcase]}, [], ["FOO"]} + In the case of calls to anonymous functions, the inner tuple with the dot + special form has only one argument, reflecting the fact that the operator is + unary: - The tuple containing `:.` is wrapped in another tuple, which actually - represents the function call, and has `"FOO"` as argument. + iex> quote do + ...> negate.(0) + ...> end + {{:., [], [{:negate, [], __MODULE__}]}, [], [0]} When the right side is an alias (i.e. starts with uppercase), we get instead: - iex> quote do: Hello.World + iex> quote do + ...> Hello.World + ...> end {:__aliases__, [alias: false], [:Hello, :World]} - We got into more details about aliases in the `__aliases__` special form + We go into more details about aliases in the `__aliases__/1` special form documentation. ## Unquoting @@ -401,27 +473,31 @@ defmodule Kernel.SpecialForms do We can also use unquote to generate a remote call in a quoted expression: iex> x = :downcase - iex> quote do: String.unquote(x)("FOO") + iex> quote do + ...> String.unquote(x)("FOO") + ...> end {{:., [], [{:__aliases__, [alias: false], [:String]}, :downcase]}, [], ["FOO"]} - Similar to `Kernel."HELLO"`, `unquote(x)` will always generate a remote call, + Similar to `Kernel."FUNCTION_NAME"`, `unquote(x)` will always generate a remote call, independent of the value of `x`. To generate an alias via the quoted expression, one needs to rely on `Module.concat/2`: iex> x = Sample - iex> quote do: Module.concat(String, unquote(x)) + iex> quote do + ...> Module.concat(String, unquote(x)) + ...> end {{:., [], [{:__aliases__, [alias: false], [:Module]}, :concat]}, [], [{:__aliases__, [alias: false], [:String]}, Sample]} """ - defmacro unquote(:.)(left, right) + defmacro unquote(:.)(left, right), do: error!([left, right]) @doc """ - `alias` is used to setup aliases, often useful with modules names. + `alias/2` is used to set up aliases, often useful with modules' names. ## Examples - `alias` can be used to setup an alias for any module: + `alias/2` can be used to set up an alias for any module: defmodule Math do alias MyKeyword, as: Keyword @@ -434,10 +510,10 @@ defmodule Kernel.SpecialForms do In case one wants to access the original `Keyword`, it can be done by accessing `Elixir`: - Keyword.values #=> uses MyKeyword.values + Keyword.values #=> uses MyKeyword.values Elixir.Keyword.values #=> uses Keyword.values - Notice that calling `alias` without the `as:` option automatically + Note that calling `alias` without the `:as` option automatically sets an alias based on the last part of the module. For example: alias Foo.Bar.Baz @@ -446,9 +522,19 @@ defmodule Kernel.SpecialForms do alias Foo.Bar.Baz, as: Baz + We can also alias multiple modules in one line: + + alias Foo.{Bar, Baz, Biz} + + Is the same as: + + alias Foo.Bar + alias Foo.Baz + alias Foo.Biz + ## Lexical scope - `import`, `require` and `alias` are called directives and all + `import/2`, `require/2` and `alias/2` are called directives and all have lexical scope. This means you can set up aliases inside specific functions and it won't affect the overall scope. @@ -462,20 +548,20 @@ defmodule Kernel.SpecialForms do was not explicitly defined. Both warning behaviours could be changed by explicitly - setting the `:warn` option to true or false. + setting the `:warn` option to `true` or `false`. + """ - defmacro alias(module, opts) + defmacro alias(module, opts), do: error!([module, opts]) @doc """ - Requires a given module to be compiled and loaded. + Requires a module in order to use its macros. ## Examples - Notice that usually modules should not be required before usage, - the only exception is if you want to use the macros from a module. - In such cases, you need to explicitly require them. + Public functions in modules are globally available, but in order to use + macros, you need to opt-in by requiring the module they are defined in. - Let's suppose you created your own `if` implementation in the module + Let's suppose you created your own `if/2` implementation in the module `MyMacros`. If you want to invoke it, you need to first explicitly require the `MyMacros`: @@ -488,17 +574,17 @@ defmodule Kernel.SpecialForms do ## Alias shortcut - `require` also accepts `as:` as an option so it automatically sets - up an alias. Please check `alias` for more information. + `require/2` also accepts `:as` as an option so it automatically sets + up an alias. Please check `alias/2` for more information. """ - defmacro require(module, opts) + defmacro require(module, opts), do: error!([module, opts]) @doc """ - Imports function and macros from other modules. + Imports functions and macros from other modules. - `import` allows one to easily access functions or macros from - others modules without using the qualified name. + `import/2` allows one to easily access functions or macros from + other modules without using the qualified name. ## Examples @@ -508,21 +594,22 @@ defmodule Kernel.SpecialForms do iex> import List iex> flatten([1, [2], 3]) - [1,2,3] + [1, 2, 3] ## Selector By default, Elixir imports functions and macros from the given - module, except the ones starting with underscore (which are + module, except the ones starting with an underscore (which are usually callbacks): import List - A developer can filter to import only macros or functions via - the only option: + A developer can filter to import only functions, macros, or sigils + (which can be functions or macros) via the `:only` option: import List, only: :functions import List, only: :macros + import Kernel, only: :sigils Alternatively, Elixir allows a developer to pass pairs of name/arities to `:only` or `:except` as a fine grained control @@ -531,26 +618,37 @@ defmodule Kernel.SpecialForms do import List, only: [flatten: 1] import String, except: [split: 2] - Notice that calling `except` for a previously declared `import` - simply filters the previously imported elements. For example: + Importing the same module again will erase the previous imports, + except when the `except` option is used, which is always exclusive + on a previously declared `import/2`. If there is no previous import, + then it applies to all functions and macros in the module. For + example: - import List, only: [flatten: 1, keyfind: 3] + import List, only: [flatten: 1, keyfind: 4] import List, except: [flatten: 1] - After the two import calls above, only `List.keyfind/3` will be + After the two import calls above, only `List.keyfind/4` will be imported. + ## Underscore functions + + By default functions starting with `_` are not imported. If you really want + to import a function starting with `_` you must explicitly include it in the + `:only` selector. + + import File.Stream, only: [__build__: 3] + ## Lexical scope - It is important to notice that `import` is lexical. This means you + It is important to note that `import/2` is lexical. This means you can import specific macros inside specific functions: defmodule Math do def some_function do - # 1) Disable `if/2` from Kernel + # 1) Disable "if/2" from Kernel import Kernel, except: [if: 2] - # 2) Require the new `if` macro from MyMacros + # 2) Require the new "if/2" macro from MyMacros import MyMacros # 3) Use the new macro @@ -574,7 +672,7 @@ defmodule Kernel.SpecialForms do was not explicitly defined. Both warning behaviours could be changed by explicitly - setting the `:warn` option to true or false. + setting the `:warn` option to `true` or `false`. ## Ambiguous function/macro names @@ -583,7 +681,7 @@ defmodule Kernel.SpecialForms do if an ambiguous call to `foo/1` is actually made; that is, the errors are emitted lazily, not eagerly. """ - defmacro import(module, opts) + defmacro import(module, opts), do: error!([module, opts]) @doc """ Returns the current environment information as a `Macro.Env` struct. @@ -591,23 +689,23 @@ defmodule Kernel.SpecialForms do In the environment you can access the current filename, line numbers, set up aliases, the current function and others. """ - defmacro __ENV__ + defmacro __ENV__, do: error!([]) @doc """ Returns the current module name as an atom or `nil` otherwise. - Although the module can be accessed in the `__ENV__`, this macro + Although the module can be accessed in the `__ENV__/0`, this macro is a convenient shortcut. """ - defmacro __MODULE__ + defmacro __MODULE__, do: error!([]) @doc """ - Returns the current directory as a binary. + Returns the absolute path of the directory of the current file as a binary. Although the directory can be accessed as `Path.dirname(__ENV__.file)`, this macro is a convenient shortcut. """ - defmacro __DIR__ + defmacro __DIR__, do: error!([]) @doc """ Returns the current calling environment as a `Macro.Env` struct. @@ -615,29 +713,42 @@ defmodule Kernel.SpecialForms do In the environment you can access the filename, line numbers, set up aliases, the function and others. """ - defmacro __CALLER__ + defmacro __CALLER__, do: error!([]) + + @doc """ + Returns the stacktrace for the currently handled exception. + + It is available only in the `catch` and `rescue` clauses of `try/1` + expressions. + + To retrieve the stacktrace of the current process, use + `Process.info(self(), :current_stacktrace)` instead. + """ + @doc since: "1.7.0" + defmacro __STACKTRACE__, do: error!([]) @doc """ - Accesses an already bound variable in match clauses. + Pin operator. Accesses an already bound variable in match clauses. ## Examples Elixir allows variables to be rebound via static single assignment: iex> x = 1 - iex> x = 2 + iex> x = x + 1 iex> x 2 However, in some situations, it is useful to match against an existing - value, instead of rebinding. This can be done with the `^` special form: + value, instead of rebinding. This can be done with the `^` special form, + colloquially known as the pin operator: iex> x = 1 iex> ^x = List.first([1]) iex> ^x = List.first([2]) ** (MatchError) no match of right hand side value: 2 - Note that `^` always refers to the value of x prior to the match. The + Note that `^x` always refers to the value of `x` prior to the match. The following example will match: iex> x = 0 @@ -646,17 +757,45 @@ defmodule Kernel.SpecialForms do 1 """ - defmacro ^(var) + defmacro ^var, do: error!([var]) + + @doc """ + Match operator. Matches the value on the right against the pattern on the left. + """ + defmacro left = right, do: error!([left, right]) + + @doc """ + Type operator. Used by types and bitstrings to specify types. + + This operator is used in two distinct occasions in Elixir. + It is used in typespecs to specify the type of a variable, + function or of a type itself: + + @type number :: integer | float + @spec add(number, number) :: number + + It may also be used in bit strings to specify the type + of a given bit segment: + + <> = bits + + Read the documentation on the `Typespec` page and + `<<>>/1` for more information on typespecs and + bitstrings respectively. + """ + defmacro left :: right, do: error!([left, right]) @doc ~S""" Gets the representation of any expression. ## Examples - quote do: sum(1, 2, 3) - #=> {:sum, [], [1, 2, 3]} + iex> quote do + ...> sum(1, 2, 3) + ...> end + {:sum, [], [1, 2, 3]} - ## Explanation + ## Elixir's AST (Abstract Syntax Tree) Any Elixir code can be represented using Elixir data structures. The building block of Elixir macros is a tuple with three elements, @@ -670,30 +809,15 @@ defmodule Kernel.SpecialForms do * The first element of the tuple is always an atom or another tuple in the same representation. - * The second element of the tuple represents metadata. + * The second element of the tuple represents [metadata](`t:Macro.metadata/0`). * The third element of the tuple are the arguments for the function call. The third argument may be an atom, which is usually a variable (or a local call). - ## Options - - * `:unquote` - when false, disables unquoting. Useful when you have a quote - inside another quote and want to control what quote is able to unquote. - - * `:location` - when set to `:keep`, keeps the current line and file from - quote. Read the Stacktrace information section below for more - information. - - * `:context` - sets the resolution context. - - * `:bind_quoted` - passes a binding to the macro. Whenever a binding is - given, `unquote` is automatically disabled. - - ## Quote literals - Besides the tuple described above, Elixir has a few literals that - when quoted return themselves. They are: + are also part of its AST. Those literals return themselves when + quoted. They are: :sum #=> Atoms 1 #=> Integers @@ -702,11 +826,46 @@ defmodule Kernel.SpecialForms do "strings" #=> Strings {key, value} #=> Tuples with two elements + Any other value, such as a map or a four-element tuple, must be escaped + (`Macro.escape/1`) before being introduced into an AST. + + ## Options + + * `:bind_quoted` - passes a binding to the macro. Whenever a binding is + given, `unquote/1` is automatically disabled. + + * `:context` - sets the resolution context. + + * `:generated` - marks the given chunk as generated so it does not emit warnings. + Currently it only works on special forms (for example, you can annotate a `case` + but not an `if`). + + * `:file` - sets the quoted expressions to have the given file. + + * `:line` - sets the quoted expressions to have the given line. + + * `:location` - when set to `:keep`, keeps the current line and file from + quote. Read the "Stacktrace information" section below for more information. + + * `:unquote` - when `false`, disables unquoting. This means any `unquote` + call will be kept as is in the AST, instead of replaced by the `unquote` + arguments. For example: + + iex> quote do + ...> unquote("hello") + ...> end + "hello" + + iex> quote unquote: false do + ...> unquote("hello") + ...> end + {:unquote, [], ["hello"]} + ## Quote and macros - `quote` is commonly used with macros for code generation. As an exercise, - let's define a macro that multiplies a number by itself (squared). Note - there is no reason to define such as a macro (and it would actually be + `quote/2` is commonly used with macros for code generation. As an exercise, + let's define a macro that multiplies a number by itself (squared). In practice, + there is no reason to define such a macro (and it would actually be seen as a bad practice), but it is simple enough that it allows us to focus on the important aspects of quotes and macros: @@ -721,7 +880,7 @@ defmodule Kernel.SpecialForms do We can invoke it as: import Math - IO.puts "Got #{squared(5)}" + IO.puts("Got #{squared(5)}") At first, there is nothing in this example that actually reveals it is a macro. But what is happening is that, at compilation time, `squared(5)` @@ -731,16 +890,16 @@ defmodule Kernel.SpecialForms do import Math my_number = fn -> - IO.puts "Returning 5" + IO.puts("Returning 5") 5 end - IO.puts "Got #{squared(my_number.())}" + IO.puts("Got #{squared(my_number.())}") The example above will print: Returning 5 Returning 5 - 25 + Got 25 Notice how "Returning 5" was printed twice, instead of just once. This is because a macro receives an expression and not a value (which is what we @@ -768,11 +927,11 @@ defmodule Kernel.SpecialForms do end end - Now invoking `square(my_number.())` as before will print the value just + Now invoking `squared(my_number.())` as before will print the value just once. In fact, this pattern is so common that most of the times you will want - to use the `bind_quoted` option with `quote`: + to use the `bind_quoted` option with `quote/2`: defmodule Math do defmacro squared(x) do @@ -784,7 +943,7 @@ defmodule Kernel.SpecialForms do `:bind_quoted` will translate to the same code as the example above. `:bind_quoted` can be used in many cases and is seen as good practice, - not only because it helps us from running into common mistakes but also + not only because it helps prevent us from running into common mistakes, but also because it allows us to leverage other tools exposed by macros, such as unquote fragments discussed in some sections below. @@ -800,7 +959,8 @@ defmodule Kernel.SpecialForms do import Math squared(5) - x #=> ** (RuntimeError) undefined function or variable: x + x + ** (CompileError) undefined variable x or undefined function x/0 We can see that `x` did not leak to the user context. This happens because Elixir macros are hygienic, a topic we will discuss at length @@ -812,36 +972,42 @@ defmodule Kernel.SpecialForms do defmodule Hygiene do defmacro no_interference do - quote do: a = 1 + quote do + a = 1 + end end end require Hygiene a = 10 - Hygiene.no_interference - a #=> 10 + Hygiene.no_interference() + a + #=> 10 In the example above, `a` returns 10 even if the macro is apparently setting it to 1 because variables defined - in the macro does not affect the context the macro is executed in. + in the macro do not affect the context the macro is executed in. If you want to set or get a variable in the caller's context, you can do it with the help of the `var!` macro: defmodule NoHygiene do defmacro interference do - quote do: var!(a) = 1 + quote do + var!(a) = 1 + end end end require NoHygiene a = 10 - NoHygiene.interference - a #=> 1 + NoHygiene.interference() + a + #=> 1 - Note that you cannot even access variables defined in the same - module unless you explicitly give it a context: + You cannot even access variables defined in the same module unless + you explicitly give it a context: defmodule Hygiene do defmacro write do @@ -857,9 +1023,9 @@ defmodule Kernel.SpecialForms do end end - Hygiene.write - Hygiene.read - #=> ** (RuntimeError) undefined function or variable: a + Hygiene.write() + Hygiene.read() + ** (RuntimeError) undefined variable a or undefined function a/0 For such, you can explicitly pass the current module scope as argument: @@ -878,8 +1044,8 @@ defmodule Kernel.SpecialForms do end end - ContextHygiene.write - ContextHygiene.read + ContextHygiene.write() + ContextHygiene.read() #=> 1 ## Hygiene in aliases @@ -888,48 +1054,58 @@ defmodule Kernel.SpecialForms do Consider the following example: defmodule Hygiene do - alias HashDict, as: D + alias Map, as: M defmacro no_interference do - quote do: D.new + quote do + M.new() + end end end require Hygiene - Hygiene.no_interference #=> #HashDict<[]> + Hygiene.no_interference() + #=> %{} - Notice that, even though the alias `D` is not available + Note that, even though the alias `M` is not available in the context the macro is expanded, the code above works - because `D` still expands to `HashDict`. + because `M` still expands to `Map`. Similarly, even if we defined an alias with the same name before invoking a macro, it won't affect the macro's result: defmodule Hygiene do - alias HashDict, as: D + alias Map, as: M defmacro no_interference do - quote do: D.new + quote do + M.new() + end end end require Hygiene - alias SomethingElse, as: D - Hygiene.no_interference #=> #HashDict<[]> + alias SomethingElse, as: M + Hygiene.no_interference() + #=> %{} In some cases, you want to access an alias or a module defined in the caller. For such, you can use the `alias!` macro: defmodule Hygiene do - # This will expand to Elixir.Nested.hello + # This will expand to Elixir.Nested.hello() defmacro no_interference do - quote do: Nested.hello + quote do + Nested.hello() + end end - # This will expand to Nested.hello for + # This will expand to Nested.hello() for # whatever is Nested in the caller defmacro interference do - quote do: alias!(Nested).hello + quote do + alias!(Nested).hello() + end end end @@ -939,10 +1115,10 @@ defmodule Kernel.SpecialForms do end require Hygiene - Hygiene.no_interference - #=> ** (UndefinedFunctionError) ... + Hygiene.no_interference() + ** (UndefinedFunctionError) ... - Hygiene.interference + Hygiene.interference() #=> "world" end @@ -952,54 +1128,56 @@ defmodule Kernel.SpecialForms do following code: defmodule Hygiene do - defmacrop get_size do + defmacrop get_length do quote do - size("hello") + length([1, 2, 3]) end end - def return_size do - import Kernel, except: [size: 1] - get_size + def return_length do + import Kernel, except: [length: 1] + get_length end end - Hygiene.return_size #=> 5 + Hygiene.return_length() + #=> 3 - Notice how `return_size` returns 5 even though the `size/1` - function is not imported. In fact, even if `return_size` imported - a function from another module, it wouldn't affect the function - result: + Notice how `Hygiene.return_length/0` returns `3` even though the `Kernel.length/1` + function is not imported. In fact, even if `return_length/0` + imported a function with the same name and arity from another + module, it wouldn't affect the function result: - def return_size do - import Dict, only: [size: 1] - get_size + def return_length do + import String, only: [length: 1] + get_length end - Calling this new `return_size` will still return 5 as result. + Calling this new `return_length/0` will still return `3` as result. Elixir is smart enough to delay the resolution to the latest - moment possible. So, if you call `size("hello")` inside quote, - but no `size/1` function is available, it is then expanded in + possible moment. So, if you call `length([1, 2, 3])` inside quote, + but no `length/1` function is available, it is then expanded in the caller: defmodule Lazy do - defmacrop get_size do - import Kernel, except: [size: 1] + defmacrop get_length do + import Kernel, except: [length: 1] quote do - size([a: 1, b: 2]) + length("hello") end end - def return_size do - import Kernel, except: [size: 1] - import Dict, only: [size: 1] - get_size + def return_length do + import Kernel, except: [length: 1] + import String, only: [length: 1] + get_length end end - Lazy.return_size #=> 2 + Lazy.return_length() + #=> 5 ## Stacktrace information @@ -1023,33 +1201,44 @@ defmodule Kernel.SpecialForms do defadd end + require Sample + Sample.add(:one, :two) + ** (ArithmeticError) bad argument in arithmetic expression + adder.ex:5: Sample.add/2 + When using `location: :keep` and invalid arguments are given to `Sample.add/2`, the stacktrace information will point to the file and line inside the quote. Without `location: :keep`, the error is - reported to where `defadd` was invoked. Note `location: :keep` affects + reported to where `defadd` was invoked. `location: :keep` affects only definitions inside the quote. + > **Important:** do not use location: :keep if the function definition + > also `unquote`s some of the macro arguments. If you do so, Elixir + > will store the file definition of the current location but the + > unquoted arguments may contain line information of the macro caller, + > leading to erroneous stacktraces. + ## Binding and unquote fragments - Elixir quote/unquote mechanisms provides a functionality called + Elixir quote/unquote mechanisms provide a functionality called unquote fragments. Unquote fragments provide an easy way to generate functions on the fly. Consider this example: kv = [foo: 1, bar: 2] - Enum.each kv, fn {k, v} -> + Enum.each(kv, fn {k, v} -> def unquote(k)(), do: unquote(v) - end + end) In the example above, we have generated the functions `foo/0` and - `bar/0` dynamically. Now, imagine that, we want to convert this + `bar/0` dynamically. Now, imagine that we want to convert this functionality into a macro: defmacro defkv(kv) do - Enum.map kv, fn {k, v} -> + Enum.map(kv, fn {k, v} -> quote do def unquote(k)(), do: unquote(v) end - end + end) end We can invoke this macro as: @@ -1072,15 +1261,15 @@ defmodule Kernel.SpecialForms do defmacro defkv(kv) do quote do - Enum.each unquote(kv), fn {k, v} -> + Enum.each(unquote(kv), fn {k, v} -> def unquote(k)(), do: unquote(v) - end + end) end end If you try to run our new macro, you will notice it won't even compile, complaining that the variables `k` and `v` - does not exist. This is because of the ambiguity: `unquote(k)` + do not exist. This is because of the ambiguity: `unquote(k)` can either be an unquote fragment, as previously, or a regular unquote as in `unquote(kv)`. @@ -1093,54 +1282,86 @@ defmodule Kernel.SpecialForms do defmacro defkv(kv) do quote bind_quoted: [kv: kv] do - Enum.each kv, fn {k, v} -> + Enum.each(kv, fn {k, v} -> def unquote(k)(), do: unquote(v) - end + end) end end In fact, the `:bind_quoted` option is recommended every time one desires to inject a value into the quote. """ - defmacro quote(opts, block) + defmacro quote(opts, block), do: error!([opts, block]) @doc """ - Unquotes the given expression from inside a macro. + Unquotes the given expression inside a quoted expression. + + This function expects a valid Elixir AST, also known as + quoted expression, as argument. If you would like to `unquote` + any value, such as a map or a four-element tuple, you should + call `Macro.escape/1` before unquoting. ## Examples - Imagine the situation you have a variable `name` and + Imagine the situation you have a quoted expression and you want to inject it inside some quote. The first attempt would be: - value = 13 - quote do: sum(1, value, 3) + value = + quote do + 13 + end + + quote do + sum(1, value, 3) + end + + + Which the argument for the `:sum` function call is not the + expected result: - Which would then return: + {:sum, [], [1, {:value, [if_undefined: :apply], Elixir}, 3]} - {:sum, [], [1, {:value, [], quoted}, 3]} + For this, we use `unquote`: - Which is not the expected result. For this, we use unquote: + iex> value = + ...> quote do + ...> 13 + ...> end + iex> quote do + ...> sum(1, unquote(value), 3) + ...> end + {:sum, [], [1, 13, 3]} - value = 13 - quote do: sum(1, unquote(value), 3) - #=> {:sum, [], [1, 13, 3]} + If you want to unquote a value that is not a quoted expression, + such as a map, you need to call `Macro.escape/1` before: + + iex> value = %{foo: :bar} + iex> quote do + ...> process_map(unquote(Macro.escape(value))) + ...> end + {:process_map, [], [{:%{}, [], [foo: :bar]}]} + If you forget to escape it, Elixir will raise an error + when compiling the code. """ - defmacro unquote(:unquote)(expr) + defmacro unquote(:unquote)(expr), do: error!([expr]) @doc """ - Unquotes the given list expanding its arguments. Similar - to unquote. + Unquotes the given list expanding its arguments. + + Similar to `unquote/1`. ## Examples - values = [2, 3, 4] - quote do: sum(1, unquote_splicing(values), 5) - #=> {:sum, [], [1, 2, 3, 4, 5]} + iex> values = [2, 3, 4] + iex> quote do + ...> sum(1, unquote_splicing(values), 5) + ...> end + {:sum, [], [1, 2, 3, 4, 5]} """ - defmacro unquote(:unquote_splicing)(expr) + defmacro unquote(:unquote_splicing)(expr), do: error!([expr]) @doc ~S""" Comprehensions allow you to quickly build a data structure from @@ -1159,7 +1380,7 @@ defmodule Kernel.SpecialForms do [2, 4, 6, 8] # A comprehension with two generators - iex> for x <- [1, 2], y <- [2, 3], do: x*y + iex> for x <- [1, 2], y <- [2, 3], do: x * y [2, 3, 4, 6] Filters can also be given: @@ -1168,26 +1389,60 @@ defmodule Kernel.SpecialForms do iex> for n <- [1, 2, 3, 4, 5, 6], rem(n, 2) == 0, do: n [2, 4, 6] - Note generators can also be used to filter as it removes any value - that doesn't match the left side of `<-`: + Filters must evaluate to truthy values (everything but `nil` + and `false`). If a filter is falsy, then the current value is + discarded. + + Generators can also be used to filter as it removes any value + that doesn't match the pattern on the left side of `<-`: - iex> for {:user, name} <- [user: "jose", admin: "john", user: "eric"] do + iex> users = [user: "john", admin: "meg", guest: "barbara"] + iex> for {type, name} when type != :guest <- users do ...> String.upcase(name) ...> end - ["JOSE", "ERIC"] + ["JOHN", "MEG"] Bitstring generators are also supported and are very useful when you need to organize bitstring streams: iex> pixels = <<213, 45, 132, 64, 76, 32, 76, 0, 0, 234, 32, 15>> - iex> for <>, do: {r, g, b} - [{213,45,132},{64,76,32},{76,0,0},{234,32,15}] + iex> for <>, do: {r, g, b} + [{213, 45, 132}, {64, 76, 32}, {76, 0, 0}, {234, 32, 15}] Variable assignments inside the comprehension, be it in generators, filters or inside the block, are not reflected outside of the comprehension. - ## Into + Variable assignments inside filters must still return a truthy value, + otherwise values are discarded. Let's see an example. Imagine you have + a keyword list where the key is a programming language and the value + is its direct parent. Then let's try to compute the grandparent of each + language. You could try this: + + iex> languages = [elixir: :erlang, erlang: :prolog, prolog: nil] + iex> for {language, parent} <- languages, grandparent = languages[parent], do: {language, grandparent} + [elixir: :prolog] + + Given the grandparents of Erlang and Prolog were nil, those values were + filtered out. If you don't want this behaviour, a simple option is to + move the filter inside the do-block: + + iex> languages = [elixir: :erlang, erlang: :prolog, prolog: nil] + iex> for {language, parent} <- languages do + ...> grandparent = languages[parent] + ...> {language, grandparent} + ...> end + [elixir: :prolog, erlang: nil, prolog: nil] + + However, such option is not always available, as you may have further + filters. An alternative is to convert the filter into a generator by + wrapping the right side of `=` in a list: + + iex> languages = [elixir: :erlang, erlang: :prolog, prolog: nil] + iex> for {language, parent} <- languages, grandparent <- [languages[parent]], do: {language, grandparent} + [elixir: :prolog, erlang: nil, prolog: nil] + + ## The `:into` and `:uniq` options In the examples above, the result returned by the comprehension was always a list. The returned result can be configured by passing an @@ -1203,24 +1458,222 @@ defmodule Kernel.SpecialForms do The `IO` module provides streams, that are both `Enumerable` and `Collectable`, here is an upcase echo server using comprehensions: - for line <- IO.stream(:stdio, :line), into: IO.stream(:stdio, :line) do + for line <- IO.stream(), into: IO.stream() do String.upcase(line) end + Similarly, `uniq: true` can also be given to comprehensions to guarantee + the results are only added to the collection if they were not returned + before. For example: + + iex> for x <- [1, 1, 2, 3], uniq: true, do: x * 2 + [2, 4, 6] + + iex> for <>, uniq: true, into: "", do: <> + "ABC" + + ## The `:reduce` option + + While the `:into` option allows us to customize the comprehension behaviour + to a given data type, such as putting all of the values inside a map or inside + a binary, it is not always enough. + + For example, imagine that you have a binary with letters where you want to + count how many times each lowercase letter happens, ignoring all uppercase + ones. For instance, for the string `"AbCabCABc"`, we want to return the map + `%{"a" => 1, "b" => 2, "c" => 1}`. + + If we were to use `:into`, we would need a data type that computes the + frequency of each element it holds. While there is no such data type in + Elixir, you could implement one yourself. + + A simpler option would be to use comprehensions for the mapping and + filtering of letters, and then we invoke `Enum.reduce/3` to build a map, + for example: + + iex> letters = for <>, x in ?a..?z, do: <> + iex> Enum.reduce(letters, %{}, fn x, acc -> Map.update(acc, x, 1, & &1 + 1) end) + %{"a" => 1, "b" => 2, "c" => 1} + + While the above is straight-forward, it has the downside of traversing the + data at least twice. If you are expecting long strings as inputs, this can + be quite expensive. + + Luckily, comprehensions also support the `:reduce` option, which would allow + us to fuse both steps above into a single step: + + iex> for <>, x in ?a..?z, reduce: %{} do + ...> acc -> Map.update(acc, <>, 1, & &1 + 1) + ...> end + %{"a" => 1, "b" => 2, "c" => 1} + + When the `:reduce` key is given, its value is used as the initial accumulator + and the `do` block must be changed to use `->` clauses, where the left side + of `->` receives the accumulated value of the previous iteration and the + expression on the right side must return the new accumulator value. Once there are no more + elements, the final accumulated value is returned. If there are no elements + at all, then the initial accumulator value is returned. + """ + defmacro for(args), do: error!([args]) + + @doc """ + Used to combine matching clauses. + + Let's start with an example: + + iex> opts = %{width: 10, height: 15} + iex> with {:ok, width} <- Map.fetch(opts, :width), + ...> {:ok, height} <- Map.fetch(opts, :height) do + ...> {:ok, width * height} + ...> end + {:ok, 150} + + If all clauses match, the `do` block is executed, returning its result. + Otherwise the chain is aborted and the non-matched value is returned: + + iex> opts = %{width: 10} + iex> with {:ok, width} <- Map.fetch(opts, :width), + ...> {:ok, height} <- Map.fetch(opts, :height) do + ...> {:ok, width * height} + ...> end + :error + + Guards can be used in patterns as well: + + iex> users = %{"melany" => "guest", "bob" => :admin} + iex> with {:ok, role} when not is_binary(role) <- Map.fetch(users, "bob") do + ...> {:ok, to_string(role)} + ...> end + {:ok, "admin"} + + As in `for/1`, variables bound inside `with/1` won't leak. + Expressions without `<-` may also be used in clauses. For instance, + you can perform regular matches with the `=` operator: + + iex> width = nil + iex> opts = %{width: 10, height: 15} + iex> with {:ok, width} <- Map.fetch(opts, :width), + ...> double_width = width * 2, + ...> {:ok, height} <- Map.fetch(opts, :height) do + ...> {:ok, double_width * height} + ...> end + {:ok, 300} + iex> width + nil + + The behaviour of any expression in a clause is the same as if it was + written outside of `with`. For example, `=` will raise a `MatchError` + instead of returning the non-matched value: + + with :foo = :bar, do: :ok + ** (MatchError) no match of right hand side value: :bar + + As with any other function or macro call in Elixir, explicit parens can + also be used around the arguments before the `do`-`end` block: + + iex> opts = %{width: 10, height: 15} + iex> with( + ...> {:ok, width} <- Map.fetch(opts, :width), + ...> {:ok, height} <- Map.fetch(opts, :height) + ...> ) do + ...> {:ok, width * height} + ...> end + {:ok, 150} + + The choice between parens and no parens is a matter of preference. + + ## Else clauses + + An `else` option can be given to modify what is being returned from + `with` in the case of a failed match: + + iex> opts = %{width: 10} + iex> with {:ok, width} <- Map.fetch(opts, :width), + ...> {:ok, height} <- Map.fetch(opts, :height) do + ...> {:ok, width * height} + ...> else + ...> :error -> + ...> {:error, :wrong_data} + ...> + ...> _other_error -> + ...> :unexpected_error + ...> end + {:error, :wrong_data} + + The `else` block works like a `case` clause: it can have multiple clauses, + and the first match will be used. Variables bound inside `with` (such as + `width` in this example) are not available in the `else` block. + + If an `else` block is used and there are no matching clauses, a `WithClauseError` + exception is raised. + + ### Beware! + + Keep in mind that, one of potential drawback of `with` is that all + failure clauses are flattened into a single `else` block. For example, + take this code that checks if a given path points to an Elixir file + and that it exists before creating a backup copy: + + with ".ex" <- Path.extname(path), + true <- File.exists?(path) do + backup_path = path <> ".backup" + File.cp!(path, backup_path) + {:ok, backup_path} + else + binary when is_binary(binary) -> + {:error, :invalid_extension} + + false -> + {:error, :missing_file} + end + + Note how we are having to reconstruct the result types of `Path.extname/1` + and `File.exists?/1` to build error messages. In this case, it is better + to change the with clauses to already return the desired format, like this: + + with :ok <- validate_extension(path), + :ok <- validate_exists(path) do + backup_path = path <> ".backup" + File.cp!(path, backup_path) + {:ok, backup_path} + end + + defp validate_extension(path) do + if Path.extname(path) == ".ex", do: :ok, else: {:error, :invalid_extension} + end + + defp validate_exists(path) do + if File.exists?(path), do: :ok, else: {:error, :missing_file} + end + + Note how the code above is better organized and clearer once we + make sure each clause in `with` returns a normalized format. """ - defmacro for(args) + defmacro with(args), do: error!([args]) @doc """ Defines an anonymous function. + See `Function` for more information. + ## Examples iex> add = fn a, b -> a + b end iex> add.(1, 2) 3 + Anonymous functions can also have multiple clauses. All clauses + should expect the same number of arguments: + + iex> negate = fn + ...> true -> false + ...> false -> true + ...> end + iex> negate.(false) + true + """ - defmacro unquote(:fn)(clauses) + defmacro unquote(:fn)(clauses), do: error!([clauses]) @doc """ Internal special form for block expressions. @@ -1229,14 +1682,18 @@ defmodule Kernel.SpecialForms do of expressions in Elixir. This special form is private and should not be invoked directly: - iex> quote do: (1; 2; 3) + iex> quote do + ...> 1 + ...> 2 + ...> 3 + ...> end {:__block__, [], [1, 2, 3]} """ - defmacro __block__(args) + defmacro unquote(:__block__)(args), do: error!([args]) @doc """ - Captures or creates an anonymous function. + Capture operator. Captures or creates an anonymous function. ## Capture @@ -1258,6 +1715,8 @@ defmodule Kernel.SpecialForms do &local_function/1 + See also `Function.capture/3`. + ## Anonymous functions The capture operator can also be used to partially apply @@ -1269,11 +1728,18 @@ defmodule Kernel.SpecialForms do 4 In other words, `&(&1 * 2)` is equivalent to `fn x -> x * 2 end`. - Another example using a local function: - iex> fun = &is_atom(&1) - iex> fun.(:atom) - true + We can partially apply a remote function with placeholder: + + iex> take_five = &Enum.take(&1, 5) + iex> take_five.(1..10) + [1, 2, 3, 4, 5] + + Another example while using an imported or local function: + + iex> first_elem = &elem(&1, 0) + iex> first_elem.({0, 1}) + 0 The `&` operator can be used with more complex expressions: @@ -1287,120 +1753,165 @@ defmodule Kernel.SpecialForms do iex> fun.(1, 2) {1, 2} - iex> fun = &[&1|&2] - iex> fun.(1, 2) - [1|2] + iex> fun = &[&1 | &2] + iex> fun.(1, [2, 3]) + [1, 2, 3] The only restrictions when creating anonymous functions is that at least one placeholder must be present, i.e. it must contain at least - `&1`: + `&1`, and that block expressions are not supported: - # No placeholder fails to compile - &var + # No placeholder, fails to compile. + &(:foo) - # Block expressions are also not supported - &(foo(&1, &2); &3 + &4) + # Block expression, fails to compile. + &(&1; &2) """ - defmacro unquote(:&)(expr) + defmacro unquote(:&)(expr), do: error!([expr]) @doc """ Internal special form to hold aliases information. It is usually compiled to an atom: - iex> quote do: Foo.Bar + iex> quote do + ...> Foo.Bar + ...> end {:__aliases__, [alias: false], [:Foo, :Bar]} Elixir represents `Foo.Bar` as `__aliases__` so calls can be unambiguously identified by the operator `:.`. For example: - iex> quote do: Foo.bar + iex> quote do + ...> Foo.bar() + ...> end {{:., [], [{:__aliases__, [alias: false], [:Foo]}, :bar]}, [], []} Whenever an expression iterator sees a `:.` as the tuple key, it can be sure that it represents a call and the second argument in the list is an atom. - On the other hand, aliases holds some properties: + On the other hand, aliases hold some properties: - 1. The head element of aliases can be any term. + 1. The head element of aliases can be any term that must expand to + an atom at compilation time. 2. The tail elements of aliases are guaranteed to always be atoms. - 3. When the head element of aliases is the atom `:Elixir`, no expansion happen. - - 4. When the head element of aliases is not an atom, it is expanded at runtime: - - quote do: some_var.Foo - {:__aliases__, [], [{:some_var, [], Elixir}, :Foo]} - - Since `some_var` is not available at compilation time, the compiler - expands such expression to: - - Module.concat [some_var, Foo] + 3. When the head element of aliases is the atom `:Elixir`, no expansion happens. """ - defmacro __aliases__(args) + defmacro unquote(:__aliases__)(args), do: error!([args]) @doc """ - Calls the overriden function when overriding it with `defoverridable`. - See `Kernel.defoverridable` for more information and documentation. + Calls the overridden function when overriding it with `Kernel.defoverridable/1`. + + See `Kernel.defoverridable/1` for more information and documentation. """ - defmacro super(args) + defmacro super(args), do: error!([args]) - @doc """ + @doc ~S""" Matches the given expression against the given clauses. ## Examples - case thing do - {:selector, i, value} when is_integer(i) -> - value - value -> - value + case File.read(file) do + {:ok, contents} when is_binary(contents) -> + String.split(contents, "\n") + + {:error, _reason} -> + Logger.warning "could not find #{file}, assuming empty..." + [] end - In the example above, we match `thing` against each clause "head" - and execute the clause "body" corresponding to the first clause - that matches. If no clause matches, an error is raised. + In the example above, we match the result of `File.read/1` + against each clause "head" and execute the clause "body" + corresponding to the first clause that matches. + + If no clause matches, an error is raised. For this reason, + it may be necessary to add a final catch-all clause (like `_`) + which will always match. - ## Variables handling + x = 10 - Notice that variables bound in a clause "head" do not leak to the - outer context: + case x do + 0 -> + "This clause won't match" + + _ -> + "This clause would match any value (x = #{x})" + end + #=> "This clause would match any value (x = 10)" + + ## Variable handling + + Note that variables bound in a clause do not leak to the outer context: case data do {:ok, value} -> value :error -> nil end - value #=> unbound variable value + value + #=> unbound variable value - However, variables explicitly bound in the clause "body" are - accessible from the outer context: + Variables in the outer context cannot be overridden either: value = 7 case lucky? do false -> value = 13 - true -> true + true -> true + end + + value + #=> 7 + + In the example above, `value` is going to be `7` regardless of the value of + `lucky?`. The variable `value` bound in the clause and the variable `value` + bound in the outer context are two entirely separate variables. + + If you want to pattern match against an existing variable, + you need to use the `^/1` operator: + + x = 1 + + case 10 do + ^x -> "Won't match" + _ -> "Will match" end + #=> "Will match" + + ## Using guards to match against multiple values + + While it is not possible to match against multiple patterns in a single + clause, it's possible to match against multiple values by using guards: - value #=> 7 or 13 + case data do + value when value in [:one, :two] -> + "#{value} has been matched" + + :three -> + "three has been matched" + end - In the example above, value is going to be `7` or `13` depending on - the value of `lucky?`. In case `value` has no previous value before - case, clauses that do not explicitly bind a value have the variable - bound to nil. """ - defmacro case(condition, clauses) + defmacro case(condition, clauses), do: error!([condition, clauses]) @doc """ Evaluates the expression corresponding to the first clause that - evaluates to truth value. + evaluates to a truthy value. - Raises an error if all conditions evaluate to to nil or false. + cond do + hd([1, 2, 3]) -> + "1 is considered as true" + end + #=> "1 is considered as true" + + Raises an error if all conditions evaluate to `nil` or `false`. + For this reason, it may be necessary to add a final always-truthy condition + (anything non-`false` and non-`nil`), which will always match. ## Examples @@ -1412,12 +1923,13 @@ defmodule Kernel.SpecialForms do true -> "This will" end + #=> "This will" """ - defmacro cond(clauses) + defmacro cond(clauses), do: error!([clauses]) @doc ~S""" - Evaluate the given expressions and handle any error, exit + Evaluates the given expressions and handles any error, exit, or throw that may have happened. ## Examples @@ -1426,80 +1938,88 @@ defmodule Kernel.SpecialForms do do_something_that_may_fail(some_arg) rescue ArgumentError -> - IO.puts "Invalid argument given" + IO.puts("Invalid argument given") catch value -> - IO.puts "caught #{value}" + IO.puts("Caught #{inspect(value)}") else value -> - IO.puts "Success! The result was #{value}" + IO.puts("Success! The result was #{inspect(value)}") after - IO.puts "This is printed regardless if it failed or succeed" + IO.puts("This is printed regardless if it failed or succeeded") end - The rescue clause is used to handle exceptions, while the catch - clause can be used to catch thrown values. The else clause can - be used to control flow based on the result of the expression. - Catch, rescue and else clauses work based on pattern matching. + The `rescue` clause is used to handle exceptions while the `catch` + clause can be used to catch thrown values and exits. + The `else` clause can be used to control flow based on the result of + the expression. `catch`, `rescue`, and `else` clauses work based on + pattern matching (similar to the `case` special form). - Note that calls inside `try` are not tail recursive since the VM - needs to keep the stacktrace in case an exception happens. + Calls inside `try/1` are not tail recursive since the VM needs to keep + the stacktrace in case an exception happens. To retrieve the stacktrace, + access `__STACKTRACE__/0` inside the `rescue` or `catch` clause. - ## Rescue clauses + ## `rescue` clauses - Besides relying on pattern matching, rescue clauses provides some - conveniences around exceptions that allows one to rescue an - exception by its name. All the following formats are valid rescue - expressions: + Besides relying on pattern matching, `rescue` clauses provide some + conveniences around exceptions that allow one to rescue an + exception by its name. All the following formats are valid patterns + in `rescue` clauses: + # Rescue a single exception without binding the exception + # to a variable try do UndefinedModule.undefined_function rescue UndefinedFunctionError -> nil end + # Rescue any of the given exception without binding try do UndefinedModule.undefined_function rescue - [UndefinedFunctionError] -> nil + [UndefinedFunctionError, ArgumentError] -> nil end - # rescue and bind to x + # Rescue and bind the exception to the variable "x" try do UndefinedModule.undefined_function rescue x in [UndefinedFunctionError] -> nil end - # rescue all and bind to x + # Rescue all kinds of exceptions and bind the rescued exception + # to the variable "x" try do UndefinedModule.undefined_function rescue x -> nil end - ## Erlang errors + ### Erlang errors - Erlang errors are transformed into Elixir ones during rescue: + Erlang errors are transformed into Elixir ones when rescuing: try do :erlang.error(:badarg) rescue ArgumentError -> :ok end + #=> :ok The most common Erlang errors will be transformed into their - Elixir counter-part. Those which are not will be transformed - into `ErlangError`: + Elixir counterpart. Those which are not will be transformed + into the more generic `ErlangError`: try do :erlang.error(:unknown) rescue ErlangError -> :ok end + #=> :ok - In fact, ErlangError can be used to rescue any error that is - not an Elixir error proper. For example, it can be used to rescue + In fact, `ErlangError` can be used to rescue any error that is + not a proper Elixir error. For example, it can be used to rescue the earlier `:badarg` error too, prior to transformation: try do @@ -1507,30 +2027,94 @@ defmodule Kernel.SpecialForms do rescue ErlangError -> :ok end + #=> :ok + + ## `catch` clauses + + The `catch` clause can be used to catch thrown values, exits, and errors. + + ### Catching thrown values + + `catch` can be used to catch values thrown by `Kernel.throw/1`: + + try do + throw(:some_value) + catch + thrown_value -> + IO.puts("A value was thrown: #{inspect(thrown_value)}") + end - ## Catching throws and exits + ### Catching values of any kind - The catch clause can be used to catch throws values and exits. + The `catch` clause also supports catching exits and errors. To do that, it + allows matching on both the *kind* of the caught value as well as the value + itself: try do - exit(1) + exit(:shutdown) catch - :exit, 1 -> IO.puts "Exited with 1" + :exit, value -> + IO.puts("Exited with value #{inspect(value)}") end try do - throw(:sample) + exit(:shutdown) catch - :throw, :sample -> - IO.puts "sample thrown" + kind, value when kind in [:exit, :throw] -> + IO.puts("Caught exit or throw with value #{inspect(value)}") end - catch values also support `:error`, as in Erlang, although it is - commonly avoided in favor of raise/rescue control mechanisms. + The `catch` clause also supports `:error` alongside `:exit` and `:throw` as + in Erlang, although this is commonly avoided in favor of `raise`/`rescue` control + mechanisms. One reason for this is that when catching `:error`, the error is + not automatically transformed into an Elixir error: - ## Else clauses + try do + :erlang.error(:badarg) + catch + :error, :badarg -> :ok + end + #=> :ok + + ## `after` clauses + + An `after` clause allows you to define cleanup logic that will be invoked both + when the block of code passed to `try/1` succeeds and also when an error is raised. Note + that the process will exit as usual when receiving an exit signal that causes + it to exit abruptly and so the `after` clause is not guaranteed to be executed. + Luckily, most resources in Elixir (such as open files, ETS tables, ports, sockets, + and so on) are linked to or monitor the owning process and will automatically clean + themselves up if that process exits. + + File.write!("tmp/story.txt", "Hello, World") + try do + do_something_with("tmp/story.txt") + after + File.rm("tmp/story.txt") + end - Else clauses allow the result of the expression to be pattern + Although `after` clauses are invoked whether or not there was an error, they do not + modify the return value. All of the following examples return `:return_me`: + + try do + :return_me + after + IO.puts("I will be printed") + :not_returned + end + + try do + raise "boom" + rescue + _ -> :return_me + after + IO.puts("I will be printed") + :not_returned + end + + ## `else` clauses + + `else` clauses allow the result of the body passed to `try/1` to be pattern matched on: x = 2 @@ -1546,8 +2130,8 @@ defmodule Kernel.SpecialForms do :large end - If an else clause is not present the result of the expression will - be return, if no exceptions are raised: + If an `else` clause is not present and no exceptions are raised, + the result of the expression will be returned: x = 1 ^x = @@ -1558,16 +2142,16 @@ defmodule Kernel.SpecialForms do :infinity end - However when an else clause is present but the result of the expression - does not match any of the patterns an exception will be raised. This - exception will not be caught by a catch or rescue in the same try: + However, when an `else` clause is present but the result of the expression + does not match any of the patterns then an exception will be raised. This + exception will not be caught by a `catch` or `rescue` in the same `try`: x = 1 try do try do 1 / x rescue - # The TryClauseError can not be rescued here: + # The TryClauseError cannot be rescued here: TryClauseError -> :error_a else @@ -1580,8 +2164,8 @@ defmodule Kernel.SpecialForms do :error_b end - Similarly an exception inside an else clause is not caught or rescued - inside the same try: + Similarly, an exception inside an `else` clause is not caught or rescued + inside the same `try`: try do try do @@ -1601,9 +2185,24 @@ defmodule Kernel.SpecialForms do end This means the VM no longer needs to keep the stacktrace once inside - an else clause and so tail recursion is possible when using a `try` - with a tail call as the final call inside an else clause. The same - is true for rescue and catch clauses. + an `else` clause and so tail recursion is possible when using a `try` + with a tail call as the final call inside an `else` clause. The same + is true for `rescue` and `catch` clauses. + + Only the result of the tried expression falls down to the `else` clause. + If the `try` ends up in the `rescue` or `catch` clauses, their result + will not fall down to `else`: + + try do + throw(:catch_this) + catch + :throw, :catch_this -> + :it_was_caught + else + # :it_was_caught will not fall down to this "else" clause. + other -> + {:else, other} + end ## Variable handling @@ -1619,7 +2218,8 @@ defmodule Kernel.SpecialForms do _, _ -> :failed end - x #=> unbound variable `x` + x + #=> unbound variable "x" In the example above, `x` cannot be accessed since it was defined inside the `try` clause. A common practice to address this issue @@ -1635,7 +2235,7 @@ defmodule Kernel.SpecialForms do end """ - defmacro try(args) + defmacro try(args), do: error!([args]) @doc """ Checks if there is a message matching the given clauses @@ -1647,42 +2247,47 @@ defmodule Kernel.SpecialForms do ## Examples receive do - {:selector, i, value} when is_integer(i) -> - value - value when is_atom(value) -> - value + {:selector, number, name} when is_integer(number) -> + name + name when is_atom(name) -> + name _ -> - IO.puts :stderr, "Unexpected message received" + IO.puts(:stderr, "Unexpected message received") end - An optional after clause can be given in case the message was not - received after the specified period of time: + An optional `after` clause can be given in case the message was not + received after the given timeout period, specified in milliseconds: receive do - {:selector, i, value} when is_integer(i) -> - value - value when is_atom(value) -> - value + {:selector, number, name} when is_integer(number) -> + name + name when is_atom(name) -> + name _ -> - IO.puts :stderr, "Unexpected message received" + IO.puts(:stderr, "Unexpected message received") after 5000 -> - IO.puts :stderr, "No message in 5 seconds" + IO.puts(:stderr, "No message in 5 seconds") end The `after` clause can be specified even if there are no match clauses. - There are two special cases for the timeout value given to `after` + The timeout value given to `after` can be any expression evaluating to + one of the allowed values: * `:infinity` - the process should wait indefinitely for a matching - message, this is the same as not using a timeout + message, this is the same as not using the after clause - * 0 - if there is no matching message in the mailbox, the timeout + * `0` - if there is no matching message in the mailbox, the timeout will occur immediately - ## Variables handling + * positive integer smaller than or equal to `4_294_967_295` (`0xFFFFFFFF` + in hexadecimal notation) - it should be possible to represent the timeout + value as an unsigned 32-bit integer. + + ## Variable handling - The `receive` special form handles variables exactly as the `case` + The `receive/1` special form handles variables exactly as the `case/2` special macro. For more information, check the docs for `case/2`. """ - defmacro receive(args) + defmacro receive(args), do: error!([args]) end diff --git a/lib/elixir/lib/kernel/typespec.ex b/lib/elixir/lib/kernel/typespec.ex index f75870c323d..9b0e66cf0ae 100644 --- a/lib/elixir/lib/kernel/typespec.ex +++ b/lib/elixir/lib/kernel/typespec.ex @@ -1,968 +1,1049 @@ defmodule Kernel.Typespec do - @moduledoc """ - Provides macros and functions for working with typespecs. + @moduledoc false - Elixir comes with a notation for declaring types and specifications. Elixir is - dynamically typed, as such typespecs are never used by the compiler to - optimize or modify code. Still, using typespecs is useful as documentation and - tools such as [Dialyzer](http://www.erlang.org/doc/man/dialyzer.html) can - analyze the code with typespecs to find bugs. + ## Deprecated API moved to Code.Typespec - The attributes `@type`, `@opaque`, `@typep`, `@spec` and `@callback` available - in modules are handled by the equivalent macros defined by this module. See - sub-sections "Defining a type" and "Defining a specification" below. + @doc false + @deprecated "Use Code.Typespec.spec_to_quoted/2 instead" + def spec_to_ast(name, spec) do + Code.Typespec.spec_to_quoted(name, spec) + end - ## Types and their syntax + @doc false + @deprecated "Use Code.Typespec.type_to_quoted/1 instead" + def type_to_ast(type) do + Code.Typespec.type_to_quoted(type) + end - The type syntax provided by Elixir is fairly similar to the one in - [Erlang](http://www.erlang.org/doc/reference_manual/typespec.html). + @doc false + @deprecated "Use Code.fetch_docs/1 instead" + def beam_typedocs(module) when is_atom(module) or is_binary(module) do + case Code.fetch_docs(module) do + {:docs_v1, _, _, _, _, _, docs} -> + for {{:type, name, arity}, _, _, doc, _} <- docs do + case doc do + %{"en" => doc_string} -> {{name, arity}, doc_string} + :hidden -> {{name, arity}, false} + _ -> {{name, arity}, nil} + end + end - Most of the built-in types provided in Erlang (for example, `pid()`) are - expressed the same way: `pid()` or simply `pid`. Parametrized types are also - supported (`list(integer)`) and so are remote types (`Enum.t`). + {:error, _} -> + nil + end + end - Integers and atom literals are allowed as types (ex. `1`, `:atom` or - `false`). All other types are built of unions of predefined types. Certain - shorthands are allowed, such as `[...]`, `<<>>` and `{...}`. - - ### Predefined types - - Type :: any # the top type, the set of all terms - | none # the bottom type, contains no terms - | pid - | port - | reference - | Atom - | Bitstring - | float - | Fun - | Integer - | List - | Tuple - | Union - | UserDefined # Described in section "Defining a type" + @doc false + @deprecated "Use Code.Typespec.fetch_types/1 instead" + def beam_types(module) when is_atom(module) or is_binary(module) do + case Code.Typespec.fetch_types(module) do + {:ok, types} -> types + :error -> nil + end + end - Atom :: atom - | ElixirAtom # `:foo`, `:bar`, ... + @doc false + @deprecated "Use Code.Typespec.fetch_specs/1 instead" + def beam_specs(module) when is_atom(module) or is_binary(module) do + case Code.Typespec.fetch_specs(module) do + {:ok, specs} -> specs + :error -> nil + end + end - Bitstring :: <<>> - | << _ :: M >> # M is a positive integer - | << _ :: _ * N >> # N is a positive integer - | << _ :: M, _ :: _ * N >> + @doc false + @deprecated "Use Code.Typespec.fetch_callbacks/1 instead" + def beam_callbacks(module) when is_atom(module) or is_binary(module) do + case Code.Typespec.fetch_callbacks(module) do + {:ok, callbacks} -> callbacks + :error -> nil + end + end - Fun :: (... -> any) # any function - | (... -> Type) # any arity, returning Type - | (() -> Type)) - | (TList -> Type) + ## Hooks for Module functions - Integer :: integer - | ElixirInteger # ..., -1, 0, 1, ... 42 ... - | ElixirInteger..ElixirInteger # an integer range + def defines_type?(module, {name, arity} = signature) + when is_atom(module) and is_atom(name) and arity in 0..255 do + {_set, bag} = :elixir_module.data_tables(module) - List :: list(Type) # proper list ([]-terminated) - | improper_list(Type1, Type2) # Type1=contents, Type2=termination - | maybe_improper_list(Type1, Type2) # Type1 and Type2 as above - | nonempty_list(Type) # proper non-empty list - | [] # empty list - | [Type] # shorthand for list(Type) - | [Type, ...] # shorthand for nonempty_list(Type) + finder = fn {_kind, expr, _caller} -> + type_to_signature(expr) == signature + end - Tuple :: tuple # a tuple of any size - | {} # empty tuple - | {TList} + :lists.any(finder, get_typespecs(bag, [:type, :opaque, :typep])) + end - TList :: Type - | Type, TList + def spec_to_callback(module, {name, arity} = signature) + when is_atom(module) and is_atom(name) and arity in 0..255 do + {set, bag} = :elixir_module.data_tables(module) - Union :: Type1 | Type2 + filter = fn {:spec, expr, pos} -> + if spec_to_signature(expr) == signature do + kind = :callback + store_typespec(bag, kind, expr, pos) - ### Bit strings + case :ets.lookup(set, {:function, name, arity}) do + [{{:function, ^name, ^arity}, _, line, _, doc, doc_meta}] -> + store_doc(set, kind, name, arity, line, :doc, doc, doc_meta) - Bit string with a base size of 3: + _ -> + nil + end - << _ :: 3 >> + true + else + false + end + end - Bit string with a unit size of 8: + :lists.filter(filter, get_typespecs(bag, :spec)) != [] + end - << _ :: _ * 8 >> + ## Typespec definition and storage - ### Anonymous functions + @doc """ + Defines a typespec. - Any anonymous function: + Invoked by `@/1` expansion. + """ + def deftypespec(:spec, expr, _line, _file, module, pos) do + {_set, bag} = :elixir_module.data_tables(module) + store_typespec(bag, :spec, expr, pos) + end - ((...) -> any) - (... -> any) + def deftypespec(kind, expr, line, _file, module, pos) + when kind in [:callback, :macrocallback] do + {set, bag} = :elixir_module.data_tables(module) - Anonymous function with arity of zero: + case spec_to_signature(expr) do + {name, arity} -> + # store doc only once in case callback has multiple clauses + unless :ets.member(set, {kind, name, arity}) do + {line, doc} = get_doc_info(set, :doc, line) + store_doc(set, kind, name, arity, line, :doc, doc, %{}) + end - (() -> type) + :error -> + :error + end - Anonymous function with some arity: + store_typespec(bag, kind, expr, pos) + end - ((type, type) -> type) - (type, type -> type) + @reserved_signatures [required: 1, optional: 1] + def deftypespec(kind, expr, line, file, module, pos) + when kind in [:type, :typep, :opaque] do + {set, bag} = :elixir_module.data_tables(module) - ## Built-in types + case type_to_signature(expr) do + {name, arity} = signature when signature in @reserved_signatures -> + compile_error( + :elixir_locals.get_cached_env(pos), + "type #{name}/#{arity} is a reserved type and it cannot be defined" + ) - Built-in type | Defined as - :-------------------- | :--------- - `term` | `any` - `binary` | `<< _ :: _ * 8 >>` - `bitstring` | `<< _ :: _ * 1 >>` - `boolean` | `false` | `true` - `byte` | `0..255` - `char` | `0..0xffff` - `number` | `integer` | `float` - `list` | `[any]` - `maybe_improper_list` | `maybe_improper_list(any, any)` - `nonempty_list` | `nonempty_list(any)` - `iodata` | `iolist` | `binary` - `iolist` | `maybe_improper_list(byte` | `binary` | `iolist, binary` | `[])` - `module` | `atom` - `mfa` | `{atom, atom, arity}` - `arity` | `0..255` - `node` | `atom` - `timeout` | `:infinity` | `non_neg_integer` - `no_return` | `none` - `fun` | `(... -> any)` + {name, arity} when kind == :typep -> + {line, doc} = get_doc_info(set, :typedoc, line) - Some built-in types cannot be expressed with valid syntax according to the - language defined above. + if doc do + warning = + "type #{name}/#{arity} is private, @typedoc's are always discarded for private types" - Built-in type | Can be interpreted as - :---------------- | :-------------------- - `non_neg_integer` | `0..` - `pos_integer` | `1..` - `neg_integer` | `..-1` + :elixir_errors.erl_warn(line, file, warning) + end - Types defined in other modules are referred to as "remote types", they are - referenced as `Module.type_name` (ex. `Enum.t` or `String.t`). + {name, arity} -> + {line, doc} = get_doc_info(set, :typedoc, line) + spec_meta = if kind == :opaque, do: %{opaque: true}, else: %{} + store_doc(set, :type, name, arity, line, :typedoc, doc, spec_meta) - ## Defining a type + :error -> + :error + end - @type type_name :: type - @typep type_name :: type - @opaque type_name :: type + store_typespec(bag, kind, expr, pos) + end - A type defined with `@typep` is private. An opaque type, defined with - `@opaque` is a type where the internal structure of the type will not be - visible, but the type is still public. + defp get_typespecs(bag, keys) when is_list(keys) do + :lists.flatmap(&get_typespecs(bag, &1), keys) + end - Types can be parametrised by defining variables as parameters, these variables - can then be used to define the type. + defp get_typespecs(bag, key) do + :ets.lookup_element(bag, {:accumulate, key}, 2) + catch + :error, :badarg -> [] + end - @type dict(key, value) :: [{key, value}] + defp take_typespecs(bag, keys) when is_list(keys) do + :lists.flatmap(&take_typespecs(bag, &1), keys) + end - ## Defining a specification + defp take_typespecs(bag, key) do + :lists.map(&elem(&1, 1), :ets.take(bag, {:accumulate, key})) + end - @spec function_name(type1, type2) :: return_type - @callback function_name(type1, type2) :: return_type + defp store_typespec(bag, key, expr, pos) do + :ets.insert(bag, {{:accumulate, key}, {key, expr, pos}}) + :ok + end - Callbacks are used to define the callbacks functions of behaviours (see - `Behaviour`). + defp store_doc(set, kind, name, arity, line, doc_kind, doc, spec_meta) do + doc_meta = get_doc_meta(spec_meta, doc_kind, set) + :ets.insert(set, {{kind, name, arity}, line, doc, doc_meta}) + end - Guards can be used to restrict type variables given as arguments to the - function. + defp get_doc_info(set, attr, line) do + case :ets.take(set, attr) do + [{^attr, {line, doc}, _}] -> {line, doc} + [] -> {line, nil} + end + end - @spec function(arg) :: [arg] when arg: atom + defp get_doc_meta(spec_meta, doc_kind, set) do + case :ets.take(set, {doc_kind, :meta}) do + [{{^doc_kind, :meta}, metadata, _}] -> Map.merge(metadata, spec_meta) + [] -> spec_meta + end + end - Type variables with no restriction can also be defined. + defp spec_to_signature({:when, _, [spec, _]}), do: type_to_signature(spec) + defp spec_to_signature(other), do: type_to_signature(other) - @spec function(arg) :: [arg] when arg: var + defp type_to_signature({:"::", _, [{name, _, context}, _]}) + when is_atom(name) and name != :"::" and is_atom(context), + do: {name, 0} - Specifications can be overloaded just like ordinary functions. + defp type_to_signature({:"::", _, [{name, _, args}, _]}) when is_atom(name) and name != :"::", + do: {name, length(args)} - @spec function(integer) :: atom - @spec function(atom) :: integer + defp type_to_signature(_), do: :error - ## Notes + ## Translation from Elixir AST to typespec AST - Elixir discourages the use of type `string` as it might be confused with - binaries which are referred to as "strings" in Elixir (as opposed to character - lists). In order to use the type that is called `string` in Erlang, one has to - use the `char_list` type which is a synonym for `string`. If you use `string`, - you'll get a warning from the compiler. + @doc false + def translate_typespecs_for_module(_set, bag) do + type_typespecs = take_typespecs(bag, [:type, :opaque, :typep]) + defined_type_pairs = collect_defined_type_pairs(type_typespecs) - If you want to refer to the "string" type (the one operated on by functions in - the `String` module), use `String.t` type instead. - """ + state = %{ + defined_type_pairs: defined_type_pairs, + used_type_pairs: [], + local_vars: %{}, + undefined_type_error_enabled?: true + } - @doc """ - Defines a type. - This macro is responsible for handling the attribute `@type`. + {types, state} = :lists.mapfoldl(&translate_type/2, state, type_typespecs) + {specs, state} = :lists.mapfoldl(&translate_spec/2, state, take_typespecs(bag, :spec)) + {callbacks, state} = :lists.mapfoldl(&translate_spec/2, state, take_typespecs(bag, :callback)) - ## Examples + {macrocallbacks, state} = + :lists.mapfoldl(&translate_spec/2, state, take_typespecs(bag, :macrocallback)) - @type my_type :: atom + optional_callbacks = :lists.flatten(get_typespecs(bag, :optional_callbacks)) + used_types = filter_used_types(types, state) - """ - defmacro deftype(type) do - quote do - Kernel.Typespec.deftype(:type, unquote(Macro.escape(type, unquote: true)), __ENV__) - end + {used_types, specs, callbacks, macrocallbacks, optional_callbacks} end - @doc """ - Defines an opaque type. - This macro is responsible for handling the attribute `@opaque`. - - ## Examples + defp collect_defined_type_pairs(type_typespecs) do + fun = fn {_kind, expr, pos}, type_pairs -> + %{file: file, line: line} = env = :elixir_locals.get_cached_env(pos) - @opaque my_type :: atom + case type_to_signature(expr) do + {name, arity} = type_pair -> + if built_in_type?(name, arity) do + message = "type #{name}/#{arity} is a built-in type and it cannot be redefined" + compile_error(env, message) + end - """ - defmacro defopaque(type) do - quote do - Kernel.Typespec.deftype(:opaque, unquote(Macro.escape(type, unquote: true)), __ENV__) - end - end + if Map.has_key?(type_pairs, type_pair) do + {error_full_path, error_line} = type_pairs[type_pair] + error_relative_path = Path.relative_to_cwd(error_full_path) - @doc """ - Defines a private type. - This macro is responsible for handling the attribute `@typep`. - - ## Examples + compile_error( + env, + "type #{name}/#{arity} is already defined in #{error_relative_path}:#{error_line}" + ) + end - @typep my_type :: atom + Map.put(type_pairs, type_pair, {file, line}) - """ - defmacro deftypep(type) do - quote do - Kernel.Typespec.deftype(:typep, unquote(Macro.escape(type, unquote: true)), __ENV__) + :error -> + compile_error(env, "invalid type specification: #{Macro.to_string(expr)}") + end end - end - @doc """ - Defines a spec. - This macro is responsible for handling the attribute `@spec`. - - ## Examples - - @spec add(number, number) :: number + :lists.foldl(fun, %{}, type_typespecs) + end - """ - defmacro defspec(spec) do - quote do - Kernel.Typespec.defspec(:spec, unquote(Macro.escape(spec, unquote: true)), __ENV__) + defp filter_used_types(types, state) do + fun = fn {_kind, {name, arity} = type_pair, _line, _type, export} -> + if not export and not :lists.member(type_pair, state.used_type_pairs) do + %{^type_pair => {file, line}} = state.defined_type_pairs + :elixir_errors.erl_warn(line, file, "type #{name}/#{arity} is unused") + false + else + true + end end + + :lists.filter(fun, types) end - @doc """ - Defines a callback. - This macro is responsible for handling the attribute `@callback`. + defp translate_type({kind, {:"::", _, [{name, _, args}, definition]}, pos}, state) do + caller = :elixir_locals.get_cached_env(pos) + state = clean_local_state(state) - ## Examples + args = + if is_atom(args) do + [] + else + :lists.map(&variable/1, args) + end - @callback add(number, number) :: number + vars = :lists.filter(&match?({:var, _, _}, &1), args) + var_names = :lists.map(&elem(&1, 2), vars) + state = :lists.foldl(&update_local_vars(&2, &1), state, var_names) + {spec, state} = typespec(definition, var_names, caller, state) + type = {name, spec, vars} + arity = length(args) - """ - defmacro defcallback(spec) do - quote do - Kernel.Typespec.defspec(:callback, unquote(Macro.escape(spec, unquote: true)), __ENV__) - end - end + ensure_no_underscore_local_vars!(caller, var_names) + ensure_no_unused_local_vars!(caller, state.local_vars) - @doc """ - Defines a `type`, `typep` or `opaque` by receiving a typespec expression. - """ - def define_type(kind, expr, doc \\ nil, env) do - Module.store_typespec(env.module, kind, {kind, expr, doc, env}) - end + {kind, export} = + case kind do + :type -> {:type, true} + :typep -> {:type, false} + :opaque -> {:opaque, true} + end - @doc """ - Defines a `spec` by receiving a typespec expression. - """ - def define_spec(kind, expr, env) do - Module.store_typespec(env.module, kind, {kind, expr, env}) - end + invalid_args = :lists.filter(&(not valid_variable_ast?(&1)), args) - @doc """ - Returns `true` if the current module defines a given type - (private, opaque or not). This function is only available - for modules being compiled. - """ - def defines_type?(module, name, arity) do - finder = fn {_kind, expr, _doc, _caller} -> - type_to_signature(expr) == {name, arity} - end + unless invalid_args == [] do + invalid_args = :lists.join(", ", :lists.map(&Macro.to_string/1, invalid_args)) - :lists.any(finder, Module.get_attribute(module, :type)) or - :lists.any(finder, Module.get_attribute(module, :opaque)) - end + message = + "@type definitions expect all arguments to be variables. The type " <> + "#{name}/#{arity} has an invalid argument(s): #{invalid_args}" - @doc """ - Returns `true` if the current module defines a given spec. - This function is only available for modules being compiled. - """ - def defines_spec?(module, name, arity) do - finder = fn {_kind, expr, _caller} -> - spec_to_signature(expr) == {name, arity} + compile_error(caller, message) end - :lists.any(finder, Module.get_attribute(module, :spec)) - end - @doc """ - Returns `true` if the current module defines a callback. - This function is only available for modules being compiled. - """ - def defines_callback?(module, name, arity) do - finder = fn {_kind, expr, _caller} -> - spec_to_signature(expr) == {name, arity} + if underspecified?(kind, arity, spec) do + message = "@#{kind} type #{name}/#{arity} is underspecified and therefore meaningless" + :elixir_errors.erl_warn(caller.line, caller.file, message) end - :lists.any(finder, Module.get_attribute(module, :callback)) + + {{kind, {name, arity}, caller.line, type, export}, state} end - @doc """ - Converts a spec clause back to Elixir AST. - """ - def spec_to_ast(name, {:type, line, :fun, [{:type, _, :product, args}, result]}) do - meta = [line: line] - body = {name, meta, Enum.map(args, &typespec_to_ast/1)} + defp valid_variable_ast?({variable_name, _, context}) + when is_atom(variable_name) and is_atom(context), + do: true - vars = args ++ [result] - |> Enum.flat_map(&collect_vars/1) - |> Enum.uniq - |> Enum.map(&{&1, {:var, meta, nil}}) + defp valid_variable_ast?(_), do: false - spec = {:::, meta, [body, typespec_to_ast(result)]} + defp underspecified?(:opaque, 0, {:type, _, type, []}) when type in [:any, :term], do: true + defp underspecified?(_kind, _arity, _spec), do: false - if vars == [] do - spec - else - {:when, meta, [spec, vars]} - end + defp translate_spec({kind, {:when, _meta, [spec, guard]}, pos}, state) do + caller = :elixir_locals.get_cached_env(pos) + translate_spec(kind, spec, guard, caller, state) end - def spec_to_ast(name, {:type, line, :fun, []}) do - {:::, [line: line], [{name, [line: line], []}, quote(do: term)]} + defp translate_spec({kind, spec, pos}, state) do + caller = :elixir_locals.get_cached_env(pos) + translate_spec(kind, spec, [], caller, state) end - def spec_to_ast(name, {:type, line, :bounded_fun, [{:type, _, :fun, [{:type, _, :product, args}, result]}, constraints]}) do - guards = - for {:type, _, :constraint, [{:atom, _, :is_subtype}, [{:var, _, var}, type]]} <- constraints do - {var, typespec_to_ast(type)} - end - - meta = [line: line] - - vars = args ++ [result] - |> Enum.flat_map(&collect_vars/1) - |> Enum.uniq - |> Kernel.--(Keyword.keys(guards)) - |> Enum.map(&{&1, {:var, meta, nil}}) - - args = for arg <- args, do: typespec_to_ast(arg) - - {:when, meta, [ - {:::, meta, [{name, [line: line], args}, typespec_to_ast(result)]}, - guards ++ vars - ]} + defp translate_spec(kind, {:"::", meta, [{name, _, args}, return]}, guard, caller, state) + when is_atom(name) and name != :"::" do + translate_spec(kind, meta, name, args, return, guard, caller, state) end - @doc """ - Converts a type clause back to Elixir AST. - """ - def type_to_ast({{:record, record}, fields, args}) when is_atom(record) do - fields = for field <- fields, do: typespec_to_ast(field) - args = for arg <- args, do: typespec_to_ast(arg) - type = {:{}, [], [record|fields]} - quote do: unquote(record)(unquote_splicing(args)) :: unquote(type) + defp translate_spec(_kind, {name, _meta, _args} = spec, _guard, caller, _state) + when is_atom(name) and name != :"::" do + spec = Macro.to_string(spec) + compile_error(caller, "type specification missing return type: #{spec}") end - def type_to_ast({name, type, args}) do - args = for arg <- args, do: typespec_to_ast(arg) - quote do: unquote(name)(unquote_splicing(args)) :: unquote(typespec_to_ast(type)) + defp translate_spec(_kind, spec, _guard, caller, _state) do + spec = Macro.to_string(spec) + compile_error(caller, "invalid type specification: #{spec}") end - @doc """ - Returns all type docs available from the module's beam code. + defp translate_spec(kind, meta, name, args, return, guard, caller, state) when is_atom(args), + do: translate_spec(kind, meta, name, [], return, guard, caller, state) - The result is returned as a list of tuples where the first element is the pair of type - name and arity and the second element is the documentation. + defp translate_spec(kind, meta, name, args, return, guard, caller, state) do + ensure_no_defaults!(args) + state = clean_local_state(state) - The module must have a corresponding beam file which can be - located by the runtime system. - """ - @spec beam_typedocs(module | binary) :: [tuple] | nil - def beam_typedocs(module) when is_atom(module) or is_binary(module) do - case abstract_code(module) do - {:ok, abstract_code} -> - type_docs = for {:attribute, _, :typedoc, tup} <- abstract_code, do: tup - :lists.flatten(type_docs) - _ -> - nil + unless Keyword.keyword?(guard) do + error = "expected keywords as guard in type specification, got: #{Macro.to_string(guard)}" + compile_error(caller, error) end - end - @doc """ - Returns all types available from the module's beam code. + line = line(meta) + vars = Keyword.keys(guard) + {fun_args, state} = fn_args(meta, args, return, vars, caller, state) + spec = {:type, line, :fun, fun_args} - The result is returned as a list of tuples where the first - element is the type (`:typep`, `:type` and `:opaque`). + {spec, state} = + case guard_to_constraints(guard, vars, meta, caller, state) do + {[], state} -> {spec, state} + {constraints, state} -> {{:type, line, :bounded_fun, [spec, constraints]}, state} + end - The module must have a corresponding beam file which can be - located by the runtime system. - """ - @spec beam_types(module | binary) :: [tuple] | nil - def beam_types(module) when is_atom(module) or is_binary(module) do - case abstract_code(module) do - {:ok, abstract_code} -> - exported_types = for {:attribute, _, :export_type, types} <- abstract_code, do: types - exported_types = :lists.flatten(exported_types) - - for {:attribute, _, kind, {name, _, args} = type} <- abstract_code, kind in [:opaque, :type] do - cond do - kind == :opaque -> {:opaque, type} - {name, length(args)} in exported_types -> {:type, type} - true -> {:typep, type} - end - end - _ -> - nil - end - end + ensure_no_unused_local_vars!(caller, state.local_vars) - @doc """ - Returns all specs available from the module's beam code. + arity = length(args) + {{kind, {name, arity}, caller.line, spec}, state} + end + + # TODO: Remove char_list type by v2.0 + defp built_in_type?(:char_list, 0), do: true + defp built_in_type?(:charlist, 0), do: true + defp built_in_type?(:as_boolean, 1), do: true + defp built_in_type?(:struct, 0), do: true + defp built_in_type?(:nonempty_charlist, 0), do: true + defp built_in_type?(:keyword, 0), do: true + defp built_in_type?(:keyword, 1), do: true + defp built_in_type?(:var, 0), do: true + defp built_in_type?(name, arity), do: :erl_internal.is_type(name, arity) + + defp ensure_no_defaults!(args) do + fun = fn + {:"::", _, [left, right]} -> + ensure_not_default(left) + ensure_not_default(right) + left + + other -> + ensure_not_default(other) + other + end - The result is returned as a list of tuples where the first - element is spec name and arity and the second is the spec. + :lists.foreach(fun, args) + end - The module must have a corresponding beam file which can be - located by the runtime system. - """ - @spec beam_specs(module | binary) :: [tuple] | nil - def beam_specs(module) when is_atom(module) or is_binary(module) do - from_abstract_code(module, :spec) + defp ensure_not_default({:\\, _, [_, _]}) do + raise ArgumentError, "default arguments \\\\ not supported in typespecs" end - @doc """ - Returns all callbacks available from the module's beam code. + defp ensure_not_default(_), do: :ok - The result is returned as a list of tuples where the first - element is spec name and arity and the second is the spec. + defp guard_to_constraints(guard, vars, meta, caller, state) do + line = line(meta) - The module must have a corresponding beam file - which can be located by the runtime system. - """ - @spec beam_callbacks(module | binary) :: [tuple] | nil - def beam_callbacks(module) when is_atom(module) or is_binary(module) do - from_abstract_code(module, :callback) - end + fun = fn + {_name, {:var, _, context}}, {constraints, state} when is_atom(context) -> + {constraints, state} - defp from_abstract_code(module, kind) do - case abstract_code(module) do - {:ok, abstract_code} -> - for {:attribute, _, abs_kind, value} <- abstract_code, kind == abs_kind, do: value - :error -> - nil + {name, type}, {constraints, state} -> + {spec, state} = typespec(type, vars, caller, state) + constraint = [{:atom, line, :is_subtype}, [{:var, line, name}, spec]] + state = update_local_vars(state, name) + {[{:type, line, :constraint, constraint} | constraints], state} end + + {constraints, state} = :lists.foldl(fun, {[], state}, guard) + {:lists.reverse(constraints), state} end - defp abstract_code(module) do - case :beam_lib.chunks(abstract_code_beam(module), [:abstract_code]) do - {:ok, {_, [{:abstract_code, {_raw_abstract_v1, abstract_code}}]}} -> - {:ok, abstract_code} - _ -> - :error - end + ## To typespec conversion + + defp line(meta) do + Keyword.get(meta, :line, 0) end - defp abstract_code_beam(module) when is_atom(module) do - case :code.get_object_code(module) do - {^module, beam, _filename} -> beam - :error -> module - end + # Handle unions + defp typespec({:|, meta, [_, _]} = exprs, vars, caller, state) do + exprs = collect_union(exprs) + {union, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, exprs) + {{:type, line(meta), :union, union}, state} end - defp abstract_code_beam(binary) when is_binary(binary) do - binary + # Handle binaries + defp typespec({:<<>>, meta, []}, _, _, state) do + line = line(meta) + {{:type, line, :binary, [{:integer, line, 0}, {:integer, line, 0}]}, state} end - ## Helpers + defp typespec( + {:<<>>, meta, [{:"::", unit_meta, [{:_, _, ctx1}, {:*, _, [{:_, _, ctx2}, unit]}]}]}, + _, + _, + state + ) + when is_atom(ctx1) and is_atom(ctx2) and unit in 1..256 do + line = line(meta) + {{:type, line, :binary, [{:integer, line, 0}, {:integer, line(unit_meta), unit}]}, state} + end - @doc false - def spec_to_signature({:when, _, [spec, _]}), - do: type_to_signature(spec) - def spec_to_signature(other), - do: type_to_signature(other) + defp typespec({:<<>>, meta, [{:"::", size_meta, [{:_, _, ctx}, size]}]}, _, _, state) + when is_atom(ctx) and is_integer(size) and size >= 0 do + line = line(meta) + {{:type, line, :binary, [{:integer, line(size_meta), size}, {:integer, line, 0}]}, state} + end - @doc false - def type_to_signature({:::, _, [{name, _, nil}, _]}), - do: {name, 0} - def type_to_signature({:::, _, [{name, _, args}, _]}), - do: {name, length(args)} + defp typespec( + { + :<<>>, + meta, + [ + {:"::", size_meta, [{:_, _, ctx1}, size]}, + {:"::", unit_meta, [{:_, _, ctx2}, {:*, _, [{:_, _, ctx3}, unit]}]} + ] + }, + _, + _, + state + ) + when is_atom(ctx1) and is_atom(ctx2) and is_atom(ctx3) and is_integer(size) and + size >= 0 and unit in 1..256 do + args = [{:integer, line(size_meta), size}, {:integer, line(unit_meta), unit}] + {{:type, line(meta), :binary, args}, state} + end - ## Macro callbacks + defp typespec({:<<>>, _meta, _args}, _vars, caller, _state) do + message = + "invalid binary specification, expected <<_::size>>, <<_::_*unit>>, " <> + "or <<_::size, _::_*unit>> with size being non-negative integers, and unit being an integer between 1 and 256" - @doc false - def defspec(kind, expr, caller) do - Module.store_typespec(caller.module, kind, {kind, expr, caller}) + compile_error(caller, message) end - @doc false - def deftype(kind, expr, caller) do - module = caller.module - doc = Module.get_attribute(module, :typedoc) - - Module.delete_attribute(module, :typedoc) - Module.store_typespec(module, kind, {kind, expr, doc, caller}) + ## Handle maps and structs + defp typespec({:map, meta, args}, _vars, _caller, state) when args == [] or is_atom(args) do + {{:type, line(meta), :map, :any}, state} end - ## Translation from Elixir AST to typespec AST + defp typespec({:%{}, meta, fields} = map, vars, caller, state) do + fun = fn + {{:required, meta2, [k]}, v}, state -> + {arg1, state} = typespec(k, vars, caller, state) + {arg2, state} = typespec(v, vars, caller, state) + {{:type, line(meta2), :map_field_exact, [arg1, arg2]}, state} - @doc false - def translate_type(kind, {:::, _, [{name, _, args}, definition]}, doc, caller) when is_atom(name) and name != ::: do - args = - if is_atom(args) do - [] - else - for(arg <- args, do: variable(arg)) - end + {{:optional, meta2, [k]}, v}, state -> + {arg1, state} = typespec(k, vars, caller, state) + {arg2, state} = typespec(v, vars, caller, state) + {{:type, line(meta2), :map_field_assoc, [arg1, arg2]}, state} - vars = for {:var, _, var} <- args, do: var - spec = typespec(definition, vars, caller) + {k, v}, state -> + {arg1, state} = typespec(k, vars, caller, state) + {arg2, state} = typespec(v, vars, caller, state) + {{:type, line(meta), :map_field_exact, [arg1, arg2]}, state} - vars = for {:var, _, _} = var <- args, do: var - type = {name, spec, vars} - arity = length(vars) + {:|, _, [_, _]}, _state -> + error = + "invalid map specification. When using the | operator in the map key, " <> + "make sure to wrap the key type in parentheses: #{Macro.to_string(map)}" - {kind, export} = - case kind do - :type -> {:type, true} - :typep -> {:type, false} - :opaque -> {:opaque, true} - end + compile_error(caller, error) - if not export and doc do - :elixir_errors.warn(caller.line, caller.file, "type #{name}/#{arity} is private, " <> - "@typedoc's are always discarded for private types\n") + _, _state -> + compile_error(caller, "invalid map specification: #{Macro.to_string(map)}") end - {{kind, {name, arity}, type}, caller.line, export, doc} - end - - def translate_type(_kind, other, _doc, caller) do - type_spec = Macro.to_string(other) - compile_error caller, "invalid type specification: #{type_spec}" + {fields, state} = :lists.mapfoldl(fun, state, fields) + {{:type, line(meta), :map, fields}, state} end - @doc false - def translate_spec(kind, {:when, _meta, [spec, guard]}, caller) do - translate_spec(kind, spec, guard, caller) - end + defp typespec({:%, _, [name, {:%{}, meta, fields}]}, vars, caller, state) do + module = Macro.expand(name, %{caller | function: {:__info__, 1}}) - def translate_spec(kind, spec, caller) do - translate_spec(kind, spec, [], caller) - end + struct = + module + |> Macro.struct!(caller) + |> Map.delete(:__struct__) + |> Map.to_list() - defp translate_spec(kind, {:::, meta, [{name, _, args}, return]}, guard, caller) when is_atom(name) and name != ::: do - if is_atom(args), do: args = [] + unless Keyword.keyword?(fields) do + compile_error(caller, "expected key-value pairs in struct #{Macro.to_string(name)}") + end - unless Keyword.keyword?(guard) do - guard = Macro.to_string(guard) - compile_error caller, "expected keywords as guard in function type specification, got: #{guard}" + types = + :lists.map( + fn + {:__exception__ = field, true} -> {field, Keyword.get(fields, field, true)} + {field, _} -> {field, Keyword.get(fields, field, quote(do: term()))} + end, + :lists.sort(struct) + ) + + fun = fn {field, _} -> + unless Keyword.has_key?(struct, field) do + compile_error( + caller, + "undefined field #{inspect(field)} on struct #{inspect(module)}" + ) + end end - vars = Keyword.keys(guard) - constraints = guard_to_constraints(guard, vars, meta, caller) + :lists.foreach(fun, fields) + typespec({:%{}, meta, [__struct__: module] ++ types}, vars, caller, state) + end + + # Handle records + defp typespec({:record, meta, [atom]}, vars, caller, state) do + typespec({:record, meta, [atom, []]}, vars, caller, state) + end + + defp typespec({:record, meta, [tag, field_specs]}, vars, caller, state) + when is_atom(tag) and is_list(field_specs) do + # We cannot set a function name to avoid tracking + # as a compile time dependency because for records it actually is one. + case Macro.expand({tag, [], [{:{}, [], []}]}, caller) do + {_, _, [name, fields | _]} when is_list(fields) -> + types = + :lists.map( + fn {field, _} -> + {:"::", [], + [ + {field, [], nil}, + Keyword.get(field_specs, field, quote(do: term())) + ]} + end, + fields + ) + + fun = fn {field, _} -> + unless Keyword.has_key?(fields, field) do + compile_error(caller, "undefined field #{field} on record #{inspect(tag)}") + end + end - spec = {:type, line(meta), :fun, fn_args(meta, args, return, vars, caller)} - if constraints != [] do - spec = {:type, line(meta), :bounded_fun, [spec, constraints]} - end + :lists.foreach(fun, field_specs) + typespec({:{}, meta, [name | types]}, vars, caller, state) - arity = length(args) - {{kind, {name, arity}, spec}, caller.line} + _ -> + compile_error(caller, "unknown record #{inspect(tag)}") + end end - defp translate_spec(_kind, spec, _guard, caller) do - spec = Macro.to_string(spec) - compile_error caller, "invalid function type specification: #{spec}" + defp typespec({:record, _meta, [_tag, _field_specs]}, _vars, caller, _state) do + message = "invalid record specification, expected the record name to be an atom literal" + compile_error(caller, message) end - defp guard_to_constraints(guard, vars, meta, caller) do - line = line(meta) + # Handle ranges + defp typespec({:.., meta, [left, right]}, vars, caller, state) do + {left, state} = typespec(left, vars, caller, state) + {right, state} = typespec(right, vars, caller, state) + :ok = validate_range(left, right, caller) - :lists.foldl(fn - {_name, {:var, _, context}}, acc when is_atom(context) -> - acc - {name, type}, acc -> - constraint = [{:atom, line, :is_subtype}, [{:var, line, name}, typespec(type, vars, caller)]] - type = {:type, line, :constraint, constraint} - [type|acc] - end, [], guard) |> :lists.reverse + {{:type, line(meta), :range, [left, right]}, state} end - ## To AST conversion - - defp collect_vars({:ann_type, _line, args}) when is_list(args) do - [] + # Handle special forms + defp typespec({:__MODULE__, _, atom}, vars, caller, state) when is_atom(atom) do + typespec(caller.module, vars, caller, state) end - defp collect_vars({:type, _line, _kind, args}) when is_list(args) do - Enum.flat_map(args, &collect_vars/1) + defp typespec({:__aliases__, _, _} = alias, vars, caller, state) do + typespec(expand_remote(alias, caller), vars, caller, state) end - defp collect_vars({:remote_type, _line, args}) when is_list(args) do - Enum.flat_map(args, &collect_vars/1) + # Handle funs + defp typespec([{:->, meta, [args, return]}], vars, caller, state) + when is_list(args) do + {args, state} = fn_args(meta, args, return, vars, caller, state) + {{:type, line(meta), :fun, args}, state} end - defp collect_vars({:typed_record_field, _line, type}) do - collect_vars(type) + # Handle type operator + defp typespec( + {:"::", meta, [{var_name, var_meta, context}, expr]} = ann_type, + vars, + caller, + state + ) + when is_atom(var_name) and is_atom(context) do + case typespec(expr, vars, caller, state) do + {{:ann_type, _, _}, _state} -> + message = + "invalid type annotation. Type annotations cannot be nested: " <> + "#{Macro.to_string(ann_type)}" + + # TODO: Make this an error on v2.0 and remove the code below + :elixir_errors.erl_warn(caller.line, caller.file, message) + + # This may be generating an invalid typespec but we need to generate it + # to avoid breaking existing code that was valid but only broke Dialyzer + {right, state} = typespec(expr, vars, caller, state) + {{:ann_type, line(meta), [{:var, line(var_meta), var_name}, right]}, state} + + {right, state} -> + {{:ann_type, line(meta), [{:var, line(var_meta), var_name}, right]}, state} + end end - defp collect_vars({:paren_type, _line, [type]}) do - collect_vars(type) - end + defp typespec({:"::", meta, [left, right]}, vars, caller, state) do + message = + "invalid type annotation. The left side of :: must be a variable, got: #{Macro.to_string(left)}" - defp collect_vars({:var, _line, var}) do - [erl_to_ex_var(var)] - end + message = + case left do + {:|, _, _} -> + message <> + ". Note \"left | right :: ann\" is the same as \"(left | right) :: ann\". " <> + "To solve this, use parentheses around the union operands: \"left | (right :: ann)\"" - defp collect_vars(_) do - [] - end + _ -> + message + end - defp typespec_to_ast({:type, line, :tuple, :any}) do - {:tuple, [line: line], []} - end + # TODO: Make this an error on v2.0, and remove the code below and + # the :undefined_type_error_enabled? key from the state + :elixir_errors.erl_warn(caller.line, caller.file, message) - defp typespec_to_ast({:type, line, :tuple, args}) do - args = for arg <- args, do: typespec_to_ast(arg) - {:{}, [line: line], args} + # This may be generating an invalid typespec but we need to generate it + # to avoid breaking existing code that was valid but only broke Dialyzer + state = %{state | undefined_type_error_enabled?: false} + {left, state} = typespec(left, vars, caller, state) + state = %{state | undefined_type_error_enabled?: true} + {right, state} = typespec(right, vars, caller, state) + {{:ann_type, line(meta), [left, right]}, state} end - defp typespec_to_ast({:type, _line, :list, [{:type, _, :union, unions} = arg]}) do - case unpack_typespec_kw(unions, []) do - {:ok, ast} -> ast - :error -> [typespec_to_ast(arg)] + # Handle unary ops + defp typespec({op, meta, [integer]}, _, _, state) when op in [:+, :-] and is_integer(integer) do + line = line(meta) + {{:op, line, op, {:integer, line, integer}}, state} + end + + # Handle remote calls in the form of @module_attribute.type. + # These are not handled by the general remote type clause as calling + # Macro.expand/2 on the remote does not expand module attributes (but expands + # things like __MODULE__). + defp typespec( + {{:., meta, [{:@, _, [{attr, _, _}]}, name]}, _, args} = orig, + vars, + caller, + state + ) do + remote = Module.get_attribute(caller.module, attr) + + unless is_atom(remote) and remote != nil do + message = + "invalid remote in typespec: #{Macro.to_string(orig)} (@#{attr} is #{inspect(remote)})" + + compile_error(caller, message) end - end - defp typespec_to_ast({:type, _line, :list, args}) do - for arg <- args, do: typespec_to_ast(arg) + {remote_spec, state} = typespec(remote, vars, caller, state) + {name_spec, state} = typespec(name, vars, caller, state) + type = {remote_spec, meta, name_spec, args} + remote_type(type, vars, caller, state) end - defp typespec_to_ast({:type, line, :map, fields}) do - fields = Enum.map fields, fn {:type, _, :map_field_assoc, k, v} -> - {typespec_to_ast(k), typespec_to_ast(v)} - end + # Handle remote calls + defp typespec({{:., meta, [remote, name]}, _, args} = orig, vars, caller, state) do + remote = expand_remote(remote, caller) - {struct, fields} = Keyword.pop(fields, :__struct__) - map = {:%{}, [line: line], fields} + cond do + not is_atom(remote) -> + compile_error(caller, "invalid remote in typespec: #{Macro.to_string(orig)}") - if struct do - {:%, [line: line], [struct, map]} - else - map - end - end + remote == caller.module -> + typespec({name, meta, args}, vars, caller, state) - defp typespec_to_ast({:type, line, :binary, [arg1, arg2]}) do - [arg1, arg2] = for arg <- [arg1, arg2], do: typespec_to_ast(arg) - cond do - arg2 == 0 -> - quote line: line, do: <<_ :: unquote(arg1)>> - arg1 == 0 -> - quote line: line, do: <<_ :: _ * unquote(arg2)>> true -> - quote line: line, do: <<_ :: unquote(arg1) * unquote(arg2)>> + {remote_spec, state} = typespec(remote, vars, caller, state) + {name_spec, state} = typespec(name, vars, caller, state) + type = {remote_spec, meta, name_spec, args} + remote_type(type, vars, caller, state) end end - defp typespec_to_ast({:type, line, :union, args}) do - args = for arg <- args, do: typespec_to_ast(arg) - Enum.reduce Enum.reverse(args), fn(arg, expr) -> {:|, [line: line], [arg, expr]} end + # Handle tuples + defp typespec({:tuple, meta, []}, _vars, _caller, state) do + {{:type, line(meta), :tuple, :any}, state} end - defp typespec_to_ast({:type, line, :fun, [{:type, _, :product, args}, result]}) do - args = for arg <- args, do: typespec_to_ast(arg) - [{:->, [line: line], [args, typespec_to_ast(result)]}] + defp typespec({:{}, meta, t}, vars, caller, state) when is_list(t) do + {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, t) + {{:type, line(meta), :tuple, args}, state} end - defp typespec_to_ast({:type, line, :fun, [args, result]}) do - [{:->, [line: line], [[typespec_to_ast(args)], typespec_to_ast(result)]}] + defp typespec({left, right}, vars, caller, state) do + typespec({:{}, [], [left, right]}, vars, caller, state) end - defp typespec_to_ast({:type, line, :fun, []}) do - typespec_to_ast({:type, line, :fun, [{:type, line, :any}, {:type, line, :any, []} ]}) + # Handle blocks + defp typespec({:__block__, _meta, [arg]}, vars, caller, state) do + typespec(arg, vars, caller, state) end - defp typespec_to_ast({:type, line, :range, [left, right]}) do - {:"..", [line: line], [typespec_to_ast(left), typespec_to_ast(right)]} + # Handle variables or local calls + defp typespec({name, meta, atom}, vars, caller, state) when is_atom(atom) do + if :lists.member(name, vars) do + state = update_local_vars(state, name) + {{:var, line(meta), name}, state} + else + typespec({name, meta, []}, vars, caller, state) + end end - defp typespec_to_ast({:type, line, name, args}) do - args = for arg <- args, do: typespec_to_ast(arg) - {name, [line: line], args} - end + # Handle local calls + defp typespec({:string, meta, args}, vars, caller, state) do + warning = + "string() type use is discouraged. " <> + "For character lists, use charlist() type, for strings, String.t()\n" <> + Exception.format_stacktrace(Macro.Env.stacktrace(caller)) - defp typespec_to_ast({:var, line, var}) do - {erl_to_ex_var(var), line, nil} + :elixir_errors.erl_warn(caller.line, caller.file, warning) + {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, args) + {{:type, line(meta), :string, args}, state} end - defp typespec_to_ast({:op, line, op, arg}) do - {op, [line: line], [typespec_to_ast(arg)]} - end + defp typespec({:nonempty_string, meta, args}, vars, caller, state) do + warning = + "nonempty_string() type use is discouraged. " <> + "For non-empty character lists, use nonempty_charlist() type, for strings, String.t()\n" <> + Exception.format_stacktrace(Macro.Env.stacktrace(caller)) - # Special shortcut(s) - defp typespec_to_ast({:remote_type, line, [{:atom, _, :elixir}, {:atom, _, :char_list}, []]}) do - typespec_to_ast({:type, line, :char_list, []}) + :elixir_errors.erl_warn(caller.line, caller.file, warning) + {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, args) + {{:type, line(meta), :nonempty_string, args}, state} end - defp typespec_to_ast({:remote_type, line, [{:atom, _, :elixir}, {:atom, _, :as_boolean}, [arg]]}) do - typespec_to_ast({:type, line, :as_boolean, [arg]}) - end + defp typespec({type, _meta, []}, vars, caller, state) when type in [:charlist, :char_list] do + if type == :char_list do + warning = "the char_list() type is deprecated, use charlist()" + :elixir_errors.erl_warn(caller.line, caller.file, warning) + end - defp typespec_to_ast({:remote_type, line, [mod, name, args]}) do - args = for arg <- args, do: typespec_to_ast(arg) - dot = {:., [line: line], [typespec_to_ast(mod), typespec_to_ast(name)]} - {dot, [line: line], args} + typespec(quote(do: :elixir.charlist()), vars, caller, state) end - defp typespec_to_ast({:ann_type, line, [var, type]}) do - {:::, [line: line], [typespec_to_ast(var), typespec_to_ast(type)]} + defp typespec({:nonempty_charlist, _meta, []}, vars, caller, state) do + typespec(quote(do: :elixir.nonempty_charlist()), vars, caller, state) end - defp typespec_to_ast({:typed_record_field, - {:record_field, line, {:atom, line1, name}}, - type}) do - typespec_to_ast({:ann_type, line, [{:var, line1, name}, type]}) + defp typespec({:struct, _meta, []}, vars, caller, state) do + typespec(quote(do: :elixir.struct()), vars, caller, state) end - defp typespec_to_ast({:type, _, :any}) do - quote do: ... + defp typespec({:as_boolean, _meta, [arg]}, vars, caller, state) do + typespec(quote(do: :elixir.as_boolean(unquote(arg))), vars, caller, state) end - defp typespec_to_ast({:paren_type, _, [type]}) do - typespec_to_ast(type) + defp typespec({:keyword, _meta, args}, vars, caller, state) when length(args) <= 1 do + typespec(quote(do: :elixir.keyword(unquote_splicing(args))), vars, caller, state) end - defp typespec_to_ast({t, _line, atom}) when is_atom(t) do - atom + defp typespec({:fun, meta, args}, vars, caller, state) do + {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, args) + {{:type, line(meta), :fun, args}, state} end - defp typespec_to_ast(other), do: other + defp typespec({name, meta, args}, vars, caller, state) do + {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, args) + arity = length(args) - defp erl_to_ex_var(var) do - case Atom.to_string(var) do - <<"_", c :: [binary, size(1)], rest :: binary>> -> - String.to_atom("_#{String.downcase(c)}#{rest}") - <> -> - String.to_atom("#{String.downcase(c)}#{rest}") - end - end + case :erl_internal.is_type(name, arity) do + true -> + {{:type, line(meta), name, args}, state} + + false -> + if state.undefined_type_error_enabled? and + not Map.has_key?(state.defined_type_pairs, {name, arity}) do + compile_error( + caller, + "type #{name}/#{arity} undefined (no such type in #{inspect(caller.module)})" + ) + end - ## To typespec conversion + state = + if :lists.member({name, arity}, state.used_type_pairs) do + state + else + %{state | used_type_pairs: [{name, arity} | state.used_type_pairs]} + end - defp line(meta) do - case :lists.keyfind(:line, 1, meta) do - {:line, line} -> line - false -> 0 + {{:user_type, line(meta), name, args}, state} end end - # Handle unions - defp typespec({:|, meta, [_, _]} = exprs, vars, caller) do - exprs = collect_union(exprs) - union = for e <- exprs, do: typespec(e, vars, caller) - {:type, line(meta), :union, union} + # Handle literals + defp typespec(atom, _, _, state) when is_atom(atom) do + {{:atom, 0, atom}, state} end - # Handle binaries - defp typespec({:<<>>, meta, []}, _, _) do - {:type, line(meta), :binary, [{:integer, line(meta), 0}, {:integer, line(meta), 0}]} + defp typespec(integer, _, _, state) when is_integer(integer) do + {{:integer, 0, integer}, state} end - defp typespec({:<<>>, meta, [{:::, _, [{:_, meta1, atom}, {:*, _, [{:_, meta2, atom}, unit]}]}]}, _, _) when is_atom(atom) do - {:type, line(meta), :binary, [{:integer, line(meta1), 0}, {:integer, line(meta2), unit}]} + defp typespec([], vars, caller, state) do + typespec({nil, [], []}, vars, caller, state) end - defp typespec({:<<>>, meta, [{:::, meta1, [{:_, meta2, atom}, base]}]}, _, _) when is_atom(atom) do - {:type, line(meta), :binary, [{:integer, line(meta1), base}, {:integer, line(meta2), 0}]} + defp typespec([{:..., _, atom}], vars, caller, state) when is_atom(atom) do + typespec({:nonempty_list, [], []}, vars, caller, state) end - ## Handle maps and structs - defp typespec({:%{}, meta, fields}, vars, caller) do - fields = :lists.map(fn {k, v} -> - {:type, line(meta), :map_field_assoc, typespec(k, vars, caller), typespec(v, vars, caller)} - end, fields) - {:type, line(meta), :map, fields} + defp typespec([spec, {:..., _, atom}], vars, caller, state) when is_atom(atom) do + typespec({:nonempty_list, [], [spec]}, vars, caller, state) end - defp typespec({:%, _, [name, {:%{}, meta, fields}]}, vars, caller) do - typespec({:%{}, meta, [{:__struct__, name}|fields]}, vars, caller) + defp typespec([spec], vars, caller, state) do + typespec({:list, [], [spec]}, vars, caller, state) end - # Handle ranges - defp typespec({:.., meta, args}, vars, caller) do - typespec({:range, meta, args}, vars, caller) - end + defp typespec(list, vars, caller, state) when is_list(list) do + [head | tail] = :lists.reverse(list) - # Handle special forms - defp typespec({:__MODULE__, _, atom}, vars, caller) when is_atom(atom) do - typespec(caller.module, vars, caller) - end + union = + :lists.foldl( + fn elem, acc -> {:|, [], [validate_kw(elem, list, caller), acc]} end, + validate_kw(head, list, caller), + tail + ) - defp typespec({:__aliases__, _, _} = alias, vars, caller) do - atom = Macro.expand alias, caller - typespec(atom, vars, caller) + typespec({:list, [], [union]}, vars, caller, state) end - # Handle funs - defp typespec([{:->, meta, [arguments, return]}], vars, caller) when is_list(arguments) do - args = fn_args(meta, arguments, return, vars, caller) - {:type, line(meta), :fun, args} + defp typespec(other, _vars, caller, _state) do + compile_error(caller, "unexpected expression in typespec: #{Macro.to_string(other)}") end - # Handle type operator - defp typespec({:::, meta, [var, expr]}, vars, caller) do - left = typespec(var, [elem(var, 0)|vars], caller) - right = typespec(expr, vars, caller) - {:ann_type, line(meta), [left, right]} - end + ## Helpers - # Handle unary ops - defp typespec({op, meta, [integer]}, _, _) when op in [:+, :-] and is_integer(integer) do - {:op, line(meta), op, {:integer, line(meta), integer}} - end + # This is a backport of Macro.expand/2 because we want to expand + # aliases but we don't them to become compile-time references. + defp expand_remote({:__aliases__, _, _} = alias, env) do + case :elixir_aliases.expand(alias, env) do + receiver when is_atom(receiver) -> + receiver - # Handle remote calls - defp typespec({{:., meta, [remote, name]}, _, args} = orig, vars, caller) do - remote = Macro.expand remote, caller - unless is_atom(remote) do - compile_error(caller, "invalid remote in typespec: #{Macro.to_string(orig)}") + aliases -> + aliases = :lists.map(&Macro.expand_once(&1, env), aliases) + + case :lists.all(&is_atom/1, aliases) do + true -> :elixir_aliases.concat(aliases) + false -> alias + end end - remote_type({typespec(remote, vars, caller), meta, typespec(name, vars, caller), args}, vars, caller) end - # Handle tuples - defp typespec({:tuple, meta, args}, _vars, _caller) when args == [] or is_atom(args) do - {:type, line(meta), :tuple, :any} - end + defp expand_remote(other, env), do: Macro.expand(other, env) - defp typespec({:{}, meta, t}, vars, caller) when is_list(t) do - args = for e <- t, do: typespec(e, vars, caller) - {:type, line(meta), :tuple, args} + defp compile_error(caller, desc) do + raise CompileError, file: caller.file, line: caller.line, description: desc end - defp typespec({left, right}, vars, caller) do - typespec({:{}, [], [left, right]}, vars, caller) + defp remote_type({remote, meta, name, args}, vars, caller, state) do + {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, args) + {{:remote_type, line(meta), [remote, name, args]}, state} end - # Handle blocks - defp typespec({:__block__, _meta, [arg]}, vars, caller) do - typespec(arg, vars, caller) - end + defp collect_union({:|, _, [a, b]}), do: [a | collect_union(b)] + defp collect_union(v), do: [v] - # Handle variables or local calls - defp typespec({name, meta, atom}, vars, caller) when is_atom(atom) do - if :lists.member(name, vars) do - {:var, line(meta), name} - else - typespec({name, meta, []}, vars, caller) - end - end + defp validate_kw({key, _} = t, _, _caller) when is_atom(key), do: t - # Handle local calls - defp typespec({:string, meta, arguments}, vars, caller) do - :elixir_errors.warn caller.line, caller.file, "string() type use is discouraged. For character lists, use " <> - "char_list() type, for strings, String.t()\n#{Exception.format_stacktrace(Macro.Env.stacktrace(caller))}" - arguments = for arg <- arguments, do: typespec(arg, vars, caller) - {:type, line(meta), :string, arguments} + defp validate_kw(_, original, caller) do + compile_error(caller, "unexpected list in typespec: #{Macro.to_string(original)}") end - defp typespec({:char_list, _meta, []}, vars, caller) do - typespec((quote do: :elixir.char_list()), vars, caller) + defp validate_range({:op, _, :-, {:integer, meta, first}}, last, caller) do + validate_range({:integer, meta, -first}, last, caller) end - defp typespec({:as_boolean, _meta, [arg]}, vars, caller) do - typespec((quote do: :elixir.as_boolean(unquote(arg))), vars, caller) + defp validate_range(first, {:op, _, :-, {:integer, meta, last}}, caller) do + validate_range(first, {:integer, meta, -last}, caller) end - defp typespec({name, meta, arguments}, vars, caller) do - arguments = for arg <- arguments, do: typespec(arg, vars, caller) - {:type, line(meta), name, arguments} + defp validate_range({:integer, _, first}, {:integer, _, last}, _caller) when first < last do + :ok end - # Handle literals - defp typespec(atom, _, _) when is_atom(atom) do - {:atom, 0, atom} - end + defp validate_range(_, _, caller) do + message = + "invalid range specification, expected both sides to be integers, " <> + "with the left side lower than the right side" - defp typespec(integer, _, _) when is_integer(integer) do - {:integer, 0, integer} + compile_error(caller, message) end - defp typespec([], vars, caller) do - typespec({nil, [], []}, vars, caller) + defp fn_args(meta, args, return, vars, caller, state) do + {fun_args, state} = fn_args(meta, args, vars, caller, state) + {spec, state} = typespec(return, vars, caller, state) + + case [fun_args, spec] do + [{:type, _, :any}, {:type, _, :any, []}] -> {[], state} + x -> {x, state} + end end - defp typespec([spec], vars, caller) do - typespec({:list, [], [spec]}, vars, caller) + defp fn_args(meta, [{:..., _, _}], _vars, _caller, state) do + {{:type, line(meta), :any}, state} end - defp typespec([spec, {:"...", _, quoted}], vars, caller) when is_atom(quoted) do - typespec({:nonempty_list, [], [spec]}, vars, caller) + defp fn_args(meta, args, vars, caller, state) do + {args, state} = :lists.mapfoldl(&typespec(&1, vars, caller, &2), state, args) + {{:type, line(meta), :product, args}, state} end - defp typespec(list, vars, caller) do - [h|t] = :lists.reverse(list) - union = :lists.foldl(fn(x, acc) -> - {:|, [], [validate_kw(x, list, caller), acc]} - end, validate_kw(h, list, caller), t) - typespec({:list, [], [union]}, vars, caller) + defp variable({name, meta, args}) when is_atom(name) and is_atom(args) do + {:var, line(meta), name} end - ## Helpers + defp variable(expr), do: expr - defp compile_error(caller, desc) do - raise CompileError, file: caller.file, line: caller.line, description: desc + defp clean_local_state(state) do + %{state | local_vars: %{}} end - defp remote_type({remote, meta, name, arguments}, vars, caller) do - arguments = for arg <- arguments, do: typespec(arg, vars, caller) - {:remote_type, line(meta), [ remote, name, arguments ]} + defp update_local_vars(%{local_vars: local_vars} = state, var_name) do + case Map.fetch(local_vars, var_name) do + {:ok, :used_once} -> %{state | local_vars: Map.put(local_vars, var_name, :used_multiple)} + {:ok, :used_multiple} -> state + :error -> %{state | local_vars: Map.put(local_vars, var_name, :used_once)} + end end - defp collect_union({:|, _, [a, b]}), do: [a|collect_union(b)] - defp collect_union(v), do: [v] - - defp validate_kw({key, _} = t, _, _caller) when is_atom(key), do: t - defp validate_kw(_, original, caller) do - compile_error(caller, "unexpected list in typespec: #{Macro.to_string original}") - end + defp ensure_no_underscore_local_vars!(caller, var_names) do + case :lists.member(:_, var_names) do + true -> + compile_error(caller, "type variable '_' is invalid") - defp fn_args(meta, args, return, vars, caller) do - case [fn_args(meta, args, vars, caller), typespec(return, vars, caller)] do - [{:type, _, :any}, {:type, _, :any, []}] -> [] - x -> x + false -> + :ok end end - defp fn_args(meta, [{:"...", _, _}], _vars, _caller) do - {:type, line(meta), :any} - end + defp ensure_no_unused_local_vars!(caller, local_vars) do + fun = fn {name, used_times} -> + case {:erlang.atom_to_list(name), used_times} do + {[?_ | _], :used_once} -> + :ok - defp fn_args(meta, args, vars, caller) do - args = for arg <- args, do: typespec(arg, vars, caller) - {:type, line(meta), :product, args} - end + {[?_ | _], :used_multiple} -> + warning = + "the underscored type variable \"#{name}\" is used more than once in the " <> + "type specification. A leading underscore indicates that the value of the " <> + "variable should be ignored. If this is intended please rename the variable to " <> + "remove the underscore" - defp variable({name, meta, _}) do - {:var, line(meta), name} - end + :elixir_errors.erl_warn(caller.line, caller.file, warning) - defp unpack_typespec_kw([{:type, _, :tuple, [{:atom, _, atom}, type]}|t], acc) do - unpack_typespec_kw(t, [{atom, typespec_to_ast(type)}|acc]) - end + {_, :used_once} -> + compile_error( + caller, + "type variable #{name} is used only once. Type variables in typespecs " <> + "must be referenced at least twice, otherwise it is equivalent to term()" + ) - defp unpack_typespec_kw([], acc) do - {:ok, :lists.reverse(acc)} - end + _ -> + :ok + end + end - defp unpack_typespec_kw(_, _acc) do - :error + :lists.foreach(fun, :maps.to_list(local_vars)) end end diff --git a/lib/elixir/lib/kernel/utils.ex b/lib/elixir/lib/kernel/utils.ex new file mode 100644 index 00000000000..b5ef70eea5f --- /dev/null +++ b/lib/elixir/lib/kernel/utils.ex @@ -0,0 +1,398 @@ +import Kernel, except: [destructure: 2, defdelegate: 2, defstruct: 2] + +defmodule Kernel.Utils do + @moduledoc false + + @doc """ + Callback for destructure. + """ + def destructure(list, count) + when is_list(list) and is_integer(count) and count >= 0, + do: destructure_list(list, count) + + def destructure(nil, count) + when is_integer(count) and count >= 0, + do: destructure_nil(count) + + defp destructure_list(_, 0), do: [] + defp destructure_list([], count), do: destructure_nil(count) + defp destructure_list([h | t], count), do: [h | destructure_list(t, count - 1)] + + defp destructure_nil(0), do: [] + defp destructure_nil(count), do: [nil | destructure_nil(count - 1)] + + @doc """ + Callback for defdelegate entry point. + """ + def defdelegate_all(funs, opts, env) do + to = Keyword.get(opts, :to) || raise ArgumentError, "expected to: to be given as argument" + as = Keyword.get(opts, :as) + + if to == env.module and is_nil(as) do + raise ArgumentError, + "defdelegate function is calling itself, which will lead to an infinite loop. You should either change the value of the :to option or specify the :as option" + end + + if is_list(funs) do + IO.warn( + "passing a list to Kernel.defdelegate/2 is deprecated, please define each delegate separately", + Macro.Env.stacktrace(env) + ) + end + + if Keyword.has_key?(opts, :append_first) do + IO.warn( + "Kernel.defdelegate/2 :append_first option is deprecated", + Macro.Env.stacktrace(env) + ) + end + + to + end + + @doc """ + Callback for each function in defdelegate. + """ + def defdelegate_each(fun, opts) when is_list(opts) do + # TODO: Remove on v2.0 + append_first? = Keyword.get(opts, :append_first, false) + + {name, args} = + case fun do + {:when, _, [_left, right]} -> + raise ArgumentError, + "guards are not allowed in defdelegate/2, got: when #{Macro.to_string(right)}" + + _ -> + case Macro.decompose_call(fun) do + {_, _} = pair -> pair + _ -> raise ArgumentError, "invalid syntax in defdelegate #{Macro.to_string(fun)}" + end + end + + as = Keyword.get(opts, :as, name) + as_args = build_as_args(args, append_first?) + + {name, args, as, as_args} + end + + defp build_as_args(args, append_first?) do + as_args = :lists.map(&build_as_arg/1, args) + + case append_first? do + true -> tl(as_args) ++ [hd(as_args)] + false -> as_args + end + end + + defp build_as_arg({:\\, _, [arg, _default_arg]}), do: validate_arg(arg) + defp build_as_arg(arg), do: validate_arg(arg) + + defp validate_arg({name, _, mod} = arg) when is_atom(name) and is_atom(mod) do + arg + end + + defp validate_arg(ast) do + raise ArgumentError, + "defdelegate/2 only accepts function parameters, got: #{Macro.to_string(ast)}" + end + + @doc """ + Callback for defstruct. + """ + def defstruct(module, fields, bootstrapped?) do + if Module.has_attribute?(module, :__struct__) do + raise ArgumentError, + "defstruct has already been called for " <> + "#{Kernel.inspect(module)}, defstruct can only be called once per module" + end + + case fields do + fs when is_list(fs) -> + :ok + + other -> + raise ArgumentError, "struct fields definition must be list, got: #{inspect(other)}" + end + + mapper = fn + {key, val} when is_atom(key) -> + try do + Macro.escape(val) + rescue + e in [ArgumentError] -> + raise ArgumentError, "invalid value for struct field #{key}, " <> Exception.message(e) + else + _ -> {key, val} + end + + key when is_atom(key) -> + {key, nil} + + other -> + raise ArgumentError, "struct field names must be atoms, got: #{inspect(other)}" + end + + fields = :lists.map(mapper, fields) + enforce_keys = List.wrap(Module.get_attribute(module, :enforce_keys)) + + # TODO: Make it raise on v2.0 + warn_on_duplicate_struct_key(:lists.keysort(1, fields)) + + foreach = fn + key when is_atom(key) -> + :ok + + key -> + raise ArgumentError, "keys given to @enforce_keys must be atoms, got: #{inspect(key)}" + end + + :lists.foreach(foreach, enforce_keys) + struct = :maps.put(:__struct__, module, :maps.from_list(fields)) + + body = + case bootstrapped? do + true -> + case enforce_keys do + [] -> + quote do + Enum.reduce(var!(kv), @__struct__, fn {key, val}, map -> + %{map | key => val} + end) + end + + _ -> + quote do + {map, keys} = + Enum.reduce(var!(kv), {@__struct__, unquote(enforce_keys)}, fn + {key, val}, {map, keys} -> + {%{map | key => val}, List.delete(keys, key)} + end) + + case keys do + [] -> + map + + _ -> + raise ArgumentError, + "the following keys must also be given when building " <> + "struct #{inspect(__MODULE__)}: #{inspect(keys)}" + end + end + end + + false -> + quote do + :lists.foldl( + fn {key, val}, acc -> Map.replace!(acc, key, val) end, + @__struct__, + var!(kv) + ) + end + end + + case enforce_keys -- :maps.keys(struct) do + [] -> + # The __struct__ field is used for expansion and for loading remote structs + Module.put_attribute(module, :__struct__, struct) + + # Finally store all field metadata to go into __info__(:struct) + mapper = fn {key, val} -> + %{field: key, default: val, required: :lists.member(key, enforce_keys)} + end + + {set, _} = :elixir_module.data_tables(module) + :ets.insert(set, {{:elixir, :struct}, :lists.map(mapper, fields)}) + {struct, Module.delete_attribute(module, :derive), body} + + error_keys -> + raise ArgumentError, + "@enforce_keys required keys (#{inspect(error_keys)}) that are not defined in defstruct: " <> + "#{inspect(fields)}" + end + end + + defp warn_on_duplicate_struct_key([]) do + :ok + end + + defp warn_on_duplicate_struct_key([{key, _} | [{key, _} | _] = rest]) do + IO.warn("duplicate key #{inspect(key)} found in struct") + warn_on_duplicate_struct_key(rest) + end + + defp warn_on_duplicate_struct_key([_ | rest]) do + warn_on_duplicate_struct_key(rest) + end + + @doc """ + Announcing callback for defstruct. + """ + def announce_struct(module) do + case :erlang.get(:elixir_compiler_info) do + :undefined -> :ok + {pid, _} -> send(pid, {:available, :struct, module}) + end + end + + @doc """ + Callback for raise. + """ + def raise(msg) when is_binary(msg) do + RuntimeError.exception(msg) + end + + def raise(module) when is_atom(module) do + module.exception([]) + end + + def raise(%_{__exception__: true} = exception) do + exception + end + + def raise(other) do + ArgumentError.exception( + "raise/1 and reraise/2 expect a module name, string or exception " <> + "as the first argument, got: #{inspect(other)}" + ) + end + + @doc """ + Callback for defguard. + + Rewrites an expression so it can be used both inside and outside a guard. + + Take, for example, the expression: + + is_integer(value) and rem(value, 2) == 0 + + If we wanted to create a macro, `is_even`, from this expression, that could be + used in guards, we'd have to take several things into account. + + First, if this expression is being used inside a guard, `value` needs to be + unquoted each place it occurs, since it has not yet been at that point in our + macro. + + Secondly, if the expression is being used outside of a guard, we want to unquote + `value`, but only once, and then re-use the unquoted form throughout the expression. + + This helper does exactly that: takes the AST for an expression and a list of + variable references it should be aware of, and rewrites it into a new expression + that checks for its presence in a guard, then unquotes the variable references as + appropriate. + + The following code + + expression = quote do: is_integer(value) and rem(value, 2) == 0 + variable_references = [value: Elixir] + Kernel.Utils.defguard(expression, variable_references) |> Macro.to_string() |> IO.puts() + + would print a code similar to: + + case Macro.Env.in_guard?(__CALLER__) do + true -> + quote do + is_integer(unquote(value)) and rem(unquote(value), 2) == 0 + end + + false -> + quote do + value = unquote(value) + is_integer(value) and rem(value, 2) == 0 + end + end + + """ + defmacro defguard(args, expr) do + defguard(args, expr, __CALLER__) + end + + @spec defguard([Macro.t()], Macro.t(), Macro.Env.t()) :: Macro.t() + def defguard(args, expr, env) do + {^args, vars} = extract_refs_from_args(args) + env = :elixir_env.with_vars(%{env | context: :guard}, vars) + {expr, _, _} = :elixir_expand.expand(expr, :elixir_env.env_to_ex(env), env) + + quote do + case Macro.Env.in_guard?(__CALLER__) do + true -> unquote(literal_quote(unquote_every_ref(expr, vars))) + false -> unquote(literal_quote(unquote_refs_once(expr, vars))) + end + end + end + + defp extract_refs_from_args(args) do + Macro.postwalk(args, [], fn + {ref, meta, context} = var, acc when is_atom(ref) and is_atom(context) -> + {var, [{ref, var_context(meta, context)} | acc]} + + node, acc -> + {node, acc} + end) + end + + # Finds every reference to `refs` in `guard` and wraps them in an unquote. + defp unquote_every_ref(guard, refs) do + Macro.postwalk(guard, fn + {ref, meta, context} = var when is_atom(ref) and is_atom(context) -> + case {ref, var_context(meta, context)} in refs do + true -> literal_unquote(var) + false -> var + end + + node -> + node + end) + end + + # Prefaces `guard` with unquoted versions of `refs`. + defp unquote_refs_once(guard, refs) do + {guard, used_refs} = + Macro.postwalk(guard, %{}, fn + {ref, meta, context} = var, acc when is_atom(ref) and is_atom(context) -> + pair = {ref, var_context(meta, context)} + + case pair in refs do + true -> + case acc do + %{^pair => {new_var, _}} -> + {new_var, acc} + + %{} -> + generated = String.to_atom("arg" <> Integer.to_string(map_size(acc) + 1)) + new_var = Macro.var(generated, Elixir) + {new_var, Map.put(acc, pair, {new_var, var})} + end + + false -> + {var, acc} + end + + node, acc -> + {node, acc} + end) + + all_used = for ref <- :lists.reverse(refs), used = :maps.get(ref, used_refs, nil), do: used + {vars, exprs} = :lists.unzip(all_used) + + quote do + {unquote_splicing(vars)} = {unquote_splicing(Enum.map(exprs, &literal_unquote/1))} + unquote(guard) + end + end + + defp literal_quote(ast) do + {:quote, [], [[do: ast]]} + end + + defp literal_unquote(ast) do + {:unquote, [], List.wrap(ast)} + end + + defp var_context(meta, kind) do + case :lists.keyfind(:counter, 1, meta) do + {:counter, counter} -> counter + false -> kind + end + end +end diff --git a/lib/elixir/lib/keyword.ex b/lib/elixir/lib/keyword.ex index a991c6a4ccb..fcd6a6b0187 100644 --- a/lib/elixir/lib/keyword.ex +++ b/lib/elixir/lib/keyword.ex @@ -1,31 +1,101 @@ defmodule Keyword do @moduledoc """ - A keyword is a list of tuples where the first element - of the tuple is an atom and the second element can be - any value. + A keyword list is a list that consists exclusively of two-element tuples. - A keyword may have duplicated keys so it is not strictly - a dictionary. However most of the functions in this module - behave exactly as a dictionary and mimic the API defined - by the `Dict` behaviour. + The first element of these tuples is known as the *key*, and it must be an atom. + The second element, known as the *value*, can be any term. - For example, `Keyword.get` will get the first entry matching - the given key, regardless if duplicated entries exist. - Similarly, `Keyword.put` and `Keyword.delete` ensure all - duplicated entries for a given key are removed when invoked. + Keywords are mostly used to work with optional values. - A handful of functions exist to handle duplicated keys, in - particular, `from_enum` allows creating a new keywords without - removing duplicated keys, `get_values` returns all values for - a given key and `delete_first` deletes just one of the existing - entries. + ## Examples + + For example, the following is a keyword list: + + [{:exit_on_close, true}, {:active, :once}, {:packet_size, 1024}] + + Elixir provides a special and more concise syntax for keyword lists: + + [exit_on_close: true, active: :once, packet_size: 1024] + + The two syntaxes return the exact same value. + + A *key* can be any atom, consisting of Unicode letters, numbers, + an underscore or the `@` sign. If the *key* should have any other + characters, such as spaces, you can wrap it in quotes: + + iex> ["exit on close": true] + ["exit on close": true] + + Wrapping an atom in quotes does not make it a string. Keyword list + *keys* are always atoms. Quotes should only be used when necessary + or Elixir will issue a warning. + + ## Duplicate keys and ordering + + A keyword may have duplicate keys so it is not strictly a key-value + data type. However most of the functions in this module work on a + key-value structure and behave similar to the functions you would + find in the `Map` module. For example, `Keyword.get/3` will get the first + entry matching the given key, regardless if duplicate entries exist. + Similarly, `Keyword.put/3` and `Keyword.delete/2` ensure all duplicate + entries for a given key are removed when invoked. Note, however, that + keyword list operations need to traverse the whole list in order to find + keys, so these operations are slower than their map counterparts. + + A handful of functions exist to handle duplicate keys, for example, + `get_values/2` returns all values for a given key and `delete_first/2` + deletes just the first entry of the existing ones. + + Even though lists preserve the existing order, the functions in + `Keyword` do not guarantee any ordering. For example, if you invoke + `Keyword.put(opts, new_key, new_value)`, there is no guarantee for + where `new_key` will be added to (the front, the end or anywhere else). + + Given ordering is not guaranteed, it is not recommended to pattern + match on keyword lists either. For example, a function such as: + + def my_function([some_key: value, another_key: another_value]) + + will match + + my_function([some_key: :foo, another_key: :bar]) + + but it won't match + + my_function([another_key: :bar, some_key: :foo]) + + Most of the functions in this module work in linear time. This means + that the time it takes to perform an operation grows at the same + rate as the length of the list. + + ## Call syntax + + When keyword lists are passed as the last argument to a function, + the square brackets around the keyword list can be omitted. For + example, the keyword list syntax: + + String.split("1-0", "-", [trim: true, parts: 2]) + + can be written without the enclosing brackets whenever it is the last + argument of a function call: + + String.split("1-0", "-", trim: true, parts: 2) + + Since tuples, lists and maps are treated similarly to function + arguments in Elixir syntax, this property is also available to them: + + iex> {1, 2, foo: :bar} + {1, 2, [{:foo, :bar}]} + + iex> [1, 2, foo: :bar] + [1, 2, {:foo, :bar}] + + iex> %{1 => 2, foo: :bar} + %{1 => 2, :foo => :bar} - Since a keyword list is simply a list, all the operations defined - in `Enum` and `List` can also be applied. """ @compile :inline_list_funcs - @behaviour Dict @type key :: atom @type value :: any @@ -34,86 +104,286 @@ defmodule Keyword do @type t(value) :: [{key, value}] @doc """ - Checks if the given argument is a keywords list or not. + Builds a keyword from the given `keys` and the fixed `value`. + + ## Examples + + iex> Keyword.from_keys([:foo, :bar, :baz], :atom) + [foo: :atom, bar: :atom, baz: :atom] + """ - @spec keyword?(term) :: boolean - def keyword?([{key, _value} | rest]) when is_atom(key) do - keyword?(rest) + @doc since: "1.14.0" + @spec from_keys([key], value) :: t(value) + def from_keys(keys, value) when is_list(keys) do + :lists.map(&{&1, value}, keys) end - def keyword?([]), do: true + @doc """ + Returns `true` if `term` is a keyword list, otherwise `false`. + + When `term` is a list it is traversed to the end. + + ## Examples + + iex> Keyword.keyword?([]) + true + iex> Keyword.keyword?(a: 1) + true + iex> Keyword.keyword?([{Foo, 1}]) + true + iex> Keyword.keyword?([{}]) + false + iex> Keyword.keyword?([:key]) + false + iex> Keyword.keyword?(%{}) + false + + """ + @spec keyword?(term) :: boolean + def keyword?(term) + + def keyword?([{key, _value} | rest]) when is_atom(key), do: keyword?(rest) + def keyword?([]), do: true def keyword?(_other), do: false @doc """ Returns an empty keyword list, i.e. an empty list. + + ## Examples + + iex> Keyword.new() + [] + """ - @spec new :: t - def new do - [] - end + @spec new :: [] + def new, do: [] @doc """ - Creates a keyword from an enumerable. + Creates a keyword list from an enumerable. - Duplicated entries are removed, the latest one prevails. - I.e. differently from `Enum.into(enumerable, [])`, - `Keyword.new(enumerable)` guarantees the keys are unique. + Removes duplicate entries and the last one prevails. + Unlike `Enum.into(enumerable, [])`, `Keyword.new(enumerable)` + guarantees the keys are unique. ## Examples iex> Keyword.new([{:b, 1}, {:a, 2}]) - [a: 2, b: 1] + [b: 1, a: 2] + + iex> Keyword.new([{:a, 1}, {:a, 2}, {:a, 3}]) + [a: 3] """ - @spec new(Enum.t) :: t + @spec new(Enumerable.t()) :: t def new(pairs) do - Enum.reduce pairs, [], fn {k, v}, keywords -> - put(keywords, k, v) - end + new(pairs, fn pair -> pair end) end @doc """ - Creates a keyword from an enumerable via the transformation function. + Creates a keyword list from an enumerable via the transformation function. - Duplicated entries are removed, the latest one prevails. - I.e. differently from `Enum.into(enumerable, [], fun)`, + Removes duplicate entries and the last one prevails. + Unlike `Enum.into(enumerable, [], fun)`, `Keyword.new(enumerable, fun)` guarantees the keys are unique. ## Examples - iex> Keyword.new([:a, :b], fn (x) -> {x, x} end) |> Enum.sort + iex> Keyword.new([:a, :b], fn x -> {x, x} end) [a: :a, b: :b] """ - @spec new(Enum.t, ({key, value} -> {key, value})) :: t - def new(pairs, transform) do - Enum.reduce pairs, [], fn i, keywords -> - {k, v} = transform.(i) - put(keywords, k, v) + @spec new(Enumerable.t(), (term -> {key, value})) :: t + def new(pairs, transform) when is_function(transform, 1) do + fun = fn el, acc -> + {k, v} = transform.(el) + put_new(acc, k, v) + end + + :lists.foldl(fun, [], Enum.reverse(pairs)) + end + + @doc """ + Ensures the given `keyword` has only the keys given in `values`. + + The second argument must be a list of atoms, specifying + a given key, or tuples specifying a key and a default value. + + If the keyword list has only the given keys, it returns + `{:ok, keyword}` with default values applied. Otherwise it + returns `{:error, invalid_keys}` with invalid keys. + + See also: `validate!/2`. + + ## Examples + + iex> {:ok, result} = Keyword.validate([], [one: 1, two: 2]) + iex> Enum.sort(result) + [one: 1, two: 2] + + iex> {:ok, result} = Keyword.validate([two: 3], [one: 1, two: 2]) + iex> Enum.sort(result) + [one: 1, two: 3] + + If atoms are given, they are supported as keys but do not + provide a default value: + + iex> {:ok, result} = Keyword.validate([], [:one, two: 2]) + iex> Enum.sort(result) + [two: 2] + + iex> {:ok, result} = Keyword.validate([one: 1], [:one, two: 2]) + iex> Enum.sort(result) + [one: 1, two: 2] + + Passing unknown keys returns an error: + + iex> Keyword.validate([three: 3, four: 4], [one: 1, two: 2]) + {:error, [:four, :three]} + + Passing the same key multiple times also errors: + + iex> Keyword.validate([one: 1, two: 2, one: 1], [:one, :two]) + {:error, [:one]} + + """ + @doc since: "1.13.0" + @spec validate(keyword(), values :: [atom() | {atom(), term()}]) :: + {:ok, keyword()} | {:error, [atom]} + def validate(keyword, values) when is_list(keyword) and is_list(values) do + validate(keyword, values, [], [], []) + end + + defp validate([{key, _} = pair | keyword], values1, values2, acc, bad_keys) when is_atom(key) do + case find_key!(key, values1, values2) do + {values1, values2} -> + validate(keyword, values1, values2, [pair | acc], bad_keys) + + :error -> + case find_key!(key, values2, values1) do + {values1, values2} -> + validate(keyword, values1, values2, [pair | acc], bad_keys) + + :error -> + validate(keyword, values1, values2, acc, [key | bad_keys]) + end + end + end + + defp validate([], values1, values2, acc, []) do + {:ok, move_pairs!(values1, move_pairs!(values2, acc))} + end + + defp validate([], _values1, _values2, _acc, bad_keys) do + {:error, bad_keys} + end + + defp validate([pair | _], _values1, _values2, _acc, []) do + raise ArgumentError, + "expected a keyword list as first argument, got invalid entry: #{inspect(pair)}" + end + + defp find_key!(key, [key | rest], acc), do: {rest, acc} + defp find_key!(key, [{key, _} | rest], acc), do: {rest, acc} + defp find_key!(key, [head | tail], acc), do: find_key!(key, tail, [head | acc]) + defp find_key!(_key, [], _acc), do: :error + + defp move_pairs!([key | rest], acc) when is_atom(key), + do: move_pairs!(rest, acc) + + defp move_pairs!([{key, _} = pair | rest], acc) when is_atom(key), + do: move_pairs!(rest, [pair | acc]) + + defp move_pairs!([], acc), + do: acc + + defp move_pairs!([other | _], _) do + raise ArgumentError, + "expected the second argument to be a list of atoms or tuples, got: #{inspect(other)}" + end + + @doc """ + Similar to `validate/2` but returns the keyword or raises an error. + + ## Examples + + iex> Keyword.validate!([], [one: 1, two: 2]) |> Enum.sort() + [one: 1, two: 2] + iex> Keyword.validate!([two: 3], [one: 1, two: 2]) |> Enum.sort() + [one: 1, two: 3] + + If atoms are given, they are supported as keys but do not + provide a default value: + + iex> Keyword.validate!([], [:one, two: 2]) |> Enum.sort() + [two: 2] + iex> Keyword.validate!([one: 1], [:one, two: 2]) |> Enum.sort() + [one: 1, two: 2] + + Passing unknown keys raises an error: + + iex> Keyword.validate!([three: 3], [one: 1, two: 2]) + ** (ArgumentError) unknown keys [:three] in [three: 3], the allowed keys are: [:one, :two] + + Passing the same key multiple times also errors: + + iex> Keyword.validate!([one: 1, two: 2, one: 1], [:one, :two]) + ** (ArgumentError) duplicate keys [:one] in [one: 1, two: 2, one: 1] + + """ + @doc since: "1.13.0" + @spec validate!(keyword(), values :: [atom() | {atom(), term()}]) :: keyword() + def validate!(keyword, values) do + case validate(keyword, values) do + {:ok, kw} -> + kw + + {:error, invalid_keys} -> + keys = + for value <- values, + do: if(is_atom(value), do: value, else: elem(value, 0)) + + message = + case Enum.split_with(invalid_keys, &(&1 in keys)) do + {_, [_ | _] = unknown} -> + "unknown keys #{inspect(unknown)} in #{inspect(keyword)}, " <> + "the allowed keys are: #{inspect(keys)}" + + {[_ | _] = known, _} -> + "duplicate keys #{inspect(known)} in #{inspect(keyword)}" + end + + raise ArgumentError, message end end @doc """ - Gets the value for a specific `key`. + Gets the value under the given `key`. - If `key` does not exist, return default value (`nil` if no default value). + Returns the default value if `key` does not exist + (`nil` if no default value is provided). - If duplicated entries exist, the first one is returned. + If duplicate entries exist, it returns the first one. Use `get_values/2` to retrieve all entries. ## Examples + iex> Keyword.get([], :a) + nil iex> Keyword.get([a: 1], :a) 1 - iex> Keyword.get([a: 1], :b) nil - iex> Keyword.get([a: 1], :b, 3) 3 + With duplicate keys: + + iex> Keyword.get([a: 1, a: 2], :a, 3) + 1 + iex> Keyword.get([a: 1, a: 2], :b, 3) + 3 + """ - @spec get(t, key) :: value @spec get(t, key, value) :: value def get(keywords, key, default \\ nil) when is_list(keywords) and is_atom(key) do case :lists.keyfind(key, 1, keywords) do @@ -122,20 +392,182 @@ defmodule Keyword do end end + @doc """ + Gets the value under the given `key`. + + If `key` does not exist, lazily evaluates `fun` and returns its result. + + This is useful if the default value is very expensive to calculate or + generally difficult to set up and tear down again. + + If duplicate entries exist, it returns the first one. + Use `get_values/2` to retrieve all entries. + + ## Examples + + iex> keyword = [a: 1] + iex> fun = fn -> + ...> # some expensive operation here + ...> 13 + ...> end + iex> Keyword.get_lazy(keyword, :a, fun) + 1 + iex> Keyword.get_lazy(keyword, :b, fun) + 13 + + """ + @spec get_lazy(t, key, (() -> value)) :: value + def get_lazy(keywords, key, fun) + when is_list(keywords) and is_atom(key) and is_function(fun, 0) do + case :lists.keyfind(key, 1, keywords) do + {^key, value} -> value + false -> fun.() + end + end + + @doc """ + Gets the value from `key` and updates it, all in one pass. + + The `fun` argument receives the value of `key` (or `nil` if `key` + is not present) and must return a two-element tuple: the current value + (the retrieved value, which can be operated on before being returned) + and the new value to be stored under `key`. The `fun` may also + return `:pop`, implying the current value shall be removed from the + keyword list and returned. + + Returns a tuple that contains the current value returned by + `fun` and a new keyword list with the updated value under `key`. + + ## Examples + + iex> Keyword.get_and_update([a: 1], :a, fn current_value -> + ...> {current_value, "new value!"} + ...> end) + {1, [a: "new value!"]} + + iex> Keyword.get_and_update([a: 1], :b, fn current_value -> + ...> {current_value, "new value!"} + ...> end) + {nil, [b: "new value!", a: 1]} + + iex> Keyword.get_and_update([a: 2], :a, fn number -> + ...> {2 * number, 3 * number} + ...> end) + {4, [a: 6]} + + iex> Keyword.get_and_update([a: 1], :a, fn _ -> :pop end) + {1, []} + + iex> Keyword.get_and_update([a: 1], :b, fn _ -> :pop end) + {nil, [a: 1]} + + """ + @spec get_and_update(t, key, (value | nil -> {current_value, new_value :: value} | :pop)) :: + {current_value, new_keywords :: t} + when current_value: value + def get_and_update(keywords, key, fun) + when is_list(keywords) and is_atom(key), + do: get_and_update(keywords, [], key, fun) + + defp get_and_update([{key, current} | t], acc, key, fun) do + case fun.(current) do + {get, value} -> + {get, :lists.reverse(acc, [{key, value} | t])} + + :pop -> + {current, :lists.reverse(acc, t)} + + other -> + raise "the given function must return a two-element tuple or :pop, got: #{inspect(other)}" + end + end + + defp get_and_update([{_, _} = h | t], acc, key, fun), do: get_and_update(t, [h | acc], key, fun) + + defp get_and_update([], acc, key, fun) do + case fun.(nil) do + {get, update} -> + {get, [{key, update} | :lists.reverse(acc)]} + + :pop -> + {nil, :lists.reverse(acc)} + + other -> + raise "the given function must return a two-element tuple or :pop, got: #{inspect(other)}" + end + end + + @doc """ + Gets the value under `key` and updates it. Raises if there is no `key`. + + The `fun` argument receives the value under `key` and must return a + two-element tuple: the current value (the retrieved value, which can be + operated on before being returned) and the new value to be stored under + `key`. + + Returns a tuple that contains the current value returned by + `fun` and a new keyword list with the updated value under `key`. + + ## Examples + + iex> Keyword.get_and_update!([a: 1], :a, fn current_value -> + ...> {current_value, "new value!"} + ...> end) + {1, [a: "new value!"]} + + iex> Keyword.get_and_update!([a: 1], :b, fn current_value -> + ...> {current_value, "new value!"} + ...> end) + ** (KeyError) key :b not found in: [a: 1] + + iex> Keyword.get_and_update!([a: 1], :a, fn _ -> + ...> :pop + ...> end) + {1, []} + + """ + @spec get_and_update!(t, key, (value | nil -> {current_value, new_value :: value} | :pop)) :: + {current_value, new_keywords :: t} + when current_value: value + def get_and_update!(keywords, key, fun) do + get_and_update!(keywords, key, fun, []) + end + + defp get_and_update!([{key, value} | keywords], key, fun, acc) do + case fun.(value) do + {get, value} -> + {get, :lists.reverse(acc, [{key, value} | delete(keywords, key)])} + + :pop -> + {value, :lists.reverse(acc, keywords)} + + other -> + raise "the given function must return a two-element tuple or :pop, got: #{inspect(other)}" + end + end + + defp get_and_update!([{_, _} = e | keywords], key, fun, acc) do + get_and_update!(keywords, key, fun, [e | acc]) + end + + defp get_and_update!([], key, _fun, acc) when is_atom(key) do + raise KeyError, key: key, term: acc + end + @doc """ Fetches the value for a specific `key` and returns it in a tuple. - If the `key` does not exist, returns `:error`. + + If the `key` does not exist, it returns `:error`. ## Examples iex> Keyword.fetch([a: 1], :a) {:ok, 1} - iex> Keyword.fetch([a: 1], :b) :error """ - @spec fetch(t, key) :: {:ok, value} + @spec fetch(t, key) :: {:ok, value} | :error def fetch(keywords, key) when is_list(keywords) and is_atom(key) do case :lists.keyfind(key, 1, keywords) do {^key, value} -> {:ok, value} @@ -144,70 +576,92 @@ defmodule Keyword do end @doc """ - Fetches the value for specific `key`. If `key` does not exist, - a `KeyError` is raised. + Fetches the value for specific `key`. + + If the `key` does not exist, it raises a `KeyError`. ## Examples iex> Keyword.fetch!([a: 1], :a) 1 - iex> Keyword.fetch!([a: 1], :b) ** (KeyError) key :b not found in: [a: 1] """ - @spec fetch!(t, key) :: value | no_return + @spec fetch!(t, key) :: value def fetch!(keywords, key) when is_list(keywords) and is_atom(key) do case :lists.keyfind(key, 1, keywords) do {^key, value} -> value - false -> raise(KeyError, key: key, term: keywords) + false -> raise KeyError, key: key, term: keywords end end @doc """ - Gets all values for a specific `key`. + Gets all values under a specific `key`. ## Examples + iex> Keyword.get_values([], :a) + [] + iex> Keyword.get_values([a: 1], :a) + [1] iex> Keyword.get_values([a: 1, a: 2], :a) - [1,2] + [1, 2] """ @spec get_values(t, key) :: [value] def get_values(keywords, key) when is_list(keywords) and is_atom(key) do - fun = fn - {k, v} when k === key -> {true, v} - {_, _} -> false - end - - :lists.filtermap(fun, keywords) + get_values(keywords, key, []) end + defp get_values([{key, value} | tail], key, values), do: get_values(tail, key, [value | values]) + defp get_values([{_, _} | tail], key, values), do: get_values(tail, key, values) + defp get_values([], _key, values), do: :lists.reverse(values) + @doc """ - Returns all keys from the keyword list. Duplicated - keys appear duplicated in the final list of keys. + Returns all keys from the keyword list. + + Keeps duplicate keys in the resulting list of keys. ## Examples - iex> Keyword.keys([a: 1, b: 2]) - [:a,:b] + iex> Keyword.keys(a: 1, b: 2) + [:a, :b] - iex> Keyword.keys([a: 1, b: 2, a: 3]) - [:a,:b,:a] + iex> Keyword.keys(a: 1, b: 2, a: 3) + [:a, :b, :a] + + iex> Keyword.keys([{:a, 1}, {"b", 2}, {:c, 3}]) + ** (ArgumentError) expected a keyword list, but an entry in the list is not a two-element tuple with an atom as its first element, got: {"b", 2} """ @spec keys(t) :: [key] def keys(keywords) when is_list(keywords) do - :lists.map(fn {k, _} -> k end, keywords) + :lists.map( + fn + {key, _} when is_atom(key) -> key + element -> throw(element) + end, + keywords + ) + catch + element -> + raise ArgumentError, + "expected a keyword list, but an entry in the list is not a two-element tuple " <> + "with an atom as its first element, got: #{inspect(element)}" end @doc """ Returns all values from the keyword list. + Keeps values from duplicate keys in the resulting list of values. + ## Examples - iex> Keyword.values([a: 1, b: 2]) - [1,2] + iex> Keyword.values(a: 1, b: 2) + [1, 2] + iex> Keyword.values(a: 1, b: 2, a: 3) + [1, 2, 3] """ @spec values(t) :: [value] @@ -215,97 +669,147 @@ defmodule Keyword do :lists.map(fn {_, v} -> v end, keywords) end - @doc """ - Deletes the entries in the keyword list for a `key` with `value`. - If no `key` with `value` exists, returns the keyword list unchanged. - - ## Examples - - iex> Keyword.delete([a: 1, b: 2], :a, 1) - [b: 2] + @doc false + @deprecated "Use Keyword.fetch/2 + Keyword.delete/2 instead" + def delete(keywords, key, value) when is_list(keywords) and is_atom(key) do + case :lists.keymember(key, 1, keywords) do + true -> delete_key_value(keywords, key, value) + _ -> keywords + end + end - iex> Keyword.delete([a: 1, b: 2, a: 3], :a, 3) - [a: 1, b: 2] + defp delete_key_value([{key, value} | tail], key, value) do + delete_key_value(tail, key, value) + end - iex> Keyword.delete([b: 2], :a, 5) - [b: 2] + defp delete_key_value([{_, _} = pair | tail], key, value) do + [pair | delete_key_value(tail, key, value)] + end - """ - @spec delete(t, key, value) :: t - def delete(keywords, key, value) when is_list(keywords) and is_atom(key) do - :lists.filter(fn {k, v} -> k != key or v != value end, keywords) + defp delete_key_value([], _key, _value) do + [] end @doc """ - Deletes the entries in the keyword list for a specific `key`. - If the `key` does not exist, returns the keyword list unchanged. - Use `delete_first` to delete just the first entry in case of - duplicated keys. + Deletes the entries in the keyword list under a specific `key`. + + If the `key` does not exist, it returns the keyword list unchanged. + Use `delete_first/2` to delete just the first entry in case of + duplicate keys. ## Examples iex> Keyword.delete([a: 1, b: 2], :a) [b: 2] - iex> Keyword.delete([a: 1, b: 2, a: 3], :a) [b: 2] - iex> Keyword.delete([b: 2], :a) [b: 2] """ @spec delete(t, key) :: t + @compile {:inline, delete: 2} def delete(keywords, key) when is_list(keywords) and is_atom(key) do - :lists.filter(fn {k, _} -> k != key end, keywords) + case :lists.keymember(key, 1, keywords) do + true -> delete_key(keywords, key) + _ -> keywords + end end + defp delete_key([{key, _} | tail], key), do: delete_key(tail, key) + defp delete_key([{_, _} = pair | tail], key), do: [pair | delete_key(tail, key)] + defp delete_key([], _key), do: [] + @doc """ - Deletes the first entry in the keyword list for a specific `key`. - If the `key` does not exist, returns the keyword list unchanged. + Deletes the first entry in the keyword list under a specific `key`. + + If the `key` does not exist, it returns the keyword list unchanged. ## Examples iex> Keyword.delete_first([a: 1, b: 2, a: 3], :a) [b: 2, a: 3] - iex> Keyword.delete_first([b: 2], :a) [b: 2] """ @spec delete_first(t, key) :: t def delete_first(keywords, key) when is_list(keywords) and is_atom(key) do - :lists.keydelete(key, 1, keywords) + case :lists.keymember(key, 1, keywords) do + true -> delete_first_key(keywords, key) + _ -> keywords + end + end + + defp delete_first_key([{key, _} | tail], key) do + tail + end + + defp delete_first_key([{_, _} = pair | tail], key) do + [pair | delete_first_key(tail, key)] + end + + defp delete_first_key([], _key) do + [] end @doc """ - Puts the given `value` under `key`. + Puts the given `value` under the specified `key`. - If a previous value is already stored, all entries are - removed and the value is overridden. + If a value under `key` already exists, it overrides the value + and removes all duplicate entries. ## Examples + iex> Keyword.put([a: 1], :b, 2) + [b: 2, a: 1] iex> Keyword.put([a: 1, b: 2], :a, 3) [a: 3, b: 2] - iex> Keyword.put([a: 1, b: 2, a: 4], :a, 3) [a: 3, b: 2] """ @spec put(t, key, value) :: t def put(keywords, key, value) when is_list(keywords) and is_atom(key) do - [{key, value}|delete(keywords, key)] + [{key, value} | delete(keywords, key)] + end + + @doc """ + Evaluates `fun` and puts the result under `key` + in keyword list unless `key` is already present. + + This is useful if the value is very expensive to calculate or + generally difficult to set up and tear down again. + + ## Examples + + iex> keyword = [a: 1] + iex> fun = fn -> + ...> # some expensive operation here + ...> 13 + ...> end + iex> Keyword.put_new_lazy(keyword, :a, fun) + [a: 1] + iex> Keyword.put_new_lazy(keyword, :b, fun) + [b: 13, a: 1] + + """ + @spec put_new_lazy(t, key, (() -> value)) :: t + def put_new_lazy(keywords, key, fun) + when is_list(keywords) and is_atom(key) and is_function(fun, 0) do + case :lists.keyfind(key, 1, keywords) do + {^key, _} -> keywords + false -> [{key, fun.()} | keywords] + end end @doc """ - Puts the given `value` under `key` unless the entry `key` - already exists. + Puts the given `value` under `key`, unless the entry `key` already exists. ## Examples iex> Keyword.put_new([a: 1], :b, 2) [b: 2, a: 1] - iex> Keyword.put_new([a: 1, b: 2], :a, 3) [a: 1, b: 2] @@ -314,64 +818,249 @@ defmodule Keyword do def put_new(keywords, key, value) when is_list(keywords) and is_atom(key) do case :lists.keyfind(key, 1, keywords) do {^key, _} -> keywords - false -> [{key, value}|keywords] + false -> [{key, value} | keywords] end end @doc """ - Checks if two keywords are equal. I.e. they contain + Puts a value under `key` only if the `key` already exists in `keywords`. + + In case a key exists multiple times in the keyword list, + it removes later occurrences. + + ## Examples + + iex> Keyword.replace([a: 1, b: 2, a: 4], :a, 3) + [a: 3, b: 2] + + iex> Keyword.replace([a: 1], :b, 2) + [a: 1] + + """ + @doc since: "1.11.0" + @spec replace(t, key, value) :: t + def replace(keywords, key, value) when is_list(keywords) and is_atom(key) do + do_replace(keywords, key, value) + end + + defp do_replace([{key, _} | keywords], key, value) do + [{key, value} | delete(keywords, key)] + end + + defp do_replace([{_, _} = e | keywords], key, value) do + [e | do_replace(keywords, key, value)] + end + + defp do_replace([], _key, _value) do + [] + end + + @doc """ + Puts a value under `key` only if the `key` already exists in `keywords`. + + If `key` is not present in `keywords`, it raises a `KeyError`. + + ## Examples + + iex> Keyword.replace!([a: 1, b: 2, a: 3], :a, :new) + [a: :new, b: 2] + iex> Keyword.replace!([a: 1, b: 2, c: 3, b: 4], :b, :new) + [a: 1, b: :new, c: 3] + + iex> Keyword.replace!([a: 1], :b, 2) + ** (KeyError) key :b not found in: [a: 1] + + """ + @doc since: "1.5.0" + @spec replace!(t, key, value) :: t + def replace!(keywords, key, value) when is_list(keywords) and is_atom(key) do + replace!(keywords, key, value, keywords) + end + + defp replace!([{key, _} | keywords], key, value, _original) do + [{key, value} | delete(keywords, key)] + end + + defp replace!([{_, _} = e | keywords], key, value, original) do + [e | replace!(keywords, key, value, original)] + end + + defp replace!([], key, _value, original) do + raise KeyError, key: key, term: original + end + + @doc """ + Replaces the value under `key` using the given function only if + `key` already exists in `keywords`. + + In comparison to `replace/3`, this can be useful when it's expensive to calculate the value. + + If `key` does not exist, the original keyword list is returned unchanged. + + ## Examples + + iex> Keyword.replace_lazy([a: 1, b: 2], :a, fn v -> v * 4 end) + [a: 4, b: 2] + + iex> Keyword.replace_lazy([a: 2, b: 2, a: 1], :a, fn v -> v * 4 end) + [a: 8, b: 2] + + iex> Keyword.replace_lazy([a: 1, b: 2], :c, fn v -> v * 4 end) + [a: 1, b: 2] + + """ + @doc since: "1.14.0" + @spec replace_lazy(t, key, (existing_value :: value -> new_value :: value)) :: t + def replace_lazy(keywords, key, fun) + when is_list(keywords) and is_atom(key) and is_function(fun, 1) do + do_replace_lazy(keywords, key, fun) + end + + defp do_replace_lazy([{key, value} | keywords], key, fun) do + [{key, fun.(value)} | delete(keywords, key)] + end + + defp do_replace_lazy([{_, _} = e | keywords], key, fun) do + [e | do_replace_lazy(keywords, key, fun)] + end + + defp do_replace_lazy([], _key, _value), do: [] + + @doc """ + Checks if two keywords are equal. + + Considers two keywords to be equal if they contain the same keys and those keys contain the same values. ## Examples iex> Keyword.equal?([a: 1, b: 2], [b: 2, a: 1]) true + iex> Keyword.equal?([a: 1, b: 2], [b: 1, a: 2]) + false + iex> Keyword.equal?([a: 1, b: 2, a: 3], [b: 2, a: 3, a: 1]) + true + + Comparison between values is done with `===/3`, + which means integers are not equivalent to floats: + + iex> Keyword.equal?([a: 1.0], [a: 1]) + false """ @spec equal?(t, t) :: boolean def equal?(left, right) when is_list(left) and is_list(right) do - :lists.sort(left) == :lists.sort(right) + :lists.sort(left) === :lists.sort(right) end @doc """ - Merges two keyword lists into one. If they have duplicated - entries, the one given as second argument wins. + Merges two keyword lists into one. + + Adds all keys, including duplicate keys, given in `keywords2` + to `keywords1`, overriding any existing ones. + + There are no guarantees about the order of the keys in the returned keyword. ## Examples - iex> Keyword.merge([a: 1, b: 2], [a: 3, d: 4]) |> Enum.sort - [a: 3, b: 2, d: 4] + iex> Keyword.merge([a: 1, b: 2], [a: 3, d: 4]) + [b: 2, a: 3, d: 4] + + iex> Keyword.merge([a: 1, b: 2], [a: 3, d: 4, a: 5]) + [b: 2, a: 3, d: 4, a: 5] + + iex> Keyword.merge([a: 1], [2, 3]) + ** (ArgumentError) expected a keyword list as the second argument, got: [2, 3] """ @spec merge(t, t) :: t - def merge(d1, d2) when is_list(d1) and is_list(d2) do - fun = fn {k, _v} -> not has_key?(d2, k) end - d2 ++ :lists.filter(fun, d1) + def merge(keywords1, keywords2) + + def merge(keywords1, []) when is_list(keywords1), do: keywords1 + def merge([], keywords2) when is_list(keywords2), do: keywords2 + + def merge(keywords1, keywords2) when is_list(keywords1) and is_list(keywords2) do + if keyword?(keywords2) do + fun = fn + {key, _value} when is_atom(key) -> + not has_key?(keywords2, key) + + _ -> + raise ArgumentError, + "expected a keyword list as the first argument, got: #{inspect(keywords1)}" + end + + :lists.filter(fun, keywords1) ++ keywords2 + else + raise ArgumentError, + "expected a keyword list as the second argument, got: #{inspect(keywords2)}" + end end @doc """ - Merges two keyword lists into one. If they have duplicated - entries, the given function is invoked to solve conflicts. + Merges two keyword lists into one. + + Adds all keys, including duplicate keys, given in `keywords2` + to `keywords1`. Invokes the given function to solve conflicts. + + If `keywords2` has duplicate keys, it invokes the given function + for each matching pair in `keywords1`. + + There are no guarantees about the order of the keys in the returned keyword. ## Examples - iex> Keyword.merge([a: 1, b: 2], [a: 3, d: 4], fn (_k, v1, v2) -> - ...> v1 + v2 + iex> Keyword.merge([a: 1, b: 2], [a: 3, d: 4], fn _k, v1, v2 -> + ...> v1 + v2 ...> end) - [a: 4, b: 2, d: 4] + [b: 2, a: 4, d: 4] + + iex> Keyword.merge([a: 1, b: 2], [a: 3, d: 4, a: 5], fn :a, v1, v2 -> + ...> v1 + v2 + ...> end) + [b: 2, a: 4, d: 4, a: 5] + + iex> Keyword.merge([a: 1, b: 2, a: 3], [a: 3, d: 4, a: 5], fn :a, v1, v2 -> + ...> v1 + v2 + ...> end) + [b: 2, a: 4, d: 4, a: 8] + + iex> Keyword.merge([a: 1, b: 2], [:a, :b], fn :a, v1, v2 -> + ...> v1 + v2 + ...> end) + ** (ArgumentError) expected a keyword list as the second argument, got: [:a, :b] """ @spec merge(t, t, (key, value, value -> value)) :: t - def merge(d1, d2, fun) when is_list(d1) and is_list(d2) do - do_merge(d2, d1, fun) + def merge(keywords1, keywords2, fun) + when is_list(keywords1) and is_list(keywords2) and is_function(fun, 3) do + if keyword?(keywords1) do + do_merge(keywords2, [], keywords1, keywords1, fun, keywords2) + else + raise ArgumentError, + "expected a keyword list as the first argument, got: #{inspect(keywords1)}" + end end - defp do_merge([{k, v2}|t], acc, fun) do - do_merge t, update(acc, k, v2, fn(v1) -> fun.(k, v1, v2) end), fun + defp do_merge([{key, value2} | tail], acc, rest, original, fun, keywords2) when is_atom(key) do + case :lists.keyfind(key, 1, original) do + {^key, value1} -> + acc = [{key, fun.(key, value1, value2)} | acc] + original = :lists.keydelete(key, 1, original) + do_merge(tail, acc, delete(rest, key), original, fun, keywords2) + + false -> + do_merge(tail, [{key, value2} | acc], rest, original, fun, keywords2) + end + end + + defp do_merge([], acc, rest, _original, _fun, _keywords2) do + rest ++ :lists.reverse(acc) end - defp do_merge([], acc, _fun) do - acc + defp do_merge(_other, _acc, _rest, _original, _fun, keywords2) do + raise ArgumentError, + "expected a keyword list as the second argument, got: #{inspect(keywords2)}" end @doc """ @@ -381,7 +1070,6 @@ defmodule Keyword do iex> Keyword.has_key?([a: 1], :a) true - iex> Keyword.has_key?([a: 1], :b) false @@ -392,92 +1080,103 @@ defmodule Keyword do end @doc """ - Updates the `key` with the given function. If the `key` does - not exist, raises `KeyError`. + Updates the value under `key` using the given function. + + Raises `KeyError` if the `key` does not exist. - If there are duplicated entries, they are all removed and only the first one - is updated. + Removes all duplicate keys and only updates the first one. ## Examples - iex> Keyword.update!([a: 1], :a, &(&1 * 2)) - [a: 2] + iex> Keyword.update!([a: 1, b: 2, a: 3], :a, &(&1 * 2)) + [a: 2, b: 2] + iex> Keyword.update!([a: 1, b: 2, c: 3], :b, &(&1 * 2)) + [a: 1, b: 4, c: 3] iex> Keyword.update!([a: 1], :b, &(&1 * 2)) ** (KeyError) key :b not found in: [a: 1] """ - @spec update!(t, key, (value -> value)) :: t | no_return - def update!(keywords, key, fun) do + @spec update!(t, key, (current_value :: value -> new_value :: value)) :: t + def update!(keywords, key, fun) + when is_list(keywords) and is_atom(key) and is_function(fun, 1) do update!(keywords, key, fun, keywords) end - defp update!([{key, value}|keywords], key, fun, _dict) do - [{key, fun.(value)}|delete(keywords, key)] + defp update!([{key, value} | keywords], key, fun, _original) do + [{key, fun.(value)} | delete(keywords, key)] end - defp update!([{_, _} = e|keywords], key, fun, dict) do - [e|update!(keywords, key, fun, dict)] + defp update!([{_, _} = pair | keywords], key, fun, original) do + [pair | update!(keywords, key, fun, original)] end - defp update!([], key, _fun, dict) when is_atom(key) do - raise(KeyError, key: key, term: dict) + defp update!([], key, _fun, original) do + raise KeyError, key: key, term: original end @doc """ - Updates the `key` with the given function. If the `key` does - not exist, inserts the given `initial` value. + Updates the value under `key` in `keywords` using the given function. + + If the `key` does not exist, it inserts the given `default` value. + Does not pass the `default` value through the update function. - If there are duplicated entries, they are all removed and only the first one - is updated. + Removes all duplicate keys and only updates the first one. ## Examples - iex> Keyword.update([a: 1], :a, 13, &(&1 * 2)) + iex> Keyword.update([a: 1], :a, 13, fn existing_value -> existing_value * 2 end) + [a: 2] + + iex> Keyword.update([a: 1, a: 2], :a, 13, fn existing_value -> existing_value * 2 end) [a: 2] - iex> Keyword.update([a: 1], :b, 11, &(&1 * 2)) + iex> Keyword.update([a: 1], :b, 11, fn existing_value -> existing_value * 2 end) [a: 1, b: 11] """ - @spec update(t, key, value, (value -> value)) :: t - def update([{key, value}|keywords], key, _initial, fun) do - [{key, fun.(value)}|delete(keywords, key)] + @spec update(t, key, default :: value, (existing_value :: value -> new_value :: value)) :: t + def update(keywords, key, default, fun) + when is_list(keywords) and is_atom(key) and is_function(fun, 1) do + update_guarded(keywords, key, default, fun) + end + + defp update_guarded([{key, value} | keywords], key, _default, fun) do + [{key, fun.(value)} | delete(keywords, key)] end - def update([{_, _} = e|keywords], key, initial, fun) do - [e|update(keywords, key, initial, fun)] + defp update_guarded([{_, _} = pair | keywords], key, default, fun) do + [pair | update_guarded(keywords, key, default, fun)] end - def update([], key, initial, _fun) when is_atom(key) do - [{key, initial}] + defp update_guarded([], key, default, _fun) do + [{key, default}] end @doc """ - Takes all entries corresponding to the given keys and extracts them into a - separate keyword list. Returns a tuple with the new list and the old list - with removed keys. + Takes all entries corresponding to the given `keys` and extracts them into a + separate keyword list. - Keys for which there are no entires in the keyword list are ignored. + Returns a tuple with the new list and the old list with removed keys. - Entries with duplicated keys end up in the same keyword list. + Ignores keys for which there are no entries in the keyword list. - ## Examples + Entries with duplicate keys end up in the same keyword list. - iex> d = [a: 1, b: 2, c: 3, d: 4] - iex> Keyword.split(d, [:a, :c, :e]) - {[a: 1, c: 3], [b: 2, d: 4]} + ## Examples - iex> d = [a: 1, b: 2, c: 3, d: 4, a: 5] - iex> Keyword.split(d, [:a, :c, :e]) - {[a: 1, c: 3, a: 5], [b: 2, d: 4]} + iex> Keyword.split([a: 1, b: 2, c: 3], [:a, :c, :e]) + {[a: 1, c: 3], [b: 2]} + iex> Keyword.split([a: 1, b: 2, c: 3, a: 4], [:a, :c, :e]) + {[a: 1, c: 3, a: 4], [b: 2]} """ - def split(keywords, keys) when is_list(keywords) do + @spec split(t, [key]) :: {t, t} + def split(keywords, keys) when is_list(keywords) and is_list(keys) do fun = fn {k, v}, {take, drop} -> case k in keys do - true -> {[{k, v}|take], drop} - false -> {take, [{k, v}|drop]} + true -> {[{k, v} | take], drop} + false -> {take, [{k, v} | drop]} end end @@ -487,113 +1186,270 @@ defmodule Keyword do end @doc """ - Takes all entries corresponding to the given keys and returns them in a new + Takes all entries corresponding to the given `keys` and returns them as a new keyword list. - Duplicated keys are preserved in the new keyword list. + Preserves duplicate keys in the new keyword list. ## Examples - iex> d = [a: 1, b: 2, c: 3, d: 4] - iex> Keyword.take(d, [:a, :c, :e]) + iex> Keyword.take([a: 1, b: 2, c: 3], [:a, :c, :e]) [a: 1, c: 3] - - iex> d = [a: 1, b: 2, c: 3, d: 4, a: 5] - iex> Keyword.take(d, [:a, :c, :e]) + iex> Keyword.take([a: 1, b: 2, c: 3, a: 5], [:a, :c, :e]) [a: 1, c: 3, a: 5] """ - def take(keywords, keys) when is_list(keywords) do + @spec take(t, [key]) :: t + def take(keywords, keys) when is_list(keywords) and is_list(keys) do :lists.filter(fn {k, _} -> k in keys end, keywords) end @doc """ - Drops the given keys from the dict. + Drops the given `keys` from the keyword list. - Duplicated keys are preserved in the new keyword list. + Removes duplicate keys from the new keyword list. ## Examples - iex> d = [a: 1, b: 2, c: 3, d: 4] - iex> Keyword.drop(d, [:b, :d]) + iex> Keyword.drop([a: 1, a: 2], [:a]) + [] + iex> Keyword.drop([a: 1, b: 2, c: 3], [:b, :d]) [a: 1, c: 3] - - iex> d = [a: 1, b: 2, b: 3, c: 3, d: 4, a: 5] - iex> Keyword.drop(d, [:b, :d]) + iex> Keyword.drop([a: 1, b: 2, b: 3, c: 3, a: 5], [:b, :d]) [a: 1, c: 3, a: 5] """ - def drop(keywords, keys) when is_list(keywords) do - :lists.filter(fn {k, _} -> not k in keys end, keywords) + @spec drop(t, [key]) :: t + def drop(keywords, keys) when is_list(keywords) and is_list(keys) do + :lists.filter(fn {k, _} -> k not in keys end, keywords) + end + + @doc """ + Returns the first value for `key` and removes all associated entries in the keyword list. + + It returns a tuple where the first element is the first value for `key` and the + second element is a keyword list with all entries associated with `key` removed. + If the `key` is not present in the keyword list, it returns `{default, keyword_list}`. + + If you don't want to remove all the entries associated with `key` use `pop_first/3` + instead, which will remove only the first entry. + + ## Examples + + iex> Keyword.pop([a: 1], :a) + {1, []} + iex> Keyword.pop([a: 1], :b) + {nil, [a: 1]} + iex> Keyword.pop([a: 1], :b, 3) + {3, [a: 1]} + iex> Keyword.pop([a: 1, a: 2], :a) + {1, []} + + """ + @spec pop(t, key, value) :: {value, t} + def pop(keywords, key, default \\ nil) when is_list(keywords) and is_atom(key) do + case fetch(keywords, key) do + {:ok, value} -> {value, delete(keywords, key)} + :error -> {default, keywords} + end end @doc """ - Returns the first value associated with `key` in the keyword - list as well as the keyword list without `key`. + Returns the first value for `key` and removes all associated entries in the keyword list, + raising if `key` is not present. - All duplicated entries are removed. See `pop_first/3` for - removing only the first entry. + This function behaves like `pop/3`, but raises in case the `key` is not present in the + given `keywords`. ## Examples - iex> Keyword.pop [a: 1], :a - {1,[]} + iex> Keyword.pop!([a: 1], :a) + {1, []} + iex> Keyword.pop!([a: 1, a: 2], :a) + {1, []} + iex> Keyword.pop!([a: 1], :b) + ** (KeyError) key :b not found in: [a: 1] + + """ + @doc since: "1.10.0" + @spec pop!(t, key) :: {value, t} + def pop!(keywords, key) when is_list(keywords) and is_atom(key) do + case fetch(keywords, key) do + {:ok, value} -> {value, delete(keywords, key)} + :error -> raise KeyError, key: key, term: keywords + end + end + + @doc """ + Returns all values for `key` and removes all associated entries in the keyword list. - iex> Keyword.pop [a: 1], :b - {nil,[a: 1]} + It returns a tuple where the first element is a list of values for `key` and the + second element is a keyword list with all entries associated with `key` removed. + If the `key` is not present in the keyword list, it returns `{[], keyword_list}`. - iex> Keyword.pop [a: 1], :b, 3 - {3,[a: 1]} + If you don't want to remove all the entries associated with `key` use `pop_first/3` + instead, which will remove only the first entry. - iex> Keyword.pop [a: 1], :b, 3 - {3,[a: 1]} + ## Examples - iex> Keyword.pop [a: 1, a: 2], :a - {1,[]} + iex> Keyword.pop_values([a: 1], :a) + {[1], []} + iex> Keyword.pop_values([a: 1], :b) + {[], [a: 1]} + iex> Keyword.pop_values([a: 1, a: 2], :a) + {[1, 2], []} """ - def pop(keywords, key, default \\ nil) when is_list(keywords) do - {get(keywords, key, default), delete(keywords, key)} + @doc since: "1.10.0" + @spec pop_values(t, key) :: {[value], t} + def pop_values(keywords, key) when is_list(keywords) and is_atom(key) do + pop_values(:lists.reverse(keywords), key, [], []) end + defp pop_values([{key, value} | tail], key, values, acc), + do: pop_values(tail, key, [value | values], acc) + + defp pop_values([{_, _} = pair | tail], key, values, acc), + do: pop_values(tail, key, values, [pair | acc]) + + defp pop_values([], _key, values, acc), + do: {values, acc} + @doc """ - Returns the first value associated with `key` in the keyword - list as well as the keyword list without that particular ocurrence - of `key`. + Lazily returns and removes all values associated with `key` in the keyword list. + + This is useful if the default value is very expensive to calculate or + generally difficult to set up and tear down again. - Duplicated entries are not removed. + Removes all duplicate keys. See `pop_first/3` for removing only the first entry. ## Examples - iex> Keyword.pop_first [a: 1], :a - {1,[]} + iex> keyword = [a: 1] + iex> fun = fn -> + ...> # some expensive operation here + ...> 13 + ...> end + iex> Keyword.pop_lazy(keyword, :a, fun) + {1, []} + iex> Keyword.pop_lazy(keyword, :b, fun) + {13, [a: 1]} - iex> Keyword.pop_first [a: 1], :b - {nil,[a: 1]} + """ + @spec pop_lazy(t, key, (() -> value)) :: {value, t} + def pop_lazy(keywords, key, fun) + when is_list(keywords) and is_atom(key) and is_function(fun, 0) do + case fetch(keywords, key) do + {:ok, value} -> {value, delete(keywords, key)} + :error -> {fun.(), keywords} + end + end - iex> Keyword.pop_first [a: 1], :b, 3 - {3,[a: 1]} + @doc """ + Returns and removes the first value associated with `key` in the keyword list. - iex> Keyword.pop_first [a: 1], :b, 3 - {3,[a: 1]} + Keeps duplicate keys in the resulting keyword list. - iex> Keyword.pop_first [a: 1, a: 2], :a - {1,[a: 2]} + ## Examples + + iex> Keyword.pop_first([a: 1], :a) + {1, []} + iex> Keyword.pop_first([a: 1], :b) + {nil, [a: 1]} + iex> Keyword.pop_first([a: 1], :b, 3) + {3, [a: 1]} + iex> Keyword.pop_first([a: 1, a: 2], :a) + {1, [a: 2]} """ - def pop_first(keywords, key, default \\ nil) when is_list(keywords) do - {get(keywords, key, default), delete_first(keywords, key)} + @spec pop_first(t, key, value) :: {value, t} + def pop_first(keywords, key, default \\ nil) when is_list(keywords) and is_atom(key) do + case :lists.keytake(key, 1, keywords) do + {:value, {^key, value}, rest} -> {value, rest} + false -> {default, keywords} + end end - # Dict callbacks + @doc """ + Returns the keyword list itself. + + ## Examples + + iex> Keyword.to_list(a: 1) + [a: 1] + + """ + @spec to_list(t) :: t + def to_list(keywords) when is_list(keywords) do + keywords + end @doc false - def size(keyword) do - length(keyword) + @deprecated "Use Kernel.length/1 instead" + def size(keywords) do + length(keywords) + end + + @doc """ + Returns a keyword list containing only the entries from `keywords` + for which the function `fun` returns a truthy value. + + See also `reject/2` which discards all entries where the function + returns a truthy value. + + ## Examples + + iex> Keyword.filter([one: 1, two: 2, three: 3], fn {_key, val} -> rem(val, 2) == 1 end) + [one: 1, three: 3] + + """ + @doc since: "1.13.0" + @spec filter(t, ({key, value} -> as_boolean(term))) :: t + def filter(keywords, fun) when is_list(keywords) and is_function(fun, 1) do + do_filter(keywords, fun) + end + + defp do_filter([], _fun), do: [] + + defp do_filter([{_, _} = entry | entries], fun) do + if fun.(entry) do + [entry | do_filter(entries, fun)] + else + do_filter(entries, fun) + end + end + + @doc """ + Returns a keyword list excluding the entries from `keywords` + for which the function `fun` returns a truthy value. + + See also `filter/2`. + + ## Examples + + iex> Keyword.reject([one: 1, two: 2, three: 3], fn {_key, val} -> rem(val, 2) == 1 end) + [two: 2] + + """ + @doc since: "1.13.0" + @spec reject(t, ({key, value} -> as_boolean(term))) :: t + def reject(keywords, fun) when is_list(keywords) and is_function(fun, 1) do + do_reject(keywords, fun) + end + + defp do_reject([], _fun), do: [] + + defp do_reject([{_, _} = entry | entries], fun) do + if fun.(entry) do + do_reject(entries, fun) + else + [entry | do_reject(entries, fun)] + end end @doc false - def to_list(keyword) do - keyword + @deprecated "Use Keyword.new/2 instead" + def map(keywords, fun) when is_list(keywords) do + Enum.map(keywords, fn {k, v} -> {k, fun.({k, v})} end) end end diff --git a/lib/elixir/lib/list.ex b/lib/elixir/lib/list.ex index f4990bdc337..88571b11c57 100644 --- a/lib/elixir/lib/list.ex +++ b/lib/elixir/lib/list.ex @@ -1,55 +1,188 @@ defmodule List do @moduledoc """ - Implements functions that only make sense for lists - and cannot be part of the Enum protocol. In general, - favor using the Enum API instead of List. + Linked lists hold zero, one, or more elements in the chosen order. - Some functions in this module expect an index. Index - access for list is linear. Negative indexes are also - supported but they imply the list will be iterated twice, - one to calculate the proper index and another to the - operation. + Lists in Elixir are specified between square brackets: - A decision was taken to delegate most functions to - Erlang's standard library but follow Elixir's convention - of receiving the target (in this case, a list) as the - first argument. + iex> [1, "two", 3, :four] + [1, "two", 3, :four] + + Two lists can be concatenated and subtracted using the + `++/2` and `--/2` operators: + + iex> [1, 2, 3] ++ [4, 5, 6] + [1, 2, 3, 4, 5, 6] + iex> [1, true, 2, false, 3, true] -- [true, false] + [1, 2, 3, true] + + An element can be prepended to a list using `|`: + + iex> new = 0 + iex> list = [1, 2, 3] + iex> [new | list] + [0, 1, 2, 3] + + Lists in Elixir are effectively linked lists, which means + they are internally represented in pairs containing the + head and the tail of a list: + + iex> [head | tail] = [1, 2, 3] + iex> head + 1 + iex> tail + [2, 3] + + Similarly, we could write the list `[1, 2, 3]` using only + such pairs (called cons cells): + + iex> [1 | [2 | [3 | []]]] + [1, 2, 3] + + Some lists, called improper lists, do not have an empty list as + the second element in the last cons cell: + + iex> [1 | [2 | [3 | 4]]] + [1, 2, 3 | 4] + + Although improper lists are generally avoided, they are used in some + special circumstances like iodata and chardata entities (see the `IO` module). + + Due to their cons cell based representation, prepending an element + to a list is always fast (constant time), while appending becomes + slower as the list grows in size (linear time): + + iex> list = [1, 2, 3] + iex> [0 | list] # fast + [0, 1, 2, 3] + iex> list ++ [4] # slow + [1, 2, 3, 4] + + Most of the functions in this module work in linear time. This means that, + that the time it takes to perform an operation grows at the same rate as the + length of the list. For example `length/1` and `last/1` will run in linear + time because they need to iterate through every element of the list, but + `first/1` will run in constant time because it only needs the first element. + + Lists also implement the `Enumerable` protocol, so many functions to work with + lists are found in the `Enum` module. Additionally, the following functions and + operators for lists are found in `Kernel`: + + * `++/2` + * `--/2` + * `hd/1` + * `tl/1` + * `in/2` + * `length/1` + + ## Charlists + + If a list is made of non-negative integers, where each integer represents a + Unicode code point, the list can also be called a charlist. These integers + must: + + * be within the range `0..0x10FFFF` (`0..1_114_111`); + * and be out of the range `0xD800..0xDFFF` (`55_296..57_343`), which is + reserved in Unicode for UTF-16 surrogate pairs. + + Elixir uses single quotes to define charlists: + + iex> 'héllo' + [104, 233, 108, 108, 111] + + In particular, charlists will be printed back by default in single + quotes if they contain only printable ASCII characters: + + iex> 'abc' + 'abc' + + Even though the representation changed, the raw data does remain a list of + numbers, which can be handled as such: + + iex> inspect('abc', charlists: :as_list) + "[97, 98, 99]" + iex> Enum.map('abc', fn num -> 1000 + num end) + [1097, 1098, 1099] + + You can use the `IEx.Helpers.i/1` helper to get a condensed rundown on + charlists in IEx when you encounter them, which shows you the type, description + and also the raw representation in one single summary. + + The rationale behind this behaviour is to better support + Erlang libraries which may return text as charlists + instead of Elixir strings. In Erlang, charlists are the default + way of handling strings, while in Elixir it's binaries. One + example of such functions is `Application.loaded_applications/0`: + + Application.loaded_applications() + #=> [ + #=> {:stdlib, 'ERTS CXC 138 10', '2.6'}, + #=> {:compiler, 'ERTS CXC 138 10', '6.0.1'}, + #=> {:elixir, 'elixir', '1.0.0'}, + #=> {:kernel, 'ERTS CXC 138 10', '4.1'}, + #=> {:logger, 'logger', '1.0.0'} + #=> ] + + A list can be checked if it is made of only printable ASCII + characters with `ascii_printable?/2`. + + Improper lists are never deemed as charlists. """ @compile :inline_list_funcs @doc """ - Deletes the given item from the list. Returns a list without - the item. If the item occurs more than once in the list, just + Deletes the given `element` from the `list`. Returns a new list without + the element. + + If the `element` occurs more than once in the `list`, just the first occurrence is removed. ## Examples - iex> List.delete([1, 2, 3], 1) - [2,3] + iex> List.delete([:a, :b, :c], :a) + [:b, :c] - iex> List.delete([1, 2, 2, 3], 2) - [1, 2, 3] + iex> List.delete([:a, :b, :c], :d) + [:a, :b, :c] + + iex> List.delete([:a, :b, :b, :c], :b) + [:a, :b, :c] + + iex> List.delete([], :b) + [] """ - @spec delete(list, any) :: list - def delete(list, item) do - :lists.delete(item, list) - end + @spec delete([], any) :: [] + @spec delete([...], any) :: list + def delete(list, element) + def delete([element | list], element), do: list + def delete([other | list], element), do: [other | delete(list, element)] + def delete([], _element), do: [] @doc """ Duplicates the given element `n` times in a list. + `n` is an integer greater than or equal to `0`. + + If `n` is `0`, an empty list is returned. + ## Examples - iex> List.duplicate("hello", 3) - ["hello","hello","hello"] + iex> List.duplicate("hello", 0) + [] + + iex> List.duplicate("hi", 1) + ["hi"] + + iex> List.duplicate("bye", 2) + ["bye", "bye"] - iex> List.duplicate([1, 2], 2) - [[1,2],[1,2]] + iex> List.duplicate([1, 2], 3) + [[1, 2], [1, 2], [1, 2]] """ - @spec duplicate(elem, non_neg_integer) :: [elem] when elem: var + @spec duplicate(any, 0) :: [] + @spec duplicate(elem, pos_integer) :: [elem, ...] when elem: var def duplicate(elem, n) do :lists.duplicate(n, elem) end @@ -57,10 +190,15 @@ defmodule List do @doc """ Flattens the given `list` of nested lists. + Empty list elements are discarded. + ## Examples iex> List.flatten([1, [[2], 3]]) - [1,2,3] + [1, 2, 3] + + iex> List.flatten([[], [[], []]]) + [] """ @spec flatten(deep_list) :: list when deep_list: [any | deep_list] @@ -73,10 +211,16 @@ defmodule List do The list `tail` will be added at the end of the flattened list. + Empty list elements from `list` are discarded, + but not the ones from `tail`. + ## Examples iex> List.flatten([1, [[2], 3]], [4, 5]) - [1,2,3,4,5] + [1, 2, 3, 4, 5] + + iex> List.flatten([1, [], 2], [3, [], 4]) + [1, 2, 3, [], 4] """ @spec flatten(deep_list, [elem]) :: [elem] when elem: var, deep_list: [elem | deep_list] @@ -85,46 +229,57 @@ defmodule List do end @doc """ - Folds (reduces) the given list to the left with - a function. Requires an accumulator. + Folds (reduces) the given list from the left with + a function. Requires an accumulator, which can be any value. ## Examples - iex> List.foldl([5, 5], 10, fn (x, acc) -> x + acc end) + iex> List.foldl([5, 5], 10, fn x, acc -> x + acc end) 20 - iex> List.foldl([1, 2, 3, 4], 0, fn (x, acc) -> x - acc end) + iex> List.foldl([1, 2, 3, 4], 0, fn x, acc -> x - acc end) 2 + iex> List.foldl([1, 2, 3], {0, 0}, fn x, {a1, a2} -> {a1 + x, a2 - x} end) + {6, -6} + """ @spec foldl([elem], acc, (elem, acc -> acc)) :: acc when elem: var, acc: var - def foldl(list, acc, function) when is_list(list) and is_function(function) do - :lists.foldl(function, acc, list) + def foldl(list, acc, fun) when is_list(list) and is_function(fun) do + :lists.foldl(fun, acc, list) end @doc """ - Folds (reduces) the given list to the right with - a function. Requires an accumulator. + Folds (reduces) the given list from the right with + a function. Requires an accumulator, which can be any value. ## Examples - iex> List.foldr([1, 2, 3, 4], 0, fn (x, acc) -> x - acc end) + iex> List.foldr([1, 2, 3, 4], 0, fn x, acc -> x - acc end) -2 + iex> List.foldr([1, 2, 3, 4], %{sum: 0, product: 1}, fn x, %{sum: a1, product: a2} -> %{sum: a1 + x, product: a2 * x} end) + %{product: 24, sum: 10} + """ @spec foldr([elem], acc, (elem, acc -> acc)) :: acc when elem: var, acc: var - def foldr(list, acc, function) when is_list(list) and is_function(function) do - :lists.foldr(function, acc, list) + def foldr(list, acc, fun) when is_list(list) and is_function(fun) do + :lists.foldr(fun, acc, list) end @doc """ - Returns the first element in `list` or `nil` if `list` is empty. + Returns the first element in `list` or `default` if `list` is empty. + + `first/2` has been introduced in Elixir v1.12.0, while `first/1` has been available since v1.0.0. ## Examples iex> List.first([]) nil + iex> List.first([], 1) + 1 + iex> List.first([1]) 1 @@ -132,18 +287,25 @@ defmodule List do 1 """ - @spec first([elem]) :: nil | elem when elem: var - def first([]), do: nil - def first([h|_]), do: h + @spec first([], any) :: any + @spec first([elem, ...], any) :: elem when elem: var + def first(list, default \\ nil) + def first([], default), do: default + def first([head | _], _default), do: head @doc """ - Returns the last element in `list` or `nil` if `list` is empty. + Returns the last element in `list` or `default` if `list` is empty. + + `last/2` has been introduced in Elixir v1.12.0, while `last/1` has been available since v1.0.0. ## Examples iex> List.last([]) nil + iex> List.last([], 1) + 1 + iex> List.last([1]) 1 @@ -151,15 +313,20 @@ defmodule List do 3 """ - @spec last([elem]) :: nil | elem when elem: var - def last([]), do: nil - def last([h]), do: h - def last([_|t]), do: last(t) + @spec last([], any) :: any + @spec last([elem, ...], any) :: elem when elem: var + @compile {:inline, last: 2} + def last(list, default \\ nil) + def last([], default), do: default + def last([head], _default), do: head + def last([_ | tail], default), do: last(tail, default) @doc """ Receives a list of tuples and returns the first tuple - where the item at `position` in the tuple matches the - given `item`. + where the element at `position` in the tuple matches the + given `key`. + + If no matching tuple is found, `default` is returned. ## Examples @@ -172,16 +339,56 @@ defmodule List do iex> List.keyfind([a: 1, b: 2], :c, 0) nil + This function works for any list of tuples: + + iex> List.keyfind([{22, "SSH"}, {80, "HTTP"}], 22, 0) + {22, "SSH"} + """ @spec keyfind([tuple], any, non_neg_integer, any) :: any - def keyfind(list, key, position, default \\ nil) do + def keyfind(list, key, position, default \\ nil) when is_integer(position) do :lists.keyfind(key, position + 1, list) || default end + @doc """ + Receives a list of tuples and returns the first tuple + where the element at `position` in the tuple matches the + given `key`. + + If no matching tuple is found, an error is raised. + + ## Examples + + iex> List.keyfind!([a: 1, b: 2], :a, 0) + {:a, 1} + + iex> List.keyfind!([a: 1, b: 2], 2, 1) + {:b, 2} + + iex> List.keyfind!([a: 1, b: 2], :c, 0) + ** (KeyError) key :c at position 0 not found in: [a: 1, b: 2] + + This function works for any list of tuples: + + iex> List.keyfind!([{22, "SSH"}, {80, "HTTP"}], 22, 0) + {22, "SSH"} + + """ + @doc since: "1.13.0" + @spec keyfind!([tuple], any, non_neg_integer) :: any + def keyfind!(list, key, position) when is_integer(position) do + :lists.keyfind(key, position + 1, list) || + raise KeyError, + key: key, + term: list, + message: + "key #{inspect(key)} at position #{inspect(position)} not found in: #{inspect(list)}" + end + @doc """ Receives a list of tuples and returns `true` if there is - a tuple where the item at `position` in the tuple matches - the given `item`. + a tuple where the element at `position` in the tuple matches + the given `key`. ## Examples @@ -194,30 +401,49 @@ defmodule List do iex> List.keymember?([a: 1, b: 2], :c, 0) false + This function works for any list of tuples: + + iex> List.keymember?([{22, "SSH"}, {80, "HTTP"}], 22, 0) + true + """ - @spec keymember?([tuple], any, non_neg_integer) :: any - def keymember?(list, key, position) do + @spec keymember?([tuple], any, non_neg_integer) :: boolean + def keymember?(list, key, position) when is_integer(position) do :lists.keymember(key, position + 1, list) end @doc """ - Receives a list of tuples and replaces the item - identified by `key` at `position` if it exists. + Receives a list of tuples and if the identified element by `key` at `position` + exists, it is replaced with `new_tuple`. ## Examples iex> List.keyreplace([a: 1, b: 2], :a, 0, {:a, 3}) [a: 3, b: 2] + iex> List.keyreplace([a: 1, b: 2], :a, 1, {:a, 3}) + [a: 1, b: 2] + + This function works for any list of tuples: + + iex> List.keyreplace([{22, "SSH"}, {80, "HTTP"}], 22, 0, {22, "Secure Shell"}) + [{22, "Secure Shell"}, {80, "HTTP"}] + """ @spec keyreplace([tuple], any, non_neg_integer, tuple) :: [tuple] - def keyreplace(list, key, position, new_tuple) do + def keyreplace(list, key, position, new_tuple) when is_integer(position) do :lists.keyreplace(key, position + 1, list, new_tuple) end @doc """ - Receives a list of tuples and sorts the items - at `position` of the tuples. The sort is stable. + Receives a list of tuples and sorts the elements + at `position` of the tuples. + + The sort is stable. + + A `sorter` argument is available since Elixir v1.14.0. Similar to + `Enum.sort/2`, the sorter can be an anonymous function, the atoms + `:asc` or `:desc`, or module that implements a compare function. ## Examples @@ -227,16 +453,74 @@ defmodule List do iex> List.keysort([a: 5, c: 1, b: 3], 0) [a: 5, b: 3, c: 1] + To sort in descending order: + + iex> List.keysort([a: 5, c: 1, b: 3], 0, :desc) + [c: 1, b: 3, a: 5] + + As in `Enum.sort/2`, avoid using the default sorting function to sort + structs, as by default it performs structural comparison instead of a + semantic one. In such cases, you shall pass a sorting function as third + element or any module that implements a `compare/2` function. For example, + if you have tuples with user names and their birthday, and you want to + sort on their birthday, in both ascending and descending order, you should + do: + + iex> users = [ + ...> {"Ellis", ~D[1943-05-11]}, + ...> {"Lovelace", ~D[1815-12-10]}, + ...> {"Turing", ~D[1912-06-23]} + ...> ] + iex> List.keysort(users, 1, Date) + [ + {"Lovelace", ~D[1815-12-10]}, + {"Turing", ~D[1912-06-23]}, + {"Ellis", ~D[1943-05-11]} + ] + iex> List.keysort(users, 1, {:desc, Date}) + [ + {"Ellis", ~D[1943-05-11]}, + {"Turing", ~D[1912-06-23]}, + {"Lovelace", ~D[1815-12-10]} + ] + """ - @spec keysort([tuple], non_neg_integer) :: [tuple] - def keysort(list, position) do + @doc since: "1.14.0" + @spec keysort( + [tuple], + non_neg_integer, + (any, any -> boolean) | :asc | :desc | module() | {:asc | :desc, module()} + ) :: [tuple] + def keysort(list, position, sorter \\ :asc) + + def keysort(list, position, :asc) when is_list(list) and is_integer(position) do :lists.keysort(position + 1, list) end + def keysort(list, position, sorter) when is_list(list) and is_integer(position) do + :lists.sort(keysort_fun(sorter, position + 1), list) + end + + defp keysort_fun(sorter, position) when is_function(sorter, 2), + do: &sorter.(:erlang.element(position, &1), :erlang.element(position, &2)) + + defp keysort_fun(:desc, position), + do: &(:erlang.element(position, &1) >= :erlang.element(position, &2)) + + defp keysort_fun(module, position) when is_atom(module), + do: &(module.compare(:erlang.element(position, &1), :erlang.element(position, &2)) != :gt) + + defp keysort_fun({:asc, module}, position) when is_atom(module), + do: &(module.compare(:erlang.element(position, &1), :erlang.element(position, &2)) != :gt) + + defp keysort_fun({:desc, module}, position) when is_atom(module), + do: &(module.compare(:erlang.element(position, &1), :erlang.element(position, &2)) != :lt) + @doc """ - Receives a list of tuples and replaces the item - identified by `key` at `position`. If the item - does not exist, it is added to the end of the list. + Receives a `list` of tuples and replaces the element + identified by `key` at `position` with `new_tuple`. + + If the element does not exist, it is added to the end of the `list`. ## Examples @@ -246,16 +530,21 @@ defmodule List do iex> List.keystore([a: 1, b: 2], :c, 0, {:c, 3}) [a: 1, b: 2, c: 3] + This function works for any list of tuples: + + iex> List.keystore([{22, "SSH"}], 80, 0, {80, "HTTP"}) + [{22, "SSH"}, {80, "HTTP"}] + """ - @spec keystore([tuple], any, non_neg_integer, tuple) :: [tuple] - def keystore(list, key, position, new_tuple) do + @spec keystore([tuple], any, non_neg_integer, tuple) :: [tuple, ...] + def keystore(list, key, position, new_tuple) when is_integer(position) do :lists.keystore(key, position + 1, list, new_tuple) end @doc """ - Receives a list of tuples and deletes the first tuple - where the item at `position` matches the - given `item`. Returns the new list. + Receives a `list` of tuples and deletes the first tuple + where the element at `position` matches the + given `key`. Returns the new list. ## Examples @@ -268,16 +557,54 @@ defmodule List do iex> List.keydelete([a: 1, b: 2], :c, 0) [a: 1, b: 2] + This function works for any list of tuples: + + iex> List.keydelete([{22, "SSH"}, {80, "HTTP"}], 80, 0) + [{22, "SSH"}] + """ @spec keydelete([tuple], any, non_neg_integer) :: [tuple] - def keydelete(list, key, position) do + def keydelete(list, key, position) when is_integer(position) do :lists.keydelete(key, position + 1, list) end @doc """ - Wraps the argument in a list. - If the argument is already a list, returns the list. - If the argument is `nil`, returns an empty list. + Receives a `list` of tuples and returns the first tuple + where the element at `position` in the tuple matches the + given `key`, as well as the `list` without found tuple. + + If such a tuple is not found, `nil` will be returned. + + ## Examples + + iex> List.keytake([a: 1, b: 2], :a, 0) + {{:a, 1}, [b: 2]} + + iex> List.keytake([a: 1, b: 2], 2, 1) + {{:b, 2}, [a: 1]} + + iex> List.keytake([a: 1, b: 2], :c, 0) + nil + + This function works for any list of tuples: + + iex> List.keytake([{22, "SSH"}, {80, "HTTP"}], 80, 0) + {{80, "HTTP"}, [{22, "SSH"}]} + + """ + @spec keytake([tuple], any, non_neg_integer) :: {tuple, [tuple]} | nil + def keytake(list, key, position) when is_integer(position) do + case :lists.keytake(key, position + 1, list) do + {:value, element, list} -> {element, list} + false -> nil + end + end + + @doc """ + Wraps `term` in a list if this is not list. + + If `term` is already a list, it returns the list. + If `term` is `nil`, it returns an empty list. ## Examples @@ -285,13 +612,15 @@ defmodule List do ["hello"] iex> List.wrap([1, 2, 3]) - [1,2,3] + [1, 2, 3] iex> List.wrap(nil) [] """ - @spec wrap(list | any) :: list + @spec wrap(term) :: maybe_improper_list() + def wrap(term) + def wrap(list) when is_list(list) do list end @@ -307,6 +636,8 @@ defmodule List do @doc """ Zips corresponding elements from each list in `list_of_lists`. + The zipping finishes as soon as any list terminates. + ## Examples iex> List.zip([[1, 2], [3, 4], [5, 6]]) @@ -318,32 +649,101 @@ defmodule List do """ @spec zip([list]) :: [tuple] def zip([]), do: [] + def zip(list_of_lists) when is_list(list_of_lists) do do_zip(list_of_lists, []) end - @doc """ - Unzips the given list of lists or tuples into separate lists and returns a - list of lists. + @doc ~S""" + Checks if `list` is a charlist made only of printable ASCII characters. + + Takes an optional `limit` as a second argument. `ascii_printable?/2` only + checks the printability of the list up to the `limit`. + + A printable charlist in Elixir contains only the printable characters in the + standard seven-bit ASCII character encoding, which are characters ranging from + 32 to 126 in decimal notation, plus the following control characters: + + * `?\a` - Bell + * `?\b` - Backspace + * `?\t` - Horizontal tab + * `?\n` - Line feed + * `?\v` - Vertical tab + * `?\f` - Form feed + * `?\r` - Carriage return + * `?\e` - Escape + + For more information read the [Character groups](https://en.wikipedia.org/wiki/ASCII#Character_groups) + section in the Wikipedia article of the [ASCII](https://en.wikipedia.org/wiki/ASCII) standard. ## Examples - iex> List.unzip([{1, 2}, {3, 4}]) - [[1, 3], [2, 4]] + iex> List.ascii_printable?('abc') + true + + iex> List.ascii_printable?('abc' ++ [0]) + false + + iex> List.ascii_printable?('abc' ++ [0], 2) + true + + Improper lists are not printable, even if made only of ASCII characters: - iex> List.unzip([{1, :a, "apple"}, {2, :b, "banana"}, {3, :c}]) - [[1, 2, 3], [:a, :b, :c]] + iex> List.ascii_printable?('abc' ++ ?d) + false """ - @spec unzip([tuple]) :: [list] - def unzip(list) when is_list(list) do - :lists.map &Tuple.to_list/1, zip(list) + @doc since: "1.6.0" + @spec ascii_printable?(list, 0) :: true + @spec ascii_printable?([], limit) :: true + when limit: :infinity | pos_integer + @spec ascii_printable?([...], limit) :: boolean + when limit: :infinity | pos_integer + def ascii_printable?(list, limit \\ :infinity) + when is_list(list) and (limit == :infinity or (is_integer(limit) and limit >= 0)) do + ascii_printable_guarded?(list, limit) + end + + defp ascii_printable_guarded?(_, 0) do + true + end + + defp ascii_printable_guarded?([char | rest], counter) + # 7..13 is the range '\a\b\t\n\v\f\r'. 32..126 are ASCII printables. + when is_integer(char) and + ((char >= 7 and char <= 13) or char == ?\e or (char >= 32 and char <= 126)) do + ascii_printable_guarded?(rest, decrement(counter)) end + defp ascii_printable_guarded?([], _counter), do: true + defp ascii_printable_guarded?(_, _counter), do: false + + @compile {:inline, decrement: 1} + defp decrement(:infinity), do: :infinity + defp decrement(counter), do: counter - 1 + + @doc """ + Returns `true` if `list` is an improper list. Otherwise returns `false`. + + ## Examples + + iex> List.improper?([1, 2 | 3]) + true + + iex> List.improper?([1, 2, 3]) + false + + """ + @doc since: "1.8.0" + @spec improper?(maybe_improper_list) :: boolean + def improper?(list) when is_list(list) and length(list) >= 0, do: false + def improper?(list) when is_list(list), do: true + @doc """ Returns a list with `value` inserted at the specified `index`. + Note that `index` is capped at the list length. Negative indices - indicate an offset from the end of the list. + indicate an offset from the end of the `list`. ## Examples @@ -361,17 +761,26 @@ defmodule List do """ @spec insert_at(list, integer, any) :: list - def insert_at(list, index, value) do - if index < 0 do - do_insert_at(list, length(list) + index + 1, value) - else - do_insert_at(list, index, value) + def insert_at(list, index, value) when is_list(list) and is_integer(index) do + case index do + -1 -> + list ++ [value] + + _ when index < 0 -> + case length(list) + index + 1 do + index when index < 0 -> [value | list] + index -> do_insert_at(list, index, value) + end + + _ -> + do_insert_at(list, index, value) end end @doc """ Returns a list with a replaced value at the specified `index`. - Negative indices indicate an offset from the end of the list. + + Negative indices indicate an offset from the end of the `list`. If `index` is out of bounds, the original `list` is returned. ## Examples @@ -390,9 +799,12 @@ defmodule List do """ @spec replace_at(list, integer, any) :: list - def replace_at(list, index, value) do + def replace_at(list, index, value) when is_list(list) and is_integer(index) do if index < 0 do - do_replace_at(list, length(list) + index, value) + case length(list) + index do + index when index < 0 -> list + index -> do_replace_at(list, index, value) + end else do_replace_at(list, index, value) end @@ -400,7 +812,8 @@ defmodule List do @doc """ Returns a list with an updated value at the specified `index`. - Negative indices indicate an offset from the end of the list. + + Negative indices indicate an offset from the end of the `list`. If `index` is out of bounds, the original `list` is returned. ## Examples @@ -419,9 +832,12 @@ defmodule List do """ @spec update_at([elem], integer, (elem -> any)) :: list when elem: var - def update_at(list, index, fun) do + def update_at(list, index, fun) when is_list(list) and is_function(fun) and is_integer(index) do if index < 0 do - do_update_at(list, length(list) + index, fun) + case length(list) + index do + index when index < 0 -> list + index -> do_update_at(list, index, fun) + end else do_update_at(list, index, fun) end @@ -429,7 +845,8 @@ defmodule List do @doc """ Produces a new list by removing the value at the specified `index`. - Negative indices indicate an offset from the end of the list. + + Negative indices indicate an offset from the end of the `list`. If `index` is out of bounds, the original `list` is returned. ## Examples @@ -437,7 +854,7 @@ defmodule List do iex> List.delete_at([1, 2, 3], 0) [2, 3] - iex List.delete_at([1, 2, 3], 10) + iex> List.delete_at([1, 2, 3], 10) [1, 2, 3] iex> List.delete_at([1, 2, 3], -1) @@ -445,48 +862,125 @@ defmodule List do """ @spec delete_at(list, integer) :: list - def delete_at(list, index) do + def delete_at(list, index) when is_integer(index) do + elem(pop_at(list, index), 1) + end + + @doc """ + Returns and removes the value at the specified `index` in the `list`. + + Negative indices indicate an offset from the end of the `list`. + If `index` is out of bounds, the original `list` is returned. + + ## Examples + + iex> List.pop_at([1, 2, 3], 0) + {1, [2, 3]} + iex> List.pop_at([1, 2, 3], 5) + {nil, [1, 2, 3]} + iex> List.pop_at([1, 2, 3], 5, 10) + {10, [1, 2, 3]} + iex> List.pop_at([1, 2, 3], -1) + {3, [1, 2]} + + """ + @doc since: "1.4.0" + @spec pop_at(list, integer, any) :: {any, list} + def pop_at(list, index, default \\ nil) when is_integer(index) do if index < 0 do - do_delete_at(list, length(list) + index) + do_pop_at(list, length(list) + index, default, []) else - do_delete_at(list, index) + do_pop_at(list, index, default, []) end end @doc """ - Converts a char list to an atom. + Returns `true` if `list` starts with the given `prefix` list; otherwise returns `false`. + + If `prefix` is an empty list, it returns `true`. + + ### Examples + + iex> List.starts_with?([1, 2, 3], [1, 2]) + true + + iex> List.starts_with?([1, 2], [1, 2, 3]) + false + + iex> List.starts_with?([:alpha], []) + true + + iex> List.starts_with?([], [:alpha]) + false + + """ + @doc since: "1.5.0" + @spec starts_with?(nonempty_list, nonempty_list) :: boolean + @spec starts_with?(list, []) :: true + @spec starts_with?([], nonempty_list) :: false + def starts_with?(list, prefix) + + def starts_with?([head | tail], [head | prefix_tail]), do: starts_with?(tail, prefix_tail) + def starts_with?(list, []) when is_list(list), do: true + def starts_with?(list, [_ | _]) when is_list(list), do: false + + @doc """ + Converts a charlist to an atom. - Currently Elixir does not support conversions from char lists - which contains Unicode codepoints greater than 0xFF. + Elixir supports conversions from charlists which contains any Unicode + code point. Inlined by the compiler. ## Examples - iex> List.to_atom('elixir') - :elixir + iex> List.to_atom('Elixir') + :Elixir + + iex> List.to_atom('🌢 Elixir') + :"🌢 Elixir" """ - @spec to_atom(char_list) :: atom - def to_atom(char_list) do - :erlang.list_to_atom(char_list) + @spec to_atom(charlist) :: atom + def to_atom(charlist) do + :erlang.list_to_atom(charlist) end @doc """ - Converts a char list to an existing atom. + Converts a charlist to an existing atom. - Currently Elixir does not support conversions from char lists - which contains Unicode codepoints greater than 0xFF. + Elixir supports conversions from charlists which contains any Unicode + code point. Raises an `ArgumentError` if the atom does not exist. Inlined by the compiler. + + > #### Atoms and modules {: .info} + > + > Since Elixir is a compiled language, the atoms defined in a module + > will only exist after said module is loaded, which typically happens + > whenever a function in the module is executed. Therefore, it is + > generally recommended to call `List.to_existing_atom/1` only to + > convert atoms defined within the module making the function call + > to `to_existing_atom/1`. + + ## Examples + + iex> _ = :my_atom + iex> List.to_existing_atom('my_atom') + :my_atom + + iex> _ = :"🌢 Elixir" + iex> List.to_existing_atom('🌢 Elixir') + :"🌢 Elixir" + """ - @spec to_existing_atom(char_list) :: atom - def to_existing_atom(char_list) do - :erlang.list_to_existing_atom(char_list) + @spec to_existing_atom(charlist) :: atom + def to_existing_atom(charlist) do + :erlang.list_to_existing_atom(charlist) end @doc """ - Returns the float whose text representation is `char_list`. + Returns the float whose text representation is `charlist`. Inlined by the compiler. @@ -496,13 +990,13 @@ defmodule List do 2.2017764 """ - @spec to_float(char_list) :: float - def to_float(char_list) do - :erlang.list_to_float(char_list) + @spec to_float(charlist) :: float + def to_float(charlist) do + :erlang.list_to_float(charlist) end @doc """ - Returns an integer whose text representation is `char_list`. + Returns an integer whose text representation is `charlist`. Inlined by the compiler. @@ -512,25 +1006,27 @@ defmodule List do 123 """ - @spec to_integer(char_list) :: integer - def to_integer(char_list) do - :erlang.list_to_integer(char_list) + @spec to_integer(charlist) :: integer + def to_integer(charlist) do + :erlang.list_to_integer(charlist) end @doc """ - Returns an integer whose text representation is `char_list` in base `base`. + Returns an integer whose text representation is `charlist` in base `base`. Inlined by the compiler. + The base needs to be between `2` and `36`. + ## Examples iex> List.to_integer('3FF', 16) 1023 """ - @spec to_integer(char_list, non_neg_integer) :: integer - def to_integer(char_list, base) do - :erlang.list_to_integer(char_list, base) + @spec to_integer(charlist, 2..36) :: integer + def to_integer(charlist, base) do + :erlang.list_to_integer(charlist, base) end @doc """ @@ -550,12 +1046,19 @@ defmodule List do end @doc """ - Converts a list of integers representing codepoints, lists or + Converts a list of integers representing code points, lists or strings into a string. - Notice that this function expect a list of integer representing - UTF-8 codepoints. If you have a list of bytes, you must instead use - [the `:binary` module](http://erlang.org/doc/man/binary.html). + To be converted to a string, a list must either be empty or only + contain the following elements: + + * strings + * integers representing Unicode code points + * a list containing one of these three elements + + Note that this function expects a list of integers representing + Unicode code points. If you have a list of bytes, you must instead use + the [`:binary` module](`:binary`). ## Examples @@ -565,10 +1068,34 @@ defmodule List do iex> List.to_string([0x0061, "bc"]) "abc" + iex> List.to_string([0x0064, "ee", ['p']]) + "deep" + + iex> List.to_string([]) + "" + """ - @spec to_string(:unicode.char_list) :: String.t + @spec to_string(:unicode.charlist()) :: String.t() def to_string(list) when is_list(list) do - case :unicode.characters_to_binary(list) do + try do + :unicode.characters_to_binary(list) + rescue + ArgumentError -> + raise ArgumentError, """ + cannot convert the given list to a string. + + To be converted to a string, a list must either be empty or only + contain the following elements: + + * strings + * integers representing Unicode code points + * a list containing one of these three elements + + Please check the given list or call inspect/1 to get the list representation, got: + + #{inspect(list)} + """ + else result when is_binary(result) -> result @@ -580,6 +1107,214 @@ defmodule List do end end + @doc """ + Converts a list of integers representing Unicode code points, lists or + strings into a charlist. + + Note that this function expects a list of integers representing + Unicode code points. If you have a list of bytes, you must instead use + the [`:binary` module](`:binary`). + + ## Examples + + iex> List.to_charlist([0x00E6, 0x00DF]) + 'æß' + + iex> List.to_charlist([0x0061, "bc"]) + 'abc' + + iex> List.to_charlist([0x0064, "ee", ['p']]) + 'deep' + + """ + @doc since: "1.8.0" + @spec to_charlist(:unicode.charlist()) :: charlist() + def to_charlist(list) when is_list(list) do + try do + :unicode.characters_to_list(list) + rescue + ArgumentError -> + raise ArgumentError, """ + cannot convert the given list to a charlist. + + To be converted to a charlist, a list must contain only: + + * strings + * integers representing Unicode code points + * or a list containing one of these three elements + + Please check the given list or call inspect/1 to get the list representation, got: + + #{inspect(list)} + """ + else + result when is_list(result) -> + result + + {:error, encoded, rest} -> + raise UnicodeConversionError, encoded: encoded, rest: rest, kind: :invalid + + {:incomplete, encoded, rest} -> + raise UnicodeConversionError, encoded: encoded, rest: rest, kind: :incomplete + end + end + + @doc """ + Returns a keyword list that represents an *edit script*. + + The algorithm is outlined in the + "An O(ND) Difference Algorithm and Its Variations" paper by E. Myers. + + An *edit script* is a keyword list. Each key describes the "editing action" to + take in order to bring `list1` closer to being equal to `list2`; a key can be + `:eq`, `:ins`, or `:del`. Each value is a sublist of either `list1` or `list2` + that should be inserted (if the corresponding key `:ins`), deleted (if the + corresponding key is `:del`), or left alone (if the corresponding key is + `:eq`) in `list1` in order to be closer to `list2`. + + See `myers_difference/3` if you want to handle nesting in the diff scripts. + + ## Examples + + iex> List.myers_difference([1, 4, 2, 3], [1, 2, 3, 4]) + [eq: [1], del: [4], eq: [2, 3], ins: [4]] + + """ + @doc since: "1.4.0" + @spec myers_difference(list, list) :: [{:eq | :ins | :del, list}] + def myers_difference(list1, list2) when is_list(list1) and is_list(list2) do + myers_difference_with_diff_script(list1, list2, nil) + end + + @doc """ + Returns a keyword list that represents an *edit script* with nested diffs. + + This is an extension of `myers_difference/2` where a `diff_script` function + can be given in case it is desired to compute nested differences. The function + may return a list with the inner edit script or `nil` in case there is no + such script. The returned inner edit script will be under the `:diff` key. + + ## Examples + + iex> List.myers_difference(["a", "db", "c"], ["a", "bc"], &String.myers_difference/2) + [eq: ["a"], diff: [del: "d", eq: "b", ins: "c"], del: ["c"]] + + """ + @doc since: "1.8.0" + @spec myers_difference(list, list, (term, term -> script | nil)) :: script + when script: [{:eq | :ins | :del | :diff, list}] + def myers_difference(list1, list2, diff_script) + when is_list(list1) and is_list(list2) and is_function(diff_script) do + myers_difference_with_diff_script(list1, list2, diff_script) + end + + defp myers_difference_with_diff_script(list1, list2, diff_script) do + path = {0, list1, list2, []} + find_script(0, length(list1) + length(list2), [path], diff_script) + end + + defp find_script(envelope, max, paths, diff_script) do + case each_diagonal(-envelope, envelope, paths, [], diff_script) do + {:done, edits} -> compact_reverse(edits, []) + {:next, paths} -> find_script(envelope + 1, max, paths, diff_script) + end + end + + defp compact_reverse([], acc), do: acc + + defp compact_reverse([{:diff, _} = fragment | rest], acc) do + compact_reverse(rest, [fragment | acc]) + end + + defp compact_reverse([{kind, elem} | rest], [{kind, result} | acc]) do + compact_reverse(rest, [{kind, [elem | result]} | acc]) + end + + defp compact_reverse(rest, [{:eq, elem}, {:ins, elem}, {:eq, other} | acc]) do + compact_reverse(rest, [{:ins, elem}, {:eq, elem ++ other} | acc]) + end + + defp compact_reverse([{kind, elem} | rest], acc) do + compact_reverse(rest, [{kind, [elem]} | acc]) + end + + defp each_diagonal(diag, limit, _paths, next_paths, _diff_script) when diag > limit do + {:next, :lists.reverse(next_paths)} + end + + defp each_diagonal(diag, limit, paths, next_paths, diff_script) do + {path, rest} = proceed_path(diag, limit, paths, diff_script) + + case follow_snake(path) do + {:cont, path} -> each_diagonal(diag + 2, limit, rest, [path | next_paths], diff_script) + {:done, edits} -> {:done, edits} + end + end + + defp proceed_path(0, 0, [path], _diff_script), do: {path, []} + + defp proceed_path(diag, limit, [path | _] = paths, diff_script) when diag == -limit do + {move_down(path, diff_script), paths} + end + + defp proceed_path(diag, limit, [path], diff_script) when diag == limit do + {move_right(path, diff_script), []} + end + + defp proceed_path(_diag, _limit, [path1, path2 | rest], diff_script) do + if elem(path1, 0) > elem(path2, 0) do + {move_right(path1, diff_script), [path2 | rest]} + else + {move_down(path2, diff_script), [path2 | rest]} + end + end + + defp move_right({y, [elem1 | rest1] = list1, [elem2 | rest2], edits}, diff_script) + when diff_script != nil do + if diff = diff_script.(elem1, elem2) do + {y + 1, rest1, rest2, [{:diff, diff} | edits]} + else + {y, list1, rest2, [{:ins, elem2} | edits]} + end + end + + defp move_right({y, list1, [elem | rest], edits}, _diff_script) do + {y, list1, rest, [{:ins, elem} | edits]} + end + + defp move_right({y, list1, [], edits}, _diff_script) do + {y, list1, [], edits} + end + + defp move_down({y, [elem1 | rest1], [elem2 | rest2] = list2, edits}, diff_script) + when diff_script != nil do + if diff = diff_script.(elem1, elem2) do + {y + 1, rest1, rest2, [{:diff, diff} | edits]} + else + {y + 1, rest1, list2, [{:del, elem1} | edits]} + end + end + + defp move_down({y, [elem | rest], list2, edits}, _diff_script) do + {y + 1, rest, list2, [{:del, elem} | edits]} + end + + defp move_down({y, [], list2, edits}, _diff_script) do + {y + 1, [], list2, edits} + end + + defp follow_snake({y, [elem | rest1], [elem | rest2], edits}) do + follow_snake({y + 1, rest1, rest2, [{:eq, elem} | edits]}) + end + + defp follow_snake({_y, [], [], edits}) do + {:done, edits} + end + + defp follow_snake(path) do + {:cont, path} + end + ## Helpers # replace_at @@ -588,77 +1323,67 @@ defmodule List do [] end - defp do_replace_at(list, index, _value) when index < 0 do - list - end - - defp do_replace_at([_old|rest], 0, value) do - [ value | rest ] + defp do_replace_at([_old | rest], 0, value) do + [value | rest] end - defp do_replace_at([h|t], index, value) do - [ h | do_replace_at(t, index - 1, value) ] + defp do_replace_at([head | tail], index, value) do + [head | do_replace_at(tail, index - 1, value)] end # insert_at defp do_insert_at([], _index, value) do - [ value ] + [value] end - defp do_insert_at(list, index, value) when index <= 0 do - [ value | list ] + defp do_insert_at(list, 0, value) do + [value | list] end - defp do_insert_at([h|t], index, value) do - [ h | do_insert_at(t, index - 1, value) ] + defp do_insert_at([head | tail], index, value) do + [head | do_insert_at(tail, index - 1, value)] end # update_at - defp do_update_at([value|list], 0, fun) do - [ fun.(value) | list ] + defp do_update_at([value | list], 0, fun) do + [fun.(value) | list] end - defp do_update_at(list, index, _fun) when index < 0 do - list - end - - defp do_update_at([h|t], index, fun) do - [ h | do_update_at(t, index - 1, fun) ] + defp do_update_at([head | tail], index, fun) do + [head | do_update_at(tail, index - 1, fun)] end defp do_update_at([], _index, _fun) do [] end - # delete_at - - defp do_delete_at([], _index) do - [] - end + # pop_at - defp do_delete_at([_|t], 0) do - t + defp do_pop_at([], _index, default, acc) do + {default, :lists.reverse(acc)} end - defp do_delete_at(list, index) when index < 0 do - list + defp do_pop_at([head | tail], 0, _default, acc) do + {head, :lists.reverse(acc, tail)} end - defp do_delete_at([h|t], index) do - [h | do_delete_at(t, index-1)] + defp do_pop_at([head | tail], index, default, acc) do + do_pop_at(tail, index - 1, default, [head | acc]) end # zip defp do_zip(list, acc) do converter = fn x, acc -> do_zip_each(to_list(x), acc) end - {mlist, heads} = :lists.mapfoldl converter, [], list - case heads do - nil -> :lists.reverse acc - _ -> do_zip mlist, [:erlang.list_to_tuple(:lists.reverse(heads))|acc] + case :lists.mapfoldl(converter, [], list) do + {_, nil} -> + :lists.reverse(acc) + + {mlist, heads} -> + do_zip(mlist, [to_tuple(:lists.reverse(heads)) | acc]) end end @@ -666,8 +1391,8 @@ defmodule List do {nil, nil} end - defp do_zip_each([h|t], acc) do - {t, [h|acc]} + defp do_zip_each([head | tail], acc) do + {tail, [head | acc]} end defp do_zip_each([], _) do @@ -675,5 +1400,5 @@ defmodule List do end defp to_list(tuple) when is_tuple(tuple), do: Tuple.to_list(tuple) - defp to_list(list) when is_list(list), do: list + defp to_list(list) when is_list(list), do: list end diff --git a/lib/elixir/lib/list/chars.ex b/lib/elixir/lib/list/chars.ex index cfd42f42ddb..08fd2b50df9 100644 --- a/lib/elixir/lib/list/chars.ex +++ b/lib/elixir/lib/list/chars.ex @@ -1,56 +1,63 @@ defprotocol List.Chars do @moduledoc ~S""" - The List.Chars protocol is responsible for - converting a structure to a list (only if applicable). - The only function required to be implemented is - `to_char_list` which does the conversion. + The `List.Chars` protocol is responsible for + converting a structure to a charlist (only if applicable). - The `to_char_list` function automatically imported - by Kernel invokes this protocol. + The only function that must be implemented is + `to_charlist/1` which does the conversion. + + The `to_charlist/1` function automatically imported + by `Kernel` invokes this protocol. """ - def to_char_list(thing) + @doc """ + Converts `term` to a charlist. + """ + @spec to_charlist(t) :: charlist + def to_charlist(term) + + @doc false + @deprecated "Use List.Chars.to_charlist/1 instead" + Kernel.def to_char_list(term) do + __MODULE__.to_charlist(term) + end end defimpl List.Chars, for: Atom do - def to_char_list(atom), do: Atom.to_char_list(atom) + def to_charlist(nil), do: '' + + def to_charlist(atom), do: Atom.to_charlist(atom) end defimpl List.Chars, for: BitString do @doc """ - Returns the given binary converted to a char list. + Returns the given binary `term` converted to a charlist. """ - def to_char_list(thing) when is_binary(thing) do - String.to_char_list(thing) + def to_charlist(term) when is_binary(term) do + String.to_charlist(term) end - def to_char_list(thing) do + def to_charlist(term) do raise Protocol.UndefinedError, - protocol: @protocol, - value: thing, - description: "cannot convert a bitstring to a char list" + protocol: @protocol, + value: term, + description: "cannot convert a bitstring to a charlist" end end defimpl List.Chars, for: List do - def to_char_list(list), do: list + # Note that same inlining is used for the rewrite rule. + def to_charlist(list), do: list end defimpl List.Chars, for: Integer do - def to_char_list(thing) do - Integer.to_char_list(thing) + def to_charlist(term) do + Integer.to_charlist(term) end end defimpl List.Chars, for: Float do - @digits 20 - @limit :math.pow(10, @digits) - - def to_char_list(thing) when thing > @limit do - Float.to_char_list(thing, scientific: @digits) - end - - def to_char_list(thing) do - Float.to_char_list(thing, compact: true, decimals: @digits) + def to_charlist(term) do + :io_lib_format.fwrite_g(term) end end diff --git a/lib/elixir/lib/macro.ex b/lib/elixir/lib/macro.ex index 6e75221c54e..f082c1a4d5b 100644 --- a/lib/elixir/lib/macro.ex +++ b/lib/elixir/lib/macro.ex @@ -1,60 +1,239 @@ import Kernel, except: [to_string: 1] defmodule Macro do - @moduledoc """ - Conveniences for working with macros. - """ + @moduledoc ~S""" + Functions for manipulating AST and implementing macros. - @typedoc "Abstract Syntax Tree (AST)" - @type t :: expr | {t, t} | atom | number | binary | pid | fun | [t] + Macros are compile-time constructs that receive Elixir's AST as input + and return Elixir's AST as output. - @typedoc "Expr node (remaining ones are literals)" - @type expr :: {expr | atom, Keyword.t, atom | [t]} + Many of the functions in this module exist precisely to work with Elixir + AST, to traverse, query, and transform it. - @binary_ops [:===, :!==, - :==, :!=, :<=, :>=, - :&&, :||, :<>, :++, :--, :\\, :::, :<-, :.., :|>, :=~, - :<, :>, :->, - :+, :-, :*, :/, :=, :|, :., - :and, :or, :xor, :when, :in, - :<<<, :>>>, :|||, :&&&, :^^^, :~~~] + Let's see a simple example that shows the difference between functions + and macros: - @doc false - defmacro binary_ops, do: @binary_ops + defmodule Example do + defmacro macro_inspect(value) do + IO.inspect(value) + value + end - @unary_ops [:!, :@, :^, :not, :+, :-, :~~~, :&] + def fun_inspect(value) do + IO.inspect(value) + value + end + end - @doc false - defmacro unary_ops, do: @unary_ops - - @spec binary_op_props(atom) :: {:left | :right, precedence :: integer} - defp binary_op_props(o) do - case o do - o when o in [:<-, :\\] -> {:left, 40} - :when -> {:right, 50} - ::: -> {:right, 60} - :| -> {:right, 70} - := -> {:right, 90} - o when o in [:||, :|||, :or, :xor] -> {:left, 130} - o when o in [:&&, :&&&, :and] -> {:left, 140} - o when o in [:==, :!=, :=~, :===, :!==] -> {:left, 150} - o when o in [:<, :<=, :>=, :>] -> {:left, 160} - o when o in [:|>, :<<<, :>>>] -> {:left, 170} - :in -> {:left, 180} - o when o in [:++, :--, :.., :<>] -> {:right, 200} - o when o in [:+, :-] -> {:left, 210} - o when o in [:*, :/] -> {:left, 220} - :^^^ -> {:left, 250} - :. -> {:left, 310} - end - end + Now let's give it a try: + + import Example + + macro_inspect(1) + #=> 1 + #=> 1 + + fun_inspect(1) + #=> 1 + #=> 1 + + So far they behave the same, as we are passing an integer as argument. + But let's see what happens when we pass an expression: + + macro_inspect(1 + 2) + #=> {:+, [line: 3], [1, 2]} + #=> 3 + + fun_inspect(1 + 2) + #=> 3 + #=> 3 + + The macro receives the representation of the code given as argument, + while a function receives the result of the code given as argument. + A macro must return a superset of the code representation. See + `t:input/0` and `t:output/0` for more information. + + To learn more about Elixir's AST and how to build them programmatically, + see `quote/2`. + + > Note: the functions in this module do not evaluate code. In fact, + > evaluating code from macros is often an anti-pattern. For code + > evaluation, see the `Code` module. + + ## Custom Sigils + + Macros are also commonly used to implement custom sigils. To create a custom + sigil, define a macro with the name `sigil_{identifier}` that takes two + arguments. The first argument will be the string, the second will be a charlist + containing any modifiers. If the sigil is lower case (such as `sigil_x`) then + the string argument will allow interpolation. If the sigil is upper case + (such as `sigil_X`) then the string will not be interpolated. + + Valid modifiers include only lower and upper case letters. Other characters + will cause a syntax error. + + The module containing the custom sigil must be imported before the sigil + syntax can be used. + + ### Examples + + defmodule MySigils do + defmacro sigil_x(term, [?r]) do + quote do + unquote(term) |> String.reverse() + end + end + defmacro sigil_x(term, _modifiers) do + term + end + defmacro sigil_X(term, [?r]) do + quote do + unquote(term) |> String.reverse() + end + end + defmacro sigil_X(term, _modifiers) do + term + end + end + + import MySigils + + ~x(with #{"inter" <> "polation"}) + #=>"with interpolation" + + ~x(with #{"inter" <> "polation"})r + #=>"noitalopretni htiw" + + ~X(without #{"interpolation"}) + #=>"without \#{"interpolation"}" + + ~X(without #{"interpolation"})r + #=>"}\"noitalopretni\"{# tuohtiw" + + """ + + alias Code.Identifier + + @typedoc "Abstract Syntax Tree (AST)" + @type t :: input + + @typedoc "The inputs of a macro" + @type input :: + input_expr + | {input, input} + | [input] + | atom + | number + | binary + + @typep input_expr :: {input_expr | atom, metadata, atom | [input]} + + @typedoc "The output of a macro" + @type output :: + output_expr + | {output, output} + | [output] + | atom + | number + | binary + | captured_remote_function + | pid + + @typep output_expr :: {output_expr | atom, metadata, atom | [output]} + + @typedoc """ + A keyword list of AST metadata. + + The metadata in Elixir AST is a keyword list of values. Any key can be used + and different parts of the compiler may use different keys. For example, + the AST received by a macro will always include the `:line` annotation, + while the AST emitted by `quote/2` will only have the `:line` annotation if + the `:line` option is provided. + + The following metadata keys are public: + + * `:context` - Defines the context in which the AST was generated. + For example, `quote/2` will include the module calling `quote/2` + as the context. This is often used to distinguish regular code from code + generated by a macro or by `quote/2`. + * `:counter` - The variable counter used for variable hygiene. In terms of + the compiler, each variable is identified by the combination of either + `name` and `metadata[:counter]`, or `name` and `context`. + * `:generated` - Whether the code should be considered as generated by + the compiler or not. This means the compiler and tools like Dialyzer may not + emit certain warnings. + * `:keep` - Used by `quote/2` with the option `location: :keep` to annotate + the file and the line number of the quoted source. + * `:line` - The line number of the AST node. + + The following metadata keys are enabled by `Code.string_to_quoted/2`: + + * `:closing` - contains metadata about the closing pair, such as a `}` + in a tuple or in a map, or such as the closing `)` in a function call + with parens. The `:closing` does not delimit the end of expression if + there are `:do` and `:end` metadata (when `:token_metadata` is true) + * `:column` - the column number of the AST node (when `:columns` is true) + * `:delimiter` - contains the opening delimiter for sigils, strings, + and charlists as a string (such as `"{"`, `"/"`, `"'"`, and the like) + * `:format` - set to `:keyword` when an atom is defined as a keyword + * `:do` - contains metadata about the `do` location in a function call with + `do`-`end` blocks (when `:token_metadata` is true) + * `:end` - contains metadata about the `end` location in a function call with + `do`-`end` blocks (when `:token_metadata` is true) + * `:end_of_expression` - denotes when the end of expression effectively + happens. Available for all expressions except the last one inside a + `__block__` (when `:token_metadata` is true) + * `:indentation` - indentation of a sigil heredoc + + The following metadata keys are private: + + * `:alias` - Used for alias hygiene. + * `:ambiguous_op` - Used for improved error messages in the compiler. + * `:imports` - Used for import hygiene. + * `:var` - Used for improved error messages on undefined variables. + + Do not rely on them as they may change or be fully removed in future versions + of the language. They are often used by `quote/2` and the compiler to provide + features like hygiene, better error messages, and so forth. + + If you introduce custom keys into the AST metadata, please make sure to prefix + them with the name of your library or application, so that they will not conflict + with keys that could potentially be introduced by the compiler in the future. + """ + @type metadata :: keyword + + @typedoc "A captured remote function in the format of &Mod.fun/arity" + @type captured_remote_function :: fun @doc """ Breaks a pipeline expression into a list. - Raises if the pipeline is ill-formed. + The AST for a pipeline (a sequence of applications of `|>/2`) is similar to the + AST of a sequence of binary operators or function applications: the top-level + expression is the right-most `:|>` (which is the last one to be executed), and + its left-hand and right-hand sides are its arguments: + + quote do: 100 |> div(5) |> div(2) + #=> {:|>, _, [arg1, arg2]} + + In the example above, the `|>/2` pipe is the right-most pipe; `arg1` is the AST + for `100 |> div(5)`, and `arg2` is the AST for `div(2)`. + + It's often useful to have the AST for such a pipeline as a list of function + applications. This function does exactly that: + + Macro.unpipe(quote do: 100 |> div(5) |> div(2)) + #=> [{100, 0}, {{:div, [], [5]}, 0}, {{:div, [], [2]}, 0}] + + We get a list that follows the pipeline directly: first the `100`, then the + `div(5)` (more precisely, its AST), then `div(2)`. The `0` as the second + element of the tuples is the position of the previous element in the pipeline + inside the current function application: `{{:div, [], [5]}, 0}` means that the + previous element (`100`) will be inserted as the 0th (first) argument to the + `div/2` function, so that the AST for that function will become `{:div, [], + [100, 5]}` (`div(100, 5)`). """ - @spec unpipe(Macro.t) :: [Macro.t] + @spec unpipe(t()) :: [t()] def unpipe(expr) do :lists.reverse(unpipe(expr, [])) end @@ -64,35 +243,96 @@ defmodule Macro do end defp unpipe(other, acc) do - [{other, 0}|acc] + [{other, 0} | acc] end @doc """ Pipes `expr` into the `call_args` at the given `position`. + + `expr` is the AST of an expression. `call_args` must be the AST *of a call*, + otherwise this function will raise an error. As an example, consider the pipe + operator `|>/2`, which uses this function to build pipelines. + + Even if the expression is piped into the AST, it doesn't necessarily mean that + the AST is valid. For example, you could pipe an argument to `div/2`, effectively + turning it into a call to `div/3`, which is a function that doesn't exist by + default. The code will raise unless a `div/3` function is locally defined. """ - @spec pipe(Macro.t, Macro.t, integer) :: Macro.t | no_return + @spec pipe(t(), t(), integer) :: t() def pipe(expr, call_args, position) def pipe(expr, {:&, _, _} = call_args, _integer) do - raise ArgumentError, "cannot pipe #{to_string expr} into #{to_string call_args}" + raise ArgumentError, bad_pipe(expr, call_args) + end + + def pipe(expr, {tuple_or_map, _, _} = call_args, _integer) when tuple_or_map in [:{}, :%{}] do + raise ArgumentError, bad_pipe(expr, call_args) + end + + # Without this, `Macro |> Env == Macro.Env`. + def pipe(expr, {:__aliases__, _, _} = call_args, _integer) do + raise ArgumentError, bad_pipe(expr, call_args) + end + + def pipe(expr, {:<<>>, _, _} = call_args, _integer) do + raise ArgumentError, bad_pipe(expr, call_args) + end + + def pipe(expr, {unquote, _, []}, _integer) when unquote in [:unquote, :unquote_splicing] do + raise ArgumentError, + "cannot pipe #{to_string(expr)} into the special form #{unquote}/1 " <> + "since #{unquote}/1 is used to build the Elixir AST itself" + end + + # {:fn, _, _} is what we get when we pipe into an anonymous function without + # calling it, for example, `:foo |> (fn x -> x end)`. + def pipe(expr, {:fn, _, _}, _integer) do + raise ArgumentError, + "cannot pipe #{to_string(expr)} into an anonymous function without" <> + " calling the function; use Kernel.then/2 instead or" <> + " define the anonymous function as a regular private function" end def pipe(expr, {call, line, atom}, integer) when is_atom(atom) do {call, line, List.insert_at([], integer, expr)} end - def pipe(expr, {call, line, args}, integer) when is_list(args) do - {call, line, List.insert_at(args, integer, expr)} + def pipe(_expr, {op, _line, [arg]}, _integer) when op == :+ or op == :- do + raise ArgumentError, + "piping into a unary operator is not supported, please use the qualified name: " <> + "Kernel.#{op}(#{to_string(arg)}), instead of #{op}#{to_string(arg)}" + end + + def pipe(expr, {op, line, args} = op_args, integer) when is_list(args) do + cond do + is_atom(op) and operator?(op, 1) -> + raise ArgumentError, + "cannot pipe #{to_string(expr)} into #{to_string(op_args)}, " <> + "the #{to_string(op)} operator can only take one argument" + + is_atom(op) and operator?(op, 2) -> + raise ArgumentError, + "cannot pipe #{to_string(expr)} into #{to_string(op_args)}, " <> + "the #{to_string(op)} operator can only take two arguments" + + true -> + {op, line, List.insert_at(args, integer, expr)} + end end def pipe(expr, call_args, _integer) do - raise ArgumentError, "cannot pipe #{to_string expr} into #{to_string call_args}" + raise ArgumentError, bad_pipe(expr, call_args) + end + + defp bad_pipe(expr, call_args) do + "cannot pipe #{to_string(expr)} into #{to_string(call_args)}, " <> + "can only pipe into local calls foo(), remote calls Foo.bar() or anonymous function calls foo.()" end @doc """ Applies the given function to the node metadata if it contains one. - This is often useful when used with `Macro.prewalk/1` to remove + This is often useful when used with `Macro.prewalk/2` to remove information like lines and hygienic counters from the expression for either storage or comparison. @@ -104,7 +344,7 @@ defmodule Macro do {:sample, [], []} """ - @spec update_meta(t, (Keyword.t -> Keyword.t)) :: t + @spec update_meta(t, (keyword -> keyword)) :: t def update_meta(quoted, fun) def update_meta({left, meta, right}, fun) when is_list(meta) do @@ -116,14 +356,69 @@ defmodule Macro do end @doc """ - Genrates a AST node representing the variable given + Generates AST nodes for a given number of required argument + variables using `Macro.var/2`. + + Note the arguments are not unique. If you later on want + to access the same variables, you can invoke this function + with the same inputs. Use `generate_unique_arguments/2` to + generate a unique arguments that can't be overridden. + + ## Examples + + iex> Macro.generate_arguments(2, __MODULE__) + [{:arg1, [], __MODULE__}, {:arg2, [], __MODULE__}] + + """ + @doc since: "1.5.0" + @spec generate_arguments(0, context :: atom) :: [] + @spec generate_arguments(pos_integer, context) :: [{atom, [], context}, ...] when context: atom + def generate_arguments(amount, context), do: generate_arguments(amount, context, &var/2) + + @doc """ + Generates AST nodes for a given number of required argument + variables using `Macro.unique_var/2`. + + ## Examples + + iex> [var1, var2] = Macro.generate_unique_arguments(2, __MODULE__) + iex> {:arg1, [counter: c1], __MODULE__} = var1 + iex> {:arg2, [counter: c2], __MODULE__} = var2 + iex> is_integer(c1) and is_integer(c2) + true + + """ + @doc since: "1.11.3" + @spec generate_unique_arguments(0, context :: atom) :: [] + @spec generate_unique_arguments(pos_integer, context) :: [ + {atom, [counter: integer], context}, + ... + ] + when context: atom + def generate_unique_arguments(amount, context), + do: generate_arguments(amount, context, &unique_var/2) + + defp generate_arguments(0, context, _fun) when is_atom(context), do: [] + + defp generate_arguments(amount, context, fun) + when is_integer(amount) and amount > 0 and is_atom(context) do + for id <- 1..amount, do: fun.(String.to_atom("arg" <> Integer.to_string(id)), context) + end + + @doc """ + Generates an AST node representing the variable given by the atoms `var` and `context`. + Note this variable is not unique. If you later on want + to access this same variable, you can invoke `var/2` + again with the same arguments. Use `unique_var/2` to + generate a unique variable that can't be overridden. + ## Examples In order to build a variable, a context is expected. Most of the times, in order to preserve hygiene, the - context must be `__MODULE__`: + context must be `__MODULE__/0`: iex> Macro.var(:foo, __MODULE__) {:foo, [], __MODULE__} @@ -141,100 +436,183 @@ defmodule Macro do end @doc """ - Performs a depth-first, pre-order traversal of quoted expressions. + Generates an AST node representing a unique variable + given by the atoms `var` and `context`. + + Calling this function with the same arguments will + generate another variable, with its own unique counter. + See `var/2` for an alternative. + + ## Examples + + iex> {:foo, [counter: c], __MODULE__} = Macro.unique_var(:foo, __MODULE__) + iex> is_integer(c) + true + """ - @spec prewalk(t, (t -> t)) :: t - def prewalk(ast, fun) when is_function(fun, 1) do - elem(prewalk(ast, nil, fn x, nil -> {fun.(x), nil} end), 0) + @doc since: "1.11.3" + @spec unique_var(var, context) :: {var, [counter: integer], context} + when var: atom, context: atom + def unique_var(var, context) when is_atom(var) and is_atom(context) do + {var, [counter: :elixir_module.next_counter(context)], context} end @doc """ - Performs a depth-first, pre-order traversal of quoted expressions + Performs a depth-first traversal of quoted expressions using an accumulator. + + Returns a tuple where the first element is a new AST and the second one is + the final accumulator. The new AST is the result of invoking `pre` on each + node of `ast` during the pre-order phase and `post` during the post-order + phase. + + ## Examples + + iex> ast = quote do: 5 + 3 * 7 + iex> {:+, _, [5, {:*, _, [3, 7]}]} = ast + iex> {new_ast, acc} = + ...> Macro.traverse( + ...> ast, + ...> [], + ...> fn + ...> {:+, meta, children}, acc -> {{:-, meta, children}, [:- | acc]} + ...> {:*, meta, children}, acc -> {{:/, meta, children}, [:/ | acc]} + ...> other, acc -> {other, acc} + ...> end, + ...> fn + ...> {:-, meta, children}, acc -> {{:min, meta, children}, [:min | acc]} + ...> {:/, meta, children}, acc -> {{:max, meta, children}, [:max | acc]} + ...> other, acc -> {other, acc} + ...> end + ...> ) + iex> {:min, _, [5, {:max, _, [3, 7]}]} = new_ast + iex> [:min, :max, :/, :-] = acc + iex> Code.eval_quoted(new_ast) + {5, []} + """ - @spec prewalk(t, any, (t, any -> {t, any})) :: {t, any} - def prewalk(ast, acc, fun) when is_function(fun, 2) do - {ast, acc} = fun.(ast, acc) - do_prewalk(ast, acc, fun) + @spec traverse(t, any, (t, any -> {t, any}), (t, any -> {t, any})) :: {t, any} + def traverse(ast, acc, pre, post) when is_function(pre, 2) and is_function(post, 2) do + {ast, acc} = pre.(ast, acc) + do_traverse(ast, acc, pre, post) end - defp do_prewalk({form, meta, args}, acc, fun) do - unless is_atom(form) do - {form, acc} = fun.(form, acc) - {form, acc} = do_prewalk(form, acc, fun) - end + defp do_traverse({form, meta, args}, acc, pre, post) when is_atom(form) do + {args, acc} = do_traverse_args(args, acc, pre, post) + post.({form, meta, args}, acc) + end - unless is_atom(args) do - {args, acc} = Enum.map_reduce(args, acc, fn x, acc -> - {x, acc} = fun.(x, acc) - do_prewalk(x, acc, fun) - end) - end + defp do_traverse({form, meta, args}, acc, pre, post) do + {form, acc} = pre.(form, acc) + {form, acc} = do_traverse(form, acc, pre, post) + {args, acc} = do_traverse_args(args, acc, pre, post) + post.({form, meta, args}, acc) + end - {{form, meta, args}, acc} + defp do_traverse({left, right}, acc, pre, post) do + {left, acc} = pre.(left, acc) + {left, acc} = do_traverse(left, acc, pre, post) + {right, acc} = pre.(right, acc) + {right, acc} = do_traverse(right, acc, pre, post) + post.({left, right}, acc) end - defp do_prewalk({left, right}, acc, fun) do - {left, acc} = fun.(left, acc) - {left, acc} = do_prewalk(left, acc, fun) - {right, acc} = fun.(right, acc) - {right, acc} = do_prewalk(right, acc, fun) - {{left, right}, acc} + defp do_traverse(list, acc, pre, post) when is_list(list) do + {list, acc} = do_traverse_args(list, acc, pre, post) + post.(list, acc) end - defp do_prewalk(list, acc, fun) when is_list(list) do - Enum.map_reduce(list, acc, fn x, acc -> - {x, acc} = fun.(x, acc) - do_prewalk(x, acc, fun) - end) + defp do_traverse(x, acc, _pre, post) do + post.(x, acc) + end + + defp do_traverse_args(args, acc, _pre, _post) when is_atom(args) do + {args, acc} end - defp do_prewalk(x, acc, _fun) do - {x, acc} + defp do_traverse_args(args, acc, pre, post) when is_list(args) do + :lists.mapfoldl( + fn x, acc -> + {x, acc} = pre.(x, acc) + do_traverse(x, acc, pre, post) + end, + acc, + args + ) end @doc """ - Performs a depth-first, post-order traversal of quoted expressions. + Performs a depth-first, pre-order traversal of quoted expressions. + + Returns a new AST where each node is the result of invoking `fun` on each + corresponding node of `ast`. + + ## Examples + + iex> ast = quote do: 5 + 3 * 7 + iex> {:+, _, [5, {:*, _, [3, 7]}]} = ast + iex> new_ast = Macro.prewalk(ast, fn + ...> {:+, meta, children} -> {:*, meta, children} + ...> {:*, meta, children} -> {:+, meta, children} + ...> other -> other + ...> end) + iex> {:*, _, [5, {:+, _, [3, 7]}]} = new_ast + iex> Code.eval_quoted(ast) + {26, []} + iex> Code.eval_quoted(new_ast) + {50, []} + """ - @spec postwalk(t, (t -> t)) :: t - def postwalk(ast, fun) when is_function(fun, 1) do - elem(postwalk(ast, nil, fn x, nil -> {fun.(x), nil} end), 0) + @spec prewalk(t, (t -> t)) :: t + def prewalk(ast, fun) when is_function(fun, 1) do + elem(prewalk(ast, nil, fn x, nil -> {fun.(x), nil} end), 0) end @doc """ - Performs a depth-first, post-order traversal of quoted expressions + Performs a depth-first, pre-order traversal of quoted expressions using an accumulator. - """ - @spec postwalk(t, any, (t, any -> {t, any})) :: {t, any} - def postwalk(ast, acc, fun) when is_function(fun, 2) do - do_postwalk(ast, acc, fun) - end - defp do_postwalk({form, meta, args}, acc, fun) do - unless is_atom(form) do - {form, acc} = do_postwalk(form, acc, fun) - end + Returns a tuple where the first element is a new AST where each node is the + result of invoking `fun` on each corresponding node and the second one is the + final accumulator. - unless is_atom(args) do - {args, acc} = Enum.map_reduce(args, acc, &do_postwalk(&1, &2, fun)) - end + ## Examples - fun.({form, meta, args}, acc) - end + iex> ast = quote do: 5 + 3 * 7 + iex> {:+, _, [5, {:*, _, [3, 7]}]} = ast + iex> {new_ast, acc} = Macro.prewalk(ast, [], fn + ...> {:+, meta, children}, acc -> {{:*, meta, children}, [:+ | acc]} + ...> {:*, meta, children}, acc -> {{:+, meta, children}, [:* | acc]} + ...> other, acc -> {other, acc} + ...> end) + iex> {{:*, _, [5, {:+, _, [3, 7]}]}, [:*, :+]} = {new_ast, acc} + iex> Code.eval_quoted(ast) + {26, []} + iex> Code.eval_quoted(new_ast) + {50, []} - defp do_postwalk({left, right}, acc, fun) do - {left, acc} = do_postwalk(left, acc, fun) - {right, acc} = do_postwalk(right, acc, fun) - fun.({left, right}, acc) + """ + @spec prewalk(t, any, (t, any -> {t, any})) :: {t, any} + def prewalk(ast, acc, fun) when is_function(fun, 2) do + traverse(ast, acc, fun, fn x, a -> {x, a} end) end - defp do_postwalk(list, acc, fun) when is_list(list) do - {list, acc} = Enum.map_reduce(list, acc, &do_postwalk(&1, &2, fun)) - fun.(list, acc) + @doc """ + This function behaves like `prewalk/2`, but performs a depth-first, + post-order traversal of quoted expressions. + """ + @spec postwalk(t, (t -> t)) :: t + def postwalk(ast, fun) when is_function(fun, 1) do + elem(postwalk(ast, nil, fn x, nil -> {fun.(x), nil} end), 0) end - defp do_postwalk(x, acc, fun) do - fun.(x, acc) + @doc """ + This functions behaves like `prewalk/3`, but performs a depth-first, + post-order traversal of quoted expressions using an accumulator. + """ + @spec postwalk(t, any, (t, any -> {t, any})) :: {t, any} + def postwalk(ast, acc, fun) when is_function(fun, 2) do + traverse(ast, acc, fn x, a -> {x, a} end, fun) end @doc """ @@ -245,42 +623,42 @@ defmodule Macro do ## Examples - iex> Macro.decompose_call(quote do: foo) + iex> Macro.decompose_call(quote(do: foo)) {:foo, []} - iex> Macro.decompose_call(quote do: foo()) + iex> Macro.decompose_call(quote(do: foo())) {:foo, []} - iex> Macro.decompose_call(quote do: foo(1, 2, 3)) + iex> Macro.decompose_call(quote(do: foo(1, 2, 3))) {:foo, [1, 2, 3]} - iex> Macro.decompose_call(quote do: Elixir.M.foo(1, 2, 3)) + iex> Macro.decompose_call(quote(do: Elixir.M.foo(1, 2, 3))) {{:__aliases__, [], [:Elixir, :M]}, :foo, [1, 2, 3]} - iex> Macro.decompose_call(quote do: 42) + iex> Macro.decompose_call(quote(do: 42)) + :error + + iex> Macro.decompose_call(quote(do: {:foo, [], []})) :error """ - @spec decompose_call(Macro.t) :: {atom, [Macro.t]} | {Macro.t, atom, [Macro.t]} | :error - def decompose_call({{:., _, [remote, function]}, _, args}) when is_tuple(remote) or is_atom(remote), - do: {remote, function, args} + @spec decompose_call(t()) :: {atom, [t()]} | {t(), atom, [t()]} | :error + def decompose_call(ast) - def decompose_call({name, _, args}) when is_atom(name) and is_atom(args), - do: {name, []} + def decompose_call({:{}, _, args}) when is_list(args), do: :error - def decompose_call({name, _, args}) when is_atom(name) and is_list(args), - do: {name, args} + def decompose_call({{:., _, [remote, function]}, _, args}) + when is_tuple(remote) or is_atom(remote), + do: {remote, function, args} - def decompose_call(_), - do: :error + def decompose_call({name, _, args}) when is_atom(name) and is_atom(args), do: {name, []} - @doc """ - Recursively escapes a value so it can be inserted - into a syntax tree. + def decompose_call({name, _, args}) when is_atom(name) and is_list(args), do: {name, args} - One may pass `unquote: true` to `escape/2` - which leaves `unquote` statements unescaped, effectively - unquoting the contents on escape. + def decompose_call(_), do: :error + + @doc """ + Recursively escapes a value so it can be inserted into a syntax tree. ## Examples @@ -293,27 +671,268 @@ defmodule Macro do iex> Macro.escape({:unquote, [], [1]}, unquote: true) 1 + ## Options + + * `:unquote` - when true, this function leaves `unquote/1` and + `unquote_splicing/1` statements unescaped, effectively unquoting + the contents on escape. This option is useful only when escaping + ASTs which may have quoted fragments in them. Defaults to false. + + * `:prune_metadata` - when true, removes metadata from escaped AST + nodes. Note this option changes the semantics of escaped code and + it should only be used when escaping ASTs. Defaults to false. + + As an example, `ExUnit` stores the AST of every assertion, so when + an assertion fails we can show code snippets to users. Without this + option, each time the test module is compiled, we get a different + MD5 of the module bytecode, because the AST contains metadata, + such as counters, specific to the compilation environment. By pruning + the metadata, we ensure that the module is deterministic and reduce + the amount of data `ExUnit` needs to keep around. Only the minimal + amount of metadata is kept, such as `:line` and `:no_parens`. + + ## Comparison to `quote/2` + + The `escape/2` function is sometimes confused with `quote/2`, + because the above examples behave the same with both. The key difference is + best illustrated when the value to escape is stored in a variable. + + iex> Macro.escape({:a, :b, :c}) + {:{}, [], [:a, :b, :c]} + iex> quote do: {:a, :b, :c} + {:{}, [], [:a, :b, :c]} + + iex> value = {:a, :b, :c} + iex> Macro.escape(value) + {:{}, [], [:a, :b, :c]} + + iex> quote do: value + {:value, [], __MODULE__} + + iex> value = {:a, :b, :c} + iex> quote do: unquote(value) + {:a, :b, :c} + + `escape/2` is used to escape *values* (either directly passed or variable + bound), while `quote/2` produces syntax trees for + expressions. """ - @spec escape(term) :: Macro.t - @spec escape(term, Keyword.t) :: Macro.t + @spec escape(term, keyword) :: t() def escape(expr, opts \\ []) do - elem(:elixir_quote.escape(expr, Keyword.get(opts, :unquote, false)), 0) + unquote = Keyword.get(opts, :unquote, false) + kind = if Keyword.get(opts, :prune_metadata, false), do: :prune_metadata, else: :none + :elixir_quote.escape(expr, kind, unquote) + end + + @doc """ + Expands the struct given by `module` in the given `env`. + + This is useful when a struct needs to be expanded at + compilation time and the struct being expanded may or may + not have been compiled. This function is also capable of + expanding structs defined under the module being compiled. + + It will raise `CompileError` if the struct is not available. + From Elixir v1.12, calling this function also adds an export + dependency on the given struct. + """ + @doc since: "1.8.0" + @spec struct!(module, Macro.Env.t()) :: + %{required(:__struct__) => module, optional(atom) => any} + when module: module() + def struct!(module, env) when is_atom(module) do + if module == env.module do + Module.get_attribute(module, :__struct__) + end || :elixir_map.load_struct([line: env.line], module, [], [], env) + end + + @doc """ + Validates the given expressions are valid quoted expressions. + + Check the type `t:Macro.t/0` for a complete specification of a + valid quoted expression. + + It returns `:ok` if the expression is valid. Otherwise it returns + a tuple in the form of `{:error, remainder}` where `remainder` is + the invalid part of the quoted expression. + + ## Examples + + iex> Macro.validate({:two_element, :tuple}) + :ok + iex> Macro.validate({:three, :element, :tuple}) + {:error, {:three, :element, :tuple}} + + iex> Macro.validate([1, 2, 3]) + :ok + iex> Macro.validate([1, 2, 3, {4}]) + {:error, {4}} + + """ + @spec validate(term) :: :ok | {:error, term} + def validate(expr) do + find_invalid(expr) || :ok + end + + defp find_invalid({left, right}), do: find_invalid(left) || find_invalid(right) + + defp find_invalid({left, meta, right}) + when is_list(meta) and (is_atom(right) or is_list(right)), + do: find_invalid(left) || find_invalid(right) + + defp find_invalid(list) when is_list(list), do: Enum.find_value(list, &find_invalid/1) + + defp find_invalid(pid) when is_pid(pid), do: nil + defp find_invalid(atom) when is_atom(atom), do: nil + defp find_invalid(num) when is_number(num), do: nil + defp find_invalid(bin) when is_binary(bin), do: nil + + defp find_invalid(fun) when is_function(fun) do + unless Function.info(fun, :env) == {:env, []} and + Function.info(fun, :type) == {:type, :external} do + {:error, fun} + end + end + + defp find_invalid(other), do: {:error, other} + + @doc """ + Returns an enumerable that traverses the `ast` in depth-first, + pre-order traversal. + + ## Examples + + iex> ast = quote do: foo(1, "abc") + iex> Enum.map(Macro.prewalker(ast), & &1) + [{:foo, [], [1, "abc"]}, 1, "abc"] + + """ + @doc since: "1.13.0" + @spec prewalker(t()) :: Enumerable.t() + def prewalker(ast) do + &prewalker([ast], &1, &2) + end + + defp prewalker(_buffer, {:halt, acc}, _fun) do + {:halted, acc} + end + + defp prewalker(buffer, {:suspend, acc}, fun) do + {:suspended, acc, &prewalker(buffer, &1, fun)} + end + + defp prewalker([], {:cont, acc}, _fun) do + {:done, acc} + end + + defp prewalker([{left, right} = node | tail], {:cont, acc}, fun) do + prewalker([left, right | tail], fun.(node, acc), fun) + end + + defp prewalker([{left, meta, right} = node | tail], {:cont, acc}, fun) + when is_atom(left) and is_list(meta) do + if is_atom(right) do + prewalker(tail, fun.(node, acc), fun) + else + prewalker(right ++ tail, fun.(node, acc), fun) + end + end + + defp prewalker([{left, meta, right} = node | tail], {:cont, acc}, fun) when is_list(meta) do + if is_atom(right) do + prewalker([left | tail], fun.(node, acc), fun) + else + prewalker([left | right] ++ tail, fun.(node, acc), fun) + end + end + + defp prewalker([list | tail], {:cont, acc}, fun) when is_list(list) do + prewalker(list ++ tail, fun.(list, acc), fun) + end + + defp prewalker([head | tail], {:cont, acc}, fun) do + prewalker(tail, fun.(head, acc), fun) + end + + @doc """ + Returns an enumerable that traverses the `ast` in depth-first, + post-order traversal. + + ## Examples + + iex> ast = quote do: foo(1, "abc") + iex> Enum.map(Macro.postwalker(ast), & &1) + [1, "abc", {:foo, [], [1, "abc"]}] + + """ + @doc since: "1.13.0" + @spec postwalker(t()) :: Enumerable.t() + def postwalker(ast) do + &postwalker([ast], make_ref(), &1, &2) + end + + defp postwalker(_buffer, _ref, {:halt, acc}, _fun) do + {:halted, acc} + end + + defp postwalker(buffer, ref, {:suspend, acc}, fun) do + {:suspended, acc, &postwalker(buffer, ref, &1, fun)} + end + + defp postwalker([], _ref, {:cont, acc}, _fun) do + {:done, acc} + end + + defp postwalker([{ref, head} | tail], ref, {:cont, acc}, fun) do + postwalker(tail, ref, fun.(head, acc), fun) + end + + defp postwalker([{left, right} = node | tail], ref, {:cont, acc}, fun) do + postwalker([right, {ref, node} | tail], ref, fun.(left, acc), fun) + end + + defp postwalker([{left, meta, right} = node | tail], ref, {:cont, acc}, fun) + when is_atom(left) and is_list(meta) do + if is_atom(right) do + postwalker(tail, ref, fun.(node, acc), fun) + else + postwalker(right ++ [{ref, node} | tail], ref, {:cont, acc}, fun) + end + end + + defp postwalker([{left, meta, right} = node | tail], ref, cont_acc, fun) + when is_list(meta) do + if is_atom(right) do + postwalker([left, {ref, node} | tail], ref, cont_acc, fun) + else + postwalker([left | right] ++ [{ref, node} | tail], ref, cont_acc, fun) + end + end + + defp postwalker([list | tail], ref, cont_acc, fun) when is_list(list) do + postwalker(list ++ [{ref, list} | tail], ref, cont_acc, fun) + end + + defp postwalker([head | tail], ref, {:cont, acc}, fun) do + postwalker(tail, ref, fun.(head, acc), fun) end @doc ~S""" - Unescape the given chars. + Unescapes characters in a string. This is the unescaping behaviour used by default in Elixir single- and double-quoted strings. Check `unescape_string/2` for information on how to customize the escaping map. - In this setup, Elixir will escape the following: `\a`, `\b`, - `\d`, `\e`, `\f`, `\n`, `\r`, `\s`, `\t` and `\v`. Octals are - also escaped according to the latin1 set they represent. + In this setup, Elixir will escape the following: `\0`, `\a`, `\b`, + `\d`, `\e`, `\f`, `\n`, `\r`, `\s`, `\t` and `\v`. Bytes can be + given as hexadecimals via `\xNN` and Unicode code points as + `\uNNNN` escapes. This function is commonly used on sigil implementations - (like `~r`, `~s` and others) which receive a raw, unescaped - string. + (like `~r`, `~s` and others), which receive a raw, unescaped + string, and it can be used anywhere that needs to mimic how + Elixir parses strings. ## Examples @@ -323,23 +942,30 @@ defmodule Macro do In the example above, we pass a string with `\n` escaped and return a version with it unescaped. """ - @spec unescape_string(String.t) :: String.t - def unescape_string(chars) do - :elixir_interpolation.unescape_chars(chars) + @spec unescape_string(String.t()) :: String.t() + def unescape_string(string) do + :elixir_interpolation.unescape_string(string) end @doc ~S""" - Unescape the given chars according to the map given. + Unescapes characters in a string according to the given mapping. - Check `unescape_string/1` if you want to use the same map + Check `unescape_string/1` if you want to use the same mapping as Elixir single- and double-quoted strings. - ## Map + ## Mapping function + + The mapping function receives an integer representing the code point + of the character it wants to unescape. There are also the special atoms + `:newline`, `:unicode`, and `:hex`, which control newline, unicode, + and escaping respectively. - The map must be a function. The function receives an integer - representing the codepoint of the character it wants to unescape. Here is the default mapping function implemented by Elixir: + def unescape_map(:newline), do: true + def unescape_map(:unicode), do: true + def unescape_map(:hex), do: true + def unescape_map(?0), do: ?0 def unescape_map(?a), do: ?\a def unescape_map(?b), do: ?\b def unescape_map(?d), do: ?\d @@ -350,74 +976,87 @@ defmodule Macro do def unescape_map(?s), do: ?\s def unescape_map(?t), do: ?\t def unescape_map(?v), do: ?\v - def unescape_map(e), do: e + def unescape_map(e), do: e - If the `unescape_map` function returns `false`. The char is - not escaped and `\` is kept in the char list. - - ## Octals - - Octals will by default be escaped unless the map function - returns `false` for `?0`. - - ## Hex - - Hexadecimals will by default be escaped unless the map function - returns `false` for `?x`. + If the `unescape_map/1` function returns `false`, the char is + not escaped and the backslash is kept in the string. ## Examples - Using the `unescape_map` function defined above is easy: + Using the `unescape_map/1` function defined above is easy: - Macro.unescape_string "example\\n", &unescape_map(&1) + Macro.unescape_string("example\\n", &unescape_map(&1)) """ - @spec unescape_string(String.t, (non_neg_integer -> non_neg_integer | false)) :: String.t - def unescape_string(chars, map) do - :elixir_interpolation.unescape_chars(chars, map) + @spec unescape_string(String.t(), (non_neg_integer -> non_neg_integer | false)) :: String.t() + def unescape_string(string, map) do + :elixir_interpolation.unescape_string(string, map) end - @doc """ - Unescape the given tokens according to the default map. - - Check `unescape_string/1` and `unescape_string/2` for more - information about unescaping. - - Only tokens that are binaries are unescaped, all others are - ignored. This function is useful when implementing your own - sigils. Check the implementation of `Kernel.sigil_s/2` - for examples. - """ - @spec unescape_tokens([Macro.t]) :: [Macro.t] + @doc false + @deprecated "Traverse over the arguments using Enum.map/2 instead" def unescape_tokens(tokens) do - :elixir_interpolation.unescape_tokens(tokens) + for token <- tokens do + if is_binary(token), do: unescape_string(token), else: token + end + end + + @doc false + @deprecated "Traverse over the arguments using Enum.map/2 instead" + def unescape_tokens(tokens, map) do + for token <- tokens do + if is_binary(token), do: unescape_string(token, map), else: token + end end @doc """ - Unescape the given tokens according to the given map. + Converts the given expression AST to a string. + + This is a convenience function for converting AST into + a string, which discards all formatting of the original + code and wraps newlines around 98 characters. See + `Code.quoted_to_algebra/2` as a lower level function + with more control around formatting. + + ## Examples + + iex> Macro.to_string(quote(do: foo.bar(1, 2, 3))) + "foo.bar(1, 2, 3)" - Check `unescape_tokens/1` and `unescape_string/2` for more information. """ - @spec unescape_tokens([Macro.t], (non_neg_integer -> non_neg_integer | false)) :: [Macro.t] - def unescape_tokens(tokens, map) do - :elixir_interpolation.unescape_tokens(tokens, map) + @spec to_string(t()) :: String.t() + # TODO: Allow line_length to be configurable on v1.17 + def to_string(tree) do + doc = Inspect.Algebra.format(Code.quoted_to_algebra(tree), 98) + IO.iodata_to_binary(doc) end @doc """ - Converts the given expression to a binary. + Converts the given expression AST to a string. + + The given `fun` is called for every node in the AST with two arguments: the + AST of the node being printed and the string representation of that same + node. The return value of this function is used as the final string + representation for that AST node. + + This function discards all formatting of the original code. ## Examples - iex> Macro.to_string(quote do: foo.bar(1, 2, 3)) - "foo.bar(1, 2, 3)" + Macro.to_string(quote(do: 1 + 2), fn + 1, _string -> "one" + 2, _string -> "two" + _ast, string -> string + end) + #=> "one + two" """ - @spec to_string(Macro.t) :: String.t - @spec to_string(Macro.t, (Macro.t, String.t -> String.t)) :: String.t - def to_string(tree, fun \\ fn(_ast, string) -> string end) + @deprecated "Use Macro.to_string/1 instead" + @spec to_string(t(), (t(), String.t() -> String.t())) :: String.t() + def to_string(tree, fun) # Variables - def to_string({var, _, atom} = ast, fun) when is_atom(atom) do + def to_string({var, _, context} = ast, fun) when is_atom(var) and is_atom(context) do fun.(ast, Atom.to_string(var)) end @@ -426,149 +1065,394 @@ defmodule Macro do fun.(ast, Enum.map_join(refs, ".", &call_to_string(&1, fun))) end - # Blocks - def to_string({:__block__, _, [expr]} = ast, fun) do - fun.(ast, to_string(expr, fun)) + # Blocks + def to_string({:__block__, _, [expr]} = ast, fun) do + fun.(ast, to_string(expr, fun)) + end + + def to_string({:__block__, _, _} = ast, fun) do + block = adjust_new_lines(block_to_string(ast, fun), "\n ") + fun.(ast, "(\n " <> block <> "\n)") + end + + # Bits containers + def to_string({:<<>>, _, parts} = ast, fun) do + if interpolated?(ast) do + fun.(ast, interpolate(ast, fun)) + else + result = + Enum.map_join(parts, ", ", fn part -> + str = bitpart_to_string(part, fun) + + if :binary.first(str) == ?< or :binary.last(str) == ?> do + "(" <> str <> ")" + else + str + end + end) + + fun.(ast, "<<" <> result <> ">>") + end + end + + # Tuple containers + def to_string({:{}, _, args} = ast, fun) do + tuple = "{" <> Enum.map_join(args, ", ", &to_string(&1, fun)) <> "}" + fun.(ast, tuple) + end + + # Map containers + def to_string({:%{}, _, args} = ast, fun) do + map = "%{" <> map_to_string(args, fun) <> "}" + fun.(ast, map) + end + + def to_string({:%, _, [struct_name, map]} = ast, fun) do + {:%{}, _, args} = map + struct = "%" <> to_string(struct_name, fun) <> "{" <> map_to_string(args, fun) <> "}" + fun.(ast, struct) + end + + # Fn keyword + def to_string({:fn, _, [{:->, _, [_, tuple]}] = arrow} = ast, fun) + when not is_tuple(tuple) or elem(tuple, 0) != :__block__ do + fun.(ast, "fn " <> arrow_to_string(arrow, fun) <> " end") + end + + def to_string({:fn, _, [{:->, _, _}] = block} = ast, fun) do + fun.(ast, "fn " <> block_to_string(block, fun) <> "\nend") + end + + def to_string({:fn, _, block} = ast, fun) do + block = adjust_new_lines(block_to_string(block, fun), "\n ") + fun.(ast, "fn\n " <> block <> "\nend") + end + + # left -> right + def to_string([{:->, _, _} | _] = ast, fun) do + fun.(ast, "(" <> arrow_to_string(ast, fun, true) <> ")") + end + + # left when right + def to_string({:when, _, [left, right]} = ast, fun) do + right = + if right != [] and Keyword.keyword?(right) do + kw_list_to_string(right, fun) + else + fun.(ast, op_to_string(right, fun, :when, :right)) + end + + fun.(ast, op_to_string(left, fun, :when, :left) <> " when " <> right) + end + + # Splat when + def to_string({:when, _, args} = ast, fun) do + {left, right} = split_last(args) + + result = + "(" <> Enum.map_join(left, ", ", &to_string(&1, fun)) <> ") when " <> to_string(right, fun) + + fun.(ast, result) + end + + # Capture + def to_string({:&, _, [{:/, _, [{name, _, ctx}, arity]}]} = ast, fun) + when is_atom(name) and is_atom(ctx) and is_integer(arity) do + result = "&" <> Atom.to_string(name) <> "/" <> to_string(arity, fun) + fun.(ast, result) + end + + def to_string({:&, _, [{:/, _, [{{:., _, [mod, name]}, _, []}, arity]}]} = ast, fun) + when is_atom(name) and is_integer(arity) do + result = + "&" <> to_string(mod, fun) <> "." <> Atom.to_string(name) <> "/" <> to_string(arity, fun) + + fun.(ast, result) + end + + def to_string({:&, _, [arg]} = ast, fun) when not is_integer(arg) do + fun.(ast, "&(" <> to_string(arg, fun) <> ")") + end + + # left not in right + def to_string({:not, _, [{:in, _, [left, right]}]} = ast, fun) do + fun.(ast, to_string(left, fun) <> " not in " <> to_string(right, fun)) + end + + # Access + def to_string({{:., _, [Access, :get]}, _, [left, right]} = ast, fun) do + if op_expr?(left) do + fun.(ast, "(" <> to_string(left, fun) <> ")" <> to_string([right], fun)) + else + fun.(ast, to_string(left, fun) <> to_string([right], fun)) + end + end + + # foo.{bar, baz} + def to_string({{:., _, [left, :{}]}, _, args} = ast, fun) do + fun.(ast, to_string(left, fun) <> ".{" <> args_to_string(args, fun) <> "}") + end + + # All other calls + def to_string({{:., _, [left, _]} = target, meta, []} = ast, fun) do + to_string = call_to_string(target, fun) + + if is_tuple(left) && meta[:no_parens] do + fun.(ast, to_string) + else + fun.(ast, to_string <> "()") + end + end + + def to_string({target, _, args} = ast, fun) when is_list(args) do + with :error <- unary_call(ast, fun), + :error <- op_call(ast, fun), + :error <- sigil_call(ast, fun) do + {list, last} = split_last(args) + + result = + if kw_blocks?(last) do + case list do + [] -> call_to_string(target, fun) <> kw_blocks_to_string(last, fun) + _ -> call_to_string_with_args(target, list, fun) <> kw_blocks_to_string(last, fun) + end + else + call_to_string_with_args(target, args, fun) + end + + fun.(ast, result) + else + {:ok, value} -> value + end + end + + # Two-element tuples + def to_string({left, right}, fun) do + to_string({:{}, [], [left, right]}, fun) + end + + # Lists + def to_string(list, fun) when is_list(list) do + result = + cond do + list == [] -> + "[]" + + :io_lib.printable_list(list) -> + {escaped, _} = Identifier.escape(IO.chardata_to_string(list), ?') + IO.iodata_to_binary([?', escaped, ?']) + + Inspect.List.keyword?(list) -> + "[" <> kw_list_to_string(list, fun) <> "]" + + true -> + "[" <> Enum.map_join(list, ", ", &to_string(&1, fun)) <> "]" + end + + fun.(list, result) + end + + # All other structures + def to_string(other, fun) do + fun.(other, inspect_no_limit(other)) + end + + defp inspect_no_limit(value) do + Kernel.inspect(value, limit: :infinity, printable_limit: :infinity) + end + + defp bitpart_to_string({:"::", meta, [left, right]} = ast, fun) do + result = + if meta[:inferred_bitstring_spec] do + to_string(left, fun) + else + op_to_string(left, fun, :"::", :left) <> + "::" <> bitmods_to_string(right, fun, :"::", :right) + end + + fun.(ast, result) + end + + defp bitpart_to_string(ast, fun) do + to_string(ast, fun) + end + + defp bitmods_to_string({op, _, [left, right]} = ast, fun, _, _) when op in [:*, :-] do + result = + bitmods_to_string(left, fun, op, :left) <> + Atom.to_string(op) <> bitmods_to_string(right, fun, op, :right) + + fun.(ast, result) + end + + defp bitmods_to_string(other, fun, parent_op, side) do + op_to_string(other, fun, parent_op, side) end - def to_string({:__block__, _, _} = ast, fun) do - block = adjust_new_lines block_to_string(ast, fun), "\n " - fun.(ast, "(\n " <> block <> "\n)") + # Block keywords + kw_keywords = [:do, :rescue, :catch, :else, :after] + + defp kw_blocks?([{:do, _} | _] = kw) do + Enum.all?(kw, &match?({x, _} when x in unquote(kw_keywords), &1)) end - # Bits containers - def to_string({:<<>>, _, args} = ast, fun) do - fun.(ast, case Enum.map_join(args, ", ", &to_string(&1, fun)) do - "<" <> rest -> "<< <" <> rest <> " >>" - rest -> "<<" <> rest <> ">>" + defp kw_blocks?(_), do: false + + # Check if we have an interpolated string. + defp interpolated?({:<<>>, _, [_ | _] = parts}) do + Enum.all?(parts, fn + {:"::", _, [{{:., _, [Kernel, :to_string]}, _, [_]}, {:binary, _, _}]} -> true + binary when is_binary(binary) -> true + _ -> false end) end - # Tuple containers - def to_string({:{}, _, args} = ast, fun) do - tuple = "{" <> Enum.map_join(args, ", ", &to_string(&1, fun)) <> "}" - fun.(ast, tuple) + defp interpolated?(_) do + false end - # Map containers - def to_string({:%{}, _, args} = ast, fun) do - map = "%{" <> map_to_string(args, fun) <> "}" - fun.(ast, map) - end + defp interpolate(ast, fun), do: interpolate(ast, "\"", "\"", fun) - def to_string({:%, _, [structname, map]} = ast, fun) do - {:%{}, _, args} = map - struct = "%" <> to_string(structname, fun) <> "{" <> map_to_string(args, fun) <> "}" - fun.(ast, struct) + defp interpolate({:<<>>, _, [parts]}, left, right, _) when left in [~s["""\n], ~s['''\n]] do + <> end - # Fn keyword - def to_string({:fn, _, [{:->, _, [_, tuple]}] = arrow} = ast, fun) - when not is_tuple(tuple) or elem(tuple, 0) != :__block__ do - fun.(ast, "fn " <> arrow_to_string(arrow, fun) <> " end") + defp interpolate({:<<>>, _, parts}, left, right, fun) do + parts = + Enum.map_join(parts, "", fn + {:"::", _, [{{:., _, [Kernel, :to_string]}, _, [arg]}, {:binary, _, _}]} -> + "\#{" <> to_string(arg, fun) <> "}" + + binary when is_binary(binary) -> + escape_sigil(binary, left) + end) + + <> end - def to_string({:fn, _, [{:->, _, _}] = block} = ast, fun) do - fun.(ast, "fn " <> block_to_string(block, fun) <> "\nend") + defp escape_sigil(parts, "("), do: String.replace(parts, ")", ~S"\)") + defp escape_sigil(parts, "{"), do: String.replace(parts, "}", ~S"\}") + defp escape_sigil(parts, "["), do: String.replace(parts, "]", ~S"\]") + defp escape_sigil(parts, "<"), do: String.replace(parts, ">", ~S"\>") + defp escape_sigil(parts, delimiter), do: String.replace(parts, delimiter, "\\#{delimiter}") + + defp module_to_string(atom, _fun) when is_atom(atom) do + inspect_no_limit(atom) end - def to_string({:fn, _, block} = ast, fun) do - block = adjust_new_lines block_to_string(block, fun), "\n " - fun.(ast, "fn\n " <> block <> "\nend") + defp module_to_string({:&, _, [val]} = expr, fun) when not is_integer(val) do + "(" <> to_string(expr, fun) <> ")" end - # left -> right - def to_string([{:->, _, _}|_] = ast, fun) do - fun.(ast, "(" <> arrow_to_string(ast, fun, true) <> ")") + defp module_to_string({:fn, _, _} = expr, fun) do + "(" <> to_string(expr, fun) <> ")" end - # left when right - def to_string({:when, _, [left, right]} = ast, fun) do - if right != [] and Keyword.keyword?(right) do - right = kw_list_to_string(right, fun) + defp module_to_string({_, _, [_ | _] = args} = expr, fun) do + if kw_blocks?(List.last(args)) do + "(" <> to_string(expr, fun) <> ")" else - right = fun.(ast, op_to_string(right, fun, :when, :right)) + to_string(expr, fun) end - - fun.(ast, op_to_string(left, fun, :when, :left) <> " when " <> right) end - # Binary ops - def to_string({op, _, [left, right]} = ast, fun) when op in unquote(@binary_ops) do - fun.(ast, op_to_string(left, fun, op, :left) <> " #{op} " <> op_to_string(right, fun, op, :right)) + defp module_to_string(expr, fun) do + to_string(expr, fun) end - # Splat when - def to_string({:when, _, args} = ast, fun) do - {left, right} = :elixir_utils.split_last(args) - fun.(ast, "(" <> Enum.map_join(left, ", ", &to_string(&1, fun)) <> ") when " <> to_string(right, fun)) + defp unary_call({op, _, [arg]} = ast, fun) when is_atom(op) do + if operator?(op, 1) do + if op == :not or op_expr?(arg) do + {:ok, fun.(ast, Atom.to_string(op) <> "(" <> to_string(arg, fun) <> ")")} + else + {:ok, fun.(ast, Atom.to_string(op) <> to_string(arg, fun))} + end + else + :error + end end - # Unary ops - def to_string({unary, _, [{binary, _, [_, _]} = arg]} = ast, fun) - when unary in unquote(@unary_ops) and binary in unquote(@binary_ops) do - fun.(ast, Atom.to_string(unary) <> "(" <> to_string(arg, fun) <> ")") + defp unary_call(_, _) do + :error end - def to_string({:not, _, [arg]} = ast, fun) do - fun.(ast, "not " <> to_string(arg, fun)) + defp op_call({:"..//", _, [left, middle, right]} = ast, fun) do + left = op_to_string(left, fun, :.., :left) + middle = op_to_string(middle, fun, :.., :right) + right = op_to_string(right, fun, :"//", :right) + {:ok, fun.(ast, left <> ".." <> middle <> "//" <> right)} end - def to_string({op, _, [arg]} = ast, fun) when op in unquote(@unary_ops) do - fun.(ast, Atom.to_string(op) <> to_string(arg, fun)) + defp op_call({op, _, [left, right]} = ast, fun) when is_atom(op) do + if operator?(op, 2) do + left = op_to_string(left, fun, op, :left) + right = op_to_string(right, fun, op, :right) + op = if op in [:..], do: "#{op}", else: " #{op} " + {:ok, fun.(ast, left <> op <> right)} + else + :error + end end - # Access - def to_string({{:., _, [Access, :get]}, _, [left, right]} = ast, fun) do - fun.(ast, to_string(left, fun) <> to_string([right], fun)) + defp op_call(_, _) do + :error end - # All other calls - def to_string({target, _, args} = ast, fun) when is_list(args) do - {list, last} = :elixir_utils.split_last(args) - fun.(ast, case kw_blocks?(last) do - true -> call_to_string_with_args(target, list, fun) <> kw_blocks_to_string(last, fun) - false -> call_to_string_with_args(target, args, fun) - end) - end + defp sigil_call({sigil, meta, [{:<<>>, _, _} = parts, args]} = ast, fun) + when is_atom(sigil) and is_list(args) do + delimiter = Keyword.get(meta, :delimiter, "\"") + {left, right} = delimiter_pair(delimiter) - # Two-item tuples - def to_string({left, right}, fun) do - to_string({:{}, [], [left, right]}, fun) + case Atom.to_string(sigil) do + <<"sigil_", name>> when name >= ?A and name <= ?Z -> + args = sigil_args(args, fun) + {:<<>>, _, [binary]} = parts + formatted = <> + {:ok, fun.(ast, formatted)} + + <<"sigil_", name>> when name >= ?a and name <= ?z -> + args = sigil_args(args, fun) + formatted = "~" <> <> <> interpolate(parts, left, right, fun) <> args + {:ok, fun.(ast, formatted)} + + _ -> + :error + end end - # Lists - def to_string(list, fun) when is_list(list) do - fun.(list, cond do - list == [] -> - "[]" - :io_lib.printable_list(list) -> - "'" <> Inspect.BitString.escape(IO.chardata_to_string(list), ?') <> "'" - Keyword.keyword?(list) -> - "[" <> kw_list_to_string(list, fun) <> "]" - true -> - "[" <> Enum.map_join(list, ", ", &to_string(&1, fun)) <> "]" - end) + defp sigil_call(_other, _fun) do + :error end - # All other structures - def to_string(other, fun), do: fun.(other, inspect(other, [])) + defp delimiter_pair("["), do: {"[", "]"} + defp delimiter_pair("{"), do: {"{", "}"} + defp delimiter_pair("("), do: {"(", ")"} + defp delimiter_pair("<"), do: {"<", ">"} + defp delimiter_pair("\"\"\""), do: {"\"\"\"\n", "\"\"\""} + defp delimiter_pair("'''"), do: {"'''\n", "'''"} + defp delimiter_pair(str), do: {str, str} - # Block keywords - @kw_keywords [:do, :catch, :rescue, :after, :else] + defp sigil_args([], _fun), do: "" + defp sigil_args(args, fun), do: fun.(args, List.to_string(args)) - defp kw_blocks?([_|_] = kw) do - Enum.all?(kw, &match?({x, _} when x in unquote(@kw_keywords), &1)) + defp op_expr?(expr) do + case expr do + {op, _, [_, _]} -> operator?(op, 2) + {op, _, [_]} -> operator?(op, 1) + _ -> false + end end - defp kw_blocks?(_), do: false - - defp module_to_string(atom, _fun) when is_atom(atom), do: inspect(atom, []) - defp module_to_string(other, fun), do: call_to_string(other, fun) defp call_to_string(atom, _fun) when is_atom(atom), do: Atom.to_string(atom) - defp call_to_string({:., _, [arg]}, fun), do: module_to_string(arg, fun) <> "." - defp call_to_string({:., _, [left, right]}, fun), do: module_to_string(left, fun) <> "." <> call_to_string(right, fun) - defp call_to_string(other, fun), do: to_string(other, fun) + defp call_to_string({:., _, [arg]}, fun), do: module_to_string(arg, fun) <> "." + + defp call_to_string({:., _, [left, right]}, fun) when is_atom(right), + do: module_to_string(left, fun) <> "." <> call_to_string_for_atom(right) + + defp call_to_string({:., _, [left, right]}, fun), + do: module_to_string(left, fun) <> "." <> call_to_string(right, fun) + + defp call_to_string(other, fun), do: to_string(other, fun) defp call_to_string_with_args(target, args, fun) do target = call_to_string(target, fun) @@ -576,36 +1460,44 @@ defmodule Macro do target <> "(" <> args <> ")" end + defp call_to_string_for_atom(atom) do + Macro.inspect_atom(:remote_call, atom) + end + defp args_to_string(args, fun) do - {list, last} = :elixir_utils.split_last(args) + {list, last} = split_last(args) - if last != [] and Keyword.keyword?(last) do - args = Enum.map_join(list, ", ", &to_string(&1, fun)) - if list != [], do: args = args <> ", " - args <> kw_list_to_string(last, fun) + if last != [] and Inspect.List.keyword?(last) do + prefix = + case list do + [] -> "" + _ -> Enum.map_join(list, ", ", &to_string(&1, fun)) <> ", " + end + + prefix <> kw_list_to_string(last, fun) else Enum.map_join(args, ", ", &to_string(&1, fun)) end end defp kw_blocks_to_string(kw, fun) do - Enum.reduce(@kw_keywords, " ", fn(x, acc) -> + Enum.reduce(unquote(kw_keywords), " ", fn x, acc -> case Keyword.has_key?(kw, x) do - true -> acc <> kw_block_to_string(x, Keyword.get(kw, x), fun) + true -> acc <> kw_block_to_string(x, Keyword.get(kw, x), fun) false -> acc end end) <> "end" end defp kw_block_to_string(key, value, fun) do - block = adjust_new_lines block_to_string(value, fun), "\n " + block = adjust_new_lines(block_to_string(value, fun), "\n ") Atom.to_string(key) <> "\n " <> block <> "\n" end - defp block_to_string([{:->, _, _}|_] = block, fun) do - Enum.map_join(block, "\n", fn({:->, _, [left, right]}) -> + defp block_to_string([{:->, _, _} | _] = block, fun) do + Enum.map_join(block, "\n", fn {:->, _, [left, right]} -> left = comma_join_or_empty_paren(left, fun, false) - left <> "->\n " <> adjust_new_lines block_to_string(right, fun), "\n " + left <> "->\n " <> adjust_new_lines(block_to_string(right, fun), "\n ") end) end @@ -621,67 +1513,74 @@ defmodule Macro do defp map_to_string(list, fun) do cond do - Keyword.keyword?(list) -> kw_list_to_string(list, fun) + Inspect.List.keyword?(list) -> kw_list_to_string(list, fun) true -> map_list_to_string(list, fun) end end defp kw_list_to_string(list, fun) do Enum.map_join(list, ", ", fn {key, value} -> - atom_name = case Inspect.Atom.inspect(key) do - ":" <> rest -> rest - other -> other - end - atom_name <> ": " <> to_string(value, fun) + Macro.inspect_atom(:key, key) <> " " <> to_string(value, fun) end) end defp map_list_to_string(list, fun) do - Enum.map_join(list, ", ", fn {key, value} -> - to_string(key, fun) <> " => " <> to_string(value, fun) + Enum.map_join(list, ", ", fn + {key, value} -> to_string(key, fun) <> " => " <> to_string(value, fun) + other -> to_string(other, fun) end) end - defp parenthise(expr, fun) do + defp wrap_in_parenthesis(expr, fun) do "(" <> to_string(expr, fun) <> ")" end - defp op_to_string({op, _, [_, _]} = expr, fun, parent_op, side) when op in unquote(@binary_ops) do - {parent_assoc, parent_prec} = binary_op_props(parent_op) - {_, prec} = binary_op_props(op) - cond do - parent_prec < prec -> to_string(expr, fun) - parent_prec > prec -> parenthise(expr, fun) - true -> - # parent_prec == prec, so look at associativity. - if parent_assoc == side do - to_string(expr, fun) - else - parenthise(expr, fun) + defp op_to_string({op, _, [_, _]} = expr, fun, parent_op, side) when is_atom(op) do + case Identifier.binary_op(op) do + {_, prec} -> + {parent_assoc, parent_prec} = Identifier.binary_op(parent_op) + + cond do + parent_prec < prec -> to_string(expr, fun) + parent_prec > prec -> wrap_in_parenthesis(expr, fun) + parent_assoc == side -> to_string(expr, fun) + true -> wrap_in_parenthesis(expr, fun) end + + :error -> + to_string(expr, fun) end end defp op_to_string(expr, fun, _, _), do: to_string(expr, fun) defp arrow_to_string(pairs, fun, paren \\ false) do - Enum.map_join(pairs, "; ", fn({:->, _, [left, right]}) -> + Enum.map_join(pairs, "; ", fn {:->, _, [left, right]} -> left = comma_join_or_empty_paren(left, fun, paren) left <> "-> " <> to_string(right, fun) end) end - defp comma_join_or_empty_paren([], _fun, true), do: "() " + defp comma_join_or_empty_paren([], _fun, true), do: "() " defp comma_join_or_empty_paren([], _fun, false), do: "" defp comma_join_or_empty_paren(left, fun, _) do Enum.map_join(left, ", ", &to_string(&1, fun)) <> " " end + defp split_last([]) do + {[], []} + end + + defp split_last(args) do + {left, [right]} = Enum.split(args, -1) + {left, right} + end + defp adjust_new_lines(block, replacement) do for <>, into: "" do case x == ?\n do - true -> replacement + true -> replacement false -> <> end end @@ -694,13 +1593,15 @@ defmodule Macro do * Macros (local or remote) * Aliases are expanded (if possible) and return atoms - * Pseudo-variables (`__ENV__`, `__MODULE__` and `__DIR__`) + * Compilation environment macros (`__CALLER__/0`, `__DIR__/0`, `__ENV__/0` and `__MODULE__/0`) * Module attributes reader (`@foo`) If the expression cannot be expanded, it returns the expression - itself. Notice that `expand_once/2` performs the expansion just - once and it is not recursive. Check `expand/2` for expansion - until the node can no longer be expanded. + itself. This function does not traverse the AST, only the root + node is expanded. + + `expand_once/2` performs the expansion just once. Check `expand/2` + to perform expansion until the node can no longer be expanded. ## Examples @@ -712,7 +1613,7 @@ defmodule Macro do Consider the implementation below: defmacro defmodule_with_length(name, do: block) do - length = length(Atom.to_char_list(name)) + length = length(Atom.to_charlist(name)) quote do defmodule unquote(name) do @@ -729,13 +1630,13 @@ defmodule Macro do end The compilation will fail because `My.Module` when quoted - is not an atom, but a syntax tree as follow: + is not an atom, but a syntax tree as follows: {:__aliases__, [], [:My, :Module]} That said, we need to expand the aliases node above to an atom, so we can retrieve its length. Expanding the node is - not straight-forward because we also need to expand the + not straightforward because we also need to expand the caller aliases. For example: alias MyHelpers, as: My @@ -752,7 +1653,7 @@ defmodule Macro do defmacro defmodule_with_length(name, do: block) do expanded = Macro.expand(name, __CALLER__) - length = length(Atom.to_char_list(expanded)) + length = length(Atom.to_charlist(expanded)) quote do defmodule unquote(name) do @@ -763,46 +1664,43 @@ defmodule Macro do end """ + @spec expand_once(t(), Macro.Env.t()) :: t() def expand_once(ast, env) do elem(do_expand_once(ast, env), 0) end - defp do_expand_once({:__aliases__, _, _} = original, env) do - case :elixir_aliases.expand(original, env.aliases, env.macro_aliases, env.lexical_tracker) do + defp do_expand_once({:__aliases__, meta, _} = original, env) do + case :elixir_aliases.expand_or_concat(original, env) do receiver when is_atom(receiver) -> - :elixir_lexical.record_remote(receiver, env.lexical_tracker) + :elixir_env.trace({:alias_reference, meta, receiver}, env) {receiver, true} + aliases -> - aliases = for alias <- aliases, do: elem(do_expand_once(alias, env), 0) + aliases = :lists.map(&elem(do_expand_once(&1, env), 0), aliases) case :lists.all(&is_atom/1, aliases) do true -> receiver = :elixir_aliases.concat(aliases) - :elixir_lexical.record_remote(receiver, env.lexical_tracker) + :elixir_env.trace({:alias_reference, meta, receiver}, env) {receiver, true} + false -> {original, false} end end end - # Expand @ calls - defp do_expand_once({:@, _, [{name, _, args}]} = original, env) when is_atom(args) or args == [] do - case (module = env.module) && Module.open?(module) do - true -> {escape(Module.get_attribute(module, name)), true} - false -> {original, false} - end - end + # Expand compilation environment macros + defp do_expand_once({:__MODULE__, _, atom}, env) when is_atom(atom), do: {env.module, true} - # Expand pseudo-variables - defp do_expand_once({:__MODULE__, _, atom}, env) when is_atom(atom), - do: {env.module, true} defp do_expand_once({:__DIR__, _, atom}, env) when is_atom(atom), do: {:filename.dirname(env.file), true} + defp do_expand_once({:__ENV__, _, atom}, env) when is_atom(atom), do: {{:%{}, [], Map.to_list(env)}, true} - defp do_expand_once({{:., _, [{:__ENV__, _, atom}, field]}, _, []} = original, env) when - is_atom(atom) and is_atom(field) do + + defp do_expand_once({{:., _, [{:__ENV__, _, atom}, field]}, _, []} = original, env) + when is_atom(atom) and is_atom(field) do if Map.has_key?(env, field) do {Map.get(env, field), true} else @@ -810,42 +1708,46 @@ defmodule Macro do end end - # Expand possible macro import invocation - defp do_expand_once({atom, meta, context} = original, env) - when is_atom(atom) and is_list(meta) and is_atom(context) do - if :lists.member({atom, Keyword.get(meta, :counter, context)}, env.vars) do - {original, false} - else - case do_expand_once({atom, meta, []}, env) do - {_, true} = exp -> exp - {_, false} -> {original, false} - end - end + defp do_expand_once({atom, meta, context} = original, _env) + when is_atom(atom) and is_list(meta) and is_atom(context) do + {original, false} end defp do_expand_once({atom, meta, args} = original, env) - when is_atom(atom) and is_list(args) and is_list(meta) do + when is_atom(atom) and is_list(args) and is_list(meta) do arity = length(args) - if :elixir_import.special_form(atom, arity) do + if special_form?(atom, arity) do {original, false} else module = env.module - extra = if function_exported?(module, :__info__, 1) do - [{module, module.__info__(:macros)}] - else - [] - end - expand = :elixir_dispatch.expand_import(meta, {atom, length(args)}, args, - env, extra) + extra = + if function_exported?(module, :__info__, 1) do + [{module, module.__info__(:macros)}] + else + [] + end + + s = :elixir_env.env_to_ex(env) + + expand = + :elixir_dispatch.expand_import(meta, {atom, length(args)}, args, s, env, extra, true) case expand do {:ok, receiver, quoted} -> - next = :elixir_counter.next + next = :elixir_module.next_counter(module) {:elixir_quote.linify_with_context_counter(0, {receiver, next}, quoted), true} + + {:ok, Kernel, op, [arg]} when op in [:+, :-] -> + case expand_once(arg, env) do + integer when is_integer(integer) -> {apply(Kernel, op, [integer]), true} + _ -> {original, false} + end + {:ok, _receiver, _name, _args} -> {original, false} + :error -> {original, false} end @@ -857,14 +1759,19 @@ defmodule Macro do {receiver, _} = do_expand_once(left, env) case is_atom(receiver) do - false -> {original, false} - true -> - expand = :elixir_dispatch.expand_require(meta, receiver, {right, length(args)}, args, env) + false -> + {original, false} + + true -> + s = :elixir_env.env_to_ex(env) + name_arity = {right, length(args)} + expand = :elixir_dispatch.expand_require(meta, receiver, name_arity, args, s, env) case expand do {:ok, receiver, quoted} -> - next = :elixir_counter.next + next = :elixir_module.next_counter(env.module) {:elixir_quote.linify_with_context_counter(0, {receiver, next}, quoted), true} + :error -> {original, false} end @@ -874,22 +1781,491 @@ defmodule Macro do # Anything else is just returned defp do_expand_once(other, _env), do: {other, false} + @doc """ + Returns `true` if the given name and arity is a special form. + """ + @doc since: "1.7.0" + @spec special_form?(name :: atom(), arity()) :: boolean() + def special_form?(name, arity) when is_atom(name) and is_integer(arity) do + :elixir_import.special_form(name, arity) + end + + @doc """ + Returns `true` if the given name and arity is an operator. + + ## Examples + + iex> Macro.operator?(:not_an_operator, 3) + false + iex> Macro.operator?(:.., 0) + true + iex> Macro.operator?(:+, 1) + true + iex> Macro.operator?(:++, 2) + true + iex> Macro.operator?(:..//, 3) + true + + """ + @doc since: "1.7.0" + @spec operator?(name :: atom(), arity()) :: boolean() + def operator?(name, arity) + + def operator?(:"..//", 3), + do: true + + # Code.Identifier treats :// as a binary operator for precedence + # purposes but it isn't really one, so we explicitly skip it. + def operator?(name, 2) when is_atom(name), + do: Identifier.binary_op(name) != :error and name != :"//" + + def operator?(name, 1) when is_atom(name), + do: Identifier.unary_op(name) != :error + + def operator?(:.., 0), + do: true + + def operator?(name, arity) when is_atom(name) and is_integer(arity), do: false + + @doc """ + Returns `true` if the given quoted expression represents a quoted literal. + + Atoms and numbers are always literals. Binaries, lists, tuples, + maps, and structs are only literals if all of their terms are also literals. + + ## Examples + + iex> Macro.quoted_literal?(quote(do: "foo")) + true + iex> Macro.quoted_literal?(quote(do: {"foo", 1})) + true + iex> Macro.quoted_literal?(quote(do: {"foo", 1, :baz})) + true + iex> Macro.quoted_literal?(quote(do: %{foo: "bar"})) + true + iex> Macro.quoted_literal?(quote(do: %URI{path: "/"})) + true + iex> Macro.quoted_literal?(quote(do: URI.parse("/"))) + false + iex> Macro.quoted_literal?(quote(do: {foo, var})) + false + + """ + @doc since: "1.7.0" + @spec quoted_literal?(t) :: boolean + def quoted_literal?(term) + + def quoted_literal?({:__aliases__, _, args}), + do: quoted_literal?(args) + + def quoted_literal?({:%, _, [left, right]}), + do: quoted_literal?(left) and quoted_literal?(right) + + def quoted_literal?({:%{}, _, args}), do: quoted_literal?(args) + def quoted_literal?({:{}, _, args}), do: quoted_literal?(args) + def quoted_literal?({left, right}), do: quoted_literal?(left) and quoted_literal?(right) + def quoted_literal?(list) when is_list(list), do: Enum.all?(list, "ed_literal?/1) + def quoted_literal?(term), do: is_atom(term) or is_number(term) or is_binary(term) + @doc """ Receives an AST node and expands it until it can no longer be expanded. + Note this function does not traverse the AST, only the root + node is expanded. + This function uses `expand_once/2` under the hood. Check - `expand_once/2` for more information and exmaples. + it out for more information and examples. + """ + @spec expand(t(), Macro.Env.t()) :: t() + def expand(ast, env) do + expand_until({ast, true}, env) + end + + defp expand_until({ast, true}, env) do + expand_until(do_expand_once(ast, env), env) + end + + defp expand_until({ast, false}, _env) do + ast + end + + @doc """ + Converts the given argument to a string with the underscore-slash format. + + The argument must either be an atom or a string. + If an atom is given, it is assumed to be an Elixir module, + so it is converted to a string and then processed. + + This function was designed to format language identifiers/tokens with the underscore-slash format, + that's why it belongs to the `Macro` module. Do not use it as a general + mechanism for underscoring strings as it does not support Unicode or + characters that are not valid in Elixir identifiers. + + ## Examples + + iex> Macro.underscore("FooBar") + "foo_bar" + + iex> Macro.underscore("Foo.Bar") + "foo/bar" + + iex> Macro.underscore(Foo.Bar) + "foo/bar" + + In general, `underscore` can be thought of as the reverse of + `camelize`, however, in some cases formatting may be lost: + + iex> Macro.underscore("SAPExample") + "sap_example" + + iex> Macro.camelize("sap_example") + "SapExample" + + iex> Macro.camelize("hello_10") + "Hello10" + + iex> Macro.camelize("foo/bar") + "Foo.Bar" + + """ + @spec underscore(module() | atom() | String.t()) :: String.t() + def underscore(atom_or_string) + + def underscore(atom) when is_atom(atom) do + "Elixir." <> rest = Atom.to_string(atom) + underscore(rest) + end + + def underscore(<>) do + <> <> do_underscore(t, h) + end + + def underscore("") do + "" + end + + defp do_underscore(<>, _) + when h >= ?A and h <= ?Z and not (t >= ?A and t <= ?Z) and not (t >= ?0 and t <= ?9) and + t != ?. and t != ?_ do + <> <> do_underscore(rest, t) + end + + defp do_underscore(<>, prev) + when h >= ?A and h <= ?Z and not (prev >= ?A and prev <= ?Z) and prev != ?_ do + <> <> do_underscore(t, h) + end + + defp do_underscore(<>, _) do + <> <> underscore(t) + end + + defp do_underscore(<>, _) do + <> <> do_underscore(t, h) + end + + defp do_underscore(<<>>, _) do + <<>> + end + + @doc """ + Converts the given string to CamelCase format. + + This function was designed to camelize language identifiers/tokens, + that's why it belongs to the `Macro` module. Do not use it as a general + mechanism for camelizing strings as it does not support Unicode or + characters that are not valid in Elixir identifiers. + + ## Examples + + iex> Macro.camelize("foo_bar") + "FooBar" + + iex> Macro.camelize("foo/bar") + "Foo.Bar" + + If uppercase characters are present, they are not modified in any way + as a mechanism to preserve acronyms: + + iex> Macro.camelize("API.V1") + "API.V1" + iex> Macro.camelize("API_SPEC") + "API_SPEC" + + """ + @spec camelize(String.t()) :: String.t() + def camelize(string) + + def camelize(""), do: "" + def camelize(<>), do: camelize(t) + def camelize(<>), do: <> <> do_camelize(t) + + defp do_camelize(<>), do: do_camelize(<>) + + defp do_camelize(<>) when h >= ?a and h <= ?z, + do: <> <> do_camelize(t) + + defp do_camelize(<>) when h >= ?0 and h <= ?9, do: <> <> do_camelize(t) + defp do_camelize(<>), do: <<>> + defp do_camelize(<>), do: <> <> camelize(t) + defp do_camelize(<>), do: <> <> do_camelize(t) + defp do_camelize(<<>>), do: <<>> + + defp to_upper_char(char) when char >= ?a and char <= ?z, do: char - 32 + defp to_upper_char(char), do: char + + defp to_lower_char(char) when char >= ?A and char <= ?Z, do: char + 32 + defp to_lower_char(char), do: char + + ## Atom handling + + @doc """ + Classifies a runtime `atom` based on its possible AST placement. + + It returns one of the following atoms: + + * `:alias` - the atom represents an alias + + * `:identifier` - the atom can be used as a variable or local function + call (as well as be an unquoted atom) + + * `:unquoted` - the atom can be used in its unquoted form, + includes operators and atoms with `@` in them + + * `:quoted` - all other atoms which can only be used in their quoted form + + Most operators are going to be `:unquoted`, such as `:+`, with + some exceptions returning `:quoted` due to ambiguity, such as + `:"::"`. Use `operator?/2` to check if a given atom is an operator. + + ## Examples + + iex> Macro.classify_atom(:foo) + :identifier + iex> Macro.classify_atom(Foo) + :alias + iex> Macro.classify_atom(:foo@bar) + :unquoted + iex> Macro.classify_atom(:+) + :unquoted + iex> Macro.classify_atom(:Foo) + :unquoted + iex> Macro.classify_atom(:"with spaces") + :quoted + + """ + @doc since: "1.14.0" + @spec classify_atom(atom) :: :alias | :identifier | :quoted | :unquoted + def classify_atom(atom) do + case inner_classify(atom) do + :alias -> :alias + :identifier -> :identifier + type when type in [:unquoted_operator, :not_callable] -> :unquoted + _ -> :quoted + end + end + + @doc ~S""" + Inspects `atom` according to different source formats. + + The atom can be inspected according to the three different + formats it appears in the AST: as a literal (`:literal`), + as a key (`:key`), or as the function name of a remote call + (`:remote_call`). + + ## Examples + + ### As a literal + + Literals include regular atoms, quoted atoms, operators, + aliases, and the special `nil`, `true`, and `false` atoms. + + iex> Macro.inspect_atom(:literal, nil) + "nil" + iex> Macro.inspect_atom(:literal, :foo) + ":foo" + iex> Macro.inspect_atom(:literal, :<>) + ":<>" + iex> Macro.inspect_atom(:literal, :Foo) + ":Foo" + iex> Macro.inspect_atom(:literal, Foo.Bar) + "Foo.Bar" + iex> Macro.inspect_atom(:literal, :"with spaces") + ":\"with spaces\"" + + ### As a key + + Inspect an atom as a key of a keyword list or a map. + + iex> Macro.inspect_atom(:key, :foo) + "foo:" + iex> Macro.inspect_atom(:key, :<>) + "<>:" + iex> Macro.inspect_atom(:key, :Foo) + "Foo:" + iex> Macro.inspect_atom(:key, :"with spaces") + "\"with spaces\":" + + ### As a remote call + + Inspect an atom the function name of a remote call. + + iex> Macro.inspect_atom(:remote_call, :foo) + "foo" + iex> Macro.inspect_atom(:remote_call, :<>) + "<>" + iex> Macro.inspect_atom(:remote_call, :Foo) + "\"Foo\"" + iex> Macro.inspect_atom(:remote_call, :"with spaces") + "\"with spaces\"" + """ - def expand(tree, env) do - expand_until({tree, true}, env) + @doc since: "1.14.0" + @spec inspect_atom(:literal | :key | :remote_call, atom) :: binary + def inspect_atom(source_format, atom) + + def inspect_atom(:literal, atom) when is_nil(atom) or is_boolean(atom) do + Atom.to_string(atom) + end + + def inspect_atom(:literal, atom) when is_atom(atom) do + binary = Atom.to_string(atom) + + case classify_atom(atom) do + :alias -> + case binary do + binary when binary in ["Elixir", "Elixir.Elixir"] -> binary + "Elixir.Elixir." <> _rest -> binary + "Elixir." <> rest -> rest + end + + :quoted -> + {escaped, _} = Code.Identifier.escape(binary, ?") + IO.iodata_to_binary([?:, ?", escaped, ?"]) + + _ -> + ":" <> binary + end + end + + def inspect_atom(:key, atom) when is_atom(atom) do + binary = Atom.to_string(atom) + + case classify_atom(atom) do + :alias -> + IO.iodata_to_binary([?", binary, ?", ?:]) + + :quoted -> + {escaped, _} = Code.Identifier.escape(binary, ?") + IO.iodata_to_binary([?", escaped, ?", ?:]) + + _ -> + IO.iodata_to_binary([binary, ?:]) + end + end + + def inspect_atom(:remote_call, atom) when is_atom(atom) do + binary = Atom.to_string(atom) + + case inner_classify(atom) do + type when type in [:identifier, :unquoted_operator, :quoted_operator] -> + binary + + type -> + escaped = + if type in [:not_callable, :alias] do + binary + else + elem(Code.Identifier.escape(binary, ?"), 0) + end + + IO.iodata_to_binary([?", escaped, ?"]) + end end - defp expand_until({tree, true}, env) do - expand_until(do_expand_once(tree, env), env) + # Classifies the given atom into one of the following categories: + # + # * `:alias` - a valid Elixir alias, like `Foo`, `Foo.Bar` and so on + # + # * `:identifier` - an atom that can be used as a variable/local call; + # this category includes identifiers like `:foo` + # + # * `:unquoted_operator` - all callable operators, such as `:<>`. Note + # operators such as `:..` are not callable because of ambiguity + # + # * `:quoted_operator` - callable operators that must be wrapped in quotes when + # defined as an atom. For example, `::` must be written as `:"::"` to avoid + # the ambiguity between the atom and the keyword identifier + # + # * `:not_callable` - an atom that cannot be used as a function call after the + # `.` operator. Those are typically AST nodes that are special forms (such as + # `:%{}` and `:<<>>>`) as well as nodes that are ambiguous in calls (such as + # `:..` and `:...`). This category also includes atoms like `:Foo`, since + # they are valid identifiers but they need quotes to be used in function + # calls (`Foo."Bar"`) + # + # * `:other` - any other atom (these are usually escaped when inspected, like + # `:"foo and bar"`) + # + defp inner_classify(atom) when is_atom(atom) do + cond do + atom in [:%, :%{}, :{}, :<<>>, :..., :.., :., :"..//", :->] -> + :not_callable + + # <|>, ^^^, and ~~~ are deprecated + atom in [:"::", :"^^^", :"~~~", :"<|>"] -> + :quoted_operator + + operator?(atom, 1) or operator?(atom, 2) -> + :unquoted_operator + + true -> + charlist = Atom.to_charlist(atom) + + if valid_alias?(charlist) do + :alias + else + case :elixir_config.identifier_tokenizer().tokenize(charlist) do + {kind, _acc, [], _, _, special} -> + cond do + kind != :identifier or :lists.member(:at, special) -> + :not_callable + + # identifier_tokenizer used to return errors for non-nfc, but + # now it nfc-normalizes everything. However, lack of nfc is + # still a good reason to quote an atom when printing. + :lists.member(:nfkc, special) -> + :other + + true -> + :identifier + end + + _ -> + :other + end + end + end + end + + defp valid_alias?('Elixir' ++ rest), do: valid_alias_piece?(rest) + defp valid_alias?(_other), do: false + + defp valid_alias_piece?([?., char | rest]) when char >= ?A and char <= ?Z, + do: valid_alias_piece?(trim_leading_while_valid_identifier(rest)) + + defp valid_alias_piece?([]), do: true + defp valid_alias_piece?(_other), do: false + + defp trim_leading_while_valid_identifier([char | rest]) + when char >= ?a and char <= ?z + when char >= ?A and char <= ?Z + when char >= ?0 and char <= ?9 + when char == ?_ do + trim_leading_while_valid_identifier(rest) end - defp expand_until({tree, false}, _env) do - tree + defp trim_leading_while_valid_identifier(other) do + other end end diff --git a/lib/elixir/lib/macro/env.ex b/lib/elixir/lib/macro/env.ex index 98c5ed341c4..263c4ee8d04 100644 --- a/lib/elixir/lib/macro/env.ex +++ b/lib/elixir/lib/macro/env.ex @@ -3,112 +3,312 @@ defmodule Macro.Env do A struct that holds compile time environment information. The current environment can be accessed at any time as - `__ENV__`. Inside macros, the caller environment can be - accessed as `__CALLER__`. It contains the following fields: + `__ENV__/0`. Inside macros, the caller environment can be + accessed as `__CALLER__/0`. + + An instance of `Macro.Env` must not be modified by hand. If you need to + create a custom environment to pass to `Code.eval_quoted/3`, use the + following trick: + + def make_custom_env do + import SomeModule, only: [some_function: 2] + alias A.B.C + __ENV__ + end + + You may then call `make_custom_env()` to get a struct with the desired + imports and aliases included. + + It contains the following fields: - * `module` - the current module name - * `file` - the current file name as a binary - * `line` - the current line as an integer - * `function` - a tuple as `{atom, integer`}, where the first - element is the function name and the seconds its arity; returns - `nil` if not inside a function * `context` - the context of the environment; it can be `nil` - (default context), inside a guard or inside an assign - * `aliases` - a list of two item tuples, where the first - item is the aliased name and the second the actual name - * `requires` - the list of required modules - * `functions` - a list of functions imported from each module - * `macros` - a list of macros imported from each module - * `macro_aliases` - a list of aliases defined inside the current macro + (default context), `:guard` (inside a guard) or `:match` (inside a match) * `context_modules` - a list of modules defined in the current context - * `vars` - a list keeping all defined variables as `{var, context}` - * `export_vars` - a list keeping all variables to be exported in a - construct (may be `nil`) - * `lexical_tracker` - PID of the lexical tracker which is responsible to - keep user info - * `local` - the module to expand local functions to + * `file` - the current file name as a binary + * `function` - a tuple as `{atom, integer}`, where the first + element is the function name and the second its arity; returns + `nil` if not inside a function + * `line` - the current line as an integer + * `module` - the current module name + + The following fields are private to Elixir's macro expansion mechanism and + must not be accessed directly: + + * `aliases` + * `functions` + * `macro_aliases` + * `macros` + * `lexical_tracker` + * `requires` + * `tracers` + * `versioned_vars` + """ - @type name_arity :: {atom, non_neg_integer} - @type file :: binary - @type line :: non_neg_integer - @type aliases :: [{module, module}] - @type macro_aliases :: [{module, {integer, module}}] @type context :: :match | :guard | nil - @type requires :: [module] - @type functions :: [{module, [name_arity]}] - @type macros :: [{module, [name_arity]}] @type context_modules :: [module] - @type vars :: [{atom, atom | non_neg_integer}] - @type export_vars :: vars | nil - @type lexical_tracker :: pid - @type local :: module | nil - - @type t :: %{__struct__: __MODULE__, - module: module, - file: file, - line: line, - function: name_arity | nil, - context: context, - requires: requires, - aliases: aliases, - functions: functions, - macros: macros, - macro_aliases: aliases, - context_modules: context_modules, - vars: vars, - export_vars: export_vars, - lexical_tracker: lexical_tracker, - local: local} + @type file :: binary + @type line :: non_neg_integer + @type name_arity :: {atom, arity} + @type variable :: {atom, atom | term} + @typep aliases :: [{module, module}] + @typep functions :: [{module, [name_arity]}] + @typep lexical_tracker :: pid | nil + @typep macro_aliases :: [{module, {term, module}}] + @typep macros :: [{module, [name_arity]}] + @typep requires :: [module] + @typep tracers :: [module] + @typep versioned_vars :: %{optional(variable) => var_version :: non_neg_integer} + + @type t :: %{ + __struct__: __MODULE__, + aliases: aliases, + context: context, + context_modules: context_modules, + file: file, + function: name_arity | nil, + functions: functions, + lexical_tracker: lexical_tracker, + line: line, + macro_aliases: macro_aliases, + macros: macros, + module: module, + requires: requires, + tracers: tracers, + versioned_vars: versioned_vars + } + + # Define the __struct__ callbacks by hand for bootstrap reasons. + @doc false def __struct__ do - %{__struct__: __MODULE__, - module: nil, + %{ + __struct__: __MODULE__, + aliases: [], + context: nil, + context_modules: [], file: "nofile", - line: 0, function: nil, - context: nil, - requires: [], - aliases: [], functions: [], - macros: [], - macro_aliases: [], - context_modules: [], - vars: [], - export_vars: nil, lexical_tracker: nil, - local: nil} + line: 0, + macro_aliases: [], + macros: [], + module: nil, + requires: [], + tracers: [], + versioned_vars: %{} + } + end + + @doc false + def __struct__(kv) do + Enum.reduce(kv, __struct__(), fn {k, v}, acc -> :maps.update(k, v, acc) end) + end + + @doc """ + Prunes compile information from the environment. + + This happens when the environment is captured at compilation + time, for example, in the module body, and then used to + evaluate code after the module has been defined. + """ + @doc since: "1.14.0" + @spec prune_compile_info(t) :: t + def prune_compile_info(env) do + %{env | lexical_tracker: nil, tracers: []} + end + + @doc """ + Returns a list of variables in the current environment. + + Each variable is identified by a tuple of two elements, + where the first element is the variable name as an atom + and the second element is its context, which may be an + atom or an integer. + """ + @doc since: "1.7.0" + @spec vars(t) :: [variable] + def vars(env) + + def vars(%{__struct__: Macro.Env, versioned_vars: vars}) do + Map.keys(vars) + end + + @doc """ + Checks if a variable belongs to the environment. + + ## Examples + + iex> x = 13 + iex> x + 13 + iex> Macro.Env.has_var?(__ENV__, {:x, nil}) + true + iex> Macro.Env.has_var?(__ENV__, {:unknown, nil}) + false + + """ + @doc since: "1.7.0" + @spec has_var?(t, variable) :: boolean() + def has_var?(env, var) + + def has_var?(%{__struct__: Macro.Env, versioned_vars: vars}, var) do + Map.has_key?(vars, var) end @doc """ Returns a keyword list containing the file and line information as keys. """ + @spec location(t) :: keyword + def location(env) + def location(%{__struct__: Macro.Env, file: file, line: line}) do [file: file, line: line] end + @doc """ + Fetches the alias for the given atom. + + Returns `{:ok, alias}` if the alias exists, `:error` + otherwise. + + ## Examples + + iex> alias Foo.Bar, as: Baz + iex> Baz + Foo.Bar + iex> Macro.Env.fetch_alias(__ENV__, :Baz) + {:ok, Foo.Bar} + iex> Macro.Env.fetch_alias(__ENV__, :Unknown) + :error + + """ + @doc since: "1.13.0" + @spec fetch_alias(t, atom) :: {:ok, atom} | :error + def fetch_alias(%{__struct__: Macro.Env, aliases: aliases}, atom) when is_atom(atom), + do: Keyword.fetch(aliases, :"Elixir.#{atom}") + + @doc """ + Fetches the macro alias for the given atom. + + Returns `{:ok, macro_alias}` if the alias exists, `:error` + otherwise. + + A macro alias is only used inside quoted expansion. See + `fetch_alias/2` for a more general example. + """ + @doc since: "1.13.0" + @spec fetch_macro_alias(t, atom) :: {:ok, atom} | :error + def fetch_macro_alias(%{__struct__: Macro.Env, macro_aliases: aliases}, atom) + when is_atom(atom), + do: Keyword.fetch(aliases, :"Elixir.#{atom}") + + @doc """ + Returns the modules from which the given `{name, arity}` was + imported. + + It returns a list of two element tuples in the shape of + `{:function | :macro, module}`. The elements in the list + are in no particular order and the order is not guaranteed. + + ## Examples + + iex> Macro.Env.lookup_import(__ENV__, {:duplicate, 2}) + [] + iex> import Tuple, only: [duplicate: 2], warn: false + iex> Macro.Env.lookup_import(__ENV__, {:duplicate, 2}) + [{:function, Tuple}] + iex> import List, only: [duplicate: 2], warn: false + iex> Macro.Env.lookup_import(__ENV__, {:duplicate, 2}) + [{:function, List}, {:function, Tuple}] + + iex> Macro.Env.lookup_import(__ENV__, {:def, 1}) + [{:macro, Kernel}] + + """ + @doc since: "1.13.0" + @spec lookup_import(t, name_arity) :: [{:function | :macro, module}] + def lookup_import( + %{__struct__: Macro.Env, functions: functions, macros: macros}, + {name, arity} = pair + ) + when is_atom(name) and is_integer(arity) do + f = for {mod, pairs} <- functions, :ordsets.is_element(pair, pairs), do: {:function, mod} + m = for {mod, pairs} <- macros, :ordsets.is_element(pair, pairs), do: {:macro, mod} + f ++ m + end + + @doc """ + Returns true if the given module has been required. + + ## Examples + + iex> Macro.Env.required?(__ENV__, Integer) + false + iex> require Integer + iex> Macro.Env.required?(__ENV__, Integer) + true + + iex> Macro.Env.required?(__ENV__, Kernel) + true + """ + @doc since: "1.13.0" + @spec required?(t, module) :: boolean + def required?(%{__struct__: Macro.Env, requires: requires}, mod) when is_atom(mod), + do: mod in requires + + @doc """ + Prepend a tracer to the list of tracers in the environment. + + ## Examples + + Macro.Env.prepend_tracer(__ENV__, MyCustomTracer) + + """ + @doc since: "1.13.0" + @spec prepend_tracer(t, module) :: t + def prepend_tracer(%{__struct__: Macro.Env, tracers: tracers} = env, tracer) do + %{env | tracers: [tracer | tracers]} + end + + @doc """ + Returns a `Macro.Env` in the match context. + """ + @spec to_match(t) :: t + def to_match(%{__struct__: Macro.Env} = env) do + %{env | context: :match} + end + @doc """ Returns whether the compilation environment is currently inside a guard. """ + @spec in_guard?(t) :: boolean + def in_guard?(env) def in_guard?(%{__struct__: Macro.Env, context: context}), do: context == :guard @doc """ Returns whether the compilation environment is currently inside a match clause. """ + @spec in_match?(t) :: boolean + def in_match?(env) def in_match?(%{__struct__: Macro.Env, context: context}), do: context == :match @doc """ Returns the environment stacktrace. """ + @spec stacktrace(t) :: list def stacktrace(%{__struct__: Macro.Env} = env) do cond do - nil?(env.module) -> + is_nil(env.module) -> [{:elixir_compiler, :__FILE__, 1, relative_location(env)}] - nil?(env.function) -> + + is_nil(env.function) -> [{env.module, :__MODULE__, 0, relative_location(env)}] + true -> {name, arity} = env.function [{env.module, name, arity, relative_location(env)}] @@ -116,6 +316,6 @@ defmodule Macro.Env do end defp relative_location(env) do - [file: Path.relative_to_cwd(env.file), line: env.line] + [file: String.to_charlist(Path.relative_to_cwd(env.file)), line: env.line] end end diff --git a/lib/elixir/lib/map.ex b/lib/elixir/lib/map.ex index d93e88f194b..e2bd11f0b4d 100644 --- a/lib/elixir/lib/map.ex +++ b/lib/elixir/lib/map.ex @@ -1,42 +1,1113 @@ defmodule Map do @moduledoc """ - A Dict implementation that works on maps. + Maps are the "go to" key-value data structure in Elixir. - Maps are key-value stores where keys are compared using - the match operator (`===`). Maps can be created with - the `%{}` special form defined in the `Kernel.SpecialForms` - module. + Maps can be created with the `%{}` syntax, and key-value pairs can be + expressed as `key => value`: + + iex> %{} + %{} + iex> %{"one" => :two, 3 => "four"} + %{3 => "four", "one" => :two} + + Key-value pairs in a map do not follow any order (that's why the printed map + in the example above has a different order than the map that was created). + + Maps do not impose any restriction on the key type: anything can be a key in a + map. As a key-value structure, maps do not allow duplicated keys. Keys are + compared using the exact-equality operator (`===/2`). If colliding keys are defined + in a map literal, the last one prevails. + + When the key in a key-value pair is an atom, the `key: value` shorthand syntax + can be used (as in many other special forms): + + iex> %{a: 1, b: 2} + %{a: 1, b: 2} + + If you want to mix the shorthand syntax with `=>`, the shorthand syntax must come + at the end: + + iex> %{"hello" => "world", a: 1, b: 2} + %{:a => 1, :b => 2, "hello" => "world"} + + Keys in maps can be accessed through some of the functions in this module + (such as `Map.get/3` or `Map.fetch/2`) or through the `map[]` syntax provided + by the `Access` module: + + iex> map = %{a: 1, b: 2} + iex> Map.fetch(map, :a) + {:ok, 1} + iex> map[:b] + 2 + iex> map["non_existing_key"] + nil + + To access atom keys, one may also use the `map.key` notation. Note that `map.key` + will raise a `KeyError` if the `map` doesn't contain the key `:key`, compared to + `map[:key]`, that would return `nil`. + + map = %{foo: "bar", baz: "bong"} + map.foo + #=> "bar" + map.non_existing_key + ** (KeyError) key :non_existing_key not found in: %{baz: "bong", foo: "bar"} + + > Note: do not add parens when accessing fields, such as in `data.key()`. + > If parenthesis are used, Elixir will expect `data` to be an atom representing + > a module and attempt to call the *function* `key/0` in it. + + The two syntaxes for accessing keys reveal the dual nature of maps. The `map[key]` + syntax is used for dynamically created maps that may have any key, of any type. + `map.key` is used with maps that hold a predetermined set of atoms keys, which are + expected to always be present. Structs, defined via `defstruct/1`, are one example + of such "static maps", where the keys can also be checked during compile time. + + Maps can be pattern matched on. When a map is on the left-hand side of a + pattern match, it will match if the map on the right-hand side contains the + keys on the left-hand side and their values match the ones on the left-hand + side. This means that an empty map matches every map. + + iex> %{} = %{foo: "bar"} + %{foo: "bar"} + iex> %{a: a} = %{:a => 1, "b" => 2, [:c, :e, :e] => 3} + iex> a + 1 + + But this will raise a `MatchError` exception: + + %{:c => 3} = %{:a => 1, 2 => :b} + + Variables can be used as map keys both when writing map literals as well as + when matching: + + iex> n = 1 + 1 + iex> %{n => :one} + %{1 => :one} + iex> %{^n => :one} = %{1 => :one, 2 => :two, 3 => :three} + %{1 => :one, 2 => :two, 3 => :three} + + Maps also support a specific update syntax to update the value stored under + *existing* atom keys: + + iex> map = %{one: 1, two: 2} + iex> %{map | one: "one"} + %{one: "one", two: 2} + + When a key that does not exist in the map is updated a `KeyError` exception will be raised: + + %{map | three: 3} + + The functions in this module that need to find a specific key work in logarithmic time. + This means that the time it takes to find keys grows as the map grows, but it's not + directly proportional to the map size. In comparison to finding an element in a list, + it performs better because lists have a linear time complexity. Some functions, + such as `keys/1` and `values/1`, run in linear time because they need to get to every + element in the map. + + Maps also implement the `Enumerable` protocol, so many functions to work with maps + are found in the `Enum` module. Additionally, the following functions for maps are + found in `Kernel`: + + * `map_size/1` + + """ + + @type key :: any + @type value :: any + @compile {:inline, fetch: 2, fetch!: 2, get: 2, put: 3, delete: 2, has_key?: 2, replace!: 3} + + # TODO: Remove conditional on Erlang/OTP 24+ + @compile {:no_warn_undefined, {:maps, :from_keys, 2}} + @doc """ + Builds a map from the given `keys` and the fixed `value`. + + ## Examples + + iex> Map.from_keys([1, 2, 3], :number) + %{1 => :number, 2 => :number, 3 => :number} + + """ + @doc since: "1.14.0" + @spec from_keys([key], value) :: map + def from_keys(keys, value) do + if function_exported?(:maps, :from_keys, 2) do + :maps.from_keys(keys, value) + else + :maps.from_list(:lists.map(&{&1, value}, keys)) + end + end + + @doc """ + Returns all keys from `map`. + + Inlined by the compiler. + + ## Examples + + iex> Map.keys(%{a: 1, b: 2}) + [:a, :b] - For more information about the functions in this module and - their APIs, please consult the `Dict` module. """ + @spec keys(map) :: [key] + defdelegate keys(map), to: :maps + + @doc """ + Returns all values from `map`. + + Inlined by the compiler. - use Dict + ## Examples - defdelegate [keys(map), values(map), size(map), merge(map1, map2), to_list(map)], to: :maps + iex> Map.values(%{a: 1, b: 2}) + [1, 2] - @compile {:inline, fetch: 2, put: 3, delete: 2, has_key?: 2} + """ + @spec values(map) :: [value] + defdelegate values(map), to: :maps + + @doc """ + Converts `map` to a list. + + Each key-value pair in the map is converted to a two-element tuple `{key, + value}` in the resulting list. + + Inlined by the compiler. + + ## Examples + + iex> Map.to_list(%{a: 1}) + [a: 1] + iex> Map.to_list(%{1 => 2}) + [{1, 2}] + + """ + @spec to_list(map) :: [{term, term}] + defdelegate to_list(map), to: :maps @doc """ Returns a new empty map. + + ## Examples + + iex> Map.new() + %{} + """ + @spec new :: map def new, do: %{} + @doc """ + Creates a map from an `enumerable`. + + Duplicated keys are removed; the latest one prevails. + + ## Examples + + iex> Map.new([{:b, 1}, {:a, 2}]) + %{a: 2, b: 1} + iex> Map.new(a: 1, a: 2, a: 3) + %{a: 3} + + """ + @spec new(Enumerable.t()) :: map + def new(enumerable) + def new(list) when is_list(list), do: :maps.from_list(list) + def new(%_{} = struct), do: new_from_enum(struct) + def new(%{} = map), do: map + def new(enum), do: new_from_enum(enum) + + defp new_from_enum(enumerable) do + enumerable + |> Enum.to_list() + |> :maps.from_list() + end + + @doc """ + Creates a map from an `enumerable` via the given transformation function. + + Duplicated keys are removed; the latest one prevails. + + ## Examples + + iex> Map.new([:a, :b], fn x -> {x, x} end) + %{a: :a, b: :b} + + """ + @spec new(Enumerable.t(), (term -> {key, value})) :: map + def new(enumerable, transform) + def new(%_{} = enumerable, transform), do: new_from_enum(enumerable, transform) + def new(%{} = map, transform), do: new_from_map(map, transform) + def new(enumerable, transform), do: new_from_enum(enumerable, transform) + + defp new_from_map(map, transform) when is_function(transform, 1) do + iter = :maps.iterator(map) + next = :maps.next(iter) + :maps.from_list(do_map(next, transform)) + end + + defp do_map(:none, _fun), do: [] + + defp do_map({key, value, iter}, transform) do + [transform.({key, value}) | do_map(:maps.next(iter), transform)] + end + + defp new_from_enum(enumerable, transform) when is_function(transform, 1) do + enumerable + |> Enum.map(transform) + |> :maps.from_list() + end + + @doc """ + Returns whether the given `key` exists in the given `map`. + + Inlined by the compiler. + + ## Examples + + iex> Map.has_key?(%{a: 1}, :a) + true + iex> Map.has_key?(%{a: 1}, :b) + false + + """ + @spec has_key?(map, key) :: boolean def has_key?(map, key), do: :maps.is_key(key, map) + @doc """ + Fetches the value for a specific `key` in the given `map`. + + If `map` contains the given `key` then its value is returned in the shape of `{:ok, value}`. + If `map` doesn't contain `key`, `:error` is returned. + + Inlined by the compiler. + + ## Examples + + iex> Map.fetch(%{a: 1}, :a) + {:ok, 1} + iex> Map.fetch(%{a: 1}, :b) + :error + + """ + @spec fetch(map, key) :: {:ok, value} | :error def fetch(map, key), do: :maps.find(key, map) - def put(map, key, val) do - :maps.put(key, val, map) + @doc """ + Fetches the value for a specific `key` in the given `map`, erroring out if + `map` doesn't contain `key`. + + If `map` contains `key`, the corresponding value is returned. If + `map` doesn't contain `key`, a `KeyError` exception is raised. + + Inlined by the compiler. + + ## Examples + + iex> Map.fetch!(%{a: 1}, :a) + 1 + + """ + @spec fetch!(map, key) :: value + def fetch!(map, key) do + :maps.get(key, map) + end + + @doc """ + Puts the given `value` under `key` unless the entry `key` + already exists in `map`. + + ## Examples + + iex> Map.put_new(%{a: 1}, :b, 2) + %{a: 1, b: 2} + iex> Map.put_new(%{a: 1, b: 2}, :a, 3) + %{a: 1, b: 2} + + """ + @spec put_new(map, key, value) :: map + def put_new(map, key, value) do + case map do + %{^key => _value} -> + map + + %{} -> + put(map, key, value) + + other -> + :erlang.error({:badmap, other}) + end + end + + @doc """ + Puts a value under `key` only if the `key` already exists in `map`. + + ## Examples + + iex> Map.replace(%{a: 1, b: 2}, :a, 3) + %{a: 3, b: 2} + + iex> Map.replace(%{a: 1}, :b, 2) + %{a: 1} + + """ + @doc since: "1.11.0" + @spec replace(map, key, value) :: map + def replace(map, key, value) do + case map do + %{^key => _value} -> + %{map | key => value} + + %{} -> + map + + other -> + :erlang.error({:badmap, other}) + end + end + + @doc """ + Puts a value under `key` only if the `key` already exists in `map`. + + If `key` is not present in `map`, a `KeyError` exception is raised. + + Inlined by the compiler. + + ## Examples + + iex> Map.replace!(%{a: 1, b: 2}, :a, 3) + %{a: 3, b: 2} + + iex> Map.replace!(%{a: 1}, :b, 2) + ** (KeyError) key :b not found in: %{a: 1} + + """ + @doc since: "1.5.0" + @spec replace!(map, key, value) :: map + def replace!(map, key, value) do + :maps.update(key, value, map) + end + + @doc """ + Replaces the value under `key` using the given function only if + `key` already exists in `map`. + + In comparison to `replace/3`, this can be useful when it's expensive to calculate the value. + + If `key` does not exist, the original map is returned unchanged. + + ## Examples + + iex> Map.replace_lazy(%{a: 1, b: 2}, :a, fn v -> v * 4 end) + %{a: 4, b: 2} + + iex> Map.replace_lazy(%{a: 1, b: 2}, :c, fn v -> v * 4 end) + %{a: 1, b: 2} + + """ + @doc since: "1.14.0" + @spec replace_lazy(map, key, (existing_value :: value -> new_value :: value)) :: map + def replace_lazy(map, key, fun) when is_map(map) and is_function(fun, 1) do + case map do + %{^key => val} -> %{map | key => fun.(val)} + %{} -> map + end + end + + @doc """ + Evaluates `fun` and puts the result under `key` + in `map` unless `key` is already present. + + This function is useful in case you want to compute the value to put under + `key` only if `key` is not already present, as for example, when the value is expensive to + calculate or generally difficult to setup and teardown again. + + ## Examples + + iex> map = %{a: 1} + iex> fun = fn -> + ...> # some expensive operation here + ...> 3 + ...> end + iex> Map.put_new_lazy(map, :a, fun) + %{a: 1} + iex> Map.put_new_lazy(map, :b, fun) + %{a: 1, b: 3} + + """ + @spec put_new_lazy(map, key, (() -> value)) :: map + def put_new_lazy(map, key, fun) when is_function(fun, 0) do + case map do + %{^key => _value} -> + map + + %{} -> + put(map, key, fun.()) + + other -> + :erlang.error({:badmap, other}) + end + end + + @doc """ + Returns a new map with all the key-value pairs in `map` where the key + is in `keys`. + + If `keys` contains keys that are not in `map`, they're simply ignored. + + ## Examples + + iex> Map.take(%{a: 1, b: 2, c: 3}, [:a, :c, :e]) + %{a: 1, c: 3} + + """ + @spec take(map, [key]) :: map + def take(map, keys) + + def take(map, keys) when is_map(map) and is_list(keys) do + take(keys, map, _acc = []) + end + + def take(map, keys) when is_map(map) do + IO.warn( + "Map.take/2 with an Enumerable of keys that is not a list is deprecated. " <> + " Use a list of keys instead." + ) + + take(map, Enum.to_list(keys)) + end + + def take(non_map, _keys) do + :erlang.error({:badmap, non_map}) + end + + defp take([], _map, acc) do + :maps.from_list(acc) + end + + defp take([key | rest], map, acc) do + acc = + case map do + %{^key => value} -> [{key, value} | acc] + %{} -> acc + end + + take(rest, map, acc) + end + + @doc """ + Gets the value for a specific `key` in `map`. + + If `key` is present in `map` then its value `value` is + returned. Otherwise, `default` is returned. + + If `default` is not provided, `nil` is used. + + ## Examples + + iex> Map.get(%{}, :a) + nil + iex> Map.get(%{a: 1}, :a) + 1 + iex> Map.get(%{a: 1}, :b) + nil + iex> Map.get(%{a: 1}, :b, 3) + 3 + + """ + @spec get(map, key, value) :: value + def get(map, key, default \\ nil) do + case map do + %{^key => value} -> + value + + %{} -> + default + + other -> + :erlang.error({:badmap, other}, [map, key, default]) + end + end + + @doc """ + Gets the value for a specific `key` in `map`. + + If `key` is present in `map` then its value `value` is + returned. Otherwise, `fun` is evaluated and its result is returned. + + This is useful if the default value is very expensive to calculate or + generally difficult to setup and teardown again. + + ## Examples + + iex> map = %{a: 1} + iex> fun = fn -> + ...> # some expensive operation here + ...> 13 + ...> end + iex> Map.get_lazy(map, :a, fun) + 1 + iex> Map.get_lazy(map, :b, fun) + 13 + + """ + @spec get_lazy(map, key, (() -> value)) :: value + def get_lazy(map, key, fun) when is_function(fun, 0) do + case map do + %{^key => value} -> + value + + %{} -> + fun.() + + other -> + :erlang.error({:badmap, other}, [map, key, fun]) + end + end + + @doc """ + Puts the given `value` under `key` in `map`. + + Inlined by the compiler. + + ## Examples + + iex> Map.put(%{a: 1}, :b, 2) + %{a: 1, b: 2} + iex> Map.put(%{a: 1, b: 2}, :a, 3) + %{a: 3, b: 2} + + """ + @spec put(map, key, value) :: map + def put(map, key, value) do + :maps.put(key, value, map) end + @doc """ + Deletes the entry in `map` for a specific `key`. + + If the `key` does not exist, returns `map` unchanged. + + Inlined by the compiler. + + ## Examples + + iex> Map.delete(%{a: 1, b: 2}, :a) + %{b: 2} + iex> Map.delete(%{b: 2}, :a) + %{b: 2} + + """ + @spec delete(map, key) :: map def delete(map, key), do: :maps.remove(key, map) - def merge(map1, map2, callback) do - :maps.fold fn k, v2, acc -> - update(acc, k, v2, fn(v1) -> callback.(k, v1, v2) end) - end, map1, map2 + @doc """ + Merges two maps into one. + + All keys in `map2` will be added to `map1`, overriding any existing one + (i.e., the keys in `map2` "have precedence" over the ones in `map1`). + + If you have a struct and you would like to merge a set of keys into the + struct, do not use this function, as it would merge all keys on the right + side into the struct, even if the key is not part of the struct. Instead, + use `struct/2`. + + Inlined by the compiler. + + ## Examples + + iex> Map.merge(%{a: 1, b: 2}, %{a: 3, d: 4}) + %{a: 3, b: 2, d: 4} + + """ + @spec merge(map, map) :: map + defdelegate merge(map1, map2), to: :maps + + @doc """ + Merges two maps into one, resolving conflicts through the given `fun`. + + All keys in `map2` will be added to `map1`. The given function will be invoked + when there are duplicate keys; its arguments are `key` (the duplicate key), + `value1` (the value of `key` in `map1`), and `value2` (the value of `key` in + `map2`). The value returned by `fun` is used as the value under `key` in + the resulting map. + + ## Examples + + iex> Map.merge(%{a: 1, b: 2}, %{a: 3, d: 4}, fn _k, v1, v2 -> + ...> v1 + v2 + ...> end) + %{a: 4, b: 2, d: 4} + + """ + @spec merge(map, map, (key, value, value -> value)) :: map + def merge(map1, map2, fun) when is_function(fun, 3) do + if map_size(map1) > map_size(map2) do + folder = fn key, val2, acc -> + update(acc, key, val2, fn val1 -> fun.(key, val1, val2) end) + end + + :maps.fold(folder, map1, map2) + else + folder = fn key, val2, acc -> + update(acc, key, val2, fn val1 -> fun.(key, val2, val1) end) + end + + :maps.fold(folder, map2, map1) + end + end + + @doc """ + Updates the `key` in `map` with the given function. + + If `key` is present in `map` then the existing value is passed to `fun` and its result is + used as the updated value of `key`. If `key` is + not present in `map`, `default` is inserted as the value of `key`. The default + value will not be passed through the update function. + + ## Examples + + iex> Map.update(%{a: 1}, :a, 13, fn existing_value -> existing_value * 2 end) + %{a: 2} + iex> Map.update(%{a: 1}, :b, 11, fn existing_value -> existing_value * 2 end) + %{a: 1, b: 11} + + """ + @spec update(map, key, default :: value, (existing_value :: value -> new_value :: value)) :: + map + def update(map, key, default, fun) when is_function(fun, 1) do + case map do + %{^key => value} -> + %{map | key => fun.(value)} + + %{} -> + put(map, key, default) + + other -> + :erlang.error({:badmap, other}, [map, key, default, fun]) + end + end + + @doc """ + Removes the value associated with `key` in `map` and returns the value and the updated map. + + If `key` is present in `map`, it returns `{value, updated_map}` where `value` is the value of + the key and `updated_map` is the result of removing `key` from `map`. If `key` + is not present in `map`, `{default, map}` is returned. + + ## Examples + + iex> Map.pop(%{a: 1}, :a) + {1, %{}} + iex> Map.pop(%{a: 1}, :b) + {nil, %{a: 1}} + iex> Map.pop(%{a: 1}, :b, 3) + {3, %{a: 1}} + + """ + @spec pop(map, key, default) :: {value, updated_map :: map} | {default, map} when default: value + def pop(map, key, default \\ nil) do + case :maps.take(key, map) do + {_, _} = tuple -> tuple + :error -> {default, map} + end + end + + @doc """ + Removes the value associated with `key` in `map` and returns the value + and the updated map, or it raises if `key` is not present. + + Behaves the same as `pop/3` but raises if `key` is not present in `map`. + + ## Examples + + iex> Map.pop!(%{a: 1}, :a) + {1, %{}} + iex> Map.pop!(%{a: 1, b: 2}, :a) + {1, %{b: 2}} + iex> Map.pop!(%{a: 1}, :b) + ** (KeyError) key :b not found in: %{a: 1} + + """ + @doc since: "1.10.0" + @spec pop!(map, key) :: {value, updated_map :: map} + def pop!(map, key) do + case :maps.take(key, map) do + {_, _} = tuple -> tuple + :error -> raise KeyError, key: key, term: map + end + end + + @doc """ + Lazily returns and removes the value associated with `key` in `map`. + + If `key` is present in `map`, it returns `{value, new_map}` where `value` is the value of + the key and `new_map` is the result of removing `key` from `map`. If `key` + is not present in `map`, `{fun_result, map}` is returned, where `fun_result` + is the result of applying `fun`. + + This is useful if the default value is very expensive to calculate or + generally difficult to setup and teardown again. + + ## Examples + + iex> map = %{a: 1} + iex> fun = fn -> + ...> # some expensive operation here + ...> 13 + ...> end + iex> Map.pop_lazy(map, :a, fun) + {1, %{}} + iex> Map.pop_lazy(map, :b, fun) + {13, %{a: 1}} + + """ + @spec pop_lazy(map, key, (() -> value)) :: {value, map} + def pop_lazy(map, key, fun) when is_function(fun, 0) do + case :maps.take(key, map) do + {_, _} = tuple -> tuple + :error -> {fun.(), map} + end + end + + @doc """ + Drops the given `keys` from `map`. + + If `keys` contains keys that are not in `map`, they're simply ignored. + + ## Examples + + iex> Map.drop(%{a: 1, b: 2, c: 3}, [:b, :d]) + %{a: 1, c: 3} + + """ + @spec drop(map, [key]) :: map + def drop(map, keys) + + def drop(map, keys) when is_map(map) and is_list(keys) do + drop_keys(keys, map) + end + + def drop(map, keys) when is_map(map) do + IO.warn( + "Map.drop/2 with an Enumerable of keys that is not a list is deprecated. " <> + " Use a list of keys instead." + ) + + drop(map, Enum.to_list(keys)) + end + + def drop(non_map, keys) do + :erlang.error({:badmap, non_map}, [non_map, keys]) + end + + defp drop_keys([], acc), do: acc + + defp drop_keys([key | rest], acc) do + drop_keys(rest, delete(acc, key)) + end + + @doc """ + Takes all entries corresponding to the given `keys` in `map` and extracts + them into a separate map. + + Returns a tuple with the new map and the old map with removed keys. + + Keys for which there are no entries in `map` are ignored. + + ## Examples + + iex> Map.split(%{a: 1, b: 2, c: 3}, [:a, :c, :e]) + {%{a: 1, c: 3}, %{b: 2}} + + """ + @spec split(map, [key]) :: {map, map} + def split(map, keys) + + def split(map, keys) when is_map(map) and is_list(keys) do + split(keys, [], map) + end + + def split(map, keys) when is_map(map) do + IO.warn( + "Map.split/2 with an Enumerable of keys that is not a list is deprecated. " <> + " Use a list of keys instead." + ) + + split(map, Enum.to_list(keys)) + end + + def split(non_map, keys) do + :erlang.error({:badmap, non_map}, [non_map, keys]) + end + + defp split([], included, excluded) do + {:maps.from_list(included), excluded} + end + + defp split([key | rest], included, excluded) do + case excluded do + %{^key => value} -> + split(rest, [{key, value} | included], delete(excluded, key)) + + _other -> + split(rest, included, excluded) + end + end + + @doc """ + Updates `key` with the given function. + + If `key` is present in `map` then the existing value is passed to `fun` and its result is + used as the updated value of `key`. If `key` is + not present in `map`, a `KeyError` exception is raised. + + ## Examples + + iex> Map.update!(%{a: 1}, :a, &(&1 * 2)) + %{a: 2} + + iex> Map.update!(%{a: 1}, :b, &(&1 * 2)) + ** (KeyError) key :b not found in: %{a: 1} + + """ + @spec update!(map, key, (existing_value :: value -> new_value :: value)) :: map + def update!(map, key, fun) when is_function(fun, 1) do + value = fetch!(map, key) + %{map | key => fun.(value)} + end + + @doc """ + Gets the value from `key` and updates it, all in one pass. + + `fun` is called with the current value under `key` in `map` (or `nil` if `key` + is not present in `map`) and must return a two-element tuple: the current value + (the retrieved value, which can be operated on before being returned) and the + new value to be stored under `key` in the resulting new map. `fun` may also + return `:pop`, which means the current value shall be removed from `map` and + returned (making this function behave like `Map.pop(map, key)`). + + The returned value is a two-element tuple with the current value returned by + `fun` and a new map with the updated value under `key`. + + ## Examples + + iex> Map.get_and_update(%{a: 1}, :a, fn current_value -> + ...> {current_value, "new value!"} + ...> end) + {1, %{a: "new value!"}} + + iex> Map.get_and_update(%{a: 1}, :b, fn current_value -> + ...> {current_value, "new value!"} + ...> end) + {nil, %{a: 1, b: "new value!"}} + + iex> Map.get_and_update(%{a: 1}, :a, fn _ -> :pop end) + {1, %{}} + + iex> Map.get_and_update(%{a: 1}, :b, fn _ -> :pop end) + {nil, %{a: 1}} + + """ + @spec get_and_update(map, key, (value | nil -> {current_value, new_value :: value} | :pop)) :: + {current_value, new_map :: map} + when current_value: value + def get_and_update(map, key, fun) when is_function(fun, 1) do + current = get(map, key) + + case fun.(current) do + {get, update} -> + {get, put(map, key, update)} + + :pop -> + {current, delete(map, key)} + + other -> + raise "the given function must return a two-element tuple or :pop, got: #{inspect(other)}" + end + end + + @doc """ + Gets the value from `key` and updates it, all in one pass. Raises if there is no `key`. + + Behaves exactly like `get_and_update/3`, but raises a `KeyError` exception if + `key` is not present in `map`. + + ## Examples + + iex> Map.get_and_update!(%{a: 1}, :a, fn current_value -> + ...> {current_value, "new value!"} + ...> end) + {1, %{a: "new value!"}} + + iex> Map.get_and_update!(%{a: 1}, :b, fn current_value -> + ...> {current_value, "new value!"} + ...> end) + ** (KeyError) key :b not found in: %{a: 1} + + iex> Map.get_and_update!(%{a: 1}, :a, fn _ -> + ...> :pop + ...> end) + {1, %{}} + + """ + @spec get_and_update!(map, key, (value | nil -> {current_value, new_value :: value} | :pop)) :: + {current_value, map} + when current_value: value + def get_and_update!(map, key, fun) when is_function(fun, 1) do + value = fetch!(map, key) + + case fun.(value) do + {get, update} -> + {get, %{map | key => update}} + + :pop -> + {value, delete(map, key)} + + other -> + raise "the given function must return a two-element tuple or :pop, got: #{inspect(other)}" + end end + @doc """ + Converts a `struct` to map. + + It accepts the struct module or a struct itself and + simply removes the `__struct__` field from the given struct + or from a new struct generated from the given module. + + ## Example + + defmodule User do + defstruct [:name] + end + + Map.from_struct(User) + #=> %{name: nil} + + Map.from_struct(%User{name: "john"}) + #=> %{name: "john"} + + """ + @spec from_struct(atom | struct) :: map + def from_struct(struct) when is_atom(struct) do + delete(struct.__struct__(), :__struct__) + end + + def from_struct(%_{} = struct) do + delete(struct, :__struct__) + end + + @doc """ + Checks if two maps are equal. + + Two maps are considered to be equal if they contain + the same keys and those keys contain the same values. + + Note this function exists for completeness so the `Map` + and `Keyword` modules provide similar APIs. In practice, + developers often compare maps using `==/2` or `===/2` + directly. + + ## Examples + + iex> Map.equal?(%{a: 1, b: 2}, %{b: 2, a: 1}) + true + iex> Map.equal?(%{a: 1, b: 2}, %{b: 1, a: 2}) + false + + Comparison between keys and values is done with `===/3`, + which means integers are not equivalent to floats: + + iex> Map.equal?(%{a: 1.0}, %{a: 1}) + false + + """ + @spec equal?(map, map) :: boolean + def equal?(map1, map2) + def equal?(%{} = map1, %{} = map2), do: map1 === map2 + def equal?(%{} = map1, map2), do: :erlang.error({:badmap, map2}, [map1, map2]) + def equal?(term, other), do: :erlang.error({:badmap, term}, [term, other]) + + @doc false + @deprecated "Use Kernel.map_size/1 instead" + def size(map) do + map_size(map) + end + + @doc """ + Returns a map containing only those pairs from `map` + for which `fun` returns a truthy value. + + `fun` receives the key and value of each of the + elements in the map as a key-value pair. + + See also `reject/2` which discards all elements where the + function returns a truthy value. + + > Note: if you find yourself doing multiple calls to `Map.filter/2` + > and `Map.reject/2` in a pipeline, it is likely more efficient + > to use `Enum.map/2` and `Enum.filter/2` instead and convert to + > a map at the end using `Map.new/1`. + + ## Examples + + iex> Map.filter(%{one: 1, two: 2, three: 3}, fn {_key, val} -> rem(val, 2) == 1 end) + %{one: 1, three: 3} + + """ + @doc since: "1.13.0" + @spec filter(map, ({key, value} -> as_boolean(term))) :: map + def filter(map, fun) when is_map(map) and is_function(fun, 1) do + iter = :maps.iterator(map) + next = :maps.next(iter) + :maps.from_list(do_filter(next, fun)) + end + + defp do_filter(:none, _fun), do: [] + + defp do_filter({key, value, iter}, fun) do + if fun.({key, value}) do + [{key, value} | do_filter(:maps.next(iter), fun)] + else + do_filter(:maps.next(iter), fun) + end + end + + @doc """ + Returns map excluding the pairs from `map` for which `fun` returns + a truthy value. + + See also `filter/2`. + + ## Examples + + iex> Map.reject(%{one: 1, two: 2, three: 3}, fn {_key, val} -> rem(val, 2) == 1 end) + %{two: 2} + + """ + @doc since: "1.13.0" + @spec reject(map, ({key, value} -> as_boolean(term))) :: map + def reject(map, fun) when is_map(map) and is_function(fun, 1) do + iter = :maps.iterator(map) + next = :maps.next(iter) + :maps.from_list(do_reject(next, fun)) + end + + defp do_reject(:none, _fun), do: [] + + defp do_reject({key, value, iter}, fun) do + if fun.({key, value}) do + do_reject(:maps.next(iter), fun) + else + [{key, value} | do_reject(:maps.next(iter), fun)] + end + end + + @doc false + @deprecated "Use Map.new/2 instead (invoke Map.from_struct/1 before if you have a struct)" + def map(map, fun) when is_map(map) do + :maps.map(fn k, v -> fun.({k, v}) end, map) + end end diff --git a/lib/elixir/lib/map_set.ex b/lib/elixir/lib/map_set.ex new file mode 100644 index 00000000000..4b850779dab --- /dev/null +++ b/lib/elixir/lib/map_set.ex @@ -0,0 +1,476 @@ +defmodule MapSet do + @moduledoc """ + Functions that work on sets. + + A set is a data structure that can contain unique elements of any kind, + without any particular order. `MapSet` is the "go to" set data structure in Elixir. + + A set can be constructed using `MapSet.new/0`: + + iex> MapSet.new() + MapSet.new([]) + + Elements in a set don't have to be of the same type and they can be + populated from an [enumerable](`t:Enumerable.t/0`) using `MapSet.new/1`: + + iex> MapSet.new([1, :two, {"three"}]) + MapSet.new([1, :two, {"three"}]) + + Elements can be inserted using `MapSet.put/2`: + + iex> MapSet.new([2]) |> MapSet.put(4) |> MapSet.put(0) + MapSet.new([0, 2, 4]) + + By definition, sets can't contain duplicate elements: when + inserting an element in a set where it's already present, the insertion is + simply a no-op. + + iex> map_set = MapSet.new() + iex> MapSet.put(map_set, "foo") + MapSet.new(["foo"]) + iex> map_set |> MapSet.put("foo") |> MapSet.put("foo") + MapSet.new(["foo"]) + + A `MapSet` is represented internally using the `%MapSet{}` struct. This struct + can be used whenever there's a need to pattern match on something being a `MapSet`: + + iex> match?(%MapSet{}, MapSet.new()) + true + + Note that, however, the struct fields are private and must not be accessed + directly; use the functions in this module to perform operations on sets. + + `MapSet`s can also be constructed starting from other collection-type data + structures: for example, see `MapSet.new/1` or `Enum.into/2`. + + `MapSet` is built on top of `Map`, this means that they share many properties, + including logarithmic time complexity. See the documentation for `Map` for more + information on its execution time complexity. + """ + + # MapSets have an underlying Map. MapSet elements are keys of said map, + # and this empty list is their associated dummy value. + @dummy_value [] + + @type value :: term + + @opaque internal(value) :: %{optional(value) => []} + @type t(value) :: %__MODULE__{map: internal(value)} + @type t :: t(term) + + # TODO: Remove version key when we require Erlang/OTP 24 + # TODO: Implement the functions in this module using Erlang/OTP 24 new sets + defstruct map: %{}, version: 2 + + @doc """ + Returns a new set. + + ## Examples + + iex> MapSet.new() + MapSet.new([]) + + """ + @spec new :: t + def new(), do: %MapSet{} + + @doc """ + Creates a set from an enumerable. + + ## Examples + + iex> MapSet.new([:b, :a, 3]) + MapSet.new([3, :a, :b]) + iex> MapSet.new([3, 3, 3, 2, 2, 1]) + MapSet.new([1, 2, 3]) + + """ + @spec new(Enumerable.t()) :: t + def new(enumerable) + + def new(%__MODULE__{} = map_set), do: map_set + + def new(enumerable) do + keys = Enum.to_list(enumerable) + %MapSet{map: Map.from_keys(keys, @dummy_value)} + end + + @doc """ + Creates a set from an enumerable via the transformation function. + + ## Examples + + iex> MapSet.new([1, 2, 1], fn x -> 2 * x end) + MapSet.new([2, 4]) + + """ + @spec new(Enumerable.t(), (term -> val)) :: t(val) when val: value + def new(enumerable, transform) when is_function(transform, 1) do + keys = Enum.map(enumerable, transform) + %MapSet{map: Map.from_keys(keys, @dummy_value)} + end + + @doc """ + Deletes `value` from `map_set`. + + Returns a new set which is a copy of `map_set` but without `value`. + + ## Examples + + iex> map_set = MapSet.new([1, 2, 3]) + iex> MapSet.delete(map_set, 4) + MapSet.new([1, 2, 3]) + iex> MapSet.delete(map_set, 2) + MapSet.new([1, 3]) + + """ + @spec delete(t(val1), val2) :: t(val1) when val1: value, val2: value + def delete(%MapSet{map: map} = map_set, value) do + %{map_set | map: Map.delete(map, value)} + end + + @doc """ + Returns a set that is `map_set1` without the members of `map_set2`. + + ## Examples + + iex> MapSet.difference(MapSet.new([1, 2]), MapSet.new([2, 3, 4])) + MapSet.new([1]) + + """ + @spec difference(t(val1), t(val2)) :: t(val1) when val1: value, val2: value + def difference(map_set1, map_set2) + + # If the first set is less than twice the size of the second map, it is fastest + # to re-accumulate elements in the first set that are not present in the second set. + def difference(%MapSet{map: map1}, %MapSet{map: map2}) + when map_size(map1) < map_size(map2) * 2 do + map = + map1 + |> :maps.iterator() + |> :maps.next() + |> filter_not_in(map2, []) + + %MapSet{map: map} + end + + # If the second set is less than half the size of the first set, it's fastest + # to simply iterate through each element in the second set, deleting them from + # the first set. + def difference(%MapSet{map: map1} = map_set, %MapSet{map: map2}) do + %{map_set | map: Map.drop(map1, Map.keys(map2))} + end + + defp filter_not_in(:none, _map2, acc), do: Map.new(acc) + + defp filter_not_in({key, _val, iter}, map2, acc) do + if :erlang.is_map_key(key, map2) do + filter_not_in(:maps.next(iter), map2, acc) + else + filter_not_in(:maps.next(iter), map2, [{key, @dummy_value} | acc]) + end + end + + @doc """ + Checks if `map_set1` and `map_set2` have no members in common. + + ## Examples + + iex> MapSet.disjoint?(MapSet.new([1, 2]), MapSet.new([3, 4])) + true + iex> MapSet.disjoint?(MapSet.new([1, 2]), MapSet.new([2, 3])) + false + + """ + @spec disjoint?(t, t) :: boolean + def disjoint?(%MapSet{map: map1}, %MapSet{map: map2}) do + {map1, map2} = order_by_size(map1, map2) + + map1 + |> :maps.iterator() + |> :maps.next() + |> none_in?(map2) + end + + defp none_in?(:none, _), do: true + + defp none_in?({key, _val, iter}, map2) do + not :erlang.is_map_key(key, map2) and none_in?(:maps.next(iter), map2) + end + + @doc """ + Checks if two sets are equal. + + The comparison between elements is done using `===/2`, + which a set with `1` is not equivalent to a set with + `1.0`. + + ## Examples + + iex> MapSet.equal?(MapSet.new([1, 2]), MapSet.new([2, 1, 1])) + true + iex> MapSet.equal?(MapSet.new([1, 2]), MapSet.new([3, 4])) + false + iex> MapSet.equal?(MapSet.new([1]), MapSet.new([1.0])) + false + + """ + @spec equal?(t, t) :: boolean + def equal?(%MapSet{map: map1, version: version}, %MapSet{map: map2, version: version}) do + map1 === map2 + end + + # Elixir v1.5 changed the map representation, so on + # version mismatch we need to compare the keys directly. + def equal?(%MapSet{map: map1}, %MapSet{map: map2}) do + map_size(map1) == map_size(map2) and all_in?(map1, map2) + end + + @doc """ + Returns a set containing only members that `map_set1` and `map_set2` have in common. + + ## Examples + + iex> MapSet.intersection(MapSet.new([1, 2]), MapSet.new([2, 3, 4])) + MapSet.new([2]) + + iex> MapSet.intersection(MapSet.new([1, 2]), MapSet.new([3, 4])) + MapSet.new([]) + + """ + @spec intersection(t(val), t(val)) :: t(val) when val: value + def intersection(%MapSet{map: map1} = map_set, %MapSet{map: map2}) do + {map1, map2} = order_by_size(map1, map2) + %{map_set | map: Map.take(map2, Map.keys(map1))} + end + + @doc """ + Checks if `map_set` contains `value`. + + ## Examples + + iex> MapSet.member?(MapSet.new([1, 2, 3]), 2) + true + iex> MapSet.member?(MapSet.new([1, 2, 3]), 4) + false + + """ + @spec member?(t, value) :: boolean + def member?(%MapSet{map: map}, value) do + :erlang.is_map_key(value, map) + end + + @doc """ + Inserts `value` into `map_set` if `map_set` doesn't already contain it. + + ## Examples + + iex> MapSet.put(MapSet.new([1, 2, 3]), 3) + MapSet.new([1, 2, 3]) + iex> MapSet.put(MapSet.new([1, 2, 3]), 4) + MapSet.new([1, 2, 3, 4]) + + """ + @spec put(t(val), new_val) :: t(val | new_val) when val: value, new_val: value + def put(%MapSet{map: map} = map_set, value) do + %{map_set | map: Map.put(map, value, @dummy_value)} + end + + @doc """ + Returns the number of elements in `map_set`. + + ## Examples + + iex> MapSet.size(MapSet.new([1, 2, 3])) + 3 + + """ + @spec size(t) :: non_neg_integer + def size(%MapSet{map: map}) do + map_size(map) + end + + @doc """ + Checks if `map_set1`'s members are all contained in `map_set2`. + + This function checks if `map_set1` is a subset of `map_set2`. + + ## Examples + + iex> MapSet.subset?(MapSet.new([1, 2]), MapSet.new([1, 2, 3])) + true + iex> MapSet.subset?(MapSet.new([1, 2, 3]), MapSet.new([1, 2])) + false + + """ + @spec subset?(t, t) :: boolean + def subset?(%MapSet{map: map1}, %MapSet{map: map2}) do + map_size(map1) <= map_size(map2) and all_in?(map1, map2) + end + + defp all_in?(:none, _), do: true + + defp all_in?({key, _val, iter}, map2) do + :erlang.is_map_key(key, map2) and all_in?(:maps.next(iter), map2) + end + + defp all_in?(map1, map2) when is_map(map1) and is_map(map2) do + map1 + |> :maps.iterator() + |> :maps.next() + |> all_in?(map2) + end + + @doc """ + Converts `map_set` to a list. + + ## Examples + + iex> MapSet.to_list(MapSet.new([1, 2, 3])) + [1, 2, 3] + + """ + @spec to_list(t(val)) :: [val] when val: value + def to_list(%MapSet{map: map}) do + Map.keys(map) + end + + @doc """ + Returns a set containing all members of `map_set1` and `map_set2`. + + ## Examples + + iex> MapSet.union(MapSet.new([1, 2]), MapSet.new([2, 3, 4])) + MapSet.new([1, 2, 3, 4]) + + """ + @spec union(t(val1), t(val2)) :: t(val1 | val2) when val1: value, val2: value + def union(map_set1, map_set2) + + def union(%MapSet{map: map1, version: version} = map_set, %MapSet{map: map2, version: version}) do + %{map_set | map: Map.merge(map1, map2)} + end + + def union(%MapSet{map: map1}, %MapSet{map: map2}) do + keys = Map.keys(map1) ++ Map.keys(map2) + %MapSet{map: Map.from_keys(keys, @dummy_value)} + end + + @compile {:inline, [order_by_size: 2]} + defp order_by_size(map1, map2) when map_size(map1) > map_size(map2), do: {map2, map1} + defp order_by_size(map1, map2), do: {map1, map2} + + @doc """ + Filters the set by returning only the elements from `set` for which invoking + `fun` returns a truthy value. + + Also see `reject/2` which discards all elements where the function returns + a truthy value. + + > Note: if you find yourself doing multiple calls to `MapSet.filter/2` + > and `MapSet.reject/2` in a pipeline, it is likely more efficient + > to use `Enum.map/2` and `Enum.filter/2` instead and convert to + > a map at the end using `Map.new/1`. + + ## Examples + + iex> MapSet.filter(MapSet.new(1..5), fn x -> x > 3 end) + MapSet.new([4, 5]) + + iex> MapSet.filter(MapSet.new(["a", :b, "c"]), &is_atom/1) + MapSet.new([:b]) + + """ + @doc since: "1.14.0" + @spec filter(t(a), (a -> as_boolean(term))) :: t(a) when a: value + def filter(%MapSet{map: map}, fun) when is_map(map) and is_function(fun) do + iter = :maps.iterator(map) + next = :maps.next(iter) + keys = filter_keys(next, fun) + %MapSet{map: Map.from_keys(keys, @dummy_value)} + end + + defp filter_keys(:none, _fun), do: [] + + defp filter_keys({key, _value, iter}, fun) do + if fun.(key) do + [key | filter_keys(:maps.next(iter), fun)] + else + filter_keys(:maps.next(iter), fun) + end + end + + @doc """ + Returns a set by excluding the elements from `set` for which invoking `fun` + returns a truthy value. + + See also `filter/2`. + + ## Examples + + iex> MapSet.reject(MapSet.new(1..5), fn x -> rem(x, 2) != 0 end) + MapSet.new([2, 4]) + + iex> MapSet.reject(MapSet.new(["a", :b, "c"]), &is_atom/1) + MapSet.new(["a", "c"]) + + """ + @doc since: "1.14.0" + @spec reject(t(a), (a -> as_boolean(term))) :: t(a) when a: value + def reject(%MapSet{map: map}, fun) when is_map(map) and is_function(fun) do + iter = :maps.iterator(map) + next = :maps.next(iter) + keys = reject_keys(next, fun) + %MapSet{map: Map.from_keys(keys, @dummy_value)} + end + + defp reject_keys(:none, _fun), do: [] + + defp reject_keys({key, _value, iter}, fun) do + if fun.(key) do + reject_keys(:maps.next(iter), fun) + else + [key | reject_keys(:maps.next(iter), fun)] + end + end + + defimpl Enumerable do + def count(map_set) do + {:ok, MapSet.size(map_set)} + end + + def member?(map_set, val) do + {:ok, MapSet.member?(map_set, val)} + end + + def slice(map_set) do + size = MapSet.size(map_set) + {:ok, size, &MapSet.to_list/1} + end + + def reduce(map_set, acc, fun) do + Enumerable.List.reduce(MapSet.to_list(map_set), acc, fun) + end + end + + defimpl Collectable do + def into(map_set) do + fun = fn + list, {:cont, x} -> [x | list] + list, :done -> %{map_set | map: Map.merge(map_set.map, Map.from_keys(list, []))} + _, :halt -> :ok + end + + {[], fun} + end + end + + defimpl Inspect do + import Inspect.Algebra + + def inspect(map_set, opts) do + opts = %Inspect.Opts{opts | charlists: :as_lists} + concat(["MapSet.new(", Inspect.List.inspect(MapSet.to_list(map_set), opts), ")"]) + end + end +end diff --git a/lib/elixir/lib/module.ex b/lib/elixir/lib/module.ex index 3853bfeac8c..cb0ce5fbf29 100644 --- a/lib/elixir/lib/module.ex +++ b/lib/elixir/lib/module.ex @@ -1,314 +1,671 @@ defmodule Module do @moduledoc ~S''' - This module provides many functions to deal with modules during - compilation time. It allows a developer to dynamically attach - documentation, add, delete and register attributes and so forth. + Provides functions to deal with modules during compilation time. + + It allows a developer to dynamically add, delete and register + attributes, attach documentation and so forth. After a module is compiled, using many of the functions in this module will raise errors, since it is out of their scope to inspect runtime data. Most of the runtime data can be inspected - via the `__info__(attr)` function attached to each compiled module. + via the [`__info__/1`](`c:Module.__info__/1`) function attached to + each compiled module. ## Module attributes Each module can be decorated with one or more attributes. The following ones are currently defined by Elixir: - * `@after_compile` + ### `@after_compile` - A hook that will be invoked right after the current module is compiled. + A hook that will be invoked right after the current module is compiled. + Accepts a module or a `{module, function_name}`. See the "Compile callbacks" + section below. - Accepts a module or a tuple `{, }`. The function - must take two arguments: the module environment and its bytecode. - When just a module is provided, the function is assumed to be - `__after_compile__/2`. + ### `@before_compile` - ### Example + A hook that will be invoked before the module is compiled. + Accepts a module or a `{module, function_or_macro_name}` tuple. + See the "Compile callbacks" section below. - defmodule M do - @after_compile __MODULE__ + ### `@behaviour` - def __after_compile__(env, _bytecode) do - IO.inspect env - end - end + Note the British spelling! - * `@before_compile` + Behaviours can be referenced by modules to ensure they implement + required specific function signatures defined by `@callback`. - A hook that will be invoked before the module is compiled. + For example, you could specify a `URI.Parser` behaviour as follows: - Accepts a module or a tuple `{, }`. The - function/macro must take one argument: the module environment. If it's a - macro, its returned value will be injected at the end of the module definition - before the compilation starts. + defmodule URI.Parser do + @doc "Defines a default port" + @callback default_port() :: integer + + @doc "Parses the given URL" + @callback parse(uri_info :: URI.t()) :: URI.t() + end - When just a module is provided, the function/macro is assumed to be - `__before_compile__/1`. + And then a module may use it as: - Note: unlike `@after_compile`, the callback function/macro must - be placed in a separate module (because when the callback is invoked, - the current module does not yet exist). + defmodule URI.HTTP do + @behaviour URI.Parser + def default_port(), do: 80 + def parse(info), do: info + end - ### Example + If the behaviour changes or `URI.HTTP` does not implement + one of the callbacks, a warning will be raised. - defmodule A do - defmacro __before_compile__(_env) do - quote do - def hello, do: "world" - end - end - end + For detailed documentation, see the + [behaviour typespec documentation](typespecs.md#behaviours). - defmodule B do - @before_compile A - end + ### `@impl` - * `@behaviour` (notice the British spelling) + To aid in the correct implementation of behaviours, you may optionally declare + `@impl` for implemented callbacks of a behaviour. This makes callbacks + explicit and can help you to catch errors in your code. The compiler will warn + in these cases: - Specify an OTP or user-defined behaviour. + * if you mark a function with `@impl` when that function is not a callback. - ### Example + * if you don't mark a function with `@impl` when other functions are marked + with `@impl`. If you mark one function with `@impl`, you must mark all + other callbacks for that behaviour as `@impl`. - defmodule M do - @behaviour gen_event + `@impl` works on a per-context basis. If you generate a function through a macro + and mark it with `@impl`, that won't affect the module where that function is + generated in. - # ... - end + `@impl` also helps with maintainability by making it clear to other developers + that the function is implementing a callback. - * `@compile` + Using `@impl`, the example above can be rewritten as: - Define options for module compilation that are passed to the Erlang - compiler. + defmodule URI.HTTP do + @behaviour URI.Parser - Accepts an atom, a tuple, or a list of atoms and tuples. + @impl true + def default_port(), do: 80 - See http://www.erlang.org/doc/man/compile.html for the list of supported - options. + @impl true + def parse(info), do: info + end - ### Example + You may pass either `false`, `true`, or a specific behaviour to `@impl`. - defmodule M do - @compile {:inline, myfun: 1} + defmodule Foo do + @behaviour Bar + @behaviour Baz - def myfun(arg) do - to_string(arg) - end - end + # Will warn if neither Bar nor Baz specify a callback named bar/0. + @impl true + def bar(), do: :ok - * `@doc` + # Will warn if Baz does not specify a callback named baz/0. + @impl Baz + def baz(), do: :ok + end - Provide documentation for the function or macro that follows the - attribute. + The code is now more readable, as it is now clear which functions are + part of your API and which ones are callback implementations. To reinforce this + idea, `@impl true` automatically marks the function as `@doc false`, disabling + documentation unless `@doc` is explicitly set. - Accepts a string (often a heredoc) or `false` where `@doc false` will - make the function/macro invisible to the documentation extraction tools - like ExDoc. + ### `@compile` - Can be invoked more than once. + Defines options for module compilation. This is used to configure + both Elixir and Erlang compilers, as any other compilation pass + added by external tools. For example: - ### Example + defmodule MyModule do + @compile {:inline, my_fun: 1} - defmodule M do - @doc "Hello world" - def hello do - "world" - end + def my_fun(arg) do + to_string(arg) + end + end - @doc """ - Sum. - """ - def sum(a, b) do - a + b - end - end + Multiple uses of `@compile` will accumulate instead of overriding + previous ones. See the "Compile options" section below. - * `@file` + ### `@deprecated` (since 1.6.0) - Change the filename used in stacktraces for the function or macro that - follows the attribute. + Provides the deprecation reason for a function. For example: - Accepts a string. Can be used more than once. + defmodule Keyword do + @deprecated "Use Kernel.length/1 instead" + def size(keyword) do + length(keyword) + end + end - ### Example + The Mix compiler automatically looks for calls to deprecated modules + and emit warnings during compilation. - defmodule M do - @doc "Hello world" - @file "hello.ex" - def hello do - "world" - end - end + Using the `@deprecated` attribute will also be reflected in the + documentation of the given function and macro. You can choose between + the `@deprecated` attribute and the documentation metadata to provide + hard-deprecations (with warnings) and soft-deprecations (without warnings): - * `@moduledoc` + This is a soft-deprecation as it simply annotates the documentation + as deprecated: - Provide documentation for the current module. + @doc deprecated: "Use Kernel.length/1 instead" + def size(keyword) - Accepts a string (which is often a heredoc) or `false` where - `@moduledoc false` will make the module invisible to the - documentation extraction tools like ExDoc. + This is a hard-deprecation as it emits warnings and annotates the + documentation as deprecated: - ### Example + @deprecated "Use Kernel.length/1 instead" + def size(keyword) - defmodule M do - @moduledoc """ - A very useful module - """ - end + Currently `@deprecated` only supports functions and macros. However + you can use the `:deprecated` key in the annotation metadata to + annotate the docs of modules, types and callbacks too. + We recommend using this feature with care, especially library authors. + Deprecating code always pushes the burden towards library users. We + also recommend for deprecated functionality to be maintained for long + periods of time, even after deprecation, giving developers plenty of + time to update (except for cases where keeping the deprecated API is + undesired, such as in the presence of security issues). - * `@on_definition` + ### `@doc` and `@typedoc` - A hook that will be invoked when each function or macro in the current - module is defined. Useful when annotating functions. + Provides documentation for the entity that follows the attribute. + `@doc` is to be used with a function, macro, callback, or + macrocallback, while `@typedoc` with a type (public or opaque). - Accepts a module or a tuple `{, }`. The function - must take 6 arguments: + Accepts one of these: - - the module environment - - kind: `:def`, `:defp`, `:defmacro`, or `:defmacrop` - - function/macro name - - list of expanded arguments - - list of expanded guards - - expanded function body + * a string (often a heredoc) + * `false`, which will make the entity invisible to documentation-extraction + tools like [`ExDoc`](https://hexdocs.pm/ex_doc/) + * a keyword list, since Elixir 1.7.0 - Note the hook receives the expanded arguments and it is invoked before - the function is stored in the module. So `Module.defines?/2` will return - false for the first clause of every function. + For example: - If the function/macro being defined has multiple clauses, the hook will - be called for each clause. + defmodule MyModule do + @typedoc "This type" + @typedoc since: "1.1.0" + @type t :: term + + @doc "Hello world" + @doc since: "1.1.0" + def hello do + "world" + end - Unlike other hooks, `@on_definition` will only invoke functions - and never macros. This is because the hook is invoked inside the context - of the function (and nested function definitions are not allowed in - Elixir). + @doc """ + Sums `a` to `b`. + """ + def sum(a, b) do + a + b + end + end - When just a module is provided, the function is assumed to be - `__on_definition__/6`. + As can be seen in the example above, since Elixir 1.7.0 `@doc` and `@typedoc` + also accept a keyword list that serves as a way to provide arbitrary metadata + about the entity. Tools like [`ExDoc`](https://hexdocs.pm/ex_doc/) and + `IEx` may use this information to display annotations. A common use + case is the `:since` key, which may be used to annotate in which version the + function was introduced. - ### Example + As illustrated in the example, it is possible to use these attributes + more than once before an entity. However, the compiler will warn if + used twice with binaries as that replaces the documentation text from + the preceding use. Multiple uses with keyword lists will merge the + lists into one. - defmodule H do - def on_def(_env, kind, name, args, guards, body) do - IO.puts "Defining #{kind} named #{name} with args:" - IO.inspect args - IO.puts "and guards" - IO.inspect guards - IO.puts "and body" - IO.puts Macro.to_string(body) - end - end + Note that since the compiler also defines some additional metadata, + there are a few reserved keys that will be ignored and warned if used. + Currently these are: `:opaque` and `:defaults`. - defmodule M do - @on_definition {H, :on_def} + Once this module is compiled, this information becomes available via + the `Code.fetch_docs/1` function. - def hello(arg) when is_binary(arg) or is_list(arg) do - "Hello" <> to_string(arg) - end + ### `@dialyzer` - def hello(_) do - :ok - end - end + Defines warnings to request or suppress when using `:dialyzer`. - * `@on_load` + Accepts an atom, a tuple, or a list of atoms and tuples. For example: - A hook that will be invoked whenever the module is loaded. + defmodule MyModule do + @dialyzer {:nowarn_function, my_fun: 1} - Accepts a function atom of a function in the current module. The function - must have arity 0 (no arguments) and has to return `:ok`, otherwise the - loading of the module will be aborted. + def my_fun(arg) do + M.not_a_function(arg) + end + end - ### Example + For the list of supported warnings, see + [`:dialyzer` module](`:dialyzer`). - defmodule M do - @on_load :load_check + Multiple uses of `@dialyzer` will accumulate instead of overriding + previous ones. - def load_check do - if some_condition() do - :ok - else - nil - end - end + ### `@external_resource` - def some_condition do - false - end - end + Specifies an external resource for the current module. - * `@vsn` + Sometimes a module embeds information from an external file. This + attribute allows the module to annotate which external resources + have been used. - Specify the module version. Accepts any valid Elixir value. + Tools may use this information to ensure the module is recompiled + in case any of the external resources change, see for example: + [`mix compile.elixir`](https://hexdocs.pm/mix/Mix.Tasks.Compile.Elixir.html). - ### Example + If the external resource does not exist, the module still has + a dependency on it, causing the module to be recompiled as soon + as the file is added. - defmodule M do - @vsn "1.0" - end + ### `@file` - * `@external_resource` + Changes the filename used in stacktraces for the function or macro that + follows the attribute, such as: - Specify an external resource to the current module. + defmodule MyModule do + @doc "Hello world" + @file "hello.ex" + def hello do + "world" + end + end - Many times a module embeds information from an external file. This - attribute allows the module to annotate which external resources - have been used. + ### `@moduledoc` - Tools like Mix may use this information to ensure the module is - recompiled in case any of the external resources change. + Provides documentation for the current module. - The following attributes are part of typespecs and are also reserved by - Elixir (see `Kernel.Typespec` for more information about typespecs): + defmodule MyModule do + @moduledoc """ + A very useful module. + """ + @moduledoc authors: ["Alice", "Bob"] + end - * `@type` - defines a type to be used in `@spec` - * `@typep` - defines a private type to be used in `@spec` - * `@opaque` - defines an opaque type to be used in `@spec` - * `@spec` - provides a specification for a function - * `@callback` - provides a specification for the behaviour callback + Accepts a string (often a heredoc) or `false` where `@moduledoc false` + will make the module invisible to documentation extraction tools like + [`ExDoc`](https://hexdocs.pm/ex_doc/). - In addition to the built-in attributes outlined above, custom attributes may - also be added. A custom attribute is any valid identifier prefixed with an - `@` and followed by a valid Elixir value: + Similarly to `@doc` also accepts a keyword list to provide metadata + about the module. For more details, see the documentation of `@doc` + above. - defmodule M do - @custom_attr [some: "stuff"] + Once this module is compiled, this information becomes available via + the `Code.fetch_docs/1` function. + + ### `@on_definition` + + A hook that will be invoked when each function or macro in the current + module is defined. Useful when annotating functions. + + Accepts a module or a `{module, function_name}` tuple. See the + "Compile callbacks" section below. + + ### `@on_load` + + A hook that will be invoked whenever the module is loaded. + + Accepts the function name (as an atom) of a function in the current module or + `{function_name, 0}` tuple where `function_name` is the name of a function in + the current module. The function must have an arity of 0 (no arguments). If + the function does not return `:ok`, the loading of the module will be aborted. + For example: + + defmodule MyModule do + @on_load :load_check + + def load_check do + if some_condition() do + :ok + else + :abort + end + end + + def some_condition do + false end + end + + ### `@vsn` + + Specify the module version. Accepts any valid Elixir value, for example: + + defmodule MyModule do + @vsn "1.0" + end + + ### Struct attributes + + * `@derive` - derives an implementation for the given protocol for the + struct defined in the current module + + * `@enforce_keys` - ensures the given keys are always set when building + the struct defined in the current module + + See `defstruct/1` for more information on building and using structs. + + ### Typespec attributes + + The following attributes are part of typespecs and are also built-in in + Elixir: + + * `@type` - defines a type to be used in `@spec` + * `@typep` - defines a private type to be used in `@spec` + * `@opaque` - defines an opaque type to be used in `@spec` + * `@spec` - provides a specification for a function + * `@callback` - provides a specification for a behaviour callback + * `@macrocallback` - provides a specification for a macro behaviour callback + * `@optional_callbacks` - specifies which behaviour callbacks and macro + behaviour callbacks are optional + * `@impl` - declares an implementation of a callback function or macro + + For detailed documentation, see the [typespec documentation](typespecs.md). + + ### Custom attributes + + In addition to the built-in attributes outlined above, custom attributes may + also be added. Custom attributes are expressed using the `@/1` operator followed + by a valid variable name. The value given to the custom attribute must be a valid + Elixir value: + + defmodule MyModule do + @custom_attr [some: "stuff"] + end For more advanced options available when defining custom attributes, see `register_attribute/3`. - ## Runtime information about a module + ## Compile callbacks + + There are three callbacks that are invoked when functions are defined, + as well as before and immediately after the module bytecode is generated. + + ### `@after_compile` + + A hook that will be invoked right after the current module is compiled. + + Accepts a module or a `{module, function_name}` tuple. The function + must take two arguments: the module environment and its bytecode. + When just a module is provided, the function is assumed to be + `__after_compile__/2`. + + Callbacks will run in the order they are registered. + + `Module` functions expecting not yet compiled modules (such as `definitions_in/1`) + are still available at the time `@after_compile` is invoked. + + #### Example + + defmodule MyModule do + @after_compile __MODULE__ + + def __after_compile__(env, _bytecode) do + IO.inspect(env) + end + end + + ### `@before_compile` + + A hook that will be invoked before the module is compiled. + + Accepts a module or a `{module, function_or_macro_name}` tuple. The + function/macro must take one argument: the module environment. If + it's a macro, its returned value will be injected at the end of the + module definition before the compilation starts. + + When just a module is provided, the function/macro is assumed to be + `__before_compile__/1`. + + Callbacks will run in the order they are registered. Any overridable + definition will be made concrete before the first callback runs. + A definition may be made overridable again in another before compile + callback and it will be made concrete one last time after all callbacks + run. + + *Note*: unlike `@after_compile`, the callback function/macro must + be placed in a separate module (because when the callback is invoked, + the current module does not yet exist). + + #### Example + + defmodule A do + defmacro __before_compile__(_env) do + quote do + def hello, do: "world" + end + end + end + + defmodule B do + @before_compile A + end + + B.hello() + #=> "world" + + ### `@on_definition` + + A hook that will be invoked when each function or macro in the current + module is defined. Useful when annotating functions. + + Accepts a module or a `{module, function_name}` tuple. The function + must take 6 arguments: + + * the module environment + * the kind of the function/macro: `:def`, `:defp`, `:defmacro`, or `:defmacrop` + * the function/macro name + * the list of quoted arguments + * the list of quoted guards + * the quoted function body + + If the function/macro being defined has multiple clauses, the hook will + be called for each clause. + + Unlike other hooks, `@on_definition` will only invoke functions and + never macros. This is to avoid `@on_definition` callbacks from + redefining functions that have just been defined in favor of more + explicit approaches. + + When just a module is provided, the function is assumed to be + `__on_definition__/6`. + + #### Example + + defmodule Hooks do + def on_def(_env, kind, name, args, guards, body) do + IO.puts("Defining #{kind} named #{name} with args:") + IO.inspect(args) + IO.puts("and guards") + IO.inspect(guards) + IO.puts("and body") + IO.puts(Macro.to_string(body)) + end + end + + defmodule MyModule do + @on_definition {Hooks, :on_def} + + def hello(arg) when is_binary(arg) or is_list(arg) do + "Hello" <> to_string(arg) + end + + def hello(_) do + :ok + end + end + + ## Compile options + + The `@compile` attribute accepts different options that are used by both + Elixir and Erlang compilers. Some of the common use cases are documented + below: + + * `@compile :debug_info` - includes `:debug_info` regardless of the + corresponding setting in `Code.get_compiler_option/1` + + * `@compile {:debug_info, false}` - disables `:debug_info` regardless + of the corresponding setting in `Code.get_compiler_option/1` + + * `@compile {:inline, some_fun: 2, other_fun: 3}` - inlines the given + name/arity pairs. Inlining is applied locally, calls from another + module are not affected by this option + + * `@compile {:autoload, false}` - disables automatic loading of + modules after compilation. Instead, the module will be loaded after + it is dispatched to + + * `@compile {:no_warn_undefined, Mod}` or + `@compile {:no_warn_undefined, {Mod, fun, arity}}` - does not warn if + the given module or the given `Mod.fun/arity` are not defined - It is possible to query a module at runtime to find out which functions and - macros it defines, extract its docstrings, etc. See `__info__/1`. ''' + @type definition :: {atom, arity} + @type def_kind :: :def | :defp | :defmacro | :defmacrop + + @extra_error_msg_defines? "Use Kernel.function_exported?/3 and Kernel.macro_exported?/3 " <> + "to check for public functions and macros instead" + + @extra_error_msg_definitions_in "Use the Module.__info__/1 callback to get public functions and macros instead" + @doc """ - Provides runtime information about functions and macros defined by the - module, enables docstring extraction, etc. + Provides runtime information about functions, macros, and other information + defined by the module. Each module gets an `__info__/1` function when it's compiled. The function - takes one of the following atoms: + takes one of the following items: + + * `:attributes` - a keyword list with all persisted attributes + + * `:compile` - a list with compiler metadata + + * `:functions` - a keyword list of public functions and their arities + + * `:macros` - a keyword list of public macros and their arities + + * `:md5` - the MD5 of the module + + * `:module` - the module atom name + + * `:struct` - if the module defines a struct and if so each field in order + + """ + @callback __info__(:attributes) :: keyword() + @callback __info__(:compile) :: [term()] + @callback __info__(:functions) :: keyword() + @callback __info__(:macros) :: keyword() + @callback __info__(:md5) :: binary() + @callback __info__(:module) :: module() + @callback __info__(:struct) :: list(%{field: atom(), required: boolean()}) | nil - * `:functions` - keyword list of public functions along with their arities + @doc """ + Returns information about module attributes used by Elixir. + + See the "Module attributes" section in the module documentation for more + information on each attribute. - * `:macros` - keyword list of public macros along with their arities + ## Examples - * `:module` - module name (`Module == Module.__info__(:module)`) + iex> map = Module.reserved_attributes() + iex> Map.has_key?(map, :moduledoc) + true + iex> Map.has_key?(map, :doc) + true - In addition to the above, you may also pass to `__info__/1` any atom supported - by Erlang's `module_info` function which also gets defined for each compiled - module. See http://erlang.org/doc/reference_manual/modules.html#id69430 for - more information. """ - def __info__(kind) + @doc since: "1.12.0" + def reserved_attributes() do + %{ + after_compile: %{ + doc: "A hook that will be invoked right after the current module is compiled." + }, + before_compile: %{ + doc: "A hook that will be invoked before the module is compiled." + }, + behaviour: %{ + doc: "Specifies that the current module implements a given behaviour." + }, + on_definition: %{ + doc: + "A hook that will be invoked when each function or macro in the current module is defined." + }, + impl: %{ + doc: "Declares an implementation of a callback function or macro." + }, + compile: %{ + doc: "Defines options for module compilation." + }, + deprecated: %{ + doc: "Provides the deprecation reason for a function." + }, + moduledoc: %{ + doc: "Provides documentation for the current module." + }, + doc: %{ + doc: "Provides documentation for a function/macro/callback." + }, + typedoc: %{ + doc: "Provides documentation for a type." + }, + dialyzer: %{ + doc: "Defines Dialyzer warnings to request or suppress." + }, + external_resource: %{ + doc: "Specifies an external resource for the current module." + }, + file: %{ + doc: + "Changes the filename used in stacktraces for the function or macro that follows the attribute." + }, + on_load: %{ + doc: "A hook that will be invoked whenever the module is loaded." + }, + vsn: %{ + doc: "Specify the module version." + }, + type: %{ + doc: "Defines a type to be used in `@spec`." + }, + typep: %{ + doc: "Defines a private type to be used in `@spec`." + }, + opaque: %{ + doc: "Defines an opaque type to be used in `@spec`." + }, + spec: %{ + doc: "Provides a specification for a function." + }, + callback: %{ + doc: "Provides a specification for a behaviour callback." + }, + macrocallback: %{ + doc: "Provides a specification for a macro behaviour callback." + }, + optional_callbacks: %{ + doc: "Specifies which behaviour callbacks and macro behaviour callbacks are optional." + }, + derive: %{ + doc: + "Derives an implementation for the given protocol for the struct defined in the current module." + }, + enforce_keys: %{ + doc: + "Ensures the given keys are always set when building the struct defined in the current module." + } + } + end @doc """ - Check if a module is open, i.e. it is currently being defined - and its attributes and functions can be modified. + Checks if a module is open. + + A module is "open" if it is currently being defined and its attributes and + functions can be modified. """ - def open?(module) do + @spec open?(module) :: boolean + def open?(module) when is_atom(module) do :elixir_module.is_open(module) end @@ -323,38 +680,61 @@ defmodule Module do ## Examples defmodule Foo do - contents = quote do: (def sum(a, b), do: a + b) - Module.eval_quoted __MODULE__, contents + contents = + quote do + def sum(a, b), do: a + b + end + + Module.eval_quoted(__MODULE__, contents) end - Foo.sum(1, 2) #=> 3 + Foo.sum(1, 2) + #=> 3 - For convenience, you can my pass `__ENV__` as argument and - all options will be automatically extracted from the environment: + For convenience, you can pass any `Macro.Env` struct, such + as `__ENV__/0`, as the first argument or as options. Both + the module and all options will be automatically extracted + from the environment: defmodule Foo do - contents = quote do: (def sum(a, b), do: a + b) - Module.eval_quoted __MODULE__, contents, [], __ENV__ + contents = + quote do + def sum(a, b), do: a + b + end + + Module.eval_quoted(__ENV__, contents) end - Foo.sum(1, 2) #=> 3 + Foo.sum(1, 2) + #=> 3 + Note that if you pass a `Macro.Env` struct as first argument + while also passing `opts`, they will be merged with `opts` + having precedence. """ - def eval_quoted(module, quoted, binding \\ [], opts \\ []) + @spec eval_quoted(module | Macro.Env.t(), Macro.t(), list, keyword | Macro.Env.t()) :: term + def eval_quoted(module_or_env, quoted, binding \\ [], opts \\ []) + + def eval_quoted(%Macro.Env{} = env, quoted, binding, opts) + when is_list(binding) and is_list(opts) do + validated_eval_quoted(env.module, quoted, binding, struct!(env, opts)) + end - def eval_quoted(%Macro.Env{} = env, quoted, binding, opts) do - eval_quoted(env.module, quoted, binding, Keyword.merge(Map.to_list(env), opts)) + def eval_quoted(module, quoted, binding, %Macro.Env{} = env) + when is_atom(module) and is_list(binding) do + validated_eval_quoted(module, quoted, binding, env) end - def eval_quoted(module, quoted, binding, %Macro.Env{} = env) do - eval_quoted(module, quoted, binding, Map.to_list(env)) + def eval_quoted(module, quoted, binding, opts) + when is_atom(module) and is_list(binding) and is_list(opts) do + validated_eval_quoted(module, quoted, binding, opts) end - def eval_quoted(module, quoted, binding, opts) do - assert_not_compiled!(:eval_quoted, module) + defp validated_eval_quoted(module, quoted, binding, env_or_opts) do + assert_not_compiled!({:eval_quoted, 4}, module) :elixir_def.reset_last(module) - {value, binding, _env, _scope} = - :elixir.eval_quoted quoted, binding, Keyword.put(opts, :module, module) + env = :elixir.env_for_eval(env_or_opts) + {value, binding, _env} = :elixir.eval_quoted(quoted, binding, %{env | module: module}) {value, binding} end @@ -365,6 +745,15 @@ defmodule Module do The line where the module is defined and its file **must** be passed as options. + It returns a tuple of shape `{:module, module, binary, term}` + where `module` is the module name, `binary` is the module + bytecode and `term` is the result of the last expression in + `quoted`. + + Similar to `Kernel.defmodule/2`, the binary will only be + written to disk as a `.beam` file if `Module.create/3` is + invoked in a file that is currently being compiled. + ## Examples contents = @@ -374,24 +763,26 @@ defmodule Module do Module.create(Hello, contents, Macro.Env.location(__ENV__)) - Hello.world #=> true + Hello.world() + #=> true ## Differences from `defmodule` - `Module.create` works similarly to `defmodule` and - return the same results. While one could also use - `defmodule` to define modules dynamically, this - function is preferred when the module body is given - by a quoted expression. + `Module.create/3` works similarly to `Kernel.defmodule/2` + and return the same results. While one could also use + `Kernel.defmodule/2` to define modules dynamically, this function + is preferred when the module body is given by a quoted + expression. - Another important distinction is that `Module.create` + Another important distinction is that `Module.create/3` allows you to control the environment variables used - when defining the module, while `defmodule` automatically - shares the same environment. + when defining the module, while `Kernel.defmodule/2` + automatically uses the environment it is invoked at. """ + @spec create(module, Macro.t(), Macro.Env.t() | keyword) :: {:module, module, binary, term} def create(module, quoted, opts) - def create(module, quoted, %Macro.Env{} = env) do + def create(module, quoted, %Macro.Env{} = env) when is_atom(module) do create(module, quoted, Map.to_list(env)) end @@ -399,12 +790,18 @@ defmodule Module do unless Keyword.has_key?(opts, :file) do raise ArgumentError, "expected :file to be given as option" end + + next = :elixir_module.next_counter(nil) + line = Keyword.get(opts, :line, 0) + quoted = :elixir_quote.linify_with_context_counter(line, {module, next}, quoted) :elixir_module.compile(module, quoted, [], :elixir.env_for_eval(opts)) end @doc """ Concatenates a list of aliases and returns a new alias. + It handles binaries and atoms. + ## Examples iex> Module.concat([Foo, Bar]) @@ -422,6 +819,8 @@ defmodule Module do @doc """ Concatenates two aliases and returns a new alias. + It handles binaries and atoms. + ## Examples iex> Module.concat(Foo, Bar) @@ -432,165 +831,200 @@ defmodule Module do """ @spec concat(binary | atom, binary | atom) :: atom - def concat(left, right) do + def concat(left, right) + when (is_binary(left) or is_atom(left)) and (is_binary(right) or is_atom(right)) do :elixir_aliases.concat([left, right]) end @doc """ - Concatenates a list of aliases and returns a new alias only - if the alias was already referenced. If the alias was not - referenced yet, fails with `ArgumentError`. - It handles char lists, binaries and atoms. + Concatenates a list of aliases and returns a new alias only if the alias + was already referenced. - ## Examples + If the alias was not referenced yet, fails with `ArgumentError`. + It handles binaries and atoms. - iex> Module.safe_concat([Unknown, Module]) - ** (ArgumentError) argument error + ## Examples iex> Module.safe_concat([List, Chars]) List.Chars """ - @spec safe_concat([binary | atom]) :: atom | no_return + @spec safe_concat([binary | atom]) :: atom def safe_concat(list) when is_list(list) do :elixir_aliases.safe_concat(list) end @doc """ - Concatenates two aliases and returns a new alias only - if the alias was already referenced. If the alias was not - referenced yet, fails with `ArgumentError`. - It handles char lists, binaries and atoms. + Concatenates two aliases and returns a new alias only if the alias was + already referenced. - ## Examples + If the alias was not referenced yet, fails with `ArgumentError`. + It handles binaries and atoms. - iex> Module.safe_concat(Unknown, Module) - ** (ArgumentError) argument error + ## Examples iex> Module.safe_concat(List, Chars) List.Chars """ - @spec safe_concat(binary | atom, binary | atom) :: atom | no_return - def safe_concat(left, right) do + @spec safe_concat(binary | atom, binary | atom) :: atom + def safe_concat(left, right) + when (is_binary(left) or is_atom(left)) and (is_binary(right) or is_atom(right)) do :elixir_aliases.safe_concat([left, right]) end - @doc """ - Gets an anonymous function from the given module, function - and arity. The module and function are not verified to exist. + # Build signatures to be stored in docs - iex> fun = Module.function(Kernel, :is_atom, 1) - iex> fun.(:hello) - true + defp build_signature(args, env) do + {reverse_args, counters} = simplify_args(args, %{}, [], env) + expand_keys(reverse_args, counters, []) + end - """ - def function(mod, fun, arity) do - :erlang.make_fun(mod, fun, arity) + defp simplify_args([arg | args], counters, acc, env) do + {arg, counters} = simplify_arg(arg, counters, env) + simplify_args(args, counters, [arg | acc], env) end - @doc """ - Attaches documentation to a given function or type. It expects - the module the function/type belongs to, the line (a non negative - integer), the kind (`def` or `defmacro`), a tuple representing - the function and its arity, the function signature (the signature - should be omitted for types) and the documentation, which should - be either a binary or a boolean. + defp simplify_args([], counters, reverse_args, _env) do + {reverse_args, counters} + end - ## Examples + defp simplify_arg({:\\, _, [left, right]}, counters, env) do + {left, counters} = simplify_arg(left, counters, env) - defmodule MyModule do - Module.add_doc(__MODULE__, __ENV__.line + 1, :def, {:version, 0}, [], "Manually added docs") - def version, do: 1 - end + right = + Macro.prewalk(right, fn + {:@, _, _} = attr -> Macro.expand_once(attr, env) + other -> other + end) - """ - def add_doc(module, line, kind, tuple, signature \\ [], doc) + {{:\\, [], [left, right]}, counters} + end - def add_doc(_module, _line, kind, _tuple, _signature, doc) when kind in [:defp, :defmacrop, :typep] do - if doc, do: {:error, :private_doc}, else: :ok + # If the variable is being used explicitly for naming, + # we always give it a higher priority (nil) even if it + # starts with underscore. + defp simplify_arg({:=, _, [{var, _, atom}, _]}, counters, _env) when is_atom(atom) do + {simplify_var(var, nil), counters} end - def add_doc(module, line, kind, tuple, signature, doc) when - kind in [:def, :defmacro, :type, :opaque] and (is_binary(doc) or is_boolean(doc) or doc == nil) do - assert_not_compiled!(:add_doc, module) - table = docs_table_for(module) + defp simplify_arg({:=, _, [_, {var, _, atom}]}, counters, _env) when is_atom(atom) do + {simplify_var(var, nil), counters} + end - {signature, _} = :lists.mapfoldl fn(x, acc) -> - {simplify_signature(x, acc), acc + 1} - end, 1, signature + # If we have only the variable as argument, it also gets + # higher priority. However, if the variable starts with an + # underscore, we give it a secondary context (Elixir) with + # lower priority. + defp simplify_arg({var, _, atom}, counters, _env) when is_atom(atom) do + {simplify_var(var, Elixir), counters} + end - case :ets.lookup(table, tuple) do - [] -> - :ets.insert(table, {tuple, line, kind, signature, doc}) - :ok - [{tuple, line, _old_kind, old_sign, old_doc}] -> - :ets.insert(table, { - tuple, - line, - kind, - merge_signatures(old_sign, signature, 1), - if(nil?(doc), do: old_doc, else: doc) - }) - :ok + defp simplify_arg({:%, _, [left, _]}, counters, env) do + case Macro.expand_once(left, env) do + module when is_atom(module) -> autogenerated_key(counters, simplify_module_name(module)) + _ -> autogenerated_key(counters, :struct) end end - # Simplify signatures to be stored in docs - - defp simplify_signature({:\\, _, [left, right ]}, i) do - {:\\, [], [simplify_signature(left, i), right]} + defp simplify_arg({:%{}, _, _}, counters, _env) do + autogenerated_key(counters, :map) end - defp simplify_signature({:%, _, [left, _]}, _i) when is_atom(left) do - last = List.last(String.split(Atom.to_string(left), ".")) - atom = String.to_atom(downcase(last)) - {atom, [], nil} + defp simplify_arg({:@, _, _} = attr, counters, env) do + simplify_arg(Macro.expand_once(attr, env), counters, env) end - defp simplify_signature({:=, _, [_, right]}, i) do - simplify_signature(right, i) + defp simplify_arg({:var!, _, [{var, _, atom} | _]}, counters, _env) when is_atom(atom) do + {simplify_var(var, Elixir), counters} end - defp simplify_signature({var, _, atom}, _i) when is_atom(atom) do + defp simplify_arg(other, counters, _env) when is_integer(other), + do: autogenerated_key(counters, :int) + + defp simplify_arg(other, counters, _env) when is_boolean(other), + do: autogenerated_key(counters, :bool) + + defp simplify_arg(other, counters, _env) when is_atom(other), + do: autogenerated_key(counters, :atom) + + defp simplify_arg(other, counters, _env) when is_list(other), + do: autogenerated_key(counters, :list) + + defp simplify_arg(other, counters, _env) when is_float(other), + do: autogenerated_key(counters, :float) + + defp simplify_arg(other, counters, _env) when is_binary(other), + do: autogenerated_key(counters, :binary) + + defp simplify_arg(_, counters, _env), do: autogenerated_key(counters, :arg) + + defp simplify_var(var, guess_priority) do case Atom.to_string(var) do - "_" <> rest -> {String.to_atom(rest), [], Elixir} - _ -> {var, [], nil} + "_" -> {:_, [], guess_priority} + "_" <> rest -> {String.to_atom(rest), [], guess_priority} + _ -> {var, [], nil} + end + end + + defp simplify_module_name(module) when is_atom(module) do + try do + split(module) + rescue + ArgumentError -> module + else + module_name -> String.to_atom(Macro.underscore(List.last(module_name))) + end + end + + defp autogenerated_key(counters, key) do + case counters do + %{^key => :once} -> {key, %{counters | key => 2}} + %{^key => value} -> {key, %{counters | key => value + 1}} + %{} -> {key, Map.put(counters, key, :once)} end end - defp simplify_signature(other, i) when is_integer(other), do: {:"int#{i}", [], Elixir} - defp simplify_signature(other, i) when is_boolean(other), do: {:"bool#{i}", [], Elixir} - defp simplify_signature(other, i) when is_atom(other), do: {:"atom#{i}", [], Elixir} - defp simplify_signature(other, i) when is_list(other), do: {:"list#{i}", [], Elixir} - defp simplify_signature(other, i) when is_float(other), do: {:"float#{i}", [], Elixir} - defp simplify_signature(other, i) when is_binary(other), do: {:"binary#{i}", [], Elixir} - defp simplify_signature(_, i), do: {:"arg#{i}", [], Elixir} + defp expand_keys([{:\\, meta, [key, default]} | keys], counters, acc) when is_atom(key) do + {var, counters} = expand_key(key, counters) + expand_keys(keys, counters, [{:\\, meta, [var, default]} | acc]) + end + + defp expand_keys([key | keys], counters, acc) when is_atom(key) do + {var, counters} = expand_key(key, counters) + expand_keys(keys, counters, [var | acc]) + end - defp downcase(<>) when c >= ?A and c <= ?Z do - <> + defp expand_keys([arg | args], counters, acc) do + expand_keys(args, counters, [arg | acc]) end - defp downcase(<>) do - <> + defp expand_keys([], _counters, acc) do + acc end - defp downcase(<<>>) do - <<>> + defp expand_key(key, counters) do + case counters do + %{^key => count} when is_integer(count) and count >= 1 -> + {{:"#{key}#{count}", [], Elixir}, Map.put(counters, key, count - 1)} + + _ -> + {{key, [], Elixir}, counters} + end end # Merge - defp merge_signatures([h1|t1], [h2|t2], i) do - [merge_signature(h1, h2, i)|merge_signatures(t1, t2, i + 1)] + defp merge_signatures([h1 | t1], [h2 | t2], i) do + [merge_signature(h1, h2, i) | merge_signatures(t1, t2, i + 1)] end defp merge_signatures([], [], _) do [] end - defp merge_signature({:\\, line, [left, right]}, newer, i) do - {:\\, line, [merge_signature(left, newer, i), right]} + defp merge_signature({:\\, meta, [left, right]}, newer, i) do + {:\\, meta, [merge_signature(left, newer, i), right]} end defp merge_signature(older, {:\\, _, [left, _]}, i) do @@ -598,270 +1032,556 @@ defmodule Module do end # The older signature, when given, always have higher precedence - defp merge_signature({_, _, nil} = older, _newer, _), do: older - defp merge_signature(_older, {_, _, nil} = newer, _), do: newer + defp merge_signature({_, _, nil} = older, _newer, _), do: older + defp merge_signature(_older, {_, _, nil} = newer, _), do: newer # Both are a guess, so check if they are the same guess defp merge_signature({var, _, _} = older, {var, _, _}, _), do: older # Otherwise, returns a generic guess - defp merge_signature({_, line, _}, _newer, i), do: {:"arg#{i}", line, Elixir} + defp merge_signature({_, meta, _}, _newer, i), do: {:"arg#{i}", meta, Elixir} @doc """ Checks if the module defines the given function or macro. + Use `defines?/3` to assert for a specific type. + This function can only be used on modules that have not yet been compiled. + Use `Kernel.function_exported?/3` and `Kernel.macro_exported?/3` to check for + public functions and macros respectively in compiled modules. + + Note that `defines?` returns false for functions and macros that have + been defined but then marked as overridable and no other implementation + has been provided. You can check the overridable status by calling + `overridable?/2`. + ## Examples defmodule Example do - Module.defines? __MODULE__, {:version, 0} #=> false + Module.defines?(__MODULE__, {:version, 0}) #=> false def version, do: 1 - Module.defines? __MODULE__, {:version, 0} #=> true + Module.defines?(__MODULE__, {:version, 0}) #=> true end """ - def defines?(module, tuple) when is_tuple(tuple) do - assert_not_compiled!(:defines?, module) - table = function_table_for(module) - :ets.lookup(table, tuple) != [] + @spec defines?(module, definition) :: boolean + def defines?(module, {name, arity} = tuple) + when is_atom(module) and is_atom(name) and is_integer(arity) and arity >= 0 and arity <= 255 do + assert_not_compiled!(__ENV__.function, module, @extra_error_msg_defines?) + {set, _bag} = data_tables_for(module) + :ets.member(set, {:def, tuple}) end @doc """ Checks if the module defines a function or macro of the - given `kind`. `kind` can be any of `:def`, `:defp`, - `:defmacro` or `:defmacrop`. + given `kind`. + + `kind` can be any of `:def`, `:defp`, `:defmacro`, or `:defmacrop`. + + This function can only be used on modules that have not yet been compiled. + Use `Kernel.function_exported?/3` and `Kernel.macro_exported?/3` to check for + public functions and macros respectively in compiled modules. ## Examples defmodule Example do - Module.defines? __MODULE__, {:version, 0}, :defp #=> false + Module.defines?(__MODULE__, {:version, 0}, :def) #=> false def version, do: 1 - Module.defines? __MODULE__, {:version, 0}, :defp #=> false + Module.defines?(__MODULE__, {:version, 0}, :def) #=> true end """ - def defines?(module, tuple, kind) do - assert_not_compiled!(:defines?, module) - table = function_table_for(module) - case :ets.lookup(table, tuple) do - [{_, ^kind, _, _, _, _, _}] -> true + @spec defines?(module, definition, def_kind) :: boolean + def defines?(module, {name, arity} = tuple, def_kind) + when is_atom(module) and is_atom(name) and is_integer(arity) and arity >= 0 and arity <= 255 and + def_kind in [:def, :defp, :defmacro, :defmacrop] do + assert_not_compiled!(__ENV__.function, module, @extra_error_msg_defines?) + + {set, _bag} = data_tables_for(module) + + case :ets.lookup(set, {:def, tuple}) do + [{_, ^def_kind, _, _, _, _}] -> true _ -> false end end @doc """ - Return all functions defined in `module`. + Checks if the current module defines the given type (private, opaque or not). - ## Examples + This function is only available for modules being compiled. + """ + @doc since: "1.7.0" + @spec defines_type?(module, definition) :: boolean + def defines_type?(module, definition) when is_atom(module) do + Kernel.Typespec.defines_type?(module, definition) + end - defmodule Example do - def version, do: 1 - Module.definitions_in __MODULE__ #=> [{:version,0}] - end + @doc """ + Copies the given spec as a callback. + Returns `true` if there is such a spec and it was copied as a callback. + If the function associated to the spec has documentation defined prior to + invoking this function, the docs are copied too. """ - def definitions_in(module) do - assert_not_compiled!(:definitions_in, module) - table = function_table_for(module) - for {tuple, _, _, _, _, _, _} <- :ets.tab2list(table), do: tuple + @doc since: "1.7.0" + @spec spec_to_callback(module, definition) :: boolean + def spec_to_callback(module, definition) do + Kernel.Typespec.spec_to_callback(module, definition) end @doc """ - Returns all functions defined in `module`, according - to its kind. + Returns all module attributes names defined in `module`. + + This function can only be used on modules that have not yet been compiled. ## Examples defmodule Example do - def version, do: 1 - Module.definitions_in __MODULE__, :def #=> [{:version,0}] - Module.definitions_in __MODULE__, :defp #=> [] + @foo 1 + Module.register_attribute(__MODULE__, :bar, accumulate: true) + + :foo in Module.attributes_in(__MODULE__) + #=> true + + :bar in Module.attributes_in(__MODULE__) + #=> true end """ - def definitions_in(module, kind) do - assert_not_compiled!(:definitions_in, module) - table = function_table_for(module) - for {tuple, stored_kind, _, _, _, _, _} <- :ets.tab2list(table), stored_kind == kind, do: tuple + @doc since: "1.13.0" + @spec attributes_in(module) :: [atom] + def attributes_in(module) when is_atom(module) do + assert_not_compiled!(__ENV__.function, module) + {set, _} = data_tables_for(module) + :ets.select(set, [{{:"$1", :_, :_}, [{:is_atom, :"$1"}], [:"$1"]}]) end @doc """ - Makes the given functions in `module` overridable. - An overridable function is lazily defined, allowing a - developer to customize it. See `Kernel.defoverridable/1` for - more information and documentation. - """ - def make_overridable(module, tuples) do - assert_not_compiled!(:make_overridable, module) - - for tuple <- tuples do - case :elixir_def.lookup_definition(module, tuple) do - false -> - {name, arity} = tuple - raise "Cannot make function #{name}/#{arity} overridable because it was not defined" - clause -> - :elixir_def.delete_definition(module, tuple) - - neighbours = if loaded?(Module.LocalsTracker) do - Module.LocalsTracker.yank(module, tuple) - else - [] - end + Returns all overridable definitions in `module`. + + Note a definition is included even if it was was already overridden. + You can use `defines?/2` to see if a definition exists or one is pending. + + This function can only be used on modules that have not yet been compiled. + + ## Examples + + defmodule Example do + def foo, do: 1 + def bar, do: 2 - old = get_attribute(module, :__overridable) - merged = :orddict.update(tuple, fn({count, _, _, _}) -> - {count + 1, clause, neighbours, false} - end, {1, clause, neighbours, false}, old) + defoverridable foo: 0, bar: 0 + def foo, do: 3 - put_attribute(module, :__overridable, merged) + [bar: 0, foo: 0] = Module.overridables_in(__MODULE__) |> Enum.sort() end - end - end - @doc """ - Returns `true` if `tuple` in `module` is marked as overridable. """ - def overridable?(module, tuple) do - !!List.keyfind(get_attribute(module, :__overridable), tuple, 0) + @doc since: "1.13.0" + @spec overridables_in(module) :: [atom] + def overridables_in(module) when is_atom(module) do + assert_not_compiled!(__ENV__.function, module) + :elixir_overridable.overridables_for(module) end @doc """ - Puts an Erlang attribute to the given module with the given - key and value. The semantics of putting the attribute depends - if the attribute was registered or not via `register_attribute/3`. + Returns all functions and macros defined in `module`. + + It returns a list with all defined functions and macros, public and private, + in the shape of `[{name, arity}, ...]`. + + This function can only be used on modules that have not yet been compiled. + Use the `c:Module.__info__/1` callback to get the public functions and macros in + compiled modules. ## Examples - defmodule MyModule do - Module.put_attribute __MODULE__, :custom_threshold_for_lib, 10 + defmodule Example do + def version, do: 1 + defmacrop test(arg), do: arg + Module.definitions_in(__MODULE__) #=> [{:version, 0}, {:test, 1}] end """ - def put_attribute(module, key, value) when is_atom(key) do - assert_not_compiled!(:put_attribute, module) - table = data_table_for(module) - value = normalize_attribute(key, value) - acc = :ets.lookup_element(table, :__acc_attributes, 2) - - new = - if :lists.member(key, acc) do - case :ets.lookup(table, key) do - [{^key, old}] -> [value|old] - [] -> [value] - end - else - value - end - - :ets.insert(table, {key, new}) + @spec definitions_in(module) :: [definition] + def definitions_in(module) when is_atom(module) do + assert_not_compiled!(__ENV__.function, module, @extra_error_msg_definitions_in) + {_, bag} = data_tables_for(module) + bag_lookup_element(bag, :defs, 2) end @doc """ - Gets the given attribute from a module. If the attribute - was marked with `accumulate` with `Module.register_attribute/3`, - a list is always returned. + Returns all functions defined in `module`, according + to its kind. - The `@` macro compiles to a call to this function. For example, - the following code: + This function can only be used on modules that have not yet been compiled. + Use the `c:Module.__info__/1` callback to get the public functions and macros in + compiled modules. - @foo + ## Examples + + defmodule Example do + def version, do: 1 + Module.definitions_in(__MODULE__, :def) #=> [{:version, 0}] + Module.definitions_in(__MODULE__, :defp) #=> [] + end - Expands to: + """ + @spec definitions_in(module, def_kind) :: [definition] + def definitions_in(module, kind) + when is_atom(module) and kind in [:def, :defp, :defmacro, :defmacrop] do + assert_not_compiled!(__ENV__.function, module, @extra_error_msg_definitions_in) + {set, _} = data_tables_for(module) + :ets.select(set, [{{{:def, :"$1"}, kind, :_, :_, :_, :_}, [], [:"$1"]}]) + end - Module.get_attribute(__MODULE__, :foo, true) + @doc """ + Returns the definition for the given name-arity pair. - Notice the third argument may be given to indicate a stacktrace - to be emitted when the attribute was not previously defined. - The default value for `warn` is nil for direct calls but the `@foo` - macro sets it to the proper stacktrace automatically, warning - every time `@foo` is used but not set previously. + It returns a tuple with the `version`, the `kind`, + the definition `metadata`, and a list with each clause. + Each clause is a four-element tuple with metadata, + the arguments, the guards, and the clause AST. - ## Examples + The clauses are returned in the Elixir AST but a subset + that has already been expanded and normalized. This makes + it useful for analyzing code but it cannot be reinjected + into the module as it will have lost some of its original + context. Given this AST representation is mostly internal, + it is versioned and it may change at any time. Therefore, + **use this API with caution**. - defmodule Foo do - Module.put_attribute __MODULE__, :value, 1 - Module.get_attribute __MODULE__, :value #=> 1 + ## Options - Module.register_attribute __MODULE__, :value, accumulate: true - Module.put_attribute __MODULE__, :value, 1 - Module.get_attribute __MODULE__, :value #=> [1] - end + * `:nillify_clauses` (since v1.13.0) - returns `nil` instead + of returning the clauses. This is useful when there is + only an interest in fetching the kind and metadata """ - @spec get_attribute(module, atom, warn :: nil | [tuple]) :: term - def get_attribute(module, key, warn \\ nil) when - is_atom(key) and (is_list(warn) or nil?(warn)) do - assert_not_compiled!(:get_attribute, module) - table = data_table_for(module) + @spec get_definition(module, definition, keyword) :: + {:v1, def_kind, meta :: keyword, + [{meta :: keyword, arguments :: [Macro.t()], guards :: [Macro.t()], Macro.t()}] | nil} + @doc since: "1.12.0" + def get_definition(module, {name, arity}, options \\ []) + when is_atom(module) and is_atom(name) and is_integer(arity) and is_list(options) do + assert_not_compiled!(__ENV__.function, module, "") + {set, bag} = data_tables_for(module) + + case :ets.lookup(set, {:def, {name, arity}}) do + [{_key, kind, meta, _, _, _}] -> + clauses = + if options[:nillify_clauses], + do: nil, + else: bag_lookup_element(bag, {:clauses, {name, arity}}, 2) + + {:v1, kind, meta, clauses} - case :ets.lookup(table, key) do - [{^key, val}] -> val [] -> - acc = :ets.lookup_element(table, :__acc_attributes, 2) - - cond do - :lists.member(key, acc) -> - [] - is_list(warn) -> - :elixir_errors.warn warn_info(warn), "undefined module attribute @#{key}, " <> - "please remove access to @#{key} or explicitly set it to nil before access\n" - nil - true -> - nil - end + nil end end - defp warn_info([entry|_]) do - opts = elem(entry, tuple_size(entry) - 1) - Exception.format_file_line(Keyword.get(opts, :file), Keyword.get(opts, :line)) <> " " - end + @doc """ + Deletes a definition from a module. - defp warn_info([]) do - "" + It returns true if the definition exists and it was removed, + otherwise it returns false. + """ + @doc since: "1.12.0" + @spec delete_definition(module, definition) :: boolean() + def delete_definition(module, {name, arity}) + when is_atom(module) and is_atom(name) and is_integer(arity) do + assert_not_readonly!(__ENV__.function, module) + + case :elixir_def.take_definition(module, {name, arity}) do + false -> + false + + _ -> + :elixir_locals.yank({name, arity}, module) + true + end end @doc """ - Deletes all attributes that match the given key. + Makes the given functions in `module` overridable. + + An overridable function is lazily defined, allowing a + developer to customize it. See `Kernel.defoverridable/1` for + more information and documentation. + + Once a function or a macro is marked as overridable, it will + no longer be listed under `definitions_in/1` or return true + when given to `defines?/2` until another implementation is + given. + """ + @spec make_overridable(module, [definition]) :: :ok + def make_overridable(module, tuples) when is_atom(module) and is_list(tuples) do + assert_not_readonly!(__ENV__.function, module) + + func = fn + {function_name, arity} = tuple + when is_atom(function_name) and is_integer(arity) and arity >= 0 and arity <= 255 -> + case :elixir_def.take_definition(module, tuple) do + false -> + raise ArgumentError, + "cannot make function #{function_name}/#{arity} " <> + "overridable because it was not defined" + + clause -> + neighbours = :elixir_locals.yank(tuple, module) + :elixir_overridable.record_overridable(module, tuple, clause, neighbours) + end + + other -> + raise ArgumentError, + "each element in tuple list has to be a " <> + "{function_name :: atom, arity :: 0..255} tuple, got: #{inspect(other)}" + end + + :lists.foreach(func, tuples) + end + + @spec make_overridable(module, module) :: :ok + def make_overridable(module, behaviour) when is_atom(module) and is_atom(behaviour) do + case check_module_for_overridable(module, behaviour) do + :ok -> + :ok + + {:error, error_explanation} -> + raise ArgumentError, + "cannot pass module #{inspect(behaviour)} as argument " <> + "to defoverridable/1 because #{error_explanation}" + end + + behaviour_callbacks = + for callback <- behaviour_info(behaviour, :callbacks) do + {pair, _kind} = normalize_macro_or_function_callback(callback) + pair + end + + tuples = + for definition <- definitions_in(module), + definition in behaviour_callbacks, + do: definition + + make_overridable(module, tuples) + end + + defp check_module_for_overridable(module, behaviour) do + {_, bag} = data_tables_for(module) + behaviour_definitions = bag_lookup_element(bag, {:accumulate, :behaviour}, 2) + + cond do + not Code.ensure_loaded?(behaviour) -> + {:error, "it was not defined"} + + not function_exported?(behaviour, :behaviour_info, 1) -> + {:error, "it does not define any callbacks"} + + behaviour not in behaviour_definitions -> + error_message = + "its corresponding behaviour is missing. Did you forget to " <> + "add @behaviour #{inspect(behaviour)}?" + + {:error, error_message} + + true -> + :ok + end + end + + defp normalize_macro_or_function_callback({function_name, arity}) do + case :erlang.atom_to_list(function_name) do + # Macros are always provided one extra argument in behaviour_info/1 + 'MACRO-' ++ tail -> + {{:erlang.list_to_atom(tail), arity - 1}, :defmacro} + + _ -> + {{function_name, arity}, :def} + end + end + + defp behaviour_info(module, key) do + case module.behaviour_info(key) do + list when is_list(list) -> list + :undefined -> [] + end + end + + @doc """ + Returns `true` if `tuple` in `module` was marked as overridable + at some point. + + Note `overridable?/2` returns true even if the definition was + already overridden. You can use `defines?/2` to see if a definition + exists or one is pending. + """ + @spec overridable?(module, definition) :: boolean + def overridable?(module, {function_name, arity} = tuple) + when is_atom(function_name) and is_integer(arity) and arity >= 0 and arity <= 255 do + :elixir_overridable.overridable_for(module, tuple) != :not_overridable + end + + @doc """ + Puts a module attribute with `key` and `value` in the given `module`. + + ## Examples + + defmodule MyModule do + Module.put_attribute(__MODULE__, :custom_threshold_for_lib, 10) + end + + """ + @spec put_attribute(module, atom, term) :: :ok + def put_attribute(module, key, value) when is_atom(module) and is_atom(key) do + __put_attribute__(module, key, value, nil) + end + + @doc """ + Gets the given attribute from a module. + + If the attribute was marked with `accumulate` with + `Module.register_attribute/3`, a list is always returned. + `nil` is returned if the attribute has not been marked with + `accumulate` and has not been set to any value. + + The `@` macro compiles to a call to this function. For example, + the following code: + + @foo + + Expands to something akin to: + + Module.get_attribute(__MODULE__, :foo) + + This function can only be used on modules that have not yet been compiled. + Use the `c:Module.__info__/1` callback to get all persisted attributes, or + `Code.fetch_docs/1` to retrieve all documentation related attributes in + compiled modules. + + ## Examples + + defmodule Foo do + Module.put_attribute(__MODULE__, :value, 1) + Module.get_attribute(__MODULE__, :value) #=> 1 + + Module.get_attribute(__MODULE__, :value, :default) #=> 1 + Module.get_attribute(__MODULE__, :not_found, :default) #=> :default + + Module.register_attribute(__MODULE__, :value, accumulate: true) + Module.put_attribute(__MODULE__, :value, 1) + Module.get_attribute(__MODULE__, :value) #=> [1] + end + + """ + @spec get_attribute(module, atom, term) :: term + def get_attribute(module, key, default \\ nil) when is_atom(module) and is_atom(key) do + case __get_attribute__(module, key, nil) do + nil -> default + value -> value + end + end + + @doc """ + Checks if the given attribute has been defined. + + An attribute is defined if it has been registered with `register_attribute/3` + or assigned a value. If an attribute has been deleted with `delete_attribute/2` + it is no longer considered defined. + + This function can only be used on modules that have not yet been compiled. + + ## Examples + + defmodule MyModule do + @value 1 + Module.register_attribute(__MODULE__, :other_value) + Module.put_attribute(__MODULE__, :another_value, 1) + + Module.has_attribute?(__MODULE__, :value) #=> true + Module.has_attribute?(__MODULE__, :other_value) #=> true + Module.has_attribute?(__MODULE__, :another_value) #=> true + + Module.has_attribute?(__MODULE__, :undefined) #=> false + + Module.delete_attribute(__MODULE__, :value) + Module.has_attribute?(__MODULE__, :value) #=> false + end + + """ + @doc since: "1.10.0" + @spec has_attribute?(module, atom) :: boolean + def has_attribute?(module, key) when is_atom(module) and is_atom(key) do + assert_not_compiled!(__ENV__.function, module) + {set, _bag} = data_tables_for(module) + + :ets.member(set, key) + end + + @doc """ + Deletes the entry (or entries) for the given module attribute. + + It returns the deleted attribute value. If the attribute has not + been set nor configured to accumulate, it returns `nil`. + + If the attribute is set to accumulate, then this function always + returns a list. Deleting the attribute removes existing entries + but the attribute will still accumulate. ## Examples defmodule MyModule do - Module.put_attribute __MODULE__, :custom_threshold_for_lib, 10 - Module.delete_attribute __MODULE__, :custom_threshold_for_lib + Module.put_attribute(__MODULE__, :custom_threshold_for_lib, 10) + Module.delete_attribute(__MODULE__, :custom_threshold_for_lib) end """ - def delete_attribute(module, key) when is_atom(key) do - assert_not_compiled!(:delete_attribute, module) - table = data_table_for(module) - :ets.delete(table, key) + @spec delete_attribute(module, atom) :: term + def delete_attribute(module, key) when is_atom(module) and is_atom(key) do + assert_not_readonly!(__ENV__.function, module) + {set, bag} = data_tables_for(module) + + case :ets.lookup(set, key) do + [{_, _, :accumulate}] -> + reverse_values(:ets.take(bag, {:accumulate, key}), []) + + [{_, value, _}] -> + :ets.delete(set, key) + value + + [] -> + nil + end end + defp reverse_values([{_, value} | tail], acc), do: reverse_values(tail, [value | acc]) + defp reverse_values([], acc), do: acc + @doc """ - Registers an attribute. By registering an attribute, a developer - is able to customize how Elixir will store and accumulate the - attribute values. + Registers an attribute. + + By registering an attribute, a developer is able to customize + how Elixir will store and accumulate the attribute values. ## Options When registering an attribute, two options can be given: * `:accumulate` - several calls to the same attribute will - accumulate instead of override the previous one. New attributes + accumulate instead of overriding the previous one. New attributes are always added to the top of the accumulated list. * `:persist` - the attribute will be persisted in the Erlang Abstract Format. Useful when interfacing with Erlang libraries. - By default, both options are `false`. + By default, both options are `false`. Once an attribute has been + set to accumulate or persist, the behaviour cannot be reverted. ## Examples defmodule MyModule do - Module.register_attribute __MODULE__, - :custom_threshold_for_lib, - accumulate: true, persist: false + Module.register_attribute(__MODULE__, :custom_threshold_for_lib, accumulate: true) @custom_threshold_for_lib 10 @custom_threshold_for_lib 20 @@ -869,116 +1589,849 @@ defmodule Module do end """ - def register_attribute(module, new, opts) when is_atom(new) do - assert_not_compiled!(:register_attribute, module) - table = data_table_for(module) - - if Keyword.get(opts, :persist) do - old = :ets.lookup_element(table, :__persisted_attributes, 2) - :ets.insert(table, {:__persisted_attributes, [new|old]}) + @spec register_attribute(module, atom, [{:accumulate, boolean}, {:persist, boolean}]) :: :ok + def register_attribute(module, attribute, options) + when is_atom(module) and is_atom(attribute) and is_list(options) do + assert_not_readonly!(__ENV__.function, module) + {set, bag} = data_tables_for(module) + + if Keyword.get(options, :persist) do + :ets.insert(bag, {:persisted_attributes, attribute}) end - if Keyword.get(opts, :accumulate) do - old = :ets.lookup_element(table, :__acc_attributes, 2) - :ets.insert(table, {:__acc_attributes, [new|old]}) + if Keyword.get(options, :accumulate) do + :ets.insert_new(set, {attribute, [], :accumulate}) || + :ets.update_element(set, attribute, {3, :accumulate}) + else + :ets.insert_new(bag, {:warn_attributes, attribute}) + :ets.insert_new(set, {attribute, nil, :unset}) end + + :ok end @doc """ - Split the given module name into binary parts. + Splits the given module name into binary parts. + + `module` has to be an Elixir module, as `split/1` won't work with Erlang-style + modules (for example, `split(:lists)` raises an error). + + `split/1` also supports splitting the string representation of Elixir modules + (that is, the result of calling `Atom.to_string/1` with the module name). ## Examples - Module.split Very.Long.Module.Name.And.Even.Longer - #=> ["Very", "Long", "Module", "Name", "And", "Even", "Longer"] + iex> Module.split(Very.Long.Module.Name.And.Even.Longer) + ["Very", "Long", "Module", "Name", "And", "Even", "Longer"] + iex> Module.split("Elixir.String.Chars") + ["String", "Chars"] """ - def split(module) do - tl(String.split(String.Chars.to_string(module), ".")) + @spec split(module | String.t()) :: [String.t(), ...] + def split(module) + + def split(module) when is_atom(module) do + split(Atom.to_string(module), _original = module) + end + + def split(module) when is_binary(module) do + split(module, _original = module) + end + + defp split("Elixir." <> name, _original) do + String.split(name, ".") + end + + defp split(_module, original) do + raise ArgumentError, "expected an Elixir module, got: #{inspect(original)}" end @doc false - # Used internally to compile documentation. This function - # is private and must be used only internally. - def compile_doc(env, kind, name, args, _guards, _body) do - module = env.module - line = env.line - arity = length(args) - pair = {name, arity} - doc = get_attribute(module, :doc) - - case add_doc(module, line, kind, pair, args, doc) do - :ok -> - :ok - {:error, :private_doc} -> - :elixir_errors.warn line, env.file, "function #{name}/#{arity} is private, @doc's are always discarded for private functions\n" + @deprecated "Use @doc instead" + def add_doc(module, line, kind, {name, arity}, signature \\ [], doc) do + assert_not_compiled!(__ENV__.function, module) + + if kind in [:defp, :defmacrop, :typep] do + if doc, do: {:error, :private_doc}, else: :ok + else + {set, _bag} = data_tables_for(module) + compile_doc(set, nil, line, kind, name, arity, signature, nil, doc, %{}, __ENV__, false) + :ok + end + end + + @doc false + # Used internally to compile documentation. + # This function is private and must be used only internally. + def compile_definition_attributes(env, kind, name, args, _guards, body) do + %{module: module} = env + {set, bag} = data_tables_for(module) + {arity, defaults} = args_count(args, 0, 0) + + context = Keyword.get(:ets.lookup_element(set, {:def, {name, arity}}, 3), :context) + impl = compile_impl(set, bag, context, name, env, kind, arity, defaults) + doc_meta = compile_doc_meta(set, bag, name, arity, defaults) + + {line, doc} = get_doc_info(set, env) + compile_doc(set, context, line, kind, name, arity, args, body, doc, doc_meta, env, impl) + + :ok + end + + defp compile_doc(_table, _ctx, line, kind, name, arity, _args, _body, doc, _meta, env, _impl) + when kind in [:defp, :defmacrop] do + if doc do + message = + "#{kind} #{name}/#{arity} is private, " <> + "@doc attribute is always discarded for private functions/macros/types" + + IO.warn(message, %{env | line: line}) + end + end + + defp compile_doc(table, ctx, line, kind, name, arity, args, body, doc, doc_meta, env, impl) do + key = {doc_key(kind), name, arity} + signature = build_signature(args, env) + + case :ets.lookup(table, key) do + [] -> + doc = if is_nil(doc) && impl, do: false, else: doc + :ets.insert(table, {key, ctx, line, signature, doc, doc_meta}) + + [{_, current_ctx, current_line, current_sign, current_doc, current_doc_meta}] -> + if is_binary(current_doc) and is_binary(doc) and body != nil and is_nil(current_ctx) do + message = ~s''' + redefining @doc attribute previously set at line #{current_line}. + + Please remove the duplicate docs. If instead you want to override a \ + previously defined @doc, attach the @doc attribute to a function head \ + (the function signature not followed by any do-block). For example: + + @doc """ + new docs + """ + def #{name}(...) + ''' + + IO.warn(message, %{env | line: line}) + end + + signature = merge_signatures(current_sign, signature, 1) + doc = if is_nil(doc), do: current_doc, else: doc + doc = if is_nil(doc) && impl, do: false, else: doc + doc_meta = Map.merge(current_doc_meta, doc_meta) + :ets.insert(table, {key, ctx, current_line, signature, doc, doc_meta}) + end + end + + defp doc_key(:def), do: :function + defp doc_key(:defmacro), do: :macro + + defp compile_doc_meta(set, bag, name, arity, defaults) do + doc_meta = compile_deprecated(%{}, set, bag, name, arity, defaults) + doc_meta = get_doc_meta(doc_meta, set) + add_defaults_count(doc_meta, defaults) + end + + defp get_doc_meta(existing_meta, set) do + case :ets.take(set, {:doc, :meta}) do + [{{:doc, :meta}, metadata, _}] -> Map.merge(existing_meta, metadata) + [] -> existing_meta + end + end + + defp compile_deprecated(doc_meta, set, bag, name, arity, defaults) do + case :ets.take(set, :deprecated) do + [{:deprecated, reason, _}] when is_binary(reason) -> + :ets.insert(bag, deprecated_reasons(defaults, name, arity, reason)) + Map.put(doc_meta, :deprecated, reason) + + _ -> + doc_meta + end + end + + defp add_defaults_count(doc_meta, 0), do: doc_meta + defp add_defaults_count(doc_meta, n), do: Map.put(doc_meta, :defaults, n) + + defp deprecated_reasons(0, name, arity, reason) do + [deprecated_reason(name, arity, reason)] + end + + defp deprecated_reasons(defaults, name, arity, reason) do + [ + deprecated_reason(name, arity - defaults, reason) + | deprecated_reasons(defaults - 1, name, arity, reason) + ] + end + + defp deprecated_reason(name, arity, reason), + do: {:deprecated, {{name, arity}, reason}} + + defp compile_impl(set, bag, context, name, env, kind, arity, defaults) do + %{line: line, file: file} = env + + case :ets.take(set, :impl) do + [{:impl, value, _}] -> + impl = {{name, arity}, context, defaults, kind, line, file, value} + :ets.insert(bag, {:impls, impl}) + value + + [] -> + false end + end + + defp args_count([{:\\, _, _} | tail], total, defaults) do + args_count(tail, total + 1, defaults + 1) + end - delete_attribute(module, :doc) + defp args_count([_head | tail], total, defaults) do + args_count(tail, total + 1, defaults) end + defp args_count([], total, defaults), do: {total, defaults} + @doc false - # Used internally to compile types. This function - # is private and must be used only internally. - def store_typespec(module, key, value) when is_atom(key) do - assert_not_compiled!(:put_attribute, module) - table = data_table_for(module) + def check_derive_behaviours_and_impls(env, set, bag, all_definitions) do + check_derive(env, set, bag) + behaviours = bag_lookup_element(bag, {:accumulate, :behaviour}, 2) + impls = bag_lookup_element(bag, :impls, 2) + callbacks = check_behaviours(env, behaviours) + + pending_callbacks = + if impls != [] do + {non_implemented_callbacks, contexts} = check_impls(env, behaviours, callbacks, impls) + warn_missing_impls(env, non_implemented_callbacks, contexts, all_definitions) + non_implemented_callbacks + else + callbacks + end + + check_callbacks(env, pending_callbacks, all_definitions) + :ok + end + + defp check_derive(env, set, bag) do + case bag_lookup_element(bag, {:accumulate, :derive}, 2) do + [] -> + :ok - new = - case :ets.lookup(table, key) do - [{^key, old}] -> [value|old] - [] -> [value] + _ -> + message = + case :ets.lookup(set, :__struct__) do + [] -> + "warning: module attribute @derive was set but never used (it must come before defstruct)" + + _ -> + "warning: module attribute @derive was set after defstruct, all @derive calls must come before defstruct" + end + + IO.warn(message, env) + end + end + + defp check_behaviours(env, behaviours) do + Enum.reduce(behaviours, %{}, fn behaviour, acc -> + cond do + not Code.ensure_loaded?(behaviour) -> + message = + "@behaviour #{inspect(behaviour)} does not exist (in module #{inspect(env.module)})" + + IO.warn(message, env) + acc + + not function_exported?(behaviour, :behaviour_info, 1) -> + message = + "module #{inspect(behaviour)} is not a behaviour (in module #{inspect(env.module)})" + + IO.warn(message, env) + acc + + true -> + :elixir_env.trace({:require, [], behaviour, []}, env) + optional_callbacks = behaviour_info(behaviour, :optional_callbacks) + callbacks = behaviour_info(behaviour, :callbacks) + Enum.reduce(callbacks, acc, &add_callback(&1, behaviour, env, optional_callbacks, &2)) end + end) + end - :ets.insert(table, {key, new}) + defp add_callback(original, behaviour, env, optional_callbacks, acc) do + {callback, kind} = normalize_macro_or_function_callback(original) + + case acc do + %{^callback => {_kind, conflict, _optional?}} -> + message = + if conflict == behaviour do + "the behavior #{inspect(conflict)} has been declared twice " <> + "(conflict in #{format_definition(kind, callback)} in module #{inspect(env.module)})" + else + "conflicting behaviours found. #{format_definition(kind, callback)} is required by " <> + "#{inspect(conflict)} and #{inspect(behaviour)} (in module #{inspect(env.module)})" + end + + IO.warn(message, env) + + %{} -> + :ok + end + + Map.put(acc, callback, {kind, behaviour, original in optional_callbacks}) + end + + defp check_callbacks(env, callbacks, all_definitions) do + for {callback, {kind, behaviour, optional?}} <- callbacks do + case :lists.keyfind(callback, 1, all_definitions) do + false when not optional? -> + message = + format_callback(callback, kind, behaviour) <> + " is not implemented (in module #{inspect(env.module)})" + + IO.warn(message, env) + + {_, wrong_kind, _, _} when kind != wrong_kind -> + message = + format_callback(callback, kind, behaviour) <> + " was implemented as \"#{wrong_kind}\" but should have been \"#{kind}\" " <> + "(in module #{inspect(env.module)})" + + IO.warn(message, env) + + _ -> + :ok + end + end + + :ok + end + + defp format_callback(callback, kind, module) do + protocol_or_behaviour = if protocol?(module), do: "protocol ", else: "behaviour " + + format_definition(kind, callback) <> + " required by " <> protocol_or_behaviour <> inspect(module) + end + + defp protocol?(module) do + Code.ensure_loaded?(module) and function_exported?(module, :__protocol__, 1) and + module.__protocol__(:module) == module + end + + defp check_impls(env, behaviours, callbacks, impls) do + acc = {callbacks, %{}} + + Enum.reduce(impls, acc, fn {fa, context, defaults, kind, line, file, value}, acc -> + case impl_behaviours(fa, defaults, kind, value, behaviours, callbacks) do + {:ok, impl_behaviours} -> + Enum.reduce(impl_behaviours, acc, fn {fa, behaviour}, {callbacks, contexts} -> + callbacks = Map.delete(callbacks, fa) + contexts = Map.update(contexts, behaviour, [context], &[context | &1]) + {callbacks, contexts} + end) + + {:error, message} -> + formatted = format_impl_warning(fa, kind, message) + IO.warn(formatted, %{env | line: line, file: file}) + acc + end + end) + end + + defp impl_behaviours({function, arity}, defaults, kind, value, behaviours, callbacks) do + impls = for n <- arity..(arity - defaults), do: {function, n} + impl_behaviours(impls, kind, value, behaviours, callbacks) + end + + defp impl_behaviours(_, kind, _, _, _) when kind in [:defp, :defmacrop] do + {:error, :private_function} + end + + defp impl_behaviours(_, _, value, [], _) do + {:error, {:no_behaviours, value}} + end + + defp impl_behaviours(impls, _, false, _, callbacks) do + case callbacks_for_impls(impls, callbacks) do + [] -> {:ok, []} + [impl | _] -> {:error, {:impl_not_defined, impl}} + end + end + + defp impl_behaviours(impls, _, true, _, callbacks) do + case callbacks_for_impls(impls, callbacks) do + [] -> {:error, {:impl_defined, callbacks}} + impls -> {:ok, impls} + end + end + + defp impl_behaviours(impls, _, behaviour, behaviours, callbacks) do + filtered = behaviour_callbacks_for_impls(impls, behaviour, callbacks) + + cond do + filtered != [] -> + {:ok, filtered} + + behaviour not in behaviours -> + {:error, {:behaviour_not_declared, behaviour}} + + true -> + {:error, {:behaviour_not_defined, behaviour, callbacks}} + end + end + + defp behaviour_callbacks_for_impls([], _behaviour, _callbacks) do + [] + end + + defp behaviour_callbacks_for_impls([fa | tail], behaviour, callbacks) do + case callbacks[fa] do + {_, ^behaviour, _} -> + [{fa, behaviour} | behaviour_callbacks_for_impls(tail, behaviour, callbacks)] + + _ -> + behaviour_callbacks_for_impls(tail, behaviour, callbacks) + end + end + + defp callbacks_for_impls([], _) do + [] + end + + defp callbacks_for_impls([fa | tail], callbacks) do + case callbacks[fa] do + {_, behaviour, _} -> [{fa, behaviour} | callbacks_for_impls(tail, callbacks)] + nil -> callbacks_for_impls(tail, callbacks) + end + end + + defp format_impl_warning(fa, kind, :private_function) do + "#{format_definition(kind, fa)} is private, @impl attribute is always discarded for private functions/macros" + end + + defp format_impl_warning(fa, kind, {:no_behaviours, value}) do + "got \"@impl #{inspect(value)}\" for #{format_definition(kind, fa)} but no behaviour was declared" + end + + defp format_impl_warning(_, kind, {:impl_not_defined, {fa, behaviour}}) do + "got \"@impl false\" for #{format_definition(kind, fa)} " <> + "but it is a callback specified in #{inspect(behaviour)}" + end + + defp format_impl_warning(fa, kind, {:impl_defined, callbacks}) do + "got \"@impl true\" for #{format_definition(kind, fa)} " <> + "but no behaviour specifies such callback#{known_callbacks(callbacks)}" + end + + defp format_impl_warning(fa, kind, {:behaviour_not_declared, behaviour}) do + "got \"@impl #{inspect(behaviour)}\" for #{format_definition(kind, fa)} " <> + "but this behaviour was not declared with @behaviour" + end + + defp format_impl_warning(fa, kind, {:behaviour_not_defined, behaviour, callbacks}) do + "got \"@impl #{inspect(behaviour)}\" for #{format_definition(kind, fa)} " <> + "but this behaviour does not specify such callback#{known_callbacks(callbacks)}" + end + + defp warn_missing_impls(_env, callbacks, _contexts, _defs) when map_size(callbacks) == 0 do + :ok + end + + defp warn_missing_impls(env, non_implemented_callbacks, contexts, defs) do + for {pair, kind, meta, _clauses} <- defs, + kind in [:def, :defmacro] do + with {:ok, {_, behaviour, _}} <- Map.fetch(non_implemented_callbacks, pair), + true <- missing_impl_in_context?(meta, behaviour, contexts) do + message = + "module attribute @impl was not set for #{format_definition(kind, pair)} " <> + "callback (specified in #{inspect(behaviour)}). " <> + "This either means you forgot to add the \"@impl true\" annotation before the " <> + "definition or that you are accidentally overriding this callback" + + IO.warn(message, %{env | line: :elixir_utils.get_line(meta)}) + end + end + + :ok + end + + defp missing_impl_in_context?(meta, behaviour, contexts) do + case contexts do + %{^behaviour => known} -> Keyword.get(meta, :context) in known + %{} -> not Keyword.has_key?(meta, :context) + end + end + + defp format_definition(kind, {name, arity}) do + format_definition(kind) <> " #{name}/#{arity}" + end + + defp format_definition(:defmacro), do: "macro" + defp format_definition(:defmacrop), do: "macro" + defp format_definition(:def), do: "function" + defp format_definition(:defp), do: "function" + + defp known_callbacks(callbacks) when map_size(callbacks) == 0 do + ". There are no known callbacks, please specify the proper @behaviour " <> + "and make sure it defines callbacks" + end + + defp known_callbacks(callbacks) do + formatted_callbacks = + for {{name, arity}, {kind, module, _}} <- callbacks do + "\n * " <> Exception.format_mfa(module, name, arity) <> " (#{format_definition(kind)})" + end + + ". The known callbacks are:\n#{formatted_callbacks}\n" + end + + @doc false + # Used internally by Kernel's @. + # This function is private and must be used only internally. + def __get_attribute__(module, key, line) when is_atom(key) do + assert_not_compiled!( + {:get_attribute, 2}, + module, + "Use the Module.__info__/1 callback or Code.fetch_docs/1 instead" + ) + + {set, bag} = data_tables_for(module) + + case :ets.lookup(set, key) do + [{_, _, :accumulate}] -> + :lists.reverse(bag_lookup_element(bag, {:accumulate, key}, 2)) + + [{_, val, line}] when is_integer(line) -> + :ets.update_element(set, key, {3, :used}) + val + + [{_, val, _}] -> + val + + [] when is_integer(line) -> + # TODO: Consider raising instead of warning on v2.0 as it usually cascades + error_message = + "undefined module attribute @#{key}, " <> + "please remove access to @#{key} or explicitly set it before access" + + IO.warn(error_message, attribute_stack(module, line)) + nil + + [] -> + nil + end + end + + @doc false + # Used internally by Kernel's @. + # This function is private and must be used only internally. + def __put_attribute__(module, key, value, line) when is_atom(key) do + assert_not_readonly!(__ENV__.function, module) + {set, bag} = data_tables_for(module) + value = preprocess_attribute(key, value) + put_attribute(module, key, value, line, set, bag) + :ok + end + + # If any of the doc attributes are called with a keyword list that + # will become documentation metadata. Multiple calls will be merged + # into the same map overriding duplicate keys. + defp put_attribute(module, key, {_, metadata}, line, set, _bag) + when key in [:doc, :typedoc, :moduledoc] and is_list(metadata) do + metadata_map = preprocess_doc_meta(metadata, module, line, %{}) + + case :ets.insert_new(set, {{key, :meta}, metadata_map, line}) do + true -> + :ok + + false -> + current_metadata = :ets.lookup_element(set, {key, :meta}, 2) + :ets.update_element(set, {key, :meta}, {2, Map.merge(current_metadata, metadata_map)}) + end + end + + # Optimize some attributes by avoiding writing to the attributes key + # in the bag table since we handle them internally. + defp put_attribute(module, key, value, line, set, _bag) + when key in [:doc, :typedoc, :moduledoc, :impl, :deprecated] do + try do + :ets.lookup_element(set, key, 3) + catch + :error, :badarg -> :ok + else + unread_line when is_integer(line) and is_integer(unread_line) -> + message = "redefining @#{key} attribute previously set at line #{unread_line}" + IO.warn(message, attribute_stack(module, line)) + + _ -> + :ok + end + + :ets.insert(set, {key, value, line}) + end + + defp put_attribute(_module, :on_load, value, line, set, bag) do + try do + :ets.lookup_element(set, :on_load, 3) + catch + :error, :badarg -> + :ets.insert(set, {:on_load, value, line}) + :ets.insert(bag, {:warn_attributes, :on_load}) + else + _ -> raise ArgumentError, "the @on_load attribute can only be set once per module" + end + end + + defp put_attribute(_module, key, value, line, set, bag) do + try do + :ets.lookup_element(set, key, 3) + catch + :error, :badarg -> + :ets.insert(set, {key, value, line}) + :ets.insert(bag, {:warn_attributes, key}) + else + :accumulate -> :ets.insert(bag, {{:accumulate, key}, value}) + _ -> :ets.insert(set, {key, value, line}) + end + end + + defp attribute_stack(module, line) do + file = String.to_charlist(Path.relative_to_cwd(:elixir_module.file(module))) + [{module, :__MODULE__, 0, file: file, line: line}] end ## Helpers - defp normalize_attribute(:on_load, atom) when is_atom(atom) do - {atom, 0} + defp preprocess_attribute(key, value) when key in [:moduledoc, :typedoc, :doc] do + case value do + {line, doc} when is_integer(line) and (is_binary(doc) or doc == false or is_nil(doc)) -> + value + + {line, [{key, _} | _]} when is_integer(line) and is_atom(key) -> + value + + {line, doc} when is_integer(line) -> + raise ArgumentError, + "@#{key} is a built-in module attribute for documentation. It should be either " <> + "false, nil, a string, or a keyword list, got: #{inspect(doc)}" + + _other -> + raise ArgumentError, + "@#{key} is a built-in module attribute for documentation. When set dynamically, " <> + "it should be {line, doc} (where \"doc\" is either false, nil, a string, or a keyword list), " <> + "got: #{inspect(value)}" + end end - defp normalize_attribute(:behaviour, atom) when is_atom(atom) do - Code.ensure_compiled(atom) - atom + defp preprocess_attribute(:behaviour, value) do + if is_atom(value) do + Code.ensure_compiled(value) + value + else + raise ArgumentError, "@behaviour expects a module, got: #{inspect(value)}" + end end - defp normalize_attribute(:file, file) when is_binary(file) do - file + defp preprocess_attribute(:on_load, value) do + case value do + _ when is_atom(value) -> + {value, 0} + + {atom, 0} = tuple when is_atom(atom) -> + tuple + + _ -> + raise ArgumentError, + "@on_load is a built-in module attribute that annotates a function to be invoked " <> + "when the module is loaded. It should be an atom or an {atom, 0} tuple, " <> + "got: #{inspect(value)}" + end end - defp normalize_attribute(key, atom) when is_atom(atom) and - key in [:before_compile, :after_compile, :on_definition] do - {atom, :"__#{key}__"} + defp preprocess_attribute(:impl, value) do + if is_boolean(value) or (is_atom(value) and value != nil) do + value + else + raise ArgumentError, + "@impl is a built-in module attribute that marks the next definition " <> + "as a callback implementation. It should be a module or a boolean, " <> + "got: #{inspect(value)}" + end + end + + defp preprocess_attribute(:before_compile, atom) when is_atom(atom), + do: {atom, :__before_compile__} + + defp preprocess_attribute(:after_compile, atom) when is_atom(atom), + do: {atom, :__after_compile__} + + defp preprocess_attribute(:on_definition, atom) when is_atom(atom), + do: {atom, :__on_definition__} + + defp preprocess_attribute(key, _value) + when key in [:type, :typep, :opaque, :spec, :callback, :macrocallback] do + raise ArgumentError, + "attributes type, typep, opaque, spec, callback, and macrocallback " <> + "must be set directly via the @ notation" end - defp normalize_attribute(key, _value) when key in [:type, :typep, :export_type, :opaque, :callback] do - raise ArgumentError, "attributes type, typep, export_type, opaque and callback " <> - "must be set via Kernel.Typespec" + defp preprocess_attribute(:external_resource, value) when not is_binary(value) do + raise ArgumentError, + "@external_resource is a built-in module attribute used for specifying file " <> + "dependencies. It should be a string path to a file, got: #{inspect(value)}" end - defp normalize_attribute(_key, value) do + defp preprocess_attribute(:deprecated, value) when not is_binary(value) do + raise ArgumentError, + "@deprecated is a built-in module attribute that annotates a definition as deprecated. " <> + "It should be a string with the reason for the deprecation, got: #{inspect(value)}" + end + + defp preprocess_attribute(:file, value) do + case value do + _ when is_binary(value) -> + value + + {file, line} when is_binary(file) and is_integer(line) -> + value + + _ -> + raise ArgumentError, + "@file is a built-in module attribute that annotates the file and line the next " <> + "definition comes from. It should be a string or {string, line} tuple as value, " <> + "got: #{inspect(value)}" + end + end + + defp preprocess_attribute(:dialyzer, value) do + # From https://github.com/erlang/otp/blob/master/lib/stdlib/src/erl_lint.erl + :lists.foreach( + fn attr -> + if not valid_dialyzer_attribute?(attr) do + raise ArgumentError, "invalid value for @dialyzer attribute: #{inspect(attr)}" + end + end, + List.wrap(value) + ) + value end - defp data_table_for(module) do - module + defp preprocess_attribute(_key, value) do + value end - defp function_table_for(module) do - :elixir_def.table(module) + defp valid_dialyzer_attribute?({key, fun_arities}) when is_atom(key) do + (key == :nowarn_function or valid_dialyzer_attribute?(key)) and + :lists.all( + fn + {fun, arity} when is_atom(fun) and is_integer(arity) -> true + _ -> false + end, + List.wrap(fun_arities) + ) end - defp docs_table_for(module) do - :elixir_module.docs_table(module) + defp valid_dialyzer_attribute?(attr) do + :lists.member( + attr, + [:no_return, :no_unused, :no_improper_lists, :no_fun_app] ++ + [:no_match, :no_opaque, :no_fail_call, :no_contracts] ++ + [:no_behaviours, :no_undefined_callbacks, :unmatched_returns] ++ + [:error_handling, :race_conditions, :no_missing_calls] ++ + [:specdiffs, :overspecs, :underspecs, :unknown, :no_underspecs] + ) end - defp assert_not_compiled!(fun, module) do + defp preprocess_doc_meta([], _module, _line, map), do: map + + defp preprocess_doc_meta([{key, _} | tail], module, line, map) + when key in [:opaque, :defaults] do + message = "ignoring reserved documentation metadata key: #{inspect(key)}" + IO.warn(message, attribute_stack(module, line)) + preprocess_doc_meta(tail, module, line, map) + end + + defp preprocess_doc_meta([{key, value} | tail], module, line, map) when is_atom(key) do + validate_doc_meta(key, value) + preprocess_doc_meta(tail, module, line, Map.put(map, key, value)) + end + + defp validate_doc_meta(:since, value) when not is_binary(value) do + raise ArgumentError, + ":since is a built-in documentation metadata key. It should be a string representing " <> + "the version in which the documented entity was added, got: #{inspect(value)}" + end + + defp validate_doc_meta(:deprecated, value) when not is_binary(value) do + raise ArgumentError, + ":deprecated is a built-in documentation metadata key. It should be a string " <> + "representing the replacement for the deprecated entity, got: #{inspect(value)}" + end + + defp validate_doc_meta(:delegate_to, value) do + case value do + {m, f, a} when is_atom(m) and is_atom(f) and is_integer(a) and a >= 0 -> + :ok + + _ -> + raise ArgumentError, + ":delegate_to is a built-in documentation metadata key. It should be a three-element " <> + "tuple in the form of {module, function, arity}, got: #{inspect(value)}" + end + end + + defp validate_doc_meta(_, _), do: :ok + + defp get_doc_info(table, env) do + case :ets.take(table, :doc) do + [{:doc, {_, _} = pair, _}] -> + pair + + [] -> + {env.line, nil} + end + end + + defp data_tables_for(module) do + :elixir_module.data_tables(module) + end + + defp bag_lookup_element(table, key, pos) do + :ets.lookup_element(table, key, pos) + catch + :error, :badarg -> [] + end + + defp assert_not_compiled!(function_name_arity, module, extra_msg \\ "") do open?(module) || raise ArgumentError, - "could not call #{fun} on module #{inspect module} because it was already compiled" + assert_not_compiled_message(function_name_arity, module, extra_msg) end - defp loaded?(module), do: is_tuple :code.is_loaded(module) + defp assert_not_readonly!({function_name, arity}, module) do + case :elixir_module.mode(module) do + :all -> + :ok + + :readonly -> + raise ArgumentError, + "could not call Module.#{function_name}/#{arity} because the module " <> + "#{inspect(module)} is in read-only mode (@after_compile)" + + :closed -> + raise ArgumentError, + assert_not_compiled_message({function_name, arity}, module, "") + end + end + + defp assert_not_compiled_message({function_name, arity}, module, extra_msg) do + mfa = "Module.#{function_name}/#{arity}" + + "could not call #{mfa} because the module #{inspect(module)} is already compiled" <> + case extra_msg do + "" -> "" + _ -> ". " <> extra_msg + end + end end diff --git a/lib/elixir/lib/module/locals_tracker.ex b/lib/elixir/lib/module/locals_tracker.ex index 2a2da62ad8c..483b22f0eab 100644 --- a/lib/elixir/lib/module/locals_tracker.ex +++ b/lib/elixir/lib/module/locals_tracker.ex @@ -1,330 +1,253 @@ -# This is a module Elixir responsible for tracking +# This is an Elixir module responsible for tracking # calls in order to extract Elixir modules' behaviour # during compilation time. # # ## Implementation # -# The implementation uses the digraph module to track -# all dependencies. The graph starts with one main vertice: +# The implementation uses ETS to track all dependencies +# resembling a graph. The keys and what they point to are: # -# * `:local` - points to local functions +# * `:reattach` points to `{name, arity}` +# * `{:local, {name, arity}}` points to `{{name, arity}, line, macro_dispatch?}` +# * `{:import, {name, arity}}` points to `Module` # -# We also have can the following vertices: -# -# * `Module` - a module that was invoked via an import -# * `{name, arity}` - a local function/arity pair -# * `{:import, name, arity}` - an invoked function/arity import -# -# Each of those vertices can associate to other vertices -# as described below: -# -# * `Module` -# * in neighbours: `{:import, name, arity}` -# -# * `{name, arity}` -# * in neighbours: `:local`, `{name, arity}` -# * out neighbours: `{:import, name, arity}` -# -# * `{:import, name, arity}` -# * in neighbours: `{name, arity}` -# * out neighbours: `Module` -# -# Note that since this is required for bootstrap, we can't use -# any of the `GenServer.Behaviour` conveniences. +# This is built on top of the internal module tables. defmodule Module.LocalsTracker do @moduledoc false - @timeout 30_000 - @behaviour :gen_server + @defmacros [:defmacro, :defmacrop] - @type ref :: pid | module - @type name :: atom - @type name_arity :: {name, arity} - - @type local :: {name, arity} - @type import :: {:import, name, arity} + @doc """ + Adds and tracks defaults for a definition into the tracker. + """ + def add_defaults({_set, bag}, kind, {name, arity} = pair, defaults, meta) do + for i <- :lists.seq(arity - defaults, arity - 1) do + put_edge(bag, {:local, {name, i}}, {pair, get_line(meta), kind in @defmacros}) + end - # Public API + :ok + end @doc """ - Returns all imported modules that had the given - `{name, arity}` invoked. + Adds a local dispatch from-to the given target. """ - @spec imports_with_dispatch(ref, name_arity) :: [module] - def imports_with_dispatch(ref, {name, arity}) do - d = :gen_server.call(to_pid(ref), :digraph, @timeout) - :digraph.out_neighbours(d, {:import, name, arity}) + def add_local({_set, bag}, from, to, meta, macro_dispatch?) + when is_tuple(from) and is_tuple(to) and is_boolean(macro_dispatch?) do + put_edge(bag, {:local, from}, {to, get_line(meta), macro_dispatch?}) + :ok end @doc """ - Returns all locals that are reachable. - - By default, all public functions are reachable. - A private function is only reachable if it has - a public function that it invokes directly. + Adds an import dispatch to the given target. """ - @spec reachable(ref) :: [local] - def reachable(ref) do - d = :gen_server.call(to_pid(ref), :digraph, @timeout) - reduce_reachable(d, :local, []) + def add_import({set, _bag}, function, module, imported) + when is_tuple(function) and is_atom(module) do + put_edge(set, {:import, imported}, module) + :ok end - defp reduce_reachable(d, vertex, vertices) do - neighbours = :digraph.out_neighbours(d, vertex) - neighbours = (for {_, _} = t <- neighbours, do: t) |> :ordsets.from_list - remaining = :ordsets.subtract(neighbours, vertices) - vertices = :ordsets.union(neighbours, vertices) - :lists.foldl(&reduce_reachable(d, &1, &2), vertices, remaining) + @doc """ + Yanks a local node. Returns its in and out vertices in a tuple. + """ + def yank({_set, bag}, local) do + :lists.usort(take_out_neighbours(bag, {:local, local})) end - defp to_pid(pid) when is_pid(pid), do: pid - defp to_pid(mod) when is_atom(mod) do - table = :elixir_module.data_table(mod) - [{_, val}] = :ets.lookup(table, :__locals_tracker) - val - end + @doc """ + Reattach a previously yanked node. + """ + def reattach({_set, bag}, tuple, kind, function, out_neighbours, meta) do + for out_neighbour <- out_neighbours do + put_edge(bag, {:local, function}, out_neighbour) + end - # Internal API + # Make a call from the old function to the new one + if function != tuple do + put_edge(bag, {:local, function}, {tuple, get_line(meta), kind in @defmacros}) + end - # Starts the tracker and returns its pid. - @doc false - def start_link do - {:ok, pid} = :gen_server.start_link(__MODULE__, [], []) - pid + # Finally marked the new one as reattached + put_edge(bag, :reattach, tuple) + :ok end - # Adds a definition into the tracker. A public - # definition is connected with the :local node - # while a private one is left unreachable until - # a call is made to. - @doc false - def add_definition(pid, kind, tuple) when kind in [:def, :defp, :defmacro, :defmacrop] do - :gen_server.cast(pid, {:add_definition, kind, tuple}) + @doc """ + Collect all conflicting imports with the given functions + """ + def collect_imports_conflicts({set, _bag}, all_defined) do + for {pair, _, meta, _} <- all_defined, n = out_neighbour(set, {:import, pair}) do + {meta, {n, pair}} + end end - # Adds and tracks defaults for a definition into the tracker. - @doc false - def add_defaults(pid, kind, tuple, defaults) when kind in [:def, :defp, :defmacro, :defmacrop] do - :gen_server.cast(pid, {:add_defaults, kind, tuple, defaults}) + @doc """ + Collect all unused definitions based on the private + given, also accounting the expected number of default + clauses a private function have. + """ + def collect_unused_locals({_set, bag}, all_defined, private) do + reachable = + Enum.reduce(all_defined, %{}, fn {pair, kind, _, _}, acc -> + if kind in [:def, :defmacro] do + reachable_from(bag, pair, acc) + else + acc + end + end) + + reattached = :lists.usort(out_neighbours(bag, :reattach)) + {unreachable(reachable, reattached, private), collect_warnings(reachable, private)} end - # Adds a local dispatch to the given target. - def add_local(pid, to) when is_tuple(to) do - :gen_server.cast(pid, {:add_local, :local, to}) + @doc """ + Collect undefined functions based on local calls and existing definitions. + """ + def collect_undefined_locals({set, bag}, all_defined) do + undefined = + for {pair, _, meta, _} <- all_defined, + {local, line, macro_dispatch?} <- out_neighbours(bag, {:local, pair}), + error = undefined_local_error(set, local, macro_dispatch?), + do: {build_meta(line, meta), local, error} + + :lists.usort(undefined) end - # Adds a local dispatch from-to the given target. - @doc false - def add_local(pid, from, to) when is_tuple(from) and is_tuple(to) do - :gen_server.cast(pid, {:add_local, from, to}) + defp undefined_local_error(set, local, true) do + case :ets.member(set, {:def, local}) do + true -> false + false -> :undefined_function + end end - # Adds a import dispatch to the given target. - @doc false - def add_import(pid, function, module, target) when is_atom(module) and is_tuple(target) do - :gen_server.cast(pid, {:add_import, function, module, target}) + defp undefined_local_error(set, local, false) do + try do + if :ets.lookup_element(set, {:def, local}, 2) in @defmacros do + :incorrect_dispatch + else + false + end + catch + _, _ -> :undefined_function + end end - # Yanks a local node. Returns its in and out vertices in a tuple. - @doc false - def yank(pid, local) do - :gen_server.call(to_pid(pid), {:yank, local}, @timeout) + defp unreachable(reachable, reattached, private) do + for {tuple, kind, _, _} <- private, + not reachable?(tuple, kind, reachable, reattached), + do: tuple end - # Reattach a previously yanked node - @doc false - def reattach(pid, kind, tuple, neighbours) do - pid = to_pid(pid) - add_definition(pid, kind, tuple) - :gen_server.cast(pid, {:reattach, tuple, neighbours}) + defp reachable?(tuple, :defmacrop, reachable, reattached) do + # All private macros are unreachable unless they have been + # reattached and they are reachable. + :lists.member(tuple, reattached) and Map.has_key?(reachable, tuple) end - # Collecting all conflicting imports with the given functions - @doc false - def collect_imports_conflicts(pid, all_defined) do - d = :gen_server.call(pid, :digraph, @timeout) + defp reachable?(tuple, :defp, reachable, _reattached) do + Map.has_key?(reachable, tuple) + end - for {name, arity} <- all_defined, - :digraph.in_neighbours(d, {:import, name, arity}) != [], - n = :digraph.out_neighbours(d, {:import, name, arity}), - n != [] do - {n, name, arity} - end + defp collect_warnings(reachable, private) do + :lists.foldl(&collect_warnings(&1, &2, reachable), [], private) end - # Collect all unused definitions based on the private - # given also accounting the expected amount of default - # clauses a private function have. - @doc false - def collect_unused_locals(pid, private) do - reachable = reachable(pid) - :lists.foldl(&collect_unused_locals(&1, &2, reachable), [], private) + defp collect_warnings({_, _, false, _}, acc, _reachable) do + acc end - defp collect_unused_locals({tuple, kind, 0}, acc, reachable) do - if :lists.member(tuple, reachable) do + defp collect_warnings({tuple, kind, meta, 0}, acc, reachable) do + if Map.has_key?(reachable, tuple) do acc else - [{:unused_def, tuple, kind}|acc] + [{meta, {:unused_def, tuple, kind}} | acc] end end - defp collect_unused_locals({tuple, kind, default}, acc, reachable) when default > 0 do + defp collect_warnings({tuple, kind, meta, default}, acc, reachable) when default > 0 do {name, arity} = tuple min = arity - default max = arity - invoked = for {n, a} <- reachable, n == name, a in min..max, do: a - - if invoked == [] do - [{:unused_def, tuple, kind}|acc] - else - case :lists.min(invoked) - min do - 0 -> acc - ^default -> [{:unused_args, tuple}|acc] - unused_args -> [{:unused_args, tuple, unused_args}|acc] - end + case min_reachable_default(max, min, :none, name, reachable) do + :none -> [{meta, {:unused_def, tuple, kind}} | acc] + ^min -> acc + ^max -> [{meta, {:unused_args, tuple}} | acc] + diff -> [{meta, {:unused_args, tuple, diff}} | acc] end end - @doc false - def cache_env(pid, env) do - :gen_server.call(pid, {:cache_env, env}, @timeout) - end - - @doc false - def get_cached_env(pid, ref) do - :gen_server.call(pid, {:get_cached_env, ref}, @timeout) - end - - # Stops the gen server - @doc false - def stop(pid) do - :gen_server.cast(pid, :stop) - end - - # Callbacks - - def init([]) do - d = :digraph.new([:protected]) - :digraph.add_vertex(d, :local) - {:ok, {d, []}} - end - - def handle_call({:cache_env, env}, _from, {d, cache}) do - case cache do - [{i,^env}|_] -> - {:reply, i, {d, cache}} - t -> - i = length(t) - {:reply, i, {d, [{i,env}|t]}} + defp min_reachable_default(max, min, last, name, reachable) when max >= min do + case Map.has_key?(reachable, {name, max}) do + true -> min_reachable_default(max - 1, min, max, name, reachable) + false -> min_reachable_default(max - 1, min, last, name, reachable) end end - def handle_call({:get_cached_env, ref}, _from, {_, cache} = state) do - {^ref, env} = :lists.keyfind(ref, 1, cache) - {:reply, env, state} + defp min_reachable_default(_max, _min, last, _name, _reachable) do + last end - def handle_call({:yank, local}, _from, {d, _} = state) do - in_vertices = :digraph.in_neighbours(d, local) - out_vertices = :digraph.out_neighbours(d, local) - :digraph.del_vertex(d, local) - {:reply, {in_vertices, out_vertices}, state} - end + @doc """ + Returns all local nodes reachable from `vertex`. - def handle_call(:digraph, _from, {d, _} = state) do - {:reply, d, state} + By default, all public functions are reachable. + A private function is only reachable if it has + a public function that it invokes directly. + """ + def reachable_from({_, bag}, local) do + bag + |> reachable_from(local, %{}) + |> Map.keys() end - def handle_call(request, _from, state) do - {:stop, {:bad_call, request}, state} - end + defp reachable_from(bag, local, vertices) do + vertices = Map.put(vertices, local, true) - def handle_info(_msg, state) do - {:noreply, state} + Enum.reduce(out_neighbours(bag, {:local, local}), vertices, fn {local, _line, _}, acc -> + case acc do + %{^local => true} -> acc + _ -> reachable_from(bag, local, acc) + end + end) end - def handle_cast({:add_local, from, to}, {d, _} = state) do - handle_add_local(d, from, to) - {:noreply, state} - end + defp get_line(meta), do: Keyword.get(meta, :line) - def handle_cast({:add_import, function, module, {name, arity}}, {d, _} = state) do - handle_import(d, function, module, name, arity) - {:noreply, state} - end + defp build_meta(nil, _meta), do: [] - def handle_cast({:add_definition, kind, tuple}, {d, _} = state) do - handle_add_definition(d, kind, tuple) - {:noreply, state} - end - - def handle_cast({:add_defaults, kind, {name, arity}, defaults}, {d, _} = state) do - for i <- :lists.seq(arity - defaults, arity - 1) do - handle_add_definition(d, kind, {name, i}) - handle_add_local(d, {name, i}, {name, i + 1}) + # We need to transform any file annotation in the function + # definition into a keep annotation that is used by the + # error handling system in order to respect line/file. + defp build_meta(line, meta) do + case Keyword.get(meta, :file) do + {file, _} -> [keep: {file, line}] + _ -> [line: line] end - {:noreply, state} - end - - def handle_cast({:reattach, tuple, {in_neigh, out_neigh}}, {d, _} = state) do - for from <- in_neigh, do: replace_edge(d, from, tuple) - for to <- out_neigh, do: replace_edge(d, tuple, to) - {:noreply, state} end - def handle_cast(:stop, state) do - {:stop, :normal, state} - end - - def handle_cast(msg, state) do - {:stop, {:bad_cast, msg}, state} - end - - def terminate(_reason, _state) do - :ok - end + ## Lightweight digraph implementation - def code_change(_old, state, _extra) do - {:ok, state} + defp put_edge(d, from, to) do + :ets.insert(d, {from, to}) end - defp handle_import(d, function, module, name, arity) do - :digraph.add_vertex(d, module) - - tuple = {:import, name, arity} - :digraph.add_vertex(d, tuple) - replace_edge!(d, tuple, module) - - if function != nil do - replace_edge!(d, function, tuple) + defp out_neighbour(d, from) do + try do + :ets.lookup_element(d, from, 2) + catch + :error, :badarg -> nil end end - defp handle_add_local(d, from, to) do - :digraph.add_vertex(d, to) - replace_edge!(d, from, to) - end - - defp handle_add_definition(d, public, tuple) when public in [:def, :defmacro] do - :digraph.add_vertex(d, tuple) - replace_edge!(d, :local, tuple) - end - - defp handle_add_definition(d, private, tuple) when private in [:defp, :defmacrop] do - :digraph.add_vertex(d, tuple) - end - - defp replace_edge!(d, from, to) do - unless :lists.member(to, :digraph.out_neighbours(d, from)) do - [:"$e"|_] = :digraph.add_edge(d, from, to) + defp out_neighbours(d, from) do + try do + :ets.lookup_element(d, from, 2) + catch + :error, :badarg -> [] end end - defp replace_edge(d, from, to) do - unless :lists.member(to, :digraph.out_neighbours(d, from)) do - :digraph.add_edge(d, from, to) - end + defp take_out_neighbours(d, from) do + Keyword.values(:ets.take(d, from)) end end diff --git a/lib/elixir/lib/module/parallel_checker.ex b/lib/elixir/lib/module/parallel_checker.ex new file mode 100644 index 00000000000..7fb199e4684 --- /dev/null +++ b/lib/elixir/lib/module/parallel_checker.ex @@ -0,0 +1,496 @@ +defmodule Module.ParallelChecker do + @moduledoc false + + import Kernel, except: [spawn: 3] + + @type cache() :: {pid(), :ets.tid()} + @type warning() :: term() + @type kind() :: :def | :defmacro + @type mode() :: :elixir | :erlang + + @doc """ + Initializes the parallel checker process. + """ + def start_link(schedulers \\ nil) do + :gen_server.start_link(__MODULE__, schedulers, []) + end + + @doc """ + Stops the parallel checker process. + """ + def stop(checker) do + send(checker, {__MODULE__, :stop}) + :ok + end + + @doc """ + Gets the parallel checker data from pdict. + """ + def get do + {_, checker} = :erlang.get(:elixir_checker_info) + checker + end + + @doc """ + Stores the parallel checker information. + """ + def put(pid, checker) do + :erlang.put(:elixir_checker_info, {pid, checker}) + end + + @doc """ + Spawns a process that runs the parallel checker. + """ + def spawn({pid, checker}, module, info) do + ref = make_ref() + + spawned = + spawn(fn -> + Process.link(pid) + mon_ref = Process.monitor(pid) + + receive do + {^ref, :cache, ets} -> + loaded_info = + if is_map(info) do + cache_from_module_map(ets, info) + info + else + info = File.read!(info) + cache_from_chunk(ets, module, info) + info + end + + send(checker, {ref, :cached}) + + receive do + {^ref, :check} -> + warnings = check_module(module, loaded_info, {checker, ets}) + send(pid, {__MODULE__, module, warnings}) + send(checker, {__MODULE__, :done}) + end + + {:DOWN, ^mon_ref, _, _, _} -> + :ok + end + end) + + {spawned, ref} + end + + @doc """ + Verifies the given compilation function + by starting a checker if one does not exist. + See `verify/3`. + """ + def verify(fun) do + case :erlang.get(:elixir_compiler_info) do + :undefined -> + previous = :erlang.get(:elixir_checker_info) + {:ok, checker} = start_link() + put(self(), checker) + + try do + {result, compile_info} = fun.() + _ = verify(checker, compile_info, []) + result + after + if previous != :undefined do + :erlang.put(:elixir_checker_info, previous) + else + :erlang.erase(:elixir_checker_info) + end + + stop(checker) + end + + _ -> + # If we are during compilation, then they will be + # reported to the compiler, which will validate them. + {result, _info} = fun.() + result + end + end + + @doc """ + Receives pairs of module maps and BEAM binaries. In parallel it verifies + the modules and adds the ExCk chunk to the binaries. Returns the updated + list of warnings from the verification. + """ + @spec verify(pid(), [{pid(), reference()}], [{module(), binary()}]) :: [warning()] + def verify(checker, compiled_info, runtime_files) do + runtime_info = + for {module, file} <- runtime_files do + spawn({self(), checker}, module, file) + end + + modules = compiled_info ++ runtime_info + :gen_server.cast(checker, {:start, modules}) + collect_results(modules, []) + end + + defp collect_results([], warnings) do + warnings + end + + defp collect_results([_ | modules], warnings) do + receive do + {__MODULE__, _module, new_warnings} -> + collect_results(modules, new_warnings ++ warnings) + end + end + + @doc """ + Test cache. + """ + def test_cache do + {:ok, checker} = start_link() + {checker, :gen_server.call(checker, :ets)} + end + + @doc """ + Preloads a module into the cache. Call this function before any other + cache lookups for the module. + """ + @spec preload_module(cache(), module()) :: :ok + def preload_module({server, ets}, module) do + case :ets.lookup(ets, {:cached, module}) do + [{_key, _}] -> :ok + [] -> cache_module({server, ets}, module) + end + end + + @doc """ + Returns the export kind and deprecation reason for the given MFA from + the cache. If the module does not exist return `{:error, :module}`, + or if the function does not exist return `{:error, :function}`. + """ + @spec fetch_export(cache(), module(), atom(), arity()) :: + {:ok, mode(), kind(), binary() | nil} | {:error, :function | :module} + def fetch_export({_server, ets}, module, fun, arity) do + case :ets.lookup(ets, {:cached, module}) do + [{_key, false}] -> + {:error, :module} + + [{_key, mode}] -> + case :ets.lookup(ets, {:export, module, {fun, arity}}) do + [{_key, kind, reason}] -> {:ok, mode, kind, reason} + [] -> {:error, :function} + end + end + end + + @doc """ + Returns all exported functions and macros for the given module from + the cache. + """ + @spec all_exports(cache(), module()) :: [{atom(), arity()}] + def all_exports({_server, ets}, module) do + # This is only called after we get a deprecation notice + # so we can assume it's a cached module + ets + |> :ets.match({{:export, module, :"$1"}, :_, :_}) + |> Enum.flat_map(& &1) + |> Enum.sort() + end + + ## Module checking + + defp check_module(module, info, cache) do + case extract_definitions(module, info) do + {:ok, module, file, definitions, no_warn_undefined} -> + Module.Types.warnings(module, file, definitions, no_warn_undefined, cache) + |> group_warnings() + |> emit_warnings() + + :error -> + [] + end + end + + defp extract_definitions(module, module_map) when is_map(module_map) do + no_warn_undefined = + module_map.compile_opts + |> extract_no_warn_undefined() + |> merge_compiler_no_warn_undefined() + + {:ok, module, module_map.file, module_map.definitions, no_warn_undefined} + end + + defp extract_definitions(module, binary) when is_binary(binary) do + with {:ok, {_, [debug_info: chunk]}} <- :beam_lib.chunks(binary, [:debug_info]), + {:debug_info_v1, backend, data} <- chunk, + {:ok, module_map} <- backend.debug_info(:elixir_v1, module, data, []) do + extract_definitions(module, module_map) + else + _ -> :error + end + end + + defp extract_no_warn_undefined(compile_opts) do + for( + {:no_warn_undefined, values} <- compile_opts, + value <- List.wrap(values), + do: value + ) + end + + defp merge_compiler_no_warn_undefined(no_warn_undefined) do + case Code.get_compiler_option(:no_warn_undefined) do + :all -> + :all + + list when is_list(list) -> + no_warn_undefined ++ list + end + end + + ## Warning helpers + + def group_warnings(warnings) do + warnings + |> Enum.reduce(%{}, fn {module, warning, location}, acc -> + locations = MapSet.new([location]) + Map.update(acc, {module, warning}, locations, &MapSet.put(&1, location)) + end) + |> Enum.map(fn {{module, warning}, locations} -> {module, warning, Enum.sort(locations)} end) + |> Enum.sort() + end + + def emit_warnings(warnings) do + Enum.flat_map(warnings, fn {module, warning, locations} -> + message = module.format_warning(warning) + print_warning([message, ?\n, format_locations(locations)]) + + Enum.map(locations, fn {file, line, _mfa} -> + {file, line, message} + end) + end) + end + + defp format_locations([location]) do + format_location(location) + end + + defp format_locations(locations) do + [ + "Invalid call found at #{length(locations)} locations:\n", + Enum.map(locations, &format_location/1) + ] + end + + defp format_location({file, line, {module, fun, arity}}) do + mfa = Exception.format_mfa(module, fun, arity) + [format_file_line(file, line), ": ", mfa, ?\n] + end + + defp format_location({file, line, nil}) do + [format_file_line(file, line), ?\n] + end + + defp format_location({file, line, module}) do + [format_file_line(file, line), ": ", inspect(module), ?\n] + end + + defp format_file_line(file, line) do + file = Path.relative_to_cwd(file) + line = if line > 0, do: [?: | Integer.to_string(line)], else: [] + [" ", file, line] + end + + defp print_warning(message) do + IO.puts(:stderr, [:elixir_errors.warning_prefix(), message]) + end + + ## Cache + + defp cache_module({server, ets}, module) do + if lock(server, module) do + cache_from_chunk(ets, module) || cache_from_info(ets, module) + unlock(server, module) + end + end + + defp cache_from_chunk(ets, module) do + case :code.get_object_code(module) do + {^module, binary, _filename} -> cache_from_chunk(ets, module, binary) + _other -> false + end + end + + defp cache_from_chunk(ets, module, binary) do + with {:ok, {_, [{'ExCk', chunk}]}} <- :beam_lib.chunks(binary, ['ExCk']), + {:elixir_checker_v1, contents} <- :erlang.binary_to_term(chunk) do + cache_chunk(ets, module, contents.exports) + true + else + _ -> false + end + end + + defp cache_from_module_map(ets, map) do + exports = + [{{:__info__, 1}, :def}] ++ + behaviour_exports(map) ++ + definitions_to_exports(map.definitions) + + deprecated = Map.new(map.deprecated) + cache_info(ets, map.module, exports, deprecated, :elixir) + end + + defp cache_from_info(ets, module) do + if Code.ensure_loaded?(module) do + {mode, exports} = info_exports(module) + deprecated = info_deprecated(module) + cache_info(ets, module, exports, deprecated, mode) + else + :ets.insert(ets, {{:cached, module}, false}) + end + end + + defp info_exports(module) do + map = + Map.new( + [{{:__info__, 1}, :def}] ++ + behaviour_exports(module) ++ + Enum.map(module.__info__(:macros), &{&1, :defmacro}) ++ + Enum.map(module.__info__(:functions), &{&1, :def}) + ) + + {:elixir, map} + rescue + _ -> {:erlang, Map.new(Enum.map(module.module_info(:exports), &{&1, :def}))} + end + + defp info_deprecated(module) do + Map.new(module.__info__(:deprecated)) + rescue + _ -> %{} + end + + defp cache_info(ets, module, exports, deprecated, mode) do + Enum.each(exports, fn {{fun, arity}, kind} -> + reason = Map.get(deprecated, {fun, arity}) + :ets.insert(ets, {{:export, module, {fun, arity}}, kind, reason}) + {{fun, arity}, kind} + end) + + :ets.insert(ets, {{:cached, module}, mode}) + end + + defp cache_chunk(ets, module, exports) do + Enum.each(exports, fn {{fun, arity}, %{kind: kind, deprecated_reason: reason}} -> + :ets.insert(ets, {{:export, module, {fun, arity}}, kind, reason}) + + {{fun, arity}, kind} + end) + + :ets.insert(ets, {{:export, module, {:__info__, 1}}, :def, nil}) + :ets.insert(ets, {{:cached, module}, :elixir}) + end + + defp behaviour_exports(%{is_behaviour: true}), do: [{{:behaviour_info, 1}, :def}] + defp behaviour_exports(%{is_behaviour: false}), do: [] + + defp behaviour_exports(module) when is_atom(module) do + if function_exported?(module, :behaviour_info, 1) do + [{{:behaviour_info, 1}, :def}] + else + [] + end + end + + defp definitions_to_exports(definitions) do + Enum.flat_map(definitions, fn {function, kind, _meta, _clauses} -> + if kind in [:def, :defmacro] do + [{function, kind}] + else + [] + end + end) + end + + defp lock(server, module) do + :gen_server.call(server, {:lock, module}, :infinity) + end + + defp unlock(server, module) do + :gen_server.call(server, {:unlock, module}) + end + + ## Server callbacks + + def init(schedulers) do + ets = :ets.new(__MODULE__, [:set, :public, {:read_concurrency, true}]) + + state = %{ + ets: ets, + waiting: %{}, + modules: [], + spawned: 0, + schedulers: schedulers || max(:erlang.system_info(:schedulers_online), 2) + } + + {:ok, state} + end + + def handle_call(:ets, _from, state) do + {:reply, state.ets, state} + end + + def handle_call({:lock, module}, from, %{waiting: waiting} = state) do + case waiting do + %{^module => froms} -> + waiting = Map.put(state.waiting, module, [from | froms]) + {:noreply, %{state | waiting: waiting}} + + %{} -> + waiting = Map.put(state.waiting, module, []) + {:reply, true, %{state | waiting: waiting}} + end + end + + def handle_call({:unlock, module}, _from, %{waiting: waiting} = state) do + froms = Map.fetch!(waiting, module) + Enum.each(froms, &:gen_server.reply(&1, false)) + waiting = Map.delete(waiting, module) + {:reply, :ok, %{state | waiting: waiting}} + end + + def handle_info({__MODULE__, :done}, state) do + state = %{state | spawned: state.spawned - 1} + {:noreply, run_checkers(state)} + end + + def handle_info({__MODULE__, :stop}, state) do + {:stop, :normal, state} + end + + def handle_cast({:start, modules}, %{ets: ets} = state) do + for {pid, ref} <- modules do + send(pid, {ref, :cache, ets}) + end + + for {_pid, ref} <- modules do + receive do + {^ref, :cached} -> :ok + end + end + + {:noreply, run_checkers(%{state | modules: modules})} + end + + defp run_checkers(%{modules: []} = state) do + state + end + + defp run_checkers(%{spawned: spawned, schedulers: schedulers} = state) + when spawned >= schedulers do + state + end + + defp run_checkers(%{modules: [{pid, ref} | modules]} = state) do + send(pid, {ref, :check}) + run_checkers(%{state | modules: modules, spawned: state.spawned + 1}) + end +end diff --git a/lib/elixir/lib/module/types.ex b/lib/elixir/lib/module/types.ex new file mode 100644 index 00000000000..45dd9120c89 --- /dev/null +++ b/lib/elixir/lib/module/types.ex @@ -0,0 +1,522 @@ +defmodule Module.Types do + @moduledoc false + + defmodule Error do + defexception [:message] + end + + import Module.Types.Helpers + alias Module.Types.{Expr, Pattern, Unify} + + @doc false + def warnings(module, file, defs, no_warn_undefined, cache) do + stack = stack() + + Enum.flat_map(defs, fn {{fun, arity} = function, kind, meta, clauses} -> + context = context(with_file_meta(meta, file), module, function, no_warn_undefined, cache) + + Enum.flat_map(clauses, fn {_meta, args, guards, body} -> + def_expr = {kind, meta, [guards_to_expr(guards, {fun, [], args})]} + + try do + warnings_from_clause(args, guards, body, def_expr, stack, context) + rescue + e -> + def_expr = {kind, meta, [guards_to_expr(guards, {fun, [], args}), [do: body]]} + + error = + Error.exception(""" + found error while checking types for #{Exception.format_mfa(module, fun, arity)} + + #{Macro.to_string(def_expr)} + + Please report this bug: https://github.com/elixir-lang/elixir/issues + + #{Exception.format_banner(:error, e, __STACKTRACE__)}\ + """) + + reraise error, __STACKTRACE__ + end + end) + end) + end + + defp with_file_meta(meta, file) do + case Keyword.fetch(meta, :file) do + {:ok, {meta_file, _}} -> meta_file + :error -> file + end + end + + defp guards_to_expr([], left) do + left + end + + defp guards_to_expr([guard | guards], left) do + guards_to_expr(guards, {:when, [], [left, guard]}) + end + + defp warnings_from_clause(args, guards, body, def_expr, stack, context) do + head_stack = Unify.push_expr_stack(def_expr, stack) + + with {:ok, _types, context} <- Pattern.of_head(args, guards, head_stack, context), + {:ok, _type, context} <- Expr.of_expr(body, :dynamic, stack, context) do + context.warnings + else + {:error, {type, error, context}} -> + [error_to_warning(type, error, context) | context.warnings] + end + end + + @doc false + def context(file, module, function, no_warn_undefined, cache) do + %{ + # File of module + file: file, + # Module of definitions + module: module, + # Current function + function: function, + # List of calls to not warn on as undefined + no_warn_undefined: no_warn_undefined, + # A list of cached modules received from the parallel compiler + cache: cache, + # Expression variable to type variable + vars: %{}, + # Type variable to expression variable + types_to_vars: %{}, + # Type variable to type + types: %{}, + # Trace of all variables that have been refined to a type, + # including the type they were refined to, why, and where + traces: %{}, + # Counter to give type variables unique names + counter: 0, + # Track if a variable was inferred from a type guard function such is_tuple/1 + # or a guard function that fails such as elem/2, possible values are: + # `:guarded` when `is_tuple(x)` + # `:guarded` when `is_tuple and elem(x, 0)` + # `:fail` when `elem(x, 0)` + guard_sources: %{}, + # A list with all warnings from the running the code + warnings: [] + } + end + + @doc false + def stack() do + %{ + # Stack of variables we have refined during unification, + # used for creating relevant traces + unify_stack: [], + # Last expression we have recursed through during inference, + # used for tracing + last_expr: nil, + # When false do not add a trace when a type variable is refined, + # useful when merging contexts where the variables already have traces + trace: true, + # There are two factors that control how we track guards. + # + # * consider_type_guards?: if type guards should be considered. + # This applies only at the root and root-based "and" and "or" nodes. + # + # * keep_guarded? - if a guarded clause should remain as guarded + # even on failure. Used on the right side of and. + # + type_guards: {_consider_type_guards? = true, _keep_guarded? = false}, + # Context used to determine if unification is bi-directional, :expr + # is directional, :pattern is bi-directional + context: nil + } + end + + ## ERROR TO WARNING + + # Collect relevant information from context and traces to report error + def error_to_warning(:unable_apply, {mfa, args, expected, signature, stack}, context) do + {fun, arity} = context.function + line = get_meta(stack.last_expr)[:line] + location = {context.file, line, {context.module, fun, arity}} + + traces = type_traces(stack, context) + {[signature | args], traces} = lift_all_types([signature | args], traces, context) + error = {:unable_apply, mfa, args, expected, signature, {location, stack.last_expr, traces}} + {Module.Types, error, location} + end + + def error_to_warning(:unable_unify, {left, right, stack}, context) do + {fun, arity} = context.function + line = get_meta(stack.last_expr)[:line] + location = {context.file, line, {context.module, fun, arity}} + + traces = type_traces(stack, context) + {[left, right], traces} = lift_all_types([left, right], traces, context) + error = {:unable_unify, left, right, {location, stack.last_expr, traces}} + {Module.Types, error, location} + end + + # Collect relevant traces from context.traces using stack.unify_stack + defp type_traces(stack, context) do + # TODO: Do we need the unify_stack or is enough to only get the last variable + # in the stack since we get related variables anyway? + stack = + stack.unify_stack + |> Enum.flat_map(&[&1 | related_variables(&1, context.types)]) + |> Enum.uniq() + + Enum.flat_map(stack, fn var_index -> + with %{^var_index => traces} <- context.traces, + %{^var_index => expr_var} <- context.types_to_vars do + Enum.map(traces, &tag_trace(expr_var, &1, context)) + else + _other -> [] + end + end) + end + + defp related_variables(var, types) do + Enum.flat_map(types, fn + {related_var, {:var, ^var}} -> + [related_var | related_variables(related_var, types)] + + _ -> + [] + end) + end + + # Tag if trace is for a concrete type or type variable + defp tag_trace(var, {type, expr, location}, context) do + with {:var, var_index} <- type, + %{^var_index => expr_var} <- context.types_to_vars do + {:var, var, expr_var, expr, location} + else + _ -> {:type, var, type, expr, location} + end + end + + defp lift_all_types(types, traces, context) do + trace_types = for({:type, _, type, _, _} <- traces, do: type) + {types, lift_context} = Unify.lift_types(types, context) + {trace_types, _lift_context} = Unify.lift_types(trace_types, lift_context) + + {traces, []} = + Enum.map_reduce(traces, trace_types, fn + {:type, var, _, expr, location}, [type | acc] -> {{:type, var, type, expr, location}, acc} + other, acc -> {other, acc} + end) + + {types, traces} + end + + ## FORMAT WARNINGS + + def format_warning({:unable_apply, mfa, args, expected, signature, {location, expr, traces}}) do + {module, function, arity} = mfa + mfa_args = Macro.generate_arguments(arity, __MODULE__) + {module, function, ^arity} = call_to_mfa(erl_to_ex(module, function, mfa_args, [])) + format_mfa = Exception.format_mfa(module, function, arity) + {traces, [] = _hints} = format_traces(traces, [], false) + + clauses = + Enum.map( + signature, + &String.slice(IO.iodata_to_binary(Unify.format_type({:fun, [&1]}, false)), 1..-2) + ) + + [ + "expected #{format_mfa} to have signature:\n\n ", + Enum.map_join(args, ", ", &Unify.format_type(&1, false)), + " -> #{Unify.format_type(expected, false)}", + "\n\nbut it has signature:\n\n ", + indent(Enum.join(clauses, "\n")), + "\n\n", + format_expr(expr, location), + traces, + "Conflict found at" + ] + end + + def format_warning({:unable_unify, left, right, {location, expr, traces}}) do + if map_type?(left) and map_type?(right) and match?({:ok, _, _}, missing_field(left, right)) do + {:ok, atom, known_atoms} = missing_field(left, right) + + # Drop the last trace which is the expression map.foo + traces = Enum.drop(traces, 1) + {traces, hints} = format_traces(traces, [left, right], true) + + [ + "undefined field \"#{atom}\" ", + format_expr(expr, location), + "expected one of the following fields: ", + Enum.map_join(Enum.sort(known_atoms), ", ", & &1), + "\n\n", + traces, + format_message_hints(hints), + "Conflict found at" + ] + else + simplify_left? = simplify_type?(left, right) + simplify_right? = simplify_type?(right, left) + + {traces, hints} = format_traces(traces, [left, right], simplify_left? or simplify_right?) + + [ + "incompatible types:\n\n ", + Unify.format_type(left, simplify_left?), + " !~ ", + Unify.format_type(right, simplify_right?), + "\n\n", + format_expr(expr, location), + traces, + format_message_hints(hints), + "Conflict found at" + ] + end + end + + defp missing_field( + {:map, [{:required, {:atom, atom} = type, _}, {:optional, :dynamic, :dynamic}]}, + {:map, fields} + ) do + matched_missing_field(fields, type, atom) + end + + defp missing_field( + {:map, fields}, + {:map, [{:required, {:atom, atom} = type, _}, {:optional, :dynamic, :dynamic}]} + ) do + matched_missing_field(fields, type, atom) + end + + defp missing_field(_, _), do: :error + + defp matched_missing_field(fields, type, atom) do + if List.keymember?(fields, type, 1) do + :error + else + known_atoms = for {_, {:atom, atom}, _} <- fields, do: atom + {:ok, atom, known_atoms} + end + end + + defp format_traces([], _types, _simplify?) do + {[], []} + end + + defp format_traces(traces, types, simplify?) do + traces + |> Enum.uniq() + |> Enum.reverse() + |> Enum.map_reduce([], fn + {:type, var, type, expr, location}, hints -> + {hint, hints} = format_type_hint(type, types, expr, hints) + + trace = [ + "where \"", + Macro.to_string(var), + "\" was given the type ", + Unify.format_type(type, simplify?), + hint, + " in:\n\n # ", + format_location(location), + " ", + indent(expr_to_string(expr)), + "\n\n" + ] + + {trace, hints} + + {:var, var1, var2, expr, location}, hints -> + trace = [ + "where \"", + Macro.to_string(var1), + "\" was given the same type as \"", + Macro.to_string(var2), + "\" in:\n\n # ", + format_location(location), + " ", + indent(expr_to_string(expr)), + "\n\n" + ] + + {trace, hints} + end) + end + + defp format_location({file, line, _mfa}) do + format_location({file, line}) + end + + defp format_location({file, line}) do + file = Path.relative_to_cwd(file) + line = if line, do: [Integer.to_string(line)], else: [] + [file, ?:, line, ?\n] + end + + defp simplify_type?(type, other) do + map_like_type?(type) and not map_like_type?(other) + end + + ## EXPRESSION FORMATTING + + defp format_expr(nil, _location) do + [] + end + + defp format_expr(expr, location) do + [ + "in expression:\n\n # ", + format_location(location), + " ", + indent(expr_to_string(expr)), + "\n\n" + ] + end + + @doc false + def expr_to_string(expr) do + expr + |> reverse_rewrite() + |> Macro.to_string() + end + + defp reverse_rewrite(guard) do + Macro.prewalk(guard, fn + {{:., _, [mod, fun]}, meta, args} -> erl_to_ex(mod, fun, args, meta) + other -> other + end) + end + + defp erl_to_ex(mod, fun, args, meta) do + case :elixir_rewrite.erl_to_ex(mod, fun, args) do + {Kernel, fun, args} -> {fun, meta, args} + {mod, fun, args} -> {{:., [], [mod, fun]}, meta, args} + end + end + + ## Hints + + defp format_message_hints(hints) do + hints + |> Enum.uniq() + |> Enum.reverse() + |> Enum.map(&[format_message_hint(&1), "\n"]) + end + + defp format_message_hint(:inferred_dot) do + """ + HINT: "var.field" (without parentheses) implies "var" is a map() while \ + "var.fun()" (with parentheses) implies "var" is an atom() + """ + end + + defp format_message_hint(:inferred_bitstring_spec) do + """ + HINT: all expressions given to binaries are assumed to be of type \ + integer() unless said otherwise. For example, <> assumes "expr" \ + is an integer. Pass a modifier, such as <> or <>, \ + to change the default behaviour. + """ + end + + defp format_message_hint({:sized_and_unsize_tuples, {size, var}}) do + """ + HINT: use pattern matching or "is_tuple(#{Macro.to_string(var)}) and \ + tuple_size(#{Macro.to_string(var)}) == #{size}" to guard a sized tuple. + """ + end + + defp format_type_hint(type, types, expr, hints) do + case format_type_hint(type, types, expr) do + {message, hint} -> {message, [hint | hints]} + :error -> {[], hints} + end + end + + defp format_type_hint(type, types, expr) do + cond do + dynamic_map_dot?(type, expr) -> + {" (due to calling var.field)", :inferred_dot} + + dynamic_remote_call?(type, expr) -> + {" (due to calling var.fun())", :inferred_dot} + + inferred_bitstring_spec?(type, expr) -> + {[], :inferred_bitstring_spec} + + message = sized_and_unsize_tuples(expr, types) -> + {[], {:sized_and_unsize_tuples, message}} + + true -> + :error + end + end + + defp dynamic_map_dot?(type, expr) do + with true <- map_type?(type), + {{:., _meta1, [_map, _field]}, meta2, []} <- expr, + true <- Keyword.get(meta2, :no_parens, false) do + true + else + _ -> false + end + end + + defp dynamic_remote_call?(type, expr) do + with true <- atom_type?(type), + {{:., _meta1, [_module, _field]}, meta2, []} <- expr, + false <- Keyword.get(meta2, :no_parens, false) do + true + else + _ -> false + end + end + + defp inferred_bitstring_spec?(type, expr) do + with true <- integer_type?(type), + {:<<>>, _, args} <- expr, + true <- Enum.any?(args, &match?({:"::", [{:inferred_bitstring_spec, true} | _], _}, &1)) do + true + else + _ -> false + end + end + + defp sized_and_unsize_tuples({{:., _, [:erlang, :is_tuple]}, _, [var]}, types) do + case Enum.find(types, &match?({:tuple, _, _}, &1)) do + {:tuple, size, _} -> + {size, var} + + nil -> + nil + end + end + + defp sized_and_unsize_tuples(_expr, _types) do + nil + end + + ## Formatting helpers + + defp indent(string) do + String.replace(string, "\n", "\n ") + end + + defp map_type?({:map, _}), do: true + defp map_type?(_other), do: false + + defp map_like_type?({:map, _}), do: true + defp map_like_type?({:union, union}), do: Enum.any?(union, &map_like_type?/1) + defp map_like_type?(_other), do: false + + defp atom_type?(:atom), do: true + defp atom_type?({:atom, _}), do: false + defp atom_type?({:union, union}), do: Enum.all?(union, &atom_type?/1) + defp atom_type?(_other), do: false + + defp integer_type?(:integer), do: true + defp integer_type?(_other), do: false + + defp call_to_mfa({{:., _, [mod, fun]}, _, args}), do: {mod, fun, length(args)} + defp call_to_mfa({fun, _, args}) when is_atom(fun), do: {Kernel, fun, length(args)} +end diff --git a/lib/elixir/lib/module/types/expr.ex b/lib/elixir/lib/module/types/expr.ex new file mode 100644 index 00000000000..1fb7ea3651a --- /dev/null +++ b/lib/elixir/lib/module/types/expr.ex @@ -0,0 +1,525 @@ +defmodule Module.Types.Expr do + @moduledoc false + + alias Module.Types.{Of, Pattern} + import Module.Types.{Helpers, Unify} + + def of_expr(expr, expected, %{context: stack_context} = stack, context) + when stack_context != :expr do + of_expr(expr, expected, %{stack | context: :expr}, context) + end + + # :atom + def of_expr(atom, _expected, _stack, context) when is_atom(atom) do + {:ok, {:atom, atom}, context} + end + + # 12 + def of_expr(literal, _expected, _stack, context) when is_integer(literal) do + {:ok, :integer, context} + end + + # 1.2 + def of_expr(literal, _expected, _stack, context) when is_float(literal) do + {:ok, :float, context} + end + + # "..." + def of_expr(literal, _expected, _stack, context) when is_binary(literal) do + {:ok, :binary, context} + end + + # #PID<...> + def of_expr(literal, _expected, _stack, context) when is_pid(literal) do + {:ok, :dynamic, context} + end + + # <<...>>> + def of_expr({:<<>>, _meta, args}, _expected, stack, context) do + case Of.binary(args, stack, context, &of_expr/4) do + {:ok, context} -> {:ok, :binary, context} + {:error, reason} -> {:error, reason} + end + end + + # left | [] + def of_expr({:|, _meta, [left_expr, []]} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + of_expr(left_expr, :dynamic, stack, context) + end + + # left | right + def of_expr({:|, _meta, [left_expr, right_expr]} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + + case of_expr(left_expr, :dynamic, stack, context) do + {:ok, left, context} -> + case of_expr(right_expr, :dynamic, stack, context) do + {:ok, {:list, right}, context} -> + {:ok, to_union([left, right], context), context} + + {:ok, right, context} -> + {:ok, to_union([left, right], context), context} + + {:error, reason} -> + {:error, reason} + end + + {:error, reason} -> + {:error, reason} + end + end + + # [] + def of_expr([], _expected, _stack, context) do + {:ok, {:list, :dynamic}, context} + end + + # [expr, ...] + def of_expr(exprs, _expected, stack, context) when is_list(exprs) do + stack = push_expr_stack(exprs, stack) + + case map_reduce_ok(exprs, context, &of_expr(&1, :dynamic, stack, &2)) do + {:ok, types, context} -> {:ok, {:list, to_union(types, context)}, context} + {:error, reason} -> {:error, reason} + end + end + + # __CALLER__ + def of_expr({:__CALLER__, _meta, var_context}, _expected, _stack, context) + when is_atom(var_context) do + struct_pair = {:required, {:atom, :__struct__}, {:atom, Macro.Env}} + + pairs = + Enum.map(Map.from_struct(Macro.Env.__struct__()), fn {key, _value} -> + {:required, {:atom, key}, :dynamic} + end) + + {:ok, {:map, [struct_pair | pairs]}, context} + end + + # __STACKTRACE__ + def of_expr({:__STACKTRACE__, _meta, var_context}, _expected, _stack, context) + when is_atom(var_context) do + file = {:tuple, 2, [{:atom, :file}, {:list, :integer}]} + line = {:tuple, 2, [{:atom, :line}, :integer]} + file_line = {:list, {:union, [file, line]}} + type = {:list, {:tuple, 4, [:atom, :atom, :integer, file_line]}} + {:ok, type, context} + end + + # var + def of_expr(var, _expected, _stack, context) when is_var(var) do + {:ok, get_var!(var, context), context} + end + + # {left, right} + def of_expr({left, right}, expected, stack, context) do + of_expr({:{}, [], [left, right]}, expected, stack, context) + end + + # {...} + def of_expr({:{}, _meta, exprs} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + + case map_reduce_ok(exprs, context, &of_expr(&1, :dynamic, stack, &2)) do + {:ok, types, context} -> {:ok, {:tuple, length(types), types}, context} + {:error, reason} -> {:error, reason} + end + end + + # left = right + def of_expr({:=, _meta, [left_expr, right_expr]} = expr, _expected, stack, context) do + # TODO: We might want to bring the expected type forward in case the type of this + # pattern is not useful. For example: 1 = _ = expr + + stack = push_expr_stack(expr, stack) + + with {:ok, left_type, context} <- + Pattern.of_pattern(left_expr, stack, context), + {:ok, right_type, context} <- of_expr(right_expr, left_type, stack, context), + do: unify(right_type, left_type, stack, context) + end + + # %{map | ...} + def of_expr({:%{}, _, [{:|, _, [map, args]}]} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + map_type = {:map, [{:optional, :dynamic, :dynamic}]} + + with {:ok, map_type, context} <- of_expr(map, map_type, stack, context), + {:ok, {:map, arg_pairs}, context} <- Of.closed_map(args, stack, context, &of_expr/4), + dynamic_value_pairs = + Enum.map(arg_pairs, fn {:required, key, _value} -> {:required, key, :dynamic} end), + args_type = {:map, dynamic_value_pairs ++ [{:optional, :dynamic, :dynamic}]}, + {:ok, type, context} <- unify(map_type, args_type, stack, context) do + # Retrieve map type and overwrite with the new value types from the map update + {:map, pairs} = resolve_var(type, context) + + updated_pairs = + Enum.reduce(arg_pairs, pairs, fn {:required, key, value}, pairs -> + List.keyreplace(pairs, key, 1, {:required, key, value}) + end) + + {:ok, {:map, updated_pairs}, context} + end + end + + # %Struct{map | ...} + def of_expr( + {:%, meta, [module, {:%{}, _, [{:|, _, [_, _]}]} = update]} = expr, + _expected, + stack, + context + ) do + stack = push_expr_stack(expr, stack) + map_type = {:map, [{:optional, :dynamic, :dynamic}]} + + with {:ok, struct, context} <- Of.struct(module, meta, context), + {:ok, update, context} <- of_expr(update, map_type, stack, context) do + unify(update, struct, stack, context) + end + end + + # %{...} + def of_expr({:%{}, _meta, args} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + Of.closed_map(args, stack, context, &of_expr/4) + end + + # %Struct{...} + def of_expr({:%, meta1, [module, {:%{}, _meta2, args}]} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + + with {:ok, struct, context} <- Of.struct(module, meta1, context), + {:ok, map, context} <- Of.open_map(args, stack, context, &of_expr/4) do + unify(map, struct, stack, context) + end + end + + # () + def of_expr({:__block__, _meta, []}, _expected, _stack, context) do + {:ok, {:atom, nil}, context} + end + + # (expr; expr) + def of_expr({:__block__, _meta, exprs}, expected, stack, context) do + expected_types = List.duplicate(:dynamic, length(exprs) - 1) ++ [expected] + + result = + map_reduce_ok(Enum.zip(exprs, expected_types), context, fn {expr, expected}, context -> + of_expr(expr, expected, stack, context) + end) + + case result do + {:ok, expr_types, context} -> {:ok, Enum.at(expr_types, -1), context} + {:error, reason} -> {:error, reason} + end + end + + # case expr do pat -> expr end + def of_expr({:case, _meta, [case_expr, [{:do, clauses}]]} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + + with {:ok, _expr_type, context} <- of_expr(case_expr, :dynamic, stack, context), + {:ok, context} <- of_clauses(clauses, stack, context), + do: {:ok, :dynamic, context} + end + + # fn pat -> expr end + def of_expr({:fn, _meta, clauses} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + + case of_clauses(clauses, stack, context) do + {:ok, context} -> {:ok, :dynamic, context} + {:error, reason} -> {:error, reason} + end + end + + @try_blocks [:do, :after] + @try_clause_blocks [:catch, :else, :after] + + # try do expr end + def of_expr({:try, _meta, [blocks]} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + + {result, context} = + reduce_ok(blocks, context, fn + {:rescue, clauses}, context -> + reduce_ok(clauses, context, fn + {:->, _, [[{:in, _, [var, _exceptions]}], body]}, context = acc -> + {_type, context} = new_pattern_var(var, context) + + with {:ok, context} <- of_expr_context(body, :dynamic, stack, context) do + {:ok, keep_warnings(acc, context)} + end + + {:->, _, [[var], body]}, context = acc -> + {_type, context} = new_pattern_var(var, context) + + with {:ok, context} <- of_expr_context(body, :dynamic, stack, context) do + {:ok, keep_warnings(acc, context)} + end + end) + + {block, body}, context = acc when block in @try_blocks -> + with {:ok, context} <- of_expr_context(body, :dynamic, stack, context) do + {:ok, keep_warnings(acc, context)} + end + + {block, clauses}, context when block in @try_clause_blocks -> + of_clauses(clauses, stack, context) + end) + + case result do + :ok -> {:ok, :dynamic, context} + :error -> {:error, context} + end + end + + # receive do pat -> expr end + def of_expr({:receive, _meta, [blocks]} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + + {result, context} = + reduce_ok(blocks, context, fn + {:do, {:__block__, _, []}}, context -> + {:ok, context} + + {:do, clauses}, context -> + of_clauses(clauses, stack, context) + + {:after, [{:->, _meta, [head, body]}]}, context = acc -> + with {:ok, _type, context} <- of_expr(head, :dynamic, stack, context), + {:ok, _type, context} <- of_expr(body, :dynamic, stack, context), + do: {:ok, keep_warnings(acc, context)} + end) + + case result do + :ok -> {:ok, :dynamic, context} + :error -> {:error, context} + end + end + + # for pat <- expr do expr end + def of_expr({:for, _meta, [_ | _] = args} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + {clauses, [[{:do, block} | opts]]} = Enum.split(args, -1) + + with {:ok, context} <- reduce_ok(clauses, context, &for_clause(&1, stack, &2)), + {:ok, context} <- reduce_ok(opts, context, &for_option(&1, stack, &2)) do + if Keyword.has_key?(opts, :reduce) do + with {:ok, context} <- of_clauses(block, stack, context) do + {:ok, :dynamic, context} + end + else + with {:ok, _type, context} <- of_expr(block, :dynamic, stack, context) do + {:ok, :dynamic, context} + end + end + end + end + + # with pat <- expr do expr end + def of_expr({:with, _meta, [_ | _] = clauses} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + + case reduce_ok(clauses, context, &with_clause(&1, stack, &2)) do + {:ok, context} -> {:ok, :dynamic, context} + {:error, reason} -> {:error, reason} + end + end + + # fun.(args) + def of_expr({{:., _meta1, [fun]}, _meta2, args} = expr, _expected, stack, context) do + # TODO: Use expected type to infer intersection return type + stack = push_expr_stack(expr, stack) + + with {:ok, _fun_type, context} <- of_expr(fun, :dynamic, stack, context), + {:ok, _arg_types, context} <- + map_reduce_ok(args, context, &of_expr(&1, :dynamic, stack, &2)) do + {:ok, :dynamic, context} + end + end + + # expr.key_or_fun + def of_expr({{:., _meta1, [expr1, key_or_fun]}, meta2, []} = expr2, _expected, stack, context) + when not is_atom(expr1) do + stack = push_expr_stack(expr2, stack) + + if Keyword.get(meta2, :no_parens, false) do + with {:ok, expr_type, context} <- of_expr(expr1, :dynamic, stack, context), + {value_var, context} = add_var(context), + pair_type = {:required, {:atom, key_or_fun}, value_var}, + optional_type = {:optional, :dynamic, :dynamic}, + map_field_type = {:map, [pair_type, optional_type]}, + {:ok, _map_type, context} <- unify(map_field_type, expr_type, stack, context), + do: {:ok, value_var, context} + else + # TODO: Use expected type to infer intersection return type + with {:ok, expr_type, context} <- of_expr(expr1, :dynamic, stack, context), + {:ok, _map_type, context} <- unify(expr_type, :atom, stack, context), + do: {:ok, :dynamic, context} + end + end + + # expr.fun(arg) + def of_expr({{:., meta1, [expr1, fun]}, _meta2, args} = expr2, _expected, stack, context) do + # TODO: Use expected type to infer intersection return type + + context = Of.remote(expr1, fun, length(args), meta1, context) + stack = push_expr_stack(expr2, stack) + + with {:ok, _expr_type, context} <- of_expr(expr1, :dynamic, stack, context), + {:ok, _fun_type, context} <- of_expr(fun, :dynamic, stack, context), + {:ok, _arg_types, context} <- + map_reduce_ok(args, context, &of_expr(&1, :dynamic, stack, &2)) do + {:ok, :dynamic, context} + end + end + + # &Foo.bar/1 + def of_expr( + {:&, meta, [{:/, _, [{{:., _, [module, fun]}, _, []}, arity]}]}, + _expected, + _stack, + context + ) + when is_atom(module) and is_atom(fun) do + context = Of.remote(module, fun, arity, meta, context) + {:ok, :dynamic, context} + end + + # &foo/1 + # & &1 + def of_expr({:&, _meta, _arg}, _expected, _stack, context) do + # TODO: Function type + {:ok, :dynamic, context} + end + + # fun(arg) + def of_expr({fun, _meta, args} = expr, _expected, stack, context) + when is_atom(fun) and is_list(args) do + # TODO: Use expected type to infer intersection return type + + stack = push_expr_stack(expr, stack) + + case map_reduce_ok(args, context, &of_expr(&1, :dynamic, stack, &2)) do + {:ok, _arg_types, context} -> {:ok, :dynamic, context} + {:error, reason} -> {:error, reason} + end + end + + defp for_clause({:<-, _, [left, expr]}, stack, context) do + {pattern, guards} = extract_head([left]) + + with {:ok, _pattern_type, context} <- Pattern.of_head([pattern], guards, stack, context), + {:ok, _expr_type, context} <- of_expr(expr, :dynamic, stack, context), + do: {:ok, context} + end + + defp for_clause({:<<>>, _, [{:<-, _, [pattern, expr]}]}, stack, context) do + # TODO: the compiler guarantees pattern is a binary but we need to check expr is a binary + with {:ok, _pattern_type, context} <- Pattern.of_pattern(pattern, stack, context), + {:ok, _expr_type, context} <- of_expr(expr, :dynamic, stack, context), + do: {:ok, context} + end + + defp for_clause(list, stack, context) when is_list(list) do + reduce_ok(list, context, &for_option(&1, stack, &2)) + end + + defp for_clause(expr, stack, context) do + of_expr_context(expr, :dynamic, stack, context) + end + + defp for_option({:into, expr}, stack, context) do + of_expr_context(expr, :dynamic, stack, context) + end + + defp for_option({:reduce, expr}, stack, context) do + of_expr_context(expr, :dynamic, stack, context) + end + + defp for_option({:uniq, _}, _stack, context) do + {:ok, context} + end + + defp with_clause({:<-, _, [left, expr]}, stack, context) do + {pattern, guards} = extract_head([left]) + + with {:ok, _pattern_type, context} <- Pattern.of_head([pattern], guards, stack, context), + {:ok, _expr_type, context} <- of_expr(expr, :dynamic, stack, context), + do: {:ok, context} + end + + defp with_clause(list, stack, context) when is_list(list) do + reduce_ok(list, context, &with_option(&1, stack, &2)) + end + + defp with_clause(expr, stack, context) do + of_expr_context(expr, :dynamic, stack, context) + end + + defp with_option({:do, body}, stack, context) do + of_expr_context(body, :dynamic, stack, context) + end + + defp with_option({:else, clauses}, stack, context) do + of_clauses(clauses, stack, context) + end + + defp of_clauses(clauses, stack, context) do + reduce_ok(clauses, context, fn {:->, meta, [head, body]}, context = acc -> + {patterns, guards} = extract_head(head) + + case Pattern.of_head(patterns, guards, stack, context) do + {:ok, _, context} -> + with {:ok, _expr_type, context} <- of_expr(body, :dynamic, stack, context) do + {:ok, keep_warnings(acc, context)} + end + + error -> + # Skip the clause if it the head has an error + if meta[:generated], do: {:ok, acc}, else: error + end + end) + end + + defp keep_warnings(context, %{warnings: warnings}) do + %{context | warnings: warnings} + end + + defp extract_head([{:when, _meta, args}]) do + case Enum.split(args, -1) do + {patterns, [guards]} -> {patterns, flatten_when(guards)} + {patterns, []} -> {patterns, []} + end + end + + defp extract_head(other) do + {other, []} + end + + defp flatten_when({:when, _meta, [left, right]}) do + [left | flatten_when(right)] + end + + defp flatten_when(other) do + [other] + end + + defp of_expr_context(expr, expected, stack, context) do + case of_expr(expr, expected, stack, context) do + {:ok, _type, context} -> {:ok, context} + {:error, reason} -> {:error, reason} + end + end + + defp new_pattern_var({:_, _meta, var_context}, context) when is_atom(var_context) do + {:dynamic, context} + end + + defp new_pattern_var(var, context) do + new_var(var, context) + end +end diff --git a/lib/elixir/lib/module/types/helpers.ex b/lib/elixir/lib/module/types/helpers.ex new file mode 100644 index 00000000000..d4203ed38dc --- /dev/null +++ b/lib/elixir/lib/module/types/helpers.ex @@ -0,0 +1,196 @@ +defmodule Module.Types.Helpers do + # AST and enumeration helpers. + @moduledoc false + + @doc """ + Guard function to check if an AST node is a variable. + """ + defmacro is_var(expr) do + quote do + is_tuple(unquote(expr)) and + tuple_size(unquote(expr)) == 3 and + is_atom(elem(unquote(expr), 0)) and + is_atom(elem(unquote(expr), 2)) + end + end + + @doc """ + Returns unique identifier for the current assignment of the variable. + """ + def var_name({_name, meta, _context}), do: Keyword.fetch!(meta, :version) + + @doc """ + Returns the AST metadata. + """ + def get_meta({_, meta, _}), do: meta + def get_meta(_other), do: [] + + @doc """ + Like `Enum.reduce/3` but only continues while `fun` returns `{:ok, acc}` + and stops on `{:error, reason}`. + """ + def reduce_ok(list, acc, fun) do + do_reduce_ok(list, acc, fun) + end + + defp do_reduce_ok([head | tail], acc, fun) do + case fun.(head, acc) do + {:ok, acc} -> + do_reduce_ok(tail, acc, fun) + + {:error, reason} -> + {:error, reason} + end + end + + defp do_reduce_ok([], acc, _fun), do: {:ok, acc} + + @doc """ + Like `Enum.unzip/1` but only continues while `fun` returns `{:ok, elem1, elem2}` + and stops on `{:error, reason}`. + """ + def unzip_ok(list) do + do_unzip_ok(list, [], []) + end + + defp do_unzip_ok([{:ok, head1, head2} | tail], acc1, acc2) do + do_unzip_ok(tail, [head1 | acc1], [head2 | acc2]) + end + + defp do_unzip_ok([{:error, reason} | _tail], _acc1, _acc2), do: {:error, reason} + + defp do_unzip_ok([], acc1, acc2), do: {:ok, Enum.reverse(acc1), Enum.reverse(acc2)} + + @doc """ + Like `Enum.map/2` but only continues while `fun` returns `{:ok, elem}` + and stops on `{:error, reason}`. + """ + def map_ok(list, fun) do + do_map_ok(list, [], fun) + end + + defp do_map_ok([head | tail], acc, fun) do + case fun.(head) do + {:ok, elem} -> + do_map_ok(tail, [elem | acc], fun) + + {:error, reason} -> + {:error, reason} + end + end + + defp do_map_ok([], acc, _fun), do: {:ok, Enum.reverse(acc)} + + @doc """ + Like `Enum.map_reduce/3` but only continues while `fun` returns `{:ok, elem, acc}` + and stops on `{:error, reason}`. + """ + def map_reduce_ok(list, acc, fun) do + do_map_reduce_ok(list, {[], acc}, fun) + end + + defp do_map_reduce_ok([head | tail], {list, acc}, fun) do + case fun.(head, acc) do + {:ok, elem, acc} -> + do_map_reduce_ok(tail, {[elem | list], acc}, fun) + + {:error, reason} -> + {:error, reason} + end + end + + defp do_map_reduce_ok([], {list, acc}, _fun), do: {:ok, Enum.reverse(list), acc} + + @doc """ + Like `Enum.flat_map/2` but only continues while `fun` returns `{:ok, list}` + and stops on `{:error, reason}`. + """ + def flat_map_ok(list, fun) do + do_flat_map_ok(list, [], fun) + end + + defp do_flat_map_ok([head | tail], acc, fun) do + case fun.(head) do + {:ok, elem} -> + do_flat_map_ok(tail, [elem | acc], fun) + + {:error, reason} -> + {:error, reason} + end + end + + defp do_flat_map_ok([], acc, _fun), do: {:ok, Enum.reverse(Enum.concat(acc))} + + @doc """ + Like `Enum.flat_map_reduce/3` but only continues while `fun` returns `{:ok, list, acc}` + and stops on `{:error, reason}`. + """ + def flat_map_reduce_ok(list, acc, fun) do + do_flat_map_reduce_ok(list, {[], acc}, fun) + end + + defp do_flat_map_reduce_ok([head | tail], {list, acc}, fun) do + case fun.(head, acc) do + {:ok, elems, acc} -> + do_flat_map_reduce_ok(tail, {[elems | list], acc}, fun) + + {:error, reason} -> + {:error, reason} + end + end + + defp do_flat_map_reduce_ok([], {list, acc}, _fun), + do: {:ok, Enum.reverse(Enum.concat(list)), acc} + + @doc """ + Given a list of `[{:ok, term()} | {:error, term()}]` it returns a list of + errors `{:error, [term()]}` in case of at least one error or `{:ok, [term()]}` + if there are no errors. + """ + def oks_or_errors(list) do + case Enum.split_with(list, &match?({:ok, _}, &1)) do + {oks, []} -> {:ok, Enum.map(oks, fn {:ok, ok} -> ok end)} + {_oks, errors} -> {:error, Enum.map(errors, fn {:error, error} -> error end)} + end + end + + @doc """ + Combines a list of guard expressions `when x when y when z` to an expression + combined with `or`, `x or y or z`. + """ + # TODO: Remove this and let multiple when be treated as multiple clauses, + # meaning they will be intersection types + def guards_to_or([]) do + [] + end + + def guards_to_or(guards) do + Enum.reduce(guards, fn guard, acc -> {{:., [], [:erlang, :orelse]}, [], [guard, acc]} end) + end + + @doc """ + Like `Enum.zip/1` but will zip multiple lists together instead of only two. + """ + def zip_many(lists) do + zip_many(lists, [], [[]]) + end + + defp zip_many([], [], [[] | acc]) do + map_reverse(acc, [], &Enum.reverse/1) + end + + defp zip_many([], remain, [last | acc]) do + zip_many(Enum.reverse(remain), [], [[] | [last | acc]]) + end + + defp zip_many([[] | _], remain, [last | acc]) do + zip_many(Enum.reverse(remain), [], [last | acc]) + end + + defp zip_many([[elem | list1] | list2], remain, [last | acc]) do + zip_many(list2, [list1 | remain], [[elem | last] | acc]) + end + + defp map_reverse([], acc, _fun), do: acc + defp map_reverse([head | tail], acc, fun), do: map_reverse(tail, [fun.(head) | acc], fun) +end diff --git a/lib/elixir/lib/module/types/of.ex b/lib/elixir/lib/module/types/of.ex new file mode 100644 index 00000000000..690f4f0fda0 --- /dev/null +++ b/lib/elixir/lib/module/types/of.ex @@ -0,0 +1,365 @@ +defmodule Module.Types.Of do + # Typing functionality shared between Expr and Pattern. + # Generic AST and Enum helpers go to Module.Types.Helpers. + @moduledoc false + + @prefix quote(do: ...) + @suffix quote(do: ...) + + alias Module.ParallelChecker + + import Module.Types.Helpers + import Module.Types.Unify + + # There are important assumptions on how we work with maps. + # + # First, the keys in the map must be ordered by subtyping. + # + # Second, optional keys must be a superset of the required + # keys, i.e. %{required(atom) => integer, optional(:foo) => :bar} + # is forbidden. + # + # Third, in order to preserve co/contra-variance, a supertype + # must satisfy its subtypes. I.e. %{foo: :bar, atom() => :baz} + # is forbidden, it must be %{foo: :bar, atom() => :baz | :bar}. + # + # Once we support user declared maps, we need to validate these + # assumptions. + + @doc """ + Handles open maps (with dynamic => dynamic). + """ + def open_map(args, stack, context, of_fun) do + with {:ok, pairs, context} <- map_pairs(args, stack, context, of_fun) do + # If we match on a map such as %{"foo" => "bar"}, we cannot + # assert that %{binary() => binary()}, since we are matching + # only a single binary of infinite possible values. Therefore, + # the correct would be to match it to %{binary() => binary() | var}. + # + # We can skip this in two cases: + # + # 1. If the key is a singleton, then we know that it has no + # other value than the current one + # + # 2. If the value is a variable, then there is no benefit in + # creating another variable, so we can skip it + # + # For now, we skip generating the var itself and introduce + # :dynamic instead. + pairs = + for {key, value} <- pairs, not has_unbound_var?(key, context) do + if singleton?(key, context) or match?({:var, _}, value) do + {key, value} + else + {key, to_union([value, :dynamic], context)} + end + end + + triplets = pairs_to_unions(pairs, [], context) ++ [{:optional, :dynamic, :dynamic}] + {:ok, {:map, triplets}, context} + end + end + + @doc """ + Handles closed maps (without dynamic => dynamic). + """ + def closed_map(args, stack, context, of_fun) do + with {:ok, pairs, context} <- map_pairs(args, stack, context, of_fun) do + {:ok, {:map, closed_to_unions(pairs, context)}, context} + end + end + + defp map_pairs(pairs, stack, context, of_fun) do + map_reduce_ok(pairs, context, fn {key, value}, context -> + with {:ok, key_type, context} <- of_fun.(key, :dynamic, stack, context), + {:ok, value_type, context} <- of_fun.(value, :dynamic, stack, context), + do: {:ok, {key_type, value_type}, context} + end) + end + + defp closed_to_unions([{key, value}], _context), do: [{:required, key, value}] + + defp closed_to_unions(pairs, context) do + case Enum.split_with(pairs, fn {key, _value} -> has_unbound_var?(key, context) end) do + {[], pairs} -> pairs_to_unions(pairs, [], context) + {[_ | _], pairs} -> pairs_to_unions([{:dynamic, :dynamic} | pairs], [], context) + end + end + + defp pairs_to_unions([{key, value} | ahead], behind, context) do + {matched_ahead, values} = find_matching_values(ahead, key, [], []) + + # In case nothing matches, use the original ahead + ahead = matched_ahead || ahead + + all_values = + [value | values] ++ + find_subtype_values(ahead, key, context) ++ + find_subtype_values(behind, key, context) + + pairs_to_unions(ahead, [{key, to_union(all_values, context)} | behind], context) + end + + defp pairs_to_unions([], acc, context) do + acc + |> Enum.sort(&subtype?(elem(&1, 0), elem(&2, 0), context)) + |> Enum.map(fn {key, value} -> {:required, key, value} end) + end + + defp find_subtype_values(pairs, key, context) do + for {pair_key, pair_value} <- pairs, subtype?(pair_key, key, context), do: pair_value + end + + defp find_matching_values([{key, value} | ahead], key, acc, values) do + find_matching_values(ahead, key, acc, [value | values]) + end + + defp find_matching_values([{_, _} = pair | ahead], key, acc, values) do + find_matching_values(ahead, key, [pair | acc], values) + end + + defp find_matching_values([], _key, acc, [_ | _] = values), do: {Enum.reverse(acc), values} + defp find_matching_values([], _key, _acc, []), do: {nil, []} + + @doc """ + Handles structs. + """ + def struct(struct, meta, context) do + context = remote(struct, :__struct__, 0, meta, context) + + entries = + for key <- Map.keys(struct.__struct__()), key != :__struct__ do + {:required, {:atom, key}, :dynamic} + end + + {:ok, {:map, [{:required, {:atom, :__struct__}, {:atom, struct}} | entries]}, context} + end + + ## Binary + + @doc """ + Handles binaries. + + In the stack, we add nodes such as <>, <<..., expr>>, etc, + based on the position of the expression within the binary. + """ + def binary([], _stack, context, _of_fun) do + {:ok, context} + end + + def binary([head], stack, context, of_fun) do + head_stack = push_expr_stack({:<<>>, get_meta(head), [head]}, stack) + binary_segment(head, head_stack, context, of_fun) + end + + def binary([head | tail], stack, context, of_fun) do + head_stack = push_expr_stack({:<<>>, get_meta(head), [head, @suffix]}, stack) + + case binary_segment(head, head_stack, context, of_fun) do + {:ok, context} -> binary_many(tail, stack, context, of_fun) + {:error, reason} -> {:error, reason} + end + end + + defp binary_many([last], stack, context, of_fun) do + last_stack = push_expr_stack({:<<>>, get_meta(last), [@prefix, last]}, stack) + binary_segment(last, last_stack, context, of_fun) + end + + defp binary_many([head | tail], stack, context, of_fun) do + head_stack = push_expr_stack({:<<>>, get_meta(head), [@prefix, head, @suffix]}, stack) + + case binary_segment(head, head_stack, context, of_fun) do + {:ok, context} -> binary_many(tail, stack, context, of_fun) + {:error, reason} -> {:error, reason} + end + end + + defp binary_segment({:"::", _meta, [expr, specifiers]}, stack, context, of_fun) do + expected_type = + collect_binary_specifier(specifiers, &binary_type(stack.context, &1)) || :integer + + utf? = collect_binary_specifier(specifiers, &utf_type?/1) + float? = collect_binary_specifier(specifiers, &float_type?/1) + + # Special case utf and float specifiers because they can be two types as literals + # but only a specific type as a variable in a pattern + cond do + stack.context == :pattern and utf? and is_binary(expr) -> + {:ok, context} + + stack.context == :pattern and float? and is_integer(expr) -> + {:ok, context} + + true -> + with {:ok, type, context} <- of_fun.(expr, expected_type, stack, context), + {:ok, _type, context} <- unify(type, expected_type, stack, context), + do: {:ok, context} + end + end + + # Collect binary type specifiers, + # from `<>` collect `integer` + defp collect_binary_specifier({:-, _meta, [left, right]}, fun) do + collect_binary_specifier(left, fun) || collect_binary_specifier(right, fun) + end + + defp collect_binary_specifier(other, fun) do + fun.(other) + end + + defp binary_type(:expr, {:float, _, _}), do: {:union, [:integer, :float]} + defp binary_type(:expr, {:utf8, _, _}), do: {:union, [:integer, :binary]} + defp binary_type(:expr, {:utf16, _, _}), do: {:union, [:integer, :binary]} + defp binary_type(:expr, {:utf32, _, _}), do: {:union, [:integer, :binary]} + defp binary_type(:pattern, {:utf8, _, _}), do: :integer + defp binary_type(:pattern, {:utf16, _, _}), do: :integer + defp binary_type(:pattern, {:utf32, _, _}), do: :integer + defp binary_type(:pattern, {:float, _, _}), do: :float + defp binary_type(_context, {:integer, _, _}), do: :integer + defp binary_type(_context, {:bits, _, _}), do: :binary + defp binary_type(_context, {:bitstring, _, _}), do: :binary + defp binary_type(_context, {:bytes, _, _}), do: :binary + defp binary_type(_context, {:binary, _, _}), do: :binary + defp binary_type(_context, _specifier), do: nil + + defp utf_type?({specifier, _, _}), do: specifier in [:utf8, :utf16, :utf32] + defp utf_type?(_), do: false + + defp float_type?({:float, _, _}), do: true + defp float_type?(_), do: false + + ## Remote + + @doc """ + Handles remote calls. + """ + def remote(module, fun, arity, meta, context) when is_atom(module) do + # TODO: In the future we may want to warn for modules defined + # in the local context + if Keyword.get(meta, :context_module, false) do + context + else + ParallelChecker.preload_module(context.cache, module) + check_export(module, fun, arity, meta, context) + end + end + + def remote(_module, _fun, _arity, _meta, context), do: context + + defp check_export(module, fun, arity, meta, context) do + case ParallelChecker.fetch_export(context.cache, module, fun, arity) do + {:ok, mode, :def, reason} -> + check_deprecated(mode, module, fun, arity, reason, meta, context) + + {:ok, mode, :defmacro, reason} -> + context = warn(meta, context, {:unrequired_module, module, fun, arity}) + check_deprecated(mode, module, fun, arity, reason, meta, context) + + {:error, :module} -> + if warn_undefined?(module, fun, arity, context) do + warn(meta, context, {:undefined_module, module, fun, arity}) + else + context + end + + {:error, :function} -> + if warn_undefined?(module, fun, arity, context) do + exports = ParallelChecker.all_exports(context.cache, module) + warn(meta, context, {:undefined_function, module, fun, arity, exports}) + else + context + end + end + end + + defp check_deprecated(:elixir, module, fun, arity, reason, meta, context) do + if reason do + warn(meta, context, {:deprecated, module, fun, arity, reason}) + else + context + end + end + + defp check_deprecated(:erlang, module, fun, arity, _reason, meta, context) do + case :otp_internal.obsolete(module, fun, arity) do + {:deprecated, string} when is_list(string) -> + reason = string |> List.to_string() |> String.capitalize() + warn(meta, context, {:deprecated, module, fun, arity, reason}) + + {:deprecated, string, removal} when is_list(string) and is_list(removal) -> + reason = string |> List.to_string() |> String.capitalize() + reason = "It will be removed in #{removal}. #{reason}" + warn(meta, context, {:deprecated, module, fun, arity, reason}) + + _ -> + context + end + end + + # The protocol code dispatches to unknown modules, so we ignore them here. + # + # try do + # SomeProtocol.Atom.__impl__ + # rescue + # ... + # end + # + # But for protocols we don't want to traverse the protocol code anyway. + # TODO: remove this clause once we no longer traverse the protocol code. + defp warn_undefined?(_module, :__impl__, 1, _context), do: false + defp warn_undefined?(_module, :module_info, 0, _context), do: false + defp warn_undefined?(_module, :module_info, 1, _context), do: false + defp warn_undefined?(:erlang, :orelse, 2, _context), do: false + defp warn_undefined?(:erlang, :andalso, 2, _context), do: false + + defp warn_undefined?(_, _, _, %{no_warn_undefined: :all}) do + false + end + + defp warn_undefined?(module, fun, arity, context) do + not Enum.any?(context.no_warn_undefined, &(&1 == module or &1 == {module, fun, arity})) + end + + defp warn(meta, context, warning) do + {fun, arity} = context.function + location = {context.file, meta[:line] || 0, {context.module, fun, arity}} + %{context | warnings: [{__MODULE__, warning, location} | context.warnings]} + end + + ## Warning formatting + + def format_warning({:undefined_module, module, fun, arity}) do + [ + Exception.format_mfa(module, fun, arity), + " is undefined (module ", + inspect(module), + " is not available or is yet to be defined)" + ] + end + + def format_warning({:undefined_function, module, fun, arity, exports}) do + [ + Exception.format_mfa(module, fun, arity), + " is undefined or private", + UndefinedFunctionError.hint_for_loaded_module(module, fun, arity, exports) + ] + end + + def format_warning({:deprecated, module, fun, arity, reason}) do + [ + Exception.format_mfa(module, fun, arity), + " is deprecated. ", + reason + ] + end + + def format_warning({:unrequired_module, module, fun, arity}) do + [ + "you must require ", + inspect(module), + " before invoking the macro ", + Exception.format_mfa(module, fun, arity) + ] + end +end diff --git a/lib/elixir/lib/module/types/pattern.ex b/lib/elixir/lib/module/types/pattern.ex new file mode 100644 index 00000000000..c7c83a0c14c --- /dev/null +++ b/lib/elixir/lib/module/types/pattern.ex @@ -0,0 +1,759 @@ +defmodule Module.Types.Pattern do + @moduledoc false + + alias Module.Types.Of + import Module.Types.{Helpers, Unify} + + @doc """ + Handles patterns and guards at once. + """ + def of_head(patterns, guards, stack, context) do + with {:ok, types, context} <- + map_reduce_ok(patterns, context, &of_pattern(&1, stack, &2)), + # TODO: Check that of_guard/4 returns boolean() | :fail + {:ok, _, context} <- of_guard(guards_to_or(guards), :dynamic, stack, context), + do: {:ok, types, context} + end + + @doc """ + Return the type and typing context of a pattern expression or an error + in case of a typing conflict. + """ + def of_pattern(pattern, %{context: stack_context} = stack, context) + when stack_context != :pattern do + of_pattern(pattern, %{stack | context: :pattern}, context) + end + + # _ + def of_pattern({:_, _meta, atom}, _stack, context) when is_atom(atom) do + {:ok, :dynamic, context} + end + + # ^var + def of_pattern({:^, _meta, [var]}, _stack, context) do + {:ok, get_var!(var, context), context} + end + + # var + def of_pattern(var, _stack, context) when is_var(var) do + {type, context} = new_var(var, context) + {:ok, type, context} + end + + # left = right + def of_pattern({:=, _meta, [left_expr, right_expr]} = expr, stack, context) do + stack = push_expr_stack(expr, stack) + + with {:ok, left_type, context} <- of_pattern(left_expr, stack, context), + {:ok, right_type, context} <- of_pattern(right_expr, stack, context), + do: unify(left_type, right_type, stack, context) + end + + # %_{...} + def of_pattern( + {:%, _meta1, [{:_, _meta2, var_context}, {:%{}, _meta3, args}]} = expr, + stack, + context + ) + when is_atom(var_context) do + stack = push_expr_stack(expr, stack) + expected_fun = fn arg, _expected, stack, context -> of_pattern(arg, stack, context) end + + with {:ok, {:map, pairs}, context} <- Of.open_map(args, stack, context, expected_fun) do + {:ok, {:map, [{:required, {:atom, :__struct__}, :atom} | pairs]}, context} + end + end + + # %var{...} and %^var{...} + def of_pattern({:%, _meta1, [var, {:%{}, _meta2, args}]} = expr, stack, context) + when not is_atom(var) do + stack = push_expr_stack(expr, stack) + expected_fun = fn arg, _expected, stack, context -> of_pattern(arg, stack, context) end + + with {:ok, var_type, context} = of_pattern(var, stack, context), + {:ok, _, context} <- unify(var_type, :atom, stack, context), + {:ok, {:map, pairs}, context} <- Of.open_map(args, stack, context, expected_fun) do + {:ok, {:map, [{:required, {:atom, :__struct__}, var_type} | pairs]}, context} + end + end + + def of_pattern(expr, stack, context) do + of_shared(expr, stack, context, &of_pattern/3) + end + + ## GUARDS + + # TODO: Some guards can be changed to intersection types or higher order types + @boolean {:union, [{:atom, true}, {:atom, false}]} + @number {:union, [:integer, :float]} + @unary_number_fun [{[:integer], :integer}, {[@number], :float}] + @binary_number_fun [ + {[:integer, :integer], :integer}, + {[:float, @number], :float}, + {[@number, :float], :float} + ] + + @guard_functions %{ + {:is_atom, 1} => [{[:atom], @boolean}], + {:is_binary, 1} => [{[:binary], @boolean}], + {:is_bitstring, 1} => [{[:binary], @boolean}], + {:is_boolean, 1} => [{[@boolean], @boolean}], + {:is_float, 1} => [{[:float], @boolean}], + {:is_function, 1} => [{[:fun], @boolean}], + {:is_function, 2} => [{[:fun, :integer], @boolean}], + {:is_integer, 1} => [{[:integer], @boolean}], + {:is_list, 1} => [{[{:list, :dynamic}], @boolean}], + {:is_map, 1} => [{[{:map, [{:optional, :dynamic, :dynamic}]}], @boolean}], + {:is_map_key, 2} => [{[:dynamic, {:map, [{:optional, :dynamic, :dynamic}]}], :dynamic}], + {:is_number, 1} => [{[@number], @boolean}], + {:is_pid, 1} => [{[:pid], @boolean}], + {:is_port, 1} => [{[:port], @boolean}], + {:is_reference, 1} => [{[:reference], @boolean}], + {:is_tuple, 1} => [{[:tuple], @boolean}], + {:<, 2} => [{[:dynamic, :dynamic], @boolean}], + {:"=<", 2} => [{[:dynamic, :dynamic], @boolean}], + {:>, 2} => [{[:dynamic, :dynamic], @boolean}], + {:>=, 2} => [{[:dynamic, :dynamic], @boolean}], + {:"/=", 2} => [{[:dynamic, :dynamic], @boolean}], + {:"=/=", 2} => [{[:dynamic, :dynamic], @boolean}], + {:==, 2} => [{[:dynamic, :dynamic], @boolean}], + {:"=:=", 2} => [{[:dynamic, :dynamic], @boolean}], + {:*, 2} => @binary_number_fun, + {:+, 1} => @unary_number_fun, + {:+, 2} => @binary_number_fun, + {:-, 1} => @unary_number_fun, + {:-, 2} => @binary_number_fun, + {:/, 2} => @binary_number_fun, + {:abs, 1} => @unary_number_fun, + {:ceil, 1} => [{[@number], :integer}], + {:floor, 1} => [{[@number], :integer}], + {:round, 1} => [{[@number], :integer}], + {:trunc, 1} => [{[@number], :integer}], + {:element, 2} => [{[:integer, :tuple], :dynamic}], + {:hd, 1} => [{[{:list, :dynamic}], :dynamic}], + {:length, 1} => [{[{:list, :dynamic}], :integer}], + {:map_get, 2} => [{[:dynamic, {:map, [{:optional, :dynamic, :dynamic}]}], :dynamic}], + {:map_size, 1} => [{[{:map, [{:optional, :dynamic, :dynamic}]}], :integer}], + {:tl, 1} => [{[{:list, :dynamic}], :dynamic}], + {:tuple_size, 1} => [{[:tuple], :integer}], + {:node, 1} => [{[{:union, [:pid, :reference, :port]}], :atom}], + {:binary_part, 3} => [{[:binary, :integer, :integer], :binary}], + {:bit_size, 1} => [{[:binary], :integer}], + {:byte_size, 1} => [{[:binary], :integer}], + {:size, 1} => [{[{:union, [:binary, :tuple]}], @boolean}], + {:div, 2} => [{[:integer, :integer], :integer}], + {:rem, 2} => [{[:integer, :integer], :integer}], + {:node, 0} => [{[], :atom}], + {:self, 0} => [{[], :pid}], + {:bnot, 1} => [{[:integer], :integer}], + {:band, 2} => [{[:integer, :integer], :integer}], + {:bor, 2} => [{[:integer, :integer], :integer}], + {:bxor, 2} => [{[:integer, :integer], :integer}], + {:bsl, 2} => [{[:integer, :integer], :integer}], + {:bsr, 2} => [{[:integer, :integer], :integer}], + {:or, 2} => [{[@boolean, @boolean], @boolean}], + {:and, 2} => [{[@boolean, @boolean], @boolean}], + {:xor, 2} => [{[@boolean, @boolean], @boolean}], + {:not, 1} => [{[@boolean], @boolean}] + + # Following guards are matched explicitly to handle + # type guard functions such as is_atom/1 + # {:andalso, 2} => {[@boolean, @boolean], @boolean} + # {:orelse, 2} => {[@boolean, @boolean], @boolean} + } + + @type_guards [ + :is_atom, + :is_binary, + :is_bitstring, + :is_boolean, + :is_float, + :is_function, + :is_integer, + :is_list, + :is_map, + :is_number, + :is_pid, + :is_port, + :is_reference, + :is_tuple + ] + + @doc """ + Refines the type variables in the typing context using type check guards + such as `is_integer/1`. + """ + def of_guard(expr, expected, %{context: stack_context} = stack, context) + when stack_context != :pattern do + of_guard(expr, expected, %{stack | context: :pattern}, context) + end + + def of_guard({{:., _, [:erlang, :andalso]}, _, [left, right]} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + + with {:ok, left_type, context} <- of_guard(left, @boolean, stack, context), + {:ok, _, context} <- unify(left_type, @boolean, stack, context), + {:ok, right_type, context} <- of_guard(right, :dynamic, keep_guarded(stack), context), + do: {:ok, to_union([@boolean, right_type], context), context} + end + + def of_guard({{:., _, [:erlang, :orelse]}, _, [left, right]} = expr, _expected, stack, context) do + stack = push_expr_stack(expr, stack) + left_indexes = collect_var_indexes_from_expr(left, context) + right_indexes = collect_var_indexes_from_expr(right, context) + + with {:ok, left_type, left_context} <- of_guard(left, @boolean, stack, context), + {:ok, _right_type, right_context} <- of_guard(right, :dynamic, stack, context), + context = + merge_context_or( + left_indexes, + right_indexes, + context, + stack, + left_context, + right_context + ), + {:ok, _, context} <- unify(left_type, @boolean, stack, context), + do: {:ok, @boolean, context} + end + + # The unary operators + and - are special cased to avoid common warnings until + # we add support for intersection types for the guard functions + # -integer / +integer + def of_guard({{:., _, [:erlang, guard]}, _, [integer]}, _expected, _stack, context) + when guard in [:+, :-] and is_integer(integer) do + {:ok, :integer, context} + end + + # -float / +float + def of_guard({{:., _, [:erlang, guard]}, _, [float]}, _expected, _stack, context) + when guard in [:+, :-] and is_float(float) do + {:ok, :float, context} + end + + # tuple_size(arg) == integer + def of_guard( + {{:., _, [:erlang, :==]}, _, [{{:., _, [:erlang, :tuple_size]}, _, [var]}, size]} = expr, + expected, + stack, + context + ) + when is_var(var) and is_integer(size) do + of_tuple_size(var, size, expr, expected, stack, context) + end + + # integer == tuple_size(arg) + def of_guard( + {{:., _, [:erlang, :==]}, _, [size, {{:., _, [:erlang, :tuple_size]}, _, [var]}]} = expr, + expected, + stack, + context + ) + when is_var(var) and is_integer(size) do + of_tuple_size(var, size, expr, expected, stack, context) + end + + # fun(args) + def of_guard({{:., _, [:erlang, guard]}, _, args} = expr, expected, stack, context) do + type_guard? = type_guard?(guard) + {consider_type_guards?, keep_guarded?} = stack.type_guards + signature = guard_signature(guard, length(args)) + + # Only check type guards in the context of and/or/not, + # a type guard in the context of is_tuple(x) > :foo + # should not affect the inference of x + if not type_guard? or consider_type_guards? do + stack = push_expr_stack(expr, stack) + expected_clauses = filter_clauses(signature, expected, stack, context) + param_unions = signature_to_param_unions(expected_clauses, context) + arg_stack = %{stack | type_guards: {false, keep_guarded?}} + mfa = {:erlang, guard, length(args)} + + with {:ok, arg_types, context} <- + map_reduce_ok(Enum.zip(args, param_unions), context, fn {arg, param}, context -> + of_guard(arg, param, arg_stack, context) + end), + {:ok, return_type, context} <- + unify_call( + arg_types, + expected_clauses, + expected, + mfa, + signature, + stack, + context, + type_guard? + ) do + guard_sources = guard_sources(arg_types, type_guard?, keep_guarded?, context) + {:ok, return_type, %{context | guard_sources: guard_sources}} + end + else + # Assume that type guards always return boolean + boolean = {:union, [atom: true, atom: false]} + [{_params, ^boolean}] = signature + {:ok, boolean, context} + end + end + + # map.field + def of_guard({{:., meta1, [map, field]}, meta2, []}, expected, stack, context) do + of_guard({{:., meta1, [:erlang, :map_get]}, meta2, [field, map]}, expected, stack, context) + end + + # var + def of_guard(var, _expected, _stack, context) when is_var(var) do + {:ok, get_var!(var, context), context} + end + + def of_guard(expr, _expected, stack, context) do + of_shared(expr, stack, context, &of_guard(&1, :dynamic, &2, &3)) + end + + defp of_tuple_size(var, size, expr, _expected, stack, context) do + {consider_type_guards?, _keep_guarded?} = stack.type_guards + + result = + if consider_type_guards? do + stack = push_expr_stack(expr, stack) + tuple_elems = Enum.map(1..size//1, fn _ -> :dynamic end) + + with {:ok, type, context} <- of_guard(var, :dynamic, stack, context), + {:ok, _type, context} <- unify({:tuple, size, tuple_elems}, type, stack, context), + do: {:ok, context} + else + {:ok, context} + end + + case result do + {:ok, context} -> + boolean = {:union, [atom: true, atom: false]} + {:ok, boolean, context} + + {:error, reason} -> + {:error, reason} + end + end + + defp signature_to_param_unions(signature, context) do + signature + |> Enum.map(fn {params, _return} -> params end) + |> zip_many() + |> Enum.map(&to_union(&1, context)) + end + + # Collect guard sources from argument types, see type context documentation + # for more information + defp guard_sources(arg_types, type_guard?, keep_guarded?, context) do + {arg_types, guard_sources} = + case arg_types do + [{:var, index} | rest_arg_types] when type_guard? -> + guard_sources = Map.put_new(context.guard_sources, index, :guarded) + {rest_arg_types, guard_sources} + + _ -> + {arg_types, context.guard_sources} + end + + Enum.reduce(arg_types, guard_sources, fn + {:var, index}, guard_sources -> + Map.update(guard_sources, index, :fail, &guarded_if_keep_guarded(&1, keep_guarded?)) + + _, guard_sources -> + guard_sources + end) + end + + defp collect_var_indexes_from_expr(expr, context) do + {_, vars} = + Macro.prewalk(expr, %{}, fn + var, acc when is_var(var) -> + var_name = var_name(var) + %{^var_name => type} = context.vars + {var, collect_var_indexes(type, context, acc)} + + other, acc -> + {other, acc} + end) + + Map.keys(vars) + end + + defp unify_call(args, clauses, _expected, _mfa, _signature, stack, context, true = _type_guard?) do + unify_type_guard_call(args, clauses, stack, context) + end + + defp unify_call(args, clauses, expected, mfa, signature, stack, context, false = _type_guard?) do + unify_call(args, clauses, expected, mfa, signature, stack, context) + end + + defp unify_call([], [{[], return}], _expected, _mfa, _signature, _stack, context) do + {:ok, return, context} + end + + defp unify_call(args, clauses, expected, mfa, signature, stack, context) do + # Given the arguments: + # foo | bar, {:ok, baz | bat} + + # Expand unions in arguments: + # foo | bar, {:ok, baz} | {:ok, bat} + + # Permute arguments: + # foo, {:ok, baz} + # foo, {:ok, bat} + # bar, {:ok, baz} + # bar, {:ok, bat} + + flatten_args = Enum.map(args, &flatten_union(&1, context)) + cartesian_args = cartesian_product(flatten_args) + + # Remove clauses that do not match the expected type + # Ignore type variables in parameters by changing them to dynamic + + clauses = + clauses + |> filter_clauses(expected, stack, context) + |> Enum.map(fn {params, return} -> + {Enum.map(params, &var_to_dynamic/1), return} + end) + + # For each permuted argument find the clauses they match + # All arguments must match at least one clause, but all clauses + # do not need to match + # Collect the return values from clauses that matched and collect + # the type contexts from unifying argument and parameter to + # infer type variables in arguments + result = + flat_map_ok(cartesian_args, fn cartesian_args -> + result = + Enum.flat_map(clauses, fn {params, return} -> + result = + map_ok(Enum.zip(cartesian_args, params), fn {arg, param} -> + case unify(arg, param, stack, context) do + {:ok, _type, context} -> {:ok, context} + {:error, reason} -> {:error, reason} + end + end) + + case result do + {:ok, contexts} -> [{return, contexts}] + {:error, _reason} -> [] + end + end) + + if result != [] do + {:ok, result} + else + {:error, args} + end + end) + + case result do + {:ok, returns_contexts} -> + {success_returns, contexts} = Enum.unzip(returns_contexts) + contexts = Enum.concat(contexts) + + indexes = + for types <- flatten_args, + type <- types, + index <- collect_var_indexes_from_type(type), + do: index, + uniq: true + + # Build unions from collected type contexts to unify with + # type variables from arguments + result = + map_reduce_ok(indexes, context, fn index, context -> + union = + contexts + |> Enum.map(&Map.fetch!(&1.types, index)) + |> Enum.reject(&(&1 == :unbound)) + + if union == [] do + {:ok, {:var, index}, context} + else + unify({:var, index}, to_union(union, context), stack, context) + end + end) + + case result do + {:ok, _types, context} -> {:ok, to_union(success_returns, context), context} + {:error, reason} -> {:error, reason} + end + + {:error, args} -> + error(:unable_apply, {mfa, args, expected, signature, stack}, context) + end + end + + defp unify_type_guard_call(args, [{params, return}], stack, context) do + result = + reduce_ok(Enum.zip(args, params), context, fn {arg, param}, context -> + case unify(arg, param, stack, context) do + {:ok, _, context} -> {:ok, context} + {:error, reason} -> {:error, reason} + end + end) + + case result do + {:ok, context} -> {:ok, return, context} + {:error, reason} -> {:error, reason} + end + end + + defp cartesian_product(lists) do + List.foldr(lists, [[]], fn list, acc -> + for elem_list <- list, + list_acc <- acc, + do: [elem_list | list_acc] + end) + end + + defp var_to_dynamic(type) do + {type, _acc} = + walk(type, :ok, fn + {:var, _index}, :ok -> + {:dynamic, :ok} + + other, :ok -> + {other, :ok} + end) + + type + end + + defp collect_var_indexes_from_type(type) do + {_type, indexes} = + walk(type, [], fn + {:var, index}, indexes -> + {{:var, index}, [index | indexes]} + + other, indexes -> + {other, indexes} + end) + + indexes + end + + defp merge_context_or(left_indexes, right_indexes, context, stack, left, right) do + left_different = filter_different_indexes(left_indexes, left, right) + right_different = filter_different_indexes(right_indexes, left, right) + + case {left_different, right_different} do + {[index], [index]} -> merge_context_or_equal(index, stack, left, right) + {_, _} -> merge_context_or_diff(left_different, context, left) + end + end + + defp filter_different_indexes(indexes, left, right) do + Enum.filter(indexes, fn index -> + %{^index => left_type} = left.types + %{^index => right_type} = right.types + left_type != right_type + end) + end + + defp merge_context_or_equal(index, stack, left, right) do + %{^index => left_type} = left.types + %{^index => right_type} = right.types + + cond do + left_type == :unbound -> + refine_var!(index, right_type, stack, left) + + right_type == :unbound -> + left + + true -> + # Only include right side if left side is from type guard such as is_list(x), + # do not refine in case of length(x) + if left.guard_sources[index] == :fail do + guard_sources = Map.put(left.guard_sources, index, :fail) + left = %{left | guard_sources: guard_sources} + refine_var!(index, left_type, stack, left) + else + guard_sources = merge_guard_sources([left.guard_sources, right.guard_sources]) + left = %{left | guard_sources: guard_sources} + refine_var!(index, to_union([left_type, right_type], left), stack, left) + end + end + end + + # If the variable failed, we can keep them from the left side as is. + # If they didn't fail, then we need to restore them to their original value. + defp merge_context_or_diff(indexes, old_context, new_context) do + Enum.reduce(indexes, new_context, fn index, context -> + if new_context.guard_sources[index] == :fail do + context + else + restore_var!(index, new_context, old_context) + end + end) + end + + defp merge_guard_sources(sources) do + Enum.reduce(sources, fn left, right -> + Map.merge(left, right, fn + _index, :guarded, :guarded -> :guarded + _index, _, _ -> :fail + end) + end) + end + + defp guarded_if_keep_guarded(:guarded, true), do: :guarded + defp guarded_if_keep_guarded(_, _), do: :fail + + defp keep_guarded(%{type_guards: {consider?, _}} = stack), + do: %{stack | type_guards: {consider?, true}} + + defp filter_clauses(signature, expected, stack, context) do + Enum.filter(signature, fn {_params, return} -> + match?({:ok, _type, _context}, unify(return, expected, stack, context)) + end) + end + + Enum.each(@guard_functions, fn {{name, arity}, signature} -> + defp guard_signature(unquote(name), unquote(arity)), do: unquote(Macro.escape(signature)) + end) + + Enum.each(@type_guards, fn name -> + defp type_guard?(unquote(name)), do: true + end) + + defp type_guard?(name) when is_atom(name), do: false + + ## Shared + + # :atom + defp of_shared(atom, _stack, context, _fun) when is_atom(atom) do + {:ok, {:atom, atom}, context} + end + + # 12 + defp of_shared(literal, _stack, context, _fun) when is_integer(literal) do + {:ok, :integer, context} + end + + # 1.2 + defp of_shared(literal, _stack, context, _fun) when is_float(literal) do + {:ok, :float, context} + end + + # "..." + defp of_shared(literal, _stack, context, _fun) when is_binary(literal) do + {:ok, :binary, context} + end + + # <<...>>> + defp of_shared({:<<>>, _meta, args}, stack, context, fun) do + expected_fun = fn arg, _expected, stack, context -> fun.(arg, stack, context) end + + case Of.binary(args, stack, context, expected_fun) do + {:ok, context} -> {:ok, :binary, context} + {:error, reason} -> {:error, reason} + end + end + + # left | [] + defp of_shared({:|, _meta, [left_expr, []]} = expr, stack, context, fun) do + stack = push_expr_stack(expr, stack) + fun.(left_expr, stack, context) + end + + # left | right + defp of_shared({:|, _meta, [left_expr, right_expr]} = expr, stack, context, fun) do + stack = push_expr_stack(expr, stack) + + case fun.(left_expr, stack, context) do + {:ok, left, context} -> + case fun.(right_expr, stack, context) do + {:ok, {:list, right}, context} -> + {:ok, to_union([left, right], context), context} + + {:ok, right, context} -> + {:ok, to_union([left, right], context), context} + + {:error, reason} -> + {:error, reason} + end + + {:error, reason} -> + {:error, reason} + end + end + + # [] + defp of_shared([], _stack, context, _fun) do + {:ok, {:list, :dynamic}, context} + end + + # [expr, ...] + defp of_shared(exprs, stack, context, fun) when is_list(exprs) do + stack = push_expr_stack(exprs, stack) + + case map_reduce_ok(exprs, context, &fun.(&1, stack, &2)) do + {:ok, types, context} -> {:ok, {:list, to_union(types, context)}, context} + {:error, reason} -> {:error, reason} + end + end + + # left ++ right + defp of_shared( + {{:., _meta1, [:erlang, :++]}, _meta2, [left_expr, right_expr]} = expr, + stack, + context, + fun + ) do + stack = push_expr_stack(expr, stack) + + case fun.(left_expr, stack, context) do + {:ok, {:list, left}, context} -> + case fun.(right_expr, stack, context) do + {:ok, {:list, right}, context} -> + {:ok, {:list, to_union([left, right], context)}, context} + + {:ok, right, context} -> + {:ok, {:list, to_union([left, right], context)}, context} + + {:error, reason} -> + {:error, reason} + end + + {:error, reason} -> + {:error, reason} + end + end + + # {left, right} + defp of_shared({left, right}, stack, context, fun) do + of_shared({:{}, [], [left, right]}, stack, context, fun) + end + + # {...} + defp of_shared({:{}, _meta, exprs} = expr, stack, context, fun) do + stack = push_expr_stack(expr, stack) + + case map_reduce_ok(exprs, context, &fun.(&1, stack, &2)) do + {:ok, types, context} -> {:ok, {:tuple, length(types), types}, context} + {:error, reason} -> {:error, reason} + end + end + + # %{...} + defp of_shared({:%{}, _meta, args} = expr, stack, context, fun) do + stack = push_expr_stack(expr, stack) + expected_fun = fn arg, _expected, stack, context -> fun.(arg, stack, context) end + Of.open_map(args, stack, context, expected_fun) + end + + # %Struct{...} + defp of_shared({:%, meta1, [module, {:%{}, _meta2, args}]} = expr, stack, context, fun) + when is_atom(module) do + stack = push_expr_stack(expr, stack) + expected_fun = fn arg, _expected, stack, context -> fun.(arg, stack, context) end + + with {:ok, struct, context} <- Of.struct(module, meta1, context), + {:ok, map, context} <- Of.open_map(args, stack, context, expected_fun) do + unify(map, struct, stack, context) + end + end +end diff --git a/lib/elixir/lib/module/types/unify.ex b/lib/elixir/lib/module/types/unify.ex new file mode 100644 index 00000000000..f26e338aeab --- /dev/null +++ b/lib/elixir/lib/module/types/unify.ex @@ -0,0 +1,1012 @@ +defmodule Module.Types.Unify do + @moduledoc false + + import Module.Types.Helpers + + # Those are the simple types known to the system: + # + # :dynamic + # {:var, var} + # {:atom, atom} < :atom + # :integer + # :float + # :binary + # :pid + # :port + # :reference + # + # Those are the composite types: + # + # {:list, type} + # {:tuple, size, [type]} < :tuple + # {:union, [type]} + # {:map, [{:required | :optional, key_type, value_type}]} + # {:fun, [{params, return}]} + # + # Once new types are added, they should be considered in: + # + # * unify (all) + # * format_type (all) + # * subtype? (subtypes only) + # * recursive_type? (composite only) + # * collect_var_indexes (composite only) + # * lift_types (composite only) + # * flatten_union (composite only) + # * walk (composite only) + # + + @doc """ + Unifies two types and returns the unified type and an updated typing context + or an error in case of a typing conflict. + """ + def unify(same, same, _stack, context) do + {:ok, same, context} + end + + def unify({:var, var}, type, stack, context) do + unify_var(var, type, stack, context, _var_source = true) + end + + def unify(type, {:var, var}, stack, context) do + unify_var(var, type, stack, context, _var_source = false) + end + + def unify({:tuple, n, sources}, {:tuple, n, targets}, stack, context) do + result = + map_reduce_ok(Enum.zip(sources, targets), context, fn {source, target}, context -> + unify(source, target, stack, context) + end) + + case result do + {:ok, types, context} -> {:ok, {:tuple, n, types}, context} + {:error, reason} -> {:error, reason} + end + end + + def unify({:list, source}, {:list, target}, stack, context) do + case unify(source, target, stack, context) do + {:ok, type, context} -> {:ok, {:list, type}, context} + {:error, reason} -> {:error, reason} + end + end + + def unify({:map, source_pairs}, {:map, target_pairs}, stack, context) do + unify_maps(source_pairs, target_pairs, stack, context) + end + + def unify(source, :dynamic, _stack, context) do + {:ok, source, context} + end + + def unify(:dynamic, target, _stack, context) do + {:ok, target, context} + end + + def unify({:union, types}, target, stack, context) do + unify_result = + map_reduce_ok(types, context, fn type, context -> + unify(type, target, stack, context) + end) + + case unify_result do + {:ok, types, context} -> {:ok, to_union(types, context), context} + {:error, context} -> {:error, context} + end + end + + def unify(source, target, stack, context) do + cond do + # TODO: This condition exists to handle unions with unbound vars. + match?({:union, _}, target) and has_unbound_var?(target, context) -> + {:ok, source, context} + + subtype?(source, target, context) -> + {:ok, source, context} + + true -> + error(:unable_unify, {source, target, stack}, context) + end + end + + def unify_var(var, :dynamic, _stack, context, _var_source?) do + {:ok, {:var, var}, context} + end + + def unify_var(var, type, stack, context, var_source?) do + case context.types do + %{^var => :unbound} -> + context = refine_var!(var, type, stack, context) + stack = push_unify_stack(var, stack) + + if recursive_type?(type, [], context) do + if var_source? do + error(:unable_unify, {{:var, var}, type, stack}, context) + else + error(:unable_unify, {type, {:var, var}, stack}, context) + end + else + {:ok, {:var, var}, context} + end + + %{^var => {:var, new_var} = var_type} -> + unify_result = + if var_source? do + unify(var_type, type, stack, context) + else + unify(type, var_type, stack, context) + end + + case unify_result do + {:ok, type, context} -> + {:ok, type, context} + + {:error, {type, reason, %{traces: error_traces} = error_context}} -> + old_var_traces = Map.get(context.traces, new_var, []) + new_var_traces = Map.get(error_traces, new_var, []) + add_var_traces = Enum.drop(new_var_traces, -length(old_var_traces)) + + error_traces = + error_traces + |> Map.update(var, add_var_traces, &(add_var_traces ++ &1)) + |> Map.put(new_var, old_var_traces) + + {:error, {type, reason, %{error_context | traces: error_traces}}} + end + + %{^var => var_type} -> + # Only add trace if the variable wasn't already "expanded" + context = + if variable_expanded?(var, stack, context) do + context + else + trace_var(var, type, stack, context) + end + + stack = push_unify_stack(var, stack) + + unify_result = + if var_source? do + unify(var_type, type, stack, context) + else + unify(type, var_type, stack, context) + end + + case unify_result do + {:ok, {:var, ^var}, context} -> + {:ok, {:var, var}, context} + + {:ok, res_type, context} -> + context = refine_var!(var, res_type, stack, context) + {:ok, {:var, var}, context} + + {:error, reason} -> + {:error, reason} + end + end + end + + # * All required keys on each side need to match to the other side. + # * All optional keys on each side that do not match must be discarded. + + def unify_maps(source_pairs, target_pairs, stack, context) do + {source_required, source_optional} = split_pairs(source_pairs) + {target_required, target_optional} = split_pairs(target_pairs) + + with {:ok, source_required_pairs, context} <- + unify_source_required(source_required, target_pairs, stack, context), + {:ok, target_required_pairs, context} <- + unify_target_required(target_required, source_pairs, stack, context), + {:ok, source_optional_pairs, context} <- + unify_source_optional(source_optional, target_optional, stack, context), + {:ok, target_optional_pairs, context} <- + unify_target_optional(target_optional, source_optional, stack, context) do + # Remove duplicate pairs from matching in both left and right directions + pairs = + Enum.uniq( + source_required_pairs ++ + target_required_pairs ++ + source_optional_pairs ++ + target_optional_pairs + ) + + {:ok, {:map, pairs}, context} + else + {:error, :unify} -> + error(:unable_unify, {{:map, source_pairs}, {:map, target_pairs}, stack}, context) + + {:error, context} -> + {:error, context} + end + end + + def unify_source_required(source_required, target_pairs, stack, context) do + map_reduce_ok(source_required, context, fn {source_key, source_value}, context -> + Enum.find_value(target_pairs, fn {target_kind, target_key, target_value} -> + with {:ok, key, context} <- unify(source_key, target_key, stack, context) do + case unify(source_value, target_value, stack, context) do + {:ok, value, context} -> + {:ok, {:required, key, value}, context} + + {:error, _reason} -> + source_map = {:map, [{:required, source_key, source_value}]} + target_map = {:map, [{target_kind, target_key, target_value}]} + error(:unable_unify, {source_map, target_map, stack}, context) + end + else + {:error, _reason} -> nil + end + end) || {:error, :unify} + end) + end + + def unify_target_required(target_required, source_pairs, stack, context) do + map_reduce_ok(target_required, context, fn {target_key, target_value}, context -> + Enum.find_value(source_pairs, fn {source_kind, source_key, source_value} -> + with {:ok, key, context} <- unify(source_key, target_key, stack, context) do + case unify(source_value, target_value, stack, context) do + {:ok, value, context} -> + {:ok, {:required, key, value}, context} + + {:error, _reason} -> + source_map = {:map, [{source_kind, source_key, source_value}]} + target_map = {:map, [{:required, target_key, target_value}]} + error(:unable_unify, {source_map, target_map, stack}, context) + end + else + {:error, _reason} -> nil + end + end) || {:error, :unify} + end) + end + + def unify_source_optional(source_optional, target_optional, stack, context) do + flat_map_reduce_ok(source_optional, context, fn {source_key, source_value}, context -> + Enum.find_value(target_optional, fn {target_key, target_value} -> + with {:ok, key, context} <- unify(source_key, target_key, stack, context) do + case unify(source_value, target_value, stack, context) do + {:ok, value, context} -> + {:ok, [{:optional, key, value}], context} + + {:error, _reason} -> + source_map = {:map, [{:optional, source_key, source_value}]} + target_map = {:map, [{:optional, target_key, target_value}]} + error(:unable_unify, {source_map, target_map, stack}, context) + end + else + _ -> nil + end + end) || {:ok, [], context} + end) + end + + def unify_target_optional(target_optional, source_optional, stack, context) do + flat_map_reduce_ok(target_optional, context, fn {target_key, target_value}, context -> + Enum.find_value(source_optional, fn {source_key, source_value} -> + with {:ok, key, context} <- unify(source_key, target_key, stack, context) do + case unify(source_value, target_value, stack, context) do + {:ok, value, context} -> + {:ok, [{:optional, key, value}], context} + + {:error, _reason} -> + source_map = {:map, [{:optional, source_key, source_value}]} + target_map = {:map, [{:optional, target_key, target_value}]} + error(:unable_unify, {source_map, target_map, stack}, context) + end + else + _ -> nil + end + end) || {:ok, [], context} + end) + end + + defp split_pairs(pairs) do + {required, optional} = + Enum.split_with(pairs, fn {kind, _key, _value} -> kind == :required end) + + required = Enum.map(required, fn {_kind, key, value} -> {key, value} end) + optional = Enum.map(optional, fn {_kind, key, value} -> {key, value} end) + {required, optional} + end + + def error(type, reason, context), do: {:error, {type, reason, context}} + + @doc """ + Push expression to stack. + + The expression stack is used to give the context where a type variable + was refined when show a type conflict error. + """ + def push_expr_stack(expr, stack) do + %{stack | last_expr: expr} + end + + @doc """ + Gets a variable. + """ + def get_var!(var, context) do + Map.fetch!(context.vars, var_name(var)) + end + + @doc """ + Adds a variable to the typing context and returns its type variable. + If the variable has already been added, return the existing type variable. + """ + def new_var(var, context) do + var_name = var_name(var) + + case context.vars do + %{^var_name => type} -> + {type, context} + + %{} -> + type = {:var, context.counter} + vars = Map.put(context.vars, var_name, type) + types_to_vars = Map.put(context.types_to_vars, context.counter, var) + types = Map.put(context.types, context.counter, :unbound) + traces = Map.put(context.traces, context.counter, []) + + context = %{ + context + | vars: vars, + types_to_vars: types_to_vars, + types: types, + traces: traces, + counter: context.counter + 1 + } + + {type, context} + end + end + + @doc """ + Adds an internal variable to the typing context and returns its type variable. + An internal variable is used to help unify complex expressions, + it does not belong to a specific AST expression. + """ + def add_var(context) do + type = {:var, context.counter} + types = Map.put(context.types, context.counter, :unbound) + traces = Map.put(context.traces, context.counter, []) + + context = %{ + context + | types: types, + traces: traces, + counter: context.counter + 1 + } + + {type, context} + end + + @doc """ + Resolves a variable raising if it is unbound. + """ + def resolve_var({:var, var}, context) do + case context.types do + %{^var => :unbound} -> raise "cannot resolve unbound var" + %{^var => type} -> resolve_var(type, context) + end + end + + def resolve_var(other, _context), do: other + + # Check unify stack to see if variable was already expanded + defp variable_expanded?(var, stack, context) do + Enum.any?(stack.unify_stack, &variable_same?(var, &1, context)) + end + + defp variable_same?(left, right, context) do + case context.types do + %{^left => {:var, new_left}} -> + variable_same?(new_left, right, context) + + %{^right => {:var, new_right}} -> + variable_same?(left, new_right, context) + + %{} -> + false + end + end + + defp push_unify_stack(var, stack) do + %{stack | unify_stack: [var | stack.unify_stack]} + end + + @doc """ + Restores the variable information from the old context into new context. + """ + def restore_var!(var, new_context, old_context) do + %{^var => type} = old_context.types + %{^var => trace} = old_context.traces + types = Map.put(new_context.types, var, type) + traces = Map.put(new_context.traces, var, trace) + %{new_context | types: types, traces: traces} + end + + @doc """ + Set the type for a variable and add trace. + """ + def refine_var!(var, type, stack, context) do + types = Map.put(context.types, var, type) + context = %{context | types: types} + trace_var(var, type, stack, context) + end + + @doc """ + Remove type variable and all its traces. + """ + def remove_var(var, context) do + types = Map.delete(context.types, var) + traces = Map.delete(context.traces, var) + %{context | types: types, traces: traces} + end + + defp trace_var(var, type, %{trace: true, last_expr: last_expr} = _stack, context) do + line = get_meta(last_expr)[:line] + trace = {type, last_expr, {context.file, line}} + traces = Map.update!(context.traces, var, &[trace | &1]) + %{context | traces: traces} + end + + defp trace_var(_var, _type, %{trace: false} = _stack, context) do + context + end + + # Check if a variable is recursive and incompatible with itself + # Bad: `{var} = var` + # Good: `x = y; y = z; z = x` + defp recursive_type?({:var, var} = parent, parents, context) do + case context.types do + %{^var => :unbound} -> + false + + %{^var => type} -> + if type in parents do + not Enum.all?(parents, &match?({:var, _}, &1)) + else + recursive_type?(type, [parent | parents], context) + end + end + end + + defp recursive_type?({:list, type} = parent, parents, context) do + recursive_type?(type, [parent | parents], context) + end + + defp recursive_type?({:union, types} = parent, parents, context) do + Enum.any?(types, &recursive_type?(&1, [parent | parents], context)) + end + + defp recursive_type?({:tuple, _, types} = parent, parents, context) do + Enum.any?(types, &recursive_type?(&1, [parent | parents], context)) + end + + defp recursive_type?({:map, pairs} = parent, parents, context) do + Enum.any?(pairs, fn {_kind, key, value} -> + recursive_type?(key, [parent | parents], context) or + recursive_type?(value, [parent | parents], context) + end) + end + + defp recursive_type?({:fun, clauses}, parents, context) do + Enum.any?(clauses, fn {args, return} -> + Enum.any?([return | args], &recursive_type?(&1, [clauses | parents], context)) + end) + end + + defp recursive_type?(_other, _parents, _context) do + false + end + + @doc """ + Collects all type vars recursively. + """ + def collect_var_indexes(type, context, acc \\ %{}) do + {_type, indexes} = + walk(type, acc, fn + {:var, var}, acc -> + case acc do + %{^var => _} -> + {{:var, var}, acc} + + %{} -> + case context.types do + %{^var => :unbound} -> + {{:var, var}, Map.put(acc, var, true)} + + %{^var => type} -> + {{:var, var}, collect_var_indexes(type, context, Map.put(acc, var, true))} + end + end + + other, acc -> + {other, acc} + end) + + indexes + end + + @doc """ + Checks if the type has a type var. + """ + def has_unbound_var?(type, context) do + walk(type, :ok, fn + {:var, var}, acc -> + case context.types do + %{^var => :unbound} -> + throw(:has_unbound_var?) + + %{^var => type} -> + has_unbound_var?(type, context) + {{:var, var}, acc} + end + + other, acc -> + {other, acc} + end) + + false + catch + :throw, :has_unbound_var? -> true + end + + @doc """ + Returns true if it is a singleton type. + + Only atoms are singleton types. Unbound vars are not + considered singleton types. + """ + def singleton?({:var, var}, context) do + case context.types do + %{^var => :unbound} -> false + %{^var => type} -> singleton?(type, context) + end + end + + def singleton?({:atom, _}, _context), do: true + def singleton?(_type, _context), do: false + + @doc """ + Checks if the first argument is a subtype of the second argument. + + This function assumes that: + + * unbound variables are not subtype of anything + + * dynamic is not considered a subtype of all other types but the top type. + This allows this function can be used for ordering, in other cases, you + may need to check for both sides + + """ + def subtype?(type, type, _context), do: true + + def subtype?({:var, var}, other, context) do + case context.types do + %{^var => :unbound} -> false + %{^var => type} -> subtype?(type, other, context) + end + end + + def subtype?(other, {:var, var}, context) do + case context.types do + %{^var => :unbound} -> false + %{^var => type} -> subtype?(other, type, context) + end + end + + def subtype?(_, :dynamic, _context), do: true + def subtype?({:atom, atom}, :atom, _context) when is_atom(atom), do: true + + # Composite + + def subtype?({:tuple, _, _}, :tuple, _context), do: true + + def subtype?({:tuple, n, left_types}, {:tuple, n, right_types}, context) do + left_types + |> Enum.zip(right_types) + |> Enum.all?(fn {left, right} -> subtype?(left, right, context) end) + end + + def subtype?({:map, left_pairs}, {:map, right_pairs}, context) do + Enum.all?(left_pairs, fn + {:required, left_key, left_value} -> + Enum.any?(right_pairs, fn {_, right_key, right_value} -> + subtype?(left_key, right_key, context) and subtype?(left_value, right_value, context) + end) + + {:optional, _, _} -> + true + end) + end + + def subtype?({:list, left}, {:list, right}, context) do + subtype?(left, right, context) + end + + def subtype?({:union, left_types}, {:union, _} = right_union, context) do + Enum.all?(left_types, &subtype?(&1, right_union, context)) + end + + def subtype?(left, {:union, right_types}, context) do + Enum.any?(right_types, &subtype?(left, &1, context)) + end + + def subtype?({:union, left_types}, right, context) do + Enum.all?(left_types, &subtype?(&1, right, context)) + end + + def subtype?(_left, _right, _context), do: false + + @doc """ + Returns a "simplified" union using `subtype?/3` to remove redundant types. + + Due to limitations in `subtype?/3` some overlapping types may still be + included. For example unions with overlapping non-concrete types such as + `{boolean()} | {atom()}` will not be merged or types with variables that + are distinct but equivalent such as `a | b when a ~ b`. + """ + def to_union([type], _context), do: type + + def to_union(types, context) when types != [] do + case unique_super_types(unnest_unions(types), context) do + [type] -> type + types -> {:union, types} + end + end + + defp unnest_unions(types) do + Enum.flat_map(types, fn + {:union, types} -> unnest_unions(types) + type -> [type] + end) + end + + # Filter subtypes + # + # `boolean() | atom()` => `atom()` + # `:foo | atom()` => `atom()` + # + # Does not merge `true | false` => `boolean()` + defp unique_super_types([type | types], context) do + types = Enum.reject(types, &subtype?(&1, type, context)) + + if Enum.any?(types, &subtype?(type, &1, context)) do + unique_super_types(types, context) + else + [type | unique_super_types(types, context)] + end + end + + defp unique_super_types([], _context) do + [] + end + + ## Type lifting + + @doc """ + Lifts type variables to their inferred types from the context. + """ + def lift_types(types, %{lifted_types: _} = context) do + Enum.map_reduce(types, context, &lift_type/2) + end + + def lift_types(types, context) do + context = %{ + types: context.types, + lifted_types: %{}, + lifted_counter: 0 + } + + Enum.map_reduce(types, context, &lift_type/2) + end + + # Lift type variable to its inferred (hopefully concrete) types from the context + defp lift_type({:var, var}, context) do + case context.lifted_types do + %{^var => lifted_var} -> + {{:var, lifted_var}, context} + + %{} -> + case context.types do + %{^var => :unbound} -> + new_lifted_var(var, context) + + %{^var => type} -> + if recursive_type?(type, [], context) do + new_lifted_var(var, context) + else + # Remove visited types to avoid infinite loops + # then restore after we are done recursing on vars + types = context.types + context = put_in(context.types[var], :unbound) + {type, context} = lift_type(type, context) + {type, %{context | types: types}} + end + + %{} -> + new_lifted_var(var, context) + end + end + end + + defp lift_type({:union, types}, context) do + {types, context} = Enum.map_reduce(types, context, &lift_type/2) + {{:union, types}, context} + end + + defp lift_type({:tuple, n, types}, context) do + {types, context} = Enum.map_reduce(types, context, &lift_type/2) + {{:tuple, n, types}, context} + end + + defp lift_type({:map, pairs}, context) do + {pairs, context} = + Enum.map_reduce(pairs, context, fn {kind, key, value}, context -> + {key, context} = lift_type(key, context) + {value, context} = lift_type(value, context) + {{kind, key, value}, context} + end) + + {{:map, pairs}, context} + end + + defp lift_type({:list, type}, context) do + {type, context} = lift_type(type, context) + {{:list, type}, context} + end + + defp lift_type({:fun, clauses}, context) do + clauses = + Enum.map_reduce(clauses, context, fn {args, return}, context -> + {[return | args], context} = Enum.map_reduce([return | args], context, &lift_type/2) + {{args, return}, context} + end) + + {{:fun, clauses}, context} + end + + defp lift_type(other, context) do + {other, context} + end + + defp new_lifted_var(original_var, context) do + types = Map.put(context.lifted_types, original_var, context.lifted_counter) + counter = context.lifted_counter + 1 + + type = {:var, context.lifted_counter} + context = %{context | lifted_types: types, lifted_counter: counter} + {type, context} + end + + # TODO: Figure out function expansion + + @doc """ + Expand unions so that all unions are at the top level. + + {integer() | float()} => {integer()} | {float()} + """ + def flatten_union({:union, types}, context) do + Enum.flat_map(types, &flatten_union(&1, context)) + end + + def flatten_union(type, context) do + List.wrap(do_flatten_union(type, context)) + end + + def do_flatten_union({:tuple, num, types}, context) do + flatten_union_tuple(types, num, context, []) + end + + def do_flatten_union({:list, type}, context) do + case do_flatten_union(type, context) do + {:union, union_types} -> Enum.map(union_types, &{:list, &1}) + _type -> [{:list, type}] + end + end + + def do_flatten_union({:map, pairs}, context) do + flatten_union_map(pairs, context, []) + end + + def do_flatten_union({:var, var}, context) do + if looping_var?(var, context, []) do + {:var, var} + else + case context.types do + %{^var => :unbound} -> {:var, var} + %{^var => {:union, types}} -> Enum.map(types, &do_flatten_union(&1, context)) + %{^var => type} -> do_flatten_union(type, context) + end + end + end + + def do_flatten_union(type, _context) do + type + end + + defp flatten_union_tuple([type | types], num, context, acc) do + case do_flatten_union(type, context) do + {:union, union_types} -> + Enum.flat_map(union_types, &flatten_union_tuple(types, num, context, [&1 | acc])) + + type -> + flatten_union_tuple(types, num, context, [type | acc]) + end + end + + defp flatten_union_tuple([], num, _context, acc) do + [{:tuple, num, Enum.reverse(acc)}] + end + + defp flatten_union_map([{kind, key, value} | pairs], context, acc) do + case do_flatten_union(key, context) do + {:union, union_types} -> + Enum.flat_map(union_types, &flatten_union_map_value(kind, &1, value, pairs, context, acc)) + + type -> + flatten_union_map_value(kind, type, value, pairs, context, acc) + end + end + + defp flatten_union_map([], _context, acc) do + [{:map, Enum.reverse(acc)}] + end + + defp flatten_union_map_value(kind, key, value, pairs, context, acc) do + case do_flatten_union(value, context) do + {:union, union_types} -> + Enum.flat_map(union_types, &flatten_union_map(pairs, context, [{kind, key, &1} | acc])) + + value -> + flatten_union_map(pairs, context, [{kind, key, value} | acc]) + end + end + + defp looping_var?(var, context, parents) do + case context.types do + %{^var => :unbound} -> + false + + %{^var => {:var, type}} -> + if var in parents do + true + else + looping_var?(type, context, [var | parents]) + end + + %{^var => _type} -> + false + end + end + + @doc """ + Formats types. + + The second argument says when complex types such as maps and + structs should be simplified and not shown. + """ + def format_type({:map, pairs}, true) do + case List.keyfind(pairs, {:atom, :__struct__}, 1) do + {:required, {:atom, :__struct__}, {:atom, struct}} -> + ["%", inspect(struct), "{}"] + + _ -> + "map()" + end + end + + def format_type({:union, types}, simplify?) do + types + |> Enum.map(&format_type(&1, simplify?)) + |> Enum.intersperse(" | ") + end + + def format_type({:tuple, _, types}, simplify?) do + format = + types + |> Enum.map(&format_type(&1, simplify?)) + |> Enum.intersperse(", ") + + ["{", format, "}"] + end + + def format_type({:list, type}, simplify?) do + ["[", format_type(type, simplify?), "]"] + end + + def format_type({:map, pairs}, false) do + case List.keytake(pairs, {:atom, :__struct__}, 1) do + {{:required, {:atom, :__struct__}, {:atom, struct}}, pairs} -> + ["%", inspect(struct), "{", format_map_pairs(pairs), "}"] + + _ -> + ["%{", format_map_pairs(pairs), "}"] + end + end + + def format_type({:atom, literal}, _simplify?) do + inspect(literal) + end + + def format_type({:var, index}, _simplify?) do + ["var", Integer.to_string(index + 1)] + end + + def format_type({:fun, clauses}, simplify?) do + format = + Enum.map(clauses, fn {params, return} -> + params = Enum.intersperse(Enum.map(params, &format_type(&1, simplify?)), ", ") + params = if params == [], do: params, else: [params, " "] + return = format_type(return, simplify?) + [params, "-> ", return] + end) + + ["(", Enum.intersperse(format, "; "), ")"] + end + + def format_type(atom, _simplify?) when is_atom(atom) do + [Atom.to_string(atom), "()"] + end + + defp format_map_pairs(pairs) do + {atoms, others} = Enum.split_with(pairs, &match?({:required, {:atom, _}, _}, &1)) + {required, optional} = Enum.split_with(others, &match?({:required, _, _}, &1)) + + (atoms ++ required ++ optional) + |> Enum.map(fn + {:required, {:atom, atom}, right} -> + [Atom.to_string(atom), ": ", format_type(right, false)] + + {:required, left, right} -> + [format_type(left, false), " => ", format_type(right, false)] + + {:optional, left, right} -> + ["optional(", format_type(left, false), ") => ", format_type(right, false)] + end) + |> Enum.intersperse(", ") + end + + @doc """ + Performs a depth-first, pre-order traversal of the type tree using an accumulator. + """ + def walk({:map, pairs}, acc, fun) do + {pairs, acc} = + Enum.map_reduce(pairs, acc, fn {kind, key, value}, acc -> + {key, acc} = walk(key, acc, fun) + {value, acc} = walk(value, acc, fun) + {{kind, key, value}, acc} + end) + + fun.({:map, pairs}, acc) + end + + def walk({:union, types}, acc, fun) do + {types, acc} = Enum.map_reduce(types, acc, &walk(&1, &2, fun)) + fun.({:union, types}, acc) + end + + def walk({:tuple, num, types}, acc, fun) do + {types, acc} = Enum.map_reduce(types, acc, &walk(&1, &2, fun)) + fun.({:tuple, num, types}, acc) + end + + def walk({:list, type}, acc, fun) do + {type, acc} = walk(type, acc, fun) + fun.({:list, type}, acc) + end + + def walk({:fun, clauses}, acc, fun) do + {clauses, acc} = + Enum.map_reduce(clauses, acc, fn {params, return}, acc -> + {params, acc} = Enum.map_reduce(params, acc, &walk(&1, &2, fun)) + {return, acc} = walk(return, acc, fun) + {{params, return}, acc} + end) + + fun.({:fun, clauses}, acc) + end + + def walk(type, acc, fun) do + fun.(type, acc) + end +end diff --git a/lib/elixir/lib/node.ex b/lib/elixir/lib/node.ex index 9b52e451f0c..812fd998270 100644 --- a/lib/elixir/lib/node.ex +++ b/lib/elixir/lib/node.ex @@ -13,11 +13,25 @@ defmodule Node do @doc """ Turns a non-distributed node into a distributed node. - This functionality starts the `:net_kernel` and other - related processes. + This functionality starts the `:net_kernel` and other related + processes. + + This function is rarely invoked in practice. Instead, nodes are + named and started via the command line by using the `--sname` and + `--name` flags. If you need to use this function to dynamically + name a node, please make sure the `epmd` operating system process + is running by calling `epmd -daemon`. + + Invoking this function when the distribution has already been started, + either via the command line interface or dynamically, will return an + error. + + ## Examples + + {:ok, pid} = Node.start(:example, :shortnames, 15000) + """ - @spec start(node, :longnames | :shortnames, non_neg_integer) :: - {:ok, pid} | {:error, term} + @spec start(node, :longnames | :shortnames, non_neg_integer) :: {:ok, pid} | {:error, term} def start(name, type \\ :longnames, tick_time \\ 15000) do :net_kernel.start([name, type, tick_time]) end @@ -30,7 +44,7 @@ defmodule Node do returns `{:error, :not_allowed}`. Returns `{:error, :not_found}` if the local node is not alive. """ - @spec stop() :: :ok | {:error, term} + @spec stop() :: :ok | {:error, :not_allowed | :not_found} def stop() do :net_kernel.stop() end @@ -60,6 +74,8 @@ defmodule Node do the local node. Same as `list(:visible)`. + + Inlined by the compiler. """ @spec list :: [t] def list do @@ -72,9 +88,11 @@ defmodule Node do The result returned when the argument is a list, is the list of nodes satisfying the disjunction(s) of the list elements. - See http://www.erlang.org/doc/man/erlang.html#nodes-1 for more info. + For more information, see `:erlang.nodes/1`. + + Inlined by the compiler. """ - @typep state :: :visible | :hidden | :connected | :this | :known + @type state :: :visible | :hidden | :connected | :this | :known @spec list(state | [state]) :: [t] def list(args) do :erlang.nodes(args) @@ -86,7 +104,9 @@ defmodule Node do If `flag` is `true`, monitoring is turned on. If `flag` is `false`, monitoring is turned off. - See http://www.erlang.org/doc/man/erlang.html#monitor_node-2 for more info. + For more information, see `:erlang.monitor_node/2`. + + For monitoring status changes of all nodes, see `:net_kernel.monitor_nodes/2`. """ @spec monitor(t, boolean) :: true def monitor(node, flag) do @@ -97,7 +117,9 @@ defmodule Node do Behaves as `monitor/2` except that it allows an extra option to be given, namely `:allow_passive_connect`. - See http://www.erlang.org/doc/man/erlang.html#monitor_node-3 for more info. + For more information, see `:erlang.monitor_node/3`. + + For monitoring status changes of all nodes, see `:net_kernel.monitor_nodes/2`. """ @spec monitor(t, boolean, [:allow_passive_connect]) :: true def monitor(node, flag, options) do @@ -128,7 +150,7 @@ defmodule Node do protocols. Returns `true` if disconnection succeeds, otherwise `false`. If the local node is not alive, the function returns `:ignored`. - See http://www.erlang.org/doc/man/erlang.html#disconnect_node-1 for more info. + For more information, see `:erlang.disconnect_node/1`. """ @spec disconnect(t) :: boolean | :ignored def disconnect(node) do @@ -141,7 +163,7 @@ defmodule Node do Returns `true` if successful, `false` if not, and the atom `:ignored` if the local node is not alive. - See http://erlang.org/doc/man/net_kernel.html#connect_node-1 for more info. + For more information, see `:net_kernel.connect_node/1`. """ @spec connect(t) :: boolean | :ignored def connect(node) do @@ -149,11 +171,10 @@ defmodule Node do end @doc """ - Returns the pid of a new process started by the application of `fun` - on `node`. If `node` does not exist, a useless pid is returned. + Returns the PID of a new process started by the application of `fun` + on `node`. If `node` does not exist, a useless PID is returned. - Check http://www.erlang.org/doc/man/erlang.html#spawn-2 for - the list of available options. + For the list of available options, see `:erlang.spawn/2`. Inlined by the compiler. """ @@ -163,27 +184,27 @@ defmodule Node do end @doc """ - Returns the pid of a new process started by the application of `fun` + Returns the PID of a new process started by the application of `fun` on `node`. - If `node` does not exist, a useless pid is returned. Check - http://www.erlang.org/doc/man/erlang.html#spawn_opt-3 for the list of - available options. + If `node` does not exist, a useless PID is returned. + + For the list of available options, see `:erlang.spawn_opt/3`. Inlined by the compiler. """ - @spec spawn(t, (() -> any), Process.spawn_opts) :: pid | {pid, reference} + @spec spawn(t, (() -> any), Process.spawn_opts()) :: pid | {pid, reference} def spawn(node, fun, opts) do :erlang.spawn_opt(node, fun, opts) end @doc """ - Returns the pid of a new process started by the application of + Returns the PID of a new process started by the application of `module.function(args)` on `node`. - If `node` does not exist, a useless pid is returned. Check - http://www.erlang.org/doc/man/erlang.html#spawn-4 for the list of - available options. + If `node` does not exist, a useless PID is returned. + + For the list of available options, see `:erlang.spawn/4`. Inlined by the compiler. """ @@ -193,25 +214,25 @@ defmodule Node do end @doc """ - Returns the pid of a new process started by the application of + Returns the PID of a new process started by the application of `module.function(args)` on `node`. - If `node` does not exist, a useless pid is returned. Check - http://www.erlang.org/doc/man/erlang.html#spawn_opt-5 for the list of - available options. + If `node` does not exist, a useless PID is returned. + + For the list of available options, see `:erlang.spawn/4`. Inlined by the compiler. """ - @spec spawn(t, module, atom, [any], Process.spawn_opts) :: pid | {pid, reference} + @spec spawn(t, module, atom, [any], Process.spawn_opts()) :: pid | {pid, reference} def spawn(node, module, fun, args, opts) do :erlang.spawn_opt(node, module, fun, args, opts) end @doc """ - Returns the pid of a new linked process started by the application of `fun` on `node`. + Returns the PID of a new linked process started by the application of `fun` on `node`. A link is created between the calling process and the new process, atomically. - If `node` does not exist, a useless pid is returned (and due to the link, an exit + If `node` does not exist, a useless PID is returned (and due to the link, an exit signal with exit reason `:noconnection` will be received). Inlined by the compiler. @@ -222,11 +243,11 @@ defmodule Node do end @doc """ - Returns the pid of a new linked process started by the application of + Returns the PID of a new linked process started by the application of `module.function(args)` on `node`. A link is created between the calling process and the new process, atomically. - If `node` does not exist, a useless pid is returned (and due to the link, an exit + If `node` does not exist, a useless PID is returned (and due to the link, an exit signal with exit reason `:noconnection` will be received). Inlined by the compiler. @@ -236,15 +257,46 @@ defmodule Node do :erlang.spawn_link(node, module, fun, args) end + @doc """ + Spawns the given function on a node, monitors it and returns its PID + and monitoring reference. + + This functionality was added on Erlang/OTP 23. Using this function to + communicate with nodes running on earlier versions will fail. + + Inlined by the compiler. + """ + @doc since: "1.14.0" + @spec spawn_monitor(t, (() -> any)) :: {pid, reference} + def spawn_monitor(node, fun) do + :erlang.spawn_monitor(node, fun) + end + + @doc """ + Spawns the given module and function passing the given args on a node, + monitors it and returns its PID and monitoring reference. + + This functionality was added on Erlang/OTP 23. Using this function + to communicate with nodes running on earlier versions will fail. + + Inlined by the compiler. + """ + @doc since: "1.14.0" + @spec spawn_monitor(t, module, atom, [any]) :: {pid, reference} + def spawn_monitor(node, module, fun, args) do + :erlang.spawn_monitor(node, module, fun, args) + end + @doc """ Sets the magic cookie of `node` to the atom `cookie`. - The default node is `Node.self`, the local node. If `node` is the local node, + The default node is `Node.self/0`, the local node. If `node` is the local node, the function also sets the cookie of all other unknown nodes to `cookie`. This function will raise `FunctionClauseError` if the given `node` is not alive. """ - def set_cookie(node \\ Node.self, cookie) when is_atom(cookie) do + @spec set_cookie(t, atom) :: true + def set_cookie(node \\ Node.self(), cookie) when is_atom(cookie) do :erlang.set_cookie(node, cookie) end @@ -253,6 +305,7 @@ defmodule Node do Returns the cookie if the node is alive, otherwise `:nocookie`. """ + @spec get_cookie() :: atom def get_cookie() do :erlang.get_cookie() end diff --git a/lib/elixir/lib/option_parser.ex b/lib/elixir/lib/option_parser.ex index ebfa6e752dc..89a76f0570c 100644 --- a/lib/elixir/lib/option_parser.ex +++ b/lib/elixir/lib/option_parser.ex @@ -1,79 +1,206 @@ defmodule OptionParser do @moduledoc """ - This module contains functions to parse command line arguments. + Functions for parsing command line arguments. + + When calling a command, it's possible to pass command line options + to modify what the command does. In this documentation, those are + called "switches", in other situations they may be called "flags" + or simply "options". A switch can be given a value, also called an + "argument". + + The main function in this module is `parse/2`, which parses a list + of command line options and arguments into a keyword list: + + iex> OptionParser.parse(["--debug"], strict: [debug: :boolean]) + {[debug: true], [], []} + + `OptionParser` provides some conveniences out of the box, + such as aliases and automatic handling of negation switches. + + The `parse_head/2` function is an alternative to `parse/2` + which stops parsing as soon as it finds a value that is not + a switch nor a value for a previous switch. + + This module also provides low-level functions, such as `next/2`, + for parsing switches manually, as well as `split/1` and `to_argv/1` + for parsing from and converting switches to strings. """ - @type argv :: [String.t] - @type parsed :: Keyword.t - @type errors :: [{String.t, String.t | nil}] - @type options :: [switches: Keyword.t, strict: Keyword.t, aliases: Keyword.t] + @type argv :: [String.t()] + @type parsed :: keyword + @type errors :: [{String.t(), String.t() | nil}] + @type options :: [ + switches: keyword, + strict: keyword, + aliases: keyword, + allow_nonexistent_atoms: boolean + ] + + defmodule ParseError do + defexception [:message] + end @doc """ - Parses `argv` into a keywords list. + Parses `argv` into a keyword list. - It returns the parsed values, remaining arguments and the - invalid options. + It returns a three-element tuple with the form `{parsed, args, invalid}`, where: - ## Examples + * `parsed` is a keyword list of parsed switches with `{switch_name, value}` + tuples in it; `switch_name` is the atom representing the switch name while + `value` is the value for that switch parsed according to `opts` (see the + "Examples" section for more information) + * `args` is a list of the remaining arguments in `argv` as strings + * `invalid` is a list of invalid options as `{option_name, value}` where + `option_name` is the raw option and `value` is `nil` if the option wasn't + expected or the string value if the value didn't have the expected type for + the corresponding option + + Elixir converts switches to underscored atoms, so `--source-path` becomes + `:source_path`. This is done to better suit Elixir conventions. However, this + means that switches can't contain underscores and switches that do contain + underscores are always returned in the list of invalid switches. - iex> OptionParser.parse(["--debug"]) + When parsing, it is common to list switches and their expected types: + + iex> OptionParser.parse(["--debug"], strict: [debug: :boolean]) {[debug: true], [], []} - iex> OptionParser.parse(["--source", "lib"]) + iex> OptionParser.parse(["--source", "lib"], strict: [source: :string]) {[source: "lib"], [], []} - iex> OptionParser.parse(["--source-path", "lib", "test/enum_test.exs", "--verbose"]) + iex> OptionParser.parse( + ...> ["--source-path", "lib", "test/enum_test.exs", "--verbose"], + ...> strict: [source_path: :string, verbose: :boolean] + ...> ) {[source_path: "lib", verbose: true], ["test/enum_test.exs"], []} - By default, Elixir will try to automatically parse switches. - Switches without an argument, like `--debug` will automatically - be set to true. Switches followed by a value will be assigned - to the value, always as strings. + We will explore the valid switches and operation modes of option parser below. + + ## Options + + The following options are supported: - Note Elixir also converts the switches to underscore atoms, as - `--source-path` becomes `:source_path`, to better suit Elixir - conventions. This means that option names on the command line cannot contain - underscores; such options will be reported as `:undefined` (in strict mode) - or `:invalid` (in basic mode). + * `:switches` or `:strict` - see the "Switch definitions" section below + * `:allow_nonexistent_atoms` - see the "Parsing unknown switches" section below + * `:aliases` - see the "Aliases" section below - ## Switches + ## Switch definitions - Many times though, it is better to explicitly list the available - switches and their formats. The switches can be specified via two - different options: + Switches can be specified via one of two options: - * `:strict` - the switches are strict. Any switch that does not - exist in the switch list is treated as an error. + * `:strict` - defines strict switches and their types. Any switch + in `argv` that is not specified in the list is returned in the + invalid options list. This is the preferred way to parse options. - * `:switches` - configure some switches. Switches that does not - exist in the switch list are still attempted to be parsed. + * `:switches` - defines switches and their types. This function + still attempts to parse switches that are not in this list. - Note only `:strict` or `:switches` may be given at once. + Both these options accept a keyword list where the key is an atom + defining the name of the switch and value is the `type` of the + switch (see the "Types" section below for more information). - For each switch, the following types are supported: + Note that you should only supply the `:switches` or the `:strict` option. + If you supply both, an `ArgumentError` exception will be raised. - * `:boolean` - marks the given switch as a boolean. Boolean switches - never consume the following value unless it is `true` or - `false`. - * `:integer` - parses the switch as an integer. - * `:float` - parses the switch as a float. - * `:string` - returns the switch as a string. + ### Types - If a switch can't be parsed or is not specfied in the strict case, - the option is returned in the invalid options list (third element - of the returned tuple). + Switches parsed by `OptionParser` may take zero or one arguments. - The following extra "types" are supported: + The following switches types take no arguments: + + * `:boolean` - sets the value to `true` when given (see also the + "Negation switches" section below) + * `:count` - counts the number of times the switch is given + + The following switches take one argument: + + * `:integer` - parses the value as an integer + * `:float` - parses the value as a float + * `:string` - parses the value as a string + + If a switch can't be parsed according to the given type, it is + returned in the invalid options list. + + ### Modifiers + + Switches can be specified with modifiers, which change how + they behave. The following modifiers are supported: + + * `:keep` - keeps duplicated elements instead of overriding them; + works with all types except `:count`. Specifying `switch_name: :keep` + assumes the type of `:switch_name` will be `:string`. + + To use `:keep` with a type other than `:string`, use a list as the type + for the switch. For example: `[foo: [:integer, :keep]]`. + + ### Negation switches + + In case a switch `SWITCH` is specified to have type `:boolean`, it may be + passed as `--no-SWITCH` as well which will set the option to `false`: + + iex> OptionParser.parse(["--no-op", "path/to/file"], switches: [op: :boolean]) + {[op: false], ["path/to/file"], []} - * `:keep` - keeps duplicated items in the list instead of overriding + ### Parsing unknown switches - Examples: + When the `:switches` option is given, `OptionParser` will attempt to parse + unknown switches: + + iex> OptionParser.parse(["--debug"], switches: [key: :string]) + {[debug: true], [], []} + + Even though we haven't specified `--debug` in the list of switches, it is part + of the returned options. This would also work: + + iex> OptionParser.parse(["--debug", "value"], switches: [key: :string]) + {[debug: "value"], [], []} + + Switches followed by a value will be assigned the value, as a string. Switches + without an argument will be set automatically to `true`. Since we cannot assert + the type of the switch value, it is preferred to use the `:strict` option that + accepts only known switches and always verify their types. + + If you do want to parse unknown switches, remember that Elixir converts switches + to atoms. Since atoms are not garbage-collected, OptionParser will only parse + switches that translate to atoms used by the runtime to avoid leaking atoms. + For instance, the code below will discard the `--option-parser-example` switch + because the `:option_parser_example` atom is never used anywhere: + + OptionParser.parse(["--option-parser-example"], switches: [debug: :boolean]) + # The :option_parser_example atom is not used anywhere below + + However, the code below would work as long as `:option_parser_example` atom is + used at some point later (or earlier) **in the same module**. For example: + + {opts, _, _} = OptionParser.parse(["--option-parser-example"], switches: [debug: :boolean]) + # ... then somewhere in the same module you access it ... + opts[:option_parser_example] + + In other words, Elixir will only parse options that are used by the runtime, + ignoring all others. If you would like to parse all switches, regardless if + they exist or not, you can force creation of atoms by passing + `allow_nonexistent_atoms: true` as option. Use this option with care. It is + only useful when you are building command-line applications that receive + dynamically-named arguments and must be avoided in long-running systems. + + ## Aliases + + A set of aliases can be specified in the `:aliases` option: + + iex> OptionParser.parse(["-d"], aliases: [d: :debug], strict: [debug: :boolean]) + {[debug: true], [], []} + + ## Examples + + Here are some examples of working with different types and modifiers: iex> OptionParser.parse(["--unlock", "path/to/file"], strict: [unlock: :boolean]) {[unlock: true], ["path/to/file"], []} - iex> OptionParser.parse(["--unlock", "--limit", "0", "path/to/file"], - ...> strict: [unlock: :boolean, limit: :integer]) + iex> OptionParser.parse( + ...> ["--unlock", "--limit", "0", "path/to/file"], + ...> strict: [unlock: :boolean, limit: :integer] + ...> ) {[unlock: true, limit: 0], ["path/to/file"], []} iex> OptionParser.parse(["--limit", "3"], strict: [limit: :integer]) @@ -82,39 +209,71 @@ defmodule OptionParser do iex> OptionParser.parse(["--limit", "xyz"], strict: [limit: :integer]) {[], [], [{"--limit", "xyz"}]} + iex> OptionParser.parse(["--verbose"], switches: [verbose: :count]) + {[verbose: 1], [], []} + + iex> OptionParser.parse(["-v", "-v"], aliases: [v: :verbose], strict: [verbose: :count]) + {[verbose: 2], [], []} + iex> OptionParser.parse(["--unknown", "xyz"], strict: []) {[], ["xyz"], [{"--unknown", nil}]} - iex> OptionParser.parse(["--limit", "3", "--unknown", "xyz"], - ...> switches: [limit: :integer]) + iex> OptionParser.parse( + ...> ["--limit", "3", "--unknown", "xyz"], + ...> switches: [limit: :integer] + ...> ) {[limit: 3, unknown: "xyz"], [], []} - ## Negation switches + iex> OptionParser.parse( + ...> ["--unlock", "path/to/file", "--unlock", "path/to/another/file"], + ...> strict: [unlock: :keep] + ...> ) + {[unlock: "path/to/file", unlock: "path/to/another/file"], [], []} + + """ + @spec parse(argv, options) :: {parsed, argv, errors} + def parse(argv, opts \\ []) when is_list(argv) and is_list(opts) do + do_parse(argv, build_config(opts), [], [], [], true) + end + + @doc """ + The same as `parse/2` but raises an `OptionParser.ParseError` + exception if any invalid options are given. - All switches starting with `--no-` are considered to be booleans and never - parse the next value: + If there are no errors, returns a `{parsed, rest}` tuple where: - iex> OptionParser.parse(["--no-op", "path/to/file"]) - {[no_op: true], ["path/to/file"], []} + * `parsed` is the list of parsed switches (same as in `parse/2`) + * `rest` is the list of arguments (same as in `parse/2`) - However, in case the base switch exists, it sets that particular switch to - false: + ## Examples - iex> OptionParser.parse(["--no-op", "path/to/file"], switches: [op: :boolean]) - {[op: false], ["path/to/file"], []} + iex> OptionParser.parse!(["--debug", "path/to/file"], strict: [debug: :boolean]) + {[debug: true], ["path/to/file"]} - ## Aliases + iex> OptionParser.parse!(["--limit", "xyz"], strict: [limit: :integer]) + ** (OptionParser.ParseError) 1 error found! + --limit : Expected type integer, got "xyz" - A set of aliases can be given as options too: + iex> OptionParser.parse!(["--unknown", "xyz"], strict: []) + ** (OptionParser.ParseError) 1 error found! + --unknown : Unknown option - iex> OptionParser.parse(["-d"], aliases: [d: :debug]) - {[debug: true], [], []} + iex> OptionParser.parse!( + ...> ["-l", "xyz", "-f", "bar"], + ...> switches: [limit: :integer, foo: :integer], + ...> aliases: [l: :limit, f: :foo] + ...> ) + ** (OptionParser.ParseError) 2 errors found! + -l : Expected type integer, got "xyz" + -f : Expected type integer, got "bar" """ - @spec parse(argv, options) :: {parsed, argv, errors} - def parse(argv, opts \\ []) when is_list(argv) and is_list(opts) do - config = compile_config(opts, true) - do_parse(argv, config, [], [], []) + @spec parse!(argv, options) :: {parsed, argv} + def parse!(argv, opts \\ []) when is_list(argv) and is_list(opts) do + case parse(argv, opts) do + {parsed, args, []} -> {parsed, args} + {_, _, errors} -> raise ParseError, format_errors(errors, opts) + end end @doc """ @@ -125,46 +284,92 @@ defmodule OptionParser do ## Example - iex> OptionParser.parse_head(["--source", "lib", "test/enum_test.exs", "--verbose"]) + iex> OptionParser.parse_head( + ...> ["--source", "lib", "test/enum_test.exs", "--verbose"], + ...> switches: [source: :string, verbose: :boolean] + ...> ) {[source: "lib"], ["test/enum_test.exs", "--verbose"], []} - iex> OptionParser.parse_head(["--verbose", "--source", "lib", "test/enum_test.exs", "--unlock"]) + iex> OptionParser.parse_head( + ...> ["--verbose", "--source", "lib", "test/enum_test.exs", "--unlock"], + ...> switches: [source: :string, verbose: :boolean, unlock: :boolean] + ...> ) {[verbose: true, source: "lib"], ["test/enum_test.exs", "--unlock"], []} """ @spec parse_head(argv, options) :: {parsed, argv, errors} def parse_head(argv, opts \\ []) when is_list(argv) and is_list(opts) do - config = compile_config(opts, false) - do_parse(argv, config, [], [], []) + do_parse(argv, build_config(opts), [], [], [], false) + end + + @doc """ + The same as `parse_head/2` but raises an `OptionParser.ParseError` + exception if any invalid options are given. + + If there are no errors, returns a `{parsed, rest}` tuple where: + + * `parsed` is the list of parsed switches (same as in `parse_head/2`) + * `rest` is the list of arguments (same as in `parse_head/2`) + + ## Examples + + iex> OptionParser.parse_head!( + ...> ["--source", "lib", "path/to/file", "--verbose"], + ...> switches: [source: :string, verbose: :boolean] + ...> ) + {[source: "lib"], ["path/to/file", "--verbose"]} + + iex> OptionParser.parse_head!( + ...> ["--number", "lib", "test/enum_test.exs", "--verbose"], + ...> strict: [number: :integer] + ...> ) + ** (OptionParser.ParseError) 1 error found! + --number : Expected type integer, got "lib" + + iex> OptionParser.parse_head!( + ...> ["--verbose", "--source", "lib", "test/enum_test.exs", "--unlock"], + ...> strict: [verbose: :integer, source: :integer] + ...> ) + ** (OptionParser.ParseError) 2 errors found! + --verbose : Missing argument of type integer + --source : Expected type integer, got "lib" + + """ + @spec parse_head!(argv, options) :: {parsed, argv} + def parse_head!(argv, opts \\ []) when is_list(argv) and is_list(opts) do + case parse_head(argv, opts) do + {parsed, args, []} -> {parsed, args} + {_, _, errors} -> raise ParseError, format_errors(errors, opts) + end end - defp do_parse([], _config, opts, args, invalid) do + defp do_parse([], _config, opts, args, invalid, _all?) do {Enum.reverse(opts), Enum.reverse(args), Enum.reverse(invalid)} end - defp do_parse(argv, {aliases, switches, strict, all}=config, opts, args, invalid) do - case next(argv, aliases, switches, strict) do + defp do_parse(argv, %{switches: switches} = config, opts, args, invalid, all?) do + case next_with_config(argv, config) do {:ok, option, value, rest} -> - # the option exist and it was successfully parsed - kinds = List.wrap Keyword.get(switches, option) - new_opts = do_store_option(opts, option, value, kinds) - do_parse(rest, config, new_opts, args, invalid) + # the option exists and it was successfully parsed + kinds = List.wrap(Keyword.get(switches, option)) + new_opts = store_option(opts, option, value, kinds) + do_parse(rest, config, new_opts, args, invalid, all?) {:invalid, option, value, rest} -> # the option exist but it has wrong value - do_parse(rest, config, opts, args, [{option, value}|invalid]) + do_parse(rest, config, opts, args, [{option, value} | invalid], all?) {:undefined, option, _value, rest} -> - # the option does not exist (for strict cases) - do_parse(rest, config, opts, args, [{option, nil}|invalid]) + invalid = if config.strict?, do: [{option, nil} | invalid], else: invalid + do_parse(rest, config, opts, args, invalid, all?) - {:error, ["--"|rest]} -> + {:error, ["--" | rest]} -> {Enum.reverse(opts), Enum.reverse(args, rest), Enum.reverse(invalid)} - {:error, [arg|rest]=remaining_args} -> + {:error, [arg | rest] = remaining_args} -> # there is no option - if all do - do_parse(rest, config, opts, [arg|args], invalid) + if all? do + do_parse(rest, config, opts, [arg | args], invalid, all?) else {Enum.reverse(opts), Enum.reverse(args, remaining_args), Enum.reverse(invalid)} end @@ -175,249 +380,497 @@ defmodule OptionParser do Low-level function that parses one option. It accepts the same options as `parse/2` and `parse_head/2` - as both functions are built on top of next. This function + as both functions are built on top of this function. This function may return: * `{:ok, key, value, rest}` - the option `key` with `value` was successfully parsed * `{:invalid, key, value, rest}` - the option `key` is invalid with `value` - (returned when the switch type does not match the one given via the - command line) + (returned when the value cannot be parsed according to the switch type) * `{:undefined, key, value, rest}` - the option `key` is undefined - (returned on strict cases and the switch is unknown) + (returned in strict mode when the switch is unknown or on nonexistent atoms) - * `{:error, rest}` - there are no switches at the top of the given argv - """ + * `{:error, rest}` - there are no switches at the head of the given `argv` + """ @spec next(argv, options) :: - {:ok, key :: atom, value :: term, argv} | - {:invalid, key :: atom, value :: term, argv} | - {:undefined, key :: atom, value :: term, argv} | - {:error, argv} + {:ok, key :: atom, value :: term, argv} + | {:invalid, String.t(), String.t() | nil, argv} + | {:undefined, String.t(), String.t() | nil, argv} + | {:error, argv} def next(argv, opts \\ []) when is_list(argv) and is_list(opts) do - {aliases, switches, strict, _} = compile_config(opts, true) - next(argv, aliases, switches, strict) + next_with_config(argv, build_config(opts)) end - defp next([], _aliases, _switches, _strict) do + defp next_with_config([], _config) do {:error, []} end - defp next(["--"|_]=argv, _aliases, _switches, _strict) do + defp next_with_config(["--" | _] = argv, _config) do {:error, argv} end - defp next(["-"|_]=argv, _aliases, _switches, _strict) do + defp next_with_config(["-" | _] = argv, _config) do {:error, argv} end - defp next(["- " <> _|_]=argv, _aliases, _switches, _strict) do + defp next_with_config(["- " <> _ | _] = argv, _config) do {:error, argv} end - defp next(["-" <> option|rest], aliases, switches, strict) do + # Handles --foo or --foo=bar + defp next_with_config(["--" <> option | rest], config) do {option, value} = split_option(option) - opt_name_bin = "-" <> option - tagged = tag_option(option, value, switches, aliases) - if strict and not option_defined?(tagged, switches) do - {:undefined, opt_name_bin, value, rest} + if String.contains?(option, ["_"]) do + {:undefined, "--" <> option, value, rest} else - {opt_name, kinds, value} = normalize_option(tagged, value, switches) - {value, kinds, rest} = normalize_value(value, kinds, rest, strict) + tagged = tag_option(option, config) + next_tagged(tagged, value, "--" <> option, rest, config) + end + end + + # Handles -a, -abc, -abc=something, -n2 + defp next_with_config(["-" <> option | rest] = argv, config) do + {option, value} = split_option(option) + original = "-" <> option + + cond do + is_nil(value) and starts_with_number?(option) -> + {:error, argv} + + String.contains?(option, ["-", "_"]) -> + {:undefined, original, value, rest} + + String.length(option) == 1 -> + # We have a regular one-letter alias here + tagged = tag_oneletter_alias(option, config) + next_tagged(tagged, value, original, rest, config) + + true -> + key = get_option_key(option, config.allow_nonexistent_atoms?) + option_key = config.aliases[key] + + if key && option_key do + IO.warn("multi-letter aliases are deprecated, got: #{inspect(key)}") + next_tagged({:default, option_key}, value, original, rest, config) + else + next_with_config(expand_multiletter_alias(option, value) ++ rest, config) + end + end + end + + defp next_with_config(argv, _config) do + {:error, argv} + end + + defp next_tagged(:unknown, value, original, rest, _) do + {value, _kinds, rest} = normalize_value(value, [], rest) + {:undefined, original, value, rest} + end + + defp next_tagged({tag, option}, value, original, rest, %{switches: switches, strict?: strict?}) do + if strict? and not Keyword.has_key?(switches, option) do + {:undefined, original, value, rest} + else + {kinds, value} = normalize_tag(tag, option, value, switches) + {value, kinds, rest} = normalize_value(value, kinds, rest) + case validate_option(value, kinds) do - {:ok, new_value} -> {:ok, opt_name, new_value, rest} - :invalid -> {:invalid, opt_name_bin, value, rest} + {:ok, new_value} -> {:ok, option, new_value, rest} + :invalid -> {:invalid, original, value, rest} end end end - defp next(argv, _aliases, _switches, _strict) do - {:error, argv} + @doc """ + Receives a key-value enumerable and converts it to `t:argv/0`. + + Keys must be atoms. Keys with `nil` value are discarded, + boolean values are converted to `--key` or `--no-key` + (if the value is `true` or `false`, respectively), + and all other values are converted using `to_string/1`. + + It is advised to pass to `to_argv/2` the same set of `options` + given to `parse/2`. Some switches can only be reconstructed + correctly with the `:switches` information in hand. + + ## Examples + + iex> OptionParser.to_argv(foo_bar: "baz") + ["--foo-bar", "baz"] + iex> OptionParser.to_argv(bool: true, bool: false, discarded: nil) + ["--bool", "--no-bool"] + + Some switches will output different values based on the switches + types: + + iex> OptionParser.to_argv([number: 2], switches: []) + ["--number", "2"] + iex> OptionParser.to_argv([number: 2], switches: [number: :count]) + ["--number", "--number"] + + """ + @spec to_argv(Enumerable.t(), options) :: argv + def to_argv(enum, options \\ []) do + switches = Keyword.get(options, :switches, []) + + Enum.flat_map(enum, fn + {_key, nil} -> [] + {key, true} -> [to_switch(key)] + {key, false} -> [to_switch(key, "--no-")] + {key, value} -> to_argv(key, value, switches) + end) + end + + defp to_argv(key, value, switches) do + if switches[key] == :count do + List.duplicate(to_switch(key), value) + else + [to_switch(key), to_string(value)] + end + end + + defp to_switch(key, prefix \\ "--") when is_atom(key) do + prefix <> String.replace(Atom.to_string(key), "_", "-") + end + + @doc ~S""" + Splits a string into `t:argv/0` chunks. + + This function splits the given `string` into a list of strings in a similar + way to many shells. + + ## Examples + + iex> OptionParser.split("foo bar") + ["foo", "bar"] + + iex> OptionParser.split("foo \"bar baz\"") + ["foo", "bar baz"] + + """ + @spec split(String.t()) :: argv + def split(string) when is_binary(string) do + do_split(String.trim_leading(string, " "), "", [], nil) + end + + # If we have an escaped quote, simply remove the escape + defp do_split(<>, buffer, acc, quote), + do: do_split(t, <>, acc, quote) + + # If we have a quote and we were not in a quote, start one + defp do_split(<>, buffer, acc, nil) when quote in [?", ?'], + do: do_split(t, buffer, acc, quote) + + # If we have a quote and we were inside it, close it + defp do_split(<>, buffer, acc, quote), do: do_split(t, buffer, acc, nil) + + # If we have an escaped quote/space, simply remove the escape as long as we are not inside a quote + defp do_split(<>, buffer, acc, nil) when h in [?\s, ?', ?"], + do: do_split(t, <>, acc, nil) + + # If we have space and we are outside of a quote, start new segment + defp do_split(<>, buffer, acc, nil), + do: do_split(String.trim_leading(t, " "), "", [buffer | acc], nil) + + # All other characters are moved to buffer + defp do_split(<>, buffer, acc, quote) do + do_split(t, <>, acc, quote) + end + + # Finish the string expecting a nil marker + defp do_split(<<>>, "", acc, nil), do: Enum.reverse(acc) + + defp do_split(<<>>, buffer, acc, nil), do: Enum.reverse([buffer | acc]) + + # Otherwise raise + defp do_split(<<>>, _, _acc, marker) do + raise "argv string did not terminate properly, a #{<>} was opened but never closed" end ## Helpers - defp compile_config(opts, all) do - aliases = opts[:aliases] || [] + defp build_config(opts) do + {switches, strict?} = + cond do + opts[:switches] && opts[:strict] -> + raise ArgumentError, ":switches and :strict cannot be given together" - {switches, strict} = cond do - s = opts[:switches] -> - {s, false} - s = opts[:strict] -> - {s, true} - true -> - {[], false} - end + switches = opts[:switches] -> + validate_switches(switches) + {switches, false} + + strict = opts[:strict] -> + validate_switches(strict) + {strict, true} + + true -> + IO.warn("not passing the :switches or :strict option to OptionParser is deprecated") + {[], false} + end - {aliases, switches, strict, all} + %{ + aliases: opts[:aliases] || [], + allow_nonexistent_atoms?: opts[:allow_nonexistent_atoms] || false, + strict?: strict?, + switches: switches + } end - defp validate_option(value, kinds) do - {is_invalid, value} = cond do - :invalid in kinds -> - {true, value} - :boolean in kinds -> - case value do - t when t in [true, "true"] -> {nil, true} - f when f in [false, "false"] -> {nil, false} - _ -> {true, value} - end - :integer in kinds -> - case Integer.parse(value) do - {value, ""} -> {nil, value} - _ -> {true, value} - end - :float in kinds -> - case Float.parse(value) do - {value, ""} -> {nil, value} - _ -> {true, value} - end - true -> - {nil, value} + defp validate_switches(switches) do + Enum.map(switches, &validate_switch/1) + end + + defp validate_switch({_name, type_or_type_and_modifiers}) do + valid = [:boolean, :count, :integer, :float, :string, :keep] + invalid = List.wrap(type_or_type_and_modifiers) -- valid + + if invalid != [] do + raise ArgumentError, + "invalid switch types/modifiers: " <> Enum.map_join(invalid, ", ", &inspect/1) end + end - if is_invalid do + defp validate_option(value, kinds) do + {invalid?, value} = + cond do + :invalid in kinds -> + {true, value} + + :boolean in kinds -> + case value do + t when t in [true, "true"] -> {false, true} + f when f in [false, "false"] -> {false, false} + _ -> {true, value} + end + + :count in kinds -> + case value do + nil -> {false, 1} + _ -> {true, value} + end + + :integer in kinds -> + case Integer.parse(value) do + {value, ""} -> {false, value} + _ -> {true, value} + end + + :float in kinds -> + case Float.parse(value) do + {value, ""} -> {false, value} + _ -> {true, value} + end + + true -> + {false, value} + end + + if invalid? do :invalid else {:ok, value} end end - defp do_store_option(dict, option, value, kinds) do + defp store_option(dict, option, value, kinds) do cond do + :count in kinds -> + Keyword.update(dict, option, value, &(&1 + 1)) + :keep in kinds -> - [{option, value}|dict] + [{option, value} | dict] + true -> - [{option, value}|Keyword.delete(dict, option)] + [{option, value} | Keyword.delete(dict, option)] end end - defp tag_option(<>, value, switches, _aliases) do - get_negated(option, value, switches) + defp tag_option("no-" <> option = original, config) do + %{switches: switches, allow_nonexistent_atoms?: allow_nonexistent_atoms?} = config + + cond do + (negated = get_option_key(option, allow_nonexistent_atoms?)) && + :boolean in List.wrap(switches[negated]) -> + {:negated, negated} + + option_key = get_option_key(original, allow_nonexistent_atoms?) -> + {:default, option_key} + + true -> + :unknown + end end - defp tag_option(option, _value, _switches, aliases) when is_binary(option) do - opt = get_option(option) - if alias = aliases[opt] do - {:default, alias} + defp tag_option(option, config) do + %{allow_nonexistent_atoms?: allow_nonexistent_atoms?} = config + + if option_key = get_option_key(option, allow_nonexistent_atoms?) do + {:default, option_key} else :unknown end end - defp option_defined?(:unknown, _switches) do - false - end - - defp option_defined?({:negated, option}, switches) do - Keyword.has_key?(switches, option) - end + defp tag_oneletter_alias(alias, config) when is_binary(alias) do + %{aliases: aliases, allow_nonexistent_atoms?: allow_nonexistent_atoms?} = config - defp option_defined?({:default, option}, switches) do - Keyword.has_key?(switches, option) + if option_key = aliases[to_existing_key(alias, allow_nonexistent_atoms?)] do + {:default, option_key} + else + :unknown + end end - defp normalize_option(:unknown, value, _switches) do - {nil, [:invalid], value} + defp expand_multiletter_alias(options, value) do + {options, maybe_integer} = + options + |> String.to_charlist() + |> Enum.split_while(&(&1 not in ?0..?9)) + + {last, expanded} = + options + |> List.to_string() + |> String.graphemes() + |> Enum.map(&("-" <> &1)) + |> List.pop_at(-1) + + expanded ++ + [ + last <> + if(maybe_integer != [], do: "=#{maybe_integer}", else: "") <> + if(value, do: "=#{value}", else: "") + ] end - defp normalize_option({:negated, option}, nil, switches) do - kinds = List.wrap(switches[option]) - - cond do - :boolean in kinds -> - {option, kinds, false} - kinds == [] -> - {option, kinds, true} - true -> - {reverse_negated(option), [:invalid], nil} + defp normalize_tag(:negated, option, value, switches) do + if value do + {[:invalid], value} + else + {List.wrap(switches[option]), false} end end - defp normalize_option({:negated, option}, value, _switches) do - {option, [:invalid], value} + defp normalize_tag(:default, option, value, switches) do + {List.wrap(switches[option]), value} end - defp normalize_option({:default, option}, value, switches) do - {option, List.wrap(switches[option]), value} - end - - defp normalize_value(nil, kinds, t, strict) do - nil_or_true = if strict, do: nil, else: true + defp normalize_value(nil, kinds, t) do cond do :boolean in kinds -> {true, kinds, t} + + :count in kinds -> + {nil, kinds, t} + value_in_tail?(t) -> - [h|t] = t + [h | t] = t {h, kinds, t} + kinds == [] -> - {nil_or_true, kinds, t} + {true, kinds, t} + true -> {nil, [:invalid], t} end end - defp normalize_value(value, kinds, t, _) do + defp normalize_value(value, kinds, t) do {value, kinds, t} end - defp value_in_tail?(["-"|_]), do: true - defp value_in_tail?(["- " <> _|_]), do: true - defp value_in_tail?(["-" <> _|_]), do: false - defp value_in_tail?([]), do: false - defp value_in_tail?(_), do: true + defp value_in_tail?(["-" | _]), do: true + defp value_in_tail?(["- " <> _ | _]), do: true + defp value_in_tail?(["-" <> arg | _]), do: starts_with_number?(arg) + defp value_in_tail?([]), do: false + defp value_in_tail?(_), do: true defp split_option(option) do case :binary.split(option, "=") do - [h] -> {h, nil} + [h] -> {h, nil} [h, t] -> {h, t} end end defp to_underscore(option), do: to_underscore(option, <<>>) + defp to_underscore("-" <> rest, acc), do: to_underscore(rest, acc <> "_") + defp to_underscore(<> <> rest, acc), do: to_underscore(rest, <>) + defp to_underscore(<<>>, acc), do: acc - defp to_underscore("_" <> _rest, _acc), do: nil - - defp to_underscore("-" <> rest, acc), - do: to_underscore(rest, acc <> "_") - - defp to_underscore(<> <> rest, acc), - do: to_underscore(rest, <>) + defp get_option_key(option, allow_nonexistent_atoms?) do + option + |> to_underscore() + |> to_existing_key(allow_nonexistent_atoms?) + end - defp to_underscore(<<>>, acc), do: acc + defp to_existing_key(option, true), do: String.to_atom(option) - defp get_option(option) do - if str = to_underscore(option) do - String.to_atom(str) + defp to_existing_key(option, false) do + try do + String.to_existing_atom(option) + rescue + ArgumentError -> nil end end - defp reverse_negated(negated) do - String.to_atom("no_" <> Atom.to_string(negated)) + defp starts_with_number?(<>) when char in ?0..?9, do: true + defp starts_with_number?(_), do: false + + defp format_errors([_ | _] = errors, opts) do + types = opts[:switches] || opts[:strict] + error_count = length(errors) + error = if error_count == 1, do: "error", else: "errors" + + "#{error_count} #{error} found!\n" <> + Enum.map_join(errors, "\n", &format_error(&1, opts, types)) end - defp get_negated("no-" <> rest = option, value, switches) do - if negated = get_option(rest) do - option = if Keyword.has_key?(switches, negated) and value == nil do - negated - else - get_option(option) - end - {:negated, option} + defp format_error({option, nil}, opts, types) do + if type = get_type(option, opts, types) do + "#{option} : Missing argument of type #{type}" else - :unknown + msg = "#{option} : Unknown option" + + case did_you_mean(option, types) do + {similar, score} when score > 0.8 -> + msg <> ". Did you mean --#{similar}?" + + _ -> + msg + end end end - defp get_negated(rest, _value, _switches) do - if option = get_option(rest) do - {:default, option} + defp format_error({option, value}, opts, types) do + type = get_type(option, opts, types) + "#{option} : Expected type #{type}, got #{inspect(value)}" + end + + defp get_type(option, opts, types) do + allow_nonexistent_atoms? = opts[:allow_nonexistent_atoms] || false + key = option |> String.trim_leading("-") |> get_option_key(allow_nonexistent_atoms?) + + if option_key = opts[:aliases][key] do + types[option_key] else - :unknown + types[key] end end + + defp did_you_mean(option, types) do + key = option |> String.trim_leading("-") |> String.replace("-", "_") + Enum.reduce(types, {nil, 0}, &max_similar(&1, key, &2)) + end + + defp max_similar({source, _}, target, {_, current} = best) do + source = Atom.to_string(source) + + score = String.jaro_distance(source, target) + option = String.replace(source, "_", "-") + if score < current, do: best, else: {option, score} + end end diff --git a/lib/elixir/lib/partition_supervisor.ex b/lib/elixir/lib/partition_supervisor.ex new file mode 100644 index 00000000000..32a9f40a754 --- /dev/null +++ b/lib/elixir/lib/partition_supervisor.ex @@ -0,0 +1,386 @@ +defmodule PartitionSupervisor do + @moduledoc """ + A supervisor that starts multiple partitions of the same child. + + Certain processes may become bottlenecks in large systems. + If those processes can have their state trivially partitioned, + in a way there is no dependency between them, then they can use + the `PartitionSupervisor` to create multiple isolated and + independent partitions. + + Once the `PartitionSupervisor` starts, you can dispatch to its + children using `{:via, PartitionSupervisor, {name, key}}`, where + `name` is the name of the `PartitionSupervisor` and key is used + for routing. + + ## Example + + The `DynamicSupervisor` is a single process responsible for starting + other processes. In some applications, the `DynamicSupervisor` may + become a bottleneck. To address this, you can start multiple instances + of the `DynamicSupervisor` through a `PartitionSupervisor`, and then + pick a "random" instance to start the child on. + + Instead of starting a single `DynamicSupervisor`: + + children = [ + {DynamicSupervisor, name: MyApp.DynamicSupervisor} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + + and starting children on that dynamic supervisor directly: + + DynamicSupervisor.start_child(MyApp.DynamicSupervisor, {Agent, fn -> %{} end}) + + You can do start the dynamic supervisors under a `PartitionSupervisor`: + + children = [ + {PartitionSupervisor, + child_spec: DynamicSupervisor, + name: MyApp.DynamicSupervisors} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + + and then: + + DynamicSupervisor.start_child( + {:via, PartitionSupervisor, {MyApp.DynamicSupervisors, self()}}, + {Agent, fn -> %{} end} + ) + + In the code above, we start a partition supervisor that will by default + start a dynamic supervisor for each core in your machine. Then, instead + of calling the `DynamicSupervisor` by name, you call it through the + partition supervisor using the `{:via, PartitionSupervisor, {name, key}}` + format. We picked `self()` as the routing key, which means each process + will be assigned one of the existing dynamic supervisors. See `start_link/1` + to see all options supported by the `PartitionSupervisor`. + + ## Implementation notes + + The `PartitionSupervisor` uses either an ETS table or a `Registry` to + manage all of the partitions. Under the hood, the `PartitionSupervisor` + generates a child spec for each partition and then acts as a regular + supervisor. The ID of each child spec is the partition number. + + For routing, two strategies are used. If `key` is an integer, it is routed + using `rem(abs(key), partitions)` where `partitions` is the number of + partitions. Otherwise it uses `:erlang.phash2(key, partitions)`. + The particular routing may change in the future, and therefore must not + be relied on. If you want to retrieve a particular PID for a certain key, + you can use `GenServer.whereis({:via, PartitionSupervisor, {name, key}})`. + """ + + @behaviour Supervisor + + @registry PartitionSupervisor.Registry + + @typedoc """ + The name of the `PartitionSupervisor`. + """ + @type name :: atom() | {:via, module(), term()} + + @doc false + def child_spec(opts) when is_list(opts) do + id = + case Keyword.get(opts, :name, DynamicSupervisor) do + name when is_atom(name) -> name + {:via, _module, name} -> name + end + + %{ + id: id, + start: {PartitionSupervisor, :start_link, [opts]}, + type: :supervisor + } + end + + @doc """ + Starts a partition supervisor with the given options. + + This function is typically not invoked directly, instead it is invoked + when using a `PartitionSupervisor` as a child of another supervisor: + + children = [ + {PartitionSupervisor, child_spec: SomeChild, name: MyPartitionSupervisor} + ] + + If the supervisor is successfully spawned, this function returns + `{:ok, pid}`, where `pid` is the PID of the supervisor. If the given name + for the partition supervisor is already assigned to a process, + the function returns `{:error, {:already_started, pid}}`, where `pid` + is the PID of that process. + + Note that a supervisor started with this function is linked to the parent + process and exits not only on crashes but also if the parent process exits + with `:normal` reason. + + ## Options + + * `:name` - an atom or via tuple representing the name of the partition + supervisor (see `t:name/0`). + + * `:partitions` - a positive integer with the number of partitions. + Defaults to `System.schedulers_online()` (typically the number of cores). + + * `:strategy` - the restart strategy option, defaults to `:one_for_one`. + You can learn more about strategies in the `Supervisor` module docs. + + * `:max_restarts` - the maximum number of restarts allowed in + a time frame. Defaults to `3`. + + * `:max_seconds` - the time frame in which `:max_restarts` applies. + Defaults to `5`. + + * `:with_arguments` - a two-argument anonymous function that allows + the partition to be given to the child starting function. See the + `:with_arguments` section below. + + ## `:with_arguments` + + Sometimes you want each partition to know their partition assigned number. + This can be done with the `:with_arguments` option. This function receives + the list of arguments of the child specification and the partition. It + must return a new list of arguments that will be passed to the child specification + of children. + + For example, most processes are started by calling `start_link(opts)`, + where `opts` is a keyword list. You could inject the partition into the + options given to the child: + + with_arguments: fn [opts], partition -> + [Keyword.put(opts, :partition, partition)] + end + + """ + @doc since: "1.14.0" + @spec start_link(keyword) :: Supervisor.on_start() + def start_link(opts) when is_list(opts) do + name = opts[:name] + + unless name do + raise ArgumentError, "the :name option must be given to PartitionSupervisor" + end + + {child_spec, opts} = Keyword.pop(opts, :child_spec) + + unless child_spec do + raise ArgumentError, "the :child_spec option must be given to PartitionSupervisor" + end + + {partitions, opts} = Keyword.pop(opts, :partitions, System.schedulers_online()) + + unless is_integer(partitions) and partitions >= 1 do + raise ArgumentError, + "the :partitions option must be a positive integer, got: #{inspect(partitions)}" + end + + {with_arguments, opts} = Keyword.pop(opts, :with_arguments, fn args, _partition -> args end) + + unless is_function(with_arguments, 2) do + raise ArgumentError, + "the :with_arguments option must be a function that receives two arguments, " <> + "the current call arguments and the partition, got: #{inspect(with_arguments)}" + end + + %{start: {mod, fun, args}} = map = Supervisor.child_spec(child_spec, []) + modules = map[:modules] || [mod] + + children = + for partition <- 0..(partitions - 1) do + args = with_arguments.(args, partition) + + unless is_list(args) do + raise "the call to the function in :with_arguments must return a list, got: #{inspect(args)}" + end + + start = {__MODULE__, :start_child, [mod, fun, args, name, partition]} + Map.merge(map, %{id: partition, start: start, modules: modules}) + end + + {init_opts, start_opts} = Keyword.split(opts, [:strategy, :max_seconds, :max_restarts]) + Supervisor.start_link(__MODULE__, {name, partitions, children, init_opts}, start_opts) + end + + @doc false + def start_child(mod, fun, args, name, partition) do + case apply(mod, fun, args) do + {:ok, pid} -> + register_child(name, partition, pid) + {:ok, pid} + + {:ok, pid, info} -> + register_child(name, partition, pid) + {:ok, pid, info} + + other -> + other + end + end + + defp register_child(name, partition, pid) when is_atom(name) do + :ets.insert(name, {partition, pid}) + end + + defp register_child({:via, _, _}, partition, pid) do + Registry.register(@registry, {self(), partition}, pid) + end + + @impl true + def init({name, partitions, children, init_opts}) do + init_partitions(name, partitions) + Supervisor.init(children, Keyword.put_new(init_opts, :strategy, :one_for_one)) + end + + defp init_partitions(name, partitions) when is_atom(name) do + :ets.new(name, [:set, :named_table, :protected, read_concurrency: true]) + :ets.insert(name, {:partitions, partitions}) + end + + defp init_partitions({:via, _, _}, partitions) do + child_spec = {Registry, keys: :unique, name: @registry} + + unless Process.whereis(@registry) do + Supervisor.start_child(:elixir_sup, child_spec) + end + + Registry.register(@registry, self(), partitions) + end + + @doc """ + Returns the number of partitions for the partition supervisor. + """ + @doc since: "1.14.0" + @spec partitions(name()) :: pos_integer() + def partitions(name) do + {_name, partitions} = name_partitions(name) + partitions + end + + # For whereis_name, we want to lookup on GenServer.whereis/1 + # just once, so we lookup the name and partitions together. + defp name_partitions(name) when is_atom(name) do + try do + {name, :ets.lookup_element(name, :partitions, 2)} + rescue + _ -> exit({:noproc, {__MODULE__, :partitions, [name]}}) + end + end + + defp name_partitions(name) when is_tuple(name) do + with pid when is_pid(pid) <- GenServer.whereis(name), + [name_partitions] <- Registry.lookup(@registry, pid) do + name_partitions + else + _ -> exit({:noproc, {__MODULE__, :partitions, [name]}}) + end + end + + @doc """ + Returns a list with information about all children. + + This function returns a list of tuples containing: + + * `id` - the partition number + + * `child` - the PID of the corresponding child process or the + atom `:restarting` if the process is about to be restarted + + * `type` - `:worker` or `:supervisor` as defined in the child + specification + + * `modules` - as defined in the child specification + + """ + @doc since: "1.14.0" + @spec which_children(name()) :: [ + # Inlining [module()] | :dynamic here because :supervisor.modules() is not exported + {:undefined, pid | :restarting, :worker | :supervisor, [module()] | :dynamic} + ] + def which_children(name) when is_atom(name) or elem(name, 0) == :via do + Supervisor.which_children(name) + end + + @doc """ + Returns a map containing count values for the supervisor. + + The map contains the following keys: + + * `:specs` - the number of partitions (children processes) + + * `:active` - the count of all actively running child processes managed by + this supervisor + + * `:supervisors` - the count of all supervisors whether or not the child + process is still alive + + * `:workers` - the count of all workers, whether or not the child process + is still alive + + """ + @doc since: "1.14.0" + @spec count_children(name()) :: %{ + specs: non_neg_integer, + active: non_neg_integer, + supervisors: non_neg_integer, + workers: non_neg_integer + } + def count_children(supervisor) when is_atom(supervisor) do + Supervisor.count_children(supervisor) + end + + @doc """ + Synchronously stops the given partition supervisor with the given `reason`. + + It returns `:ok` if the supervisor terminates with the given + reason. If it terminates with another reason, the call exits. + + This function keeps OTP semantics regarding error reporting. + If the reason is any other than `:normal`, `:shutdown` or + `{:shutdown, _}`, an error report is logged. + """ + @doc since: "1.14.0" + @spec stop(name(), reason :: term, timeout) :: :ok + def stop(supervisor, reason \\ :normal, timeout \\ :infinity) when is_atom(supervisor) do + Supervisor.stop(supervisor, reason, timeout) + end + + ## Via callbacks + + @doc false + def whereis_name({name, key}) when is_atom(name) or is_tuple(name) do + {name, partitions} = name_partitions(name) + + partition = + if is_integer(key), do: rem(abs(key), partitions), else: :erlang.phash2(key, partitions) + + whereis_name(name, partition) + end + + defp whereis_name(name, partition) when is_atom(name) do + :ets.lookup_element(name, partition, 2) + end + + defp whereis_name(name, partition) when is_pid(name) do + @registry + |> Registry.values({name, partition}, name) + |> List.first(:undefined) + end + + @doc false + def send(name_key, msg) do + Kernel.send(whereis_name(name_key), msg) + end + + @doc false + def register_name(_, _) do + raise "{:via, PartitionSupervisor, _} cannot be given on registration" + end + + @doc false + def unregister_name(_, _) do + raise "{:via, PartitionSupervisor, _} cannot be given on unregistration" + end +end diff --git a/lib/elixir/lib/path.ex b/lib/elixir/lib/path.ex index 1dd4068a18b..2fd703dd2f4 100644 --- a/lib/elixir/lib/path.ex +++ b/lib/elixir/lib/path.ex @@ -3,23 +3,29 @@ defmodule Path do This module provides conveniences for manipulating or retrieving file system paths. - The functions in this module may receive a char data as - argument (i.e. a string or a list of characters / string) - and will always return a string (encoded in UTF-8). + The functions in this module may receive chardata as + arguments and will always return a string encoded in UTF-8. Chardata + is a string or a list of characters and strings, see `t:IO.chardata/0`. + If a binary is given, in whatever encoding, its encoding will be kept. The majority of the functions in this module do not interact with the file system, except for a few functions - that require it (like `wildcard/1` and `expand/1`). + that require it (like `wildcard/2` and `expand/1`). """ - alias :filename, as: FN - @type t :: :unicode.chardata() + @typedoc """ + A path. + """ + @type t :: IO.chardata() @doc """ - Converts the given path to an absolute one. Unlike - `expand/1`, no attempt is made to resolve `..`, `.` or `~`. + Converts the given path to an absolute one. + + Unlike `expand/1`, no attempt is made to resolve `..`, `.`, or `~`. - ## Unix examples + ## Examples + + ### Unix-like operating systems Path.absname("foo") #=> "/usr/local/foo" @@ -27,22 +33,25 @@ defmodule Path do Path.absname("../x") #=> "/usr/local/../x" - ## Windows + ### Windows - Path.absname("foo"). - "D:/usr/local/foo" - Path.absname("../x"). - "D:/usr/local/../x" + Path.absname("foo") + #=> "D:/usr/local/foo" + + Path.absname("../x") + #=> "D:/usr/local/../x" """ @spec absname(t) :: binary def absname(path) do - absname(path, System.cwd!) + absname(path, File.cwd!()) end @doc """ - Builds a path from `relative_to` to `path`. If `path` is already - an absolute path, `relative_to` is ignored. See also `relative_to/2`. + Builds a path from `relative_to` to `path`. + + If `path` is already an absolute path, `relative_to` is ignored. See also + `relative_to/2`. Unlike `expand/2`, no attempt is made to resolve `..`, `.` or `~`. @@ -59,62 +68,111 @@ defmodule Path do @spec absname(t, t) :: binary def absname(path, relative_to) do path = IO.chardata_to_string(path) + case type(path) do - :relative -> join(relative_to, path) + :relative -> + absname_join([relative_to, path]) + :absolute -> - cond do - path == "/" -> - path - :binary.last(path) == ?/ -> - binary_part(path, 0, byte_size(path) - 1) - true -> - path - end + absname_join([path]) + :volumerelative -> relative_to = IO.chardata_to_string(relative_to) absname_vr(split(path), split(relative_to), relative_to) end end - ## Absolute path on current drive - defp absname_vr(["/"|rest], [volume|_], _relative), - do: join([volume|rest]) + # Absolute path on current drive + defp absname_vr(["/" | rest], [volume | _], _relative), do: absname_join([volume | rest]) - ## Relative to current directory on current drive. - defp absname_vr([<>|rest], [<>|_], relative), - do: absname(join(rest), relative) + # Relative to current directory on current drive + defp absname_vr([<> | rest], [<> | _], relative), + do: absname(absname_join(rest), relative) - ## Relative to current directory on another drive. - defp absname_vr([<>|name], _, _relative) do + # Relative to current directory on another drive + defp absname_vr([<> | name], _, _relative) do cwd = case :file.get_cwd([x, ?:]) do - {:ok, dir} -> IO.chardata_to_string(dir) + {:ok, dir} -> IO.chardata_to_string(dir) {:error, _} -> <> end - absname(join(name), cwd) + + absname(absname_join(name), cwd) + end + + @slash [?/, ?\\] + + defp absname_join([]), do: "" + defp absname_join(list), do: absname_join(list, major_os_type()) + + defp absname_join([name1, name2 | rest], os_type) do + joined = do_absname_join(IO.chardata_to_string(name1), relative(name2), [], os_type) + absname_join([joined | rest], os_type) + end + + defp absname_join([name], os_type) do + do_absname_join(IO.chardata_to_string(name), <<>>, [], os_type) end + defp do_absname_join(<>, relativename, [], :win32) + when uc_letter in ?A..?Z, + do: do_absname_join(rest, relativename, [?:, uc_letter + ?a - ?A], :win32) + + defp do_absname_join(<>, relativename, [], :win32) + when c1 in @slash and c2 in @slash, + do: do_absname_join(rest, relativename, '//', :win32) + + defp do_absname_join(<>, relativename, result, :win32), + do: do_absname_join(<>, relativename, result, :win32) + + defp do_absname_join(<>, relativename, [?., ?/ | result], os_type), + do: do_absname_join(rest, relativename, [?/ | result], os_type) + + defp do_absname_join(<>, relativename, [?/ | result], os_type), + do: do_absname_join(rest, relativename, [?/ | result], os_type) + + defp do_absname_join(<<>>, <<>>, result, os_type), + do: IO.iodata_to_binary(reverse_maybe_remove_dir_sep(result, os_type)) + + defp do_absname_join(<<>>, relativename, [?: | rest], :win32), + do: do_absname_join(relativename, <<>>, [?: | rest], :win32) + + defp do_absname_join(<<>>, relativename, [?/ | result], os_type), + do: do_absname_join(relativename, <<>>, [?/ | result], os_type) + + defp do_absname_join(<<>>, relativename, result, os_type), + do: do_absname_join(relativename, <<>>, [?/ | result], os_type) + + defp do_absname_join(<>, relativename, result, os_type), + do: do_absname_join(rest, relativename, [char | result], os_type) + + defp reverse_maybe_remove_dir_sep([?/, ?:, letter], :win32), do: [letter, ?:, ?/] + defp reverse_maybe_remove_dir_sep([?/], _), do: [?/] + defp reverse_maybe_remove_dir_sep([?/ | name], _), do: :lists.reverse(name) + defp reverse_maybe_remove_dir_sep(name, _), do: :lists.reverse(name) + @doc """ - Converts the path to an absolute one and expands - any `.` and `..` characters and a leading `~`. + Converts the path to an absolute one, expanding + any `.` and `..` components and a leading `~`. ## Examples - Path.expand("/foo/bar/../bar") - "/foo/bar" + Path.expand("/foo/bar/../baz") + #=> "/foo/baz" """ @spec expand(t) :: binary def expand(path) do - normalize absname(expand_home(path), System.cwd!) + expand_dot(absname(expand_home(path), File.cwd!())) end @doc """ Expands the path relative to the path given as the second argument - expanding any `.` and `..` characters. If the path is already an - absolute path, `relative_to` is ignored. + expanding any `.` and `..` characters. - Note, that this function treats `path` with a leading `~` as + If the path is already an absolute path, `relative_to` is ignored. + + Note that this function treats a `path` with a leading `~` as an absolute one. The second argument is first expanded to an absolute path. @@ -126,20 +184,23 @@ defmodule Path do #=> "/quux/baz/foo/bar" Path.expand("foo/bar/../bar", "/baz") - "/baz/foo/bar" + #=> "/baz/foo/bar" + Path.expand("/foo/bar/../bar", "/baz") - "/foo/bar" + #=> "/foo/bar" """ @spec expand(t, t) :: binary def expand(path, relative_to) do - normalize absname(absname(expand_home(path), expand_home(relative_to)), System.cwd!) + expand_dot(absname(absname(expand_home(path), expand_home(relative_to)), File.cwd!())) end @doc """ Returns the path type. - ## Unix examples + ## Examples + + ### Unix-like operating systems Path.type("/") #=> :absolute Path.type("/usr/local/bin") #=> :absolute @@ -147,7 +208,7 @@ defmodule Path do Path.type("../usr/local/bin") #=> :relative Path.type("~/file") #=> :relative - ## Windows examples + ### Windows Path.type("D:/usr/local/bin") #=> :absolute Path.type("usr/local/bin") #=> :relative @@ -156,23 +217,24 @@ defmodule Path do """ @spec type(t) :: :absolute | :relative | :volumerelative - def type(name) when is_list(name) or is_binary(name) do - case :os.type() do - {:win32, _} -> win32_pathtype(name) - _ -> unix_pathtype(name) - end |> elem(0) + def type(name) + when is_list(name) + when is_binary(name) do + pathtype(name, major_os_type()) |> elem(0) end @doc """ Forces the path to be a relative path. - ## Unix examples + ## Examples + + ### Unix-like operating systems Path.relative("/usr/local/bin") #=> "usr/local/bin" Path.relative("usr/local/bin") #=> "usr/local/bin" Path.relative("../usr/local/bin") #=> "../usr/local/bin" - ## Windows examples + ### Windows Path.relative("D:/usr/local/bin") #=> "usr/local/bin" Path.relative("usr/local/bin") #=> "usr/local/bin" @@ -182,55 +244,65 @@ defmodule Path do """ @spec relative(t) :: binary def relative(name) do - case :os.type() do - {:win32, _} -> win32_pathtype(name) - _ -> unix_pathtype(name) - end |> elem(1) |> IO.chardata_to_string + relative(name, major_os_type()) end - defp unix_pathtype(<>), do: - {:absolute, relative} - defp unix_pathtype([?/|relative]), do: - {:absolute, relative} - defp unix_pathtype([list|rest]) when is_list(list), do: - unix_pathtype(list ++ rest) - defp unix_pathtype(relative), do: - {:relative, relative} + defp relative(name, os_type) do + pathtype(name, os_type) + |> elem(1) + |> IO.chardata_to_string() + end - @slash [?/, ?\\] + defp pathtype(name, os_type) do + case os_type do + :win32 -> win32_pathtype(name) + _ -> unix_pathtype(name) + end + end + + defp unix_pathtype(path) when path in ["/", '/'], do: {:absolute, "."} + defp unix_pathtype(<>), do: {:absolute, relative} + defp unix_pathtype([?/ | relative]), do: {:absolute, relative} + defp unix_pathtype([list | rest]) when is_list(list), do: unix_pathtype(list ++ rest) + defp unix_pathtype(relative), do: {:relative, relative} + + defp win32_pathtype([list | rest]) when is_list(list), do: win32_pathtype(list ++ rest) + + defp win32_pathtype([char, list | rest]) when is_list(list), + do: win32_pathtype([char | list ++ rest]) + + defp win32_pathtype(<>) when c1 in @slash and c2 in @slash, + do: {:absolute, relative} + + defp win32_pathtype(<>) when char in @slash, + do: {:volumerelative, relative} + + defp win32_pathtype(<<_letter, ?:, char, relative::binary>>) when char in @slash, + do: {:absolute, relative} + + defp win32_pathtype(<<_letter, ?:, relative::binary>>), do: {:volumerelative, relative} - defp win32_pathtype([list|rest]) when is_list(list), do: - win32_pathtype(list++rest) - defp win32_pathtype([char, list|rest]) when is_list(list), do: - win32_pathtype([char|list++rest]) - defp win32_pathtype(<>) when c1 in @slash and c2 in @slash, do: - {:absolute, relative} - defp win32_pathtype(<>) when c in @slash, do: - {:volumerelative, relative} - defp win32_pathtype(<<_letter, ?:, c, relative :: binary>>) when c in @slash, do: - {:absolute, relative} - defp win32_pathtype(<<_letter, ?:, relative :: binary>>), do: - {:volumerelative, relative} - - defp win32_pathtype([c1, c2 | relative]) when c1 in @slash and c2 in @slash, do: - {:absolute, relative} - defp win32_pathtype([c | relative]) when c in @slash, do: - {:volumerelative, relative} - defp win32_pathtype([c1, c2, list|rest]) when is_list(list), do: - win32_pathtype([c1, c2|list++rest]) - defp win32_pathtype([_letter, ?:, c | relative]) when c in @slash, do: - {:absolute, relative} - defp win32_pathtype([_letter, ?: | relative]), do: - {:volumerelative, relative} - defp win32_pathtype(relative), do: - {:relative, relative} + defp win32_pathtype([c1, c2 | relative]) when c1 in @slash and c2 in @slash, + do: {:absolute, relative} + + defp win32_pathtype([char | relative]) when char in @slash, do: {:volumerelative, relative} + + defp win32_pathtype([c1, c2, list | rest]) when is_list(list), + do: win32_pathtype([c1, c2 | list ++ rest]) + + defp win32_pathtype([_letter, ?:, char | relative]) when char in @slash, + do: {:absolute, relative} + + defp win32_pathtype([_letter, ?: | relative]), do: {:volumerelative, relative} + defp win32_pathtype(relative), do: {:relative, relative} @doc """ Returns the given `path` relative to the given `from` path. - In other words, it tries to strip the `from` prefix from `path`. + + In other words, this function tries to strip the `from` prefix from `path`. This function does not query the file system, so it assumes - no symlinks in between the paths. + no symlinks between the paths. In case a direct relative path cannot be found, it returns the original path. @@ -246,6 +318,9 @@ defmodule Path do iex> Path.relative_to("/usr/local/foo", "/etc") "/usr/local/foo" + iex> Path.relative_to("/usr/local/foo", "/usr/local/foo") + "." + """ @spec relative_to(t, t) :: binary def relative_to(path, from) do @@ -253,11 +328,15 @@ defmodule Path do relative_to(split(path), split(from), path) end - defp relative_to([h|t1], [h|t2], original) do + defp relative_to(path, path, _original) do + "." + end + + defp relative_to([h | t1], [h | t2], original) do relative_to(t1, t2, original) end - defp relative_to([_|_] = l1, [], _original) do + defp relative_to([_ | _] = l1, [], _original) do join(l1) end @@ -267,12 +346,14 @@ defmodule Path do @doc """ Convenience to get the path relative to the current working - directory. If, for some reason, the current working directory - cannot be retrieved, returns the full path. + directory. + + If, for some reason, the current working directory + cannot be retrieved, this function returns the given `path`. """ @spec relative_to_cwd(t) :: binary def relative_to_cwd(path) do - case :file.get_cwd do + case :file.get_cwd() do {:ok, base} -> relative_to(path, IO.chardata_to_string(base)) _ -> path end @@ -290,19 +371,24 @@ defmodule Path do iex> Path.basename("foo/bar") "bar" + iex> Path.basename("lib/module/submodule.ex") + "submodule.ex" + iex> Path.basename("/") "" """ @spec basename(t) :: binary def basename(path) do - FN.basename(IO.chardata_to_string(path)) + :filename.basename(IO.chardata_to_string(path)) end @doc """ Returns the last component of `path` with the `extension` - stripped. This function should be used to remove a specific - extension which may, or may not, be there. + stripped. + + This function should be used to remove a specific + extension which may or may not be there. ## Examples @@ -318,7 +404,7 @@ defmodule Path do """ @spec basename(t, t) :: binary def basename(path, extension) do - FN.basename(IO.chardata_to_string(path), IO.chardata_to_string(extension)) + :filename.basename(IO.chardata_to_string(path), IO.chardata_to_string(extension)) end @doc """ @@ -326,20 +412,37 @@ defmodule Path do ## Examples - Path.dirname("/foo/bar.ex") - #=> "/foo" - Path.dirname("/foo/bar/baz.ex") - #=> "/foo/bar" + iex> Path.dirname("/foo/bar.ex") + "/foo" + + iex> Path.dirname("/foo/bar/baz.ex") + "/foo/bar" + + iex> Path.dirname("/foo/bar/") + "/foo/bar" + + iex> Path.dirname("bar.ex") + "." """ @spec dirname(t) :: binary def dirname(path) do - FN.dirname(IO.chardata_to_string(path)) + :filename.dirname(IO.chardata_to_string(path)) end @doc """ Returns the extension of the last component of `path`. + The behaviour of this function changed in Erlang/OTP 24 for filenames + starting with a dot and without an extension. For example, for a file + named `.gitignore`, `extname/1` now returns an empty string, while it + would return `".gitignore"` in previous Erlang/OTP versions. This was + done to match the behaviour of `rootname/1`, which would return + `".gitignore"` as its name (and therefore it cannot also be an extension). + + See `basename/1` and `rootname/1` for related functions to extract + information from paths. + ## Examples iex> Path.extname("foo.erl") @@ -351,7 +454,7 @@ defmodule Path do """ @spec extname(t) :: binary def extname(path) do - FN.extension(IO.chardata_to_string(path)) + :filename.extension(IO.chardata_to_string(path)) end @doc """ @@ -368,12 +471,14 @@ defmodule Path do """ @spec rootname(t) :: binary def rootname(path) do - FN.rootname(IO.chardata_to_string(path)) + :filename.rootname(IO.chardata_to_string(path)) end @doc """ - Returns the `path` with the `extension` stripped. This function should be used to - remove a specific extension which might, or might not, be there. + Returns the `path` with the `extension` stripped. + + This function should be used to remove a specific extension which may + or may not be there. ## Examples @@ -386,14 +491,16 @@ defmodule Path do """ @spec rootname(t, t) :: binary def rootname(path, extension) do - FN.rootname(IO.chardata_to_string(path), IO.chardata_to_string(extension)) + :filename.rootname(IO.chardata_to_string(path), IO.chardata_to_string(extension)) end @doc """ - Returns a string with one or more path components joined by the path separator. + Joins a list of paths. - This function should be used to convert a list of strings to a path. - Note that any trailing slash is removed on join. + This function should be used to convert a list of paths to a path. + Note that any trailing slash is removed when joining. + + Raises an error if the given list of paths is empty. ## Examples @@ -407,139 +514,149 @@ defmodule Path do "/foo/bar" """ - @spec join([t]) :: binary - def join([name1, name2|rest]), do: - join([join(name1, name2)|rest]) - def join([name]), do: - do_join(IO.chardata_to_string(name), <<>>, [], major_os_type()) + @spec join(nonempty_list(t)) :: binary + def join([name1, name2 | rest]), do: join([join(name1, name2) | rest]) + def join([name]), do: IO.chardata_to_string(name) @doc """ Joins two paths. + The right path will always be expanded to its relative format + and any trailing slash will be removed when joining. + ## Examples iex> Path.join("foo", "bar") "foo/bar" + iex> Path.join("/foo", "/bar/") + "/foo/bar" + + The functions in this module support chardata, so giving a list will + treat it as a single entity: + + iex> Path.join("foo", ["bar", "fiz"]) + "foo/barfiz" + + iex> Path.join(["foo", "bar"], "fiz") + "foobar/fiz" + + Use `join/1` if you need to join a list of paths instead. """ @spec join(t, t) :: binary - def join(left, right), - do: do_join(IO.chardata_to_string(left), relative(right), [], major_os_type()) + def join(left, right) do + left = IO.chardata_to_string(left) + os_type = major_os_type() + do_join(left, right, os_type) |> remove_dir_sep(os_type) + end - defp major_os_type do - :os.type |> elem(0) - end - - defp do_join(<>, relativename, [], :win32) when uc_letter in ?A..?Z, do: - do_join(rest, relativename, [?:, uc_letter+?a-?A], :win32) - defp do_join(<>, relativename, result, :win32), do: - do_join(<>, relativename, result, :win32) - defp do_join(<>, relativename, [?., ?/|result], os_type), do: - do_join(rest, relativename, [?/|result], os_type) - defp do_join(<>, relativename, [?/|result], os_type), do: - do_join(rest, relativename, [?/|result], os_type) - defp do_join(<<>>, <<>>, result, os_type), do: - IO.iodata_to_binary(maybe_remove_dirsep(result, os_type)) - defp do_join(<<>>, relativename, [?:|rest], :win32), do: - do_join(relativename, <<>>, [?:|rest], :win32) - defp do_join(<<>>, relativename, [?/|result], os_type), do: - do_join(relativename, <<>>, [?/|result], os_type) - defp do_join(<<>>, relativename, result, os_type), do: - do_join(relativename, <<>>, [?/|result], os_type) - defp do_join(<>, relativename, result, os_type), do: - do_join(rest, relativename, [char|result], os_type) - - defp maybe_remove_dirsep([?/, ?:, letter], :win32), do: - [letter, ?:, ?/] - defp maybe_remove_dirsep([?/], _), do: - [?/] - defp maybe_remove_dirsep([?/|name], _), do: - :lists.reverse(name) - defp maybe_remove_dirsep(name, _), do: - :lists.reverse(name) + defp do_join(left, "/", os_type), do: remove_dir_sep(left, os_type) + defp do_join("", right, os_type), do: relative(right, os_type) + defp do_join("/", right, os_type), do: "/" <> relative(right, os_type) - @doc """ - Returns a list with the path split by the path separator. - If an empty string is given, returns the root path. + defp do_join(left, right, os_type), + do: remove_dir_sep(left, os_type) <> "/" <> relative(right, os_type) + + defp remove_dir_sep("", _os_type), do: "" + defp remove_dir_sep("/", _os_type), do: "/" + + defp remove_dir_sep(bin, os_type) do + last = :binary.last(bin) + + if last == ?/ or (last == ?\\ and os_type == :win32) do + binary_part(bin, 0, byte_size(bin) - 1) + else + bin + end + end + + @doc ~S""" + Splits the path into a list at the path separator. + + If an empty string is given, returns an empty list. + + On Windows, path is split on both `"\"` and `"/"` separators + and the driver letter, if there is one, is always returned + in lowercase. ## Examples - iex> Path.split("") - [] + iex> Path.split("") + [] - iex> Path.split("foo") - ["foo"] + iex> Path.split("foo") + ["foo"] - iex> Path.split("/foo/bar") - ["/", "foo", "bar"] + iex> Path.split("/foo/bar") + ["/", "foo", "bar"] """ @spec split(t) :: [binary] - - # Work around a bug in Erlang on UNIX - def split(""), do: [] - def split(path) do - FN.split(IO.chardata_to_string(path)) + :filename.split(IO.chardata_to_string(path)) end defmodule Wildcard do @moduledoc false - def read_file_info(file) do - call({:read_link_info, file}) + def read_link_info(file) do + :file.read_link_info(file) end def list_dir(dir) do - case call({:list_dir, dir}) do - {:ok, files} -> - {:ok, for(file <- files, hd(file) != ?., do: file)} - other -> - other + case :file.list_dir(dir) do + {:ok, files} -> {:ok, for(file <- files, hd(file) != ?., do: file)} + other -> other end end - - @compile {:inline, call: 1} - - defp call(tuple) do - x = :erlang.dt_spread_tag(true) - y = :gen_server.call(:file_server_2, tuple) - :erlang.dt_restore_tag(x) - y - end end @doc """ - Traverses paths according to the given `glob` expression. + Traverses paths according to the given `glob` expression and returns a + list of matches. - The wildcard looks like an ordinary path, except that certain - "wildcard characters" are interpreted in a special way. The - following characters are special: + The wildcard looks like an ordinary path, except that the following + "wildcard characters" are interpreted in a special way: - * `?` - matches one character + * `?` - matches one character. * `*` - matches any number of characters up to the end of the filename, the - next dot, or the next slash + next dot, or the next slash. * `**` - two adjacent `*`'s used as a single pattern will match all - files and zero or more directories and subdirectories + files and zero or more directories and subdirectories. * `[char1,char2,...]` - matches any of the characters listed; two - characters separated by a hyphen will match a range of characters + characters separated by a hyphen will match a range of characters. + Do not add spaces before and after the comma as it would then match + paths containing the space character itself. - * `{item1,item2,...}` - matches one of the alternatives + * `{item1,item2,...}` - matches one of the alternatives. + Do not add spaces before and after the comma as it would then match + paths containing the space character itself. Other characters represent themselves. Only paths that have exactly the same character in the same position will match. Note - that matching is case-sensitive; i.e. "a" will not match "A". + that matching is case-sensitive: `"a"` will not match `"A"`. + + Directory separators must always be written as `/`, even on Windows. + You may call `Path.expand/1` to normalize the path before invoking + this function. By default, the patterns `*` and `?` do not match files starting - with a dot `.` unless `match_dot: true` is given. + with a dot `.`. See the `:match_dot` option in the "Options" section + below. + + ## Options + + * `:match_dot` - (boolean) if `false`, the special wildcard characters `*` and `?` + will not match files starting with a dot (`.`). If `true`, files starting with + a `.` will not be treated specially. Defaults to `false`. ## Examples Imagine you have a directory called `projects` with three Elixir projects - inside of it: `elixir`, `ex_doc` and `dynamo`. You can find all `.beam` files + inside of it: `elixir`, `ex_doc`, and `plug`. You can find all `.beam` files inside the `ebin` directory of each project as follows: Path.wildcard("projects/*/ebin/**/*.beam") @@ -549,21 +666,25 @@ defmodule Path do Path.wildcard("projects/*/ebin/**/*.{beam,app}") """ - @spec wildcard(t) :: [binary] - def wildcard(glob, opts \\ []) do + @spec wildcard(t, keyword) :: [binary] + def wildcard(glob, opts \\ []) when is_list(opts) do mod = if Keyword.get(opts, :match_dot), do: :file, else: Path.Wildcard + glob - |> chardata_to_list() + |> chardata_to_list!() |> :filelib.wildcard(mod) |> Enum.map(&IO.chardata_to_string/1) end - # Normalize the given path by expanding "..", "." and "~". - - defp chardata_to_list(chardata) do + defp chardata_to_list!(chardata) do case :unicode.characters_to_list(chardata) do result when is_list(result) -> - result + if 0 in result do + raise ArgumentError, + "cannot execute Path.wildcard/2 for path with null byte, got: #{inspect(chardata)}" + else + result + end {:error, encoded, rest} -> raise UnicodeConversionError, encoded: encoded, rest: rest, kind: :invalid @@ -575,34 +696,108 @@ defmodule Path do defp expand_home(type) do case IO.chardata_to_string(type) do - "~" <> rest -> System.user_home! <> rest - rest -> rest + "~" <> rest -> resolve_home(rest) + rest -> rest end end - defp normalize(path), do: normalize(split(path), []) + defp resolve_home(""), do: System.user_home!() - defp normalize([".."|t], ["/"|_] = acc) do - normalize t, acc + defp resolve_home(rest) do + case {rest, major_os_type()} do + {"\\" <> _, :win32} -> System.user_home!() <> rest + {"/" <> _, _} -> System.user_home!() <> rest + _ -> "~" <> rest + end end - defp normalize([".."|t], [<>|_] = acc) when letter in ?a..?z do - normalize t, acc - end + # expand_dot the given path by expanding "..", "." and "~". + defp expand_dot(<<"/", rest::binary>>), do: "/" <> do_expand_dot(rest) - defp normalize([".."|t], [_|acc]) do - normalize t, acc - end + defp expand_dot(<>) when letter in ?a..?z, + do: <> <> do_expand_dot(rest) + + defp expand_dot(path), do: do_expand_dot(path) + + defp do_expand_dot(path), do: do_expand_dot(:binary.split(path, "/", [:global]), []) + defp do_expand_dot([".." | t], [_, _ | acc]), do: do_expand_dot(t, acc) + defp do_expand_dot([".." | t], []), do: do_expand_dot(t, []) + defp do_expand_dot(["." | t], acc), do: do_expand_dot(t, acc) + defp do_expand_dot([h | t], acc), do: do_expand_dot(t, ["/", h | acc]) + defp do_expand_dot([], []), do: "" + defp do_expand_dot([], ["/" | acc]), do: IO.iodata_to_binary(:lists.reverse(acc)) - defp normalize(["."|t], acc) do - normalize t, acc + defp major_os_type do + :os.type() |> elem(0) end - defp normalize([h|t], acc) do - normalize t, [h|acc] + @doc """ + Returns a relative path that is protected from directory-traversal attacks. + + The given relative path is sanitized by eliminating `..` and `.` components. + + This function checks that, after expanding those components, the path is still "safe". + Paths are considered unsafe if either of these is true: + + * The path is not relative, such as `"/foo/bar"`. + + * A `..` component would make it so that the path would travers up above + the root of `relative_to`. + + * A symbolic link in the path points to something above the root of `relative_to`. + + ## Examples + + iex> Path.safe_relative_to("deps/my_dep/app.beam", "deps") + {:ok, "deps/my_dep/app.beam"} + + iex> Path.safe_relative_to("deps/my_dep/./build/../app.beam", "deps") + {:ok, "deps/my_dep/app.beam"} + + iex> Path.safe_relative_to("my_dep/../..", "deps") + :error + + iex> Path.safe_relative_to("/usr/local", ".") + :error + + """ + @doc since: "1.14.0" + @spec safe_relative_to(t, t) :: {:ok, binary} | :error + def safe_relative_to(path, relative_to) do + path = IO.chardata_to_string(path) + + case :filelib.safe_relative_path(path, relative_to) do + :unsafe -> :error + relative_path -> {:ok, IO.chardata_to_string(relative_path)} + end end - defp normalize([], acc) do - join :lists.reverse(acc) + @doc """ + Returns a path relative to the current working directory that is + protected from directory-traversal attacks. + + Same as `safe_relative_to/2` with the current working directory as + the second argument. If there is an issue retrieving the current working + directory, this function raises an error. + + ## Examples + + iex> Path.safe_relative("foo") + {:ok, "foo"} + + iex> Path.safe_relative("foo/../bar") + {:ok, "bar"} + + iex> Path.safe_relative("foo/../..") + :error + + iex> Path.safe_relative("/usr/local") + :error + + """ + @doc since: "1.14.0" + @spec safe_relative(t) :: {:ok, binary} | :error + def safe_relative(path) do + safe_relative_to(path, File.cwd!()) end end diff --git a/lib/elixir/lib/port.ex b/lib/elixir/lib/port.ex index e7f465a26bb..cd230ee8fbe 100644 --- a/lib/elixir/lib/port.ex +++ b/lib/elixir/lib/port.ex @@ -1,68 +1,329 @@ defmodule Port do - @moduledoc """ - Functions related to Erlang ports. + @moduledoc ~S""" + Functions for interacting with the external world through ports. + + Ports provide a mechanism to start operating system processes external + to the Erlang VM and communicate with them via message passing. + + ## Example + + iex> port = Port.open({:spawn, "cat"}, [:binary]) + iex> send(port, {self(), {:command, "hello"}}) + iex> send(port, {self(), {:command, "world"}}) + iex> flush() + {#Port<0.1444>, {:data, "hello"}} + {#Port<0.1444>, {:data, "world"}} + iex> send(port, {self(), :close}) + :ok + iex> flush() + {#Port<0.1464>, :closed} + :ok + + In the example above, we have created a new port that executes the + program `cat`. `cat` is a program available on Unix-like operating systems that + receives data from multiple inputs and concatenates them in the output. + + After the port was created, we sent it two commands in the form of + messages using `send/2`. The first command has the binary payload + of "hello" and the second has "world". + + After sending those two messages, we invoked the IEx helper `flush()`, + which printed all messages received from the port, in this case we got + "hello" and "world" back. Note that the messages are in binary because we + passed the `:binary` option when opening the port in `Port.open/2`. Without + such option, it would have yielded a list of bytes. + + Once everything was done, we closed the port. + + Elixir provides many conveniences for working with ports and some drawbacks. + We will explore those below. + + ## Message and function APIs + + There are two APIs for working with ports. It can be either asynchronous via + message passing, as in the example above, or by calling the functions on this + module. + + The messages supported by ports and their counterpart function APIs are + listed below: + + * `{pid, {:command, binary}}` - sends the given data to the port. + See `command/3`. + + * `{pid, :close}` - closes the port. Unless the port is already closed, + the port will reply with `{port, :closed}` message once it has flushed + its buffers and effectively closed. See `close/1`. + + * `{pid, {:connect, new_pid}}` - sets the `new_pid` as the new owner of + the port. Once a port is opened, the port is linked and connected to the + caller process and communication to the port only happens through the + connected process. This message makes `new_pid` the new connected processes. + Unless the port is dead, the port will reply to the old owner with + `{port, :connected}`. See `connect/2`. + + On its turn, the port will send the connected process the following messages: + + * `{port, {:data, data}}` - data sent by the port + * `{port, :closed}` - reply to the `{pid, :close}` message + * `{port, :connected}` - reply to the `{pid, {:connect, new_pid}}` message + * `{:EXIT, port, reason}` - exit signals in case the port crashes. If reason + is not `:normal`, this message will only be received if the owner process + is trapping exits + + ## Open mechanisms + + The port can be opened through four main mechanisms. + + As a short summary, prefer to using the `:spawn` and `:spawn_executable` + options mentioned below. The other two options, `:spawn_driver` and `:fd` + are for advanced usage within the VM. Also consider using `System.cmd/3` + if all you want is to execute a program and retrieve its return value. + + ### spawn + + The `:spawn` tuple receives a binary that is going to be executed as a + full invocation. For example, we can use it to invoke "echo hello" directly: + + iex> port = Port.open({:spawn, "echo hello"}, [:binary]) + iex> flush() + {#Port<0.1444>, {:data, "hello\n"}} + + `:spawn` will retrieve the program name from the argument and traverse your + operating system `$PATH` environment variable looking for a matching program. + + Although the above is handy, it means it is impossible to invoke an executable + that has whitespaces on its name or in any of its arguments. For those reasons, + most times it is preferable to execute `:spawn_executable`. + + ### spawn_executable + + Spawn executable is a more restricted and explicit version of spawn. It expects + full file paths to the executable you want to execute. If they are in your `$PATH`, + they can be retrieved by calling `System.find_executable/1`: + + iex> path = System.find_executable("echo") + iex> port = Port.open({:spawn_executable, path}, [:binary, args: ["hello world"]]) + iex> flush() + {#Port<0.1380>, {:data, "hello world\n"}} + + When using `:spawn_executable`, the list of arguments can be passed via + the `:args` option as done above. For the full list of options, see the + documentation for the Erlang function `:erlang.open_port/2`. + + ### fd + + The `:fd` name option allows developers to access `in` and `out` file + descriptors used by the Erlang VM. You would use those only if you are + reimplementing core part of the Runtime System, such as the `:user` and + `:shell` processes. + + ## Zombie operating system processes + + A port can be closed via the `close/1` function or by sending a `{pid, :close}` + message. However, if the VM crashes, a long-running program started by the port + will have its stdin and stdout channels closed but **it won't be automatically + terminated**. + + While most Unix command line tools will exit once its communication channels + are closed, not all command line applications will do so. You can easily check + this by starting the port and then shutting down the VM and inspecting your + operating system to see if the port process is still running. + + While we encourage graceful termination by detecting if stdin/stdout has been + closed, we do not always have control over how third-party software terminates. + In those cases, you can wrap the application in a script that checks for stdin. + Here is such script that has been verified to work on bash shells: + + #!/usr/bin/env bash + + # Start the program in the background + exec "$@" & + pid1=$! + + # Silence warnings from here on + exec >/dev/null 2>&1 + + # Read from stdin in the background and + # kill running program when stdin closes + exec 0<&0 $( + while read; do :; done + kill -KILL $pid1 + ) & + pid2=$! + + # Clean up + wait $pid1 + ret=$? + kill -KILL $pid2 + exit $ret + + Note the program above hijacks stdin, so you won't be able to communicate + with the underlying software via stdin (on the positive side, software that + reads from stdin typically terminates when stdin closes). + + Now instead of: + + Port.open( + {:spawn_executable, "/path/to/program"}, + args: ["a", "b", "c"] + ) + + You may invoke: + + Port.open( + {:spawn_executable, "/path/to/wrapper"}, + args: ["/path/to/program", "a", "b", "c"] + ) + """ + @type name :: + {:spawn, charlist | binary} + | {:spawn_driver, charlist | binary} + | {:spawn_executable, :file.name_all()} + | {:fd, non_neg_integer, non_neg_integer} + @doc """ - See http://www.erlang.org/doc/man/erlang.html#open_port-2. + Opens a port given a tuple `name` and a list of `options`. + + The module documentation above contains documentation and examples + for the supported `name` values, summarized below: + + * `{:spawn, command}` - runs an external program. `command` must contain + the program name and optionally a list of arguments separated by space. + If passing programs or arguments with space in their name, use the next option. + * `{:spawn_executable, filename}` - runs the executable given by the absolute + file name `filename`. Arguments can be passed via the `:args` option. + * `{:spawn_driver, command}` - spawns so-called port drivers. + * `{:fd, fd_in, fd_out}` - accesses file descriptors, `fd_in` and `fd_out` + opened by the VM. + + For more information and the list of options, see `:erlang.open_port/2`. + + Inlined by the compiler. """ - def open(name, settings) do - :erlang.open_port(name, settings) + @spec open(name, list) :: port + def open(name, options) do + :erlang.open_port(name, options) end @doc """ - See http://www.erlang.org/doc/man/erlang.html#port_close-1. + Closes the `port`. + + For more information, see `:erlang.port_close/1`. + + Inlined by the compiler. """ + @spec close(port) :: true def close(port) do :erlang.port_close(port) end @doc """ - See http://www.erlang.org/doc/man/erlang.html#port_command-2. + Sends `data` to the port driver `port`. + + For more information, see `:erlang.port_command/3`. + + Inlined by the compiler. """ + @spec command(port, iodata, [:force | :nosuspend]) :: boolean def command(port, data, options \\ []) do :erlang.port_command(port, data, options) end @doc """ - See http://www.erlang.org/doc/man/erlang.html#port_connect-2. + Associates the `port` identifier with a `pid`. + + For more information, see `:erlang.port_connect/2`. + + Inlined by the compiler. """ + @spec connect(port, pid) :: true def connect(port, pid) do :erlang.port_connect(port, pid) end @doc """ - See http://www.erlang.org/doc/man/erlang.html#port_control-3. + Returns information about the `port` or `nil` if the port is closed. + + For more information, see `:erlang.port_info/1`. """ - def control(port, operation, data) do - :erlang.port_control(port, operation, data) + @spec info(port) :: keyword | nil + def info(port) do + nilify(:erlang.port_info(port)) end @doc """ - See http://www.erlang.org/doc/man/erlang.html#port_call-3. + Returns information about the `port` or `nil` if the port is closed. + + For more information, see `:erlang.port_info/2`. """ - def call(port, operation, data) do - :erlang.port_call(port, operation, data) + @spec info(port, atom) :: {atom, term} | nil + def info(port, spec) + + def info(port, :registered_name) do + case :erlang.port_info(port, :registered_name) do + :undefined -> nil + [] -> {:registered_name, []} + other -> other + end + end + + def info(port, item) do + nilify(:erlang.port_info(port, item)) end @doc """ - See http://www.erlang.org/doc/man/erlang.html#port_info-1. + Starts monitoring the given `port` from the calling process. + + Once the monitored port process dies, a message is delivered to the + monitoring process in the shape of: + + {:DOWN, ref, :port, object, reason} + + where: + + * `ref` is a monitor reference returned by this function; + * `object` is either the `port` being monitored (when monitoring by port ID) + or `{name, node}` (when monitoring by a port name); + * `reason` is the exit reason. + + See `:erlang.monitor/2` for more information. + + Inlined by the compiler. """ - def info(port) do - :erlang.port_info(port) + @doc since: "1.6.0" + @spec monitor(port | {name, node} | name) :: reference when name: atom + def monitor(port) do + :erlang.monitor(:port, port) end @doc """ - See http://www.erlang.org/doc/man/erlang.html#port_info-2. + Demonitors the monitor identified by the given `reference`. + + If `monitor_ref` is a reference which the calling process + obtained by calling `monitor/1`, that monitoring is turned off. + If the monitoring is already turned off, nothing happens. + + See `:erlang.demonitor/2` for more information. + + Inlined by the compiler. """ - def info(port, item) do - :erlang.port_info(port, item) - end + @doc since: "1.6.0" + @spec demonitor(reference, options :: [:flush | :info]) :: boolean + defdelegate demonitor(monitor_ref, options \\ []), to: :erlang @doc """ - See http://www.erlang.org/doc/man/erlang.html#ports-0. + Returns a list of all ports in the current node. + + Inlined by the compiler. """ + @spec list :: [port] def list do - :erlang.ports + :erlang.ports() end -end \ No newline at end of file + + @compile {:inline, nilify: 1} + defp nilify(:undefined), do: nil + defp nilify(other), do: other +end diff --git a/lib/elixir/lib/process.ex b/lib/elixir/lib/process.ex index ce73ca7bb16..d440330bccb 100644 --- a/lib/elixir/lib/process.ex +++ b/lib/elixir/lib/process.ex @@ -4,7 +4,7 @@ defmodule Process do Besides the functions available in this module, the `Kernel` module exposes and auto-imports some basic functionality related to processes - available through the functions: + available through the following functions: * `Kernel.spawn/1` and `Kernel.spawn/3` * `Kernel.spawn_link/1` and `Kernel.spawn_link/3` @@ -12,304 +12,668 @@ defmodule Process do * `Kernel.self/0` * `Kernel.send/2` + While this module provides low-level conveniences to work with processes, + developers typically use abstractions such as `Agent`, `GenServer`, + `Registry`, `Supervisor` and `Task` for building their systems and + resort to this module for gathering information, trapping exits, links + and monitoring. """ + @typedoc """ + A process destination. + + A remote or local PID, a local port, a locally registered name, or a tuple in + the form of `{registered_name, node}` for a registered name at another node. + """ + @type dest :: pid | port | (registered_name :: atom) | {registered_name :: atom, node} + @doc """ - Returns true if the process exists and is alive, that is, - is not exiting and has not exited. Otherwise, returns false. + Tells whether the given process is alive on the local node. - `pid` must refer to a process at the local node. + If the process identified by `pid` is alive (that is, it's not exiting and has + not exited yet) than this function returns `true`. Otherwise, it returns + `false`. + + `pid` must refer to a process running on the local node or `ArgumentError` is raised. + + Inlined by the compiler. """ @spec alive?(pid) :: boolean - def alive?(pid) do - :erlang.is_process_alive(pid) - end + defdelegate alive?(pid), to: :erlang, as: :is_process_alive @doc """ - Returns all key-values in the dictionary. + Returns all key-value pairs in the process dictionary. + + Inlined by the compiler. """ - @spec get :: [{term, term}] - def get do - :erlang.get() - end + @spec get() :: [{term, term}] + defdelegate get(), to: :erlang @doc """ - Returns the value for the given key. + Returns the value for the given `key` in the process dictionary, + or `default` if `key` is not set. + + ## Examples + + # Assuming :locale was not set + iex> Process.get(:locale, "pt") + "pt" + iex> Process.put(:locale, "fr") + nil + iex> Process.get(:locale, "pt") + "fr" + """ - @spec get(term) :: term @spec get(term, default :: term) :: term def get(key, default \\ nil) do case :erlang.get(key) do - :undefined -> - default - value -> - value + :undefined -> default + value -> value end end @doc """ - Returns all keys that have the given `value`. + Returns all keys in the process dictionary. + + Inlined by the compiler. + + ## Examples + + # Assuming :locale was not set + iex> :locale in Process.get_keys() + false + iex> Process.put(:locale, "pt") + nil + iex> :locale in Process.get_keys() + true + + """ + @spec get_keys() :: [term] + defdelegate get_keys(), to: :erlang + + @doc """ + Returns all keys in the process dictionary that have the given `value`. + + Inlined by the compiler. """ @spec get_keys(term) :: [term] - def get_keys(value) do - :erlang.get_keys(value) - end + defdelegate get_keys(value), to: :erlang @doc """ - Stores the given key-value in the process dictionary. + Stores the given `key`-`value` pair in the process dictionary. + + The return value of this function is the value that was previously stored + under `key`, or `nil` in case no value was stored under it. + + ## Examples + + # Assuming :locale was not set + iex> Process.put(:locale, "en") + nil + iex> Process.put(:locale, "fr") + "en" + """ @spec put(term, term) :: term | nil def put(key, value) do - nillify :erlang.put(key, value) + nilify(:erlang.put(key, value)) end @doc """ - Deletes the given key from the dictionary. + Deletes the given `key` from the process dictionary. + + Returns the value that was under `key` in the process dictionary, + or `nil` if `key` was not stored in the process dictionary. + + ## Examples + + iex> Process.put(:comments, ["comment", "other comment"]) + iex> Process.delete(:comments) + ["comment", "other comment"] + iex> Process.delete(:comments) + nil + """ @spec delete(term) :: term | nil def delete(key) do - nillify :erlang.erase(key) + nilify(:erlang.erase(key)) end @doc """ - Sends an exit signal with the given reason to the pid. + Sends an exit signal with the given `reason` to `pid`. - The following behaviour applies if reason is any term except `:normal` or `:kill`: + The following behaviour applies if `reason` is any term except `:normal` + or `:kill`: - 1. If pid is not trapping exits, pid will exit with the given reason. + 1. If `pid` is not trapping exits, `pid` will exit with the given + `reason`. - 2. If pid is trapping exits, the exit signal is transformed into a message - {:EXIT, from, reason} and delivered to the message queue of pid. + 2. If `pid` is trapping exits, the exit signal is transformed into a + message `{:EXIT, from, reason}` and delivered to the message queue + of `pid`. - 3. If reason is the atom `:normal`, pid will not exit. If it is trapping - exits, the exit signal is transformed into a message {:EXIT, from, - :normal} and delivered to its message queue. + If `reason` is the atom `:normal`, `pid` will not exit (unless `pid` is + the calling process, in which case it will exit with the reason `:normal`). + If it is trapping exits, the exit signal is transformed into a message + `{:EXIT, from, :normal}` and delivered to its message queue. - 4. If reason is the atom `:kill`, that is if `exit(pid, :kill)` is called, - an untrappable exit signal is sent to pid which will unconditionally - exit with exit reason `:killed`. + If `reason` is the atom `:kill`, that is if `Process.exit(pid, :kill)` is called, + an untrappable exit signal is sent to `pid` which will unconditionally exit + with reason `:killed`. Inlined by the compiler. ## Examples Process.exit(pid, :kill) + #=> true """ @spec exit(pid, term) :: true - def exit(pid, reason) do - :erlang.exit(pid, reason) + defdelegate exit(pid, reason), to: :erlang + + @doc """ + Sleeps the current process for the given `timeout`. + + `timeout` is either the number of milliseconds to sleep as an + integer or the atom `:infinity`. When `:infinity` is given, + the current process will sleep forever, and not + consume or reply to messages. + + **Use this function with extreme care**. For almost all situations + where you would use `sleep/1` in Elixir, there is likely a + more correct, faster and precise way of achieving the same with + message passing. + + For example, if you are waiting for a process to perform some + action, it is better to communicate the progress of such action + with messages. + + In other words, **do not**: + + Task.start_link(fn -> + do_something() + ... + end) + + # Wait until work is done + Process.sleep(2000) + + But **do**: + + parent = self() + + Task.start_link(fn -> + do_something() + send(parent, :work_is_done) + ... + end) + + receive do + :work_is_done -> :ok + after + # Optional timeout + 30_000 -> :timeout + end + + For cases like the one above, `Task.async/1` and `Task.await/2` are + preferred. + + Similarly, if you are waiting for a process to terminate, + monitor that process instead of sleeping. **Do not**: + + Task.start_link(fn -> + ... + end) + + # Wait until task terminates + Process.sleep(2000) + + Instead **do**: + + {:ok, pid} = + Task.start_link(fn -> + ... + end) + + ref = Process.monitor(pid) + + receive do + {:DOWN, ^ref, _, _, _} -> :task_is_down + after + # Optional timeout + 30_000 -> :timeout + end + + """ + @spec sleep(timeout) :: :ok + def sleep(timeout) + when is_integer(timeout) and timeout >= 0 + when timeout == :infinity do + receive after: (timeout -> :ok) end @doc """ - Sends a message to the given process. + Sends a message to the given `dest`. - If the option `:noconnect` is used and sending the message would require an - auto-connection to another node the message is not sent and `:noconnect` is - returned. + `dest` may be a remote or local PID, a local port, a locally + registered name, or a tuple in the form of `{registered_name, node}` for a + registered name at another node. - If the option `:nosuspend` is used and sending the message would cause the - sender to be suspended the message is not sent and `:nosuspend` is returned. + Inlined by the compiler. + + ## Options + + * `:noconnect` - when used, if sending the message would require an + auto-connection to another node the message is not sent and `:noconnect` is + returned. + + * `:nosuspend` - when used, if sending the message would cause the sender to + be suspended the message is not sent and `:nosuspend` is returned. Otherwise the message is sent and `:ok` is returned. ## Examples - iex> Process.send({:name, :node_does_not_exist}, :hi, [:noconnect]) + iex> Process.send({:name, :node_that_does_not_exist}, :hi, [:noconnect]) :noconnect """ - @spec send(dest, msg, [option]) :: result when - dest: pid | port | atom | {atom, node}, - msg: any, - option: :noconnect | :nosuspend, - result: :ok | :noconnect | :nosuspend - def send(dest, msg, options) do - :erlang.send(dest, msg, options) - end + @spec send(dest, msg, [option]) :: :ok | :noconnect | :nosuspend + when dest: dest(), + msg: any, + option: :noconnect | :nosuspend + defdelegate send(dest, msg, options), to: :erlang @doc """ - Sends `msg` to `dest` after `time` millisecons. + Sends `msg` to `dest` after `time` milliseconds. - If `dest` is a pid, it has to be a pid of a local process, dead or alive. - If `dest` is an atom, it is supposed to be the name of a registered process - which is looked up at the time of delivery. No error is given if the name does + If `dest` is a PID, it must be the PID of a local process, dead or alive. + If `dest` is an atom, it must be the name of a registered process + which is looked up at the time of delivery. No error is produced if the name does not refer to a process. - This function returns a timer reference, which can be read or canceled with - `:erlang.read_timer/1`, `:erlang.start_timer/3` and `:erlang.cancel_timer/1`. - Note `time` cannot be greater than `4294967295`. + The message is not sent immediately. Therefore, `dest` can receive other messages + in-between even when `time` is `0`. + + This function returns a timer reference, which can be read with `read_timer/1` + or canceled with `cancel_timer/1`. - Finally, the timer will be automatically canceled if the given `dest` is a pid - which is not alive or when the given pid exits. Note that timers will not be + The timer will be automatically canceled if the given `dest` is a PID + which is not alive or when the given PID exits. Note that timers will not be automatically canceled when `dest` is an atom (as the atom resolution is done on delivery). + + Inlined by the compiler. + + ## Options + + * `:abs` - (boolean) when `false`, `time` is treated as relative to the + current monotonic time. When `true`, `time` is the absolute value of the + Erlang monotonic time at which `msg` should be delivered to `dest`. + To read more about Erlang monotonic time and other time-related concepts, + look at the documentation for the `System` module. Defaults to `false`. + + ## Examples + + timer_ref = Process.send_after(pid, :hi, 1000) + """ - @spec send_after(pid | atom, term, non_neg_integer) :: reference - def send_after(dest, msg, time) do - :erlang.send_after(time, dest, msg) + @spec send_after(pid | atom, term, non_neg_integer, [option]) :: reference + when option: {:abs, boolean} + def send_after(dest, msg, time, opts \\ []) do + :erlang.send_after(time, dest, msg, opts) end - @type spawn_opt :: :link | :monitor | {:priority, :low | :normal | :high} | - {:fullsweep_after, non_neg_integer} | - {:min_heap_size, non_neg_integer} | - {:min_bin_vheap_size, non_neg_integer} + @doc """ + Cancels a timer returned by `send_after/3`. + + When the result is an integer, it represents the time in milliseconds + left until the timer would have expired. + + When the result is `false`, a timer corresponding to `timer_ref` could not be + found. This can happen either because the timer expired, because it has + already been canceled, or because `timer_ref` never corresponded to a timer. + + Even if the timer had expired and the message was sent, this function does not + tell you if the timeout message has arrived at its destination yet. + + Inlined by the compiler. + + ## Options + + * `:async` - (boolean) when `false`, the request for cancellation is + synchronous. When `true`, the request for cancellation is asynchronous, + meaning that the request to cancel the timer is issued and `:ok` is + returned right away. Defaults to `false`. + + * `:info` - (boolean) whether to return information about the timer being + cancelled. When the `:async` option is `false` and `:info` is `true`, then + either an integer or `false` (like described above) is returned. If + `:async` is `false` and `:info` is `false`, `:ok` is returned. If `:async` + is `true` and `:info` is `true`, a message in the form `{:cancel_timer, + timer_ref, result}` (where `result` is an integer or `false` like + described above) is sent to the caller of this function when the + cancellation has been performed. If `:async` is `true` and `:info` is + `false`, no message is sent. Defaults to `true`. + + """ + @spec cancel_timer(reference, options) :: non_neg_integer | false | :ok + when options: [async: boolean, info: boolean] + defdelegate cancel_timer(timer_ref, options \\ []), to: :erlang + + @doc """ + Reads a timer created by `send_after/3`. + + When the result is an integer, it represents the time in milliseconds + left until the timer will expire. + + When the result is `false`, a timer corresponding to `timer_ref` could not be + found. This can be either because the timer expired, because it has already + been canceled, or because `timer_ref` never corresponded to a timer. + + Even if the timer had expired and the message was sent, this function does not + tell you if the timeout message has arrived at its destination yet. + + Inlined by the compiler. + """ + @spec read_timer(reference) :: non_neg_integer | false + defdelegate read_timer(timer_ref), to: :erlang + + @type spawn_opt :: + :link + | :monitor + | {:monitor, monitor_option()} + | {:priority, :low | :normal | :high} + | {:fullsweep_after, non_neg_integer} + | {:min_heap_size, non_neg_integer} + | {:min_bin_vheap_size, non_neg_integer} + | {:max_heap_size, heap_size} + | {:message_queue_data, :off_heap | :on_heap} @type spawn_opts :: [spawn_opt] + # TODO: Use :erlang.monitor_option() on Erlang/OTP 24+ + @typep monitor_option :: + [alias: :explicit_unalias | :demonitor | :reply_demonitor, tag: term()] + @doc """ - Spawns the given module and function passing the given args - according to the given options. + Spawns the given function according to the given options. The result depends on the given options. In particular, if `:monitor` is given as an option, it will return a tuple - containing the pid and the monitoring reference, otherwise - just the spawned process pid. + containing the PID and the monitoring reference, otherwise + just the spawned process PID. - It also accepts extra options, for the list of available options - check http://www.erlang.org/doc/man/erlang.html#spawn_opt-4 + More options are available; for the comprehensive list of available options + check `:erlang.spawn_opt/4`. Inlined by the compiler. + + ## Examples + + Process.spawn(fn -> 1 + 2 end, [:monitor]) + #=> {#PID<0.93.0>, #Reference<0.18808174.1939079169.202418>} + Process.spawn(fn -> 1 + 2 end, [:link]) + #=> #PID<0.95.0> + """ @spec spawn((() -> any), spawn_opts) :: pid | {pid, reference} - def spawn(fun, opts) do - :erlang.spawn_opt(fun, opts) - end + defdelegate spawn(fun, opts), to: :erlang, as: :spawn_opt @doc """ - Spawns the given module and function passing the given args + Spawns the given function `fun` from module `mod`, passing the given `args` according to the given options. The result depends on the given options. In particular, if `:monitor` is given as an option, it will return a tuple - containing the pid and the monitoring reference, otherwise - just the spawned process pid. + containing the PID and the monitoring reference, otherwise + just the spawned process PID. It also accepts extra options, for the list of available options - check http://www.erlang.org/doc/man/erlang.html#spawn_opt-4 + check `:erlang.spawn_opt/4`. Inlined by the compiler. """ @spec spawn(module, atom, list, spawn_opts) :: pid | {pid, reference} - def spawn(mod, fun, args, opts) do - :erlang.spawn_opt(mod, fun, args, opts) - end + defdelegate spawn(mod, fun, args, opts), to: :erlang, as: :spawn_opt @doc """ - The calling process starts monitoring the item given. - It returns the monitor reference. + Starts monitoring the given `item` from the calling process. + + Once the monitored process dies, a message is delivered to the + monitoring process in the shape of: + + {:DOWN, ref, :process, object, reason} + + where: + + * `ref` is a monitor reference returned by this function; + * `object` is either a `pid` of the monitored process (if monitoring + a PID) or `{name, node}` (if monitoring a remote or local name); + * `reason` is the exit reason. + + If the process is already dead when calling `Process.monitor/1`, a + `:DOWN` message is delivered immediately. - See http://www.erlang.org/doc/man/erlang.html#monitor-2 for more info. + See ["The need for monitoring"](https://elixir-lang.org/getting-started/mix-otp/genserver.html#the-need-for-monitoring) + for an example. See `:erlang.monitor/2` for more information. Inlined by the compiler. + + ## Examples + + pid = spawn(fn -> 1 + 2 end) + #=> #PID<0.118.0> + Process.monitor(pid) + #=> #Reference<0.906660723.3006791681.40191> + Process.exit(pid, :kill) + #=> true + receive do + msg -> msg + end + #=> {:DOWN, #Reference<0.906660723.3006791681.40191>, :process, #PID<0.118.0>, :noproc} + """ - @spec monitor(pid | {reg_name :: atom, node :: atom} | reg_name :: atom) :: reference + @spec monitor(pid | {name, node} | name) :: reference when name: atom def monitor(item) do :erlang.monitor(:process, item) end @doc """ - If monitor_ref is a reference which the calling process - obtained by calling monitor/1, this monitoring is turned off. + Demonitors the monitor identified by the given `reference`. + + If `monitor_ref` is a reference which the calling process + obtained by calling `monitor/1`, that monitoring is turned off. If the monitoring is already turned off, nothing happens. - See http://www.erlang.org/doc/man/erlang.html#demonitor-2 for more info. + See `:erlang.demonitor/2` for more information. Inlined by the compiler. + + ## Examples + + pid = spawn(fn -> 1 + 2 end) + ref = Process.monitor(pid) + Process.demonitor(ref) + #=> true + """ - @spec demonitor(reference) :: true @spec demonitor(reference, options :: [:flush | :info]) :: boolean - def demonitor(monitor_ref, options \\ []) do - :erlang.demonitor(monitor_ref, options) - end + defdelegate demonitor(monitor_ref, options \\ []), to: :erlang @doc """ - Returns a list of process identifiers corresponding to all the + Returns a list of PIDs corresponding to all the processes currently existing on the local node. - Note that a process that is exiting, exists but is not alive, i.e., - alive?/1 will return false for a process that is exiting, - but its process identifier will be part of the result returned. + Note that if a process is exiting, it is considered to exist but not be + alive. This means that for such process, `alive?/1` will return `false` but + its PID will be part of the list of PIDs returned by this function. + + See `:erlang.processes/0` for more information. + + Inlined by the compiler. + + ## Examples + + Process.list() + #=> [#PID<0.0.0>, #PID<0.1.0>, #PID<0.2.0>, #PID<0.3.0>, ...] - See http://www.erlang.org/doc/man/erlang.html#processes-0 for more info. """ - @spec list :: [pid] - def list do - :erlang.processes() - end + @spec list() :: [pid] + defdelegate list(), to: :erlang, as: :processes @doc """ - Creates a link between the calling process and another process - (or port) `pid`, if there is not such a link already. + Creates a link between the calling process and the given item (process or + port). - See http://www.erlang.org/doc/man/erlang.html#link-1 for more info. + Links are bidirectional. Linked processes can be unlinked by using `unlink/1`. + + If such a link exists already, this function does nothing since there can only + be one link between two given processes. If a process tries to create a link + to itself, nothing will happen. + + When two processes are linked, each one receives exit signals from the other + (see also `exit/2`). Let's assume `pid1` and `pid2` are linked. If `pid2` + exits with a reason other than `:normal` (which is also the exit reason used + when a process finishes its job) and `pid1` is not trapping exits (see + `flag/2`), then `pid1` will exit with the same reason as `pid2` and in turn + emit an exit signal to all its other linked processes. The behaviour when + `pid1` is trapping exits is described in `exit/2`. + + See `:erlang.link/1` for more information. Inlined by the compiler. """ @spec link(pid | port) :: true - def link(pid) do - :erlang.link(pid) - end + defdelegate link(pid_or_port), to: :erlang @doc """ - Removes the link, if there is one, between the calling process and - the process or port referred to by `pid`. Returns true and does not - fail, even if there is no link or `id` does not exist + Removes the link between the calling process and the given item (process or + port). + + If there is no such link, this function does nothing. If `pid_or_port` does + not exist, this function does not produce any errors and simply does nothing. + + The return value of this function is always `true`. - See http://www.erlang.org/doc/man/erlang.html#unlink-1 for more info. + See `:erlang.unlink/1` for more information. Inlined by the compiler. """ @spec unlink(pid | port) :: true - def unlink(pid) do - :erlang.unlink(pid) - end + defdelegate unlink(pid_or_port), to: :erlang @doc """ - Associates the name with a pid or a port identifier. name, which must - be an atom, can be used instead of the pid / port identifier with the - `Kernel.send/2` function. + Registers the given `pid_or_port` under the given `name`. + + `name` must be an atom and can then be used instead of the + PID/port identifier when sending messages with `Kernel.send/2`. + + `register/2` will fail with `ArgumentError` in any of the following cases: + + * the PID/Port is not existing locally and alive + * the name is already registered + * the `pid_or_port` is already registered under a different `name` + + The following names are reserved and cannot be assigned to + processes nor ports: + + * `nil` + * `false` + * `true` + * `:undefined` + + ## Examples + + Process.register(self(), :test) + #=> true + send(:test, :hello) + #=> :hello + send(:wrong_name, :hello) + ** (ArgumentError) argument error - `Process.register/2` will fail with `ArgumentError` if the pid supplied - is no longer alive, (check with `alive?/1`) or if the name is - already registered (check with `registered?/1`). """ @spec register(pid | port, atom) :: true - def register(pid, name) when not name in [nil, false, true] do - :erlang.register(name, pid) + def register(pid_or_port, name) + when is_atom(name) and name not in [nil, false, true, :undefined] do + :erlang.register(name, pid_or_port) + catch + :error, :badarg when node(pid_or_port) != node() -> + message = "could not register #{inspect(pid_or_port)} because it belongs to another node" + :erlang.error(ArgumentError.exception(message), [pid_or_port, name]) + + :error, :badarg -> + message = + "could not register #{inspect(pid_or_port)} with " <> + "name #{inspect(name)} because it is not alive, the name is already " <> + "taken, or it has already been given another name" + + :erlang.error(ArgumentError.exception(message), [pid_or_port, name]) end @doc """ - Removes the registered name, associated with a pid or a port identifier. + Removes the registered `name`, associated with a PID + or a port identifier. + + Fails with `ArgumentError` if the name is not registered + to any PID or port. + + Inlined by the compiler. + + ## Examples + + Process.register(self(), :test) + #=> true + Process.unregister(:test) + #=> true + Process.unregister(:wrong_name) + ** (ArgumentError) argument error - See http://www.erlang.org/doc/man/erlang.html#unregister-1 for more info. """ @spec unregister(atom) :: true - def unregister(name) do - :erlang.unregister(name) - end + defdelegate unregister(name), to: :erlang @doc """ - Returns the pid or port identifier with the registered name. - Returns nil if the name is not registered. + Returns the PID or port identifier registered under `name` or `nil` if the + name is not registered. + + See `:erlang.whereis/1` for more information. + + ## Examples + + Process.register(self(), :test) + Process.whereis(:test) + #=> #PID<0.84.0> + Process.whereis(:wrong_name) + #=> nil - See http://www.erlang.org/doc/man/erlang.html#whereis-1 for more info. """ @spec whereis(atom) :: pid | port | nil def whereis(name) do - nillify :erlang.whereis(name) + nilify(:erlang.whereis(name)) end @doc """ - Returns the pid of the group leader for the process which evaluates the function. + Returns the PID of the group leader for the calling process. + + Inlined by the compiler. + + ## Examples + + Process.group_leader() + #=> #PID<0.53.0> + """ - @spec group_leader :: pid - def group_leader do - :erlang.group_leader - end + @spec group_leader() :: pid + defdelegate group_leader(), to: :erlang @doc """ - Sets the group leader of `pid` to `leader`. Typically, this is used when a processes - started from a certain shell should have another group leader than `:init`. + Sets the group leader of the given `pid` to `leader`. + + Typically, this is used when a process started from a certain shell should + have a group leader other than `:init`. + + Inlined by the compiler. """ @spec group_leader(pid, leader :: pid) :: true def group_leader(pid, leader) do @@ -317,58 +681,84 @@ defmodule Process do end @doc """ - Returns a list of names which have been registered using register/2. + Returns a list of names which have been registered using `register/2`. + + Inlined by the compiler. + + ## Examples + + Process.register(self(), :test) + Process.registered() + #=> [:test, :elixir_config, :inet_db, ...] + """ - @spec registered :: [atom] - def registered do - :erlang.registered() - end + @spec registered() :: [atom] + defdelegate registered(), to: :erlang + + @typep heap_size :: + non_neg_integer + | %{size: non_neg_integer, kill: boolean, error_logger: boolean} + + @typep priority_level :: :low | :normal | :high | :max - @typep process_flag :: :trap_exit | :error_handler | :min_heap_size | - :min_bin_vheap_size | :priority | :save_calls | - :sensitive @doc """ - Sets certain flags for the process which calls this function. - Returns the old value of the flag. + Sets the given `flag` to `value` for the calling process. + + Returns the old value of `flag`. - See http://www.erlang.org/doc/man/erlang.html#process_flag-2 for more info. + See `:erlang.process_flag/2` for more information. + + Inlined by the compiler. """ - @spec flag(process_flag, term) :: term - def flag(flag, value) do - :erlang.process_flag(flag, value) - end + @spec flag(:error_handler, module) :: module + @spec flag(:max_heap_size, heap_size) :: heap_size + # :off_heap | :on_heap twice because :erlang.message_queue_data() is not exported + @spec flag(:message_queue_data, :off_heap | :on_heap) :: :off_heap | :on_heap + @spec flag(:min_bin_vheap_size, non_neg_integer) :: non_neg_integer + @spec flag(:min_heap_size, non_neg_integer) :: non_neg_integer + @spec flag(:priority, priority_level) :: priority_level + @spec flag(:save_calls, 0..10000) :: 0..10000 + @spec flag(:sensitive, boolean) :: boolean + @spec flag(:trap_exit, boolean) :: boolean + defdelegate flag(flag, value), to: :erlang, as: :process_flag @doc """ - Sets certain flags for the process Pid, in the same manner as flag/2. - Returns the old value of the flag. The allowed values for Flag are - only a subset of those allowed in flag/2, namely: save_calls. + Sets the given `flag` to `value` for the given process `pid`. + + Returns the old value of `flag`. + + It raises `ArgumentError` if `pid` is not a local process. + + The allowed values for `flag` are only a subset of those allowed in `flag/2`, + namely `:save_calls`. - See http://www.erlang.org/doc/man/erlang.html#process_flag-3 for more info. + See `:erlang.process_flag/3` for more information. + + Inlined by the compiler. """ - @spec flag(pid, process_flag, term) :: term - def flag(pid, flag, value) do - :erlang.process_flag(pid, flag, value) - end + @spec flag(pid, :save_calls, 0..10000) :: 0..10000 + defdelegate flag(pid, flag, value), to: :erlang, as: :process_flag @doc """ - Returns information about the process identified by pid or nil if the process + Returns information about the process identified by `pid`, or returns `nil` if the process is not alive. + Use this only for debugging information. - See http://www.erlang.org/doc/man/erlang.html#process_info-1 for more info. + See `:erlang.process_info/1` for more information. """ - @spec info(pid) :: Keyword.t + @spec info(pid) :: keyword | nil def info(pid) do - nillify :erlang.process_info(pid) + nilify(:erlang.process_info(pid)) end @doc """ - Returns information about the process identified by pid - or nil if the process is not alive. + Returns information about the process identified by `pid`, + or returns `nil` if the process is not alive. - See http://www.erlang.org/doc/man/erlang.html#process_info-2 for more info. + See `:erlang.process_info/2` for more information. """ - @spec info(pid, atom) :: {atom, term} | nil + @spec info(pid, atom | [atom]) :: {atom, term} | [{atom, term}] | nil def info(pid, spec) def info(pid, :registered_name) do @@ -379,11 +769,26 @@ defmodule Process do end end - def info(pid, spec) when is_atom(spec) do - nillify :erlang.process_info(pid, spec) + def info(pid, spec) when is_atom(spec) or is_list(spec) do + nilify(:erlang.process_info(pid, spec)) end - @compile {:inline, nillify: 1} - defp nillify(:undefined), do: nil - defp nillify(other), do: other + @doc """ + Puts the calling process into a "hibernation" state. + + The calling process is put into a waiting state + where its memory allocation has been reduced as much as possible, + which is useful if the process does not expect to receive any messages + in the near future. + + See `:erlang.hibernate/3` for more information. + + Inlined by the compiler. + """ + @spec hibernate(module, atom, list) :: no_return + defdelegate hibernate(mod, fun_name, args), to: :erlang + + @compile {:inline, nilify: 1} + defp nilify(:undefined), do: nil + defp nilify(other), do: other end diff --git a/lib/elixir/lib/protocol.ex b/lib/elixir/lib/protocol.ex index e29f31e1517..2522025f043 100644 --- a/lib/elixir/lib/protocol.ex +++ b/lib/elixir/lib/protocol.ex @@ -1,15 +1,267 @@ defmodule Protocol do - @moduledoc """ - Functions for working with protocols. - """ + @moduledoc ~S""" + Reference and functions for working with protocols. + + A protocol specifies an API that should be defined by its + implementations. A protocol is defined with `Kernel.defprotocol/2` + and its implementations with `Kernel.defimpl/3`. + + ## A real case + + In Elixir, we have two nouns for checking how many items there + are in a data structure: `length` and `size`. `length` means the + information must be computed. For example, `length(list)` needs to + traverse the whole list to calculate its length. On the other hand, + `tuple_size(tuple)` and `byte_size(binary)` do not depend on the + tuple and binary size as the size information is precomputed in + the data structure. + + Although Elixir includes specific functions such as `tuple_size`, + `binary_size` and `map_size`, sometimes we want to be able to + retrieve the size of a data structure regardless of its type. + In Elixir we can write polymorphic code, i.e. code that works + with different shapes/types, by using protocols. A size protocol + could be implemented as follows: + + defprotocol Size do + @doc "Calculates the size (and not the length!) of a data structure" + def size(data) + end - @doc """ - Defines a new protocol function. + Now that the protocol can be implemented for every data structure + the protocol may have a compliant implementation for: + + defimpl Size, for: BitString do + def size(binary), do: byte_size(binary) + end + + defimpl Size, for: Map do + def size(map), do: map_size(map) + end + + defimpl Size, for: Tuple do + def size(tuple), do: tuple_size(tuple) + end + + Note that we didn't implement it for lists as we don't have the + `size` information on lists, rather its value needs to be + computed with `length`. + + The data structure you are implementing the protocol for + must be the first argument to all functions defined in the + protocol. + + It is possible to implement protocols for all Elixir types: + + * Structs (see the "Protocols and Structs" section below) + * `Tuple` + * `Atom` + * `List` + * `BitString` + * `Integer` + * `Float` + * `Function` + * `PID` + * `Map` + * `Port` + * `Reference` + * `Any` (see the "Fallback to `Any`" section below) + + ## Protocols and Structs + + The real benefit of protocols comes when mixed with structs. + For instance, Elixir ships with many data types implemented as + structs, like `MapSet`. We can implement the `Size` protocol + for those types as well: + + defimpl Size, for: MapSet do + def size(map_set), do: MapSet.size(map_set) + end + + When implementing a protocol for a struct, the `:for` option can + be omitted if the `defimpl/3` call is inside the module that defines + the struct: + + defmodule User do + defstruct [:email, :name] + + defimpl Size do + # two fields + def size(%User{}), do: 2 + end + end + + If a protocol implementation is not found for a given type, + invoking the protocol will raise unless it is configured to + fall back to `Any`. Conveniences for building implementations + on top of existing ones are also available, look at `defstruct/1` + for more information about deriving + protocols. + + ## Fallback to `Any` + + In some cases, it may be convenient to provide a default + implementation for all types. This can be achieved by setting + the `@fallback_to_any` attribute to `true` in the protocol + definition: + + defprotocol Size do + @fallback_to_any true + def size(data) + end + + The `Size` protocol can now be implemented for `Any`: + + defimpl Size, for: Any do + def size(_), do: 0 + end + + Although the implementation above is arguably not a reasonable + one. For example, it makes no sense to say a PID or an integer + have a size of `0`. That's one of the reasons why `@fallback_to_any` + is an opt-in behaviour. For the majority of protocols, raising + an error when a protocol is not implemented is the proper behaviour. + + ## Multiple implementations + + Protocols can also be implemented for multiple types at once: + + defprotocol Reversible do + def reverse(term) + end + + defimpl Reversible, for: [Map, List] do + def reverse(term), do: Enum.reverse(term) + end + + Inside `defimpl/3`, you can use `@protocol` to access the protocol + being implemented and `@for` to access the module it is being + defined for. + + ## Types + + Defining a protocol automatically defines a zero-arity type named `t`, which + can be used as follows: + + @spec print_size(Size.t()) :: :ok + def print_size(data) do + result = + case Size.size(data) do + 0 -> "data has no items" + 1 -> "data has one item" + n -> "data has #{n} items" + end + + IO.puts(result) + end + + The `@spec` above expresses that all types allowed to implement the + given protocol are valid argument types for the given function. + + ## Reflection + + Any protocol module contains three extra functions: + + * `__protocol__/1` - returns the protocol information. The function takes + one of the following atoms: + + * `:consolidated?` - returns whether the protocol is consolidated + + * `:functions` - returns a keyword list of protocol functions and their arities + + * `:impls` - if consolidated, returns `{:consolidated, modules}` with the list of modules + implementing the protocol, otherwise `:not_consolidated` + + * `:module` - the protocol module atom name + + * `impl_for/1` - returns the module that implements the protocol for the given argument, + `nil` otherwise + + * `impl_for!/1` - same as above but raises `Protocol.UndefinedError` if an implementation is + not found - Protocols do not allow functions to be defined directly, instead, the - regular `Kernel.def/*` macros are replaced by this macro which - defines the protocol functions with the appropriate callbacks. + For example, for the `Enumerable` protocol we have: + + iex> Enumerable.__protocol__(:functions) + [count: 1, member?: 2, reduce: 3, slice: 1] + + iex> Enumerable.impl_for([]) + Enumerable.List + + iex> Enumerable.impl_for(42) + nil + + In addition, every protocol implementation module contains the `__impl__/1` + function. The function takes one of the following atoms: + + * `:for` - returns the module responsible for the data structure of the + protocol implementation + + * `:protocol` - returns the protocol module for which this implementation + is provided + + For example, the module implementing the `Enumerable` protocol for lists is + `Enumerable.List`. Therefore, we can invoke `__impl__/1` on this module: + + iex(1)> Enumerable.List.__impl__(:for) + List + + iex(2)> Enumerable.List.__impl__(:protocol) + Enumerable + + ## Consolidation + + In order to speed up protocol dispatching, whenever all protocol implementations + are known up-front, typically after all Elixir code in a project is compiled, + Elixir provides a feature called *protocol consolidation*. Consolidation directly + links protocols to their implementations in a way that invoking a function from a + consolidated protocol is equivalent to invoking two remote functions. + + Protocol consolidation is applied by default to all Mix projects during compilation. + This may be an issue during test. For instance, if you want to implement a protocol + during test, the implementation will have no effect, as the protocol has already been + consolidated. One possible solution is to include compilation directories that are + specific to your test environment in your mix.exs: + + def project do + ... + elixirc_paths: elixirc_paths(Mix.env()) + ... + end + + defp elixirc_paths(:test), do: ["lib", "test/support"] + defp elixirc_paths(_), do: ["lib"] + + And then you can define the implementations specific to the test environment + inside `test/support/some_file.ex`. + + Another approach is to disable protocol consolidation during tests in your + mix.exs: + + def project do + ... + consolidate_protocols: Mix.env() != :test + ... + end + + If you are using `Mix.install/2`, you can do by passing the `consolidate_protocols` + option: + + Mix.install( + deps, + consolidate_protocols: false + ) + + Although doing so is not recommended as it may affect the performance of + your code. + + Finally, note all protocols are compiled with `debug_info` set to `true`, + regardless of the option set by the `elixirc` compiler. The debug info is + used for consolidation and it is removed after consolidation unless + globally set. """ + + @doc false defmacro def(signature) defmacro def({_, _, args}) when args == [] or is_atom(args) do @@ -19,60 +271,63 @@ defmodule Protocol do defmacro def({name, _, args}) when is_atom(name) and is_list(args) do arity = length(args) - type_args = for _ <- :lists.seq(2, arity), do: quote(do: term) + type_args = :lists.map(fn _ -> quote(do: term) end, :lists.seq(2, arity)) type_args = [quote(do: t) | type_args] - call_args = for i <- :lists.seq(2, arity), - do: {String.to_atom(<>), [], __MODULE__} - call_args = [quote(do: t) | call_args] + varify = fn pos -> Macro.var(String.to_atom("arg" <> Integer.to_string(pos)), __MODULE__) end + + call_args = :lists.map(varify, :lists.seq(2, arity)) + call_args = [quote(do: term) | call_args] quote do - name = unquote(name) + name = unquote(name) arity = unquote(arity) - @functions [{name, arity}|@functions] + @__functions__ [{name, arity} | @__functions__] # Generate a fake definition with the user # signature that will be used by docs - Kernel.def unquote(name)(unquote_splicing(args)) + Kernel.def(unquote(name)(unquote_splicing(args))) # Generate the actual implementation Kernel.def unquote(name)(unquote_splicing(call_args)) do - impl_for!(t).unquote(name)(unquote_splicing(call_args)) + impl_for!(term).unquote(name)(unquote_splicing(call_args)) end - # Convert the spec to callback if possible, + # Copy spec as callback if possible, # otherwise generate a dummy callback - Protocol.__spec__?(__MODULE__, name, arity) || + Module.spec_to_callback(__MODULE__, {name, arity}) || @callback unquote(name)(unquote_splicing(type_args)) :: term end end defmacro def(_) do - raise ArgumentError, "invalid args for def inside defprotocol" + raise ArgumentError, "invalid arguments for def inside defprotocol" end @doc """ Checks if the given module is loaded and is protocol. - Returns `:ok` if so, otherwise raises ArgumentError. + Returns `:ok` if so, otherwise raises `ArgumentError`. """ - @spec assert_protocol!(module) :: :ok | no_return + @spec assert_protocol!(module) :: :ok def assert_protocol!(module) do assert_protocol!(module, "") end defp assert_protocol!(module, extra) do - case Code.ensure_compiled(module) do - {:module, ^module} -> :ok - _ -> raise ArgumentError, "#{inspect module} is not available" <> extra + try do + Code.ensure_compiled!(module) + rescue + e in ArgumentError -> + raise ArgumentError, e.message <> extra end try do - module.__protocol__(:name) + module.__protocol__(:module) rescue UndefinedFunctionError -> - raise ArgumentError, "#{inspect module} is not a protocol" <> extra + raise ArgumentError, "#{inspect(module)} is not a protocol" <> extra end :ok @@ -82,112 +337,163 @@ defmodule Protocol do Checks if the given module is loaded and is an implementation of the given protocol. - Returns `:ok` if so, otherwise raises ArgumentError. + Returns `:ok` if so, otherwise raises `ArgumentError`. """ - @spec assert_impl!(module, module) :: :ok | no_return - def assert_impl!(protocol, impl) do - assert_impl!(protocol, impl, "") + @spec assert_impl!(module, module) :: :ok + def assert_impl!(protocol, base) do + assert_impl!(protocol, base, "") end - defp assert_impl!(protocol, impl, extra) do - case Code.ensure_compiled(impl) do - {:module, ^impl} -> :ok - _ -> raise ArgumentError, - "#{inspect impl} is not available" <> extra + defp assert_impl!(protocol, base, extra) do + impl = Module.concat(protocol, base) + + try do + Code.ensure_compiled!(impl) + rescue + e in ArgumentError -> + raise ArgumentError, e.message <> extra end try do impl.__impl__(:protocol) rescue UndefinedFunctionError -> - raise ArgumentError, - "#{inspect impl} is not an implementation of a protocol" <> extra + raise ArgumentError, "#{inspect(impl)} is not an implementation of a protocol" <> extra else ^protocol -> :ok + other -> raise ArgumentError, - "expected #{inspect impl} to be an implementation of #{inspect protocol}, got: #{inspect other}" <> extra + "expected #{inspect(impl)} to be an implementation of #{inspect(protocol)}" <> + ", got: #{inspect(other)}" <> extra end end @doc """ - Derive the `protocol` for `module` with the given options. + Derives the `protocol` for `module` with the given options. + + If your implementation passes options or if you are generating + custom code based on the struct, you will also need to implement + a macro defined as `__deriving__(module, struct, options)` + to get the options that were passed. + + ## Examples + + defprotocol Derivable do + def ok(arg) + end + + defimpl Derivable, for: Any do + defmacro __deriving__(module, struct, options) do + quote do + defimpl Derivable, for: unquote(module) do + def ok(arg) do + {:ok, arg, unquote(Macro.escape(struct)), unquote(options)} + end + end + end + end + + def ok(arg) do + {:ok, arg} + end + end + + defmodule ImplStruct do + @derive [Derivable] + defstruct a: 0, b: 0 + end + + Derivable.ok(%ImplStruct{}) + #=> {:ok, %ImplStruct{a: 0, b: 0}, %ImplStruct{a: 0, b: 0}, []} + + Explicit derivations can now be called via `__deriving__/3`: + + # Explicitly derived via `__deriving__/3` + Derivable.ok(%ImplStruct{a: 1, b: 1}) + #=> {:ok, %ImplStruct{a: 1, b: 1}, %ImplStruct{a: 0, b: 0}, []} + + # Explicitly derived by API via `__deriving__/3` + require Protocol + Protocol.derive(Derivable, ImplStruct, :oops) + Derivable.ok(%ImplStruct{a: 1, b: 1}) + #=> {:ok, %ImplStruct{a: 1, b: 1}, %ImplStruct{a: 0, b: 0}, :oops} + """ defmacro derive(protocol, module, options \\ []) do quote do - module = unquote(module) - Protocol.__derive__([{unquote(protocol), unquote(options)}], module, __ENV__) + Protocol.__derive__([{unquote(protocol), unquote(options)}], unquote(module), __ENV__) end end ## Consolidation @doc """ - Extract all protocols from the given paths. + Extracts all protocols from the given paths. - The paths can be either a char list or a string. Internally - they are worked on as char lists, so passing them as lists + The paths can be either a charlist or a string. Internally + they are worked on as charlists, so passing them as lists avoid extra conversion. Does not load any of the protocols. ## Examples - # Get Elixir's ebin and retrieve all protocols + # Get Elixir's ebin directory path and retrieve all protocols iex> path = :code.lib_dir(:elixir, :ebin) iex> mods = Protocol.extract_protocols([path]) iex> Enumerable in mods true """ - @spec extract_protocols([char_list | String.t]) :: [atom] + @spec extract_protocols([charlist | String.t()]) :: [atom] def extract_protocols(paths) do - extract_matching_by_attribute paths, 'Elixir.', - fn module, attributes -> - case attributes[:protocol] do - [fallback_to_any: _, consolidated: _] -> module - _ -> nil - end + extract_matching_by_attribute(paths, 'Elixir.', fn module, attributes -> + case attributes[:__protocol__] do + [fallback_to_any: _] -> module + _ -> nil end + end) end @doc """ - Extract all types implemented for the given protocol from + Extracts all types implemented for the given protocol from the given paths. - The paths can be either a char list or a string. Internally - they are worked on as char lists, so passing them as lists + The paths can be either a charlist or a string. Internally + they are worked on as charlists, so passing them as lists avoid extra conversion. Does not load any of the implementations. ## Examples - # Get Elixir's ebin and retrieve all protocols + # Get Elixir's ebin directory path and retrieve all protocols iex> path = :code.lib_dir(:elixir, :ebin) iex> mods = Protocol.extract_impls(Enumerable, [path]) iex> List in mods true """ - @spec extract_impls(module, [char_list | String.t]) :: [atom] + @spec extract_impls(module, [charlist | String.t()]) :: [atom] def extract_impls(protocol, paths) when is_atom(protocol) do - prefix = Atom.to_char_list(protocol) ++ '.' - extract_matching_by_attribute paths, prefix, fn - _mod, attributes -> - case attributes[:impl] do - [protocol: ^protocol, for: for] -> for - _ -> nil - end - end + prefix = Atom.to_charlist(protocol) ++ '.' + + extract_matching_by_attribute(paths, prefix, fn _mod, attributes -> + case attributes[:__impl__] do + [protocol: ^protocol, for: for] -> for + _ -> nil + end + end) end defp extract_matching_by_attribute(paths, prefix, callback) do for path <- paths, - file <- list_dir(path), - mod = extract_from_file(path, file, prefix, callback), - do: mod + path = to_charlist(path), + file <- list_dir(path), + mod = extract_from_file(path, file, prefix, callback), + do: mod end defp list_dir(path) when is_list(path) do @@ -197,8 +503,6 @@ defmodule Protocol do end end - defp list_dir(path), do: list_dir(to_char_list(path)) - defp extract_from_file(path, file, prefix, callback) do if :lists.prefix(prefix, file) and :filename.extension(file) == '.beam' do extract_from_beam(:filename.join(path, file), callback) @@ -209,26 +513,18 @@ defmodule Protocol do case :beam_lib.chunks(file, [:attributes]) do {:ok, {module, [attributes: attributes]}} -> callback.(module, attributes) - _ -> - nil - end - end - defmacrop if_ok(expr, call) do - quote do - case unquote(expr) do - {:ok, var} -> unquote(Macro.pipe(quote(do: var), call, 0)) - other -> other - end + _ -> + nil end end @doc """ - Returns true if the protocol was consolidated. + Returns `true` if the protocol was consolidated. """ @spec consolidated?(module) :: boolean def consolidated?(protocol) do - protocol.__info__(:attributes)[:protocol][:consolidated] + protocol.__protocol__(:consolidated?) end @doc """ @@ -241,45 +537,52 @@ defmodule Protocol do are retrieved with the help of `extract_impls/2`. It returns the updated version of the protocol bytecode. + If the first element of the tuple is `:ok`, it means + the protocol was consolidated. + A given bytecode or protocol implementation can be checked to be consolidated or not by analyzing the protocol attribute: Protocol.consolidated?(Enumerable) - If the first element of the tuple is true, it means - the protocol was consolidated. - This function does not load the protocol at any point nor loads the new bytecode for the compiled module. However each implementation must be available and it will be loaded. """ @spec consolidate(module, [module]) :: - {:ok, binary} | - {:error, :not_a_protocol} | - {:error, :no_beam_info} + {:ok, binary} + | {:error, :not_a_protocol} + | {:error, :no_beam_info} def consolidate(protocol, types) when is_atom(protocol) do - beam_protocol(protocol) - |> if_ok(change_debug_info types) - |> if_ok(compile) - end + # Ensure the types are sorted so the compiled beam is deterministic + types = Enum.sort(types) - @docs_chunk 'ExDc' + with {:ok, ast_info, specs, compile_info} <- beam_protocol(protocol), + {:ok, definitions} <- change_debug_info(protocol, ast_info, types), + do: compile(definitions, specs, compile_info) + end defp beam_protocol(protocol) do - chunk_ids = [:abstract_code, :attributes, @docs_chunk] + chunk_ids = [:debug_info, 'Docs', 'ExCk'] opts = [:allow_missing_chunks] + case :beam_lib.chunks(beam_file(protocol), chunk_ids, opts) do - {:ok, {^protocol, [{:abstract_code, {_raw, abstract_code}}, - {:attributes, attributes}, - {@docs_chunk, docs}]}} -> - case attributes[:protocol] do - [fallback_to_any: any, consolidated: _] -> - {:ok, {protocol, any, abstract_code, docs}} + {:ok, {^protocol, [{:debug_info, debug_info} | chunks]}} -> + {:debug_info_v1, _backend, {:elixir_v1, info, specs}} = debug_info + %{attributes: attributes, definitions: definitions} = info + chunks = :lists.filter(fn {_name, value} -> value != :missing_chunk end, chunks) + chunks = :lists.map(fn {name, value} -> {List.to_string(name), value} end, chunks) + + case attributes[:__protocol__] do + [fallback_to_any: any] -> + {:ok, {any, definitions}, specs, {info, chunks}} + _ -> {:error, :not_a_protocol} end + _ -> {:error, :no_beam_info} end @@ -287,96 +590,90 @@ defmodule Protocol do defp beam_file(module) when is_atom(module) do case :code.which(module) do - :non_existing -> module - file -> file + [_ | _] = file -> file + _ -> module end end # Change the debug information to the optimized # impl_for/1 dispatch version. - defp change_debug_info({protocol, any, code, docs}, types) do - types = if any, do: types, else: List.delete(types, Any) - all = [Any] ++ for {_guard, mod} <- builtin, do: mod + defp change_debug_info(protocol, {any, definitions}, types) do + types = if any, do: types, else: List.delete(types, Any) + all = [Any] ++ for {_guard, mod} <- __built_in__(), do: mod structs = types -- all - case change_impl_for(code, protocol, types, structs, false, []) do - {:ok, ret} -> {:ok, {ret, docs}} - other -> other - end - end - - defp change_impl_for([{:attribute, line, :protocol, opts}|t], protocol, types, structs, _, acc) do - opts = [fallback_to_any: opts[:fallback_to_any], consolidated: true] - change_impl_for(t, protocol, types, structs, true, - [{:attribute, line, :protocol, opts}|acc]) - end - defp change_impl_for([{:function, line, :impl_for, 1, _}|t], protocol, types, structs, is_protocol, acc) do - fallback = if Any in types, do: load_impl(protocol, Any), else: nil + case List.keytake(definitions, {:__protocol__, 1}, 0) do + {protocol_def, definitions} -> + {impl_for, definitions} = List.keytake(definitions, {:impl_for, 1}, 0) + {struct_impl_for, definitions} = List.keytake(definitions, {:struct_impl_for, 1}, 0) - clauses = for {guard, mod} <- builtin, - mod in types, - do: builtin_clause_for(mod, guard, protocol, line) + protocol_def = change_protocol(protocol_def, types) + impl_for = change_impl_for(impl_for, protocol, types) + struct_impl_for = change_struct_impl_for(struct_impl_for, protocol, types, structs) - clauses = [struct_clause_for(line)|clauses] ++ - [fallback_clause_for(fallback, protocol, line)] + {:ok, [protocol_def, impl_for, struct_impl_for] ++ definitions} - change_impl_for(t, protocol, types, structs, is_protocol, - [{:function, line, :impl_for, 1, clauses}|acc]) + nil -> + {:error, :not_a_protocol} + end end - defp change_impl_for([{:function, line, :struct_impl_for, 1, _}|t], protocol, types, structs, is_protocol, acc) do - fallback = if Any in types, do: load_impl(protocol, Any), else: nil - clauses = for struct <- structs, do: each_struct_clause_for(struct, protocol, line) - clauses = clauses ++ [fallback_clause_for(fallback, protocol, line)] + defp change_protocol({_name, _kind, meta, clauses}, types) do + clauses = + Enum.map(clauses, fn + {meta, [:consolidated?], [], _} -> {meta, [:consolidated?], [], true} + {meta, [:impls], [], _} -> {meta, [:impls], [], {:consolidated, types}} + clause -> clause + end) - change_impl_for(t, protocol, types, structs, is_protocol, - [{:function, line, :struct_impl_for, 1, clauses}|acc]) + {{:__protocol__, 1}, :def, meta, clauses} end - defp change_impl_for([h|t], protocol, info, types, is_protocol, acc) do - change_impl_for(t, protocol, info, types, is_protocol, [h|acc]) + defp change_impl_for({_name, _kind, meta, _clauses}, protocol, types) do + fallback = if Any in types, do: load_impl(protocol, Any) + line = meta[:line] + + clauses = + for {guard, mod} <- __built_in__(), + mod in types, + do: built_in_clause_for(mod, guard, protocol, meta, line) + + struct_clause = struct_clause_for(meta, line) + fallback_clause = fallback_clause_for(fallback, protocol, meta) + clauses = [struct_clause] ++ clauses ++ [fallback_clause] + + {{:impl_for, 1}, :def, meta, clauses} end - defp change_impl_for([], protocol, _info, _types, is_protocol, acc) do - if is_protocol do - {:ok, {protocol, Enum.reverse(acc)}} - else - {:error, :not_a_protocol} - end + defp change_struct_impl_for({_name, _kind, meta, _clauses}, protocol, types, structs) do + fallback = if Any in types, do: load_impl(protocol, Any) + clauses = for struct <- structs, do: each_struct_clause_for(struct, protocol, meta) + clauses = clauses ++ [fallback_clause_for(fallback, protocol, meta)] + + {{:struct_impl_for, 1}, :defp, meta, clauses} end - defp builtin_clause_for(mod, guard, protocol, line) do - {:clause, line, - [{:var, line, :x}], - [[{:call, line, - {:remote, line, {:atom, line, :erlang}, {:atom, line, guard}}, - [{:var, line, :x}], - }]], - [{:atom, line, load_impl(protocol, mod)}]} + defp built_in_clause_for(mod, guard, protocol, meta, line) do + x = {:x, [line: line, version: -1], __MODULE__} + guard = quote(line: line, do: :erlang.unquote(guard)(unquote(x))) + body = load_impl(protocol, mod) + {meta, [x], [guard], body} end - defp struct_clause_for(line) do - {:clause, line, - [{:map, line, [ - {:map_field_exact, line, {:atom, line, :__struct__}, {:var, line, :x}} - ]}], - [[{:call, line, - {:remote, line, {:atom, line, :erlang}, {:atom, line, :is_atom}}, - [{:var, line, :x}], - }]], - [{:call, line, - {:atom, line, :struct_impl_for}, - [{:var, line, :x}]}]} + defp struct_clause_for(meta, line) do + x = {:x, [line: line, version: -1], __MODULE__} + head = quote(line: line, do: %{__struct__: unquote(x)}) + guard = quote(line: line, do: :erlang.is_atom(unquote(x))) + body = quote(line: line, do: struct_impl_for(unquote(x))) + {meta, [head], [guard], body} end - defp each_struct_clause_for(other, protocol, line) do - {:clause, line, [{:atom, line, other}], [], - [{:atom, line, load_impl(protocol, other)}]} + defp each_struct_clause_for(struct, protocol, meta) do + {meta, [struct], [], load_impl(protocol, struct)} end - defp fallback_clause_for(value, _protocol, line) do - {:clause, line, [{:var, line, :_}], [], - [{:atom, line, value}]} + defp fallback_clause_for(value, _protocol, meta) do + {meta, [quote(do: _)], [], value} end defp load_impl(protocol, for) do @@ -384,26 +681,34 @@ defmodule Protocol do end # Finally compile the module and emit its bytecode. - defp compile({{protocol, code}, docs}) do - opts = if Code.compiler_options[:debug_info], do: [:debug_info], else: [] - {:ok, ^protocol, binary, _warnings} = :compile.forms(code, [:return|opts]) - unless docs == :missing_chunk do - binary = :elixir_module.add_beam_chunk(binary, @docs_chunk, docs) - end - {:ok, binary} + defp compile(definitions, specs, {info, chunks}) do + info = %{info | definitions: definitions} + {:ok, :elixir_erl.consolidate(info, specs, chunks)} end ## Definition callbacks @doc false - def __protocol__(name, [do: block]) do + def __protocol__(name, do: block) do quote do defmodule unquote(name) do + @before_compile Protocol + # We don't allow function definition inside protocols - import Kernel, except: [ - defmacrop: 1, defmacrop: 2, defmacro: 1, defmacro: 2, - defp: 1, defp: 2, def: 1, def: 2 - ] + import Kernel, + except: [ + def: 1, + def: 2, + defp: 1, + defp: 2, + defdelegate: 2, + defguard: 1, + defguardp: 1, + defmacro: 1, + defmacro: 2, + defmacrop: 1, + defmacrop: 2 + ] # Import the new dsl that holds the new def import Protocol, only: [def: 1] @@ -412,207 +717,339 @@ defmodule Protocol do @compile :debug_info # Set up a clear slate to store defined functions - @functions [] + @__functions__ [] @fallback_to_any false # Invoke the user given block - unquote(block) + _ = unquote(block) # Finalize expansion - unquote(after_defprotocol) + unquote(after_defprotocol()) end end end + defp callback_ast_to_fa({kind, {:"::", meta, [{name, _, args}, _return]}, _pos}) + when kind in [:callback, :macrocallback] do + [{{name, length(List.wrap(args))}, meta}] + end + + defp callback_ast_to_fa( + {kind, {:when, _, [{:"::", meta, [{name, _, args}, _return]}, _vars]}, _pos} + ) + when kind in [:callback, :macrocallback] do + [{{name, length(List.wrap(args))}, meta}] + end + + defp callback_ast_to_fa({kind, _, _pos}) when kind in [:callback, :macrocallback] do + [] + end + + defp callback_metas(module, kind) + when kind in [:callback, :macrocallback] do + :lists.flatmap(&callback_ast_to_fa/1, Module.get_attribute(module, kind)) + |> :maps.from_list() + end + + defp get_callback_line(fa, metas), + do: :maps.get(fa, metas, [])[:line] + + defp warn(message, env, nil) do + IO.warn(message, env) + end + + defp warn(message, env, line) when is_integer(line) do + stacktrace = :maps.update(:line, line, env) |> Macro.Env.stacktrace() + IO.warn(message, stacktrace) + end + + def __before_compile__(env) do + callback_metas = callback_metas(env.module, :callback) + callbacks = :maps.keys(callback_metas) + functions = Module.get_attribute(env.module, :__functions__) + + if functions == [] do + warn( + "protocols must define at least one function, but none was defined", + env, + nil + ) + end + + # TODO: Convert the following warnings into errors in future Elixir versions + :lists.map( + fn {name, arity} = fa -> + warn( + "cannot define @callback #{name}/#{arity} inside protocol, use def/1 to outline your protocol definition", + env, + get_callback_line(fa, callback_metas) + ) + end, + callbacks -- functions + ) + + # Macro Callbacks + macrocallback_metas = callback_metas(env.module, :macrocallback) + macrocallbacks = :maps.keys(macrocallback_metas) + + :lists.map( + fn {name, arity} = fa -> + warn( + "cannot define @macrocallback #{name}/#{arity} inside protocol, use def/1 to outline your protocol definition", + env, + get_callback_line(fa, macrocallback_metas) + ) + end, + macrocallbacks + ) + + # Optional Callbacks + optional_callbacks = Module.get_attribute(env.module, :optional_callbacks) + + if optional_callbacks != [] do + warn( + "cannot define @optional_callbacks inside protocol, all of the protocol definitions are required", + env, + nil + ) + end + end + defp after_defprotocol do - quote bind_quoted: [builtin: builtin] do - @spec impl_for(term) :: module | nil - Kernel.def impl_for(data) + quote bind_quoted: [built_in: __built_in__()] do + any_impl_for = + if @fallback_to_any do + quote do: unquote(__MODULE__.Any).__impl__(:target) + else + nil + end + + # Disable Dialyzer checks - before and after consolidation + # the types could be more strict + @dialyzer {:nowarn_function, __protocol__: 1, impl_for: 1, impl_for!: 1} + + @doc false + @spec impl_for(term) :: atom | nil + Kernel.def(impl_for(data)) # Define the implementation for structs. # # It simply delegates to struct_impl_for which is then # optimized during protocol consolidation. - Kernel.def impl_for(%{__struct__: struct}) when :erlang.is_atom(struct) do + Kernel.def impl_for(%struct{}) do struct_impl_for(struct) end - # Define the implementation for builtins. - for {guard, mod} <- builtin do - target = Module.concat(__MODULE__, mod) - - Kernel.def impl_for(data) when :erlang.unquote(guard)(data) do - case impl_for?(unquote(target)) do - true -> unquote(target).__impl__(:target) - false -> any_impl_for + # Define the implementation for built-ins + :lists.foreach( + fn {guard, mod} -> + target = Module.concat(__MODULE__, mod) + + Kernel.def impl_for(data) when :erlang.unquote(guard)(data) do + try do + unquote(target).__impl__(:target) + rescue + UndefinedFunctionError -> + unquote(any_impl_for) + end end - end - end - - @spec impl_for!(term) :: module | no_return - Kernel.def impl_for!(data) do - impl_for(data) || raise(Protocol.UndefinedError, protocol: __MODULE__, value: data) + end, + built_in + ) + + # Define a catch-all impl_for/1 clause to pacify Dialyzer (since + # destructuring opaque types is illegal, Dialyzer will think none of the + # previous clauses matches opaque types, and without this clause, will + # conclude that impl_for can't handle an opaque argument). This is a hack + # since it relies on Dialyzer not being smart enough to conclude that all + # opaque types will get the any_impl_for/0 implementation. + Kernel.def impl_for(_) do + unquote(any_impl_for) end - # Internal handler for Any - if @fallback_to_any do - Kernel.defp any_impl_for do - case impl_for?(__MODULE__.Any) do - true -> __MODULE__.Any.__impl__(:target) - false -> nil - end + @doc false + @spec impl_for!(term) :: atom + if any_impl_for do + Kernel.def impl_for!(data) do + impl_for(data) end else - Kernel.defp any_impl_for, do: nil + Kernel.def impl_for!(data) do + impl_for(data) || raise(Protocol.UndefinedError, protocol: __MODULE__, value: data) + end end # Internal handler for Structs Kernel.defp struct_impl_for(struct) do target = Module.concat(__MODULE__, struct) - case impl_for?(target) do - true -> target.__impl__(:target) - false -> any_impl_for - end - end - # Check if compilation is available internally - Kernel.defp impl_for?(target) do - Code.ensure_compiled?(target) and - function_exported?(target, :__impl__, 1) + try do + target.__impl__(:target) + rescue + UndefinedFunctionError -> + unquote(any_impl_for) + end end - # Inline any and struct implementations - @compile {:inline, any_impl_for: 0, struct_impl_for: 1, impl_for?: 1} + # Inline struct implementation for performance + @compile {:inline, struct_impl_for: 1} - unless Kernel.Typespec.defines_type?(__MODULE__, :t, 0) do + unless Module.defines_type?(__MODULE__, {:t, 0}) do @type t :: term end # Store information as an attribute so it # can be read without loading the module. - Module.register_attribute(__MODULE__, :protocol, persist: true) - @protocol [fallback_to_any: !!@fallback_to_any, consolidated: false] + Module.register_attribute(__MODULE__, :__protocol__, persist: true) + @__protocol__ [fallback_to_any: !!@fallback_to_any] @doc false - @spec __protocol__(atom) :: term - Kernel.def __protocol__(:name), do: __MODULE__ - Kernel.def __protocol__(:functions), do: unquote(:lists.sort(@functions)) + @spec __protocol__(:module) :: __MODULE__ + @spec __protocol__(:functions) :: unquote(Protocol.__functions_spec__(@__functions__)) + @spec __protocol__(:consolidated?) :: boolean + @spec __protocol__(:impls) :: :not_consolidated | {:consolidated, [module]} + Kernel.def(__protocol__(:module), do: __MODULE__) + Kernel.def(__protocol__(:functions), do: unquote(:lists.sort(@__functions__))) + Kernel.def(__protocol__(:consolidated?), do: false) + Kernel.def(__protocol__(:impls), do: :not_consolidated) end end + @doc false + def __functions_spec__([]), do: [] + + def __functions_spec__([head | tail]), + do: [:lists.foldl(&{:|, [], [&1, &2]}, head, tail), quote(do: ...)] + @doc false def __impl__(protocol, opts) do do_defimpl(protocol, :lists.keysort(1, opts)) end - defp do_defimpl(protocol, [do: block, for: for]) when is_list(for) do - for f <- for, do: do_defimpl(protocol, [do: block, for: f]) + defp do_defimpl(protocol, do: block, for: for) when is_list(for) do + for f <- for, do: do_defimpl(protocol, do: block, for: f) end - defp do_defimpl(protocol, [do: block, for: for]) do + defp do_defimpl(protocol, do: block, for: for) do + # Unquote the implementation just later + # when all variables will already be injected + # into the module body. + impl = + quote unquote: false do + @doc false + @spec __impl__(:for) :: unquote(for) + @spec __impl__(:target) :: __MODULE__ + @spec __impl__(:protocol) :: unquote(protocol) + def __impl__(:for), do: unquote(for) + def __impl__(:target), do: __MODULE__ + def __impl__(:protocol), do: unquote(protocol) + end + quote do protocol = unquote(protocol) - for = unquote(for) - name = Module.concat(protocol, for) + for = unquote(for) + name = Module.concat(protocol, for) Protocol.assert_protocol!(protocol) + Protocol.__ensure_defimpl__(protocol, for, __ENV__) defmodule name do @behaviour protocol - @protocol protocol - @for for + @protocol protocol + @for for unquote(block) - Module.register_attribute(__MODULE__, :impl, persist: true) - @impl [protocol: @protocol, for: @for] + Module.register_attribute(__MODULE__, :__impl__, persist: true) + @__impl__ [protocol: @protocol, for: @for] - @doc false - @spec __impl__(atom) :: term - def __impl__(:target), do: __MODULE__ - def __impl__(:protocol), do: @protocol - def __impl__(:for), do: @for + unquote(impl) end end end @doc false def __derive__(derives, for, %Macro.Env{} = env) when is_atom(for) do - struct = - if for == env.module do - Module.get_attribute(for, :struct) || - raise "struct is not defined for #{inspect for}" - else - for.__struct__ - end + struct = Macro.struct!(for, env) - :lists.foreach(fn + foreach = fn proto when is_atom(proto) -> derive(proto, for, struct, [], env) + {proto, opts} when is_atom(proto) -> derive(proto, for, struct, opts, env) - end, :lists.flatten(derives)) + end + + :lists.foreach(foreach, :lists.flatten(derives)) :ok end defp derive(protocol, for, struct, opts, env) do - impl = Module.concat(protocol, Map) - extra = ", cannot derive #{inspect protocol} for #{inspect for}" + extra = ", cannot derive #{inspect(protocol)} for #{inspect(for)}" assert_protocol!(protocol, extra) - assert_impl!(protocol, impl, extra) + __ensure_defimpl__(protocol, for, env) + assert_impl!(protocol, Any, extra) # Clean up variables from eval context - env = %{env | vars: [], export_vars: nil} + env = :elixir_env.reset_vars(env) args = [for, struct, opts] + impl = Module.concat(protocol, Any) - :elixir_module.expand_callback(env.line, impl, :__deriving__, args, env, fn - mod, fun, args -> - if function_exported?(mod, fun, length(args)) do - apply(mod, fun, args) - else - Module.create(Module.concat(protocol, for), quote do - Module.register_attribute(__MODULE__, :impl, persist: true) - @impl [protocol: unquote(protocol), for: unquote(for)] + :elixir_module.expand_callback(env.line, impl, :__deriving__, args, env, fn mod, fun, args -> + if function_exported?(mod, fun, length(args)) do + apply(mod, fun, args) + else + quoted = + quote do + Module.register_attribute(__MODULE__, :__impl__, persist: true) + @__impl__ [protocol: unquote(protocol), for: unquote(for)] @doc false - @spec __impl__(atom) :: term - def __impl__(:target), do: unquote(impl) + @spec __impl__(:target) :: unquote(impl) + @spec __impl__(:protocol) :: unquote(protocol) + @spec __impl__(:for) :: unquote(for) + def __impl__(:target), do: unquote(impl) def __impl__(:protocol), do: unquote(protocol) - def __impl__(:for), do: unquote(for) - end, Macro.Env.location(env)) - end + def __impl__(:for), do: unquote(for) + end + + Module.create(Module.concat(protocol, for), quoted, Macro.Env.location(env)) + end end) end @doc false - def __spec__?(module, name, arity) do - signature = {name, arity} - specs = Module.get_attribute(module, :spec) - - found = - for {:spec, expr, caller} <- specs, - Kernel.Typespec.spec_to_signature(expr) == signature do - Kernel.Typespec.define_spec(:callback, expr, caller) - true - end + def __ensure_defimpl__(protocol, for, env) do + if Protocol.consolidated?(protocol) do + message = + "the #{inspect(protocol)} protocol has already been consolidated, an " <> + "implementation for #{inspect(for)} has no effect. If you want to " <> + "implement protocols after compilation or during tests, check the " <> + "\"Consolidation\" section in the Protocol module documentation" + + IO.warn(message, env) + end - found != [] + :ok end ## Helpers - defp builtin do - [is_tuple: Tuple, - is_atom: Atom, - is_list: List, - is_map: Map, - is_bitstring: BitString, - is_integer: Integer, - is_float: Float, - is_function: Function, - is_pid: PID, - is_port: Port, - is_reference: Reference] + @doc false + def __built_in__ do + [ + is_tuple: Tuple, + is_atom: Atom, + is_list: List, + is_map: Map, + is_bitstring: BitString, + is_integer: Integer, + is_float: Float, + is_function: Function, + is_pid: PID, + is_port: Port, + is_reference: Reference + ] end end diff --git a/lib/elixir/lib/range.ex b/lib/elixir/lib/range.ex index 55c09855305..3f745ab93cb 100644 --- a/lib/elixir/lib/range.ex +++ b/lib/elixir/lib/range.ex @@ -1,124 +1,449 @@ defmodule Range do @moduledoc """ - Defines a Range. + Ranges represent a sequence of zero, one or many, ascending + or descending integers with a common difference called step. - A Range are represented internally as a struct. However, - the most common form of creating and matching on ranges - is via the `../2` macro, auto-imported from Kernel: + The most common form of creating and matching on ranges is + via the [`first..last`](`../2`) and [`first..last//step`](`..///3`) + notations, auto-imported from `Kernel`: - iex> range = 1..3 - 1..3 - iex> first .. last = range + iex> 1 in 1..10 + true + iex> 5 in 1..10 + true + iex> 10 in 1..10 + true + + Ranges are always inclusive in Elixir. When a step is defined, + integers will only belong to the range if they match the step: + + iex> 5 in 1..10//2 + true + iex> 4 in 1..10//2 + false + + When defining a range without a step, the step will be + defined based on the first and last position of the + range, If `first >= last`, it will be an increasing range + with a step of 1. Otherwise, it is a decreasing range. + Note however implicit decreasing ranges are deprecated. + Therefore, if you need a decreasing range from `3` to `1`, + prefer to write `3..1//-1` instead. + + `../0` can also be used as a shortcut to create the range `0..-1//1`, + also known as the full-slice range: + + iex> .. + 0..-1//1 + + ## Use cases + + Ranges typically have two uses in Elixir: as a collection or + to represent a slice of another data structure. + + ### Ranges as collections + + Ranges in Elixir are enumerables and therefore can be used + with the `Enum` module: + + iex> Enum.to_list(1..3) + [1, 2, 3] + iex> Enum.to_list(3..1//-1) + [3, 2, 1] + iex> Enum.to_list(1..5//2) + [1, 3, 5] + + Ranges may also have a single element: + + iex> Enum.to_list(1..1) + [1] + iex> Enum.to_list(1..1//2) + [1] + + Or even no elements at all: + + iex> Enum.to_list(10..0//1) + [] + iex> Enum.to_list(0..10//-1) + [] + + The full-slice range, returned by `../0`, is an empty collection: + + iex> Enum.to_list(..) + [] + + ### Ranges as slices + + Ranges are also frequently used to slice collections. + You can slice strings or any enumerable: + + iex> String.slice("elixir", 1..4) + "lixi" + iex> Enum.slice([0, 1, 2, 3, 4, 5], 1..4) + [1, 2, 3, 4] + + In those cases, the first and last values of the range + are mapped to positions in the collections. + + If a negative number is given, it maps to a position + from the back: + + iex> String.slice("elixir", 1..-2//1) + "lixi" + iex> Enum.slice([0, 1, 2, 3, 4, 5], 1..-2//1) + [1, 2, 3, 4] + + The range `0..-1//1`, returned by `../0`, returns the + collection as is, which is why it is called the full-slice + range: + + iex> String.slice("elixir", ..) + "elixir" + iex> Enum.slice([0, 1, 2, 3, 4, 5], ..) + [0, 1, 2, 3, 4, 5] + + ## Definition + + An increasing range `first..last//step` is a range from `first` + to `last` increasing by `step` where `step` must be a positive + integer and all values `v` must be `first <= v and v <= last`. + Therefore, a range `10..0//1` is an empty range because there + is no value `v` that is `10 <= v and v <= 0`. + + Similarly, a decreasing range `first..last//step` is a range + from `first` to `last` decreasing by `step` where `step` must + be a negative integer and values `v` must be `first >= v and v >= last`. + Therefore, a range `0..10//-1` is an empty range because there + is no value `v` that is `0 >= v and v >= 10`. + + ## Representation + + Internally, ranges are represented as structs: + + iex> range = 1..9//2 + 1..9//2 + iex> first..last//step = range iex> first 1 iex> last - 3 + 9 + iex> step + 2 + iex> range.step + 2 + You can access the range fields (`first`, `last`, and `step`) + directly but you should not modify nor create ranges by hand. + Instead use the proper operators or `new/2` and `new/3`. + + Ranges implement the `Enumerable` protocol with memory + efficient versions of all `Enumerable` callbacks: + + iex> range = 1..10 + 1..10 + iex> Enum.reduce(range, 0, fn i, acc -> i * i + acc end) + 385 + iex> Enum.count(range) + 10 + iex> Enum.member?(range, 11) + false + iex> Enum.member?(range, 8) + true + + Such function calls are efficient memory-wise no matter the + size of the range. The implementation of the `Enumerable` + protocol uses logic based solely on the endpoints and does + not materialize the whole list of integers. """ - defstruct first: nil, last: nil + @enforce_keys [:first, :last, :step] + defstruct first: nil, last: nil, step: nil - @type t(first, last) :: %{__struct__: Range, first: first, last: last} + @type limit :: integer + @type step :: pos_integer | neg_integer + @type t :: %__MODULE__{first: limit, last: limit, step: step} + @type t(first, last) :: %__MODULE__{first: first, last: last, step: step} @doc """ Creates a new range. + + If `first` is less than `last`, the range will be increasing from + `first` to `last`. If `first` is equal to `last`, the range will contain + one element, which is the number itself. + + If `first` is greater than `last`, the range will be decreasing from `first` + to `last`, albeit this behaviour is deprecated. Therefore, it is advised to + explicitly list the step with `new/3`. + + ## Examples + + iex> Range.new(-100, 100) + -100..100 + """ + + @spec new(limit, limit) :: t + def new(first, last) when is_integer(first) and is_integer(last) do + # TODO: Deprecate inferring a range with a step of -1 on Elixir v1.17 + step = if first <= last, do: 1, else: -1 + %Range{first: first, last: last, step: step} + end + def new(first, last) do - %Range{first: first, last: last} + raise ArgumentError, + "ranges (first..last) expect both sides to be integers, " <> + "got: #{inspect(first)}..#{inspect(last)}" end @doc """ - Returns true if the given argument is a range. + Creates a new range with `step`. ## Examples - iex> Range.range?(1..3) - true - - iex> Range.range?(0) - false + iex> Range.new(-100, 100, 2) + -100..100//2 """ - def range?(%Range{}), do: true - def range?(_), do: false -end + @doc since: "1.12.0" + @spec new(limit, limit, step) :: t + def new(first, last, step) + when is_integer(first) and is_integer(last) and is_integer(step) and step != 0 do + %Range{first: first, last: last, step: step} + end + + def new(first, last, step) do + raise ArgumentError, + "ranges (first..last//step) expect both sides to be integers and the step to be a " <> + "non-zero integer, got: #{inspect(first)}..#{inspect(last)}//#{inspect(step)}" + end + + @doc """ + Returns the size of `range`. + + ## Examples + + iex> Range.size(1..10) + 10 + iex> Range.size(1..10//2) + 5 + iex> Range.size(1..10//3) + 4 + iex> Range.size(1..10//-1) + 0 + + iex> Range.size(10..1) + 10 + iex> Range.size(10..1//-1) + 10 + iex> Range.size(10..1//-2) + 5 + iex> Range.size(10..1//-3) + 4 + iex> Range.size(10..1//1) + 0 -defprotocol Range.Iterator do - @moduledoc """ - A protocol used for iterating range elements. """ + @doc since: "1.12.0" + def size(range) + def size(first..last//step) when step > 0 and first > last, do: 0 + def size(first..last//step) when step < 0 and first < last, do: 0 + def size(first..last//step), do: abs(div(last - first, step)) + 1 + + # TODO: Remove me on v2.0 + def size(%{__struct__: Range, first: first, last: last} = range) do + step = if first <= last, do: 1, else: -1 + size(Map.put(range, :step, step)) + end @doc """ - Returns the function that calculates the next item. + Shifts a range by the given number of steps. + + ## Examples + + iex> Range.shift(0..10, 1) + 1..11 + iex> Range.shift(0..10, 2) + 2..12 + + iex> Range.shift(0..10//2, 2) + 4..14//2 """ - def next(first, range) + @doc since: "1.14.0" + def shift(first..last//step, steps_to_shift) + when is_integer(first) and is_integer(last) and is_integer(step) and + is_integer(steps_to_shift) do + new(first + steps_to_shift * step, last + steps_to_shift * step, step) + end @doc """ - Count how many items are in the range. + Checks if two ranges are disjoint. + + ## Examples + + iex> Range.disjoint?(1..5, 6..9) + true + iex> Range.disjoint?(5..1, 6..9) + true + iex> Range.disjoint?(1..5, 5..9) + false + iex> Range.disjoint?(1..5, 2..7) + false + + Steps are also considered when computing the ranges to be disjoint: + + iex> Range.disjoint?(1..10//2, 2..10//2) + true + + # First element in common in all below is 29 + iex> Range.disjoint?(2..100//3, 9..100//5) + false + iex> Range.disjoint?(101..2//-3, 99..9//-5) + false + iex> Range.disjoint?(1..100//14, 8..100//21) + false + iex> Range.disjoint?(57..-1//-14, 8..100//21) + false + iex> Range.disjoint?(1..100//14, 51..8//-21) + false + + # If 29 is out of range + iex> Range.disjoint?(1..28//14, 8..28//21) + true + iex> Range.disjoint?(2..28//3, 9..28//5) + true + """ - def count(first, range) + @doc since: "1.8.0" + @spec disjoint?(t, t) :: boolean + def disjoint?(first1..last1//step1 = range1, first2..last2//step2 = range2) do + if size(range1) == 0 or size(range2) == 0 do + true + else + {first1, last1, step1} = normalize(first1, last1, step1) + {first2, last2, step2} = normalize(first2, last2, step2) + + cond do + last2 < first1 or last1 < first2 -> + true + + abs(step1) == 1 and abs(step2) == 1 -> + false + + true -> + # We need to find the first intersection of two arithmetic + # progressions and see if they belong within the ranges + # https://math.stackexchange.com/questions/1656120/formula-to-find-the-first-intersection-of-two-arithmetic-progressions + {gcd, u, v} = Integer.extended_gcd(-step1, step2) + c = first1 - first2 + step2 - step1 + t1 = -c / step1 * u + t2 = -c / step2 * v + t = max(floor(t1) + 1, floor(t2) + 1) + x = div(c * u + t * step2, gcd) - 1 + y = div(c * v + t * step1, gcd) - 1 + + x < 0 or first1 + x * step1 > last1 or + y < 0 or first2 + y * step2 > last2 + end + end + end + + @compile inline: [normalize: 3] + defp normalize(first, last, step) when first > last, do: {last, first, -step} + defp normalize(first, last, step), do: {first, last, step} + + @doc false + @deprecated "Pattern match on first..last//step instead" + def range?(term) + def range?(first..last) when is_integer(first) and is_integer(last), do: true + def range?(_), do: false end defimpl Enumerable, for: Range do - def reduce(first .. last = range, acc, fun) do - reduce(first, last, acc, fun, Range.Iterator.next(first, range), last >= first) + def reduce(first..last//step, acc, fun) do + reduce(first, last, acc, fun, step) end - defp reduce(_x, _y, {:halt, acc}, _fun, _next, _up) do - {:halted, acc} + # TODO: Remove me on v2.0 + def reduce(%{__struct__: Range, first: first, last: last} = range, acc, fun) do + step = if first <= last, do: 1, else: -1 + reduce(Map.put(range, :step, step), acc, fun) end - defp reduce(x, y, {:suspend, acc}, fun, next, up) do - {:suspended, acc, &reduce(x, y, &1, fun, next, up)} + defp reduce(_first, _last, {:halt, acc}, _fun, _step) do + {:halted, acc} end - defp reduce(x, y, {:cont, acc}, fun, next, true) when x <= y do - reduce(next.(x), y, fun.(x, acc), fun, next, true) + defp reduce(first, last, {:suspend, acc}, fun, step) do + {:suspended, acc, &reduce(first, last, &1, fun, step)} end - defp reduce(x, y, {:cont, acc}, fun, next, false) when x >= y do - reduce(next.(x), y, fun.(x, acc), fun, next, false) + defp reduce(first, last, {:cont, acc}, fun, step) + when step > 0 and first <= last + when step < 0 and first >= last do + reduce(first + step, last, fun.(first, acc), fun, step) end - defp reduce(_, _, {:cont, acc}, _fun, _next, _up) do + defp reduce(_, _, {:cont, acc}, _fun, _up) do {:done, acc} end - def member?(first .. last, value) do - if first <= last do - {:ok, first <= value and value <= last} - else - {:ok, last <= value and value <= first} + def member?(first..last//step = range, value) when is_integer(value) do + cond do + Range.size(range) == 0 -> + {:ok, false} + + first <= last -> + {:ok, first <= value and value <= last and rem(value - first, step) == 0} + + true -> + {:ok, last <= value and value <= first and rem(value - first, step) == 0} end end - def count(first .. _ = range) do - {:ok, Range.Iterator.count(first, range)} + # TODO: Remove me on v2.0 + def member?(%{__struct__: Range, first: first, last: last} = range, value) + when is_integer(value) do + step = if first <= last, do: 1, else: -1 + member?(Map.put(range, :step, step), value) end -end -defimpl Range.Iterator, for: Integer do - def next(first, _ .. last) when is_integer(last) do - if last >= first do - &(&1 + 1) - else - &(&1 - 1) - end + def member?(_, _value) do + {:ok, false} end - def count(first, _ .. last) when is_integer(last) do - if last >= first do - last - first + 1 - else - first - last + 1 - end + def count(range) do + {:ok, Range.size(range)} + end + + def slice(first.._//step = range) do + {:ok, Range.size(range), &slice(first + &1 * step, step + &3 - 1, &2)} end + + # TODO: Remove me on v2.0 + def slice(%{__struct__: Range, first: first, last: last} = range) do + step = if first <= last, do: 1, else: -1 + slice(Map.put(range, :step, step)) + end + + defp slice(current, _step, 1), do: [current] + defp slice(current, step, remaining), do: [current | slice(current + step, step, remaining - 1)] end defimpl Inspect, for: Range do import Inspect.Algebra + import Kernel, except: [inspect: 2] + + def inspect(first..last//1, opts) when last >= first do + concat([to_doc(first, opts), "..", to_doc(last, opts)]) + end + + def inspect(first..last//step, opts) do + concat([to_doc(first, opts), "..", to_doc(last, opts), "//", to_doc(step, opts)]) + end - def inspect(first .. last, opts) do - concat [to_doc(first, opts), "..", to_doc(last, opts)] + # TODO: Remove me on v2.0 + def inspect(%{__struct__: Range, first: first, last: last} = range, opts) do + step = if first <= last, do: 1, else: -1 + inspect(Map.put(range, :step, step), opts) end end diff --git a/lib/elixir/lib/record.ex b/lib/elixir/lib/record.ex index c58081870f9..6ee963677d5 100644 --- a/lib/elixir/lib/record.ex +++ b/lib/elixir/lib/record.ex @@ -1,74 +1,141 @@ defmodule Record do @moduledoc """ - Module to work, define and import records. + Module to work with, define, and import records. Records are simply tuples where the first element is an atom: - iex> Record.record? {User, "jose", 27} + iex> Record.is_record({User, "john", 27}) true This module provides conveniences for working with records at compilation time, where compile-time field names are used to manipulate the tuples, providing fast operations on top of - the tuples compact structure. + the tuples' compact structure. In Elixir, records are used mostly in two situations: 1. to work with short, internal data 2. to interface with Erlang records - The macros `defrecord/3` and `defrecordp/3` can be used to create - records while `extract/2` can be used to extract records from Erlang - files. + The macros `defrecord/3` and `defrecordp/3` can be used to create records + while `extract/2` and `extract_all/1` can be used to extract records from + Erlang files. + + ## Types + + Types can be defined for tuples with the `record/2` macro (only available in + typespecs). This macro will expand to a tuple as seen in the example below: + + defmodule MyModule do + require Record + Record.defrecord(:user, name: "john", age: 25) + + @type user :: record(:user, name: String.t(), age: integer) + # expands to: "@type user :: {:user, String.t(), integer}" + end + + ## Reflection + + A list of all records in a module, if any, can be retrieved by reading the + `@__records__` module attribute. It returns a list of maps with the record + kind, name, tag, and fields. The attribute is only available inside the + module definition. """ @doc """ Extracts record information from an Erlang file. Returns a quoted expression containing the fields as a list - of tuples. It expects the record name to be an atom and the - library path to be a string at expansion time. + of tuples. + + `name`, which is the name of the extracted record, is expected to be an atom + *at compile time*. + + ## Options + + This function requires one of the following options, which are exclusive to each + other (i.e., only one of them can be used in the same call): + + * `:from` - (binary representing a path to a file) path to the Erlang file + that contains the record definition to extract; with this option, this + function uses the same path lookup used by the `-include` attribute used in + Erlang modules. + + * `:from_lib` - (binary representing a path to a file) path to the Erlang + file that contains the record definition to extract; with this option, + this function uses the same path lookup used by the `-include_lib` + attribute used in Erlang modules. + + It additionally accepts the following optional, non-exclusive options: + + * `:includes` - (a list of directories as binaries) if the record being + extracted depends on relative includes, this option allows developers + to specify the directory where those relative includes exist. + + * `:macros` - (keyword list of macro names and values) if the record + being extracted depends on the values of macros, this option allows + the value of those macros to be set. + + These options are expected to be literals (including the binary values) at + compile time. ## Examples iex> Record.extract(:file_info, from_lib: "kernel/include/file.hrl") - [size: :undefined, type: :undefined, access: :undefined, atime: :undefined, - mtime: :undefined, ctime: :undefined, mode: :undefined, links: :undefined, - major_device: :undefined, minor_device: :undefined, inode: :undefined, - uid: :undefined, gid: :undefined] + [ + size: :undefined, + type: :undefined, + access: :undefined, + atime: :undefined, + mtime: :undefined, + ctime: :undefined, + mode: :undefined, + links: :undefined, + major_device: :undefined, + minor_device: :undefined, + inode: :undefined, + uid: :undefined, + gid: :undefined + ] + + """ + @spec extract(name :: atom, keyword) :: keyword + def extract(name, opts) when is_atom(name) and is_list(opts) do + Record.Extractor.extract(name, opts) + end + + @doc """ + Extracts all records information from an Erlang file. + + Returns a keyword list of `{record_name, fields}` tuples where `record_name` + is the name of an extracted record and `fields` is a list of `{field, value}` + tuples representing the fields for that record. + + ## Options + + Accepts the same options as listed for `Record.extract/2`. """ - defmacro extract(name, opts) when is_atom(name) and is_list(opts) do - Macro.escape Record.Extractor.extract(name, opts) + @spec extract_all(keyword) :: [{name :: atom, keyword}] + def extract_all(opts) when is_list(opts) do + Record.Extractor.extract_all(opts) end @doc """ - Checks if the given `data` is a record of `kind`. + Checks if the given `data` is a record of kind `kind`. This is implemented as a macro so it can be used in guard clauses. ## Examples - iex> record = {User, "jose", 27} - iex> Record.record?(record, User) + iex> record = {User, "john", 27} + iex> Record.is_record(record, User) true """ - defmacro record?(data, kind) do - case Macro.Env.in_guard?(__CALLER__) do - true -> - quote do - is_tuple(unquote(data)) and tuple_size(unquote(data)) > 0 - and :erlang.element(1, unquote(data)) == unquote(kind) - end - false -> - quote do - result = unquote(data) - is_tuple(result) and tuple_size(result) > 0 - and :erlang.element(1, result) == unquote(kind) - end - end - end + defguard is_record(data, kind) + when is_atom(kind) and is_tuple(data) and tuple_size(data) > 0 and + elem(data, 0) == kind @doc """ Checks if the given `data` is a record. @@ -77,77 +144,120 @@ defmodule Record do ## Examples - iex> record = {User, "jose", 27} - iex> Record.record?(record) - true - iex> tuple = {} - iex> Record.record?(tuple) - false + Record.is_record({User, "john", 27}) + #=> true + + Record.is_record({}) + #=> false """ - defmacro record?(data) do - case Macro.Env.in_guard?(__CALLER__) do - true -> - quote do - is_tuple(unquote(data)) and tuple_size(unquote(data)) > 0 - and is_atom(:erlang.element(1, unquote(data))) - end - false -> - quote do - result = unquote(data) - is_tuple(result) and tuple_size(result) > 0 - and is_atom(:erlang.element(1, result)) - end - end - end + defguard is_record(data) + when is_tuple(data) and tuple_size(data) > 0 and is_atom(elem(data, 0)) @doc """ - Defines a set of macros to create and access a record. + Defines a set of macros to create, access, and pattern match + on a record. + + The name of the generated macros will be `name` (which has to be an + atom). `tag` is also an atom and is used as the "tag" for the record (i.e., + the first element of the record tuple); by default (if `nil`), it's the same + as `name`. `kv` is a keyword list of `name: default_value` fields for the + new record. + + The following macros are generated: + + * `name/0` to create a new record with default values for all fields + * `name/1` to create a new record with the given fields and values, + to get the zero-based index of the given field in a record or to + convert the given record to a keyword list + * `name/2` to update an existing record with the given fields and values + or to access a given field in a given record - The macros are going to have `name`, a tag (which defaults) - to the name if none is given, and a set of fields given by - `kv`. + All these macros are public macros (as defined by `defmacro`). + + See the "Examples" section for examples on how to use these macros. ## Examples defmodule User do - Record.defrecord :user, [name: "José", age: "25"] + require Record + Record.defrecord(:user, name: "meg", age: "25") end In the example above, a set of macros named `user` but with different - arities will be defined to manipulate the underlying record: + arities will be defined to manipulate the underlying record. + + # Import the module to make the user macros locally available + import User # To create records - user() #=> {:user, "José", 25} - user(age: 26) #=> {:user, "José", 26} + record = user() #=> {:user, "meg", 25} + record = user(age: 26) #=> {:user, "meg", 26} # To get a field from the record - user(record, :name) #=> "José" + user(record, :name) #=> "meg" # To update the record - user(record, age: 26) #=> {:user, "José", 26} + user(record, age: 26) #=> {:user, "meg", 26} + + # To get the zero-based index of the field in record tuple + # (index 0 is occupied by the record "tag") + user(:name) #=> 1 - By default, Elixir uses the record name as the first element of - the tuple (the tag). But it can be changed to something else: + # Convert a record to a keyword list + user(record) #=> [name: "meg", age: 26] + + The generated macros can also be used in order to pattern match on records and + to bind variables during the match: + + record = user() #=> {:user, "meg", 25} + + user(name: name) = record + name #=> "meg" + + By default, Elixir uses the record name as the first element of the tuple (the "tag"). + However, a different tag can be specified when defining a record, + as in the following example, in which we use `Customer` as the second argument of `defrecord/3`: defmodule User do - Record.defrecord :user, User, name: nil + require Record + Record.defrecord(:user, Customer, name: nil) end require User - User.user() #=> {User, nil} + User.user() #=> {Customer, nil} + + ## Defining extracted records with anonymous functions in the values + + If a record defines an anonymous function in the default values, an + `ArgumentError` will be raised. This can happen unintentionally when defining + a record after extracting it from an Erlang library that uses anonymous + functions for defaults. + + Record.defrecord(:my_rec, Record.extract(...)) + ** (ArgumentError) invalid value for record field fun_field, + cannot escape #Function<12.90072148/2 in :erl_eval.expr/5>. + + To work around this error, redefine the field with your own &M.f/a function, + like so: + + defmodule MyRec do + require Record + Record.defrecord(:my_rec, Record.extract(...) |> Keyword.merge(fun_field: &__MODULE__.foo/2)) + def foo(bar, baz), do: IO.inspect({bar, baz}) + end """ defmacro defrecord(name, tag \\ nil, kv) do quote bind_quoted: [name: name, tag: tag, kv: kv] do tag = tag || name - fields = Macro.escape Record.__fields__(:defrecord, kv) + fields = Record.__record__(__MODULE__, :defrecord, name, tag, kv) - defmacro(unquote(name)(args \\ [])) do + defmacro unquote(name)(args \\ []) do Record.__access__(unquote(tag), unquote(fields), args, __CALLER__) end - defmacro(unquote(name)(record, args)) do + defmacro unquote(name)(record, args) do Record.__access__(unquote(tag), unquote(fields), record, args, __CALLER__) end end @@ -159,124 +269,258 @@ defmodule Record do defmacro defrecordp(name, tag \\ nil, kv) do quote bind_quoted: [name: name, tag: tag, kv: kv] do tag = tag || name - fields = Macro.escape Record.__fields__(:defrecordp, kv) + fields = Record.__record__(__MODULE__, :defrecordp, name, tag, kv) - defmacrop(unquote(name)(args \\ [])) do + defmacrop unquote(name)(args \\ []) do Record.__access__(unquote(tag), unquote(fields), args, __CALLER__) end - defmacrop(unquote(name)(record, args)) do + defmacrop unquote(name)(record, args) do Record.__access__(unquote(tag), unquote(fields), record, args, __CALLER__) end end end - # Normalizes of record fields to have default values. + defp error_on_duplicate_record(module, name) do + defined_arity = + Enum.find(0..2, fn arity -> + Module.defines?(module, {name, arity}) + end) + + if defined_arity do + raise ArgumentError, + "cannot define record #{inspect(name)} because a definition #{name}/#{defined_arity} already exists" + end + end + + defp warn_on_duplicate_key([]) do + :ok + end + + defp warn_on_duplicate_key([{key, _} | [{key, _} | _] = rest]) do + IO.warn("duplicate key #{inspect(key)} found in record") + warn_on_duplicate_key(rest) + end + + defp warn_on_duplicate_key([_ | rest]) do + warn_on_duplicate_key(rest) + end + + # Callback invoked from the record/2 macro. @doc false - def __fields__(type, fields) do - :lists.map(fn - { key, _ } = pair when is_atom(key) -> pair - key when is_atom(key) -> { key, nil } - other -> raise ArgumentError, "#{type} fields must be atoms, got: #{inspect other}" - end, fields) + def __record__(module, kind, name, tag, kv) do + error_on_duplicate_record(module, name) + + fields = fields(kind, kv) + Module.register_attribute(module, :__records__, accumulate: true) + + Module.put_attribute(module, :__records__, %{ + kind: kind, + name: name, + tag: tag, + fields: :lists.map(&elem(&1, 0), fields) + }) + + # TODO: Make it raise on v2.0 + warn_on_duplicate_key(:lists.keysort(1, fields)) + fields + end + + # Normalizes of record fields to have default values. + defp fields(kind, fields) do + normalizer_fun = fn + {key, value} when is_atom(key) -> + try do + Macro.escape(value) + rescue + e in [ArgumentError] -> + raise ArgumentError, "invalid value for record field #{key}, " <> Exception.message(e) + else + value -> {key, value} + end + + key when is_atom(key) -> + {key, nil} + + other -> + raise ArgumentError, "#{kind} fields must be atoms, got: #{inspect(other)}" + end + + :lists.map(normalizer_fun, fields) end # Callback invoked from record/0 and record/1 macros. @doc false - def __access__(atom, fields, args, caller) do + def __access__(tag, fields, args, caller) do cond do is_atom(args) -> - index(atom, fields, args) + index(tag, fields, args) + Keyword.keyword?(args) -> - create(atom, fields, args, caller) + create(tag, fields, args, caller) + true -> - msg = "expected arguments to be a compile time atom or keywords, got: #{Macro.to_string args}" - raise ArgumentError, msg + fields = Macro.escape(fields) + + case Macro.expand(args, caller) do + {:{}, _, [^tag | list]} when length(list) == length(fields) -> + record = List.to_tuple([tag | list]) + Record.__keyword__(tag, fields, record) + + {^tag, arg} when length(fields) == 1 -> + Record.__keyword__(tag, fields, {tag, arg}) + + _ -> + quote(do: Record.__keyword__(unquote(tag), unquote(fields), unquote(args))) + end end end # Callback invoked from the record/2 macro. @doc false - def __access__(atom, fields, record, args, caller) do + def __access__(tag, fields, record, args, caller) do cond do is_atom(args) -> - get(atom, fields, record, args) + get(tag, fields, record, args) + Keyword.keyword?(args) -> - update(atom, fields, record, args, caller) + update(tag, fields, record, args, caller) + true -> - msg = "expected arguments to be a compile time atom or keywords, got: #{Macro.to_string args}" - raise ArgumentError, msg + raise ArgumentError, + "expected arguments to be a compile time atom or a keyword list, got: " <> + Macro.to_string(args) end end # Gets the index of field. - defp index(atom, fields, field) do - if index = find_index(fields, field, 0) do - index - 1 # Convert to Elixir index - else - raise ArgumentError, "record #{inspect atom} does not have the key: #{inspect field}" - end + defp index(tag, fields, field) do + find_index(fields, field, 1) || + raise ArgumentError, "record #{inspect(tag)} does not have the key: #{inspect(field)}" end # Creates a new record with the given default fields and keyword values. - defp create(atom, fields, keyword, caller) do - in_match = Macro.Env.in_match?(caller) - - {match, remaining} = - Enum.map_reduce(fields, keyword, fn({field, default}, each_keyword) -> - new_fields = - case Keyword.has_key?(each_keyword, field) do - true -> Keyword.get(each_keyword, field) - false -> - case in_match do - true -> {:_, [], nil} - false -> Macro.escape(default) - end - end - - {new_fields, Keyword.delete(each_keyword, field)} + defp create(tag, fields, keyword, caller) do + # Using {} here is safe, since it's not valid AST + default = if Macro.Env.in_match?(caller), do: {:_, [], nil}, else: {} + {default, keyword} = Keyword.pop(keyword, :_, default) + {keyword, exprs} = hoist_expressions(keyword, caller) + + {elements, remaining} = + Enum.map_reduce(fields, keyword, fn {key, field_default}, remaining -> + case Keyword.pop(remaining, key, default) do + {{}, remaining} -> {Macro.escape(field_default), remaining} + {default, remaining} -> {default, remaining} + end end) case remaining do [] -> - {:{}, [], [atom|match]} - _ -> - keys = for {key, _} <- remaining, do: key - raise ArgumentError, "record #{inspect atom} does not have the key: #{inspect hd(keys)}" + quote(do: {unquote(tag), unquote_splicing(elements)}) + |> maybe_prepend_reversed_exprs(exprs) + + [{key, _} | _] -> + raise ArgumentError, "record #{inspect(tag)} does not have the key: #{inspect(key)}" end end # Updates a record given by var with the given keyword. - defp update(atom, fields, var, keyword, caller) do + defp update(tag, fields, var, keyword, caller) do if Macro.Env.in_match?(caller) do raise ArgumentError, "cannot invoke update style macro inside match" end - Enum.reduce keyword, var, fn({key, value}, acc) -> - index = find_index(fields, key, 0) - if index do - quote do - :erlang.setelement(unquote(index), unquote(acc), unquote(value)) - end + {keyword, exprs} = hoist_expressions(keyword, caller) + + if Keyword.has_key?(keyword, :_) do + message = "updating a record with a default (:_) is equivalent to creating a new record" + IO.warn(message, caller) + create(tag, fields, keyword, caller) + else + updates = + Enum.map(keyword, fn {key, value} -> + if index = find_index(fields, key, 2) do + {index, value} + else + raise ArgumentError, "record #{inspect(tag)} does not have the key: #{inspect(key)}" + end + end) + + build_update(updates, var) |> maybe_prepend_reversed_exprs(exprs) + end + end + + defp hoist_expressions(keyword, %{context: nil, module: module}) do + Enum.map_reduce(keyword, [], fn {key, expr}, acc -> + if simple_argument?(expr) do + {{key, expr}, acc} else - raise ArgumentError, "record #{inspect atom} does not have the key: #{inspect key}" + var = Macro.unique_var(key, module) + {{key, var}, [{:=, [], [var, expr]} | acc]} end - end + end) end + defp hoist_expressions(keyword, _), do: {keyword, []} + + defp maybe_prepend_reversed_exprs(expr, []), + do: expr + + defp maybe_prepend_reversed_exprs(expr, exprs), + do: {:__block__, [], :lists.reverse([expr | exprs])} + + defp build_update(updates, initial) do + updates + |> Enum.sort(fn {left, _}, {right, _} -> right <= left end) + |> Enum.reduce(initial, fn {key, value}, acc -> + quote(do: :erlang.setelement(unquote(key), unquote(acc), unquote(value))) + end) + end + + defp simple_argument?({name, _, ctx}) when is_atom(name) and is_atom(ctx), do: true + defp simple_argument?(other), do: Macro.quoted_literal?(other) + # Gets a record key from the given var. - defp get(atom, fields, var, key) do - index = find_index(fields, key, 0) - if index do - quote do - :erlang.element(unquote(index), unquote(var)) + defp get(tag, fields, var, key) do + index = + find_index(fields, key, 2) || + raise ArgumentError, "record #{inspect(tag)} does not have the key: #{inspect(key)}" + + quote do + :erlang.element(unquote(index), unquote(var)) + end + end + + defp find_index([{k, _} | _], k, i), do: i + defp find_index([{_, _} | t], k, i), do: find_index(t, k, i + 1) + defp find_index([], _k, _i), do: nil + + # Returns a keyword list of the record + @doc false + def __keyword__(tag, fields, record) do + if is_record(record, tag) do + [_tag | values] = Tuple.to_list(record) + + case join_keyword(fields, values, []) do + kv when is_list(kv) -> + kv + + expected_fields -> + raise ArgumentError, + "expected argument to be a #{inspect(tag)} record with " <> + "#{expected_fields} fields, got: " <> inspect(record) end else - raise ArgumentError, "record #{inspect atom} does not have the key: #{inspect key}" + raise ArgumentError, + "expected argument to be a literal atom, literal keyword or " <> + "a #{inspect(tag)} record, got runtime: " <> inspect(record) end end - defp find_index([{k, _}|_], k, i), do: i + 2 - defp find_index([{_, _}|t], k, i), do: find_index(t, k, i + 1) - defp find_index([], _k, _i), do: nil + # Returns a keyword list, or expected number of fields on size mismatch + defp join_keyword([{field, _default} | fields], [value | values], acc), + do: join_keyword(fields, values, [{field, value} | acc]) + + defp join_keyword([], [], acc), do: :lists.reverse(acc) + defp join_keyword(rest_fields, _rest_values, acc), do: length(acc) + length(rest_fields) end diff --git a/lib/elixir/lib/record/extractor.ex b/lib/elixir/lib/record/extractor.ex index 9de1ad4f0dd..380cc95339e 100644 --- a/lib/elixir/lib/record/extractor.ex +++ b/lib/elixir/lib/record/extractor.ex @@ -1,56 +1,85 @@ defmodule Record.Extractor do @moduledoc false - # Retrieve a record definition from an Erlang file using - # the same lookup as the *include* attribute from Erlang modules. - def extract(name, from: file) when is_binary(file) do - file = String.to_char_list(file) - - realfile = - case :code.where_is_file(file) do - :non_existing -> file - realfile -> realfile - end - - extract_record(name, realfile) + def extract(name, opts) do + extract_record(name, from_or_from_lib_file(opts)) end - # Retrieve a record definition from an Erlang file using - # the same lookup as the *include_lib* attribute from Erlang modules. - def extract(name, from_lib: file) when is_binary(file) do - [app|path] = :filename.split(String.to_char_list(file)) + def extract_all(opts) do + extract_all_records(from_or_from_lib_file(opts)) + end + + defp from_or_from_lib_file(opts) do + cond do + file = opts[:from] -> + {from_file(file), Keyword.delete(opts, :from)} + + file = opts[:from_lib] -> + {from_lib_file(file), Keyword.delete(opts, :from_lib)} + + true -> + raise ArgumentError, "expected :from or :from_lib to be given as option" + end + end + + # Find file using the same lookup as the *include* attribute from Erlang modules. + defp from_file(file) do + file = String.to_charlist(file) + + case :code.where_is_file(file) do + :non_existing -> file + realfile -> realfile + end + end + + # Find file using the same lookup as the *include_lib* attribute from Erlang modules. + defp from_lib_file(file) do + [app | path] = :filename.split(String.to_charlist(file)) case :code.lib_dir(List.to_atom(app)) do {:error, _} -> raise ArgumentError, "lib file #{file} could not be found" + libpath -> - extract_record name, :filename.join([libpath|path]) + :filename.join([libpath | path]) end end # Retrieve the record with the given name from the given file - defp extract_record(name, file) do - form = read_file(file) + defp extract_record(name, {file, opts}) do + form = read_file(file, opts) records = extract_records(form) + if record = List.keyfind(records, name, 0) do parse_record(record, form) else - raise ArgumentError, "no record #{name} found at #{file}" + raise ArgumentError, + "no record #{name} found at #{file}. Or the record does not exist or " <> + "its entry is malformed or depends on other include files" end end + # Retrieve all records from the given file + defp extract_all_records({file, opts}) do + form = read_file(file, opts) + records = extract_records(form) + for rec = {name, _fields} <- records, do: {name, parse_record(rec, form)} + end + # Parse the given file and extract all existent records. defp extract_records(form) do for {:attribute, _, :record, record} <- form, do: record end # Read a file and return its abstract syntax form that also - # includes record and other preprocessor modules. This is done - # by using Erlang's epp_dodger. - defp read_file(file) do - case :epp_dodger.quick_parse_file(file) do + # includes record but with macros and other attributes expanded, + # such as "-include(...)" and "-include_lib(...)". This is done + # by using Erlang's epp. + defp read_file(file, opts) do + case :epp.parse_file(file, opts) do {:ok, form} -> form + other -> raise "error parsing file #{file}, got: #{inspect(other)}" end @@ -60,9 +89,7 @@ defmodule Record.Extractor do # list of tuples where the first element is the field # and the second is its default value. defp parse_record({_name, fields}, form) do - cons = List.foldr fields, {nil, 0}, fn f, acc -> - {:cons, 0, parse_field(f), acc} - end + cons = List.foldr(fields, {nil, 0}, fn f, acc -> {:cons, 0, parse_field(f), acc} end) eval_record(cons, form) end @@ -79,12 +106,10 @@ defmodule Record.Extractor do end defp eval_record(cons, form) do - form = form ++ - [ {:function, 0, :hello, 0, [ - {:clause, 0, [], [], [ cons ]} ]} ] + form = form ++ [{:function, 0, :hello, 0, [{:clause, 0, [], [], [cons]}]}] - {:function, 0, :hello, 0, [ - {:clause, 0, [], [], [ record_ast ]} ]} = :erl_expand_records.module(form, []) |> List.last + {:function, 0, :hello, 0, [{:clause, 0, [], [], [record_ast]}]} = + :erl_expand_records.module(form, []) |> List.last() {:value, record, _} = :erl_eval.expr(record_ast, []) record diff --git a/lib/elixir/lib/regex.ex b/lib/elixir/lib/regex.ex index b6591de723e..9a420cac2b8 100644 --- a/lib/elixir/lib/regex.ex +++ b/lib/elixir/lib/regex.ex @@ -1,35 +1,53 @@ defmodule Regex do @moduledoc ~S""" - Regular expressions for Elixir built on top of Erlang's `re` module. + Provides regular expressions for Elixir. - As the `re` module, Regex is based on PCRE - (Perl Compatible Regular Expressions). More information can be - found in the [`re` documentation](http://www.erlang.org/doc/man/re.html). + Regex is based on PCRE (Perl Compatible Regular Expressions) and + built on top of Erlang's `:re` module. More information can be found + in the [`:re` module documentation](`:re`). - Regular expressions in Elixir can be created using `Regex.compile!/2` - or using the special form with [`~r`](Kernel.html#sigil_r/2): + Regular expressions in Elixir can be created using the sigils + `~r` (see `sigil_r/2`) or `~R` (see `sigil_R/2`): - # A simple regular expressions that matches foo anywhere in the string + # A simple regular expression that matches foo anywhere in the string ~r/foo/ - # A regular expression with case insensitive and unicode options + # A regular expression with case insensitive and Unicode options ~r/foo/iu + Regular expressions created via sigils are pre-compiled and stored + in the `.beam` file. Note that this may be a problem if you are precompiling + Elixir, see the "Precompilation" section for more information. + A Regex is represented internally as the `Regex` struct. Therefore, `%Regex{}` can be used whenever there is a need to match on them. + Keep in mind that all of the structs fields are private. There is + also not guarantee two regular expressions from the same source are + equal, for example: + + ~r/(?.)(?.)/ == ~r/(?.)(?.)/ + + may return `true` or `false` depending on your machine, endianness, + available optimizations and others. You can, however, retrieve the source + of a compiled regular expression by accessing the `source` field, and then + compare those directly: + + ~r/(?.)(?.)/.source == ~r/(?.)(?.)/.source ## Modifiers The modifiers available when creating a Regex are: - * `unicode` (u) - enables unicode specific patterns like `\p`; it expects - valid unicode strings to be given on match + * `unicode` (u) - enables Unicode specific patterns like `\p` and causes + character classes like `\w`, `\W`, `\s`, and the like to also match on Unicode + (see examples below in "Character classes"). It expects valid Unicode + strings to be given on match - * `caseless` (i) - add case insensitivity + * `caseless` (i) - adds case insensitivity * `dotall` (s) - causes dot to match newlines and also set newline to anycrlf; the new line setting can be overridden by setting `(*CR)` or - `(*LF)` or `(*CRLF)` or `(*ANY)` according to re documentation + `(*LF)` or `(*CRLF)` or `(*ANY)` according to `:re` documentation * `multiline` (m) - causes `^` and `$` to mark the beginning and end of each line; use `\A` and `\z` to match the end or beginning of the string @@ -40,7 +58,8 @@ defmodule Regex do * `firstline` (f) - forces the unanchored pattern to match before or at the first newline, though the matched text may continue over the newline - * `ungreedy` (r) - inverts the "greediness" of the regexp + * `ungreedy` (U) - inverts the "greediness" of the regexp + (the previous `r` option is deprecated in favor of `U`) The options not available are: @@ -49,11 +68,11 @@ defmodule Regex do * `no_auto_capture` - not available, use `?:` instead * `newline` - not available, use `(*CR)` or `(*LF)` or `(*CRLF)` or `(*ANYCRLF)` or `(*ANY)` at the beginning of the regexp according to the - re documentation + `:re` documentation ## Captures - Many functions in this module allows what to capture in a regex + Many functions in this module handle what to capture in a regex match via the `:capture` option. The supported values are: * `:all` - all captured subpatterns including the complete matching string @@ -63,19 +82,80 @@ defmodule Regex do complete matching part of the string; all explicitly captured subpatterns are discarded - * `:all_but_first`- all but the first matching subpattern, i.e. all + * `:all_but_first` - all but the first matching subpattern, i.e. all explicitly captured subpatterns, but not the complete matching part of the string - * `:none` - do not return matching subpatterns at all + * `:none` - does not return matching subpatterns at all - * `:all_names` - captures all names in the Regex + * `:all_names` - captures all named subpattern matches in the Regex as a list + ordered **alphabetically** by the names of the subpatterns * `list(binary)` - a list of named captures to capture + ## Character classes + + Regex supports several built in named character classes. These are used by + enclosing the class name in `[: :]` inside a group. For example: + + iex> String.match?("123", ~r/^[[:alnum:]]+$/) + true + iex> String.match?("123 456", ~r/^[[:alnum:][:blank:]]+$/) + true + + The supported class names are: + + * alnum - Letters and digits + * alpha - Letters + * blank - Space or tab only + * cntrl - Control characters + * digit - Decimal digits (same as \\d) + * graph - Printing characters, excluding space + * lower - Lowercase letters + * print - Printing characters, including space + * punct - Printing characters, excluding letters, digits, and space + * space - Whitespace (the same as \s from PCRE 8.34) + * upper - Uppercase letters + * word - "Word" characters (same as \w) + * xdigit - Hexadecimal digits + + There is another character class, `ascii`, that erroneously matches + Latin-1 characters instead of the 0-127 range specified by POSIX. This + cannot be fixed without altering the behaviour of other classes, so we + recommend matching the range with `[\\0-\x7f]` instead. + + Note the behaviour of those classes may change according to the Unicode + and other modifiers: + + iex> String.match?("josé", ~r/^[[:lower:]]+$/) + false + iex> String.match?("josé", ~r/^[[:lower:]]+$/u) + true + iex> Regex.replace(~r/\s/, "Unicode\u00A0spaces", "-") + "Unicode spaces" + iex> Regex.replace(~r/\s/u, "Unicode\u00A0spaces", "-") + "Unicode-spaces" + + ## Precompilation + + Regular expressions built with sigil are precompiled and stored in `.beam` + files. Precompiled regexes will be checked in runtime and may work slower + between operating systems and OTP releases. This is rarely a problem, as most Elixir code + shared during development is compiled on the target (such as dependencies, + archives, and escripts) and, when running in production, the code must either + be compiled on the target (via `mix compile` or similar) or released on the + host (via `mix releases` or similar) with a matching OTP, operating system + and architecture as the target. + + If you know you are running on a different system than the current one and + you are doing multiple matches with the regex, you can manually invoke + `Regex.recompile/1` or `Regex.recompile!/1` to perform a runtime version + check and recompile the regex if necessary. """ - defstruct re_pattern: nil :: term, source: "" :: binary, opts: "" :: binary + defstruct re_pattern: nil, source: "", opts: "", re_version: "" + + @type t :: %__MODULE__{re_pattern: term, source: binary, opts: binary} defmodule CompileError do defexception message: "regex could not be compiled" @@ -85,8 +165,9 @@ defmodule Regex do Compiles the regular expression. The given options can either be a binary with the characters - representing the same regex options given to the `~r` sigil, - or a list of options, as expected by the [Erlang `re` docs](http://www.erlang.org/doc/man/re.html). + representing the same regex options given to the + `~r` (see `sigil_r/2`) sigil, or a list of options, as + expected by the Erlang's `:re` module. It returns `{:ok, regex}` in case of success, `{:error, reason}` otherwise. @@ -94,77 +175,121 @@ defmodule Regex do ## Examples iex> Regex.compile("foo") - {:ok, ~r"foo"} + {:ok, ~r/foo/} iex> Regex.compile("*foo") {:error, {'nothing to repeat', 0}} """ @spec compile(binary, binary | [term]) :: {:ok, t} | {:error, any} - def compile(source, options \\ "") + def compile(source, options \\ "") when is_binary(source) do + compile(source, options, version()) + end - def compile(source, options) when is_binary(options) do - case translate_options(options) do + defp compile(source, options, version) when is_binary(options) do + case translate_options(options, []) do {:error, rest} -> {:error, {:invalid_option, rest}} translated_options -> - compile(source, translated_options, options) + compile(source, translated_options, options, version) end end - def compile(source, options) when is_list(options) do - compile(source, options, "") + defp compile(source, options, version) when is_list(options) do + compile(source, options, "", version) end - defp compile(source, opts, doc_opts) when is_binary(source) do + defp compile(source, opts, doc_opts, version) do case :re.compile(source, opts) do {:ok, re_pattern} -> - {:ok, %Regex{re_pattern: re_pattern, source: source, opts: doc_opts}} + {:ok, %Regex{re_pattern: re_pattern, re_version: version, source: source, opts: doc_opts}} + error -> error end end @doc """ - Compiles the regular expression according to the given options. - Fails with `Regex.CompileError` if the regex cannot be compiled. + Compiles the regular expression and raises `Regex.CompileError` in case of errors. """ - def compile!(source, options \\ "") do + @spec compile!(binary, binary | [term]) :: t + def compile!(source, options \\ "") when is_binary(source) do case compile(source, options) do {:ok, regex} -> regex - {:error, {reason, at}} -> raise Regex.CompileError, message: "#{reason} at position #{at}" + {:error, {reason, at}} -> raise Regex.CompileError, "#{reason} at position #{at}" end end @doc """ - Returns a boolean indicating whether there was a match or not. + Recompiles the existing regular expression if necessary. - ## Examples - - iex> Regex.match?(~r/foo/, "foo") - true + This checks the version stored in the regular expression + and recompiles the regex in case of version mismatch. + """ + @doc since: "1.4.0" + @spec recompile(t) :: {:ok, t} | {:error, any} + def recompile(%Regex{} = regex) do + version = version() + + case regex do + %{re_version: ^version} -> + {:ok, regex} + + _ -> + %{source: source, opts: opts} = regex + compile(source, opts, version) + end + end - iex> Regex.match?(~r/foo/, "bar") - false + @doc """ + Recompiles the existing regular expression and raises `Regex.CompileError` in case of errors. + """ + @doc since: "1.4.0" + @spec recompile!(t) :: t + def recompile!(regex) do + case recompile(regex) do + {:ok, regex} -> regex + {:error, {reason, at}} -> raise Regex.CompileError, "#{reason} at position #{at}" + end + end + @doc """ + Returns the version of the underlying Regex engine. """ - def match?(%Regex{re_pattern: compiled}, string) when is_binary(string) do - :re.run(string, compiled, [{:capture, :none}]) == :match + @doc since: "1.4.0" + @spec version :: term() + def version do + {:re.version(), :erlang.system_info(:endian)} end @doc """ - Returns true if the given argument is a regex. + Returns a boolean indicating whether there was a match or not. ## Examples - iex> Regex.regex?(~r/foo/) + iex> Regex.match?(~r/foo/, "foo") true - iex> Regex.regex?(0) + iex> Regex.match?(~r/foo/, "bar") false + Elixir also provides text-based match operator `=~/2` and function `String.match?/2` as + an alternative to test strings against regular expressions and + strings. + """ + @spec match?(t, String.t()) :: boolean + def match?(%Regex{} = regex, string) when is_binary(string) do + safe_run(regex, string, [{:capture, :none}]) == :match + end + + @doc """ + Returns `true` if the given `term` is a regex. + Otherwise returns `false`. """ + # TODO: deprecate permanently on Elixir v1.15 + @doc deprecated: "Use Kernel.is_struct/2 or pattern match on %Regex{} instead" + def regex?(term) def regex?(%Regex{}), do: true def regex?(_), do: false @@ -174,9 +299,12 @@ defmodule Regex do ## Options - * `:return` - set to `:index` to return indexes. Defaults to `:binary`. + * `:return` - when set to `:index`, returns byte index and match length. + Defaults to `:binary`. * `:capture` - what to capture in the result. Check the moduledoc for `Regex` - to see the possible capture values. + to see the possible capture values. + * `:offset` - (since v1.12.0) specifies the starting offset to match in the given string. + Defaults to zero. ## Examples @@ -187,26 +315,31 @@ defmodule Regex do nil iex> Regex.run(~r/c(d)/, "abcd", return: :index) - [{2,2},{3,1}] + [{2, 2}, {3, 1}] """ + @spec run(t, binary, [term]) :: nil | [binary] | [{integer, integer}] def run(regex, string, options \\ []) - def run(%Regex{re_pattern: compiled}, string, options) when is_binary(string) do - return = Keyword.get(options, :return, :binary) + def run(%Regex{} = regex, string, options) when is_binary(string) do + return = Keyword.get(options, :return, :binary) captures = Keyword.get(options, :capture, :all) + offset = Keyword.get(options, :offset, 0) - case :re.run(string, compiled, [{:capture, captures, return}]) do + case safe_run(regex, string, [{:capture, captures, return}, {:offset, offset}]) do :nomatch -> nil - :match -> [] + :match -> [] {:match, results} -> results end end @doc """ - Returns the given captures as a map or `nil` if no captures are - found. The option `:return` can be set to `:index` to get indexes - back. + Returns the given captures as a map or `nil` if no captures are found. + + ## Options + + * `:return` - when set to `:index`, returns byte index and match length. + Defaults to `:binary`. ## Examples @@ -220,6 +353,7 @@ defmodule Regex do nil """ + @spec named_captures(t, String.t(), [term]) :: map | nil def named_captures(regex, string, options \\ []) when is_binary(string) do names = names(regex) options = Keyword.put(options, :capture, names) @@ -230,6 +364,7 @@ defmodule Regex do @doc """ Returns the underlying `re_pattern` in the regular expression. """ + @spec re_pattern(t) :: term def re_pattern(%Regex{re_pattern: compiled}) do compiled end @@ -239,10 +374,11 @@ defmodule Regex do ## Examples - iex> Regex.source(~r(foo)) + iex> Regex.source(~r/foo/) "foo" """ + @spec source(t) :: String.t() def source(%Regex{source: source}) do source end @@ -252,10 +388,11 @@ defmodule Regex do ## Examples - iex> Regex.opts(~r(foo)m) + iex> Regex.opts(~r/foo/m) "m" """ + @spec opts(t) :: String.t() def opts(%Regex{opts: opts}) do opts end @@ -269,22 +406,37 @@ defmodule Regex do ["foo"] """ - def names(%Regex{re_pattern: re_pattern}) do + @spec names(t) :: [String.t()] + def names(%Regex{re_pattern: compiled, re_version: version, source: source}) do + re_pattern = + case version() do + ^version -> + compiled + + _ -> + {:ok, recompiled} = :re.compile(source) + recompiled + end + {:namelist, names} = :re.inspect(re_pattern, :namelist) names end - @doc """ + @doc ~S""" Same as `run/3`, but scans the target several times collecting all - matches of the regular expression. A list of lists is returned, - where each entry in the primary list represents a match and each - entry in the secondary list represents the captured contents. + matches of the regular expression. + + A list of lists is returned, where each entry in the primary list represents a + match and each entry in the secondary list represents the captured contents. ## Options - * `:return` - set to `:index` to return indexes. Defaults to `:binary`. + * `:return` - when set to `:index`, returns byte index and match length. + Defaults to `:binary`. * `:capture` - what to capture in the result. Check the moduledoc for `Regex` - to see the possible capture values. + to see the possible capture values. + * `:offset` - (since v1.12.0) specifies the starting offset to match in the given string. + Defaults to zero. ## Examples @@ -297,77 +449,177 @@ defmodule Regex do iex> Regex.scan(~r/e/, "abcd") [] + iex> Regex.scan(~r/\p{Sc}/u, "$, £, and €") + [["$"], ["£"], ["€"]] + + iex> Regex.scan(~r/=+/, "=ü†ƒ8===", return: :index) + [[{0, 1}], [{9, 3}]] + """ + @spec scan(t(), String.t(), [term()]) :: [[String.t()]] | [[{integer(), integer()}]] def scan(regex, string, options \\ []) - def scan(%Regex{re_pattern: compiled}, string, options) when is_binary(string) do - return = Keyword.get(options, :return, :binary) + def scan(%Regex{} = regex, string, options) when is_binary(string) do + return = Keyword.get(options, :return, :binary) captures = Keyword.get(options, :capture, :all) - options = [{:capture, captures, return}, :global] + offset = Keyword.get(options, :offset, 0) + options = [{:capture, captures, return}, :global, {:offset, offset}] - case :re.run(string, compiled, options) do + case safe_run(regex, string, options) do :match -> [] :nomatch -> [] {:match, results} -> results end end + defp safe_run( + %Regex{re_pattern: compiled, source: source, re_version: version, opts: compile_opts}, + string, + options + ) do + case version() do + ^version -> :re.run(string, compiled, options) + _ -> :re.run(string, source, translate_options(compile_opts, options)) + end + end + @doc """ - Splits the given target into the number of parts specified. + Splits the given target based on the given pattern and in the given number of + parts. ## Options * `:parts` - when specified, splits the string into the given number of - parts. If not specified, `:parts` is defaulted to `:infinity`, which will + parts. If not specified, `:parts` defaults to `:infinity`, which will split the string into the maximum number of parts possible based on the given pattern. - * `:trim` - when true, remove blank strings from the result. + * `:trim` - when `true`, removes empty strings (`""`) from the result. + Defaults to `false`. + + * `:on` - specifies which captures to split the string on, and in what + order. Defaults to `:first` which means captures inside the regex do not + affect the splitting process. + + * `:include_captures` - when `true`, includes in the result the matches of + the regular expression. The matches are not counted towards the maximum + number of parts if combined with the `:parts` option. Defaults to `false`. ## Examples - iex> Regex.split(~r/-/, "a-b-c") - ["a","b","c"] + iex> Regex.split(~r{-}, "a-b-c") + ["a", "b", "c"] - iex> Regex.split(~r/-/, "a-b-c", [parts: 2]) - ["a","b-c"] + iex> Regex.split(~r{-}, "a-b-c", parts: 2) + ["a", "b-c"] - iex> Regex.split(~r/-/, "abc") + iex> Regex.split(~r{-}, "abc") ["abc"] - iex> Regex.split(~r//, "abc") - ["a", "b", "c", ""] + iex> Regex.split(~r{}, "abc") + ["", "a", "b", "c", ""] + + iex> Regex.split(~r{a(?b)c}, "abc") + ["", ""] - iex> Regex.split(~r//, "abc", trim: true) + iex> Regex.split(~r{a(?b)c}, "abc", on: [:second]) + ["a", "c"] + + iex> Regex.split(~r{(x)}, "Elixir", include_captures: true) + ["Eli", "x", "ir"] + + iex> Regex.split(~r{a(?b)c}, "abc", on: [:second], include_captures: true) ["a", "b", "c"] """ - + @spec split(t, String.t(), [term]) :: [String.t()] def split(regex, string, options \\ []) - def split(%Regex{re_pattern: compiled}, string, options) when is_binary(string) do - parts = Keyword.get(options, :parts, :infinity) - opts = [return: :binary, parts: zero_to_infinity(parts)] - splits = :re.split(string, compiled, opts) + def split(%Regex{}, "", opts) do + if Keyword.get(opts, :trim, false) do + [] + else + [""] + end + end - if Keyword.get(options, :trim, false) do - for split <- splits, split != "", do: split + def split(%Regex{} = regex, string, opts) + when is_binary(string) and is_list(opts) do + on = Keyword.get(opts, :on, :first) + + case safe_run(regex, string, [:global, capture: on]) do + {:match, matches} -> + index = parts_to_index(Keyword.get(opts, :parts, :infinity)) + trim = Keyword.get(opts, :trim, false) + include_captures = Keyword.get(opts, :include_captures, false) + do_split(matches, string, 0, index, trim, include_captures) + + :match -> + [string] + + :nomatch -> + [string] + end + end + + defp parts_to_index(:infinity), do: 0 + defp parts_to_index(n) when is_integer(n) and n > 0, do: n + + defp do_split(_, string, offset, _counter, true, _with_captures) + when byte_size(string) <= offset do + [] + end + + defp do_split(_, string, offset, 1, _trim, _with_captures), + do: [binary_part(string, offset, byte_size(string) - offset)] + + defp do_split([], string, offset, _counter, _trim, _with_captures), + do: [binary_part(string, offset, byte_size(string) - offset)] + + defp do_split([[{pos, _} | h] | t], string, offset, counter, trim, with_captures) + when pos - offset < 0 do + do_split([h | t], string, offset, counter, trim, with_captures) + end + + defp do_split([[] | t], string, offset, counter, trim, with_captures), + do: do_split(t, string, offset, counter, trim, with_captures) + + defp do_split([[{pos, length} | h] | t], string, offset, counter, trim, true) do + new_offset = pos + length + keep = pos - offset + + <<_::binary-size(offset), part::binary-size(keep), match::binary-size(length), _::binary>> = + string + + if keep == 0 and trim do + [match | do_split([h | t], string, new_offset, counter - 1, trim, true)] else - splits + [part, match | do_split([h | t], string, new_offset, counter - 1, trim, true)] end end - defp zero_to_infinity(0), do: :infinity - defp zero_to_infinity(n), do: n + defp do_split([[{pos, length} | h] | t], string, offset, counter, trim, false) do + new_offset = pos + length + keep = pos - offset + + if keep == 0 and trim do + do_split([h | t], string, new_offset, counter, trim, false) + else + <<_::binary-size(offset), part::binary-size(keep), _::binary>> = string + [part | do_split([h | t], string, new_offset, counter - 1, trim, false)] + end + end @doc ~S""" Receives a regex, a binary and a replacement, returns a new - binary where the all matches are replaced by replacement. + binary where all matches are replaced by the replacement. The replacement can be either a string or a function. The string is used as a replacement for every match and it allows specific - captures to be accessed via `\N`, where `N` is the capture. In - case `\0` is used, the whole match is inserted. + captures to be accessed via `\N` or `\g{N}`, where `N` is the + capture. In case `\0` is used, the whole match is inserted. Note + that in regexes the backslash needs to be escaped, hence in practice + you'll need to use `\\N` and `\\g{N}`. When the replacement is a function, the function may have arity N where each argument maps to a capture, with the first argument @@ -377,7 +629,7 @@ defmodule Regex do ## Options * `:global` - when `false`, replaces only the first occurrence - (defaults to true) + (defaults to `true`) ## Examples @@ -393,64 +645,70 @@ defmodule Regex do iex> Regex.replace(~r/a(b|d)c/, "abcadc", "[\\1]") "[b][d]" + iex> Regex.replace(~r/\.(\d)$/, "500.5", ".\\g{1}0") + "500.50" + iex> Regex.replace(~r/a(b|d)c/, "abcadc", fn _, x -> "[#{x}]" end) "[b][d]" - """ - def replace(regex, string, replacement, options \\ []) + iex> Regex.replace(~r/a/, "abcadc", "A", global: false) + "Abcadc" - def replace(regex, string, replacement, options) when is_binary(replacement) do - do_replace(regex, string, precompile_replacement(replacement), options) - end - - def replace(regex, string, replacement, options) when is_function(replacement) do - {:arity, arity} = :erlang.fun_info(replacement, :arity) - do_replace(regex, string, {replacement, arity}, options) - end - - defp do_replace(%Regex{re_pattern: compiled}, string, replacement, options) do + """ + @spec replace(t, String.t(), String.t() | (... -> String.t()), [term]) :: String.t() + def replace(%Regex{} = regex, string, replacement, options \\ []) + when is_binary(string) and is_list(options) do opts = if Keyword.get(options, :global) != false, do: [:global], else: [] - opts = [{:capture, :all, :index}|opts] + opts = [{:capture, :all, :index} | opts] - case :re.run(string, compiled, opts) do + case safe_run(regex, string, opts) do :nomatch -> string - {:match, [mlist|t]} when is_list(mlist) -> - apply_list(string, replacement, [mlist|t]) |> IO.iodata_to_binary + + {:match, [mlist | t]} when is_list(mlist) -> + apply_list(string, precompile_replacement(replacement), [mlist | t]) + |> IO.iodata_to_binary() + {:match, slist} -> - apply_list(string, replacement, [slist]) |> IO.iodata_to_binary + apply_list(string, precompile_replacement(replacement), [slist]) + |> IO.iodata_to_binary() end end - defp precompile_replacement(""), - do: [] + defp precompile_replacement(replacement) when is_function(replacement) do + {:arity, arity} = Function.info(replacement, :arity) + {replacement, arity} + end - defp precompile_replacement(<>) when x < ?0 or x > ?9 do - case precompile_replacement(rest) do - [head | t] when is_binary(head) -> - [<> | t] - other -> - [<> | other] - end + defp precompile_replacement(""), do: [] + + defp precompile_replacement(<>) when byte_size(rest) > 0 do + {ns, <>} = pick_int(rest) + [List.to_integer(ns) | precompile_replacement(rest)] end - defp precompile_replacement(<>) when byte_size(rest) > 0 do + defp precompile_replacement(<>) do + [<> | precompile_replacement(rest)] + end + + defp precompile_replacement(<>) when x in ?0..?9 do {ns, rest} = pick_int(rest) - [List.to_integer(ns) | precompile_replacement(rest)] + [List.to_integer([x | ns]) | precompile_replacement(rest)] end - defp precompile_replacement(<>) do + defp precompile_replacement(<>) do case precompile_replacement(rest) do [head | t] when is_binary(head) -> - [<> | t] + [<> | t] + other -> [<> | other] end end - defp pick_int(<>) when x in ?0..?9 do + defp pick_int(<>) when x in ?0..?9 do {found, rest} = pick_int(rest) - {[x|found], rest} + {[x | found], rest} end defp pick_int(bin) do @@ -469,14 +727,15 @@ defmodule Regex do string end - defp apply_list(whole, string, pos, replacement, [[{mpos, _} | _] | _] = list) when mpos > pos do + defp apply_list(whole, string, pos, replacement, [[{mpos, _} | _] | _] = list) + when mpos > pos do length = mpos - pos - <> = string + <> = string [untouched | apply_list(whole, rest, mpos, replacement, list)] end - defp apply_list(whole, string, pos, replacement, [[{mpos, length} | _] = head | tail]) when mpos == pos do - <<_ :: [size(length), binary], rest :: binary>> = string + defp apply_list(whole, string, pos, replacement, [[{pos, length} | _] = head | tail]) do + <<_::size(length)-binary, rest::binary>> = string new_data = apply_replace(whole, replacement, head) [new_data | apply_list(whole, rest, pos + length, replacement, tail)] end @@ -496,20 +755,22 @@ defmodule Regex do cond do is_binary(part) -> part - part > tuple_size(indexes) -> + + part >= tuple_size(indexes) -> "" + true -> get_index(string, elem(indexes, part)) end end end - defp get_index(_string, {pos, _len}) when pos < 0 do + defp get_index(_string, {pos, _length}) when pos < 0 do "" end - defp get_index(string, {pos, len}) do - <<_ :: [size(pos), binary], res :: [size(len), binary], _ :: binary>> = string + defp get_index(string, {pos, length}) do + <<_::size(pos)-binary, res::size(length)-binary, _::binary>> = string res end @@ -518,16 +779,13 @@ defmodule Regex do end defp get_indexes(string, [], arity) do - [""|get_indexes(string, [], arity - 1)] + ["" | get_indexes(string, [], arity - 1)] end - defp get_indexes(string, [h|t], arity) do - [get_index(string, h)|get_indexes(string, t, arity - 1)] + defp get_indexes(string, [h | t], arity) do + [get_index(string, h) | get_indexes(string, t, arity - 1)] end - {:ok, pattern} = :re.compile(~S"[.^$*+?()[{\\\|\s#]", [:unicode]) - @escape_pattern pattern - @doc ~S""" Escapes a string to be literally matched in a regex. @@ -540,37 +798,66 @@ defmodule Regex do "\\\\what\\ if" """ - @spec escape(String.t) :: String.t + @spec escape(String.t()) :: String.t() def escape(string) when is_binary(string) do - :re.replace(string, @escape_pattern, "\\\\&", [:global, {:return, :binary}]) + string + |> escape(_length = 0, string) + |> IO.iodata_to_binary() + end + + @escapable '.^$*+?()[]{}|#-\\\t\n\v\f\r\s' + + defp escape(<>, length, original) when char in @escapable do + escape_char(rest, length, original, char) + end + + defp escape(<<_, rest::binary>>, length, original) do + escape(rest, length + 1, original) + end + + defp escape(<<>>, _length, original) do + original + end + + defp escape_char(<>, 0, _original, char) do + [?\\, char | escape(rest, 0, rest)] + end + + defp escape_char(<>, length, original, char) do + [binary_part(original, 0, length), ?\\, char | escape(rest, 0, rest)] end # Helpers @doc false # Unescape map function used by Macro.unescape_string. + def unescape_map(:newline), do: true def unescape_map(?f), do: ?\f def unescape_map(?n), do: ?\n def unescape_map(?r), do: ?\r def unescape_map(?t), do: ?\t def unescape_map(?v), do: ?\v def unescape_map(?a), do: ?\a - def unescape_map(_), do: false + def unescape_map(_), do: false # Private Helpers - defp translate_options(<>) do - IO.write :stderr, "The /g flag for regular expressions is no longer needed\n#{Exception.format_stacktrace}" - translate_options(t) + defp translate_options(<>, acc), do: translate_options(t, [:unicode, :ucp | acc]) + defp translate_options(<>, acc), do: translate_options(t, [:caseless | acc]) + defp translate_options(<>, acc), do: translate_options(t, [:extended | acc]) + defp translate_options(<>, acc), do: translate_options(t, [:firstline | acc]) + defp translate_options(<>, acc), do: translate_options(t, [:ungreedy | acc]) + + defp translate_options(<>, acc), + do: translate_options(t, [:dotall, {:newline, :anycrlf} | acc]) + + defp translate_options(<>, acc), do: translate_options(t, [:multiline | acc]) + + defp translate_options(<>, acc) do + IO.warn("the /r modifier in regular expressions is deprecated, please use /U instead") + translate_options(t, [:ungreedy | acc]) end - defp translate_options(<>), do: [:unicode|translate_options(t)] - defp translate_options(<>), do: [:caseless|translate_options(t)] - defp translate_options(<>), do: [:extended|translate_options(t)] - defp translate_options(<>), do: [:firstline|translate_options(t)] - defp translate_options(<>), do: [:ungreedy|translate_options(t)] - defp translate_options(<>), do: [:dotall, {:newline, :anycrlf}|translate_options(t)] - defp translate_options(<>), do: [:multiline|translate_options(t)] - defp translate_options(<<>>), do: [] - defp translate_options(rest), do: {:error, rest} + defp translate_options(<<>>, acc), do: acc + defp translate_options(rest, _acc), do: {:error, rest} end diff --git a/lib/elixir/lib/registry.ex b/lib/elixir/lib/registry.ex new file mode 100644 index 00000000000..8e7578a25b5 --- /dev/null +++ b/lib/elixir/lib/registry.ex @@ -0,0 +1,1596 @@ +defmodule Registry do + @moduledoc ~S""" + A local, decentralized and scalable key-value process storage. + + It allows developers to lookup one or more processes with a given key. + If the registry has `:unique` keys, a key points to 0 or 1 process. + If the registry allows `:duplicate` keys, a single key may point to any + number of processes. In both cases, different keys could identify the + same process. + + Each entry in the registry is associated to the process that has + registered the key. If the process crashes, the keys associated to that + process are automatically removed. All key comparisons in the registry + are done using the match operation (`===/2`). + + The registry can be used for different purposes, such as name lookups (using + the `:via` option), storing properties, custom dispatching rules, or a pubsub + implementation. We explore some of those use cases below. + + The registry may also be transparently partitioned, which provides + more scalable behaviour for running registries on highly concurrent + environments with thousands or millions of entries. + + ## Using in `:via` + + Once the registry is started with a given name using + `Registry.start_link/1`, it can be used to register and access named + processes using the `{:via, Registry, {registry, key}}` tuple: + + {:ok, _} = Registry.start_link(keys: :unique, name: Registry.ViaTest) + name = {:via, Registry, {Registry.ViaTest, "agent"}} + {:ok, _} = Agent.start_link(fn -> 0 end, name: name) + Agent.get(name, & &1) + #=> 0 + Agent.update(name, &(&1 + 1)) + Agent.get(name, & &1) + #=> 1 + + In the previous example, we were not interested in associating a value to the + process: + + Registry.lookup(Registry.ViaTest, "agent") + #=> [{self(), nil}] + + However, in some cases it may be desired to associate a value to the process + using the alternate `{:via, Registry, {registry, key, value}}` tuple: + + {:ok, _} = Registry.start_link(keys: :unique, name: Registry.ViaTest) + name = {:via, Registry, {Registry.ViaTest, "agent", :hello}} + {:ok, agent_pid} = Agent.start_link(fn -> 0 end, name: name) + Registry.lookup(Registry.ViaTest, "agent") + #=> [{agent_pid, :hello}] + + To this point, we have been starting `Registry` using `start_link/1`. + Typically the registry is started as part of a supervision tree though: + + {Registry, keys: :unique, name: Registry.ViaTest} + + Only registries with unique keys can be used in `:via`. If the name is + already taken, the case-specific `start_link` function (`Agent.start_link/2` + in the example above) will return `{:error, {:already_started, current_pid}}`. + + ## Using as a dispatcher + + `Registry` has a dispatch mechanism that allows developers to implement custom + dispatch logic triggered from the caller. For example, let's say we have a + duplicate registry started as so: + + {:ok, _} = Registry.start_link(keys: :duplicate, name: Registry.DispatcherTest) + + By calling `register/3`, different processes can register under a given key + and associate any value under that key. In this case, let's register the + current process under the key `"hello"` and attach the `{IO, :inspect}` tuple + to it: + + {:ok, _} = Registry.register(Registry.DispatcherTest, "hello", {IO, :inspect}) + + Now, an entity interested in dispatching events for a given key may call + `dispatch/3` passing in the key and a callback. This callback will be invoked + with a list of all the values registered under the requested key, alongside + the PID of the process that registered each value, in the form of `{pid, + value}` tuples. In our example, `value` will be the `{module, function}` tuple + in the code above: + + Registry.dispatch(Registry.DispatcherTest, "hello", fn entries -> + for {pid, {module, function}} <- entries, do: apply(module, function, [pid]) + end) + # Prints #PID<...> where the PID is for the process that called register/3 above + #=> :ok + + Dispatching happens in the process that calls `dispatch/3` either serially or + concurrently in case of multiple partitions (via spawned tasks). The + registered processes are not involved in dispatching unless involving them is + done explicitly (for example, by sending them a message in the callback). + + Furthermore, if there is a failure when dispatching, due to a bad + registration, dispatching will always fail and the registered process will not + be notified. Therefore let's make sure we at least wrap and report those + errors: + + require Logger + + Registry.dispatch(Registry.DispatcherTest, "hello", fn entries -> + for {pid, {module, function}} <- entries do + try do + apply(module, function, [pid]) + catch + kind, reason -> + formatted = Exception.format(kind, reason, __STACKTRACE__) + Logger.error("Registry.dispatch/3 failed with #{formatted}") + end + end + end) + # Prints #PID<...> + #=> :ok + + You could also replace the whole `apply` system by explicitly sending + messages. That's the example we will see next. + + ## Using as a PubSub + + Registries can also be used to implement a local, non-distributed, scalable + PubSub by relying on the `dispatch/3` function, similarly to the previous + section: in this case, however, we will send messages to each associated + process, instead of invoking a given module-function. + + In this example, we will also set the number of partitions to the number of + schedulers online, which will make the registry more performant on highly + concurrent environments: + + {:ok, _} = + Registry.start_link( + keys: :duplicate, + name: Registry.PubSubTest, + partitions: System.schedulers_online() + ) + + {:ok, _} = Registry.register(Registry.PubSubTest, "hello", []) + + Registry.dispatch(Registry.PubSubTest, "hello", fn entries -> + for {pid, _} <- entries, do: send(pid, {:broadcast, "world"}) + end) + #=> :ok + + The example above broadcasted the message `{:broadcast, "world"}` to all + processes registered under the "topic" (or "key" as we called it until now) + `"hello"`. + + The third argument given to `register/3` is a value associated to the + current process. While in the previous section we used it when dispatching, + in this particular example we are not interested in it, so we have set it + to an empty list. You could store a more meaningful value if necessary. + + ## Registrations + + Looking up, dispatching and registering are efficient and immediate at + the cost of delayed unsubscription. For example, if a process crashes, + its keys are automatically removed from the registry but the change may + not propagate immediately. This means certain operations may return processes + that are already dead. When such may happen, it will be explicitly stated + in the function documentation. + + However, keep in mind those cases are typically not an issue. After all, a + process referenced by a PID may crash at any time, including between getting + the value from the registry and sending it a message. Many parts of the standard + library are designed to cope with that, such as `Process.monitor/1` which will + deliver the `:DOWN` message immediately if the monitored process is already dead + and `send/2` which acts as a no-op for dead processes. + + ## ETS + + Note that the registry uses one ETS table plus two ETS tables per partition. + """ + + @keys [:unique, :duplicate] + @all_info -1 + @key_info -2 + + @typedoc "The registry identifier" + @type registry :: atom + + @typedoc "The type of the registry" + @type keys :: :unique | :duplicate + + @typedoc "The type of keys allowed on registration" + @type key :: term + + @typedoc "The type of values allowed on registration" + @type value :: term + + @typedoc "The type of registry metadata keys" + @type meta_key :: atom | tuple + + @typedoc "The type of registry metadata values" + @type meta_value :: term + + @typedoc "A pattern to match on objects in a registry" + @type match_pattern :: atom | term + + @typedoc "A guard to be evaluated when matching on objects in a registry" + @type guard :: atom | tuple + + @typedoc "A list of guards to be evaluated when matching on objects in a registry" + @type guards :: [guard] + + @typedoc "A pattern used to representing the output format part of a match spec" + @type body :: [term] + + @typedoc "A full match spec used when selecting objects in the registry" + @type spec :: [{match_pattern, guards, body}] + + @typedoc "Options used for `child_spec/1` and `start_link/1`" + @type start_option :: + {:keys, keys} + | {:name, registry} + | {:partitions, pos_integer} + | {:listeners, [atom]} + | {:meta, [{meta_key, meta_value}]} + + ## Via callbacks + + @doc false + def whereis_name({registry, key}), do: whereis_name(registry, key) + def whereis_name({registry, key, _value}), do: whereis_name(registry, key) + + defp whereis_name(registry, key) do + case key_info!(registry) do + {:unique, partitions, key_ets} -> + key_ets = key_ets || key_ets!(registry, key, partitions) + + case safe_lookup_second(key_ets, key) do + {pid, _} -> + if Process.alive?(pid), do: pid, else: :undefined + + _ -> + :undefined + end + + {kind, _, _} -> + raise ArgumentError, ":via is not supported for #{kind} registries" + end + end + + @doc false + def register_name({registry, key}, pid), do: register_name(registry, key, nil, pid) + def register_name({registry, key, value}, pid), do: register_name(registry, key, value, pid) + + defp register_name(registry, key, value, pid) when pid == self() do + case register(registry, key, value) do + {:ok, _} -> :yes + {:error, _} -> :no + end + end + + @doc false + def send({registry, key}, msg) do + case lookup(registry, key) do + [{pid, _}] -> Kernel.send(pid, msg) + [] -> :erlang.error(:badarg, [{registry, key}, msg]) + end + end + + def send({registry, key, _value}, msg) do + Registry.send({registry, key}, msg) + end + + @doc false + def unregister_name({registry, key}), do: unregister(registry, key) + def unregister_name({registry, key, _value}), do: unregister(registry, key) + + ## Registry API + + @doc """ + Starts the registry as a supervisor process. + + Manually it can be started as: + + Registry.start_link(keys: :unique, name: MyApp.Registry) + + In your supervisor tree, you would write: + + Supervisor.start_link([ + {Registry, keys: :unique, name: MyApp.Registry} + ], strategy: :one_for_one) + + For intensive workloads, the registry may also be partitioned (by specifying + the `:partitions` option). If partitioning is required then a good default is to + set the number of partitions to the number of schedulers available: + + Registry.start_link( + keys: :unique, + name: MyApp.Registry, + partitions: System.schedulers_online() + ) + + or: + + Supervisor.start_link([ + {Registry, keys: :unique, name: MyApp.Registry, partitions: System.schedulers_online()} + ], strategy: :one_for_one) + + ## Options + + The registry requires the following keys: + + * `:keys` - chooses if keys are `:unique` or `:duplicate` + * `:name` - the name of the registry and its tables + + The following keys are optional: + + * `:partitions` - the number of partitions in the registry. Defaults to `1`. + * `:listeners` - a list of named processes which are notified of `:register` + and `:unregister` events. The registered process must be monitored by the + listener if the listener wants to be notified if the registered process + crashes. + * `:meta` - a keyword list of metadata to be attached to the registry. + + """ + @doc since: "1.5.0" + @spec start_link([start_option]) :: {:ok, pid} | {:error, term} + def start_link(options) do + keys = Keyword.get(options, :keys) + + unless keys in @keys do + raise ArgumentError, + "expected :keys to be given and be one of :unique or :duplicate, got: #{inspect(keys)}" + end + + name = + case Keyword.fetch(options, :name) do + {:ok, name} when is_atom(name) -> + name + + {:ok, other} -> + raise ArgumentError, "expected :name to be an atom, got: #{inspect(other)}" + + :error -> + raise ArgumentError, "expected :name option to be present" + end + + meta = Keyword.get(options, :meta, []) + + unless Keyword.keyword?(meta) do + raise ArgumentError, "expected :meta to be a keyword list, got: #{inspect(meta)}" + end + + partitions = Keyword.get(options, :partitions, 1) + + unless is_integer(partitions) and partitions >= 1 do + raise ArgumentError, + "expected :partitions to be a positive integer, got: #{inspect(partitions)}" + end + + listeners = Keyword.get(options, :listeners, []) + + unless is_list(listeners) and Enum.all?(listeners, &is_atom/1) do + raise ArgumentError, + "expected :listeners to be a list of named processes, got: #{inspect(listeners)}" + end + + compressed = Keyword.get(options, :compressed, false) + + unless is_boolean(compressed) do + raise ArgumentError, + "expected :compressed to be a boolean, got: #{inspect(compressed)}" + end + + # The @info format must be kept in sync with Registry.Partition optimization. + entries = [ + {@all_info, {keys, partitions, nil, nil, listeners}}, + {@key_info, {keys, partitions, nil}} | meta + ] + + Registry.Supervisor.start_link(keys, name, partitions, listeners, entries, compressed) + end + + @doc false + @deprecated "Use Registry.start_link/1 instead" + def start_link(keys, name, options \\ []) when keys in @keys and is_atom(name) do + start_link([keys: keys, name: name] ++ options) + end + + @doc """ + Returns a specification to start a registry under a supervisor. + + See `Supervisor`. + """ + @doc since: "1.5.0" + @spec child_spec([start_option]) :: Supervisor.child_spec() + def child_spec(options) do + %{ + id: Keyword.get(options, :name, Registry), + start: {Registry, :start_link, [options]}, + type: :supervisor + } + end + + @doc """ + Updates the value for `key` for the current process in the unique `registry`. + + Returns a `{new_value, old_value}` tuple or `:error` if there + is no such key assigned to the current process. + + If a non-unique registry is given, an error is raised. + + ## Examples + + iex> Registry.start_link(keys: :unique, name: Registry.UpdateTest) + iex> {:ok, _} = Registry.register(Registry.UpdateTest, "hello", 1) + iex> Registry.lookup(Registry.UpdateTest, "hello") + [{self(), 1}] + iex> Registry.update_value(Registry.UpdateTest, "hello", &(&1 + 1)) + {2, 1} + iex> Registry.lookup(Registry.UpdateTest, "hello") + [{self(), 2}] + + """ + @doc since: "1.4.0" + @spec update_value(registry, key, (value -> value)) :: + {new_value :: term, old_value :: term} | :error + def update_value(registry, key, callback) when is_atom(registry) and is_function(callback, 1) do + case key_info!(registry) do + {:unique, partitions, key_ets} -> + key_ets = key_ets || key_ets!(registry, key, partitions) + + try do + :ets.lookup_element(key_ets, key, 2) + catch + :error, :badarg -> :error + else + {pid, old_value} when pid == self() -> + new_value = callback.(old_value) + :ets.insert(key_ets, {key, {pid, new_value}}) + {new_value, old_value} + + {_, _} -> + :error + end + + {kind, _, _} -> + raise ArgumentError, "Registry.update_value/3 is not supported for #{kind} registries" + end + end + + @doc """ + Invokes the callback with all entries under `key` in each partition + for the given `registry`. + + The list of `entries` is a non-empty list of two-element tuples where + the first element is the PID and the second element is the value + associated to the PID. If there are no entries for the given key, + the callback is never invoked. + + If the registry is partitioned, the callback is invoked multiple times + per partition. If the registry is partitioned and `parallel: true` is + given as an option, the dispatching happens in parallel. In both cases, + the callback is only invoked if there are entries for that partition. + + See the module documentation for examples of using the `dispatch/3` + function for building custom dispatching or a pubsub system. + """ + @doc since: "1.4.0" + @spec dispatch(registry, key, dispatcher, keyword) :: :ok + when dispatcher: (entries :: [{pid, value}] -> term) | {module(), atom(), [any()]} + def dispatch(registry, key, mfa_or_fun, opts \\ []) + when is_atom(registry) and is_function(mfa_or_fun, 1) + when is_atom(registry) and tuple_size(mfa_or_fun) == 3 do + case key_info!(registry) do + {:unique, partitions, key_ets} -> + (key_ets || key_ets!(registry, key, partitions)) + |> safe_lookup_second(key) + |> List.wrap() + |> apply_non_empty_to_mfa_or_fun(mfa_or_fun) + + {:duplicate, 1, key_ets} -> + key_ets + |> safe_lookup_second(key) + |> apply_non_empty_to_mfa_or_fun(mfa_or_fun) + + {:duplicate, partitions, _} -> + if Keyword.get(opts, :parallel, false) do + registry + |> dispatch_parallel(key, mfa_or_fun, partitions) + |> Enum.each(&Task.await(&1, :infinity)) + else + dispatch_serial(registry, key, mfa_or_fun, partitions) + end + end + + :ok + end + + defp dispatch_serial(_registry, _key, _mfa_or_fun, 0) do + :ok + end + + defp dispatch_serial(registry, key, mfa_or_fun, partition) do + partition = partition - 1 + + registry + |> key_ets!(partition) + |> safe_lookup_second(key) + |> apply_non_empty_to_mfa_or_fun(mfa_or_fun) + + dispatch_serial(registry, key, mfa_or_fun, partition) + end + + defp dispatch_parallel(_registry, _key, _mfa_or_fun, 0) do + [] + end + + defp dispatch_parallel(registry, key, mfa_or_fun, partition) do + partition = partition - 1 + parent = self() + + task = + Task.async(fn -> + registry + |> key_ets!(partition) + |> safe_lookup_second(key) + |> apply_non_empty_to_mfa_or_fun(mfa_or_fun) + + Process.unlink(parent) + :ok + end) + + [task | dispatch_parallel(registry, key, mfa_or_fun, partition)] + end + + defp apply_non_empty_to_mfa_or_fun([], _mfa_or_fun) do + :ok + end + + defp apply_non_empty_to_mfa_or_fun(entries, {module, function, args}) do + apply(module, function, [entries | args]) + end + + defp apply_non_empty_to_mfa_or_fun(entries, fun) do + fun.(entries) + end + + @doc """ + Finds the `{pid, value}` pair for the given `key` in `registry` in no particular order. + + An empty list if there is no match. + + For unique registries, a single partition lookup is necessary. For + duplicate registries, all partitions must be looked up. + + ## Examples + + In the example below we register the current process and look it up + both from itself and other processes: + + iex> Registry.start_link(keys: :unique, name: Registry.UniqueLookupTest) + iex> Registry.lookup(Registry.UniqueLookupTest, "hello") + [] + iex> {:ok, _} = Registry.register(Registry.UniqueLookupTest, "hello", :world) + iex> Registry.lookup(Registry.UniqueLookupTest, "hello") + [{self(), :world}] + iex> Task.async(fn -> Registry.lookup(Registry.UniqueLookupTest, "hello") end) |> Task.await() + [{self(), :world}] + + The same applies to duplicate registries: + + iex> Registry.start_link(keys: :duplicate, name: Registry.DuplicateLookupTest) + iex> Registry.lookup(Registry.DuplicateLookupTest, "hello") + [] + iex> {:ok, _} = Registry.register(Registry.DuplicateLookupTest, "hello", :world) + iex> Registry.lookup(Registry.DuplicateLookupTest, "hello") + [{self(), :world}] + iex> {:ok, _} = Registry.register(Registry.DuplicateLookupTest, "hello", :another) + iex> Enum.sort(Registry.lookup(Registry.DuplicateLookupTest, "hello")) + [{self(), :another}, {self(), :world}] + + """ + @doc since: "1.4.0" + @spec lookup(registry, key) :: [{pid, value}] + def lookup(registry, key) when is_atom(registry) do + case key_info!(registry) do + {:unique, partitions, key_ets} -> + key_ets = key_ets || key_ets!(registry, key, partitions) + + case safe_lookup_second(key_ets, key) do + {_, _} = pair -> + [pair] + + _ -> + [] + end + + {:duplicate, 1, key_ets} -> + safe_lookup_second(key_ets, key) + + {:duplicate, partitions, _key_ets} -> + for partition <- 0..(partitions - 1), + pair <- safe_lookup_second(key_ets!(registry, partition), key), + do: pair + end + end + + @doc """ + Returns `{pid, value}` pairs under the given `key` in `registry` that match `pattern`. + + Pattern must be an atom or a tuple that will match the structure of the + value stored in the registry. The atom `:_` can be used to ignore a given + value or tuple element, while the atom `:"$1"` can be used to temporarily assign part + of pattern to a variable for a subsequent comparison. + + Optionally, it is possible to pass a list of guard conditions for more precise matching. + Each guard is a tuple, which describes checks that should be passed by assigned part of pattern. + For example the `$1 > 1` guard condition would be expressed as the `{:>, :"$1", 1}` tuple. + Please note that guard conditions will work only for assigned + variables like `:"$1"`, `:"$2"`, and so forth. + Avoid usage of special match variables `:"$_"` and `:"$$"`, because it might not work as expected. + + An empty list will be returned if there is no match. + + For unique registries, a single partition lookup is necessary. For + duplicate registries, all partitions must be looked up. + + ## Examples + + In the example below we register the current process under the same + key in a duplicate registry but with different values: + + iex> Registry.start_link(keys: :duplicate, name: Registry.MatchTest) + iex> {:ok, _} = Registry.register(Registry.MatchTest, "hello", {1, :atom, 1}) + iex> {:ok, _} = Registry.register(Registry.MatchTest, "hello", {2, :atom, 2}) + iex> Registry.match(Registry.MatchTest, "hello", {1, :_, :_}) + [{self(), {1, :atom, 1}}] + iex> Registry.match(Registry.MatchTest, "hello", {2, :_, :_}) + [{self(), {2, :atom, 2}}] + iex> Registry.match(Registry.MatchTest, "hello", {:_, :atom, :_}) |> Enum.sort() + [{self(), {1, :atom, 1}}, {self(), {2, :atom, 2}}] + iex> Registry.match(Registry.MatchTest, "hello", {:"$1", :_, :"$1"}) |> Enum.sort() + [{self(), {1, :atom, 1}}, {self(), {2, :atom, 2}}] + iex> guards = [{:>, :"$1", 1}] + iex> Registry.match(Registry.MatchTest, "hello", {:_, :_, :"$1"}, guards) + [{self(), {2, :atom, 2}}] + iex> guards = [{:is_atom, :"$1"}] + iex> Registry.match(Registry.MatchTest, "hello", {:_, :"$1", :_}, guards) |> Enum.sort() + [{self(), {1, :atom, 1}}, {self(), {2, :atom, 2}}] + + """ + @doc since: "1.4.0" + @spec match(registry, key, match_pattern, guards) :: [{pid, term}] + def match(registry, key, pattern, guards \\ []) when is_atom(registry) and is_list(guards) do + guards = [{:"=:=", {:element, 1, :"$_"}, {:const, key}} | guards] + spec = [{{:_, {:_, pattern}}, guards, [{:element, 2, :"$_"}]}] + + case key_info!(registry) do + {:unique, partitions, key_ets} -> + key_ets = key_ets || key_ets!(registry, key, partitions) + :ets.select(key_ets, spec) + + {:duplicate, 1, key_ets} -> + :ets.select(key_ets, spec) + + {:duplicate, partitions, _key_ets} -> + for partition <- 0..(partitions - 1), + pair <- :ets.select(key_ets!(registry, partition), spec), + do: pair + end + end + + @doc """ + Returns the known keys for the given `pid` in `registry` in no particular order. + + If the registry is unique, the keys are unique. Otherwise + they may contain duplicates if the process was registered + under the same key multiple times. The list will be empty + if the process is dead or it has no keys in this registry. + + ## Examples + + Registering under a unique registry does not allow multiple entries: + + iex> Registry.start_link(keys: :unique, name: Registry.UniqueKeysTest) + iex> Registry.keys(Registry.UniqueKeysTest, self()) + [] + iex> {:ok, _} = Registry.register(Registry.UniqueKeysTest, "hello", :world) + iex> Registry.register(Registry.UniqueKeysTest, "hello", :later) # registry is :unique + {:error, {:already_registered, self()}} + iex> Registry.keys(Registry.UniqueKeysTest, self()) + ["hello"] + + Such is possible for duplicate registries though: + + iex> Registry.start_link(keys: :duplicate, name: Registry.DuplicateKeysTest) + iex> Registry.keys(Registry.DuplicateKeysTest, self()) + [] + iex> {:ok, _} = Registry.register(Registry.DuplicateKeysTest, "hello", :world) + iex> {:ok, _} = Registry.register(Registry.DuplicateKeysTest, "hello", :world) + iex> Registry.keys(Registry.DuplicateKeysTest, self()) + ["hello", "hello"] + + """ + @doc since: "1.4.0" + @spec keys(registry, pid) :: [key] + def keys(registry, pid) when is_atom(registry) and is_pid(pid) do + {kind, partitions, _, pid_ets, _} = info!(registry) + {_, pid_ets} = pid_ets || pid_ets!(registry, pid, partitions) + + keys = + try do + spec = [{{pid, :"$1", :"$2", :_}, [], [{{:"$1", :"$2"}}]}] + :ets.select(pid_ets, spec) + catch + :error, :badarg -> [] + end + + # Handle the possibility of fake keys + keys = gather_keys(keys, [], false) + + cond do + kind == :unique -> Enum.uniq(keys) + true -> keys + end + end + + defp gather_keys([{key, {_, remaining}} | rest], acc, _fake) do + gather_keys(rest, [key | acc], {key, remaining}) + end + + defp gather_keys([{key, _} | rest], acc, fake) do + gather_keys(rest, [key | acc], fake) + end + + defp gather_keys([], acc, {key, remaining}) do + List.duplicate(key, remaining) ++ Enum.reject(acc, &(&1 === key)) + end + + defp gather_keys([], acc, false) do + acc + end + + @doc """ + Reads the values for the given `key` for `pid` in `registry`. + + For unique registries, it is either an empty list or a list + with a single element. For duplicate registries, it is a list + with zero, one, or multiple elements. + + ## Examples + + In the example below we register the current process and look it up + both from itself and other processes: + + iex> Registry.start_link(keys: :unique, name: Registry.UniqueLookupTest) + iex> Registry.values(Registry.UniqueLookupTest, "hello", self()) + [] + iex> {:ok, _} = Registry.register(Registry.UniqueLookupTest, "hello", :world) + iex> Registry.values(Registry.UniqueLookupTest, "hello", self()) + [:world] + iex> Task.async(fn -> Registry.values(Registry.UniqueLookupTest, "hello", self()) end) |> Task.await() + [] + iex> parent = self() + iex> Task.async(fn -> Registry.values(Registry.UniqueLookupTest, "hello", parent) end) |> Task.await() + [:world] + + The same applies to duplicate registries: + + iex> Registry.start_link(keys: :duplicate, name: Registry.DuplicateLookupTest) + iex> Registry.values(Registry.DuplicateLookupTest, "hello", self()) + [] + iex> {:ok, _} = Registry.register(Registry.DuplicateLookupTest, "hello", :world) + iex> Registry.values(Registry.DuplicateLookupTest, "hello", self()) + [:world] + iex> {:ok, _} = Registry.register(Registry.DuplicateLookupTest, "hello", :another) + iex> Enum.sort(Registry.values(Registry.DuplicateLookupTest, "hello", self())) + [:another, :world] + + """ + @doc since: "1.12.0" + @spec values(registry, key, pid) :: [value] + def values(registry, key, pid) when is_atom(registry) do + case key_info!(registry) do + {:unique, partitions, key_ets} -> + key_ets = key_ets || key_ets!(registry, key, partitions) + + case safe_lookup_second(key_ets, key) do + {^pid, value} -> + [value] + + _ -> + [] + end + + {:duplicate, partitions, key_ets} -> + key_ets = key_ets || key_ets!(registry, pid, partitions) + for {^pid, value} <- safe_lookup_second(key_ets, key), do: value + end + end + + @doc """ + Unregisters all entries for the given `key` associated to the current + process in `registry`. + + Always returns `:ok` and automatically unlinks the current process from + the owner if there are no more keys associated to the current process. See + also `register/3` to read more about the "owner". + + ## Examples + + For unique registries: + + iex> Registry.start_link(keys: :unique, name: Registry.UniqueUnregisterTest) + iex> Registry.register(Registry.UniqueUnregisterTest, "hello", :world) + iex> Registry.keys(Registry.UniqueUnregisterTest, self()) + ["hello"] + iex> Registry.unregister(Registry.UniqueUnregisterTest, "hello") + :ok + iex> Registry.keys(Registry.UniqueUnregisterTest, self()) + [] + + For duplicate registries: + + iex> Registry.start_link(keys: :duplicate, name: Registry.DuplicateUnregisterTest) + iex> Registry.register(Registry.DuplicateUnregisterTest, "hello", :world) + iex> Registry.register(Registry.DuplicateUnregisterTest, "hello", :world) + iex> Registry.keys(Registry.DuplicateUnregisterTest, self()) + ["hello", "hello"] + iex> Registry.unregister(Registry.DuplicateUnregisterTest, "hello") + :ok + iex> Registry.keys(Registry.DuplicateUnregisterTest, self()) + [] + + """ + @doc since: "1.4.0" + @spec unregister(registry, key) :: :ok + def unregister(registry, key) when is_atom(registry) do + self = self() + {kind, partitions, key_ets, pid_ets, listeners} = info!(registry) + {key_partition, pid_partition} = partitions(kind, key, self, partitions) + key_ets = key_ets || key_ets!(registry, key_partition) + {pid_server, pid_ets} = pid_ets || pid_ets!(registry, pid_partition) + + # Remove first from the key_ets because in case of crashes + # the pid_ets will still be able to clean up. The last step is + # to clean if we have no more entries. + true = __unregister__(key_ets, {key, {self, :_}}, 1) + true = __unregister__(pid_ets, {self, key, key_ets, :_}, 2) + + unlink_if_unregistered(pid_server, pid_ets, self) + + for listener <- listeners do + Kernel.send(listener, {:unregister, registry, key, self}) + end + + :ok + end + + @doc """ + Unregisters entries for keys matching a pattern associated to the current + process in `registry`. + + ## Examples + + For unique registries it can be used to conditionally unregister a key on + the basis of whether or not it matches a particular value. + + iex> Registry.start_link(keys: :unique, name: Registry.UniqueUnregisterMatchTest) + iex> Registry.register(Registry.UniqueUnregisterMatchTest, "hello", :world) + iex> Registry.keys(Registry.UniqueUnregisterMatchTest, self()) + ["hello"] + iex> Registry.unregister_match(Registry.UniqueUnregisterMatchTest, "hello", :foo) + :ok + iex> Registry.keys(Registry.UniqueUnregisterMatchTest, self()) + ["hello"] + iex> Registry.unregister_match(Registry.UniqueUnregisterMatchTest, "hello", :world) + :ok + iex> Registry.keys(Registry.UniqueUnregisterMatchTest, self()) + [] + + For duplicate registries: + + iex> Registry.start_link(keys: :duplicate, name: Registry.DuplicateUnregisterMatchTest) + iex> Registry.register(Registry.DuplicateUnregisterMatchTest, "hello", :world_a) + iex> Registry.register(Registry.DuplicateUnregisterMatchTest, "hello", :world_b) + iex> Registry.register(Registry.DuplicateUnregisterMatchTest, "hello", :world_c) + iex> Registry.keys(Registry.DuplicateUnregisterMatchTest, self()) + ["hello", "hello", "hello"] + iex> Registry.unregister_match(Registry.DuplicateUnregisterMatchTest, "hello", :world_a) + :ok + iex> Registry.keys(Registry.DuplicateUnregisterMatchTest, self()) + ["hello", "hello"] + iex> Registry.lookup(Registry.DuplicateUnregisterMatchTest, "hello") + [{self(), :world_b}, {self(), :world_c}] + + """ + @doc since: "1.5.0" + @spec unregister_match(registry, key, match_pattern, guards) :: :ok + def unregister_match(registry, key, pattern, guards \\ []) when is_list(guards) do + self = self() + + {kind, partitions, key_ets, pid_ets, listeners} = info!(registry) + {key_partition, pid_partition} = partitions(kind, key, self, partitions) + key_ets = key_ets || key_ets!(registry, key_partition) + {pid_server, pid_ets} = pid_ets || pid_ets!(registry, pid_partition) + + # Remove first from the key_ets because in case of crashes + # the pid_ets will still be able to clean up. The last step is + # to clean if we have no more entries. + + # Here we want to count all entries for this pid under this key, regardless of pattern. + underscore_guard = {:"=:=", {:element, 1, :"$_"}, {:const, key}} + total_spec = [{{:_, {self, :_}}, [underscore_guard], [true]}] + total = :ets.select_count(key_ets, total_spec) + + # We only want to delete things that match the pattern + delete_spec = [{{:_, {self, pattern}}, [underscore_guard | guards], [true]}] + + case :ets.select_delete(key_ets, delete_spec) do + # We deleted everything, we can just delete the object + ^total -> + true = __unregister__(pid_ets, {self, key, key_ets, :_}, 2) + unlink_if_unregistered(pid_server, pid_ets, self) + + for listener <- listeners do + Kernel.send(listener, {:unregister, registry, key, self}) + end + + 0 -> + :ok + + deleted -> + # There are still entries remaining for this pid. delete_object/2 with + # duplicate_bag tables will remove every entry, but we only want to + # remove those we have deleted. The solution is to introduce a temp_entry + # that indicates how many keys WILL be remaining after the delete operation. + counter = System.unique_integer() + remaining = total - deleted + temp_entry = {self, key, {key_ets, remaining}, counter} + true = :ets.insert(pid_ets, temp_entry) + true = __unregister__(pid_ets, {self, key, key_ets, :_}, 2) + real_keys = List.duplicate({self, key, key_ets, counter}, remaining) + true = :ets.insert(pid_ets, real_keys) + # We've recreated the real remaining key entries, so we can now delete + # our temporary entry. + true = :ets.delete_object(pid_ets, temp_entry) + end + + :ok + end + + @doc """ + Registers the current process under the given `key` in `registry`. + + A value to be associated with this registration must also be given. + This value will be retrieved whenever dispatching or doing a key + lookup. + + This function returns `{:ok, owner}` or `{:error, reason}`. + The `owner` is the PID in the registry partition responsible for + the PID. The owner is automatically linked to the caller. + + If the registry has unique keys, it will return `{:ok, owner}` unless + the key is already associated to a PID, in which case it returns + `{:error, {:already_registered, pid}}`. + + If the registry has duplicate keys, multiple registrations from the + current process under the same key are allowed. + + ## Examples + + Registering under a unique registry does not allow multiple entries: + + iex> Registry.start_link(keys: :unique, name: Registry.UniqueRegisterTest) + iex> {:ok, _} = Registry.register(Registry.UniqueRegisterTest, "hello", :world) + iex> Registry.register(Registry.UniqueRegisterTest, "hello", :later) + {:error, {:already_registered, self()}} + iex> Registry.keys(Registry.UniqueRegisterTest, self()) + ["hello"] + + Such is possible for duplicate registries though: + + iex> Registry.start_link(keys: :duplicate, name: Registry.DuplicateRegisterTest) + iex> {:ok, _} = Registry.register(Registry.DuplicateRegisterTest, "hello", :world) + iex> {:ok, _} = Registry.register(Registry.DuplicateRegisterTest, "hello", :world) + iex> Registry.keys(Registry.DuplicateRegisterTest, self()) + ["hello", "hello"] + + """ + @doc since: "1.4.0" + @spec register(registry, key, value) :: {:ok, pid} | {:error, {:already_registered, pid}} + def register(registry, key, value) when is_atom(registry) do + self = self() + {kind, partitions, key_ets, pid_ets, listeners} = info!(registry) + {key_partition, pid_partition} = partitions(kind, key, self, partitions) + key_ets = key_ets || key_ets!(registry, key_partition) + {pid_server, pid_ets} = pid_ets || pid_ets!(registry, pid_partition) + + # Note that we write first to the pid_ets table because it will + # always be able to do the cleanup. If we register first to the + # key one and the process crashes, the key will stay there forever. + Process.link(pid_server) + + counter = System.unique_integer() + true = :ets.insert(pid_ets, {self, key, key_ets, counter}) + + case register_key(kind, key_ets, key, {key, {self, value}}) do + :ok -> + for listener <- listeners do + Kernel.send(listener, {:register, registry, key, self, value}) + end + + {:ok, pid_server} + + {:error, {:already_registered, ^self}} = error -> + true = :ets.delete_object(pid_ets, {self, key, key_ets, counter}) + error + + {:error, _} = error -> + true = :ets.delete_object(pid_ets, {self, key, key_ets, counter}) + unlink_if_unregistered(pid_server, pid_ets, self) + error + end + end + + defp register_key(:duplicate, key_ets, _key, entry) do + true = :ets.insert(key_ets, entry) + :ok + end + + defp register_key(:unique, key_ets, key, entry) do + if :ets.insert_new(key_ets, entry) do + :ok + else + # Note that we have to call register_key recursively + # because we are always at odds of a race condition. + case :ets.lookup(key_ets, key) do + [{^key, {pid, _}} = current] -> + if Process.alive?(pid) do + {:error, {:already_registered, pid}} + else + :ets.delete_object(key_ets, current) + register_key(:unique, key_ets, key, entry) + end + + [] -> + register_key(:unique, key_ets, key, entry) + end + end + end + + @doc """ + Reads registry metadata given on `start_link/1`. + + Atoms and tuples are allowed as keys. + + ## Examples + + iex> Registry.start_link(keys: :unique, name: Registry.MetaTest, meta: [custom_key: "custom_value"]) + iex> Registry.meta(Registry.MetaTest, :custom_key) + {:ok, "custom_value"} + iex> Registry.meta(Registry.MetaTest, :unknown_key) + :error + + """ + @doc since: "1.4.0" + @spec meta(registry, meta_key) :: {:ok, meta_value} | :error + def meta(registry, key) when is_atom(registry) and (is_atom(key) or is_tuple(key)) do + try do + :ets.lookup(registry, key) + catch + :error, :badarg -> + raise ArgumentError, + "unknown registry: #{inspect(registry)}. Either the registry name is invalid or " <> + "the registry is not running, possibly because its application isn't started" + else + [{^key, value}] -> {:ok, value} + _ -> :error + end + end + + @doc """ + Stores registry metadata. + + Atoms and tuples are allowed as keys. + + ## Examples + + iex> Registry.start_link(keys: :unique, name: Registry.PutMetaTest) + iex> Registry.put_meta(Registry.PutMetaTest, :custom_key, "custom_value") + :ok + iex> Registry.meta(Registry.PutMetaTest, :custom_key) + {:ok, "custom_value"} + iex> Registry.put_meta(Registry.PutMetaTest, {:tuple, :key}, "tuple_value") + :ok + iex> Registry.meta(Registry.PutMetaTest, {:tuple, :key}) + {:ok, "tuple_value"} + + """ + @doc since: "1.4.0" + @spec put_meta(registry, meta_key, meta_value) :: :ok + def put_meta(registry, key, value) when is_atom(registry) and (is_atom(key) or is_tuple(key)) do + try do + :ets.insert(registry, {key, value}) + :ok + catch + :error, :badarg -> + raise ArgumentError, "unknown registry: #{inspect(registry)}" + end + end + + @doc """ + Deletes registry metadata for the given `key` in `registry`. + + ## Examples + + iex> Registry.start_link(keys: :unique, name: Registry.DeleteMetaTest) + iex> Registry.put_meta(Registry.DeleteMetaTest, :custom_key, "custom_value") + :ok + iex> Registry.meta(Registry.DeleteMetaTest, :custom_key) + {:ok, "custom_value"} + iex> Registry.delete_meta(Registry.DeleteMetaTest, :custom_key) + :ok + iex> Registry.meta(Registry.DeleteMetaTest, :custom_key) + :error + + """ + @doc since: "1.11.0" + @spec delete_meta(registry, meta_key) :: :ok + def delete_meta(registry, key) when is_atom(registry) and (is_atom(key) or is_tuple(key)) do + try do + :ets.delete(registry, key) + :ok + catch + :error, :badarg -> + raise ArgumentError, "unknown registry: #{inspect(registry)}" + end + end + + @doc """ + Returns the number of registered keys in a registry. + It runs in constant time. + + ## Examples + In the example below we register the current process and ask for the + number of keys in the registry: + + iex> Registry.start_link(keys: :unique, name: Registry.UniqueCountTest) + iex> Registry.count(Registry.UniqueCountTest) + 0 + iex> {:ok, _} = Registry.register(Registry.UniqueCountTest, "hello", :world) + iex> {:ok, _} = Registry.register(Registry.UniqueCountTest, "world", :world) + iex> Registry.count(Registry.UniqueCountTest) + 2 + + The same applies to duplicate registries: + + iex> Registry.start_link(keys: :duplicate, name: Registry.DuplicateCountTest) + iex> Registry.count(Registry.DuplicateCountTest) + 0 + iex> {:ok, _} = Registry.register(Registry.DuplicateCountTest, "hello", :world) + iex> {:ok, _} = Registry.register(Registry.DuplicateCountTest, "hello", :world) + iex> Registry.count(Registry.DuplicateCountTest) + 2 + + """ + @doc since: "1.7.0" + @spec count(registry) :: non_neg_integer() + def count(registry) when is_atom(registry) do + case key_info!(registry) do + {_kind, partitions, nil} -> + Enum.reduce(0..(partitions - 1), 0, fn partition_index, acc -> + acc + safe_size(key_ets!(registry, partition_index)) + end) + + {_kind, 1, key_ets} -> + safe_size(key_ets) + end + end + + defp safe_size(ets) do + try do + :ets.info(ets, :size) + catch + :error, :badarg -> 0 + end + end + + @doc """ + Returns the number of `{pid, value}` pairs under the given `key` in `registry` + that match `pattern`. + + Pattern must be an atom or a tuple that will match the structure of the + value stored in the registry. The atom `:_` can be used to ignore a given + value or tuple element, while the atom `:"$1"` can be used to temporarily assign part + of pattern to a variable for a subsequent comparison. + + Optionally, it is possible to pass a list of guard conditions for more precise matching. + Each guard is a tuple, which describes checks that should be passed by assigned part of pattern. + For example the `$1 > 1` guard condition would be expressed as the `{:>, :"$1", 1}` tuple. + Please note that guard conditions will work only for assigned + variables like `:"$1"`, `:"$2"`, and so forth. + Avoid usage of special match variables `:"$_"` and `:"$$"`, because it might not work as expected. + + Zero will be returned if there is no match. + + For unique registries, a single partition lookup is necessary. For + duplicate registries, all partitions must be looked up. + + ## Examples + + In the example below we register the current process under the same + key in a duplicate registry but with different values: + + iex> Registry.start_link(keys: :duplicate, name: Registry.CountMatchTest) + iex> {:ok, _} = Registry.register(Registry.CountMatchTest, "hello", {1, :atom, 1}) + iex> {:ok, _} = Registry.register(Registry.CountMatchTest, "hello", {2, :atom, 2}) + iex> Registry.count_match(Registry.CountMatchTest, "hello", {1, :_, :_}) + 1 + iex> Registry.count_match(Registry.CountMatchTest, "hello", {2, :_, :_}) + 1 + iex> Registry.count_match(Registry.CountMatchTest, "hello", {:_, :atom, :_}) + 2 + iex> Registry.count_match(Registry.CountMatchTest, "hello", {:"$1", :_, :"$1"}) + 2 + iex> Registry.count_match(Registry.CountMatchTest, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 1}]) + 1 + iex> Registry.count_match(Registry.CountMatchTest, "hello", {:_, :"$1", :_}, [{:is_atom, :"$1"}]) + 2 + + """ + @doc since: "1.7.0" + @spec count_match(registry, key, match_pattern, guards) :: non_neg_integer() + def count_match(registry, key, pattern, guards \\ []) + when is_atom(registry) and is_list(guards) do + guards = [{:"=:=", {:element, 1, :"$_"}, {:const, key}} | guards] + spec = [{{:_, {:_, pattern}}, guards, [true]}] + + case key_info!(registry) do + {:unique, partitions, key_ets} -> + key_ets = key_ets || key_ets!(registry, key, partitions) + :ets.select_count(key_ets, spec) + + {:duplicate, 1, key_ets} -> + :ets.select_count(key_ets, spec) + + {:duplicate, partitions, _key_ets} -> + Enum.reduce(0..(partitions - 1), 0, fn partition_index, acc -> + count = :ets.select_count(key_ets!(registry, partition_index), spec) + acc + count + end) + end + end + + @doc """ + Select key, pid, and values registered using full match specs. + + The `spec` consists of a list of three part tuples, in the shape of `[{match_pattern, guards, body}]`. + + The first part, the match pattern, must be a tuple that will match the structure of the + the data stored in the registry, which is `{key, pid, value}`. The atom `:_` can be used to + ignore a given value or tuple element, while the atom `:"$1"` can be used to temporarily + assign part of pattern to a variable for a subsequent comparison. This can be combined + like `{:"$1", :_, :_}`. + + The second part, the guards, is a list of conditions that allow filtering the results. + Each guard is a tuple, which describes checks that should be passed by assigned part of pattern. + For example the `$1 > 1` guard condition would be expressed as the `{:>, :"$1", 1}` tuple. + Please note that guard conditions will work only for assigned + variables like `:"$1"`, `:"$2"`, and so forth. + + The third part, the body, is a list of shapes of the returned entries. Like guards, you have access to + assigned variables like `:"$1"`, which you can combine with hardcoded values to freely shape entries + Note that tuples have to be wrapped in an additional tuple. To get a result format like + `%{key: key, pid: pid, value: value}`, assuming you bound those variables in order in the match part, + you would provide a body like `[%{key: :"$1", pid: :"$2", value: :"$3"}]`. Like guards, you can use + some operations like `:element` to modify the output format. + + Do not use special match variables `:"$_"` and `:"$$"`, because they might not work as expected. + + Note that for large registries with many partitions this will be costly as it builds the result by + concatenating all the partitions. + + ## Examples + + This example shows how to get everything from the registry: + + iex> Registry.start_link(keys: :unique, name: Registry.SelectAllTest) + iex> {:ok, _} = Registry.register(Registry.SelectAllTest, "hello", :value) + iex> {:ok, _} = Registry.register(Registry.SelectAllTest, "world", :value) + iex> Registry.select(Registry.SelectAllTest, [{{:"$1", :"$2", :"$3"}, [], [{{:"$1", :"$2", :"$3"}}]}]) + [{"world", self(), :value}, {"hello", self(), :value}] + + Get all keys in the registry: + + iex> Registry.start_link(keys: :unique, name: Registry.SelectAllTest) + iex> {:ok, _} = Registry.register(Registry.SelectAllTest, "hello", :value) + iex> {:ok, _} = Registry.register(Registry.SelectAllTest, "world", :value) + iex> Registry.select(Registry.SelectAllTest, [{{:"$1", :_, :_}, [], [:"$1"]}]) + ["world", "hello"] + + """ + @doc since: "1.9.0" + @spec select(registry, spec) :: [term] + def select(registry, spec) + when is_atom(registry) and is_list(spec) do + spec = group_match_headers(spec, __ENV__.function) + + case key_info!(registry) do + {_kind, partitions, nil} -> + Enum.flat_map(0..(partitions - 1), fn partition_index -> + :ets.select(key_ets!(registry, partition_index), spec) + end) + + {_kind, 1, key_ets} -> + :ets.select(key_ets, spec) + end + end + + @doc """ + Works like `select/2`, but only returns the number of matching records. + + ## Examples + + In the example below we register the current process under different + keys in a unique registry but with the same value: + + iex> Registry.start_link(keys: :unique, name: Registry.CountSelectTest) + iex> {:ok, _} = Registry.register(Registry.CountSelectTest, "hello", :value) + iex> {:ok, _} = Registry.register(Registry.CountSelectTest, "world", :value) + iex> Registry.count_select(Registry.CountSelectTest, [{{:_, :_, :value}, [], [true]}]) + 2 + """ + @doc since: "1.14.0" + @spec count_select(registry, spec) :: non_neg_integer() + def count_select(registry, spec) + when is_atom(registry) and is_list(spec) do + spec = group_match_headers(spec, __ENV__.function) + + case key_info!(registry) do + {_kind, partitions, nil} -> + Enum.reduce(0..(partitions - 1), 0, fn partition_index, acc -> + count = :ets.select_count(key_ets!(registry, partition_index), spec) + acc + count + end) + + {_kind, 1, key_ets} -> + :ets.select_count(key_ets, spec) + end + end + + defp group_match_headers(spec, {fun, arity}) do + for part <- spec do + case part do + {{key, pid, value}, guards, select} -> + {{key, {pid, value}}, guards, select} + + _ -> + raise ArgumentError, + "invalid match specification in Registry.#{fun}/#{arity}: #{inspect(spec)}" + end + end + end + + ## Helpers + + @compile {:inline, hash: 2} + + defp hash(term, limit) do + :erlang.phash2(term, limit) + end + + defp info!(registry) do + try do + :ets.lookup_element(registry, @all_info, 2) + catch + :error, :badarg -> + raise ArgumentError, "unknown registry: #{inspect(registry)}" + end + end + + defp key_info!(registry) do + try do + :ets.lookup_element(registry, @key_info, 2) + catch + :error, :badarg -> + raise ArgumentError, "unknown registry: #{inspect(registry)}" + end + end + + defp key_ets!(registry, key, partitions) do + :ets.lookup_element(registry, hash(key, partitions), 2) + end + + defp key_ets!(registry, partition) do + :ets.lookup_element(registry, partition, 2) + end + + defp pid_ets!(registry, key, partitions) do + :ets.lookup_element(registry, hash(key, partitions), 3) + end + + defp pid_ets!(registry, partition) do + :ets.lookup_element(registry, partition, 3) + end + + defp safe_lookup_second(ets, key) do + try do + :ets.lookup_element(ets, key, 2) + catch + :error, :badarg -> [] + end + end + + defp partitions(:unique, key, pid, partitions) do + {hash(key, partitions), hash(pid, partitions)} + end + + defp partitions(:duplicate, _key, pid, partitions) do + partition = hash(pid, partitions) + {partition, partition} + end + + defp unlink_if_unregistered(pid_server, pid_ets, self) do + unless :ets.member(pid_ets, self) do + Process.unlink(pid_server) + end + end + + @doc false + def __unregister__(table, match, pos) do + key = :erlang.element(pos, match) + + # We need to perform an element comparison if we have an special atom key. + if is_atom(key) and reserved_atom?(Atom.to_string(key)) do + match = :erlang.setelement(pos, match, :_) + guard = {:"=:=", {:element, pos, :"$_"}, {:const, key}} + :ets.select_delete(table, [{match, [guard], [true]}]) >= 0 + else + :ets.match_delete(table, match) + end + end + + defp reserved_atom?("_"), do: true + defp reserved_atom?("$" <> _), do: true + defp reserved_atom?(_), do: false +end + +defmodule Registry.Supervisor do + @moduledoc false + use Supervisor + + def start_link(kind, registry, partitions, listeners, entries, compressed) do + arg = {kind, registry, partitions, listeners, entries, compressed} + Supervisor.start_link(__MODULE__, arg, name: registry) + end + + def init({kind, registry, partitions, listeners, entries, compressed}) do + ^registry = :ets.new(registry, [:set, :public, :named_table, read_concurrency: true]) + true = :ets.insert(registry, entries) + + children = + for i <- 0..(partitions - 1) do + key_partition = Registry.Partition.key_name(registry, i) + pid_partition = Registry.Partition.pid_name(registry, i) + arg = {kind, registry, i, partitions, key_partition, pid_partition, listeners, compressed} + + %{ + id: pid_partition, + start: {Registry.Partition, :start_link, [pid_partition, arg]} + } + end + + Supervisor.init(children, strategy: strategy_for_kind(kind)) + end + + # Unique registries have their key partition hashed by key. + # This means that, if a PID partition crashes, it may have + # entries from all key partitions, so we need to crash all. + defp strategy_for_kind(:unique), do: :one_for_all + + # Duplicate registries have both key and pid partitions hashed + # by pid. This means that, if a PID partition crashes, all of + # its associated entries are in its sibling table, so we crash one. + defp strategy_for_kind(:duplicate), do: :one_for_one +end + +defmodule Registry.Partition do + @moduledoc false + + # This process owns the equivalent key and pid ETS tables + # and is responsible for monitoring processes that map to + # its own pid table. + use GenServer + @all_info -1 + @key_info -2 + + @doc """ + Returns the name of key partition table. + """ + @spec key_name(atom, non_neg_integer) :: atom + def key_name(registry, partition) do + Module.concat(registry, "KeyPartition" <> Integer.to_string(partition)) + end + + @doc """ + Returns the name of pid partition table. + """ + @spec pid_name(atom, non_neg_integer) :: atom + def pid_name(name, partition) do + Module.concat(name, "PIDPartition" <> Integer.to_string(partition)) + end + + @doc """ + Starts the registry partition. + + The process is only responsible for monitoring, demonitoring + and cleaning up when monitored processes crash. + """ + def start_link(registry, arg) do + GenServer.start_link(__MODULE__, arg, name: registry) + end + + ## Callbacks + + def init({kind, registry, i, partitions, key_partition, pid_partition, listeners, compressed}) do + Process.flag(:trap_exit, true) + key_ets = init_key_ets(kind, key_partition, compressed) + pid_ets = init_pid_ets(kind, pid_partition) + + # If we have only one partition, we do an optimization which + # is to write the table information alongside the registry info. + if partitions == 1 do + entries = [ + {@key_info, {kind, partitions, key_ets}}, + {@all_info, {kind, partitions, key_ets, {self(), pid_ets}, listeners}} + ] + + true = :ets.insert(registry, entries) + else + true = :ets.insert(registry, {i, key_ets, {self(), pid_ets}}) + end + + {:ok, pid_ets} + end + + # The key partition is a set for unique keys, + # duplicate bag for duplicate ones. + defp init_key_ets(:unique, key_partition, compressed) do + opts = [:set, :public, read_concurrency: true, write_concurrency: true] + :ets.new(key_partition, compression_opt(opts, compressed)) + end + + defp init_key_ets(:duplicate, key_partition, compressed) do + opts = [:duplicate_bag, :public, read_concurrency: true, write_concurrency: true] + :ets.new(key_partition, compression_opt(opts, compressed)) + end + + defp compression_opt(opts, compressed) do + if compressed, do: [:compressed] ++ opts, else: opts + end + + # A process can always have multiple keys, so the + # pid partition is always a duplicate bag. + defp init_pid_ets(_, pid_partition) do + :ets.new(pid_partition, [ + :duplicate_bag, + :public, + read_concurrency: true, + write_concurrency: true + ]) + end + + def handle_call(:sync, _, state) do + {:reply, :ok, state} + end + + def handle_info({:EXIT, pid, _reason}, ets) do + entries = :ets.take(ets, pid) + + for {_pid, key, key_ets, _counter} <- entries do + key_ets = + case key_ets do + # In case the fake key_ets is being used. See unregister_match/2. + {key_ets, _} -> + key_ets + + _ -> + key_ets + end + + try do + Registry.__unregister__(key_ets, {key, {pid, :_}}, 1) + catch + :error, :badarg -> :badarg + end + end + + {:noreply, ets} + end +end diff --git a/lib/elixir/lib/set.ex b/lib/elixir/lib/set.ex index 97d4aa4a160..b3c5f3c48a7 100644 --- a/lib/elixir/lib/set.ex +++ b/lib/elixir/lib/set.ex @@ -1,94 +1,33 @@ defmodule Set do @moduledoc ~S""" - This module specifies the Set API expected to be - implemented by different representations. + Generic API for sets. - It also provides functions that redirect to the - underlying Set, allowing a developer to work with - different Set implementations using one API. - - To create a new set, use the `new` functions defined - by each set type: - - HashSet.new #=> creates an empty HashSet - - In the examples below, `set_impl` means a specific - `Set` implementation, for example `HashSet`. - - ## Protocols - - Sets are required to implement both `Enumerable` and `Collectable` - protocols. - - ## Match - - Sets are required to implement all operations using the match (`===`) - operator. + This module is deprecated, use the `MapSet` module instead. """ - use Behaviour + @moduledoc deprecated: "Use MapSet instead" @type value :: any - @type values :: [ value ] + @type values :: [value] @type t :: map - defcallback new :: t - defcallback delete(t, value) :: t - defcallback difference(t, t) :: t - defcallback disjoint?(t, t) :: boolean - defcallback equal?(t, t) :: boolean - defcallback intersection(t, t) :: t - defcallback member?(t, value) :: boolean - defcallback put(t, value) :: t - defcallback size(t) :: non_neg_integer - defcallback subset?(t, t) :: boolean - defcallback to_list(t) :: list() - defcallback union(t, t) :: t + message = "Use the MapSet module for working with sets" defmacrop target(set) do quote do case unquote(set) do - %{__struct__: x} when is_atom(x) -> - x - x -> - unsupported_set(x) + %module{} -> module + set -> unsupported_set(set) end end end - @doc """ - Deletes `value` from `set`. - - ## Examples - - iex> s = Enum.into([1, 2, 3], set_impl.new) - iex> Set.delete(s, 4) |> Enum.sort - [1, 2, 3] - - iex> s = Enum.into([1, 2, 3], set_impl.new) - iex> Set.delete(s, 2) |> Enum.sort - [1, 3] - - """ - @spec delete(t, value) :: t + @deprecated message def delete(set, value) do target(set).delete(set, value) end - @doc """ - Returns a set that is `set1` without the members of `set2`. - - Notice this function is polymorphic as it calculates the difference - for of any type. Each set implementation also provides a `difference` - function, but they can only work with sets of the same type. - - ## Examples - - iex> Set.difference(Enum.into([1,2], set_impl.new), Enum.into([2,3,4], set_impl.new)) |> Enum.sort - [1] - - """ - @spec difference(t, t) :: t + @deprecated message def difference(set1, set2) do target1 = target(set1) target2 = target(set2) @@ -96,29 +35,14 @@ defmodule Set do if target1 == target2 do target1.difference(set1, set2) else - target2.reduce(set2, {:cont, set1}, fn v, acc -> + Enumerable.reduce(set2, {:cont, set1}, fn v, acc -> {:cont, target1.delete(acc, v)} - end) |> elem(1) + end) + |> elem(1) end end - @doc """ - Checks if `set1` and `set2` have no members in common. - - Notice this function is polymorphic as it checks for disjoint sets of - any type. Each set implementation also provides a `disjoint?` function, - but they can only work with sets of the same type. - - ## Examples - - iex> Set.disjoint?(Enum.into([1, 2], set_impl.new), Enum.into([3, 4], set_impl.new)) - true - - iex> Set.disjoint?(Enum.into([1, 2], set_impl.new), Enum.into([2, 3], set_impl.new)) - false - - """ - @spec disjoint?(t, t) :: boolean + @deprecated message def disjoint?(set1, set2) do target1 = target(set1) target2 = target(set2) @@ -126,38 +50,22 @@ defmodule Set do if target1 == target2 do target1.disjoint?(set1, set2) else - target2.reduce(set2, {:cont, true}, fn member, acc -> + Enumerable.reduce(set2, {:cont, true}, fn member, acc -> case target1.member?(set1, member) do false -> {:cont, acc} - _ -> {:halt, false} + _ -> {:halt, false} end - end) |> elem(1) + end) + |> elem(1) end end - @doc false - @spec empty(t) :: t + @deprecated message def empty(set) do target(set).empty(set) end - @doc """ - Check if two sets are equal using `===`. - - Notice this function is polymorphic as it compares sets of - any type. Each set implementation also provides an `equal?` - function, but they can only work with sets of the same type. - - ## Examples - - iex> Set.equal?(Enum.into([1, 2], set_impl.new), Enum.into([2, 1, 1], set_impl.new)) - true - - iex> Set.equal?(Enum.into([1, 2], set_impl.new), Enum.into([3, 4], set_impl.new)) - false - - """ - @spec equal?(t, t) :: boolean + @deprecated message def equal?(set1, set2) do target1 = target(set1) target2 = target(set2) @@ -174,23 +82,7 @@ defmodule Set do end end - @doc """ - Returns a set containing only members in common between `set1` and `set2`. - - Notice this function is polymorphic as it calculates the intersection of - any type. Each set implementation also provides a `intersection` function, - but they can only work with sets of the same type. - - ## Examples - - iex> Set.intersection(Enum.into([1,2], set_impl.new), Enum.into([2,3,4], set_impl.new)) |> Enum.sort - [2] - - iex> Set.intersection(Enum.into([1,2], set_impl.new), Enum.into([3,4], set_impl.new)) |> Enum.sort - [] - - """ - @spec intersection(t, t) :: t + @deprecated message def intersection(set1, set2) do target1 = target(set1) target2 = target(set2) @@ -198,77 +90,29 @@ defmodule Set do if target1 == target2 do target1.intersection(set1, set2) else - target1.reduce(set1, {:cont, Collectable.empty(set1)}, fn v, acc -> + Enumerable.reduce(set1, {:cont, target1.new}, fn v, acc -> {:cont, if(target2.member?(set2, v), do: target1.put(acc, v), else: acc)} - end) |> elem(1) + end) + |> elem(1) end end - @doc """ - Checks if `set` contains `value`. - - ## Examples - - iex> Set.member?(Enum.into([1, 2, 3], set_impl.new), 2) - true - - iex> Set.member?(Enum.into([1, 2, 3], set_impl.new), 4) - false - - """ - @spec member?(t, value) :: boolean + @deprecated message def member?(set, value) do target(set).member?(set, value) end - @doc """ - Inserts `value` into `set` if it does not already contain it. - - ## Examples - - iex> Set.put(Enum.into([1, 2, 3], set_impl.new), 3) |> Enum.sort - [1, 2, 3] - - iex> Set.put(Enum.into([1, 2, 3], set_impl.new), 4) |> Enum.sort - [1, 2, 3, 4] - - """ - @spec put(t, value) :: t + @deprecated message def put(set, value) do target(set).put(set, value) end - @doc """ - Returns the number of elements in `set`. - - ## Examples - - iex> Set.size(Enum.into([1, 2, 3], set_impl.new)) - 3 - - """ - @spec size(t) :: non_neg_integer + @deprecated message def size(set) do target(set).size(set) end - @doc """ - Checks if `set1`'s members are all contained in `set2`. - - Notice this function is polymorphic as it checks the subset for - any type. Each set implementation also provides a `subset?` function, - but they can only work with sets of the same type. - - ## Examples - - iex> Set.subset?(Enum.into([1, 2], set_impl.new), Enum.into([1, 2, 3], set_impl.new)) - true - - iex> Set.subset?(Enum.into([1, 2, 3], set_impl.new), Enum.into([1, 2], set_impl.new)) - false - - """ - @spec subset?(t, t) :: boolean + @deprecated message def subset?(set1, set2) do target1 = target(set1) target2 = target(set2) @@ -280,34 +124,12 @@ defmodule Set do end end - @doc """ - Converts `set` to a list. - - ## Examples - - iex> set_impl.to_list(Enum.into([1, 2, 3], set_impl.new)) |> Enum.sort - [1,2,3] - - """ - @spec to_list(t) :: list + @deprecated message def to_list(set) do target(set).to_list(set) end - @doc """ - Returns a set containing all members of `set1` and `set2`. - - Notice this function is polymorphic as it calculates the union of - any type. Each set implementation also provides a `union` function, - but they can only work with sets of the same type. - - ## Examples - - iex> Set.union(Enum.into([1,2], set_impl.new), Enum.into([2,3,4], set_impl.new)) |> Enum.sort - [1,2,3,4] - - """ - @spec union(t, t) :: t + @deprecated message def union(set1, set2) do target1 = target(set1) target2 = target(set2) @@ -315,22 +137,24 @@ defmodule Set do if target1 == target2 do target1.union(set1, set2) else - target2.reduce(set2, {:cont, set1}, fn v, acc -> + Enumerable.reduce(set2, {:cont, set1}, fn v, acc -> {:cont, target1.put(acc, v)} - end) |> elem(1) + end) + |> elem(1) end end - defp do_subset?(target1, target2, set1, set2) do - target1.reduce(set1, {:cont, true}, fn member, acc -> + defp do_subset?(_target1, target2, set1, set2) do + Enumerable.reduce(set1, {:cont, true}, fn member, acc -> case target2.member?(set2, member) do true -> {:cont, acc} - _ -> {:halt, false} + _ -> {:halt, false} end - end) |> elem(1) + end) + |> elem(1) end defp unsupported_set(set) do - raise ArgumentError, "unsupported set: #{inspect set}" + raise ArgumentError, "unsupported set: #{inspect(set)}" end end diff --git a/lib/elixir/lib/stream.ex b/lib/elixir/lib/stream.ex index d61de10063b..e3a0ebfe3da 100644 --- a/lib/elixir/lib/stream.ex +++ b/lib/elixir/lib/stream.ex @@ -1,15 +1,16 @@ defmodule Stream do @moduledoc """ - Module for creating and composing streams. + Functions for creating and composing streams. - Streams are composable, lazy enumerables. Any enumerable that generates - items one by one during enumeration is called a stream. For example, + Streams are composable, lazy enumerables (for an introduction on + enumerables, see the `Enum` module). Any enumerable that generates + elements one by one during enumeration is called a stream. For example, Elixir's `Range` is a stream: iex> range = 1..5 1..5 - iex> Enum.map range, &(&1 * 2) - [2,4,6,8,10] + iex> Enum.map(range, &(&1 * 2)) + [2, 4, 6, 8, 10] In the example above, as we mapped over the range, the elements being enumerated were created one by one, during enumeration. The `Stream` @@ -18,12 +19,12 @@ defmodule Stream do iex> range = 1..3 iex> stream = Stream.map(range, &(&1 * 2)) iex> Enum.map(stream, &(&1 + 1)) - [3,5,7] + [3, 5, 7] - Notice we started with a range and then we created a stream that is - meant to multiply each item in the range by 2. At this point, no - computation was done yet. Just when `Enum.map/2` is called we - enumerate over each item in the range, multiplying it by 2 and adding 1. + Note that we started with a range and then we created a stream that is + meant to multiply each element in the range by 2. At this point, no + computation was done. Only when `Enum.map/2` is called we actually + enumerate over each element in the range, multiplying it by 2 and adding 1. We say the functions in `Stream` are *lazy* and the functions in `Enum` are *eager*. @@ -33,26 +34,26 @@ defmodule Stream do computations that are executed at a later moment. Let's see another example: - 1..3 |> - Enum.map(&IO.inspect(&1)) |> - Enum.map(&(&1 * 2)) |> - Enum.map(&IO.inspect(&1)) + 1..3 + |> Enum.map(&IO.inspect(&1)) + |> Enum.map(&(&1 * 2)) + |> Enum.map(&IO.inspect(&1)) 1 2 3 2 4 6 - #=> [2,4,6] + #=> [2, 4, 6] - Notice that we first printed each item in the list, then multiplied each + Note that we first printed each element in the list, then multiplied each element by 2 and finally printed each new value. In this example, the list was enumerated three times. Let's see an example with streams: - stream = 1..3 |> - Stream.map(&IO.inspect(&1)) |> - Stream.map(&(&1 * 2)) |> - Stream.map(&IO.inspect(&1)) + stream = 1..3 + |> Stream.map(&IO.inspect(&1)) + |> Stream.map(&(&1 * 2)) + |> Stream.map(&IO.inspect(&1)) Enum.to_list(stream) 1 2 @@ -60,16 +61,23 @@ defmodule Stream do 4 3 6 - #=> [2,4,6] + #=> [2, 4, 6] - Although the end result is the same, the order in which the items were - printed changed! With streams, we print the first item and then print + Although the end result is the same, the order in which the elements were + printed changed! With streams, we print the first element and then print its double. In this example, the list was enumerated just once! - That's what we meant when we first said that streams are composable, - lazy enumerables. Notice we could call `Stream.map/2` multiple times, - effectively composing the streams and they are lazy. The computations - are performed only when you call a function from the `Enum` module. + That's what we meant when we said earlier that streams are composable, + lazy enumerables. Note that we could call `Stream.map/2` multiple times, + effectively composing the streams and keeping them lazy. The computations + are only performed when you call a function from the `Enum` module. + + Like with `Enum`, the functions in this module work in linear time. This + means that, the time it takes to perform an operation grows at the same + rate as the length of the list. This is expected on operations such as + `Stream.map/2`. After all, if we want to traverse every element on a + stream, the longer the stream, the more elements we need to traverse, + and the longer it will take. ## Creating Streams @@ -85,93 +93,112 @@ defmodule Stream do Note the functions in this module are guaranteed to return enumerables. Since enumerables can have different shapes (structs, anonymous functions, and so on), the functions in this module may return any of those shapes - and that it may change at any time. For example, a function that today + and this may change at any time. For example, a function that today returns an anonymous function may return a struct in future releases. """ + @doc false defstruct enum: nil, funs: [], accs: [], done: nil @type acc :: any @type element :: any + + @typedoc "Zero-based index." @type index :: non_neg_integer + @type default :: any + @type timer :: non_neg_integer | :infinity # Require Stream.Reducers and its callbacks require Stream.Reducers, as: R - defmacrop cont(f, entry, acc) do - quote do: unquote(f).(unquote(entry), unquote(acc)) + defmacrop skip(acc) do + {:cont, acc} + end + + defmacrop next(fun, entry, acc) do + quote(do: unquote(fun).(unquote(entry), unquote(acc))) end - defmacrop acc(h, n, t) do - quote do: [unquote(h),unquote(n)|unquote(t)] + defmacrop acc(head, state, tail) do + quote(do: [unquote(head), unquote(state) | unquote(tail)]) end - defmacrop cont_with_acc(f, entry, h, n, t) do + defmacrop next_with_acc(fun, entry, head, state, tail) do quote do - {reason, [h|t]} = unquote(f).(unquote(entry), [unquote(h)|unquote(t)]) - {reason, [h,unquote(n)|t]} + {reason, [head | tail]} = unquote(fun).(unquote(entry), [unquote(head) | unquote(tail)]) + {reason, [head, unquote(state) | tail]} end end ## Transformers + @doc false + @deprecated "Use Stream.chunk_every/2 instead" + def chunk(enum, n), do: chunk(enum, n, n, nil) + + @doc false + @deprecated "Use Stream.chunk_every/3 instead" + def chunk(enum, n, step) do + chunk_every(enum, n, step, nil) + end + + @doc false + @deprecated "Use Stream.chunk_every/4 instead" + def chunk(enum, n, step, leftover) + when is_integer(n) and n > 0 and is_integer(step) and step > 0 do + chunk_every(enum, n, step, leftover || :discard) + end + @doc """ - Shortcut to `chunk(enum, n, n)`. + Shortcut to `chunk_every(enum, count, count)`. """ - @spec chunk(Enumerable.t, non_neg_integer) :: Enumerable.t - def chunk(enum, n), do: chunk(enum, n, n, nil) + @doc since: "1.5.0" + @spec chunk_every(Enumerable.t(), pos_integer) :: Enumerable.t() + def chunk_every(enum, count), do: chunk_every(enum, count, count, []) @doc """ - Streams the enumerable in chunks, containing `n` items each, where - each new chunk starts `step` elements into the enumerable. - - `step` is optional and, if not passed, defaults to `n`, i.e. - chunks do not overlap. If the final chunk does not have `n` - elements to fill the chunk, elements are taken as necessary - from `pad` if it was passed. If `pad` is passed and does not - have enough elements to fill the chunk, then the chunk is - returned anyway with less than `n` elements. If `pad` is not - passed at all or is `nil`, then the partial chunk is discarded - from the result. + Streams the enumerable in chunks, containing `count` elements each, + where each new chunk starts `step` elements into the enumerable. + + `step` is optional and, if not passed, defaults to `count`, i.e. + chunks do not overlap. + + If the last chunk does not have `count` elements to fill the chunk, + elements are taken from `leftover` to fill in the chunk. If `leftover` + does not have enough elements to fill the chunk, then a partial chunk + is returned with less than `count` elements. + + If `:discard` is given in `leftover`, the last chunk is discarded + unless it has exactly `count` elements. ## Examples - iex> Stream.chunk([1, 2, 3, 4, 5, 6], 2) |> Enum.to_list + iex> Stream.chunk_every([1, 2, 3, 4, 5, 6], 2) |> Enum.to_list() [[1, 2], [3, 4], [5, 6]] - iex> Stream.chunk([1, 2, 3, 4, 5, 6], 3, 2) |> Enum.to_list + iex> Stream.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, :discard) |> Enum.to_list() [[1, 2, 3], [3, 4, 5]] - iex> Stream.chunk([1, 2, 3, 4, 5, 6], 3, 2, [7]) |> Enum.to_list + iex> Stream.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, [7]) |> Enum.to_list() [[1, 2, 3], [3, 4, 5], [5, 6, 7]] - iex> Stream.chunk([1, 2, 3, 4, 5, 6], 3, 3, []) |> Enum.to_list + iex> Stream.chunk_every([1, 2, 3, 4, 5, 6], 3, 3, []) |> Enum.to_list() [[1, 2, 3], [4, 5, 6]] """ - @spec chunk(Enumerable.t, non_neg_integer, non_neg_integer) :: Enumerable.t - @spec chunk(Enumerable.t, non_neg_integer, non_neg_integer, Enumerable.t | nil) :: Enumerable.t - def chunk(enum, n, step, pad \\ nil) when n > 0 and step > 0 do - limit = :erlang.max(n, step) - lazy enum, {[], 0}, - fn(f1) -> R.chunk(n, step, limit, f1) end, - fn(f1) -> &do_chunk(&1, n, pad, f1) end - end - - defp do_chunk(acc(h, {buffer, count} = old, t) = acc, n, pad, f1) do - if nil?(pad) || count == 0 do - {:cont, acc} - else - buffer = :lists.reverse(buffer) ++ Enum.take(pad, n - count) - cont_with_acc(f1, buffer, h, old, t) - end + @doc since: "1.5.0" + @spec chunk_every(Enumerable.t(), pos_integer, pos_integer, Enumerable.t() | :discard) :: + Enumerable.t() + def chunk_every(enum, count, step, leftover \\ []) + when is_integer(count) and count > 0 and is_integer(step) and step > 0 do + R.chunk_every(&chunk_while/4, enum, count, step, leftover) end @doc """ - Chunks the `enum` by buffering elements for which `fun` returns - the same value and only emit them when `fun` returns a new value - or the `enum` finishes. + Chunks the `enum` by buffering elements for which `fun` returns the same value. + + Elements are only emitted when `fun` returns a new value or the `enum` finishes. ## Examples @@ -180,93 +207,258 @@ defmodule Stream do [[1], [2, 2], [3], [4, 4, 6], [7, 7]] """ - @spec chunk_by(Enumerable.t, (element -> any)) :: Enumerable.t - def chunk_by(enum, fun) do - lazy enum, nil, - fn(f1) -> R.chunk_by(fun, f1) end, - fn(f1) -> &do_chunk_by(&1, f1) end + @spec chunk_by(Enumerable.t(), (element -> any)) :: Enumerable.t() + def chunk_by(enum, fun) when is_function(fun, 1) do + R.chunk_by(&chunk_while/4, enum, fun) end - defp do_chunk_by(acc(_, nil, _) = acc, _f1) do - {:cont, acc} + @doc """ + Chunks the `enum` with fine grained control when every chunk is emitted. + + `chunk_fun` receives the current element and the accumulator and + must return `{:cont, element, acc}` to emit the given chunk and + continue with accumulator or `{:cont, acc}` to not emit any chunk + and continue with the return accumulator. + + `after_fun` is invoked when iteration is done and must also return + `{:cont, element, acc}` or `{:cont, acc}`. + + ## Examples + + iex> chunk_fun = fn element, acc -> + ...> if rem(element, 2) == 0 do + ...> {:cont, Enum.reverse([element | acc]), []} + ...> else + ...> {:cont, [element | acc]} + ...> end + ...> end + iex> after_fun = fn + ...> [] -> {:cont, []} + ...> acc -> {:cont, Enum.reverse(acc), []} + ...> end + iex> stream = Stream.chunk_while(1..10, [], chunk_fun, after_fun) + iex> Enum.to_list(stream) + [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + """ + @doc since: "1.5.0" + @spec chunk_while( + Enumerable.t(), + acc, + (element, acc -> {:cont, chunk, acc} | {:cont, acc} | {:halt, acc}), + (acc -> {:cont, chunk, acc} | {:cont, acc}) + ) :: Enumerable.t() + when chunk: any + def chunk_while(enum, acc, chunk_fun, after_fun) + when is_function(chunk_fun, 2) and is_function(after_fun, 1) do + lazy( + enum, + [acc | after_fun], + fn f1 -> chunk_while_fun(chunk_fun, f1) end, + &after_chunk_while/2 + ) + end + + defp chunk_while_fun(callback, fun) do + fn entry, acc(head, [acc | after_fun], tail) -> + case callback.(entry, acc) do + {:cont, emit, acc} -> + # If we emit an element and then we have to halt, + # we need to disable the after_fun callback to + # avoid emitting even more elements. + case next(fun, emit, [head | tail]) do + {:halt, [head | tail]} -> {:halt, acc(head, [acc | &{:cont, &1}], tail)} + {command, [head | tail]} -> {command, acc(head, [acc | after_fun], tail)} + end + + {:cont, acc} -> + skip(acc(head, [acc | after_fun], tail)) + + {:halt, acc} -> + {:halt, acc(head, [acc | after_fun], tail)} + end + end + end + + defp after_chunk_while(acc(h, [acc | after_fun], t), f1) do + case after_fun.(acc) do + {:cont, emit, acc} -> next_with_acc(f1, emit, h, [acc | after_fun], t) + {:cont, acc} -> {:cont, acc(h, [acc | after_fun], t)} + end end - defp do_chunk_by(acc(h, {buffer, _}, t), f1) do - cont_with_acc(f1, :lists.reverse(buffer), h, nil, t) + @doc """ + Creates a stream that only emits elements if they are different from the last emitted element. + + This function only ever needs to store the last emitted element. + + Elements are compared using `===/2`. + + ## Examples + + iex> Stream.dedup([1, 2, 3, 3, 2, 1]) |> Enum.to_list() + [1, 2, 3, 2, 1] + + """ + @spec dedup(Enumerable.t()) :: Enumerable.t() + def dedup(enum) do + dedup_by(enum, fn x -> x end) end @doc """ - Lazily drops the next `n` items from the enumerable. + Creates a stream that only emits elements if the result of calling `fun` on the element is + different from the (stored) result of calling `fun` on the last emitted element. + + ## Examples + + iex> Stream.dedup_by([{1, :x}, {2, :y}, {2, :z}, {1, :x}], fn {x, _} -> x end) |> Enum.to_list() + [{1, :x}, {2, :y}, {1, :x}] - If a negative `n` is given, it will drop the last `n` items from + """ + @spec dedup_by(Enumerable.t(), (element -> term)) :: Enumerable.t() + def dedup_by(enum, fun) when is_function(fun, 1) do + lazy(enum, nil, fn f1 -> R.dedup(fun, f1) end) + end + + @doc """ + Lazily drops the next `n` elements from the enumerable. + + If a negative `n` is given, it will drop the last `n` elements from the collection. Note that the mechanism by which this is implemented - will delay the emission of any item until `n` additional items have + will delay the emission of any element until `n` additional elements have been emitted by the enum. ## Examples iex> stream = Stream.drop(1..10, 5) iex> Enum.to_list(stream) - [6,7,8,9,10] + [6, 7, 8, 9, 10] iex> stream = Stream.drop(1..10, -5) iex> Enum.to_list(stream) - [1,2,3,4,5] + [1, 2, 3, 4, 5] """ - @spec drop(Enumerable.t, non_neg_integer) :: Enumerable.t - def drop(enum, n) when n >= 0 do - lazy enum, n, fn(f1) -> R.drop(f1) end + @spec drop(Enumerable.t(), integer) :: Enumerable.t() + def drop(enum, n) when is_integer(n) and n >= 0 do + lazy(enum, n, fn f1 -> R.drop(f1) end) end - def drop(enum, n) when n < 0 do + def drop(enum, n) when is_integer(n) and n < 0 do n = abs(n) - lazy enum, {0, [], []}, fn(f1) -> + lazy(enum, {0, [], []}, fn f1 -> fn entry, [h, {count, buf1, []} | t] -> do_drop(:cont, n, entry, h, count, buf1, [], t) - entry, [h, {count, buf1, [next|buf2]} | t] -> - {reason, [h|t]} = f1.(next, [h|t]) + + entry, [h, {count, buf1, [next | buf2]} | t] -> + {reason, [h | t]} = f1.(next, [h | t]) do_drop(reason, n, entry, h, count, buf1, buf2, t) end - end + end) end defp do_drop(reason, n, entry, h, count, buf1, buf2, t) do - buf1 = [entry|buf1] + buf1 = [entry | buf1] count = count + 1 + if count == n do - {reason, [h, {0, [], :lists.reverse(buf1)}|t]} + {reason, [h, {0, [], :lists.reverse(buf1)} | t]} else - {reason, [h, {count, buf1, buf2}|t]} + {reason, [h, {count, buf1, buf2} | t]} end end + @doc """ + Creates a stream that drops every `nth` element from the enumerable. + + The first element is always dropped, unless `nth` is 0. + + `nth` must be a non-negative integer. + + ## Examples + + iex> stream = Stream.drop_every(1..10, 2) + iex> Enum.to_list(stream) + [2, 4, 6, 8, 10] + + iex> stream = Stream.drop_every(1..1000, 1) + iex> Enum.to_list(stream) + [] + + iex> stream = Stream.drop_every([1, 2, 3, 4, 5], 0) + iex> Enum.to_list(stream) + [1, 2, 3, 4, 5] + + """ + @spec drop_every(Enumerable.t(), non_neg_integer) :: Enumerable.t() + def drop_every(enum, nth) + def drop_every(enum, 0), do: %Stream{enum: enum} + def drop_every([], _nth), do: %Stream{enum: []} + + def drop_every(enum, nth) when is_integer(nth) and nth > 0 do + lazy(enum, nth, fn f1 -> R.drop_every(nth, f1) end) + end + @doc """ Lazily drops elements of the enumerable while the given - function returns `true`. + function returns a truthy value. ## Examples iex> stream = Stream.drop_while(1..10, &(&1 <= 5)) iex> Enum.to_list(stream) - [6,7,8,9,10] + [6, 7, 8, 9, 10] """ - @spec drop_while(Enumerable.t, (element -> as_boolean(term))) :: Enumerable.t - def drop_while(enum, fun) do - lazy enum, true, fn(f1) -> R.drop_while(fun, f1) end + @spec drop_while(Enumerable.t(), (element -> as_boolean(term))) :: Enumerable.t() + def drop_while(enum, fun) when is_function(fun, 1) do + lazy(enum, true, fn f1 -> R.drop_while(fun, f1) end) end @doc """ - Execute the given function for each item. + Duplicates the given element `n` times in a stream. + + `n` is an integer greater than or equal to `0`. + + If `n` is `0`, an empty stream is returned. + + ## Examples + + iex> stream = Stream.duplicate("hello", 0) + iex> Enum.to_list(stream) + [] + + iex> stream = Stream.duplicate("hi", 1) + iex> Enum.to_list(stream) + ["hi"] + + iex> stream = Stream.duplicate("bye", 2) + iex> Enum.to_list(stream) + ["bye", "bye"] + + iex> stream = Stream.duplicate([1, 2], 3) + iex> Enum.to_list(stream) + [[1, 2], [1, 2], [1, 2]] + """ + @doc since: "1.14.0" + @spec duplicate(any, non_neg_integer) :: Enumerable.t() + def duplicate(value, n) when is_integer(n) and n >= 0 do + unfold(n, fn + 0 -> nil + remaining -> {value, remaining - 1} + end) + end + + @doc """ + Executes the given function for each element. Useful for adding side effects (like printing) to a stream. ## Examples - iex> stream = Stream.each([1, 2, 3], fn(x) -> send self, x end) + iex> stream = Stream.each([1, 2, 3], fn x -> send(self(), x) end) iex> Enum.to_list(stream) iex> receive do: (x when is_integer(x) -> x) 1 @@ -276,29 +468,35 @@ defmodule Stream do 3 """ - @spec each(Enumerable.t, (element -> term)) :: Enumerable.t - def each(enum, fun) do - lazy enum, fn(f1) -> - fn(x, acc) -> + @spec each(Enumerable.t(), (element -> term)) :: Enumerable.t() + def each(enum, fun) when is_function(fun, 1) do + lazy(enum, fn f1 -> + fn x, acc -> fun.(x) f1.(x, acc) end - end + end) end @doc """ - Creates a stream that will apply the given function on enumeration and - flatten the result. + Maps the given `fun` over `enumerable` and flattens the result. + + This function returns a new stream built by appending the result of invoking `fun` + on each element of `enumerable` together. ## Examples - iex> stream = Stream.flat_map([1, 2, 3], fn(x) -> [x, x * 2] end) + iex> stream = Stream.flat_map([1, 2, 3], fn x -> [x, x * 2] end) iex> Enum.to_list(stream) [1, 2, 2, 4, 3, 6] + iex> stream = Stream.flat_map([1, 2, 3], fn x -> [[x]] end) + iex> Enum.to_list(stream) + [[1], [2], [3]] + """ - @spec flat_map(Enumerable.t, (element -> Enumerable.t)) :: Enumerable.t - def flat_map(enum, mapper) do + @spec flat_map(Enumerable.t(), (element -> Enumerable.t())) :: Enumerable.t() + def flat_map(enum, mapper) when is_function(mapper, 1) do transform(enum, nil, fn val, nil -> {mapper.(val), nil} end) end @@ -308,32 +506,48 @@ defmodule Stream do ## Examples - iex> stream = Stream.filter([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) + iex> stream = Stream.filter([1, 2, 3], fn x -> rem(x, 2) == 0 end) iex> Enum.to_list(stream) [2] """ - @spec filter(Enumerable.t, (element -> as_boolean(term))) :: Enumerable.t - def filter(enum, fun) do - lazy enum, fn(f1) -> R.filter(fun, f1) end + @spec filter(Enumerable.t(), (element -> as_boolean(term))) :: Enumerable.t() + def filter(enum, fun) when is_function(fun, 1) do + lazy(enum, fn f1 -> R.filter(fun, f1) end) + end + + @doc false + @deprecated "Use Stream.filter/2 + Stream.map/2 instead" + def filter_map(enum, filter, mapper) do + lazy(enum, fn f1 -> R.filter_map(filter, mapper, f1) end) end @doc """ - Creates a stream that filters and then maps elements according - to given functions. + Creates a stream that emits a value after the given period `n` + in milliseconds. - Exists for symmetry with `Enum.filter_map/3`. + The values emitted are an increasing counter starting at `0`. + This operation will block the caller by the given interval + every time a new element is streamed. + + Do not use this function to generate a sequence of numbers. + If blocking the caller process is not necessary, use + `Stream.iterate(0, & &1 + 1)` instead. ## Examples - iex> stream = Stream.filter_map(1..6, fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) - iex> Enum.to_list(stream) - [4,8,12] + iex> Stream.interval(10) |> Enum.take(10) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] """ - @spec filter_map(Enumerable.t, (element -> as_boolean(term)), (element -> any)) :: Enumerable.t - def filter_map(enum, filter, mapper) do - lazy enum, fn(f1) -> R.filter_map(filter, mapper, f1) end + @spec interval(timer()) :: Enumerable.t() + def interval(n) + when is_integer(n) and n >= 0 + when n == :infinity do + unfold(0, fn count -> + Process.sleep(n) + {count, count + 1} + end) end @doc """ @@ -342,33 +556,35 @@ defmodule Stream do This function is often used with `run/1` since any evaluation is delayed until the stream is executed. See `run/1` for an example. """ - @spec into(Enumerable.t, Collectable.t) :: Enumerable.t - def into(enum, collectable, transform \\ fn x -> x end) do + @spec into(Enumerable.t(), Collectable.t(), (term -> term)) :: Enumerable.t() + def into(enum, collectable, transform \\ fn x -> x end) when is_function(transform, 1) do &do_into(enum, collectable, transform, &1, &2) end defp do_into(enum, collectable, transform, acc, fun) do {initial, into} = Collectable.into(collectable) - composed = fn x, [acc|collectable] -> + + composed = fn x, [acc | collectable] -> collectable = into.(collectable, {:cont, transform.(x)}) {reason, acc} = fun.(x, acc) - {reason, [acc|collectable]} + {reason, [acc | collectable]} end + do_into(&Enumerable.reduce(enum, &1, composed), initial, into, acc) end defp do_into(reduce, collectable, into, {command, acc}) do try do - reduce.({command, [acc|collectable]}) + reduce.({command, [acc | collectable]}) catch kind, reason -> - stacktrace = System.stacktrace into.(collectable, :halt) - :erlang.raise(kind, reason, stacktrace) + :erlang.raise(kind, reason, __STACKTRACE__) else - {:suspended, [acc|collectable], continuation} -> + {:suspended, [acc | collectable], continuation} -> {:suspended, acc, &do_into(continuation, collectable, into, &1)} - {reason, [acc|collectable]} -> + + {reason, [acc | collectable]} -> into.(collectable, :done) {reason, acc} end @@ -380,14 +596,51 @@ defmodule Stream do ## Examples - iex> stream = Stream.map([1, 2, 3], fn(x) -> x * 2 end) + iex> stream = Stream.map([1, 2, 3], fn x -> x * 2 end) + iex> Enum.to_list(stream) + [2, 4, 6] + + """ + @spec map(Enumerable.t(), (element -> any)) :: Enumerable.t() + def map(enum, fun) when is_function(fun, 1) do + lazy(enum, fn f1 -> R.map(fun, f1) end) + end + + @doc """ + Creates a stream that will apply the given function on + every `nth` element from the enumerable. + + The first element is always passed to the given function. + + `nth` must be a non-negative integer. + + ## Examples + + iex> stream = Stream.map_every(1..10, 2, fn x -> x * 2 end) + iex> Enum.to_list(stream) + [2, 2, 6, 4, 10, 6, 14, 8, 18, 10] + + iex> stream = Stream.map_every([1, 2, 3, 4, 5], 1, fn x -> x * 2 end) iex> Enum.to_list(stream) - [2,4,6] + [2, 4, 6, 8, 10] + + iex> stream = Stream.map_every(1..5, 0, fn x -> x * 2 end) + iex> Enum.to_list(stream) + [1, 2, 3, 4, 5] """ - @spec map(Enumerable.t, (element -> any)) :: Enumerable.t - def map(enum, fun) do - lazy enum, fn(f1) -> R.map(fun, f1) end + @doc since: "1.4.0" + @spec map_every(Enumerable.t(), non_neg_integer, (element -> any)) :: Enumerable.t() + def map_every(enum, nth, fun) when is_integer(nth) and nth >= 0 and is_function(fun, 1) do + map_every_after_guards(enum, nth, fun) + end + + defp map_every_after_guards(enum, 1, fun), do: map(enum, fun) + defp map_every_after_guards(enum, 0, _fun), do: %Stream{enum: enum} + defp map_every_after_guards([], _nth, _fun), do: %Stream{enum: []} + + defp map_every_after_guards(enum, nth, fun) do + lazy(enum, nth, fn f1 -> R.map_every(nth, fun, f1) end) end @doc """ @@ -396,14 +649,14 @@ defmodule Stream do ## Examples - iex> stream = Stream.reject([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) + iex> stream = Stream.reject([1, 2, 3], fn x -> rem(x, 2) == 0 end) iex> Enum.to_list(stream) - [1,3] + [1, 3] """ - @spec reject(Enumerable.t, (element -> as_boolean(term))) :: Enumerable.t - def reject(enum, fun) do - lazy enum, fn(f1) -> R.reject(fun, f1) end + @spec reject(Enumerable.t(), (element -> as_boolean(term))) :: Enumerable.t() + def reject(enum, fun) when is_function(fun, 1) do + lazy(enum, fn f1 -> R.reject(fun, f1) end) end @doc """ @@ -417,35 +670,36 @@ defmodule Stream do Open up a file, replace all `#` by `%` and stream to another file without loading the whole file in memory: - stream = File.stream!("code") + File.stream!("/path/to/file") |> Stream.map(&String.replace(&1, "#", "%")) - |> Stream.into(File.stream!("new")) - |> Stream.run + |> Stream.into(File.stream!("/path/to/other/file")) + |> Stream.run() - No computation will be done until we call one of the Enum functions - or `Stream.run/1`. + No computation will be done until we call one of the `Enum` functions + or `run/1`. """ - @spec run(Enumerable.t) :: :ok + @spec run(Enumerable.t()) :: :ok def run(stream) do - Enumerable.reduce(stream, {:cont, nil}, fn(_, _) -> {:cont, nil} end) + _ = Enumerable.reduce(stream, {:cont, nil}, fn _, _ -> {:cont, nil} end) :ok end @doc """ Creates a stream that applies the given function to each element, emits the result and uses the same result as the accumulator - for the next computation. + for the next computation. Uses the first element in the enumerable + as the starting value. ## Examples iex> stream = Stream.scan(1..5, &(&1 + &2)) iex> Enum.to_list(stream) - [1,3,6,10,15] + [1, 3, 6, 10, 15] """ - @spec scan(Enumerable.t, (element, acc -> any)) :: Enumerable.t - def scan(enum, fun) do - lazy enum, :first, fn(f1) -> R.scan_2(fun, f1) end + @spec scan(Enumerable.t(), (element, acc -> any)) :: Enumerable.t() + def scan(enum, fun) when is_function(fun, 2) do + lazy(enum, :first, fn f1 -> R.scan2(fun, f1) end) end @doc """ @@ -457,223 +711,372 @@ defmodule Stream do iex> stream = Stream.scan(1..5, 0, &(&1 + &2)) iex> Enum.to_list(stream) - [1,3,6,10,15] + [1, 3, 6, 10, 15] """ - @spec scan(Enumerable.t, acc, (element, acc -> any)) :: Enumerable.t - def scan(enum, acc, fun) do - lazy enum, acc, fn(f1) -> R.scan_3(fun, f1) end + @spec scan(Enumerable.t(), acc, (element, acc -> any)) :: Enumerable.t() + def scan(enum, acc, fun) when is_function(fun, 2) do + lazy(enum, acc, fn f1 -> R.scan3(fun, f1) end) end @doc """ - Lazily takes the next `n` items from the enumerable and stops + Lazily takes the next `count` elements from the enumerable and stops enumeration. - If a negative `n` is given, the last `n` values will be taken. - For such, the collection is fully enumerated keeping up to `2 * n` + If a negative `count` is given, the last `count` values will be taken. + For such, the collection is fully enumerated keeping up to `2 * count` elements in memory. Once the end of the collection is reached, the last `count` elements will be executed. Therefore, using - a negative `n` on an infinite collection will never return. + a negative `count` on an infinite collection will never return. ## Examples iex> stream = Stream.take(1..100, 5) iex> Enum.to_list(stream) - [1,2,3,4,5] + [1, 2, 3, 4, 5] iex> stream = Stream.take(1..100, -5) iex> Enum.to_list(stream) - [96,97,98,99,100] + [96, 97, 98, 99, 100] iex> stream = Stream.cycle([1, 2, 3]) |> Stream.take(5) iex> Enum.to_list(stream) - [1,2,3,1,2] + [1, 2, 3, 1, 2] """ - @spec take(Enumerable.t, non_neg_integer) :: Enumerable.t - def take(_enum, 0), do: %Stream{enum: []} - - def take(enum, n) when n > 0 do - lazy enum, n, fn(f1) -> R.take(f1) end + @spec take(Enumerable.t(), integer) :: Enumerable.t() + def take(enum, count) when is_integer(count) do + take_after_guards(enum, count) end - def take(enum, n) when n < 0 do - &do_take(enum, abs(n), &1, &2) - end + defp take_after_guards(_enum, 0), do: %Stream{enum: []} - defp do_take(enum, n, acc, f) do - {_, {_count, buf1, buf2}} = - Enumerable.reduce(enum, {:cont, {0, [], []}}, fn - entry, {count, buf1, buf2} -> - buf1 = [entry|buf1] - count = count + 1 - if count == n do - {:cont, {0, [], buf1}} - else - {:cont, {count, buf1, buf2}} - end - end) + defp take_after_guards([], _count), do: %Stream{enum: []} - Enumerable.reduce(do_take_last(buf1, buf2, n, []), acc, f) + defp take_after_guards(enum, count) when count > 0 do + lazy(enum, count, fn f1 -> R.take(f1) end) end - defp do_take_last(_buf1, _buf2, 0, acc), - do: acc - defp do_take_last([], [], _, acc), - do: acc - defp do_take_last([], [h|t], n, acc), - do: do_take_last([], t, n-1, [h|acc]) - defp do_take_last([h|t], buf2, n, acc), - do: do_take_last(t, buf2, n-1, [h|acc]) + defp take_after_guards(enum, count) when count < 0 do + &Enumerable.reduce(Enum.take(enum, count), &1, &2) + end @doc """ - Creates a stream that takes every `n` item from the enumerable. + Creates a stream that takes every `nth` element from the enumerable. + + The first element is always included, unless `nth` is 0. - The first item is always included, unless `n` is 0. + `nth` must be a non-negative integer. ## Examples iex> stream = Stream.take_every(1..10, 2) iex> Enum.to_list(stream) - [1,3,5,7,9] + [1, 3, 5, 7, 9] + + iex> stream = Stream.take_every([1, 2, 3, 4, 5], 1) + iex> Enum.to_list(stream) + [1, 2, 3, 4, 5] + + iex> stream = Stream.take_every(1..1000, 0) + iex> Enum.to_list(stream) + [] """ - @spec take_every(Enumerable.t, non_neg_integer) :: Enumerable.t - def take_every(enum, n) when n > 0 do - lazy enum, n, fn(f1) -> R.take_every(n, f1) end + @spec take_every(Enumerable.t(), non_neg_integer) :: Enumerable.t() + def take_every(enum, nth) when is_integer(nth) and nth >= 0 do + take_every_after_guards(enum, nth) end - def take_every(_enum, 0), do: %Stream{enum: []} + defp take_every_after_guards(_enum, 0), do: %Stream{enum: []} + + defp take_every_after_guards([], _nth), do: %Stream{enum: []} + + defp take_every_after_guards(enum, nth) do + lazy(enum, nth, fn f1 -> R.take_every(nth, f1) end) + end @doc """ Lazily takes elements of the enumerable while the given - function returns `true`. + function returns a truthy value. ## Examples iex> stream = Stream.take_while(1..100, &(&1 <= 5)) iex> Enum.to_list(stream) - [1,2,3,4,5] + [1, 2, 3, 4, 5] """ - @spec take_while(Enumerable.t, (element -> as_boolean(term))) :: Enumerable.t - def take_while(enum, fun) do - lazy enum, fn(f1) -> R.take_while(fun, f1) end + @spec take_while(Enumerable.t(), (element -> as_boolean(term))) :: Enumerable.t() + def take_while(enum, fun) when is_function(fun, 1) do + lazy(enum, fn f1 -> R.take_while(fun, f1) end) + end + + @doc """ + Creates a stream that emits a single value after `n` milliseconds. + + The value emitted is `0`. This operation will block the caller by + the given time until the element is streamed. + + ## Examples + + iex> Stream.timer(10) |> Enum.to_list() + [0] + + """ + @spec timer(timer()) :: Enumerable.t() + def timer(n) + when is_integer(n) and n >= 0 + when n == :infinity do + take(interval(n), 1) end @doc """ Transforms an existing stream. - It expects an accumulator and a function that receives each stream item - and an accumulator, and must return a tuple containing a new stream - (often a list) with the new accumulator or a tuple with `:halt` as first - element and the accumulator as second. + It expects an accumulator and a function that receives two arguments, + the stream element and the updated accumulator. It must return a tuple, + where the first element is a new stream (often a list) or the atom `:halt`, + and the second element is the accumulator to be used by the next element. - Note: this function is similar to `Enum.flat_map_reduce/3` except the - latter returns both the flat list and accumulator, while this one returns - only the stream. + Note: this function is equivalent to `Enum.flat_map_reduce/3`, except this + function does not return the accumulator once the stream is processed. ## Examples - `Stream.transform/3` is a useful as it can be used as basis to implement + `Stream.transform/3` is useful as it can be used as the basis to implement many of the functions defined in this module. For example, we can implement `Stream.take(enum, n)` as follows: - iex> enum = 1..100 + iex> enum = 1001..9999 iex> n = 3 iex> stream = Stream.transform(enum, 0, fn i, acc -> ...> if acc < n, do: {[i], acc + 1}, else: {:halt, acc} ...> end) iex> Enum.to_list(stream) - [1,2,3] + [1001, 1002, 1003] + + `Stream.transform/5` further generalizes this function to allow wrapping + around resources. + """ + @spec transform(Enumerable.t(), acc, fun) :: Enumerable.t() + when fun: (element, acc -> {Enumerable.t(), acc} | {:halt, acc}), + acc: any + def transform(enum, acc, reducer) when is_function(reducer, 2) do + &do_transform(enum, fn -> acc end, reducer, &1, &2, nil, fn acc -> acc end) + end + @doc """ + Similar to `Stream.transform/5`, except `last_fun` is not supplied. + + This function can be seen as a combination of `Stream.resource/3` with + `Stream.transform/3`. """ - @spec transform(Enumerable.t, acc, fun) :: Enumerable.t when - fun: (element, acc -> {Enumerable.t, acc} | {:halt, acc}), - acc: any - def transform(enum, acc, reducer) do - &do_transform(enum, acc, reducer, &1, &2) + @spec transform(Enumerable.t(), start_fun, reducer, after_fun) :: Enumerable.t() + when start_fun: (() -> acc), + reducer: (element, acc -> {Enumerable.t(), acc} | {:halt, acc}), + after_fun: (acc -> term), + acc: any + def transform(enum, start_fun, reducer, after_fun) + when is_function(start_fun, 0) and is_function(reducer, 2) and is_function(after_fun, 1) do + &do_transform(enum, start_fun, reducer, &1, &2, nil, after_fun) end - defp do_transform(enumerables, user_acc, user, inner_acc, fun) do + @doc """ + Transforms an existing stream with function-based start, last, and after + callbacks. + + Once transformation starts, `start_fun` is invoked to compute the initial + accumulator. Then, for each element in the enumerable, the `reducer` function + is invoked with the element and the accumulator, returning new elements and a + new accumulator, as in `transform/3`. + + Once the collection is done, `last_fun` is invoked with the accumulator to + emit any remaining items. Then `after_fun` is invoked, to close any resource, + but not emitting any new items. `last_fun` is only invoked if the given + enumerable terminates successfully (either because it is done or it halted + itself). `after_fun` is always invoked, therefore `after_fun` must be the + one used for closing resources. + """ + @spec transform(Enumerable.t(), start_fun, reducer, last_fun, after_fun) :: Enumerable.t() + when start_fun: (() -> acc), + reducer: (element, acc -> {Enumerable.t(), acc} | {:halt, acc}), + last_fun: (acc -> {Enumerable.t(), acc} | {:halt, acc}), + after_fun: (acc -> term), + acc: any + def transform(enum, start_fun, reducer, last_fun, after_fun) + when is_function(start_fun, 0) and is_function(reducer, 2) and is_function(last_fun, 1) and + is_function(after_fun, 1) do + &do_transform(enum, start_fun, reducer, &1, &2, last_fun, after_fun) + end + + defp do_transform(enumerables, user_acc, user, inner_acc, fun, last_fun, after_fun) do inner = &do_transform_each(&1, &2, fun) - step = &do_transform_step(&1, &2) - next = &Enumerable.reduce(enumerables, &1, step) - do_transform(user_acc, user, fun, [], next, inner_acc, inner) - end - - defp do_transform(user_acc, user, fun, next_acc, next, inner_acc, inner) do - case next.({:cont, next_acc}) do - {:suspended, [val|next_acc], next} -> - try do - user.(val, user_acc) - catch - kind, reason -> - stacktrace = System.stacktrace - next.({:halt, next_acc}) - :erlang.raise(kind, reason, stacktrace) - else - {[], user_acc} -> - do_transform(user_acc, user, fun, next_acc, next, inner_acc, inner) - {list, user_acc} when is_list(list) -> - do_list_transform(user_acc, user, fun, next_acc, next, inner_acc, inner, &Enumerable.List.reduce(list, &1, fun)) - {:halt, _user_acc} -> - next.({:halt, next_acc}) - {:halted, elem(inner_acc, 1)} - {other, user_acc} -> - do_other_transform(user_acc, user, fun, next_acc, next, inner_acc, inner, &Enumerable.reduce(other, &1, inner)) - end - {reason, _} -> - {reason, elem(inner_acc, 1)} + step = &do_transform_step(&1, &2) + next = &Enumerable.reduce(enumerables, &1, step) + funs = {user, fun, inner, last_fun, after_fun} + do_transform(user_acc.(), :cont, next, inner_acc, funs) + end + + defp do_transform(user_acc, _next_op, next, {:halt, inner_acc}, funs) do + {_, _, _, _, after_fun} = funs + next.({:halt, []}) + after_fun.(user_acc) + {:halted, inner_acc} + end + + defp do_transform(user_acc, next_op, next, {:suspend, inner_acc}, funs) do + {:suspended, inner_acc, &do_transform(user_acc, next_op, next, &1, funs)} + end + + defp do_transform(user_acc, :cont, next, inner_acc, funs) do + {_, _, _, _, after_fun} = funs + + try do + next.({:cont, []}) + catch + kind, reason -> + after_fun.(user_acc) + :erlang.raise(kind, reason, __STACKTRACE__) + else + {:suspended, vals, next} -> + do_transform_user(:lists.reverse(vals), user_acc, :cont, next, inner_acc, funs) + + {_, vals} -> + do_transform_user(:lists.reverse(vals), user_acc, :last, next, inner_acc, funs) + end + end + + defp do_transform(user_acc, :last, next, inner_acc, funs) do + {_, _, _, last_fun, after_fun} = funs + + if last_fun do + try do + last_fun.(user_acc) + catch + kind, reason -> + next.({:halt, []}) + after_fun.(user_acc) + :erlang.raise(kind, reason, __STACKTRACE__) + else + result -> do_transform_result(result, [], :halt, next, inner_acc, funs) + end + else + do_transform(user_acc, :halt, next, inner_acc, funs) + end + end + + defp do_transform(user_acc, :halt, _next, inner_acc, funs) do + {_, _, _, _, after_fun} = funs + after_fun.(user_acc) + {:halted, elem(inner_acc, 1)} + end + + defp do_transform_user([], user_acc, next_op, next, inner_acc, funs) do + do_transform(user_acc, next_op, next, inner_acc, funs) + end + + defp do_transform_user([val | vals], user_acc, next_op, next, inner_acc, funs) do + {user, _, _, _, after_fun} = funs + + try do + user.(val, user_acc) + catch + kind, reason -> + next.({:halt, []}) + after_fun.(user_acc) + :erlang.raise(kind, reason, __STACKTRACE__) + else + result -> do_transform_result(result, vals, next_op, next, inner_acc, funs) + end + end + + defp do_transform_result(result, vals, next_op, next, inner_acc, funs) do + {_, fun, inner, _, after_fun} = funs + + case result do + {[], user_acc} -> + do_transform_user(vals, user_acc, next_op, next, inner_acc, funs) + + {list, user_acc} when is_list(list) -> + reduce = &Enumerable.List.reduce(list, &1, fun) + do_transform_inner_list(vals, user_acc, next_op, next, inner_acc, reduce, funs) + + {:halt, user_acc} -> + next.({:halt, []}) + after_fun.(user_acc) + {:halted, elem(inner_acc, 1)} + + {other, user_acc} -> + reduce = &Enumerable.reduce(other, &1, inner) + do_transform_inner_enum(vals, user_acc, next_op, next, inner_acc, reduce, funs) end end - defp do_list_transform(user_acc, user, fun, next_acc, next, inner_acc, inner, reduce) do + defp do_transform_inner_list(vals, user_acc, next_op, next, inner_acc, reduce, funs) do + {_, _, _, _, after_fun} = funs + try do reduce.(inner_acc) catch kind, reason -> - next.({:halt, next_acc}) - :erlang.raise(kind, reason, :erlang.get_stacktrace) + next.({:halt, []}) + after_fun.(user_acc) + :erlang.raise(kind, reason, __STACKTRACE__) else {:done, acc} -> - do_transform(user_acc, user, fun, next_acc, next, {:cont, acc}, inner) + do_transform_user(vals, user_acc, next_op, next, {:cont, acc}, funs) + {:halted, acc} -> - next.({:halt, next_acc}) + next.({:halt, []}) + after_fun.(user_acc) {:halted, acc} - {:suspended, acc, c} -> - {:suspended, acc, &do_list_transform(user_acc, user, fun, next_acc, next, &1, inner, c)} + + {:suspended, acc, continuation} -> + resume = &do_transform_inner_list(vals, user_acc, next_op, next, &1, continuation, funs) + {:suspended, acc, resume} end end - defp do_other_transform(user_acc, user, fun, next_acc, next, inner_acc, inner, reduce) do + defp do_transform_inner_enum(vals, user_acc, next_op, next, {op, inner_acc}, reduce, funs) do + {_, _, _, _, after_fun} = funs + try do - reduce.(inner_acc) + reduce.({op, [:outer | inner_acc]}) catch - {:stream_transform, h} -> - next.({:halt, next_acc}) - {:halted, h} kind, reason -> - stacktrace = System.stacktrace - next.({:halt, next_acc}) - :erlang.raise(kind, reason, stacktrace) + next.({:halt, []}) + after_fun.(user_acc) + :erlang.raise(kind, reason, __STACKTRACE__) else - {_, acc} -> - do_transform(user_acc, user, fun, next_acc, next, {:cont, acc}, inner) - {:suspended, acc, c} -> - {:suspended, acc, &do_other_transform(user_acc, user, fun, next_acc, next, &1, inner, c)} + # Only take into account outer halts when the op is not halt itself. + # Otherwise, we were the ones wishing to halt, so we should just stop. + {:halted, [:outer | acc]} when op != :halt -> + do_transform_user(vals, user_acc, next_op, next, {:cont, acc}, funs) + + {:halted, [_ | acc]} -> + next.({:halt, []}) + after_fun.(user_acc) + {:halted, acc} + + {:done, [_ | acc]} -> + do_transform_user(vals, user_acc, next_op, next, {:cont, acc}, funs) + + {:suspended, [_ | acc], continuation} -> + resume = &do_transform_inner_enum(vals, user_acc, next_op, next, &1, continuation, funs) + {:suspended, acc, resume} end end - defp do_transform_each(x, acc, f) do + defp do_transform_each(x, [:outer | acc], f) do case f.(x, acc) do - {:halt, h} -> throw({:stream_transform, h}) - {_, _} = o -> o + {:halt, res} -> {:halt, [:inner | res]} + {op, res} -> {op, [:outer | res]} end end defp do_transform_step(x, acc) do - {:suspend, [x|acc]} + {:suspend, [x | acc]} end @doc """ @@ -682,37 +1085,71 @@ defmodule Stream do Keep in mind that, in order to know if an element is unique or not, this function needs to store all unique values emitted by the stream. Therefore, if the stream is infinite, the number - of items stored will grow infinitely, never being garbage collected. + of elements stored will grow infinitely, never being garbage-collected. ## Examples - iex> Stream.uniq([1, 2, 3, 2, 1]) |> Enum.to_list + iex> Stream.uniq([1, 2, 3, 3, 2, 1]) |> Enum.to_list() [1, 2, 3] - iex> Stream.uniq([{1, :x}, {2, :y}, {1, :z}], fn {x, _} -> x end) |> Enum.to_list - [{1,:x}, {2,:y}] + """ + @spec uniq(Enumerable.t()) :: Enumerable.t() + def uniq(enum) do + uniq_by(enum, fn x -> x end) + end + + @doc false + @deprecated "Use Stream.uniq_by/2 instead" + def uniq(enum, fun) do + uniq_by(enum, fun) + end + + @doc """ + Creates a stream that only emits elements if they are unique, by removing the + elements for which function `fun` returned duplicate elements. + + The function `fun` maps every element to a term which is used to + determine if two elements are duplicates. + + Keep in mind that, in order to know if an element is unique + or not, this function needs to store all unique values emitted + by the stream. Therefore, if the stream is infinite, the number + of elements stored will grow infinitely, never being garbage-collected. + + ## Example + + iex> Stream.uniq_by([{1, :x}, {2, :y}, {1, :z}], fn {x, _} -> x end) |> Enum.to_list() + [{1, :x}, {2, :y}] + + iex> Stream.uniq_by([a: {:tea, 2}, b: {:tea, 2}, c: {:coffee, 1}], fn {_, y} -> y end) |> Enum.to_list() + [a: {:tea, 2}, c: {:coffee, 1}] """ - @spec uniq(Enumerable.t) :: Enumerable.t - @spec uniq(Enumerable.t, (element -> term)) :: Enumerable.t - def uniq(enum, fun \\ fn x -> x end) do - lazy enum, [], fn f1 -> R.uniq(fun, f1) end + @spec uniq_by(Enumerable.t(), (element -> term)) :: Enumerable.t() + def uniq_by(enum, fun) when is_function(fun, 1) do + lazy(enum, %{}, fn f1 -> R.uniq_by(fun, f1) end) end @doc """ - Creates a stream where each item in the enumerable will + Creates a stream where each element in the enumerable will be wrapped in a tuple alongside its index. + If an `offset` is given, we will index from the given offset instead of from zero. + ## Examples iex> stream = Stream.with_index([1, 2, 3]) iex> Enum.to_list(stream) - [{1,0},{2,1},{3,2}] + [{1, 0}, {2, 1}, {3, 2}] + + iex> stream = Stream.with_index([1, 2, 3], 3) + iex> Enum.to_list(stream) + [{1, 3}, {2, 4}, {3, 5}] """ - @spec with_index(Enumerable.t) :: Enumerable.t - def with_index(enum) do - lazy enum, 0, fn(f1) -> R.with_index(f1) end + @spec with_index(Enumerable.t(), integer) :: Enumerable.t() + def with_index(enum, offset \\ 0) when is_integer(offset) do + lazy(enum, offset, fn f1 -> R.with_index(f1) end) end ## Combiners @@ -724,12 +1161,12 @@ defmodule Stream do iex> stream = Stream.concat([1..3, 4..6, 7..9]) iex> Enum.to_list(stream) - [1,2,3,4,5,6,7,8,9] + [1, 2, 3, 4, 5, 6, 7, 8, 9] """ - @spec concat(Enumerable.t) :: Enumerable.t + @spec concat(Enumerable.t()) :: Enumerable.t() def concat(enumerables) do - flat_map(enumerables, &(&1)) + flat_map(enumerables, & &1) end @doc """ @@ -739,91 +1176,255 @@ defmodule Stream do iex> stream = Stream.concat(1..3, 4..6) iex> Enum.to_list(stream) - [1,2,3,4,5,6] + [1, 2, 3, 4, 5, 6] iex> stream1 = Stream.cycle([1, 2, 3]) iex> stream2 = Stream.cycle([4, 5, 6]) iex> stream = Stream.concat(stream1, stream2) iex> Enum.take(stream, 6) - [1,2,3,1,2,3] + [1, 2, 3, 1, 2, 3] """ - @spec concat(Enumerable.t, Enumerable.t) :: Enumerable.t + @spec concat(Enumerable.t(), Enumerable.t()) :: Enumerable.t() def concat(first, second) do - flat_map([first, second], &(&1)) + flat_map([first, second], & &1) + end + + @doc """ + Zips two enumerables together, lazily. + + The zipping finishes as soon as either enumerable completes. + + ## Examples + + iex> concat = Stream.concat(1..3, 4..6) + iex> cycle = Stream.cycle([:a, :b, :c]) + iex> Stream.zip(concat, cycle) |> Enum.to_list() + [{1, :a}, {2, :b}, {3, :c}, {4, :a}, {5, :b}, {6, :c}] + + """ + @spec zip(Enumerable.t(), Enumerable.t()) :: Enumerable.t() + def zip(enumerable1, enumerable2) do + zip_with(enumerable1, enumerable2, fn left, right -> {left, right} end) end @doc """ - Zips two collections together, lazily. + Zips corresponding elements from a finite collection of enumerables + into one stream of tuples. - The zipping finishes as soon as any enumerable completes. + The zipping finishes as soon as any enumerable in the given collection completes. ## Examples iex> concat = Stream.concat(1..3, 4..6) - iex> cycle = Stream.cycle([:a, :b, :c]) - iex> Stream.zip(concat, cycle) |> Enum.to_list - [{1,:a},{2,:b},{3,:c},{4,:a},{5,:b},{6,:c}] + iex> cycle = Stream.cycle(["foo", "bar", "baz"]) + iex> Stream.zip([concat, [:a, :b, :c], cycle]) |> Enum.to_list() + [{1, :a, "foo"}, {2, :b, "bar"}, {3, :c, "baz"}] """ - @spec zip(Enumerable.t, Enumerable.t) :: Enumerable.t - def zip(left, right) do - step = &do_zip_step(&1, &2) - left_fun = &Enumerable.reduce(left, &1, step) - right_fun = &Enumerable.reduce(right, &1, step) + @doc since: "1.4.0" + @spec zip(enumerables) :: Enumerable.t() when enumerables: [Enumerable.t()] | Enumerable.t() + def zip(enumerables) do + zip_with(enumerables, &List.to_tuple(&1)) + end + + @doc """ + Lazily zips corresponding elements from two enumerables into a new one, transforming them with + the `zip_fun` function as it goes. + + The `zip_fun` will be called with the first element from `enumerable1` and the first + element from `enumerable2`, then with the second element from each, and so on until + either one of the enumerables completes. - # Return a function as a lazy enumerator. - &do_zip([{left_fun, []}, {right_fun, []}], &1, &2) + ## Examples + + iex> concat = Stream.concat(1..3, 4..6) + iex> Stream.zip_with(concat, concat, fn a, b -> a + b end) |> Enum.to_list() + [2, 4, 6, 8, 10, 12] + + """ + @doc since: "1.12.0" + @spec zip_with(Enumerable.t(), Enumerable.t(), (term, term -> term)) :: Enumerable.t() + def zip_with(enumerable1, enumerable2, zip_fun) + when is_list(enumerable1) and is_list(enumerable2) and is_function(zip_fun, 2) do + &zip_pair(enumerable1, enumerable2, &1, &2, zip_fun) end - defp do_zip(zips, {:halt, acc}, _fun) do + def zip_with(enumerable1, enumerable2, zip_fun) when is_function(zip_fun, 2) do + zip_with([enumerable1, enumerable2], fn [left, right] -> zip_fun.(left, right) end) + end + + defp zip_pair(_list1, _list2, {:halt, acc}, _fun, _zip_fun) do + {:halted, acc} + end + + defp zip_pair(list1, list2, {:suspend, acc}, fun, zip_fun) do + {:suspended, acc, &zip_pair(list1, list2, &1, fun, zip_fun)} + end + + defp zip_pair([], _list2, {:cont, acc}, _fun, _zip_fun), do: {:done, acc} + defp zip_pair(_list1, [], {:cont, acc}, _fun, _zip_fun), do: {:done, acc} + + defp zip_pair([head1 | tail1], [head2 | tail2], {:cont, acc}, fun, zip_fun) do + zip_pair(tail1, tail2, fun.(zip_fun.(head1, head2), acc), fun, zip_fun) + end + + @doc """ + Lazily zips corresponding elements from a finite collection of enumerables into a new + enumerable, transforming them with the `zip_fun` function as it goes. + + The first element from each of the enums in `enumerables` will be put into a list which is then passed to + the one-arity `zip_fun` function. Then, the second elements from each of the enums are put into a list and passed to + `zip_fun`, and so on until any one of the enums in `enumerables` completes. + + Returns a new enumerable with the results of calling `zip_fun`. + + ## Examples + + iex> concat = Stream.concat(1..3, 4..6) + iex> Stream.zip_with([concat, concat], fn [a, b] -> a + b end) |> Enum.to_list() + [2, 4, 6, 8, 10, 12] + + iex> concat = Stream.concat(1..3, 4..6) + iex> Stream.zip_with([concat, concat, 1..3], fn [a, b, c] -> a + b + c end) |> Enum.to_list() + [3, 6, 9] + + """ + @doc since: "1.12.0" + @spec zip_with(enumerables, (Enumerable.t() -> term)) :: Enumerable.t() + when enumerables: [Enumerable.t()] | Enumerable.t() + def zip_with(enumerables, zip_fun) when is_function(zip_fun, 1) do + if is_list(enumerables) and :lists.all(&is_list/1, enumerables) do + &zip_list(enumerables, &1, &2, zip_fun) + else + &zip_enum(enumerables, &1, &2, zip_fun) + end + end + + defp zip_list(_enumerables, {:halt, acc}, _fun, _zip_fun) do + {:halted, acc} + end + + defp zip_list(enumerables, {:suspend, acc}, fun, zip_fun) do + {:suspended, acc, &zip_list(enumerables, &1, fun, zip_fun)} + end + + defp zip_list(enumerables, {:cont, acc}, fun, zip_fun) do + case zip_list_heads_tails(enumerables, [], []) do + {heads, tails} -> zip_list(tails, fun.(zip_fun.(heads), acc), fun, zip_fun) + :error -> {:done, acc} + end + end + + defp zip_list_heads_tails([[head | tail] | rest], heads, tails) do + zip_list_heads_tails(rest, [head | heads], [tail | tails]) + end + + defp zip_list_heads_tails([[] | _rest], _heads, _tails) do + :error + end + + defp zip_list_heads_tails([], heads, tails) do + {:lists.reverse(heads), :lists.reverse(tails)} + end + + defp zip_enum(enumerables, acc, fun, zip_fun) do + step = fn x, acc -> + {:suspend, :lists.reverse([x | acc])} + end + + enum_funs = + Enum.map(enumerables, fn enum -> + {&Enumerable.reduce(enum, &1, step), [], :cont} + end) + + do_zip_enum(enum_funs, acc, fun, zip_fun) + end + + # This implementation of do_zip_enum/4 works for any number of streams to zip + defp do_zip_enum(zips, {:halt, acc}, _fun, _zip_fun) do do_zip_close(zips) {:halted, acc} end - defp do_zip(zips, {:suspend, acc}, fun) do - {:suspended, acc, &do_zip(zips, &1, fun)} + defp do_zip_enum(zips, {:suspend, acc}, fun, zip_fun) do + {:suspended, acc, &do_zip_enum(zips, &1, fun, zip_fun)} end - defp do_zip(zips, {:cont, acc}, callback) do + defp do_zip_enum([], {:cont, acc}, _callback, _zip_fun) do + {:done, acc} + end + + defp do_zip_enum(zips, {:cont, acc}, callback, zip_fun) do try do - do_zip(zips, acc, callback, [], []) + do_zip_next(zips, acc, callback, [], [], zip_fun) catch kind, reason -> - stacktrace = System.stacktrace do_zip_close(zips) - :erlang.raise(kind, reason, stacktrace) + :erlang.raise(kind, reason, __STACKTRACE__) else {:next, buffer, acc} -> - do_zip(buffer, acc, callback) - {:done, _} = o -> - o + do_zip_enum(buffer, acc, callback, zip_fun) + + {:done, _acc} = other -> + other end end - defp do_zip([{fun, fun_acc}|t], acc, callback, list, buffer) do - case fun.({:cont, fun_acc}) do - {:suspended, [i|fun_acc], fun} -> - do_zip(t, acc, callback, [i|list], [{fun, fun_acc}|buffer]) - {_, _} -> - do_zip_close(:lists.reverse(buffer) ++ t) + # do_zip_next/6 computes the next tuple formed by + # the next element of each zipped stream. + defp do_zip_next( + [{_, [], :halt} | zips], + acc, + _callback, + _yielded_elems, + buffer, + _zip_fun + ) do + do_zip_close(:lists.reverse(buffer, zips)) + {:done, acc} + end + + defp do_zip_next([{fun, [], :cont} | zips], acc, callback, yielded_elems, buffer, zip_fun) do + case fun.({:cont, []}) do + {:suspended, [elem | next_acc], fun} -> + next_buffer = [{fun, next_acc, :cont} | buffer] + do_zip_next(zips, acc, callback, [elem | yielded_elems], next_buffer, zip_fun) + + {_, [elem | next_acc]} -> + next_buffer = [{fun, next_acc, :halt} | buffer] + do_zip_next(zips, acc, callback, [elem | yielded_elems], next_buffer, zip_fun) + + {_, []} -> + # The current zipped stream terminated, so we close all the streams + # and return {:halted, acc} (which is returned as is by do_zip/3). + do_zip_close(:lists.reverse(buffer, zips)) {:done, acc} end end - defp do_zip([], acc, callback, list, buffer) do - zipped = List.to_tuple(:lists.reverse(list)) - {:next, :lists.reverse(buffer), callback.(zipped, acc)} + defp do_zip_next( + [{fun, zip_acc, zip_op} | zips], + acc, + callback, + yielded_elems, + buffer, + zip_fun + ) do + [elem | rest] = zip_acc + next_buffer = [{fun, rest, zip_op} | buffer] + do_zip_next(zips, acc, callback, [elem | yielded_elems], next_buffer, zip_fun) end - defp do_zip_close([]), do: :ok - defp do_zip_close([{fun, acc}|t]) do - fun.({:halt, acc}) - do_zip_close(t) + defp do_zip_next([] = _zips, acc, callback, yielded_elems, buffer, zip_fun) do + # "yielded_elems" is a reversed list of results for the current iteration of + # zipping. That is to say, the nth element from each of the enums being zipped. + # It needs to be reversed and passed to the zipping function so it can do it's thing. + {:next, :lists.reverse(buffer), callback.(zip_fun.(:lists.reverse(yielded_elems)), acc)} end - defp do_zip_step(x, acc) do - {:suspend, [x|acc]} + defp do_zip_close(zips) do + :lists.foreach(fn {fun, _, _} -> fun.({:halt, []}) end, zips) end ## Sources @@ -834,64 +1435,88 @@ defmodule Stream do ## Examples - iex> stream = Stream.cycle([1,2,3]) + iex> stream = Stream.cycle([1, 2, 3]) iex> Enum.take(stream, 5) - [1,2,3,1,2] + [1, 2, 3, 1, 2] """ - @spec cycle(Enumerable.t) :: Enumerable.t + @spec cycle(Enumerable.t()) :: Enumerable.t() + def cycle(enumerable) + + def cycle([]) do + raise ArgumentError, "cannot cycle over an empty enumerable" + end + + def cycle(enumerable) when is_list(enumerable) do + unfold({enumerable, enumerable}, fn + {source, [h | t]} -> {h, {source, t}} + {source = [h | t], []} -> {h, {source, t}} + end) + end + def cycle(enumerable) do fn acc, fun -> - inner = &do_cycle_each(&1, &2, fun) - outer = &Enumerable.reduce(enumerable, &1, inner) - do_cycle(outer, outer, acc) + step = &do_cycle_step(&1, &2) + cycle = &Enumerable.reduce(enumerable, &1, step) + reduce = check_cycle_first_element(cycle) + do_cycle(reduce, [], cycle, acc, fun) end end - defp do_cycle(_reduce, _cycle, {:halt, acc}) do + defp do_cycle(reduce, inner_acc, _cycle, {:halt, acc}, _fun) do + reduce.({:halt, inner_acc}) {:halted, acc} end - defp do_cycle(reduce, cycle, {:suspend, acc}) do - {:suspended, acc, &do_cycle(reduce, cycle, &1)} + defp do_cycle(reduce, inner_acc, cycle, {:suspend, acc}, fun) do + {:suspended, acc, &do_cycle(reduce, inner_acc, cycle, &1, fun)} end - defp do_cycle(reduce, cycle, acc) do - try do - reduce.(acc) - catch - {:stream_cycle, acc} -> - {:halted, acc} - else - {state, acc} when state in [:done, :halted] -> - do_cycle(cycle, cycle, {:cont, acc}) - {:suspended, acc, continuation} -> - {:suspended, acc, &do_cycle(continuation, cycle, &1)} + defp do_cycle(reduce, inner_acc, cycle, {:cont, acc}, fun) do + case reduce.({:cont, inner_acc}) do + {:suspended, [element], new_reduce} -> + do_cycle(new_reduce, inner_acc, cycle, fun.(element, acc), fun) + + {_, [element]} -> + do_cycle(cycle, [], cycle, fun.(element, acc), fun) + + {_, []} -> + do_cycle(cycle, [], cycle, {:cont, acc}, fun) end end - defp do_cycle_each(x, acc, f) do - case f.(x, acc) do - {:halt, h} -> throw({:stream_cycle, h}) - {_, _} = o -> o + defp do_cycle_step(x, acc) do + {:suspend, [x | acc]} + end + + defp check_cycle_first_element(reduce) do + fn acc -> + case reduce.(acc) do + {state, []} when state in [:done, :halted] -> + raise ArgumentError, "cannot cycle over an empty enumerable" + + other -> + other + end end end @doc """ - Emit a sequence of values, starting with `start_value`. Successive + Emits a sequence of values, starting with `start_value`. Successive values are generated by calling `next_fun` on the previous value. ## Examples - iex> Stream.iterate(0, &(&1+1)) |> Enum.take(5) - [0,1,2,3,4] + iex> Stream.iterate(0, &(&1 + 1)) |> Enum.take(5) + [0, 1, 2, 3, 4] """ - @spec iterate(element, (element -> element)) :: Enumerable.t - def iterate(start_value, next_fun) do + @spec iterate(element, (element -> element)) :: Enumerable.t() + def iterate(start_value, next_fun) when is_function(next_fun, 1) do unfold({:ok, start_value}, fn {:ok, value} -> {value, {:next, value}} + {:next, value} -> next = next_fun.(value) {next, {:next, next}} @@ -903,11 +1528,13 @@ defmodule Stream do ## Examples - iex> Stream.repeatedly(&:random.uniform/0) |> Enum.take(3) - [0.4435846174457203, 0.7230402056221108, 0.94581636451987] + # Although not necessary, let's seed the random algorithm + iex> :rand.seed(:exsss, {1, 2, 3}) + iex> Stream.repeatedly(&:rand.uniform/0) |> Enum.take(3) + [0.5455598952593053, 0.6039309974353404, 0.6684893034823949] """ - @spec repeatedly((() -> element)) :: Enumerable.t + @spec repeatedly((() -> element)) :: Enumerable.t() def repeatedly(generator_fun) when is_function(generator_fun, 0) do &do_repeatedly(generator_fun, &1, &2) end @@ -927,32 +1554,52 @@ defmodule Stream do @doc """ Emits a sequence of values for the given resource. - Similar to `unfold/2` but the initial value is computed lazily via - `start_fun` and executes an `after_fun` at the end of enumeration - (both in cases of success and failure). + Similar to `transform/3` but the initial accumulated value is + computed lazily via `start_fun` and executes an `after_fun` at + the end of enumeration (both in cases of success and failure). Successive values are generated by calling `next_fun` with the previous accumulator (the initial value being the result returned - by `start_fun`) and it must return a tuple with the current and - next accumulator. The enumeration finishes if it returns `nil`. + by `start_fun`) and it must return a tuple containing a list + of elements to be emitted and the next accumulator. The enumeration + finishes if it returns `{:halt, acc}`. As the name says, this function is useful to stream values from resources. ## Examples - Stream.resource(fn -> File.open!("sample") end, - fn file -> - case IO.read(file, :line) do - data when is_binary(data) -> {data, file} - _ -> nil - end - end, - fn file -> File.close(file) end) + Stream.resource( + fn -> File.open!("sample") end, + fn file -> + case IO.read(file, :line) do + data when is_binary(data) -> {[data], file} + _ -> {:halt, file} + end + end, + fn file -> File.close(file) end + ) + + iex> Stream.resource( + ...> fn -> + ...> {:ok, pid} = StringIO.open("string") + ...> pid + ...> end, + ...> fn pid -> + ...> case IO.getn(pid, "", 1) do + ...> :eof -> {:halt, pid} + ...> char -> {[char], pid} + ...> end + ...> end, + ...> fn pid -> StringIO.close(pid) end + ...> ) |> Enum.to_list() + ["s", "t", "r", "i", "n", "g"] """ - @spec resource((() -> acc), (acc -> {element, acc} | nil), (acc -> term)) :: Enumerable.t - def resource(start_fun, next_fun, after_fun) do + @spec resource((() -> acc), (acc -> {[element], acc} | {:halt, acc}), (acc -> term)) :: + Enumerable.t() + def resource(start_fun, next_fun, after_fun) + when is_function(start_fun, 0) and is_function(next_fun, 1) and is_function(after_fun, 1) do &do_resource(start_fun.(), next_fun, &1, &2, after_fun) end @@ -967,24 +1614,93 @@ defmodule Stream do defp do_resource(next_acc, next_fun, {:cont, acc}, fun, after_fun) do try do - case next_fun.(next_acc) do - nil -> nil - {v, next_acc} -> {fun.(v, acc), next_acc} - end + next_fun.(next_acc) catch kind, reason -> - stacktrace = System.stacktrace after_fun.(next_acc) - :erlang.raise(kind, reason, stacktrace) + :erlang.raise(kind, reason, __STACKTRACE__) else - nil -> + {:halt, next_acc} -> + do_resource(next_acc, next_fun, {:halt, acc}, fun, after_fun) + + {[], next_acc} -> + do_resource(next_acc, next_fun, {:cont, acc}, fun, after_fun) + + {[v], next_acc} -> + do_element_resource(next_acc, next_fun, acc, fun, after_fun, v) + + {list, next_acc} when is_list(list) -> + reduce = &Enumerable.List.reduce(list, &1, fun) + do_list_resource(next_acc, next_fun, {:cont, acc}, fun, after_fun, reduce) + + {enum, next_acc} -> + inner = &do_resource_each(&1, &2, fun) + reduce = &Enumerable.reduce(enum, &1, inner) + do_enum_resource(next_acc, next_fun, {:cont, acc}, fun, after_fun, reduce) + end + end + + defp do_element_resource(next_acc, next_fun, acc, fun, after_fun, v) do + try do + fun.(v, acc) + catch + kind, reason -> after_fun.(next_acc) - {:done, acc} - {acc, next_acc} -> + :erlang.raise(kind, reason, __STACKTRACE__) + else + acc -> do_resource(next_acc, next_fun, acc, fun, after_fun) end end + defp do_list_resource(next_acc, next_fun, acc, fun, after_fun, reduce) do + try do + reduce.(acc) + catch + kind, reason -> + after_fun.(next_acc) + :erlang.raise(kind, reason, __STACKTRACE__) + else + {:done, acc} -> + do_resource(next_acc, next_fun, {:cont, acc}, fun, after_fun) + + {:halted, acc} -> + do_resource(next_acc, next_fun, {:halt, acc}, fun, after_fun) + + {:suspended, acc, c} -> + {:suspended, acc, &do_list_resource(next_acc, next_fun, &1, fun, after_fun, c)} + end + end + + defp do_enum_resource(next_acc, next_fun, {op, acc}, fun, after_fun, reduce) do + try do + reduce.({op, [:outer | acc]}) + catch + kind, reason -> + after_fun.(next_acc) + :erlang.raise(kind, reason, __STACKTRACE__) + else + {:halted, [:outer | acc]} -> + do_resource(next_acc, next_fun, {:cont, acc}, fun, after_fun) + + {:halted, [:inner | acc]} -> + do_resource(next_acc, next_fun, {:halt, acc}, fun, after_fun) + + {:done, [_ | acc]} -> + do_resource(next_acc, next_fun, {:cont, acc}, fun, after_fun) + + {:suspended, [_ | acc], c} -> + {:suspended, acc, &do_enum_resource(next_acc, next_fun, &1, fun, after_fun, c)} + end + end + + defp do_resource_each(x, [:outer | acc], f) do + case f.(x, acc) do + {:halt, res} -> {:halt, [:inner | res]} + {op, res} -> {op, [:outer | res]} + end + end + @doc """ Emits a sequence of values for the given accumulator. @@ -994,12 +1710,15 @@ defmodule Stream do ## Examples - iex> Stream.unfold(5, fn 0 -> nil; n -> {n, n-1} end) |> Enum.to_list() + iex> Stream.unfold(5, fn + ...> 0 -> nil + ...> n -> {n, n - 1} + ...> end) |> Enum.to_list() [5, 4, 3, 2, 1] """ - @spec unfold(acc, (acc -> {element, acc} | nil)) :: Enumerable.t - def unfold(next_acc, next_fun) do + @spec unfold(acc, (acc -> {element, acc} | nil)) :: Enumerable.t() + def unfold(next_acc, next_fun) when is_function(next_fun, 1) do &do_unfold(next_acc, next_fun, &1, &2) end @@ -1013,33 +1732,61 @@ defmodule Stream do defp do_unfold(next_acc, next_fun, {:cont, acc}, fun) do case next_fun.(next_acc) do - nil -> {:done, acc} + nil -> {:done, acc} {v, next_acc} -> do_unfold(next_acc, next_fun, fun.(v, acc), fun) end end + @doc """ + Lazily intersperses `intersperse_element` between each element of the enumeration. + + ## Examples + + iex> Stream.intersperse([1, 2, 3], 0) |> Enum.to_list() + [1, 0, 2, 0, 3] + + iex> Stream.intersperse([1], 0) |> Enum.to_list() + [1] + + iex> Stream.intersperse([], 0) |> Enum.to_list() + [] + + """ + @doc since: "1.6.0" + @spec intersperse(Enumerable.t(), any) :: Enumerable.t() + def intersperse(enumerable, intersperse_element) do + Stream.transform(enumerable, false, fn + element, true -> {[intersperse_element, element], true} + element, false -> {[element], true} + end) + end + ## Helpers @compile {:inline, lazy: 2, lazy: 3, lazy: 4} - defp lazy(%Stream{funs: funs} = lazy, fun), - do: %{lazy | funs: [fun|funs] } - defp lazy(enum, fun), - do: %Stream{enum: enum, funs: [fun]} + defp lazy(%Stream{done: nil, funs: funs} = lazy, fun), do: %{lazy | funs: [fun | funs]} + defp lazy(enum, fun), do: %Stream{enum: enum, funs: [fun]} + + defp lazy(%Stream{done: nil, funs: funs, accs: accs} = lazy, acc, fun), + do: %{lazy | funs: [fun | funs], accs: [acc | accs]} - defp lazy(%Stream{funs: funs, accs: accs} = lazy, acc, fun), - do: %{lazy | funs: [fun|funs], accs: [acc|accs] } - defp lazy(enum, acc, fun), - do: %Stream{enum: enum, funs: [fun], accs: [acc]} + defp lazy(enum, acc, fun), do: %Stream{enum: enum, funs: [fun], accs: [acc]} defp lazy(%Stream{done: nil, funs: funs, accs: accs} = lazy, acc, fun, done), - do: %{lazy | funs: [fun|funs], accs: [acc|accs], done: done} - defp lazy(enum, acc, fun, done), - do: %Stream{enum: enum, funs: [fun], accs: [acc], done: done} + do: %{lazy | funs: [fun | funs], accs: [acc | accs], done: done} + + defp lazy(enum, acc, fun, done), do: %Stream{enum: enum, funs: [fun], accs: [acc], done: done} end defimpl Enumerable, for: Stream do - @compile :inline_list_funs + @compile :inline_list_funcs + + def count(_lazy), do: {:error, __MODULE__} + + def member?(_lazy, _value), do: {:error, __MODULE__} + + def slice(_lazy), do: {:error, __MODULE__} def reduce(lazy, acc, fun) do do_reduce(lazy, acc, fn x, [acc] -> @@ -1048,37 +1795,43 @@ defimpl Enumerable, for: Stream do end) end - def count(_lazy) do - {:error, __MODULE__} - end - - def member?(_lazy, _value) do - {:error, __MODULE__} - end - defp do_reduce(%Stream{enum: enum, funs: funs, accs: accs, done: done}, acc, fun) do - composed = :lists.foldl(fn fun, acc -> fun.(acc) end, fun, funs) - do_each(&Enumerable.reduce(enum, &1, composed), - done && {done, fun}, :lists.reverse(accs), acc) + composed = :lists.foldl(fn entry_fun, acc -> entry_fun.(acc) end, fun, funs) + reduce = &Enumerable.reduce(enum, &1, composed) + do_each(reduce, done && {done, fun}, :lists.reverse(accs), acc) end defp do_each(reduce, done, accs, {command, acc}) do - case reduce.({command, [acc|accs]}) do - {:suspended, [acc|accs], continuation} -> + case reduce.({command, [acc | accs]}) do + {:suspended, [acc | accs], continuation} -> {:suspended, acc, &do_each(continuation, done, accs, &1)} - {:halted, [acc|_]} -> - {:halted, acc} - {:done, [acc|_] = accs} -> - case done do - nil -> - {:done, acc} - {done, fun} -> - case done.(fun).(accs) do - {:cont, [acc|_]} -> {:done, acc} - {:halt, [acc|_]} -> {:halted, acc} - {:suspend, [acc|_]} -> {:suspended, acc, &({:done, elem(&1, 1)})} - end - end + + {:halted, accs} -> + do_done({:halted, accs}, done) + + {:done, accs} -> + do_done({:done, accs}, done) end end + + defp do_done({reason, [acc | _]}, nil), do: {reason, acc} + + defp do_done({reason, [acc | t]}, {done, fun}) do + [h | _] = :lists.reverse(t) + + case done.([acc, h], fun) do + {:cont, [acc | _]} -> {reason, acc} + {:halt, [acc | _]} -> {:halted, acc} + {:suspend, [acc | _]} -> {:suspended, acc, &{:done, elem(&1, 1)}} + end + end +end + +defimpl Inspect, for: Stream do + import Inspect.Algebra + + def inspect(%{enum: enum, funs: funs}, opts) do + inner = [enum: enum, funs: :lists.reverse(funs)] + concat(["#Stream<", to_doc(inner, opts), ">"]) + end end diff --git a/lib/elixir/lib/stream/reducers.ex b/lib/elixir/lib/stream/reducers.ex index 18659600fd5..668cfd58bbb 100644 --- a/lib/elixir/lib/stream/reducers.ex +++ b/lib/elixir/lib/stream/reducers.ex @@ -1,163 +1,222 @@ defmodule Stream.Reducers do - # Collection of reducers shared by Enum and Stream. + # Collection of reducers and utilities shared by Enum and Stream. @moduledoc false - defmacro chunk(n, step, limit, f \\ nil) do - quote do - fn entry, acc(h, {buffer, count}, t) -> - buffer = [entry|buffer] - count = count + 1 + def chunk_every(chunk_by, enumerable, count, step, leftover) do + limit = :erlang.max(count, step) - new = - if count >= unquote(limit) do - left = count - unquote(step) - {Enum.take(buffer, left), left} - else - {buffer, count} - end + chunk_fun = fn entry, {acc_buffer, acc_count} -> + acc_buffer = [entry | acc_buffer] + acc_count = acc_count + 1 - if count == unquote(n) do - cont_with_acc(unquote(f), :lists.reverse(buffer), h, new, t) + new_state = + if acc_count >= limit do + remaining = acc_count - step + {Enum.take(acc_buffer, remaining), remaining} else - {:cont, acc(h, new, t)} + {acc_buffer, acc_count} end + + if acc_count == count do + {:cont, :lists.reverse(acc_buffer), new_state} + else + {:cont, new_state} + end + end + + after_fun = fn {acc_buffer, acc_count} -> + if leftover == :discard or acc_count == 0 or acc_count >= count do + {:cont, []} + else + {:cont, :lists.reverse(acc_buffer, Enum.take(leftover, count - acc_count)), []} end end + + chunk_by.(enumerable, {[], 0}, chunk_fun, after_fun) end - defmacro chunk_by(callback, f \\ nil) do + def chunk_by(chunk_by, enumerable, fun) do + chunk_fun = fn + entry, nil -> + {:cont, {[entry], fun.(entry)}} + + entry, {acc, value} -> + case fun.(entry) do + ^value -> {:cont, {[entry | acc], value}} + new_value -> {:cont, :lists.reverse(acc), {[entry], new_value}} + end + end + + after_fun = fn + nil -> {:cont, :done} + {acc, _value} -> {:cont, :lists.reverse(acc), :done} + end + + chunk_by.(enumerable, nil, chunk_fun, after_fun) + end + + defmacro dedup(callback, fun \\ nil) do + quote do + fn entry, acc(head, prev, tail) = acc -> + value = unquote(callback).(entry) + + case prev do + {:value, ^value} -> skip(acc) + _ -> next_with_acc(unquote(fun), entry, head, {:value, value}, tail) + end + end + end + end + + defmacro drop(fun \\ nil) do quote do fn - entry, acc(h, {buffer, value}, t) -> - new_value = unquote(callback).(entry) - if new_value == value do - {:cont, acc(h, {[entry|buffer], value}, t)} - else - cont_with_acc(unquote(f), :lists.reverse(buffer), h, {[entry], new_value}, t) - end - entry, acc(h, nil, t) -> - {:cont, acc(h, {[entry], unquote(callback).(entry)}, t)} + _entry, acc(head, amount, tail) when amount > 0 -> + skip(acc(head, amount - 1, tail)) + + entry, acc(head, amount, tail) -> + next_with_acc(unquote(fun), entry, head, amount, tail) end end end - defmacro drop(f \\ nil) do + defmacro drop_every(nth, fun \\ nil) do quote do fn - _entry, acc(h, n, t) when n > 0 -> - {:cont, acc(h, n-1, t)} - entry, acc(h, n, t) -> - cont_with_acc(unquote(f), entry, h, n, t) + entry, acc(head, curr, tail) when curr in [unquote(nth), :first] -> + skip(acc(head, 1, tail)) + + entry, acc(head, curr, tail) -> + next_with_acc(unquote(fun), entry, head, curr + 1, tail) end end end - defmacro drop_while(callback, f \\ nil) do + defmacro drop_while(callback, fun \\ nil) do quote do - fn entry, acc(h, bool, t) = orig -> + fn entry, acc(head, bool, tail) = original -> if bool and unquote(callback).(entry) do - {:cont, orig} + skip(original) else - cont_with_acc(unquote(f), entry, h, false, t) + next_with_acc(unquote(fun), entry, head, false, tail) end end end end - defmacro filter(callback, f \\ nil) do + defmacro filter(callback, fun \\ nil) do quote do - fn(entry, acc) -> + fn entry, acc -> if unquote(callback).(entry) do - cont(unquote(f), entry, acc) + next(unquote(fun), entry, acc) else - {:cont, acc} + skip(acc) end end end end - defmacro filter_map(filter, mapper, f \\ nil) do + defmacro filter_map(filter, mapper, fun \\ nil) do quote do - fn(entry, acc) -> + fn entry, acc -> if unquote(filter).(entry) do - cont(unquote(f), unquote(mapper).(entry), acc) + next(unquote(fun), unquote(mapper).(entry), acc) else - {:cont, acc} + skip(acc) end end end end - defmacro map(callback, f \\ nil) do + defmacro map(callback, fun \\ nil) do + quote do + fn entry, acc -> + next(unquote(fun), unquote(callback).(entry), acc) + end + end + end + + defmacro map_every(nth, mapper, fun \\ nil) do quote do - fn(entry, acc) -> - cont(unquote(f), unquote(callback).(entry), acc) + fn + entry, acc(head, curr, tail) when curr in [unquote(nth), :first] -> + next_with_acc(unquote(fun), unquote(mapper).(entry), head, 1, tail) + + entry, acc(head, curr, tail) -> + next_with_acc(unquote(fun), entry, head, curr + 1, tail) end end end - defmacro reject(callback, f \\ nil) do + defmacro reject(callback, fun \\ nil) do quote do - fn(entry, acc) -> + fn entry, acc -> unless unquote(callback).(entry) do - cont(unquote(f), entry, acc) + next(unquote(fun), entry, acc) else - {:cont, acc} + skip(acc) end end end end - defmacro scan_2(callback, f \\ nil) do + defmacro scan2(callback, fun \\ nil) do quote do fn - entry, acc(h, :first, t) -> - cont_with_acc(unquote(f), entry, h, {:ok, entry}, t) - entry, acc(h, {:ok, acc}, t) -> + entry, acc(head, :first, tail) -> + next_with_acc(unquote(fun), entry, head, {:ok, entry}, tail) + + entry, acc(head, {:ok, acc}, tail) -> value = unquote(callback).(entry, acc) - cont_with_acc(unquote(f), value, h, {:ok, value}, t) + next_with_acc(unquote(fun), value, head, {:ok, value}, tail) end end end - defmacro scan_3(callback, f \\ nil) do + defmacro scan3(callback, fun \\ nil) do quote do - fn(entry, acc(h, acc, t)) -> + fn entry, acc(head, acc, tail) -> value = unquote(callback).(entry, acc) - cont_with_acc(unquote(f), value, h, value, t) + next_with_acc(unquote(fun), value, head, value, tail) end end end - defmacro take(f \\ nil) do + defmacro take(fun \\ nil) do quote do - fn(entry, acc(h, n, t) = orig) -> - if n >= 1 do - cont_with_acc(unquote(f), entry, h, n-1, t) - else - {:halt, orig} + fn entry, acc(head, curr, tail) = original -> + case curr do + 0 -> + {:halt, original} + + 1 -> + {_, acc} = next_with_acc(unquote(fun), entry, head, 0, tail) + {:halt, acc} + + _ -> + next_with_acc(unquote(fun), entry, head, curr - 1, tail) end end end end - defmacro take_every(nth, f \\ nil) do + defmacro take_every(nth, fun \\ nil) do quote do fn - entry, acc(h, n, t) when n === :first - when n === unquote(nth) -> - cont_with_acc(unquote(f), entry, h, 1, t) - entry, acc(h, n, t) -> - {:cont, acc(h, n+1, t)} + entry, acc(head, curr, tail) when curr in [unquote(nth), :first] -> + next_with_acc(unquote(fun), entry, head, 1, tail) + + entry, acc(head, curr, tail) -> + skip(acc(head, curr + 1, tail)) end end end - defmacro take_while(callback, f \\ nil) do + defmacro take_while(callback, fun \\ nil) do quote do - fn(entry, acc) -> + fn entry, acc -> if unquote(callback).(entry) do - cont(unquote(f), entry, acc) + next(unquote(fun), entry, acc) else {:halt, acc} end @@ -165,23 +224,24 @@ defmodule Stream.Reducers do end end - defmacro uniq(callback, f \\ nil) do + defmacro uniq_by(callback, fun \\ nil) do quote do - fn(entry, acc(h, prev, t) = acc) -> + fn entry, acc(head, prev, tail) = original -> value = unquote(callback).(entry) - if :lists.member(value, prev) do - {:cont, acc} + + if Map.has_key?(prev, value) do + skip(original) else - cont_with_acc(unquote(f), entry, h, [value|prev], t) + next_with_acc(unquote(fun), entry, head, Map.put(prev, value, true), tail) end end end end - defmacro with_index(f \\ nil) do + defmacro with_index(fun \\ nil) do quote do - fn(entry, acc(h, counter, t)) -> - cont_with_acc(unquote(f), {entry, counter}, h, counter + 1, t) + fn entry, acc(head, counter, tail) -> + next_with_acc(unquote(fun), {entry, counter}, head, counter + 1, tail) end end end diff --git a/lib/elixir/lib/string.ex b/lib/elixir/lib/string.ex index b0895253346..c0e959f5c94 100644 --- a/lib/elixir/lib/string.ex +++ b/lib/elixir/lib/string.ex @@ -2,156 +2,359 @@ import Kernel, except: [length: 1] defmodule String do @moduledoc ~S""" - A String in Elixir is a UTF-8 encoded binary. + Strings in Elixir are UTF-8 encoded binaries. + + Strings in Elixir are a sequence of Unicode characters, + typically written between double quoted strings, such + as `"hello"` and `"héllò"`. + + In case a string must have a double-quote in itself, + the double quotes must be escaped with a backslash, + for example: `"this is a string with \"double quotes\""`. + + You can concatenate two strings with the `<>/2` operator: + + iex> "hello" <> " " <> "world" + "hello world" + + The functions in this module act according to + [The Unicode Standard, Version 14.0.0](http://www.unicode.org/versions/Unicode14.0.0/). + + ## Interpolation + + Strings in Elixir also support interpolation. This allows + you to place some value in the middle of a string by using + the `#{}` syntax: + + iex> name = "joe" + iex> "hello #{name}" + "hello joe" + + Any Elixir expression is valid inside the interpolation. + If a string is given, the string is interpolated as is. + If any other value is given, Elixir will attempt to convert + it to a string using the `String.Chars` protocol. This + allows, for example, to output an integer from the interpolation: + + iex> "2 + 2 = #{2 + 2}" + "2 + 2 = 4" + + In case the value you want to interpolate cannot be + converted to a string, because it doesn't have a human + textual representation, a protocol error will be raised. + + ## Escape characters + + Besides allowing double-quotes to be escaped with a backslash, + strings also support the following escape characters: + + * `\a` - Bell + * `\b` - Backspace + * `\t` - Horizontal tab + * `\n` - Line feed (New lines) + * `\v` - Vertical tab + * `\f` - Form feed + * `\r` - Carriage return + * `\e` - Command Escape + * `\#` - Returns the `#` character itself, skipping interpolation + * `\xNN` - A byte represented by the hexadecimal `NN` + * `\uNNNN` - A Unicode code point represented by `NNNN` + + Note it is generally not advised to use `\xNN` in Elixir + strings, as introducing an invalid byte sequence would + make the string invalid. If you have to introduce a + character by its hexadecimal representation, it is best + to work with Unicode code points, such as `\uNNNN`. In fact, + understanding Unicode code points can be essential when doing + low-level manipulations of string, so let's explore them in + detail next. + + ## Unicode and code points + + In order to facilitate meaningful communication between computers + across multiple languages, a standard is required so that the ones + and zeros on one machine mean the same thing when they are transmitted + to another. The Unicode Standard acts as an official registry of + virtually all the characters we know: this includes characters from + classical and historical texts, emoji, and formatting and control + characters as well. + + Unicode organizes all of the characters in its repertoire into code + charts, and each character is given a unique numerical index. This + numerical index is known as a Code Point. + + In Elixir you can use a `?` in front of a character literal to reveal + its code point: + + iex> ?a + 97 + iex> ?ł + 322 + + Note that most Unicode code charts will refer to a code point by its + hexadecimal (hex) representation, e.g. `97` translates to `0061` in hex, + and we can represent any Unicode character in an Elixir string by + using the `\u` escape character followed by its code point number: + + iex> "\u0061" === "a" + true + iex> 0x0061 = 97 = ?a + 97 - ## String and binary operations + The hex representation will also help you look up information about a + code point, e.g. [https://codepoints.net/U+0061](https://codepoints.net/U+0061) + has a data sheet all about the lower case `a`, a.k.a. code point 97. + Remember you can get the hex presentation of a number by calling + `Integer.to_string/2`: - The functions in this module act according to the - Unicode Standard, version 6.3.0. For example, - `capitalize/1`, `downcase/1`, `strip/1` are provided by this - module. + iex> Integer.to_string(?a, 16) + "61" - In addition to this module, Elixir provides more low-level - operations that work directly with binaries. Some - of those can be found in the `Kernel` module, as: + ## UTF-8 encoded and encodings - * `Kernel.binary_part/3` - retrieves part of the binary - * `Kernel.bit_size/1` and `Kernel.byte_size/1` - size related functions - * `Kernel.is_bitstring/1` and `Kernel.is_binary/1` - type checking function - * Plus a number of functions for working with binaries (bytes) - [in the `:binary` module](http://erlang.org/doc/man/binary.html) - - ## Codepoints and graphemes + Now that we understand what the Unicode standard is and what code points + are, we can finally talk about encodings. Whereas the code point is **what** + we store, an encoding deals with **how** we store it: encoding is an + implementation. In other words, we need a mechanism to convert the code + point numbers into bytes so they can be stored in memory, written to disk, and such. - As per the Unicode Standard, a codepoint is an Unicode - Character, which may be represented by one or more bytes. - For example, the character "é" is represented with two - bytes: + Elixir uses UTF-8 to encode its strings, which means that code points are + encoded as a series of 8-bit bytes. UTF-8 is a **variable width** character + encoding that uses one to four bytes to store each code point. It is capable + of encoding all valid Unicode code points. Let's see an example: - iex> byte_size("é") - 2 + iex> string = "héllo" + "héllo" + iex> String.length(string) + 5 + iex> byte_size(string) + 6 - However, this module returns the proper length: + Although the string above has 5 characters, it uses 6 bytes, as two bytes + are used to represent the character `é`. - iex> String.length("é") - 1 + ## Grapheme clusters - Furthermore, this module also presents the concept of - graphemes, which are multiple characters that may be - "perceived as a single character" by readers. For example, - the same "é" character written above could be represented - by the letter "e" followed by the accent ́: + This module also works with the concept of grapheme cluster + (from now on referenced as graphemes). Graphemes can consist + of multiple code points that may be perceived as a single character + by readers. For example, "é" can be represented either as a single + "e with acute" code point, as seen above in the string `"héllo"`, + or as the letter "e" followed by a "combining acute accent" + (two code points): - iex> string = "\x{0065}\x{0301}" + iex> string = "\u0065\u0301" + "é" iex> byte_size(string) 3 iex> String.length(string) 1 + iex> String.codepoints(string) + ["e", "́"] + iex> String.graphemes(string) + ["é"] - Although the example above is made of two characters, it is - perceived by users as one. + Although it looks visually the same as before, the example above + is made of two characters, it is perceived by users as one. - Graphemes can also be two characters that are interpreted - as one by some languages. For example, some languages may - consider "ch" as a grapheme. However, since this information - depends on the locale, it is not taken into account by this - module. + Graphemes can also be two characters that are interpreted as one + by some languages. For example, some languages may consider "ch" + as a single character. However, since this information depends on + the locale, it is not taken into account by this module. In general, the functions in this module rely on the Unicode - Standard, but does not contain any of the locale specific - behaviour. - + Standard, but do not contain any of the locale specific behaviour. More information about graphemes can be found in the [Unicode - Standard Annex #29](http://www.unicode.org/reports/tr29/). - This current Elixir version implements Extended Grapheme Cluster - algorithm. + Standard Annex #29](https://www.unicode.org/reports/tr29/). - ## Integer codepoints + For converting a binary to a different encoding and for Unicode + normalization mechanisms, see Erlang's `:unicode` module. - Although codepoints could be represented as integers, this - module represents all codepoints as strings. For example: + ## String and binary operations - iex> String.codepoints("josé") - ["j", "o", "s", "é"] + To act according to the Unicode Standard, many functions + in this module run in linear time, as they need to traverse + the whole string considering the proper Unicode code points. - There are a couple of ways to retrieve a character integer - codepoint. One may use the `?` special macro: + For example, `String.length/1` will take longer as + the input grows. On the other hand, `Kernel.byte_size/1` always runs + in constant time (i.e. regardless of the input size). - iex> ?j - 106 + This means often there are performance costs in using the + functions in this module, compared to the more low-level + operations that work directly with binaries: - iex> ?é - 233 + * `Kernel.binary_part/3` - retrieves part of the binary + * `Kernel.bit_size/1` and `Kernel.byte_size/1` - size related functions + * `Kernel.is_bitstring/1` and `Kernel.is_binary/1` - type-check function + * Plus a number of functions for working with binaries (bytes) + in the [`:binary` module](`:binary`) - Or also via pattern matching: + A `utf8` modifier is also available inside the binary syntax `<<>>`. + It can be used to match code points out of a binary/string: - iex> << eacute :: utf8 >> = "é" + iex> <> = "é" iex> eacute 233 - As we have seen above, codepoints can be inserted into - a string by their hexadecimal code: + You can also fully convert a string into a list of integer code points, + known as "charlists" in Elixir, by calling `String.to_charlist/1`: + + iex> String.to_charlist("héllo") + [104, 233, 108, 108, 111] + + If you would rather see the underlying bytes of a string, instead of + its codepoints, a common trick is to concatenate the null byte `<<0>>` + to it: + + iex> "héllo" <> <<0>> + <<104, 195, 169, 108, 108, 111, 0>> - "jos\x{0065}\x{0301}" #=> - "josé" + Alternatively, you can view a string's binary representation by + passing an option to `IO.inspect/2`: + + IO.inspect("héllo", binaries: :as_binaries) + #=> <<104, 195, 169, 108, 108, 111>> ## Self-synchronization The UTF-8 encoding is self-synchronizing. This means that if malformed data (i.e., data that is not possible according to the definition of the encoding) is encountered, only one - codepoint needs to be rejected. + code point needs to be rejected. This module relies on this behaviour to ignore such invalid - characters. For example, `length/1` is going to return - a correct result even if an invalid codepoint is fed into it. + characters. For example, `length/1` will return + a correct result even if an invalid code point is fed into it. In other words, this module expects invalid data to be detected - when retrieving data from the external source. For example, a - driver that reads strings from a database will be the one - responsible to check the validity of the encoding. + elsewhere, usually when retrieving data from the external source. + For example, a driver that reads strings from a database will be + responsible to check the validity of the encoding. `String.chunk/2` + can be used for breaking a string into valid and invalid parts. + + ## Compile binary patterns + + Many functions in this module work with patterns. For example, + `String.split/3` can split a string into multiple strings given + a pattern. This pattern can be a string, a list of strings or + a compiled pattern: + + iex> String.split("foo bar", " ") + ["foo", "bar"] + + iex> String.split("foo bar!", [" ", "!"]) + ["foo", "bar", ""] + + iex> pattern = :binary.compile_pattern([" ", "!"]) + iex> String.split("foo bar!", pattern) + ["foo", "bar", ""] + + The compiled pattern is useful when the same match will + be done over and over again. Note though that the compiled + pattern cannot be stored in a module attribute as the pattern + is generated at runtime and does not survive compile time. """ + @typedoc """ + A UTF-8 encoded binary. + + The types `String.t()` and `binary()` are equivalent to analysis tools. + Although, for those reading the documentation, `String.t()` implies + it is a UTF-8 encoded binary. + """ @type t :: binary + + @typedoc "A single Unicode code point encoded in UTF-8. It may be one or more bytes." @type codepoint :: t + + @typedoc "Multiple code points that may be perceived as a single character by readers" @type grapheme :: t + @typedoc """ + Pattern used in functions like `replace/4` and `split/3`. + + It must be one of: + + * a string + * an empty list + * a list containing non-empty strings + * a compiled search pattern created by `:binary.compile_pattern/1` + + """ + # TODO: Replace "nonempty_binary :: <<_::8, _::_*8>>" with "nonempty_binary()" + # when minimum requirement is >= OTP 24. + @type pattern :: + t() + | [nonempty_binary :: <<_::8, _::_*8>>] + | (compiled_search_pattern :: :binary.cp()) + + @conditional_mappings [:greek, :turkic] + @doc """ - Checks if a string is printable considering it is encoded - as UTF-8. Returns `true` if so, `false` otherwise. + Checks if a string contains only printable characters up to `character_limit`. + + Takes an optional `character_limit` as a second argument. If `character_limit` is `0`, this + function will return `true`. ## Examples iex> String.printable?("abc") true + iex> String.printable?("abc" <> <<0>>) + false + + iex> String.printable?("abc" <> <<0>>, 2) + true + + iex> String.printable?("abc" <> <<0>>, 0) + true + """ - @spec printable?(t) :: boolean + @spec printable?(t, 0) :: true + @spec printable?(t, pos_integer | :infinity) :: boolean + def printable?(string, character_limit \\ :infinity) + when is_binary(string) and + (character_limit == :infinity or + (is_integer(character_limit) and character_limit >= 0)) do + recur_printable?(string, character_limit) + end + + defp recur_printable?(_string, 0), do: true + defp recur_printable?(<<>>, _character_limit), do: true + + for char <- 0x20..0x7E do + defp recur_printable?(<>, character_limit) do + recur_printable?(rest, decrement(character_limit)) + end + end + + for char <- '\n\r\t\v\b\f\e\d\a' do + defp recur_printable?(<>, character_limit) do + recur_printable?(rest, decrement(character_limit)) + end + end - def printable?(<< h :: utf8, t :: binary >>) - when h in ?\040..?\176 - when h in 0xA0..0xD7FF - when h in 0xE000..0xFFFD - when h in 0x10000..0x10FFFF do - printable?(t) + defp recur_printable?(<>, character_limit) + when char in 0xA0..0xD7FF + when char in 0xE000..0xFFFD + when char in 0x10000..0x10FFFF do + recur_printable?(rest, decrement(character_limit)) end - def printable?(<>), do: printable?(t) - def printable?(<>), do: printable?(t) - def printable?(<>), do: printable?(t) - def printable?(<>), do: printable?(t) - def printable?(<>), do: printable?(t) - def printable?(<>), do: printable?(t) - def printable?(<>), do: printable?(t) - def printable?(<>), do: printable?(t) - def printable?(<>), do: printable?(t) + defp recur_printable?(_string, _character_limit) do + false + end - def printable?(<<>>), do: true - def printable?(b) when is_binary(b), do: false + defp decrement(:infinity), do: :infinity + defp decrement(character_limit), do: character_limit - 1 - @doc """ + @doc ~S""" Divides a string into substrings at each Unicode whitespace - occurrence with leading and trailing whitespace ignored. + occurrence with leading and trailing whitespace ignored. Groups + of whitespace are treated as a single occurrence. Divisions do + not occur on non-breaking whitespace. ## Examples @@ -161,25 +364,45 @@ defmodule String do iex> String.split("foo" <> <<194, 133>> <> "bar") ["foo", "bar"] - iex> String.split(" foo bar ") + iex> String.split(" foo bar ") ["foo", "bar"] + iex> String.split("no\u00a0break") + ["no\u00a0break"] + """ @spec split(t) :: [t] - defdelegate split(binary), to: String.Unicode + defdelegate split(binary), to: String.Break @doc ~S""" - Divides a string into substrings based on a pattern. + Divides a string into parts based on a pattern. + + Returns a list of these parts. - Returns a list of these substrings. The pattern can - be a string, a list of strings or a regular expression. + The `pattern` may be a string, a list of strings, a regular expression, or a + compiled pattern. The string is split into as many parts as possible by - default, but can be controlled via the `parts: num` option. - If you pass `parts: :infinity`, it will return all possible parts. + default, but can be controlled via the `:parts` option. Empty strings are only removed from the result if the - `trim` option is set to `true`. + `:trim` option is set to `true`. + + When the pattern used is a regular expression, the string is + split using `Regex.split/3`. + + ## Options + + * `:parts` (positive integer or `:infinity`) - the string + is split into at most as many parts as this option specifies. + If `:infinity`, the string will be split into all possible + parts. Defaults to `:infinity`. + + * `:trim` (boolean) - if `true`, empty strings are removed from + the resulting list. + + This function also accepts all options accepted by `Regex.split/3` + if `pattern` is a regular expression. ## Examples @@ -210,109 +433,367 @@ defmodule String do iex> String.split(" a b c ", ~r{\s}, trim: true) ["a", "b", "c"] - Splitting on empty patterns returns codepoints: + iex> String.split("abc", ~r{b}, include_captures: true) + ["a", "b", "c"] + + A compiled pattern: + + iex> pattern = :binary.compile_pattern([" ", ","]) + iex> String.split("1,2 3,4", pattern) + ["1", "2", "3", "4"] - iex> String.split("abc", ~r{}) - ["a", "b", "c", ""] + Splitting on empty string returns graphemes: iex> String.split("abc", "") - ["a", "b", "c", ""] + ["", "a", "b", "c", ""] iex> String.split("abc", "", trim: true) ["a", "b", "c"] - iex> String.split("abc", "", parts: 2) - ["a", "bc"] + iex> String.split("abc", "", parts: 1) + ["abc"] + + iex> String.split("abc", "", parts: 3) + ["", "a", "bc"] + + Be aware that this function can split within or across grapheme boundaries. + For example, take the grapheme "é" which is made of the characters + "e" and the acute accent. The following will split the string into two parts: + + iex> String.split(String.normalize("é", :nfd), "e") + ["", "́"] + + However, if "é" is represented by the single character "e with acute" + accent, then it will split the string into just one part: + + iex> String.split(String.normalize("é", :nfc), "e") + ["é"] """ - @spec split(t, t | [t] | Regex.t) :: [t] - @spec split(t, t | [t] | Regex.t, Keyword.t) :: [t] - def split(binary, pattern, options \\ []) + @spec split(t, pattern | Regex.t(), keyword) :: [t] + def split(string, pattern, options \\ []) - def split("", _pattern, _options), do: [""] + def split(string, %Regex{} = pattern, options) when is_binary(string) and is_list(options) do + Regex.split(pattern, string, options) + end - def split(binary, "", options), do: split(binary, ~r""u, options) + def split(string, "", options) when is_binary(string) and is_list(options) do + parts = Keyword.get(options, :parts, :infinity) + index = parts_to_index(parts) + trim = Keyword.get(options, :trim, false) - def split(binary, pattern, options) do - if Regex.regex?(pattern) do - Regex.split(pattern, binary, options) + if trim == false and index != 1 do + ["" | split_empty(string, trim, index - 1)] else - splits = - case Keyword.get(options, :parts, :infinity) do - num when is_number(num) and num > 0 -> - split_parts(binary, pattern, num - 1) - _ -> - :binary.split(binary, pattern, [:global]) - end + split_empty(string, trim, index) + end + end - if Keyword.get(options, :trim, false) do - for split <- splits, split != "", do: split - else - splits - end + def split(string, [], options) when is_binary(string) and is_list(options) do + if string == "" and Keyword.get(options, :trim, false) do + [] + else + [string] end end - defp split_parts("", _pattern, _num), do: [""] - defp split_parts(binary, pattern, num), do: split_parts(binary, pattern, num, []) - defp split_parts("", _pattern, _num, parts), do: Enum.reverse([""|parts]) - defp split_parts(binary, _pattern, 0, parts), do: Enum.reverse([binary|parts]) - defp split_parts(binary, pattern, num, parts) do - case :binary.split(binary, pattern) do - [head] -> Enum.reverse([head|parts]) - [head, rest] -> split_parts(rest, pattern, num - 1, [head|parts]) + def split(string, pattern, options) when is_binary(string) and is_list(options) do + parts = Keyword.get(options, :parts, :infinity) + trim = Keyword.get(options, :trim, false) + + case {parts, trim} do + {:infinity, false} -> + :binary.split(string, pattern, [:global]) + + {:infinity, true} -> + :binary.split(string, pattern, [:global, :trim_all]) + + {2, false} -> + :binary.split(string, pattern) + + _ -> + pattern = maybe_compile_pattern(pattern) + split_each(string, pattern, trim, parts_to_index(parts)) + end + end + + defp parts_to_index(:infinity), do: 0 + defp parts_to_index(n) when is_integer(n) and n > 0, do: n + + defp split_empty("", true, 1), do: [] + defp split_empty(string, _, 1), do: [string] + + defp split_empty(string, trim, count) do + case :unicode_util.gc(string) do + [gc] -> [grapheme_to_binary(gc) | split_empty(<<>>, trim, count - 1)] + [gc | rest] -> [grapheme_to_binary(gc) | split_empty(rest, trim, count - 1)] + [] -> split_empty("", trim, 1) + {:error, <>} -> [<> | split_empty(rest, trim, count - 1)] + end + end + + defp split_each("", _pattern, true, 1), do: [] + defp split_each(string, _pattern, _trim, 1) when is_binary(string), do: [string] + + defp split_each(string, pattern, trim, count) do + case do_splitter(string, pattern, trim) do + {h, t} -> [h | split_each(t, pattern, trim, count - 1)] + nil -> [] end end + @doc """ + Returns an enumerable that splits a string on demand. + + This is in contrast to `split/3` which splits the + entire string upfront. + + This function does not support regular expressions + by design. When using regular expressions, it is often + more efficient to have the regular expressions traverse + the string at once than in parts, like this function does. + + ## Options + + * :trim - when `true`, does not emit empty patterns + + ## Examples + + iex> String.splitter("1,2 3,4 5,6 7,8,...,99999", [" ", ","]) |> Enum.take(4) + ["1", "2", "3", "4"] + + iex> String.splitter("abcd", "") |> Enum.take(10) + ["", "a", "b", "c", "d", ""] + + iex> String.splitter("abcd", "", trim: true) |> Enum.take(10) + ["a", "b", "c", "d"] + + A compiled pattern can also be given: + + iex> pattern = :binary.compile_pattern([" ", ","]) + iex> String.splitter("1,2 3,4 5,6 7,8,...,99999", pattern) |> Enum.take(4) + ["1", "2", "3", "4"] + + """ + @spec splitter(t, pattern, keyword) :: Enumerable.t() + def splitter(string, pattern, options \\ []) + + def splitter(string, "", options) when is_binary(string) and is_list(options) do + if Keyword.get(options, :trim, false) do + Stream.unfold(string, &next_grapheme/1) + else + Stream.unfold(:match, &do_empty_splitter(&1, string)) + end + end + + def splitter(string, [], options) when is_binary(string) and is_list(options) do + if string == "" and Keyword.get(options, :trim, false) do + Stream.duplicate(string, 0) + else + Stream.duplicate(string, 1) + end + end + + def splitter(string, pattern, options) when is_binary(string) and is_list(options) do + pattern = maybe_compile_pattern(pattern) + trim = Keyword.get(options, :trim, false) + Stream.unfold(string, &do_splitter(&1, pattern, trim)) + end + + defp do_empty_splitter(:match, string), do: {"", string} + defp do_empty_splitter(:nomatch, _string), do: nil + defp do_empty_splitter("", _), do: {"", :nomatch} + defp do_empty_splitter(string, _), do: next_grapheme(string) + + defp do_splitter(:nomatch, _pattern, _), do: nil + defp do_splitter("", _pattern, false), do: {"", :nomatch} + defp do_splitter("", _pattern, true), do: nil + + defp do_splitter(bin, pattern, trim) do + case :binary.split(bin, pattern) do + ["", second] when trim -> do_splitter(second, pattern, trim) + [first, second] -> {first, second} + [first] -> {first, :nomatch} + end + end + + defp maybe_compile_pattern(pattern) when is_tuple(pattern), do: pattern + defp maybe_compile_pattern(pattern), do: :binary.compile_pattern(pattern) + @doc """ Splits a string into two at the specified offset. When the offset given is negative, location is counted from the end of the string. - The offset is capped to the length of the string. + The offset is capped to the length of the string. Returns a tuple with + two elements. - Returns a tuple with two elements. + Note: keep in mind this function splits on graphemes and for such it + has to linearly traverse the string. If you want to split a string or + a binary based on the number of bytes, use `Kernel.binary_part/3` + instead. ## Examples - iex> String.split_at "sweetelixir", 5 + iex> String.split_at("sweetelixir", 5) {"sweet", "elixir"} - iex> String.split_at "sweetelixir", -6 + iex> String.split_at("sweetelixir", -6) {"sweet", "elixir"} - iex> String.split_at "abc", 0 + iex> String.split_at("abc", 0) {"", "abc"} - iex> String.split_at "abc", 1000 + iex> String.split_at("abc", 1000) {"abc", ""} - iex> String.split_at "abc", -1000 + iex> String.split_at("abc", -1000) {"", "abc"} """ @spec split_at(t, integer) :: {t, t} - def split_at(string, offset) + def split_at(string, position) - def split_at(binary, index) when index == 0, do: - {"", binary} + def split_at(string, position) + when is_binary(string) and is_integer(position) and position >= 0 do + do_split_at(string, position) + end - def split_at(binary, index) when index > 0, do: - do_split_at(next_grapheme(binary), 0, index, "") + def split_at(string, position) + when is_binary(string) and is_integer(position) and position < 0 do + position = length(string) + position - def split_at(binary, index) when index < 0, do: - do_split_at(next_grapheme(binary), 0, max(0, byte_size(binary)+index), "") + case position >= 0 do + true -> do_split_at(string, position) + false -> {"", string} + end + end - defp do_split_at(nil, _, _, acc), do: - {acc, ""} + defp do_split_at(string, position) do + remaining = byte_size_remaining_at(string, position) + start = byte_size(string) - remaining + <> = string + {left, right} + end + + @doc ~S""" + Returns `true` if `string1` is canonically equivalent to `string2`. + + It performs Normalization Form Canonical Decomposition (NFD) on the + strings before comparing them. This function is equivalent to: - defp do_split_at({grapheme, rest}, current_pos, target_pos, acc) when current_pos < target_pos, do: - do_split_at(next_grapheme(rest), current_pos+1, target_pos, acc <> grapheme) + String.normalize(string1, :nfd) == String.normalize(string2, :nfd) - defp do_split_at({grapheme, rest}, pos, pos, acc), do: - {acc, grapheme <> rest} + If you plan to compare multiple strings, multiple times in a row, you + may normalize them upfront and compare them directly to avoid multiple + normalization passes. + + ## Examples + + iex> String.equivalent?("abc", "abc") + true + + iex> String.equivalent?("man\u0303ana", "mañana") + true + + iex> String.equivalent?("abc", "ABC") + false + + iex> String.equivalent?("nø", "nó") + false + + """ + @spec equivalent?(t, t) :: boolean + def equivalent?(string1, string2) when is_binary(string1) and is_binary(string2) do + normalize(string1, :nfd) == normalize(string2, :nfd) + end @doc """ - Convert all characters on the given string to uppercase. + Converts all characters in `string` to Unicode normalization + form identified by `form`. + + Invalid Unicode codepoints are skipped and the remaining of + the string is converted. If you want the algorithm to stop + and return on invalid codepoint, use `:unicode.characters_to_nfd_binary/1`, + `:unicode.characters_to_nfc_binary/1`, `:unicode.characters_to_nfkd_binary/1`, + and `:unicode.characters_to_nfkc_binary/1` instead. + + Normalization forms `:nfkc` and `:nfkd` should not be blindly applied + to arbitrary text. Because they erase many formatting distinctions, + they will prevent round-trip conversion to and from many legacy + character sets. + + ## Forms + + The supported forms are: + + * `:nfd` - Normalization Form Canonical Decomposition. + Characters are decomposed by canonical equivalence, and + multiple combining characters are arranged in a specific + order. + + * `:nfc` - Normalization Form Canonical Composition. + Characters are decomposed and then recomposed by canonical equivalence. + + * `:nfkd` - Normalization Form Compatibility Decomposition. + Characters are decomposed by compatibility equivalence, and + multiple combining characters are arranged in a specific + order. + + * `:nfkc` - Normalization Form Compatibility Composition. + Characters are decomposed and then recomposed by compatibility equivalence. + + ## Examples + + iex> String.normalize("yêṩ", :nfd) + "yêṩ" + + iex> String.normalize("leña", :nfc) + "leña" + + iex> String.normalize("fi", :nfkd) + "fi" + + iex> String.normalize("fi", :nfkc) + "fi" + + """ + def normalize(string, form) + + def normalize(string, :nfd) when is_binary(string) do + case :unicode.characters_to_nfd_binary(string) do + string when is_binary(string) -> string + {:error, good, <>} -> good <> <> <> normalize(rest, :nfd) + end + end + + def normalize(string, :nfc) when is_binary(string) do + case :unicode.characters_to_nfc_binary(string) do + string when is_binary(string) -> string + {:error, good, <>} -> good <> <> <> normalize(rest, :nfc) + end + end + + def normalize(string, :nfkd) when is_binary(string) do + case :unicode.characters_to_nfkd_binary(string) do + string when is_binary(string) -> string + {:error, good, <>} -> good <> <> <> normalize(rest, :nfkd) + end + end + + def normalize(string, :nfkc) when is_binary(string) do + case :unicode.characters_to_nfkc_binary(string) do + string when is_binary(string) -> string + {:error, good, <>} -> good <> <> <> normalize(rest, :nfkc) + end + end + + @doc """ + Converts all characters in the given string to uppercase according to `mode`. + + `mode` may be `:default`, `:ascii`, `:greek` or `:turkic`. The `:default` mode considers + all non-conditional transformations outlined in the Unicode standard. `:ascii` + uppercases only the letters a to z. `:greek` includes the context sensitive + mappings found in Greek. `:turkic` properly handles the letter i with the dotless variant. ## Examples @@ -322,15 +803,57 @@ defmodule String do iex> String.upcase("ab 123 xpto") "AB 123 XPTO" - iex> String.upcase("josé") - "JOSÉ" + iex> String.upcase("olá") + "OLÁ" + + The `:ascii` mode ignores Unicode characters and provides a more + performant implementation when you know the string contains only + ASCII characters: + + iex> String.upcase("olá", :ascii) + "OLá" + + And `:turkic` properly handles the letter i with the dotless variant: + + iex> String.upcase("ıi") + "II" + + iex> String.upcase("ıi", :turkic) + "Iİ" """ - @spec upcase(t) :: t - defdelegate upcase(binary), to: String.Unicode + @spec upcase(t, :default | :ascii | :greek | :turkic) :: t + def upcase(string, mode \\ :default) + + def upcase("", _mode) do + "" + end + + def upcase(string, :default) when is_binary(string) do + String.Unicode.upcase(string, [], :default) + end + + def upcase(string, :ascii) when is_binary(string) do + IO.iodata_to_binary(upcase_ascii(string)) + end + + def upcase(string, mode) when is_binary(string) and mode in @conditional_mappings do + String.Unicode.upcase(string, [], mode) + end + + defp upcase_ascii(<>) when char >= ?a and char <= ?z, + do: [char - 32 | upcase_ascii(rest)] + + defp upcase_ascii(<>), do: [char | upcase_ascii(rest)] + defp upcase_ascii(<<>>), do: [] @doc """ - Convert all characters on the given string to lowercase. + Converts all characters in the given string to lowercase according to `mode`. + + `mode` may be `:default`, `:ascii`, `:greek` or `:turkic`. The `:default` mode considers + all non-conditional transformations outlined in the Unicode standard. `:ascii` + lowercases only the letters A to Z. `:greek` includes the context sensitive + mappings found in Greek. `:turkic` properly handles the letter i with the dotless variant. ## Examples @@ -340,21 +863,66 @@ defmodule String do iex> String.downcase("AB 123 XPTO") "ab 123 xpto" - iex> String.downcase("JOSÉ") - "josé" + iex> String.downcase("OLÁ") + "olá" + + The `:ascii` mode ignores Unicode characters and provides a more + performant implementation when you know the string contains only + ASCII characters: + + iex> String.downcase("OLÁ", :ascii) + "olÁ" + + The `:greek` mode properly handles the context sensitive sigma in Greek: + + iex> String.downcase("ΣΣ") + "σσ" + + iex> String.downcase("ΣΣ", :greek) + "σς" + + And `:turkic` properly handles the letter i with the dotless variant: + + iex> String.downcase("Iİ") + "ii̇" + + iex> String.downcase("Iİ", :turkic) + "ıi" """ - @spec downcase(t) :: t - defdelegate downcase(binary), to: String.Unicode + @spec downcase(t, :default | :ascii | :greek | :turkic) :: t + def downcase(string, mode \\ :default) + + def downcase("", _mode) do + "" + end + + def downcase(string, :default) when is_binary(string) do + String.Unicode.downcase(string, [], :default) + end + + def downcase(string, :ascii) when is_binary(string) do + IO.iodata_to_binary(downcase_ascii(string)) + end + + def downcase(string, mode) when is_binary(string) and mode in @conditional_mappings do + String.Unicode.downcase(string, [], mode) + end + + defp downcase_ascii(<>) when char >= ?A and char <= ?Z, + do: [char + 32 | downcase_ascii(rest)] + + defp downcase_ascii(<>), do: [char | downcase_ascii(rest)] + defp downcase_ascii(<<>>), do: [] @doc """ Converts the first character in the given string to - uppercase and the remaining to lowercase. + uppercase and the remainder to lowercase according to `mode`. - This relies on the titlecase information provided - by the Unicode Standard. Note this function makes - no attempt to capitalize all words in the string - (usually known as titlecase). + `mode` may be `:default`, `:ascii`, `:greek` or `:turkic`. The `:default` mode considers + all non-conditional transformations outlined in the Unicode standard. `:ascii` + capitalizes only the letters A to Z. `:greek` includes the context sensitive + mappings found in Greek. `:turkic` properly handles the letter i with the dotless variant. ## Examples @@ -364,210 +932,530 @@ defmodule String do iex> String.capitalize("fin") "Fin" - iex> String.capitalize("josé") - "José" + iex> String.capitalize("olá") + "Olá" """ - @spec capitalize(t) :: t - def capitalize(string) when is_binary(string) do - {char, rest} = String.Unicode.titlecase_once(string) - char <> downcase(rest) + @spec capitalize(t, :default | :ascii | :greek | :turkic) :: t + def capitalize(string, mode \\ :default) + + def capitalize(<>, :ascii) do + char = if char >= ?a and char <= ?z, do: char - 32, else: char + <> <> downcase(rest, :ascii) + end + + def capitalize(string, mode) when is_binary(string) do + {char, rest} = String.Unicode.titlecase_once(string, mode) + char <> downcase(rest, mode) + end + + @doc false + @deprecated "Use String.trim_trailing/1 instead" + defdelegate rstrip(binary), to: String.Break, as: :trim_trailing + + @doc false + @deprecated "Use String.trim_trailing/2 with a binary as second argument instead" + def rstrip(string, char) when is_integer(char) do + replace_trailing(string, <>, "") end @doc """ - Returns a string where trailing Unicode whitespace - has been removed. + Replaces all leading occurrences of `match` by `replacement` of `match` in `string`. + + Returns the string untouched if there are no occurrences. + + If `match` is `""`, this function raises an `ArgumentError` exception: this + happens because this function replaces **all** the occurrences of `match` at + the beginning of `string`, and it's impossible to replace "multiple" + occurrences of `""`. ## Examples - iex> String.rstrip(" abc ") - " abc" + iex> String.replace_leading("hello world", "hello ", "") + "world" + iex> String.replace_leading("hello hello world", "hello ", "") + "world" + iex> String.replace_leading("hello world", "hello ", "ola ") + "ola world" + iex> String.replace_leading("hello hello world", "hello ", "ola ") + "ola ola world" + + This function can replace across grapheme boundaries. See `replace/3` + for more information and examples. """ - @spec rstrip(t) :: t - defdelegate rstrip(binary), to: String.Unicode + @spec replace_leading(t, t, t) :: t + def replace_leading(string, match, replacement) + when is_binary(string) and is_binary(match) and is_binary(replacement) do + if match == "" do + raise ArgumentError, "cannot use an empty string as the match to replace" + end + + prefix_size = byte_size(match) + suffix_size = byte_size(string) - prefix_size + replace_leading(string, match, replacement, prefix_size, suffix_size, 0) + end + + defp replace_leading(string, match, replacement, prefix_size, suffix_size, acc) + when suffix_size >= 0 do + case string do + <> when prefix == match -> + replace_leading( + suffix, + match, + replacement, + prefix_size, + suffix_size - prefix_size, + acc + 1 + ) + + _ -> + prepend_unless_empty(duplicate(replacement, acc), string) + end + end + + defp replace_leading(string, _match, replacement, _prefix_size, _suffix_size, acc) do + prepend_unless_empty(duplicate(replacement, acc), string) + end @doc """ - Returns a string where trailing `char` have been removed. + Replaces all trailing occurrences of `match` by `replacement` in `string`. + + Returns the string untouched if there are no occurrences. + + If `match` is `""`, this function raises an `ArgumentError` exception: this + happens because this function replaces **all** the occurrences of `match` at + the end of `string`, and it's impossible to replace "multiple" occurrences of + `""`. ## Examples - iex> String.rstrip(" abc _", ?_) - " abc " + iex> String.replace_trailing("hello world", " world", "") + "hello" + iex> String.replace_trailing("hello world world", " world", "") + "hello" + + iex> String.replace_trailing("hello world", " world", " mundo") + "hello mundo" + iex> String.replace_trailing("hello world world", " world", " mundo") + "hello mundo mundo" + This function can replace across grapheme boundaries. See `replace/3` + for more information and examples. """ - @spec rstrip(t, char) :: t + @spec replace_trailing(t, t, t) :: t + def replace_trailing(string, match, replacement) + when is_binary(string) and is_binary(match) and is_binary(replacement) do + if match == "" do + raise ArgumentError, "cannot use an empty string as the match to replace" + end - def rstrip("", _char), do: "" + suffix_size = byte_size(match) + prefix_size = byte_size(string) - suffix_size + replace_trailing(string, match, replacement, prefix_size, suffix_size, 0) + end + + defp replace_trailing(string, match, replacement, prefix_size, suffix_size, acc) + when prefix_size >= 0 do + case string do + <> when suffix == match -> + replace_trailing( + prefix, + match, + replacement, + prefix_size - suffix_size, + suffix_size, + acc + 1 + ) + + _ -> + append_unless_empty(string, duplicate(replacement, acc)) + end + end - # Do a quick check before we traverse the whole - # binary. :binary.last is a fast operation (it - # does not traverse the whole binary). - def rstrip(string, char) when char in 0..127 do - if :binary.last(string) == char do - do_rstrip(string, "", char) - else - string + defp replace_trailing(string, _match, replacement, _prefix_size, _suffix_size, acc) do + append_unless_empty(string, duplicate(replacement, acc)) + end + + @doc """ + Replaces prefix in `string` by `replacement` if it matches `match`. + + Returns the string untouched if there is no match. If `match` is an empty + string (`""`), `replacement` is just prepended to `string`. + + ## Examples + + iex> String.replace_prefix("world", "hello ", "") + "world" + iex> String.replace_prefix("hello world", "hello ", "") + "world" + iex> String.replace_prefix("hello hello world", "hello ", "") + "hello world" + + iex> String.replace_prefix("world", "hello ", "ola ") + "world" + iex> String.replace_prefix("hello world", "hello ", "ola ") + "ola world" + iex> String.replace_prefix("hello hello world", "hello ", "ola ") + "ola hello world" + + iex> String.replace_prefix("world", "", "hello ") + "hello world" + + This function can replace across grapheme boundaries. See `replace/3` + for more information and examples. + """ + @spec replace_prefix(t, t, t) :: t + def replace_prefix(string, match, replacement) + when is_binary(string) and is_binary(match) and is_binary(replacement) do + prefix_size = byte_size(match) + + case string do + <> when prefix == match -> + prepend_unless_empty(replacement, suffix) + + _ -> + string end end - def rstrip(string, char) when is_integer(char) do - do_rstrip(string, "", char) + @doc """ + Replaces suffix in `string` by `replacement` if it matches `match`. + + Returns the string untouched if there is no match. If `match` is an empty + string (`""`), `replacement` is just appended to `string`. + + ## Examples + + iex> String.replace_suffix("hello", " world", "") + "hello" + iex> String.replace_suffix("hello world", " world", "") + "hello" + iex> String.replace_suffix("hello world world", " world", "") + "hello world" + + iex> String.replace_suffix("hello", " world", " mundo") + "hello" + iex> String.replace_suffix("hello world", " world", " mundo") + "hello mundo" + iex> String.replace_suffix("hello world world", " world", " mundo") + "hello world mundo" + + iex> String.replace_suffix("hello", "", " world") + "hello world" + + This function can replace across grapheme boundaries. See `replace/3` + for more information and examples. + """ + @spec replace_suffix(t, t, t) :: t + def replace_suffix(string, match, replacement) + when is_binary(string) and is_binary(match) and is_binary(replacement) do + suffix_size = byte_size(match) + prefix_size = byte_size(string) - suffix_size + + case string do + <> when suffix == match -> + append_unless_empty(prefix, replacement) + + _ -> + string + end end - defp do_rstrip(<>, buffer, char) do - <>, char) :: binary>> + @compile {:inline, prepend_unless_empty: 2, append_unless_empty: 2} + + defp prepend_unless_empty("", suffix), do: suffix + defp prepend_unless_empty(prefix, suffix), do: prefix <> suffix + + defp append_unless_empty(prefix, ""), do: prefix + defp append_unless_empty(prefix, suffix), do: prefix <> suffix + + @doc false + @deprecated "Use String.trim_leading/1 instead" + defdelegate lstrip(binary), to: String.Break, as: :trim_leading + + @doc false + @deprecated "Use String.trim_leading/2 with a binary as second argument instead" + def lstrip(string, char) when is_integer(char) do + replace_leading(string, <>, "") end - defp do_rstrip(<>, buffer, another_char) do - <> + @doc false + @deprecated "Use String.trim/1 instead" + def strip(string) do + trim(string) end - defp do_rstrip(<<>>, _, _) do - <<>> + @doc false + @deprecated "Use String.trim/2 with a binary second argument instead" + def strip(string, char) do + trim(string, <>) end - @doc """ - Returns a string where leading Unicode whitespace - has been removed. + @doc ~S""" + Returns a string where all leading Unicode whitespaces + have been removed. ## Examples - iex> String.lstrip(" abc ") - "abc " + iex> String.trim_leading("\n abc ") + "abc " """ - defdelegate lstrip(binary), to: String.Unicode + @spec trim_leading(t) :: t + defdelegate trim_leading(string), to: String.Break @doc """ - Returns a string where leading `char` have been removed. + Returns a string where all leading `to_trim` characters have been removed. ## Examples - iex> String.lstrip("_ abc _", ?_) - " abc _" + iex> String.trim_leading("__ abc _", "_") + " abc _" + + iex> String.trim_leading("1 abc", "11") + "1 abc" """ + @spec trim_leading(t, t) :: t + def trim_leading(string, to_trim) + when is_binary(string) and is_binary(to_trim) do + replace_leading(string, to_trim, "") + end - @spec lstrip(t, char) :: t + @doc ~S""" + Returns a string where all trailing Unicode whitespaces + has been removed. - def lstrip(<>, char) when is_integer(char) do - <> - end + ## Examples - def lstrip(other, char) when is_integer(char) do - other - end + iex> String.trim_trailing(" abc\n ") + " abc" + + """ + @spec trim_trailing(t) :: t + defdelegate trim_trailing(string), to: String.Break @doc """ - Returns a string where leading/trailing Unicode whitespace - has been removed. + Returns a string where all trailing `to_trim` characters have been removed. ## Examples - iex> String.strip(" abc ") - "abc" + iex> String.trim_trailing("_ abc __", "_") + "_ abc " + + iex> String.trim_trailing("abc 1", "11") + "abc 1" """ - @spec strip(t) :: t + @spec trim_trailing(t, t) :: t + def trim_trailing(string, to_trim) + when is_binary(string) and is_binary(to_trim) do + replace_trailing(string, to_trim, "") + end - def strip(string) do - rstrip(lstrip(string)) + @doc ~S""" + Returns a string where all leading and trailing Unicode whitespaces + have been removed. + + ## Examples + + iex> String.trim("\n abc\n ") + "abc" + + """ + @spec trim(t) :: t + def trim(string) when is_binary(string) do + string + |> trim_leading() + |> trim_trailing() end @doc """ - Returns a string where leading/trailing `char` have been + Returns a string where all leading and trailing `to_trim` characters have been removed. ## Examples - iex> String.strip("a abc a", ?a) + iex> String.trim("a abc a", "a") " abc " """ - @spec strip(t, char) :: t - - def strip(string, char) do - rstrip(lstrip(string, char), char) + @spec trim(t, t) :: t + def trim(string, to_trim) when is_binary(string) and is_binary(to_trim) do + string + |> trim_leading(to_trim) + |> trim_trailing(to_trim) end @doc ~S""" - Returns a new string of length `len` with `subject` right justified and - padded with `padding`. If `padding` is not present, it defaults to - whitespace. When `len` is less than the length of `subject`, `subject` is - returned. + Returns a new string padded with a leading filler + which is made of elements from the `padding`. + + Passing a list of strings as `padding` will take one element of the list + for every missing entry. If the list is shorter than the number of inserts, + the filling will start again from the beginning of the list. + Passing a string `padding` is equivalent to passing the list of graphemes in it. + If no `padding` is given, it defaults to whitespace. + + When `count` is less than or equal to the length of `string`, + given `string` is returned. + + Raises `ArgumentError` if the given `padding` contains a non-string element. ## Examples - iex> String.rjust("abc", 5) + iex> String.pad_leading("abc", 5) " abc" - iex> String.rjust("abc", 5, ?-) - "--abc" + iex> String.pad_leading("abc", 4, "12") + "1abc" + + iex> String.pad_leading("abc", 6, "12") + "121abc" + + iex> String.pad_leading("abc", 5, ["1", "23"]) + "123abc" """ - @spec rjust(t, pos_integer) :: t - @spec rjust(t, pos_integer, char) :: t + @spec pad_leading(t, non_neg_integer, t | [t]) :: t + def pad_leading(string, count, padding \\ [" "]) - def rjust(subject, len) do - rjust(subject, len, ?\s) + def pad_leading(string, count, padding) when is_binary(padding) do + pad_leading(string, count, graphemes(padding)) end - def rjust(subject, len, padding) when is_integer(padding) do - do_justify(subject, len, padding, :right) + def pad_leading(string, count, [_ | _] = padding) + when is_binary(string) and is_integer(count) and count >= 0 do + pad(:leading, string, count, padding) end @doc ~S""" - Returns a new string of length `len` with `subject` left justified and padded - with `padding`. If `padding` is not present, it defaults to whitespace. When - `len` is less than the length of `subject`, `subject` is returned. + Returns a new string padded with a trailing filler + which is made of elements from the `padding`. + + Passing a list of strings as `padding` will take one element of the list + for every missing entry. If the list is shorter than the number of inserts, + the filling will start again from the beginning of the list. + Passing a string `padding` is equivalent to passing the list of graphemes in it. + If no `padding` is given, it defaults to whitespace. + + When `count` is less than or equal to the length of `string`, + given `string` is returned. + + Raises `ArgumentError` if the given `padding` contains a non-string element. ## Examples - iex> String.ljust("abc", 5) + iex> String.pad_trailing("abc", 5) "abc " - iex> String.ljust("abc", 5, ?-) - "abc--" + iex> String.pad_trailing("abc", 4, "12") + "abc1" + + iex> String.pad_trailing("abc", 6, "12") + "abc121" + + iex> String.pad_trailing("abc", 5, ["1", "23"]) + "abc123" """ - @spec ljust(t, pos_integer) :: t - @spec ljust(t, pos_integer, char) :: t + @spec pad_trailing(t, non_neg_integer, t | [t]) :: t + def pad_trailing(string, count, padding \\ [" "]) - def ljust(subject, len) do - ljust(subject, len, ?\s) + def pad_trailing(string, count, padding) when is_binary(padding) do + pad_trailing(string, count, graphemes(padding)) end - def ljust(subject, len, padding) when is_integer(padding) do - do_justify(subject, len, padding, :left) + def pad_trailing(string, count, [_ | _] = padding) + when is_binary(string) and is_integer(count) and count >= 0 do + pad(:trailing, string, count, padding) end - defp do_justify(subject, 0, _padding, _type) do - subject - end + defp pad(kind, string, count, padding) do + string_length = length(string) - defp do_justify(subject, len, padding, type) when is_integer(padding) do - subject_len = length(subject) + if string_length >= count do + string + else + filler = build_filler(count - string_length, padding, padding, 0, []) - cond do - subject_len >= len -> - subject - subject_len < len -> - fill = duplicate(<>, len - subject_len) - - case type do - :left -> subject <> fill - :right -> fill <> subject - end + case kind do + :leading -> [filler | string] + :trailing -> [string | filler] + end + |> IO.iodata_to_binary() end end + defp build_filler(0, _source, _padding, _size, filler), do: filler + + defp build_filler(count, source, [], size, filler) do + rem_filler = + rem(count, size) + |> build_filler(source, source, 0, []) + + filler = + filler + |> IO.iodata_to_binary() + |> duplicate(div(count, size) + 1) + + [filler | rem_filler] + end + + defp build_filler(count, source, [elem | rest], size, filler) + when is_binary(elem) do + build_filler(count - 1, source, rest, size + 1, [filler | elem]) + end + + defp build_filler(_count, _source, [elem | _rest], _size, _filler) do + raise ArgumentError, "expected a string padding element, got: #{inspect(elem)}" + end + + @doc false + @deprecated "Use String.pad_leading/2 instead" + def rjust(subject, length) do + rjust(subject, length, ?\s) + end + + @doc false + @deprecated "Use String.pad_leading/3 with a binary padding instead" + def rjust(subject, length, pad) when is_integer(pad) and is_integer(length) and length >= 0 do + pad(:leading, subject, length, [<>]) + end + + @doc false + @deprecated "Use String.pad_trailing/2 instead" + def ljust(subject, length) do + ljust(subject, length, ?\s) + end + + @doc false + @deprecated "Use String.pad_trailing/3 with a binary padding instead" + def ljust(subject, length, pad) when is_integer(pad) and is_integer(length) and length >= 0 do + pad(:trailing, subject, length, [<>]) + end + @doc ~S""" - Returns a new binary based on `subject` by replacing the parts - matching `pattern` by `replacement`. By default, it replaces - all entries, except if the `global` option is set to `false`. + Returns a new string created by replacing occurrences of `pattern` in + `subject` with `replacement`. + + The `subject` is always a string. - A `pattern` may be a string or a regex. + The `pattern` may be a string, a list of strings, a regular expression, or a + compiled pattern. + + The `replacement` may be a string or a function that receives the matched + pattern and must return the replacement as a string or iodata. + + By default it replaces all occurrences but this behaviour can be controlled + through the `:global` option; see the "Options" section below. + + ## Options + + * `:global` - (boolean) if `true`, all occurrences of `pattern` are replaced + with `replacement`, otherwise only the first occurrence is + replaced. Defaults to `true` ## Examples @@ -577,52 +1465,163 @@ defmodule String do iex> String.replace("a,b,c", ",", "-", global: false) "a-b,c" - The pattern can also be a regex. In those cases, one can give `\N` - in the `replacement` string to access a specific capture in the regex: + The pattern may also be a list of strings and the replacement may also + be a function that receives the matches: + + iex> String.replace("a,b,c", ["a", "c"], fn <> -> <> end) + "b,b,d" + + When the pattern is a regular expression, one can give `\N` or + `\g{N}` in the `replacement` string to access a specific capture in the + regular expression: - iex> String.replace("a,b,c", ~r/,(.)/, ",\\1\\1") + iex> String.replace("a,b,c", ~r/,(.)/, ",\\1\\g{1}") "a,bb,cc" - Notice we had to escape the escape character `\`. By giving `&`, - one can inject the whole matched pattern in the replacement string. + Note that we had to escape the backslash escape character (i.e., we used `\\N` + instead of just `\N` to escape the backslash; same thing for `\\g{N}`). By + giving `\0`, one can inject the whole match in the replacement string. - When strings are used as a pattern, a developer can also use the - replaced part inside the `replacement` via the `:insert_replaced` option: + A compiled pattern can also be given: - iex> String.replace("a,b,c", "b", "[]", insert_replaced: 1) - "a,[b],c" + iex> pattern = :binary.compile_pattern(",") + iex> String.replace("a,b,c", pattern, "[]") + "a[]b[]c" - iex> String.replace("a,b,c", ",", "[]", insert_replaced: 2) - "a[],b[],c" + When an empty string is provided as a `pattern`, the function will treat it as + an implicit empty string between each grapheme and the string will be + interspersed. If an empty string is provided as `replacement` the `subject` + will be returned: - iex> String.replace("a,b,c", ",", "[]", insert_replaced: [1, 1]) - "a[,,]b[,,]c" + iex> String.replace("ELIXIR", "", ".") + ".E.L.I.X.I.R." + + iex> String.replace("ELIXIR", "", "") + "ELIXIR" + + Be aware that this function can replace within or across grapheme boundaries. + For example, take the grapheme "é" which is made of the characters + "e" and the acute accent. The following will replace only the letter "e", + moving the accent to the letter "o": + + iex> String.replace(String.normalize("é", :nfd), "e", "o") + "ó" + + However, if "é" is represented by the single character "e with acute" + accent, then it won't be replaced at all: + + iex> String.replace(String.normalize("é", :nfc), "e", "o") + "é" """ - @spec replace(t, t, t) :: t - @spec replace(t, t, t, Keyword.t) :: t + @spec replace(t, pattern | Regex.t(), t | (t -> t | iodata), keyword) :: t + def replace(subject, pattern, replacement, options \\ []) + when is_binary(subject) and + (is_binary(replacement) or is_function(replacement, 1)) and + is_list(options) do + replace_guarded(subject, pattern, replacement, options) + end - def replace(subject, pattern, replacement, options \\ []) when is_binary(replacement) do - if Regex.regex?(pattern) do - Regex.replace(pattern, subject, replacement, global: options[:global]) + defp replace_guarded(subject, %{__struct__: Regex} = regex, replacement, options) do + Regex.replace(regex, subject, replacement, options) + end + + defp replace_guarded(subject, "", "", _) do + subject + end + + defp replace_guarded(subject, [], _, _) do + subject + end + + defp replace_guarded(subject, "", replacement_binary, options) + when is_binary(replacement_binary) do + if Keyword.get(options, :global, true) do + intersperse_bin(subject, replacement_binary, [replacement_binary]) else - opts = translate_replace_options(options) - :binary.replace(subject, pattern, replacement, opts) + replacement_binary <> subject end end - defp translate_replace_options(options) do - opts = if Keyword.get(options, :global) != false, do: [:global], else: [] + defp replace_guarded(subject, "", replacement_fun, options) do + if Keyword.get(options, :global, true) do + intersperse_fun(subject, replacement_fun, [replacement_fun.("")]) + else + IO.iodata_to_binary([replacement_fun.("") | subject]) + end + end + defp replace_guarded(subject, pattern, replacement, options) do if insert = Keyword.get(options, :insert_replaced) do - opts = [{:insert_replaced, insert}|opts] + IO.warn( + "String.replace/4 with :insert_replaced option is deprecated. " <> + "Please use :binary.replace/4 instead or pass an anonymous function as replacement" + ) + + binary_options = if Keyword.get(options, :global) != false, do: [:global], else: [] + :binary.replace(subject, pattern, replacement, [insert_replaced: insert] ++ binary_options) + else + matches = + if Keyword.get(options, :global, true) do + :binary.matches(subject, pattern) + else + case :binary.match(subject, pattern) do + :nomatch -> [] + match -> [match] + end + end + + IO.iodata_to_binary(do_replace(subject, matches, replacement, 0)) end + end - opts + defp intersperse_bin(subject, replacement, acc) do + case :unicode_util.gc(subject) do + [current | rest] -> + intersperse_bin(rest, replacement, [replacement, current | acc]) + + [] -> + reverse_characters_to_binary(acc) + + {:error, <>} -> + reverse_characters_to_binary(acc) <> + <> <> intersperse_bin(rest, replacement, [replacement]) + end end - @doc """ - Reverses the given string. Works on graphemes. + defp intersperse_fun(subject, replacement, acc) do + case :unicode_util.gc(subject) do + [current | rest] -> + intersperse_fun(rest, replacement, [replacement.(""), current | acc]) + + [] -> + reverse_characters_to_binary(acc) + + {:error, <>} -> + reverse_characters_to_binary(acc) <> + <> <> intersperse_fun(rest, replacement, [replacement.("")]) + end + end + + defp do_replace(subject, [], _, n) do + [binary_part(subject, n, byte_size(subject) - n)] + end + + defp do_replace(subject, [{start, length} | matches], replacement, n) do + prefix = binary_part(subject, n, start - n) + + middle = + if is_binary(replacement) do + replacement + else + replacement.(binary_part(subject, start, length)) + end + + [prefix, middle | do_replace(subject, matches, replacement, start + length)] + end + + @doc ~S""" + Reverses the graphemes in given string. ## Examples @@ -635,20 +1634,40 @@ defmodule String do iex> String.reverse("hello ∂og") "go∂ olleh" + Keep in mind reversing the same string twice does + not necessarily yield the original string: + + iex> "̀e" + "̀e" + iex> String.reverse("̀e") + "è" + iex> String.reverse(String.reverse("̀e")) + "è" + + In the first example the accent is before the vowel, so + it is considered two graphemes. However, when you reverse + it once, you have the vowel followed by the accent, which + becomes one grapheme. Reversing it again will keep it as + one single grapheme. """ @spec reverse(t) :: t - def reverse(string) do - do_reverse(next_grapheme(string), []) + def reverse(string) when is_binary(string) do + do_reverse(:unicode_util.gc(string), []) end - defp do_reverse({grapheme, rest}, acc) do - do_reverse(next_grapheme(rest), [grapheme|acc]) - end + defp do_reverse([grapheme | rest], acc), + do: do_reverse(:unicode_util.gc(rest), [grapheme | acc]) - defp do_reverse(nil, acc), do: IO.iodata_to_binary(acc) + defp do_reverse([], acc), + do: :unicode.characters_to_binary(acc) + + defp do_reverse({:error, <>}, acc), + do: :unicode.characters_to_binary(acc) <> <> <> do_reverse(:unicode_util.gc(rest), []) @doc """ - Returns a binary `subject` duplicated `n` times. + Returns a string `subject` repeated `n` times. + + Inlined by the compiler. ## Examples @@ -662,120 +1681,148 @@ defmodule String do "abcabc" """ - @spec duplicate(t, pos_integer) :: t - def duplicate(subject, n) when is_integer(n) and n >= 0 do + @compile {:inline, duplicate: 2} + @spec duplicate(t, non_neg_integer) :: t + def duplicate(subject, n) when is_binary(subject) and is_integer(n) and n >= 0 do :binary.copy(subject, n) end - @doc """ - Returns all codepoints in the string. + @doc ~S""" + Returns a list of code points encoded as strings. + + To retrieve code points in their natural integer + representation, see `to_charlist/1`. For details about + code points and graphemes, see the `String` module + documentation. ## Examples - iex> String.codepoints("josé") - ["j", "o", "s", "é"] + iex> String.codepoints("olá") + ["o", "l", "á"] iex> String.codepoints("оптими зации") - ["о","п","т","и","м","и"," ","з","а","ц","и","и"] + ["о", "п", "т", "и", "м", "и", " ", "з", "а", "ц", "и", "и"] iex> String.codepoints("ἅἪῼ") - ["ἅ","Ἢ","ῼ"] + ["ἅ", "Ἢ", "ῼ"] + + iex> String.codepoints("\u00e9") + ["é"] + + iex> String.codepoints("\u0065\u0301") + ["e", "́"] """ @spec codepoints(t) :: [codepoint] - defdelegate codepoints(string), to: String.Unicode - - @doc """ - Returns the next codepoint in a String. + def codepoints(string) when is_binary(string) do + do_codepoints(string) + end - The result is a tuple with the codepoint and the - remaining of the string or `nil` in case - the string reached its end. + defp do_codepoints(<>) do + [<> | do_codepoints(rest)] + end - As with other functions in the String module, this - function does not check for the validity of the codepoint. - That said, if an invalid codepoint is found, it will - be returned by this function. + defp do_codepoints(<>) do + [<> | do_codepoints(rest)] + end - ## Examples + defp do_codepoints(<<>>), do: [] - iex> String.next_codepoint("josé") - {"j", "osé"} + @doc ~S""" + Returns the next code point in a string. - """ - @compile {:inline, next_codepoint: 1} - @spec next_codepoint(t) :: {codepoint, t} | nil - defdelegate next_codepoint(string), to: String.Unicode + The result is a tuple with the code point and the + remainder of the string or `nil` in case + the string reached its end. - @doc ~S""" - Checks whether `str` contains only valid characters. + As with other functions in the `String` module, `next_codepoint/1` + works with binaries that are invalid UTF-8. If the string starts + with a sequence of bytes that is not valid in UTF-8 encoding, the + first element of the returned tuple is a binary with the first byte. ## Examples - iex> String.valid?("a") - true + iex> String.next_codepoint("olá") + {"o", "lá"} - iex> String.valid?("ø") - true - - iex> String.valid?(<<0xffff :: 16>>) - false + iex> invalid = "\x80\x80OK" # first two bytes are invalid in UTF-8 + iex> {_, rest} = String.next_codepoint(invalid) + {<<128>>, <<128, 79, 75>>} + iex> String.next_codepoint(rest) + {<<128>>, "OK"} - iex> String.valid?("asd" <> <<0xffff :: 16>>) - false + ## Comparison with binary pattern matching - """ - @spec valid?(t) :: boolean + Binary pattern matching provides a similar way to decompose + a string: - noncharacters = Enum.to_list(?\x{FDD0}..?\x{FDEF}) ++ - [ ?\x{0FFFE}, ?\x{0FFFF}, ?\x{1FFFE}, ?\x{1FFFF}, ?\x{2FFFE}, ?\x{2FFFF}, - ?\x{3FFFE}, ?\x{3FFFF}, ?\x{4FFFE}, ?\x{4FFFF}, ?\x{5FFFE}, ?\x{5FFFF}, - ?\x{6FFFE}, ?\x{6FFFF}, ?\x{7FFFE}, ?\x{7FFFF}, ?\x{8FFFE}, ?\x{8FFFF}, - ?\x{9FFFE}, ?\x{9FFFF}, ?\x{10FFFE}, ?\x{10FFFF} ] + iex> <> = "Elixir" + "Elixir" + iex> codepoint + 69 + iex> rest + "lixir" - for noncharacter <- noncharacters do - def valid?(<< unquote(noncharacter) :: utf8, _ :: binary >>), do: false - end + though not entirely equivalent because `codepoint` comes as + an integer, and the pattern won't match invalid UTF-8. - def valid?(<<_ :: utf8, t :: binary>>), do: valid?(t) - def valid?(<<>>), do: true - def valid?(_), do: false + Binary pattern matching, however, is simpler and more efficient, + so pick the option that better suits your use case. + """ + @spec next_codepoint(t) :: {codepoint, t} | nil + def next_codepoint(<>), do: {<>, rest} + def next_codepoint(<>), do: {<>, rest} + def next_codepoint(<<>>), do: nil @doc ~S""" - Checks whether `str` is a valid character. - - All characters are codepoints, but some codepoints - are not valid characters. They may be reserved, private, - or other. - - More info at: http://en.wikipedia.org/wiki/Mapping_of_Unicode_characters#Noncharacters + Checks whether `string` contains only valid characters. ## Examples - iex> String.valid_character?("a") + iex> String.valid?("a") + true + + iex> String.valid?("ø") true - iex> String.valid_character?("ø") + iex> String.valid?(<<0xFFFF::16>>) + false + + iex> String.valid?(<<0xEF, 0xB7, 0x90>>) true - iex> String.valid_character?("\x{ffff}") + iex> String.valid?("asd" <> <<0xFFFF::16>>) false """ - @spec valid_character?(t) :: boolean + @spec valid?(t) :: boolean + def valid?(string) + + def valid?(<>), do: valid_utf8?(string) + def valid?(_), do: false + + defp valid_utf8?(<<_::utf8, rest::bits>>), do: valid_utf8?(rest) + defp valid_utf8?(<<>>), do: true + defp valid_utf8?(_), do: false - def valid_character?(<<_ :: utf8>> = codepoint), do: valid?(codepoint) - def valid_character?(_), do: false + @doc false + @deprecated "Use String.valid?/1 instead" + def valid_character?(string) do + case string do + <<_::utf8>> -> valid?(string) + _ -> false + end + end @doc ~S""" Splits the string into chunks of characters that share a common trait. The trait can be one of two options: - * `:valid` – the string is split into chunks of valid and invalid character - sequences + * `:valid` - the string is split into chunks of valid and invalid + character sequences - * `:printable` – the string is split into chunks of printable and + * `:printable` - the string is split into chunks of printable and non-printable character sequences Returns a list of binaries each of which contains only one kind of @@ -786,13 +1833,13 @@ defmodule String do ## Examples iex> String.chunk(<>, :valid) - ["abc\000"] + ["abc\0"] - iex> String.chunk(<>, :valid) - ["abc\000", <<0x0ffff::utf8>>] + iex> String.chunk(<>, :valid) + ["abc\0", <<0xFFFF::utf16>>] - iex> String.chunk(<>, :printable) - ["abc", <<0, 0x0ffff::utf8>>] + iex> String.chunk(<>, :printable) + ["abc", <<0, 0x0FFFF::utf8>>] """ @spec chunk(t, :valid | :printable) :: [t] @@ -801,23 +1848,23 @@ defmodule String do def chunk("", _), do: [] - def chunk(str, trait) when trait in [:valid, :printable] do - {cp, _} = next_codepoint(str) + def chunk(string, trait) when is_binary(string) and trait in [:valid, :printable] do + {cp, _} = next_codepoint(string) pred_fn = make_chunk_pred(trait) - do_chunk(str, pred_fn.(cp), pred_fn) + do_chunk(string, pred_fn.(cp), pred_fn) end - - defp do_chunk(str, flag, pred_fn), do: do_chunk(str, [], <<>>, flag, pred_fn) + defp do_chunk(string, flag, pred_fn), do: do_chunk(string, [], <<>>, flag, pred_fn) defp do_chunk(<<>>, acc, <<>>, _, _), do: Enum.reverse(acc) defp do_chunk(<<>>, acc, chunk, _, _), do: Enum.reverse(acc, [chunk]) - defp do_chunk(str, acc, chunk, flag, pred_fn) do - {cp, rest} = next_codepoint(str) + defp do_chunk(string, acc, chunk, flag, pred_fn) do + {cp, rest} = next_codepoint(string) + if pred_fn.(cp) != flag do - do_chunk(rest, [chunk|acc], cp, not flag, pred_fn) + do_chunk(rest, [chunk | acc], cp, not flag, pred_fn) else do_chunk(rest, acc, chunk <> cp, flag, pred_fn) end @@ -826,40 +1873,81 @@ defmodule String do defp make_chunk_pred(:valid), do: &valid?/1 defp make_chunk_pred(:printable), do: &printable?/1 - @doc """ - Returns unicode graphemes in the string as per Extended Grapheme - Cluster algorithm outlined in the [Unicode Standard Annex #29, - Unicode Text Segmentation](http://www.unicode.org/reports/tr29/). + @doc ~S""" + Returns Unicode graphemes in the string as per Extended Grapheme + Cluster algorithm. + + The algorithm is outlined in the [Unicode Standard Annex #29, + Unicode Text Segmentation](https://www.unicode.org/reports/tr29/). + + For details about code points and graphemes, see the `String` module documentation. ## Examples - iex> String.graphemes("Ā̀stute") - ["Ā̀","s","t","u","t","e"] + iex> String.graphemes("Ńaïve") + ["Ń", "a", "ï", "v", "e"] + + iex> String.graphemes("\u00e9") + ["é"] + + iex> String.graphemes("\u0065\u0301") + ["é"] """ + @compile {:inline, graphemes: 1} @spec graphemes(t) :: [grapheme] - defdelegate graphemes(string), to: String.Graphemes + def graphemes(string) when is_binary(string), do: do_graphemes(string) + + defp do_graphemes(gcs) do + case :unicode_util.gc(gcs) do + [gc | rest] -> [grapheme_to_binary(gc) | do_graphemes(rest)] + [] -> [] + {:error, <>} -> [<> | do_graphemes(rest)] + end + end @doc """ - Returns the next grapheme in a String. + Returns the next grapheme in a string. The result is a tuple with the grapheme and the - remaining of the string or `nil` in case + remainder of the string or `nil` in case the String reached its end. ## Examples - iex> String.next_grapheme("josé") - {"j", "osé"} + iex> String.next_grapheme("olá") + {"o", "lá"} + + iex> String.next_grapheme("") + nil """ @compile {:inline, next_grapheme: 1} @spec next_grapheme(t) :: {grapheme, t} | nil - defdelegate next_grapheme(string), to: String.Graphemes + def next_grapheme(string) when is_binary(string) do + case :unicode_util.gc(string) do + [gc] -> {grapheme_to_binary(gc), <<>>} + [gc | rest] -> {grapheme_to_binary(gc), rest} + [] -> nil + {:error, <>} -> {<>, rest} + end + end + + @doc false + @deprecated "Use String.next_grapheme/1 instead" + @spec next_grapheme_size(t) :: {pos_integer, t} | nil + def next_grapheme_size(string) when is_binary(string) do + case :unicode_util.gc(string) do + [gc] -> {grapheme_byte_size(gc), <<>>} + [gc | rest] -> {grapheme_byte_size(gc), rest} + [] -> nil + {:error, <<_, rest::bits>>} -> {1, rest} + end + end @doc """ - Returns the first grapheme from an utf8 string, - nil if the string is empty. + Returns the first grapheme from a UTF-8 string, + `nil` if the string is empty. ## Examples @@ -869,21 +1957,30 @@ defmodule String do iex> String.first("եոգլի") "ե" + iex> String.first("") + nil + """ @spec first(t) :: grapheme | nil - def first(string) do - case next_grapheme(string) do - {char, _} -> char - nil -> nil + def first(string) when is_binary(string) do + case :unicode_util.gc(string) do + [gc | _] -> grapheme_to_binary(gc) + [] -> nil + {:error, <>} -> <> end end @doc """ - Returns the last grapheme from an utf8 string, + Returns the last grapheme from a UTF-8 string, `nil` if the string is empty. + It traverses the whole string to find its last grapheme. + ## Examples + iex> String.last("") + nil + iex> String.last("elixir") "r" @@ -892,18 +1989,16 @@ defmodule String do """ @spec last(t) :: grapheme | nil - def last(string) do - do_last(next_grapheme(string), nil) - end - - defp do_last({char, rest}, _) do - do_last(next_grapheme(rest), char) - end + def last(""), do: nil + def last(string) when is_binary(string), do: do_last(:unicode_util.gc(string), nil) - defp do_last(nil, last_char), do: last_char + defp do_last([gc | rest], _), do: do_last(:unicode_util.gc(rest), gc) + defp do_last([], acc) when is_binary(acc), do: acc + defp do_last([], acc), do: :unicode.characters_to_binary([acc]) + defp do_last({:error, <>}, _), do: do_last(:unicode_util.gc(rest), <>) @doc """ - Returns the number of unicode graphemes in an utf8 string. + Returns the number of Unicode graphemes in a UTF-8 string. ## Examples @@ -915,18 +2010,18 @@ defmodule String do """ @spec length(t) :: non_neg_integer - def length(string) do - do_length(next_grapheme(string)) - end + def length(string) when is_binary(string), do: length(string, 0) - defp do_length({_, rest}) do - 1 + do_length(next_grapheme(rest)) + defp length(gcs, acc) do + case :unicode_util.gc(gcs) do + [_ | rest] -> length(rest, acc + 1) + [] -> acc + {:error, <<_, rest::bits>>} -> length(rest, acc + 1) + end end - defp do_length(nil), do: 0 - @doc """ - Returns the grapheme in the `position` of the given utf8 `string`. + Returns the grapheme at the `position` of the given UTF-8 `string`. If `position` is greater than `string` length, then it returns `nil`. ## Examples @@ -949,33 +2044,37 @@ defmodule String do """ @spec at(t, integer) :: grapheme | nil - def at(string, position) when position >= 0 do - do_at(next_grapheme(string), position, 0) + def at(string, position) when is_binary(string) and is_integer(position) and position >= 0 do + do_at(string, position) end - def at(string, position) when position < 0 do - real_pos = length(string) - abs(position) - case real_pos >= 0 do - true -> do_at(next_grapheme(string), real_pos, 0) + def at(string, position) when is_binary(string) and is_integer(position) and position < 0 do + position = length(string) + position + + case position >= 0 do + true -> do_at(string, position) false -> nil end end - defp do_at({_ , rest}, desired_pos, current_pos) when desired_pos > current_pos do - do_at(next_grapheme(rest), desired_pos, current_pos + 1) - end + defp do_at(string, position) do + left = byte_size_remaining_at(string, position) - defp do_at({char, _}, desired_pos, current_pos) when desired_pos == current_pos do - char + string + |> binary_part(byte_size(string) - left, left) + |> first() end - defp do_at(nil, _, _), do: nil - @doc """ - Returns a substring starting at the offset given by the first, and - a length given by the second. + Returns a substring starting at the offset `start`, and of the given `length`. + If the offset is greater than string length, then it returns `""`. + Remember this function works with Unicode graphemes and considers + the slices to represent grapheme offsets. If you want to split + on raw bytes, check `Kernel.binary_part/3` or `Kernel.binary_slice/3` + instead. + ## Examples iex> String.slice("elixir", 1, 3) @@ -987,38 +2086,47 @@ defmodule String do iex> String.slice("elixir", 10, 3) "" + If the start position is negative, it is normalized + against the string length and clamped to 0: + iex> String.slice("elixir", -4, 4) "ixir" iex> String.slice("elixir", -10, 3) - "" - - iex> String.slice("a", 0, 1500) - "a" + "eli" - iex> String.slice("a", 1, 1500) - "" + If start is more than the string length, an empty + string is returned: - iex> String.slice("a", 2, 1500) + iex> String.slice("elixir", 10, 1500) "" """ - @spec slice(t, integer, integer) :: grapheme + @spec slice(t, integer, non_neg_integer) :: grapheme def slice(_, _, 0) do "" end - def slice(string, start, len) when start >= 0 and len >= 0 do - do_slice(next_grapheme(string), start, start + len - 1, 0, "") + def slice(string, start, length) + when is_binary(string) and is_integer(start) and is_integer(length) and start >= 0 and + length >= 0 do + do_slice(string, start, length) end - def slice(string, start, len) when start < 0 and len >= 0 do - real_start_pos = length(string) - abs(start) - case real_start_pos >= 0 do - true -> do_slice(next_grapheme(string), real_start_pos, real_start_pos + len - 1, 0, "") - false -> "" - end + def slice(string, start, length) + when is_binary(string) and is_integer(start) and is_integer(length) and start < 0 and + length >= 0 do + start = max(length(string) + start, 0) + do_slice(string, start, length) + end + + defp do_slice(string, start, length) do + from_start = byte_size_remaining_at(string, start) + rest = binary_part(string, byte_size(string) - from_start, from_start) + + from_end = byte_size_remaining_at(rest, length) + binary_part(rest, 0, from_start - from_end) end @doc """ @@ -1028,161 +2136,280 @@ defmodule String do If the start of the range is not a valid offset for the given string or if the range is in reverse order, returns `""`. + If the start or end of the range is negative, the whole string + is traversed first in order to convert the negative indices into + positive ones. + + Remember this function works with Unicode graphemes and considers + the slices to represent grapheme offsets. If you want to split + on raw bytes, check `Kernel.binary_part/3` or + `Kernel.binary_slice/2` instead + ## Examples iex> String.slice("elixir", 1..3) "lix" - iex> String.slice("elixir", 1..10) "lixir" - iex> String.slice("elixir", 10..3) - "" - iex> String.slice("elixir", -4..-1) "ixir" - - iex> String.slice("elixir", 2..-1) + iex> String.slice("elixir", -4..6) "ixir" + iex> String.slice("elixir", -100..100) + "elixir" - iex> String.slice("elixir", -4..6) + For ranges where `start > stop`, you need to explicitly + mark them as increasing: + + iex> String.slice("elixir", 2..-1//1) "ixir" + iex> String.slice("elixir", 1..-2//1) + "lixi" - iex> String.slice("elixir", -1..-4) - "" + You can use `../0` as a shortcut for `0..-1//1`, which returns + the whole string as is: - iex> String.slice("elixir", -10..-7) - "" + iex> String.slice("elixir", ..) + "elixir" - iex> String.slice("a", 0..1500) - "a" + The step can be any positive number. For example, to + get every 2 characters of the string: - iex> String.slice("a", 1..1500) - "" + iex> String.slice("elixir", 0..-1//2) + "eii" + + If the first position is after the string ends or after + the last position of the range, it returns an empty string: - iex> String.slice("a", 2..1500) + iex> String.slice("elixir", 10..3) + "" + iex> String.slice("a", 1..1500) "" """ - @spec slice(t, Range.t) :: t + @spec slice(t, Range.t()) :: t + def slice(string, first..last//step = range) when is_binary(string) do + # TODO: Deprecate negative steps on Elixir v1.16 + cond do + step > 0 -> + slice_range(string, first, last, step) - def slice(string, range) + step == -1 and first > last -> + slice_range(string, first, last, 1) - def slice(string, first..last) when first >= 0 and last >= 0 do - do_slice(next_grapheme(string), first, last, 0, "") + true -> + raise ArgumentError, + "String.slice/2 does not accept ranges with negative steps, got: #{inspect(range)}" + end + end + + # TODO: Remove me on v2.0 + def slice(string, %{__struct__: Range, first: first, last: last} = range) + when is_binary(string) do + step = if first <= last, do: 1, else: -1 + slice(string, Map.put(range, :step, step)) end - def slice(string, first..last) do - total = length(string) + defp slice_range("", _, _, _), do: "" - if first < 0 do - first = total + first - end + defp slice_range(_string, first, last, _step) when first >= 0 and last >= 0 and first > last do + "" + end + + defp slice_range(string, first, last, step) when first >= 0 do + from_start = byte_size_remaining_at(string, first) + rest = binary_part(string, byte_size(string) - from_start, from_start) + + cond do + last == -1 -> + slice_every(rest, byte_size(rest), step) + + last >= 0 and step == 1 -> + from_end = byte_size_remaining_at(rest, last - first + 1) + binary_part(rest, 0, from_start - from_end) - if last < 0 do - last = total + last + last >= 0 -> + slice_every(rest, last - first + 1, step) + + true -> + rest + |> slice_range_negative(0, last) + |> slice_every(byte_size(string), step) end + end - if first >= 0 do - do_slice(next_grapheme(string), first, last, 0, "") - else + defp slice_range(string, first, last, step) do + string + |> slice_range_negative(first, last) + |> slice_every(byte_size(string), step) + end + + defp slice_range_negative(string, first, last) do + {reversed_bytes, length} = acc_bytes(string, [], 0) + first = add_if_negative(first, length) |> max(0) + last = add_if_negative(last, length) + + if first > last or first > length do "" + else + last = min(last + 1, length) + reversed_bytes = Enum.drop(reversed_bytes, length - last) + {length_bytes, start_bytes} = split_bytes(reversed_bytes, 0, last - first) + binary_part(string, start_bytes, length_bytes) end end - defp do_slice(_, start_pos, last_pos, _, _) when start_pos > last_pos do - "" - end + defp slice_every(string, _count, 1), do: string + defp slice_every(string, count, step), do: slice_every(string, count, step, []) + + defp slice_every(string, count, to_drop, acc) when count > 0 do + case :unicode_util.gc(string) do + [current | rest] -> + rest + |> drop(to_drop) + |> slice_every(count - to_drop, to_drop, [current | acc]) - defp do_slice({_, rest}, start_pos, last_pos, current_pos, acc) when current_pos < start_pos do - do_slice(next_grapheme(rest), start_pos, last_pos, current_pos + 1, acc) + [] -> + reverse_characters_to_binary(acc) + + {:error, <>} -> + reverse_characters_to_binary(acc) <> + <> <> slice_every(drop(rest, to_drop), count - to_drop, to_drop, []) + end end - defp do_slice({char, rest}, start_pos, last_pos, current_pos, acc) when current_pos >= start_pos and current_pos < last_pos do - do_slice(next_grapheme(rest), start_pos, last_pos, current_pos + 1, acc <> char) + defp slice_every(_string, _count, _to_drop, acc) do + reverse_characters_to_binary(acc) end - defp do_slice({char, _}, start_pos, last_pos, current_pos, acc) when current_pos >= start_pos and current_pos == last_pos do - acc <> char + defp drop(string, 1), do: string + + defp drop(string, count) do + case :unicode_util.gc(string) do + [_ | rest] -> drop(rest, count - 1) + [] -> "" + {:error, <<_, rest::bits>>} -> drop(rest, count - 1) + end end - - defp do_slice(nil, _, _, _, acc) do - acc + + defp acc_bytes(string, bytes, length) do + case :unicode_util.gc(string) do + [gc | rest] -> acc_bytes(rest, [grapheme_byte_size(gc) | bytes], length + 1) + [] -> {bytes, length} + {:error, <<_, rest::bits>>} -> acc_bytes(rest, [1 | bytes], length + 1) + end end + defp add_if_negative(value, to_add) when value < 0, do: value + to_add + defp add_if_negative(value, _to_add), do: value + + defp split_bytes(rest, acc, 0), do: {acc, Enum.sum(rest)} + defp split_bytes([], acc, _), do: {acc, 0} + defp split_bytes([head | tail], acc, count), do: split_bytes(tail, head + acc, count - 1) + @doc """ - Returns `true` if `string` starts with any of the prefixes given, otherwise - `false`. `prefixes` can be either a single prefix or a list of prefixes. + Returns `true` if `string` starts with any of the prefixes given. + + `prefix` can be either a string, a list of strings, or a compiled + pattern. ## Examples - iex> String.starts_with? "elixir", "eli" + iex> String.starts_with?("elixir", "eli") + true + iex> String.starts_with?("elixir", ["erlang", "elixir"]) true + iex> String.starts_with?("elixir", ["erlang", "ruby"]) + false + + An empty string will always match: - iex> String.starts_with? "elixir", ["erlang", "elixir"] + iex> String.starts_with?("elixir", "") true + iex> String.starts_with?("elixir", ["", "other"]) + true + + An empty list will never match: + + iex> String.starts_with?("elixir", []) + false - iex> String.starts_with? "elixir", ["erlang", "ruby"] + iex> String.starts_with?("", []) false """ @spec starts_with?(t, t | [t]) :: boolean - - def starts_with?(string, prefixes) when is_list(prefixes) do - Enum.any?(prefixes, &do_starts_with(string, &1)) + def starts_with?(string, prefix) when is_binary(string) and is_binary(prefix) do + starts_with_string?(string, byte_size(string), prefix) end - def starts_with?(string, prefix) do - do_starts_with(string, prefix) + def starts_with?(string, prefix) when is_binary(string) and is_list(prefix) do + string_size = byte_size(string) + Enum.any?(prefix, &starts_with_string?(string, string_size, &1)) end - defp do_starts_with(string, "") when is_binary(string) do - true + def starts_with?(string, prefix) when is_binary(string) do + IO.warn("compiled patterns are deprecated in starts_with?") + Kernel.match?({0, _}, :binary.match(string, prefix)) end - defp do_starts_with(string, prefix) when is_binary(prefix) do - Kernel.match?({0, _}, :binary.match(string, prefix)) + @compile {:inline, starts_with_string?: 3} + defp starts_with_string?(string, string_size, prefix) when is_binary(prefix) do + prefix_size = byte_size(prefix) + + if prefix_size <= string_size do + prefix == binary_part(string, 0, prefix_size) + else + false + end end @doc """ - Returns `true` if `string` ends with any of the suffixes given, otherwise - `false`. `suffixes` can be either a single suffix or a list of suffixes. + Returns `true` if `string` ends with any of the suffixes given. + + `suffixes` can be either a single suffix or a list of suffixes. ## Examples - iex> String.ends_with? "language", "age" + iex> String.ends_with?("language", "age") true - - iex> String.ends_with? "language", ["youth", "age"] + iex> String.ends_with?("language", ["youth", "age"]) true - - iex> String.ends_with? "language", ["youth", "elixir"] + iex> String.ends_with?("language", ["youth", "elixir"]) false - """ - @spec ends_with?(t, t | [t]) :: boolean + An empty suffix will always match: - def ends_with?(string, suffixes) when is_list(suffixes) do - Enum.any?(suffixes, &do_ends_with(string, &1)) - end + iex> String.ends_with?("language", "") + true + iex> String.ends_with?("language", ["", "other"]) + true - def ends_with?(string, suffix) do - do_ends_with(string, suffix) + """ + @spec ends_with?(t, t | [t]) :: boolean + def ends_with?(string, suffix) when is_binary(string) and is_binary(suffix) do + ends_with_string?(string, byte_size(string), suffix) end - defp do_ends_with(string, "") when is_binary(string) do - true + def ends_with?(string, suffix) when is_binary(string) and is_list(suffix) do + string_size = byte_size(string) + Enum.any?(suffix, &ends_with_string?(string, string_size, &1)) end - defp do_ends_with(string, suffix) when is_binary(suffix) do - string_size = byte_size(string) + @compile {:inline, ends_with_string?: 3} + defp ends_with_string?(string, string_size, suffix) when is_binary(suffix) do suffix_size = byte_size(suffix) - scope = {string_size - suffix_size, suffix_size} - (suffix_size <= string_size) and (:nomatch != :binary.match(string, suffix, [scope: scope])) + + if suffix_size <= string_size do + suffix == binary_part(string, string_size - suffix_size, suffix_size) + else + false + end end @doc """ - Check if `string` matches the given regular expression. + Checks if `string` matches the given regular expression. ## Examples @@ -1192,61 +2419,111 @@ defmodule String do iex> String.match?("bar", ~r/foo/) false + Elixir also provides text-based match operator `=~/2` and function `Regex.match?/2` as + alternatives to test strings against regular expressions. """ - @spec match?(t, Regex.t) :: boolean - def match?(string, regex) do + @spec match?(t, Regex.t()) :: boolean + def match?(string, regex) when is_binary(string) do Regex.match?(regex, string) end @doc """ - Check if `string` contains any of the given `contents`. + Searches if `string` contains any of the given `contents`. - `matches` can be either a single string or a list of strings. + `contents` can be either a string, a list of strings, + or a compiled pattern. If `contents` is a list, this + function will search if any of the strings in `contents` + are part of `string`. + + > Note: if you want to check if `string` is listed in `contents`, + > where `contents` is a list, use `Enum.member?(contents, string)` + > instead. ## Examples - iex> String.contains? "elixir of life", "of" + iex> String.contains?("elixir of life", "of") + true + iex> String.contains?("elixir of life", ["life", "death"]) + true + iex> String.contains?("elixir of life", ["death", "mercury"]) + false + + The argument can also be a compiled pattern: + + iex> pattern = :binary.compile_pattern(["life", "death"]) + iex> String.contains?("elixir of life", pattern) + true + + An empty string will always match: + + iex> String.contains?("elixir of life", "") + true + iex> String.contains?("elixir of life", ["", "other"]) true - iex> String.contains? "elixir of life", ["life", "death"] + An empty list will never match: + + iex> String.contains?("elixir of life", []) + false + + iex> String.contains?("", []) + false + + Be aware that this function can match within or across grapheme boundaries. + For example, take the grapheme "é" which is made of the characters + "e" and the acute accent. The following returns `true`: + + iex> String.contains?(String.normalize("é", :nfd), "e") true - iex> String.contains? "elixir of life", ["death", "mercury"] + However, if "é" is represented by the single character "e with acute" + accent, then it will return `false`: + + iex> String.contains?(String.normalize("é", :nfc), "e") false """ - @spec contains?(t, t | [t]) :: boolean - - def contains?(string, contents) when is_list(contents) do - Enum.any?(contents, &do_contains(string, &1)) + @spec contains?(t, [t] | pattern) :: boolean + def contains?(string, contents) when is_binary(string) and is_list(contents) do + list_contains?(string, byte_size(string), contents, []) end - def contains?(string, content) do - do_contains(string, content) + def contains?(string, contents) when is_binary(string) do + "" == contents or :binary.match(string, contents) != :nomatch end - defp do_contains(string, "") when is_binary(string) do - true + defp list_contains?(string, size, [head | tail], acc) do + case byte_size(head) do + 0 -> true + head_size when head_size > size -> list_contains?(string, size, tail, acc) + _ -> list_contains?(string, size, tail, [head | acc]) + end end - defp do_contains(string, match) when is_binary(match) do - :nomatch != :binary.match(string, match) - end + defp list_contains?(_string, _size, [], []), + do: false + + defp list_contains?(string, _size, [], contents), + do: :binary.match(string, contents) != :nomatch @doc """ - Converts a string into a char list. + Converts a string into a charlist. + + Specifically, this function takes a UTF-8 encoded binary and returns a list of its integer + code points. It is similar to `codepoints/1` except that the latter returns a list of code points as + strings. + + In case you need to work with bytes, take a look at the + [`:binary` module](`:binary`). ## Examples - iex> String.to_char_list("æß") + iex> String.to_charlist("æß") 'æß' - Notice that this function expect a list of integer representing - UTF-8 codepoints. If you have a raw binary, you must instead use - [the `:binary` module](http://erlang.org/doc/man/binary.html). """ - @spec to_char_list(t) :: char_list - def to_char_list(string) when is_binary(string) do + @spec to_charlist(t) :: charlist + def to_charlist(string) when is_binary(string) do case :unicode.characters_to_list(string) do result when is_list(result) -> result @@ -1262,8 +2539,15 @@ defmodule String do @doc """ Converts a string to an atom. - Currently Elixir does not support conversions from strings - which contains Unicode codepoints greater than 0xFF. + Warning: this function creates atoms dynamically and atoms are + not garbage-collected. Therefore, `string` should not be an + untrusted value, such as input received from a socket or during + a web request. Consider using `to_existing_atom/1` instead. + + By default, the maximum number of atoms is `1_048_576`. This limit + can be raised or lowered using the VM option `+t`. + + The maximum atom size is of 255 Unicode code points. Inlined by the compiler. @@ -1273,36 +2557,47 @@ defmodule String do :my_atom """ - @spec to_atom(String.t) :: atom - def to_atom(string) do + @spec to_atom(String.t()) :: atom + def to_atom(string) when is_binary(string) do :erlang.binary_to_atom(string, :utf8) end @doc """ Converts a string to an existing atom. - Currently Elixir does not support conversions from strings - which contains Unicode codepoints greater than 0xFF. + The maximum atom size is of 255 Unicode code points. + Raises an `ArgumentError` if the atom does not exist. Inlined by the compiler. + > #### Atoms and modules {: .info} + > + > Since Elixir is a compiled language, the atoms defined in a module + > will only exist after said module is loaded, which typically happens + > whenever a function in the module is executed. Therefore, it is + > generally recommended to call `String.to_existing_atom/1` only to + > convert atoms defined within the module making the function call + > to `to_existing_atom/1`. + ## Examples - iex> :my_atom + iex> _ = :my_atom iex> String.to_existing_atom("my_atom") :my_atom - iex> String.to_existing_atom("this_atom_will_never_exist") - ** (ArgumentError) argument error - """ - @spec to_existing_atom(String.t) :: atom - def to_existing_atom(string) do + @spec to_existing_atom(String.t()) :: atom + def to_existing_atom(string) when is_binary(string) do :erlang.binary_to_existing_atom(string, :utf8) end @doc """ - Returns a integer whose text representation is `string`. + Returns an integer whose text representation is `string`. + + `string` must be the string representation of an integer. + Otherwise, an `ArgumentError` will be raised. If you want + to parse a string that may contain an ill-formatted integer, + use `Integer.parse/1`. Inlined by the compiler. @@ -1311,9 +2606,14 @@ defmodule String do iex> String.to_integer("123") 123 + Passing a string that does not represent an integer leads to an error: + + String.to_integer("invalid data") + ** (ArgumentError) argument error + """ - @spec to_integer(String.t) :: integer - def to_integer(string) do + @spec to_integer(String.t()) :: integer + def to_integer(string) when is_binary(string) do :erlang.binary_to_integer(string) end @@ -1328,14 +2628,18 @@ defmodule String do 1023 """ - @spec to_integer(String.t, pos_integer) :: integer - def to_integer(string, base) do + @spec to_integer(String.t(), 2..36) :: integer + def to_integer(string, base) when is_binary(string) and is_integer(base) do :erlang.binary_to_integer(string, base) end @doc """ Returns a float whose text representation is `string`. + `string` must be the string representation of a float including a decimal point. + In order to parse a string without decimal point as a float then `Float.parse/1` + should be used. Otherwise, an `ArgumentError` will be raised. + Inlined by the compiler. ## Examples @@ -1343,9 +2647,267 @@ defmodule String do iex> String.to_float("2.2017764e+0") 2.2017764 + iex> String.to_float("3.0") + 3.0 + + String.to_float("3") + ** (ArgumentError) argument error + """ - @spec to_float(String.t) :: float - def to_float(string) do + @spec to_float(String.t()) :: float + def to_float(string) when is_binary(string) do :erlang.binary_to_float(string) end + + @doc """ + Computes the bag distance between two strings. + + Returns a float value between 0 and 1 representing the bag + distance between `string1` and `string2`. + + The bag distance is meant to be an efficient approximation + of the distance between two strings to quickly rule out strings + that are largely different. + + The algorithm is outlined in the "String Matching with Metric + Trees Using an Approximate Distance" paper by Ilaria Bartolini, + Paolo Ciaccia, and Marco Patella. + + ## Examples + + iex> String.bag_distance("abc", "") + 0.0 + iex> String.bag_distance("abcd", "a") + 0.25 + iex> String.bag_distance("abcd", "ab") + 0.5 + iex> String.bag_distance("abcd", "abc") + 0.75 + iex> String.bag_distance("abcd", "abcd") + 1.0 + + """ + @spec bag_distance(t, t) :: float + @doc since: "1.8.0" + def bag_distance(_string, ""), do: 0.0 + def bag_distance("", _string), do: 0.0 + + def bag_distance(string1, string2) when is_binary(string1) and is_binary(string2) do + {bag1, length1} = string_to_bag(string1, %{}, 0) + {bag2, length2} = string_to_bag(string2, %{}, 0) + + diff1 = bag_difference(bag1, bag2) + diff2 = bag_difference(bag2, bag1) + + 1 - max(diff1, diff2) / max(length1, length2) + end + + defp string_to_bag(string, bag, length) do + case :unicode_util.gc(string) do + [gc | rest] -> string_to_bag(rest, bag_store(bag, gc), length + 1) + [] -> {bag, length} + {:error, <>} -> string_to_bag(rest, bag_store(bag, <>), length + 1) + end + end + + defp bag_store(bag, gc) do + case bag do + %{^gc => current} -> %{bag | gc => current + 1} + %{} -> Map.put(bag, gc, 1) + end + end + + defp bag_difference(bag1, bag2) do + Enum.reduce(bag1, 0, fn {char, count1}, sum -> + case bag2 do + %{^char => count2} -> sum + max(count1 - count2, 0) + %{} -> sum + count1 + end + end) + end + + @doc """ + Computes the Jaro distance (similarity) between two strings. + + Returns a float value between `0.0` (equates to no similarity) and `1.0` + (is an exact match) representing [Jaro](https://en.wikipedia.org/wiki/Jaro-Winkler_distance) + distance between `string1` and `string2`. + + The Jaro distance metric is designed and best suited for short + strings such as person names. Elixir itself uses this function + to provide the "did you mean?" functionality. For instance, when you + are calling a function in a module and you have a typo in the + function name, we attempt to suggest the most similar function + name available, if any, based on the `jaro_distance/2` score. + + ## Examples + + iex> String.jaro_distance("Dwayne", "Duane") + 0.8222222222222223 + iex> String.jaro_distance("even", "odd") + 0.0 + iex> String.jaro_distance("same", "same") + 1.0 + + """ + @spec jaro_distance(t, t) :: float + def jaro_distance(string1, string2) + + def jaro_distance(string, string), do: 1.0 + def jaro_distance(_string, ""), do: 0.0 + def jaro_distance("", _string), do: 0.0 + + def jaro_distance(string1, string2) when is_binary(string1) and is_binary(string2) do + {chars1, len1} = graphemes_and_length(string1) + {chars2, len2} = graphemes_and_length(string2) + + case match(chars1, len1, chars2, len2) do + {0, _trans} -> + 0.0 + + {comm, trans} -> + (comm / len1 + comm / len2 + (comm - trans) / comm) / 3 + end + end + + defp match(chars1, len1, chars2, len2) do + if len1 < len2 do + match(chars1, chars2, div(len2, 2) - 1) + else + match(chars2, chars1, div(len1, 2) - 1) + end + end + + defp match(chars1, chars2, lim) do + match(chars1, chars2, {0, lim}, {0, 0, -1}, 0) + end + + defp match([char | rest], chars, range, state, idx) do + {chars, state} = submatch(char, chars, range, state, idx) + + case range do + {lim, lim} -> match(rest, tl(chars), range, state, idx + 1) + {pre, lim} -> match(rest, chars, {pre + 1, lim}, state, idx + 1) + end + end + + defp match([], _, _, {comm, trans, _}, _), do: {comm, trans} + + defp submatch(char, chars, {pre, _} = range, state, idx) do + case detect(char, chars, range) do + nil -> + {chars, state} + + {subidx, chars} -> + {chars, proceed(state, idx - pre + subidx)} + end + end + + defp detect(char, chars, {pre, lim}) do + detect(char, chars, pre + 1 + lim, 0, []) + end + + defp detect(_char, _chars, 0, _idx, _acc), do: nil + defp detect(_char, [], _lim, _idx, _acc), do: nil + + defp detect(char, [char | rest], _lim, idx, acc), do: {idx, Enum.reverse(acc, [nil | rest])} + + defp detect(char, [other | rest], lim, idx, acc), + do: detect(char, rest, lim - 1, idx + 1, [other | acc]) + + defp proceed({comm, trans, former}, current) do + if current < former do + {comm + 1, trans + 1, current} + else + {comm + 1, trans, current} + end + end + + @doc """ + Returns a keyword list that represents an edit script. + + Check `List.myers_difference/2` for more information. + + ## Examples + + iex> string1 = "fox hops over the dog" + iex> string2 = "fox jumps over the lazy cat" + iex> String.myers_difference(string1, string2) + [eq: "fox ", del: "ho", ins: "jum", eq: "ps over the ", del: "dog", ins: "lazy cat"] + + """ + @doc since: "1.3.0" + @spec myers_difference(t, t) :: [{:eq | :ins | :del, t}] + def myers_difference(string1, string2) when is_binary(string1) and is_binary(string2) do + graphemes(string1) + |> List.myers_difference(graphemes(string2)) + |> Enum.map(fn {kind, chars} -> {kind, IO.iodata_to_binary(chars)} end) + end + + @doc false + @deprecated "Use String.to_charlist/1 instead" + @spec to_char_list(t) :: charlist + def to_char_list(string), do: String.to_charlist(string) + + ## Helpers + + @compile {:inline, + codepoint_byte_size: 1, + grapheme_byte_size: 1, + grapheme_to_binary: 1, + graphemes_and_length: 1, + reverse_characters_to_binary: 1} + + defp byte_size_unicode(binary) when is_binary(binary), do: byte_size(binary) + defp byte_size_unicode([head]), do: byte_size_unicode(head) + defp byte_size_unicode([head | tail]), do: byte_size_unicode(head) + byte_size_unicode(tail) + + defp byte_size_remaining_at(unicode, 0) do + byte_size_unicode(unicode) + end + + defp byte_size_remaining_at(unicode, n) do + case :unicode_util.gc(unicode) do + [_] -> 0 + [_ | rest] -> byte_size_remaining_at(rest, n - 1) + [] -> 0 + {:error, <<_, bin::bits>>} -> byte_size_remaining_at(bin, n - 1) + end + end + + defp codepoint_byte_size(cp) when cp <= 0x007F, do: 1 + defp codepoint_byte_size(cp) when cp <= 0x07FF, do: 2 + defp codepoint_byte_size(cp) when cp <= 0xFFFF, do: 3 + defp codepoint_byte_size(_), do: 4 + + defp grapheme_to_binary(cp) when is_integer(cp), do: <> + defp grapheme_to_binary(gc), do: :unicode.characters_to_binary(gc) + + defp grapheme_byte_size(cp) when is_integer(cp), do: codepoint_byte_size(cp) + defp grapheme_byte_size(cps), do: grapheme_byte_size(cps, 0) + + defp grapheme_byte_size([cp | cps], acc), + do: grapheme_byte_size(cps, acc + codepoint_byte_size(cp)) + + defp grapheme_byte_size([], acc), + do: acc + + defp graphemes_and_length(string), + do: graphemes_and_length(string, [], 0) + + defp graphemes_and_length(string, acc, length) do + case :unicode_util.gc(string) do + [gc | rest] -> + graphemes_and_length(rest, [gc | acc], length + 1) + + [] -> + {:lists.reverse(acc), length} + + {:error, <>} -> + graphemes_and_length(rest, [<> | acc], length + 1) + end + end + + defp reverse_characters_to_binary(acc), + do: acc |> :lists.reverse() |> :unicode.characters_to_binary() end diff --git a/lib/elixir/lib/string/chars.ex b/lib/elixir/lib/string/chars.ex index 0d02f457602..af6eabf45dd 100644 --- a/lib/elixir/lib/string/chars.ex +++ b/lib/elixir/lib/string/chars.ex @@ -2,19 +2,24 @@ import Kernel, except: [to_string: 1] defprotocol String.Chars do @moduledoc ~S""" - The String.Chars protocol is responsible for - converting a structure to a Binary (only if applicable). + The `String.Chars` protocol is responsible for + converting a structure to a binary (only if applicable). + The only function required to be implemented is - `to_string` which does the conversion. + `to_string/1`, which does the conversion. - The `to_string` function automatically imported - by Kernel invokes this protocol. String - interpolation also invokes to_string in its + The `to_string/1` function automatically imported + by `Kernel` invokes this protocol. String + interpolation also invokes `to_string/1` in its arguments. For example, `"foo#{bar}"` is the same as `"foo" <> to_string(bar)`. """ - def to_string(thing) + @doc """ + Converts `term` to a string. + """ + @spec to_string(t) :: String.t() + def to_string(term) end defimpl String.Chars, for: Atom do @@ -28,30 +33,30 @@ defimpl String.Chars, for: Atom do end defimpl String.Chars, for: BitString do - def to_string(thing) when is_binary(thing) do - thing + def to_string(term) when is_binary(term) do + term end - def to_string(thing) do + def to_string(term) do raise Protocol.UndefinedError, - protocol: @protocol, - value: thing, - description: "cannot convert a bitstring to a string" + protocol: @protocol, + value: term, + description: "cannot convert a bitstring to a string" end end defimpl String.Chars, for: List do - def to_string(char_list), do: List.to_string(char_list) + def to_string(charlist), do: List.to_string(charlist) end defimpl String.Chars, for: Integer do - def to_string(thing) do - Integer.to_string(thing) + def to_string(term) do + Integer.to_string(term) end end defimpl String.Chars, for: Float do - def to_string(thing) do - IO.iodata_to_binary(:io_lib_format.fwrite_g(thing)) + def to_string(term) do + IO.iodata_to_binary(:io_lib_format.fwrite_g(term)) end end diff --git a/lib/elixir/lib/string_io.ex b/lib/elixir/lib/string_io.ex index 8728e75d248..c07a0012671 100644 --- a/lib/elixir/lib/string_io.ex +++ b/lib/elixir/lib/string_io.ex @@ -1,6 +1,9 @@ defmodule StringIO do @moduledoc """ - This module provides an IO device that wraps a string. + Controls an IO device process that wraps a string. + + A `StringIO` IO device can be passed as a "device" to + most of the functions in the `IO` module. ## Examples @@ -10,14 +13,77 @@ defmodule StringIO do """ - use GenServer + # We're implementing the GenServer behaviour instead of using the + # `use GenServer` macro, because we don't want the `child_spec/1` + # function as it doesn't make sense to be started under a supervisor. + @behaviour GenServer - @doc """ + @doc ~S""" Creates an IO device. - If the `:capture_prompt` option is set to `true`, - prompts (specified as arguments to `IO.get*` functions) - are captured. + `string` will be the initial input of the newly created + device. + + The device will be created and sent to the function given. + When the function returns, the device will be closed. The final + result will be a tuple with `:ok` and the result of the function. + + ## Options + + * `:capture_prompt` - if set to `true`, prompts (specified as + arguments to `IO.get*` functions) are captured in the output. + Defaults to `false`. + + * `:encoding` (since v1.10.0) - encoding of the IO device. Allowed + values are `:unicode` (default) and `:latin1`. + + ## Examples + + iex> StringIO.open("foo", [], fn pid -> + ...> input = IO.gets(pid, ">") + ...> IO.write(pid, "The input was #{input}") + ...> StringIO.contents(pid) + ...> end) + {:ok, {"", "The input was foo"}} + + iex> StringIO.open("foo", [capture_prompt: true], fn pid -> + ...> input = IO.gets(pid, ">") + ...> IO.write(pid, "The input was #{input}") + ...> StringIO.contents(pid) + ...> end) + {:ok, {"", ">The input was foo"}} + + """ + @doc since: "1.7.0" + @spec open(binary, keyword, (pid -> res)) :: {:ok, res} when res: var + def open(string, options, function) + when is_binary(string) and is_list(options) and is_function(function, 1) do + {:ok, pid} = GenServer.start_link(__MODULE__, {string, options}, []) + + try do + {:ok, function.(pid)} + after + {:ok, {_input, _output}} = close(pid) + end + end + + @doc ~S""" + Creates an IO device. + + `string` will be the initial input of the newly created + device. + + `options_or_function` can be a keyword list of options or + a function. + + If options are provided, the result will be `{:ok, pid}`, returning the + IO device created. The option `:capture_prompt`, when set to `true`, causes + prompts (which are specified as arguments to `IO.get*` functions) to be + included in the device's output. + + If a function is provided, the device will be created and sent to the + function. When the function returns, the device will be closed. The final + result will be a tuple with `:ok` and the result of the function. ## Examples @@ -33,14 +99,30 @@ defmodule StringIO do iex> StringIO.contents(pid) {"", ">"} + iex> StringIO.open("foo", fn pid -> + ...> input = IO.gets(pid, ">") + ...> IO.write(pid, "The input was #{input}") + ...> StringIO.contents(pid) + ...> end) + {:ok, {"", "The input was foo"}} + """ - @spec open(binary, Keyword.t) :: {:ok, pid} - def open(string, options \\ []) when is_binary(string) do - :gen_server.start_link(__MODULE__, {string, options}, []) + @spec open(binary, keyword) :: {:ok, pid} + @spec open(binary, (pid -> res)) :: {:ok, res} when res: var + def open(string, options_or_function \\ []) + + def open(string, options_or_function) when is_binary(string) and is_list(options_or_function) do + GenServer.start_link(__MODULE__, {string, options_or_function}, []) + end + + def open(string, options_or_function) + when is_binary(string) and is_function(options_or_function, 1) do + open(string, [], options_or_function) end @doc """ - Returns current buffers. + Returns the current input/output buffers for the given IO + device. ## Examples @@ -52,11 +134,30 @@ defmodule StringIO do """ @spec contents(pid) :: {binary, binary} def contents(pid) when is_pid(pid) do - :gen_server.call(pid, :contents) + GenServer.call(pid, :contents) + end + + @doc """ + Flushes the output buffer and returns its current contents. + + ## Examples + + iex> {:ok, pid} = StringIO.open("in") + iex> IO.write(pid, "out") + iex> StringIO.flush(pid) + "out" + iex> StringIO.contents(pid) + {"in", ""} + + """ + @spec flush(pid) :: binary + def flush(pid) when is_pid(pid) do + GenServer.call(pid, :flush) end @doc """ - Stops the IO device and returns remaining buffers. + Stops the IO device and returns the remaining input/output + buffers. ## Examples @@ -68,241 +169,264 @@ defmodule StringIO do """ @spec close(pid) :: {:ok, {binary, binary}} def close(pid) when is_pid(pid) do - :gen_server.call(pid, :close) + GenServer.call(pid, :close) end ## callbacks + @impl true def init({string, options}) do capture_prompt = options[:capture_prompt] || false - {:ok, %{input: string, output: "", capture_prompt: capture_prompt}} + encoding = options[:encoding] || :unicode + {:ok, %{encoding: encoding, input: string, output: "", capture_prompt: capture_prompt}} end - def handle_info({:io_request, from, reply_as, req}, s) do - s = io_request(from, reply_as, req, s) - {:noreply, s} + @impl true + def handle_info({:io_request, from, reply_as, req}, state) do + state = io_request(from, reply_as, req, state) + {:noreply, state} end - def handle_info(msg, s) do - super(msg, s) + def handle_info(_message, state) do + {:noreply, state} end - def handle_call(:contents, _from, %{input: input, output: output} = s) do - {:reply, {input, output}, s} + @impl true + def handle_call(:contents, _from, %{input: input, output: output} = state) do + {:reply, {input, output}, state} end - def handle_call(:close, _from, %{input: input, output: output} = s) do - {:stop, :normal, {:ok, {input, output}}, s} + def handle_call(:flush, _from, %{output: output} = state) do + {:reply, output, %{state | output: ""}} end - def handle_call(request, from, s) do - super(request, from, s) + def handle_call(:close, _from, %{input: input, output: output} = state) do + {:stop, :normal, {:ok, {input, output}}, state} end - defp io_request(from, reply_as, req, s) do - {reply, s} = io_request(req, s) - io_reply(from, reply_as, to_reply(reply)) - s + defp io_request(from, reply_as, req, state) do + {reply, state} = io_request(req, state) + io_reply(from, reply_as, reply) + state end - defp io_request({:put_chars, chars}, %{output: output} = s) do - {:ok, %{s | output: << output :: binary, IO.chardata_to_string(chars) :: binary >>}} + defp io_request({:put_chars, chars} = req, state) do + put_chars(:latin1, chars, req, state) end - defp io_request({:put_chars, m, f, as}, %{output: output} = s) do - chars = apply(m, f, as) - {:ok, %{s | output: << output :: binary, IO.chardata_to_string(chars) :: binary >>}} + defp io_request({:put_chars, mod, fun, args} = req, state) do + put_chars(:latin1, apply(mod, fun, args), req, state) end - defp io_request({:put_chars, _encoding, chars}, s) do - io_request({:put_chars, chars}, s) + defp io_request({:put_chars, encoding, chars} = req, state) do + put_chars(encoding, chars, req, state) end - defp io_request({:put_chars, _encoding, mod, func, args}, s) do - io_request({:put_chars, mod, func, args}, s) + defp io_request({:put_chars, encoding, mod, fun, args} = req, state) do + put_chars(encoding, apply(mod, fun, args), req, state) end - defp io_request({:get_chars, prompt, n}, s) when n >= 0 do - io_request({:get_chars, :latin1, prompt, n}, s) + defp io_request({:get_chars, prompt, count}, state) when count >= 0 do + io_request({:get_chars, :latin1, prompt, count}, state) end - defp io_request({:get_chars, encoding, prompt, n}, s) when n >= 0 do - get_chars(encoding, prompt, n, s) + defp io_request({:get_chars, encoding, prompt, count}, state) when count >= 0 do + get_chars(encoding, prompt, count, state) end - defp io_request({:get_line, prompt}, s) do - io_request({:get_line, :latin1, prompt}, s) + defp io_request({:get_line, prompt}, state) do + io_request({:get_line, :latin1, prompt}, state) end - defp io_request({:get_line, encoding, prompt}, s) do - get_line(encoding, prompt, s) + defp io_request({:get_line, encoding, prompt}, state) do + get_line(encoding, prompt, state) end - defp io_request({:get_until, prompt, mod, fun, args}, s) do - io_request({:get_until, :latin1, prompt, mod, fun, args}, s) + defp io_request({:get_until, prompt, mod, fun, args}, state) do + io_request({:get_until, :latin1, prompt, mod, fun, args}, state) end - defp io_request({:get_until, encoding, prompt, mod, fun, args}, s) do - get_until(encoding, prompt, mod, fun, args, s) + defp io_request({:get_until, encoding, prompt, mod, fun, args}, state) do + get_until(encoding, prompt, mod, fun, args, state) end - defp io_request({:get_password, encoding}, s) do - get_line(encoding, "", s) + defp io_request({:get_password, encoding}, state) do + get_line(encoding, "", state) end - defp io_request({:setopts, _opts}, s) do - {{:error, :enotsup}, s} + defp io_request({:setopts, [encoding: encoding]}, state) when encoding in [:latin1, :unicode] do + {:ok, %{state | encoding: encoding}} end - defp io_request(:getopts, s) do - {{:ok, [binary: true, encoding: :unicode]}, s} + defp io_request({:setopts, _opts}, state) do + {{:error, :enotsup}, state} end - defp io_request({:get_geometry, :columns}, s) do - {{:error, :enotsup}, s} + defp io_request(:getopts, state) do + {[binary: true, encoding: state.encoding], state} end - defp io_request({:get_geometry, :rows}, s) do - {{:error, :enotsup}, s} + defp io_request({:get_geometry, :columns}, state) do + {{:error, :enotsup}, state} end - defp io_request({:requests, reqs}, s) do - io_requests(reqs, {:ok, s}) + defp io_request({:get_geometry, :rows}, state) do + {{:error, :enotsup}, state} end - defp io_request(_, s) do - {{:error, :request}, s} + defp io_request({:requests, reqs}, state) do + io_requests(reqs, {:ok, state}) + end + + defp io_request(_, state) do + {{:error, :request}, state} + end + + ## put_chars + + defp put_chars(encoding, chars, req, state) do + case :unicode.characters_to_binary(chars, encoding, state.encoding) do + string when is_binary(string) -> + {:ok, %{state | output: state.output <> string}} + + {_, _, _} -> + {{:error, {:no_translation, encoding, state.encoding}}, state} + end + rescue + ArgumentError -> {{:error, req}, state} end ## get_chars - defp get_chars(encoding, prompt, n, - %{input: input, output: output, capture_prompt: capture_prompt} = s) do - case do_get_chars(input, encoding, n) do + defp get_chars(encoding, prompt, count, %{input: input} = state) do + case get_chars(input, encoding, count) do {:error, _} = error -> - {error, s} - {result, input} -> - if capture_prompt do - output = << output :: binary, IO.chardata_to_string(prompt) :: binary >> - end + {error, state} - {result, %{s | input: input, output: output}} + {result, input} -> + {result, state_after_read(state, input, prompt, 1)} end end - defp do_get_chars("", _encoding, _n) do + defp get_chars("", _encoding, _count) do {:eof, ""} end - defp do_get_chars(input, :latin1, n) when byte_size(input) < n do + defp get_chars(input, :latin1, count) when byte_size(input) < count do {input, ""} end - defp do_get_chars(input, :latin1, n) do - <> = input + defp get_chars(input, :latin1, count) do + <> = input {chars, rest} end - defp do_get_chars(input, encoding, n) do - try do - case :file_io_server.count_and_find(input, n, encoding) do - {buf_count, split_pos} when buf_count < n or split_pos == :none -> - {input, ""} - {_buf_count, split_pos} -> - <> = input - {chars, rest} - end - catch - :exit, :invalid_unicode -> - {:error, :invalid_unicode} + defp get_chars(input, :unicode, count) do + with {:ok, count} <- split_at(input, count, 0) do + <> = input + {chars, rest} end end + defp split_at(_, 0, acc), + do: {:ok, acc} + + defp split_at(<>, count, acc), + do: split_at(t, count - 1, acc + byte_size(<>)) + + defp split_at(<<_, _::binary>>, _count, _acc), + do: {:error, :invalid_unicode} + + defp split_at(<<>>, _count, acc), + do: {:ok, acc} + ## get_line - defp get_line(encoding, prompt, - %{input: input, output: output, capture_prompt: capture_prompt} = s) do - case :unicode.characters_to_list(input, encoding) do - {:error, _, _} -> - {{:error, :collect_line}, s} - {:incomplete, _, _} -> - {{:error, :collect_line}, s} - chars -> - {result, input} = do_get_line(chars, encoding) - - if capture_prompt do - output = << output :: binary, IO.chardata_to_string(prompt) :: binary >> - end + defp get_line(encoding, prompt, %{input: input} = state) do + case bytes_until_eol(input, encoding, 0) do + {:split, 0} -> + {:eof, state_after_read(state, "", prompt, 1)} - {result, %{s | input: input, output: output}} - end - end + {:split, count} -> + {result, remainder} = :erlang.split_binary(input, count) + {result, state_after_read(state, remainder, prompt, 1)} - defp do_get_line('', _encoding) do - {:eof, ""} - end + {:replace_split, count} -> + {result, remainder} = :erlang.split_binary(input, count) + result = binary_part(result, 0, byte_size(result) - 2) <> "\n" + {result, state_after_read(state, remainder, prompt, 1)} - defp do_get_line(chars, encoding) do - {line, rest} = collect_line(chars) - {:unicode.characters_to_binary(line, encoding), - :unicode.characters_to_binary(rest, encoding)} + :error -> + {{:error, :collect_line}, state} + end end ## get_until - defp get_until(encoding, prompt, mod, fun, args, - %{input: input, output: output, capture_prompt: capture_prompt} = s) do - case :unicode.characters_to_list(input, encoding) do - {:error, _, _} -> - {:error, s} - {:incomplete, _, _} -> - {:error, s} - chars -> - {result, input, count} = do_get_until(chars, encoding, mod, fun, args) - - if capture_prompt do - output = << output :: binary, :binary.copy(IO.chardata_to_string(prompt), count) :: binary >> - end - + defp get_until(encoding, prompt, mod, fun, args, %{input: input} = state) do + case get_until(input, encoding, mod, fun, args, [], 0) do + {result, input, count} -> input = case input do :eof -> "" - _ -> :unicode.characters_to_binary(input, encoding) + _ -> list_to_binary(input, encoding) end - {result, %{s | input: input, output: output}} + {get_until_result(result, encoding), state_after_read(state, input, prompt, count)} + + :error -> + {:error, state} end end - defp do_get_until(chars, encoding, mod, fun, args, continuation \\ [], count \\ 0) - - defp do_get_until('', encoding, mod, fun, args, continuation, count) do + defp get_until("", encoding, mod, fun, args, continuation, count) do case apply(mod, fun, [continuation, :eof | args]) do {:done, result, rest} -> {result, rest, count + 1} + {:more, next_continuation} -> - do_get_until('', encoding, mod, fun, args, next_continuation, count + 1) + get_until("", encoding, mod, fun, args, next_continuation, count + 1) end end - defp do_get_until(chars, encoding, mod, fun, args, continuation, count) do - {line, rest} = collect_line(chars) + defp get_until(chars, encoding, mod, fun, args, continuation, count) do + case bytes_until_eol(chars, encoding, 0) do + {kind, size} when kind in [:split, :replace_split] -> + {line, rest} = :erlang.split_binary(chars, size) + + case apply(mod, fun, [continuation, binary_to_list(line, encoding) | args]) do + {:done, result, :eof} -> + {result, rest, count + 1} + + {:done, result, extra} -> + {result, extra ++ binary_to_list(rest, encoding), count + 1} - case apply(mod, fun, [continuation, line | args]) do - {:done, result, rest1} -> - unless rest1 == :eof do - rest = rest1 ++ rest + {:more, next_continuation} -> + get_until(rest, encoding, mod, fun, args, next_continuation, count + 1) end - {result, rest, count + 1} - {:more, next_continuation} -> - do_get_until(rest, encoding, mod, fun, args, next_continuation, count + 1) + + :error -> + :error end end + defp binary_to_list(data, _) when is_list(data), do: data + defp binary_to_list(data, :unicode) when is_binary(data), do: String.to_charlist(data) + defp binary_to_list(data, :latin1) when is_binary(data), do: :erlang.binary_to_list(data) + + defp list_to_binary(data, _) when is_binary(data), do: data + defp list_to_binary(data, :unicode) when is_list(data), do: List.to_string(data) + defp list_to_binary(data, :latin1) when is_list(data), do: :erlang.list_to_binary(data) + + # From https://www.erlang.org/doc/apps/stdlib/io_protocol.html: result can be any + # Erlang term, but if it is a list(), the I/O server can convert it to a binary(). + defp get_until_result(data, encoding) when is_list(data), do: list_to_binary(data, encoding) + defp get_until_result(data, _), do: data + ## io_requests - defp io_requests([r|rs], {:ok, s}) do - io_requests(rs, io_request(r, s)) + defp io_requests([req | rest], {:ok, state}) do + io_requests(rest, io_request(req, state)) end defp io_requests(_, result) do @@ -311,30 +435,30 @@ defmodule StringIO do ## helpers - defp collect_line(chars) do - collect_line(chars, []) + defp state_after_read(%{capture_prompt: false} = state, remainder, _prompt, _count) do + %{state | input: remainder} end - defp collect_line([], stack) do - {:lists.reverse(stack), []} + defp state_after_read(%{capture_prompt: true, output: output} = state, remainder, prompt, count) do + output = <> + %{state | input: remainder, output: output} end - defp collect_line([?\r, ?\n | rest], stack) do - {:lists.reverse([?\n|stack]), rest} - end + defp bytes_until_eol("", _, count), do: {:split, count} + defp bytes_until_eol(<<"\r\n"::binary, _::binary>>, _, count), do: {:replace_split, count + 2} + defp bytes_until_eol(<<"\n"::binary, _::binary>>, _, count), do: {:split, count + 1} - defp collect_line([?\n | rest], stack) do - {:lists.reverse([?\n|stack]), rest} + defp bytes_until_eol(<>, :unicode, count) do + bytes_until_eol(tail, :unicode, count + byte_size(<>)) end - defp collect_line([h|t], stack) do - collect_line(t, [h|stack]) + defp bytes_until_eol(<<_, tail::binary>>, :latin1, count) do + bytes_until_eol(tail, :latin1, count + 1) end + defp bytes_until_eol(<<_::binary>>, _, _), do: :error + defp io_reply(from, reply_as, reply) do - send from, {:io_reply, reply_as, reply} + send(from, {:io_reply, reply_as, reply}) end - - defp to_reply(list) when is_list(list), do: IO.chardata_to_string(list) - defp to_reply(other), do: other end diff --git a/lib/elixir/lib/supervisor.ex b/lib/elixir/lib/supervisor.ex index 9945e2ce5c9..863fab99750 100644 --- a/lib/elixir/lib/supervisor.ex +++ b/lib/elixir/lib/supervisor.ex @@ -1,122 +1,315 @@ defmodule Supervisor do - @moduledoc """ - A behaviour module for implementing supervision functionality. + @moduledoc ~S""" + A behaviour module for implementing supervisors. - A supervisor is a process which supervises other processes called - child processes. Supervisors are used to build an hierarchical process - structure called a supervision tree, a nice way to structure fault-tolerant - applications. + A supervisor is a process which supervises other processes, which we + refer to as *child processes*. Supervisors are used to build a hierarchical + process structure called a *supervision tree*. Supervision trees provide + fault-tolerance and encapsulate how our applications start and shutdown. - A supervisor implemented using this module will have a standard set - of interface functions and include functionality for tracing and error - reporting. It will also fit into an supervision tree. + A supervisor may be started directly with a list of child specifications via + `start_link/2` or you may define a module-based supervisor that implements + the required callbacks. The sections below use `start_link/2` to start + supervisors in most examples, but it also includes a specific section + on module-based ones. - ## Example + ## Examples - In order to define a supervisor, we need to first define a child process - that is going to be supervised. In order to do so, we will define a GenServer - that represents a stack: + In order to start a supervisor, we need to first define a child process + that will be supervised. As an example, we will define a `GenServer`, + a generic server, that keeps a counter. Other processes can then send + messages to this process to read the counter and bump its value. - defmodule Stack do + > Note: in practice you would not define a counter as a GenServer. Instead, + > if you need a counter, you would pass it around as inputs and outputs to + > the functions that need it. The reason we picked a counter in this example + > is due to its simplicity, as it allows us to focus on how supervisors work. + + defmodule Counter do use GenServer - def start_link(state) do - GenServer.start_link(__MODULE__, state, [name: :sup_stack]) + def start_link(arg) when is_integer(arg) do + GenServer.start_link(__MODULE__, arg, name: __MODULE__) + end + + ## Callbacks + + @impl true + def init(counter) do + {:ok, counter} end - def handle_call(:pop, _from, [h|t]) do - {:reply, h, t} + @impl true + def handle_call(:get, _from, counter) do + {:reply, counter, counter} end - def handle_cast({:push, h}, _from, t) do - {:noreply, [h|t]} + def handle_call({:bump, value}, _from, counter) do + {:reply, counter, counter + value} end end - We can now define our supervisor and start it as follows: + The `Counter` receives an argument on `start_link`. This argument + is passed to the `init/1` callback which becomes the initial value + of the counter. Our counter handles two operations (known as calls): + `:get`, to get the current counter value, and `:bump`, that bumps + the counter by the given `value` and returns the old counter. - # Import helpers for defining supervisors - import Supervisor.Spec + We can now start a supervisor that will start and supervise our + counter process. The first step is to define a list of **child + specifications** that control how each child behaves. Each child + specification is a map, as shown below: - # We are going to supervise the Stack server which will - # be started with a single argument [:hello] children = [ - worker(Stack, [[:hello]]) + # The Counter is a child started via Counter.start_link(0) + %{ + id: Counter, + start: {Counter, :start_link, [0]} + } ] - # Start the supervisor with our one child + # Now we start the supervisor with the children and a strategy {:ok, pid} = Supervisor.start_link(children, strategy: :one_for_one) - Notice that when starting the GenServer, we have registered it - with name `:sup_stack`, which allows us to call it directly and - get what is on the stack: + # After started, we can query the supervisor for information + Supervisor.count_children(pid) + #=> %{active: 1, specs: 1, supervisors: 0, workers: 1} + + Note that when starting the GenServer, we are registering it + with name `Counter` via the `name: __MODULE__` option. This allows + us to call it directly and get its value: - GenServer.call(:sup_stack, :pop) - #=> :hello + GenServer.call(Counter, :get) + #=> 0 - GenServer.cast(:sup_stack, {:push, :world}) - #=> :ok + GenServer.cast(Counter, {:bump, 3}) + #=> 0 - GenServer.call(:sup_stack, :pop) - #=> :world + GenServer.call(Counter, :get) + #=> 3 - However, there is a bug in our stack server. If we call `:pop` and - the stack is empty, it is going to crash because no clause matches. - Let's try it: + However, there is a bug in our counter server. If we call `:bump` with + a non-numeric value, it is going to crash: - GenServer.call(:sup_stack, :pop) - =ERROR REPORT==== + GenServer.call(Counter, {:bump, "oops"}) + ** (exit) exited in: GenServer.call(Counter, {:bump, "oops"}, 5000) Luckily, since the server is being supervised by a supervisor, the - supervisor will automatically start a new one, with the default stack - of `[:hello]` like before: + supervisor will automatically start a new one, reset back to its initial + value of `0`: - GenServer.call(:sup_stack, :pop) == :hello + GenServer.call(Counter, :get) + #=> 0 Supervisors support different strategies; in the example above, we have chosen `:one_for_one`. Furthermore, each supervisor can have many - workers and supervisors as children, each of them with their specific - configuration, shutdown values, and restart strategies. + workers and/or supervisors as children, with each one having its own + configuration (as outlined in the "Child specification" section). - Continue reading this moduledoc to learn more about supervision strategies - and then follow to the `Supervisor.Spec` module documentation to learn - about the specification for workers and supervisors. + The rest of this document will cover how child processes are specified, + how they can be started and stopped, different supervision strategies + and more. - ## Module-based supervisors + ## Child specification - In the example above, a supervisor was dynamically created by passing - the supervision structure to `start_link/2`. However, supervisors - can also be created by explicitly defining a supervision module: + The child specification describes how the supervisor starts, shuts down, + and restarts child processes. - defmodule MyApp.Supervisor do - use Supervisor + The child specification is a map containing up to 6 elements. The first two keys + in the following list are required, and the remaining ones are optional: - def start_link do - Supervisor.start_link(__MODULE__, []) - end + * `:id` - any term used to identify the child specification internally by + the supervisor; defaults to the given module. This key is required. + For supervisors, in the case of conflicting `:id` values, the supervisor + will refuse to initialize and require explicit IDs. This is not the case + for [dynamic supervisors](`DynamicSupervisor`) though. - def init([]) do - children = [ - worker(Stack, [[:hello]]) - ] + * `:start` - a tuple with the module-function-args to be invoked + to start the child process. This key is required. - supervise(children, strategy: :one_for_one) - end + * `:restart` - an atom that defines when a terminated child process + should be restarted (see the "Restart values" section below). + This key is optional and defaults to `:permanent`. + + * `:shutdown` - an integer or atom that defines how a child process should + be terminated (see the "Shutdown values" section below). This key + is optional and defaults to `5_000` if the type is `:worker` or + `:infinity` if the type is `:supervisor`. + + * `:type` - specifies that the child process is a `:worker` or a + `:supervisor`. This key is optional and defaults to `:worker`. + + * `:modules` - a list of modules used by hot code upgrade mechanisms + to determine which processes are using certain modules. It is typically + set to the callback module of behaviours like `GenServer`, `Supervisor`, + and such. It is set automatically based on the `:start` value and it is rarely + changed in practice. + + Let's understand what the `:shutdown` and `:restart` options control. + + ### Shutdown values (:shutdown) + + The following shutdown values are supported in the `:shutdown` option: + + * `:brutal_kill` - the child process is unconditionally and immediately + terminated using `Process.exit(child, :kill)`. + + * any integer >= 0 - the amount of time in milliseconds that the + supervisor will wait for its children to terminate after emitting a + `Process.exit(child, :shutdown)` signal. If the child process is + not trapping exits, the initial `:shutdown` signal will terminate + the child process immediately. If the child process is trapping + exits, it has the given amount of time to terminate. + If it doesn't terminate within the specified time, the child process + is unconditionally terminated by the supervisor via + `Process.exit(child, :kill)`. + + * `:infinity` - works as an integer except the supervisor will wait + indefinitely for the child to terminate. If the child process is a + supervisor, the recommended value is `:infinity` to give the supervisor + and its children enough time to shut down. This option can be used with + regular workers but doing so is discouraged and requires extreme care. + If not used carefully, the child process will never terminate, + preventing your application from terminating as well. + + ### Restart values (:restart) + + The `:restart` option controls what the supervisor should consider to + be a successful termination or not. If the termination is successful, + the supervisor won't restart the child. If the child process crashed, + the supervisor will start a new one. + + The following restart values are supported in the `:restart` option: + + * `:permanent` - the child process is always restarted. + + * `:temporary` - the child process is never restarted, regardless + of the supervision strategy: any termination (even abnormal) is + considered successful. + + * `:transient` - the child process is restarted only if it + terminates abnormally, i.e., with an exit reason other than + `:normal`, `:shutdown`, or `{:shutdown, term}`. + + For a more complete understanding of the exit reasons and their + impact, see the "Exit reasons and restarts" section. + + ## `child_spec/1` function + + When starting a supervisor, we may pass a list of child specifications. Those + specifications are maps that tell how the supervisor should start, stop and + restart each of its children: + + %{ + id: Counter, + start: {Counter, :start_link, [0]} + } + + The map above defines a child with `:id` of `Counter` that is started + by calling `Counter.start_link(0)`. + + However, defining the child specification for each child as a map can be + quite error prone, as we may change the `Counter` implementation and forget + to update its specification. That's why Elixir allows you to pass a tuple with + the module name and the `start_link` argument instead of the specification: + + children = [ + {Counter, 0} + ] + + The supervisor will then invoke `Counter.child_spec(0)` to retrieve a child + specification. Now the `Counter` module is responsible for building its own + specification, for example, we could write: + + def child_spec(arg) do + %{ + id: Counter, + start: {Counter, :start_link, [arg]} + } end - You may want to use a module-based supervisor if: + Luckily for us, `use GenServer` already defines a `Counter.child_spec/1` + exactly like above, so you don't need to write the definition above yourself. + If you want to customize the automatically generated `child_spec/1` function, + you can pass the options directly to `use GenServer`: + + use GenServer, restart: :transient + + Finally, note it is also possible to simply pass the `Counter` module as + a child: + + children = [ + Counter + ] + + When only the module name is given, it is equivalent to `{Counter, []}`, + which in our case would be invalid, which is why we always pass the initial + counter explicitly. + + By replacing the child specification with `{Counter, 0}`, we keep it + encapsulated in the `Counter` module. We could now share our + `Counter` implementation with other developers and they can add it directly + to their supervision tree without worrying about the low-level details of + the counter. + + Overall, a child specification can be one of the following: + + * a map representing the child specification itself - as outlined in the + "Child specification" section + + * a tuple with a module as first element and the start argument as second - + such as `{Counter, 0}`. In this case, `Counter.child_spec(0)` is called + to retrieve the child specification + + * a module - such as `Counter`. In this case, `Counter.child_spec([])` + would be called, which is invalid for the counter, but it is useful in + many other cases, especially when you want to pass a list of options + to the child process + + If you need to convert a `{module, arg}` tuple or a module child specification to a + [child specification](`t:child_spec/0`) or modify a child specification itself, + you can use the `Supervisor.child_spec/2` function. + For example, to run the counter with a different `:id` and a `:shutdown` value of + 10 seconds (10_000 milliseconds): - * You need to do some particular action on supervisor - initialization, like setting up a ETS table. + children = [ + Supervisor.child_spec({Counter, 0}, id: MyCounter, shutdown: 10_000) + ] + + ## Supervisor strategies and options + + So far we have started the supervisor passing a single child as a tuple + as well as a strategy called `:one_for_one`: + + children = [ + {Counter, 0} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + + The first argument given to `start_link/2` is a list of child + specifications as defined in the "child_spec/1" section above. + + The second argument is a keyword list of options: + + * `:strategy` - the supervision strategy option. It can be either + `:one_for_one`, `:rest_for_one` or `:one_for_all`. Required. + See the "Strategies" section. + + * `:max_restarts` - the maximum number of restarts allowed in + a time frame. Defaults to `3`. + + * `:max_seconds` - the time frame in which `:max_restarts` applies. + Defaults to `5`. - * You want to perform partial hot-code swapping of the - tree. For example, if you add or remove a children, - the module-based supervision will add and remove the - new children directly, while the dynamic supervision - requires the whole tree to be restarted in order to - perform such swaps. + * `:name` - a name to register the supervisor process. Supported values are + explained in the "Name registration" section in the documentation for + `GenServer`. Optional. - ## Strategies + ### Strategies + + Supervisors support different supervision strategies (through the + `:strategy` option, as seen above): * `:one_for_one` - if a child process terminates, only that process is restarted. @@ -125,188 +318,648 @@ defmodule Supervisor do processes are terminated and then all child processes (including the terminated one) are restarted. - * `:rest_for_one` - if a child process terminates, the "rest" of - the child processes, i.e. the child processes after the terminated - one in start order, are terminated. Then the terminated child - process and the rest of the child processes are restarted. + * `:rest_for_one` - if a child process terminates, the terminated child + process and the rest of the children started after it, are terminated and + restarted. + + In the above, process termination refers to unsuccessful termination, which + is determined by the `:restart` option. - * `:simple_one_for_one` - similar to `:one_for_one` but suits better - when dynamically attaching children. This strategy requires the - supervisor specification to contain only one children. Many functions - in this module behave slightly differently when this strategy is - used. + To dynamically supervise children, see `DynamicSupervisor`. - ## Name Registration + ### Name registration A supervisor is bound to the same name registration rules as a `GenServer`. - Read more about it in the `GenServer` docs. + Read more about these rules in the documentation for `GenServer`. + + ## Module-based supervisors + + In the example so far, the supervisor was started by passing the supervision + structure to `start_link/2`. However, supervisors can also be created by + explicitly defining a supervision module: + + defmodule MyApp.Supervisor do + # Automatically defines child_spec/1 + use Supervisor + + def start_link(init_arg) do + Supervisor.start_link(__MODULE__, init_arg, name: __MODULE__) + end + + @impl true + def init(_init_arg) do + children = [ + {Counter, 0} + ] + + Supervisor.init(children, strategy: :one_for_one) + end + end + + The difference between the two approaches is that a module-based + supervisor gives you more direct control over how the supervisor + is initialized. Instead of calling `Supervisor.start_link/2` with + a list of child specifications that are automatically initialized, we manually + initialize the children by calling `Supervisor.init/2` inside its + `c:init/1` callback. `Supervisor.init/2` accepts the same `:strategy`, + `:max_restarts`, and `:max_seconds` options as `start_link/2`. + + `use Supervisor` also defines a `child_spec/1` function which allows + us to run `MyApp.Supervisor` as a child of another supervisor or + at the top of your supervision tree as: + + children = [ + MyApp.Supervisor + ] + + Supervisor.start_link(children, strategy: :one_for_one) + + A general guideline is to use the supervisor without a callback + module only at the top of your supervision tree, generally in the + `c:Application.start/2` callback. We recommend using module-based + supervisors for any other supervisor in your application, so they + can run as a child of another supervisor in the tree. The `child_spec/1` + generated automatically by `Supervisor` can be customized with the + following options: + + * `:id` - the child specification identifier, defaults to the current module + * `:restart` - when the supervisor should be restarted, defaults to `:permanent` + + The `@doc` annotation immediately preceding `use Supervisor` will be + attached to the generated `child_spec/1` function. + + ## Start and shutdown + + When the supervisor starts, it traverses all child specifications and + then starts each child in the order they are defined. This is done by + calling the function defined under the `:start` key in the child + specification and typically defaults to `start_link/1`. + + The `start_link/1` (or a custom) is then called for each child process. + The `start_link/1` function must return `{:ok, pid}` where `pid` is the + process identifier of a new process that is linked to the supervisor. + The child process usually starts its work by executing the `c:init/1` + callback. Generally speaking, the `init` callback is where we initialize + and configure the child process. + + The shutdown process happens in reverse order. + + When a supervisor shuts down, it terminates all children in the opposite + order they are listed. The termination happens by sending a shutdown exit + signal, via `Process.exit(child_pid, :shutdown)`, to the child process and + then awaiting for a time interval for the child process to terminate. This + interval defaults to 5000 milliseconds. If the child process does not + terminate in this interval, the supervisor abruptly terminates the child + with reason `:kill`. The shutdown time can be configured in the child + specification which is fully detailed in the next section. + + If the child process is not trapping exits, it will shutdown immediately + when it receives the first exit signal. If the child process is trapping + exits, then the `terminate` callback is invoked, and the child process + must terminate in a reasonable time interval before being abruptly + terminated by the supervisor. + + In other words, if it is important that a process cleans after itself + when your application or the supervision tree is shutting down, then + this process must trap exits and its child specification should specify + the proper `:shutdown` value, ensuring it terminates within a reasonable + interval. + + ## Exit reasons and restarts + + A supervisor restarts a child process depending on its `:restart` configuration. + For example, when `:restart` is set to `:transient`, the supervisor does not + restart the child in case it exits with reason `:normal`, `:shutdown` or + `{:shutdown, term}`. + + So one may ask: which exit reason should I choose when exiting? There are + three options: + + * `:normal` - in such cases, the exit won't be logged, there is no restart + in transient mode, and linked processes do not exit + + * `:shutdown` or `{:shutdown, term}` - in such cases, the exit won't be + logged, there is no restart in transient mode, and linked processes exit + with the same reason unless they're trapping exits + + * any other term - in such cases, the exit will be logged, there are + restarts in transient mode, and linked processes exit with the same + reason unless they're trapping exits + + Note that the supervisor that reaches maximum restart intensity will exit with + `:shutdown` reason. In this case the supervisor will only be restarted if its + child specification was defined with the `:restart` option set to `:permanent` + (the default). """ @doc false - defmacro __using__(_) do - quote location: :keep do - @behaviour :supervisor + defmacro __using__(opts) do + quote location: :keep, bind_quoted: [opts: opts] do import Supervisor.Spec + @behaviour Supervisor + + unless Module.has_attribute?(__MODULE__, :doc) do + @doc """ + Returns a specification to start this module under a supervisor. + + See `Supervisor`. + """ + end + + def child_spec(init_arg) do + default = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [init_arg]}, + type: :supervisor + } + + Supervisor.child_spec(default, unquote(Macro.escape(opts))) + end + + defoverridable child_spec: 1 end end + @doc """ + Callback invoked to start the supervisor and during hot code upgrades. + + Developers typically invoke `Supervisor.init/2` at the end of their + init callback to return the proper supervision flags. + """ + @callback init(init_arg :: term) :: + {:ok, + {sup_flags(), [child_spec() | (old_erlang_child_spec :: :supervisor.child_spec())]}} + | :ignore + @typedoc "Return values of `start_link` functions" - @type on_start :: {:ok, pid} | :ignore | - {:error, {:already_started, pid} | {:shutdown, term} | term} + @type on_start :: + {:ok, pid} + | :ignore + | {:error, {:already_started, pid} | {:shutdown, term} | term} @typedoc "Return values of `start_child` functions" - @type on_start_child :: {:ok, child} | {:ok, child, info :: term} | - {:error, {:already_started, child} | :already_present | term} + @type on_start_child :: + {:ok, child} + | {:ok, child, info :: term} + | {:error, {:already_started, child} | :already_present | term} + + @typedoc """ + A child process. + It can be a PID when the child process was started, or `:undefined` when + the child was created by a [dynamic supervisor](`DynamicSupervisor`). + """ @type child :: pid | :undefined - @typedoc "The Supervisor name" + @typedoc "The supervisor name" @type name :: atom | {:global, term} | {:via, module, term} - @typedoc "Options used by the `start*` functions" - @type options :: [name: name, - strategy: Supervisor.Spec.strategy, - max_restarts: non_neg_integer, - max_seconds: non_neg_integer] + @typedoc "Option values used by the `start*` functions" + @type option :: {:name, name} + + @typedoc "The supervisor flags returned on init" + @type sup_flags() :: %{ + strategy: strategy(), + intensity: non_neg_integer(), + period: pos_integer() + } @typedoc "The supervisor reference" @type supervisor :: pid | name | {atom, node} + @typedoc "Options given to `start_link/2` and `init/2`" + @type init_option :: + {:strategy, strategy} + | {:max_restarts, non_neg_integer} + | {:max_seconds, pos_integer} + + @typedoc "Supported restart options" + @type restart :: :permanent | :transient | :temporary + + # TODO: Update :shutdown to "timeout() | :brutal_kill" when we require Erlang/OTP 24. + # Additionally apply https://github.com/elixir-lang/elixir/pull/11836 + @typedoc "Supported shutdown options" + @type shutdown :: pos_integer() | :infinity | :brutal_kill + + @typedoc "Supported strategies" + @type strategy :: :one_for_one | :one_for_all | :rest_for_one + + @typedoc """ + Supervisor type. + + Whether the supervisor is a worker or a supervisor. + """ + @type type :: :worker | :supervisor + + # Note we have inlined all types for readability + @typedoc """ + The supervisor child specification. + + It defines how the supervisor should start, stop and restart each of its children. + """ + @type child_spec :: %{ + required(:id) => atom() | term(), + required(:start) => {module(), function_name :: atom(), args :: [term()]}, + optional(:restart) => restart(), + optional(:shutdown) => shutdown(), + optional(:type) => type(), + optional(:modules) => [module()] | :dynamic + } + @doc """ Starts a supervisor with the given children. - A strategy is required to be given as an option. Furthermore, - the `:max_restarts` and `:max_seconds` value can be configured - as described in `Supervisor.Spec.supervise/2` docs. + `children` is a list of the following forms: + + * a [child specification](`t:child_spec/0`) + + * a module, where `module.child_spec([])` will be invoked to retrieve + its child specification + + * a two-element tuple in the shape of `{module, arg}`, where `module.child_spec(arg)` + will be invoked to retrieve its child specification + + A strategy is required to be provided through the `:strategy` option. See + "Supervisor strategies and options" for examples and other options. The options can also be used to register a supervisor name. - the supported values are described under the `Name Registration` + The supported values are described under the "Name registration" section in the `GenServer` module docs. - If the supervisor and its child processes are successfully created - (i.e. if the start function of all child processes returns `{:ok, child}`, - `{:ok, child, info}`, or `:ignore`) the function returns - `{:ok, pid}`, where `pid` is the pid of the supervisor. If there - already exists a process with the specified name, the function returns - `{:error, {:already_started, pid}}`, where pid is the pid of that - process. - - If any of the child process start functions fail or return an error tuple or - an erroneous value, the supervisor will first terminate all already - started child processes with reason `:shutdown` and then terminate - itself and return `{:error, {:shutdown, reason}}`. - - Note that the `Supervisor` is linked to the parent process - and will exit not only on crashes but also if the parent process - exits with `:normal` reason. + If the supervisor and all child processes are successfully spawned + (if the start function of each child process returns `{:ok, child}`, + `{:ok, child, info}`, or `:ignore`), this function returns + `{:ok, pid}`, where `pid` is the PID of the supervisor. If the supervisor + is given a name and a process with the specified name already exists, + the function returns `{:error, {:already_started, pid}}`, where `pid` + is the PID of that process. + + If the start function of any of the child processes fails or returns an error + tuple or an erroneous value, the supervisor first terminates with reason + `:shutdown` all the child processes that have already been started, and then + terminates itself and returns `{:error, {:shutdown, reason}}`. + + Note that a supervisor started with this function is linked to the parent + process and exits not only on crashes but also if the parent process exits + with `:normal` reason. """ - @spec start_link([tuple], options) :: on_start + @spec start_link( + [ + child_spec() + | {module, term} + | module + | (old_erlang_child_spec :: :supervisor.child_spec()) + ], + [option | init_option] + ) :: {:ok, pid} | {:error, {:already_started, pid} | {:shutdown, term} | term} def start_link(children, options) when is_list(children) do - spec = Supervisor.Spec.supervise(children, options) - start_link(Supervisor.Default, spec, options) + {sup_opts, start_opts} = Keyword.split(options, [:strategy, :max_seconds, :max_restarts]) + start_link(Supervisor.Default, init(children, sup_opts), start_opts) + end + + @doc """ + Receives a list of child specifications to initialize and a set of `options`. + + This is typically invoked at the end of the `c:init/1` callback of + module-based supervisors. See the sections "Supervisor strategies and options" and + "Module-based supervisors" in the module documentation for more information. + + This function returns a tuple containing the supervisor + flags and child specifications. + + ## Examples + + def init(_init_arg) do + children = [ + {Counter, 0} + ] + + Supervisor.init(children, strategy: :one_for_one) + end + + ## Options + + * `:strategy` - the supervision strategy option. It can be either + `:one_for_one`, `:rest_for_one`, or `:one_for_all` + + * `:max_restarts` - the maximum number of restarts allowed in + a time frame. Defaults to `3`. + + * `:max_seconds` - the time frame in seconds in which `:max_restarts` + applies. Defaults to `5`. + + The `:strategy` option is required and by default a maximum of 3 restarts + is allowed within 5 seconds. Check the `Supervisor` module for a detailed + description of the available strategies. + """ + @doc since: "1.5.0" + @spec init( + [ + child_spec() + | {module, term} + | module + | (old_erlang_child_spec :: :supervisor.child_spec()) + ], + [init_option] + ) :: + {:ok, + {sup_flags(), [child_spec() | (old_erlang_child_spec :: :supervisor.child_spec())]}} + def init(children, options) when is_list(children) and is_list(options) do + strategy = + case options[:strategy] do + nil -> + raise ArgumentError, "expected :strategy option to be given" + + :simple_one_for_one -> + IO.warn( + ":simple_one_for_one strategy is deprecated, please use DynamicSupervisor instead" + ) + + :simple_one_for_one + + other -> + other + end + + intensity = Keyword.get(options, :max_restarts, 3) + period = Keyword.get(options, :max_seconds, 5) + flags = %{strategy: strategy, intensity: intensity, period: period} + {:ok, {flags, Enum.map(children, &init_child/1)}} + end + + defp init_child(module) when is_atom(module) do + init_child({module, []}) + end + + defp init_child({module, arg}) when is_atom(module) do + try do + module.child_spec(arg) + rescue + e in UndefinedFunctionError -> + case __STACKTRACE__ do + [{^module, :child_spec, [^arg], _} | _] -> + raise ArgumentError, child_spec_error(module) + + stack -> + reraise e, stack + end + end + end + + defp init_child(map) when is_map(map) do + map + end + + defp init_child({_, _, _, _, _, _} = tuple) do + tuple + end + + defp init_child(other) do + raise ArgumentError, """ + supervisors expect each child to be one of the following: + + * a module + * a {module, arg} tuple + * a child specification as a map with at least the :id and :start fields + * or a tuple with 6 elements generated by Supervisor.Spec (deprecated) + + Got: #{inspect(other)} + """ + end + + defp child_spec_error(module) do + if Code.ensure_loaded?(module) do + """ + The module #{inspect(module)} was given as a child to a supervisor + but it does not implement child_spec/1. + + If you own the given module, please define a child_spec/1 function + that receives an argument and returns a child specification as a map. + For example: + + def child_spec(opts) do + %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [opts]}, + type: :worker, + restart: :permanent, + shutdown: 500 + } + end + + Note that "use Agent", "use GenServer" and so on automatically define + this function for you. + + However, if you don't own the given module and it doesn't implement + child_spec/1, instead of passing the module name directly as a supervisor + child, you will have to pass a child specification as a map: + + %{ + id: #{inspect(module)}, + start: {#{inspect(module)}, :start_link, [arg1, arg2]} + } + + See the Supervisor documentation for more information. + """ + else + "The module #{inspect(module)} was given as a child to a supervisor but it does not exist." + end + end + + @doc """ + Builds and overrides a child specification. + + Similar to `start_link/2` and `init/2`, it expects a module, `{module, arg}`, + or a [child specification](`t:child_spec/0`). + + If a two-element tuple in the shape of `{module, arg}` is given, + the child specification is retrieved by calling `module.child_spec(arg)`. + + If a module is given, the child specification is retrieved by calling + `module.child_spec([])`. + + After the child specification is retrieved, the fields on `overrides` + are directly applied on the child spec. If `overrides` has keys that + do not map to any child specification field, an error is raised. + + See the "Child specification" section in the module documentation + for all of the available keys for overriding. + + ## Examples + + This function is often used to set an `:id` option when + the same module needs to be started multiple times in the + supervision tree: + + Supervisor.child_spec({Agent, fn -> :ok end}, id: {Agent, 1}) + #=> %{id: {Agent, 1}, + #=> start: {Agent, :start_link, [fn -> :ok end]}} + + """ + @spec child_spec(child_spec() | {module, arg :: term} | module, keyword) :: child_spec() + def child_spec(module_or_map, overrides) + + def child_spec({_, _, _, _, _, _} = tuple, _overrides) do + raise ArgumentError, + "old tuple-based child specification #{inspect(tuple)} " <> + "is not supported in Supervisor.child_spec/2" + end + + def child_spec(module_or_map, overrides) do + Enum.reduce(overrides, init_child(module_or_map), fn + {key, value}, acc when key in [:id, :start, :restart, :shutdown, :type, :modules] -> + Map.put(acc, key, value) + + {key, _value}, _acc -> + raise ArgumentError, "unknown key #{inspect(key)} in child specification override" + end) end @doc """ - Starts a supervisor module with the given `arg`. + Starts a module-based supervisor process with the given `module` and `init_arg`. - To start the supervisor, the `init/1` callback will be invoked - in the given module. The `init/1` callback must return a - supervision specification which can be created with the help - of `Supervisor.Spec` module. + To start the supervisor, the `c:init/1` callback will be invoked in the given + `module`, with `init_arg` as its argument. The `c:init/1` callback must return a + supervisor specification which can be created with the help of the `init/2` + function. - If the `init/1` callback returns `:ignore`, this function returns + If the `c:init/1` callback returns `:ignore`, this function returns `:ignore` as well and the supervisor terminates with reason `:normal`. If it fails or returns an incorrect value, this function returns `{:error, term}` where `term` is a term with information about the error, and the supervisor terminates with reason `term`. The `:name` option can also be given in order to register a supervisor - name, the supported values are described under the `Name Registration` + name, the supported values are described in the "Name registration" section in the `GenServer` module docs. - - Other failure conditions are specified in `start_link/2` docs. """ - @spec start_link(module, term, options) :: on_start - def start_link(module, arg, options \\ []) when is_list(options) do + + # It is important to keep the two-arity spec because it is a catch-all + # to start_link(children, options). + @spec start_link(module, term) :: on_start + @spec start_link(module, term, [option]) :: on_start + def start_link(module, init_arg, options \\ []) when is_list(options) do case Keyword.get(options, :name) do nil -> - :supervisor.start_link(module, arg) + :supervisor.start_link(module, init_arg) + atom when is_atom(atom) -> - :supervisor.start_link({:local, atom}, module, arg) - other when is_tuple(other) -> - :supervisor.start_link(other, module, arg) + :supervisor.start_link({:local, atom}, module, init_arg) + + {:global, _term} = tuple -> + :supervisor.start_link(tuple, module, init_arg) + + {:via, via_module, _term} = tuple when is_atom(via_module) -> + :supervisor.start_link(tuple, module, init_arg) + + other -> + raise ArgumentError, """ + expected :name option to be one of the following: + + * nil + * atom + * {:global, term} + * {:via, module, term} + + Got: #{inspect(other)} + """ end end @doc """ - Dynamically adds and starts a child specification to the supervisor. + Adds a child specification to `supervisor` and starts that child. - `child_spec` should be a valid child specification (unless the supervisor - is a `:simple_one_for_one` supervisor, see below). The child process will + `child_spec` should be a valid child specification. The child process will be started as defined in the child specification. - In the case of `:simple_one_for_one`, the child specification defined in - the supervisor will be used and instead of a `child_spec`, an arbitrary list - of terms is expected. The child process will then be started by appending - the given list to the existing function arguments in the child specification. + If a child specification with the specified ID already exists, `child_spec` is + discarded and this function returns an error with `:already_started` or + `:already_present` if the corresponding child process is running or not, + respectively. - If there already exists a child specification with the specified id, - `child_spec` is discarded and the function returns an error with `:already_started` - or `:already_present` if the corresponding child process is running or not. + If the child process start function returns `{:ok, child}` or `{:ok, child, + info}`, then child specification and PID are added to the supervisor and + this function returns the same value. - If the child process start function returns `{:ok, child}` or `{:ok, child, info}`, - the child specification and pid is added to the supervisor and the function returns - the same value. - - If the child process start function returns `:ignore, the child specification is - added to the supervisor, the pid is set to undefined and the function returns - `{:ok, :undefined}`. + If the child process start function returns `:ignore`, the child specification + is added to the supervisor, the PID is set to `:undefined` and this function + returns `{:ok, :undefined}`. - If the child process start function returns an error tuple or an erroneous value, - or if it fails, the child specification is discarded and the function returns - `{:error, error}` where `error` is a term containing information about the error - and child specification. + If the child process start function returns an error tuple or an erroneous + value, or if it fails, the child specification is discarded and this function + returns `{:error, error}` where `error` is a term containing information about + the error and child specification. """ - @spec start_child(supervisor, Supervisor.Spec.spec | [term]) :: on_start_child - defdelegate start_child(supervisor, child_spec_or_args), to: :supervisor + @spec start_child( + supervisor, + child_spec() + | {module, term} + | module + | (old_erlang_child_spec :: :supervisor.child_spec()) + ) :: + on_start_child + def start_child(supervisor, {_, _, _, _, _, _} = child_spec) do + call(supervisor, {:start_child, child_spec}) + end + + def start_child(supervisor, args) when is_list(args) do + IO.warn_once( + {__MODULE__, :start_child}, + "Supervisor.start_child/2 with a list of args is deprecated, please use DynamicSupervisor instead", + _stacktrace_drop_levels = 2 + ) + + call(supervisor, {:start_child, args}) + end + + def start_child(supervisor, child_spec) do + call(supervisor, {:start_child, Supervisor.child_spec(child_spec, [])}) + end @doc """ - Terminates the given pid or child id. + Terminates the given child identified by `child_id`. - If the supervisor is not a `simple_one_for_one`, the child id is expected - and the process, if there is one, is terminated; the child specification is + The process is terminated, if there's one. The child specification is kept unless the child is temporary. - In case of a `simple_one_for_one` supervisor, a pid is expected. If the child - specification identifier is given instead of a `pid`, the function will - return `{:error, :simple_one_for_one}`. - - A non-temporary child process may later be restarted by the supervisor. The child - process can also be restarted explicitly by calling `restart_child/2`. Use - `delete_child/2` to remove the child specification. + A non-temporary child process may later be restarted by the supervisor. + The child process can also be restarted explicitly by calling `restart_child/2`. + Use `delete_child/2` to remove the child specification. - If successful, the function returns `:ok`. If there is no child specification or - pid, the function returns `{:error, :not_found}`. + If successful, this function returns `:ok`. If there is no child + specification for the given child ID, this function returns + `{:error, :not_found}`. """ - @spec terminate_child(supervisor, pid | Supervisor.Spec.child_id) :: :ok | {:error, error} - when error: :not_found | :simple_one_for_one - defdelegate terminate_child(supervisor, pid_or_child_id), to: :supervisor + @spec terminate_child(supervisor, term()) :: :ok | {:error, :not_found} + def terminate_child(supervisor, child_id) + + def terminate_child(supervisor, pid) when is_pid(pid) do + IO.warn( + "Supervisor.terminate_child/2 with a PID is deprecated, please use DynamicSupervisor instead" + ) + + call(supervisor, {:terminate_child, pid}) + end + + def terminate_child(supervisor, child_id) do + call(supervisor, {:terminate_child, child_id}) + end @doc """ Deletes the child specification identified by `child_id`. - The corresponding child process must not be running, use `terminate_child/2` - to terminate it. - - If successful, the function returns `:ok`. This function may error with an - appropriate error tuple if the `child_id` is not found, or if the current - process is running or being restarted. + The corresponding child process must not be running; use `terminate_child/2` + to terminate it if it's running. - This operation is not supported by `simple_one_for_one` supervisors. + If successful, this function returns `:ok`. This function may return an error + with an appropriate error tuple if the `child_id` is not found, or if the + current process is running or being restarted. """ - @spec delete_child(supervisor, Supervisor.Spec.child_id) :: :ok | {:error, error} - when error: :not_found | :simple_one_for_one | :running | :restarting - defdelegate delete_child(supervisor, child_id), to: :supervisor + @spec delete_child(supervisor, term()) :: :ok | {:error, error} + when error: :not_found | :running | :restarting + def delete_child(supervisor, child_id) do + call(supervisor, {:delete_child, child_id}) + end @doc """ Restarts a child process identified by `child_id`. @@ -317,54 +970,54 @@ defmodule Supervisor do Note that for temporary children, the child specification is automatically deleted when the child terminates, and thus it is not possible to restart such children. - If the child process start function returns `{:ok, child}` or - `{:ok, child, info}`, the pid is added to the supervisor and the function returns - the same value. + If the child process start function returns `{:ok, child}` or `{:ok, child, info}`, + the PID is added to the supervisor and this function returns the same value. - If the child process start function returns `:ignore`, the pid remains set to - `:undefined` and the function returns `{:ok, :undefined}`. + If the child process start function returns `:ignore`, the PID remains set to + `:undefined` and this function returns `{:ok, :undefined}`. - This function may error with an appropriate error tuple if the `child_id` is not - found, or if the current process is running or being restarted. + This function may return an error with an appropriate error tuple if the + `child_id` is not found, or if the current process is running or being + restarted. If the child process start function returns an error tuple or an erroneous value, - or if it fails, the function returns `{:error, error}`. - - This operation is not supported by `simple_one_for_one` supervisors. + or if it fails, this function returns `{:error, error}`. """ - @spec restart_child(supervisor, Supervisor.Spec.child_id) :: - {:ok, child} | {:ok, child, term} | {:error, error} - when error: :not_found | :simple_one_for_one | :running | :restarting | term - defdelegate restart_child(supervisor, child_id), to: :supervisor + @spec restart_child(supervisor, term()) :: {:ok, child} | {:ok, child, term} | {:error, error} + when error: :not_found | :running | :restarting | term + def restart_child(supervisor, child_id) do + call(supervisor, {:restart_child, child_id}) + end @doc """ - Returns a list with information about all children. + Returns a list with information about all children of the given supervisor. Note that calling this function when supervising a large number of children under low memory conditions can cause an out of memory exception. - This function returns a list of tuples containing: + This function returns a list of `{id, child, type, modules}` tuples, where: + + * `id` - as defined in the child specification - * `id` - as defined in the child specification or `:undefined` in the case - of a `simple_one_for_one` supervisor + * `child` - the PID of the corresponding child process, `:restarting` if the + process is about to be restarted, or `:undefined` if there is no such + process - * `child` - the pid of the corresponding child process, the atom - `:restarting` if the process is about to be restarted, or `:undefined` if - there is no such process + * `type` - `:worker` or `:supervisor`, as specified by the child specification - * `type` - `:worker` or `:supervisor` as defined in the child specification + * `modules` - as specified by the child specification - * `modules` – as defined in the child specification """ - @spec which_children(supervisor) :: - [{Supervisor.Spec.child_id | :undefined, - child | :restarting, - Supervisor.Spec.worker, - Supervisor.Spec.modules}] - defdelegate which_children(supervisor), to: :supervisor + @spec which_children(supervisor) :: [ + # inlining module() | :dynamic here because :supervisor.modules() is not exported + {term() | :undefined, child | :restarting, :worker | :supervisor, [module()] | :dynamic} + ] + def which_children(supervisor) do + call(supervisor, :which_children) + end @doc """ - Returns a map containing count values for the supervisor. + Returns a map containing count values for the given supervisor. The map contains the following keys: @@ -373,17 +1026,41 @@ defmodule Supervisor do * `:active` - the count of all actively running child processes managed by this supervisor - * `:supervisors` - the count of all supervisors whether or not the child - process is still alive + * `:supervisors` - the count of all supervisors whether or not these + child supervisors are still alive - * `:workers` - the count of all workers, whether or not the child process - is still alive + * `:workers` - the count of all workers, whether or not these child workers + are still alive """ - @spec count_children(supervisor) :: - [specs: non_neg_integer, active: non_neg_integer, - supervisors: non_neg_integer, workers: non_neg_integer] + @spec count_children(supervisor) :: %{ + specs: non_neg_integer, + active: non_neg_integer, + supervisors: non_neg_integer, + workers: non_neg_integer + } def count_children(supervisor) do - :supervisor.count_children(supervisor) |> :maps.from_list + call(supervisor, :count_children) |> :maps.from_list() + end + + @doc """ + Synchronously stops the given supervisor with the given `reason`. + + It returns `:ok` if the supervisor terminates with the given + reason. If it terminates with another reason, the call exits. + + This function keeps OTP semantics regarding error reporting. + If the reason is any other than `:normal`, `:shutdown` or + `{:shutdown, _}`, an error report is logged. + """ + @spec stop(supervisor, reason :: term, timeout) :: :ok + def stop(supervisor, reason \\ :normal, timeout \\ :infinity) do + GenServer.stop(supervisor, reason, timeout) + end + + @compile {:inline, call: 2} + + defp call(supervisor, req) do + GenServer.call(supervisor, req, :infinity) end end diff --git a/lib/elixir/lib/supervisor/default.ex b/lib/elixir/lib/supervisor/default.ex index 863bce65907..fc3a1011809 100644 --- a/lib/elixir/lib/supervisor/default.ex +++ b/lib/elixir/lib/supervisor/default.ex @@ -1,13 +1,7 @@ defmodule Supervisor.Default do @moduledoc false - @behaviour :supervisor - @doc """ - Supevisor callback that simply returns the given args. - - This is the supervisor used by `Supervisor.start_link/2`. - """ def init(args) do args end -end \ No newline at end of file +end diff --git a/lib/elixir/lib/supervisor/spec.ex b/lib/elixir/lib/supervisor/spec.ex index df9f2625b22..61c54d2b2f4 100644 --- a/lib/elixir/lib/supervisor/spec.ex +++ b/lib/elixir/lib/supervisor/spec.ex @@ -1,11 +1,17 @@ defmodule Supervisor.Spec do @moduledoc """ - Convenience functions for defining a supervision specification. + Outdated functions for building child specifications. + + The functions in this module are deprecated and they do not work + with the module-based child specs introduced in Elixir v1.5. + Please see the `Supervisor` documentation instead. + + Convenience functions for defining supervisor specifications. ## Example - By using the functions in this module one can define a supervisor - and start it with `Supervisor.start_link/2`: + By using the functions in this module one can specify the children + to be used under a supervisor, started with `Supervisor.start_link/2`: import Supervisor.Spec @@ -16,7 +22,7 @@ defmodule Supervisor.Spec do Supervisor.start_link(children, strategy: :one_for_one) - In many situations, it may be handy to define supervisors backed + Sometimes, it may be handy to define supervisors backed by a module: defmodule MySupervisor do @@ -35,44 +41,37 @@ defmodule Supervisor.Spec do end end - Notice in this case we don't have to explicitly import - `Supervisor.Spec` as `use Supervisor` automatically does so. - - Explicit supervisors as above are required when there is a need to: - - 1. Partialy change the supervision tree during hot-code swaps. - - 2. Define supervisors inside other supervisors. - - 3. Perform actions inside the supervision `init/1` callback. - - For example, you may want to start an ETS table that is linked to - the supervisor (i.e. if the supervision tree needs to be restarted, - the ETS table must be restarted too). + Note that in this case we don't have to explicitly import + `Supervisor.Spec` since `use Supervisor` automatically does so. + Defining a module-based supervisor can be useful, for example, + to perform initialization tasks in the `c:Supervisor.init/1` callback. ## Supervisor and worker options - In the example above, we have defined workers and supervisors - and each accepts the following options: + In the example above, we defined specs for workers and supervisors. + These specs (both for workers as well as supervisors) accept the + following options: * `:id` - a name used to identify the child specification internally by the supervisor; defaults to the given module - name + name for the child worker/supervisor * `:function` - the function to invoke on the child to start it - * `:restart` - defines when the child process should restart + * `:restart` - an atom that defines when a terminated child process should + be restarted (see the "Restart values" section below) - * `:shutdown` - defines how a child process should be terminated + * `:shutdown` - an atom that defines how a child process should be + terminated (see the "Shutdown values" section below) * `:modules` - it should be a list with one element `[module]`, where module is the name of the callback module only if the child process is a `Supervisor` or `GenServer`; if the child - process is a `GenEvent`, modules should be `:dynamic` + process is a `GenEvent`, `:modules` should be `:dynamic` - ### Restart values + ### Restart values (:restart) - The following restart values are supported: + The following restart values are supported in the `:restart` option: * `:permanent` - the child process is always restarted @@ -80,27 +79,36 @@ defmodule Supervisor.Spec do when the supervisor's strategy is `:rest_for_one` or `:one_for_all`) * `:transient` - the child process is restarted only if it - terminates abnormally, i.e. with another exit reason than + terminates abnormally, i.e., with an exit reason other than `:normal`, `:shutdown` or `{:shutdown, term}` - ### Shutdown values + Note that supervisor that reached maximum restart intensity will exit with `:shutdown` reason. + In this case the supervisor will only restart if its child specification was defined with + the `:restart` option set to `:permanent` (the default). + + ### Shutdown values (`:shutdown`) - The following shutdown values are supported: + The following shutdown values are supported in the `:shutdown` option: * `:brutal_kill` - the child process is unconditionally terminated - using `exit(child, :kill)`. + using `Process.exit(child, :kill)` - * `:infinity` - if the child process is a supervisor, it is a mechanism - to give the subtree enough time to shutdown. It can also be used with - workers with care. + * `:infinity` - if the child process is a supervisor, this is a mechanism + to give the subtree enough time to shut down; it can also be used with + workers with care + + * a non-negative integer - the amount of time in milliseconds + that the supervisor tells the child process to terminate by calling + `Process.exit(child, :shutdown)` and then waits for an exit signal back. + If no exit signal is received within the specified time, + the child process is unconditionally terminated + using `Process.exit(child, :kill)` - * Finally, it can also be any integer meaning that the supervisor tells - the child process to terminate by calling `Process.exit(child, :shutdown)` - and then waits for an exit signal back. If no exit signal is received - within the specified time (in miliseconds), the child process is - unconditionally terminated using `Process.exit(child, :kill)`. """ + @moduledoc deprecated: + "Use the new child specifications outlined in the Supervisor module instead" + @typedoc "Supported strategies" @type strategy :: :simple_one_for_one | :one_for_one | :one_for_all | :rest_for_one @@ -108,7 +116,7 @@ defmodule Supervisor.Spec do @type restart :: :permanent | :transient | :temporary @typedoc "Supported shutdown values" - @type shutdown :: :brutal_kill | :infinity | non_neg_integer + @type shutdown :: timeout | :brutal_kill @typedoc "Supported worker values" @type worker :: :worker | :supervisor @@ -116,26 +124,24 @@ defmodule Supervisor.Spec do @typedoc "Supported module values" @type modules :: :dynamic | [module] - @typedoc "Supported id values" + @typedoc "Supported ID values" @type child_id :: term @typedoc "The supervisor specification" - @type spec :: {child_id, - start_fun :: {module, atom, [term]}, - restart, - shutdown, - worker, - modules} + @type spec :: + {child_id, start_fun :: {module, atom, [term]}, restart, shutdown, worker, modules} @doc """ - Receives a list of children (workers or supervisors) to - supervise and a set of options. + Receives a list of `children` (workers or supervisors) to + supervise and a set of `options`. - Returns a tuple containing the supervisor specification. + Returns a tuple containing the supervisor specification. This tuple can be + used as the return value of the `c:Supervisor.init/1` callback when implementing a + module-based supervisor. ## Examples - supervise children, strategy: :one_for_one + supervise(children, strategy: :one_for_one) ## Options @@ -144,36 +150,52 @@ defmodule Supervisor.Spec do `:simple_one_for_one`. You can learn more about strategies in the `Supervisor` module docs. - * `:max_restarts` - the maximum amount of restarts allowed in - a time frame. Defaults to 5. + * `:max_restarts` - the maximum number of restarts allowed in + a time frame. Defaults to `3`. * `:max_seconds` - the time frame in which `:max_restarts` applies. - Defaults to 5. + Defaults to `5`. - The `:strategy` option is required and by default maximum 5 restarts - are allowed within 5 seconds. Please check the `Supervisor` module for - a complete description of the available strategies. + The `:strategy` option is required and by default a maximum of 3 restarts is + allowed within 5 seconds. Check the `Supervisor` module for a detailed + description of the available strategies. """ - @spec supervise([spec], strategy: strategy, - max_restarts: non_neg_integer, - max_seconds: non_neg_integer) :: {:ok, tuple} + @spec supervise( + [spec], + strategy: strategy, + max_restarts: non_neg_integer, + max_seconds: pos_integer + ) :: {:ok, tuple} + @deprecated "Use the new child specifications outlined in the Supervisor module instead" def supervise(children, options) do unless strategy = options[:strategy] do raise ArgumentError, "expected :strategy option to be given" end - maxR = Keyword.get(options, :max_restarts, 5) + maxR = Keyword.get(options, :max_restarts, 3) maxS = Keyword.get(options, :max_seconds, 5) - assert_unique_ids(Enum.map(children, &elem(&1, 0))) + assert_unique_ids(Enum.map(children, &get_id/1)) {:ok, {{strategy, maxR, maxS}, children}} end - defp assert_unique_ids([id|rest]) do + defp get_id({id, _, _, _, _, _}) do + id + end + + defp get_id(other) do + raise ArgumentError, + "invalid tuple specification given to supervise/2. If you are trying to use " <> + "the map child specification that is part of the Elixir v1.5, use Supervisor.init/2 " <> + "instead of Supervisor.Spec.supervise/2. See the Supervisor module for more information. " <> + "Got: #{inspect(other)}" + end + + defp assert_unique_ids([id | rest]) do if id in rest do raise ArgumentError, - "duplicated id #{inspect id} found in the supervisor specification, " <> - "please explicitly pass the :id option when defining this worker/supervisor" + "duplicated ID #{inspect(id)} found in the supervisor specification, " <> + "please explicitly pass the :id option when defining this worker/supervisor" else assert_unique_ids(rest) end @@ -187,22 +209,32 @@ defmodule Supervisor.Spec do Defines the given `module` as a worker which will be started with the given arguments. - worker ExUnit.Runner, [], restart: :permanent + worker(ExUnit.Runner, [], restart: :permanent) By default, the function `start_link` is invoked on the given module. Overall, the default values for the options are: - [id: module, - function: :start_link, - restart: :permanent, - shutdown: 5000, - modules: [module]] + [ + id: module, + function: :start_link, + restart: :permanent, + shutdown: 5000, + modules: [module] + ] - Check `Supervisor.Spec` module docs for more information on - the options. + See the "Supervisor and worker options" section in the `Supervisor.Spec` module for more + information on the available options. """ - @spec worker(module, [term], [restart: restart, shutdown: shutdown, - id: term, function: atom, modules: modules]) :: spec + @spec worker( + module, + [term], + restart: restart, + shutdown: shutdown, + id: term, + function: atom, + modules: modules + ) :: spec + @deprecated "Use the new child specifications outlined in the Supervisor module instead" def worker(module, args, options \\ []) do child(:worker, module, args, options) end @@ -211,38 +243,47 @@ defmodule Supervisor.Spec do Defines the given `module` as a supervisor which will be started with the given arguments. - supervisor ExUnit.Runner, [], restart: :permanent + supervisor(module, [], restart: :permanent) By default, the function `start_link` is invoked on the given module. Overall, the default values for the options are: - [id: module, - function: :start_link, - restart: :permanent, - shutdown: :infinity, - modules: [module]] + [ + id: module, + function: :start_link, + restart: :permanent, + shutdown: :infinity, + modules: [module] + ] - Check `Supervisor.Spec` module docs for more information on - the options. + See the "Supervisor and worker options" section in the `Supervisor.Spec` module for more + information on the available options. """ - @spec supervisor(module, [term], [restart: restart, shutdown: shutdown, - id: term, function: atom, modules: modules]) :: spec + @spec supervisor( + module, + [term], + restart: restart, + shutdown: shutdown, + id: term, + function: atom, + modules: modules + ) :: spec + @deprecated "Use the new child specifications outlined in the Supervisor module instead" def supervisor(module, args, options \\ []) do options = Keyword.put_new(options, :shutdown, :infinity) child(:supervisor, module, args, options) end defp child(type, module, args, options) do - id = Keyword.get(options, :id, module) - modules = Keyword.get(options, :modules, modules(module)) + id = Keyword.get(options, :id, module) + modules = Keyword.get(options, :modules, modules(module)) function = Keyword.get(options, :function, :start_link) - restart = Keyword.get(options, :restart, :permanent) + restart = Keyword.get(options, :restart, :permanent) shutdown = Keyword.get(options, :shutdown, 5000) - {id, {module, function, args}, - restart, shutdown, type, modules} + {id, {module, function, args}, restart, shutdown, type, modules} end defp modules(GenEvent), do: :dynamic - defp modules(module), do: [module] + defp modules(module), do: [module] end diff --git a/lib/elixir/lib/system.ex b/lib/elixir/lib/system.ex index 6b35d7abf32..625a3fab5dc 100644 --- a/lib/elixir/lib/system.ex +++ b/lib/elixir/lib/system.ex @@ -1,50 +1,186 @@ defmodule System do @moduledoc """ - The System module provides access to variables used or - maintained by the VM and to functions that interact directly + The `System` module provides functions that interact directly with the VM or the host system. + + ## Time + + The `System` module also provides functions that work with time, + returning different times kept by the system with support for + different time units. + + One of the complexities in relying on system times is that they + may be adjusted. For example, when you enter and leave daylight + saving time, the system clock will be adjusted, often adding + or removing one hour. We call such changes "time warps". In + order to understand how such changes may be harmful, imagine + the following code: + + ## DO NOT DO THIS + prev = System.os_time() + # ... execute some code ... + next = System.os_time() + diff = next - prev + + If, while the code is executing, the system clock changes, + some code that executed in 1 second may be reported as taking + over 1 hour! To address such concerns, the VM provides a + monotonic time via `System.monotonic_time/0` which never + decreases and does not leap: + + ## DO THIS + prev = System.monotonic_time() + # ... execute some code ... + next = System.monotonic_time() + diff = next - prev + + Generally speaking, the VM provides three time measurements: + + * `os_time/0` - the time reported by the operating system (OS). This time may be + adjusted forwards or backwards in time with no limitation; + + * `system_time/0` - the VM view of the `os_time/0`. The system time and operating + system time may not match in case of time warps although the VM works towards + aligning them. This time is not monotonic (i.e., it may decrease) + as its behaviour is configured [by the VM time warp + mode](https://www.erlang.org/doc/apps/erts/time_correction.html#Time_Warp_Modes); + + * `monotonic_time/0` - a monotonically increasing time provided + by the Erlang VM. + + The time functions in this module work in the `:native` unit + (unless specified otherwise), which is operating system dependent. Most of + the time, all calculations are done in the `:native` unit, to + avoid loss of precision, with `convert_time_unit/3` being + invoked at the end to convert to a specific time unit like + `:millisecond` or `:microsecond`. See the `t:time_unit/0` type for + more information. + + For a more complete rundown on the VM support for different + times, see the [chapter on time and time + correction](https://www.erlang.org/doc/apps/erts/time_correction.html) + in the Erlang docs. """ - defp strip_re(iodata, pattern) do - :re.replace(iodata, pattern, "", [return: :binary]) + @typedoc """ + The time unit to be passed to functions like `monotonic_time/1` and others. + + The `:second`, `:millisecond`, `:microsecond` and `:nanosecond` time + units controls the return value of the functions that accept a time unit. + + A time unit can also be a strictly positive integer. In this case, it + represents the "parts per second": the time will be returned in `1 / + parts_per_second` seconds. For example, using the `:millisecond` time unit + is equivalent to using `1000` as the time unit (as the time will be returned + in 1/1000 seconds - milliseconds). + """ + @type time_unit :: + :second + | :millisecond + | :microsecond + | :nanosecond + | pos_integer + + @type signal :: + :sigabrt + | :sigalrm + | :sigchld + | :sighup + | :sigquit + | :sigstop + | :sigterm + | :sigtstp + | :sigusr1 + | :sigusr2 + + @vm_signals [:sigquit, :sigterm, :sigusr1] + @os_signals [:sighup, :sigabrt, :sigalrm, :sigusr2, :sigchld, :sigstop, :sigtstp] + @signals @vm_signals ++ @os_signals + + @base_dir :filename.join(__DIR__, "../../..") + @version_file :filename.join(@base_dir, "VERSION") + + defp strip(iodata) do + :re.replace(iodata, "^[\s\r\n\t]+|[\s\r\n\t]+$", "", [:global, return: :binary]) end defp read_stripped(path) do case :file.read_file(path) do {:ok, binary} -> - strip_re(binary, "^\s+|\s+$") - _ -> "" + strip(binary) + + _ -> + "" end end - # Read and strip the version from the `VERSION` file. + # Read and strip the version from the VERSION file. defmacrop get_version do - case read_stripped(:filename.join(__DIR__, "../../../VERSION")) do - "" -> raise RuntimeError, message: "could not read the version number from VERSION" + case read_stripped(@version_file) do + "" -> raise "could not read the version number from VERSION" data -> data end end - # Tries to run `git describe --always --tags`. In the case of success returns - # the most recent tag. If that is not available, tries to read the commit hash - # from .git/HEAD. If that fails, returns an empty string. - defmacrop get_describe do - dirpath = :filename.join(__DIR__, "../../../.git") - case :file.read_file_info(dirpath) do - {:ok, _} -> - if :os.find_executable('git') do - data = :os.cmd('git describe --always --tags') - strip_re(data, "\n") - else - read_stripped(:filename.join(".git", "HEAD")) - end - _ -> "" - end + # Returns OTP version that Elixir was compiled with. + defmacrop get_otp_release do + :erlang.list_to_binary(:erlang.system_info(:otp_release)) + end + + # Tries to run "git rev-parse --short=7 HEAD". In the case of success returns + # the short revision hash. If that fails, returns an empty string. + defmacrop get_revision do + null = + case :os.type() do + {:win32, _} -> 'NUL' + _ -> '/dev/null' + end + + 'git rev-parse --short=7 HEAD 2> ' + |> Kernel.++(null) + |> :os.cmd() + |> strip end + defp revision, do: get_revision() + # Get the date at compilation time. + # Follows https://reproducible-builds.org/specs/source-date-epoch/ defmacrop get_date do - IO.iodata_to_binary :httpd_util.rfc1123_date + unix_epoch = + if source_date_epoch = :os.getenv('SOURCE_DATE_EPOCH') do + try do + List.to_integer(source_date_epoch) + rescue + _ -> nil + end + end + + unix_epoch = unix_epoch || :os.system_time(:second) + + {{year, month, day}, {hour, minute, second}} = + :calendar.gregorian_seconds_to_datetime(unix_epoch + 62_167_219_200) + + "~4..0b-~2..0b-~2..0bT~2..0b:~2..0b:~2..0bZ" + |> :io_lib.format([year, month, day, hour, minute, second]) + |> :erlang.iolist_to_binary() + end + + @doc """ + Returns the endianness. + """ + @spec endianness() :: :little | :big + def endianness do + :erlang.system_info(:endian) + end + + @doc """ + Returns the endianness the system was compiled with. + """ + @endianness :erlang.system_info(:endian) + @spec compiled_endianness() :: :little | :big + def compiled_endianness do + @endianness end @doc """ @@ -52,38 +188,105 @@ defmodule System do Returns Elixir's version as binary. """ - @spec version() :: String.t - def version, do: get_version + @spec version() :: String.t() + def version, do: get_version() @doc """ Elixir build information. - Returns a keyword list with Elixir version, git tag info and compilation date. + Returns a map with the Elixir version, the Erlang/OTP release it was compiled + with, a short Git revision hash and the date and time it was built. + + Every value in the map is a string, and these are: + + * `:build` - the Elixir version, short Git revision hash and + Erlang/OTP release it was compiled with + * `:date` - a string representation of the ISO8601 date and time it was built + * `:otp_release` - OTP release it was compiled with + * `:revision` - short Git revision hash. If Git was not available at building + time, it is set to `""` + * `:version` - the Elixir version + + One should not rely on the specific formats returned by each of those fields. + Instead one should use specialized functions, such as `version/0` to retrieve + the Elixir version and `otp_release/0` to retrieve the Erlang/OTP release. + + ## Examples + + iex> System.build_info() + %{ + build: "1.9.0-dev (772a00a0c) (compiled with Erlang/OTP 21)", + date: "2018-12-24T01:09:21Z", + otp_release: "21", + revision: "772a00a0c", + version: "1.9.0-dev" + } + """ - @spec build_info() :: map + @spec build_info() :: %{ + build: String.t(), + date: String.t(), + revision: String.t(), + version: String.t(), + otp_release: String.t() + } def build_info do - %{version: version, tag: get_describe, date: get_date} + %{ + build: build(), + date: get_date(), + revision: revision(), + version: version(), + otp_release: get_otp_release() + } + end + + # Returns a string of the build info + defp build do + {:ok, v} = Version.parse(version()) + + revision_string = if v.pre != [] and revision() != "", do: " (#{revision()})", else: "" + otp_version_string = " (compiled with Erlang/OTP #{get_otp_release()})" + + version() <> revision_string <> otp_version_string end @doc """ - List command line arguments. + Lists command line arguments. Returns the list of command line arguments passed to the program. """ - @spec argv() :: [String.t] + @spec argv() :: [String.t()] def argv do - :elixir_code_server.call :argv + :elixir_config.get(:argv) end @doc """ - Modify command line arguments. + Modifies command line arguments. Changes the list of command line arguments. Use it with caution, as it destroys any previous argv information. """ - @spec argv([String.t]) :: :ok + @spec argv([String.t()]) :: :ok def argv(args) do - :elixir_code_server.cast({:argv, args}) + :elixir_config.put(:argv, args) + end + + @doc """ + Marks if the system should halt or not at the end of ARGV processing. + """ + @doc since: "1.9.0" + @spec no_halt(boolean) :: :ok + def no_halt(boolean) when is_boolean(boolean) do + :elixir_config.put(:no_halt, boolean) + end + + @doc """ + Checks if the system will halt or not at the end of ARGV processing. + """ + @doc since: "1.9.0" + @spec no_halt() :: boolean + def no_halt() do + :elixir_config.get(:no_halt) end @doc """ @@ -92,9 +295,11 @@ defmodule System do Returns the current working directory or `nil` if one is not available. """ + @deprecated "Use File.cwd/0 instead" + @spec cwd() :: String.t() | nil def cwd do - case :file.get_cwd do - {:ok, base} -> IO.chardata_to_string(base) + case File.cwd() do + {:ok, cwd} -> cwd _ -> nil end end @@ -104,21 +309,32 @@ defmodule System do Returns the current working directory or raises `RuntimeError`. """ + @deprecated "Use File.cwd!/0 instead" + @spec cwd!() :: String.t() def cwd! do - cwd || - raise RuntimeError, message: "could not get a current working directory, the current location is not accessible" + case File.cwd() do + {:ok, cwd} -> + cwd + + _ -> + raise "could not get a current working directory, the current location is not accessible" + end end @doc """ User home directory. Returns the user home directory (platform independent). - Returns `nil` if no user home is set. """ + @spec user_home() :: String.t() | nil def user_home do - case :os.type() do - {:win32, _} -> get_windows_home - _ -> get_unix_home + case :init.get_argument(:home) do + {:ok, [[home] | _]} -> + encoding = :file.native_name_encoding() + :unicode.characters_to_binary(home, encoding, encoding) + + _ -> + nil end end @@ -128,23 +344,9 @@ defmodule System do Same as `user_home/0` but raises `RuntimeError` instead of returning `nil` if no user home is set. """ + @spec user_home!() :: String.t() def user_home! do - user_home || - raise RuntimeError, message: "could not find the user home, please set the HOME environment variable" - end - - defp get_unix_home do - get_env("HOME") - end - - defp get_windows_home do - :filename.absname( - get_env("USERPROFILE") || ( - hd = get_env("HOMEDRIVE") - hp = get_env("HOMEPATH") - hd && hp && hd <> hp - ) - ) + user_home() || raise "could not find the user home, please set the HOME environment variable" end @doc ~S""" @@ -156,17 +358,22 @@ defmodule System do 1. the directory named by the TMPDIR environment variable 2. the directory named by the TEMP environment variable 3. the directory named by the TMP environment variable - 4. `C:\TMP` on Windows or `/tmp` on Unix + 4. `C:\TMP` on Windows or `/tmp` on Unix-like operating systems 5. as a last resort, the current working directory Returns `nil` if none of the above are writable. """ + @spec tmp_dir() :: String.t() | nil def tmp_dir do - write_env_tmp_dir('TMPDIR') || - write_env_tmp_dir('TEMP') || - write_env_tmp_dir('TMP') || - write_tmp_dir('/tmp') || - ((cwd = cwd()) && write_tmp_dir(cwd)) + write_env_tmp_dir('TMPDIR') || write_env_tmp_dir('TEMP') || write_env_tmp_dir('TMP') || + write_tmp_dir('/tmp') || write_cwd_tmp_dir() + end + + defp write_cwd_tmp_dir do + case File.cwd() do + {:ok, cwd} -> write_tmp_dir(cwd) + _ -> nil + end end @doc """ @@ -175,16 +382,16 @@ defmodule System do Same as `tmp_dir/0` but raises `RuntimeError` instead of returning `nil` if no temp dir is set. """ + @spec tmp_dir!() :: String.t() def tmp_dir! do - tmp_dir || - raise RuntimeError, message: "could not get a writable temporary directory, " <> - "please set the TMPDIR environment variable" + tmp_dir() || + raise "could not get a writable temporary directory, please set the TMPDIR environment variable" end defp write_env_tmp_dir(env) do case :os.getenv(env) do false -> nil - tmp -> write_tmp_dir(tmp) + tmp -> write_tmp_dir(tmp) end end @@ -194,135 +401,343 @@ defmodule System do case {stat.type, stat.access} do {:directory, access} when access in [:read_write, :write] -> IO.chardata_to_string(dir) + _ -> nil end - {:error, _} -> nil + + {:error, _} -> + nil end end @doc """ - Register a program exit handler function. + Registers a program exit handler function. + + Registers a function that will be invoked at the end of an Elixir script. + A script is typically started via the command line via the `elixir` and + `mix` executables. - Registers a function that will be invoked - at the end of program execution. Useful for - invoking a hook in "script" mode. + The handler always executes in a different process from the one it was + registered in. As a consequence, any resources managed by the calling process + (ETS tables, open files, and others) won't be available by the time the handler + function is invoked. - The function must receive the exit status code - as an argument. + The function must receive the exit status code as an argument. + + If the VM terminates programmatically, via `System.stop/1`, `System.halt/1`, + or exit signals, the `at_exit/1` callbacks are not executed. """ + @spec at_exit((non_neg_integer -> any)) :: :ok def at_exit(fun) when is_function(fun, 1) do - :elixir_code_server.cast {:at_exit, fun} + :elixir_config.update(:at_exit, &[fun | &1]) + :ok + end + + defmodule SignalHandler do + @moduledoc false + @behaviour :gen_event + + @impl true + def init({event, fun}) do + {:ok, {event, fun}} + end + + @impl true + def handle_call(_message, state) do + {:ok, :ok, state} + end + + @impl true + def handle_event(signal, {event, fun}) do + if signal == event, do: :ok = fun.() + {:ok, {event, fun}} + end + + @impl true + def handle_info(_, {event, fun}) do + {:ok, {event, fun}} + end end @doc """ - Execute a system command. + Traps the given `signal` to execute the `fun`. + + > **Important**: Trapping signals may have strong implications + > on how a system shuts down and behave in production and + > therefore it is extremely discouraged for libraries to + > set their own traps. Instead, they should redirect users + > to configure them themselves. The only cases where it is + > acceptable for libraries to set their own traps is when + > using Elixir in script mode, such as in `.exs` files and + > via Mix tasks. + + An optional `id` that uniquely identifies the function + can be given, otherwise a unique one is automatically + generated. If a previously registered `id` is given, + this function returns an error tuple. The `id` can be + used to remove a registered signal by calling + `untrap_signal/2`. + + The given `fun` receives no arguments and it must return + `:ok`. + + It returns `{:ok, id}` in case of success, + `{:error, :already_registered}` in case the id has already + been registered for the given signal, or `{:error, :not_sup}` + in case trapping exists is not supported by the current OS. + + The first time a signal is trapped, it will override the + default behaviour from the operating system. If the same + signal is trapped multiple times, subsequent functions + given to `trap_signal` will execute *first*. In other + words, you can consider each function is prepended to + the signal handler. + + By default, the Erlang VM register traps to the three + signals: + + * `:sigstop` - gracefully shuts down the VM with `stop/0` + * `:sigquit` - halts the VM via `halt/0` + * `:sigusr1` - halts the VM via status code of 1 + + Therefore, if you add traps to the signals above, the + default behaviour above will be executed after all user + signals. + + ## Implementation notes + + All signals run from a single process. Therefore, blocking the + `fun` will block subsequent traps. It is also not possible to add + or remove traps from within a trap itself. + + Internally, this functionality is built on top of `:os.set_signal/2`. + When you register a trap, Elixir automatically sets it to `:handle` + and it reverts it back to `:default` once all traps are removed + (except for `:sigquit`, `:sigterm`, and `:sigusr1` which are always + handled). If you or a library call `:os.set_signal/2` directly, + it may disable Elixir traps (or Elixir may override your configuration). + """ + @doc since: "1.12.0" + @spec trap_signal(signal, (() -> :ok)) :: {:ok, reference()} | {:error, :not_sup} + @spec trap_signal(signal, id, (() -> :ok)) :: + {:ok, id} | {:error, :already_registered} | {:error, :not_sup} + when id: term() + def trap_signal(signal, id \\ make_ref(), fun) + when signal in @signals and is_function(fun, 0) do + :elixir_config.serial(fn -> + gen_id = {signal, id} + + if {SignalHandler, gen_id} in signal_handlers() do + {:error, :already_registered} + else + try do + :os.set_signal(signal, :handle) + rescue + _ -> {:error, :not_sup} + else + :ok -> + :ok = + :gen_event.add_handler(:erl_signal_server, {SignalHandler, gen_id}, {signal, fun}) - Executes `command` in a command shell of the target OS, - captures the standard output of the command and returns - the result as a binary. + {:ok, id} + end + end + end) + end - If `command` is a char list, a char list is returned. - Otherwise a string, correctly encoded in UTF-8, is expected. + @doc """ + Removes a previously registered `signal` with `id`. """ - @spec cmd(String.t) :: String.t - @spec cmd(char_list) :: char_list + @doc since: "1.12.0" + @spec untrap_signal(signal, id) :: :ok | {:error, :not_found} when id: term + def untrap_signal(signal, id) when signal in @signals do + :elixir_config.serial(fn -> + gen_id = {signal, id} + + case :gen_event.delete_handler(:erl_signal_server, {SignalHandler, gen_id}, :delete) do + :ok -> + if not trapping?(signal) do + :os.set_signal(signal, :default) + end + + :ok + + {:error, :module_not_found} -> + {:error, :not_found} + end + end) + end - def cmd(command) when is_list(command) do - :os.cmd(command) + defp trapping?(signal) do + signal in @vm_signals or + Enum.any?(signal_handlers(), &match?({_, {^signal, _}}, &1)) end - def cmd(command) when is_binary(command) do - List.to_string :os.cmd(String.to_char_list(command)) + defp signal_handlers do + :gen_event.which_handlers(:erl_signal_server) end @doc """ - Locate an executable on the system. + Locates an executable on the system. This function looks up an executable program given - its name using the environment variable PATH on Unix - and Windows. It also considers the proper executable - extension for each OS, so for Windows it will try to + its name using the environment variable PATH on Windows and Unix-like + operating systems. It also considers the proper executable + extension for each operating system, so for Windows it will try to lookup files with `.com`, `.cmd` or similar extensions. - - If `program` is a char list, a char list is returned. - Returns a binary otherwise. """ @spec find_executable(binary) :: binary | nil - @spec find_executable(char_list) :: char_list | nil - - def find_executable(program) when is_list(program) do - :os.find_executable(program) || nil - end - def find_executable(program) when is_binary(program) do - case :os.find_executable(String.to_char_list(program)) do + assert_no_null_byte!(program, "System.find_executable/1") + + case :os.find_executable(String.to_charlist(program)) do false -> nil other -> List.to_string(other) end end + # TODO: Remove this once we require Erlang/OTP 24+ + @compile {:no_warn_undefined, {:os, :env, 0}} + @doc """ - System environment variables. + Returns all system environment variables. - Returns a list of all environment variables. Each variable is given as a - `{name, value}` tuple where both `name` and `value` are strings. + The returned value is a map containing name-value pairs. + Variable names and their values are strings. """ - @spec get_env() :: %{String.t => String.t} + @spec get_env() :: %{optional(String.t()) => String.t()} def get_env do - Enum.into(:os.getenv, %{}, fn var -> - var = IO.chardata_to_string var - [k, v] = String.split var, "=", parts: 2 - {k, v} - end) + if function_exported?(:os, :env, 0) do + Map.new(:os.env(), fn {k, v} -> + {IO.chardata_to_string(k), IO.chardata_to_string(v)} + end) + else + Enum.into(:os.getenv(), %{}, fn var -> + var = IO.chardata_to_string(var) + [k, v] = String.split(var, "=", parts: 2) + {k, v} + end) + end end @doc """ - Environment variable value. + Returns the value of the given environment variable. + + The returned value of the environment variable + `varname` is a string. If the environment variable + is not set, returns the string specified in `default` or + `nil` if none is specified. + + ## Examples + + iex> System.get_env("PORT") + "4000" + + iex> System.get_env("NOT_SET") + nil + + iex> System.get_env("NOT_SET", "4001") + "4001" - Returns the value of the environment variable - `varname` as a binary, or `nil` if the environment - variable is undefined. """ - @spec get_env(binary) :: binary | nil - def get_env(varname) when is_binary(varname) do - case :os.getenv(String.to_char_list(varname)) do - false -> nil + @doc since: "1.9.0" + @spec get_env(String.t(), String.t() | nil) :: String.t() | nil + def get_env(varname, default \\ nil) + when is_binary(varname) and + (is_binary(default) or is_nil(default)) do + case :os.getenv(String.to_charlist(varname)) do + false -> default other -> List.to_string(other) end end + @doc """ + Returns the value of the given environment variable or `:error` if not found. + + If the environment variable `varname` is set, then `{:ok, value}` is returned + where `value` is a string. If `varname` is not set, `:error` is returned. + + ## Examples + + iex> System.fetch_env("PORT") + {:ok, "4000"} + + iex> System.fetch_env("NOT_SET") + :error + + """ + @doc since: "1.9.0" + @spec fetch_env(String.t()) :: {:ok, String.t()} | :error + def fetch_env(varname) when is_binary(varname) do + case :os.getenv(String.to_charlist(varname)) do + false -> :error + other -> {:ok, List.to_string(other)} + end + end + + @doc """ + Returns the value of the given environment variable or raises if not found. + + Same as `get_env/1` but raises instead of returning `nil` when the variable is + not set. + + ## Examples + + iex> System.fetch_env!("PORT") + "4000" + + iex> System.fetch_env!("NOT_SET") + ** (ArgumentError) could not fetch environment variable "NOT_SET" because it is not set + + """ + @doc since: "1.9.0" + @spec fetch_env!(String.t()) :: String.t() + def fetch_env!(varname) when is_binary(varname) do + get_env(varname) || + raise ArgumentError, + "could not fetch environment variable #{inspect(varname)} because it is not set" + end + @doc """ Erlang VM process identifier. Returns the process identifier of the current Erlang emulator in the format most commonly used by the operating system environment. - See http://www.erlang.org/doc/man/os.html#getpid-0 for more info. + For more information, see `:os.getpid/0`. """ + @deprecated "Use System.pid/0 instead" @spec get_pid() :: binary - def get_pid, do: IO.iodata_to_binary(:os.getpid) + def get_pid, do: IO.iodata_to_binary(:os.getpid()) @doc """ - Set an environment variable value. + Sets an environment variable value. Sets a new `value` for the environment variable `varname`. """ @spec put_env(binary, binary) :: :ok def put_env(varname, value) when is_binary(varname) and is_binary(value) do - :os.putenv String.to_char_list(varname), String.to_char_list(value) - :ok + case :binary.match(varname, "=") do + {_, _} -> + raise ArgumentError, + "cannot execute System.put_env/2 for key with \"=\", got: #{inspect(varname)}" + + :nomatch -> + :os.putenv(String.to_charlist(varname), String.to_charlist(value)) + :ok + end end @doc """ - Set multiple environment variables. + Sets multiple environment variables. Sets a new value for each environment variable corresponding - to each key in `dict`. + to each `{key, value}` pair in `enum`. """ - @spec put_env(Dict.t) :: :ok - def put_env(dict) do - Enum.each dict, fn {key, val} -> put_env key, val end + @spec put_env(Enumerable.t()) :: :ok + def put_env(enum) do + Enum.each(enum, fn {key, val} -> put_env(key, val) end) end @doc """ @@ -330,30 +745,30 @@ defmodule System do Removes the variable `varname` from the environment. """ - @spec delete_env(String.t) :: :ok + @spec delete_env(String.t()) :: :ok def delete_env(varname) do - :os.unsetenv(String.to_char_list(varname)) + :os.unsetenv(String.to_charlist(varname)) :ok end @doc """ - Last exception stacktrace. - - Note that the Erlang VM (and therefore this function) does not - return the current stacktrace but rather the stacktrace of the - latest exception. + Deprecated mechanism to retrieve the last exception stacktrace. - Inlined by the compiler into `:erlang.get_stacktrace/0`. + It always return an empty list. """ + @deprecated "Use __STACKTRACE__ instead" def stacktrace do - :erlang.get_stacktrace + [] end @doc """ - Halt the Erlang runtime system. + Immediately halts the Erlang runtime system. - Halts the Erlang runtime system where the argument `status` must be a - non-negative integer, the atom `:abort` or a binary. + Terminates the Erlang runtime system without properly shutting down + applications and ports. Please see `stop/1` for a careful shutdown of the + system. + + `status` must be a non-negative integer, the atom `:abort` or a binary. * If an integer, the runtime system exits with the integer value which is returned to the operating system. @@ -361,13 +776,13 @@ defmodule System do * If `:abort`, the runtime system aborts producing a core dump, if that is enabled in the operating system. - * If a string, an erlang crash dump is produced with status as slogan, + * If a string, an Erlang crash dump is produced with status as slogan, and then the runtime system exits with status code 1. Note that on many platforms, only the status codes 0-255 are supported by the operating system. - For more information, check: http://www.erlang.org/doc/man/erlang.html#halt-1 + For more information, see `:erlang.halt/1`. ## Examples @@ -385,6 +800,530 @@ defmodule System do end def halt(status) when is_binary(status) do - :erlang.halt(String.to_char_list(status)) + :erlang.halt(String.to_charlist(status)) + end + + @doc """ + Returns the operating system PID for the current Erlang runtime system instance. + + Returns a string containing the (usually) numerical identifier for a process. + On Unix-like operating systems, this is typically the return value of the `getpid()` system call. + On Windows, the process ID as returned by the `GetCurrentProcessId()` system + call is used. + + ## Examples + + System.pid() + + """ + @doc since: "1.9.0" + @spec pid :: String.t() + def pid do + List.to_string(:os.getpid()) + end + + @doc """ + Restarts all applications in the Erlang runtime system. + + All applications are taken down smoothly, all code is unloaded, and all ports + are closed before the system starts all applications once again. + + ## Examples + + System.restart() + + """ + @doc since: "1.9.0" + @spec restart :: :ok + defdelegate restart(), to: :init + + @doc """ + Carefully stops the Erlang runtime system. + + All applications are taken down smoothly, all code is unloaded, and all ports + are closed before the system terminates by calling `halt/1`. + + `status` must be a non-negative integer or a binary. + + * If an integer, the runtime system exits with the integer value which is + returned to the operating system. + + * If a binary, an Erlang crash dump is produced with status as slogan, and + then the runtime system exits with status code 1. + + Note that on many platforms, only the status codes 0-255 are supported + by the operating system. + + ## Examples + + System.stop(0) + System.stop(1) + + """ + @doc since: "1.5.0" + @spec stop(non_neg_integer | binary) :: no_return + def stop(status \\ 0) + + def stop(status) when is_integer(status) do + :init.stop(status) + end + + def stop(status) when is_binary(status) do + :init.stop(String.to_charlist(status)) + end + + @doc ~S""" + Executes the given `command` in the OS shell. + + It uses `sh` for Unix-like systems and `cmd` for Windows. + + > **Important:**: Use this function with care. In particular, **never + > pass untrusted user input to this function**, as the user would be + > able to perform "command injection attacks" by executing any code + > directly on the machine. Generally speaking, prefer to use `cmd/3` + > over this function. + + ## Examples + + iex> System.shell("echo hello") + {"hello\n", 0} + + If you want to stream the output to Standard IO as it arrives: + + iex> System.shell("echo hello", into: IO.stream()) + hello + {%IO.Stream{}, 0} + + ## Options + + It accepts the same options as `cmd/3`, except for `arg0`. + """ + @doc since: "1.12.0" + @spec shell(binary, keyword) :: {Collectable.t(), exit_status :: non_neg_integer} + def shell(command, opts \\ []) when is_binary(command) do + assert_no_null_byte!(command, "System.shell/2") + + # Finding shell command logic from :os.cmd in OTP + # https://github.com/erlang/otp/blob/8deb96fb1d017307e22d2ab88968b9ef9f1b71d0/lib/kernel/src/os.erl#L184 + case :os.type() do + {:unix, _} -> + shell_path = :os.find_executable('sh') || :erlang.error(:enoent, [command, opts]) + command = "(#{command}\n) + command = String.to_charlist(command) + + command = + case {System.get_env("COMSPEC"), osname} do + {nil, :windows} -> 'command.com /s /c ' ++ command + {nil, _} -> 'cmd /s /c ' ++ command + {cmd, _} -> '#{cmd} /s /c ' ++ command + end + + do_cmd({:spawn, command}, [], opts) + end + end + + @doc ~S""" + Executes the given `command` with `args`. + + `command` is expected to be an executable available in PATH + unless an absolute path is given. + + `args` must be a list of binaries which the executable will receive + as its arguments as is. This means that: + + * environment variables will not be interpolated + * wildcard expansion will not happen (unless `Path.wildcard/2` is used + explicitly) + * arguments do not need to be escaped or quoted for shell safety + + This function returns a tuple containing the collected result + and the command exit status. + + Internally, this function uses a `Port` for interacting with the + outside world. However, if you plan to run a long-running program, + ports guarantee stdin/stdout devices will be closed but it does not + automatically terminate the program. The documentation for the + `Port` module describes this problem and possible solutions under + the "Zombie processes" section. + + ## Examples + + iex> System.cmd("echo", ["hello"]) + {"hello\n", 0} + + iex> System.cmd("echo", ["hello"], env: [{"MIX_ENV", "test"}]) + {"hello\n", 0} + + If you want to stream the output to Standard IO as it arrives: + + iex> System.cmd("echo", ["hello"], into: IO.stream()) + hello + {%IO.Stream{}, 0} + + ## Options + + * `:into` - injects the result into the given collectable, defaults to `""` + * `:cd` - the directory to run the command in + * `:env` - an enumerable of tuples containing environment key-value as + binary. The child process inherits all environment variables from its + parent process, the Elixir application, except those overwritten or + cleared using this option. Specify a value of `nil` to clear (unset) an + environment variable, which is useful for preventing credentials passed + to the application from leaking into child processes. + * `:arg0` - sets the command arg0 + * `:stderr_to_stdout` - redirects stderr to stdout when `true` + * `:parallelism` - when `true`, the VM will schedule port tasks to improve + parallelism in the system. If set to `false`, the VM will try to perform + commands immediately, improving latency at the expense of parallelism. + The default can be set on system startup by passing the "+spp" argument + to `--erl`. + + ## Error reasons + + If invalid arguments are given, `ArgumentError` is raised by + `System.cmd/3`. `System.cmd/3` also expects a strict set of + options and will raise if unknown or invalid options are given. + + Furthermore, `System.cmd/3` may fail with one of the POSIX reasons + detailed below: + + * `:system_limit` - all available ports in the Erlang emulator are in use + + * `:enomem` - there was not enough memory to create the port + + * `:eagain` - there are no more available operating system processes + + * `:enametoolong` - the external command given was too long + + * `:emfile` - there are no more available file descriptors + (for the operating system process that the Erlang emulator runs in) + + * `:enfile` - the file table is full (for the entire operating system) + + * `:eacces` - the command does not point to an executable file + + * `:enoent` - the command does not point to an existing file + + ## Shell commands + + If you desire to execute a trusted command inside a shell, with pipes, + redirecting and so on, please check `shell/2`. + """ + @spec cmd(binary, [binary], keyword) :: {Collectable.t(), exit_status :: non_neg_integer} + def cmd(command, args, opts \\ []) when is_binary(command) and is_list(args) do + assert_no_null_byte!(command, "System.cmd/3") + + unless Enum.all?(args, &is_binary/1) do + raise ArgumentError, "all arguments for System.cmd/3 must be binaries" + end + + cmd = String.to_charlist(command) + + cmd = + if Path.type(cmd) == :absolute do + cmd + else + :os.find_executable(cmd) || :erlang.error(:enoent, [command, args, opts]) + end + + do_cmd({:spawn_executable, cmd}, [args: args], opts) + end + + defp do_cmd(port_init, base_opts, opts) do + {into, opts} = cmd_opts(opts, [:use_stdio, :exit_status, :binary, :hide] ++ base_opts, "") + {initial, fun} = Collectable.into(into) + + try do + do_port(Port.open(port_init, opts), initial, fun) + catch + kind, reason -> + fun.(initial, :halt) + :erlang.raise(kind, reason, __STACKTRACE__) + else + {acc, status} -> {fun.(acc, :done), status} + end + end + + defp do_port(port, acc, fun) do + receive do + {^port, {:data, data}} -> + do_port(port, fun.(acc, {:cont, data}), fun) + + {^port, {:exit_status, status}} -> + {acc, status} + end + end + + defp cmd_opts([{:into, any} | t], opts, _into), + do: cmd_opts(t, opts, any) + + defp cmd_opts([{:cd, bin} | t], opts, into) when is_binary(bin), + do: cmd_opts(t, [{:cd, bin} | opts], into) + + defp cmd_opts([{:arg0, bin} | t], opts, into) when is_binary(bin), + do: cmd_opts(t, [{:arg0, bin} | opts], into) + + defp cmd_opts([{:stderr_to_stdout, true} | t], opts, into), + do: cmd_opts(t, [:stderr_to_stdout | opts], into) + + defp cmd_opts([{:stderr_to_stdout, false} | t], opts, into), + do: cmd_opts(t, opts, into) + + defp cmd_opts([{:parallelism, bool} | t], opts, into) when is_boolean(bool), + do: cmd_opts(t, [{:parallelism, bool} | opts], into) + + defp cmd_opts([{:env, enum} | t], opts, into), + do: cmd_opts(t, [{:env, validate_env(enum)} | opts], into) + + defp cmd_opts([{key, val} | _], _opts, _into), + do: raise(ArgumentError, "invalid option #{inspect(key)} with value #{inspect(val)}") + + defp cmd_opts([], opts, into), + do: {into, opts} + + defp validate_env(enum) do + Enum.map(enum, fn + {k, nil} -> + {String.to_charlist(k), false} + + {k, v} -> + {String.to_charlist(k), String.to_charlist(v)} + + other -> + raise ArgumentError, "invalid environment key-value #{inspect(other)}" + end) + end + + @doc """ + Returns the current monotonic time in the `:native` time unit. + + This time is monotonically increasing and starts in an unspecified + point in time. + + Inlined by the compiler. + """ + @spec monotonic_time() :: integer + def monotonic_time do + :erlang.monotonic_time() + end + + @doc """ + Returns the current monotonic time in the given time unit. + + This time is monotonically increasing and starts in an unspecified + point in time. + """ + @spec monotonic_time(time_unit) :: integer + def monotonic_time(unit) do + :erlang.monotonic_time(normalize_time_unit(unit)) + end + + @doc """ + Returns the current system time in the `:native` time unit. + + It is the VM view of the `os_time/0`. They may not match in + case of time warps although the VM works towards aligning + them. This time is not monotonic. + + Inlined by the compiler. + """ + @spec system_time() :: integer + def system_time do + :erlang.system_time() + end + + @doc """ + Returns the current system time in the given time unit. + + It is the VM view of the `os_time/0`. They may not match in + case of time warps although the VM works towards aligning + them. This time is not monotonic. + """ + @spec system_time(time_unit) :: integer + def system_time(unit) do + :erlang.system_time(normalize_time_unit(unit)) + end + + @doc """ + Converts `time` from time unit `from_unit` to time unit `to_unit`. + + The result is rounded via the floor function. + + `convert_time_unit/3` accepts an additional time unit (other than the + ones in the `t:time_unit/0` type) called `:native`. `:native` is the time + unit used by the Erlang runtime system. It's determined when the runtime + starts and stays the same until the runtime is stopped, but could differ + the next time the runtime is started on the same machine. For this reason, + you should use this function to convert `:native` time units to a predictable + unit before you display them to humans. + + To determine how many seconds the `:native` unit represents in your current + runtime, you can call this function to convert 1 second to the `:native` + time unit: `System.convert_time_unit(1, :second, :native)`. + """ + @spec convert_time_unit(integer, time_unit | :native, time_unit | :native) :: integer + def convert_time_unit(time, from_unit, to_unit) do + :erlang.convert_time_unit(time, normalize_time_unit(from_unit), normalize_time_unit(to_unit)) + end + + @doc """ + Returns the current time offset between the Erlang VM monotonic + time and the Erlang VM system time. + + The result is returned in the `:native` time unit. + + See `time_offset/1` for more information. + + Inlined by the compiler. + """ + @spec time_offset() :: integer + def time_offset do + :erlang.time_offset() + end + + @doc """ + Returns the current time offset between the Erlang VM monotonic + time and the Erlang VM system time. + + The result is returned in the given time unit `unit`. The returned + offset, added to an Erlang monotonic time (for instance, one obtained with + `monotonic_time/1`), gives the Erlang system time that corresponds + to that monotonic time. + """ + @spec time_offset(time_unit) :: integer + def time_offset(unit) do + :erlang.time_offset(normalize_time_unit(unit)) + end + + @doc """ + Returns the current operating system (OS) time. + + The result is returned in the `:native` time unit. + + This time may be adjusted forwards or backwards in time + with no limitation and is not monotonic. + + Inlined by the compiler. + """ + @spec os_time() :: integer + @doc since: "1.3.0" + def os_time do + :os.system_time() + end + + @doc """ + Returns the current operating system (OS) time in the given time `unit`. + + This time may be adjusted forwards or backwards in time + with no limitation and is not monotonic. + """ + @spec os_time(time_unit) :: integer + @doc since: "1.3.0" + def os_time(unit) do + :os.system_time(normalize_time_unit(unit)) + end + + @doc """ + Returns the Erlang/OTP release number. + """ + @spec otp_release :: String.t() + @doc since: "1.3.0" + def otp_release do + :erlang.list_to_binary(:erlang.system_info(:otp_release)) + end + + @doc """ + Returns the number of schedulers in the VM. + """ + @spec schedulers :: pos_integer + @doc since: "1.3.0" + def schedulers do + :erlang.system_info(:schedulers) + end + + @doc """ + Returns the number of schedulers online in the VM. + """ + @spec schedulers_online :: pos_integer + @doc since: "1.3.0" + def schedulers_online do + :erlang.system_info(:schedulers_online) + end + + @doc """ + Generates and returns an integer that is unique in the current runtime + instance. + + "Unique" means that this function, called with the same list of `modifiers`, + will never return the same integer more than once on the current runtime + instance. + + If `modifiers` is `[]`, then a unique integer (that can be positive or negative) is returned. + Other modifiers can be passed to change the properties of the returned integer: + + * `:positive` - the returned integer is guaranteed to be positive. + * `:monotonic` - the returned integer is monotonically increasing. This + means that, on the same runtime instance (but even on different + processes), integers returned using the `:monotonic` modifier will always + be strictly less than integers returned by successive calls with the + `:monotonic` modifier. + + All modifiers listed above can be combined; repeated modifiers in `modifiers` + will be ignored. + + Inlined by the compiler. + """ + @spec unique_integer([:positive | :monotonic]) :: integer + def unique_integer(modifiers \\ []) do + :erlang.unique_integer(modifiers) + end + + defp assert_no_null_byte!(binary, operation) do + case :binary.match(binary, "\0") do + {_, _} -> + raise ArgumentError, + "cannot execute #{operation} for program with null byte, got: #{inspect(binary)}" + + :nomatch -> + binary + end + end + + defp normalize_time_unit(:native), do: :native + + defp normalize_time_unit(:second), do: :second + defp normalize_time_unit(:millisecond), do: :millisecond + defp normalize_time_unit(:microsecond), do: :microsecond + defp normalize_time_unit(:nanosecond), do: :nanosecond + + defp normalize_time_unit(:seconds), do: warn(:seconds, :second) + defp normalize_time_unit(:milliseconds), do: warn(:milliseconds, :millisecond) + defp normalize_time_unit(:microseconds), do: warn(:microseconds, :microsecond) + defp normalize_time_unit(:nanoseconds), do: warn(:nanoseconds, :nanosecond) + + defp normalize_time_unit(:milli_seconds), do: warn(:milli_seconds, :millisecond) + defp normalize_time_unit(:micro_seconds), do: warn(:micro_seconds, :microsecond) + defp normalize_time_unit(:nano_seconds), do: warn(:nano_seconds, :nanosecond) + + defp normalize_time_unit(unit) when is_integer(unit) and unit > 0, do: unit + + defp normalize_time_unit(other) do + raise ArgumentError, + "unsupported time unit. Expected :second, :millisecond, " <> + ":microsecond, :nanosecond, or a positive integer, " <> "got #{inspect(other)}" + end + + defp warn(unit, replacement_unit) do + IO.warn_once( + {__MODULE__, unit}, + "deprecated time unit: #{inspect(unit)}. A time unit should be " <> + ":second, :millisecond, :microsecond, :nanosecond, or a positive integer", + _stacktrace_drop_levels = 4 + ) + + replacement_unit end end diff --git a/lib/elixir/lib/task.ex b/lib/elixir/lib/task.ex index 0ba6df0cf89..51d6c11651f 100644 --- a/lib/elixir/lib/task.ex +++ b/lib/elixir/lib/task.ex @@ -1,172 +1,829 @@ defmodule Task do @moduledoc """ - Conveniences for spawning and awaiting for tasks. + Conveniences for spawning and awaiting tasks. Tasks are processes meant to execute one particular - action throughout their life-cycle, often with little or no + action throughout their lifetime, often with little or no communication with other processes. The most common use case - for tasks is to compute a value asynchronously: + for tasks is to convert sequential code into concurrent code + by computing a value asynchronously: task = Task.async(fn -> do_some_work() end) - res = do_some_other_work() + res = do_some_other_work() res + Task.await(task) - Tasks spawned with `async` can be awaited on by its caller - process (and only its caller) as shown in the example above. + Tasks spawned with `async` can be awaited on by their caller + process (and only their caller) as shown in the example above. They are implemented by spawning a process that sends a message to the caller once the given computation is performed. - Besides `async/1` and `await/2`, tasks can also be - started as part of supervision trees and dynamically spawned - in remote nodes. We will explore all three scenarios next. + Besides `async/1` and `await/2`, tasks can also be + started as part of a supervision tree and dynamically spawned + on remote nodes. We will explore these scenarios next. ## async and await - The most common way to spawn a task is with `Task.async/1`. A new - process will be created, linked and monitored by the caller. Once - the task action finishes, a message will be sent to the caller - with the result. + One of the common uses of tasks is to convert sequential code + into concurrent code with `Task.async/1` while keeping its semantics. + When invoked, a new process will be created, linked and monitored + by the caller. Once the task action finishes, a message will be sent + to the caller with the result. - `Task.await/2` is used to read the message sent by the task. On - `await`, Elixir will also setup a monitor to verify if the process - exited for any abnormal reason (or in case exits are being - trapped by the caller). + `Task.await/2` is used to read the message sent by the task. - ## Supervised tasks + There are two important things to consider when using `async`: - It is also possible to spawn a task inside a supervision tree - with `start_link/1` and `start_link/3`: + 1. If you are using async tasks, you **must await** a reply + as they are *always* sent. If you are not expecting a reply, + consider using `Task.start_link/1` as detailed below. - Task.start_link(fn -> IO.puts "ok" end) + 2. async tasks link the caller and the spawned process. This + means that, if the caller crashes, the task will crash + too and vice-versa. This is on purpose: if the process + meant to receive the result no longer exists, there is + no purpose in completing the computation. - Such tasks can be mounted in your supervision tree as: + If this is not desired, you will want to use supervised + tasks, described next. - import Supervisor.Spec + ## Dynamically supervised tasks - children = [ - worker(Task, [fn -> IO.puts "ok" end]) - ] + The `Task.Supervisor` module allows developers to dynamically + create multiple supervised tasks. - Since these tasks are supervised and not directly linked to - the caller, they cannot be awaited on. Note `start_link/1`, - unlike `async/1`, returns `{:ok, pid}` (which is - the result expected by supervision trees). + A short example is: - ## Supervision trees + {:ok, pid} = Task.Supervisor.start_link() - The `Task.Supervisor` module allows developers to start supervisors - that dynamically supervise tasks: + task = + Task.Supervisor.async(pid, fn -> + # Do something + end) - {:ok, pid} = Task.Supervisor.start_link() - Task.Supervisor.async(pid, fn -> do_work() end) + Task.await(task) - `Task.Supervisor` also makes it possible to spawn tasks in remote nodes as - long as the supervisor is registered locally or globally: + However, in the majority of cases, you want to add the task supervisor + to your supervision tree: - # In the remote node - Task.Supervisor.start_link(name: :tasks_sup) + Supervisor.start_link([ + {Task.Supervisor, name: MyApp.TaskSupervisor} + ], strategy: :one_for_one) - # In the client - Task.Supervisor.async({:tasks_sup, :remote@local}, fn -> do_work() end) + And now you can use async/await by passing the name of + the supervisor instead of the pid: - `Task.Supervisor` is more often started in your supervision tree as: + Task.Supervisor.async(MyApp.TaskSupervisor, fn -> + # Do something + end) + |> Task.await() - import Supervisor.Spec + We encourage developers to rely on supervised tasks as much as possible. + Supervised tasks improves the visibility of how many tasks are running + at a given moment and enable a huge variety of patterns that gives you + explicit control on how to handle the results, errors, and timeouts. + Here is a summary: - children = [ - supervisor(Task.Supervisor, [[name: :tasks_sup]]) - ] + * Using `Task.Supervisor.start_child/2` allows you to start a fire-and-forget + task that you don't care about its results or if it completes successfully or not. + + * Using `Task.Supervisor.async/2` + `Task.await/2` allows you to execute + tasks concurrently and retrieve its result. If the task fails, + the caller will also fail. + + * Using `Task.Supervisor.async_nolink/2` + `Task.yield/2` + `Task.shutdown/2` + allows you to execute tasks concurrently and retrieve their results + or the reason they failed within a given time frame. If the task fails, + the caller won't fail. You will receive the error reason either on + `yield` or `shutdown`. + + Furthermore, the supervisor guarantee all tasks first terminate, within a + configurable shutdown period, when your application shuts down. See the + `Task.Supervisor` module for details on the supported operations. + + ### Distributed tasks + + With `Task.Supervisor`, it is easy to dynamically start tasks across nodes: + + # On the remote node named :remote@local + Task.Supervisor.start_link(name: MyApp.DistSupervisor) + + # On the client + supervisor = {MyApp.DistSupervisor, :remote@local} + Task.Supervisor.async(supervisor, MyMod, :my_fun, [arg1, arg2, arg3]) + + Note that, when working with distributed tasks, one should use the + `Task.Supervisor.async/5` function that expects explicit module, function, + and arguments, instead of `Task.Supervisor.async/3` that works with anonymous + functions. That's because anonymous functions expect the same module version + to exist on all involved nodes. Check the `Agent` module documentation for + more information on distributed processes as the limitations described there + apply to the whole ecosystem. + + ## Statically supervised tasks + + The `Task` module implements the `child_spec/1` function, which + allows it to be started directly under a regular `Supervisor` - + instead of a `Task.Supervisor` - by passing a tuple with a function + to run: + + Supervisor.start_link([ + {Task, fn -> :some_work end} + ], strategy: :one_for_one) + + This is often useful when you need to execute some steps while + setting up your supervision tree. For example: to warm up caches, + log the initialization status, and such. + + If you don't want to put the Task code directly under the `Supervisor`, + you can wrap the `Task` in its own module, similar to how you would + do with a `GenServer` or an `Agent`: + + defmodule MyTask do + use Task + + def start_link(arg) do + Task.start_link(__MODULE__, :run, [arg]) + end + + def run(arg) do + # ... + end + end + + And then passing it to the supervisor: - Check `Task.Supervisor` for other operations supported by the Task supervisor. + Supervisor.start_link([ + {MyTask, arg} + ], strategy: :one_for_one) + + Since these tasks are supervised and not directly linked to the caller, + they cannot be awaited on. By default, the functions `Task.start/1` + and `Task.start_link/1` are for fire-and-forget tasks, where you don't + care about the results or if it completes successfully or not. + + `use Task` defines a `child_spec/1` function, allowing the + defined module to be put under a supervision tree. The generated + `child_spec/1` can be customized with the following options: + + * `:id` - the child specification identifier, defaults to the current module + * `:restart` - when the child should be restarted, defaults to `:temporary` + * `:shutdown` - how to shut down the child, either immediately or by giving it time to shut down + + Opposite to `GenServer`, `Agent` and `Supervisor`, a Task has + a default `:restart` of `:temporary`. This means the task will + not be restarted even if it crashes. If you desire the task to + be restarted for non-successful exits, do: + + use Task, restart: :transient + + If you want the task to always be restarted: + + use Task, restart: :permanent + + See the "Child specification" section in the `Supervisor` module + for more detailed information. The `@doc` annotation immediately + preceding `use Task` will be attached to the generated `child_spec/1` + function. + + ## Ancestor and Caller Tracking + + Whenever you start a new process, Elixir annotates the parent of that process + through the `$ancestors` key in the process dictionary. This is often used to + track the hierarchy inside a supervision tree. + + For example, we recommend developers to always start tasks under a supervisor. + This provides more visibility and allows you to control how those tasks are + terminated when a node shuts down. That might look something like + `Task.Supervisor.start_child(MySupervisor, task_function)`. This means + that, although your code is the one who invokes the task, the actual ancestor of + the task is the supervisor, as the supervisor is the one effectively starting it. + + To track the relationship between your code and the task, we use the `$callers` + key in the process dictionary. Therefore, assuming the `Task.Supervisor` call + above, we have: + + [your code] -- calls --> [supervisor] ---- spawns --> [task] + + Which means we store the following relationships: + + [your code] [supervisor] <-- ancestor -- [task] + ^ | + |--------------------- caller ---------------------| + + The list of callers of the current process can be retrieved from the Process + dictionary with `Process.get(:"$callers")`. This will return either `nil` or + a list `[pid_n, ..., pid2, pid1]` with at least one entry Where `pid_n` is + the PID that called the current process, `pid2` called `pid_n`, and `pid2` was + called by `pid1`. + + If a task crashes, the callers field is included as part of the log message + metadata under the `:callers` key. """ @doc """ The Task struct. - It contains two fields: + It contains these fields: + + * `:mfa` - a three-element tuple containing the module, function name, + and arity invoked to start the task in `async/1` and `async/3` - * `:pid` - the process reference of the task process; it may be a pid - or a tuple containing the process and node names + * `:owner` - the PID of the process that started the task + + * `:pid` - the PID of the task process; `nil` if the task does + not use a task process * `:ref` - the task monitor reference """ - defstruct pid: nil, ref: nil + @enforce_keys [:mfa, :owner, :pid, :ref] + defstruct @enforce_keys + + @typedoc """ + The Task type. + + See [`%Task{}`](`__struct__/0`) for information about each field of the structure. + """ + @type t :: %__MODULE__{ + mfa: mfa(), + owner: pid(), + pid: pid() | nil, + ref: reference() + } + + defguardp is_timeout(timeout) + when timeout == :infinity or (is_integer(timeout) and timeout >= 0) @doc """ - Starts a task as part of a supervision tree. + Returns a specification to start a task under a supervisor. + + `arg` is passed as the argument to `Task.start_link/1` in the `:start` field + of the spec. + + For more information, see the `Supervisor` module, + the `Supervisor.child_spec/2` function and the `t:Supervisor.child_spec/0` type. """ - @spec start_link(fun) :: {:ok, pid} - def start_link(fun) do + @doc since: "1.5.0" + @spec child_spec(term) :: Supervisor.child_spec() + def child_spec(arg) do + %{ + id: Task, + start: {Task, :start_link, [arg]}, + restart: :temporary + } + end + + @doc false + defmacro __using__(opts) do + quote location: :keep, bind_quoted: [opts: opts] do + unless Module.has_attribute?(__MODULE__, :doc) do + @doc """ + Returns a specification to start this module under a supervisor. + + `arg` is passed as the argument to `Task.start_link/1` in the `:start` field + of the spec. + + For more information, see the `Supervisor` module, + the `Supervisor.child_spec/2` function and the `t:Supervisor.child_spec/0` type. + """ + end + + def child_spec(arg) do + default = %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [arg]}, + restart: :temporary + } + + Supervisor.child_spec(default, unquote(Macro.escape(opts))) + end + + defoverridable child_spec: 1 + end + end + + @doc """ + Starts a task as part of a supervision tree with the given `fun`. + + `fun` must be a zero-arity anonymous function. + + This is used to start a statically supervised task under a supervision tree. + """ + @spec start_link((() -> any)) :: {:ok, pid} + def start_link(fun) when is_function(fun, 0) do start_link(:erlang, :apply, [fun, []]) end @doc """ - Starts a task as part of a supervision tree. + Starts a task as part of a supervision tree with the given + `module`, `function`, and `args`. + + This is used to start a statically supervised task under a supervision tree. """ @spec start_link(module, atom, [term]) :: {:ok, pid} - def start_link(mod, fun, args) do - Task.Supervised.start_link(get_info(self), {mod, fun, args}) + def start_link(module, function, args) + when is_atom(module) and is_atom(function) and is_list(args) do + mfa = {module, function, args} + Task.Supervised.start_link(get_owner(self()), get_callers(self()), mfa) end @doc """ - Starts a task that can be awaited on. + Starts a task. + + `fun` must be a zero-arity anonymous function. + + This should only used when the task is used for side-effects + (like I/O) and you have no interest on its results nor if it + completes successfully. + + If the current node is shutdown, the node will terminate even + if the task was not completed. For this reason, we recommend + to use `Task.Supervisor.start_child/2` instead, which allows + you to control the shutdown time via the `:shutdown` option. + """ + @spec start((() -> any)) :: {:ok, pid} + def start(fun) when is_function(fun, 0) do + start(:erlang, :apply, [fun, []]) + end + + @doc """ + Starts a task. + + This should only used when the task is used for side-effects + (like I/O) and you have no interest on its results nor if it + completes successfully. + + If the current node is shutdown, the node will terminate even + if the task was not completed. For this reason, we recommend + to use `Task.Supervisor.start_child/2` instead, which allows + you to control the shutdown time via the `:shutdown` option. + """ + @spec start(module, atom, [term]) :: {:ok, pid} + def start(module, function_name, args) + when is_atom(module) and is_atom(function_name) and is_list(args) do + mfa = {module, function_name, args} + Task.Supervised.start(get_owner(self()), get_callers(self()), mfa) + end + + @doc """ + Starts a task that must be awaited on. + + `fun` must be a zero-arity anonymous function. This function + spawns a process that is linked to and monitored by the caller + process. A `Task` struct is returned containing the relevant + information. Developers must eventually call `Task.await/2` or + `Task.yield/2` followed by `Task.shutdown/2` on the returned task. + + Read the `Task` module documentation for more information about + the general usage of async tasks. + + ## Linking This function spawns a process that is linked to and monitored - by the caller process. A `Task` struct is returned containing - the relevant information. + by the caller process. The linking part is important because it + aborts the task if the parent process dies. It also guarantees + the code before async/await has the same properties after you + add the async call. For example, imagine you have this: + + x = heavy_fun() + y = some_fun() + x + y - ## Task's message format + Now you want to make the `heavy_fun()` async: - The reply sent by the task will be in the format `{ref, msg}`, - where `ref` is the monitoring reference held by the task. + x = Task.async(&heavy_fun/0) + y = some_fun() + Task.await(x) + y + + As before, if `heavy_fun/0` fails, the whole computation will + fail, including the caller process. If you don't want the task + to fail then you must change the `heavy_fun/0` code in the + same way you would achieve it if you didn't have the async call. + For example, to either return `{:ok, val} | :error` results or, + in more extreme cases, by using `try/rescue`. In other words, + an asynchronous task should be thought of as an extension of the + caller process rather than a mechanism to isolate it from all errors. + + If you don't want to link the caller to the task, then you + must use a supervised task with `Task.Supervisor` and call + `Task.Supervisor.async_nolink/2`. + + In any case, avoid any of the following: + + * Setting `:trap_exit` to `true` - trapping exits should be + used only in special circumstances as it would make your + process immune to not only exits from the task but from + any other processes. + + Moreover, even when trapping exits, calling `await` will + still exit if the task has terminated without sending its + result back. + + * Unlinking the task process started with `async`/`await`. + If you unlink the processes and the task does not belong + to any supervisor, you may leave dangling tasks in case + the caller process dies. + + ## Metadata + + The task created with this function stores `:erlang.apply/2` in + its `:mfa` metadata field, which is used internally to apply + the anonymous function. Use `async/3` if you want another function + to be used as metadata. """ - @spec async(fun) :: t - def async(fun) do + @spec async((() -> any)) :: t + def async(fun) when is_function(fun, 0) do async(:erlang, :apply, [fun, []]) end + # TODO: Remove conditional on Erlang/OTP 24 + @compile {:no_warn_undefined, {:erlang, :monitor, 3}} + @doc """ - Starts a task that can be awaited on. + Starts a task that must be awaited on. - Similar to `async/1`, but the task is specified by the given - module, function and arguments. + Similar to `async/1` except the function to be started is + specified by the given `module`, `function_name`, and `args`. + The `module`, `function_name`, and its arity are stored as + a tuple in the `:mfa` field for reflection purposes. """ @spec async(module, atom, [term]) :: t - def async(mod, fun, args) do - mfa = {mod, fun, args} - pid = :proc_lib.spawn_link(Task.Supervised, :async, [self, get_info(self), mfa]) - ref = Process.monitor(pid) - send(pid, {self(), ref}) - %Task{pid: pid, ref: ref} + def async(module, function_name, args) + when is_atom(module) and is_atom(function_name) and is_list(args) do + mfargs = {module, function_name, args} + owner = self() + {:ok, pid} = Task.Supervised.start_link(get_owner(owner), :nomonitor) + + {reply_to, ref} = + if function_exported?(:erlang, :monitor, 3) do + ref = :erlang.monitor(:process, pid, alias: :demonitor) + {ref, ref} + else + {owner, Process.monitor(pid)} + end + + send(pid, {owner, ref, reply_to, get_callers(owner), mfargs}) + %Task{pid: pid, ref: ref, owner: owner, mfa: {module, function_name, length(args)}} end - defp get_info(self) do - {node(), - case Process.info(self, :registered_name) do - {:registered_name, []} -> self() - {:registered_name, name} -> name - end} + @doc """ + Starts a task that immediately completes with the given `result`. + + Unlike `async/1`, this task does not spawn a linked process. It can + be awaited or yielded like any other task. + + ## Usage + + In some cases, it is useful to create a "completed" task that represents + a task that has already run and generated a result. For example, when + processing data you may be able to determine that certain inputs are + invalid before dispatching them for further processing: + + def process(data) do + tasks = + for entry <- data do + if invalid_input?(entry) do + Task.completed({:error, :invalid_input}) + else + Task.async(fn -> further_process(entry) end) + end + end + + Task.await_many(tasks) + end + + In many cases, `Task.completed/1` may be avoided in favor of returning the + result directly. You should generally only require this variant when working + with mixed asynchrony, when a group of inputs will be handled partially + synchronously and partially asynchronously. + """ + @doc since: "1.13.0" + @spec completed(any) :: t + def completed(result) do + ref = make_ref() + owner = self() + + # "complete" the task immediately + send(owner, {ref, result}) + + %Task{pid: nil, ref: ref, owner: owner, mfa: {Task, :completed, 1}} end @doc """ - Awaits for a task reply. + Returns a stream where the given function (`module` and `function_name`) + is mapped concurrently on each element in `enumerable`. + + Each element of `enumerable` will be prepended to the given `args` and + processed by its own task. Those tasks will be linked to an intermediate + process that is then linked to the caller process. This means a failure + in a task terminates the caller process and a failure in the caller + process terminates all tasks. + + When streamed, each task will emit `{:ok, value}` upon successful + completion or `{:exit, reason}` if the caller is trapping exits. + It's possible to have `{:exit, {element, reason}}` for exits + using the `:zip_input_on_exit` option. The order of results depends + on the value of the `:ordered` option. + + The level of concurrency and the time tasks are allowed to run can + be controlled via options (see the "Options" section below). + + Consider using `Task.Supervisor.async_stream/6` to start tasks + under a supervisor. If you find yourself trapping exits to ensure + errors in the tasks do not terminate the caller process, consider + using `Task.Supervisor.async_stream_nolink/6` to start tasks that + are not linked to the caller process. + + ## Options + + * `:max_concurrency` - sets the maximum number of tasks to run + at the same time. Defaults to `System.schedulers_online/0`. + + * `:ordered` - whether the results should be returned in the same order + as the input stream. When the output is ordered, Elixir may need to + buffer results to emit them in the original order. Setting this option + to false disables the need to buffer at the cost of removing ordering. + This is also useful when you're using the tasks only for the side effects. + Note that regardless of what `:ordered` is set to, the tasks will + process asynchronously. If you need to process elements in order, + consider using `Enum.map/2` or `Enum.each/2` instead. Defaults to `true`. + + * `:timeout` - the maximum amount of time (in milliseconds or `:infinity`) + each task is allowed to execute for. Defaults to `5000`. + + * `:on_timeout` - what to do when a task times out. The possible + values are: + * `:exit` (default) - the caller (the process that spawned the tasks) exits. + * `:kill_task` - the task that timed out is killed. The value + emitted for that task is `{:exit, :timeout}`. + + * `:zip_input_on_exit` - (since v1.14.0) adds the original + input to `:exit` tuples. The value emitted for that task is + `{:exit, {input, reason}}`, where `input` is the collection element + that caused an exited during processing. Defaults to `false`. + + ## Example + + Let's build a stream and then enumerate it: + + stream = Task.async_stream(collection, Mod, :expensive_fun, []) + Enum.to_list(stream) + + The concurrency can be increased or decreased using the `:max_concurrency` + option. For example, if the tasks are IO heavy, the value can be increased: + + max_concurrency = System.schedulers_online() * 2 + stream = Task.async_stream(collection, Mod, :expensive_fun, [], max_concurrency: max_concurrency) + Enum.to_list(stream) + + If you do not care about the results of the computation, you can run + the stream with `Stream.run/1`. Also set `ordered: false`, as you don't + care about the order of the results either: + + stream = Task.async_stream(collection, Mod, :expensive_fun, [], ordered: false) + Stream.run(stream) + + ## First async tasks to complete + + You can also use `async_stream/3` to execute M tasks and find the N tasks + to complete. For example: + + [ + &heavy_call_1/0, + &heavy_call_2/0, + &heavy_call_3/0 + ] + |> Task.async_stream(fn fun -> fun.() end, ordered: false, max_concurrency: 3) + |> Stream.filter(&match?({:ok, _}, &1)) + |> Enum.take(2) + + In the example above, we are executing three tasks and waiting for the + first 2 to complete. We use `Stream.filter/2` to restrict ourselves only + to successfully completed tasks, and then use `Enum.take/2` to retrieve + N items. Note it is important to set both `ordered: false` and + `max_concurrency: M`, where M is the number of tasks, to make sure all + calls execute concurrently. + + ### Attention: unbound async + take + + If you want to potentially process a high number of items and keep only + part of the results, you may end-up processing more items than desired. + Let's see an example: + + 1..100 + |> Task.async_stream(fn i -> + Process.sleep(100) + IO.puts(to_string(i)) + end) + |> Enum.take(10) + + Running the example above in a machine with 8 cores will process 16 items, + even though you want only 10 elements, since `async_stream/3` process items + concurrently. That's because it will process 8 elements at once. Then all 8 + elements complete at roughly the same time, causing 8 elements to be kicked + off for processing. Out of these extra 8, only 2 will be used, and the rest + will be terminated. + + Depending on the problem, you can filter or limit the number of elements + upfront: + + 1..100 + |> Stream.take(10) + |> Task.async_stream(fn i -> + Process.sleep(100) + IO.puts(to_string(i)) + end) + |> Enum.to_list() + + In other cases, you likely want to tweak `:max_concurrency` to limit how + many elements may be over processed at the cost of reducing concurrency. + You can also set the number of elements to take to be a multiple of + `:max_concurrency`. For instance, setting `max_concurrency: 5` in the + example above. + """ + @doc since: "1.4.0" + @spec async_stream(Enumerable.t(), module, atom, [term], keyword) :: Enumerable.t() + def async_stream(enumerable, module, function_name, args, options \\ []) + when is_atom(module) and is_atom(function_name) and is_list(args) do + build_stream(enumerable, {module, function_name, args}, options) + end + + @doc """ + Returns a stream that runs the given function `fun` concurrently + on each element in `enumerable`. + + Works the same as `async_stream/5` but with an anonymous function instead of a + module-function-arguments tuple. `fun` must be a one-arity anonymous function. + + Each `enumerable` element is passed as argument to the given function `fun` and + processed by its own task. The tasks will be linked to the caller process, similarly + to `async/1`. - A timeout, in milliseconds, can be given with default value - of `5000`. In case the task process dies, this function will - exit with the same reason as the task. + ## Example + + Count the code points in each string asynchronously, then add the counts together using reduce. + + iex> strings = ["long string", "longer string", "there are many of these"] + iex> stream = Task.async_stream(strings, fn text -> text |> String.codepoints() |> Enum.count() end) + iex> Enum.reduce(stream, 0, fn {:ok, num}, acc -> num + acc end) + 47 + + See `async_stream/5` for discussion, options, and more examples. """ - @spec await(t, timeout) :: term | no_return - def await(%Task{ref: ref}=task, timeout \\ 5000) do + @doc since: "1.4.0" + @spec async_stream(Enumerable.t(), (term -> term), keyword) :: Enumerable.t() + def async_stream(enumerable, fun, options \\ []) + when is_function(fun, 1) and is_list(options) do + build_stream(enumerable, fun, options) + end + + defp build_stream(enumerable, fun, options) do + fn acc, acc_fun -> + owner = get_owner(self()) + + Task.Supervised.stream(enumerable, acc, acc_fun, get_callers(self()), fun, options, fn -> + {:ok, pid} = Task.Supervised.start_link(owner, :nomonitor) + {:ok, :link, pid} + end) + end + end + + # Returns a tuple with the node where this is executed and either the + # registered name of the given PID or the PID of where this is executed. Used + # when exiting from tasks to print out from where the task was started. + defp get_owner(pid) do + self_or_name = + case Process.info(pid, :registered_name) do + {:registered_name, name} when is_atom(name) -> name + _ -> pid + end + + {node(), self_or_name, pid} + end + + defp get_callers(owner) do + case :erlang.get(:"$callers") do + [_ | _] = list -> [owner | list] + _ -> [owner] + end + end + + @doc ~S""" + Awaits a task reply and returns it. + + In case the task process dies, the caller process will exit with the same + reason as the task. + + A timeout, in milliseconds or `:infinity`, can be given with a default value + of `5000`. If the timeout is exceeded, then the caller process will exit. + If the task process is linked to the caller process which is the case when + a task is started with `async`, then the task process will also exit. If the + task process is trapping exits or not linked to the caller process, then it + will continue to run. + + This function assumes the task's monitor is still active or the monitor's + `:DOWN` message is in the message queue. If it has been demonitored, or the + message already received, this function will wait for the duration of the + timeout awaiting the message. + + This function can only be called once for any given task. If you want + to be able to check multiple times if a long-running task has finished + its computation, use `yield/2` instead. + + ## Examples + + iex> task = Task.async(fn -> 1 + 1 end) + iex> Task.await(task) + 2 + + ## Compatibility with OTP behaviours + + It is not recommended to `await` a long-running task inside an OTP + behaviour such as `GenServer`. Instead, you should match on the message + coming from a task inside your `c:GenServer.handle_info/2` callback. + + A GenServer will receive two messages on `handle_info/2`: + + * `{ref, result}` - the reply message where `ref` is the monitor + reference returned by the `task.ref` and `result` is the task + result + + * `{:DOWN, ref, :process, pid, reason}` - since all tasks are also + monitored, you will also receive the `:DOWN` message delivered by + `Process.monitor/1`. If you receive the `:DOWN` message without a + a reply, it means the task crashed + + Another consideration to have in mind is that tasks started by `Task.async/1` + are always linked to their callers and you may not want the GenServer to + crash if the task crashes. Therefore, it is preferable to instead use + `Task.Supervisor.async_nolink/3` inside OTP behaviours. For completeness, here + is an example of a GenServer that start tasks and handles their results: + + defmodule GenServerTaskExample do + use GenServer + + def start_link(opts) do + GenServer.start_link(__MODULE__, :ok, opts) + end + + def init(_opts) do + # We will keep all running tasks in a map + {:ok, %{tasks: %{}}} + end + + # Imagine we invoke a task from the GenServer to access a URL... + def handle_call(:some_message, _from, state) do + url = ... + task = Task.Supervisor.async_nolink(MyApp.TaskSupervisor, fn -> fetch_url(url) end) + + # After we start the task, we store its reference and the url it is fetching + state = put_in(state.tasks[task.ref], url) + + {:reply, :ok, state} + end + + # If the task succeeds... + def handle_info({ref, result}, state) do + # The task succeed so we can cancel the monitoring and discard the DOWN message + Process.demonitor(ref, [:flush]) + + {url, state} = pop_in(state.tasks[ref]) + IO.puts "Got #{inspect(result)} for URL #{inspect url}" + {:noreply, state} + end + + # If the task fails... + def handle_info({:DOWN, ref, _, _, reason}, state) do + {url, state} = pop_in(state.tasks[ref]) + IO.puts "URL #{inspect url} failed with reason #{inspect(reason)}" + {:noreply, state} + end + end + + With the server defined, you will want to start the task supervisor + above and the GenServer in your supervision tree: + + children = [ + {Task.Supervisor, name: MyApp.TaskSupervisor}, + {GenServerTaskExample, name: MyApp.GenServerTaskExample} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + + """ + @spec await(t, timeout) :: term + def await(%Task{ref: ref, owner: owner} = task, timeout \\ 5000) when is_timeout(timeout) do + if owner != self() do + raise ArgumentError, invalid_owner_error(task) + end + receive do {^ref, reply} -> Process.demonitor(ref, [:flush]) reply - {:DOWN, ^ref, _, _, :noconnection} -> - mfa = {__MODULE__, :await, [task, timeout]} - exit({{:nodedown, node(task.pid)}, mfa}) - {:DOWN, ^ref, _, _, reason} -> - exit({reason, {__MODULE__, :await, [task, timeout]}}) + + {:DOWN, ^ref, _, proc, reason} -> + exit({reason(reason, proc), {__MODULE__, :await, [task, timeout]}}) after timeout -> Process.demonitor(ref, [:flush]) @@ -175,46 +832,506 @@ defmodule Task do end @doc """ - Receives a group of tasks and a message and finds - a task that matches the given message. + Ignores an existing task. + + This means the task will continue running, but it will be unlinked + and you can no longer yield, await or shut it down. - This function returns a tuple with the task and the - returned value in case the message matches a task that - exited with success, it raises in case the found task - failed or `nil` if no task was found. + Returns `{:ok, reply}` if the reply is received before ignoring the task, + `{:exit, reason}` if the task died before ignoring it, otherwise `nil`. - This function is useful in situations where multiple - tasks are spawned and their results are collected - later on. For example, a `GenServer` can spawn tasks, - store the tasks in a list and later use `Task.find/2` - to see if incoming messages are from any of the tasks. + Important: avoid using [`Task.async/1,3`](`async/1`) and then immediately ignoring + the task. If you want to start tasks you don't care about their + results, use `Task.Supervisor.start_child/2` instead. + + Requires Erlang/OTP 24+. """ - @spec find([t], any) :: {term, t} | nil | no_return - def find(tasks, msg) + @doc since: "1.13.0" + def ignore(%Task{ref: ref, pid: pid, owner: owner} = task) do + unless function_exported?(:erlang, :monitor, 3) do + raise "Task.ignore/1 requires Erlang/OTP 24+" + end - def find(tasks, {ref, reply}) when is_reference(ref) do - Enum.find_value tasks, fn - %Task{ref: task_ref} = t when ref == task_ref -> + if owner != self() do + raise ArgumentError, invalid_owner_error(task) + end + + receive do + {^ref, reply} -> + Process.unlink(pid) + Process.demonitor(ref, [:flush]) + {:ok, reply} + + {:DOWN, ^ref, _, proc, :noconnection} -> + exit({reason(:noconnection, proc), {__MODULE__, :ignore, [task]}}) + + {:DOWN, ^ref, _, _, reason} -> + {:exit, reason} + after + 0 -> + Process.unlink(pid) Process.demonitor(ref, [:flush]) - {reply, t} - %Task{} -> nil end end - def find(tasks, {:DOWN, ref, _, _, reason} = msg) when is_reference(ref) do - find = fn(%Task{ref: task_ref}) -> task_ref == ref end - case Enum.find(tasks, find) do - %Task{pid: pid} when reason == :noconnection -> - exit({{:nodedown, node(pid)}, {__MODULE__, :find, [tasks, msg]}}) + @doc """ + Awaits replies from multiple tasks and returns them. + + This function receives a list of tasks and waits for their replies in the + given time interval. It returns a list of the results, in the same order as + the tasks supplied in the `tasks` input argument. + + If any of the task processes dies, the caller process will exit with the same + reason as that task. + + A timeout, in milliseconds or `:infinity`, can be given with a default value + of `5000`. If the timeout is exceeded, then the caller process will exit. + Any task processes that are linked to the caller process (which is the case + when a task is started with `async`) will also exit. Any task processes that + are trapping exits or not linked to the caller process will continue to run. + + This function assumes the tasks' monitors are still active or the monitor's + `:DOWN` message is in the message queue. If any tasks have been demonitored, + or the message already received, this function will wait for the duration of + the timeout. + + This function can only be called once for any given task. If you want to be + able to check multiple times if a long-running task has finished its + computation, use `yield_many/2` instead. + + ## Compatibility with OTP behaviours + + It is not recommended to `await` long-running tasks inside an OTP behaviour + such as `GenServer`. See `await/2` for more information. + + ## Examples + + iex> tasks = [ + ...> Task.async(fn -> 1 + 1 end), + ...> Task.async(fn -> 2 + 3 end) + ...> ] + iex> Task.await_many(tasks) + [2, 5] + + """ + @doc since: "1.11.0" + @spec await_many([t], timeout) :: [term] + def await_many(tasks, timeout \\ 5000) when is_timeout(timeout) do + awaiting = + for task <- tasks, into: %{} do + %Task{ref: ref, owner: owner} = task + + if owner != self() do + raise ArgumentError, invalid_owner_error(task) + end + + {ref, true} + end + + timeout_ref = make_ref() + + timer_ref = + if timeout != :infinity do + Process.send_after(self(), timeout_ref, timeout) + end + + try do + await_many(tasks, timeout, awaiting, %{}, timeout_ref) + after + timer_ref && Process.cancel_timer(timer_ref) + receive do: (^timeout_ref -> :ok), after: (0 -> :ok) + end + end + + defp await_many(tasks, _timeout, awaiting, replies, _timeout_ref) + when map_size(awaiting) == 0 do + for %{ref: ref} <- tasks, do: Map.fetch!(replies, ref) + end + + defp await_many(tasks, timeout, awaiting, replies, timeout_ref) do + receive do + ^timeout_ref -> + demonitor_pending_tasks(awaiting) + exit({:timeout, {__MODULE__, :await_many, [tasks, timeout]}}) + + {:DOWN, ref, _, proc, reason} when is_map_key(awaiting, ref) -> + demonitor_pending_tasks(awaiting) + exit({reason(reason, proc), {__MODULE__, :await_many, [tasks, timeout]}}) + + {ref, reply} when is_map_key(awaiting, ref) -> + Process.demonitor(ref, [:flush]) + + await_many( + tasks, + timeout, + Map.delete(awaiting, ref), + Map.put(replies, ref, reply), + timeout_ref + ) + end + end + + defp demonitor_pending_tasks(awaiting) do + Enum.each(awaiting, fn {ref, _} -> + Process.demonitor(ref, [:flush]) + end) + end + + @doc false + @deprecated "Pattern match directly on the message instead" + def find(tasks, {ref, reply}) when is_reference(ref) do + Enum.find_value(tasks, fn + %Task{ref: ^ref} = task -> + Process.demonitor(ref, [:flush]) + {reply, task} + %Task{} -> - exit({reason, {__MODULE__, :find, [tasks, msg]}}) - nil -> nil + end) + end + + def find(tasks, {:DOWN, ref, _, proc, reason} = msg) when is_reference(ref) do + find = fn %Task{ref: task_ref} -> task_ref == ref end + + if Enum.find(tasks, find) do + exit({reason(reason, proc), {__MODULE__, :find, [tasks, msg]}}) end end def find(_tasks, _msg) do nil end + + @doc ~S""" + Temporarily blocks the caller process waiting for a task reply. + + Returns `{:ok, reply}` if the reply is received, `nil` if + no reply has arrived, or `{:exit, reason}` if the task has already + exited. Keep in mind that normally a task failure also causes + the process owning the task to exit. Therefore this function can + return `{:exit, reason}` if at least one of the conditions below apply: + + * the task process exited with the reason `:normal` + * the task isn't linked to the caller (the task was started + with `Task.Supervisor.async_nolink/2` or `Task.Supervisor.async_nolink/4`) + * the caller is trapping exits + + A timeout, in milliseconds or `:infinity`, can be given with a default value + of `5000`. If the time runs out before a message from the task is received, + this function will return `nil` and the monitor will remain active. Therefore + `yield/2` can be called multiple times on the same task. + + This function assumes the task's monitor is still active or the + monitor's `:DOWN` message is in the message queue. If it has been + demonitored or the message already received, this function will wait + for the duration of the timeout awaiting the message. + + If you intend to shut the task down if it has not responded within `timeout` + milliseconds, you should chain this together with `shutdown/1`, like so: + + case Task.yield(task, timeout) || Task.shutdown(task) do + {:ok, result} -> + result + + nil -> + Logger.warn("Failed to get a result in #{timeout}ms") + nil + end + + If you intend to check on the task but leave it running after the timeout, + you can chain this together with `ignore/1`, like so: + + case Task.yield(task, timeout) || Task.ignore(task) do + {:ok, result} -> + result + + nil -> + Logger.warn("Failed to get a result in #{timeout}ms") + nil + end + + That ensures that if the task completes after the `timeout` but before `shutdown/1` + has been called, you will still get the result, since `shutdown/1` is designed to + handle this case and return the result. + """ + @spec yield(t, timeout) :: {:ok, term} | {:exit, term} | nil + def yield(%Task{ref: ref, owner: owner} = task, timeout \\ 5000) when is_timeout(timeout) do + if owner != self() do + raise ArgumentError, invalid_owner_error(task) + end + + receive do + {^ref, reply} -> + Process.demonitor(ref, [:flush]) + {:ok, reply} + + {:DOWN, ^ref, _, proc, :noconnection} -> + exit({reason(:noconnection, proc), {__MODULE__, :yield, [task, timeout]}}) + + {:DOWN, ^ref, _, _, reason} -> + {:exit, reason} + after + timeout -> + nil + end + end + + @doc """ + Yields to multiple tasks in the given time interval. + + This function receives a list of tasks and waits for their + replies in the given time interval. It returns a list + of two-element tuples, with the task as the first element + and the yielded result as the second. The tasks in the returned + list will be in the same order as the tasks supplied in the `tasks` + input argument. + + Similarly to `yield/2`, each task's result will be + + * `{:ok, term}` if the task has successfully reported its + result back in the given time interval + * `{:exit, reason}` if the task has died + * `nil` if the task keeps running past the timeout + + A timeout, in milliseconds or `:infinity`, can be given with a default value + of `5000`. + + Check `yield/2` for more information. + + ## Example + + `Task.yield_many/2` allows developers to spawn multiple tasks + and retrieve the results received in a given timeframe. + If we combine it with `Task.shutdown/2` (or `Task.ignore/1`), + it allows us to gather those results and cancel (or ignore) + the tasks that have not replied in time. + + Let's see an example. + + tasks = + for i <- 1..10 do + Task.async(fn -> + Process.sleep(i * 1000) + i + end) + end + + tasks_with_results = Task.yield_many(tasks, 5000) + + results = + Enum.map(tasks_with_results, fn {task, res} -> + # Shut down the tasks that did not reply nor exit + res || Task.shutdown(task, :brutal_kill) + end) + + # Here we are matching only on {:ok, value} and + # ignoring {:exit, _} (crashed tasks) and `nil` (no replies) + for {:ok, value} <- results do + IO.inspect(value) + end + + In the example above, we create tasks that sleep from 1 + up to 10 seconds and return the number of seconds they slept for. + If you execute the code all at once, you should see 1 up to 5 + printed, as those were the tasks that have replied in the + given time. All other tasks will have been shut down using + the `Task.shutdown/2` call. + """ + @spec yield_many([t], timeout) :: [{t, {:ok, term} | {:exit, term} | nil}] + def yield_many(tasks, timeout \\ 5000) when is_timeout(timeout) do + timeout_ref = make_ref() + + timer_ref = + if timeout != :infinity do + Process.send_after(self(), timeout_ref, timeout) + end + + try do + yield_many(tasks, timeout_ref, :infinity) + catch + {:noconnection, reason} -> + exit({reason, {__MODULE__, :yield_many, [tasks, timeout]}}) + after + timer_ref && Process.cancel_timer(timer_ref) + receive do: (^timeout_ref -> :ok), after: (0 -> :ok) + end + end + + defp yield_many([%Task{ref: ref, owner: owner} = task | rest], timeout_ref, timeout) do + if owner != self() do + raise ArgumentError, invalid_owner_error(task) + end + + receive do + {^ref, reply} -> + Process.demonitor(ref, [:flush]) + [{task, {:ok, reply}} | yield_many(rest, timeout_ref, timeout)] + + {:DOWN, ^ref, _, proc, :noconnection} -> + throw({:noconnection, reason(:noconnection, proc)}) + + {:DOWN, ^ref, _, _, reason} -> + [{task, {:exit, reason}} | yield_many(rest, timeout_ref, timeout)] + + ^timeout_ref -> + [{task, nil} | yield_many(rest, timeout_ref, 0)] + after + timeout -> + [{task, nil} | yield_many(rest, timeout_ref, 0)] + end + end + + defp yield_many([], _timeout_ref, _timeout) do + [] + end + + @doc """ + Unlinks and shuts down the task, and then checks for a reply. + + Returns `{:ok, reply}` if the reply is received while shutting down the task, + `{:exit, reason}` if the task died, otherwise `nil`. Once shut down, + you can no longer await or yield it. + + The second argument is either a timeout or `:brutal_kill`. In case + of a timeout, a `:shutdown` exit signal is sent to the task process + and if it does not exit within the timeout, it is killed. With `:brutal_kill` + the task is killed straight away. In case the task terminates abnormally + (possibly killed by another process), this function will exit with the same reason. + + It is not required to call this function when terminating the caller, unless + exiting with reason `:normal` or if the task is trapping exits. If the caller is + exiting with a reason other than `:normal` and the task is not trapping exits, the + caller's exit signal will stop the task. The caller can exit with reason + `:shutdown` to shut down all of its linked processes, including tasks, that + are not trapping exits without generating any log messages. + + If a task's monitor has already been demonitored or received and there is not + a response waiting in the message queue this function will return + `{:exit, :noproc}` as the result or exit reason can not be determined. + """ + @spec shutdown(t, timeout | :brutal_kill) :: {:ok, term} | {:exit, term} | nil + def shutdown(task, shutdown \\ 5000) + + def shutdown(%Task{pid: nil} = task, _) do + raise ArgumentError, "task #{inspect(task)} does not have an associated task process" + end + + def shutdown(%Task{owner: owner} = task, _) when owner != self() do + raise ArgumentError, invalid_owner_error(task) + end + + def shutdown(%Task{pid: pid} = task, :brutal_kill) do + mon = Process.monitor(pid) + exit(pid, :kill) + + case shutdown_receive(task, mon, :brutal_kill, :infinity) do + {:down, proc, :noconnection} -> + exit({reason(:noconnection, proc), {__MODULE__, :shutdown, [task, :brutal_kill]}}) + + {:down, _, reason} -> + {:exit, reason} + + result -> + result + end + end + + def shutdown(%Task{pid: pid} = task, timeout) when is_timeout(timeout) do + mon = Process.monitor(pid) + exit(pid, :shutdown) + + case shutdown_receive(task, mon, :shutdown, timeout) do + {:down, proc, :noconnection} -> + exit({reason(:noconnection, proc), {__MODULE__, :shutdown, [task, timeout]}}) + + {:down, _, reason} -> + {:exit, reason} + + result -> + result + end + end + + ## Helpers + + defp reason(:noconnection, proc), do: {:nodedown, monitor_node(proc)} + defp reason(reason, _), do: reason + + defp monitor_node(pid) when is_pid(pid), do: node(pid) + defp monitor_node({_, node}), do: node + + # spawn a process to ensure task gets exit signal if process dies from exit signal + # between unlink and exit. + defp exit(task, reason) do + caller = self() + ref = make_ref() + enforcer = spawn(fn -> enforce_exit(task, reason, caller, ref) end) + Process.unlink(task) + Process.exit(task, reason) + send(enforcer, {:done, ref}) + :ok + end + + defp enforce_exit(pid, reason, caller, ref) do + mon = Process.monitor(caller) + + receive do + {:done, ^ref} -> :ok + {:DOWN, ^mon, _, _, _} -> Process.exit(pid, reason) + end + end + + defp shutdown_receive(%{ref: ref} = task, mon, type, timeout) do + receive do + {:DOWN, ^mon, _, _, :shutdown} when type in [:shutdown, :timeout_kill] -> + Process.demonitor(ref, [:flush]) + flush_reply(ref) + + {:DOWN, ^mon, _, _, :killed} when type == :brutal_kill -> + Process.demonitor(ref, [:flush]) + flush_reply(ref) + + {:DOWN, ^mon, _, proc, :noproc} -> + reason = flush_noproc(ref, proc, type) + flush_reply(ref) || reason + + {:DOWN, ^mon, _, proc, reason} -> + Process.demonitor(ref, [:flush]) + flush_reply(ref) || {:down, proc, reason} + after + timeout -> + Process.exit(task.pid, :kill) + shutdown_receive(task, mon, :timeout_kill, :infinity) + end + end + + defp flush_reply(ref) do + receive do + {^ref, reply} -> {:ok, reply} + after + 0 -> nil + end + end + + defp flush_noproc(ref, proc, type) do + receive do + {:DOWN, ^ref, _, _, :shutdown} when type in [:shutdown, :timeout_kill] -> + nil + + {:DOWN, ^ref, _, _, :killed} when type == :brutal_kill -> + nil + + {:DOWN, ^ref, _, _, reason} -> + {:down, proc, reason} + after + 0 -> + Process.demonitor(ref, [:flush]) + {:down, proc, :noproc} + end + end + + defp invalid_owner_error(task) do + "task #{inspect(task)} must be queried from the owner but was queried from #{inspect(self())}" + end end diff --git a/lib/elixir/lib/task/supervised.ex b/lib/elixir/lib/task/supervised.ex index 921e8432e8c..66fc9d2e2e1 100644 --- a/lib/elixir/lib/task/supervised.ex +++ b/lib/elixir/lib/task/supervised.ex @@ -1,28 +1,43 @@ defmodule Task.Supervised do @moduledoc false + @ref_timeout 5000 - def start_link(info, fun) do - {:ok, :proc_lib.spawn_link(__MODULE__, :noreply, [info, fun])} + def start(owner, callers, fun) do + {:ok, :proc_lib.spawn(__MODULE__, :noreply, [owner, callers, fun])} end - def start_link(caller, info, fun) do - :proc_lib.start_link(__MODULE__, :reply, [caller, info, fun]) + def start_link(owner, callers, fun) do + {:ok, :proc_lib.spawn_link(__MODULE__, :noreply, [owner, callers, fun])} end - def async(caller, info, mfa) do - initial_call(mfa) - ref = receive do: ({^caller, ref} -> ref) - send caller, {ref, do_apply(info, mfa)} + def start_link(owner, monitor) do + {:ok, :proc_lib.spawn_link(__MODULE__, :reply, [owner, monitor])} end - def reply(caller, info, mfa) do - initial_call(mfa) - :erlang.link(caller) - :proc_lib.init_ack({:ok, self()}) + def reply({_, _, owner_pid} = owner, monitor) do + case monitor do + :monitor -> + mref = Process.monitor(owner_pid) + reply(owner, owner_pid, mref, @ref_timeout) + + :nomonitor -> + reply(owner, owner_pid, nil, :infinity) + end + end - ref = - # There is a race condition on this operation when working accross - # node that manifests if a `Task.Supervisor.async/1` call is made + defp reply(owner, owner_pid, mref, timeout) do + receive do + {^owner_pid, ref, reply_to, callers, mfa} -> + initial_call(mfa) + put_callers(callers) + _ = mref && Process.demonitor(mref, [:flush]) + send(reply_to, {ref, invoke_mfa(owner, mfa)}) + + {:DOWN, ^mref, _, _, reason} -> + exit({:shutdown, reason}) + after + # There is a race condition on this operation when working across + # node that manifests if a "Task.Supervisor.async/2" call is made # while the supervisor is busy spawning previous tasks. # # Imagine the following workflow: @@ -34,21 +49,25 @@ defmodule Task.Supervised do # 5. The spawned task waits forever for the monitor reference so it can begin # # We have solved this by specifying a timeout of 5000 seconds. - # Given no work is done in the client in between the task start and + # Given no work is done in the client between the task start and # sending the reference, 5000 should be enough to not raise false # negatives unless the nodes are indeed not available. - receive do - {^caller, ref} -> ref - after - 5000 -> exit(:timeout) - end - - send caller, {ref, do_apply(info, mfa)} + # + # The same situation could occur with "Task.Supervisor.async_nolink/2", + # except a monitor is used instead of a link. + timeout -> + exit(:timeout) + end end - def noreply(info, mfa) do + def noreply(owner, callers, mfa) do initial_call(mfa) - do_apply(info, mfa) + put_callers(callers) + invoke_mfa(owner, mfa) + end + + defp put_callers(callers) do + Process.put(:"$callers", callers) end defp initial_call(mfa) do @@ -56,8 +75,8 @@ defmodule Task.Supervised do end defp get_initial_call({:erlang, :apply, [fun, []]}) when is_function(fun, 0) do - {:module, module} = :erlang.fun_info(fun, :module) - {:name, name} = :erlang.fun_info(fun, :name) + {:module, module} = Function.info(fun, :module) + {:name, name} = Function.info(fun, :name) {module, name, 0} end @@ -65,43 +84,553 @@ defmodule Task.Supervised do {mod, fun, length(args)} end - defp do_apply(info, {module, fun, args} = mfa) do + defp invoke_mfa(owner, {module, fun, args} = mfa) do try do apply(module, fun, args) catch - :error, value -> - exit(info, mfa, {value, System.stacktrace()}) - :throw, value -> - exit(info, mfa, {{:nocatch, value}, System.stacktrace()}) - :exit, value -> - exit(info, mfa, value) + :exit, value + when value == :normal + when value == :shutdown + when tuple_size(value) == 2 and elem(value, 0) == :shutdown -> + :erlang.raise(:exit, value, __STACKTRACE__) + + kind, value -> + {fun, args} = get_running(mfa) + + :logger.error( + %{ + label: {Task.Supervisor, :terminating}, + report: %{ + name: self(), + starter: get_from(owner), + function: fun, + args: args, + reason: {log_value(kind, value), __STACKTRACE__} + } + }, + %{ + domain: [:otp, :elixir], + error_logger: %{tag: :error_msg}, + report_cb: &__MODULE__.format_report/1, + callers: Process.get(:"$callers") + } + ) + + :erlang.raise(kind, value, __STACKTRACE__) end end - defp exit(_info, _mfa, reason) - when reason == :normal - when reason == :shutdown - when tuple_size(reason) == 2 and elem(reason, 0) == :shutdown do - exit(reason) + defp log_value(:throw, value), do: {:nocatch, value} + defp log_value(_, value), do: value + + @doc false + def format_report(%{ + label: {Task.Supervisor, :terminating}, + report: %{ + name: name, + starter: starter, + function: fun, + args: args, + reason: reason + } + }) do + message = + '** Task ~p terminating~n' ++ + '** Started from ~p~n' ++ + '** When function == ~p~n' ++ + '** arguments == ~p~n' ++ '** Reason for termination == ~n' ++ '** ~p~n' + + {message, [starter, name, fun, args, get_reason(reason)]} + end + + defp get_from({node, pid_or_name, _pid}) when node == node(), do: pid_or_name + defp get_from({node, name, _pid}) when is_atom(name), do: {node, name} + defp get_from({_node, _name, pid}), do: pid + + defp get_running({:erlang, :apply, [fun, []]}) when is_function(fun, 0), do: {fun, []} + defp get_running({mod, fun, args}), do: {Function.capture(mod, fun, length(args)), args} + + defp get_reason({:undef, [{mod, fun, args, _info} | _] = stacktrace} = reason) + when is_atom(mod) and is_atom(fun) do + cond do + :code.is_loaded(mod) === false -> + {:"module could not be loaded", stacktrace} + + is_list(args) and not function_exported?(mod, fun, length(args)) -> + {:"function not exported", stacktrace} + + is_integer(args) and not function_exported?(mod, fun, args) -> + {:"function not exported", stacktrace} + + true -> + reason + end + end + + defp get_reason(reason) do + reason + end + + ## Stream + + def stream(enumerable, acc, reducer, callers, mfa, options, spawn) do + next = &Enumerable.reduce(enumerable, &1, fn x, acc -> {:suspend, [x | acc]} end) + max_concurrency = Keyword.get(options, :max_concurrency, System.schedulers_online()) + + unless is_integer(max_concurrency) and max_concurrency > 0 do + raise ArgumentError, ":max_concurrency must be an integer greater than zero" + end + + ordered? = Keyword.get(options, :ordered, true) + timeout = Keyword.get(options, :timeout, 5000) + on_timeout = Keyword.get(options, :on_timeout, :exit) + zip_input_on_exit? = Keyword.get(options, :zip_input_on_exit, false) + parent = self() + + {:trap_exit, trap_exit?} = Process.info(self(), :trap_exit) + + # Start a process responsible for spawning processes and translating "down" + # messages. This process will trap exits if the current process is trapping + # exit, or it won't trap exits otherwise. + spawn_opts = [:link, :monitor] + + {monitor_pid, monitor_ref} = + Process.spawn( + fn -> stream_monitor(parent, spawn, trap_exit?, timeout) end, + spawn_opts + ) + + # Now that we have the pid of the "monitor" process and the reference of the + # monitor we use to monitor such process, we can inform the monitor process + # about our reference to it. + send(monitor_pid, {parent, monitor_ref}) + + config = %{ + reducer: reducer, + monitor_pid: monitor_pid, + monitor_ref: monitor_ref, + ordered: ordered?, + timeout: timeout, + on_timeout: on_timeout, + zip_input_on_exit: zip_input_on_exit?, + callers: callers, + mfa: mfa + } + + stream_reduce( + acc, + max_concurrency, + _spawned = 0, + _delivered = 0, + _waiting = %{}, + next, + config + ) + end + + defp stream_reduce({:halt, acc}, _max, _spawned, _delivered, _waiting, next, config) do + stream_close(config) + is_function(next) && next.({:halt, []}) + {:halted, acc} + end + + defp stream_reduce({:suspend, acc}, max, spawned, delivered, waiting, next, config) do + continuation = &stream_reduce(&1, max, spawned, delivered, waiting, next, config) + {:suspended, acc, continuation} + end + + # All spawned, all delivered, next is :done. + defp stream_reduce({:cont, acc}, _max, spawned, delivered, _waiting, next, config) + when spawned == delivered and next == :done do + stream_close(config) + {:done, acc} + end + + # No more tasks to spawn because max == 0 or next is :done. We wait for task + # responses or tasks going down. + defp stream_reduce({:cont, acc}, max, spawned, delivered, waiting, next, config) + when max == 0 + when next == :done do + %{ + monitor_pid: monitor_pid, + monitor_ref: monitor_ref, + timeout: timeout, + on_timeout: on_timeout, + zip_input_on_exit: zip_input_on_exit?, + ordered: ordered? + } = config + + receive do + # The task at position "position" replied with "value". We put the + # response in the "waiting" map and do nothing, since we'll only act on + # this response when the replying task dies (we'll see this in the :down + # message). + {{^monitor_ref, position}, reply} -> + %{^position => {pid, :running, _element}} = waiting + waiting = Map.put(waiting, position, {pid, {:ok, reply}}) + stream_reduce({:cont, acc}, max, spawned, delivered, waiting, next, config) + + # The task at position "position" died for some reason. We check if it + # replied already (then the death is peaceful) or if it's still running + # (then the reply from this task will be {:exit, reason}). This message is + # sent to us by the monitor process, not by the dying task directly. + {kind, {^monitor_ref, position}, reason} + when kind in [:down, :timed_out] -> + result = + case waiting do + # If the task replied, we don't care whether it went down for timeout + # or for normal reasons. + %{^position => {_, {:ok, _} = ok}} -> + ok + + # If the task exited by itself before replying, we emit {:exit, reason}. + %{^position => {_, :running, element}} + when kind == :down -> + if zip_input_on_exit?, do: {:exit, {element, reason}}, else: {:exit, reason} + + # If the task timed out before replying, we either exit (on_timeout: :exit) + # or emit {:exit, :timeout} (on_timeout: :kill_task) (note the task is already + # dead at this point). + %{^position => {_, :running, element}} + when kind == :timed_out -> + if on_timeout == :exit do + stream_cleanup_inbox(monitor_pid, monitor_ref) + exit({:timeout, {__MODULE__, :stream, [timeout]}}) + else + if zip_input_on_exit?, do: {:exit, {element, :timeout}}, else: {:exit, :timeout} + end + end + + if ordered? do + waiting = Map.put(waiting, position, {:done, result}) + stream_deliver({:cont, acc}, max + 1, spawned, delivered, waiting, next, config) + else + pair = deliver_now(result, acc, next, config) + waiting = Map.delete(waiting, position) + stream_reduce(pair, max + 1, spawned, delivered + 1, waiting, next, config) + end + + # The monitor process died. We just cleanup the messages from the monitor + # process and exit. + {:DOWN, ^monitor_ref, _, _, reason} -> + stream_cleanup_inbox(monitor_pid, monitor_ref) + exit({reason, {__MODULE__, :stream, [timeout]}}) + end + end + + defp stream_reduce({:cont, acc}, max, spawned, delivered, waiting, next, config) do + try do + next.({:cont, []}) + catch + kind, reason -> + stream_close(config) + :erlang.raise(kind, reason, __STACKTRACE__) + else + {:suspended, [value], next} -> + waiting = stream_spawn(value, spawned, waiting, config) + stream_reduce({:cont, acc}, max - 1, spawned + 1, delivered, waiting, next, config) + + {_, [value]} -> + waiting = stream_spawn(value, spawned, waiting, config) + stream_reduce({:cont, acc}, max - 1, spawned + 1, delivered, waiting, :done, config) + + {_, []} -> + stream_reduce({:cont, acc}, max, spawned, delivered, waiting, :done, config) + end + end + + defp deliver_now(reply, acc, next, config) do + %{reducer: reducer} = config + + try do + reducer.(reply, acc) + catch + kind, reason -> + is_function(next) && next.({:halt, []}) + stream_close(config) + :erlang.raise(kind, reason, __STACKTRACE__) + end end - defp exit(info, mfa, reason) do - {fun, args} = get_running(mfa) + defp stream_deliver({:suspend, acc}, max, spawned, delivered, waiting, next, config) do + continuation = &stream_deliver(&1, max, spawned, delivered, waiting, next, config) + {:suspended, acc, continuation} + end + + defp stream_deliver({:halt, acc}, max, spawned, delivered, waiting, next, config) do + stream_reduce({:halt, acc}, max, spawned, delivered, waiting, next, config) + end + + defp stream_deliver({:cont, acc}, max, spawned, delivered, waiting, next, config) do + %{reducer: reducer} = config + + case waiting do + %{^delivered => {:done, reply}} -> + try do + reducer.(reply, acc) + catch + kind, reason -> + is_function(next) && next.({:halt, []}) + stream_close(config) + :erlang.raise(kind, reason, __STACKTRACE__) + else + pair -> + waiting = Map.delete(waiting, delivered) + stream_deliver(pair, max, spawned, delivered + 1, waiting, next, config) + end + + %{} -> + stream_reduce({:cont, acc}, max, spawned, delivered, waiting, next, config) + end + end + + defp stream_close(%{monitor_pid: monitor_pid, monitor_ref: monitor_ref, timeout: timeout}) do + send(monitor_pid, {:stop, monitor_ref}) + + receive do + {:DOWN, ^monitor_ref, _, _, :normal} -> + stream_cleanup_inbox(monitor_pid, monitor_ref) + :ok + + {:DOWN, ^monitor_ref, _, _, reason} -> + stream_cleanup_inbox(monitor_pid, monitor_ref) + exit({reason, {__MODULE__, :stream, [timeout]}}) + end + end + + defp stream_cleanup_inbox(monitor_pid, monitor_ref) do + receive do + {:EXIT, ^monitor_pid, _} -> stream_cleanup_inbox(monitor_ref) + after + 0 -> stream_cleanup_inbox(monitor_ref) + end + end + + defp stream_cleanup_inbox(monitor_ref) do + receive do + {{^monitor_ref, _}, _} -> + stream_cleanup_inbox(monitor_ref) + + {kind, {^monitor_ref, _}, _} when kind in [:down, :timed_out] -> + stream_cleanup_inbox(monitor_ref) + after + 0 -> + :ok + end + end + + # This function spawns a task for the given "value", and puts the pid of this + # new task in the map of "waiting" tasks, which is returned. + defp stream_spawn(value, spawned, waiting, config) do + %{ + monitor_pid: monitor_pid, + monitor_ref: monitor_ref, + timeout: timeout, + callers: callers, + mfa: mfa, + zip_input_on_exit: zip_input_on_exit? + } = config + + send(monitor_pid, {:spawn, spawned}) + + receive do + {:spawned, {^monitor_ref, ^spawned}, pid} -> + mfa_with_value = normalize_mfa_with_arg(mfa, value) + send(pid, {self(), {monitor_ref, spawned}, self(), callers, mfa_with_value}) + stored_value = if zip_input_on_exit?, do: value, else: nil + Map.put(waiting, spawned, {pid, :running, stored_value}) + + {:max_children, ^monitor_ref} -> + stream_close(config) + + raise """ + reached the maximum number of tasks for this task supervisor. The maximum number \ + of tasks that are allowed to run at the same time under this supervisor can be \ + configured with the :max_children option passed to Task.Supervisor.start_link/1. When \ + using async_stream or async_stream_nolink, make sure to configure :max_concurrency to \ + be lower or equal to :max_children and pay attention to whether other tasks are also \ + spawned under the same task supervisor.\ + """ + + {:DOWN, ^monitor_ref, _, ^monitor_pid, reason} -> + stream_cleanup_inbox(monitor_pid, monitor_ref) + exit({reason, {__MODULE__, :stream, [timeout]}}) + end + end + + defp stream_monitor(parent_pid, spawn, trap_exit?, timeout) do + Process.flag(:trap_exit, trap_exit?) + parent_ref = Process.monitor(parent_pid) + + # Let's wait for the parent process to tell this process the monitor ref + # it's using to monitor this process. If the parent process dies while this + # process waits, this process dies with the same reason. + receive do + {^parent_pid, monitor_ref} -> + config = %{ + parent_pid: parent_pid, + parent_ref: parent_ref, + spawn: spawn, + monitor_ref: monitor_ref, + timeout: timeout + } + + stream_monitor_loop(_running_tasks = %{}, config) + + {:DOWN, ^parent_ref, _, _, reason} -> + exit(reason) + end + end + + defp stream_monitor_loop(running_tasks, config) do + %{ + spawn: spawn, + parent_pid: parent_pid, + monitor_ref: monitor_ref, + timeout: timeout + } = config - :error_logger.format( - "** Task ~p terminating~n" <> - "** Started from ~p~n" <> - "** When function == ~p~n" <> - "** arguments == ~p~n" <> - "** Reason for termination == ~n" <> - "** ~p~n", [self, get_from(info), fun, args, reason]) + receive do + # The parent process is telling us to spawn a new task to process + # "value". We spawn it and notify the parent about its pid. + {:spawn, position} -> + case spawn.() do + {:ok, type, pid} -> + ref = Process.monitor(pid) + + # Schedule a timeout message to ourselves, unless the timeout was set to :infinity + timer_ref = + case timeout do + :infinity -> nil + timeout -> Process.send_after(self(), {:timeout, {monitor_ref, ref}}, timeout) + end + + send(parent_pid, {:spawned, {monitor_ref, position}, pid}) + + running_tasks = + Map.put(running_tasks, ref, %{ + position: position, + type: type, + pid: pid, + timer_ref: timer_ref, + timed_out?: false + }) + + stream_monitor_loop(running_tasks, config) + + {:error, :max_children} -> + send(parent_pid, {:max_children, monitor_ref}) + stream_waiting_for_stop_loop(running_tasks, config) + end + + # One of the spawned processes went down. We inform the parent process of + # this and keep going. + {:DOWN, ref, _, _, reason} when is_map_key(running_tasks, ref) -> + {task, running_tasks} = Map.pop(running_tasks, ref) + %{position: position, timer_ref: timer_ref, timed_out?: timed_out?} = task + + if timer_ref != nil do + :ok = Process.cancel_timer(timer_ref, async: true, info: false) + end + + message_kind = if(timed_out?, do: :timed_out, else: :down) + send(parent_pid, {message_kind, {monitor_ref, position}, reason}) + stream_monitor_loop(running_tasks, config) + + # One of the spawned processes timed out. We kill that process here + # regardless of the value of :on_timeout. We then send a message to the + # parent process informing it that a task timed out, and the parent + # process decides what to do. + {:timeout, {^monitor_ref, ref}} -> + running_tasks = + case running_tasks do + %{^ref => %{pid: pid, timed_out?: false} = task_info} -> + unlink_and_kill(pid) + Map.put(running_tasks, ref, %{task_info | timed_out?: true}) + + _other -> + running_tasks + end + + stream_monitor_loop(running_tasks, config) + + {:EXIT, _, _} -> + stream_monitor_loop(running_tasks, config) + + other -> + handle_stop_or_parent_down(other, running_tasks, config) + stream_monitor_loop(running_tasks, config) + end + end + + defp stream_waiting_for_stop_loop(running_tasks, config) do + receive do + message -> + handle_stop_or_parent_down(message, running_tasks, config) + stream_waiting_for_stop_loop(running_tasks, config) + end + end + + # The parent process is telling us to stop because the stream is being + # closed. In this case, we forcibly kill all spawned processes and then + # exit gracefully ourselves. + defp handle_stop_or_parent_down( + {:stop, monitor_ref}, + running_tasks, + %{monitor_ref: monitor_ref} + ) do + Process.flag(:trap_exit, true) + + for {_ref, %{pid: pid}} <- running_tasks, do: Process.exit(pid, :kill) + + for {ref, _task} <- running_tasks do + receive do + {:DOWN, ^ref, _, _, _} -> :ok + end + end + + exit(:normal) + end + + # The parent process went down with a given reason. We kill all the + # spawned processes (that are also linked) with the same reason, and then + # exit ourselves with the same reason. + defp handle_stop_or_parent_down( + {:DOWN, parent_ref, _, _, reason}, + running_tasks, + %{parent_ref: parent_ref} + ) do + for {_ref, %{type: :link, pid: pid}} <- running_tasks do + Process.exit(pid, reason) + end exit(reason) end - defp get_from({node, pid_or_name}) when node == node(), do: pid_or_name - defp get_from(other), do: other + # We ignore all other messages. + defp handle_stop_or_parent_down(_other, _running_tasks, _config) do + :ok + end + + defp unlink_and_kill(pid) do + caller = self() + ref = make_ref() - defp get_running({:erlang, :apply, [fun, []]}) when is_function(fun, 0), do: {fun, []} - defp get_running({mod, fun, args}), do: {:erlang.make_fun(mod, fun, length(args)), args} + enforcer = + spawn(fn -> + mon = Process.monitor(caller) + + receive do + {:done, ^ref} -> :ok + {:DOWN, ^mon, _, _, _} -> Process.exit(pid, :kill) + end + end) + + Process.unlink(pid) + Process.exit(pid, :kill) + send(enforcer, {:done, ref}) + end + + defp normalize_mfa_with_arg({mod, fun, args}, arg), do: {mod, fun, [arg | args]} + defp normalize_mfa_with_arg(fun, arg), do: {:erlang, :apply, [fun, [arg]]} end diff --git a/lib/elixir/lib/task/supervisor.ex b/lib/elixir/lib/task/supervisor.ex index 8366ba7c3bc..db74f7af82e 100644 --- a/lib/elixir/lib/task/supervisor.ex +++ b/lib/elixir/lib/task/supervisor.ex @@ -1,112 +1,557 @@ defmodule Task.Supervisor do @moduledoc """ - A tasks supervisor. + A task supervisor. This module defines a supervisor which can be used to dynamically - supervise tasks. Behind the scenes, this module is implemented as a - `:simple_one_for_one` supervisor where the workers are temporary - (i.e. they are not restarted after they die). + supervise tasks. - The functions in this module allow tasks to be spawned and awaited - from a supervisor, similar to the functions defined in the `Task` module. + A task supervisor is started with no children, often under a + supervisor and a name: - ## Name Registration + children = [ + {Task.Supervisor, name: MyApp.TaskSupervisor} + ] + + Supervisor.start_link(children, strategy: :one_for_one) + + The options given in the child specification are documented in `start_link/1`. + + Once started, you can start tasks directly under the supervisor, for example: + + task = Task.Supervisor.async(MyApp.TaskSupervisor, fn -> + :do_some_work + end) + + See the `Task` module for more examples. + + ## Scalability and partitioning + + The `Task.Supervisor` is a single process responsible for starting + other processes. In some applications, the `Task.Supervisor` may + become a bottleneck. To address this, you can start multiple instances + of the `Task.Supervisor` and then pick a random instance to start + the task on. + + Instead of: + + children = [ + {Task.Supervisor, name: Task.Supervisor} + ] + + and: + + Task.Supervisor.async(MyApp.TaskSupervisor, fn -> :do_some_work end) + + You can do this: + + children = [ + {PartitionSupervisor, + child_spec: Task.Supervisor, + name: MyApp.TaskSupervisors} + ] + + and then: + + Task.Supervisor.async( + {:via, PartitionSupervisor, {MyApp.TaskSupervisors, self()}}, + fn -> :do_some_work end + ) + + In the code above, we start a partition supervisor that will by default + start a dynamic supervisor for each core in your machine. Then, instead + of calling the `Task.Supervisor` by name, you call it through the + partition supervisor using the `{:via, PartitionSupervisor, {name, key}}` + format, where `name` is the name of the partition supervisor and `key` + is the routing key. We picked `self()` as the routing key, which means + each process will be assigned one of the existing task supervisors. + Read the `PartitionSupervisor` docs for more information. + + ## Name registration A `Task.Supervisor` is bound to the same name registration rules as a - `GenServer`. Read more about it in the `GenServer` docs. + `GenServer`. Read more about them in the `GenServer` docs. """ + @typedoc "Option values used by `start_link`" + @type option :: + DynamicSupervisor.option() + | DynamicSupervisor.init_option() + + @doc false + def child_spec(opts) when is_list(opts) do + id = + case Keyword.get(opts, :name, Task.Supervisor) do + name when is_atom(name) -> name + {:global, name} -> name + {:via, _module, name} -> name + end + + %{ + id: id, + start: {Task.Supervisor, :start_link, [opts]}, + type: :supervisor + } + end + @doc """ Starts a new supervisor. - The supported options are: + ## Examples + + A task supervisor is typically started under a supervision tree using + the tuple format: + + {Task.Supervisor, name: MyApp.TaskSupervisor} + + You can also start it by calling `start_link/1` directly: + + Task.Supervisor.start_link(name: MyApp.TaskSupervisor) + + But this is recommended only for scripting and should be avoided in + production code. Generally speaking, processes should always be started + inside supervision trees. + + ## Options + + * `:name` - used to register a supervisor name, the supported values are + described under the `Name Registration` section in the `GenServer` module + docs; + + * `:max_restarts`, `:max_seconds`, and `:max_children` - as specified in + `DynamicSupervisor`; + + This function could also receive `:restart` and `:shutdown` as options + but those two options have been deprecated and it is now preferred to + give them directly to `start_child`. + """ + @spec start_link([option]) :: Supervisor.on_start() + def start_link(options \\ []) do + {restart, options} = Keyword.pop(options, :restart) + {shutdown, options} = Keyword.pop(options, :shutdown) + + if restart || shutdown do + IO.warn( + ":restart and :shutdown options in Task.Supervisor.start_link/1 " <> + "are deprecated. Please pass those options on start_child/3 instead" + ) + end + + keys = [:max_children, :max_seconds, :max_restarts] + {sup_opts, start_opts} = Keyword.split(options, keys) + restart_and_shutdown = {restart || :temporary, shutdown || 5000} + DynamicSupervisor.start_link(__MODULE__, {restart_and_shutdown, sup_opts}, start_opts) + end + + @doc false + def init({{_restart, _shutdown} = arg, options}) do + Process.put(__MODULE__, arg) + DynamicSupervisor.init([strategy: :one_for_one] ++ options) + end + + @doc """ + Starts a task that can be awaited on. + + The `supervisor` must be a reference as defined in `Supervisor`. + The task will still be linked to the caller, see `Task.async/3` for + more information and `async_nolink/3` for a non-linked variant. + + Raises an error if `supervisor` has reached the maximum number of + children. + + ## Options - * `:name` - used to register a supervisor name, the supported values are - described under the `Name Registration` section in the `GenServer` module - docs; + * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown + or an integer indicating the timeout value, defaults to 5000 milliseconds. - * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown - or an integer indicating the timeout value, defaults to 5000 milliseconds; """ - @spec start_link(Supervisor.options) :: Supervisor.on_start - def start_link(opts \\ []) do - import Supervisor.Spec - {shutdown, opts} = Keyword.pop(opts, :shutdown, 5000) - children = [worker(Task.Supervised, [], restart: :temporary, shutdown: shutdown)] - Supervisor.start_link(children, [strategy: :simple_one_for_one] ++ opts) + @spec async(Supervisor.supervisor(), (() -> any), Keyword.t()) :: Task.t() + def async(supervisor, fun, options \\ []) do + async(supervisor, :erlang, :apply, [fun, []], options) end @doc """ Starts a task that can be awaited on. - The `supervisor` must be a reference as defined in `Task.Supervisor`. - For more information on tasks, check the `Task` module. + The `supervisor` must be a reference as defined in `Supervisor`. + The task will still be linked to the caller, see `Task.async/3` for + more information and `async_nolink/3` for a non-linked variant. + + Raises an error if `supervisor` has reached the maximum number of + children. + + ## Options + + * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown + or an integer indicating the timeout value, defaults to 5000 milliseconds. + """ - @spec async(Supervisor.supervisor, fun) :: Task.t - def async(supervisor, fun) do - async(supervisor, :erlang, :apply, [fun, []]) + @spec async(Supervisor.supervisor(), module, atom, [term], Keyword.t()) :: Task.t() + def async(supervisor, module, fun, args, options \\ []) do + async(supervisor, :link, module, fun, args, options) end @doc """ Starts a task that can be awaited on. - The `supervisor` must be a reference as defined in `Task.Supervisor`. - For more information on tasks, check the `Task` module. + The `supervisor` must be a reference as defined in `Supervisor`. + The task won't be linked to the caller, see `Task.async/3` for + more information. + + Raises an error if `supervisor` has reached the maximum number of + children. + + ## Options + + * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown + or an integer indicating the timeout value, defaults to 5000 milliseconds. + + ## Compatibility with OTP behaviours + + If you create a task using `async_nolink` inside an OTP behaviour + like `GenServer`, you should match on the message coming from the + task inside your `c:GenServer.handle_info/2` callback. + + The reply sent by the task will be in the format `{ref, result}`, + where `ref` is the monitor reference held by the task struct + and `result` is the return value of the task function. + + Keep in mind that, regardless of how the task created with `async_nolink` + terminates, the caller's process will always receive a `:DOWN` message + with the same `ref` value that is held by the task struct. If the task + terminates normally, the reason in the `:DOWN` message will be `:normal`. + + ## Examples + + Typically, you use `async_nolink/3` when there is a reasonable expectation that + the task may fail, and you don't want it to take down the caller. Let's see an + example where a `GenServer` is meant to run a single task and track its status: + + defmodule MyApp.Server do + use GenServer + + # ... + + def start_task do + GenServer.call(__MODULE__, :start_task) + end + + # In this case the task is already running, so we just return :ok. + def handle_call(:start_task, _from, %{ref: ref} = state) when is_reference(ref) do + {:reply, :ok, state} + end + + # The task is not running yet, so let's start it. + def handle_call(:start_task, _from, %{ref: nil} = state) do + task = + Task.Supervisor.async_nolink(MyApp.TaskSupervisor, fn -> + ... + end) + + # We return :ok and the server will continue running + {:reply, :ok, %{state | ref: task.ref}} + end + + # The task completed successfully + def handle_info({ref, answer}, %{ref: ref} = state) do + # We don't care about the DOWN message now, so let's demonitor and flush it + Process.demonitor(ref, [:flush]) + # Do something with the result and then return + {:noreply, %{state | ref: nil}} + end + + # The task failed + def handle_info({:DOWN, ref, :process, _pid, _reason}, %{ref: ref} = state) do + # Log and possibly restart the task... + {:noreply, %{state | ref: nil}} + end + end + """ - @spec async(Supervisor.supervisor, module, atom, [term]) :: Task.t - def async(supervisor, module, fun, args) do - args = [self, get_info(self), {module, fun, args}] - {:ok, pid} = Supervisor.start_child(supervisor, args) - ref = Process.monitor(pid) - send pid, {self(), ref} - %Task{pid: pid, ref: ref} + @spec async_nolink(Supervisor.supervisor(), (() -> any), Keyword.t()) :: Task.t() + def async_nolink(supervisor, fun, options \\ []) do + async_nolink(supervisor, :erlang, :apply, [fun, []], options) + end + + @doc """ + Starts a task that can be awaited on. + + The `supervisor` must be a reference as defined in `Supervisor`. + The task won't be linked to the caller, see `Task.async/3` for + more information. + + Raises an error if `supervisor` has reached the maximum number of + children. + + Note this function requires the task supervisor to have `:temporary` + as the `:restart` option (the default), as `async_nolink/5` keeps a + direct reference to the task which is lost if the task is restarted. + """ + @spec async_nolink(Supervisor.supervisor(), module, atom, [term], Keyword.t()) :: Task.t() + def async_nolink(supervisor, module, fun, args, options \\ []) do + async(supervisor, :nolink, module, fun, args, options) + end + + @doc """ + Returns a stream where the given function (`module` and `function`) + is mapped concurrently on each element in `enumerable`. + + Each element will be prepended to the given `args` and processed by its + own task. The tasks will be spawned under the given `supervisor` and + linked to the caller process, similarly to `async/5`. + + When streamed, each task will emit `{:ok, value}` upon successful + completion or `{:exit, reason}` if the caller is trapping exits. + The order of results depends on the value of the `:ordered` option. + + The level of concurrency and the time tasks are allowed to run can + be controlled via options (see the "Options" section below). + + If you find yourself trapping exits to handle exits inside + the async stream, consider using `async_stream_nolink/6` to start tasks + that are not linked to the calling process. + + ## Options + + * `:max_concurrency` - sets the maximum number of tasks to run + at the same time. Defaults to `System.schedulers_online/0`. + + * `:ordered` - whether the results should be returned in the same order + as the input stream. This option is useful when you have large + streams and don't want to buffer results before they are delivered. + This is also useful when you're using the tasks for side effects. + Defaults to `true`. + + * `:timeout` - the maximum amount of time to wait (in milliseconds) + without receiving a task reply (across all running tasks). + Defaults to `5000`. + + * `:on_timeout` - what do to when a task times out. The possible + values are: + * `:exit` (default) - the process that spawned the tasks exits. + * `:kill_task` - the task that timed out is killed. The value + emitted for that task is `{:exit, :timeout}`. + + * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown + or an integer indicating the timeout value. Defaults to `5000` milliseconds. + + ## Examples + + Let's build a stream and then enumerate it: + + stream = Task.Supervisor.async_stream(MySupervisor, collection, Mod, :expensive_fun, []) + Enum.to_list(stream) + + """ + @doc since: "1.4.0" + @spec async_stream(Supervisor.supervisor(), Enumerable.t(), module, atom, [term], keyword) :: + Enumerable.t() + def async_stream(supervisor, enumerable, module, function, args, options \\ []) + when is_atom(module) and is_atom(function) and is_list(args) do + build_stream(supervisor, :link, enumerable, {module, function, args}, options) + end + + @doc """ + Returns a stream that runs the given function `fun` concurrently + on each element in `enumerable`. + + Each element in `enumerable` is passed as argument to the given function `fun` + and processed by its own task. The tasks will be spawned under the given + `supervisor` and linked to the caller process, similarly to `async/3`. + + See `async_stream/6` for discussion, options, and examples. + """ + @doc since: "1.4.0" + @spec async_stream(Supervisor.supervisor(), Enumerable.t(), (term -> term), keyword) :: + Enumerable.t() + def async_stream(supervisor, enumerable, fun, options \\ []) when is_function(fun, 1) do + build_stream(supervisor, :link, enumerable, fun, options) + end + + @doc """ + Returns a stream where the given function (`module` and `function`) + is mapped concurrently on each element in `enumerable`. + + Each element in `enumerable` will be prepended to the given `args` and processed + by its own task. The tasks will be spawned under the given `supervisor` and + will not be linked to the caller process, similarly to `async_nolink/5`. + + See `async_stream/6` for discussion, options, and examples. + """ + @doc since: "1.4.0" + @spec async_stream_nolink( + Supervisor.supervisor(), + Enumerable.t(), + module, + atom, + [term], + keyword + ) :: Enumerable.t() + def async_stream_nolink(supervisor, enumerable, module, function, args, options \\ []) + when is_atom(module) and is_atom(function) and is_list(args) do + build_stream(supervisor, :nolink, enumerable, {module, function, args}, options) + end + + @doc """ + Returns a stream that runs the given `function` concurrently on each + element in `enumerable`. + + Each element in `enumerable` is passed as argument to the given function `fun` + and processed by its own task. The tasks will be spawned under the given + `supervisor` and will not be linked to the caller process, similarly + to `async_nolink/3`. + + See `async_stream/6` for discussion and examples. + """ + @doc since: "1.4.0" + @spec async_stream_nolink(Supervisor.supervisor(), Enumerable.t(), (term -> term), keyword) :: + Enumerable.t() + def async_stream_nolink(supervisor, enumerable, fun, options \\ []) when is_function(fun, 1) do + build_stream(supervisor, :nolink, enumerable, fun, options) end @doc """ Terminates the child with the given `pid`. """ - @spec terminate_child(Supervisor.supervisor, pid) :: :ok + @spec terminate_child(Supervisor.supervisor(), pid) :: :ok | {:error, :not_found} def terminate_child(supervisor, pid) when is_pid(pid) do - :supervisor.terminate_child(supervisor, pid) + DynamicSupervisor.terminate_child(supervisor, pid) end @doc """ - Returns all children pids. + Returns all children PIDs except those that are restarting. + + Note that calling this function when supervising a large number + of children under low memory conditions can cause an out of memory + exception. """ - @spec children(Supervisor.supervisor) :: [pid] + @spec children(Supervisor.supervisor()) :: [pid] def children(supervisor) do - :supervisor.which_children(supervisor) |> Enum.map(&elem(&1, 1)) + for {_, pid, _, _} <- DynamicSupervisor.which_children(supervisor), is_pid(pid), do: pid end @doc """ - Starts a task as child of the given `supervisor`. + Starts a task as a child of the given `supervisor`. + + Task.Supervisor.start_child(MyTaskSupervisor, fn -> + IO.puts "I am running in a task" + end) Note that the spawned process is not linked to the caller, but only to the supervisor. This command is useful in case the - task needs to perform side-effects (like I/O) and does not need - to report back to the caller. + task needs to perform side-effects (like I/O) and you have no + interest in its results nor if it completes successfully. + + ## Options + + * `:restart` - the restart strategy, may be `:temporary` (the default), + `:transient` or `:permanent`. `:temporary` means the task is never + restarted, `:transient` means it is restarted if the exit is not + `:normal`, `:shutdown` or `{:shutdown, reason}`. A `:permanent` restart + strategy means it is always restarted. + + * `:shutdown` - `:brutal_kill` if the task must be killed directly on shutdown + or an integer indicating the timeout value, defaults to 5000 milliseconds. + """ - @spec start_child(Supervisor.supervisor, fun) :: {:ok, pid} - def start_child(supervisor, fun) do - start_child(supervisor, :erlang, :apply, [fun, []]) + @spec start_child(Supervisor.supervisor(), (() -> any), keyword) :: + DynamicSupervisor.on_start_child() + def start_child(supervisor, fun, options \\ []) do + restart = options[:restart] + shutdown = options[:shutdown] + args = [get_owner(self()), get_callers(self()), {:erlang, :apply, [fun, []]}] + start_child_with_spec(supervisor, args, restart, shutdown) end @doc """ - Starts a task as child of the given `supervisor`. + Starts a task as a child of the given `supervisor`. - Similar to `start_child/2` except the task is specified + Similar to `start_child/3` except the task is specified by the given `module`, `fun` and `args`. """ - @spec start_child(Supervisor.supervisor, module, atom, [term]) :: {:ok, pid} - def start_child(supervisor, module, fun, args) do - Supervisor.start_child(supervisor, [get_info(self), {module, fun, args}]) + @spec start_child(Supervisor.supervisor(), module, atom, [term], keyword) :: + DynamicSupervisor.on_start_child() + def start_child(supervisor, module, fun, args, options \\ []) + when is_atom(fun) and is_list(args) do + restart = options[:restart] + shutdown = options[:shutdown] + args = [get_owner(self()), get_callers(self()), {module, fun, args}] + start_child_with_spec(supervisor, args, restart, shutdown) + end + + defp start_child_with_spec(supervisor, args, restart, shutdown) do + # TODO: This only exists because we need to support reading restart/shutdown + # from two different places. Remove this, the init function and the associated + # clause in DynamicSupervisor on Elixir v2.0 + # TODO: Once we do this, we can also make it so the task arguments are never + # sent to the supervisor if the restart is temporary + GenServer.call(supervisor, {:start_task, args, restart, shutdown}, :infinity) + end + + defp get_owner(pid) do + self_or_name = + case Process.info(pid, :registered_name) do + {:registered_name, name} when is_atom(name) -> name + _ -> pid + end + + {node(), self_or_name, pid} + end + + defp get_callers(owner) do + case :erlang.get(:"$callers") do + [_ | _] = list -> [owner | list] + _ -> [owner] + end + end + + # TODO: Remove conditional on Erlang/OTP 24 + @compile {:no_warn_undefined, {:erlang, :monitor, 3}} + + defp async(supervisor, link_type, module, fun, args, options) do + owner = self() + shutdown = options[:shutdown] + + case start_child_with_spec(supervisor, [get_owner(owner), :monitor], :temporary, shutdown) do + {:ok, pid} -> + if link_type == :link, do: Process.link(pid) + + {reply_to, ref} = + if function_exported?(:erlang, :monitor, 3) do + ref = :erlang.monitor(:process, pid, alias: :demonitor) + {ref, ref} + else + {owner, Process.monitor(pid)} + end + + send(pid, {owner, ref, reply_to, get_callers(owner), {module, fun, args}}) + %Task{pid: pid, ref: ref, owner: owner, mfa: {module, fun, length(args)}} + + {:error, :max_children} -> + raise """ + reached the maximum number of tasks for this task supervisor. The maximum number \ + of tasks that are allowed to run at the same time under this supervisor can be \ + configured with the :max_children option passed to Task.Supervisor.start_link/1\ + """ + end end - defp get_info(self) do - {node(), - case Process.info(self, :registered_name) do - {:registered_name, []} -> self() - {:registered_name, name} -> name - end} + defp build_stream(supervisor, link_type, enumerable, fun, options) do + fn acc, acc_fun -> + shutdown = options[:shutdown] + owner = get_owner(self()) + + Task.Supervised.stream(enumerable, acc, acc_fun, get_callers(self()), fun, options, fn -> + args = [owner, :monitor] + + case start_child_with_spec(supervisor, args, :temporary, shutdown) do + {:ok, pid} -> + if link_type == :link, do: Process.link(pid) + {:ok, link_type, pid} + + {:error, :max_children} -> + {:error, :max_children} + end + end) + end end end diff --git a/lib/elixir/lib/tuple.ex b/lib/elixir/lib/tuple.ex index b20256aa547..a5e44db7072 100644 --- a/lib/elixir/lib/tuple.ex +++ b/lib/elixir/lib/tuple.ex @@ -1,12 +1,54 @@ defmodule Tuple do @moduledoc """ Functions for working with tuples. + + Please note the following functions for tuples are found in `Kernel`: + + * `elem/2` - accesses a tuple by index + * `put_elem/3` - inserts a value into a tuple by index + * `tuple_size/1` - gets the number of elements in a tuple + + Tuples are intended as fixed-size containers for multiple elements. + To manipulate a collection of elements, use a list instead. `Enum` + functions do not work on tuples. + + Tuples are denoted with curly braces: + + iex> {} + {} + iex> {1, :two, "three"} + {1, :two, "three"} + + A tuple may contain elements of different types, which are stored + contiguously in memory. Accessing any element takes constant time, + but modifying a tuple, which produces a shallow copy, takes linear time. + Tuples are good for reading data while lists are better for traversals. + + Tuples are typically used either when a function has multiple return values + or for error handling. `File.read/1` returns `{:ok, contents}` if reading + the given file is successful, or else `{:error, reason}` such as when + the file does not exist. + + The functions in this module that add and remove elements from tuples are + rarely used in practice, as they typically imply tuples are being used as + collections. To append to a tuple, it is preferable to extract the elements + from the old tuple with pattern matching, and then create a new tuple: + + tuple = {:ok, :example} + + # Avoid + result = Tuple.insert_at(tuple, 2, %{}) + + # Prefer + {:ok, atom} = tuple + result = {:ok, atom, %{}} + """ @doc """ Creates a new tuple. - Creates a tuple of size `size` containing the + Creates a tuple of `size` containing the given `data` at every position. Inlined by the compiler. @@ -18,16 +60,16 @@ defmodule Tuple do """ @spec duplicate(term, non_neg_integer) :: tuple - def duplicate(data, size) do + def duplicate(data, size) when is_integer(size) and size >= 0 do :erlang.make_tuple(size, data) end @doc """ Inserts an element into a tuple. - Inserts `value` into `tuple` at the given zero-based `index`. - Raises an `ArgumentError` if `index` is greater than the - length of `tuple`. + Inserts `value` into `tuple` at the given `index`. + Raises an `ArgumentError` if `index` is negative or greater than the + length of `tuple`. Index is zero-based. Inlined by the compiler. @@ -36,19 +78,41 @@ defmodule Tuple do iex> tuple = {:bar, :baz} iex> Tuple.insert_at(tuple, 0, :foo) {:foo, :bar, :baz} + iex> Tuple.insert_at(tuple, 2, :bong) + {:bar, :baz, :bong} """ @spec insert_at(tuple, non_neg_integer, term) :: tuple - def insert_at(tuple, index, term) do - :erlang.insert_element(index + 1, tuple, term) + def insert_at(tuple, index, value) when is_integer(index) and index >= 0 do + :erlang.insert_element(index + 1, tuple, value) + end + + @doc """ + Inserts an element at the end of a tuple. + + Returns a new tuple with the element appended at the end, and contains + the elements in `tuple` followed by `value` as the last element. + + Inlined by the compiler. + + ## Examples + + iex> tuple = {:foo, :bar} + iex> Tuple.append(tuple, :baz) + {:foo, :bar, :baz} + + """ + @spec append(tuple, term) :: tuple + def append(tuple, value) do + :erlang.append_element(tuple, value) end @doc """ Removes an element from a tuple. - Deletes the element at the zero-based `index` from `tuple`. - Raises an `ArgumentError` if `index` is greater than - or equal to the length of `tuple`. + Deletes the element at the given `index` from `tuple`. + Raises an `ArgumentError` if `index` is negative or greater than + or equal to the length of `tuple`. Index is zero-based. Inlined by the compiler. @@ -60,14 +124,61 @@ defmodule Tuple do """ @spec delete_at(tuple, non_neg_integer) :: tuple - def delete_at(tuple, index) do + def delete_at(tuple, index) when is_integer(index) and index >= 0 do :erlang.delete_element(index + 1, tuple) end + @doc """ + Computes a sum of tuple elements. + + ## Examples + + iex> Tuple.sum({255, 255}) + 510 + iex> Tuple.sum({255, 0.0}) + 255.0 + iex> Tuple.sum({}) + 0 + """ + @doc since: "1.12.0" + @spec sum(tuple) :: number() + def sum(tuple), do: sum(tuple, tuple_size(tuple)) + + defp sum(_tuple, 0), do: 0 + defp sum(tuple, index), do: :erlang.element(index, tuple) + sum(tuple, index - 1) + + @doc """ + Computes a product of tuple elements. + + ## Examples + + iex> Tuple.product({255, 255}) + 65025 + iex> Tuple.product({255, 1.0}) + 255.0 + iex> Tuple.product({}) + 1 + """ + @doc since: "1.12.0" + @spec product(tuple) :: number() + def product(tuple), do: product(tuple, tuple_size(tuple)) + + defp product(_tuple, 0), do: 1 + defp product(tuple, index), do: :erlang.element(index, tuple) * product(tuple, index - 1) + @doc """ Converts a tuple to a list. + Returns a new list with all the tuple elements. + Inlined by the compiler. + + ## Examples + + iex> tuple = {:foo, :bar, :baz} + iex> Tuple.to_list(tuple) + [:foo, :bar, :baz] + """ @spec to_list(tuple) :: list def to_list(tuple) do diff --git a/lib/elixir/lib/uri.ex b/lib/elixir/lib/uri.ex index 9b918874d63..abb6fa4a8fe 100644 --- a/lib/elixir/lib/uri.ex +++ b/lib/elixir/lib/uri.ex @@ -1,39 +1,62 @@ defmodule URI do @moduledoc """ - Utilities for working with and creating URIs. + Utilities for working with URIs. + + This module provides functions for working with URIs (for example, parsing + URIs or encoding query strings). The functions in this module are implemented + according to [RFC 3986](https://tools.ietf.org/html/rfc3986). """ - defstruct scheme: nil, path: nil, query: nil, - fragment: nil, authority: nil, - userinfo: nil, host: nil, port: nil + @doc """ + The URI struct. - import Bitwise + The fields are defined to match the following URI representation + (with field names between brackets): - @ports %{ - "ftp" => 21, - "http" => 80, - "https" => 443, - "ldap" => 389, - "sftp" => 22, - "tftp" => 69, - } + [scheme]://[userinfo]@[host]:[port][path]?[query]#[fragment] - Enum.each @ports, fn {scheme, port} -> - def normalize_scheme(unquote(scheme)), do: unquote(scheme) - def default_port(unquote(scheme)), do: unquote(port) - end - @doc """ - Normalizes the scheme according to the spec by downcasing it. + Note the `authority` field is deprecated. `parse/1` will still + populate it for backwards compatibility but you should generally + avoid setting or getting it. """ - def normalize_scheme(nil), do: nil - def normalize_scheme(scheme), do: String.downcase(scheme) + @derive {Inspect, optional: [:authority]} + defstruct [:scheme, :authority, :userinfo, :host, :port, :path, :query, :fragment] + + @type t :: %__MODULE__{ + scheme: nil | binary, + authority: authority, + userinfo: nil | binary, + host: nil | binary, + port: nil | :inet.port_number(), + path: nil | binary, + query: nil | binary, + fragment: nil | binary + } + + @typedoc deprecated: "The authority field is deprecated" + @opaque authority :: nil | binary + + defmodule Error do + defexception [:action, :reason, :part] + + @doc false + def message(%Error{action: action, reason: reason, part: part}) do + "cannot #{action} due to reason #{reason}: #{inspect(part)}" + end + end + + import Bitwise + + @reserved_characters ':/?#[]@!$&\'()*+,;=' + @formatted_reserved_characters Enum.map_join(@reserved_characters, ", ", &<>) @doc """ - Returns the default port for a given scheme. + Returns the default port for a given `scheme`. - If the scheme is unknown to URI, returns `nil`. - Any scheme may be registered via `default_port/2`. + If the scheme is unknown to the `URI` module, this function returns + `nil`. The default port for any scheme can be configured globally + via `default_port/2`. ## Examples @@ -44,48 +67,111 @@ defmodule URI do nil """ + @spec default_port(binary) :: nil | non_neg_integer def default_port(scheme) when is_binary(scheme) do - {:ok, dict} = Application.fetch_env(:elixir, :uri) - Map.get(dict, scheme) + :elixir_config.get({:uri, scheme}, nil) end @doc """ - Registers a scheme with a default port. + Registers the default `port` for the given `scheme`. + + After this function is called, `port` will be returned by + `default_port/1` for the given scheme `scheme`. Note that this function + changes the default port for the given `scheme` *globally*, meaning for + every application. It is recommended for this function to be invoked in your - application start callback in case you want to register + application's start callback in case you want to register new URIs. """ - def default_port(scheme, port) when is_binary(scheme) and port > 0 do - {:ok, dict} = Application.fetch_env(:elixir, :uri) - Application.put_env(:elixir, :uri, Map.put(dict, scheme, port), persistent: true) + @spec default_port(binary, non_neg_integer) :: :ok + def default_port(scheme, port) when is_binary(scheme) and is_integer(port) and port >= 0 do + :elixir_config.put({:uri, scheme}, port) end @doc """ - Encodes an enumerable into a query string. + Encodes `enumerable` into a query string using `encoding`. - Takes an enumerable (containing a sequence of two-item tuples) - and returns a string of the form "key1=value1&key2=value2..." where - keys and values are URL encoded as per `encode/1`. + Takes an enumerable that enumerates as a list of two-element + tuples (for instance, a map or a keyword list) and returns a string + in the form of `key1=value1&key2=value2...`. Keys and values can be any term that implements the `String.Chars` - protocol, except lists which are explicitly forbidden. + protocol with the exception of lists, which are explicitly forbidden. + + You can specify one of the following `encoding` strategies: + + * `:www_form` - (default, since v1.12.0) keys and values are URL encoded as + per `encode_www_form/1`. This is the format typically used by browsers on + query strings and form data. It encodes " " as "+". + + * `:rfc3986` - (since v1.12.0) the same as `:www_form` except it encodes + " " as "%20" according [RFC 3986](https://tools.ietf.org/html/rfc3986). + This is the best option if you are encoding in a non-browser situation, + since encoding spaces as "+" can be ambiguous to URI parsers. This can + inadvertently lead to spaces being interpreted as literal plus signs. + + Encoding defaults to `:www_form` for backward compatibility. ## Examples - iex> hd = %{"foo" => 1, "bar" => 2} - iex> URI.encode_query(hd) + iex> query = %{"foo" => 1, "bar" => 2} + iex> URI.encode_query(query) "bar=2&foo=1" + iex> query = %{"key" => "value with spaces"} + iex> URI.encode_query(query) + "key=value+with+spaces" + + iex> query = %{"key" => "value with spaces"} + iex> URI.encode_query(query, :rfc3986) + "key=value%20with%20spaces" + + iex> URI.encode_query(%{key: [:a, :list]}) + ** (ArgumentError) encode_query/2 values cannot be lists, got: [:a, :list] + """ - def encode_query(l), do: Enum.map_join(l, "&", &pair/1) + @spec encode_query(Enumerable.t(), :rfc3986 | :www_form) :: binary + def encode_query(enumerable, encoding \\ :www_form) do + Enum.map_join(enumerable, "&", &encode_kv_pair(&1, encoding)) + end + + defp encode_kv_pair({key, _}, _encoding) when is_list(key) do + raise ArgumentError, "encode_query/2 keys cannot be lists, got: #{inspect(key)}" + end + + defp encode_kv_pair({_, value}, _encoding) when is_list(value) do + raise ArgumentError, "encode_query/2 values cannot be lists, got: #{inspect(value)}" + end + + defp encode_kv_pair({key, value}, :rfc3986) do + encode(Kernel.to_string(key), &char_unreserved?/1) <> + "=" <> encode(Kernel.to_string(value), &char_unreserved?/1) + end + + defp encode_kv_pair({key, value}, :www_form) do + encode_www_form(Kernel.to_string(key)) <> "=" <> encode_www_form(Kernel.to_string(value)) + end @doc """ - Decodes a query string into a dictionary (by default uses a map). + Decodes `query` into a map. + + Given a query string in the form of `key1=value1&key2=value2...`, this + function inserts each key-value pair in the query string as one entry in the + given `map`. Keys and values in the resulting map will be binaries. Keys and + values will be percent-unescaped. - Given a query string of the form "key1=value1&key2=value2...", produces a - map with one entry for each key-value pair. Each key and value will be a - binary. Keys and values will be percent-unescaped. + You can specify one of the following `encoding` options: + + * `:www_form` - (default, since v1.12.0) keys and values are decoded as per + `decode_www_form/1`. This is the format typically used by browsers on + query strings and form data. It decodes "+" as " ". + + * `:rfc3986` - (since v1.12.0) keys and values are decoded as per + `decode/1`. The result is the same as `:www_form` except for leaving "+" + as is in line with [RFC 3986](https://tools.ietf.org/html/rfc3986). + + Encoding defaults to `:www_form` for backward compatibility. Use `query_decoder/1` if you want to iterate over each value manually. @@ -94,107 +180,219 @@ defmodule URI do iex> URI.decode_query("foo=1&bar=2") %{"bar" => "2", "foo" => "1"} + iex> URI.decode_query("percent=oh+yes%21", %{"starting" => "map"}) + %{"percent" => "oh yes!", "starting" => "map"} + + iex> URI.decode_query("percent=oh+yes%21", %{}, :rfc3986) + %{"percent" => "oh+yes!"} + """ - def decode_query(q, dict \\ %{}) when is_binary(q) do - Enum.reduce query_decoder(q), dict, fn({k, v}, acc) -> Dict.put(acc, k, v) end + @spec decode_query(binary, %{optional(binary) => binary}, :rfc3986 | :www_form) :: %{ + optional(binary) => binary + } + def decode_query(query, map \\ %{}, encoding \\ :www_form) + + def decode_query(query, %_{} = dict, encoding) when is_binary(query) do + IO.warn( + "URI.decode_query/3 expects the second argument to be a map, other usage is deprecated" + ) + + decode_query_into_dict(query, dict, encoding) + end + + def decode_query(query, map, encoding) when is_binary(query) and is_map(map) do + decode_query_into_map(query, map, encoding) + end + + def decode_query(query, dict, encoding) when is_binary(query) do + IO.warn( + "URI.decode_query/3 expects the second argument to be a map, other usage is deprecated" + ) + + decode_query_into_dict(query, dict, encoding) + end + + defp decode_query_into_map(query, map, encoding) do + case decode_next_query_pair(query, encoding) do + nil -> + map + + {{key, value}, rest} -> + decode_query_into_map(rest, Map.put(map, key, value), encoding) + end + end + + defp decode_query_into_dict(query, dict, encoding) do + case decode_next_query_pair(query, encoding) do + nil -> + dict + + {{key, value}, rest} -> + # Avoid warnings about Dict being deprecated + dict_module = Dict + decode_query_into_dict(rest, dict_module.put(dict, key, value), encoding) + end end @doc """ - Returns an iterator function over the query string that decodes - the query string in steps. + Returns a stream of two-element tuples representing key-value pairs in the + given `query`. + + Key and value in each tuple will be binaries and will be percent-unescaped. + + You can specify one of the following `encoding` options: + + * `:www_form` - (default, since v1.12.0) keys and values are decoded as per + `decode_www_form/1`. This is the format typically used by browsers on + query strings and form data. It decodes "+" as " ". + + * `:rfc3986` - (since v1.12.0) keys and values are decoded as per + `decode/1`. The result is the same as `:www_form` except for leaving "+" + as is in line with [RFC 3986](https://tools.ietf.org/html/rfc3986). + + Encoding defaults to `:www_form` for backward compatibility. ## Examples - iex> URI.query_decoder("foo=1&bar=2") |> Enum.map &(&1) + iex> URI.query_decoder("foo=1&bar=2") |> Enum.to_list() [{"foo", "1"}, {"bar", "2"}] + iex> URI.query_decoder("food=bread%26butter&drinks=tap%20water+please") |> Enum.to_list() + [{"food", "bread&butter"}, {"drinks", "tap water please"}] + + iex> URI.query_decoder("food=bread%26butter&drinks=tap%20water+please", :rfc3986) |> Enum.to_list() + [{"food", "bread&butter"}, {"drinks", "tap water+please"}] + """ - def query_decoder(q) when is_binary(q) do - Stream.unfold(q, &do_decoder/1) + @spec query_decoder(binary, :rfc3986 | :www_form) :: Enumerable.t() + def query_decoder(query, encoding \\ :www_form) when is_binary(query) do + Stream.unfold(query, &decode_next_query_pair(&1, encoding)) end - defp do_decoder("") do + defp decode_next_query_pair("", _encoding) do nil end - defp do_decoder(q) do - {first, next} = - case :binary.split(q, "&") do - [first, rest] -> {first, rest} - [first] -> {first, ""} + defp decode_next_query_pair(query, encoding) do + {undecoded_next_pair, rest} = + case :binary.split(query, "&") do + [next_pair, rest] -> {next_pair, rest} + [next_pair] -> {next_pair, ""} end - current = - case :binary.split(first, "=") do + next_pair = + case :binary.split(undecoded_next_pair, "=") do [key, value] -> - {decode_www_form(key), decode_www_form(value)} + {decode_with_encoding(key, encoding), decode_with_encoding(value, encoding)} + [key] -> - {decode_www_form(key), nil} + {decode_with_encoding(key, encoding), ""} end - {current, next} + {next_pair, rest} end - defp pair({k, _}) when is_list(k) do - raise ArgumentError, "encode_query/1 keys cannot be lists, got: #{inspect k}" + defp decode_with_encoding(string, :www_form) do + decode_www_form(string) end - defp pair({_, v}) when is_list(v) do - raise ArgumentError, "encode_query/1 values cannot be lists, got: #{inspect v}" + defp decode_with_encoding(string, :rfc3986) do + decode(string) end - defp pair({k, v}) do - encode_www_form(to_string(k)) <> - "=" <> encode_www_form(to_string(v)) - end + @doc ~s""" + Checks if `character` is a reserved one in a URI. - @doc """ - Checks if the character is a "reserved" character in a URI. + As specified in [RFC 3986, section 2.2](https://tools.ietf.org/html/rfc3986#section-2.2), + the following characters are reserved: #{@formatted_reserved_characters} + + ## Examples + + iex> URI.char_reserved?(?+) + true - Reserved characters are specified in RFC3986, section 2.2. """ - def char_reserved?(c) do - c in ':/?#[]@!$&\'()*+,;=' + @spec char_reserved?(byte) :: boolean + def char_reserved?(character) do + character in @reserved_characters end @doc """ - Checks if the character is a "unreserved" character in a URI. + Checks if `character` is an unreserved one in a URI. + + As specified in [RFC 3986, section 2.3](https://tools.ietf.org/html/rfc3986#section-2.3), + the following characters are unreserved: + + * Alphanumeric characters: `A-Z`, `a-z`, `0-9` + * `~`, `_`, `-`, `.` + + ## Examples + + iex> URI.char_unreserved?(?_) + true - Unreserved characters are specified in RFC3986, section 2.3. """ - def char_unreserved?(c) do - c in ?0..?9 or - c in ?a..?z or - c in ?A..?Z or - c in '~_-.' + @spec char_unreserved?(byte) :: boolean + def char_unreserved?(character) do + character in ?0..?9 or character in ?a..?z or character in ?A..?Z or character in '~_-.' end @doc """ - Checks if the character is allowed unescaped in a URI. + Checks if `character` is allowed unescaped in a URI. This is the default used by `URI.encode/2` where both - reserved and unreserved characters are kept unescaped. + [reserved](`char_reserved?/1`) and [unreserved characters](`char_unreserved?/1`) + are kept unescaped. + + ## Examples + + iex> URI.char_unescaped?(?{) + false + """ - def char_unescaped?(c) do - char_reserved?(c) or char_unreserved?(c) + @spec char_unescaped?(byte) :: boolean + def char_unescaped?(character) do + char_reserved?(character) or char_unreserved?(character) end @doc """ - Percent-escape a URI. - Accepts `predicate` function as an argument to specify if char can be left as is. + Percent-escapes all characters that require escaping in `string`. - ## Example + This means reserved characters, such as `:` and `/`, and the + so-called unreserved characters, which have the same meaning both + escaped and unescaped, won't be escaped by default. + + See `encode_www_form/1` if you are interested in escaping reserved + characters too. + + This function also accepts a `predicate` function as an optional + argument. If passed, this function will be called with each byte + in `string` as its argument and should return a truthy value (anything other + than `false` or `nil`) if the given byte should be left as is, or return a + falsy value (`false` or `nil`) if the character should be escaped. Defaults + to `URI.char_unescaped?/1`. + + ## Examples iex> URI.encode("ftp://s-ite.tld/?value=put it+й") "ftp://s-ite.tld/?value=put%20it+%D0%B9" + iex> URI.encode("a string", &(&1 != ?i)) + "a str%69ng" + """ - def encode(str, predicate \\ &char_unescaped?/1) when is_binary(str) do - for <>, into: "", do: percent(c, predicate) + @spec encode(binary, (byte -> as_boolean(term))) :: binary + def encode(string, predicate \\ &char_unescaped?/1) + when is_binary(string) and is_function(predicate, 1) do + for <>, into: "", do: percent(byte, predicate) end @doc """ - Encode a string as "x-www-urlencoded". + Encodes `string` as "x-www-form-urlencoded". + + Note "x-www-form-urlencoded" is not specified as part of + RFC 3986. However, it is a commonly used format to encode + query strings and form data by browsers. ## Example @@ -202,44 +400,47 @@ defmodule URI do "put%3A+it%2B%D0%B9" """ - def encode_www_form(str) when is_binary(str) do - for <>, into: "" do - case percent(c, &char_unreserved?/1) do + @spec encode_www_form(binary) :: binary + def encode_www_form(string) when is_binary(string) do + for <>, into: "" do + case percent(byte, &char_unreserved?/1) do "%20" -> "+" - pct -> pct + percent -> percent end end end - defp percent(c, predicate) do - if predicate.(c) do - <> + defp percent(char, predicate) do + if predicate.(char) do + <> else - "%" <> hex(bsr(c, 4)) <> hex(band(c, 15)) + <<"%", hex(bsr(char, 4)), hex(band(char, 15))>> end end - defp hex(n) when n <= 9, do: <> - defp hex(n), do: <> + defp hex(n) when n <= 9, do: n + ?0 + defp hex(n), do: n + ?A - 10 @doc """ - Percent-unescape a URI. + Percent-unescapes a URI. ## Examples - iex> URI.decode("http%3A%2F%2Felixir-lang.org") - "http://elixir-lang.org" + iex> URI.decode("https%3A%2F%2Felixir-lang.org") + "https://elixir-lang.org" """ + @spec decode(binary) :: binary def decode(uri) do - unpercent(uri) - catch - :malformed_uri -> - raise ArgumentError, "malformed URI #{inspect uri}" + unpercent(uri, "", false) end @doc """ - Decode a string as "x-www-urlencoded". + Decodes `string` as "x-www-form-urlencoded". + + Note "x-www-form-urlencoded" is not specified as part of + RFC 3986. However, it is a commonly used format to encode + query strings and form data by browsers. ## Examples @@ -247,116 +448,568 @@ defmodule URI do " Enum.map_join(" ", &unpercent/1) - catch - :malformed_uri -> - raise ArgumentError, "malformed URI #{inspect str}" + @spec decode_www_form(binary) :: binary + def decode_www_form(string) when is_binary(string) do + unpercent(string, "", true) + end + + defp unpercent(<>, acc, spaces = true) do + unpercent(tail, <>, spaces) end - defp unpercent(<>) do - <> <> unpercent(tail) + defp unpercent(<>, acc, spaces) do + with <> <- tail, + dec1 when is_integer(dec1) <- hex_to_dec(hex1), + dec2 when is_integer(dec2) <- hex_to_dec(hex2) do + unpercent(tail, <>, spaces) + else + _ -> unpercent(tail, <>, spaces) + end end - defp unpercent(<>), do: throw(:malformed_uri) - defp unpercent(<>), do: throw(:malformed_uri) - defp unpercent(<>) do - <> <> unpercent(tail) + defp unpercent(<>, acc, spaces) do + unpercent(tail, <>, spaces) end - defp unpercent(<<>>), do: <<>> + defp unpercent(<<>>, acc, _spaces), do: acc + @compile {:inline, hex_to_dec: 1} defp hex_to_dec(n) when n in ?A..?F, do: n - ?A + 10 defp hex_to_dec(n) when n in ?a..?f, do: n - ?a + 10 defp hex_to_dec(n) when n in ?0..?9, do: n - ?0 - defp hex_to_dec(_n), do: throw(:malformed_uri) + defp hex_to_dec(_n), do: nil @doc """ - Parses a URI into components. + Creates a new URI struct from a URI or a string. - URIs have portions that are handled specially for the particular - scheme of the URI. For example, http and https have different - default ports. Such values can be accessed and registered via - `URI.default_port/1` and `URI.default_port/2`. + If a `%URI{}` struct is given, it returns `{:ok, uri}`. If a string is + given, it will parse and validate it. If the string is valid, it returns + `{:ok, uri}`, otherwise it returns `{:error, part}` with the invalid part + of the URI. For parsing URIs without further validation, see `parse/1`. - ## Examples + This function can parse both absolute and relative URLs. You can check + if a URI is absolute or relative by checking if the `scheme` field is + `nil` or not. - iex> URI.parse("http://elixir-lang.org/") - %URI{scheme: "http", path: "/", query: nil, fragment: nil, - authority: "elixir-lang.org", userinfo: nil, - host: "elixir-lang.org", port: 80} + When a URI is given without a port, the value returned by `URI.default_port/1` + for the URI's scheme is used for the `:port` field. The scheme is also + normalized to lowercase. + ## Examples + + iex> URI.new("https://elixir-lang.org/") + {:ok, %URI{ + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: 443, + query: nil, + scheme: "https", + userinfo: nil + }} + + iex> URI.new("//elixir-lang.org/") + {:ok, %URI{ + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + }} + + iex> URI.new("/foo/bar") + {:ok, %URI{ + fragment: nil, + host: nil, + path: "/foo/bar", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + }} + + iex> URI.new("foo/bar") + {:ok, %URI{ + fragment: nil, + host: nil, + path: "foo/bar", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + }} + + iex> URI.new("//[fe80::]/") + {:ok, %URI{ + fragment: nil, + host: "fe80::", + path: "/", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + }} + + iex> URI.new("https:?query") + {:ok, %URI{ + fragment: nil, + host: nil, + path: nil, + port: 443, + query: "query", + scheme: "https", + userinfo: nil + }} + + iex> URI.new("/invalid_greater_than_in_path/>") + {:error, ">"} + + Giving an existing URI simply returns it wrapped in a tuple: + + iex> {:ok, uri} = URI.new("https://elixir-lang.org/") + iex> URI.new(uri) + {:ok, %URI{ + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: 443, + query: nil, + scheme: "https", + userinfo: nil + }} """ - def parse(%URI{} = uri), do: uri + @doc since: "1.13.0" + @spec new(t() | String.t()) :: {:ok, t()} | {:error, String.t()} + def new(%URI{} = uri), do: {:ok, uri} + + def new(binary) when is_binary(binary) do + case :uri_string.parse(binary) do + %{} = map -> {:ok, uri_from_map(map)} + {:error, :invalid_uri, term} -> {:error, Kernel.to_string(term)} + end + end - def parse(s) when is_binary(s) do - # From http://tools.ietf.org/html/rfc3986#appendix-B - regex = ~r/^(([^:\/?#]+):)?(\/\/([^\/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?/ - parts = nillify(Regex.run(regex, s)) + @doc """ + Similar to `new/1` but raises `URI.Error` if an invalid string is given. + + ## Examples - destructure [_, _, scheme, _, authority, path, _, query, _, fragment], parts - {userinfo, host, port} = split_authority(authority) + iex> URI.new!("https://elixir-lang.org/") + %URI{ + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: 443, + query: nil, + scheme: "https", + userinfo: nil + } + + iex> URI.new!("/invalid_greater_than_in_path/>") + ** (URI.Error) cannot parse due to reason invalid_uri: ">" + + Giving an existing URI simply returns it: + + iex> uri = URI.new!("https://elixir-lang.org/") + iex> URI.new!(uri) + %URI{ + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: 443, + query: nil, + scheme: "https", + userinfo: nil + } + """ + @doc since: "1.13.0" + @spec new!(t() | String.t()) :: t() + def new!(%URI{} = uri), do: uri - if authority do - authority = "" + def new!(binary) when is_binary(binary) do + case :uri_string.parse(binary) do + %{} = map -> + uri_from_map(map) - if userinfo, do: authority = authority <> userinfo <> "@" - if host, do: authority = authority <> host - if port, do: authority = authority <> ":" <> Integer.to_string(port) + {:error, reason, part} -> + raise Error, action: :parse, reason: reason, part: Kernel.to_string(part) end + end + + defp uri_from_map(%{path: ""} = map), do: uri_from_map(%{map | path: nil}) - scheme = normalize_scheme(scheme) + defp uri_from_map(map) do + uri = Map.merge(%URI{}, map) - if nil?(port) and not nil?(scheme) do - port = default_port(scheme) + case map do + %{scheme: scheme} -> + scheme = String.downcase(scheme, :ascii) + + case map do + %{port: _} -> + %{uri | scheme: scheme} + + %{} -> + case default_port(scheme) do + nil -> %{uri | scheme: scheme} + port -> %{uri | scheme: scheme, port: port} + end + end + + %{} -> + uri end + end + + @doc """ + Parses a URI into its components, without further validation. + + This function can parse both absolute and relative URLs. You can check + if a URI is absolute or relative by checking if the `scheme` field is + nil or not. Furthermore, this function expects both absolute and + relative URIs to be well-formed and does not perform any validation. + See the "Examples" section below. Use `new/1` if you want to validate + the URI fields after parsing. + + When a URI is given without a port, the value returned by `URI.default_port/1` + for the URI's scheme is used for the `:port` field. The scheme is also + normalized to lowercase. + + If a `%URI{}` struct is given to this function, this function returns it + unmodified. + + > Note: this function sets the field :authority for backwards + > compatibility reasons but it is deprecated. + + ## Examples + + iex> URI.parse("https://elixir-lang.org/") + %URI{ + authority: "elixir-lang.org", + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: 443, + query: nil, + scheme: "https", + userinfo: nil + } + + iex> URI.parse("//elixir-lang.org/") + %URI{ + authority: "elixir-lang.org", + fragment: nil, + host: "elixir-lang.org", + path: "/", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + } + + iex> URI.parse("/foo/bar") + %URI{ + fragment: nil, + host: nil, + path: "/foo/bar", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + } + + iex> URI.parse("foo/bar") + %URI{ + fragment: nil, + host: nil, + path: "foo/bar", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + } + + In contrast to `URI.new/1`, this function will parse poorly-formed + URIs, for example: + + iex> URI.parse("/invalid_greater_than_in_path/>") + %URI{ + fragment: nil, + host: nil, + path: "/invalid_greater_than_in_path/>", + port: nil, + query: nil, + scheme: nil, + userinfo: nil + } + + Another example is a URI with brackets in query strings. It is accepted + by `parse/1`, it is commonly accepted by browsers, but it will be refused + by `new/1`: + + iex> URI.parse("/?foo[bar]=baz") + %URI{ + fragment: nil, + host: nil, + path: "/", + port: nil, + query: "foo[bar]=baz", + scheme: nil, + userinfo: nil + } + + """ + @spec parse(t | binary) :: t + def parse(%URI{} = uri), do: uri + + def parse(string) when is_binary(string) do + # From https://tools.ietf.org/html/rfc3986#appendix-B + # Parts: 12 3 4 5 6 7 8 9 + regex = ~r{^(([a-z][a-z0-9\+\-\.]*):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?}i + + parts = Regex.run(regex, string) + + destructure [ + _full, + # 1 + _scheme_with_colon, + # 2 + scheme, + # 3 + authority_with_slashes, + # 4 + _authority, + # 5 + path, + # 6 + query_with_question_mark, + # 7 + _query, + # 8 + _fragment_with_hash, + # 9 + fragment + ], + parts + + path = nilify(path) + scheme = nilify(scheme) + query = nilify_query(query_with_question_mark) + {authority, userinfo, host, port} = split_authority(authority_with_slashes) + + scheme = scheme && String.downcase(scheme) + port = port || (scheme && default_port(scheme)) %URI{ - scheme: scheme, path: path, query: query, - fragment: fragment, authority: authority, - userinfo: userinfo, host: host, port: port + scheme: scheme, + path: path, + query: query, + fragment: fragment, + authority: authority, + userinfo: userinfo, + host: host, + port: port } end + defp nilify_query("?" <> query), do: query + defp nilify_query(_other), do: nil + # Split an authority into its userinfo, host and port parts. - defp split_authority(s) do - s = s || "" - components = Regex.run ~r/(^(.*)@)?(\[[a-zA-Z0-9:.]*\]|[^:]*)(:(\d*))?/, s + # + # Note that the host field is returned *without* [] even if, according to + # RFC3986 grammar, a native IPv6 address requires them. + defp split_authority("") do + {nil, nil, nil, nil} + end + + defp split_authority("//") do + {"", nil, "", nil} + end + + defp split_authority("//" <> authority) do + regex = ~r/(^(.*)@)?(\[[a-zA-Z0-9:.]*\]|[^:]*)(:(\d*))?/ + components = Regex.run(regex, authority) - destructure [_, _, userinfo, host, _, port], nillify(components) - port = if port, do: String.to_integer(port) - host = if host, do: host |> String.lstrip(?[) |> String.rstrip(?]) + destructure [_, _, userinfo, host, _, port], components + userinfo = nilify(userinfo) + host = if nilify(host), do: host |> String.trim_leading("[") |> String.trim_trailing("]") + port = if nilify(port), do: String.to_integer(port) - {userinfo, host, port} + {authority, userinfo, host, port} end # Regex.run returns empty strings sometimes. We want # to replace those with nil for consistency. - defp nillify(l) do - for s <- l do - if byte_size(s) > 0, do: s, else: nil - end + defp nilify(""), do: nil + defp nilify(other), do: other + + @doc """ + Returns the string representation of the given [URI struct](`t:t/0`). + + ## Examples + + iex> uri = URI.parse("http://google.com") + iex> URI.to_string(uri) + "http://google.com" + + iex> uri = URI.parse("foo://bar.baz") + iex> URI.to_string(uri) + "foo://bar.baz" + + """ + @spec to_string(t) :: binary + defdelegate to_string(uri), to: String.Chars.URI + + @doc ~S""" + Merges two URIs. + + This function merges two URIs as per + [RFC 3986, section 5.2](https://tools.ietf.org/html/rfc3986#section-5.2). + + ## Examples + + iex> URI.merge(URI.parse("http://google.com"), "/query") |> to_string() + "http://google.com/query" + + iex> URI.merge("http://example.com", "http://google.com") |> to_string() + "http://google.com" + + """ + @spec merge(t | binary, t | binary) :: t + def merge(uri, rel) + + def merge(%URI{host: nil}, _rel) do + raise ArgumentError, "you must merge onto an absolute URI" + end + + def merge(_base, %URI{scheme: rel_scheme} = rel) when rel_scheme != nil do + %{rel | path: remove_dot_segments_from_path(rel.path)} + end + + def merge(base, %URI{host: host} = rel) when host != nil do + %{rel | scheme: base.scheme, path: remove_dot_segments_from_path(rel.path)} + end + + def merge(%URI{} = base, %URI{path: nil} = rel) do + %{base | query: rel.query || base.query, fragment: rel.fragment} + end + + def merge(%URI{} = base, %URI{} = rel) do + new_path = merge_paths(base.path, rel.path) + %{base | path: new_path, query: rel.query, fragment: rel.fragment} + end + + def merge(base, rel) do + merge(parse(base), parse(rel)) + end + + defp merge_paths(nil, rel_path), do: merge_paths("/", rel_path) + defp merge_paths(_, "/" <> _ = rel_path), do: remove_dot_segments_from_path(rel_path) + + defp merge_paths(base_path, rel_path) do + [_ | base_segments] = path_to_segments(base_path) + + path_to_segments(rel_path) + |> Kernel.++(base_segments) + |> remove_dot_segments([]) + |> Enum.join("/") + end + + defp remove_dot_segments_from_path(nil) do + nil + end + + defp remove_dot_segments_from_path(path) do + path + |> path_to_segments() + |> remove_dot_segments([]) + |> Enum.join("/") + end + + defp remove_dot_segments([], [head, ".." | acc]), do: remove_dot_segments([], [head | acc]) + defp remove_dot_segments([], acc), do: acc + defp remove_dot_segments(["." | tail], acc), do: remove_dot_segments(tail, acc) + + defp remove_dot_segments([head | tail], ["..", ".." | _] = acc), + do: remove_dot_segments(tail, [head | acc]) + + defp remove_dot_segments(segments, [_, ".." | acc]), do: remove_dot_segments(segments, acc) + defp remove_dot_segments([head | tail], acc), do: remove_dot_segments(tail, [head | acc]) + + defp path_to_segments(path) do + path |> String.split("/") |> Enum.reverse() + end + + @doc """ + Appends `query` to the given `uri`. + + The given `query` is not automatically encoded, use `encode/2` or `encode_www_form/1`. + + ## Examples + + iex> URI.append_query(URI.parse("http://example.com/"), "x=1") |> URI.to_string() + "http://example.com/?x=1" + + iex> URI.append_query(URI.parse("http://example.com/?x=1"), "y=2") |> URI.to_string() + "http://example.com/?x=1&y=2" + + iex> URI.append_query(URI.parse("http://example.com/?x=1"), "x=2") |> URI.to_string() + "http://example.com/?x=1&x=2" + """ + @doc since: "1.14.0" + @spec append_query(t(), binary()) :: t() + def append_query(%URI{} = uri, query) when is_binary(query) and uri.query in [nil, ""] do + %{uri | query: query} + end + + def append_query(%URI{} = uri, query) when is_binary(query) do + %{uri | query: uri.query <> "&" <> query} end end defimpl String.Chars, for: URI do - def to_string(uri) do - scheme = uri.scheme + def to_string(%{host: host, path: path} = uri) + when host != nil and is_binary(path) and + path != "" and binary_part(path, 0, 1) != "/" do + raise ArgumentError, + ":path in URI must be empty or an absolute path if URL has a :host, got: #{inspect(uri)}" + end - if scheme && (port = URI.default_port(scheme)) do - if uri.port == port, do: uri = %{uri | port: nil} - end + def to_string(%{scheme: scheme, port: port, path: path, query: query, fragment: fragment} = uri) do + uri = + case scheme && URI.default_port(scheme) do + ^port -> %{uri | port: nil} + _ -> uri + end - result = "" + # Based on https://tools.ietf.org/html/rfc3986#section-5.3 + authority = extract_authority(uri) - if uri.scheme, do: result = result <> uri.scheme <> "://" - if uri.userinfo, do: result = result <> uri.userinfo <> "@" - if uri.host, do: result = result <> uri.host - if uri.port, do: result = result <> ":" <> Integer.to_string(uri.port) - if uri.path, do: result = result <> uri.path - if uri.query, do: result = result <> "?" <> uri.query - if uri.fragment, do: result = result <> "#" <> uri.fragment + IO.iodata_to_binary([ + if(scheme, do: [scheme, ?:], else: []), + if(authority, do: ["//" | authority], else: []), + if(path, do: path, else: []), + if(query, do: ["?" | query], else: []), + if(fragment, do: ["#" | fragment], else: []) + ]) + end + + defp extract_authority(%{host: nil, authority: authority}) do + authority + end - result + defp extract_authority(%{host: host, userinfo: userinfo, port: port}) do + # According to the grammar at + # https://tools.ietf.org/html/rfc3986#appendix-A, a "host" can have a colon + # in it only if it's an IPv6 or "IPvFuture" address, so if there's a colon + # in the host we can safely surround it with []. + [ + if(userinfo, do: [userinfo | "@"], else: []), + if(String.contains?(host, ":"), do: ["[", host | "]"], else: host), + if(port, do: [":" | Integer.to_string(port)], else: []) + ] end end diff --git a/lib/elixir/lib/version.ex b/lib/elixir/lib/version.ex index 19289de4c8f..61e9d760234 100644 --- a/lib/elixir/lib/version.ex +++ b/lib/elixir/lib/version.ex @@ -5,35 +5,34 @@ defmodule Version do A version is a string in a specific format or a `Version` generated after parsing via `Version.parse/1`. - `Version` parsing and requirements follow - [SemVer 2.0 schema](http://semver.org/). + Although Elixir projects are not required to follow SemVer, + they must follow the format outlined on [SemVer 2.0 schema](https://semver.org/). ## Versions - In a nutshell, a version is given by three numbers: + In a nutshell, a version is represented by three numbers: MAJOR.MINOR.PATCH - Pre-releases are supported by appending `-[0-9A-Za-z-\.]`: + Pre-releases are supported by optionally appending a hyphen and a series of + period-separated identifiers immediately following the patch version. + Identifiers consist of only ASCII alphanumeric characters and hyphens (`[0-9A-Za-z-]`): "1.0.0-alpha.3" - Build information can be added by appending `+[0-9A-Za-z-\.]`: + Build information can be added by appending a plus sign and a series of + dot-separated identifiers immediately following the patch or pre-release version. + Identifiers consist of only ASCII alphanumeric characters and hyphens (`[0-9A-Za-z-]`): - "1.0.0-alpha.3+20130417140000" - - ## Struct - - The version is represented by the Version struct and it has its - fields named according to Semver: `:major`, `:minor`, `:patch`, - `:pre` and `:build`. + "1.0.0-alpha.3+20130417140000.amd64" ## Requirements Requirements allow you to specify which versions of a given - dependency you are willing to work against. It supports common - operators like `>=`, `<=`, `>`, `==` and friends that - work as one would expect: + dependency you are willing to work against. Requirements support the common + comparison operators such as `>`, `>=`, `<`, `<=`, and `==` that work as one + would expect, and additionally the special operator `~>` described in detail + further below. # Only version 2.0.0 "== 2.0.0" @@ -51,99 +50,299 @@ defmodule Version do "~> 2.0.0" + `~>` will never include pre-release versions of its upper bound, + regardless of the usage of the `:allow_pre` option, or whether the operand + is a pre-release version. It can also be used to set an upper bound on only the major + version part. See the table below for `~>` requirements and + their corresponding translations. + + `~>` | Translation + :------------- | :--------------------- + `~> 2.0.0` | `>= 2.0.0 and < 2.1.0` + `~> 2.1.2` | `>= 2.1.2 and < 2.2.0` + `~> 2.1.3-dev` | `>= 2.1.3-dev and < 2.2.0` + `~> 2.0` | `>= 2.0.0 and < 3.0.0` + `~> 2.1` | `>= 2.1.0 and < 3.0.0` + + The requirement operand after the `~>` is allowed to omit the patch version, + allowing us to express `~> 2.1` or `~> 2.1-dev`, something that wouldn't be allowed + when using the common comparison operators. + + When the `:allow_pre` option is set `false` in `Version.match?/3`, the requirement + will not match a pre-release version unless the operand is a pre-release version. + The default is to always allow pre-releases but note that in + Hex `:allow_pre` is set to `false`. See the table below for examples. + + Requirement | Version | `:allow_pre` | Matches + :------------- | :---------- | :---------------- | :------ + `~> 2.0` | `2.1.0` | `true` or `false` | `true` + `~> 2.0` | `3.0.0` | `true` or `false` | `false` + `~> 2.0.0` | `2.0.5` | `true` or `false` | `true` + `~> 2.0.0` | `2.1.0` | `true` or `false` | `false` + `~> 2.1.2` | `2.1.6-dev` | `true` | `true` + `~> 2.1.2` | `2.1.6-dev` | `false` | `false` + `~> 2.1-dev` | `2.2.0-dev` | `true` or `false` | `true` + `~> 2.1.2-dev` | `2.1.6-dev` | `true` or `false` | `true` + `>= 2.1.0` | `2.2.0-dev` | `true` | `true` + `>= 2.1.0` | `2.2.0-dev` | `false` | `false` + `>= 2.1.0-dev` | `2.2.6-dev` | `true` or `false` | `true` + """ import Kernel, except: [match?: 2] - defstruct [:major, :minor, :patch, :pre, :build] - @type version :: String.t | t - @type requirement :: String.t | Version.Requirement.t - @type matchable :: {major :: String.t | non_neg_integer, - minor :: non_neg_integer | nil, - patch :: non_neg_integer | nil, - pre :: [String.t]} + @doc """ + The Version struct. + + It contains the fields `:major`, `:minor`, `:patch`, `:pre`, and + `:build` according to SemVer 2.0, where `:pre` is a list. + + You can read those fields but you should not create a new `Version` + directly via the struct syntax. Instead use the functions in this + module. + """ + @enforce_keys [:major, :minor, :patch] + @derive {Inspect, optional: [:pre, :build]} + defstruct [:major, :minor, :patch, pre: [], build: nil] + + @type version :: String.t() | t + @type requirement :: String.t() | Version.Requirement.t() + @type major :: non_neg_integer + @type minor :: non_neg_integer + @type patch :: non_neg_integer + @type pre :: [String.t() | non_neg_integer] + @type build :: String.t() | nil + @type t :: %__MODULE__{major: major, minor: minor, patch: patch, pre: pre, build: build} defmodule Requirement do - @moduledoc false - defstruct [:source, :matchspec] + @moduledoc """ + A struct that holds version requirement information. + + The struct fields are private and should not be accessed. + + See the "Requirements" section in the `Version` module + for more information. + """ + + defstruct [:source, :lexed] + + @opaque t :: %__MODULE__{ + source: String.t(), + lexed: [atom | matchable] + } + + @typep matchable :: + {Version.major(), Version.minor(), Version.patch(), Version.pre(), Version.build()} + + @compile inline: [compare: 2] + + @doc false + @spec new(String.t(), [atom | matchable]) :: t + def new(source, lexed) do + %__MODULE__{source: source, lexed: lexed} + end + + @doc false + @spec compile_requirement(t) :: t + def compile_requirement(%Requirement{} = requirement) do + requirement + end + + @doc false + @spec match?(t, tuple) :: boolean + def match?(%__MODULE__{lexed: [operator, req | rest]}, version) do + match_lexed?(rest, version, match_op?(operator, req, version)) + end + + defp match_lexed?([:and, operator, req | rest], version, acc), + do: match_lexed?(rest, version, acc and match_op?(operator, req, version)) + + defp match_lexed?([:or, operator, req | rest], version, acc), + do: acc or match_lexed?(rest, version, match_op?(operator, req, version)) + + defp match_lexed?([], _version, acc), + do: acc + + defp match_op?(:==, req, version) do + compare(version, req) == :eq + end + + defp match_op?(:!=, req, version) do + compare(version, req) != :eq + end + + defp match_op?(:~>, {major, minor, nil, req_pre, _}, {_, _, _, pre, allow_pre} = version) do + compare(version, {major, minor, 0, req_pre, nil}) in [:eq, :gt] and + compare(version, {major + 1, 0, 0, [0], nil}) == :lt and + (allow_pre or req_pre != [] or pre == []) + end + + defp match_op?(:~>, {major, minor, _, req_pre, _} = req, {_, _, _, pre, allow_pre} = version) do + compare(version, req) in [:eq, :gt] and + compare(version, {major, minor + 1, 0, [0], nil}) == :lt and + (allow_pre or req_pre != [] or pre == []) + end + + defp match_op?(:>, {_, _, _, req_pre, _} = req, {_, _, _, pre, allow_pre} = version) do + compare(version, req) == :gt and (allow_pre or req_pre != [] or pre == []) + end + + defp match_op?(:>=, {_, _, _, req_pre, _} = req, {_, _, _, pre, allow_pre} = version) do + compare(version, req) in [:eq, :gt] and (allow_pre or req_pre != [] or pre == []) + end + + defp match_op?(:<, req, version) do + compare(version, req) == :lt + end + + defp match_op?(:<=, req, version) do + compare(version, req) in [:eq, :lt] + end + + defp compare({major1, minor1, patch1, pre1, _}, {major2, minor2, patch2, pre2, _}) do + cond do + major1 > major2 -> :gt + major1 < major2 -> :lt + minor1 > minor2 -> :gt + minor1 < minor2 -> :lt + patch1 > patch2 -> :gt + patch1 < patch2 -> :lt + pre1 == [] and pre2 != [] -> :gt + pre1 != [] and pre2 == [] -> :lt + pre1 > pre2 -> :gt + pre1 < pre2 -> :lt + true -> :eq + end + end end defmodule InvalidRequirementError do - defexception [:message] + defexception [:requirement] + + @impl true + def exception(requirement) when is_binary(requirement) do + %__MODULE__{requirement: requirement} + end + + @impl true + def message(%{requirement: requirement}) do + "invalid requirement: #{inspect(requirement)}" + end end defmodule InvalidVersionError do - defexception [:message] + defexception [:version] + + @impl true + def exception(version) when is_binary(version) do + %__MODULE__{version: version} + end + + @impl true + def message(%{version: version}) do + "invalid version: #{inspect(version)}" + end end @doc """ - Check if the given version matches the specification. + Checks if the given version matches the specification. Returns `true` if `version` satisfies `requirement`, `false` otherwise. Raises a `Version.InvalidRequirementError` exception if `requirement` is not - parseable, or `Version.InvalidVersionError` if `version` is not parseable. + parsable, or a `Version.InvalidVersionError` exception if `version` is not parsable. If given an already parsed version and requirement this function won't raise. + ## Options + + * `:allow_pre` (boolean) - when `false`, pre-release versions will not match + unless the operand is a pre-release version. Defaults to `true`. + For examples, please refer to the table above under the "Requirements" section. + ## Examples - iex> Version.match?("2.0.0", ">1.0.0") + iex> Version.match?("2.0.0", "> 1.0.0") + true + + iex> Version.match?("2.0.0", "== 1.0.0") + false + + iex> Version.match?("2.1.6-dev", "~> 2.1.2") true - iex> Version.match?("2.0.0", "==1.0.0") + iex> Version.match?("2.1.6-dev", "~> 2.1.2", allow_pre: false) false - iex> Version.match?("foo", "==1.0.0") - ** (Version.InvalidVersionError) foo + iex> Version.match?("foo", "== 1.0.0") + ** (Version.InvalidVersionError) invalid version: "foo" - iex> Version.match?("2.0.0", "== ==1.0.0") - ** (Version.InvalidRequirementError) == ==1.0.0 + iex> Version.match?("2.0.0", "== == 1.0.0") + ** (Version.InvalidRequirementError) invalid requirement: "== == 1.0.0" """ - @spec match?(version, requirement) :: boolean - def match?(vsn, req) when is_binary(req) do - case parse_requirement(req) do - {:ok, req} -> - match?(vsn, req) - :error -> - raise InvalidRequirementError, message: req - end + @spec match?(version, requirement, keyword) :: boolean + def match?(version, requirement, opts \\ []) + + def match?(version, requirement, opts) when is_binary(requirement) do + match?(version, parse_requirement!(requirement), opts) end - def match?(version, %Requirement{matchspec: spec}) do - {:ok, result} = :ets.test_ms(to_matchable(version), spec) - result != false + def match?(version, requirement, opts) do + allow_pre = Keyword.get(opts, :allow_pre, true) + matchable_pattern = to_matchable(version, allow_pre) + + Requirement.match?(requirement, matchable_pattern) end @doc """ - Compares two versions. Returns `:gt` if first version is greater than - the second and `:lt` for vice versa. If the two versions are equal `:eq` - is returned + Compares two versions. + + Returns `:gt` if the first version is greater than the second one, and `:lt` + for vice versa. If the two versions are equal, `:eq` is returned. - Raises a `Version.InvalidVersionError` exception if `version` is not parseable. - If given an already parsed version this function won't raise. + Pre-releases are strictly less than their corresponding release versions. + + Patch segments are compared lexicographically if they are alphanumeric, and + numerically otherwise. + + Build segments are ignored: if two versions differ only in their build segment + they are considered to be equal. + + Raises a `Version.InvalidVersionError` exception if any of the two given + versions are not parsable. If given an already parsed version this function + won't raise. ## Examples iex> Version.compare("2.0.1-alpha1", "2.0.0") :gt + iex> Version.compare("1.0.0-beta", "1.0.0-rc1") + :lt + + iex> Version.compare("1.0.0-10", "1.0.0-2") + :gt + iex> Version.compare("2.0.1+build0", "2.0.1") :eq iex> Version.compare("invalid", "2.0.1") - ** (Version.InvalidVersionError) invalid + ** (Version.InvalidVersionError) invalid version: "invalid" """ @spec compare(version, version) :: :gt | :eq | :lt - def compare(vsn1, vsn2) do - do_compare(to_matchable(vsn1), to_matchable(vsn2)) + def compare(version1, version2) do + do_compare(to_matchable(version1, true), to_matchable(version2, true)) end - defp do_compare({major1, minor1, patch1, pre1}, {major2, minor2, patch2, pre2}) do + defp do_compare({major1, minor1, patch1, pre1, _}, {major2, minor2, patch2, pre2, _}) do cond do - {major1, minor1, patch1} > {major2, minor2, patch2} -> :gt - {major1, minor1, patch1} < {major2, minor2, patch2} -> :lt + major1 > major2 -> :gt + major1 < major2 -> :lt + minor1 > minor2 -> :gt + minor1 < minor2 -> :lt + patch1 > patch2 -> :gt + patch1 < patch2 -> :lt pre1 == [] and pre2 != [] -> :gt pre1 != [] and pre2 == [] -> :lt pre1 > pre2 -> :gt @@ -153,355 +352,318 @@ defmodule Version do end @doc """ - Parse a version string into a `Version`. + Parses a version string into a `Version` struct. ## Examples - iex> Version.parse("2.0.1-alpha1") |> elem(1) - #Version<2.0.1-alpha1> + iex> Version.parse("2.0.1-alpha1") + {:ok, %Version{major: 2, minor: 0, patch: 1, pre: ["alpha1"]}} iex> Version.parse("2.0-alpha1") :error """ - @spec parse(String.t) :: {:ok, t} | :error + @spec parse(String.t()) :: {:ok, t} | :error def parse(string) when is_binary(string) do case Version.Parser.parse_version(string) do - {:ok, {major, minor, patch, pre}} -> - vsn = %Version{major: major, minor: minor, patch: patch, - pre: pre, build: get_build(string)} - {:ok, vsn} - :error -> - :error + {:ok, {major, minor, patch, pre, build_parts}} -> + build = if build_parts == [], do: nil, else: Enum.join(build_parts, ".") + version = %Version{major: major, minor: minor, patch: patch, pre: pre, build: build} + {:ok, version} + + :error -> + :error + end + end + + @doc """ + Parses a version string into a `Version`. + + If `string` is an invalid version, a `Version.InvalidVersionError` is raised. + + ## Examples + + iex> Version.parse!("2.0.1-alpha1") + %Version{major: 2, minor: 0, patch: 1, pre: ["alpha1"]} + + iex> Version.parse!("2.0-alpha1") + ** (Version.InvalidVersionError) invalid version: "2.0-alpha1" + + """ + @spec parse!(String.t()) :: t + def parse!(string) when is_binary(string) do + case parse(string) do + {:ok, version} -> version + :error -> raise InvalidVersionError, string end end @doc """ - Parse a version requirement string into a `Version.Requirement`. + Parses a version requirement string into a `Version.Requirement` struct. ## Examples - iex> Version.parse_requirement("== 2.0.1") |> elem(1) - #Version.Requirement<== 2.0.1> + iex> {:ok, requirement} = Version.parse_requirement("== 2.0.1") + iex> requirement + Version.parse_requirement!("== 2.0.1") iex> Version.parse_requirement("== == 2.0.1") :error """ - @spec parse_requirement(String.t) :: {:ok, Requirement.t} | :error + @spec parse_requirement(String.t()) :: {:ok, Requirement.t()} | :error def parse_requirement(string) when is_binary(string) do case Version.Parser.parse_requirement(string) do - {:ok, spec} -> - {:ok, %Requirement{source: string, matchspec: spec}} - :error -> - :error + {:ok, lexed} -> {:ok, Requirement.new(string, lexed)} + :error -> :error end end - defp to_matchable(%Version{major: major, minor: minor, patch: patch, pre: pre}) do - {major, minor, patch, pre} - end + @doc """ + Parses a version requirement string into a `Version.Requirement` struct. - defp to_matchable(string) do - case Version.Parser.parse_version(string) do - {:ok, vsn} -> vsn - :error -> raise InvalidVersionError, message: string - end - end + If `string` is an invalid requirement, a `Version.InvalidRequirementError` is raised. - defp get_build(string) do - case Regex.run(~r/\+([^\s]+)$/, string) do - nil -> - nil + # Examples - [_, build] -> - build - end - end + iex> Version.parse_requirement!("== 2.0.1") + Version.parse_requirement!("== 2.0.1") - defmodule Parser.DSL do - @moduledoc false + iex> Version.parse_requirement!("== == 2.0.1") + ** (Version.InvalidRequirementError) invalid requirement: "== == 2.0.1" - defmacro deflexer(match, do: body) when is_binary(match) do - quote do - def lexer(unquote(match) <> rest, acc) do - lexer(rest, [unquote(body) | acc]) - end - end + """ + @doc since: "1.8.0" + @spec parse_requirement!(String.t()) :: Requirement.t() + def parse_requirement!(string) when is_binary(string) do + case parse_requirement(string) do + {:ok, requirement} -> requirement + :error -> raise InvalidRequirementError, string end + end - defmacro deflexer(acc, do: body) do - quote do - def lexer("", unquote(acc)) do - unquote(body) - end - end - end + @doc """ + Compiles a requirement to an internal representation that may optimize matching. - defmacro deflexer(char, acc, do: body) do - quote do - def lexer(<< unquote(char) :: utf8, rest :: binary >>, unquote(acc)) do - unquote(char) = << unquote(char) :: utf8 >> + The internal representation is opaque. + """ + @spec compile_requirement(Requirement.t()) :: Requirement.t() + defdelegate compile_requirement(requirement), to: Requirement - lexer(rest, unquote(body)) - end - end - end + defp to_matchable(%Version{major: major, minor: minor, patch: patch, pre: pre}, allow_pre?) do + {major, minor, patch, pre, allow_pre?} end - defmodule Parser do - @moduledoc false - import Parser.DSL - - deflexer ">=", do: :'>=' - deflexer "<=", do: :'<=' - deflexer "~>", do: :'~>' - deflexer ">", do: :'>' - deflexer "<", do: :'<' - deflexer "==", do: :'==' - deflexer "!=", do: :'!=' - deflexer "!", do: :'!=' - deflexer " or ", do: :'||' - deflexer " and ", do: :'&&' - deflexer " ", do: :' ' + defp to_matchable(string, allow_pre?) do + case Version.Parser.parse_version(string) do + {:ok, {major, minor, patch, pre, _build_parts}} -> + {major, minor, patch, pre, allow_pre?} - deflexer x, [] do - [x, :'=='] + :error -> + raise InvalidVersionError, string end + end - deflexer x, [h | acc] do - cond do - is_binary h -> - [h <> x | acc] + @doc """ + Converts the given version to a string. - h in [:'||', :'&&'] -> - [x, :'==', h | acc] + ### Examples - true -> - [x, h | acc] - end - end + iex> Version.to_string(%Version{major: 1, minor: 2, patch: 3}) + "1.2.3" + iex> Version.to_string(Version.parse!("1.14.0-rc.0+build0")) + "1.14.0-rc.0+build0" + """ + @doc since: "1.14.0" + @spec to_string(Version.t()) :: String.t() + def to_string(%Version{} = version) do + pre = pre_to_string(version.pre) + build = if build = version.build, do: "+#{build}" + "#{version.major}.#{version.minor}.#{version.patch}#{pre}#{build}" + end - deflexer acc do - Enum.filter(Enum.reverse(acc), &(&1 != :' ')) - end + defp pre_to_string([]) do + "" + end - @version_regex ~r/^ - (\d+) # major - (?:\.(\d+))? # minor - (?:\.(\d+))? # patch - (?:\-([\d\w\.\-]+))? # pre - (?:\+([\d\w\-]+))? # build - $/x + defp pre_to_string(pre) do + "-" <> + Enum.map_join(pre, ".", fn + int when is_integer(int) -> Integer.to_string(int) + string when is_binary(string) -> string + end) + end - @spec parse_requirement(String.t) :: {:ok, Version.Requirement.t} | :error - def parse_requirement(source) do - lexed = lexer(source, []) - to_matchspec(lexed) - end + defmodule Parser do + @moduledoc false - defp nillify(""), do: nil - defp nillify(o), do: o + operators = [ + {">=", :>=}, + {"<=", :<=}, + {"~>", :~>}, + {">", :>}, + {"<", :<}, + {"==", :==}, + {" or ", :or}, + {" and ", :and} + ] - @spec parse_version(String.t) :: {:ok, Version.matchable} | :error - def parse_version(string, approximate? \\ false) when is_binary(string) do - if parsed = Regex.run(@version_regex, string) do - destructure [_, major, minor, patch, pre], parsed - patch = nillify(patch) - pre = nillify(pre) - - if nil?(minor) or (nil?(patch) and not approximate?) do - :error - else - major = String.to_integer(major) - minor = String.to_integer(minor) - patch = patch && String.to_integer(patch) - - case parse_pre(pre) do - {:ok, pre} -> - {:ok, {major, minor, patch, pre}} - :error -> - :error - end - end - else - :error - end + def lexer(string) do + lexer(string, "", []) end - defp parse_pre(nil), do: {:ok, []} - defp parse_pre(pre), do: parse_pre(String.split(pre, "."), []) - - defp parse_pre([piece|t], acc) do - cond do - piece =~ ~r/^(0|[1-9][0-9]*)$/ -> - parse_pre(t, [String.to_integer(piece)|acc]) - piece =~ ~r/^[0-9]*$/ -> - :error - true -> - parse_pre(t, [piece|acc]) + for {string_op, atom_op} <- operators do + defp lexer(unquote(string_op) <> rest, buffer, acc) do + lexer(rest, "", [unquote(atom_op) | maybe_prepend_buffer(buffer, acc)]) end end - defp parse_pre([], acc) do - {:ok, Enum.reverse(acc)} + defp lexer("!=" <> rest, buffer, acc) do + IO.warn("!= inside Version requirements is deprecated, use ~> or >= instead") + lexer(rest, "", [:!= | maybe_prepend_buffer(buffer, acc)]) end - defp valid_requirement?([]), do: false - defp valid_requirement?([a | next]), do: valid_requirement?(a, next) - - # it must finish with a version - defp valid_requirement?(a, []) when is_binary(a) do - true + defp lexer("!" <> rest, buffer, acc) do + IO.warn("! inside Version requirements is deprecated, use ~> or >= instead") + lexer(rest, "", [:!= | maybe_prepend_buffer(buffer, acc)]) end - # or | and - defp valid_requirement?(a, [b | next]) when is_atom(a) and is_atom(b) and a in [:'||', :'&&'] do - valid_requirement?(b, next) + defp lexer(" " <> rest, buffer, acc) do + lexer(rest, "", maybe_prepend_buffer(buffer, acc)) end - # or | and - defp valid_requirement?(a, [b | next]) when is_binary(a) and is_atom(b) and b in [:'||', :'&&'] do - valid_requirement?(b, next) + defp lexer(<>, buffer, acc) do + lexer(rest, <>, acc) end - # or | and - defp valid_requirement?(a, [b | next]) when is_atom(a) and is_binary(b) and a in [:'||', :'&&'] do - valid_requirement?(b, next) + defp lexer(<<>>, buffer, acc) do + maybe_prepend_buffer(buffer, acc) end - # - defp valid_requirement?(a, [b | next]) when is_atom(a) and is_binary(b) do - valid_requirement?(b, next) - end + defp maybe_prepend_buffer("", acc), do: acc - defp valid_requirement?(_, _) do - false - end + defp maybe_prepend_buffer(buffer, [head | _] = acc) + when is_atom(head) and head not in [:and, :or], + do: [buffer | acc] - defp approximate_upper(version) do - case version do - {major, _minor, nil, _} -> - {major + 1, 0, 0, [0]} + defp maybe_prepend_buffer(buffer, acc), + do: [buffer, :== | acc] - {major, minor, _patch, _} -> - {major, minor + 1, 0, [0]} + defp revert_lexed([version, op, cond | rest], acc) + when is_binary(version) and is_atom(op) and cond in [:or, :and] do + with {:ok, version} <- validate_requirement(op, version) do + revert_lexed(rest, [cond, op, version | acc]) end end - defp to_matchspec(lexed) do - if valid_requirement?(lexed) do - first = to_condition(lexed) - rest = Enum.drop(lexed, 2) - {:ok, [{{:'$1', :'$2', :'$3', :'$4'}, [to_condition(first, rest)], [:'$_']}]} - else - :error + defp revert_lexed([version, op], acc) when is_binary(version) and is_atom(op) do + with {:ok, version} <- validate_requirement(op, version) do + {:ok, [op, version | acc]} end - catch - :invalid_matchspec -> :error end - defp to_condition([:'==', version | _]) do - version = parse_condition(version) - {:'==', :'$_', {:const, version}} - end + defp revert_lexed(_rest, _acc), do: :error - defp to_condition([:'!=', version | _]) do - version = parse_condition(version) - {:'/=', :'$_', {:const, version}} + defp validate_requirement(op, version) do + case parse_version(version, true) do + {:ok, version} when op == :~> -> {:ok, version} + {:ok, {_, _, patch, _, _} = version} when is_integer(patch) -> {:ok, version} + _ -> :error + end end - defp to_condition([:'~>', version | _]) do - from = parse_condition(version, true) - to = approximate_upper(from) + @spec parse_requirement(String.t()) :: {:ok, term} | :error + def parse_requirement(source) do + revert_lexed(lexer(source), []) + end - {:andalso, to_condition([:'>=', matchable_to_string(from)]), - to_condition([:'<', matchable_to_string(to)])} + def parse_version(string, approximate? \\ false) when is_binary(string) do + destructure [version_with_pre, build], String.split(string, "+", parts: 2) + destructure [version, pre], String.split(version_with_pre, "-", parts: 2) + destructure [major, minor, patch, next], String.split(version, ".") + + with nil <- next, + {:ok, major} <- require_digits(major), + {:ok, minor} <- require_digits(minor), + {:ok, patch} <- maybe_patch(patch, approximate?), + {:ok, pre_parts} <- optional_dot_separated(pre), + {:ok, pre_parts} <- convert_parts_to_integer(pre_parts, []), + {:ok, build_parts} <- optional_dot_separated(build) do + {:ok, {major, minor, patch, pre_parts, build_parts}} + else + _other -> :error + end end - defp to_condition([:'>', version | _]) do - {major, minor, patch, pre} = parse_condition(version) + defp require_digits(nil), do: :error - {:orelse, {:'>', {{:'$1', :'$2', :'$3'}}, - {:const, {major, minor, patch}}}, - {:andalso, {:'==', {{:'$1', :'$2', :'$3'}}, - {:const, {major, minor, patch}}}, - {:orelse, {:andalso, {:'==', {:length, :'$4'}, 0}, - {:'/=', length(pre), 0}}, - {:andalso, {:'/=', length(pre), 0}, - {:orelse, {:'>', {:length, :'$4'}, length(pre)}, - {:andalso, {:'==', {:length, :'$4'}, length(pre)}, - {:'>', :'$4', {:const, pre}}}}}}}} + defp require_digits(string) do + if leading_zero?(string), do: :error, else: parse_digits(string, "") end - defp to_condition([:'>=', version | _]) do - matchable = parse_condition(version) + defp leading_zero?(<>), do: true + defp leading_zero?(_), do: false - {:orelse, {:'==', :'$_', {:const, matchable}}, - to_condition([:'>', version])} - end + defp parse_digits(<>, acc) when char in ?0..?9, + do: parse_digits(rest, <>) - defp to_condition([:'<', version | _]) do - {major, minor, patch, pre} = parse_condition(version) + defp parse_digits(<<>>, acc) when byte_size(acc) > 0, do: {:ok, String.to_integer(acc)} + defp parse_digits(_, _acc), do: :error - {:orelse, {:'<', {{:'$1', :'$2', :'$3'}}, - {:const, {major, minor, patch}}}, - {:andalso, {:'==', {{:'$1', :'$2', :'$3'}}, - {:const, {major, minor, patch}}}, - {:orelse, {:andalso, {:'/=', {:length, :'$4'}, 0}, - {:'==', length(pre), 0}}, - {:andalso, {:'/=', {:length, :'$4'}, 0}, - {:orelse, {:'<', {:length, :'$4'}, length(pre)}, - {:andalso, {:'==', {:length, :'$4'}, length(pre)}, - {:'<', :'$4', {:const, pre}}}}}}}} - end + defp maybe_patch(patch, approximate?) + defp maybe_patch(nil, true), do: {:ok, nil} + defp maybe_patch(patch, _), do: require_digits(patch) + + defp optional_dot_separated(nil), do: {:ok, []} - defp to_condition([:'<=', version | _]) do - matchable = parse_condition(version) + defp optional_dot_separated(string) do + parts = String.split(string, ".") - {:orelse, {:'==', :'$_', {:const, matchable}}, - to_condition([:'<', version])} + if Enum.all?(parts, &(&1 != "" and valid_identifier?(&1))) do + {:ok, parts} + else + :error + end end - defp to_condition(current, []) do - current + defp convert_parts_to_integer([part | rest], acc) do + case parse_digits(part, "") do + {:ok, integer} -> + if leading_zero?(part) do + :error + else + convert_parts_to_integer(rest, [integer | acc]) + end + + :error -> + convert_parts_to_integer(rest, [part | acc]) + end end - defp to_condition(current, [:'&&', operator, version | rest]) do - to_condition({:andalso, current, to_condition([operator, version])}, rest) + defp convert_parts_to_integer([], acc) do + {:ok, Enum.reverse(acc)} end - defp to_condition(current, [:'||', operator, version | rest]) do - to_condition({:orelse, current, to_condition([operator, version])}, rest) + defp valid_identifier?(<>) + when char in ?0..?9 + when char in ?a..?z + when char in ?A..?Z + when char == ?- do + valid_identifier?(rest) end - defp parse_condition(version, approximate? \\ false) do - case parse_version(version, approximate?) do - {:ok, version} -> version - :error -> throw :invalid_matchspec - end + defp valid_identifier?(<<>>) do + true end - defp matchable_to_string({major, minor, patch, pre}) do - patch = if patch, do: "#{patch}", else: "0" - pre = if pre != [], do: "-#{Enum.join(pre, ".")}" - "#{major}.#{minor}.#{patch}#{pre}" + defp valid_identifier?(_other) do + false end end end defimpl String.Chars, for: Version do - def to_string(version) do - pre = unless Enum.empty?(pre = version.pre), do: "-#{pre}" - build = if build = version.build, do: "+#{build}" - "#{version.major}.#{version.minor}.#{version.patch}#{pre}#{build}" - end -end - -defimpl Inspect, for: Version do - def inspect(self, _opts) do - "#Version<" <> to_string(self) <> ">" - end + defdelegate to_string(version), to: Version end defimpl String.Chars, for: Version.Requirement do @@ -511,7 +673,9 @@ defimpl String.Chars, for: Version.Requirement do end defimpl Inspect, for: Version.Requirement do - def inspect(%Version.Requirement{source: source}, _opts) do - "#Version.Requirement<" <> source <> ">" + def inspect(%Version.Requirement{source: source}, opts) do + colorized = Inspect.Algebra.color("\"" <> source <> "\"", :string, opts) + + Inspect.Algebra.concat(["Version.parse_requirement!(", colorized, ")"]) end end diff --git a/lib/elixir/mix.exs b/lib/elixir/mix.exs index 4b39a746602..1d657827e4b 100644 --- a/lib/elixir/mix.exs +++ b/lib/elixir/mix.exs @@ -1,12 +1,11 @@ -defmodule Elixir.Mixfile do +defmodule Elixir.MixProject do use Mix.Project def project do - [app: :elixir, - version: System.version, - build_per_environment: false, - escript_embed_elixir: false, - escript_main_module: :elixir, - escript_emu_args: "%%! -noshell\n"] + [ + app: :elixir, + version: System.version(), + build_per_environment: false + ] end end diff --git a/lib/elixir/pages/compatibility-and-deprecations.md b/lib/elixir/pages/compatibility-and-deprecations.md new file mode 100644 index 00000000000..d7dcae7d2e9 --- /dev/null +++ b/lib/elixir/pages/compatibility-and-deprecations.md @@ -0,0 +1,198 @@ +# Compatibility and Deprecations + +Elixir is versioned according to a vMAJOR.MINOR.PATCH schema. + +Elixir is currently at major version v1. A new backwards compatible minor release happens every 6 months. Patch releases are not scheduled and are made whenever there are bug fixes or security patches. + +Elixir applies bug fixes only to the latest minor branch. Security patches are available for the last 5 minor branches: + +Elixir version | Support +:------------- | :----------------------------- +1.14 | Development +1.13 | Bug fixes and security patches +1.12 | Security patches only +1.11 | Security patches only +1.10 | Security patches only +1.9 | Security patches only + +New releases are announced in the read-only [announcements mailing list](https://groups.google.com/group/elixir-lang-ann). All security releases [will be tagged with `[security]`](https://groups.google.com/forum/#!searchin/elixir-lang-ann/%5Bsecurity%5D%7Csort:date). + +There are currently no plans for a major v2 release. + +## Compatibility between non-major Elixir versions + +Elixir minor and patch releases are backwards compatible: well-defined behaviours and documented APIs in a given version will continue working on future versions. + +Although we expect the vast majority of programs to remain compatible over time, it is impossible to guarantee that no future change will break any program. Under some unlikely circumstances, we may introduce changes that break existing code: + + * Security: a security issue in the implementation may arise whose resolution requires backwards incompatible changes. We reserve the right to address such security issues. + + * Bugs: if an API has undesired behaviour, a program that depends on the buggy behaviour may break if the bug is fixed. We reserve the right to fix such bugs. + + * Compiler front-end: improvements may be done to the compiler, introducing new warnings for ambiguous modes and providing more detailed error messages. Those can lead to compilation errors (when running with `--warning-as-errors`) or tooling failures when asserting on specific error messages (although one should avoid such). We reserve the right to do such improvements. + + * Imports: new functions may be added to the `Kernel` module, which is auto-imported. They may collide with local functions defined in your modules. Collisions can be resolved in a backwards compatible fashion using `import Kernel, except: [...]` with a list of all functions you don't want to be imported from `Kernel`. We reserve the right to do such additions. + +In order to continue evolving the language without introducing breaking changes, Elixir will rely on deprecations to demote certain practices and promote new ones. Our deprecation policy is outlined in the ["Deprecations" section](#deprecations). + +The only exception to the compatibility guarantees above are experimental features, which will be explicitly marked as such, and do not provide any compatibility guarantee until they are stabilized. + +## Compatibility between Elixir and Erlang/OTP + +Erlang/OTP versioning is independent from the versioning of Elixir. Erlang releases a new major version yearly. Our goal is to support the last three Erlang major versions by the time Elixir is released. The compatibility table is shown below. + +Elixir version | Supported Erlang/OTP versions +:------------- | :------------------------------- +1.14 | 23 - 25 +1.13 | 22 - 24 (and Erlang/OTP 25 from v1.13.4) +1.12 | 22 - 24 +1.11 | 21 - 23 (and Erlang/OTP 24 from v1.11.4) +1.10 | 21 - 22 (and Erlang/OTP 23 from v1.10.3) +1.9 | 20 - 22 +1.8 | 20 - 22 +1.7 | 19 - 22 +1.6 | 19 - 20 (and Erlang/OTP 21 from v1.6.6) +1.5 | 18 - 20 +1.4 | 18 - 19 (and Erlang/OTP 20 from v1.4.5) +1.3 | 18 - 19 +1.2 | 18 - 18 (and Erlang/OTP 19 from v1.2.6) +1.1 | 17 - 18 +1.0 | 17 - 17 (and Erlang/OTP 18 from v1.0.5) + +Elixir may add compatibility to new Erlang/OTP versions on patch releases, such as support for Erlang/OTP 20 in v1.4.5. Those releases are made for convenience and typically contain the minimum changes for Elixir to run without errors, if any changes are necessary. Only the next minor release, in this case v1.5.0, effectively leverages the new features provided by the latest Erlang/OTP release. + +## Deprecations + +### Policy + +Elixir deprecations happen in 3 steps: + + 1. The feature is soft-deprecated. It means both CHANGELOG and documentation must list the feature as deprecated but no warning is effectively emitted by running the code. There is no requirement to soft-deprecate a feature. + + 2. The feature is effectively deprecated by emitting warnings on usage. This is also known as hard-deprecation. In order to deprecate a feature, the proposed alternative MUST exist for AT LEAST THREE minor versions. For example, `Enum.uniq/2` was soft-deprecated in favor of `Enum.uniq_by/2` in Elixir v1.1. This means a deprecation warning may only be emitted by Elixir v1.4 or later. + + 3. The feature is removed. This can only happen on major releases. This means deprecated features in Elixir v1.x shall only be removed by Elixir v2.x. + +### Table of deprecations + +The first column is the version the feature was hard deprecated. The second column shortly describes the deprecated feature and the third column explains the replacement and from which the version the replacement is available from. + +Version | Deprecated feature | Replaced by (available since) +:-------| :-------------------------------------------------- | :--------------------------------------------------------------- +[v1.14] | `use Bitwise` | `import Bitwise` (v1.0) +[v1.14] | `~~~/1` | `bnot/2` (v1.0) +[v1.14] | `Application.get_env/3` and similar in module body | `Application.compile_env/3` (v1.10) +[v1.14] | Compiled patterns in `String.starts_with?/2` | Pass a list of strings instead (v1.0) +[v1.14] | `Mix.Tasks.Xref.calls/1` | Compilation tracers (outlined in `Code`) (v1.10) +[v1.13] | `!` and `!=` in Version requirements | `~>` or `>=` (v1.0) +[v1.13] | `Mix.Config` | `Config` (v1.9) +[v1.13] | `:strip_beam` config to `mix escript.build` | `:strip_beams` (v1.9) +[v1.13] | `Macro.to_string/2` | `Macro.to_string/1` (v1.0) +[v1.13] | `System.get_pid/0` | `System.pid/0` (v1.9) +[v1.12] | `^^^/2` | `bxor/2` (v1.0) +[v1.12] | `@foo()` to read module attributes | Remove the parenthesis (v1.0) +[v1.12] | `use EEx.Engine` | Explicitly delegate to EEx.Engine instead (v1.0) +[v1.12] | `:xref` compiler in Mix | Nothing (it always runs as part of the compiler now) +[v1.11] | `Mix.Project.compile/2` | `Mix.Task.run("compile", args)` (v1.0) +[v1.11] | `Supervisor.Spec.worker/3` and `Supervisor.Spec.supervisor/3` | The new child specs outlined in `Supervisor` (v1.5) +[v1.11] | `Supervisor.start_child/2` and `Supervisor.terminate_child/2` | `DynamicSupervisor` (v1.6) +[v1.11] | `System.stacktrace/1` | `__STACKTRACE__` in `try/catch/rescue` (v1.7) +[v1.10] | `Code.ensure_compiled?/1` | `Code.ensure_compiled/1` (v1.0) +[v1.10] | `Code.load_file/2` | `Code.require_file/2` (v1.0) or `Code.compile_file/2` (v1.7) +[v1.10] | `Code.loaded_files/0` | `Code.required_files/0` (v1.7) +[v1.10] | `Code.unload_file/1` | `Code.unrequire_files/1` (v1.7) +[v1.10] | Passing non-chardata to `Logger.log/2` | Explicitly convert to string with `to_string/1` (v1.0) +[v1.10] | `:compile_time_purge_level` in `Logger` app environment | `:compile_time_purge_matching` in `Logger` app environment (v1.7) +[v1.10] | `Supervisor.Spec.supervise/2` | The new child specs outlined in `Supervisor` (v1.5) +[v1.10] | `:simple_one_for_one` strategy in `Supervisor` | `DynamicSupervisor` (v1.6) +[v1.10] | `:restart` and `:shutdown` in `Task.Supervisor.start_link/1` | `:restart` and `:shutdown` in `Task.Supervisor.start_child/3` (v1.6) +[v1.9] | Enumerable keys in `Map.drop/2`, `Map.split/2`, and `Map.take/2` | Call `Enum.to_list/1` on the second argument before hand (v1.0) +[v1.9] | `Mix.Project.load_paths/1` | `Mix.Project.compile_path/1` (v1.0) +[v1.9] | Passing `:insert_replaced` to `String.replace/4` | Use `:binary.replace/4` (v1.0) +[v1.8] | Passing a non-empty list to `Collectable.into/1` | `++/2` or `Keyword.merge/2` (v1.0) +[v1.8] | Passing a non-empty list to `:into` in `for/1` | `++/2` or `Keyword.merge/2` (v1.0) +[v1.8] | Passing a non-empty list to `Enum.into/2` | `++/2` or `Keyword.merge/2` (v1.0) +[v1.8] | Time units in its plural form, such as: `:seconds`, `:milliseconds`, and the like | Use the singular form, such as: `:second`, `:millisecond`, and so on (v1.4) +[v1.8] | `Inspect.Algebra.surround/3` | `Inspect.Algebra.concat/2` and `Inspect.Algebra.nest/2` (v1.0) +[v1.8] | `Inspect.Algebra.surround_many/6` | `Inspect.Algebra.container_doc/6` (v1.6) +[v1.9] | `--detached` in `Kernel.CLI` | `--erl "-detached"` (v1.0) +[v1.8] | `Kernel.ParallelCompiler.files/2` | `Kernel.ParallelCompiler.compile/2` (v1.6) +[v1.8] | `Kernel.ParallelCompiler.files_to_path/2` | `Kernel.ParallelCompiler.compile_to_path/2` (v1.6) +[v1.8] | `Kernel.ParallelRequire.files/2` | `Kernel.ParallelCompiler.require/2` (v1.6) +[v1.8] | Returning `{:ok, contents}` or `:error` from `Mix.Compilers.Erlang.compile/6`'s callback | Return `{:ok, contents, warnings}` or `{:error, errors, warnings}` (v1.6) +[v1.8] | `System.cwd/0` and `System.cwd!/0` | `File.cwd/0` and `File.cwd!/0` (v1.0) +[v1.7] | `Code.get_docs/2` | `Code.fetch_docs/1` (v1.7) +[v1.7] | `Enum.chunk/2,3,4` | `Enum.chunk_every/2` and [`Enum.chunk_every/3,4`](`Enum.chunk_every/4`) (v1.5) +[v1.7] | Calling `super/1` in`GenServer` callbacks | Implementing the behaviour explicitly without calling `super/1` (v1.0) +[v1.7] | [`not left in right`](`in/2`) | [`left not in right`](`in/2`) (v1.5) +[v1.7] | `Registry.start_link/3` | `Registry.start_link/1` (v1.5) +[v1.7] | `Stream.chunk/2,3,4` | `Stream.chunk_every/2` and [`Stream.chunk_every/3,4`](`Stream.chunk_every/4`) (v1.5) +[v1.6] | `Enum.partition/2` | `Enum.split_with/2` (v1.4) +[v1.6] | `Macro.unescape_tokens/1,2` | Use `Enum.map/2` to traverse over the arguments (v1.0) +[v1.6] | `Module.add_doc/6` | [`@doc`](`Module`) module attribute (v1.0) +[v1.6] | `Range.range?/1` | Pattern match on [`_.._`](`../2`) (v1.0) +[v1.5] | `()` to mean `nil` | `nil` (v1.0) +[v1.5] | `char_list/0` type | `t:charlist/0` type (v1.3) +[v1.5] | `Atom.to_char_list/1` | `Atom.to_charlist/1` (v1.3) +[v1.5] | `Enum.filter_map/3` | `Enum.filter/2` + `Enum.map/2` or `for/1` comprehensions (v1.0) +[v1.5] | `Float.to_char_list/1` | `Float.to_charlist/1` (v1.3) +[v1.5] | `GenEvent` module | `Supervisor` and `GenServer` (v1.0);
[`GenStage`](https://hex.pm/packages/gen_stage) (v1.3);
[`:gen_event`](`:gen_event`) (Erlang/OTP 17) +[v1.5] | `<%=` in middle and end expressions in `EEx` | Use `<%` (`<%=` is allowed only in start expressions) (v1.0) +[v1.5] | `:as_char_lists` value in `t:Inspect.Opts.t/0` type | `:as_charlists` value (v1.3) +[v1.5] | `:char_lists` key in `t:Inspect.Opts.t/0` type | `:charlists` key (v1.3) +[v1.5] | `Integer.to_char_list/1,2` | `Integer.to_charlist/1` and `Integer.to_charlist/2` (v1.3) +[v1.5] | `to_char_list/1` | `to_charlist/1` (v1.3) +[v1.5] | `List.Chars.to_char_list/1` | `List.Chars.to_charlist/1` (v1.3) +[v1.5] | `@compile {:parse_transform, _}` in `Module` | *None* +[v1.5] | `Stream.filter_map/3` | `Stream.filter/2` + `Stream.map/2` (v1.0) +[v1.5] | `String.ljust/3` and `String.rjust/3` | Use `String.pad_leading/3` and `String.pad_trailing/3` with a binary padding (v1.3) +[v1.5] | `String.lstrip/1` and `String.rstrip/1` | `String.trim_leading/1` and `String.trim_trailing/1` (v1.3) +[v1.5] | `String.lstrip/2` and `String.rstrip/2` | Use `String.trim_leading/2` and `String.trim_trailing/2` with a binary as second argument (v1.3) +[v1.5] | `String.strip/1` and `String.strip/2` | `String.trim/1` and `String.trim/2` (v1.3) +[v1.5] | `String.to_char_list/1` | `String.to_charlist/1` (v1.3) +[v1.4] | [Anonymous functions](`fn/1`) with no expression after `->` | Use an expression or explicitly return `nil` (v1.0) +[v1.4] | Support for making [private functions](`defp/2`) overridable | Use [public functions](`def/2`) (v1.0) +[v1.4] | Variable used as function call | Use parentheses (v1.0) +[v1.4] | `Access.key/1` | `Access.key/2` (v1.3) +[v1.4] | `Behaviour` module | `@callback` module attribute (v1.0) +[v1.4] | `Enum.uniq/2` | `Enum.uniq_by/2` (v1.2) +[v1.4] | `Float.to_char_list/2` | `:erlang.float_to_list/2` (Erlang/OTP 17) +[v1.4] | `Float.to_string/2` | `:erlang.float_to_binary/2` (Erlang/OTP 17) +[v1.4] | `HashDict` module | `Map` (v1.2) +[v1.4] | `HashSet` module | `MapSet` (v1.1) +[v1.4] | `IEx.Helpers.import_file/2` | `IEx.Helpers.import_file_if_available/1` (v1.3) +[v1.4] | `Mix.Utils.camelize/1` | `Macro.camelize/1` (v1.2) +[v1.4] | `Mix.Utils.underscore/1` | `Macro.underscore/1` (v1.2) +[v1.4] | Multi-letter aliases in `OptionParser` | Use single-letter aliases (v1.0) +[v1.4] | `Set` module | `MapSet` (v1.1) +[v1.4] | `Stream.uniq/2` | `Stream.uniq_by/2` (v1.2) +[v1.3] | `\x{X*}` inside strings/sigils/charlists | `\uXXXX` or `\u{X*}` (v1.1) +[v1.3] | `Dict` module | `Keyword` (v1.0) or `Map` (v1.2) +[v1.3] | `:append_first` option in `defdelegate/2` | Define the function explicitly (v1.0) +[v1.3] | Map/dictionary as 2nd argument in `Enum.group_by/3` | `Enum.reduce/3` (v1.0) +[v1.3] | `Keyword.size/1` | `length/1` (v1.0) +[v1.3] | `Map.size/1` | `map_size/1` (v1.0) +[v1.3] | `/r` option in `Regex` | `/U` (v1.1) +[v1.3] | `Set` behaviour | `MapSet` data structure (v1.1) +[v1.3] | `String.valid_character?/1` | `String.valid?/1` (v1.0) +[v1.3] | `Task.find/2` | Use direct message matching (v1.0) +[v1.3] | Non-map as 2nd argument in `URI.decode_query/2` | Use a map (v1.0) +[v1.2] | `Dict` behaviour | `Map` and `Keyword` (v1.0) +[v1.1] | `?\xHEX` | `0xHEX` (v1.0) +[v1.1] | `Access` protocol | `Access` behaviour (v1.1) +[v1.1] | `as: true \| false` in `alias/2` and `require/2` | *None* + +[v1.1]: https://github.com/elixir-lang/elixir/blob/v1.1/CHANGELOG.md#4-deprecations +[v1.2]: https://github.com/elixir-lang/elixir/blob/v1.2/CHANGELOG.md#changelog-for-elixir-v12 +[v1.3]: https://github.com/elixir-lang/elixir/blob/v1.3/CHANGELOG.md#4-deprecations +[v1.4]: https://github.com/elixir-lang/elixir/blob/v1.4/CHANGELOG.md#4-deprecations +[v1.5]: https://github.com/elixir-lang/elixir/blob/v1.5/CHANGELOG.md#4-deprecations +[v1.6]: https://github.com/elixir-lang/elixir/blob/v1.6/CHANGELOG.md#4-deprecations +[v1.7]: https://github.com/elixir-lang/elixir/blob/v1.7/CHANGELOG.md#4-hard-deprecations +[v1.8]: https://github.com/elixir-lang/elixir/blob/v1.8/CHANGELOG.md#4-hard-deprecations +[v1.9]: https://github.com/elixir-lang/elixir/blob/v1.9/CHANGELOG.md#4-hard-deprecations +[v1.10]: https://github.com/elixir-lang/elixir/blob/v1.10/CHANGELOG.md#4-hard-deprecations +[v1.11]: https://github.com/elixir-lang/elixir/blob/v1.11/CHANGELOG.md#4-hard-deprecations +[v1.12]: https://github.com/elixir-lang/elixir/blob/v1.12/CHANGELOG.md#4-hard-deprecations +[v1.13]: https://github.com/elixir-lang/elixir/blob/v1.13/CHANGELOG.md#4-hard-deprecations +[v1.14]: CHANGELOG.md#4-hard-deprecations diff --git a/lib/elixir/pages/library-guidelines.md b/lib/elixir/pages/library-guidelines.md new file mode 100644 index 00000000000..6c6333be478 --- /dev/null +++ b/lib/elixir/pages/library-guidelines.md @@ -0,0 +1,299 @@ +# Library Guidelines + +This document outlines general guidelines, anti-patterns, and rules for those writing and publishing Elixir libraries meant to be consumed by other developers. + +## Getting started + +You can create a new Elixir library by running the `mix new` command: + + $ mix new my_library + +The project name is given in the `snake_case` convention where all letters are lowercase and words are separate with underscores. This is the same convention used by variables, function names and atoms in Elixir. See the [Naming Conventions](naming-conventions.md) document for more information. + +Every project has a `mix.exs` file, with instructions on how to build, compile, run tests, and so on. Libraries commonly have a `lib` directory, which includes Elixir source code, and a `test` directory. A `src` directory may also exist for Erlang sources. + +For more information on running your project, see the official [Mix & OTP guide](https://elixir-lang.org/getting-started/mix-otp/introduction-to-mix.html) or [Mix documentation](https://hexdocs.pm/mix/Mix.html). + +### Applications with supervision tree + +The `mix new` command also allows the `--sup` option to scaffold an application with a supervision tree out of the box. We talk about supervision trees later on when discussing one of the common anti-patterns when writing libraries. + +## Publishing + +Writing code is only the first of many steps to publish a package. We strongly recommend developers to: + + * Choose a versioning schema. Elixir requires versions to be in the format `MAJOR.MINOR.PATCH` but the meaning of those numbers is up to you. Most projects choose [Semantic Versioning](https://semver.org/). + + * Choose a [license](https://choosealicense.com/). The most common licenses in the Elixir community are the [MIT License](https://choosealicense.com/licenses/mit/) and the [Apache License 2.0](https://choosealicense.com/licenses/apache-2.0/). The latter is also the one used by Elixir itself. + + * Run the [code formatter](https://hexdocs.pm/mix/Mix.Tasks.Format.html). The code formatter formats your code according to a consistent style shared by your library and the whole community, making it easier for other developers to understand your code and contribute. + + * Write tests. Elixir ships with a test-framework named [ExUnit](https://hexdocs.pm/ex_unit/ExUnit.html). The project generated by `mix new` includes sample tests and doctests. + + * Write documentation. The Elixir community is proud of treating documentation as a first-class citizen and making documentation easily accessible. Libraries contribute to the status quo by providing complete API documentation with examples for their modules, types and functions. See the [Writing Documentation](writing-documentation.md) guide for more information. Projects like [ExDoc](https://github.com/elixir-lang/ex_doc) can be used to generate HTML and EPUB documents from the documentation. ExDoc also supports "extra pages", like this one that you are reading. Such pages augment the documentation with tutorials, guides and references. + +Projects are often made available to other developers [by publishing a Hex package](https://hex.pm/docs/publish). Hex also [supports private packages for organizations](https://hex.pm/pricing). If ExDoc is configured for the Mix project, publishing a package on Hex will also automatically publish the generated documentation to [HexDocs](https://hexdocs.pm). + +## Dependency handling + +When your library is published and used as a dependency, its [lockfile](https://hexdocs.pm/mix/Mix.Project.html#module-configuration) (usually named `mix.lock`) is _ignored by the host project_. Running `mix deps.get` in the host project attempts to get the latest possible versions of your library’s dependencies, as specified by the requirements in the `deps` section of your `mix.exs`. These versions might be greater than those stored in your `mix.lock` (and hence used in your tests / CI). + +On the other hand, contributors of your library, need a deterministic build, which implies the presence of `mix.lock` in your Version Control System (VCS). + +The best practice of handling `mix.lock` file therefore would be to keep it in VCS, and run two different Continuous Integration (CI) workflows: the usual deterministic one, and another one, that starts with `mix deps.unlock --all` and always compiles your library and runs tests against latest versions of dependencies. The latter one might be even run nightly or otherwise recurrently to stay notified about any possible issue in regard to dependencies updates. + +## Anti-patterns + +In this section we document common anti-patterns to avoid when writing libraries. + +### Avoid using exceptions for control-flow + +You should avoid using exceptions for control-flow. For example, instead of: + +```elixir +try do + contents = File.read!("some_path_that_may_or_may_not_exist") + {:it_worked, contents} +rescue + File.Error -> + :it_failed +end +``` + +you should prefer: + +```elixir +case File.read("some_path_that_may_or_may_not_exist") do + {:ok, contents} -> {:it_worked, contents} +  {:error, _} -> :it_failed +end +``` + +As a library author, it is your responsibility to make sure users are not required to use exceptions for control-flow in their applications. You can follow the same convention as Elixir here, using the name without `!` for returning `:ok`/`:error` tuples and appending `!` for a version of the function which raises an exception. + +It is important to note that a name without `!` does not mean a function will never raise. For example, even `File.read/1` can fail in case of bad arguments: + +```iex +iex> File.read(1) +** (FunctionClauseError) no function clause matching in IO.chardata_to_string/1 +``` + +The usage of `:ok`/`:error` tuples is about the domain that the function works on, in this case, file system access. Bad arguments, logical errors, invalid options should raise regardless of the function name. If in doubt, prefer to return tuples instead of raising, as users of your library can always match on the results and raise if necessary. + +### Avoid working with invalid data + +Elixir programs should prefer to validate data as close to the end user as possible, so the errors are easy to locate and fix. This practice also saves you from writing defensive code in the internals of the library. + +For example, imagine you have an API that receives a filename as a binary. At some point you will want to write to this file. You could have a function like this: + +```elixir +def my_fun(some_arg, file_to_write_to, options \\ []) do + ...some code... + AnotherModuleInLib.invoke_something_that_will_eventually_write_to_file(file_to_write_to) + ...more code... +end +``` + +The problem with the code above is that, if the user supplies an invalid input, the error will be raised deep inside the library, which makes it confusing for users. Furthermore, when you don't validate the values at the boundary, the internals of your library are never quite sure which kind of values they are working with. + +A better function definition would be: + +```elixir +def my_fun(some_arg, file_to_write_to, options \\ []) when is_binary(file_to_write_to) do +``` + +Elixir also leverages pattern matching and guards in function clauses to provide clear error messages in case invalid arguments are given. + +This advice does not only apply to libraries, but to any Elixir code. Every time you receive multiple options or work with external data, you should validate the data at the boundary and convert it to structured data. For example, if you provide a `GenServer` that can be started with multiple options, you want to validate those options when the server starts and rely only on structured data throughout the process life cycle. Similarly, if a database or a socket gives you a map of strings, after you receive the data, you should validate it and potentially convert it to a struct or a map of atoms. + +### Avoid application configuration + +You should avoid using the application environment (see `Application.get_env/2`) as the configuration mechanism for libraries. The application environment is **global** which means it becomes impossible for two dependencies to use your library in two different ways. + +Let's see a simple example. Imagine that you implement a library that breaks a string in two parts based on the first occurrence of the dash `-` character: + +```elixir +defmodule DashSplitter do + def split(string) when is_binary(string) do + String.split(string, "-", parts: 2) + end +end +``` + +Now imagine someone wants to split the string in three parts. You decide to make the number of parts configurable via the application environment: + +```elixir +def split(string) when is_binary(string) do + parts = Application.get_env(:dash_splitter, :parts, 2) + String.split(string, "-", parts: parts) +end +``` + +Now users can configure your library in their `config/config.exs` file as follows: + +```elixir +config :dash_splitter, :parts, 3 +``` + +Once your library is configured, it will change the behaviour of all users of your library. If a library was expecting it to split the string in 2 parts, since the configuration is global, it will now split it in 3 parts. + +The solution is to provide configuration as close as possible to where it is used and not via the application environment. In case of a function, you could expect keyword lists as a new argument: + +```elixir +def split(string, opts \\ []) when is_binary(string) and is_list(opts) do + parts = Keyword.get(opts, :parts, 2) + String.split(string, "-", parts: parts) +end +``` + +In case you need to configure a process, the options should be passed when starting that process. + +The application environment should be reserved only for configurations that are truly global, for example, to control your application boot process and its supervision tree. And, generally speaking, it is best to avoid global configuration. If you must use configuration, then prefer runtime configuration instead of compile-time configuration. See the `Application` module for more information. + +For all remaining scenarios, libraries should not force their users to use the application environment for configuration. If the user of a library believes that certain parameter should be configured globally, then they can wrap the library functionality with their own application environment configuration. + +### Avoid defining modules that are not in your "namespace" + +Even though Elixir does not formally have the concept of namespaces, a library should use its name as a "prefix" for all of its modules (except for special cases like mix tasks). For example, if the library's OTP application name is `:my_lib`, then all of its modules should start with the `MyLib` prefix, for example `MyLib.User`, `MyLib.SubModule`, and `MyLib.Application`. + +This is important because the Erlang VM can only load one instance of a module at a time. So if there are multiple libraries that define the same module, then they are incompatible with each other due to this limitation. By always using the library name as a prefix, it avoids module name clashes due to the unique prefix. + +Furthermore, when writing a library that is an extension of another library, you should avoid defining modules inside the parent's library namespace. For example, if you are writing a package that adds authentication to [`Plug`](https://github.com/elixir-plug/plug) called `plug_auth`, its modules should be namespaced under `PlugAuth` instead of `Plug.Auth`, so it avoids conflicts with `Plug` if it were to ever define its own authentication functionality. + +### Avoid `use` when an `import` is enough + +A library should not provide `use MyLib` functionality if all `use MyLib` does is to `import`/`alias` the module itself. For example, this is an anti-pattern: + +```elixir +defmodule MyLib do + defmacro __using__(_) do + quote do + import MyLib + end + end + + def some_fun(arg1, arg2) do + ... + end +end +``` + +The reason why defining the `__using__` macro above should be avoided is because when a developer writes: + +```elixir +defmodule MyApp do + use MyLib +end +``` + +It allows `use MyLib` to run *any* code into the `MyApp` module. For someone reading the code, it is impossible to assess the impact that `use MyLib` has in a module without looking at the implementation of `__using__`. + +The following code is clearer: + +```elixir +defmodule MyApp do + import MyLib +end +``` + +The code above says we are only bringing in the functions from `MyLib` so we can invoke `some_fun(arg1, arg2)` directly without the `MyLib.` prefix. Even more important, `import MyLib` says that we have an option to not `import MyLib` at all as we can simply invoke the function as `MyLib.some_fun(arg1, arg2)`. + +If the module you want to invoke a function on has a long name, such as `SomeLibrary.Namespace.MyLib`, and you find it verbose, you can leverage the `alias/2` special form and still refer to the module as `MyLib`. + +While there are situations where `use SomeModule` is necessary, `use` should be skipped when all it does is to `import` or `alias` other modules. In a nutshell, `alias` should be preferred, as it is simpler and clearer than `import`, while `import` is simpler and clearer than `use`. + +### Avoid macros + +Although the previous section could be summarized as "avoid macros", both topics are important enough to deserve their own sections. + +To quote [the official guide on macros](https://elixir-lang.org/getting-started/meta/macros.html): + +> Even though Elixir attempts its best to provide a safe environment for macros, the major responsibility of writing clean code with macros falls on developers. Macros are harder to write than ordinary Elixir functions and it's considered to be bad style to use them when they're not necessary. So write macros responsibly. +> +> Elixir already provides mechanisms to write your everyday code in a simple and readable fashion by using its data structures and functions. Macros should only be used as a last resort. Remember that **explicit is better than implicit**. **Clear code is better than concise code**. + +When you absolutely have to use a macro, make sure that a macro is not the only way the user can interface with your library and keep the amount of code generated by a macro to a minimum. For example, the `Logger` module provides `Logger.debug/2`, `Logger.info/2` and friends as macros that are capable of extracting environment information, but a low-level mechanism for logging is still available with `Logger.bare_log/3`. + +### Avoid using processes for code organization + +A developer must never use a process for code organization purposes. A process must be used to model runtime properties such as: + + * Mutable state and access to shared resources (such as ETS, files, and others) + * Concurrency and distribution + * Initialization, shutdown and restart logic (as seen in supervisors) + * System messages such as timer messages and monitoring events + +In Elixir, code organization is done by modules and functions, processes are not necessary. For example, imagine you are implementing a calculator and you decide to put all the calculator operations behind a `GenServer`: + +```elixir +def add(a, b) do + GenServer.call(__MODULE__, {:add, a, b}) +end + +def handle_call({:add, a, b}, _from, state) do + {:reply, a + b, state} +end + +def handle_call({:subtract, a, b}, _from, state) do + {:reply, a - b, state} +end +``` + +This is an anti-pattern not only because it convolutes the calculator logic but also because you put the calculator logic behind a single process that will potentially become a bottleneck in your system, especially as the number of calls grow. Instead, just define the functions directly: + +```elixir +def add(a, b) do + a + b +end + +def subtract(a, b) do + a - b +end +``` + +Use processes only to model runtime properties, never for code organization. And even when you think something could be done in parallel with processes, often it is best to let the callers of your library decide how to parallelize, rather than impose a certain execution flow in users of your code. + +### Avoid spawning unsupervised processes + +You should avoid spawning processes outside of a supervision tree, especially long-running ones. Instead, processes must be started inside supervision trees. This guarantees developers have full control over the initialization, restarts, and shutdown of the system. + +If your application does not have a supervision tree, one can be added by changing `def application` inside `mix.exs` to include a `:mod` key with the application callback name: + +```elixir +def application do + [ + extra_applications: [:logger], + mod: {MyApp.Application, []} + ] +end +``` + +and then defining a `my_app/application.ex` file with the following template: + +```elixir +defmodule MyApp.Application do + # See https://hexdocs.pm/elixir/Application.html + # for more information on OTP Applications + @moduledoc false + + use Application + + def start(_type, _args) do + children = [ + # Starts a worker by calling: MyApp.Worker.start_link(arg) + # {MyApp.Worker, arg} + ] + + # See https://hexdocs.pm/elixir/Supervisor.html + # for other strategies and supported options + opts = [strategy: :one_for_one, name: MyApp.Supervisor] + Supervisor.start_link(children, opts) + end +end +``` + +This is the same template generated by `mix new --sup`. + +Each process started with the application must be listed as a child under the `Supervisor` above. We call those "static processes" because they are known upfront. For handling dynamic processes, such as the ones started during requests and other user inputs, look at the `DynamicSupervisor` module. + +One of the few times where it is acceptable to start a process outside of a supervision tree is with `Task.async/1` and `Task.await/2`. Opposite to `Task.start_link/1`, the `async/await` mechanism gives you full control over the spawned process life cycle - which is also why you must always call `Task.await/2` after starting a task with `Task.async/1`. Even though, if your application is spawning multiple async processes, you should consider using `Task.Supervisor` for better visibility when instrumenting and monitoring the system. diff --git a/lib/elixir/pages/naming-conventions.md b/lib/elixir/pages/naming-conventions.md new file mode 100644 index 00000000000..d7ecf8857b1 --- /dev/null +++ b/lib/elixir/pages/naming-conventions.md @@ -0,0 +1,108 @@ +# Naming Conventions + +This document covers some naming conventions in Elixir code, from casing to punctuation characters. + +## Casing + +Elixir developers must use `snake_case` when defining variables, function names, module attributes, and the like: + + some_map = %{this_is_a_key: "and a value"} + is_map(some_map) + +Aliases, commonly used as module names, are an exception as they must be capitalized and written in `CamelCase`, like `OptionParser`. For aliases, capital letters are kept in acronyms, like `ExUnit.CaptureIO` or `Mix.SCM`. + +Atoms can be written either in `:snake_case` or `:CamelCase`, although the convention is to use the snake case version throughout Elixir. + +Generally speaking, filenames follow the `snake_case` convention of the module they define. For example, `MyApp` should be defined inside the `my_app.ex` file. However, this is only a convention. At the end of the day any filename can be used as they do not affect the compiled code in any way. + +## Underscore (`_foo`) + +Elixir relies on underscores in different situations. + +For example, a value that is not meant to be used must be assigned to `_` or to a variable starting with underscore: + + iex> {:ok, _contents} = File.read("README.md") + +Function names may also start with an underscore. Such functions are never imported by default: + + iex> defmodule Example do + ...> def _wont_be_imported do + ...> :oops + ...> end + ...> end + + iex> import Example + iex> _wont_be_imported() + ** (CompileError) iex:1: undefined function _wont_be_imported/0 + +Due to this property, Elixir relies on functions starting with underscore to attach compile-time metadata to modules. Such functions are most often in the `__foo__` format. For example, every module in Elixir has an [`__info__/1`](`c:Module.__info__/1`) function: + + iex> String.__info__(:functions) + [at: 2, capitalize: 1, chunk: 2, ...] + +Elixir also includes five special forms that follow the double underscore format: `__CALLER__/0`, `__DIR__/0`, `__ENV__/0`and `__MODULE__/0` retrieve compile-time information about the current environment, while `__STACKTRACE__/0` retrieves the stacktrace for the current exception. + +## Trailing bang (`foo!`) + +A trailing bang (exclamation mark) signifies a function or macro where failure cases raise an exception. + +Many functions come in pairs, such as `File.read/1` and `File.read!/1`. `File.read/1` will return a success or failure tuple, whereas `File.read!/1` will return a plain value or else raise an exception: + + iex> File.read("file.txt") + {:ok, "file contents"} + iex> File.read("no_such_file.txt") + {:error, :enoent} + + iex> File.read!("file.txt") + "file contents" + iex> File.read!("no_such_file.txt") + ** (File.Error) could not read file no_such_file.txt: no such file or directory + +The version without `!` is preferred when you want to handle different outcomes using pattern matching: + + case File.read(file) do + {:ok, body} -> # do something with the `body` + {:error, reason} -> # handle the error caused by `reason` + end + +However, if you expect the outcome to always to be successful (for instance, if you expect the file always to exist), the bang variation can be more convenient and will raise a more helpful error message (than a failed pattern match) on failure. + +More examples of paired functions: `Base.decode16/2` and `Base.decode16!/2`, `File.cwd/0` and `File.cwd!/0`. + +There are also some non-paired functions, with no non-bang variant. The bang still signifies that it will raise an exception on failure. Example: `Protocol.assert_protocol!/1`. + +In macro code, the bang on `alias!/1` and `var!/2` signifies that [macro hygiene](https://elixir-lang.org/getting-started/meta/macros.html#macro-hygiene) is set aside. + +## Trailing question mark (`foo?`) + +Functions that return a boolean are named with a trailing question mark. + +Examples: `Keyword.keyword?/1`, `Mix.debug?/0`, `String.contains?/2` + +However, functions that return booleans and are valid in guards follow another convention, described next. + +## `is_` prefix (`is_foo`) + +Type checks and other boolean checks that are allowed in guard clauses are named with an `is_` prefix. + +Examples: `Integer.is_even/1`, `is_list/1` + +These functions and macros follow the Erlang convention of an `is_` prefix, instead of a trailing question mark, precisely to indicate that they are allowed in guard clauses. + +Note that type checks that are not valid in guard clauses do not follow this convention. For example: `Keyword.keyword?/1`. + +## Special names + +Some names have specific meaning in Elixir. We detail those cases below. + +### length and size + +When you see `size` in a function name, it means the operation runs in constant time (also written as "O(1) time") because the size is stored alongside the data structure. + +Examples: `map_size/1`, `tuple_size/1` + +When you see `length`, the operation runs in linear time ("O(n) time") because the entire data structure has to be traversed. + +Examples: `length/1`, `String.length/1` + +In other words, functions using the word "size" in its name will take the same amount of time whether the data structure is tiny or huge. Conversely, functions having "length" in its name will take more time as the data structure grows in size. diff --git a/lib/elixir/pages/operators.md b/lib/elixir/pages/operators.md new file mode 100644 index 00000000000..eb6889870c4 --- /dev/null +++ b/lib/elixir/pages/operators.md @@ -0,0 +1,164 @@ +# Operators + +This document covers operators in Elixir, how they are parsed, how they can be defined, and how they can be overridden. + +## Operator precedence and associativity + +The following is a list of all operators that Elixir is capable of parsing, ordered from higher to lower precedence, alongside their associativity: + +Operator | Associativity +---------------------------------------------- | ------------- +`@` | Unary +`.` | Left +`+` `-` `!` `^` `not` | Unary +`**` | Left +`*` `/` | Left +`+` `-` | Left +`++` `--` `+++` `---` `..` `<>` | Right +`in` `not in` | Left +`\|>` `<<<` `>>>` `<<~` `~>>` `<~` `~>` `<~>` | Left +`<` `>` `<=` `>=` | Left +`==` `!=` `=~` `===` `!==` | Left +`&&` `&&&` `and` | Left +`\|\|` `\|\|\|` `or` | Left +`=` | Right +`&` | Unary +`=>` (valid only inside `%{}`) | Right +`\|` | Right +`::` | Right +`when` | Right +`<-` `\\` | Left + +## General operators + +Elixir provides the following built-in operators: + + * [`+`](`+/1`) and [`-`](`-/1`) - unary positive/negative + * [`+`](`+/2`), [`-`](`-/2`), [`*`](`*/2`), and [`/`](`//2`) - basic arithmetic operations + * [`++`](`++/2`) and [`--`](`--/2`) - list concatenation and subtraction + * [`and`](`and/2`) and [`&&`](`&&/2`) - strict and relaxed boolean "and" + * [`or`](`or/2`) and [`||`](`||/2`) - strict and relaxed boolean "or" + * [`not`](`not/1`) and [`!`](`!/1`) - strict and relaxed boolean "not" + * [`in`](`in/2`) and [`not in`](`in/2`) - membership + * [`@`](`@/1`) - module attribute + * [`..`](`../2`) - range creation + * [`<>`](`<>/2`) - binary concatenation + * [`|>`](`|>/2`) - pipeline + * [`=~`](`=~/2`) - text-based match + +Many of those can be used in guards; consult the [list of allowed guard functions and operators](patterns-and-guards.md#list-of-allowed-functions-and-operators). + +Additionally, there are a few other operators that Elixir parses but doesn't actually use. +See [Custom and overridden operators](#custom-and-overridden-operators) below for a list and for guidelines about their use. + +Some other operators are special forms and cannot be overridden: + + * [`^`](`^/1`) - pin operator + * [`.`](`./2`) - dot operator + * [`=`](`=/2`) - match operator + * [`&`](`&/1`) - capture operator + * [`::`](`::/2`) - type operator + +Finally, these operators appear in the precedence table above but are only meaningful within certain constructs: + + * `=>` - see [`%{}`](`%{}/1`) + * `when` - see [Guards](patterns-and-guards.md#guards) + * `<-` - see [`for`](`for/1`) and [`with`](`with/1`) + * `\\` - see [Default arguments](Kernel.html#def/2-default-arguments) + +## Comparison operators + +Elixir provides the following built-in comparison operators (all of which can be used in guards): + + * [`==`](`==/2`) - equal to + * [`===`](`===/2`) - strictly equal to + * [`!=`](`!=/2`) - inequal to + * [`!==`](`!==/2`) - strictly inequal to + * [`<`](``](`>/2`) - greater-than + * [`<=`](`<=/2`) - less-than or equal to + * [`>=`](`>=/2`) - greater-than or equal to + +The only difference between [`==`](`==/2`) and [`===`](`===/2`) is that [`===`](`===/2`) is strict when it comes to comparing integers and floats: + +```elixir +iex> 1 == 1.0 +true +iex> 1 === 1.0 +false +``` + +[`!=`](`!=/2`) and [`!==`](`!==/2`) act as the negation of [`==`](`==/2`) and [`===`](`===/2`), respectively. + +### Term ordering + +In Elixir, different data types can be compared using comparison operators: + +```elixir +iex> 1 < :an_atom +true +``` + +The reason we can compare different data types is pragmatism. Sorting algorithms don't need to worry about different data types in order to sort. For reference, the overall sorting order is defined below: + +``` +number < atom < reference < function < port < pid < tuple < map < list < bitstring +``` + +When comparing two numbers of different types (a number being either an integer or a float), a conversion to the type with greater precision will always occur, unless the comparison operator used is either [`===`](`===/2`) or [`!==`](`!==/2`). A float will be considered more precise than an integer, unless the float is greater/less than +/-9007199254740992.0 respectively, at which point all the significant figures of the float are to the left of the decimal point. This behavior exists so that the comparison of large numbers remains transitive. + +The collection types are compared using the following rules: + +* Tuples are compared by size, then element by element. +* Maps are compared by size, then by keys in ascending term order, then by values in key order. In the specific case of maps' key ordering, integers are always considered to be less than floats. +* Lists are compared element by element. +* Bitstrings are compared byte by byte, incomplete bytes are compared bit by bit. +* Atoms are compared using their string value, codepoint by codepoint. + +## Custom and overridden operators + +### Defining custom operators + +Elixir is capable of parsing a predefined set of operators. It's not possible to define new operators (as supported by some languages). However, not all operators that Elixir can parse are *used* by Elixir: for example, `+` and `||` are used by Elixir for addition and boolean *or*, but `<~>` is not used (but valid). + +To define an operator, you can use the usual `def*` constructs (`def`, `defp`, `defmacro`, and so on) but with a syntax similar to how the operator is used: + +```elixir +defmodule MyOperators do + # We define ~> to return the maximum of the given two numbers, + # and <~ to return the minimum. + + def a ~> b, do: max(a, b) + def a <~ b, do: min(a, b) +end +``` + +To use the newly defined operators, you **have to** import the module that defines them: + +```elixir +iex> import MyOperators +iex> 1 ~> 2 +2 +iex> 1 <~ 2 +1 +``` + +The following is a table of all the operators that Elixir is capable of parsing, but that are not used by default: + + * `|||` + * `&&&` + * `<<<` + * `>>>` + * `<<~` + * `~>>` + * `<~` + * `~>` + * `<~>` + * `+++` + * `---` + +The following operators are used by the `Bitwise` module when imported: [`&&&`](`Bitwise.&&&/2`), [`<<<`](`Bitwise.<<>>`](`Bitwise.>>>/2`), and [`|||`](`Bitwise.|||/2`). See the documentation for `Bitwise` for more information. + +Note that the Elixir community generally discourages custom operators. They can be hard to read and even more to understand, as they don't have a descriptive name like functions do. That said, some specific cases or custom domain specific languages (DSLs) may justify these practices. + +It is also possible to replace predefined operators, such as `+`, but doing so is extremely discouraged. diff --git a/lib/elixir/pages/patterns-and-guards.md b/lib/elixir/pages/patterns-and-guards.md new file mode 100644 index 00000000000..aec6280f3f4 --- /dev/null +++ b/lib/elixir/pages/patterns-and-guards.md @@ -0,0 +1,509 @@ +# Patterns and Guards + +Elixir provides pattern matching, which allows us to assert on the shape or extract values from data structures. Patterns are often augmented with guards, which give developers the ability to perform more complex checks, albeit limited. + +This page describes the semantics of patterns and guards, where they are all allowed, and how to extend them. + +## Patterns + +Patterns in Elixir are made of variables, literals, and data structure specific syntax. One of the most used constructs to perform pattern matching is the match operator ([`=`](`=/2`)): + +```iex +iex> x = 1 +1 +iex> 1 = x +1 +``` + +In the example above, `x` starts without a value and has `1` assigned to it. Then, we compare the value of `x` to the literal `1`, which succeeds as they are both `1`. + +Matching `x` against 2 would raise: + +```iex +iex> 2 = x +** (MatchError) no match of right hand side value: 1 +``` + +Patterns are not bidirectional. If you have a variable `y` that was never assigned to (often called an unbound variable) and you write `1 = y`, an error will be raised: + +```iex +iex> 1 = y +** (CompileError) iex:2: undefined function y/0 +``` + +In other words, patterns are allowed only on the left side of `=`. The right side of `=` follows the regular evaluation semantics of the language. + +Now let's cover the pattern matching rules for each construct and then for each relevant data types. + +### Variables + +Variables in patterns are always assigned to: + +```iex +iex> x = 1 +1 +iex> x = 2 +2 +iex> x +2 +``` + +In other words, Elixir supports rebinding. In case you don't want the value of a variable to change, you can use the pin operator (`^`): + +```iex +iex> x = 1 +1 +iex> ^x = 2 +** (MatchError) no match of right hand side value: 2 +``` + +If the same variable appears twice in the same pattern, then they must be bound to the same value: + +```iex +iex> {x, x} = {1, 1} +{1, 1} +iex> {x, x} = {1, 2} +** (MatchError) no match of right hand side value: {1, 2} +``` + +The underscore variable (`_`) has a special meaning as it can never be bound to any value. It is especially useful when you don't care about certain value in a pattern: + +```iex +iex> {_, integer} = {:not_important, 1} +{:not_important, 1} +iex> integer +1 +iex> _ +** (CompileError) iex:3: invalid use of _ +``` + +### Literals (numbers and atoms) + +Atoms and numbers (integers and floats) can appear in patterns and they are always represented as is. For example, an atom will only match an atom if they are the same atom: + +```iex +iex> :atom = :atom +:atom +iex> :atom = :another_atom +** (MatchError) no match of right hand side value: :another_atom +``` + +Similar rule applies to numbers. Finally, note that numbers in patterns perform strict comparison. In other words, integers to do not match floats: + +```iex +iex> 1 = 1.0 +** (MatchError) no match of right hand side value: 1.0 +``` + +### Tuples + +Tuples may appear in patterns using the curly brackets syntax (`{}`). A tuple in a pattern will match only tuples of the same size, where each individual tuple element must also match: + +```iex +iex> {:ok, integer} = {:ok, 13} +{:ok, 13} + +# won't match due to different size +iex> {:ok, integer} = {:ok, 11, 13} +** (MatchError) no match of right hand side value: {:ok, 11, 13} + +# won't match due to mismatch on first element +iex> {:ok, binary} = {:error, :enoent} +** (MatchError) no match of right hand side value: {:error, :enoent} +``` + +### Lists + +Lists may appear in patterns using the square brackets syntax (`[]`). A list in a pattern will match only lists of the same size, where each individual list element must also match: + +```iex +iex> [:ok, integer] = [:ok, 13] +[:ok, 13] + +# won't match due to different size +iex> [:ok, integer] = [:ok, 11, 13] +** (MatchError) no match of right hand side value: [:ok, 11, 13] + +# won't match due to mismatch on first element +iex> [:ok, binary] = [:error, :enoent] +** (MatchError) no match of right hand side value: [:error, :enoent] +``` + +Opposite to tuples, lists also allow matching on non-empty lists by using the `[head | tail]` notation, which matches on the `head` and `tail` of a list: + +```iex +iex> [head | tail] = [1, 2, 3] +[1, 2, 3] +iex> head +1 +iex> tail +[2, 3] +``` + +Multiple elements may prefix the `| tail` construct: + +```iex +iex> [first, second | tail] = [1, 2, 3] +[1, 2, 3] +iex> tail +[3] +``` + +Note `[head | tail]` does not match empty lists: + +```elixir +iex> [head | tail] = [] +** (MatchError) no match of right hand side value: [] +``` + +Given charlists are represented as a list of integers, one can also perform prefix matches on charlists using the list concatenation operator ([`++`](`++/2`)): + +```elixir +iex> 'hello ' ++ world = 'hello world' +'hello world' +iex> world +'world' +``` + +Which is equivalent to matching on `[?h, ?e, ?l, ?l, ?o, ?\s | world]`. Suffix matches (`hello ++ ' world'`) are not valid patterns. + +### Maps + +Maps may appear in patterns using the percentage sign followed by the curly brackets syntax (`%{}`). Opposite to lists and tuples, maps perform a subset match. This means a map pattern will match any other map that has at least all of the keys in the pattern. + +Here is an example where all keys match: + +```iex +iex> %{name: name} = %{name: "meg"} +%{name: "meg"} +iex> name +"meg" +``` + +Here is when a subset of the keys match: + +```iex +iex> %{name: name} = %{name: "meg", age: 23} +%{age: 23, name: "meg"} +iex> name +"meg" +``` + +If a key in the pattern is not available in the map, then they won't match: + +```iex +iex> %{name: name, age: age} = %{name: "meg"} +** (MatchError) no match of right hand side value: %{name: "meg"} +``` + +Note that the empty map will match all maps, which is a contrast to tuples and lists, where an empty tuple or an empty list will only match empty tuples and empty lists respectively: + +```iex +iex> %{} = %{name: "meg"} +%{name: "meg"} +``` + +Finally, note map keys in patterns must always be literals or previously bound variables matched with the pin operator. + +### Structs + +Structs may appear in patterns using the percentage sign, the struct module name or a variable followed by the curly brackets syntax (`%{}`). + +Given the following struct: + +```elixir +defmodule User do + defstruct [:name] +end +``` + +Here is an example where all keys match: + +```iex +iex> %User{name: name} = %User{name: "meg"} +%User{name: "meg"} +iex> name +"meg" +``` + +If an unknown key is given, the compiler will raise an error: + +```iex +iex> %User{type: type} = %User{name: "meg"} +** (CompileError) iex: unknown key :type for struct User +``` + +The struct name can be extracted when putting a variable instead of a module name: + +``` +iex> %struct_name{} = %User{name: "meg"} +%User{name: "meg"} +iex> struct_name +User +``` + +### Binaries + +Binaries may appear in patterns using the double less-than/greater-than syntax ([`<<>>`](`<<>>/1`)). A binary in a pattern can match multiple segments at the same, each with different type, size, and unit: + +```iex +iex> <> = <<123, 56>> +"{8" +iex> val +31544 +``` + +See the documentation for [`<<>>`](`<<>>/1`) for a complete definition of pattern matching for binaries. + +Finally, remember that strings in Elixir are UTF-8 encoded binaries. This means that, similar to charlists, prefix matches on strings are also possible with the binary concatenation operator ([`<>`](`<>/2`)): + +```elixir +iex> "hello " <> world = "hello world" +"hello world" +iex> world +"world" +``` + +Suffix matches (`hello <> " world"`) are not valid patterns. + +## Guards + +Guards are a way to augment pattern matching with more complex checks. They are allowed in a predefined set of constructs where pattern matching is allowed, such as function definitions, case clauses, and others. + +Not all expressions are allowed in guard clauses, but only a handful of them. This is a deliberate choice. This way, Elixir (and Erlang) can make sure that nothing bad happens while executing guards and no mutations happen anywhere. It also allows the compiler to optimize the code related to guards efficiently. + +### List of allowed functions and operators + +You can find the built-in list of guards [in the `Kernel` module](Kernel.html#guards). Here is an overview: + + * comparison operators ([`==`](`==/2`), [`!=`](`!=/2`), [`===`](`===/2`), [`!==`](`!==/2`), + [`<`](``](`>/2`), [`>=`](`>=/2`)) + * strictly boolean operators ([`and`](`and/2`), [`or`](`or/2`), [`not`](`not/1`)). Note [`&&`](`&&/2`), [`||`](`||/2`), and [`!`](`!/1`) sibling operators are **not allowed** as they're not *strictly* boolean - meaning they don't require arguments to be booleans + * arithmetic unary operators ([`+`](`+/1`), [`-`](`-/1`)) + * arithmetic binary operators [`+`](`+/2`), [`-`](`-/2`), [`*`](`*/2`), [`/`](`//2`)) + * [`in`](`in/2`) and [`not in`](`in/2`) operators (as long as the right-hand side is a list or a range) + * "type-check" functions (`is_list/1`, `is_number/1`, and the like) + * functions that work on built-in datatypes (`abs/1`, `hd/1`, `map_size/1`, and others) + * the `map.field` syntax + +The module `Bitwise` also includes a handful of [Erlang bitwise operations as guards](Bitwise.html#guards). + +Macros constructed out of any combination of the above guards are also valid guards - for example, `Integer.is_even/1`. For more information, see the "Custom patterns and guards expressions" section shown below. + +### Why guards + +Let's see an example of a guard used in a function clause: + +```elixir +def empty_map?(map) when map_size(map) == 0, do: true +def empty_map?(map) when is_map(map), do: false +``` + +Guards start with the `when` operator, followed by a guard expression. The clause will be executed if and only if the guard expression returns `true`. Multiple boolean conditions can be combined with the [`and`](`and/2`) and [`or`](`or/2`) operators. + +Writing the `empty_map?/1` function by only using pattern matching would not be possible (as pattern matching on `%{}` would match *any* map, not only the empty ones). + +### Failing guards + +A function clause will be executed if and only if its guard expression evaluates to `true`. If any other value is returned, the function clause will be skipped. In particular, guards have no concept of "truthy" or "falsy". + +For example, imagine a function that checks that the head of a list is not `nil`: + +```elixir +def not_nil_head?([head | _]) when head, do: true +def not_nil_head?(_), do: false + +not_nil_head?(["some_value", "another_value"]) +#=> false +``` + +Even though the head of the list is not `nil`, the first clause for `not_nil_head?/1` fails because the expression does not evaluate to `true`, but to `"some_value"`, therefore triggering the second clause which returns `false`. To make the guard behave correctly, you must ensure that the guard evaluates to `true`, like so: + +```elixir +def not_nil_head?([head | _]) when head != nil, do: true +def not_nil_head?(_), do: false + +not_nil_head?(["some_value", "another_value"]) +#=> true +``` + +### Errors in guards + +In guards, when functions would normally raise exceptions, they cause the guard to fail instead. + +For example, the `tuple_size/1` function only works with tuples. If we use it with anything else, an argument error is raised: + +```elixir +iex> tuple_size("hello") +** (ArgumentError) argument error +``` + +However, when used in guards, the corresponding clause will fail to match instead of raising an error: + +```elixir +iex> case "hello" do +...> something when tuple_size(something) == 2 -> +...> :worked +...> _anything_else -> +...> :failed +...> end +:failed +``` + +In many cases, we can take advantage of this. In the code above, we used `tuple_size/1` to both check that the given value is a tuple *and* check its size (instead of using `is_tuple(something) and tuple_size(something) == 2`). + +However, if your guard has multiple conditions, such as checking for tuples or maps, it is best to call type-check functions like `is_tuple/1` before `tuple_size/1`, otherwise the whole guard will fail if a tuple is not given. Alternatively, your function clause can use multiple guards as shown in the following section. + +### Multiple guards in the same clause + +There exists an additional way to simplify a chain of `or` expressions in guards: Elixir supports writing "multiple guards" in the same clause. The following code: + +```elixir +def is_number_or_nil(term) when is_integer(term) or is_float(term) or is_nil(term), + do: :maybe_number +def is_number_or_nil(_other), + do: :something_else +``` + +can be alternatively written as: + +```elixir +def is_number_or_nil(term) + when is_integer(term) + when is_float(term) + when is_nil(term) do + :maybe_number +end + +def is_number_or_nil(_other) do + :something_else +end +``` + +If each guard expression always returns a boolean, the two forms are equivalent. However, recall that if any function call in a guard raises an exception, the entire guard fails. To illustrate this, the following function will not detect empty tuples: + +```elixir +defmodule Check do + # If given a tuple, map_size/1 will raise, and tuple_size/1 will not be evaluated + def empty?(val) when map_size(val) == 0 or tuple_size(val) == 0, do: true + def empty?(_val), do: false +end + +Check.empty?(%{}) +#=> true + +Check.empty?({}) +#=> false # true was expected! +``` + +This could be corrected by ensuring that no exception is raised, either via type checks like `is_map(val) and map_size(val) == 0`, or by using multiple guards, so that if an exception causes one guard to fail, the next one is evaluated. + +```elixir +defmodule Check do + # If given a tuple, map_size/1 will raise, and the second guard will be evaluated + def empty?(val) + when map_size(val) == 0 + when tuple_size(val) == 0, + do: true + + def empty?(_val), do: false +end + +Check.empty?(%{}) +#=> true + +Check.empty?({}) +#=> true +``` + +## Where patterns and guards can be used + +In the examples above, we have used the match operator ([`=`](`=/2`)) and function clauses to showcase patterns and guards respectively. Here is the list of the built-in constructs in Elixir that support patterns and guards. + + * `match?/2`: + + ```elixir + match?({:ok, value} when value > 0, {:ok, 13}) + ``` + + * function clauses: + + ```elixir + def type(term) when is_integer(term), do: :integer + def type(term) when is_float(term), do: :float + ``` + + * [`case`](`case/2`) expressions: + + ```elixir + case x do + 1 -> :one + 2 -> :two + n when is_integer(n) and n > 2 -> :larger_than_two + end + ``` + + * anonymous functions (`fn/1`): + + ```elixir + larger_than_two? = fn + n when is_integer(n) and n > 2 -> true + n when is_integer(n) -> false + end + ``` + + * [`for`](`for/1`) and [`with`](`with/1`) support patterns and guards on the left side of `<-`: + + ```elixir + for x when x >= 0 <- [1, -2, 3, -4], do: x + ``` + + `with` also supports the `else` keyword, which supports patterns matching and guards. + + * [`try`](`try/1`) supports patterns and guards on `catch` and `else` + + * [`receive`](`receive/1`) supports patterns and guards to match on the received messages. + + * custom guards can also be defined with `defguard/1` and `defguardp/1`. A custom guard can only be defined based on existing guards. + +Note that the match operator ([`=`](`=/2`)) does *not* support guards: + +```elixir +{:ok, binary} = File.read("some/file") +``` + +## Custom patterns and guards expressions + +Only the constructs listed in this page are allowed in patterns and guards. However, we can take advantage of macros to write custom patterns guards that can simplify our programs or make them more domain-specific. At the end of the day, what matters is that the *output* of the macros boils down to a combination of the constructs above. + +For example, the `Record` module in Elixir provides a series of macros to be used in patterns and guards that allows tuples to have named fields during compilation. + +For defining your own guards, Elixir even provides conveniences in `defguard` and `defguardp`. Let's look at a quick case study: we want to check whether an argument is an even or an odd integer. With pattern matching this is impossible because there is an infinite number of integers, and therefore we can't pattern match on every single one of them. Therefore we must use guards. We will just focus on checking for even numbers since checking for the odd ones is almost identical. + +Such a guard would look like this: + +```elixir +def my_function(number) when is_integer(number) and rem(number, 2) == 0 do + # do stuff +end +``` + +It would be repetitive to write every time we need this check. Instead, you can use `defguard/1` and `defguardp/1` to create guard macros. Here's an example how: + +```elixir +defmodule MyInteger do + defguard is_even(term) when is_integer(term) and rem(term, 2) == 0 +end +``` + +and then: + +```elixir +import MyInteger, only: [is_even: 1] + +def my_function(number) when is_even(number) do + # do stuff +end +``` + +While it's possible to create custom guards with macros, it's recommended to define them using `defguard/1` and `defguardp/1` which perform additional compile-time checks. diff --git a/lib/elixir/pages/syntax-reference.md b/lib/elixir/pages/syntax-reference.md new file mode 100644 index 00000000000..6af5b49ea6d --- /dev/null +++ b/lib/elixir/pages/syntax-reference.md @@ -0,0 +1,556 @@ +# Syntax reference + +Elixir syntax was designed to have a straightforward conversion to an abstract syntax tree (AST). This means the Elixir syntax is mostly uniform with a handful of "syntax sugar" constructs to reduce the noise in common Elixir idioms. + +This document covers all of Elixir syntax constructs as a reference and then discuss their exact AST representation. + +## Reserved words + +These are the reserved words in the Elixir language. They are detailed throughout this guide but summed up here for convenience: + + * `true`, `false`, `nil` - used as atoms + * `when`, `and`, `or`, `not`, `in` - used as operators + * `fn` - used for anonymous function definitions + * `do`, `end`, `catch`, `rescue`, `after`, `else` - used in do-end blocks + +## Data types + +### Numbers + +Integers (`1234`) and floats (`123.4`) in Elixir are represented as a sequence of digits that may be separated by underscore for readability purposes, such as `1_000_000`. Integers never contain a dot (`.`) in their representation. Floats contain a dot and at least one other digit after the dot. Floats also support the scientific notation, such as `123.4e10` or `123.4E10`. + +### Atoms + +Unquoted atoms start with a colon (`:`) which must be immediately followed by a Unicode letter or an underscore. The atom may continue using a sequence of Unicode letters, numbers, underscores, and `@`. Atoms may end in `!` or `?`. Valid unquoted atoms are: `:ok`, `:ISO8601`, and `:integer?`. + +If the colon is immediately followed by a pair of double- or single-quotes surrounding the atom name, the atom is considered quoted. In contrast with an unquoted atom, this one can be made of any Unicode character (not only letters), such as `:'🌢 Elixir'`, `:"++olá++"`, and `:"123"`. + +Quoted and unquoted atoms with the same name are considered equivalent, so `:atom`, `:"atom"`, and `:'atom'` represent the same atom. The only catch is that the compiler will warn when quotes are used in atoms that do not need to be quoted. + +All operators in Elixir are also valid atoms. Valid examples are `:foo`, `:FOO`, `:foo_42`, `:foo@bar`, and `:++`. Invalid examples are `:@foo` (`@` is not allowed at start), `:123` (numbers are not allowed at start), and `:(*)` (not a valid operator). + +`true`, `false`, and `nil` are reserved words that are represented by the atoms `:true`, `:false` and `:nil` respectively. + +To learn more about all Unicode characters allowed in atom, see the [Unicode syntax](unicode-syntax.md) document. + +### Strings + +Single-line strings in Elixir are written between double-quotes, such as `"foo"`. Any double-quote inside the string must be escaped with `\ `. Strings support Unicode characters and are stored as UTF-8 encoded binaries. + +Multi-line strings in Elixir are written with three double-quotes, and can have unescaped quotes within them. The resulting string will end with a newline. The indentation of the last `"""` is used to strip indentation from the inner string. For example: + +``` +iex> test = """ +...> this +...> is +...> a +...> test +...> """ +" this\n is\n a\n test\n" +iex> test = """ +...> This +...> Is +...> A +...> Test +...> """ +"This\nIs\nA\nTest\n" +``` + +Strings are always represented as themselves in the AST. + +### Charlists + +Charlists in Elixir are written in single-quotes, such as `'foo'`. Any single-quote inside the string must be escaped with `\ `. Charlists are made of non-negative integers, where each integer represents a Unicode code point. + +Multi-line charlists are written with three single-quotes (`'''`), the same way multi-line strings are. + +Charlists are always represented as themselves in the AST. + +For more in-depth information, please read the "Charlists" section in the `List` module. + +### Lists, tuples and binaries + +Data structures such as lists, tuples, and binaries are marked respectively by the delimiters `[...]`, `{...}`, and `<<...>>`. Each element is separated by comma. A trailing comma is also allowed, such as in `[1, 2, 3,]`. + +### Maps and keyword lists + +Maps use the `%{...}` notation and each key-value is given by pairs marked with `=>`, such as `%{"hello" => 1, 2 => "world"}`. + +Both keyword lists (list of two-element tuples where the first element is atom) and maps with atom keys support a keyword notation where the colon character `:` is moved to the end of the atom. `%{hello: "world"}` is equivalent to `%{:hello => "world"}` and `[foo: :bar]` is equivalent to `[{:foo, :bar}]`. This notation is a syntax sugar that emits the same AST representation. It will be explained in later sections. + +### Structs + +Structs built on the map syntax by passing the struct name between `%` and `{`. For example, `%User{...}`. + +## Expressions + +### Variables + +Variables in Elixir must start with an underscore or a Unicode letter that is not in uppercase or titlecase. The variable may continue using a sequence of Unicode letters, numbers, and underscores. Variables may end in `?` or `!`. To learn more about all Unicode characters allowed in variables, see the [Unicode syntax](unicode-syntax.md) document. + +[Elixir's naming conventions](naming-conventions.md) recommend variables to be in `snake_case` format. + +### Non-qualified calls (local calls) + +Non-qualified calls, such as `add(1, 2)`, must start with an underscore or a Unicode letter that is not in uppercase or titlecase. The call may continue using a sequence of Unicode letters, numbers, and underscore. Calls may end in `?` or `!`. To learn more about all Unicode characters allowed in calls, see the [Unicode syntax](unicode-syntax.md) document. + +Parentheses for non-qualified calls are optional, except for zero-arity calls, which would then be ambiguous with variables. If parentheses are used, they must immediately follow the function name *without spaces*. For example, `add (1, 2)` is a syntax error, since `(1, 2)` is treated as an invalid block which is attempted to be given as a single argument to `add`. + +[Elixir's naming conventions](naming-conventions.md) recommend calls to be in `snake_case` format. + +### Operators + +As many programming languages, Elixir also support operators as non-qualified calls with their precedence and associativity rules. Constructs such as `=`, `when`, `&` and `@` are simply treated as operators. See [the Operators page](operators.md) for a full reference. + +### Qualified calls (remote calls) + +Qualified calls, such as `Math.add(1, 2)`, must start with an underscore or a Unicode letter that is not in uppercase or titlecase. The call may continue using a sequence of Unicode letters, numbers, and underscores. Calls may end in `?` or `!`. To learn more about all Unicode characters allowed in calls, see the [Unicode syntax](unicode-syntax.md) document. + +[Elixir's naming conventions](naming-conventions.md) recommend calls to be in `snake_case` format. + +For qualified calls, Elixir also allows the function name to be written between double- or single-quotes, allowing calls such as `Math."++add++"(1, 2)`. Operators can be used as qualified calls without a need for quote, such as `Kernel.+(1, 2)`. + +Parentheses for qualified calls are optional. If parentheses are used, they must immediately follow the function name *without spaces*. + +### Aliases + +Aliases are constructs that expand to atoms at compile-time. The alias `String` expands to the atom `:"Elixir.String"`. Aliases must start with an ASCII uppercase character which may be followed by any ASCII letter, number, or underscore. Non-ASCII characters are not supported in aliases. + +[Elixir's naming conventions](naming-conventions.md) recommend aliases to be in `CamelCase` format. + +### Blocks + +Blocks are multiple Elixir expressions separated by newlines or semi-colons. A new block may be created at any moment by using parentheses. + +### Left to right arrow + +The left to right arrow (`->`) is used to establish a relationship between left and right, commonly referred as clauses. The left side may have zero, one, or more arguments; the right side is zero, one, or more expressions separated by new line. The `->` may appear one or more times between one of the following terminators: `do`-`end`, `fn`-`end` or `(`-`)`. When `->` is used, only other clauses are allowed between those terminators. Mixing clauses and regular expressions is invalid syntax. + +It is seen on `case` and `cond` constructs between `do` and `end`: + +```elixir +case 1 do + 2 -> 3 + 4 -> 5 +end + +cond do + true -> false +end +``` + +Seen in typespecs between `(` and `)`: + +```elixir +(integer(), boolean() -> integer()) +``` + +It is also used between `fn` and `end` for building anonymous functions: + +```elixir +fn + x, y -> x + y +end +``` + +### Sigils + +Sigils start with `~` and are followed by a letter and one of the following pairs: + + * `(` and `)` + * `{` and `}` + * `[` and `]` + * `<` and `>` + * `"` and `"` + * `'` and `'` + * `|` and `|` + * `/` and `/` + +After closing the pair, zero or more ASCII letters can be given as a modifier. Sigils are expressed as non-qualified calls prefixed with `sigil_` where the first argument is the sigil contents as a string and the second argument is a list of integers as modifiers: + +If the sigil letter is in uppercase, no interpolation is allowed in the sigil, otherwise its contents may be dynamic. Compare the results of the sigils below for more information: + +```elixir +~s/f#{"o"}o/ +~S/f#{"o"}o/ +``` + +Sigils are useful to encode text with their own escaping rules, such as regular expressions, datetimes, and others. + +## The Elixir AST + +Elixir syntax was designed to have a straightforward conversion to an abstract syntax tree (AST). Elixir's AST is a regular Elixir data structure composed of the following elements: + + * atoms - such as `:foo` + * integers - such as `42` + * floats - such as `13.1` + * strings - such as `"hello"` + * lists - such as `[1, 2, 3]` + * tuples with two elements - such as `{"hello", :world}` + * tuples with three elements, representing calls or variables, as explained next + +The building block of Elixir's AST is a call, such as: + +```elixir +sum(1, 2, 3) +``` + +which is represented as a tuple with three elements: + +```elixir +{:sum, meta, [1, 2, 3]} +``` + +the first element is an atom (or another tuple), the second element is a list of two-element tuples with metadata (such as line numbers) and the third is a list of arguments. + +We can retrieve the AST for any Elixir expression by calling `quote`: + +```elixir +quote do + sum() +end +#=> {:sum, [], []} +``` + +Variables are also represented using a tuple with three elements and a combination of lists and atoms, for example: + +```elixir +quote do + sum +end +#=> {:sum, [], Elixir} +``` + +You can see that variables are also represented with a tuple, except the third element is an atom expressing the variable context. + +Over the course of this section, we will explore many Elixir syntax constructs alongside their AST representations. + +### Operators + +Operators are treated as non-qualified calls: + +```elixir +quote do + 1 + 2 +end +#=> {:+, [], [1, 2]} +``` + +Note that `.` is also an operator. Remote calls use the dot in the AST with two arguments, where the second argument is always an atom: + +```elixir +quote do + foo.bar(1, 2, 3) +end +#=> {{:., [], [{:foo, [], Elixir}, :bar]}, [], [1, 2, 3]} +``` + +Calling anonymous functions uses the dot in the AST with a single argument, mirroring the fact the function name is "missing" from right side of the dot: + +```elixir +quote do + foo.(1, 2, 3) +end +#=> {{:., [], [{:foo, [], Elixir}]}, [], [1, 2, 3]} +``` + +### Aliases + +Aliases are represented by an `__aliases__` call with each segment separated by a dot as an argument: + +```elixir +quote do + Foo.Bar.Baz +end +#=> {:__aliases__, [], [:Foo, :Bar, :Baz]} + +quote do + __MODULE__.Bar.Baz +end +#=> {:__aliases__, [], [{:__MODULE__, [], Elixir}, :Bar, :Baz]} +``` + +All arguments, except the first, are guaranteed to be atoms. + +### Data structures + +Remember that lists are literals, so they are represented as themselves in the AST: + +```elixir +quote do + [1, 2, 3] +end +#=> [1, 2, 3] +``` + +Tuples have their own representation, except for two-element tuples, which are represented as themselves: + +```elixir +quote do + {1, 2} +end +#=> {1, 2} + +quote do + {1, 2, 3} +end +#=> {:{}, [], [1, 2, 3]} +``` + +Binaries have a representation similar to tuples, except they are tagged with `:<<>>` instead of `:{}`: + +```elixir +quote do + <<1, 2, 3>> +end +#=> {:<<>>, [], [1, 2, 3]} +``` + +The same applies to maps, where each pair is treated as a list of tuples with two elements: + +```elixir +quote do + %{1 => 2, 3 => 4} +end +#=> {:%{}, [], [{1, 2}, {3, 4}]} +``` + +### Blocks + +Blocks are represented as a `__block__` call with each line as a separate argument: + +```elixir +quote do + 1 + 2 + 3 +end +#=> {:__block__, [], [1, 2, 3]} + +quote do 1; 2; 3; end +#=> {:__block__, [], [1, 2, 3]} +``` + +### Left to right arrow + +The left to right arrow (`->`) is represented similar to operators except that they are always part of a list, its left side represents a list of arguments and the right side is an expression. + +For example, in `case` and `cond`: + +```elixir +quote do + case 1 do + 2 -> 3 + 4 -> 5 + end +end +#=> {:case, [], [1, [do: [{:->, [], [[2], 3]}, {:->, [], [[4], 5]}]]]} + +quote do + cond do + true -> false + end +end +#=> {:cond, [], [[do: [{:->, [], [[true], false]}]]]} +``` + +Between `(` and `)`: + +```elixir +quote do + (1, 2 -> 3 + 4, 5 -> 6) +end +#=> [{:->, [], [[1, 2], 3]}, {:->, [], [[4, 5], 6]}] +``` + +Between `fn` and `end`: + +```elixir +quote do + fn + 1, 2 -> 3 + 4, 5 -> 6 + end +end +#=> {:fn, [], [{:->, [], [[1, 2], 3]}, {:->, [], [[4, 5], 6]}]} +``` + +### Qualified tuples + +Qualified tuples (`foo.{bar, baz}`) are represented by a `{:., [], [expr, :{}]}` call, where the `expr` represents the left hand side of the dot, and the arguments represent the elements inside the curly braces. This is used in Elixir to provide multi aliases: + +```elixir +quote do + Foo.{Bar, Baz} +end +#=> {{:., [], [{:__aliases__, [], [:Foo]}, :{}]}, [], [{:__aliases__, [], [:Bar]}, {:__aliases__, [], [:Baz]}]} +``` + +## Syntactic sugar + +All of the constructs above are part of Elixir's syntax and have their own representation as part of the Elixir AST. This section will discuss the remaining constructs that "desugar" to one of the constructs explored above. In other words, the constructs below can be represented in more than one way in your Elixir code and retain AST equivalence. + +### Integers in other bases and Unicode code points + +Elixir allows integers to contain `_` to separate digits and provides conveniences to represent integers in other bases: + +```elixir +1_000_000 +#=> 1000000 + +0xABCD +#=> 43981 (Hexadecimal base) + +0o01234567 +#=> 342391 (Octal base) + +0b10101010 +#=> 170 (Binary base) + +?é +#=> 233 (Unicode code point) +``` + +Those constructs exist only at the syntax level. All of the examples above are represented as their underlying integers in the AST. + +### Access syntax + +The access syntax is represented as a call to `Access.get/2`: + +```elixir +quote do + opts[arg] +end +#=> {{:., [], [Access, :get]}, [], [{:opts, [], Elixir}, {:arg, [], Elixir}]} +``` + +### Optional parentheses + +Elixir provides optional parentheses: + +```elixir +quote do + sum 1, 2, 3 +end +#=> {:sum, [], [1, 2, 3]} +``` + +The above is treated the same as `sum(1, 2, 3)` by the parser. You can remove the parentheses on all calls with at least one argument. + +You can also skip parentheses on qualified calls, such as `Foo.bar 1, 2, 3`. Parentheses are required when invoking anonymous functions, such as `f.(1, 2, 3)`. + +In practice, developers prefer to add parentheses to most of their calls. They are skipped mainly in Elixir's control-flow constructs, such as `defmodule`, `if`, `case`, etc, and in certain DSLs. + +### Keywords + +Keywords in Elixir are a list of tuples of two elements, where the first element is an atom. Using the base constructs, they would be represented as: + +```elixir +[{:foo, 1}, {:bar, 2}] +``` + +However, Elixir introduces a syntax sugar where the keywords above may be written as follows: + +```elixir +[foo: 1, bar: 2] +``` + +Atoms with foreign characters, such as whitespace, must be wrapped in quotes. This rule applies to keywords as well: + +```elixir +[{:"foo bar", 1}, {:"bar baz", 2}] == ["foo bar": 1, "bar baz": 2] +``` + +Remember that, because lists and two-element tuples are quoted literals, by definition keywords are also literals (in fact, the only reason tuples with two elements are quoted literals is to support keywords as literals). + +### Keywords as last arguments + +Elixir also supports a syntax where if the last argument of a call is a keyword list then the square brackets can be skipped. This means that the following: + +```elixir +if(condition, do: this, else: that) +``` + +is the same as + +```elixir +if(condition, [do: this, else: that]) +``` + +which in turn is the same as + +```elixir +if(condition, [{:do, this}, {:else, that}]) +``` + +### `do`-`end` blocks + +The last syntax convenience are `do`-`end` blocks. `do`-`end` blocks are equivalent to keywords as the last argument of a function call, where the block contents are wrapped in parentheses. For example: + +```elixir +if true do + this +else + that +end +``` + +is the same as: + +```elixir +if(true, do: (this), else: (that)) +``` + +which we have explored in the previous section. + +Parentheses are important to support multiple expressions. This: + +```elixir +if true do + this + that +end +``` + +is the same as: + +```elixir +if(true, do: ( + this + that +)) +``` + +Inside `do`-`end` blocks you may introduce other keywords, such as `else` used in the `if` above. The supported keywords between `do`-`end` are static and are: + + * `after` + * `catch` + * `else` + * `rescue` + +You can see them being used in constructs such as `receive`, `try`, and others. + +## Summary + +This document provides a reference to Elixir syntax, exploring its constructs and their AST equivalents. + +We have also discussed a handful of syntax conveniences provided by Elixir. Those conveniences are what allow us to write + +```elixir +defmodule Math do + def add(a, b) do + a + b + end +end +``` + +instead of + +```elixir +defmodule(Math, [ + {:do, def(add(a, b), [{:do, a + b}])} +]) +``` + +The mapping between code and data (the underlying AST) is what allows Elixir to implement `defmodule`, `def`, `if`, and others in Elixir itself. Elixir makes the constructs available for building the language accessible to developers who want to extend the language to new domains. diff --git a/lib/elixir/pages/typespecs.md b/lib/elixir/pages/typespecs.md new file mode 100644 index 00000000000..b645c3b7878 --- /dev/null +++ b/lib/elixir/pages/typespecs.md @@ -0,0 +1,280 @@ +# Typespecs + +Elixir comes with a notation for declaring types and specifications. Elixir is a dynamically typed language, and as such, type specifications are never used by the compiler to optimize or modify code. Still, using type specifications is useful because: + + * they provide documentation (for example, tools such as [`ExDoc`](https://hexdocs.pm/ex_doc/) show type specifications in the documentation) + * they're used by tools such as [Dialyzer](`:dialyzer`), that can analyze code with typespec to find type inconsistencies and possible bugs + +Type specifications (sometimes referred to as *typespecs*) are defined in different contexts using the following attributes: + + * `@type` + * `@opaque` + * `@typep` + * `@spec` + * `@callback` + * `@macrocallback` + +See the "User-defined types" and "Defining a specification" sub-sections below for more information on defining types and typespecs. + +## A simple example + + defmodule StringHelpers do + @type word() :: String.t() + + @spec long_word?(word()) :: boolean() + def long_word?(word) when is_binary(word) do + String.length(word) > 8 + end + end + +In the example above, this happens: + + * we declare a new type (`word()`) that is equivalent to the string type (`String.t()`); + + * we specify that the `long_word?/1` function takes an argument of type `word()` and + returns a boolean (`boolean()`), that is, either `true` or `false`. + +## Types and their syntax + +The syntax Elixir provides for type specifications is similar to [the one in Erlang](https://www.erlang.org/doc/reference_manual/typespec.html). Most of the built-in types provided in Erlang (for example, `pid()`) are expressed in the same way: `pid()` (or simply `pid`). Parameterized types (such as `list(integer)`) are supported as well and so are remote types (such as [`Enum.t()`](`t:Enum.t/0`)). Integers and atom literals are allowed as types (for example, `1`, `:atom`, or `false`). All other types are built out of unions of predefined types. Some shorthands are allowed, such as `[...]`, `<<>>`, and `{...}`. + +The notation to represent the union of types is the pipe `|`. For example, the typespec `type :: atom() | pid() | tuple()` creates a type `type` that can be either an `atom`, a `pid`, or a `tuple`. This is usually called a [sum type](https://en.wikipedia.org/wiki/Tagged_union) in other languages + +### Basic types + + type :: + any() # the top type, the set of all terms + | none() # the bottom type, contains no terms + | atom() + | map() # any map + | pid() # process identifier + | port() # port identifier + | reference() + | tuple() # tuple of any size + + ## Numbers + | float() + | integer() + | neg_integer() # ..., -3, -2, -1 + | non_neg_integer() # 0, 1, 2, 3, ... + | pos_integer() # 1, 2, 3, ... + + ## Lists + | list(type) # proper list ([]-terminated) + | nonempty_list(type) # non-empty proper list + | maybe_improper_list(content_type, termination_type) # proper or improper list + | nonempty_improper_list(content_type, termination_type) # improper list + | nonempty_maybe_improper_list(content_type, termination_type) # non-empty proper or improper list + + | Literals # Described in section "Literals" + | BuiltIn # Described in section "Built-in types" + | Remotes # Described in section "Remote types" + | UserDefined # Described in section "User-defined types" + +### Literals + +The following literals are also supported in typespecs: + + type :: ## Atoms + :atom # atoms: :foo, :bar, ... + | true | false | nil # special atom literals + + ## Bitstrings + | <<>> # empty bitstring + | <<_::size>> # size is 0 or a positive integer + | <<_::_*unit>> # unit is an integer from 1 to 256 + | <<_::size, _::_*unit>> + + ## (Anonymous) Functions + | (-> type) # zero-arity, returns type + | (type1, type2 -> type) # two-arity, returns type + | (... -> type) # any arity, returns type + + ## Integers + | 1 # integer + | 1..10 # integer from 1 to 10 + + ## Lists + | [type] # list with any number of type elements + | [] # empty list + | [...] # shorthand for nonempty_list(any()) + | [type, ...] # shorthand for nonempty_list(type) + | [key: value_type] # keyword list with optional key :key of value_type + + ## Maps + | %{} # empty map + | %{key: value_type} # map with required key :key of value_type + | %{key_type => value_type} # map with required pairs of key_type and value_type + | %{required(key_type) => value_type} # map with required pairs of key_type and value_type + | %{optional(key_type) => value_type} # map with optional pairs of key_type and value_type + | %SomeStruct{} # struct with all fields of any type + | %SomeStruct{key: value_type} # struct with required key :key of value_type + + ## Tuples + | {} # empty tuple + | {:ok, type} # two-element tuple with an atom and any type + +### Built-in types + +The following types are also provided by Elixir as shortcuts on top of the basic and literal types described above. + +Built-in type | Defined as +:---------------------- | :--------- +`term()` | `any()` +`arity()` | `0..255` +`as_boolean(t)` | `t` +`binary()` | `<<_::_*8>>` +`bitstring()` | `<<_::_*1>>` +`boolean()` | `true` \| `false` +`byte()` | `0..255` +`char()` | `0..0x10FFFF` +`charlist()` | `[char()]` +`nonempty_charlist()` | `[char(), ...]` +`fun()` | `(... -> any)` +`function()` | `fun()` +`identifier()` | `pid()` \| `port()` \| `reference()` +`iodata()` | `iolist()` \| `binary()` +`iolist()` | `maybe_improper_list(byte() \| binary() \| iolist(), binary() \| [])` +`keyword()` | `[{atom(), any()}]` +`keyword(t)` | `[{atom(), t}]` +`list()` | `[any()]` +`nonempty_list()` | `nonempty_list(any())` +`maybe_improper_list()` | `maybe_improper_list(any(), any())` +`nonempty_maybe_improper_list()` | `nonempty_maybe_improper_list(any(), any())` +`mfa()` | `{module(), atom(), arity()}` +`module()` | `atom()` +`no_return()` | `none()` +`node()` | `atom()` +`number()` | `integer()` \| `float()` +`struct()` | `%{:__struct__ => atom(), optional(atom()) => any()}` +`timeout()` | `:infinity` \| `non_neg_integer()` + +`as_boolean(t)` exists to signal users that the given value will be treated as a boolean, where `nil` and `false` will be evaluated as `false` and everything else is `true`. For example, `Enum.filter/2` has the following specification: `filter(t, (element -> as_boolean(term))) :: list`. + +### Remote types + +Any module is also able to define its own types and the modules in Elixir are no exception. For example, the `Range` module defines a `t/0` type that represents a range: this type can be referred to as `t:Range.t/0`. In a similar fashion, a string is `t:String.t/0`, and so on. + +### Maps + +The key types in maps are allowed to overlap, and if they do, the leftmost key takes precedence. +A map value does not belong to this type if it contains a key that is not in the allowed map keys. + +If you want to denote that keys that were not previously defined in the map are allowed, +it is common to end a map type with `optional(any) => any`. + +Note that the syntactic representation of `map()` is `%{optional(any) => any}`, not `%{}`. The notation `%{}` specifies the singleton type for the empty map. + +### User-defined types + +The `@type`, `@typep`, and `@opaque` module attributes can be used to define new types: + + @type type_name :: type + @typep type_name :: type + @opaque type_name :: type + +A type defined with `@typep` is private. An opaque type, defined with `@opaque` is a type where the internal structure of the type will not be visible, but the type is still public. + +Types can be parameterized by defining variables as parameters; these variables can then be used to define the type. + + @type dict(key, value) :: [{key, value}] + +## Defining a specification + +A specification for a function can be defined as follows: + + @spec function_name(type1, type2) :: return_type + +Guards can be used to restrict type variables given as arguments to the function. + + @spec function(arg) :: [arg] when arg: atom + +If you want to specify more than one variable, you separate them by a comma. + + @spec function(arg1, arg2) :: {arg1, arg2} when arg1: atom, arg2: integer + +Type variables with no restriction can also be defined using `var`. + + @spec function(arg) :: [arg] when arg: var + +This guard notation only works with `@spec`, `@callback`, and `@macrocallback`. + +You can also name your arguments in a typespec using `arg_name :: arg_type` syntax. This is particularly useful in documentation as a way to differentiate multiple arguments of the same type (or multiple elements of the same type in a type definition): + + @spec days_since_epoch(year :: integer, month :: integer, day :: integer) :: integer + @type color :: {red :: integer, green :: integer, blue :: integer} + +Specifications can be overloaded, just like ordinary functions. + + @spec function(integer) :: atom + @spec function(atom) :: integer + +## Behaviours + +Behaviours in Elixir (and Erlang) are a way to separate and abstract the generic part of a component (which becomes the *behaviour module*) from the specific part (which becomes the *callback module*). + +A behaviour module defines a set of functions and macros (referred to as *callbacks*) that callback modules implementing that behaviour must export. This "interface" identifies the specific part of the component. For example, the `GenServer` behaviour and functions abstract away all the message-passing (sending and receiving) and error reporting that a "server" process will likely want to implement from the specific parts such as the actions that this server process has to perform. + +To define a behaviour module, it's enough to define one or more callbacks in that module. To define callbacks, the `@callback` and `@macrocallback` module attributes can be used (for function callbacks and macro callbacks respectively). + + defmodule MyBehaviour do + @callback my_fun(arg :: any) :: any + @macrocallback my_macro(arg :: any) :: Macro.t + end + +As seen in the example above, defining a callback is a matter of defining a specification for that callback, made of: + + * the callback name (`my_fun` or `my_macro` in the example) + * the arguments that the callback must accept (`arg :: any` in the example) + * the *expected* type of the callback return value + +### Optional callbacks + +Optional callbacks are callbacks that callback modules may implement if they want to, but are not required to. Usually, behaviour modules know if they should call those callbacks based on configuration, or they check if the callbacks are defined with `function_exported?/3` or `macro_exported?/3`. + +Optional callbacks can be defined through the `@optional_callbacks` module attribute, which has to be a keyword list with function or macro name as key and arity as value. For example: + + defmodule MyBehaviour do + @callback vital_fun() :: any + @callback non_vital_fun() :: any + @macrocallback non_vital_macro(arg :: any) :: Macro.t + @optional_callbacks non_vital_fun: 0, non_vital_macro: 1 + end + +One example of optional callback in Elixir's standard library is `c:GenServer.format_status/2`. + +### Implementing behaviours + +To specify that a module implements a given behaviour, the `@behaviour` attribute must be used: + + defmodule MyBehaviour do + @callback my_fun(arg :: any) :: any + end + + defmodule MyCallbackModule do + @behaviour MyBehaviour + def my_fun(arg), do: arg + end + +If a callback module that implements a given behaviour doesn't export all the functions and macros defined by that behaviour, the user will be notified through warnings during the compilation process (no errors will happen). + +Elixir's standard library contains a few frequently used behaviours such as `GenServer`, `Supervisor`, and `Application`. + +### Inspecting behaviours + +The `@callback` and `@optional_callbacks` attributes are used to create a `behaviour_info/1` function available on the defining module. This function can be used to retrieve the callbacks and optional callbacks defined by that module. + +For example, for the `MyBehaviour` module defined in "Optional callbacks" above: + + MyBehaviour.behaviour_info(:callbacks) + #=> [vital_fun: 0, "MACRO-non_vital_macro": 2, non_vital_fun: 0] + MyBehaviour.behaviour_info(:optional_callbacks) + #=> ["MACRO-non_vital_macro": 2, non_vital_fun: 0] + +When using `iex`, the `IEx.Helpers.b/1` helper is also available. + +## The `string()` type + +Elixir discourages the use of the `string()` type. The `string()` type refers to Erlang strings, which are known as "charlists" in Elixir. They do not refer to Elixir strings, which are UTF-8 encoded binaries. To avoid confusion, if you attempt to use the type `string()`, Elixir will emit a warning. You should use `charlist()`, `nonempty_charlist()`, `binary()` or `String.t()` accordingly, or any of the several literal representations for these types. + +Note that `String.t()` and `binary()` are equivalent to analysis tools. Although, for those reading the documentation, `String.t()` implies it is a UTF-8 encoded binary. diff --git a/lib/elixir/pages/unicode-syntax.md b/lib/elixir/pages/unicode-syntax.md new file mode 100644 index 00000000000..71908e4ae4b --- /dev/null +++ b/lib/elixir/pages/unicode-syntax.md @@ -0,0 +1,158 @@ +# Unicode Syntax + +Elixir supports Unicode throughout the language. + +Strings (`"olá"`) and charlists (`'olá'`) support Unicode since Elixir v1.0. Strings are UTF-8 encoded. Charlists are lists of Unicode code points. In such cases, the contents are kept as written by developers, without any transformation. + +Elixir also supports Unicode in variables, atoms, and calls since Elixir v1.5. The focus of this document is to provide a high-level introduction to how Elixir allows Unicode in its syntax. We also provide technical documentation describing how Elixir complies with the Unicode specification. + +To check the Unicode version of your current Elixir installation, run `String.Unicode.version()`. + +## Introduction + +Elixir allows Unicode characters in its variables, atoms, and calls. However, the Unicode characters must still obey the rules of the language syntax. In particular, variables and calls cannot start with an uppercase letter. From now on, we will refer to those terms as identifiers. + +The characters allowed in identifiers are the ones specified by Unicode. Generally speaking, it is restricted to characters typically used by the writing system of human languages still in activity. In particular, it excludes symbols such as emojis, alternate numeric representations, musical notes, and the like. + +Elixir imposes many restrictions on identifiers for security purposes. For example, the word "josé" can be written in two ways in Unicode: as the combination of the characters `j o s é` and as a combination of the characters `j o s e ́ `, where the accent is its own character. The former is called NFC form and the latter is the NFD form. Elixir normalizes all characters to be the in the NFC form. + +Elixir also disallows mixed-scripts in most scenarios. For example, it is not possible to name a variable `аdmin`, where `а` is in Cyrillic and the remaining characters are in Latin. Doing so will raise the following error: + +``` +** (SyntaxError) invalid mixed-script identifier found: аdmin + +Mixed-script identifiers are not supported for security reasons. 'аdmin' is made of the following scripts: + + \u0430 а {Cyrillic} + \u0064 d {Latin} + \u006D m {Latin} + \u0069 i {Latin} + \u006E n {Latin} + +Make sure all characters in the identifier resolve to a single script or a highly +restrictive script. See https://hexdocs.pm/elixir/unicode-syntax.html for more information. +``` + +The character must either be all in Cyrillic or all in Latin. The only mixed-scripts that Elixir allows, according to the Highly Restrictive Unicode recommendations, are: + + * Latin and Han with Bopomofo + * Latin and Japanese + * Latin and Korean + +Finally, Elixir will also warn on confusable identifiers in the same file. For example, Elixir will emit a warning if you use both variables `а` (Cyrillic) and `а` (Latin) in your code. + +That's the overall introduction of how Unicode is used in Elixir identifiers. In a nutshell, its goal is to support different writing systems in use today while keeping the Elixir languague itself clear and secure. + +For the technical details, see the next sections that cover the technical Unicode requirements. + +## Unicode Annex #31 + +Elixir implements the requirements outlined in the [Unicode Annex #31](https://unicode.org/reports/tr31/). + +### R1. Default Identifiers + +The general Elixir identifier rule is specified as: + + := * ? + +where `` uses the same categories as the spec but restricts them to the NFC form (see R6): + +> characters derived from the Unicode General Category of uppercase letters, lowercase letters, titlecase letters, modifier letters, other letters, letter numbers, plus `Other_ID_Start`, minus `Pattern_Syntax` and `Pattern_White_Space` code points +> +> In set notation: `[\p{L}\p{Nl}\p{Other_ID_Start}-\p{Pattern_Syntax}-\p{Pattern_White_Space}]`. + +and `` uses the same categories as the spec but restricts them to the NFC form (see R6): + +> ID_Start characters, plus characters having the Unicode General Category of nonspacing marks, spacing combining marks, decimal number, connector punctuation, plus `Other_ID_Continue`, minus `Pattern_Syntax` and `Pattern_White_Space` code points. +> +> In set notation: `[\p{ID_Start}\p{Mn}\p{Mc}\p{Nd}\p{Pc}\p{Other_ID_Continue}-\p{Pattern_Syntax}-\p{Pattern_White_Space}]`. + +`` is an addition specific to Elixir that includes only the code points `?` (003F) and `!` (0021). + +The spec also provides a `` set, but Elixir does not include any character on this set. Therefore, the identifier rule has been simplified to consider this. + +Elixir does not allow the use of ZWJ or ZWNJ in identifiers and therefore does not implement R1a. R1b is guaranteed for backwards compatibility purposes. + +#### Atoms + +Unicode atoms in Elixir follow the identifier rule above with the following modifications: + + * `` additionally includes the code point `_` (005F) + * `` additionally includes the code point `@` (0040) + +Note atoms can also be quoted, which allows any characters, such as `:"hello elixir"`. All Elixir operators are also valid atoms, such as `:+`, `:@`, `:|>`, and others. The full description of valid atoms is available in the ["Atoms" section in the syntax reference](syntax-reference.html#atoms). + +#### Variables + +Variables in Elixir follow the identifier rule above with the following modifications: + + * `` additionally includes the code point `_` (005F) + * `` additionally excludes Lu (letter uppercase) and Lt (letter titlecase) characters + +In set notation: `[\u{005F}\p{Ll}\p{Lm}\p{Lo}\p{Nl}\p{Other_ID_Start}-\p{Pattern_Syntax}-\p{Pattern_White_Space}]`. + +### R3. Pattern_White_Space and Pattern_Syntax Characters + +Elixir supports only code points `\t` (0009), `\n` (000A), `\r` (000D) and `\s` (0020) as whitespace and therefore does not follow requirement R3. R3 requires a wider variety of whitespace and syntax characters to be supported. + +### R6. Filtered Normalized Identifiers + +Identifiers in Elixir are case sensitive. + +Elixir requires all atoms and variables to be in NFC form. If another form is given, NFC is automatically applied. Quoted-atoms and strings can, however, be in any form and are not verified by the parser. + +In other words, the atom `:josé` can only be written with the code points `006A 006F 0073 00E9` or `006A 006F 0073 0065 0301`, but Elixir will rewrite it to the former (from Elixir 1.14). On the other hand, `:"josé"` may be written as `006A 006F 0073 00E9` or `006A 006F 0073 0065 0301` and its form will be retained, since it is written between quotes. + +Choosing requirement R6 automatically excludes requirements R4, R5 and R7. + +## Unicode Technical Standard #39 + +Elixir conforms to the clauses outlined in the [Unicode Technical Standard #39](https://unicode.org/reports/tr39/) on Security. + +### C1. General Security Profile for Identifiers + +Elixir will not allow tokenization of identifiers with codepoints in `\p{Identifier_Status=Restricted}`. + +> An implementation following the General Security Profile does not permit any characters in \p{Identifier_Status=Restricted}, ... + +For instance, the 'HANGUL FILLER' (`ㅤ`) character, which is often invisible, is an uncommon codepoint and will trigger this warning. + +See the note below about additional normalizations, which can perform automatic replacement of some Restricted identifiers. + +### C2. Confusable detection + +Elixir will warn on identifiers that look the same, but aren't. Examples: in `а = a = 1`, the two 'a' characters are Cyrillic and Latin, and could be confused for each other; in `力 = カ = 1`, both are Japanese, but different codepoints, in different scripts of that writing system. Confusable identifiers can lead to hard-to-catch bugs (say, due to copy-pasted code) and can be unsafe, so we will warn about identifiers within a single file that could be confused with each other. + +We use the means described in Section 4, 'Confusable Detection', with one noted modification + +> Alternatively, it shall declare that it uses a modification, and provide a precise list of character mappings that are added to or removed from the provided ones. + +Elixir will not warn on confusability for identifiers made up exclusively of characters in a-z, A-Z, 0-9, and _. This is because ASCII identifiers have existed for so long that the programming community has had their own means of dealing with confusability between identifiers like `l,1` or `O,0` (for instance, fonts designed for programming usually make it easy to differentiate between those characters). + +### C3. Mixed Script Detection + +Elixir will not allow tokenization of mixed-script identifiers unless the mixings is one of the exceptions defined in UTS 39 5.2, 'Highly Restrictive'. We use the means described in Section 5.1, Mixed-Script Detection, to determine if script mixing is occurring, with the modification documented in the section 'Additional Normalizations', below. + +Examples: Elixir allows an identifiers like `幻ㄒㄧㄤ`, even though it includes characters from multiple 'scripts', because those scripts all 'resolve' to Japanese when applying the resolution rules from UTS 39 5.1. It also allows an atom like `:Tシャツ`, the Japanese word for 't-shirt', which incorporates a Latin capital T, because {Latn, Jpan} is one of the allowed script mixings in the definition of 'Highly Restrictive' in UTS 39 5.2, and it 'covers' the string. + +However, Elixir would prevent tokenization in code like `if аdmin, do: :ok, else: :err`, where the scriptset for the 'a' character is {Cyrillic} but all other characters have scriptsets of {Latin}. The scriptsets fail to resolve, and the scriptsets from the definition of 'Highly Restrictive' in UTS 39 5.2 do not cover the string either, so a descriptive error is shown. + +### C4, C5 (inapplicable) + +'C4 - Restriction Level detection' conformance is not claimed and does not apply to identifiers in code; rather, it applies to classifying the level of safety of a given arbitrary string into one of 5 restriction levels. + +'C5 - Mixed number detection' conformance is inapplicable as Elixir does not support Unicode numbers. + +### Addition normalizations and documented UTS 39 modifications + +As of Elixir 1.14, some codepoints in `\p{Identifier_Status=Restricted}` are *normalized* to other, unrestricted codepoints. + +Initially this is only done to translate MICRO SIGN `µ` to Greek lowercase mu, `μ`. + +This is not a modification of UTS39 clauses C1 (General Security Profile) or C2 (Confusability Detection); however it is a documented modification of C3, 'Mixed-Script detection'. + +Mixed-script detection is modified by these normalizations to the extent that the normalized codepoint is given the union of scriptsets from both characters. + +- For instance, in the example of MICRO => MU, Micro was a 'Common'-script character -- the same script given to the '_' underscore codepoint -- and thus the normalized character's scriptset will be {Greek, Common}. 'Common' intersects with all non-empty scriptsets, and thus the normalized character can be used in tokens written in any script without causing script mixing. + +- The code points normalized in this fashion are those that are in use in the community, and judged not likely to cause issues with unsafe script mixing. For instance, the MICRO or MU codepoint may be used in an atom or variable dealing with microseconds. diff --git a/lib/elixir/pages/writing-documentation.md b/lib/elixir/pages/writing-documentation.md new file mode 100644 index 00000000000..4f8fcdba121 --- /dev/null +++ b/lib/elixir/pages/writing-documentation.md @@ -0,0 +1,145 @@ +# Writing Documentation + +Elixir treats documentation as a first-class citizen. This means documentation should be easy to write and easy to read. In this document you will learn how to write documentation in Elixir, covering constructs like module attributes, style practices and doctests. + +## Markdown + +Elixir documentation is written using Markdown. There are plenty of guides on Markdown online, we recommend the ones available at GitHub as a getting started point: + + * [Basic writing and formatting syntax](https://help.github.com/articles/basic-writing-and-formatting-syntax/) + * [Mastering Markdown](https://guides.github.com/features/mastering-markdown/) + +## Module Attributes + +Documentation in Elixir is usually attached to module attributes. Let's see an example: + + defmodule MyApp.Hello do + @moduledoc """ + This is the Hello module. + """ + @moduledoc since: "1.0.0" + + @doc """ + Says hello to the given `name`. + + Returns `:ok`. + + ## Examples + + iex> MyApp.Hello.world(:john) + :ok + + """ + @doc since: "1.3.0" + def world(name) do + IO.puts("hello #{name}") + end + end + +The `@moduledoc` attribute is used to add documentation to the module. `@doc` is used before a function to provide documentation for it. Besides the attributes above, `@typedoc` can also be used to attach documentation to types defined as part of typespecs. Elixir also allows metadata to be attached to documentation, by passing a keyword list to `@doc` and friends. + +## Function Arguments + +When documenting a function, argument names are inferred by the compiler. For example: + + def size(%{size: size}) do + size + end + +The compiler will infer this argument as `map`. Sometimes the inference will be suboptimal, especially if the function contains multiple clauses with the argument matching on different values each time. You can specify the proper names for documentation by declaring only the function head at any moment before the implementation: + + def size(map_with_size) + def size(%{size: size}) do + size + end + +## Documentation metadata + +Elixir allows developers to attach arbitrary metadata to the documentation. This is done by passing a keyword list to the relevant attribute (such as `@moduledoc`, `@typedoc`, and `@doc`). A commonly used metadata is `:since`, which annotates in which version that particular module, function, type, or callback was added, as shown in the example above. + +Another common metadata is `:deprecated`, which emits a warning in the documentation, explaining that its usage is discouraged: + + @doc deprecated: "Use Foo.bar/2 instead" + +Note that the `:deprecated` key does not warn when a developer invokes the functions. If you want the code to also emit a warning, you can use the `@deprecated` attribute: + + @deprecated "Use Foo.bar/2 instead" + +Metadata can have any key. Documentation tools often use metadata to provide more data to readers and to enrich the user experience. + +## Recommendations + +When writing documentation: + + * Keep the first paragraph of the documentation concise and simple, typically one-line. Tools like [ExDoc](https://github.com/elixir-lang/ex_doc/) use the first line to generate a summary. + + * Reference modules by their full name. + + Markdown uses backticks (`` ` ``) to quote code. Elixir builds on top of that to automatically generate links when module or function names are referenced. For this reason, always use full module names. If you have a module called `MyApp.Hello`, always reference it as `` `MyApp.Hello` `` and never as `` `Hello` ``. + + * Reference functions by name and arity if they are local, as in `` `world/1` ``, or by module, name and arity if pointing to an external module: `` `MyApp.Hello.world/1` ``. + + * Reference a `@callback` by prepending `c:`, as in `` `c:world/1` ``. + + * Reference a `@type` by prepending `t:`, as in `` `t:values/0` ``. + + * Start new sections with second level Markdown headers `##`. First level headers are reserved for module and function names. + + * Place documentation before the first clause of multi-clause functions. Documentation is always per function and arity and not per clause. + + * Use the `:since` key in the documentation metadata to annotate whenever new functions or modules are added to your API. + +## Doctests + +We recommend that developers include examples in their documentation, often under their own `## Examples` heading. To ensure examples do not get out of date, Elixir's test framework (ExUnit) provides a feature called doctests that allows developers to test the examples in their documentation. Doctests work by parsing out code samples starting with `iex>` from the documentation. You can read more about it at `ExUnit.DocTest`. + +Note that doctests have limitations. When you cannot doctest a function, because it relies on state or side-effects, we recommend developers include examples directly without the `iex>` prompt. + +## Documentation != Code comments + +Elixir treats documentation and code comments as different concepts. Documentation is an explicit contract between you and users of your Application Programming Interface (API), be them third-party developers, co-workers, or your future self. Modules and functions must always be documented if they are part of your API. + +Code comments are aimed at developers reading the code. They are useful for marking improvements, leaving notes (for example, why you had to resort to a workaround due to a bug in a library), and so forth. They are tied to the source code: you can completely rewrite a function and remove all existing code comments, and it will continue to behave the same, with no change to either its behaviour or its documentation. + +Because private functions cannot be accessed externally, Elixir will warn if a private function has a `@doc` attribute and will discard its content. However, you can add code comments to private functions, as with any other piece of code, and we recommend developers to do so whenever they believe it will add relevant information to the readers and maintainers of such code. + +Finally, beware of redundant code comments, such as the ones describing the exact same that the code does: + + # Total is the sum of the batch and individual entries + total = batch_sum + individual_sum + +In summary, documentation is a contract with users of your API, who may not necessarily have access to the source code; whereas code comments are for those who interact directly with the source. You can learn and express different guarantees about your software by separating those two concepts. + +## Hiding Internal Modules and Functions + +Besides the modules and functions libraries provide as part of their public interface, libraries may also implement important functionality that is not part of their API. While these modules and functions can be accessed, they are meant to be internal to the library and thus should not have documentation for end users. + +Conveniently, Elixir allows developers to hide modules and functions from the documentation, by setting `@doc false` to hide a particular function, or `@moduledoc false` to hide the whole module. If a module is hidden, you may even document the functions in the module, but the module itself won't be listed in the documentation: + + defmodule MyApp.Hidden do + @moduledoc false + + @doc """ + This function won't be listed in docs. + """ + def function_that_wont_be_listed_in_docs do + # ... + end + end + +In case you don't want to hide a whole module, you can hide functions individually: + + defmodule MyApp.Sample do + @doc false + def add(a, b), do: a + b + end + +However, keep in mind `@moduledoc false` or `@doc false` do not make a function private. The function above can still be invoked as `MyApp.Sample.add(1, 2)`. Not only that, if `MyApp.Sample` is imported, the `add/2` function will also be imported into the caller. For those reasons, be cautious when adding `@doc false` to functions, instead use one of these two options: + + * Move the undocumented function to a module with `@moduledoc false`, like `MyApp.Hidden`, ensuring the function won't be accidentally exposed or imported. Remember that you can use `@moduledoc false` to hide a whole module and still document each function with `@doc`. Tools will still ignore the module. + + * Start the function name with one or two underscores, for example, `__add__/2`. Functions starting with underscore are automatically treated as hidden, although you can also be explicit and add `@doc false`. The compiler does not import functions with leading underscores and they hint to anyone reading the code of their intended private usage. + +## Code.fetch_docs/1 + +Elixir stores documentation inside pre-defined chunks in the bytecode. It can be accessed from Elixir by using the `Code.fetch_docs/1` function. This also means documentation is only accessed when required and not when modules are loaded by the Virtual Machine. The only downside is that modules defined in-memory, like the ones defined in IEx, cannot have their documentation accessed as they do not have their bytecode written to disk. diff --git a/lib/elixir/rebar.config b/lib/elixir/rebar.config deleted file mode 100644 index 902a412104c..00000000000 --- a/lib/elixir/rebar.config +++ /dev/null @@ -1,23 +0,0 @@ -{erl_opts, [ - warn_unused_vars, - warn_export_all, - warn_shadow_vars, - warn_unused_import, - warn_unused_function, - warn_bif_clash, - warn_unused_record, - warn_deprecated_function, - warn_obsolete_guard, - strict_validation, - warn_exported_vars, - %% warn_export_vars, - %% warn_missing_spec, - %% warn_untyped_record, - %% warnings_as_errors, - debug_info - ]}. - -{yrl_opts, [ - {report, true}, - {verbose, false} - ]}. diff --git a/lib/elixir/src/elixir.app.src b/lib/elixir/src/elixir.app.src new file mode 100644 index 00000000000..a1684f54b80 --- /dev/null +++ b/lib/elixir/src/elixir.app.src @@ -0,0 +1,9 @@ +{application, elixir, +[{description, "elixir"}, + {vsn, '$will-be-replaced'}, + {modules, '$will-be-replaced'}, + {registered, [elixir_sup, elixir_config, elixir_code_server]}, + {applications, [kernel,stdlib,compiler]}, + {mod, {elixir,[]}}, + {env, [{check_endianness, true}, {ansi_enabled, false}, {time_zone_database, 'Elixir.Calendar.UTCOnlyTimeZoneDatabase'}]} +]}. diff --git a/lib/elixir/src/elixir.erl b/lib/elixir/src/elixir.erl index a7548c56489..82d37de048e 100644 --- a/lib/elixir/src/elixir.erl +++ b/lib/elixir/src/elixir.erl @@ -2,35 +2,140 @@ %% private to the Elixir compiler and reserved to be used by Elixir only. -module(elixir). -behaviour(application). --export([main/1, start_cli/0, - string_to_quoted/4, 'string_to_quoted!'/4, - env_for_eval/1, env_for_eval/2, quoted_to_erl/2, quoted_to_erl/3, - eval/2, eval/3, eval_forms/3, eval_forms/4, eval_quoted/3]). +-export([start_cli/0, + string_to_tokens/5, tokens_to_quoted/3, 'string_to_quoted!'/5, + env_for_eval/1, quoted_to_erl/2, eval_forms/3, eval_quoted/3]). -include("elixir.hrl"). +-define(system, 'Elixir.System'). %% Top level types --export_type([char_list/0, as_boolean/1]). +%% TODO: Remove char_list type on v2.0 +-export_type([charlist/0, char_list/0, nonempty_charlist/0, struct/0, as_boolean/1, keyword/0, keyword/1]). +-type charlist() :: string(). -type char_list() :: string(). +-type nonempty_charlist() :: nonempty_string(). -type as_boolean(T) :: T. +-type keyword() :: [{atom(), any()}]. +-type keyword(T) :: [{atom(), T}]. +-type struct() :: #{'__struct__' := atom(), atom() => any()}. %% OTP Application API -export([start/2, stop/1, config_change/3]). start(_Type, _Args) -> + _ = parse_otp_release(), + preload_common_modules(), + set_stdio_and_stderr_to_binary_and_maybe_utf8(), + check_file_encoding(file:native_name_encoding()), + + case application:get_env(elixir, check_endianness, true) of + true -> check_endianness(); + false -> ok + end, + + Tokenizer = case code:ensure_loaded('Elixir.String.Tokenizer') of + {module, Mod} -> Mod; + _ -> elixir_tokenizer + end, + + URIConfig = [ + {{uri, <<"ftp">>}, 21}, + {{uri, <<"sftp">>}, 22}, + {{uri, <<"tftp">>}, 69}, + {{uri, <<"http">>}, 80}, + {{uri, <<"https">>}, 443}, + {{uri, <<"ldap">>}, 389}, + {{uri, <<"ws">>}, 80}, + {{uri, <<"wss">>}, 443} + ], + + Config = [ + %% ARGV options + {at_exit, []}, + {argv, []}, + {no_halt, false}, + + %% Compiler options + {docs, true}, + {ignore_module_conflict, false}, + {parser_options, []}, + {debug_info, true}, + {warnings_as_errors, false}, + {relative_paths, true}, + {no_warn_undefined, []}, + {tracers, []} + | URIConfig + ], + + elixir_config:static(#{bootstrap => false, identifier_tokenizer => Tokenizer}), + + Tab = elixir_config:new(Config), + case elixir_sup:start_link() of + {ok, Sup} -> + {ok, Sup, Tab}; + {error, _Reason} = Error -> + elixir_config:delete(Tab), + Error + end. + +stop(Tab) -> + elixir_config:delete(Tab). + +config_change(_Changed, _New, _Remove) -> + ok. + +set_stdio_and_stderr_to_binary_and_maybe_utf8() -> %% In case there is a shell, we can't really change its %% encoding, so we just set binary to true. Otherwise %% we must set the encoding as the user with no shell %% has encoding set to latin1. Opts = case init:get_argument(noshell) of - {ok, _} -> [binary,{encoding,utf8}]; + {ok, _} -> [binary, {encoding, utf8}]; error -> [binary] end, - io:setopts(standard_io, Opts), - io:setopts(standard_error, [{unicode,true}]), - case file:native_name_encoding() of + ok = io:setopts(standard_io, Opts), + ok = io:setopts(standard_error, [{encoding, utf8}]), + ok. + +preload_common_modules() -> + %% We attempt to load those modules here so throughout + %% the codebase we can avoid code:ensure_loaded/1 checks. + _ = code:ensure_loaded('Elixir.Kernel'), + _ = code:ensure_loaded('Elixir.Macro.Env'), + ok. + +parse_otp_release() -> + %% Whenever we change this check, we should also change Makefile. + case string:to_integer(erlang:system_info(otp_release)) of + {Num, _} when Num >= 23 -> + Num; + _ -> + io:format(standard_error, "ERROR! Unsupported Erlang/OTP version, expected Erlang/OTP 23+~n", []), + erlang:halt(1) + end. + +check_endianness() -> + case code:ensure_loaded(?system) of + {module, ?system} -> + Endianness = ?system:endianness(), + case ?system:compiled_endianness() of + Endianness -> + ok; + _ -> + io:format(standard_error, + "warning: Elixir is running in a system with a different endianness than the one its " + "source code was compiled in. Please make sure Elixir and all source files were compiled " + "in a machine with the same endianness as the current one: ~ts~n", [Endianness]) + end; + {error, _} -> + ok + end. + +check_file_encoding(Encoding) -> + case Encoding of latin1 -> io:format(standard_error, "warning: the VM is running with native name encoding of latin1 which may cause " @@ -38,72 +143,82 @@ start(_Type, _Args) -> "(which can be verified by running \"locale\" in your shell)~n", []); _ -> ok - end, - - elixir_sup:start_link(). - -stop(_S) -> - ok. - -config_change(_Changed, _New, _Remove) -> - ok. - -%% escript entry point - -main(Args) -> - application:start(?MODULE), - 'Elixir.Kernel.CLI':main(Args). + end. %% Boot and process given options. Invoked by Elixir's script. start_cli() -> - application:start(?MODULE), + {ok, _} = application:ensure_all_started(?MODULE), + + %% We start the Logger so tools that depend on Elixir + %% always have the Logger directly accessible. However + %% Logger is not a dependency of the Elixir application, + %% which means releases that want to use Logger must + %% always list it as part of its applications. + _ = case code:ensure_loaded('Elixir.Logger') of + {module, _} -> application:start(logger); + {error, _} -> ok + end, + 'Elixir.Kernel.CLI':main(init:get_plain_arguments()). %% EVAL HOOKS -env_for_eval(Opts) -> - env_for_eval((elixir_env:new())#{ - local := nil, - requires := elixir_dispatch:default_requires(), - functions := elixir_dispatch:default_functions(), - macros := elixir_dispatch:default_macros() - }, Opts). +env_for_eval(#{lexical_tracker := Pid} = Env) -> + NewEnv = Env#{ + context := nil, + context_modules := [], + macro_aliases := [], + versioned_vars := #{} + }, + + case is_pid(Pid) of + true -> + case is_process_alive(Pid) of + true -> + NewEnv; + false -> + 'Elixir.IO':warn( + <<"an __ENV__ with outdated compilation information was given to eval, " + "call Macro.Env.prune_compile_info/1 to prune it">> + ), + NewEnv#{lexical_tracker := nil, tracers := []} + end; + false -> + NewEnv#{tracers := []} + end; +%% TODO: Deprecate all options except line and file on v1.15. +env_for_eval(Opts) when is_list(Opts) -> + Env = elixir_env:new(), -env_for_eval(Env, Opts) -> Line = case lists:keyfind(line, 1, Opts) of {line, LineOpt} when is_integer(LineOpt) -> LineOpt; - false -> ?m(Env, line) + false -> ?key(Env, line) end, File = case lists:keyfind(file, 1, Opts) of {file, FileOpt} when is_binary(FileOpt) -> FileOpt; - false -> ?m(Env, file) - end, - - Local = case lists:keyfind(delegate_locals_to, 1, Opts) of - {delegate_locals_to, LocalOpt} when is_atom(LocalOpt) -> LocalOpt; - false -> ?m(Env, local) + false -> ?key(Env, file) end, Aliases = case lists:keyfind(aliases, 1, Opts) of {aliases, AliasesOpt} when is_list(AliasesOpt) -> AliasesOpt; - false -> ?m(Env, aliases) + false -> ?key(Env, aliases) end, Requires = case lists:keyfind(requires, 1, Opts) of {requires, RequiresOpt} when is_list(RequiresOpt) -> ordsets:from_list(RequiresOpt); - false -> ?m(Env, requires) + false -> ?key(Env, requires) end, Functions = case lists:keyfind(functions, 1, Opts) of {functions, FunctionsOpt} when is_list(FunctionsOpt) -> FunctionsOpt; - false -> ?m(Env, functions) + false -> ?key(Env, functions) end, Macros = case lists:keyfind(macros, 1, Opts) of {macros, MacrosOpt} when is_list(MacrosOpt) -> MacrosOpt; - false -> ?m(Env, macros) + false -> ?key(Env, macros) end, Module = case lists:keyfind(module, 1, Opts) of @@ -111,111 +226,238 @@ env_for_eval(Env, Opts) -> false -> nil end, - Env#{ - file := File, local := Local, module := Module, - macros := Macros, functions := Functions, - requires := Requires, aliases := Aliases, line := Line - }. + TempTracers = case lists:keyfind(tracers, 1, Opts) of + {tracers, TracersOpt} when is_list(TracersOpt) -> TracersOpt; + false -> [] + end, -%% String evaluation + %% If there is a dead PID or lexical tracker is nil, + %% we assume the tracers also cannot be (re)used. + {LexicalTracker, Tracers} = case lists:keyfind(lexical_tracker, 1, Opts) of + {lexical_tracker, Pid} when is_pid(Pid) -> + case is_process_alive(Pid) of + true -> {Pid, TempTracers}; + false -> {nil, []} + end; + {lexical_tracker, nil} -> + {nil, []}; + false -> + {nil, TempTracers} + end, -eval(String, Binding) -> - eval(String, Binding, []). + FA = case lists:keyfind(function, 1, Opts) of + {function, {Function, Arity}} when is_atom(Function), is_integer(Arity) -> {Function, Arity}; + {function, nil} -> nil; + false -> nil + end, -eval(String, Binding, Opts) when is_list(Opts) -> - eval(String, Binding, env_for_eval(Opts)); -eval(String, Binding, #{line := Line, file := File} = E) when - is_list(String), is_list(Binding), is_integer(Line), is_binary(File) -> - Forms = 'string_to_quoted!'(String, Line, File, []), - eval_forms(Forms, Binding, E). + Env#{ + file := File, module := Module, function := FA, tracers := Tracers, + macros := Macros, functions := Functions, lexical_tracker := LexicalTracker, + requires := Requires, aliases := Aliases, line := Line + }. %% Quoted evaluation eval_quoted(Tree, Binding, Opts) when is_list(Opts) -> eval_quoted(Tree, Binding, env_for_eval(Opts)); eval_quoted(Tree, Binding, #{line := Line} = E) -> - eval_forms(elixir_quote:linify(Line, Tree), Binding, E). - -%% Handle forms evaluation. The main difference to -%% to eval_quoted is that it does not linefy the given -%% args. + eval_forms(elixir_quote:linify(Line, line, Tree), Binding, E). eval_forms(Tree, Binding, Opts) when is_list(Opts) -> eval_forms(Tree, Binding, env_for_eval(Opts)); -eval_forms(Tree, Binding, E) -> - eval_forms(Tree, Binding, E, elixir_env:env_to_scope(E)). -eval_forms(Tree, Binding, Env, Scope) -> - {ParsedBinding, ParsedScope} = elixir_scope:load_binding(Binding, Scope), - ParsedEnv = Env#{vars := [K || {K,_} <- ParsedScope#elixir_scope.vars]}, - {Erl, NewEnv, NewScope} = quoted_to_erl(Tree, ParsedEnv, ParsedScope), +eval_forms(Tree, RawBinding, OrigE) -> + {Vars, Binding} = normalize_binding(RawBinding, #{}, [], 0), + E = elixir_env:with_vars(OrigE, Vars), + {_, S} = elixir_env:env_to_erl(E), + {Erl, NewErlS, NewExS, NewE} = quoted_to_erl(Tree, E, S), case Erl of {atom, _, Atom} -> - {Atom, Binding, NewEnv, NewScope}; + {Atom, RawBinding, NewE}; + _ -> - {value, Value, NewBinding} = erl_eval(Erl, ParsedBinding), - {Value, elixir_scope:dump_binding(NewBinding, NewScope), NewEnv, NewScope} + Exprs = + case Erl of + {block, _, BlockExprs} -> BlockExprs; + _ -> [Erl] + end, + + ErlBinding = elixir_erl_var:load_binding(Binding, E, S), + ExternalHandler = eval_external_handler(NewE), + {value, Value, NewBinding} = erl_eval:exprs(Exprs, ErlBinding, none, ExternalHandler), + {Value, elixir_erl_var:dump_binding(NewBinding, NewExS, NewErlS), NewE} end. -erl_eval(Erl, ParsedBinding) -> - % Below must be all one line for locations to be the same when the stacktrace - % needs to be extended to the full stacktrace. - try erl_eval:expr(Erl, ParsedBinding) catch Class:Exception -> erlang:raise(Class, Exception, get_stacktrace()) end. - -get_stacktrace() -> - Stacktrace = erlang:get_stacktrace(), - % eval_eval and eval_bits can call :erlang.raise/3 without the full - % stacktrace. When this occurs re-add the current stacktrace so that no - % stack information is lost. - try - throw(stack) - catch - throw:stack -> - % Ignore stack item for current function. - [_ | CurrentStack] = erlang:get_stacktrace(), - get_stacktrace(Stacktrace, CurrentStack) +normalize_binding([Binding | NextBindings], VarsMap, Normalized, Counter) -> + {Pair, Value} = normalize_pair(Binding), + case VarsMap of + #{Pair := _} -> + normalize_binding(NextBindings, VarsMap, [{Pair, Value} | Normalized], Counter); + #{} -> + normalize_binding(NextBindings, VarsMap#{Pair => Counter}, [{Pair, Value} | Normalized], Counter + 1) + end; +normalize_binding([], VarsMap, Normalized, _Counter) -> + {VarsMap, Normalized}. + +normalize_pair({Key, Value}) when is_atom(Key) -> {{Key, nil}, Value}; +normalize_pair({Pair, Value}) -> {Pair, Value}. + +%% TODO: Remove conditional once we require Erlang/OTP 25+. +-if(?OTP_RELEASE >= 25). +eval_external_handler(Env) -> + Fun = fun(Ann, FunOrModFun, Args) -> + try + case FunOrModFun of + {Mod, Fun} -> apply(Mod, Fun, Args); + Fun -> apply(Fun, Args) + end + catch + Kind:Reason:Stacktrace -> + %% Take everything up to the Elixir module + Pruned = + lists:takewhile(fun + ({elixir,_,_,_}) -> false; + (_) -> true + end, Stacktrace), + + Caller = + lists:dropwhile(fun + ({elixir,_,_,_}) -> false; + (_) -> true + end, Stacktrace), + + %% Now we prune any shared code path from erl_eval + {current_stacktrace, Current} = + erlang:process_info(self(), current_stacktrace), + + %% We need to make sure that we don't generate more + %% frames than supported. So we do our best to drop + %% from the Caller, but if the caller has no frames, + %% we need to drop from Pruned. + {DroppedCaller, ToDrop} = + case Caller of + [] -> {[], true}; + _ -> {lists:droplast(Caller), false} + end, + + Reversed = drop_common(lists:reverse(Current), lists:reverse(Pruned), ToDrop), + File = elixir_utils:characters_to_list(?key(Env, file)), + Location = [{file, File}, {line, erl_anno:line(Ann)}], + + %% Add file+line information at the bottom + Custom = lists:reverse([{elixir_eval, '__FILE__', 1, Location} | Reversed], DroppedCaller), + erlang:raise(Kind, Reason, Custom) + end + end, + {value, Fun}. + +%% We need to check if we have dropped any frames. +%% If we have not dropped frames, then we need to drop one +%% at the end so we can put the elixir_eval frame in. If +%% we have more traces then depth, Erlang would discard +%% the whole stacktrace. +drop_common([H | T1], [H | T2], _ToDrop) -> drop_common(T1, T2, false); +drop_common([_ | T1], T2, ToDrop) -> drop_common(T1, T2, ToDrop); +drop_common([], [{?MODULE, _, _, _} | T2], _ToDrop) -> T2; +drop_common([], [_ | T2], true) -> T2; +drop_common([], T2, _) -> T2. +-else. +eval_external_handler(_Env) -> + none. +-endif. + +%% Converts a quoted expression to Erlang abstract format + +quoted_to_erl(Quoted, E) -> + {_, S} = elixir_env:env_to_erl(E), + quoted_to_erl(Quoted, E, S). + +quoted_to_erl(Quoted, Env, Scope) -> + {Expanded, #elixir_ex{vars={ReadVars, _}} = NewExS, NewEnv} = + elixir_expand:expand(Quoted, elixir_env:env_to_ex(Env), Env), + {Erl, NewErlS} = elixir_erl_pass:translate(Expanded, erl_anno:new(?key(Env, line)), Scope), + {Erl, NewErlS, NewExS, NewEnv#{versioned_vars := ReadVars}}. + +%% Converts a given string (charlist) into quote expression + +string_to_tokens(String, StartLine, StartColumn, File, Opts) when is_integer(StartLine), is_binary(File) -> + case elixir_tokenizer:tokenize(String, StartLine, StartColumn, Opts) of + {ok, _Line, _Column, [], Tokens} -> + {ok, Tokens}; + {ok, _Line, _Column, Warnings, Tokens} -> + (lists:keyfind(emit_warnings, 1, Opts) /= {emit_warnings, false}) andalso + [elixir_errors:erl_warn(L, File, M) || {L, M} <- lists:reverse(Warnings)], + {ok, Tokens}; + {error, {Line, Column, {ErrorPrefix, ErrorSuffix}, Token}, _Rest, _Warnings, _SoFar} -> + Location = [{line, Line}, {column, Column}], + {error, {Location, {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)}}; + {error, {Line, Column, Error, Token}, _Rest, _Warnings, _SoFar} -> + Location = [{line, Line}, {column, Column}], + {error, {Location, to_binary(Error), to_binary(Token)}} end. -% The stacktrace did not include the current stack, re-add it. -get_stacktrace([], CurrentStack) -> - CurrentStack; -% The stacktrace includes the current stack. -get_stacktrace(CurrentStack, CurrentStack) -> - CurrentStack; -get_stacktrace([StackItem | Stacktrace], CurrentStack) -> - [StackItem | get_stacktrace(Stacktrace, CurrentStack)]. +tokens_to_quoted(Tokens, WarningFile, Opts) -> + handle_parsing_opts(WarningFile, Opts), -%% Converts a quoted expression to erlang abstract format + try elixir_parser:parse(Tokens) of + {ok, Forms} -> + {ok, Forms}; + {error, {Line, _, [{ErrorPrefix, ErrorSuffix}, Token]}} -> + {error, {parser_location(Line), {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)}}; + {error, {Line, _, [Error, Token]}} -> + {error, {parser_location(Line), to_binary(Error), to_binary(Token)}} + after + erase(elixir_parser_warning_file), + erase(elixir_parser_columns), + erase(elixir_token_metadata), + erase(elixir_literal_encoder) + end. -quoted_to_erl(Quoted, Env) -> - quoted_to_erl(Quoted, Env, elixir_env:env_to_scope(Env)). +parser_location({Line, Column, _}) -> + [{line, Line}, {column, Column}]; +parser_location(Meta) -> + Line = + case lists:keyfind(line, 1, Meta) of + {line, L} -> L; + false -> 0 + end, -quoted_to_erl(Quoted, Env, Scope) -> - {Expanded, NewEnv} = elixir_exp:expand(Quoted, Env), - {Erl, NewScope} = elixir_translator:translate(Expanded, Scope), - {Erl, NewEnv, NewScope}. - -%% Converts a given string (char list) into quote expression - -string_to_quoted(String, StartLine, File, Opts) when is_integer(StartLine), is_binary(File) -> - case elixir_tokenizer:tokenize(String, StartLine, [{file, File}|Opts]) of - {ok, _Line, Tokens} -> - try elixir_parser:parse(Tokens) of - {ok, Forms} -> {ok, Forms}; - {error, {Line, _, [Error, Token]}} -> {error, {Line, to_binary(Error), to_binary(Token)}} - catch - {error, {Line, _, [Error, Token]}} -> {error, {Line, to_binary(Error), to_binary(Token)}} - end; - {error, {Line, Error, Token}, _Rest, _SoFar} -> {error, {Line, to_binary(Error), to_binary(Token)}} + case lists:keyfind(column, 1, Meta) of + {column, C} -> [{line, Line}, {column, C}]; + false -> [{line, Line}] end. -'string_to_quoted!'(String, StartLine, File, Opts) -> - case string_to_quoted(String, StartLine, File, Opts) of - {ok, Forms} -> - Forms; - {error, {Line, Error, Token}} -> - elixir_errors:parse_error(Line, File, Error, Token) +'string_to_quoted!'(String, StartLine, StartColumn, File, Opts) -> + case string_to_tokens(String, StartLine, StartColumn, File, Opts) of + {ok, Tokens} -> + case tokens_to_quoted(Tokens, File, Opts) of + {ok, Forms} -> + Forms; + {error, {Meta, Error, Token}} -> + elixir_errors:parse_error(Meta, File, Error, Token, {String, StartLine, StartColumn}) + end; + {error, {Meta, Error, Token}} -> + elixir_errors:parse_error(Meta, File, Error, Token, {String, StartLine, StartColumn}) end. to_binary(List) when is_list(List) -> elixir_utils:characters_to_binary(List); -to_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). +to_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom). + +handle_parsing_opts(File, Opts) -> + WarningFile = + case lists:keyfind(emit_warnings, 1, Opts) of + {emit_warnings, false} -> nil; + _ -> File + end, + LiteralEncoder = + case lists:keyfind(literal_encoder, 1, Opts) of + {literal_encoder, Fun} -> Fun; + false -> false + end, + TokenMetadata = lists:keyfind(token_metadata, 1, Opts) == {token_metadata, true}, + Columns = lists:keyfind(columns, 1, Opts) == {columns, true}, + put(elixir_parser_warning_file, WarningFile), + put(elixir_parser_columns, Columns), + put(elixir_token_metadata, TokenMetadata), + put(elixir_literal_encoder, LiteralEncoder). diff --git a/lib/elixir/src/elixir.hrl b/lib/elixir/src/elixir.hrl new file mode 100644 index 00000000000..d238cdee4e4 --- /dev/null +++ b/lib/elixir/src/elixir.hrl @@ -0,0 +1,40 @@ +-define(key(M, K), maps:get(K, M)). +-define(ann(Meta), elixir_erl:get_ann(Meta)). +-define(line(Meta), elixir_utils:get_line(Meta)). +-define(generated(Meta), [{generated, true} | Meta]). +-define(var_context, ?MODULE). +-define(remote(Ann, Module, Function, Args), {call, Ann, {remote, Ann, {atom, Ann, Module}, {atom, Ann, Function}}, Args}). + +-record(elixir_ex, { + caller=false, %% stores if __CALLER__ is allowed + prematch=warn, %% {Read, Counter} | warn | raise | pin | {bitsize,PreVars,OriginalVars} + stacktrace=false, %% stores if __STACKTRACE__ is allowed + unused={#{}, 0}, %% a map of unused vars and a version counter for vars + vars={#{}, false} %% a tuple with maps of read and optional write current vars +}). + +-record(elixir_erl, { + context=nil, %% can be match, guards or nil + extra=nil, %% extra information about the context, like pin_guard and map_key + caller=false, %% when true, it means caller was invoked + var_names=#{}, %% maps of defined variables and their alias + extra_guards=[], %% extra guards from args expansion + counter=#{}, %% a map counting the variables defined + expand_captures=false, %% a boolean to control if captures should be expanded + stacktrace=nil %% holds information about the stacktrace variable +}). + +-record(elixir_tokenizer, { + terminators=[], + unescape=true, + cursor_completion=false, + existing_atoms_only=false, + static_atoms_encoder=nil, + preserve_comments=nil, + identifier_tokenizer=elixir_tokenizer, + ascii_identifiers_only=true, + indentation=0, + mismatch_hints=[], + warn_on_unnecessary_quotes=true, + warnings=[] +}). diff --git a/lib/elixir/src/elixir_aliases.erl b/lib/elixir/src/elixir_aliases.erl index 8f06fffc8cb..0eba5cb8f7d 100644 --- a/lib/elixir/src/elixir_aliases.erl +++ b/lib/elixir/src/elixir_aliases.erl @@ -1,101 +1,138 @@ -module(elixir_aliases). -export([inspect/1, last/1, concat/1, safe_concat/1, format_error/1, - ensure_loaded/3, expand/4, store/7]). + ensure_loaded/3, expand/2, expand_or_concat/2, store/5]). -include("elixir.hrl"). inspect(Atom) when is_atom(Atom) -> - case elixir_compiler:get_opt(internal) of - true -> atom_to_binary(Atom, utf8); - false -> 'Elixir.Inspect.Atom':inspect(Atom) + case elixir_config:is_bootstrap() of + true -> atom_to_binary(Atom); + false -> 'Elixir.Macro':inspect_atom(literal, Atom) end. %% Store an alias in the given scope -store(_Meta, New, New, _TKV, Aliases, MacroAliases, _Lexical) -> - {Aliases, MacroAliases}; -store(Meta, New, Old, TKV, Aliases, MacroAliases, Lexical) -> - record_warn(Meta, New, TKV, Lexical), - {store_alias(New, Old, Aliases), - store_macro_alias(Meta, New, Old, MacroAliases)}. +store(Meta, New, New, _TOpts, #{aliases := Aliases, macro_aliases := MacroAliases}) -> + {remove_alias(New, Aliases), remove_macro_alias(Meta, New, MacroAliases)}; +store(Meta, New, Old, TOpts, #{aliases := Aliases, macro_aliases := MacroAliases} = E) -> + elixir_env:trace({alias, Meta, Old, New, TOpts}, E), + {store_alias(New, Old, Aliases), store_macro_alias(Meta, New, Old, MacroAliases)}. store_alias(New, Old, Aliases) -> lists:keystore(New, 1, Aliases, {New, Old}). + store_macro_alias(Meta, New, Old, Aliases) -> - case lists:keymember(context, 1, Meta) andalso - lists:keyfind(counter, 1, Meta) of - {counter, Counter} when is_integer(Counter) -> + case lists:keyfind(counter, 1, Meta) of + {counter, Counter} -> lists:keystore(New, 1, Aliases, {New, {Counter, Old}}); - _ -> + false -> Aliases end. -record_warn(Meta, Ref, Opts, Lexical) -> - Warn = - case lists:keyfind(warn, 1, Opts) of - {warn, false} -> false; - {warn, true} -> true; - false -> not lists:keymember(context, 1, Meta) - end, - elixir_lexical:record_alias(Ref, ?line(Meta), Warn, Lexical). +remove_alias(Atom, Aliases) -> + lists:keydelete(Atom, 1, Aliases). + +remove_macro_alias(Meta, Atom, Aliases) -> + case lists:keyfind(counter, 1, Meta) of + {counter, _Counter} -> + lists:keydelete(Atom, 1, Aliases); + false -> + Aliases + end. %% Expand an alias. It returns an atom (meaning that there %% was an expansion) or a list of atoms. -expand({'__aliases__', _Meta, ['Elixir'|_] = List}, _Aliases, _MacroAliases, _LexicalTracker) -> +expand({'__aliases__', _Meta, ['Elixir' | _] = List}, _E) -> concat(List); -expand({'__aliases__', Meta, _} = Alias, Aliases, MacroAliases, LexicalTracker) -> +expand({'__aliases__', Meta, _} = Alias, #{aliases := Aliases, macro_aliases := MacroAliases} = E) -> case lists:keyfind(alias, 1, Meta) of {alias, false} -> - expand(Alias, MacroAliases, LexicalTracker); + expand(Alias, MacroAliases, E); {alias, Atom} when is_atom(Atom) -> Atom; false -> - expand(Alias, Aliases, LexicalTracker) + expand(Alias, Aliases, E) end. -expand({'__aliases__', Meta, [H|T]}, Aliases, LexicalTracker) when is_atom(H) -> +expand({'__aliases__', Meta, [H | T]}, Aliases, E) when is_atom(H) -> Lookup = list_to_atom("Elixir." ++ atom_to_list(H)), + Counter = case lists:keyfind(counter, 1, Meta) of {counter, C} -> C; _ -> nil end, + case lookup(Lookup, Aliases, Counter) of - Lookup -> [H|T]; + Lookup -> [H | T]; Atom -> - elixir_lexical:record_alias(Lookup, LexicalTracker), + elixir_env:trace({alias_expansion, Meta, Lookup, Atom}, E), case T of [] -> Atom; - _ -> concat([Atom|T]) + _ -> concat([Atom | T]) end end; -expand({'__aliases__', _Meta, List}, _Aliases, _LexicalTracker) -> +expand({'__aliases__', _Meta, List}, _Aliases, _E) -> List. +%% Expands or concat if possible. + +expand_or_concat(Aliases, E) -> + case expand(Aliases, E) of + [H | T] when is_atom(H) -> concat([H | T]); + AtomOrList -> AtomOrList + end. + %% Ensure a module is loaded before its usage. ensure_loaded(_Meta, 'Elixir.Kernel', _E) -> ok; -ensure_loaded(Meta, Ref, E) -> - try - Ref:module_info(compile) - catch - error:undef -> - Kind = case lists:member(Ref, ?m(E, context_modules)) of - true -> scheduled_module; - false -> unloaded_module - end, - elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, {Kind, Ref}) +ensure_loaded(Meta, Module, E) -> + case code:ensure_loaded(Module) of + {module, Module} -> + ok; + + _ -> + case wait_for_module(Module) of + found -> + ok; + + Wait -> + Kind = case lists:member(Module, ?key(E, context_modules)) of + true -> + case ?key(E, module) of + Module -> circular_module; + _ -> scheduled_module + end; + false when Wait == deadlock -> + deadlock_module; + false -> + unloaded_module + end, + + elixir_errors:form_error(Meta, E, ?MODULE, {Kind, Module}) + end + end. + +wait_for_module(Module) -> + case erlang:get(elixir_compiler_info) of + undefined -> not_found; + _ -> 'Elixir.Kernel.ErrorHandler':ensure_compiled(Module, module, hard) end. %% Receives an atom and returns the last bit as an alias. last(Atom) -> - Last = last(lists:reverse(atom_to_list(Atom)), []), - list_to_atom("Elixir." ++ Last). + case atom_to_list(Atom) of + ("Elixir." ++ [FirstLetter | _]) = List when FirstLetter >= $A, FirstLetter =< $Z -> + Last = last(lists:reverse(List), []), + {ok, list_to_atom("Elixir." ++ Last)}; + _ -> + error + end. -last([$.|_], Acc) -> Acc; -last([H|T], Acc) -> last(T, [H|Acc]); -last([], Acc) -> Acc. +last([$. | _], Acc) -> Acc; +last([H | T], Acc) -> last(T, [H | Acc]); +last([], Acc) -> Acc. %% Receives a list of atoms, binaries or lists %% representing modules and concatenates them. @@ -103,20 +140,20 @@ last([], Acc) -> Acc. concat(Args) -> binary_to_atom(do_concat(Args), utf8). safe_concat(Args) -> binary_to_existing_atom(do_concat(Args), utf8). -do_concat([H|T]) when is_atom(H), H /= nil -> - do_concat([atom_to_binary(H, utf8)|T]); -do_concat([<<"Elixir.", _/binary>>=H|T]) -> +do_concat([H | T]) when is_atom(H), H /= nil -> + do_concat([atom_to_binary(H) | T]); +do_concat([<<"Elixir.", _/binary>>=H | T]) -> do_concat(T, H); -do_concat([<<"Elixir">>=H|T]) -> +do_concat([<<"Elixir">>=H | T]) -> do_concat(T, H); do_concat(T) -> do_concat(T, <<"Elixir">>). -do_concat([nil|T], Acc) -> +do_concat([nil | T], Acc) -> do_concat(T, Acc); -do_concat([H|T], Acc) when is_atom(H) -> - do_concat(T, <>); -do_concat([H|T], Acc) when is_binary(H) -> +do_concat([H | T], Acc) when is_atom(H) -> + do_concat(T, <>); +do_concat([H | T], Acc) when is_binary(H) -> do_concat(T, <>); do_concat([], Acc) -> Acc. @@ -137,8 +174,55 @@ lookup(Else, Dict, Counter) -> %% Errors format_error({unloaded_module, Module}) -> - io_lib:format("module ~ts is not loaded and could not be found", [elixir_aliases:inspect(Module)]); + io_lib:format("module ~ts is not loaded and could not be found", [inspect(Module)]); + +format_error({deadlock_module, Module}) -> + io_lib:format("module ~ts is not loaded and could not be found. " + "This may be happening because the module you are trying to load " + "directly or indirectly depends on the current module", + [inspect(Module)]); format_error({scheduled_module, Module}) -> - io_lib:format("module ~ts is not loaded but was defined. This happens because you are trying to use a module in the same context it is defined. Try defining the module outside the context that requires it.", - [inspect(Module)]). \ No newline at end of file + io_lib:format( + "module ~ts is not loaded but was defined. This happens when you depend on " + "a module in the same context in which it is defined. For example:\n" + "\n" + " defmodule MyApp do\n" + " defmodule Mod do\n" + " end\n" + "\n" + " use Mod\n" + " end\n" + "\n" + "Try defining the module outside the context that uses it:\n" + "\n" + " defmodule MyApp.Mod do\n" + " end\n" + "\n" + " defmodule MyApp do\n" + " use MyApp.Mod\n" + " end\n" + "\n" + "If the module is defined at the top-level and you are trying to " + "use it at the top-level, this is not supported by Elixir", + [inspect(Module)]); + +format_error({circular_module, Module}) -> + io_lib:format( + "you are trying to use the module ~ts which is currently being defined.\n" + "\n" + "This may happen if you accidentally override the module you want to use. For example:\n" + "\n" + " defmodule MyApp do\n" + " defmodule Supervisor do\n" + " use Supervisor\n" + " end\n" + " end\n" + "\n" + "In the example above, the new Supervisor conflicts with Elixir's Supervisor. " + "This may be fixed by using the fully qualified name in the definition:\n" + "\n" + " defmodule MyApp.Supervisor do\n" + " use Supervisor\n" + " end\n", + [inspect(Module)]). diff --git a/lib/elixir/src/elixir_bitstring.erl b/lib/elixir/src/elixir_bitstring.erl index fadb4fec843..b3c3c1081c2 100644 --- a/lib/elixir/src/elixir_bitstring.erl +++ b/lib/elixir/src/elixir_bitstring.erl @@ -1,223 +1,392 @@ -module(elixir_bitstring). --export([translate/3, expand/3, has_size/1]). +-export([expand/5, format_error/1]). +-import(elixir_errors, [form_error/4]). -include("elixir.hrl"). -%% Expansion +expand_match(Expr, {S, OriginalS}, E) -> + {EExpr, SE, EE} = elixir_expand:expand(Expr, S, E), + {EExpr, {SE, OriginalS}, EE}. -expand(Meta, Args, E) -> - case ?m(E, context) of +expand(Meta, Args, S, E, RequireSize) -> + case ?key(E, context) of match -> - {EArgs, EA} = expand_bitstr(fun elixir_exp:expand/2, Args, [], E), - {{'<<>>', Meta, EArgs}, EA}; + {EArgs, Alignment, {SA, _}, EA} = + expand(Meta, fun expand_match/3, Args, [], {S, S}, E, 0, RequireSize), + + case find_match(EArgs) of + false -> + {{'<<>>', [{alignment, Alignment} | Meta], EArgs}, SA, EA}; + Match -> + form_error(Meta, EA, ?MODULE, {nested_match, Match}) + end; _ -> - {EArgs, {EC, EV}} = expand_bitstr(fun elixir_exp:expand_arg/2, Args, [], {E, E}), - {{'<<>>', Meta, EArgs}, elixir_env:mergea(EV, EC)} - end. + PairS = {elixir_env:prepare_write(S), S}, + + {EArgs, Alignment, {SA, _}, EA} = + expand(Meta, fun elixir_expand:expand_arg/3, Args, [], PairS, E, 0, RequireSize), -expand_bitstr(_Fun, [], Acc, E) -> - {lists:reverse(Acc), E}; -expand_bitstr(Fun, [{'::',Meta,[Left,Right]}|T], Acc, E) -> - {ELeft, EL} = Fun(Left, E), + {{'<<>>', [{alignment, Alignment} | Meta], EArgs}, elixir_env:close_write(SA, S), EA} + end. - %% Variables defined outside the binary can be accounted - %% on subparts, however we can't assign new variables. +expand(_BitstrMeta, _Fun, [], Acc, S, E, Alignment, _RequireSize) -> + {lists:reverse(Acc), Alignment, S, E}; +expand(BitstrMeta, Fun, [{'::', Meta, [Left, Right]} | T], Acc, S, E, Alignment, RequireSize) -> + {ELeft, {SL, OriginalS}, EL} = expand_expr(Meta, Left, Fun, S, E), + + MatchOrRequireSize = RequireSize or is_match_size(T, EL), + EType = expr_type(ELeft), + {ERight, EAlignment, SS, ES} = expand_specs(EType, Meta, Right, SL, OriginalS, EL, MatchOrRequireSize), + + EAcc = concat_or_prepend_bitstring(Meta, ELeft, ERight, Acc, ES, MatchOrRequireSize), + expand(BitstrMeta, Fun, T, EAcc, {SS, OriginalS}, ES, alignment(Alignment, EAlignment), RequireSize); +expand(BitstrMeta, Fun, [H | T], Acc, S, E, Alignment, RequireSize) -> + Meta = extract_meta(H, BitstrMeta), + {ELeft, {SS, OriginalS}, ES} = expand_expr(Meta, H, Fun, S, E), + + MatchOrRequireSize = RequireSize or is_match_size(T, ES), + EType = expr_type(ELeft), + ERight = infer_spec(EType, Meta), + + InferredMeta = [{inferred_bitstring_spec, true} | Meta], + EAcc = concat_or_prepend_bitstring(InferredMeta, ELeft, ERight, Acc, ES, MatchOrRequireSize), + expand(Meta, Fun, T, EAcc, {SS, OriginalS}, ES, Alignment, RequireSize). + +extract_meta({_, Meta, _}, _) -> Meta; +extract_meta(_, Meta) -> Meta. + +%% Variables defined outside the binary can be accounted +%% on subparts, however we can't assign new variables. +is_match_size([_ | _], #{context := match}) -> true; +is_match_size(_, _) -> false. + +expr_type(Integer) when is_integer(Integer) -> integer; +expr_type(Float) when is_float(Float) -> float; +expr_type(Binary) when is_binary(Binary) -> binary; +expr_type({'<<>>', _, _}) -> bitstring; +expr_type(_) -> default. + +infer_spec(bitstring, Meta) -> {bitstring, Meta, []}; +infer_spec(binary, Meta) -> {binary, Meta, []}; +infer_spec(float, Meta) -> {float, Meta, []}; +infer_spec(integer, Meta) -> {integer, Meta, []}; +infer_spec(default, Meta) -> {integer, Meta, []}. + +concat_or_prepend_bitstring(_Meta, {'<<>>', _, []}, _ERight, Acc, _E, _RequireSize) -> + Acc; +concat_or_prepend_bitstring(Meta, {'<<>>', PartsMeta, Parts} = ELeft, ERight, Acc, E, RequireSize) -> case E of - {ER, _} -> ok; %% expand_arg, no assigns - _ -> ER = E#{context := nil} %% expand_each, revert assigns - end, + #{context := match} when RequireSize -> + case lists:last(Parts) of + {'::', SpecMeta, [Bin, {binary, _, []}]} when not is_binary(Bin) -> + form_error(SpecMeta, E, ?MODULE, unsized_binary); - ERight = expand_bit_info(Meta, Right, ER), - expand_bitstr(Fun, T, [{'::',Meta,[ELeft,ERight]}|Acc], EL); + {'::', SpecMeta, [_, {bitstring, _, []}]} -> + form_error(SpecMeta, E, ?MODULE, unsized_binary); -expand_bitstr(Fun, [H|T], Acc, E) -> - {Expr, ES} = Fun(H, E), - expand_bitstr(Fun, T, [Expr|Acc], ES). + _ -> + ok + end; + _ -> + ok + end, -%% Expand bit info + case ERight of + {binary, _, []} -> + {alignment, Alignment} = lists:keyfind(alignment, 1, PartsMeta), -expand_bit_info(Meta, Info, E) when is_list(Info) -> - expand_bit_info(Meta, Info, default, [], E); + if + Alignment == 0 -> + lists:reverse(Parts, Acc); -expand_bit_info(Meta, Info, E) -> - expand_bit_info(Meta, [Info], E). + is_integer(Alignment) -> + form_error(Meta, E, ?MODULE, {unaligned_binary, ELeft}); -expand_bit_info(Meta, [{Expr, ExprMeta, Args}|T], Size, Types, E) when is_atom(Expr) -> - ListArgs = if is_atom(Args) -> []; is_list(Args) -> Args end, - case expand_bit_type_or_size(Expr, ListArgs) of - type -> - {EArgs, EE} = elixir_exp:expand_args(ListArgs, E), - expand_bit_info(Meta, T, Size, [{Expr, [], EArgs}|Types], EE); - size -> - case Size of - default -> ok; - _ -> elixir_errors:compile_error(Meta, ?m(E, file), "duplicated size definition in bitstring") - end, - {EArgs, EE} = elixir_exp:expand_args(ListArgs, E), - expand_bit_info(Meta, T, {Expr, [], EArgs}, Types, EE); - none -> - handle_unknown_bit_info(Meta, {Expr, ExprMeta, ListArgs}, T, Size, Types, E) + true -> + [{'::', Meta, [ELeft, ERight]} | Acc] + end; + {bitstring, _, []} -> + lists:reverse(Parts, Acc) end; - -expand_bit_info(Meta, [Int|T], Size, Types, E) when is_integer(Int) -> - expand_bit_info(Meta, [{size, [], [Int]}|T], Size, Types, E); - -expand_bit_info(Meta, [Expr|_], _Size, _Types, E) -> - elixir_errors:compile_error(Meta, ?m(E, file), - "unknown bitstring specifier ~ts", ['Elixir.Kernel':inspect(Expr)]); - -expand_bit_info(_Meta, [], Size, Types, _) -> - case Size of - default -> lists:reverse(Types); - _ -> [Size|lists:reverse(Types)] - end. - -expand_bit_type_or_size(binary, []) -> type; -expand_bit_type_or_size(integer, []) -> type; -expand_bit_type_or_size(float, []) -> type; -expand_bit_type_or_size(bitstring, []) -> type; -expand_bit_type_or_size(bytes, []) -> type; -expand_bit_type_or_size(bits, []) -> type; -expand_bit_type_or_size(utf8, []) -> type; -expand_bit_type_or_size(utf16, []) -> type; -expand_bit_type_or_size(utf32, []) -> type; -expand_bit_type_or_size(signed, []) -> type; -expand_bit_type_or_size(unsigned, []) -> type; -expand_bit_type_or_size(big, []) -> type; -expand_bit_type_or_size(little, []) -> type; -expand_bit_type_or_size(native, []) -> type; -expand_bit_type_or_size(unit, [_]) -> type; -expand_bit_type_or_size(size, [_]) -> size; -expand_bit_type_or_size(_, _) -> none. - -handle_unknown_bit_info(Meta, {_, ExprMeta, _} = Expr, T, Size, Types, E) -> - case 'Elixir.Macro':expand(Expr, elixir_env:linify({?line(ExprMeta), E})) of - Expr -> - elixir_errors:compile_error(ExprMeta, ?m(E, file), - "unknown bitstring specifier ~ts", ['Elixir.Macro':to_string(Expr)]); - Other -> - List = case is_list(Other) of true -> Other; false -> [Other] end, - expand_bit_info(Meta, List ++ T, Size, Types, E) - end. - -%% Translation - -has_size({bin, _, Elements}) -> - not lists:any(fun({bin_element, _Line, _Expr, Size, Types}) -> - (Types /= default) andalso (Size == default) andalso - lists:any(fun(X) -> lists:member(X, Types) end, - [bits, bytes, bitstring, binary]) - end, Elements). - -translate(Meta, Args, S) -> - case S#elixir_scope.context of - match -> - build_bitstr(fun elixir_translator:translate/2, Args, Meta, S); - _ -> - build_bitstr(fun(X, Acc) -> elixir_translator:translate_arg(X, Acc, S) end, Args, Meta, S) +concat_or_prepend_bitstring(Meta, ELeft, ERight, Acc, _E, _RequireSize) -> + [{'::', Meta, [ELeft, ERight]} | Acc]. + +%% Handling of alignment + +alignment(Left, Right) when is_integer(Left), is_integer(Right) -> (Left + Right) rem 8; +alignment(_, _) -> unknown. + +compute_alignment(_, Size, Unit) when is_integer(Size), is_integer(Unit) -> (Size * Unit) rem 8; +compute_alignment(default, Size, Unit) -> compute_alignment(integer, Size, Unit); +compute_alignment(integer, default, Unit) -> compute_alignment(integer, 8, Unit); +compute_alignment(integer, Size, default) -> compute_alignment(integer, Size, 1); +compute_alignment(bitstring, Size, default) -> compute_alignment(bitstring, Size, 1); +compute_alignment(binary, Size, default) -> compute_alignment(binary, Size, 8); +compute_alignment(binary, _, _) -> 0; +compute_alignment(float, _, _) -> 0; +compute_alignment(utf32, _, _) -> 0; +compute_alignment(utf16, _, _) -> 0; +compute_alignment(utf8, _, _) -> 0; +compute_alignment(_, _, _) -> unknown. + +%% Expands the expression of a bitstring, that is, the LHS of :: or +%% an argument of the bitstring (such as "foo" in "<>"). +%% If we are inside a match/guard, we inline interpolations explicitly, +%% otherwise they are inlined by elixir_rewrite.erl. + +expand_expr(_Meta, {{'.', _, [Mod, to_string]}, _, [Arg]} = AST, Fun, S, #{context := Context} = E) + when Context /= nil, (Mod == 'Elixir.Kernel') orelse (Mod == 'Elixir.String.Chars') -> + case Fun(Arg, S, E) of + {EBin, SE, EE} when is_binary(EBin) -> {EBin, SE, EE}; + _ -> Fun(AST, S, E) % Let it raise + end; +expand_expr(Meta, Component, Fun, S, E) -> + case Fun(Component, S, E) of + {EComponent, _, ErrorE} when is_list(EComponent); is_atom(EComponent) -> + form_error(Meta, ErrorE, ?MODULE, {invalid_literal, EComponent}); + {_, _, _} = Expanded -> + Expanded end. -build_bitstr(Fun, Exprs, Meta, S) -> - {Final, FinalS} = build_bitstr_each(Fun, Exprs, Meta, S, []), - {{bin, ?line(Meta), lists:reverse(Final)}, FinalS}. - -build_bitstr_each(_Fun, [], _Meta, S, Acc) -> - {Acc, S}; - -build_bitstr_each(Fun, [{'::',_,[H,V]}|T], Meta, S, Acc) -> - {Size, Types} = extract_bit_info(Meta, V, S#elixir_scope{context=nil}), - build_bitstr_each(Fun, T, Meta, S, Acc, H, Size, Types); - -build_bitstr_each(Fun, [H|T], Meta, S, Acc) -> - build_bitstr_each(Fun, T, Meta, S, Acc, H, default, default). - -build_bitstr_each(Fun, T, Meta, S, Acc, H, default, Types) when is_binary(H) -> - Element = - case types_allow_splice(Types, []) of - true -> - %% See explanation in elixir_utils:elixir_to_erl/1 to know - %% why we can simply convert the binary to a list. - {bin_element, ?line(Meta), {string, 0, binary_to_list(H)}, default, default}; - false -> - case types_require_conversion(Types) of - true -> - {bin_element, ?line(Meta), {string, 0, elixir_utils:characters_to_list(H)}, default, Types}; - false -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, "invalid types for literal string in <<>>. " - "Accepted types are: little, big, utf8, utf16, utf32, bits, bytes, binary, bitstring") - end - end, +%% Expands and normalizes types of a bitstring. + +expand_specs(ExprType, Meta, Info, S, OriginalS, E, RequireSize) -> + Default = + #{size => default, + unit => default, + sign => default, + type => default, + endianness => default}, + {#{size := Size, unit := Unit, type := Type, endianness := Endianness, sign := Sign}, SS, ES} = + expand_each_spec(Meta, unpack_specs(Info, []), Default, S, OriginalS, E), + + MergedType = type(Meta, ExprType, Type, E), + validate_size_required(Meta, RequireSize, ExprType, MergedType, Size, ES), + SizeAndUnit = size_and_unit(Meta, ExprType, Size, Unit, ES), + Alignment = compute_alignment(MergedType, Size, Unit), + + [H | T] = build_spec(Meta, Size, Unit, MergedType, Endianness, Sign, SizeAndUnit, ES), + {lists:foldl(fun(I, Acc) -> {'-', Meta, [Acc, I]} end, H, T), Alignment, SS, ES}. + +type(_, default, default, _) -> + integer; +type(_, ExprType, default, _) -> + ExprType; +type(_, binary, Type, _) when Type == binary; Type == bitstring; Type == utf8; Type == utf16; Type == utf32 -> + Type; +type(_, bitstring, Type, _) when Type == binary; Type == bitstring -> + Type; +type(_, integer, Type, _) when Type == integer; Type == float; Type == utf8; Type == utf16; Type == utf32 -> + Type; +type(_, float, Type, _) when Type == float -> + Type; +type(_, default, Type, _) -> + Type; +type(Meta, Other, Value, E) -> + form_error(Meta, E, ?MODULE, {bittype_mismatch, Value, Other, type}). + +expand_each_spec(Meta, [{Expr, _, Args} = H | T], Map, S, OriginalS, E) when is_atom(Expr) -> + case validate_spec(Expr, Args) of + {Key, Arg} -> + {Value, SE, EE} = expand_spec_arg(Arg, S, OriginalS, E), + validate_spec_arg(Meta, Key, Value, SE, OriginalS, EE), + + case maps:get(Key, Map) of + default -> ok; + Value -> ok; + Other -> form_error(Meta, E, ?MODULE, {bittype_mismatch, Value, Other, Key}) + end, - build_bitstr_each(Fun, T, Meta, S, [Element|Acc]); + expand_each_spec(Meta, T, maps:put(Key, Value, Map), SE, OriginalS, EE); -build_bitstr_each(_Fun, _T, Meta, S, _Acc, H, _Size, _Types) when is_binary(H) -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, "size is not supported for literal string in <<>>"); + none -> + case 'Elixir.Macro':expand(H, E#{line := ?line(Meta)}) of + H -> + form_error(Meta, E, ?MODULE, {undefined_bittype, H}); -build_bitstr_each(_Fun, _T, Meta, S, _Acc, H, _Size, _Types) when is_list(H); is_atom(H) -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, "invalid literal ~ts in <<>>", - ['Elixir.Macro':to_string(H)]); + NewTypes -> + expand_each_spec(Meta, unpack_specs(NewTypes, []) ++ T, Map, S, OriginalS, E) + end + end; +expand_each_spec(Meta, [Expr | _], _Map, _S, _OriginalS, E) -> + form_error(Meta, E, ?MODULE, {undefined_bittype, Expr}); +expand_each_spec(_Meta, [], Map, S, _OriginalS, E) -> + {Map, S, E}. + +unpack_specs({'-', _, [H, T]}, Acc) -> + unpack_specs(H, unpack_specs(T, Acc)); +unpack_specs({'*', _, [{'_', _, Atom}, Unit]}, Acc) when is_atom(Atom) -> + [{unit, [], [Unit]} | Acc]; +unpack_specs({'*', _, [Size, Unit]}, Acc) -> + [{size, [], [Size]}, {unit, [], [Unit]} | Acc]; +unpack_specs(Size, Acc) when is_integer(Size) -> + [{size, [], [Size]} | Acc]; +unpack_specs({Expr, Meta, Args}, Acc) when is_atom(Expr) -> + ListArgs = if is_atom(Args) -> []; is_list(Args) -> Args end, + [{Expr, Meta, ListArgs} | Acc]; +unpack_specs(Other, Acc) -> + [Other | Acc]. + +validate_spec(big, []) -> {endianness, big}; +validate_spec(little, []) -> {endianness, little}; +validate_spec(native, []) -> {endianness, native}; +validate_spec(size, [Size]) -> {size, Size}; +validate_spec(unit, [Unit]) -> {unit, Unit}; +validate_spec(integer, []) -> {type, integer}; +validate_spec(float, []) -> {type, float}; +validate_spec(binary, []) -> {type, binary}; +validate_spec(bytes, []) -> {type, binary}; +validate_spec(bitstring, []) -> {type, bitstring}; +validate_spec(bits, []) -> {type, bitstring}; +validate_spec(utf8, []) -> {type, utf8}; +validate_spec(utf16, []) -> {type, utf16}; +validate_spec(utf32, []) -> {type, utf32}; +validate_spec(signed, []) -> {sign, signed}; +validate_spec(unsigned, []) -> {sign, unsigned}; +validate_spec(_, _) -> none. + +expand_spec_arg(Expr, S, _OriginalS, E) when is_atom(Expr); is_integer(Expr) -> + {Expr, S, E}; +expand_spec_arg(Expr, S, OriginalS, #{context := match} = E) -> + %% We can only access variables that are either on prematch or not in original + #elixir_ex{prematch={PreRead, Counter}} = S, + #elixir_ex{vars={OriginalRead, _}} = OriginalS, + Prematch = {bitsize, PreRead, OriginalRead}, + {EExpr, SE, EE} = elixir_expand:expand(Expr, S#elixir_ex{prematch=Prematch}, E#{context := guard}), + {EExpr, SE#elixir_ex{prematch={PreRead, Counter}}, EE#{context := match}}; +expand_spec_arg(Expr, S, OriginalS, E) -> + elixir_expand:expand(Expr, elixir_env:reset_read(S, OriginalS), E). + +validate_spec_arg(Meta, unit, Value, _S, _OriginalS, E) when not is_integer(Value) -> + form_error(Meta, E, ?MODULE, {bad_unit_argument, Value}); +validate_spec_arg(_Meta, _Key, _Value, _S, _OriginalS, _E) -> + ok. + +validate_size_required(Meta, true, default, Type, default, E) when Type == binary; Type == bitstring -> + form_error(Meta, E, ?MODULE, unsized_binary); +validate_size_required(_, _, _, _, _, _) -> + ok. + +size_and_unit(Meta, bitstring, Size, Unit, E) when Size /= default; Unit /= default -> + form_error(Meta, E, ?MODULE, bittype_literal_bitstring); +size_and_unit(Meta, binary, Size, Unit, E) when Size /= default; Unit /= default -> + form_error(Meta, E, ?MODULE, bittype_literal_string); +size_and_unit(_Meta, _ExprType, Size, Unit, _E) -> + add_arg(unit, Unit, add_arg(size, Size, [])). + +add_arg(_Key, default, Spec) -> Spec; +add_arg(Key, Arg, Spec) -> [{Key, [], [Arg]} | Spec]. + +build_spec(Meta, Size, Unit, Type, Endianness, Sign, Spec, E) when Type == utf8; Type == utf16; Type == utf32 -> + if + Size /= default; Unit /= default -> + form_error(Meta, E, ?MODULE, bittype_utf); + Sign /= default -> + form_error(Meta, E, ?MODULE, bittype_signed); + true -> + add_spec(Type, add_spec(Endianness, Spec)) + end; -build_bitstr_each(Fun, T, Meta, S, Acc, H, Size, Types) -> - {Expr, NS} = Fun(H, S), +build_spec(Meta, _Size, Unit, Type, _Endianness, Sign, Spec, E) when Type == binary; Type == bitstring -> + if + Type == bitstring, Unit /= default, Unit /= 1 -> + form_error(Meta, E, ?MODULE, {bittype_mismatch, Unit, 1, unit}); + Sign /= default -> + form_error(Meta, E, ?MODULE, bittype_signed); + true -> + %% Endianness is supported but has no effect, so we just ignore it. + add_spec(Type, Spec) + end; - case Expr of - {bin, _, Elements} -> - case (Size == default) andalso types_allow_splice(Types, Elements) of - true -> build_bitstr_each(Fun, T, Meta, NS, lists:reverse(Elements) ++ Acc); - false -> build_bitstr_each(Fun, T, Meta, NS, [{bin_element, ?line(Meta), Expr, Size, Types}|Acc]) +build_spec(Meta, Size, Unit, Type, Endianness, Sign, Spec, E) when Type == integer; Type == float -> + NumberSize = number_size(Size, Unit), + if + Type == float, is_integer(NumberSize) -> + case valid_float_size(NumberSize) of + true -> + add_spec(Type, add_spec(Endianness, add_spec(Sign, Spec))); + false -> + form_error(Meta, E, ?MODULE, {bittype_float_size, NumberSize}) end; - _ -> - build_bitstr_each(Fun, T, Meta, NS, [{bin_element, ?line(Meta), Expr, Size, Types}|Acc]) + Size == default, Unit /= default -> + form_error(Meta, E, ?MODULE, bittype_unit); + true -> + add_spec(Type, add_spec(Endianness, add_spec(Sign, Spec))) end. -types_require_conversion([End|T]) when End == little; End == big -> types_require_conversion(T); -types_require_conversion([UTF|T]) when UTF == utf8; UTF == utf16; UTF == utf32 -> types_require_conversion(T); -types_require_conversion([]) -> true; -types_require_conversion(_) -> false. - -types_allow_splice([bytes], Elements) -> is_byte_size(Elements, 0); -types_allow_splice([binary], Elements) -> is_byte_size(Elements, 0); -types_allow_splice([bits], _) -> true; -types_allow_splice([bitstring], _) -> true; -types_allow_splice(default, _) -> true; -types_allow_splice(_, _) -> false. - -is_byte_size([Element|T], Acc) -> - case elem_size(Element) of - {unknown, Unit} when Unit rem 8 == 0 -> is_byte_size(T, Acc); - {unknown, _Unit} -> false; - {Size, Unit} -> is_byte_size(T, Size*Unit + Acc) - end; -is_byte_size([], Size) -> - Size rem 8 == 0. - -elem_size({bin_element, _, _, default, _}) -> {0, 0}; -elem_size({bin_element, _, _, {integer,_,Size}, Types}) -> {Size, unit_size(Types, 1)}; -elem_size({bin_element, _, _, _Size, Types}) -> {unknown, unit_size(Types, 1)}. - -unit_size([binary|T], _) -> unit_size(T, 8); -unit_size([{unit, Size}|_], _) -> Size; -unit_size([_|T], Guess) -> unit_size(T, Guess); -unit_size([], Guess) -> Guess. - -%% Extra bitstring specifiers - -extract_bit_info(Meta, [{size, _, [Arg]}|T], S) -> - case elixir_translator:translate(Arg, S) of - {{Kind, _, _} = Size, _} when Kind == integer; Kind == var -> - {Size, extract_bit_type(Meta, T, S)}; - _ -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, - "size in bitstring expects an integer or a variable as argument, got: ~ts", ['Elixir.Macro':to_string(Arg)]) +number_size(Size, default) when is_integer(Size) -> Size; +number_size(Size, Unit) when is_integer(Size) -> Size * Unit; +number_size(Size, _) -> Size. + +%% TODO: Simplify when we require Erlang/OTP 24 +valid_float_size(16) -> erlang:system_info(otp_release) >= "24"; +valid_float_size(32) -> true; +valid_float_size(64) -> true; +valid_float_size(_) -> false. + +add_spec(default, Spec) -> Spec; +add_spec(Key, Spec) -> [{Key, [], []} | Spec]. + +find_match([{'=', _, [_Left, _Right]} = Expr | _Rest]) -> + Expr; +find_match([{_, _, Args} | Rest]) when is_list(Args) -> + case find_match(Args) of + false -> find_match(Rest); + Match -> Match end; -extract_bit_info(Meta, T, S) -> - {default, extract_bit_type(Meta, T, S)}. - -extract_bit_type(Meta, [{unit, _, [Arg]}|T], S) when is_integer(Arg) -> - [{unit, Arg}|extract_bit_type(Meta, T, S)]; -extract_bit_type(Meta, [{unit, _, [Arg]}|_], S) -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, - "unit in bitstring expects an integer as argument, got: ~ts", ['Elixir.Macro':to_string(Arg)]); -extract_bit_type(Meta, [{Other, _, []}|T], S) -> - [Other|extract_bit_type(Meta, T, S)]; -extract_bit_type(_Meta, [], _S) -> - []. +find_match([_Arg | Rest]) -> + find_match(Rest); +find_match([]) -> + false. + +format_error({unaligned_binary, Expr}) -> + Message = "expected ~ts to be a binary but its number of bits is not divisible by 8", + io_lib:format(Message, ['Elixir.Macro':to_string(Expr)]); +format_error(unsized_binary) -> + "a binary field without size is only allowed at the end of a binary pattern, " + "at the right side of binary concatenation and and never allowed in binary generators. " + "The following examples are invalid:\n\n" + " rest <> \"foo\"\n" + " <>\n\n" + "They are invalid because there is a bits/bitstring component not at the end. " + "However, the \"reverse\" would work:\n\n" + " \"foo\" <> rest\n" + " <<\"foo\", rest::binary>>\n\n"; +format_error(bittype_literal_bitstring) -> + "literal <<>> in bitstring supports only type specifiers, which must be one of: " + "binary or bitstring"; +format_error(bittype_literal_string) -> + "literal string in bitstring supports only endianness and type specifiers, which must be one of: " + "little, big, native, utf8, utf16, utf32, bits, bytes, binary or bitstring"; +format_error(bittype_utf) -> + "size and unit are not supported on utf types"; +format_error(bittype_signed) -> + "signed and unsigned specifiers are supported only on integer and float types"; +format_error(bittype_unit) -> + "integer and float types require a size specifier if the unit specifier is given"; +format_error({bittype_float_size, Other}) -> + Message = + case erlang:system_info(otp_release) >= "24" of + true -> "16, 32, or 64"; + false -> "32 or 64" + end, + io_lib:format("float requires size*unit to be ~s (default), got: ~p", [Message, Other]); +format_error({invalid_literal, Literal}) -> + io_lib:format("invalid literal ~ts in <<>>", ['Elixir.Macro':to_string(Literal)]); +format_error({undefined_bittype, Expr}) -> + io_lib:format("unknown bitstring specifier: ~ts", ['Elixir.Macro':to_string(Expr)]); +format_error({bittype_mismatch, Val1, Val2, Where}) -> + io_lib:format("conflicting ~ts specification for bit field: \"~p\" and \"~p\"", [Where, Val1, Val2]); +format_error({bad_unit_argument, Unit}) -> + io_lib:format("unit in bitstring expects an integer as argument, got: ~ts", + ['Elixir.Macro':to_string(Unit)]); +format_error({nested_match, Expr}) -> + Message = + "cannot pattern match inside a bitstring " + "that is already in match, got: ~ts", + io_lib:format(Message, ['Elixir.Macro':to_string(Expr)]); +format_error({undefined_var_in_spec, Var}) -> + Message = + "undefined variable \"~ts\" in bitstring segment. If the size of the binary is a " + "variable, the variable must be defined prior to its use in the binary/bitstring match " + "itself, or outside the pattern match", + io_lib:format(Message, ['Elixir.Macro':to_string(Var)]). diff --git a/lib/elixir/src/elixir_bootstrap.erl b/lib/elixir/src/elixir_bootstrap.erl index 97cf07fb215..e7dbe9a3f9a 100644 --- a/lib/elixir/src/elixir_bootstrap.erl +++ b/lib/elixir/src/elixir_bootstrap.erl @@ -1,5 +1,5 @@ %% An Erlang module that behaves like an Elixir module -%% used for bootstraping. +%% used for bootstrapping. -module(elixir_bootstrap). -export(['MACRO-def'/2, 'MACRO-def'/3, 'MACRO-defp'/3, 'MACRO-defmodule'/3, 'MACRO-defmacro'/2, 'MACRO-defmacro'/3, 'MACRO-defmacrop'/3, @@ -18,8 +18,8 @@ 'MACRO-defmacro'(Caller, Call, Expr) -> define(Caller, defmacro, Call, Expr). 'MACRO-defmacrop'(Caller, Call, Expr) -> define(Caller, defmacrop, Call, Expr). -'MACRO-defmodule'(_Caller, Alias, [{do,Block}]) -> - {Escaped, _} = elixir_quote:escape(Block, false), +'MACRO-defmodule'(_Caller, Alias, [{do, Block}]) -> + Escaped = elixir_quote:escape(Block, none, false), Args = [Alias, Escaped, [], env()], {{'.', [], [elixir_module, compile]}, [], Args}. @@ -27,25 +27,27 @@ []; '__info__'(macros) -> [{'@', 1}, - {def,1}, - {def,2}, - {defmacro,1}, - {defmacro,2}, - {defmacrop,2}, - {defmodule,2}, - {defp,2}]. - -define({Line,E}, Kind, Call, Expr) -> - {EscapedCall, UC} = elixir_quote:escape(Call, true), - {EscapedExpr, UE} = elixir_quote:escape(Expr, true), - Args = [Line, Kind, not(UC or UE), EscapedCall, EscapedExpr, elixir_locals:cache_env(E)], + {def, 1}, + {def, 2}, + {defmacro, 1}, + {defmacro, 2}, + {defmacrop, 2}, + {defmodule, 2}, + {defp, 2}]. + +define({Line, _S, E}, Kind, Call, Expr) -> + UC = elixir_quote:has_unquotes(Call), + UE = elixir_quote:has_unquotes(Expr), + EscapedCall = elixir_quote:escape(Call, none, true), + EscapedExpr = elixir_quote:escape(Expr, none, true), + Args = [Kind, not(UC or UE), EscapedCall, EscapedExpr, elixir_locals:cache_env(E#{line := Line})], {{'.', [], [elixir_def, store_definition]}, [], Args}. unless_loaded(Fun, Args, Callback) -> case code:is_loaded(?kernel) of {_, _} -> apply(?kernel, Fun, Args); - false -> Callback() + false -> Callback() end. env() -> - {'__ENV__', [], nil}. \ No newline at end of file + {'__ENV__', [], nil}. diff --git a/lib/elixir/src/elixir_clauses.erl b/lib/elixir/src/elixir_clauses.erl index f5b3ef9e688..8ab3dda1225 100644 --- a/lib/elixir/src/elixir_clauses.erl +++ b/lib/elixir/src/elixir_clauses.erl @@ -1,226 +1,416 @@ %% Handle code related to args, guard and -> matching for case, %% fn, receive and friends. try is handled in elixir_try. -module(elixir_clauses). --export([match/3, clause/7, clauses/4, guards/4, get_pairs/3, get_pairs/4, - extract_splat_guards/1, extract_guards/1]). +-export([match/5, clause/6, def/3, head/3, + 'case'/4, 'receive'/4, 'try'/4, 'cond'/4, with/4, + format_error/1]). +-import(elixir_errors, [form_error/4, form_warn/4]). -include("elixir.hrl"). -%% Get pairs from a clause. +match(Fun, Expr, AfterS, _BeforeS, #{context := match} = E) -> + Fun(Expr, AfterS, E); +match(Fun, Expr, AfterS, BeforeS, E) -> + #elixir_ex{vars=Current, unused={_, Counter} = Unused} = AfterS, + #elixir_ex{vars={Read, _}, prematch=Prematch} = BeforeS, -get_pairs(Key, Clauses, As) -> - get_pairs(Key, Clauses, As, false). -get_pairs(Key, Clauses, As, AllowNil) -> - case lists:keyfind(Key, 1, Clauses) of - {Key, Pairs} when is_list(Pairs) -> - [{As, Meta, Left, Right} || {'->', Meta, [Left, Right]} <- Pairs]; - {Key, nil} when AllowNil -> - []; - false -> - [] - end. - -%% Translate matches - -match(Fun, Args, #elixir_scope{context=Context, match_vars=MatchVars, - backup_vars=BackupVars, vars=Vars} = S) when Context /= match -> - {Result, NewS} = match(Fun, Args, S#elixir_scope{context=match, - match_vars=ordsets:new(), backup_vars=Vars}), - {Result, NewS#elixir_scope{context=Context, - match_vars=MatchVars, backup_vars=BackupVars}}; -match(Fun, Args, S) -> Fun(Args, S). - -%% Translate clauses with args, guards and expressions - -clause(Line, Fun, Args, Expr, Guards, Return, S) when is_integer(Line) -> - {TArgs, SA} = match(Fun, Args, S#elixir_scope{extra_guards=[]}), - {TExpr, SE} = elixir_translator:translate_block(Expr, Return, SA#elixir_scope{extra_guards=nil}), + CallS = BeforeS#elixir_ex{ + prematch={Read, Counter}, + unused=Unused, + vars=Current + }, - Extra = SA#elixir_scope.extra_guards, - TGuards = guards(Line, Guards, Extra, SA), - {{clause, Line, TArgs, TGuards, unblock(TExpr)}, SE}. + CallE = E#{context := match}, + {EExpr, #elixir_ex{vars=NewCurrent, unused=NewUnused}, EE} = Fun(Expr, CallS, CallE), -% Translate/Extract guards from the given expression. + EndS = AfterS#elixir_ex{ + prematch=Prematch, + unused=NewUnused, + vars=NewCurrent + }, -guards(Line, Guards, Extra, S) -> - SG = S#elixir_scope{context=guard, extra_guards=nil}, + EndE = EE#{context := ?key(E, context)}, + {EExpr, EndS, EndE}. + +def({Meta, Args, Guards, Body}, S, E) -> + {EArgs, SA, EA} = elixir_expand:expand_args(Args, S#elixir_ex{prematch={#{}, 0}}, E#{context := match}), + {EGuards, SG, EG} = guard(Guards, SA#elixir_ex{prematch=warn}, EA#{context := guard}), + {EBody, SB, EB} = elixir_expand:expand(Body, SG, EG#{context := nil}), + elixir_env:check_unused_vars(SB, EB), + {Meta, EArgs, EGuards, EBody}. + +clause(Meta, Kind, Fun, {'->', ClauseMeta, [_, _]} = Clause, S, E) when is_function(Fun, 4) -> + clause(Meta, Kind, fun(X, SA, EA) -> Fun(ClauseMeta, X, SA, EA) end, Clause, S, E); +clause(_Meta, _Kind, Fun, {'->', Meta, [Left, Right]}, S, E) -> + {ELeft, SL, EL} = Fun(Left, S, E), + {ERight, SR, ER} = elixir_expand:expand(Right, SL, EL), + {{'->', Meta, [ELeft, ERight]}, SR, ER}; +clause(Meta, Kind, _Fun, _, _, E) -> + form_error(Meta, E, ?MODULE, {bad_or_missing_clauses, Kind}). + +head([{'when', Meta, [_ | _] = All}], S, E) -> + {Args, Guard} = elixir_utils:split_last(All), + Prematch = S#elixir_ex.prematch, + + {{EArgs, EGuard}, SG, EG} = + match(fun(ok, SM, EM) -> + {EArgs, SA, EA} = elixir_expand:expand_args(Args, SM, EM), + {EGuard, SG, EG} = guard(Guard, SA#elixir_ex{prematch=Prematch}, EA#{context := guard}), + {{EArgs, EGuard}, SG, EG} + end, ok, S, S, E), + + {[{'when', Meta, EArgs ++ [EGuard]}], SG, EG}; +head(Args, S, E) -> + match(fun elixir_expand:expand_args/3, Args, S, S, E). + +guard({'when', Meta, [Left, Right]}, S, E) -> + {ELeft, SL, EL} = guard(Left, S, E), + {ERight, SR, ER} = guard(Right, SL, EL), + {{'when', Meta, [ELeft, ERight]}, SR, ER}; +guard(Guard, S, E) -> + {EGuard, SG, EG} = elixir_expand:expand(Guard, S, E), + warn_zero_length_guard(EGuard, EG), + {EGuard, SG, EG}. + +warn_zero_length_guard({{'.', _, [erlang, Op]}, Meta, + [{{'.', _, [erlang, length]}, _, [Arg]}, 0]}, E) when Op == '=='; Op == '>' -> + Warn = + case Op of + '==' -> {zero_list_length_in_guard, Arg}; + '>' -> {positive_list_length_in_guard, Arg} + end, + form_warn(Meta, ?key(E, file), ?MODULE, Warn); +warn_zero_length_guard({Op, _, [L, R]}, E) when Op == 'or'; Op == 'and' -> + warn_zero_length_guard(L, E), + warn_zero_length_guard(R, E); +warn_zero_length_guard(_, _) -> + ok. + +%% Case + +'case'(Meta, [], _S, E) -> + form_error(Meta, E, elixir_expand, {missing_option, 'case', [do]}); +'case'(Meta, Opts, _S, E) when not is_list(Opts) -> + form_error(Meta, E, elixir_expand, {invalid_args, 'case'}); +'case'(Meta, Opts, S, E) -> + ok = assert_at_most_once('do', Opts, 0, fun(Key) -> + form_error(Meta, E, ?MODULE, {duplicated_clauses, 'case', Key}) + end), + {Case, SA} = lists:mapfoldl(fun(X, SA) -> expand_case(Meta, X, SA, E) end, S, Opts), + {Case, SA, E}. + +expand_case(Meta, {'do', _} = Do, S, E) -> + Fun = expand_head(Meta, 'case', 'do'), + expand_clauses(Meta, 'case', Fun, Do, S, E); +expand_case(Meta, {Key, _}, _S, E) -> + form_error(Meta, E, ?MODULE, {unexpected_option, 'case', Key}). + +%% Cond + +'cond'(Meta, [], _S, E) -> + form_error(Meta, E, elixir_expand, {missing_option, 'cond', [do]}); +'cond'(Meta, Opts, _S, E) when not is_list(Opts) -> + form_error(Meta, E, elixir_expand, {invalid_args, 'cond'}); +'cond'(Meta, Opts, S, E) -> + ok = assert_at_most_once('do', Opts, 0, fun(Key) -> + form_error(Meta, E, ?MODULE, {duplicated_clauses, 'cond', Key}) + end), + {Cond, SA} = lists:mapfoldl(fun(X, SA) -> expand_cond(Meta, X, SA, E) end, S, Opts), + {Cond, SA, E}. + +expand_cond(Meta, {'do', _} = Do, S, E) -> + Fun = expand_one(Meta, 'cond', 'do', fun elixir_expand:expand_args/3), + expand_clauses(Meta, 'cond', Fun, Do, S, E); +expand_cond(Meta, {Key, _}, _S, E) -> + form_error(Meta, E, ?MODULE, {unexpected_option, 'cond', Key}). + +%% Receive + +'receive'(Meta, [], _S, E) -> + form_error(Meta, E, elixir_expand, {missing_option, 'receive', [do, 'after']}); +'receive'(Meta, Opts, _S, E) when not is_list(Opts) -> + form_error(Meta, E, elixir_expand, {invalid_args, 'receive'}); +'receive'(Meta, Opts, S, E) -> + RaiseError = fun(Key) -> + form_error(Meta, E, ?MODULE, {duplicated_clauses, 'receive', Key}) + end, + ok = assert_at_most_once('do', Opts, 0, RaiseError), + ok = assert_at_most_once('after', Opts, 0, RaiseError), + {Receive, SA} = lists:mapfoldl(fun(X, SA) -> expand_receive(Meta, X, SA, E) end, S, Opts), + {Receive, SA, E}. + +expand_receive(_Meta, {'do', {'__block__', _, []}} = Do, S, _E) -> + {Do, S}; +expand_receive(Meta, {'do', _} = Do, S, E) -> + Fun = expand_head(Meta, 'receive', 'do'), + expand_clauses(Meta, 'receive', Fun, Do, S, E); +expand_receive(Meta, {'after', [_]} = After, S, E) -> + Fun = expand_one(Meta, 'receive', 'after', fun elixir_expand:expand_args/3), + expand_clauses(Meta, 'receive', Fun, After, S, E); +expand_receive(Meta, {'after', _}, _S, E) -> + form_error(Meta, E, ?MODULE, multiple_after_clauses_in_receive); +expand_receive(Meta, {Key, _}, _S, E) -> + form_error(Meta, E, ?MODULE, {unexpected_option, 'receive', Key}). + +%% With + +with(Meta, Args, S, E) -> + {Exprs, Opts0} = + case elixir_utils:split_last(Args) of + {_, LastArg} = SplitResult when is_list(LastArg) -> + SplitResult; + _ -> + {Args, []} + end, + + S0 = elixir_env:reset_unused_vars(S), + {EExprs, {S1, E1, HasMatch}} = lists:mapfoldl(fun expand_with/2, {S0, E, false}, Exprs), + {EDo, Opts1, S2} = expand_with_do(Meta, Opts0, S, S1, E1), + {EOpts, Opts2, S3} = expand_with_else(Meta, Opts1, S2, E, HasMatch), + + case Opts2 of + [{Key, _} | _] -> + form_error(Meta, E, elixir_clauses, {unexpected_option, with, Key}); + [] -> + ok + end, - case Guards of - [] -> case Extra of [] -> []; _ -> [Extra] end; - _ -> [translate_guard(Line, Guard, Extra, SG) || Guard <- Guards] + {{with, Meta, EExprs ++ [[{do, EDo} | EOpts]]}, S3, E}. + +expand_with({'<-', Meta, [Left, Right]}, {S, E, _HasMatch}) -> + {ERight, SR, ER} = elixir_expand:expand(Right, S, E), + SM = elixir_env:reset_read(SR, S), + {[ELeft], SL, EL} = head([Left], SM, ER), + {{'<-', Meta, [ELeft, ERight]}, {SL, EL, true}}; +expand_with(Expr, {S, E, HasMatch}) -> + {EExpr, SE, EE} = elixir_expand:expand(Expr, S, E), + {EExpr, {SE, EE, HasMatch}}. + +expand_with_do(Meta, Opts, S, Acc, E) -> + case lists:keytake(do, 1, Opts) of + {value, {do, Expr}, RestOpts} -> + {EExpr, SAcc, EAcc} = elixir_expand:expand(Expr, Acc, E), + {EExpr, RestOpts, elixir_env:merge_and_check_unused_vars(SAcc, S, EAcc)}; + false -> + form_error(Meta, E, elixir_expand, {missing_option, 'with', [do]}) end. -translate_guard(Line, Guard, Extra, S) -> - [element(1, elixir_translator:translate(elixir_quote:linify(Line, Guard), S))|Extra]. - -extract_guards({'when', _, [Left, Right]}) -> {Left, extract_or_guards(Right)}; -extract_guards(Else) -> {Else, []}. - -extract_or_guards({'when', _, [Left, Right]}) -> [Left|extract_or_guards(Right)]; -extract_or_guards(Term) -> [Term]. +expand_with_else(Meta, Opts, S, E, HasMatch) -> + case lists:keytake(else, 1, Opts) of + {value, Pair, RestOpts} -> + if + HasMatch -> ok; + true -> form_warn(Meta, ?key(E, file), ?MODULE, unmatchable_else_in_with) + end, + Fun = expand_head(Meta, 'with', 'else'), + {EPair, SE} = expand_clauses(Meta, 'with', Fun, Pair, S, E), + {[EPair], RestOpts, SE}; + false -> + {[], Opts, S} + end. -% Extract guards when multiple left side args are allowed. +%% Try + +'try'(Meta, [], _S, E) -> + form_error(Meta, E, elixir_expand, {missing_option, 'try', [do]}); +'try'(Meta, [{do, _}], _S, E) -> + form_error(Meta, E, elixir_expand, {missing_option, 'try', ['catch', 'rescue', 'after']}); +'try'(Meta, Opts, _S, E) when not is_list(Opts) -> + form_error(Meta, E, elixir_expand, {invalid_args, 'try'}); +'try'(Meta, Opts, S, E) -> + % TODO: Make this an error on v2.0 + case Opts of + [{do, _}, {else, _}] -> + form_warn(Meta, ?key(E, file), ?MODULE, {try_with_only_else_clause, origin(Meta, 'try')}); + _ -> + ok + end, + RaiseError = fun(Key) -> + form_error(Meta, E, ?MODULE, {duplicated_clauses, 'try', Key}) + end, + ok = assert_at_most_once('do', Opts, 0, RaiseError), + ok = assert_at_most_once('rescue', Opts, 0, RaiseError), + ok = assert_at_most_once('catch', Opts, 0, RaiseError), + ok = assert_at_most_once('else', Opts, 0, RaiseError), + ok = assert_at_most_once('after', Opts, 0, RaiseError), + ok = warn_catch_before_rescue(Opts, Meta, E, false), + {Try, SA} = lists:mapfoldl(fun(X, SA) -> expand_try(Meta, X, SA, E) end, S, Opts), + {Try, SA, E}. + +expand_try(_Meta, {'do', Expr}, S, E) -> + {EExpr, SE, EE} = elixir_expand:expand(Expr, elixir_env:reset_unused_vars(S), E), + {{'do', EExpr}, elixir_env:merge_and_check_unused_vars(SE, S, EE)}; +expand_try(_Meta, {'after', Expr}, S, E) -> + {EExpr, SE, EE} = elixir_expand:expand(Expr, elixir_env:reset_unused_vars(S), E), + {{'after', EExpr}, elixir_env:merge_and_check_unused_vars(SE, S, EE)}; +expand_try(Meta, {'else', _} = Else, S, E) -> + Fun = expand_head(Meta, 'try', 'else'), + expand_clauses(Meta, 'try', Fun, Else, S, E); +expand_try(Meta, {'catch', _} = Catch, S, E) -> + expand_clauses_with_stacktrace(Meta, fun expand_catch/4, Catch, S, E); +expand_try(Meta, {'rescue', _} = Rescue, S, E) -> + expand_clauses_with_stacktrace(Meta, fun expand_rescue/4, Rescue, S, E); +expand_try(Meta, {Key, _}, _S, E) -> + form_error(Meta, E, ?MODULE, {unexpected_option, 'try', Key}). + +expand_clauses_with_stacktrace(Meta, Fun, Clauses, S, E) -> + OldStacktrace = S#elixir_ex.stacktrace, + SS = S#elixir_ex{stacktrace=true}, + {Ret, SE} = expand_clauses(Meta, 'try', Fun, Clauses, SS, E), + {Ret, SE#elixir_ex{stacktrace=OldStacktrace}}. + +expand_catch(_Meta, [_] = Args, S, E) -> + head(Args, S, E); +expand_catch(_Meta, [_, _] = Args, S, E) -> + head(Args, S, E); +expand_catch(Meta, _, _, E) -> + Error = {wrong_number_of_args_for_clause, "one or two args", origin(Meta, 'try'), 'catch'}, + form_error(Meta, E, ?MODULE, Error). + +expand_rescue(Meta, [Arg], S, E) -> + case expand_rescue(Arg, S, E) of + {EArg, SA, EA} -> + {[EArg], SA, EA}; + false -> + form_error(Meta, E, ?MODULE, invalid_rescue_clause) + end; +expand_rescue(Meta, _, _, E) -> + Error = {wrong_number_of_args_for_clause, "one argument", origin(Meta, 'try'), 'rescue'}, + form_error(Meta, E, ?MODULE, Error). + +%% rescue var +expand_rescue({Name, _, Atom} = Var, S, E) when is_atom(Name), is_atom(Atom) -> + match(fun elixir_expand:expand/3, Var, S, S, E); + +%% rescue var in _ => rescue var +expand_rescue({in, _, [{Name, _, VarContext} = Var, {'_', _, UnderscoreContext}]}, S, E) + when is_atom(Name), is_atom(VarContext), is_atom(UnderscoreContext) -> + expand_rescue(Var, S, E); + +%% rescue var in [Exprs] +expand_rescue({in, Meta, [Left, Right]}, S, E) -> + {ELeft, SL, EL} = match(fun elixir_expand:expand/3, Left, S, S, E), + {ERight, SR, ER} = elixir_expand:expand(Right, SL, EL), + + case ELeft of + {Name, _, Atom} when is_atom(Name), is_atom(Atom) -> + case normalize_rescue(ERight) of + false -> false; + Other -> {{in, Meta, [ELeft, Other]}, SR, ER} + end; + _ -> + false + end; -extract_splat_guards([{'when', _, [_,_|_] = Args}]) -> - {Left, Right} = elixir_utils:split_last(Args), - {Left, extract_or_guards(Right)}; -extract_splat_guards(Else) -> - {Else, []}. +%% rescue Error => _ in [Error] +expand_rescue(Arg, S, E) -> + expand_rescue({in, [], [{'_', [], ?key(E, module)}, Arg]}, S, E). + +normalize_rescue({'_', _, Atom} = N) when is_atom(Atom) -> N; +normalize_rescue(Atom) when is_atom(Atom) -> [Atom]; +normalize_rescue(Other) -> + is_list(Other) andalso lists:all(fun is_atom/1, Other) andalso Other. + +%% Expansion helpers + +expand_head(Meta, Kind, Key) -> + fun + ([{'when', _, [_, _, _ | _]}], _, E) -> + form_error(Meta, E, ?MODULE, {wrong_number_of_args_for_clause, "one argument", Kind, Key}); + ([_] = Args, S, E) -> + head(Args, S, E); + (_, _, E) -> + form_error(Meta, E, ?MODULE, {wrong_number_of_args_for_clause, "one argument", Kind, Key}) + end. -% Function for translating macros with match style like case and receive. +%% Returns a function that expands arguments +%% considering we have at maximum one entry. +expand_one(Meta, Kind, Key, Fun) -> + fun + ([_] = Args, S, E) -> + Fun(Args, S, E); + (_, _, E) -> + form_error(Meta, E, ?MODULE, {wrong_number_of_args_for_clause, "one argument", Kind, Key}) + end. -clauses(Meta, Clauses, Return, #elixir_scope{export_vars=CV} = S) -> - {TC, TS} = do_clauses(Meta, Clauses, Return, S#elixir_scope{export_vars=[]}), - {TC, TS#elixir_scope{export_vars=elixir_scope:merge_opt_vars(CV, TS#elixir_scope.export_vars)}}. +%% Expands all -> pairs in a given key but do not keep the overall vars. +expand_clauses(Meta, Kind, Fun, Clauses, S, E) -> + NewKind = origin(Meta, Kind), + expand_clauses_origin(Meta, NewKind, Fun, Clauses, S, E). -do_clauses(_Meta, [], _Return, S) -> - {[], S}; +expand_clauses_origin(Meta, Kind, Fun, {Key, [_ | _] = Clauses}, S, E) -> + Transformer = fun(Clause, SA) -> + {EClause, SAcc, EAcc} = + clause(Meta, {Kind, Key}, Fun, Clause, elixir_env:reset_unused_vars(SA), E), -do_clauses(Meta, DecoupledClauses, Return, S) -> - % Transform tree just passing the variables counter forward - % and storing variables defined inside each clause. - Transformer = fun(X, {SAcc, VAcc}) -> - {TX, TS} = each_clause(Meta, X, Return, SAcc), - {TX, {elixir_scope:mergec(S, TS), [TS#elixir_scope.export_vars|VAcc]}} + {EClause, elixir_env:merge_and_check_unused_vars(SAcc, SA, EAcc)} end, + {Values, SE} = lists:mapfoldl(Transformer, S, Clauses), + {{Key, Values}, SE}; +expand_clauses_origin(Meta, Kind, _Fun, {Key, _}, _, E) -> + form_error(Meta, E, ?MODULE, {bad_or_missing_clauses, {Kind, Key}}). + +assert_at_most_once(_Kind, [], _Count, _Fun) -> ok; +assert_at_most_once(Kind, [{Kind, _} | _], 1, ErrorFun) -> + ErrorFun(Kind); +assert_at_most_once(Kind, [{Kind, _} | Rest], Count, Fun) -> + assert_at_most_once(Kind, Rest, Count + 1, Fun); +assert_at_most_once(Kind, [_ | Rest], Count, Fun) -> + assert_at_most_once(Kind, Rest, Count, Fun). + +warn_catch_before_rescue([], _, _, _) -> + ok; +warn_catch_before_rescue([{'rescue', _} | _], Meta, E, true) -> + form_warn(Meta, ?key(E, file), ?MODULE, {catch_before_rescue, origin(Meta, 'try')}); +warn_catch_before_rescue([{'catch', _} | Rest], Meta, E, _) -> + warn_catch_before_rescue(Rest, Meta, E, true); +warn_catch_before_rescue([_ | Rest], Meta, E, Found) -> + warn_catch_before_rescue(Rest, Meta, E, Found). + +origin(Meta, Default) -> + case lists:keyfind(origin, 1, Meta) of + {origin, Origin} -> Origin; + false -> Default + end. - {TClauses, {TS, ReverseCV}} = - lists:mapfoldl(Transformer, {S, []}, DecoupledClauses), - - % Now get all the variables defined inside each clause - CV = lists:reverse(ReverseCV), - AllVars = lists:foldl(fun elixir_scope:merge_vars/2, [], CV), - - % Create a new scope that contains a list of all variables - % defined inside all the clauses. It returns this new scope and - % a list of tuples where the first element is the variable name, - % the second one is the new pointer to the variable and the third - % is the old pointer. - {FinalVars, FS} = lists:mapfoldl(fun({Key, Val}, Acc) -> - normalize_vars(Key, Val, Acc) - end, TS, AllVars), - - % Expand all clauses by adding a match operation at the end - % that defines variables missing in one clause to the others. - expand_clauses(?line(Meta), TClauses, CV, FinalVars, [], FS). - -expand_clauses(Line, [Clause|T], [ClauseVars|V], FinalVars, Acc, S) -> - case generate_match_vars(FinalVars, ClauseVars, [], []) of - {[], []} -> - expand_clauses(Line, T, V, FinalVars, [Clause|Acc], S); - {Left, Right} -> - MatchExpr = generate_match(Line, Left, Right), - ClauseExprs = element(5, Clause), - [Final|RawClauseExprs] = lists:reverse(ClauseExprs), - - % If the last sentence has a match clause, we need to assign its value - % in the variable list. If not, we insert the variable list before the - % final clause in order to keep it tail call optimized. - {FinalClauseExprs, FS} = case has_match_tuple(Final) of - true -> - case Final of - {match, _, {var, _, UserVarName} = UserVar, _} when UserVarName /= '_' -> - {[UserVar,MatchExpr,Final|RawClauseExprs], S}; - _ -> - {VarName, _, SS} = elixir_scope:build_var('_', S), - StorageVar = {var, Line, VarName}, - StorageExpr = {match, Line, StorageVar, Final}, - {[StorageVar,MatchExpr,StorageExpr|RawClauseExprs], SS} - end; - false -> - {[Final,MatchExpr|RawClauseExprs], S} - end, - - FinalClause = setelement(5, Clause, lists:reverse(FinalClauseExprs)), - expand_clauses(Line, T, V, FinalVars, [FinalClause|Acc], FS) - end; +format_error({bad_or_missing_clauses, {Kind, Key}}) -> + io_lib:format("expected -> clauses for :~ts in \"~ts\"", [Key, Kind]); +format_error({bad_or_missing_clauses, Kind}) -> + io_lib:format("expected -> clauses in \"~ts\"", [Kind]); -expand_clauses(_Line, [], [], _FinalVars, Acc, S) -> - {lists:reverse(Acc), S}. +format_error({duplicated_clauses, Kind, Key}) -> + io_lib:format("duplicated :~ts clauses given for \"~ts\"", [Key, Kind]); -% Handle each key/value clause pair and translate them accordingly. +format_error({unexpected_option, Kind, Option}) -> + io_lib:format("unexpected option ~ts in \"~ts\"", ['Elixir.Macro':to_string(Option), Kind]); -each_clause(Export, {match, Meta, [Condition], Expr}, Return, S) -> - Fun = wrap_export_fun(Export, fun elixir_translator:translate_args/2), - {Arg, Guards} = extract_guards(Condition), - clause(?line(Meta), Fun, [Arg], Expr, Guards, Return, S); +format_error({wrong_number_of_args_for_clause, Expected, Kind, Key}) -> + io_lib:format("expected ~ts for :~ts clauses (->) in \"~ts\"", [Expected, Key, Kind]); -each_clause(Export, {expr, Meta, [Condition], Expr}, Return, S) -> - {TCondition, SC} = (wrap_export_fun(Export, fun elixir_translator:translate/2))(Condition, S), - {TExpr, SB} = elixir_translator:translate_block(Expr, Return, SC), - {{clause, ?line(Meta), [TCondition], [], unblock(TExpr)}, SB}. +format_error(multiple_after_clauses_in_receive) -> + "expected a single -> clause for :after in \"receive\""; -wrap_export_fun(Meta, Fun) -> - case lists:keyfind(export_head, 1, Meta) of - {export_head, true} -> - Fun; - _ -> - fun(Args, S) -> - {TArgs, TS} = Fun(Args, S), - {TArgs, TS#elixir_scope{export_vars = S#elixir_scope.export_vars}} - end - end. +format_error(invalid_rescue_clause) -> + "invalid \"rescue\" clause. The clause should match on an alias, a variable " + "or be in the \"var in [alias]\" format"; -% Check if the given expression is a match tuple. -% This is a small optimization to allow us to change -% existing assignments instead of creating new ones every time. - -has_match_tuple({'receive', _, _, _, _}) -> - true; -has_match_tuple({'receive', _, _}) -> - true; -has_match_tuple({'case', _, _, _}) -> - true; -has_match_tuple({match, _, _, _}) -> - true; -has_match_tuple({'fun', _, {clauses, _}}) -> - false; -has_match_tuple(H) when is_tuple(H) -> - has_match_tuple(tuple_to_list(H)); -has_match_tuple(H) when is_list(H) -> - lists:any(fun has_match_tuple/1, H); -has_match_tuple(_) -> false. - -% Normalize the given var in between clauses -% by picking one value as reference and retriving -% its previous value. - -normalize_vars(Key, Value, #elixir_scope{vars=Vars,export_vars=ClauseVars} = S) -> - VS = S#elixir_scope{ - vars=orddict:store(Key, Value, Vars), - export_vars=orddict:store(Key, Value, ClauseVars) - }, - - Expr = case orddict:find(Key, Vars) of - {ok, {PreValue, _}} -> {var, 0, PreValue}; - error -> {atom, 0, nil} - end, - - {{Key, Value, Expr}, VS}. - -% Generate match vars by checking if they were updated -% or not and assigning the previous value. - -generate_match_vars([{Key, Value, Expr}|T], ClauseVars, Left, Right) -> - case orddict:find(Key, ClauseVars) of - {ok, Value} -> - generate_match_vars(T, ClauseVars, Left, Right); - {ok, Clause} -> - generate_match_vars(T, ClauseVars, - [{var, 0, element(1, Value)}|Left], - [{var, 0, element(1, Clause)}|Right]); - error -> - generate_match_vars(T, ClauseVars, - [{var, 0, element(1, Value)}|Left], [Expr|Right]) - end; +format_error({catch_before_rescue, Origin}) -> + io_lib:format("\"catch\" should always come after \"rescue\" in ~ts", [Origin]); -generate_match_vars([], _ClauseVars, Left, Right) -> - {Left, Right}. +format_error({try_with_only_else_clause, Origin}) -> + io_lib:format("\"else\" shouldn't be used as the only clause in \"~ts\", use \"case\" instead", + [Origin]); -generate_match(Line, [Left], [Right]) -> - {match, Line, Left, Right}; +format_error(unmatchable_else_in_with) -> + "\"else\" clauses will never match because all patterns in \"with\" will always match"; -generate_match(Line, LeftVars, RightVars) -> - {match, Line, {tuple, Line, LeftVars}, {tuple, Line, RightVars}}. +format_error({zero_list_length_in_guard, ListArg}) -> + Arg = 'Elixir.Macro':to_string(ListArg), + io_lib:format("do not use \"length(~ts) == 0\" to check if a list is empty since length " + "always traverses the whole list. Prefer to pattern match on an empty list or " + "use \"~ts == []\" as a guard", [Arg, Arg]); -unblock({'block', _, Exprs}) -> Exprs; -unblock(Exprs) -> [Exprs]. +format_error({positive_list_length_in_guard, ListArg}) -> + Arg = 'Elixir.Macro':to_string(ListArg), + io_lib:format("do not use \"length(~ts) > 0\" to check if a list is not empty since length " + "always traverses the whole list. Prefer to pattern match on a non-empty list, " + "such as [_ | _], or use \"~ts != []\" as a guard", [Arg, Arg]). diff --git a/lib/elixir/src/elixir_code_server.erl b/lib/elixir/src/elixir_code_server.erl index 1a1f5e08bf5..7b1c9656519 100644 --- a/lib/elixir/src/elixir_code_server.erl +++ b/lib/elixir/src/elixir_code_server.erl @@ -4,15 +4,11 @@ handle_info/2, terminate/2, code_change/3]). -behaviour(gen_server). --define(timeout, 30000). +-define(timeout, infinity). -record(elixir_code_server, { - compilation_status=[], - argv=[], - loaded=[], - at_exit=[], - pool={[],0}, - compiler_options=[{docs,true},{debug_info,true},{warnings_as_errors,false}], - erl_compiler_options=nil + required=#{}, + mod_pool={[], [], 0}, + mod_ets=#{} }). call(Args) -> @@ -27,114 +23,92 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, ok, []). init(ok) -> - code:ensure_loaded('Elixir.Macro.Env'), - code:ensure_loaded('Elixir.Module.LocalsTracker'), - code:ensure_loaded('Elixir.Kernel.LexicalTracker'), + %% The table where we store module definitions + _ = ets:new(elixir_modules, [set, public, named_table, {read_concurrency, true}]), {ok, #elixir_code_server{}}. +handle_call({defmodule, Module, Pid, Tuple}, _From, Config) -> + case ets:lookup(elixir_modules, Module) of + [] -> + {Ref, NewConfig} = defmodule(Pid, Tuple, Config), + {reply, {ok, Ref}, NewConfig}; + [CurrentTuple] -> + {reply, {error, CurrentTuple}, Config} + end; + +handle_call({undefmodule, Ref}, _From, Config) -> + {reply, ok, undefmodule(Ref, Config)}; + handle_call({acquire, Path}, From, Config) -> - Current = Config#elixir_code_server.loaded, - case orddict:find(Path, Current) of + Current = Config#elixir_code_server.required, + case maps:find(Path, Current) of {ok, true} -> - {reply, loaded, Config}; - {ok, {Ref, List}} when is_list(List), is_reference(Ref) -> - Queued = orddict:store(Path, {Ref, [From|List]}, Current), - {reply, {queued, Ref}, Config#elixir_code_server{loaded=Queued}}; + {reply, required, Config}; + {ok, Queued} when is_list(Queued) -> + Required = maps:put(Path, [From | Queued], Current), + {noreply, Config#elixir_code_server{required=Required}}; error -> - Queued = orddict:store(Path, {make_ref(), []}, Current), - {reply, proceed, Config#elixir_code_server{loaded=Queued}} + Required = maps:put(Path, [], Current), + {reply, proceed, Config#elixir_code_server{required=Required}} end; -handle_call(loaded, _From, Config) -> - {reply, [F || {F, true} <- Config#elixir_code_server.loaded], Config}; - -handle_call(at_exit, _From, Config) -> - {reply, Config#elixir_code_server.at_exit, Config}; - -handle_call(flush_at_exit, _From, Config) -> - {reply, Config#elixir_code_server.at_exit, Config#elixir_code_server{at_exit=[]}}; - -handle_call(argv, _From, Config) -> - {reply, Config#elixir_code_server.argv, Config}; +handle_call(required, _From, Config) -> + {reply, [F || {F, true} <- maps:to_list(Config#elixir_code_server.required)], Config}; -handle_call(compiler_options, _From, Config) -> - {reply, Config#elixir_code_server.compiler_options, Config}; - -handle_call({compilation_status, CompilerPid}, _From, Config) -> - CompilationStatusList = Config#elixir_code_server.compilation_status, - CompilationStatusListNew = orddict:erase(CompilerPid, CompilationStatusList), - CompilationStatus = orddict:fetch(CompilerPid, CompilationStatusList), - {reply, CompilationStatus, Config#elixir_code_server{compilation_status=CompilationStatusListNew}}; - -handle_call(retrieve_module_name, _From, Config) -> - case Config#elixir_code_server.pool of - {[H|T], Counter} -> - {reply, module_tuple(H), Config#elixir_code_server{pool={T,Counter}}}; - {[], Counter} -> - {reply, module_tuple(Counter), Config#elixir_code_server{pool={[],Counter+1}}} +handle_call(retrieve_compiler_module, _From, Config) -> + case Config#elixir_code_server.mod_pool of + {Used, [Mod | Unused], Counter} -> + {reply, Mod, Config#elixir_code_server{mod_pool={Used, Unused, Counter}}}; + {Used, [], Counter} -> + {reply, compiler_module(Counter), Config#elixir_code_server{mod_pool={Used, [], Counter+1}}} end; -handle_call(erl_compiler_options, _From, Config) -> - case Config#elixir_code_server.erl_compiler_options of - nil -> - Opts = erl_compiler_options(), - {reply, Opts, Config#elixir_code_server{erl_compiler_options=Opts}}; - Opts -> - {reply, Opts, Config} - end; +handle_call(purge_compiler_modules, _From, Config) -> + {Used, Unused, Counter} = Config#elixir_code_server.mod_pool, + _ = [code:purge(Module) || Module <- Used], + ModPool = {[], Used ++ Unused, Counter}, + {reply, {ok, length(Used)}, Config#elixir_code_server{mod_pool=ModPool}}; handle_call(Request, _From, Config) -> {stop, {badcall, Request}, Config}. -handle_cast({at_exit, AtExit}, Config) -> - {noreply, Config#elixir_code_server{at_exit=[AtExit|Config#elixir_code_server.at_exit]}}; - -handle_cast({argv, Argv}, Config) -> - {noreply, Config#elixir_code_server{argv=Argv}}; - -handle_cast({compiler_options, Options}, Config) -> - Final = orddict:merge(fun(_,_,V) -> V end, Config#elixir_code_server.compiler_options, Options), - {noreply, Config#elixir_code_server{compiler_options=Final}}; - -handle_cast({register_warning, CompilerPid}, Config) -> - CompilationStatusCurrent = Config#elixir_code_server.compilation_status, - CompilationStatusNew = orddict:store(CompilerPid, error, CompilationStatusCurrent), - case orddict:find(warnings_as_errors, Config#elixir_code_server.compiler_options) of - {ok, true} -> {noreply, Config#elixir_code_server{compilation_status=CompilationStatusNew}}; - _ -> {noreply, Config} - end; - -handle_cast({reset_warnings, CompilerPid}, Config) -> - CompilationStatusCurrent = Config#elixir_code_server.compilation_status, - CompilationStatusNew = orddict:store(CompilerPid, ok, CompilationStatusCurrent), - {noreply, Config#elixir_code_server{compilation_status=CompilationStatusNew}}; - -handle_cast({loaded, Path}, Config) -> - Current = Config#elixir_code_server.loaded, - case orddict:find(Path, Current) of +handle_cast({required, Path}, Config) -> + Current = Config#elixir_code_server.required, + case maps:find(Path, Current) of {ok, true} -> {noreply, Config}; - {ok, {Ref, List}} when is_list(List), is_reference(Ref) -> - [Pid ! {elixir_code_server, Ref, loaded} || {Pid, _Tag} <- lists:reverse(List)], - Done = orddict:store(Path, true, Current), - {noreply, Config#elixir_code_server{loaded=Done}}; + {ok, Queued} -> + _ = [gen_server:reply(From, required) || From <- lists:reverse(Queued)], + Done = maps:put(Path, true, Current), + {noreply, Config#elixir_code_server{required=Done}}; error -> - Done = orddict:store(Path, true, Current), - {noreply, Config#elixir_code_server{loaded=Done}} + Done = maps:put(Path, true, Current), + {noreply, Config#elixir_code_server{required=Done}} end; -handle_cast({unload_files, Files}, Config) -> - Current = Config#elixir_code_server.loaded, - Unloaded = lists:foldl(fun(File, Acc) -> orddict:erase(File, Acc) end, Current, Files), - {noreply, Config#elixir_code_server{loaded=Unloaded}}; +handle_cast({unrequire_files, Files}, Config) -> + Current = Config#elixir_code_server.required, + Unrequired = maps:without(Files, Current), + {noreply, Config#elixir_code_server{required=Unrequired}}; + +handle_cast({return_compiler_module, Module, Purgeable}, Config) -> + {Used, Unused, Counter} = Config#elixir_code_server.mod_pool, -handle_cast({return_module_name, H}, #elixir_code_server{pool={T,Counter}} = Config) -> - {noreply, Config#elixir_code_server{pool={[H|T],Counter}}}; + ModPool = + case Purgeable of + true -> {Used, [Module | Unused], Counter}; + false -> {[Module | Used], Unused, Counter} + end, + + {noreply, Config#elixir_code_server{mod_pool=ModPool}}; handle_cast(Request, Config) -> {stop, {badcast, Request}, Config}. -handle_info(_Request, Config) -> +handle_info({'DOWN', Ref, process, _Pid, _Reason}, Config) -> + {noreply, undefmodule(Ref, Config)}; + +handle_info(_Msg, Config) -> {noreply, Config}. terminate(_Reason, _Config) -> @@ -143,25 +117,20 @@ terminate(_Reason, _Config) -> code_change(_Old, Config, _Extra) -> {ok, Config}. -module_tuple(I) -> - {list_to_atom("elixir_compiler_" ++ integer_to_list(I)), I}. - -erl_compiler_options() -> - Key = "ERL_COMPILER_OPTIONS", - case os:getenv(Key) of - false -> []; - Str when is_list(Str) -> - case erl_scan:string(Str) of - {ok,Tokens,_} -> - case erl_parse:parse_term(Tokens ++ [{dot, 1}]) of - {ok,List} when is_list(List) -> List; - {ok,Term} -> [Term]; - {error,_Reason} -> - io:format("Ignoring bad term in ~ts\n", [Key]), - [] - end; - {error, {_,_,_Reason}, _} -> - io:format("Ignoring bad term in ~ts\n", [Key]), - [] - end +compiler_module(I) -> + list_to_atom("elixir_compiler_" ++ integer_to_list(I)). + +defmodule(Pid, Tuple, #elixir_code_server{mod_ets=ModEts} = Config) -> + ets:insert(elixir_modules, Tuple), + Ref = erlang:monitor(process, Pid), + Mod = erlang:element(1, Tuple), + {Ref, Config#elixir_code_server{mod_ets=maps:put(Ref, Mod, ModEts)}}. + +undefmodule(Ref, #elixir_code_server{mod_ets=ModEts} = Config) -> + case maps:find(Ref, ModEts) of + {ok, Mod} -> + ets:delete(elixir_modules, Mod), + Config#elixir_code_server{mod_ets=maps:remove(Ref, ModEts)}; + error -> + Config end. diff --git a/lib/elixir/src/elixir_compiler.erl b/lib/elixir/src/elixir_compiler.erl index cd5f52700a2..f87218bbb5b 100644 --- a/lib/elixir/src/elixir_compiler.erl +++ b/lib/elixir/src/elixir_compiler.erl @@ -1,224 +1,224 @@ +%% Elixir compiler front-end to the Erlang backend. -module(elixir_compiler). --export([get_opt/1, string/2, quoted/2, file/1, file_to_path/2]). --export([core/0, module/4, eval_forms/3]). +-export([string/3, quoted/3, bootstrap/0, + file/2, file_to_path/3, compile/3]). -include("elixir.hrl"). -%% Public API +string(Contents, File, Callback) -> + Forms = elixir:'string_to_quoted!'(Contents, 1, 1, File, elixir_config:get(parser_options)), + quoted(Forms, File, Callback). -get_opt(Key) -> - Dict = elixir_code_server:call(compiler_options), - case lists:keyfind(Key, 1, Dict) of - false -> false; - {Key, Value} -> Value - end. - -%% Compilation entry points. +quoted(Forms, File, Callback) -> + Previous = get(elixir_module_binaries), -string(Contents, File) when is_list(Contents), is_binary(File) -> - Forms = elixir:'string_to_quoted!'(Contents, 1, File, []), - quoted(Forms, File). + try + put(elixir_module_binaries, []), + Env = (elixir_env:new())#{line := 1, file := File, tracers := elixir_config:get(tracers)}, -quoted(Forms, File) when is_binary(File) -> - Previous = get(elixir_compiled), + elixir_lexical:run( + Env, + fun (LexicalEnv) -> eval_or_compile(Forms, [], LexicalEnv) end, + fun (#{lexical_tracker := Pid}) -> Callback(File, Pid) end + ), - try - put(elixir_compiled, []), - elixir_lexical:run(File, fun - (Pid) -> - Env = elixir:env_for_eval([{line,1},{file,File}]), - eval_forms(Forms, [], Env#{lexical_tracker := Pid}) - end), - lists:reverse(get(elixir_compiled)) + unzip_reverse(get(elixir_module_binaries), [], []) after - put(elixir_compiled, Previous) + put(elixir_module_binaries, Previous) end. -file(Relative) when is_binary(Relative) -> - File = filename:absname(Relative), - {ok, Bin} = file:read_file(File), - string(elixir_utils:characters_to_list(Bin), File). +unzip_reverse([{Mod, Info} | Tail], Mods, Infos) -> + unzip_reverse(Tail, [Mod | Mods], [Info | Infos]); +unzip_reverse([], Mods, Infos) -> + {Mods, Infos}. -file_to_path(File, Path) when is_binary(File), is_binary(Path) -> - Lists = file(File), - [binary_to_path(X, Path) || X <- Lists], - Lists. +file(File, Callback) -> + {ok, Bin} = file:read_file(File), + string(elixir_utils:characters_to_list(Bin), File, Callback). -%% Evaluation +file_to_path(File, Dest, Callback) when is_binary(File), is_binary(Dest) -> + file(File, fun(CallbackFile, CallbackLexical) -> + _ = [binary_to_path(Mod, Dest) || Mod <- get(elixir_module_binaries)], + Callback(CallbackFile, CallbackLexical) + end). -eval_forms(Forms, Vars, E) -> - case (?m(E, module) == nil) andalso allows_fast_compilation(Forms) of - true -> eval_compilation(Forms, Vars, E); - false -> code_loading_compilation(Forms, Vars, E) +%% Evaluates the given code through the Erlang compiler. +%% It may end-up evaluating the code if it is deemed a +%% more efficient strategy depending on the code snippet. +eval_or_compile(Forms, Args, E) -> + case (?key(E, module) == nil) andalso allows_fast_compilation(Forms) andalso + (not elixir_config:is_bootstrap()) of + true -> fast_compile(Forms, E); + false -> compile(Forms, Args, E) end. -eval_compilation(Forms, Vars, E) -> - Binding = [{Key, Value} || {_Name, _Kind, Key, Value} <- Vars], - {Result, _Binding, EE, _S} = elixir:eval_forms(Forms, Binding, E), - {Result, EE}. - -code_loading_compilation(Forms, Vars, #{line := Line} = E) -> - Dict = [{{Name, Kind}, {Value, 0}} || {Name, Kind, Value, _} <- Vars], - S = elixir_env:env_to_scope_with_vars(E, Dict), - {Expr, EE, _S} = elixir:quoted_to_erl(Forms, E, S), - - {Module, I} = retrieve_module_name(), - Fun = code_fun(?m(E, module)), - Form = code_mod(Fun, Expr, Line, ?m(E, file), Module, Vars), - Args = list_to_tuple([V || {_, _, _, V} <- Vars]), - - %% Pass {native, false} to speed up bootstrap - %% process when native is set to true - AllOpts = elixir_code_server:call(erl_compiler_options), - FinalOpts = AllOpts -- [native, warn_missing_spec], - module(Form, ?m(E, file), FinalOpts, true, fun(_, Binary) -> - %% If we have labeled locals, anonymous functions - %% were created and therefore we cannot ditch the - %% module - Purgeable = - case beam_lib:chunks(Binary, [labeled_locals]) of - {ok, {_, [{labeled_locals, []}]}} -> true; - _ -> false - end, - dispatch_loaded(Module, Fun, Args, Purgeable, I, EE) - end). +compile(Quoted, ArgsList, E) -> + {Expanded, SE, EE} = elixir_expand:expand(Quoted, elixir_env:env_to_ex(E), E), + elixir_env:check_unused_vars(SE, EE), + + {Module, Fun, Purgeable} = + elixir_erl_compiler:spawn(fun() -> spawned_compile(Expanded, E) end), + + Args = list_to_tuple(ArgsList), + {dispatch(Module, Fun, Args, Purgeable), EE}. + +spawned_compile(ExExprs, #{line := Line, file := File} = E) -> + {Vars, S} = elixir_env:env_to_erl(E), + {ErlExprs, _} = elixir_erl_pass:translate(ExExprs, erl_anno:new(Line), S), + + Module = retrieve_compiler_module(), + Fun = code_fun(?key(E, module)), + Forms = code_mod(Fun, ErlExprs, Line, File, Module, Vars), + + {Module, Binary} = elixir_erl_compiler:noenv_forms(Forms, File, [nowarn_nomatch, no_bool_opt, no_ssa_opt]), + code:load_binary(Module, "", Binary), + {Module, Fun, is_purgeable(Module, Binary)}. -dispatch_loaded(Module, Fun, Args, Purgeable, I, E) -> +dispatch(Module, Fun, Args, Purgeable) -> Res = Module:Fun(Args), code:delete(Module), - if Purgeable -> - code:purge(Module), - return_module_name(I); - true -> - ok - end, - {Res, E}. + Purgeable andalso code:purge(Module), + return_compiler_module(Module, Purgeable), + Res. code_fun(nil) -> '__FILE__'; code_fun(_) -> '__MODULE__'. code_mod(Fun, Expr, Line, File, Module, Vars) when is_binary(File), is_integer(Line) -> - Tuple = {tuple, Line, [{var, Line, K} || {_, _, K, _} <- Vars]}, + Ann = erl_anno:new(Line), + Tuple = {tuple, Ann, [{var, Ann, Var} || {_, Var} <- Vars]}, Relative = elixir_utils:relative_to_cwd(File), - [ - {attribute, Line, file, {elixir_utils:characters_to_list(Relative), 1}}, - {attribute, Line, module, Module}, - {attribute, Line, export, [{Fun, 1}, {'__RELATIVE__', 0}]}, - {function, Line, Fun, 1, [ - {clause, Line, [Tuple], [], [Expr]} - ]}, - {function, Line, '__RELATIVE__', 0, [ - {clause, Line, [], [], [elixir_utils:elixir_to_erl(Relative)]} - ]} - ]. + [{attribute, Ann, file, {elixir_utils:characters_to_list(Relative), 1}}, + {attribute, Ann, module, Module}, + {attribute, Ann, compile, no_auto_import}, + {attribute, Ann, export, [{Fun, 1}, {'__RELATIVE__', 0}]}, + {function, Ann, Fun, 1, [ + {clause, Ann, [Tuple], [], [Expr]} + ]}, + {function, Ann, '__RELATIVE__', 0, [ + {clause, Ann, [], [], [elixir_erl:elixir_to_erl(Relative)]} + ]}]. + +retrieve_compiler_module() -> + elixir_code_server:call(retrieve_compiler_module). -retrieve_module_name() -> - elixir_code_server:call(retrieve_module_name). +return_compiler_module(Module, Purgeable) -> + elixir_code_server:cast({return_compiler_module, Module, Purgeable}). -return_module_name(I) -> - elixir_code_server:cast({return_module_name, I}). +is_purgeable(Module, Binary) -> + beam_lib:chunks(Binary, [labeled_locals]) == {ok, {Module, [{labeled_locals, []}]}}. allows_fast_compilation({'__block__', _, Exprs}) -> lists:all(fun allows_fast_compilation/1, Exprs); -allows_fast_compilation({defmodule,_,_}) -> true; -allows_fast_compilation(_) -> false. - -%% INTERNAL API - -%% Compile the module by forms based on the scope information -%% executes the callback in case of success. This automatically -%% handles errors and warnings. Used by this module and elixir_module. -module(Forms, File, Opts, Callback) -> - Final = - case (get_opt(debug_info) == true) orelse - lists:member(debug_info, Opts) of - true -> [debug_info] ++ elixir_code_server:call(erl_compiler_options); - false -> elixir_code_server:call(erl_compiler_options) - end, - module(Forms, File, Final, false, Callback). - -module(Forms, File, Options, Bootstrap, Callback) when - is_binary(File), is_list(Forms), is_list(Options), is_boolean(Bootstrap), is_function(Callback) -> - Listname = elixir_utils:characters_to_list(File), - - case compile:noenv_forms([no_auto_import()|Forms], [return,{source,Listname}|Options]) of - {ok, ModuleName, Binary, Warnings} -> - format_warnings(Bootstrap, Warnings), - code:load_binary(ModuleName, Listname, Binary), - Callback(ModuleName, Binary); - {error, Errors, Warnings} -> - format_warnings(Bootstrap, Warnings), - format_errors(Errors) - end. - -no_auto_import() -> - {attribute, 0, compile, no_auto_import}. - -%% CORE HANDLING - -core() -> - application:start(elixir), - elixir_code_server:cast({compiler_options, [{docs,false},{internal,true}]}), - [core_file(File) || File <- core_main()]. +allows_fast_compilation({defmodule, _, [_, [{do, _}]]}) -> + true; +allows_fast_compilation(_) -> + false. + +fast_compile({'__block__', _, Exprs}, E) -> + lists:foldl(fun(Expr, _) -> fast_compile(Expr, E) end, nil, Exprs); +fast_compile({defmodule, Meta, [Mod, [{do, TailBlock}]]}, NoLineE) -> + E = NoLineE#{line := ?line(Meta)}, + + Block = {'__block__', Meta, [ + {'=', Meta, [{result, Meta, ?MODULE}, TailBlock]}, + {{'.', Meta, [elixir_utils, noop]}, Meta, []}, + {result, Meta, ?MODULE} + ]}, + + Expanded = case Mod of + {'__aliases__', _, _} -> + case elixir_aliases:expand_or_concat(Mod, E) of + Receiver when is_atom(Receiver) -> Receiver; + _ -> 'Elixir.Macro':expand(Mod, E) + end; + + _ -> + 'Elixir.Macro':expand(Mod, E) + end, -core_file(File) -> + ContextModules = [Expanded | ?key(E, context_modules)], + elixir_module:compile(Expanded, Block, [], E#{context_modules := ContextModules}). + +%% Bootstrapper + +bootstrap() -> + {ok, _} = application:ensure_all_started(elixir), + elixir_config:static(#{bootstrap => true}), + elixir_config:put(docs, false), + elixir_config:put(relative_paths, false), + elixir_config:put(ignore_module_conflict, true), + elixir_config:put(tracers, []), + elixir_config:put(parser_options, []), + {Init, Main} = bootstrap_files(), + {ok, Cwd} = file:get_cwd(), + Lib = filename:join(Cwd, "lib/elixir/lib"), + [bootstrap_file(Lib, File) || File <- [<<"kernel.ex">> | Init]], + [bootstrap_file(Lib, File) || File <- [<<"kernel.ex">> | Main]]. + +bootstrap_file(Lib, Suffix) -> try - Lists = file(File), - [binary_to_path(X, "lib/elixir/ebin") || X <- Lists], - io:format("Compiled ~ts~n", [File]) + File = filename:join(Lib, Suffix), + {Mods, _Infos} = file(File, fun(_, _) -> ok end), + _ = [binary_to_path(X, "lib/elixir/ebin") || X <- Mods], + io:format("Compiled ~ts~n", [Suffix]) catch - Kind:Reason -> - io:format("~p: ~p~nstacktrace: ~p~n", [Kind, Reason, erlang:get_stacktrace()]), + Kind:Reason:Stacktrace -> + io:format("~p: ~p~nstacktrace: ~p~n", [Kind, Reason, Stacktrace]), erlang:halt(1) end. -core_main() -> - [<<"lib/elixir/lib/kernel.ex">>, - <<"lib/elixir/lib/macro/env.ex">>, - <<"lib/elixir/lib/keyword.ex">>, - <<"lib/elixir/lib/module.ex">>, - <<"lib/elixir/lib/list.ex">>, - <<"lib/elixir/lib/macro.ex">>, - <<"lib/elixir/lib/code.ex">>, - <<"lib/elixir/lib/module/locals_tracker.ex">>, - <<"lib/elixir/lib/kernel/typespec.ex">>, - <<"lib/elixir/lib/exception.ex">>, - <<"lib/elixir/lib/protocol.ex">>, - <<"lib/elixir/lib/stream/reducers.ex">>, - <<"lib/elixir/lib/enum.ex">>, - <<"lib/elixir/lib/inspect/algebra.ex">>, - <<"lib/elixir/lib/inspect.ex">>, - <<"lib/elixir/lib/range.ex">>, - <<"lib/elixir/lib/regex.ex">>, - <<"lib/elixir/lib/string.ex">>, - <<"lib/elixir/lib/string/chars.ex">>, - <<"lib/elixir/lib/io.ex">>, - <<"lib/elixir/lib/path.ex">>, - <<"lib/elixir/lib/file.ex">>, - <<"lib/elixir/lib/system.ex">>, - <<"lib/elixir/lib/kernel/cli.ex">>, - <<"lib/elixir/lib/kernel/error_handler.ex">>, - <<"lib/elixir/lib/kernel/parallel_compiler.ex">>, - <<"lib/elixir/lib/kernel/lexical_tracker.ex">>]. +bootstrap_files() -> + { + [ + <<"macro/env.ex">>, + <<"keyword.ex">>, + <<"module.ex">>, + <<"list.ex">>, + <<"macro.ex">>, + <<"kernel/typespec.ex">>, + <<"kernel/utils.ex">>, + <<"code.ex">>, + <<"code/identifier.ex">>, + <<"protocol.ex">>, + <<"stream/reducers.ex">>, + <<"enum.ex">>, + <<"regex.ex">>, + <<"inspect/algebra.ex">>, + <<"inspect.ex">>, + <<"string.ex">>, + <<"string/chars.ex">> + ], + [ + <<"list/chars.ex">>, + <<"module/locals_tracker.ex">>, + <<"module/parallel_checker.ex">>, + <<"module/types/helpers.ex">>, + <<"module/types/unify.ex">>, + <<"module/types/of.ex">>, + <<"module/types/pattern.ex">>, + <<"module/types/expr.ex">>, + <<"module/types.ex">>, + <<"exception.ex">>, + <<"path.ex">>, + <<"file.ex">>, + <<"map.ex">>, + <<"range.ex">>, + <<"access.ex">>, + <<"io.ex">>, + <<"system.ex">>, + <<"code/formatter.ex">>, + <<"code/normalizer.ex">>, + <<"kernel/cli.ex">>, + <<"kernel/error_handler.ex">>, + <<"kernel/parallel_compiler.ex">>, + <<"kernel/lexical_tracker.ex">> + ] + }. binary_to_path({ModuleName, Binary}, CompilePath) -> Path = filename:join(CompilePath, atom_to_list(ModuleName) ++ ".beam"), - ok = file:write_file(Path, Binary), - Path. - -%% ERROR HANDLING - -format_errors([]) -> - exit({nocompile, "compilation failed but no error was raised"}); - -format_errors(Errors) -> - lists:foreach(fun ({File, Each}) -> - BinFile = elixir_utils:characters_to_binary(File), - lists:foreach(fun (Error) -> elixir_errors:handle_file_error(BinFile, Error) end, Each) - end, Errors). - -format_warnings(Bootstrap, Warnings) -> - lists:foreach(fun ({File, Each}) -> - BinFile = elixir_utils:characters_to_binary(File), - lists:foreach(fun (Warning) -> elixir_errors:handle_file_warning(Bootstrap, BinFile, Warning) end, Each) - end, Warnings). + case file:write_file(Path, Binary) of + ok -> Path; + {error, Reason} -> error('Elixir.File.Error':exception([{action, "write to"}, {path, Path}, {reason, Reason}])) + end. diff --git a/lib/elixir/src/elixir_config.erl b/lib/elixir/src/elixir_config.erl new file mode 100644 index 00000000000..d1349c11b45 --- /dev/null +++ b/lib/elixir/src/elixir_config.erl @@ -0,0 +1,84 @@ +-module(elixir_config). +-compile({no_auto_import, [get/1]}). +-export([new/1, warn/2, serial/1]). +-export([static/1, is_bootstrap/0, identifier_tokenizer/0]). +-export([delete/1, put/2, get/1, get/2, update/2, get_and_put/2]). +-export([start_link/0, init/1, handle_call/3, handle_cast/2]). +-behaviour(gen_server). + +static(Map) when is_map(Map) -> + persistent_term:put(?MODULE, maps:merge(persistent_term:get(?MODULE, #{}), Map)). +is_bootstrap() -> + maps:get(bootstrap, persistent_term:get(?MODULE, #{}), false). +identifier_tokenizer() -> + maps:get(identifier_tokenizer, persistent_term:get(?MODULE, #{}), 'Elixir.String.Tokenizer'). + +get(Key) -> + [{_, Value}] = ets:lookup(?MODULE, Key), + Value. + +get(Key, Default) -> + try ets:lookup(?MODULE, Key) of + [{_, Value}] -> Value; + [] -> Default + catch + _:_ -> Default + end. + +put(Key, Value) -> + gen_server:call(?MODULE, {put, Key, Value}). + +get_and_put(Key, Value) -> + gen_server:call(?MODULE, {get_and_put, Key, Value}). + +update(Key, Fun) -> + gen_server:call(?MODULE, {update, Key, Fun}). + +serial(Fun) -> + gen_server:call(?MODULE, {serial, Fun}). + +%% Used to guarantee warnings are emitted only once per caller. +warn(Key, [{Mod, Fun, ArgsOrArity, _} | _]) -> + EtsKey = {warn, Key, Mod, Fun, to_arity(ArgsOrArity)}, + ets:update_counter(?MODULE, EtsKey, {2, 1, 1, 1}, {EtsKey, -1}) =:= 0; + +warn(_, _) -> + true. + +to_arity(Args) when is_list(Args) -> length(Args); +to_arity(Arity) -> Arity. + +%% ets life-cycle api + +new(Opts) -> + Tab = ets:new(?MODULE, [named_table, public, {read_concurrency, true}]), + true = ets:insert_new(?MODULE, Opts), + Tab. + +delete(?MODULE) -> + ets:delete(?MODULE). + +%% gen_server api + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, ?MODULE, []). + +init(Tab) -> + {ok, Tab}. + +handle_call({serial, Fun}, _From, Tab) -> + {reply, Fun(), Tab}; +handle_call({put, Key, Value}, _From, Tab) -> + ets:insert(Tab, {Key, Value}), + {reply, ok, Tab}; +handle_call({update, Key, Fun}, _From, Tab) -> + Value = Fun(get(Key)), + ets:insert(Tab, {Key, Value}), + {reply, Value, Tab}; +handle_call({get_and_put, Key, Value}, _From, Tab) -> + OldValue = get(Key), + ets:insert(Tab, {Key, Value}), + {reply, OldValue, Tab}. + +handle_cast(Cast, Tab) -> + {stop, {bad_cast, Cast}, Tab}. diff --git a/lib/elixir/src/elixir_counter.erl b/lib/elixir/src/elixir_counter.erl deleted file mode 100644 index 35cfd160338..00000000000 --- a/lib/elixir/src/elixir_counter.erl +++ /dev/null @@ -1,38 +0,0 @@ --module(elixir_counter). --export([start_link/0, init/1, handle_call/3, handle_cast/2, - handle_info/2, terminate/2, code_change/3, next/0]). --behaviour(gen_server). - --define(timeout, 1000). %% 1 second --define(limit, 4294967296). %% 2^32 - -next() -> - gen_server:call(?MODULE, next, ?timeout). - -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, 0, []). - -init(Counter) -> - {ok, Counter}. - -handle_call(next, _From, Counter) -> - {reply, Counter, bump(Counter)}; -handle_call(Request, _From, Counter) -> - {stop, {badcall, Request}, Counter}. - -handle_cast(Request, Counter) -> - {stop, {badcast, Request}, Counter}. - -handle_info(_Request, Counter) -> - {noreply, Counter}. - -terminate(_Reason, _Counter) -> - ok. - -code_change(_Old, Counter, _Extra) -> - {ok, Counter}. - -bump(Counter) when Counter < ?limit -> - Counter + 1; -bump(_Counter) -> - 0. diff --git a/lib/elixir/src/elixir_def.erl b/lib/elixir/src/elixir_def.erl index e7c134e851a..1f237ff4e64 100644 --- a/lib/elixir/src/elixir_def.erl +++ b/lib/elixir/src/elixir_def.erl @@ -1,386 +1,449 @@ % Holds the logic responsible for function definitions (def(p) and defmacro(p)). -module(elixir_def). --export([table/1, clauses_table/1, setup/1, - cleanup/1, reset_last/1, lookup_definition/2, - delete_definition/2, store_definition/6, unwrap_definitions/1, - store_each/8, format_error/1]). +-export([setup/1, reset_last/1, local_for/4, + take_definition/2, store_definition/5, store_definition/9, + fetch_definitions/2, format_error/1]). -include("elixir.hrl"). +-define(last_def, {elixir, last_def}). --define(attr, '__def_table'). --define(clauses_attr, '__clauses_table'). - -%% Table management functions. Called internally. +setup(DataTables) -> + reset_last(DataTables), + ok. -table(Module) -> - ets:lookup_element(Module, ?attr, 2). +reset_last({DataSet, _DataBag}) -> + ets:insert(DataSet, {?last_def, none}); -clauses_table(Module) -> - ets:lookup_element(Module, ?clauses_attr, 2). +reset_last(Module) when is_atom(Module) -> + reset_last(elixir_module:data_tables(Module)). -setup(Module) -> - ets:insert(Module, {?attr, ets:new(Module, [set, public])}), - ets:insert(Module, {?clauses_attr, ets:new(Module, [bag, public])}), - reset_last(Module), - ok. +local_for(Module, Name, Arity, Kinds) -> + Tuple = {Name, Arity}, -cleanup(Module) -> - ets:delete(table(Module)), - ets:delete(clauses_table(Module)). + try + {Set, Bag} = elixir_module:data_tables(Module), + {ets:lookup(Set, {def, Tuple}), ets:lookup(Bag, {clauses, Tuple})} + of + {[{_, Kind, Meta, _, _, _}], Clauses} -> + case (Kinds == all) orelse (lists:member(Kind, Kinds)) of + true -> elixir_erl:definition_to_anonymous(Module, Kind, Meta, + [Clause || {_, Clause} <- Clauses]); + false -> false + end; + {[], _} -> + false + catch + _:_ -> false + end. -%% Reset the last item. Useful when evaling code. -reset_last(Module) -> - ets:insert(table(Module), {last, []}). +%% Take a definition out of the table -%% Looks up a definition from the database. -lookup_definition(Module, Tuple) -> - case ets:lookup(table(Module), Tuple) of - [Result] -> - CTable = clauses_table(Module), - {Result, [Clause || {_, Clause} <- ets:lookup(CTable, Tuple)]}; - _ -> +take_definition(Module, {Name, Arity} = Tuple) -> + {Set, Bag} = elixir_module:data_tables(Module), + case ets:take(Set, {def, Tuple}) of + [{{def, Tuple}, _, _, _, _, {Defaults, _, _}} = Result] -> + ets:delete_object(Bag, {defs, Tuple}), + ets:delete_object(Bag, {{default, Name}, Arity, Defaults}), + {Result, [Clause || {_, Clause} <- ets:take(Bag, {clauses, Tuple})]}; + [] -> false end. -delete_definition(Module, Tuple) -> - ets:delete(table(Module), Tuple), - ets:delete(clauses_table(Module), Tuple). +%% Fetch all available definitions + +fetch_definitions(File, Module) -> + {Set, Bag} = elixir_module:data_tables(Module), -% Invoked by the wrap definition with the function abstract tree. -% Each function is then added to the function table. + Entries = try + lists:sort(ets:lookup_element(Bag, defs, 2)) + catch + error:badarg -> [] + end, + + fetch_definition(Entries, File, Module, Set, Bag, [], []). + +fetch_definition([Tuple | T], File, Module, Set, Bag, All, Private) -> + [{_, Kind, Meta, _, Check, {MaxDefaults, _, Defaults}}] = ets:lookup(Set, {def, Tuple}), + + try ets:lookup_element(Bag, {clauses, Tuple}, 2) of + Clauses -> + NewAll = + [{Tuple, Kind, add_defaults_to_meta(MaxDefaults, Meta), Clauses} | All], + NewPrivate = + case (Kind == defp) orelse (Kind == defmacrop) of + true -> + Metas = head_and_definition_meta(Check, Meta, MaxDefaults - Defaults, All), + [{Tuple, Kind, Metas, MaxDefaults} | Private]; + false -> + Private + end, + fetch_definition(T, File, Module, Set, Bag, NewAll, NewPrivate) + catch + error:badarg -> + elixir_errors:form_error(Meta, File, ?MODULE, {function_head, Kind, Tuple}), + fetch_definition(T, File, Module, Set, Bag, All, Private) + end; + +fetch_definition([], _File, _Module, _Set, _Bag, All, Private) -> + {All, Private}. + +add_defaults_to_meta(0, Meta) -> Meta; +add_defaults_to_meta(Defaults, Meta) -> [{defaults, Defaults} | Meta]. -store_definition(Line, Kind, CheckClauses, Call, Body, Pos) -> - E = (elixir_locals:get_cached_env(Pos))#{line := Line}, - {NameAndArgs, Guards} = elixir_clauses:extract_guards(Call), +head_and_definition_meta(true, Meta, 0, _All) -> + Meta; +head_and_definition_meta(true, _Meta, _HeadDefaults, [{_, _, HeadMeta, _} | _]) -> + HeadMeta; +head_and_definition_meta(false, _Meta, _HeadDefaults, _All) -> + false. + +%% Section for storing definitions + +store_definition(Kind, CheckClauses, Call, Body, Pos) -> + #{line := Line} = E = elixir_locals:get_cached_env(Pos), + {NameAndArgs, Guards} = elixir_utils:extract_guards(Call), {Name, Args} = case NameAndArgs of {N, _, A} when is_atom(N), is_atom(A) -> {N, []}; {N, _, A} when is_atom(N), is_list(A) -> {N, A}; - _ -> elixir_errors:form_error(Line, ?m(E, file), ?MODULE, {invalid_def, Kind, NameAndArgs}) + _ -> elixir_errors:form_error([{line, Line}], E, ?MODULE, {invalid_def, Kind, NameAndArgs}) end, %% Now that we have verified the call format, %% extract meta information like file and context. {_, Meta, _} = Call, - DoCheckClauses = (not lists:keymember(context, 1, Meta)) andalso (CheckClauses), - %% Check if there is a file information in the definition. - %% If so, we assume this come from another source and we need - %% to linify taking into account keep line numbers. - {File, Key} = case lists:keyfind(file, 1, Meta) of - {file, Bin} when is_binary(Bin) -> {Bin, keep}; - _ -> {nil, line} + Context = case lists:keyfind(context, 1, Meta) of + {context, _} = ContextPair -> [ContextPair]; + _ -> [] end, - LinifyArgs = elixir_quote:linify(Line, Key, Args), - LinifyGuards = elixir_quote:linify(Line, Key, Guards), - LinifyBody = elixir_quote:linify(Line, Key, Body), - - assert_no_aliases_name(Line, Name, Args, E), - store_definition(Line, Kind, DoCheckClauses, Name, - LinifyArgs, LinifyGuards, LinifyBody, File, E). - -store_definition(Line, Kind, CheckClauses, Name, Args, Guards, Body, MetaFile, #{module := Module} = ER) -> - Arity = length(Args), - Tuple = {Name, Arity}, - E = ER#{function := Tuple}, - elixir_locals:record_definition(Tuple, Kind, Module), - - Location = retrieve_location(Line, MetaFile, Module), - {Function, Defaults, Super} = translate_definition(Kind, Line, Module, Name, Args, Guards, Body, E), - - DefaultsLength = length(Defaults), - elixir_locals:record_defaults(Tuple, Kind, Module, DefaultsLength), - - File = ?m(E, file), - Table = table(Module), - CTable = clauses_table(Module), + Generated = case lists:keyfind(generated, 1, Meta) of + {generated, true} -> ?generated(Context); + _ -> Context + end, - compile_super(Module, Super, E), - check_previous_defaults(Table, Line, Name, Arity, Kind, DefaultsLength, E), + DoCheckClauses = (Context == []) andalso (CheckClauses), - store_each(CheckClauses, Kind, File, Location, - Table, CTable, DefaultsLength, Function), - [store_each(false, Kind, File, Location, Table, CTable, 0, - default_function_for(Kind, Name, Default)) || Default <- Defaults], + %% Check if there is a file information in the definition. + %% If so, we assume this come from another source and + %% we need to linify taking into account keep line numbers. + %% + %% Line and File will always point to the caller. __ENV__.line + %% will always point to the quoted one and __ENV__.file will + %% always point to the one at @file or the quoted one. + {Location, LinifyArgs, LinifyGuards, LinifyBody} = + case elixir_utils:meta_keep(Meta) of + {_, _} = MetaFile -> + {MetaFile, + elixir_quote:linify(Line, keep, Args), + elixir_quote:linify(Line, keep, Guards), + elixir_quote:linify(Line, keep, Body)}; + nil -> + {nil, Args, Guards, Body} + end, - make_struct_available(Kind, Module, Name, Args), - {Name, Arity}. + Arity = length(Args), -%% @on_definition + {File, DefMeta} = + case retrieve_location(Location, ?key(E, module)) of + {F, L} -> + {F, [{line, Line}, {file, {F, L}} | Generated]}; + nil -> + {nil, [{line, Line} | Generated]} + end, -run_on_definition_callbacks(Kind, Line, Module, Name, Args, Guards, Expr, E) -> - case elixir_compiler:get_opt(internal) of - true -> - ok; - _ -> - Env = elixir_env:linify({Line, E}), - Callbacks = 'Elixir.Module':get_attribute(Module, on_definition), - [Mod:Fun(Env, Kind, Name, Args, Guards, Expr) || {Mod, Fun} <- Callbacks] - end. + run_with_location_change(File, E, fun(EL) -> + assert_no_aliases_name(DefMeta, Name, Args, EL), + assert_valid_name(DefMeta, Kind, Name, Args, EL), + store_definition(DefMeta, Kind, DoCheckClauses, Name, Arity, + LinifyArgs, LinifyGuards, LinifyBody, ?key(E, file), EL) + end). -make_struct_available(def, Module, '__struct__', []) -> - case erlang:get(elixir_compiler_pid) of - undefined -> ok; - Pid -> Pid ! {struct_available, Module} - end; -make_struct_available(_, _, _, _) -> - ok. +store_definition(Meta, Kind, CheckClauses, Name, Arity, DefaultsArgs, Guards, Body, File, ER) -> + Module = ?key(ER, module), + Tuple = {Name, Arity}, + {S, E} = env_for_expansion(Kind, Tuple, ER), -%% Retrieve location from meta file or @file, otherwise nil + {Args, Defaults} = unpack_defaults(Kind, Meta, Name, DefaultsArgs, S, E), + Clauses = [elixir_clauses:def(Clause, S, E) || + Clause <- def_to_clauses(Kind, Meta, Args, Guards, Body, E)], -retrieve_location(Line, File, Module) -> - case get_location_attribute(Module) of - nil when not is_binary(File) -> + DefaultsLength = length(Defaults), + elixir_locals:record_defaults(Tuple, Kind, Module, DefaultsLength, Meta), + check_previous_defaults(Meta, Module, Name, Arity, Kind, DefaultsLength, E), + + store_definition(CheckClauses, Kind, Meta, Name, Arity, File, + Module, DefaultsLength, Clauses), + [store_definition(false, Kind, Meta, Name, length(DefaultArgs), File, + Module, 0, [Default]) || {_, DefaultArgs, _, _} = Default <- Defaults], + + run_on_definition_callbacks(Kind, Module, Name, DefaultsArgs, Guards, Body, E), + Tuple. + +env_for_expansion(Kind, Tuple, E) when Kind =:= defmacro; Kind =:= defmacrop -> + S = elixir_env:env_to_ex(E), + {S#elixir_ex{caller=true}, E#{function := Tuple}}; +env_for_expansion(_Kind, Tuple, E) -> + {elixir_env:env_to_ex(E), E#{function := Tuple}}. + +retrieve_location(Location, Module) -> + {Set, _} = elixir_module:data_tables(Module), + case ets:take(Set, file) of + [] when is_tuple(Location) -> + {File, Line} = Location, + {elixir_utils:relative_to_cwd(File), Line}; + [] -> nil; - nil -> - {normalize_location(File), Line}; - X when is_binary(X) -> + [{file, File, _}] when is_binary(File) -> 'Elixir.Module':delete_attribute(Module, file), - {normalize_location(X), 0}; - {X, L} when is_binary(X) andalso is_integer(L) -> + {elixir_utils:relative_to_cwd(File), 0}; + [{file, {File, Line}, _}] when is_binary(File) andalso is_integer(Line) -> 'Elixir.Module':delete_attribute(Module, file), - {normalize_location(X), L} + {elixir_utils:relative_to_cwd(File), Line} end. -get_location_attribute(Module) -> - case elixir_compiler:get_opt(internal) of - true -> nil; - false -> 'Elixir.Module':get_attribute(Module, file) +run_with_location_change(nil, E, Callback) -> + Callback(E); +run_with_location_change(File, #{file := File} = E, Callback) -> + Callback(E); +run_with_location_change(File, E, Callback) -> + elixir_lexical:with_file(File, E, Callback). + +def_to_clauses(_Kind, Meta, Args, [], nil, E) -> + check_args_for_function_head(Meta, Args, E), + []; +def_to_clauses(_Kind, Meta, Args, Guards, [{do, Body}], _E) -> + [{Meta, Args, Guards, Body}]; +def_to_clauses(Kind, Meta, Args, Guards, Body, E) -> + case is_list(Body) andalso lists:keyfind(do, 1, Body) of + {do, _} -> + [{Meta, Args, Guards, {'try', [{origin, Kind} | Meta], [Body]}}]; + false -> + elixir_errors:form_error(Meta, E, elixir_expand, {missing_option, Kind, [do]}) end. -normalize_location(X) -> - elixir_utils:characters_to_list(elixir_utils:relative_to_cwd(X)). - -%% Compile super - -compile_super(Module, true, #{function := Function}) -> - elixir_def_overridable:store(Module, Function, true); -compile_super(_Module, _, _E) -> ok. - -%% Translate the given call and expression given -%% and then store it in memory. - -translate_definition(Kind, Line, Module, Name, Args, Guards, Body, E) when is_integer(Line) -> - Arity = length(Args), - - {EArgs, EGuards, EBody, _} = elixir_exp_clauses:def(fun elixir_def_defaults:expand/2, - Args, Guards, expr_from_body(Line, Body), E), - - Body == nil andalso check_args_for_bodyless_clause(Line, EArgs, E), - - S = elixir_env:env_to_scope(E), - {Unpacked, Defaults} = elixir_def_defaults:unpack(Kind, Name, EArgs, S), - {Clauses, Super} = translate_clause(Body, Line, Kind, Unpacked, EGuards, EBody, S), - - run_on_definition_callbacks(Kind, Line, Module, Name, EArgs, EGuards, EBody, E), - Function = {function, Line, Name, Arity, Clauses}, - {Function, Defaults, Super}. +run_on_definition_callbacks(Kind, Module, Name, Args, Guards, Body, E) -> + {_, Bag} = elixir_module:data_tables(Module), + Callbacks = ets:lookup_element(Bag, {accumulate, on_definition}, 2), + _ = [Mod:Fun(E, Kind, Name, Args, Guards, Body) || {Mod, Fun} <- lists:reverse(Callbacks)], + ok. -translate_clause(nil, _Line, _Kind, _Args, [], _Body, _S) -> - {[], false}; -translate_clause(nil, Line, Kind, _Args, _Guards, _Body, #elixir_scope{file=File}) -> - elixir_errors:form_error(Line, File, ?MODULE, {missing_do, Kind}); -translate_clause(_, Line, Kind, Args, Guards, Body, S) -> - {TClause, TS} = elixir_clauses:clause(Line, - fun elixir_translator:translate_args/2, Args, Body, Guards, true, S), +store_definition(Check, Kind, Meta, Name, Arity, File, Module, Defaults, Clauses) -> + {Set, Bag} = elixir_module:data_tables(Module), + Tuple = {Name, Arity}, + HasBody = Clauses =/= [], - FClause = case is_macro(Kind) of + if + Defaults > 0 -> + ets:insert(Bag, {{default, Name}, Arity, Defaults}); true -> - FArgs = {var, Line, '_@CALLER'}, - MClause = setelement(3, TClause, [FArgs|element(3, TClause)]), - - case TS#elixir_scope.caller of - true -> - FBody = {'match', Line, - {'var', Line, '__CALLER__'}, - elixir_utils:erl_call(Line, elixir_env, linify, [{var, Line, '_@CALLER'}]) - }, - setelement(5, MClause, [FBody|element(5, TClause)]); - false -> - MClause - end; - false -> - TClause + ok end, - {[FClause], TS#elixir_scope.super}. - -expr_from_body(_Line, nil) -> nil; -expr_from_body(_Line, [{do, Expr}]) -> Expr; -expr_from_body(Line, Else) -> {'try', [{line,Line}], [Else]}. - -is_macro(defmacro) -> true; -is_macro(defmacrop) -> true; -is_macro(_) -> false. - -% Unwrap the functions stored in the functions table. -% It returns a list of all functions to be exported, plus the macros, -% and the body of all functions. -unwrap_definitions(Module) -> - Table = table(Module), - CTable = clauses_table(Module), - ets:delete(Table, last), - unwrap_definition(ets:tab2list(Table), CTable, [], [], [], [], [], [], []). - -unwrap_definition([Fun|T], CTable, All, Exports, Private, Def, Defmacro, Functions, Tail) -> - Tuple = element(1, Fun), - Clauses = [Clause || {_, Clause} <- ets:lookup(CTable, Tuple)], - - {NewFun, NewExports, NewPrivate, NewDef, NewDefmacro} = - case Clauses of - [] -> {false, Exports, Private, Def, Defmacro}; - _ -> unwrap_definition(element(2, Fun), Tuple, Fun, Exports, Private, Def, Defmacro) + {MaxDefaults, FirstMeta} = + case ets:lookup(Set, {def, Tuple}) of + [{_, StoredKind, StoredMeta, StoredFile, StoredCheck, {StoredDefaults, LastHasBody, LastDefaults}}] -> + check_valid_kind(Meta, File, Name, Arity, Kind, StoredKind, StoredFile, StoredMeta), + check_valid_defaults(Meta, File, Name, Arity, Kind, Defaults, StoredDefaults, LastDefaults, HasBody, LastHasBody), + (Check and StoredCheck) andalso + check_valid_clause(Meta, File, Name, Arity, Kind, Set, StoredMeta, StoredFile, Clauses), + + {max(Defaults, StoredDefaults), StoredMeta}; + [] -> + ets:insert(Bag, {defs, Tuple}), + {Defaults, Meta} end, - {NewFunctions, NewTail} = case NewFun of - false -> - NewAll = All, - {Functions, Tail}; - _ -> - NewAll = [Tuple|All], - function_for_stored_definition(NewFun, Clauses, Functions, Tail) - end, - - unwrap_definition(T, CTable, NewAll, NewExports, NewPrivate, - NewDef, NewDefmacro, NewFunctions, NewTail); -unwrap_definition([], _CTable, All, Exports, Private, Def, Defmacro, Functions, Tail) -> - {All, Exports, Private, ordsets:from_list(Def), - ordsets:from_list(Defmacro), lists:reverse(Tail ++ Functions)}. - -unwrap_definition(def, Tuple, Fun, Exports, Private, Def, Defmacro) -> - {Fun, [Tuple|Exports], Private, [Tuple|Def], Defmacro}; -unwrap_definition(defmacro, {Name, Arity} = Tuple, Fun, Exports, Private, Def, Defmacro) -> - Macro = {elixir_utils:macro_name(Name), Arity + 1}, - {setelement(1, Fun, Macro), [Macro|Exports], Private, Def, [Tuple|Defmacro]}; -unwrap_definition(defp, Tuple, Fun, Exports, Private, Def, Defmacro) -> - %% {Name, Arity}, Kind, Line, Check, Defaults - Info = {Tuple, defp, element(3, Fun), element(5, Fun), element(7, Fun)}, - {Fun, Exports, [Info|Private], Def, Defmacro}; -unwrap_definition(defmacrop, Tuple, Fun, Exports, Private, Def, Defmacro) -> - %% {Name, Arity}, Kind, Line, Check, Defaults - Info = {Tuple, defmacrop, element(3, Fun), element(5, Fun), element(7, Fun)}, - {false, Exports, [Info|Private], Def, Defmacro}. - -%% Helpers - -function_for_stored_definition({{Name,Arity}, _, Line, _, _, nil, _}, Clauses, Functions, Tail) -> - {[{function, Line, Name, Arity, Clauses}|Functions], Tail}; - -function_for_stored_definition({{Name,Arity}, _, Line, _, _, Location, _}, Clauses, Functions, Tail) -> - {Functions, [ - {function, Line, Name, Arity, Clauses}, - {attribute, Line, file, Location} | Tail - ]}. - -default_function_for(Kind, Name, {clause, Line, Args, _Guards, _Exprs} = Clause) - when Kind == defmacro; Kind == defmacrop -> - {function, Line, Name, length(Args) - 1, [Clause]}; - -default_function_for(_, Name, {clause, Line, Args, _Guards, _Exprs} = Clause) -> - {function, Line, Name, length(Args), [Clause]}. - -%% Store each definition in the table. -%% This function also checks and emit warnings in case -%% the kind, of the visibility of the function changes. - -store_each(Check, Kind, File, Location, Table, CTable, Defaults, {function, Line, Name, Arity, Clauses}) -> - Tuple = {Name, Arity}, - case ets:lookup(Table, Tuple) of - [{Tuple, StoredKind, StoredLine, StoredFile, StoredCheck, StoredLocation, StoredDefaults}] -> - FinalLine = StoredLine, - FinalLocation = StoredLocation, - FinalDefaults = max(Defaults, StoredDefaults), - check_valid_kind(Line, File, Name, Arity, Kind, StoredKind), - (Check and StoredCheck) andalso - check_valid_clause(Line, File, Name, Arity, Kind, Table, StoredLine, StoredFile), - check_valid_defaults(Line, File, Name, Arity, Kind, Defaults, StoredDefaults); - [] -> - FinalLine = Line, - FinalLocation = Location, - FinalDefaults = Defaults - end, - Check andalso ets:insert(Table, {last, {Name, Arity}}), - ets:insert(CTable, [{Tuple, Clause} || Clause <- Clauses ]), - ets:insert(Table, {Tuple, Kind, FinalLine, File, Check, FinalLocation, FinalDefaults}). + Check andalso ets:insert(Set, {?last_def, Tuple}), + ets:insert(Bag, [{{clauses, Tuple}, Clause} || Clause <- Clauses]), + ets:insert(Set, {{def, Tuple}, Kind, FirstMeta, File, Check, {MaxDefaults, HasBody, Defaults}}). + +%% Handling of defaults + +unpack_defaults(Kind, Meta, Name, Args, S, E) -> + {Expanded, #elixir_ex{unused={_, VersionOffset}}} = expand_defaults(Args, S, E#{context := nil}, []), + unpack_expanded(Kind, Meta, Name, Expanded, VersionOffset, [], []). + +unpack_expanded(Kind, Meta, Name, [{'\\\\', DefaultMeta, [Expr, _]} | T] = List, VersionOffset, Acc, Clauses) -> + Base = match_defaults(Acc, length(Acc) + VersionOffset, []), + {Args, Invoke} = extract_defaults(List, length(Base), [], []), + Clause = {Meta, Base ++ Args, [], {super, [{super, {Kind, Name}} | DefaultMeta], Base ++ Invoke}}, + unpack_expanded(Kind, Meta, Name, T, VersionOffset, [Expr | Acc], [Clause | Clauses]); +unpack_expanded(Kind, Meta, Name, [H | T], VersionOffset, Acc, Clauses) -> + unpack_expanded(Kind, Meta, Name, T, VersionOffset, [H | Acc], Clauses); +unpack_expanded(_Kind, _Meta, _Name, [], _VersionOffset, Acc, Clauses) -> + {lists:reverse(Acc), lists:reverse(Clauses)}. + +expand_defaults([{'\\\\', Meta, [Expr, Default]} | Args], S, E, Acc) -> + {ExpandedDefault, SE, _} = elixir_expand:expand(Default, S, E), + expand_defaults(Args, SE, E, [{'\\\\', Meta, [Expr, ExpandedDefault]} | Acc]); +expand_defaults([Arg | Args], S, E, Acc) -> + expand_defaults(Args, S, E, [Arg | Acc]); +expand_defaults([], S, _E, Acc) -> + {lists:reverse(Acc), S}. + +extract_defaults([{'\\\\', _, [_Expr, Default]} | T], Counter, NewArgs, NewInvoke) -> + extract_defaults(T, Counter, NewArgs, [Default | NewInvoke]); +extract_defaults([_ | T], Counter, NewArgs, NewInvoke) -> + H = default_var(Counter), + extract_defaults(T, Counter + 1, [H | NewArgs], [H | NewInvoke]); +extract_defaults([], _Counter, NewArgs, NewInvoke) -> + {lists:reverse(NewArgs), lists:reverse(NewInvoke)}. + +match_defaults([], _Counter, Acc) -> + Acc; +match_defaults([_ | T], Counter, Acc) -> + NewCounter = Counter - 1, + match_defaults(T, NewCounter, [default_var(NewCounter) | Acc]). + +default_var(Counter) -> + {list_to_atom([$x | integer_to_list(Counter)]), [{generated, true}, {version, Counter}], ?var_context}. %% Validations -check_valid_kind(_Line, _File, _Name, _Arity, Kind, Kind) -> []; -check_valid_kind(Line, File, Name, Arity, Kind, StoredKind) -> - elixir_errors:form_error(Line, File, ?MODULE, - {changed_kind, {Name, Arity, StoredKind, Kind}}). +check_valid_kind(_Meta, _File, _Name, _Arity, Kind, Kind, _StoredFile, _StoredMeta) -> ok; +check_valid_kind(Meta, File, Name, Arity, Kind, StoredKind, StoredFile, StoredMeta) -> + elixir_errors:form_error(Meta, File, ?MODULE, + {changed_kind, {Name, Arity, StoredKind, Kind, StoredFile, ?line(StoredMeta)}}). -check_valid_clause(Line, File, Name, Arity, Kind, Table, StoredLine, StoredFile) -> - case ets:lookup_element(Table, last, 2) of - {Name,Arity} -> []; - [] -> []; +check_valid_clause(Meta, File, Name, Arity, Kind, Set, StoredMeta, StoredFile, Clauses) -> + case ets:lookup_element(Set, ?last_def, 2) of + none -> + ok; + {Name, Arity} when Clauses == [] -> + elixir_errors:form_warn(Meta, File, ?MODULE, + {late_function_head, {Kind, Name, Arity}}); + {Name, Arity} -> + ok; + {Name, _} -> + Relative = elixir_utils:relative_to_cwd(StoredFile), + elixir_errors:form_warn(Meta, File, ?MODULE, + {ungrouped_name, {Kind, Name, Arity, ?line(StoredMeta), Relative}}); _ -> Relative = elixir_utils:relative_to_cwd(StoredFile), - elixir_errors:handle_file_warning(File, {Line, ?MODULE, - {ungrouped_clause, {Kind, Name, Arity, StoredLine, Relative}}}) + elixir_errors:form_warn(Meta, File, ?MODULE, + {ungrouped_arity, {Kind, Name, Arity, ?line(StoredMeta), Relative}}) end. -check_valid_defaults(_Line, _File, _Name, _Arity, _Kind, 0, _) -> []; -check_valid_defaults(Line, File, Name, Arity, Kind, _, 0) -> - elixir_errors:handle_file_warning(File, {Line, ?MODULE, {out_of_order_defaults, {Kind, Name, Arity}}}); -check_valid_defaults(Line, File, Name, Arity, Kind, _, _) -> - elixir_errors:form_error(Line, File, ?MODULE, {clauses_with_defaults, {Kind, Name, Arity}}). +% Clause with defaults after clause with defaults +check_valid_defaults(Meta, File, Name, Arity, Kind, Defaults, StoredDefaults, _, _, _) + when Defaults > 0, StoredDefaults > 0 -> + elixir_errors:form_error(Meta, File, ?MODULE, {duplicate_defaults, {Kind, Name, Arity}}); +% Clause with defaults after clause without defaults +check_valid_defaults(Meta, File, Name, Arity, Kind, Defaults, 0, _, _, _) when Defaults > 0 -> + elixir_errors:form_warn(Meta, File, ?MODULE, {mixed_defaults, {Kind, Name, Arity}}); +% Clause without defaults directly after clause with defaults (bodiless does not count) +check_valid_defaults(Meta, File, Name, Arity, Kind, 0, _, LastDefaults, true, true) when LastDefaults > 0 -> + elixir_errors:form_warn(Meta, File, ?MODULE, {mixed_defaults, {Kind, Name, Arity}}); +% Clause without defaults +check_valid_defaults(_Meta, _File, _Name, _Arity, _Kind, 0, _StoredDefaults, _LastDefaults, _HasBody, _LastHasBody) -> + ok. + +check_args_for_function_head(Meta, Args, E) -> + [begin + elixir_errors:form_error(Meta, E, ?MODULE, invalid_args_for_function_head) + end || Arg <- Args, invalid_arg(Arg)]. + +invalid_arg({Name, _, Kind}) when is_atom(Name), is_atom(Kind) -> false; +invalid_arg(_) -> true. -check_previous_defaults(Table, Line, Name, Arity, Kind, Defaults, E) -> - Matches = ets:match(Table, {{Name, '$2'}, '$1', '_', '_', '_', '_', '$3'}), - [ begin - elixir_errors:form_error(Line, ?m(E, file), ?MODULE, - {defs_with_defaults, Name, {Kind, Arity}, {K, A}}) - end || [K, A, D] <- Matches, A /= Arity, D /= 0, defaults_conflict(A, D, Arity, Defaults)]. +check_previous_defaults(Meta, Module, Name, Arity, Kind, Defaults, E) -> + {_Set, Bag} = elixir_module:data_tables(Module), + Matches = ets:lookup(Bag, {default, Name}), + [begin + elixir_errors:form_error(Meta, E, ?MODULE, + {defs_with_defaults, Kind, Name, Arity, A}) + end || {_, A, D} <- Matches, A /= Arity, D /= 0, defaults_conflict(A, D, Arity, Defaults)]. defaults_conflict(A, D, Arity, Defaults) -> ((Arity >= (A - D)) andalso (Arity < A)) orelse ((A >= (Arity - Defaults)) andalso (A < Arity)). -check_args_for_bodyless_clause(Line, Args, E) -> - [ begin - elixir_errors:form_error(Line, ?m(E, file), ?MODULE, invalid_args_for_bodyless_clause) - end || Arg <- Args, invalid_arg(Arg) ]. - -invalid_arg({Name, _, Kind}) when is_atom(Name), is_atom(Kind) -> - false; -invalid_arg({'\\\\', _, [{Name, _, Kind}, _]}) when is_atom(Name), is_atom(Kind) -> - false; -invalid_arg(_) -> - true. - -assert_no_aliases_name(Line, '__aliases__', [Atom], #{file := File}) when is_atom(Atom) -> - elixir_errors:form_error(Line, File, ?MODULE, {no_alias, Atom}); - +assert_no_aliases_name(Meta, '__aliases__', [Atom], #{file := File}) when is_atom(Atom) -> + elixir_errors:form_error(Meta, File, ?MODULE, {no_alias, Atom}); assert_no_aliases_name(_Meta, _Aliases, _Args, _S) -> ok. -%% Format errors - -format_error({no_module,{Kind,Name,Arity}}) -> - io_lib:format("cannot define function outside module, invalid scope for ~ts ~ts/~B", [Kind, Name, Arity]); - -format_error({defs_with_defaults, Name, {Kind, Arity}, {K, A}}) when Arity > A -> - io_lib:format("~ts ~ts/~B defaults conflicts with ~ts ~ts/~B", - [Kind, Name, Arity, K, Name, A]); - -format_error({defs_with_defaults, Name, {Kind, Arity}, {K, A}}) when Arity < A -> - io_lib:format("~ts ~ts/~B conflicts with defaults from ~ts ~ts/~B", - [Kind, Name, Arity, K, Name, A]); +assert_valid_name(Meta, Kind, '__info__', [_], #{file := File}) -> + elixir_errors:form_error(Meta, File, ?MODULE, {'__info__', Kind}); +assert_valid_name(Meta, Kind, 'module_info', [], #{file := File}) -> + elixir_errors:form_error(Meta, File, ?MODULE, {module_info, Kind, 0}); +assert_valid_name(Meta, Kind, 'module_info', [_], #{file := File}) -> + elixir_errors:form_error(Meta, File, ?MODULE, {module_info, Kind, 1}); +assert_valid_name(Meta, Kind, is_record, [_, _], #{file := File}) when Kind == defp; Kind == def -> + elixir_errors:form_error(Meta, File, ?MODULE, {is_record, Kind}); +assert_valid_name(_Meta, _Kind, _Name, _Args, _S) -> + ok. -format_error({clauses_with_defaults,{Kind,Name,Arity}}) -> - io_lib:format("~ts ~ts/~B has default values and multiple clauses, " - "define a function head with the defaults", [Kind, Name, Arity]); +%% Format errors -format_error({out_of_order_defaults,{Kind,Name,Arity}}) -> - io_lib:format("clause with defaults should be the first clause in ~ts ~ts/~B", [Kind, Name, Arity]); +format_error({function_head, Kind, {Name, Arity}}) -> + io_lib:format("implementation not provided for predefined ~ts ~ts/~B", [Kind, Name, Arity]); -format_error({ungrouped_clause,{Kind,Name,Arity,OrigLine,OrigFile}}) -> - io_lib:format("clauses for the same ~ts should be grouped together, ~ts ~ts/~B was previously defined (~ts:~B)", - [Kind, Kind, Name, Arity, OrigFile, OrigLine]); +format_error({no_module, {Kind, Name, Arity}}) -> + io_lib:format("cannot define function outside module, invalid scope for ~ts ~ts/~B", [Kind, Name, Arity]); -format_error({changed_kind,{Name,Arity,Previous,Current}}) -> - io_lib:format("~ts ~ts/~B already defined as ~ts", [Current, Name, Arity, Previous]); +format_error({defs_with_defaults, Kind, Name, Arity, A}) when Arity > A -> + io_lib:format("~ts ~ts/~B defaults conflicts with ~ts/~B", + [Kind, Name, Arity, Name, A]); + +format_error({defs_with_defaults, Kind, Name, Arity, A}) when Arity < A -> + io_lib:format("~ts ~ts/~B conflicts with defaults from ~ts/~B", + [Kind, Name, Arity, Name, A]); + +format_error({duplicate_defaults, {Kind, Name, Arity}}) -> + io_lib:format( + "~ts ~ts/~B defines defaults multiple times. " + "Elixir allows defaults to be declared once per definition. Instead of:\n" + "\n" + " def foo(:first_clause, b \\\\ :default) do ... end\n" + " def foo(:second_clause, b \\\\ :default) do ... end\n" + "\n" + "one should write:\n" + "\n" + " def foo(a, b \\\\ :default)\n" + " def foo(:first_clause, b) do ... end\n" + " def foo(:second_clause, b) do ... end\n", + [Kind, Name, Arity]); + +format_error({mixed_defaults, {Kind, Name, Arity}}) -> + io_lib:format( + "~ts ~ts/~B has multiple clauses and also declares default values. " + "In such cases, the default values should be defined in a header. Instead of:\n" + "\n" + " def foo(:first_clause, b \\\\ :default) do ... end\n" + " def foo(:second_clause, b) do ... end\n" + "\n" + "one should write:\n" + "\n" + " def foo(a, b \\\\ :default)\n" + " def foo(:first_clause, b) do ... end\n" + " def foo(:second_clause, b) do ... end\n", + [Kind, Name, Arity]); + +format_error({ungrouped_name, {Kind, Name, Arity, OrigLine, OrigFile}}) -> + io_lib:format("clauses with the same name should be grouped together, \"~ts ~ts/~B\" was previously defined (~ts:~B)", + [Kind, Name, Arity, OrigFile, OrigLine]); + +format_error({ungrouped_arity, {Kind, Name, Arity, OrigLine, OrigFile}}) -> + io_lib:format("clauses with the same name and arity (number of arguments) should be grouped together, \"~ts ~ts/~B\" was previously defined (~ts:~B)", + [Kind, Name, Arity, OrigFile, OrigLine]); + +format_error({late_function_head, {Kind, Name, Arity}}) -> + io_lib:format("function head for ~ts ~ts/~B must come at the top of its direct implementation. Instead of:\n" + "\n" + " def add(a, b), do: a + b\n" + " def add(a, b)\n" + "\n" + "one should write:\n" + "\n" + " def add(a, b)\n" + " def add(a, b), do: a + b\n", + [Kind, Name, Arity]); + +format_error({changed_kind, {Name, Arity, Previous, Current, OrigFile, OrigLine}}) -> + OrigFileRelative = elixir_utils:relative_to_cwd(OrigFile), + io_lib:format("~ts ~ts/~B already defined as ~ts in ~ts:~B", [Current, Name, Arity, Previous, OrigFileRelative, OrigLine]); format_error({no_alias, Atom}) -> io_lib:format("function names should start with lowercase characters or underscore, invalid name ~ts", [Atom]); @@ -388,8 +451,23 @@ format_error({no_alias, Atom}) -> format_error({invalid_def, Kind, NameAndArgs}) -> io_lib:format("invalid syntax in ~ts ~ts", [Kind, 'Elixir.Macro':to_string(NameAndArgs)]); -format_error(invalid_args_for_bodyless_clause) -> - "can use only variables and \\\\ as arguments of bodyless clause"; - -format_error({missing_do, Kind}) -> - io_lib:format("missing do keyword in ~ts", [Kind]). +format_error(invalid_args_for_function_head) -> + "only variables and \\\\ are allowed as arguments in function head.\n" + "\n" + "If you did not intend to define a function head, make sure your function " + "definition has the proper syntax by wrapping the arguments in parentheses " + "and using the do instruction accordingly:\n\n" + " def add(a, b), do: a + b\n\n" + " def add(a, b) do\n" + " a + b\n" + " end\n"; + +format_error({'__info__', Kind}) -> + io_lib:format("cannot define ~ts __info__/1 as it is automatically defined by Elixir", [Kind]); + +format_error({module_info, Kind, Arity}) -> + io_lib:format("cannot define ~ts module_info/~B as it is automatically defined by Erlang", [Kind, Arity]); + +format_error({is_record, Kind}) -> + io_lib:format("cannot define ~ts is_record/2 due to compatibility " + "issues with the Erlang compiler (it is a known limitation)", [Kind]). diff --git a/lib/elixir/src/elixir_def_defaults.erl b/lib/elixir/src/elixir_def_defaults.erl deleted file mode 100644 index 0b0a2683c55..00000000000 --- a/lib/elixir/src/elixir_def_defaults.erl +++ /dev/null @@ -1,73 +0,0 @@ -% Handle default clauses for function definitions. --module(elixir_def_defaults). --export([expand/2, unpack/4]). --include("elixir.hrl"). - -expand(Args, E) -> - lists:mapfoldl(fun - ({'\\\\', Meta, [Left, Right]}, Acc) -> - {ELeft, EL} = elixir_exp:expand(Left, Acc), - {ERight, _} = elixir_exp:expand(Right, Acc#{context := nil}), - {{'\\\\', Meta, [ELeft, ERight]}, EL}; - (Left, Acc) -> - elixir_exp:expand(Left, Acc) - end, E, Args). - -unpack(Kind, Name, Args, S) -> - unpack_each(Kind, Name, Args, [], [], S). - -%% Helpers - -%% Unpack default from given args. -%% Returns the given arguments without their default -%% clauses and a list of clauses for the default calls. -unpack_each(Kind, Name, [{'\\\\', DefMeta, [Expr, _]}|T] = List, Acc, Clauses, S) -> - Base = wrap_kind(Kind, build_match(Acc, [])), - {Args, Invoke} = extract_defaults(List, length(Base), [], []), - - {DefArgs, SA} = elixir_clauses:match(fun elixir_translator:translate_args/2, Base ++ Args, S), - {DefInvoke, _} = elixir_translator:translate_args(Base ++ Invoke, SA), - - Line = ?line(DefMeta), - - Call = {call, Line, - {atom, Line, name_for_kind(Kind, Name)}, - DefInvoke - }, - - Clause = {clause, Line, DefArgs, [], [Call]}, - unpack_each(Kind, Name, T, [Expr|Acc], [Clause|Clauses], S); - -unpack_each(Kind, Name, [H|T], Acc, Clauses, S) -> - unpack_each(Kind, Name, T, [H|Acc], Clauses, S); - -unpack_each(_Kind, _Name, [], Acc, Clauses, _S) -> - {lists:reverse(Acc), lists:reverse(Clauses)}. - -% Extract default values from args following the current default clause. - -extract_defaults([{'\\\\', _, [_Expr, Default]}|T], Counter, NewArgs, NewInvoke) -> - extract_defaults(T, Counter, NewArgs, [Default|NewInvoke]); - -extract_defaults([_|T], Counter, NewArgs, NewInvoke) -> - H = {elixir_utils:atom_concat(["x", Counter]), [], nil}, - extract_defaults(T, Counter + 1, [H|NewArgs], [H|NewInvoke]); - -extract_defaults([], _Counter, NewArgs, NewInvoke) -> - {lists:reverse(NewArgs), lists:reverse(NewInvoke)}. - -% Build matches for all the previous argument until the current default clause. - -build_match([], Acc) -> Acc; - -build_match([_|T], Acc) -> - Var = {elixir_utils:atom_concat(["x", length(T)]), [], nil}, - build_match(T, [Var|Acc]). - -% Given the invoked function name based on the kind - -wrap_kind(Kind, Args) when Kind == defmacro; Kind == defmacrop -> [{c, [], nil}|Args]; -wrap_kind(_Kind, Args) -> Args. - -name_for_kind(Kind, Name) when Kind == defmacro; Kind == defmacrop -> elixir_utils:macro_name(Name); -name_for_kind(_Kind, Name) -> Name. \ No newline at end of file diff --git a/lib/elixir/src/elixir_def_overridable.erl b/lib/elixir/src/elixir_def_overridable.erl deleted file mode 100644 index f0cf2b1a7a6..00000000000 --- a/lib/elixir/src/elixir_def_overridable.erl +++ /dev/null @@ -1,75 +0,0 @@ -% Holds the logic responsible for defining overridable functions and handling super. --module(elixir_def_overridable). --export([store_pending/1, ensure_defined/4, - name/2, store/3, format_error/1]). --include("elixir.hrl"). - -overridable(Module) -> - ets:lookup_element(elixir_module:data_table(Module), '__overridable', 2). - -overridable(Module, Value) -> - ets:insert(elixir_module:data_table(Module), {'__overridable', Value}). - -%% Check if an overridable function is defined. - -ensure_defined(Meta, Module, Tuple, S) -> - Overridable = overridable(Module), - case orddict:find(Tuple, Overridable) of - {ok, {_, _, _, _}} -> ok; - _ -> elixir_errors:form_error(Meta, S#elixir_scope.file, ?MODULE, {no_super, Module, Tuple}) - end. - -%% Gets the name based on the function and stored overridables - -name(Module, Function) -> - name(Module, Function, overridable(Module)). - -name(_Module, {Name, _} = Function, Overridable) -> - {Count, _, _, _} = orddict:fetch(Function, Overridable), - elixir_utils:atom_concat([Name, " (overridable ", Count, ")"]). - -%% Store - -store(Module, Function, GenerateName) -> - Overridable = overridable(Module), - case orddict:fetch(Function, Overridable) of - {_Count, _Clause, _Neighbours, true} -> ok; - {Count, Clause, Neighbours, false} -> - overridable(Module, orddict:store(Function, {Count, Clause, Neighbours, true}, Overridable)), - {{{Name, Arity}, Kind, Line, File, _Check, Location, Defaults}, Clauses} = Clause, - - {FinalKind, FinalName} = case GenerateName of - true -> {defp, name(Module, Function, Overridable)}; - false -> {Kind, Name} - end, - - case code:is_loaded('Elixir.Module.LocalsTracker') of - {_, _} -> - 'Elixir.Module.LocalsTracker':reattach(Module, Kind, {Name, Arity}, Neighbours); - _ -> - ok - end, - - Def = {function, Line, FinalName, Arity, Clauses}, - elixir_def:store_each(false, FinalKind, File, Location, - elixir_def:table(Module), elixir_def:clauses_table(Module), Defaults, Def) - end. - -%% Store pending declarations that were not manually made concrete. - -store_pending(Module) -> - [store(Module, X, false) || {X, {_, _, _, false}} <- overridable(Module), - not 'Elixir.Module':'defines?'(Module, X)]. - -%% Error handling - -format_error({no_super, Module, {Name, Arity}}) -> - Bins = [format_fa(X) || {X, {_, _, _, _}} <- overridable(Module)], - Joined = 'Elixir.Enum':join(Bins, <<", ">>), - io_lib:format("no super defined for ~ts/~B in module ~ts. Overridable functions available are: ~ts", - [Name, Arity, elixir_aliases:inspect(Module), Joined]). - -format_fa({Name, Arity}) -> - A = atom_to_binary(Name, utf8), - B = integer_to_binary(Arity), - << A/binary, $/, B/binary >>. \ No newline at end of file diff --git a/lib/elixir/src/elixir_dispatch.erl b/lib/elixir/src/elixir_dispatch.erl index e4cbac4610f..a51b1eb6149 100644 --- a/lib/elixir/src/elixir_dispatch.erl +++ b/lib/elixir/src/elixir_dispatch.erl @@ -2,58 +2,62 @@ %% This module access the information stored on the scope %% by elixir_import and therefore assumes it is normalized (ordsets) -module(elixir_dispatch). --export([dispatch_import/5, dispatch_require/6, +-export([dispatch_import/6, dispatch_require/7, require_function/5, import_function/4, - expand_import/5, expand_require/5, + expand_import/7, expand_require/6, default_functions/0, default_macros/0, default_requires/0, - find_import/4, format_error/1]). + find_import/4, find_imports/4, format_error/1]). -include("elixir.hrl"). -import(ordsets, [is_element/2]). - --define(atom, 'Elixir.Atom'). --define(float, 'Elixir.Float'). --define(io, 'Elixir.IO'). --define(integer, 'Elixir.Integer'). -define(kernel, 'Elixir.Kernel'). --define(list, 'Elixir.List'). --define(map, 'Elixir.Map'). --define(node, 'Elixir.Node'). --define(process, 'Elixir.Process'). --define(string, 'Elixir.String'). --define(system, 'Elixir.System'). --define(tuple, 'Elixir.Tuple'). +-define(application, 'Elixir.Application'). default_functions() -> [{?kernel, elixir_imported_functions()}]. default_macros() -> [{?kernel, elixir_imported_macros()}]. default_requires() -> - ['Elixir.Kernel', 'Elixir.Kernel.Typespec']. + ['Elixir.Application', 'Elixir.Kernel', 'Elixir.Kernel.Typespec']. +%% This is used by elixir_quote. Note we don't record the +%% import locally because at that point there is no +%% ambiguity. find_import(Meta, Name, Arity, E) -> Tuple = {Name, Arity}, case find_dispatch(Meta, Tuple, [], E) of {function, Receiver} -> - elixir_lexical:record_import(Receiver, ?m(E, lexical_tracker)), - elixir_locals:record_import(Tuple, Receiver, ?m(E, module), ?m(E, function)), + elixir_env:trace({imported_function, Meta, Receiver, Name, Arity}, E), Receiver; {macro, Receiver} -> - elixir_lexical:record_import(Receiver, ?m(E, lexical_tracker)), - elixir_locals:record_import(Tuple, Receiver, ?m(E, module), ?m(E, function)), + elixir_env:trace({imported_macro, Meta, Receiver, Name, Arity}, E), Receiver; _ -> false end. +find_imports(Meta, Name, Arity, E) -> + Tuple = {Name, Arity}, + + case find_dispatch(Meta, Tuple, [], E) of + {function, Receiver} -> + elixir_env:trace({imported_function, Meta, Receiver, Name, Arity}, E), + [{Receiver, Arity}]; + {macro, Receiver} -> + elixir_env:trace({imported_macro, Meta, Receiver, Name, Arity}, E), + [{Receiver, Arity}]; + _ -> + [] + end. + %% Function retrieval import_function(Meta, Name, Arity, E) -> Tuple = {Name, Arity}, case find_dispatch(Meta, Tuple, [], E) of {function, Receiver} -> - elixir_lexical:record_import(Receiver, ?m(E, lexical_tracker)), - elixir_locals:record_import(Tuple, Receiver, ?m(E, module), ?m(E, function)), + elixir_env:trace({imported_function, Meta, Receiver, Name, Arity}, E), + elixir_locals:record_import(Tuple, Receiver, ?key(E, module), ?key(E, function)), remote_function(Meta, Receiver, Name, Arity, E); {macro, _Receiver} -> false; @@ -61,195 +65,195 @@ import_function(Meta, Name, Arity, E) -> require_function(Meta, Receiver, Name, Arity, E); false -> case elixir_import:special_form(Name, Arity) of - true -> false; + true -> + false; false -> - elixir_locals:record_local(Tuple, ?m(E, module), ?m(E, function)), - {local, Name, Arity} + Function = ?key(E, function), + + case (Function /= nil) andalso (Function /= Tuple) andalso + elixir_def:local_for(?key(E, module), Name, Arity, [defmacro, defmacrop]) of + false -> + elixir_env:trace({local_function, Meta, Name, Arity}, E), + elixir_locals:record_local(Tuple, ?key(E, module), ?key(E, function), Meta, false), + {local, Name, Arity}; + _ -> + false + end end end. require_function(Meta, Receiver, Name, Arity, E) -> - case is_element({Name, Arity}, get_optional_macros(Receiver)) of + Required = is_element(Receiver, ?key(E, requires)), + + case is_macro({Name, Arity}, Receiver, Required) of true -> false; - false -> remote_function(Meta, Receiver, Name, Arity, E) + false -> + elixir_env:trace({remote_function, Meta, Receiver, Name, Arity}, E), + remote_function(Meta, Receiver, Name, Arity, E) end. remote_function(Meta, Receiver, Name, Arity, E) -> - check_deprecation(Meta, Receiver, Name, Arity, E), + check_deprecated(Meta, function, Receiver, Name, Arity, E), - elixir_lexical:record_remote(Receiver, ?m(E, lexical_tracker)), - case inline(Receiver, Name, Arity) of + case elixir_rewrite:inline(Receiver, Name, Arity) of {AR, AN} -> {remote, AR, AN, Arity}; false -> {remote, Receiver, Name, Arity} end. %% Dispatches -dispatch_import(Meta, Name, Args, E, Callback) -> +dispatch_import(Meta, Name, Args, S, E, Callback) -> Arity = length(Args), - case expand_import(Meta, {Name, Arity}, Args, E, []) of + case expand_import(Meta, {Name, Arity}, Args, S, E, [], false) of {ok, Receiver, Quoted} -> - expand_quoted(Meta, Receiver, Name, Arity, Quoted, E); + expand_quoted(Meta, Receiver, Name, Arity, Quoted, S, E); {ok, Receiver, NewName, NewArgs} -> - elixir_exp:expand({{'.', [], [Receiver, NewName]}, Meta, NewArgs}, E); + elixir_expand:expand({{'.', Meta, [Receiver, NewName]}, Meta, NewArgs}, S, E); error -> Callback() end. -dispatch_require(Meta, Receiver, Name, Args, E, Callback) when is_atom(Receiver) -> +dispatch_require(Meta, Receiver, Name, Args, S, E, Callback) when is_atom(Receiver) -> Arity = length(Args), - case rewrite(Receiver, Name, Args, Arity) of - {ok, AR, AN, AA} -> - Callback(AR, AN, AA); + case elixir_rewrite:inline(Receiver, Name, Arity) of + {AR, AN} -> + Callback(AR, AN, Args); false -> - case expand_require(Meta, Receiver, {Name, Arity}, Args, E) of - {ok, Receiver, Quoted} -> expand_quoted(Meta, Receiver, Name, Arity, Quoted, E); + case expand_require(Meta, Receiver, {Name, Arity}, Args, S, E) of + {ok, Receiver, Quoted} -> expand_quoted(Meta, Receiver, Name, Arity, Quoted, S, E); error -> Callback(Receiver, Name, Args) end end; -dispatch_require(_Meta, Receiver, Name, Args, _E, Callback) -> +dispatch_require(_Meta, Receiver, Name, Args, _S, _E, Callback) -> Callback(Receiver, Name, Args). %% Macros expansion -expand_import(Meta, {Name, Arity} = Tuple, Args, E, Extra) -> - Module = ?m(E, module), +expand_import(Meta, {Name, Arity} = Tuple, Args, S, E, Extra, External) -> + Module = ?key(E, module), + Function = ?key(E, function), Dispatch = find_dispatch(Meta, Tuple, Extra, E), - Function = ?m(E, function), - Local = (Function /= nil) andalso (Function /= Tuple) andalso - elixir_locals:macro_for(Module, Name, Arity), case Dispatch of - %% In case it is an import, we dispatch the import. {import, _} -> - do_expand_import(Meta, Tuple, Args, Module, E, Dispatch); - - %% There is a local and an import. This is a conflict unless - %% the receiver is the same as module (happens on bootstrap). - {_, Receiver} when Local /= false, Receiver /= Module -> - Error = {macro_conflict, {Receiver, Name, Arity}}, - elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, Error); - - %% There is no local. Dispatch the import. - _ when Local == false -> - do_expand_import(Meta, Tuple, Args, Module, E, Dispatch); - - %% Dispatch to the local. + do_expand_import(Meta, Tuple, Args, Module, S, E, Dispatch); _ -> - elixir_locals:record_local(Tuple, Module, Function), - {ok, Module, expand_macro_fun(Meta, Local(), Module, Name, Args, E)} + AllowLocals = External orelse ((Function /= nil) andalso (Function /= Tuple)), + Local = AllowLocals andalso + elixir_def:local_for(Module, Name, Arity, [defmacro, defmacrop]), + + case Dispatch of + %% There is a local and an import. This is a conflict unless + %% the receiver is the same as module (happens on bootstrap). + {_, Receiver} when Local /= false, Receiver /= Module -> + Error = {macro_conflict, {Receiver, Name, Arity}}, + elixir_errors:form_error(Meta, E, ?MODULE, Error); + + %% There is no local. Dispatch the import. + _ when Local == false -> + do_expand_import(Meta, Tuple, Args, Module, S, E, Dispatch); + + %% Dispatch to the local. + _ -> + elixir_env:trace({local_macro, Meta, Name, Arity}, E), + elixir_locals:record_local(Tuple, Module, Function, Meta, true), + {ok, Module, expand_macro_fun(Meta, Local, Module, Name, Args, S, E)} + end end. -do_expand_import(Meta, {Name, Arity} = Tuple, Args, Module, E, Result) -> +do_expand_import(Meta, {Name, Arity} = Tuple, Args, Module, S, E, Result) -> case Result of {function, Receiver} -> - elixir_lexical:record_import(Receiver, ?m(E, lexical_tracker)), - elixir_locals:record_import(Tuple, Receiver, Module, ?m(E, function)), - - case rewrite(Receiver, Name, Args, Arity) of - {ok, _, _, _} = Res -> Res; - false -> {ok, Receiver, Name, Args} - end; + elixir_env:trace({imported_function, Meta, Receiver, Name, Arity}, E), + elixir_locals:record_import(Tuple, Receiver, Module, ?key(E, function)), + {ok, Receiver, Name, Args}; {macro, Receiver} -> - check_deprecation(Meta, Receiver, Name, Arity, E), - elixir_lexical:record_import(Receiver, ?m(E, lexical_tracker)), - elixir_locals:record_import(Tuple, Receiver, Module, ?m(E, function)), - {ok, Receiver, expand_macro_named(Meta, Receiver, Name, Arity, Args, E)}; + check_deprecated(Meta, macro, Receiver, Name, Arity, E), + elixir_env:trace({imported_macro, Meta, Receiver, Name, Arity}, E), + elixir_locals:record_import(Tuple, Receiver, Module, ?key(E, function)), + {ok, Receiver, expand_macro_named(Meta, Receiver, Name, Arity, Args, S, E)}; {import, Receiver} -> - case expand_require([{require,false}|Meta], Receiver, Tuple, Args, E) of + case expand_require([{required, true} | Meta], Receiver, Tuple, Args, S, E) of {ok, _, _} = Response -> Response; error -> {ok, Receiver, Name, Args} end; false when Module == ?kernel -> - case rewrite(Module, Name, Args, Arity) of - {ok, _, _, _} = Res -> Res; + case elixir_rewrite:inline(Module, Name, Arity) of + {AR, AN} -> {ok, AR, AN, Args}; false -> error end; false -> error end. -expand_require(Meta, Receiver, {Name, Arity} = Tuple, Args, E) -> - check_deprecation(Meta, Receiver, Name, Arity, E), - Module = ?m(E, module), +expand_require(Meta, Receiver, {Name, Arity} = Tuple, Args, S, E) -> + Required = + (Receiver == ?key(E, module)) orelse required(Meta) orelse + is_element(Receiver, ?key(E, requires)), - case is_element(Tuple, get_optional_macros(Receiver)) of + case is_macro(Tuple, Receiver, Required) of true -> - Requires = ?m(E, requires), - case (Receiver == Module) orelse is_element(Receiver, Requires) orelse skip_require(Meta) of - true -> - elixir_lexical:record_remote(Receiver, ?m(E, lexical_tracker)), - {ok, Receiver, expand_macro_named(Meta, Receiver, Name, Arity, Args, E)}; - false -> - Info = {unrequired_module, {Receiver, Name, length(Args), Requires}}, - elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, Info) - end; + check_deprecated(Meta, macro, Receiver, Name, Arity, E), + elixir_env:trace({remote_macro, Meta, Receiver, Name, Arity}, E), + {ok, Receiver, expand_macro_named(Meta, Receiver, Name, Arity, Args, S, E)}; false -> + check_deprecated(Meta, function, Receiver, Name, Arity, E), error end. %% Expansion helpers -expand_macro_fun(Meta, Fun, Receiver, Name, Args, E) -> +expand_macro_fun(Meta, Fun, Receiver, Name, Args, S, E) -> Line = ?line(Meta), - EArg = {Line, E}, + EArg = {Line, S, E}, try - apply(Fun, [EArg|Args]) + apply(Fun, [EArg | Args]) catch - Kind:Reason -> + Kind:Reason:Stacktrace -> Arity = length(Args), MFA = {Receiver, elixir_utils:macro_name(Name), Arity+1}, Info = [{Receiver, Name, Arity, [{file, "expanding macro"}]}, caller(Line, E)], - erlang:raise(Kind, Reason, prune_stacktrace(erlang:get_stacktrace(), MFA, Info, EArg)) + erlang:raise(Kind, Reason, prune_stacktrace(Stacktrace, MFA, Info, {ok, EArg})) end. -expand_macro_named(Meta, Receiver, Name, Arity, Args, E) -> +expand_macro_named(Meta, Receiver, Name, Arity, Args, S, E) -> ProperName = elixir_utils:macro_name(Name), ProperArity = Arity + 1, Fun = fun Receiver:ProperName/ProperArity, - expand_macro_fun(Meta, Fun, Receiver, Name, Args, E). + expand_macro_fun(Meta, Fun, Receiver, Name, Args, S, E). -expand_quoted(Meta, Receiver, Name, Arity, Quoted, E) -> +expand_quoted(Meta, Receiver, Name, Arity, Quoted, S, E) -> Line = ?line(Meta), - Next = elixir_counter:next(), + Next = elixir_module:next_counter(?key(E, module)), try - elixir_exp:expand( - elixir_quote:linify_with_context_counter(Line, {Receiver, Next}, Quoted), - E) + ToExpand = elixir_quote:linify_with_context_counter(Line, {Receiver, Next}, Quoted), + elixir_expand:expand(ToExpand, S, E) catch - Kind:Reason -> + Kind:Reason:Stacktrace -> MFA = {Receiver, elixir_utils:macro_name(Name), Arity+1}, Info = [{Receiver, Name, Arity, [{file, "expanding macro"}]}, caller(Line, E)], - erlang:raise(Kind, Reason, prune_stacktrace(erlang:get_stacktrace(), MFA, Info, nil)) + erlang:raise(Kind, Reason, prune_stacktrace(Stacktrace, MFA, Info, error)) end. -caller(Line, #{module := nil} = E) -> - {elixir_compiler, '__FILE__', 2, location(Line, E)}; -caller(Line, #{module := Module, function := nil} = E) -> - {Module, '__MODULE__', 0, location(Line, E)}; -caller(Line, #{module := Module, function := {Name, Arity}} = E) -> - {Module, Name, Arity, location(Line, E)}. - -location(Line, E) -> - [{file, elixir_utils:characters_to_list(elixir_utils:relative_to_cwd(?m(E, file)))}, - {line, Line}]. +caller(Line, E) -> + elixir_utils:caller(Line, ?key(E, file), ?key(E, module), ?key(E, function)). %% Helpers -skip_require(Meta) -> - lists:keyfind(require, 1, Meta) == {require, false}. +required(Meta) -> + lists:keyfind(required, 1, Meta) == {required, true}. -find_dispatch(Meta, Tuple, Extra, E) -> - case is_import(Meta) of +find_dispatch(Meta, {_Name, Arity} = Tuple, Extra, E) -> + case is_import(Meta, Arity) of {import, _} = Import -> Import; false -> - Funs = ?m(E, functions), - Macs = Extra ++ ?m(E, macros), + Funs = ?key(E, functions), + Macs = Extra ++ ?key(E, macros), FunMatch = find_dispatch(Tuple, Funs), MacMatch = find_dispatch(Tuple, Macs), @@ -259,69 +263,88 @@ find_dispatch(Meta, Tuple, Extra, E) -> {[], []} -> false; _ -> {Name, Arity} = Tuple, - [First, Second|_] = FunMatch ++ MacMatch, + [First, Second | _] = FunMatch ++ MacMatch, Error = {ambiguous_call, {First, Second, Name, Arity}}, - elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, Error) + elixir_errors:form_error(Meta, E, ?MODULE, Error) end end. find_dispatch(Tuple, List) -> [Receiver || {Receiver, Set} <- List, is_element(Tuple, Set)]. -is_import(Meta) -> - case lists:keyfind(import, 1, Meta) of - {import, _} = Import -> +is_import(Meta, Arity) -> + case lists:keyfind(imports, 1, Meta) of + {imports, Imports} -> case lists:keyfind(context, 1, Meta) of - {context, _} -> Import; - false -> - false + {context, _} -> + case lists:keyfind(Arity, 2, Imports) of + {Receiver, Arity} -> {import, Receiver}; + false -> false + end; + false -> false end; - false -> - false + false -> false end. % %% We've reached the macro wrapper fun, skip it with the rest -prune_stacktrace([{_, _, [E|_], _}|_], _MFA, Info, E) -> +prune_stacktrace([{_, _, [E | _], _} | _], _MFA, Info, {ok, E}) -> Info; %% We've reached the invoked macro, skip it -prune_stacktrace([{M, F, A, _}|_], {M, F, A}, Info, _E) -> +prune_stacktrace([{M, F, A, _} | _], {M, F, A}, Info, _E) -> Info; %% We've reached the elixir_dispatch internals, skip it with the rest -prune_stacktrace([{Mod, _, _, _}|_], _MFA, Info, _E) when Mod == elixir_dispatch; Mod == elixir_exp -> +prune_stacktrace([{Mod, _, _, _} | _], _MFA, Info, _E) when Mod == elixir_dispatch; Mod == elixir_exp -> Info; -prune_stacktrace([H|T], MFA, Info, E) -> - [H|prune_stacktrace(T, MFA, Info, E)]; +prune_stacktrace([H | T], MFA, Info, E) -> + [H | prune_stacktrace(T, MFA, Info, E)]; prune_stacktrace([], _MFA, Info, _E) -> Info. %% ERROR HANDLING -format_error({unrequired_module, {Receiver, Name, Arity, _Required}}) -> - Module = elixir_aliases:inspect(Receiver), - io_lib:format("you must require ~ts before invoking the macro ~ts.~ts/~B", - [Module, Module, Name, Arity]); format_error({macro_conflict, {Receiver, Name, Arity}}) -> io_lib:format("call to local macro ~ts/~B conflicts with imported ~ts.~ts/~B, " "please rename the local macro or remove the conflicting import", [Name, Arity, elixir_aliases:inspect(Receiver), Name, Arity]); format_error({ambiguous_call, {Mod1, Mod2, Name, Arity}}) -> io_lib:format("function ~ts/~B imported from both ~ts and ~ts, call is ambiguous", - [Name, Arity, elixir_aliases:inspect(Mod1), elixir_aliases:inspect(Mod2)]). + [Name, Arity, elixir_aliases:inspect(Mod1), elixir_aliases:inspect(Mod2)]); +format_error({compile_env, Name, Arity}) -> + io_lib:format("Application.~s/~B is discouraged in the module body, use Application.compile_env/3 instead", [Name, Arity]); +format_error({deprecated, Mod, '__using__', 1, Message}) -> + io_lib:format("use ~s is deprecated. ~s", [elixir_aliases:inspect(Mod), Message]); +format_error({deprecated, Mod, Fun, Arity, Message}) -> + io_lib:format("~s.~s/~B is deprecated. ~s",[elixir_aliases:inspect(Mod), Fun, Arity, Message]). %% INTROSPECTION -%% Do not try to get macros from Erlang. Speeds up compilation a bit. -get_optional_macros(erlang) -> []; +is_macro(_Tuple, _Module, false) -> + false; +is_macro(Tuple, Receiver, true) -> + try Receiver:'__info__'(macros) of + Macros -> is_element(Tuple, Macros) + catch + error:_ -> false + end. -get_optional_macros(Receiver) -> +%% Deprecations checks only happen at the module body, +%% so in there we can try to at least load the module. +get_deprecations(Receiver) -> case code:ensure_loaded(Receiver) of - {module, Receiver} -> + {module, Receiver} -> get_info(Receiver, deprecated); + _ -> [] + end. + +get_info(Receiver, Key) -> + case erlang:function_exported(Receiver, '__info__', 1) of + true -> try - Receiver:'__info__'(macros) + Receiver:'__info__'(Key) catch - error:undef -> [] + error:_ -> [] end; - {error, _} -> [] + false -> + [] end. elixir_imported_functions() -> @@ -338,184 +361,31 @@ elixir_imported_macros() -> error:undef -> [] end. -rewrite(?atom, to_string, [Arg], _) -> - {ok, erlang, atom_to_binary, [Arg, utf8]}; -rewrite(?kernel, elem, [Tuple, Index], _) -> - {ok, erlang, element, [increment(Index), Tuple]}; -rewrite(?kernel, put_elem, [Tuple, Index, Value], _) -> - {ok, erlang, setelement, [increment(Index), Tuple, Value]}; -rewrite(?map, 'has_key?', [Map, Key], _) -> - {ok, maps, is_key, [Key, Map]}; -rewrite(?map, fetch, [Map, Key], _) -> - {ok, maps, find, [Key, Map]}; -rewrite(?map, put, [Map, Key, Value], _) -> - {ok, maps, put, [Key, Value, Map]}; -rewrite(?map, delete, [Map, Key], _) -> - {ok, maps, remove, [Key, Map]}; -rewrite(?process, monitor, [Arg], _) -> - {ok, erlang, monitor, [process, Arg]}; -rewrite(?string, to_atom, [Arg], _) -> - {ok, erlang, binary_to_atom, [Arg, utf8]}; -rewrite(?string, to_existing_atom, [Arg], _) -> - {ok, erlang, binary_to_existing_atom, [Arg, utf8]}; -rewrite(?tuple, insert_at, [Tuple, Index, Term], _) -> - {ok, erlang, insert_element, [increment(Index), Tuple, Term]}; -rewrite(?tuple, delete_at, [Tuple, Index], _) -> - {ok, erlang, delete_element, [increment(Index), Tuple]}; -rewrite(?tuple, duplicate, [Data, Size], _) -> - {ok, erlang, make_tuple, [Size, Data]}; - -rewrite(Receiver, Name, Args, Arity) -> - case inline(Receiver, Name, Arity) of - {AR, AN} -> {ok, AR, AN, Args}; - false -> false - end. - -increment(Number) when is_number(Number) -> - Number + 1; -increment(Other) -> - {{'.', [], [erlang, '+']}, [], [Other, 1]}. - -inline(?atom, to_char_list, 1) -> {erlang, atom_to_list}; -inline(?io, iodata_length, 1) -> {erlang, iolist_size}; -inline(?io, iodata_to_binary, 1) -> {erlang, iolist_to_binary}; -inline(?integer, to_string, 1) -> {erlang, integer_to_binary}; -inline(?integer, to_string, 2) -> {erlang, integer_to_binary}; -inline(?integer, to_char_list, 1) -> {erlang, integer_to_list}; -inline(?integer, to_char_list, 2) -> {erlang, integer_to_list}; -inline(?float, to_string, 1) -> {erlang, float_to_binary}; -inline(?float, to_char_list, 1) -> {erlang, float_to_list}; -inline(?list, to_atom, 1) -> {erlang, list_to_atom}; -inline(?list, to_existing_atom, 1) -> {erlang, list_to_existing_atom}; -inline(?list, to_float, 1) -> {erlang, list_to_float}; -inline(?list, to_integer, 1) -> {erlang, list_to_integer}; -inline(?list, to_integer, 2) -> {erlang, list_to_integer}; -inline(?list, to_tuple, 1) -> {erlang, list_to_tuple}; - -inline(?kernel, '+', 2) -> {erlang, '+'}; -inline(?kernel, '-', 2) -> {erlang, '-'}; -inline(?kernel, '+', 1) -> {erlang, '+'}; -inline(?kernel, '-', 1) -> {erlang, '-'}; -inline(?kernel, '*', 2) -> {erlang, '*'}; -inline(?kernel, '/', 2) -> {erlang, '/'}; -inline(?kernel, '++', 2) -> {erlang, '++'}; -inline(?kernel, '--', 2) -> {erlang, '--'}; -inline(?kernel, 'not', 1) -> {erlang, 'not'}; -inline(?kernel, '<', 2) -> {erlang, '<'}; -inline(?kernel, '>', 2) -> {erlang, '>'}; -inline(?kernel, '<=', 2) -> {erlang, '=<'}; -inline(?kernel, '>=', 2) -> {erlang, '>='}; -inline(?kernel, '==', 2) -> {erlang, '=='}; -inline(?kernel, '!=', 2) -> {erlang, '/='}; -inline(?kernel, '===', 2) -> {erlang, '=:='}; -inline(?kernel, '!==', 2) -> {erlang, '=/='}; -inline(?kernel, abs, 1) -> {erlang, abs}; -inline(?kernel, apply, 2) -> {erlang, apply}; -inline(?kernel, apply, 3) -> {erlang, apply}; -inline(?kernel, binary_part, 3) -> {erlang, binary_part}; -inline(?kernel, bit_size, 1) -> {erlang, bit_size}; -inline(?kernel, byte_size, 1) -> {erlang, byte_size}; -inline(?kernel, 'div', 2) -> {erlang, 'div'}; -inline(?kernel, exit, 1) -> {erlang, exit}; -inline(?kernel, hd, 1) -> {erlang, hd}; -inline(?kernel, is_atom, 1) -> {erlang, is_atom}; -inline(?kernel, is_binary, 1) -> {erlang, is_binary}; -inline(?kernel, is_bitstring, 1) -> {erlang, is_bitstring}; -inline(?kernel, is_boolean, 1) -> {erlang, is_boolean}; -inline(?kernel, is_float, 1) -> {erlang, is_float}; -inline(?kernel, is_function, 1) -> {erlang, is_function}; -inline(?kernel, is_function, 2) -> {erlang, is_function}; -inline(?kernel, is_integer, 1) -> {erlang, is_integer}; -inline(?kernel, is_list, 1) -> {erlang, is_list}; -inline(?kernel, is_map, 1) -> {erlang, is_map}; -inline(?kernel, is_number, 1) -> {erlang, is_number}; -inline(?kernel, is_pid, 1) -> {erlang, is_pid}; -inline(?kernel, is_port, 1) -> {erlang, is_port}; -inline(?kernel, is_reference, 1) -> {erlang, is_reference}; -inline(?kernel, is_tuple, 1) -> {erlang, is_tuple}; -inline(?kernel, length, 1) -> {erlang, length}; -inline(?kernel, make_ref, 0) -> {erlang, make_ref}; -inline(?kernel, map_size, 1) -> {erlang, map_size}; -inline(?kernel, max, 2) -> {erlang, max}; -inline(?kernel, min, 2) -> {erlang, min}; -inline(?kernel, node, 0) -> {erlang, node}; -inline(?kernel, node, 1) -> {erlang, node}; -inline(?kernel, 'rem', 2) -> {erlang, 'rem'}; -inline(?kernel, round, 1) -> {erlang, round}; -inline(?kernel, self, 0) -> {erlang, self}; -inline(?kernel, send, 2) -> {erlang, send}; -inline(?kernel, spawn, 1) -> {erlang, spawn}; -inline(?kernel, spawn, 3) -> {erlang, spawn}; -inline(?kernel, spawn_link, 1) -> {erlang, spawn_link}; -inline(?kernel, spawn_link, 3) -> {erlang, spawn_link}; -inline(?kernel, spawn_monitor, 1) -> {erlang, spawn_monitor}; -inline(?kernel, spawn_monitor, 3) -> {erlang, spawn_monitor}; -inline(?kernel, throw, 1) -> {erlang, throw}; -inline(?kernel, tl, 1) -> {erlang, tl}; -inline(?kernel, trunc, 1) -> {erlang, trunc}; -inline(?kernel, tuple_size, 1) -> {erlang, tuple_size}; - -inline(?map, keys, 1) -> {maps, keys}; -inline(?map, merge, 2) -> {maps, merge}; -inline(?map, size, 1) -> {maps, size}; -inline(?map, values, 1) -> {maps, values}; -inline(?map, to_list, 1) -> {maps, to_list}; - -inline(?node, spawn, 2) -> {erlang, spawn}; -inline(?node, spawn, 3) -> {erlang, spawn_opt}; -inline(?node, spawn, 4) -> {erlang, spawn}; -inline(?node, spawn, 5) -> {erlang, spawn_opt}; -inline(?node, spawn_link, 2) -> {erlang, spawn_link}; -inline(?node, spawn_link, 4) -> {erlang, spawn_link}; - -inline(?process, exit, 2) -> {erlang, exit}; -inline(?process, spawn, 2) -> {erlang, spawn_opt}; -inline(?process, spawn, 4) -> {erlang, spawn_opt}; -inline(?process, demonitor, 1) -> {erlang, demonitor}; -inline(?process, demonitor, 2) -> {erlang, demonitor}; -inline(?process, link, 1) -> {erlang, link}; -inline(?process, unlink, 1) -> {erlang, unlink}; - -inline(?string, to_float, 1) -> {erlang, binary_to_float}; -inline(?string, to_integer, 1) -> {erlang, binary_to_integer}; -inline(?string, to_integer, 2) -> {erlang, binary_to_integer}; -inline(?system, stacktrace, 0) -> {erlang, get_stacktrace}; -inline(?tuple, to_list, 1) -> {erlang, tuple_to_list}; - -inline(_, _, _) -> false. - -check_deprecation(Meta, Receiver, Name, Arity, #{file := File}) -> - case deprecation(Receiver, Name, Arity) of - false -> ok; - Message -> - Warning = deprecation_message(Receiver, Name, Arity, Message), - elixir_errors:warn(?line(Meta), File, Warning) - end. +check_deprecated(_, _, erlang, _, _, _) -> ok; +check_deprecated(_, _, elixir_def, _, _, _) -> ok; +check_deprecated(_, _, elixir_module, _, _, _) -> ok; +check_deprecated(_, _, ?kernel, _, _, _) -> ok; +check_deprecated(Meta, Kind, ?application, Name, Arity, E) -> + case E of + #{module := Module, function := nil} + when (Module /= nil) or (Kind == macro), (Name == get_env) orelse (Name == fetch_env) orelse (Name == 'fetch_env!') -> + elixir_errors:form_warn(Meta, E, ?MODULE, {compile_env, Name, Arity}); -deprecation_message(Receiver, '__using__', _Arity, Message) -> - Warning = io_lib:format("use ~s is deprecated", [elixir_aliases:inspect(Receiver)]), - deprecation_message(Warning, Message); + _ -> + ok + end; +check_deprecated(Meta, Kind, Receiver, Name, Arity, E) -> + %% Any compile time behaviour cannot be verified by the runtime group pass. + case ((?key(E, function) == nil) or (Kind == macro)) andalso get_deprecations(Receiver) of + [_ | _] = Deprecations -> + case lists:keyfind({Name, Arity}, 1, Deprecations) of + {_, Message} -> + elixir_errors:form_warn(Meta, E, ?MODULE, {deprecated, Receiver, Name, Arity, Message}); -deprecation_message(Receiver, Name, Arity, Message) -> - Warning = io_lib:format("~s.~s/~B is deprecated", - [elixir_aliases:inspect(Receiver), Name, Arity]), - deprecation_message(Warning, Message). + false -> + false + end; -deprecation_message(Warning, Message) -> - case Message of - true -> Warning; - Message -> Warning ++ ", " ++ Message + _ -> + ok end. - -deprecation('Elixir.Mix.Generator', 'from_file', _) -> - "instead pass [from_file: file] to embed_text/2 and embed_template/2 macros. " - "Note that [from_file: file] expects paths relative to the current working " - "directory and not to the current file"; -deprecation('Elixir.EEx.TransformerEngine', '__using__', _) -> - "check EEx.SmartEngine for how to build custom engines"; -deprecation('Elixir.EEx.AssignsEngine', '__using__', _) -> - "check EEx.SmartEngine for how to build custom engines"; -deprecation('Elixir.Kernel', 'xor', _) -> - true; %% Remember to remove xor operator from tokenizer -deprecation(_, _, _) -> - false. diff --git a/lib/elixir/src/elixir_env.erl b/lib/elixir/src/elixir_env.erl index fc32e1b0599..df78f63a61b 100644 --- a/lib/elixir/src/elixir_env.erl +++ b/lib/elixir/src/elixir_env.erl @@ -1,62 +1,154 @@ -module(elixir_env). -include("elixir.hrl"). --export([new/0, linify/1, env_to_scope/1, env_to_scope_with_vars/2]). --export([mergea/2, mergev/2, merge_vars/2, merge_opt_vars/2]). +-export([ + new/0, to_caller/1, with_vars/2, reset_vars/1, + env_to_ex/1, env_to_erl/1, + reset_unused_vars/1, check_unused_vars/2, + merge_and_check_unused_vars/3, + trace/2, format_error/1, + reset_read/2, prepare_write/1, close_write/2 +]). new() -> - #{'__struct__' => 'Elixir.Macro.Env', - module => nil, %% the current module - file => <<"nofile">>, %% the current filename - line => 1, %% the current line - function => nil, %% the current function - context => nil, %% can be match_vars, guards or nil - requires => [], %% a set with modules required - aliases => [], %% an orddict with aliases by new -> old names - functions => [], %% a list with functions imported from module - macros => [], %% a list with macros imported from module - macro_aliases => [], %% keep aliases defined inside a macro - context_modules => [], %% modules defined in the current context - vars => [], %% a set of defined variables - export_vars => nil, %% a set of variables to be exported in some constructs - lexical_tracker => nil, %% holds the lexical tracker pid - local => nil}. %% the module to delegate local functions to - -linify({Line, Env}) -> - Env#{line := Line}. - -env_to_scope(#{module := Module, file := File, function := Function, context := Context}) -> - #elixir_scope{module=Module, file=File, function=Function, context=Context}. - -env_to_scope_with_vars(Env, Vars) -> - (env_to_scope(Env))#elixir_scope{ - vars=orddict:from_list(Vars), - counter=[{'_',length(Vars)}] - }. - -%% SCOPE MERGING - -%% Receives two scopes and return a new scope based on the second -%% with their variables merged. -mergev(E1, E2) when is_list(E1) -> - E2#{ - vars := merge_vars(E1, ?m(E2, vars)), - export_vars := merge_opt_vars(E1, ?m(E2, export_vars)) - }; -mergev(E1, E2) -> - E2#{ - vars := merge_vars(?m(E1, vars), ?m(E2, vars)), - export_vars := merge_opt_vars(?m(E1, export_vars), ?m(E2, export_vars)) - }. - -%% Receives two scopes and return the later scope -%% keeping the variables from the first (imports -%% and everything else are passed forward). - -mergea(E1, E2) -> - E2#{vars := ?m(E1, vars)}. - -merge_vars(V1, V2) -> ordsets:union(V1, V2). - -merge_opt_vars(_V1, nil) -> nil; -merge_opt_vars(nil, _V2) -> nil; -merge_opt_vars(V1, V2) -> ordsets:union(V1, V2). + #{ + '__struct__' => 'Elixir.Macro.Env', + module => nil, %% the current module + file => <<"nofile">>, %% the current filename + line => 1, %% the current line + function => nil, %% the current function + context => nil, %% can be match, guard or nil + aliases => [], %% a list of aliases by new -> old names + requires => elixir_dispatch:default_requires(), %% a set with modules required + functions => elixir_dispatch:default_functions(), %% a list with functions imported from module + macros => elixir_dispatch:default_macros(), %% a list with macros imported from module + macro_aliases => [], %% keep aliases defined inside a macro + context_modules => [], %% modules defined in the current context + versioned_vars => #{}, %% a map of vars with their latest versions + lexical_tracker => nil, %% lexical tracker PID + tracers => [] %% available compilation tracers + }. + +trace(Event, #{tracers := Tracers} = E) -> + [ok = Tracer:trace(Event, E) || Tracer <- Tracers], + ok. + +to_caller({Line, #elixir_ex{vars={Read, _}}, Env}) -> + Env#{line := Line, versioned_vars := Read}; +to_caller(#{} = Env) -> + Env. + +with_vars(Env, Vars) when is_list(Vars) -> + {ReversedVars, _} = + lists:foldl(fun(Var, {Acc, I}) -> {[{Var, I} | Acc], I + 1} end, {[], 0}, Vars), + Env#{versioned_vars := maps:from_list(ReversedVars)}; +with_vars(Env, #{} = Vars) -> + Env#{versioned_vars := Vars}. + +reset_vars(Env) -> + Env#{versioned_vars := #{}}. + +%% CONVERSIONS + +env_to_ex(#{context := match, versioned_vars := Vars}) -> + Counter = map_size(Vars), + #elixir_ex{prematch={Vars, Counter}, vars={Vars, false}, unused={#{}, Counter}}; +env_to_ex(#{versioned_vars := Vars}) -> + #elixir_ex{vars={Vars, false}, unused={#{}, map_size(Vars)}}. + +env_to_erl(#{context := Context, versioned_vars := Read}) -> + {VarsList, _Counter} = lists:mapfoldl(fun to_erl_var/2, 0, maps:values(Read)), + VarsMap = maps:from_list(VarsList), + Scope = #elixir_erl{ + context=Context, + var_names=VarsMap, + counter=#{'_' => map_size(VarsMap)} + }, + {VarsList, Scope}. + +to_erl_var(Version, Counter) -> + {{Version, list_to_atom("_@" ++ integer_to_list(Counter))}, Counter + 1}. + +%% VAR HANDLING + +reset_read(#elixir_ex{vars={_, Write}} = S, #elixir_ex{vars={Read, _}}) -> + S#elixir_ex{vars={Read, Write}}. + +prepare_write(#elixir_ex{vars={Read, _}} = S) -> + S#elixir_ex{vars={Read, Read}}. + +close_write(#elixir_ex{vars={_Read, Write}} = S, #elixir_ex{vars={_, false}}) -> + S#elixir_ex{vars={Write, false}}; +close_write(#elixir_ex{vars={_Read, Write}} = S, #elixir_ex{vars={_, UpperWrite}}) -> + S#elixir_ex{vars={Write, merge_vars(UpperWrite, Write)}}. + +merge_vars(V, V) -> + V; +merge_vars(V1, V2) -> + maps:fold(fun(K, M2, Acc) -> + case Acc of + #{K := M1} when M1 >= M2 -> Acc; + _ -> Acc#{K => M2} + end + end, V1, V2). + +%% UNUSED VARS + +reset_unused_vars(#elixir_ex{unused={_Unused, Version}} = S) -> + S#elixir_ex{unused={#{}, Version}}. + +check_unused_vars(#elixir_ex{unused={Unused, _Version}}, E) -> + [elixir_errors:form_warn([{line, Line}], E, ?MODULE, {unused_var, Name, Overridden}) || + {{Name, _}, {Line, Overridden}} <- maps:to_list(Unused), is_unused_var(Name)], + E. + +merge_and_check_unused_vars(S, #elixir_ex{vars={Read, Write}, unused={Unused, _Version}}, E) -> + #elixir_ex{unused={ClauseUnused, Version}} = S, + NewUnused = merge_and_check_unused_vars(Read, Unused, ClauseUnused, E), + S#elixir_ex{unused={NewUnused, Version}, vars={Read, Write}}. + +merge_and_check_unused_vars(Current, Unused, ClauseUnused, E) -> + maps:fold(fun + ({Name, Count} = Key, false, Acc) -> + Var = {Name, nil}, + + %% The parent knows it, so we have to propagate it was used up. + case Current of + #{Var := CurrentCount} when Count =< CurrentCount -> + Acc#{Key => false}; + + #{} -> + Acc + end; + + ({Name, _Count}, {Line, Overridden}, Acc) -> + case is_unused_var(Name) of + true -> + Warn = {unused_var, Name, Overridden}, + elixir_errors:form_warn([{line, Line}], E, ?MODULE, Warn); + + false -> + ok + end, + + Acc + end, Unused, ClauseUnused). + +is_unused_var(Name) -> + case atom_to_list(Name) of + "_" ++ Rest -> is_compiler_var(Rest); + _ -> true + end. + +is_compiler_var([$_]) -> true; +is_compiler_var([Var | Rest]) when Var =:= $_; Var >= $A, Var =< $Z -> is_compiler_var(Rest); +is_compiler_var(_) -> false. + +format_error({unused_var, Name, Overridden}) -> + case atom_to_list(Name) of + "_" ++ _ -> + io_lib:format("unknown compiler variable \"~ts\" (expected one of __MODULE__, __ENV__, __DIR__, __CALLER__, __STACKTRACE__)", [Name]); + _ when Overridden -> + io_lib:format("variable \"~ts\" is unused (there is a variable with the same name in the context, use the pin operator (^) to match on it or prefix this variable with underscore if it is not meant to be used)", [Name]); + _ -> + io_lib:format("variable \"~ts\" is unused (if the variable is not meant to be used, prefix it with an underscore)", [Name]) + end. diff --git a/lib/elixir/src/elixir_erl.erl b/lib/elixir/src/elixir_erl.erl new file mode 100644 index 00000000000..0f8757b0cb2 --- /dev/null +++ b/lib/elixir/src/elixir_erl.erl @@ -0,0 +1,608 @@ +%% Compiler backend to Erlang. +-module(elixir_erl). +-export([elixir_to_erl/1, elixir_to_erl/2, definition_to_anonymous/4, compile/1, consolidate/3, + get_ann/1, debug_info/4, scope/2, format_error/1]). +-include("elixir.hrl"). +-define(typespecs, 'Elixir.Kernel.Typespec'). + +%% debug_info callback + +debug_info(elixir_v1, _Module, none, _Opts) -> + {error, missing}; +debug_info(elixir_v1, _Module, {elixir_v1, Map, _Specs}, _Opts) -> + {ok, Map}; +debug_info(erlang_v1, _Module, {elixir_v1, Map, Specs}, _Opts) -> + {Prefix, Forms, _, _, _} = dynamic_form(Map), + {ok, Prefix ++ Specs ++ Forms}; +debug_info(core_v1, _Module, {elixir_v1, Map, Specs}, Opts) -> + {Prefix, Forms, _, _, _} = dynamic_form(Map), + #{compile_opts := CompileOpts} = Map, + AllOpts = CompileOpts ++ Opts, + + %% Do not rely on elixir_erl_compiler because we don't warn + %% warnings nor the other functionality provided there. + case elixir_erl_compiler:erl_to_core(Prefix ++ Specs ++ Forms, AllOpts) of + {ok, CoreForms, _} -> + try compile:noenv_forms(CoreForms, [no_spawn_compiler_process, from_core, core, return | AllOpts]) of + {ok, _, Core, _} -> {ok, Core}; + _What -> {error, failed_conversion} + catch + error:_ -> {error, failed_conversion} + end; + _ -> + {error, failed_conversion} + end; +debug_info(_, _, _, _) -> + {error, unknown_format}. + +%% Builds Erlang AST annotation. + +get_ann(Opts) when is_list(Opts) -> + get_ann(Opts, false, 0). + +get_ann([{generated, true} | T], _, Line) -> get_ann(T, true, Line); +get_ann([{line, Line} | T], Gen, _) when is_integer(Line) -> get_ann(T, Gen, Line); +get_ann([_ | T], Gen, Line) -> get_ann(T, Gen, Line); +get_ann([], Gen, Line) -> erl_anno:set_generated(Gen, erl_anno:new(Line)). + +%% Converts an Elixir definition to an anonymous function. + +definition_to_anonymous(Module, Kind, Meta, Clauses) -> + ErlClauses = [translate_clause(Kind, 0, Clause, true) || Clause <- Clauses], + Fun = {'fun', ?ann(Meta), {clauses, ErlClauses}}, + LocalHandler = fun(LocalName, LocalArgs) -> invoke_local(Module, LocalName, LocalArgs) end, + {value, Result, _Binding} = erl_eval:expr(Fun, [], {value, LocalHandler}), + Result. + +invoke_local(Module, ErlName, Args) -> + {Name, Arity} = elixir_utils:erl_fa_to_elixir_fa(ErlName, length(Args)), + + case elixir_def:local_for(Module, Name, Arity, all) of + false -> + {current_stacktrace, [_ | T]} = erlang:process_info(self(), current_stacktrace), + erlang:raise(error, undef, [{Module, Name, Arity, []} | T]); + Fun -> + apply(Fun, Args) + end. + +%% Converts Elixir quoted literals to Erlang AST. +elixir_to_erl(Tree) -> + elixir_to_erl(Tree, erl_anno:new(0)). + +elixir_to_erl(Tree, Ann) when is_tuple(Tree) -> + {tuple, Ann, [elixir_to_erl(X, Ann) || X <- tuple_to_list(Tree)]}; +elixir_to_erl([], Ann) -> + {nil, Ann}; +elixir_to_erl(<<>>, Ann) -> + {bin, Ann, []}; +elixir_to_erl(#{} = Map, Ann) -> + Assocs = [{map_field_assoc, Ann, elixir_to_erl(K, Ann), elixir_to_erl(V, Ann)} || {K, V} <- maps:to_list(Map)], + {map, Ann, Assocs}; +elixir_to_erl(Tree, Ann) when is_list(Tree) -> + elixir_to_erl_cons(Tree, Ann); +elixir_to_erl(Tree, Ann) when is_atom(Tree) -> + {atom, Ann, Tree}; +elixir_to_erl(Tree, Ann) when is_integer(Tree) -> + {integer, Ann, Tree}; +elixir_to_erl(Tree, Ann) when is_float(Tree) -> + {float, Ann, Tree}; +elixir_to_erl(Tree, Ann) when is_binary(Tree) -> + %% Note that our binaries are UTF-8 encoded and we are converting + %% to a list using binary_to_list. The reason for this is that Erlang + %% considers a string in a binary to be encoded in latin1, so the bytes + %% are not changed in any fashion. + {bin, Ann, [{bin_element, Ann, {string, Ann, binary_to_list(Tree)}, default, default}]}; +elixir_to_erl(Pid, Ann) when is_pid(Pid) -> + ?remote(Ann, erlang, binary_to_term, [elixir_to_erl(term_to_binary(Pid), Ann)]); +elixir_to_erl(_Other, _Ann) -> + error(badarg). + +elixir_to_erl_cons([H | T], Ann) -> {cons, Ann, elixir_to_erl(H, Ann), elixir_to_erl_cons(T, Ann)}; +elixir_to_erl_cons(T, Ann) -> elixir_to_erl(T, Ann). + +%% Returns a scope for translation. + +scope(_Meta, ExpandCaptures) -> + #elixir_erl{expand_captures=ExpandCaptures}. + +%% Static compilation hook, used in protocol consolidation + +consolidate(Map, TypeSpecs, Chunks) -> + {Prefix, Forms, _Def, _Defmacro, _Macros} = dynamic_form(Map), + load_form(Map, Prefix, Forms, TypeSpecs, Chunks). + +%% Dynamic compilation hook, used in regular compiler + +compile(#{module := Module, line := Line} = Map) -> + {Set, Bag} = elixir_module:data_tables(Module), + + TranslatedTypespecs = + case elixir_config:is_bootstrap() andalso + (code:ensure_loaded(?typespecs) /= {module, ?typespecs}) of + true -> {[], [], [], [], []}; + false -> ?typespecs:translate_typespecs_for_module(Set, Bag) + end, + + {Prefix, Forms, Def, Defmacro, Macros} = dynamic_form(Map), + {Types, Callbacks, TypeSpecs} = typespecs_form(Map, TranslatedTypespecs, Macros), + + DocsChunk = docs_chunk(Set, Module, Line, Def, Defmacro, Types, Callbacks), + CheckerChunk = checker_chunk(Map), + load_form(Map, Prefix, Forms, TypeSpecs, DocsChunk ++ CheckerChunk). + +dynamic_form(#{module := Module, line := Line, relative_file := RelativeFile, + attributes := Attributes, definitions := Definitions, unreachable := Unreachable, + deprecated := Deprecated, compile_opts := Opts} = Map) -> + {Def, Defmacro, Macros, Exports, Functions} = + split_definition(Definitions, Unreachable, Line, [], [], [], [], {[], []}), + + FilteredOpts = lists:filter(fun({no_warn_undefined, _}) -> false; (_) -> true end, Opts), + Location = {elixir_utils:characters_to_list(RelativeFile), Line}, + + Prefix = [{attribute, Line, file, Location}, + {attribute, Line, module, Module}, + {attribute, Line, compile, [no_auto_import | FilteredOpts]}], + + Struct = maps:get(struct, Map, nil), + Forms0 = functions_form(Line, Module, Def, Defmacro, Exports, Functions, Deprecated, Struct), + Forms1 = attributes_form(Line, Attributes, Forms0), + {Prefix, Forms1, Def, Defmacro, Macros}. + +% Definitions + +split_definition([{Tuple, Kind, Meta, Clauses} | T], Unreachable, Line, + Def, Defmacro, Macros, Exports, Functions) -> + case lists:member(Tuple, Unreachable) of + false -> + split_definition(Tuple, Kind, Meta, Clauses, T, Unreachable, Line, + Def, Defmacro, Macros, Exports, Functions); + true -> + split_definition(T, Unreachable, Line, Def, Defmacro, Macros, Exports, Functions) + end; +split_definition([], _Unreachable, _Line, Def, Defmacro, Macros, Exports, {Head, Tail}) -> + {Def, Defmacro, Macros, Exports, Head ++ Tail}. + +split_definition(Tuple, def, Meta, Clauses, T, Unreachable, Line, + Def, Defmacro, Macros, Exports, Functions) -> + {_, _, N, A, _} = Entry = translate_definition(def, Line, Meta, Tuple, Clauses), + split_definition(T, Unreachable, Line, [Tuple | Def], Defmacro, Macros, [{N, A} | Exports], + add_definition(Meta, Entry, Functions)); + +split_definition(Tuple, defp, Meta, Clauses, T, Unreachable, Line, + Def, Defmacro, Macros, Exports, Functions) -> + Entry = translate_definition(defp, Line, Meta, Tuple, Clauses), + split_definition(T, Unreachable, Line, Def, Defmacro, Macros, Exports, + add_definition(Meta, Entry, Functions)); + +split_definition(Tuple, defmacro, Meta, Clauses, T, Unreachable, Line, + Def, Defmacro, Macros, Exports, Functions) -> + {_, _, N, A, _} = Entry = translate_definition(defmacro, Line, Meta, Tuple, Clauses), + split_definition(T, Unreachable, Line, Def, [Tuple | Defmacro], [Tuple | Macros], [{N, A} | Exports], + add_definition(Meta, Entry, Functions)); + +split_definition(Tuple, defmacrop, Meta, Clauses, T, Unreachable, Line, + Def, Defmacro, Macros, Exports, Functions) -> + Entry = translate_definition(defmacro, Line, Meta, Tuple, Clauses), + split_definition(T, Unreachable, Line, Def, Defmacro, [Tuple | Macros], Exports, + add_definition(Meta, Entry, Functions)). + +add_definition(Meta, Body, {Head, Tail}) -> + case lists:keyfind(file, 1, Meta) of + {file, {F, L}} -> + %% Erlang's epp attempts to perform offsetting when generated is set to true + %% and that causes cover to fail when processing modules. Therefore we never + %% pass the generated annotation forward for file attributes. The function + %% will still be marked as generated though if that's the case. + FileMeta = erl_anno:set_generated(false, ?ann(Meta)), + Attr = {attribute, FileMeta, file, {elixir_utils:characters_to_list(F), L}}, + {Head, [Attr, Body | Tail]}; + false -> + {[Body | Head], Tail} + end. + +translate_definition(Kind, Line, Meta, {Name, Arity}, Clauses) -> + ErlClauses = [translate_clause(Kind, Line, Clause, false) || Clause <- Clauses], + + case is_macro(Kind) of + true -> {function, ?ann(Meta), elixir_utils:macro_name(Name), Arity + 1, ErlClauses}; + false -> {function, ?ann(Meta), Name, Arity, ErlClauses} + end. + +translate_clause(Kind, Line, {Meta, Args, Guards, Body}, ExpandCaptures) -> + S = scope(Meta, ExpandCaptures), + + %% If the line matches the module line, then it is most likely an + %% auto-generated function and we don't want to track its contents. + Ann = + case ?line(Meta) of + Line -> erl_anno:set_generated(true, erl_anno:new(0)); + _ -> ?ann(Meta) + end, + + {TClause, TS} = + elixir_erl_clauses:clause(Ann, fun elixir_erl_pass:translate_args/3, Args, Body, Guards, S), + + case is_macro(Kind) of + true -> + FArgs = {var, Ann, '_@CALLER'}, + MClause = setelement(3, TClause, [FArgs | element(3, TClause)]), + + case TS#elixir_erl.caller of + true -> + FBody = {'match', Ann, + {'var', Ann, '__CALLER__'}, + ?remote(Ann, elixir_env, to_caller, [{var, Ann, '_@CALLER'}]) + }, + setelement(5, MClause, [FBody | element(5, TClause)]); + false -> + MClause + end; + false -> + TClause + end. + +is_macro(defmacro) -> true; +is_macro(defmacrop) -> true; +is_macro(_) -> false. + +% Functions + +functions_form(Line, Module, Def, Defmacro, Exports, Body, Deprecated, Struct) -> + {Spec, Info} = add_info_function(Line, Module, Def, Defmacro, Deprecated, Struct), + [{attribute, Line, export, lists:sort([{'__info__', 1} | Exports])}, Spec, Info | Body]. + +add_info_function(Line, Module, Def, Defmacro, Deprecated, Struct) -> + AllowedAttrs = [attributes, compile, functions, macros, md5, exports_md5, module, deprecated, struct], + AllowedArgs = lists:map(fun(Atom) -> {atom, Line, Atom} end, AllowedAttrs), + SortedDef = lists:sort(Def), + SortedDefmacro = lists:sort(Defmacro), + + Spec = + {attribute, Line, spec, {{'__info__', 1}, + [{type, Line, 'fun', [ + {type, Line, product, [ + {type, Line, union, AllowedArgs} + ]}, + {type, Line, any, []} + ]}] + }}, + + Info = + {function, 0, '__info__', 1, [ + get_module_info(Module), + functions_info(SortedDef), + macros_info(SortedDefmacro), + struct_info(Struct), + exports_md5_info(Struct, SortedDef, SortedDefmacro), + get_module_info(Module, attributes), + get_module_info(Module, compile), + get_module_info(Module, md5), + deprecated_info(Deprecated) + ]}, + + {Spec, Info}. + +get_module_info(Module) -> + {clause, 0, [{atom, 0, module}], [], [{atom, 0, Module}]}. + +exports_md5_info(Struct, Def, Defmacro) -> + %% Deprecations do not need to be part of exports_md5 because it is always + %% checked by the runtime pass, so it is not really part of compilation. + Md5 = erlang:md5(erlang:term_to_binary({Def, Defmacro, Struct})), + {clause, 0, [{atom, 0, exports_md5}], [], [elixir_to_erl(Md5)]}. + +functions_info(Def) -> + {clause, 0, [{atom, 0, functions}], [], [elixir_to_erl(Def)]}. + +macros_info(Defmacro) -> + {clause, 0, [{atom, 0, macros}], [], [elixir_to_erl(Defmacro)]}. + +struct_info(nil) -> + {clause, 0, [{atom, 0, struct}], [], [{atom, 0, nil}]}; +struct_info(Fields) -> + FieldsWithoutDefault = [maps:remove(default, FieldInfo) || FieldInfo <- Fields], + {clause, 0, [{atom, 0, struct}], [], [elixir_to_erl(FieldsWithoutDefault)]}. + +get_module_info(Module, Key) -> + Call = ?remote(0, erlang, get_module_info, [{atom, 0, Module}, {var, 0, 'Key'}]), + {clause, 0, [{match, 0, {var, 0, 'Key'}, {atom, 0, Key}}], [], [Call]}. + +deprecated_info(Deprecated) -> + {clause, 0, [{atom, 0, deprecated}], [], [elixir_to_erl(Deprecated)]}. + +% Typespecs + +typespecs_form(Map, TranslatedTypespecs, MacroNames) -> + {Types, Specs, Callbacks, MacroCallbacks, OptionalCallbacks} = TranslatedTypespecs, + + AllCallbacks = Callbacks ++ MacroCallbacks, + MacroCallbackNames = [NameArity || {_, NameArity, _, _} <- MacroCallbacks], + validate_behaviour_info_and_attributes(Map, AllCallbacks), + validate_optional_callbacks(Map, AllCallbacks, OptionalCallbacks), + + Forms0 = [], + Forms1 = types_form(Types, Forms0), + Forms2 = callspecs_form(spec, Specs, [], MacroNames, Forms1, Map), + Forms3 = callspecs_form(callback, AllCallbacks, OptionalCallbacks, MacroCallbackNames, Forms2, Map), + + AllCallbacksWithoutSpecs = lists:usort([ + {Kind, Name, Arity} || {Kind, {Name, Arity}, _Line, _Spec} <- AllCallbacks + ]), + + {Types, AllCallbacksWithoutSpecs, Forms3}. + +%% Types + +types_form(Types, Forms) -> + Fun = fun + ({Kind, NameArity, Line, Expr, true}, Acc) -> + [{attribute, Line, export_type, [NameArity]}, {attribute, Line, Kind, Expr} | Acc]; + ({Kind, _NameArity, Line, Expr, false}, Acc) -> + [{attribute, Line, Kind, Expr} | Acc] + end, + + lists:foldl(Fun, Forms, Types). + +%% Specs and callbacks + +validate_behaviour_info_and_attributes(#{definitions := Defs} = Map, AllCallbacks) -> + case {lists:keyfind({behaviour_info, 1}, 1, Defs), AllCallbacks} of + {false, _} -> + ok; + {_, [{Kind, {Name, Arity}, _, _} | _]} when Kind == callback; Kind == macrocallback -> + form_error(Map, {callbacks_but_also_behaviour_info, {Kind, Name, Arity}}); + {_, _} -> + ok + end. + +validate_optional_callbacks(Map, AllCallbacks, Optional) -> + lists:foldl(fun(Callback, Acc) -> + case Callback of + {Name, Arity} when is_atom(Name) and is_integer(Arity) -> ok; + _ -> form_error(Map, {ill_defined_optional_callback, Callback}) + end, + + case lists:keyfind(Callback, 2, AllCallbacks) of + false -> form_error(Map, {unknown_callback, Callback}); + _ -> ok + end, + + case Acc of + #{Callback := _} -> form_error(Map, {duplicate_optional_callback, Callback}); + _ -> ok + end, + + maps:put(Callback, true, Acc) + end, #{}, Optional). + +callspecs_form(_Kind, [], _Optional, _Macros, Forms, _ModuleMap) -> + Forms; +callspecs_form(Kind, Entries, Optional, Macros, Forms, ModuleMap) -> + #{unreachable := Unreachable} = ModuleMap, + + {SpecsMap, Signatures} = + lists:foldl(fun({_, NameArity, Line, Spec}, {Acc, NA}) -> + case Kind of + spec -> validate_spec_for_existing_function(ModuleMap, NameArity, Line); + _ -> ok + end, + + case lists:member(NameArity, Unreachable) of + false -> + case Acc of + #{NameArity := List} -> {Acc#{NameArity := [{Spec, Line} | List]}, NA}; + #{} -> {Acc#{NameArity => [{Spec, Line}]}, [NameArity | NA]} + end; + true -> + {Acc, NA} + end + end, {#{}, []}, Entries), + + lists:foldl(fun(NameArity, Acc) -> + #{NameArity := ExprsLines} = SpecsMap, + {Exprs, Lines} = lists:unzip(ExprsLines), + Line = lists:min(Lines), + + {Key, Value} = + case lists:member(NameArity, Macros) of + true -> + {Name, Arity} = NameArity, + {{elixir_utils:macro_name(Name), Arity + 1}, + lists:map(fun spec_for_macro/1, Exprs)}; + false -> + {NameArity, Exprs} + end, + + case lists:member(NameArity, Optional) of + true -> + [{attribute, Line, Kind, {Key, lists:reverse(Value)}}, + {attribute, Line, optional_callbacks, [Key]} | Acc]; + false -> + [{attribute, Line, Kind, {Key, lists:reverse(Value)}} | Acc] + end + end, Forms, lists:sort(Signatures)). + +spec_for_macro({type, Line, 'bounded_fun', [H | T]}) -> + {type, Line, 'bounded_fun', [spec_for_macro(H) | T]}; +spec_for_macro({type, Line, 'fun', [{type, _, product, Args} | T]}) -> + {type, Line, 'fun', [{type, Line, product, [{type, Line, term, []} | Args]} | T]}; +spec_for_macro(Else) -> + Else. + +validate_spec_for_existing_function(ModuleMap, NameAndArity, Line) -> + #{definitions := Defs, file := File} = ModuleMap, + + case lists:keymember(NameAndArity, 1, Defs) of + true -> ok; + false -> form_error(#{line => Line, file => File}, {spec_for_undefined_function, NameAndArity}) + end. + +% Attributes + +attributes_form(Line, Attributes, Forms) -> + Fun = fun({Key, Value}, Acc) -> [{attribute, Line, Key, Value} | Acc] end, + lists:foldr(Fun, Forms, Attributes). + +% Loading forms + +load_form(#{file := File, compile_opts := Opts} = Map, Prefix, Forms, Specs, Chunks) -> + CompileOpts = extra_chunks_opts(Chunks, debug_opts(Map, Specs, Opts)), + {_, Binary} = elixir_erl_compiler:forms(Prefix ++ Specs ++ Forms, File, CompileOpts), + Binary. + +debug_opts(Map, Specs, Opts) -> + case take_debug_opts(Opts) of + {true, Rest} -> [{debug_info, {?MODULE, {elixir_v1, Map, Specs}}} | Rest]; + {false, Rest} -> [{debug_info, {?MODULE, none}} | Rest] + end. + +take_debug_opts(Opts) -> + case proplists:get_value(debug_info, Opts) of + true -> {true, proplists:delete(debug_info, Opts)}; + false -> {false, proplists:delete(debug_info, Opts)}; + undefined -> {elixir_config:get(debug_info), Opts} + end. + +extra_chunks_opts([], Opts) -> Opts; +extra_chunks_opts(Chunks, Opts) -> [{extra_chunks, Chunks} | Opts]. + +docs_chunk(Set, Module, Line, Def, Defmacro, Types, Callbacks) -> + case elixir_config:get(docs) of + true -> + {ModuleDocLine, ModuleDoc} = get_moduledoc(Line, Set), + ModuleDocMeta = get_moduledoc_meta(Set), + FunctionDocs = get_docs(Set, Module, Def, function), + MacroDocs = get_docs(Set, Module, Defmacro, macro), + CallbackDocs = get_callback_docs(Set, Callbacks), + TypeDocs = get_type_docs(Set, Types), + + DocsChunkData = term_to_binary({docs_v1, + erl_anno:new(ModuleDocLine), + elixir, + <<"text/markdown">>, + ModuleDoc, + ModuleDocMeta, + FunctionDocs ++ MacroDocs ++ CallbackDocs ++ TypeDocs + }, [compressed]), + + [{<<"Docs">>, DocsChunkData}]; + + false -> + [] + end. + +doc_value(Doc, Name) -> + case Doc of + false -> + hidden; + nil -> + case erlang:atom_to_list(Name) of + [$_ | _] -> hidden; + _ -> none + end; + Doc -> + #{<<"en">> => Doc} + end. + +get_moduledoc(Line, Set) -> + case ets:lookup_element(Set, moduledoc, 2) of + nil -> {Line, none}; + {DocLine, false} -> {DocLine, hidden}; + {DocLine, nil} -> {DocLine, none}; + {DocLine, Doc} -> {DocLine, #{<<"en">> => Doc}} + end. + +get_moduledoc_meta(Set) -> + case ets:lookup(Set, {moduledoc, meta}) of + [] -> #{}; + [{{moduledoc, meta}, Map, _}] when is_map(Map) -> Map + end. + +get_docs(Set, Module, Definitions, Kind) -> + [{Key, + erl_anno:new(Line), + [signature_to_binary(Module, Name, Signature)], + doc_value(Doc, Name), + Meta + } || {Name, Arity} <- Definitions, + {Key, _Ctx, Line, Signature, Doc, Meta} <- ets:lookup(Set, {Kind, Name, Arity})]. + +get_callback_docs(Set, Callbacks) -> + [{Key, + erl_anno:new(Line), + [], + doc_value(Doc, Name), + Meta + } || Callback <- Callbacks, {{_, Name, _} = Key, Line, Doc, Meta} <- ets:lookup(Set, Callback)]. + +get_type_docs(Set, Types) -> + [{Key, + erl_anno:new(Line), + [], + doc_value(Doc, Name), + Meta + } || {_Kind, {Name, Arity}, _, _, true} <- Types, + {Key, Line, Doc, Meta} <- ets:lookup(Set, {type, Name, Arity})]. + +signature_to_binary(_Module, Name, _Signature) when Name == '__aliases__'; Name == '__block__' -> + <<(atom_to_binary(Name))/binary, "(args)">>; + +signature_to_binary(_Module, fn, _Signature) -> + <<"fn(clauses)">>; + +signature_to_binary(_Module, Name, _Signature) + when Name == '__CALLER__'; Name == '__DIR__'; Name == '__ENV__'; + Name == '__MODULE__'; Name == '__STACKTRACE__'; Name == '%{}' -> + atom_to_binary(Name); + +signature_to_binary(_Module, '%', _) -> + <<"%struct{}">>; + +signature_to_binary(Module, '__struct__', []) -> + <<"%", ('Elixir.Kernel':inspect(Module))/binary, "{}">>; + +signature_to_binary(_, Name, Signature) -> + Quoted = {Name, [{closing, []}], Signature}, + Doc = 'Elixir.Inspect.Algebra':format('Elixir.Code':quoted_to_algebra(Quoted), infinity), + 'Elixir.IO':iodata_to_binary(Doc). + +checker_chunk(#{definitions := Definitions, deprecated := Deprecated, is_behaviour := IsBehaviour}) -> + DeprecatedMap = maps:from_list(Deprecated), + + Exports = + lists:foldl(fun({Function, Kind, _Meta, _Clauses}, Acc) -> + case Kind of + _ when Kind == def orelse Kind == defmacro -> + Reason = maps:get(Function, DeprecatedMap, nil), + [{Function, #{kind => Kind, deprecated_reason => Reason}} | Acc]; + _ -> + Acc + end + end, [], Definitions), + + Contents = #{ + exports => lists:sort(behaviour_info_exports(IsBehaviour) ++ Exports) + }, + + [{<<"ExCk">>, erlang:term_to_binary({elixir_checker_v1, Contents})}]. + +behaviour_info_exports(true) -> [{{behaviour_info, 1}, #{kind => def, deprecated_reason => nil}}]; +behaviour_info_exports(false) -> []. + +%% Errors + +form_error(#{line := Line, file := File}, Error) -> + elixir_errors:form_error([{line, Line}], File, ?MODULE, Error). + +format_error({ill_defined_optional_callback, Callback}) -> + io_lib:format("invalid optional callback ~ts. @optional_callbacks expects a " + "keyword list of callback names and arities", ['Elixir.Kernel':inspect(Callback)]); +format_error({unknown_callback, {Name, Arity}}) -> + io_lib:format("unknown callback ~ts/~B given as optional callback", [Name, Arity]); +format_error({duplicate_optional_callback, {Name, Arity}}) -> + io_lib:format("~ts/~B has been specified as optional callback more than once", [Name, Arity]); +format_error({callbacks_but_also_behaviour_info, {Type, Fun, Arity}}) -> + io_lib:format("cannot define @~ts attribute for ~ts/~B when behaviour_info/1 is defined", + [Type, Fun, Arity]); +format_error({spec_for_undefined_function, {Name, Arity}}) -> + io_lib:format("spec for undefined function ~ts/~B", [Name, Arity]). diff --git a/lib/elixir/src/elixir_erl_clauses.erl b/lib/elixir/src/elixir_erl_clauses.erl new file mode 100644 index 00000000000..84b0539360e --- /dev/null +++ b/lib/elixir/src/elixir_erl_clauses.erl @@ -0,0 +1,65 @@ +%% Handle code related to args, guard and -> matching for case, +%% fn, receive and friends. try is handled in elixir_erl_try. +-module(elixir_erl_clauses). +-export([match/4, clause/6, clauses/2, guards/4, get_clauses/3]). +-include("elixir.hrl"). + +%% Get clauses under the given key. + +get_clauses(Key, Keyword, As) -> + case lists:keyfind(Key, 1, Keyword) of + {Key, Clauses} when is_list(Clauses) -> + [{As, Meta, Left, Right} || {'->', Meta, [Left, Right]} <- Clauses]; + _ -> + [] + end. + +%% Translate matches + +match(Ann, Fun, Match, #elixir_erl{context=Context} = S) when Context =/= match -> + {Result, NewS} = Fun(Match, Ann, S#elixir_erl{context=match}), + {Result, NewS#elixir_erl{context=Context}}; +match(Ann, Fun, Match, S) -> + Fun(Match, Ann, S). + +%% Translate clauses with args, guards and expressions + +clause(Ann, Fun, Match, Expr, Guards, S) -> + {TMatch, SA} = match(Ann, Fun, Match, S), + SG = SA#elixir_erl{extra_guards=[]}, + TGuards = guards(Ann, Guards, SA#elixir_erl.extra_guards, SG), + {TExpr, SE} = elixir_erl_pass:translate(Expr, Ann, SG), + {{clause, Ann, TMatch, TGuards, unblock(TExpr)}, SE}. + +% Translate/Extract guards from the given expression. + +guards(Ann, Guards, Extra, S) -> + SG = S#elixir_erl{context=guard}, + case Guards of + [] -> case Extra of [] -> []; _ -> [Extra] end; + _ -> [translate_guard(Guard, Ann, SG, Extra) || Guard <- Guards] + end. + +translate_guard(Guard, Ann, S, Extra) -> + [element(1, elixir_erl_pass:translate(Guard, Ann, S)) | Extra]. + +% Function for translating macros with match style like case and receive. + +clauses([], S) -> + {[], S}; + +clauses(Clauses, S) -> + lists:mapfoldl(fun each_clause/2, S, Clauses). + +each_clause({match, Meta, [Condition], Expr}, S) -> + {Arg, Guards} = elixir_utils:extract_guards(Condition), + clause(?ann(Meta), fun elixir_erl_pass:translate_args/3, [Arg], Expr, Guards, S); + +each_clause({expr, Meta, [Condition], Expr}, S) -> + Ann = ?ann(Meta), + {TCondition, SC} = elixir_erl_pass:translate(Condition, Ann, S), + {TExpr, SB} = elixir_erl_pass:translate(Expr, Ann, SC), + {{clause, Ann, [TCondition], [], unblock(TExpr)}, SB}. + +unblock({'block', _, Exprs}) -> Exprs; +unblock(Exprs) -> [Exprs]. diff --git a/lib/elixir/src/elixir_erl_compiler.erl b/lib/elixir/src/elixir_erl_compiler.erl new file mode 100644 index 00000000000..d3c8983cbd2 --- /dev/null +++ b/lib/elixir/src/elixir_erl_compiler.erl @@ -0,0 +1,184 @@ +-module(elixir_erl_compiler). +-export([spawn/1, forms/3, noenv_forms/3, erl_to_core/2, format_error/1]). +-include("elixir.hrl"). + +spawn(Fun) -> + CompilerInfo = get(elixir_compiler_info), + + {_, Ref} = + spawn_monitor(fun() -> + put(elixir_compiler_info, CompilerInfo), + + try Fun() of + Result -> exit({ok, Result}) + catch + Kind:Reason:Stack -> + exit({Kind, Reason, Stack}) + end + end), + + receive + {'DOWN', Ref, process, _, {ok, Result}} -> + Result; + {'DOWN', Ref, process, _, {Kind, Reason, Stack}} -> + erlang:raise(Kind, Reason, Stack) + end. + +forms(Forms, File, Opts) -> + compile(Forms, File, Opts ++ compile:env_compiler_options()). + +noenv_forms(Forms, File, Opts) -> + compile(Forms, File, Opts). + +erl_to_core(Forms, Opts) -> + %% TODO: Remove parse transform handling on Elixir v2.0 + case [M || {parse_transform, M} <- Opts] of + [] -> + v3_core:module(Forms, Opts); + _ -> + case compile:noenv_forms(Forms, [no_spawn_compiler_process, to_core0, return, no_auto_import | Opts]) of + {ok, _Module, Core, Warnings} -> {ok, Core, Warnings}; + {error, Errors, Warnings} -> {error, Errors, Warnings} + end + end. + +compile(Forms, File, Opts) when is_list(Forms), is_list(Opts), is_binary(File) -> + Source = elixir_utils:characters_to_list(File), + + case erl_to_core(Forms, Opts) of + {ok, CoreForms, CoreWarnings} -> + format_warnings(Opts, CoreWarnings), + CompileOpts = [no_spawn_compiler_process, from_core, no_core_prepare, + no_auto_import, return, {source, Source} | Opts], + + case compile:noenv_forms(CoreForms, CompileOpts) of + {ok, Module, Binary, Warnings} when is_binary(Binary) -> + format_warnings(Opts, Warnings), + {Module, Binary}; + + {ok, Module, _Binary, _Warnings} -> + elixir_errors:form_error([], File, ?MODULE, {invalid_compilation, Module}); + + {error, Errors, Warnings} -> + format_warnings(Opts, Warnings), + format_errors(Errors) + end; + + {error, CoreErrors, CoreWarnings} -> + format_warnings(Opts, CoreWarnings), + format_errors(CoreErrors) + end. + +format_errors([]) -> + exit({nocompile, "compilation failed but no error was raised"}); +format_errors(Errors) -> + lists:foreach(fun + ({File, Each}) when is_list(File) -> + BinFile = elixir_utils:characters_to_binary(File), + lists:foreach(fun(Error) -> handle_file_error(BinFile, Error) end, Each); + ({Mod, Each}) when is_atom(Mod) -> + lists:foreach(fun(Error) -> handle_file_error(elixir_aliases:inspect(Mod), Error) end, Each) + end, Errors). + +format_warnings(Opts, Warnings) -> + NoWarnNoMatch = proplists:get_value(nowarn_nomatch, Opts, false), + lists:foreach(fun ({File, Each}) -> + BinFile = elixir_utils:characters_to_binary(File), + lists:foreach(fun(Warning) -> + handle_file_warning(NoWarnNoMatch, BinFile, Warning) + end, Each) + end, Warnings). + +%% Handle warnings from Erlang land + +%% Those we implement ourselves +handle_file_warning(_, _File, {_Line, v3_core, {map_key_repeated, _}}) -> ok; +handle_file_warning(_, _File, {_Line, sys_core_fold, {ignored, useless_building}}) -> ok; + +%% TODO: remove when we require Erlang/OTP 24 +handle_file_warning(_, _File, {_Line, sys_core_fold, useless_building}) -> ok; +handle_file_warning(true, _File, {_Line, sys_core_fold, nomatch_guard}) -> ok; +handle_file_warning(true, _File, {_Line, sys_core_fold, {nomatch_shadow, _}}) -> ok; +%% + +%% Ignore all linting errors (only come up on parse transforms) +handle_file_warning(_, _File, {_Line, erl_lint, _}) -> ok; + +handle_file_warning(_, File, {Line, Module, Desc}) -> + Message = custom_format(Module, Desc), + elixir_errors:erl_warn(Line, File, Message). + +%% Handle warnings + +handle_file_error(File, {beam_validator, Rest}) -> + elixir_errors:form_error([{line, 0}], File, beam_validator, Rest); +handle_file_error(File, {Line, Module, Desc}) -> + Message = custom_format(Module, Desc), + elixir_errors:compile_error([{line, Line}], File, Message). + +%% Mention the capture operator in make_fun +custom_format(sys_core_fold, {ignored, {no_effect, {erlang, make_fun, 3}}}) -> + "the result of the capture operator & (:erlang.make_fun/3) is never used"; + +%% Make no_effect clauses pretty +custom_format(sys_core_fold, {ignored, {no_effect, {erlang, F, A}}}) -> + {Fmt, Args} = case erl_internal:comp_op(F, A) of + true -> {"use of operator ~ts has no effect", [elixir_utils:erlang_comparison_op_to_elixir(F)]}; + false -> + case erl_internal:bif(F, A) of + false -> {"the call to :erlang.~ts/~B has no effect", [F, A]}; + true -> {"the call to ~ts/~B has no effect", [F, A]} + end + end, + io_lib:format(Fmt, Args); + +%% Rewrite nomatch to be more generic, it can happen inside if, unless, and the like +custom_format(sys_core_fold, {nomatch, X}) when X == guard; X == no_clause -> + "this check/guard will always yield the same result"; + +custom_format(sys_core_fold, {nomatch, {shadow, Line, {ErlName, ErlArity}}}) -> + {Name, Arity} = elixir_utils:erl_fa_to_elixir_fa(ErlName, ErlArity), + + io_lib:format( + "this clause for ~ts/~B cannot match because a previous clause at line ~B always matches", + [Name, Arity, Line] + ); + +%% Handle literal eval failures +custom_format(sys_core_fold, {failed, {eval_failure, {Mod, Name, Arity}, Error}}) -> + #{'__struct__' := Struct} = 'Elixir.Exception':normalize(error, Error), + {ExMod, ExName, ExArgs} = elixir_rewrite:erl_to_ex(Mod, Name, lists:duplicate(Arity, nil)), + Call = 'Elixir.Exception':format_mfa(ExMod, ExName, length(ExArgs)), + Trimmed = case Call of + <<"Kernel.", Rest/binary>> -> Rest; + _ -> Call + end, + ["the call to ", Trimmed, " will fail with ", elixir_aliases:inspect(Struct)]; + +%% TODO: remove when we require Erlang/OTP 24 +custom_format(sys_core_fold, {nomatch_shadow, Line, FA}) -> + custom_format(sys_core_fold, {nomatch, {shadow, Line, FA}}); +custom_format(sys_core_fold, nomatch_guard) -> + custom_format(sys_core_fold, {nomatch, guard}); +custom_format(sys_core_fold, {no_effect, X}) -> + custom_format(sys_core_fold, {ignored, {no_effect, X}}); +custom_format(sys_core_fold, {eval_failure, Error}) -> + #{'__struct__' := Struct} = 'Elixir.Exception':normalize(error, Error), + ["this expression will fail with ", elixir_aliases:inspect(Struct)]; +%% + +custom_format([], Desc) -> + io_lib:format("~p", [Desc]); + +custom_format(Module, Desc) -> + Module:format_error(Desc). + +%% Error formatting + +format_error({invalid_compilation, Module}) -> + io_lib:format( + "could not compile module ~ts. We expected the compiler to return a .beam binary but " + "got something else. This usually happens because ERL_COMPILER_OPTIONS or @compile " + "was set to change the compilation outcome in a way that is incompatible with Elixir", + [elixir_aliases:inspect(Module)] + ). diff --git a/lib/elixir/src/elixir_erl_for.erl b/lib/elixir/src/elixir_erl_for.erl new file mode 100644 index 00000000000..9252f43009a --- /dev/null +++ b/lib/elixir/src/elixir_erl_for.erl @@ -0,0 +1,348 @@ +-module(elixir_erl_for). +-export([translate/4]). +-include("elixir.hrl"). + +translate(Meta, Args, Return, S) -> + {Cases, [{do, Expr} | Opts]} = elixir_utils:split_last(Args), + + case lists:keyfind(reduce, 1, Opts) of + {reduce, Reduce} -> translate_reduce(Meta, Cases, Expr, Reduce, S); + false -> translate_into(Meta, Cases, Expr, Opts, Return, S) + end. + +translate_reduce(Meta, Cases, Expr, Reduce, S) -> + Ann = ?ann(Meta), + {TReduce, SR} = elixir_erl_pass:translate(Reduce, Ann, S), + {TCases, SC} = translate_gen(Meta, Cases, [], SR), + CaseExpr = {'case', Meta, [ok, [{do, Expr}]]}, + {TExpr, SE} = elixir_erl_pass:translate(CaseExpr, Ann, SC), + + InnerFun = fun + ({'case', CaseAnn, _, CaseBlock}, InnerAcc) -> {'case', CaseAnn, InnerAcc, CaseBlock} + end, + + build_reduce(Ann, TCases, InnerFun, TExpr, TReduce, false, SE). + +translate_into(Meta, Cases, Expr, Opts, Return, S) -> + Ann = ?ann(Meta), + + {TInto, SI} = + case lists:keyfind(into, 1, Opts) of + {into, Into} -> elixir_erl_pass:translate(Into, Ann, S); + false when Return -> {{nil, Ann}, S}; + false -> {false, S} + end, + + TUniq = lists:keyfind(uniq, 1, Opts) == {uniq, true}, + + {TCases, SC} = translate_gen(Meta, Cases, [], SI), + {TExpr, SE} = elixir_erl_pass:translate(wrap_expr_if_unused(Expr, TInto), Ann, SC), + + case inline_or_into(TInto) of + inline -> build_inline(Ann, TCases, TExpr, TInto, TUniq, SE); + into -> build_into(Ann, TCases, TExpr, TInto, TUniq, SE) + end. + +%% In case we have no return, we wrap the expression +%% in a block that returns nil. +wrap_expr_if_unused(Expr, false) -> {'__block__', [], [Expr, nil]}; +wrap_expr_if_unused(Expr, _) -> Expr. + +translate_gen(ForMeta, [{'<-', Meta, [Left, Right]} | T], Acc, S) -> + {TLeft, TRight, TFilters, TT, TS} = translate_gen(Meta, Left, Right, T, S), + TAcc = [{enum, Meta, TLeft, TRight, TFilters} | Acc], + translate_gen(ForMeta, TT, TAcc, TS); +translate_gen(ForMeta, [{'<<>>', _, [{'<-', Meta, [Left, Right]}]} | T], Acc, S) -> + {TLeft, TRight, TFilters, TT, TS} = translate_gen(Meta, Left, Right, T, S), + TAcc = [{bin, Meta, TLeft, TRight, TFilters} | Acc], + translate_gen(ForMeta, TT, TAcc, TS); +translate_gen(_ForMeta, [], Acc, S) -> + {lists:reverse(Acc), S}. + +translate_gen(Meta, Left, Right, T, S) -> + Ann = ?ann(Meta), + {TRight, SR} = elixir_erl_pass:translate(Right, Ann, S), + {LeftArgs, LeftGuards} = elixir_utils:extract_guards(Left), + {TLeft, SL} = elixir_erl_clauses:match(Ann, fun elixir_erl_pass:translate/3, LeftArgs, + SR#elixir_erl{extra=pin_guard}), + + TLeftGuards = elixir_erl_clauses:guards(Ann, LeftGuards, [], SL), + ExtraGuards = [{nil, X} || X <- SL#elixir_erl.extra_guards], + {Filters, TT} = collect_filters(T, []), + + {TFilters, TS} = + lists:mapfoldr( + fun(F, SF) -> translate_filter(F, Ann, SF) end, + SL#elixir_erl{extra=S#elixir_erl.extra, extra_guards=[]}, + Filters + ), + + %% The list of guards is kept in reverse order + Guards = TFilters ++ translate_guards(TLeftGuards) ++ ExtraGuards, + {TLeft, TRight, Guards, TT, TS}. + +translate_guards([]) -> + []; +translate_guards([[Guards]]) -> + [{nil, Guards}]; +translate_guards([[Left], [Right] | Rest]) -> + translate_guards([[{op, element(2, Left), 'orelse', Left, Right}] | Rest]). + +translate_filter(Filter, Ann, S) -> + {TFilter, TS} = elixir_erl_pass:translate(Filter, Ann, S), + case elixir_utils:returns_boolean(Filter) of + true -> + {{nil, TFilter}, TS}; + false -> + {Name, VS} = elixir_erl_var:build('_', TS), + {{{var, 0, Name}, TFilter}, VS} + end. + +collect_filters([{'<-', _, [_, _]} | _] = T, Acc) -> + {Acc, T}; +collect_filters([{'<<>>', _, [{'<-', _, [_, _]}]} | _] = T, Acc) -> + {Acc, T}; +collect_filters([H | T], Acc) -> + collect_filters(T, [H | Acc]); +collect_filters([], Acc) -> + {Acc, []}. + +build_inline(Ann, Clauses, Expr, Into, Uniq, S) -> + case not Uniq and lists:all(fun(Clause) -> element(1, Clause) == bin end, Clauses) of + true -> {build_comprehension(Ann, Clauses, Expr, Into), S}; + false -> build_inline_each(Ann, Clauses, Expr, Into, Uniq, S) + end. + +build_inline_each(Ann, Clauses, Expr, false, Uniq, S) -> + InnerFun = fun(InnerExpr, _InnerAcc) -> InnerExpr end, + build_reduce(Ann, Clauses, InnerFun, Expr, {nil, Ann}, Uniq, S); +build_inline_each(Ann, Clauses, Expr, {nil, _} = Into, Uniq, S) -> + InnerFun = fun(InnerExpr, InnerAcc) -> {cons, Ann, InnerExpr, InnerAcc} end, + {ReduceExpr, SR} = build_reduce(Ann, Clauses, InnerFun, Expr, Into, Uniq, S), + {?remote(Ann, lists, reverse, [ReduceExpr]), SR}; +build_inline_each(Ann, Clauses, Expr, {bin, _, []}, Uniq, S) -> + {InnerValue, SV} = build_var(Ann, S), + Generated = erl_anno:set_generated(true, Ann), + + InnerFun = fun(InnerExpr, InnerAcc) -> + {'case', Ann, InnerExpr, [ + {clause, Generated, + [InnerValue], + [[?remote(Ann, erlang, is_bitstring, [InnerValue]), + ?remote(Ann, erlang, is_list, [InnerAcc])]], + [{cons, Generated, InnerAcc, InnerValue}]}, + {clause, Generated, + [InnerValue], + [], + [?remote(Ann, erlang, error, [{tuple, Ann, [{atom, Ann, badarg}, InnerValue]}])]} + ]} + end, + + {ReduceExpr, SR} = build_reduce(Ann, Clauses, InnerFun, Expr, {nil, Ann}, Uniq, SV), + {?remote(Ann, erlang, list_to_bitstring, [ReduceExpr]), SR}. + +build_into(Ann, Clauses, Expr, {map, _, []}, Uniq, S) -> + {ReduceExpr, SR} = build_inline_each(Ann, Clauses, Expr, {nil, Ann}, Uniq, S), + {?remote(Ann, maps, from_list, [ReduceExpr]), SR}; +build_into(Ann, Clauses, Expr, Into, Uniq, S) -> + {Fun, SF} = build_var(Ann, S), + {Acc, SA} = build_var(Ann, SF), + {Kind, SK} = build_var(Ann, SA), + {Reason, SR} = build_var(Ann, SK), + {Stack, ST} = build_var(Ann, SR), + {Done, SD} = build_var(Ann, ST), + + InnerFun = fun(InnerExpr, InnerAcc) -> + {call, Ann, Fun, [InnerAcc, pair(Ann, cont, InnerExpr)]} + end, + + MatchExpr = {match, Ann, + {tuple, Ann, [Acc, Fun]}, + ?remote(Ann, 'Elixir.Collectable', into, [Into]) + }, + + {IntoReduceExpr, SN} = build_reduce(Ann, Clauses, InnerFun, Expr, Acc, Uniq, SD), + + TryExpr = + {'try', Ann, + [IntoReduceExpr], + [{clause, Ann, + [Done], + [], + [{call, Ann, Fun, [Done, {atom, Ann, done}]}]}], + [stacktrace_clause(Ann, Fun, Acc, Kind, Reason, Stack)], + []}, + + {{block, Ann, [MatchExpr, TryExpr]}, SN}. + +stacktrace_clause(Ann, Fun, Acc, Kind, Reason, Stack) -> + {clause, Ann, + [{tuple, Ann, [Kind, Reason, Stack]}], + [], + [{call, Ann, Fun, [Acc, {atom, Ann, halt}]}, + ?remote(Ann, erlang, raise, [Kind, Reason, Stack])]}. + +%% Helpers + +build_reduce(Ann, Clauses, InnerFun, Expr, Into, false, S) -> + {Acc, SA} = build_var(Ann, S), + {build_reduce_each(Clauses, InnerFun(Expr, Acc), Into, Acc, SA), SA}; +build_reduce(Ann, Clauses, InnerFun, Expr, Into, true, S) -> + %% Those variables are used only inside the anonymous function + %% so we don't need to worry about returning the scope. + {Acc, SA} = build_var(Ann, S), + {Value, SV} = build_var(Ann, SA), + {IntoAcc, SI} = build_var(Ann, SV), + {UniqAcc, SU} = build_var(Ann, SI), + + NewInto = {tuple, Ann, [Into, {map, Ann, []}]}, + AccTuple = {tuple, Ann, [IntoAcc, UniqAcc]}, + PutUniqExpr = {map, Ann, UniqAcc, [{map_field_assoc, Ann, Value, {atom, Ann, true}}]}, + + InnerExpr = {block, Ann, [ + {match, Ann, AccTuple, Acc}, + {match, Ann, Value, Expr}, + {'case', Ann, UniqAcc, [ + {clause, Ann, [{map, Ann, [{map_field_exact, Ann, Value, {atom, Ann, true}}]}], [], [AccTuple]}, + {clause, Ann, [{map, Ann, []}], [], [{tuple, Ann, [InnerFun(Value, IntoAcc), PutUniqExpr]}]} + ]} + ]}, + + EnumReduceCall = build_reduce_each(Clauses, InnerExpr, NewInto, Acc, SU), + {?remote(Ann, erlang, element, [{integer, Ann, 1}, EnumReduceCall]), SU}. + +build_reduce_each([{enum, Meta, Left, Right, Filters} | T], Expr, Arg, Acc, S) -> + Ann = ?ann(Meta), + True = build_reduce_each(T, Expr, Acc, Acc, S), + False = Acc, + Generated = erl_anno:set_generated(true, Ann), + + Clauses0 = + case is_var(Left) of + true -> []; + false -> + [{clause, Generated, + [{var, Ann, '_'}, Acc], [], + [False]}] + end, + + Clauses1 = + [{clause, Ann, + [Left, Acc], [], + [join_filters(Generated, Filters, True, False)]} | Clauses0], + + Args = [Right, Arg, {'fun', Ann, {clauses, Clauses1}}], + ?remote(Ann, 'Elixir.Enum', reduce, Args); + +build_reduce_each([{bin, Meta, Left, Right, Filters} | T], Expr, Arg, Acc, S) -> + Ann = ?ann(Meta), + Generated = erl_anno:set_generated(true, Ann), + {Tail, ST} = build_var(Ann, S), + {Fun, SF} = build_var(Ann, ST), + + True = build_reduce_each(T, Expr, Acc, Acc, SF), + False = Acc, + {bin, _, Elements} = Left, + TailElement = {bin_element, Ann, Tail, default, [bitstring]}, + + Clauses = + [{clause, Generated, + [{bin, Ann, [TailElement]}, Acc], [], + [Acc]}, + {clause, Generated, + [Tail, {var, Ann, '_'}], [], + [?remote(Ann, erlang, error, [pair(Ann, badarg, Tail)])]}], + + NoVarClauses = + case no_var(Generated, Elements) of + error -> + Clauses; + + NoVarElements -> + NoVarMatch = {bin, Ann, NoVarElements ++ [TailElement]}, + [{clause, Generated, [NoVarMatch, Acc], [], [{call, Ann, Fun, [Tail, False]}]} | Clauses] + end, + + BinMatch = {bin, Ann, Elements ++ [TailElement]}, + VarClauses = + [{clause, Ann, + [BinMatch, Acc], [], + [{call, Ann, Fun, [Tail, join_filters(Generated, Filters, True, False)]}]} | NoVarClauses], + + {call, Ann, + {named_fun, Ann, element(3, Fun), VarClauses}, + [Right, Arg]}; + +build_reduce_each([], Expr, _Arg, _Acc, _S) -> + Expr. + +is_var({var, _, _}) -> true; +is_var(_) -> false. + +pair(Ann, Atom, Arg) -> + {tuple, Ann, [{atom, Ann, Atom}, Arg]}. + +build_var(Ann, S) -> + {Name, ST} = elixir_erl_var:build('_', S), + {{var, Ann, Name}, ST}. + +no_var(ParentAnn, Elements) -> + try + [{bin_element, Ann, NoVarExpr, no_var_size(Size), Types} || + {bin_element, Ann, Expr, Size, Types} <- Elements, + NoVarExpr <- no_var_expr(ParentAnn, Expr)] + catch + unbound_size -> error + end. + +no_var_expr(Ann, {string, _, String}) -> [{var, Ann, '_'} || _ <- String]; +no_var_expr(Ann, _) -> [{var, Ann, '_'}]. +no_var_size({var, _, _}) -> throw(unbound_size); +no_var_size(Size) -> Size. + +build_comprehension(Ann, Clauses, Expr, Into) -> + {comprehension_kind(Into), Ann, Expr, comprehension_clause(Clauses)}. + +comprehension_clause([{bin, Meta, Left, Right, Filters} | T]) -> + Ann = ?ann(Meta), + [{b_generate, Ann, Left, Right}] ++ + comprehension_filter(Ann, Filters) ++ + comprehension_clause(T); +comprehension_clause([]) -> + []. + +comprehension_kind(false) -> lc; +comprehension_kind({nil, _}) -> lc; +comprehension_kind({bin, _, []}) -> bc. + +inline_or_into({bin, _, []}) -> inline; +inline_or_into({nil, _}) -> inline; +inline_or_into(false) -> inline; +inline_or_into(_) -> into. + +comprehension_filter(Ann, Filters) -> + [join_filter(Ann, Filter, {atom, Ann, true}, {atom, Ann, false}) || + Filter <- lists:reverse(Filters)]. + +join_filters(_Ann, [], True, _False) -> + True; +join_filters(Ann, [H | T], True, False) -> + lists:foldl(fun(Filter, Acc) -> + join_filter(Ann, Filter, Acc, False) + end, join_filter(Ann, H, True, False), T). + +join_filter(Ann, {nil, Filter}, True, False) -> + {'case', Ann, Filter, [ + {clause, Ann, [{atom, Ann, true}], [], [True]}, + {clause, Ann, [{atom, Ann, false}], [], [False]} + ]}; +join_filter(Ann, {Var, Filter}, True, False) -> + Guards = [ + [{op, Ann, '==', Var, {atom, Ann, false}}], + [{op, Ann, '==', Var, {atom, Ann, nil}}] + ], + + {'case', Ann, Filter, [ + {clause, Ann, [Var], Guards, [False]}, + {clause, Ann, [{var, Ann, '_'}], [], [True]} + ]}. diff --git a/lib/elixir/src/elixir_erl_pass.erl b/lib/elixir/src/elixir_erl_pass.erl new file mode 100644 index 00000000000..d562fca0c86 --- /dev/null +++ b/lib/elixir/src/elixir_erl_pass.erl @@ -0,0 +1,628 @@ +%% Translate Elixir quoted expressions to Erlang Abstract Format. +-module(elixir_erl_pass). +-export([translate/3, translate_args/3]). +-include("elixir.hrl"). + +%% = + +translate({'=', Meta, [{'_', _, Atom}, Right]}, _Ann, S) when is_atom(Atom) -> + Ann = ?ann(Meta), + {TRight, SR} = translate(Right, Ann, S), + {{match, Ann, {var, Ann, '_'}, TRight}, SR}; + +translate({'=', Meta, [Left, Right]}, _Ann, S) -> + Ann = ?ann(Meta), + {TRight, SR} = translate(Right, Ann, S), + case elixir_erl_clauses:match(Ann, fun translate/3, Left, SR) of + {TLeft, #elixir_erl{extra_guards=ExtraGuards, context=Context} = SL0} + when ExtraGuards =/= [], Context =/= match -> + SL1 = SL0#elixir_erl{extra_guards=[]}, + {ResultVarName, SL2} = elixir_erl_var:build('_', SL1), + Match = {match, Ann, TLeft, TRight}, + Generated = erl_anno:set_generated(true, Ann), + ResultVar = {var, Generated, ResultVarName}, + ResultMatch = {match, Generated, ResultVar, Match}, + True = {atom, Generated, true}, + Reason = {tuple, Generated, [{atom, Generated, badmatch}, ResultVar]}, + RaiseExpr = ?remote(Generated, erlang, error, [Reason]), + GuardsExp = {'if', Generated, [ + {clause, Generated, [], [ExtraGuards], [ResultVar]}, + {clause, Generated, [], [[True]], [RaiseExpr]} + ]}, + {{block, Generated, [ResultMatch, GuardsExp]}, SL2}; + + {TLeft, SL} -> + {{match, Ann, TLeft, TRight}, SL} + end; + +%% Containers + +translate({'{}', Meta, Args}, _Ann, S) when is_list(Args) -> + Ann = ?ann(Meta), + {TArgs, SE} = translate_args(Args, Ann, S), + {{tuple, Ann, TArgs}, SE}; + +translate({'%{}', Meta, Args}, _Ann, S) when is_list(Args) -> + translate_map(?ann(Meta), Args, S); + +translate({'%', Meta, [{'^', _, [{Name, _, Context}]} = Left, Right]}, _Ann, S) when is_atom(Name), is_atom(Context) -> + translate_struct_var_name(?ann(Meta), Left, Right, S); + +translate({'%', Meta, [{Name, _, Context} = Left, Right]}, _Ann, S) when is_atom(Name), is_atom(Context) -> + translate_struct_var_name(?ann(Meta), Left, Right, S); + +translate({'%', Meta, [Left, Right]}, _Ann, S) -> + translate_struct(?ann(Meta), Left, Right, S); + +translate({'<<>>', Meta, Args}, _Ann, S) when is_list(Args) -> + translate_bitstring(Meta, Args, S); + +%% Blocks + +translate({'__block__', Meta, Args}, _Ann, S) when is_list(Args) -> + Ann = ?ann(Meta), + {TArgs, SA} = translate_block(Args, Ann, [], S), + {{block, Ann, lists:reverse(TArgs)}, SA}; + +%% Compilation environment macros + +translate({'__CALLER__', Meta, Atom}, _Ann, S) when is_atom(Atom) -> + {{var, ?ann(Meta), '__CALLER__'}, S#elixir_erl{caller=true}}; + +translate({'__STACKTRACE__', Meta, Atom}, _Ann, S = #elixir_erl{stacktrace=Var}) when is_atom(Atom) -> + {{var, ?ann(Meta), Var}, S}; + +translate({'super', Meta, Args}, _Ann, S) when is_list(Args) -> + Ann = ?ann(Meta), + + %% In the expanded AST, super is used to invoke a function + %% in the current module originated from a default clause + %% or a super call. + {TArgs, SA} = translate_args(Args, Ann, S), + {super, {Kind, Name}} = lists:keyfind(super, 1, Meta), + + if + Kind == defmacro; Kind == defmacrop -> + MacroName = elixir_utils:macro_name(Name), + {{call, Ann, {atom, Ann, MacroName}, [{var, Ann, '__CALLER__'} | TArgs]}, SA#elixir_erl{caller=true}}; + Kind == def; Kind == defp -> + {{call, Ann, {atom, Ann, Name}, TArgs}, SA} + end; + +%% Functions + +translate({'&', Meta, [{'/', _, [{{'.', _, [Remote, Fun]}, _, []}, Arity]}]}, _Ann, S) + when is_atom(Fun), is_integer(Arity) -> + Ann = ?ann(Meta), + {TRemote, SR} = translate(Remote, Ann, S), + TFun = {atom, Ann, Fun}, + TArity = {integer, Ann, Arity}, + {{'fun', Ann, {function, TRemote, TFun, TArity}}, SR}; +translate({'&', Meta, [{'/', _, [{Fun, _, Atom}, Arity]}]}, Ann, S) + when is_atom(Fun), is_atom(Atom), is_integer(Arity) -> + case S of + #elixir_erl{expand_captures=true} -> + {Vars, SV} = lists:mapfoldl(fun(_, Acc) -> + {Var, _, AccS} = elixir_erl_var:assign(Meta, Acc), + {Var, AccS} + end, S, tl(lists:seq(0, Arity))), + translate({'fn', Meta, [{'->', Meta, [Vars, {Fun, Meta, Vars}]}]}, Ann, SV); + + #elixir_erl{expand_captures=false} -> + {{'fun', ?ann(Meta), {function, Fun, Arity}}, S} + end; + +translate({fn, Meta, Clauses}, _Ann, S) -> + Transformer = fun({'->', CMeta, [ArgsWithGuards, Expr]}, Acc) -> + {Args, Guards} = elixir_utils:extract_splat_guards(ArgsWithGuards), + elixir_erl_clauses:clause(?ann(CMeta), fun translate_fn_match/3, Args, Expr, Guards, Acc) + end, + {TClauses, NS} = lists:mapfoldl(Transformer, S, Clauses), + {{'fun', ?ann(Meta), {clauses, TClauses}}, NS}; + +%% Cond + +translate({'cond', CondMeta, [[{do, Clauses}]]}, Ann, S) -> + [{'->', Meta, [[Condition], _]} = H | T] = lists:reverse(Clauses), + + FirstMeta = + if + is_atom(Condition), Condition /= false, Condition /= nil -> ?generated(Meta); + true -> Meta + end, + + Error = {{'.', Meta, [erlang, error]}, Meta, [cond_clause]}, + {Case, SC} = build_cond_clauses([H | T], Error, FirstMeta, S), + translate(replace_case_meta(CondMeta, Case), Ann, SC); + +%% Case + +translate({'case', Meta, [Expr, Opts]}, _Ann, S) -> + translate_case(Meta, Expr, Opts, S); + +%% Try + +translate({'try', Meta, [Opts]}, _Ann, S) -> + Ann = ?ann(Meta), + Do = proplists:get_value('do', Opts, nil), + {TDo, SB} = translate(Do, Ann, S), + + Catch = [Tuple || {X, _} = Tuple <- Opts, X == 'rescue' orelse X == 'catch'], + {TCatch, SC} = elixir_erl_try:clauses(Ann, Catch, SB), + + {TAfter, SA} = case lists:keyfind('after', 1, Opts) of + {'after', After} -> + {TBlock, SAExtracted} = translate(After, Ann, SC), + {unblock(TBlock), SAExtracted}; + false -> + {[], SC} + end, + + Else = elixir_erl_clauses:get_clauses(else, Opts, match), + {TElse, SE} = elixir_erl_clauses:clauses(Else, SA), + {{'try', ?ann(Meta), unblock(TDo), TElse, TCatch, TAfter}, SE}; + +%% Receive + +translate({'receive', Meta, [Opts]}, _Ann, S) -> + Do = elixir_erl_clauses:get_clauses(do, Opts, match), + + case lists:keyfind('after', 1, Opts) of + false -> + {TClauses, SC} = elixir_erl_clauses:clauses(Do, S), + {{'receive', ?ann(Meta), TClauses}, SC}; + _ -> + After = elixir_erl_clauses:get_clauses('after', Opts, expr), + {TClauses, SC} = elixir_erl_clauses:clauses(Do ++ After, S), + {FClauses, TAfter} = elixir_utils:split_last(TClauses), + {_, _, [FExpr], _, FAfter} = TAfter, + {{'receive', ?ann(Meta), FClauses, FExpr, FAfter}, SC} + end; + +%% Comprehensions and with + +translate({for, Meta, [_ | _] = Args}, _Ann, S) -> + elixir_erl_for:translate(Meta, Args, true, S); + +translate({with, Meta, [_ | _] = Args}, _Ann, S) -> + {Exprs, [{do, Do} | Opts]} = elixir_utils:split_last(Args), + {ElseClause, SE} = translate_with_else(Meta, Opts, S), + translate_with_do(Exprs, ?ann(Meta), Do, ElseClause, SE); + +%% Variables + +translate({'^', _, [{Name, VarMeta, Kind}]}, _Ann, #elixir_erl{context=match} = S) when is_atom(Name), is_atom(Kind) -> + {Var, VS} = elixir_erl_var:translate(VarMeta, Name, Kind, S), + + case S#elixir_erl.extra of + pin_guard -> + {PinVarName, PS} = elixir_erl_var:build('_', VS), + Ann = ?ann(?generated(VarMeta)), + PinVar = {var, Ann, PinVarName}, + Guard = {op, Ann, '=:=', Var, PinVar}, + {PinVar, PS#elixir_erl{extra_guards=[Guard | PS#elixir_erl.extra_guards]}}; + _ -> + {Var, VS} + end; + +translate({Name, Meta, Kind}, _Ann, S) when is_atom(Name), is_atom(Kind) -> + elixir_erl_var:translate(Meta, Name, Kind, S); + +%% Local calls + +translate({Name, Meta, Args}, _Ann, S) when is_atom(Name), is_list(Meta), is_list(Args) -> + Ann = ?ann(Meta), + {TArgs, NS} = translate_args(Args, Ann, S), + {{call, Ann, {atom, Ann, Name}, TArgs}, NS}; + +%% Remote calls + +translate({{'.', _, [Left, Right]}, Meta, []}, _Ann, #elixir_erl{context=guard} = S) + when is_tuple(Left), is_atom(Right), is_list(Meta) -> + Ann = ?ann(Meta), + {TLeft, SL} = translate(Left, Ann, S), + TRight = {atom, Ann, Right}, + {?remote(Ann, erlang, map_get, [TRight, TLeft]), SL}; + +translate({{'.', _, [Left, Right]}, Meta, []}, _Ann, S) when is_tuple(Left), is_atom(Right), is_list(Meta) -> + Ann = ?ann(Meta), + {TLeft, SL} = translate(Left, Ann, S), + TRight = {atom, Ann, Right}, + + Generated = erl_anno:set_generated(true, Ann), + {Var, SV} = elixir_erl_var:build('_', SL), + TVar = {var, Generated, Var}, + TError = {tuple, Ann, [{atom, Ann, badkey}, TRight, TVar]}, + + {{'case', Generated, TLeft, [ + {clause, Generated, + [{map, Ann, [{map_field_exact, Ann, TRight, TVar}]}], + [], + [TVar]}, + {clause, Generated, + [TVar], + [[?remote(Generated, erlang, is_map, [TVar])]], + [?remote(Ann, erlang, error, [TError])]}, + {clause, Generated, + [TVar], + [], + [{call, Generated, {remote, Generated, TVar, TRight}, []}]} + ]}, SV}; + +translate({{'.', _, [Left, Right]}, Meta, Args}, _Ann, S) + when (is_tuple(Left) orelse is_atom(Left)), is_atom(Right), is_list(Meta), is_list(Args) -> + translate_remote(Left, Right, Meta, Args, S); + +%% Anonymous function calls + +translate({{'.', _, [Expr]}, Meta, Args}, _Ann, S) when is_list(Args) -> + Ann = ?ann(Meta), + {TExpr, SE} = translate(Expr, Ann, S), + {TArgs, SA} = translate_args(Args, Ann, SE), + {{call, Ann, TExpr, TArgs}, SA}; + +%% Literals + +translate({Left, Right}, Ann, S) -> + {TLeft, SL} = translate(Left, Ann, S), + {TRight, SR} = translate(Right, Ann, SL), + {{tuple, Ann, [TLeft, TRight]}, SR}; + +translate(List, Ann, S) when is_list(List) -> + translate_list(List, Ann, [], S); + +translate(Other, Ann, S) -> + {elixir_erl:elixir_to_erl(Other, Ann), S}. + +%% Helpers + +translate_case(Meta, Expr, Opts, S) -> + Ann = ?ann(Meta), + Clauses = elixir_erl_clauses:get_clauses(do, Opts, match), + {TExpr, SE} = translate(Expr, Ann, S), + {TClauses, SC} = elixir_erl_clauses:clauses(Clauses, SE), + {{'case', Ann, TExpr, TClauses}, SC}. + +translate_list([{'|', _, [Left, Right]}], Ann, List, Acc) -> + {TLeft, LAcc} = translate(Left, Ann, Acc), + {TRight, TAcc} = translate(Right, Ann, LAcc), + {build_list([TLeft | List], TRight, Ann), TAcc}; +translate_list([H | T], Ann, List, Acc) -> + {TH, TAcc} = translate(H, Ann, Acc), + translate_list(T, Ann, [TH | List], TAcc); +translate_list([], Ann, List, Acc) -> + {build_list(List, {nil, Ann}, Ann), Acc}. + +build_list([H | T], Acc, Ann) -> + build_list(T, {cons, Ann, H, Acc}, Ann); +build_list([], Acc, _Ann) -> + Acc. + +%% Pack a list of expressions from a block. +unblock({'block', _, Exprs}) -> Exprs; +unblock(Expr) -> [Expr]. + +translate_fn_match(Arg, Ann, S) -> + {TArg, TS} = translate_args(Arg, Ann, S#elixir_erl{extra=pin_guard}), + {TArg, TS#elixir_erl{extra=S#elixir_erl.extra}}. + +%% Translate args + +translate_args(Args, Ann, S) -> + lists:mapfoldl(fun + (Arg, SA) when is_list(Arg) -> + translate_list(Arg, Ann, [], SA); + (Arg, SA) when is_tuple(Arg) -> + translate(Arg, Ann, SA); + (Arg, SA) -> + {elixir_erl:elixir_to_erl(Arg, Ann), SA} + end, S, Args). + +%% Translate blocks + +translate_block([], _Ann, Acc, S) -> + {Acc, S}; +translate_block([H], Ann, Acc, S) -> + {TH, TS} = translate(H, Ann, S), + translate_block([], Ann, [TH | Acc], TS); +translate_block([{'__block__', Meta, Args} | T], Ann, Acc, S) when is_list(Args) -> + {TAcc, SA} = translate_block(Args, ?ann(Meta), Acc, S), + translate_block(T, Ann, TAcc, SA); +translate_block([{for, Meta, [_ | _] = Args} | T], Ann, Acc, S) -> + {TH, TS} = elixir_erl_for:translate(Meta, Args, false, S), + translate_block(T, Ann, [TH | Acc], TS); +translate_block([{'=', _, [{'_', _, Ctx}, {for, Meta, [_ | _] = Args}]} | T], Ann, Acc, S) when is_atom(Ctx) -> + {TH, TS} = elixir_erl_for:translate(Meta, Args, false, S), + translate_block(T, Ann, [TH | Acc], TS); +translate_block([H | T], Ann, Acc, S) -> + {TH, TS} = translate(H, Ann, S), + translate_block(T, Ann, [TH | Acc], TS). + +%% Cond + +build_cond_clauses([{'->', NewMeta, [[Condition], Body]} | T], Acc, OldMeta, S) -> + {NewCondition, Truthy, Other, ST} = build_truthy_clause(NewMeta, Condition, Body, S), + Falsy = {'->', OldMeta, [[Other], Acc]}, + Case = {'case', NewMeta, [NewCondition, [{do, [Truthy, Falsy]}]]}, + build_cond_clauses(T, Case, NewMeta, ST); +build_cond_clauses([], Acc, _, S) -> + {Acc, S}. + +replace_case_meta(Meta, {'case', _, Args}) -> + {'case', Meta, Args}; +replace_case_meta(_Meta, Other) -> + Other. + +build_truthy_clause(Meta, Condition, Body, S) -> + case returns_boolean(Condition, Body) of + {NewCondition, NewBody} -> + {NewCondition, {'->', Meta, [[true], NewBody]}, false, S}; + false -> + {Var, _, SV} = elixir_erl_var:assign(Meta, S), + Head = {'when', [], [Var, + {{'.', [], [erlang, 'andalso']}, [], [ + {{'.', [], [erlang, '/=']}, [], [Var, nil]}, + {{'.', [], [erlang, '/=']}, [], [Var, false]} + ]} + ]}, + {Condition, {'->', Meta, [[Head], Body]}, {'_', [], nil}, SV} + end. + +%% In case a variable is defined to match in a condition +%% but a condition returns boolean, we can replace the +%% variable directly by the boolean result. +returns_boolean({'=', _, [{Var, _, Ctx}, Condition]}, {Var, _, Ctx}) when is_atom(Var), is_atom(Ctx) -> + case elixir_utils:returns_boolean(Condition) of + true -> {Condition, true}; + false -> false + end; + +%% For all other cases, we check the condition but +%% return both condition and body untouched. +returns_boolean(Condition, Body) -> + case elixir_utils:returns_boolean(Condition) of + true -> {Condition, Body}; + false -> false + end. + +%% with + +translate_with_else(Meta, [], S) -> + Generated = ?ann(?generated(Meta)), + {VarName, SC} = elixir_erl_var:build('_', S), + Var = {var, Generated, VarName}, + {{clause, Generated, [Var], [], [Var]}, SC}; +translate_with_else(Meta, [{else, [{'->', _, [[{Var, VarMeta, Kind}], Clause]}]}], S) when is_atom(Var), is_atom(Kind) -> + Ann = ?ann(Meta), + Generated = erl_anno:set_generated(true, Ann), + {ElseVarErl, SV} = elixir_erl_var:translate(VarMeta, Var, Kind, S#elixir_erl{context=match}), + {TranslatedClause, SC} = translate(Clause, Ann, SV#elixir_erl{context=nil}), + {{clause, Generated, [ElseVarErl], [], [TranslatedClause]}, SC}; +translate_with_else(Meta, [{else, Else}], S) -> + Generated = ?generated(Meta), + {ElseVarEx, ElseVarErl, SE} = elixir_erl_var:assign(Generated, S), + {RaiseVar, _, SV} = elixir_erl_var:assign(Generated, SE), + + RaiseExpr = {{'.', Generated, [erlang, error]}, Generated, [{with_clause, RaiseVar}]}, + RaiseClause = {'->', Generated, [[RaiseVar], RaiseExpr]}, + GeneratedElse = [build_generated_clause(Generated, ElseClause) || ElseClause <- Else], + + Case = {'case', Generated, [ElseVarEx, [{do, GeneratedElse ++ [RaiseClause]}]]}, + {TranslatedCase, SC} = translate(Case, ?ann(Meta), SV), + {{clause, ?ann(Generated), [ElseVarErl], [], [TranslatedCase]}, SC}. + +build_generated_clause(Generated, {'->', _, [Args, Clause]}) -> + NewArgs = [build_generated_clause_arg(Generated, Arg) || Arg <- Args], + {'->', Generated, [NewArgs, Clause]}. + +build_generated_clause_arg(Generated, Arg) -> + {Expr, Guards} = elixir_utils:extract_guards(Arg), + NewGuards = [build_generated_guard(Generated, Guard) || Guard <- Guards], + concat_guards(Generated, Expr, NewGuards). + +build_generated_guard(Generated, {{'.', _, _} = Call, _, Args}) -> + {Call, Generated, [build_generated_guard(Generated, Arg) || Arg <- Args]}; +build_generated_guard(_, Expr) -> + Expr. + +concat_guards(_Meta, Expr, []) -> + Expr; +concat_guards(Meta, Expr, [Guard | Tail]) -> + {'when', Meta, [Expr, concat_guards(Meta, Guard, Tail)]}. + +translate_with_do([{'<-', Meta, [Left, Expr]} | Rest], _Ann, Do, Else, S) -> + Ann = ?ann(Meta), + {Args, Guards} = elixir_utils:extract_guards(Left), + {TExpr, SR} = translate(Expr, Ann, S), + {TArgs, SA} = elixir_erl_clauses:match(Ann, fun translate/3, Args, SR), + TGuards = elixir_erl_clauses:guards(Ann, Guards, SA#elixir_erl.extra_guards, SA), + {TBody, SB} = translate_with_do(Rest, Ann, Do, Else, SA#elixir_erl{extra_guards=[]}), + + Clause = {clause, Ann, [TArgs], TGuards, unblock(TBody)}, + {{'case', erl_anno:set_generated(true, Ann), TExpr, [Clause, Else]}, SB}; +translate_with_do([Expr | Rest], Ann, Do, Else, S) -> + {TExpr, TS} = translate(Expr, Ann, S), + {TRest, RS} = translate_with_do(Rest, Ann, Do, Else, TS), + {{block, Ann, [TExpr | unblock(TRest)]}, RS}; +translate_with_do([], Ann, Do, _Else, S) -> + translate(Do, Ann, S). + +%% Maps and structs + +translate_struct_var_name(Ann, Name, Args, S0) -> + {{map, MapAnn, TArgs0}, S1} = translate_struct(Ann, Name, Args, S0), + {TArgs1, S2} = generate_struct_name_guard(TArgs0, [], S1), + {{map, MapAnn, TArgs1}, S2}. + +translate_struct(Ann, Name, {'%{}', _, [{'|', _, [Update, Assocs]}]}, S) -> + Generated = erl_anno:set_generated(true, Ann), + {VarName, VS} = elixir_erl_var:build('_', S), + + Var = {var, Ann, VarName}, + Map = {map, Ann, [{map_field_exact, Ann, {atom, Ann, '__struct__'}, {atom, Ann, Name}}]}, + + Match = {match, Ann, Var, Map}, + Error = {tuple, Ann, [{atom, Ann, badstruct}, {atom, Ann, Name}, Var]}, + + {TUpdate, TU} = translate(Update, Ann, VS), + {TAssocs, TS} = translate_map(Ann, Assocs, {ok, Var}, TU), + + {{'case', Generated, TUpdate, [ + {clause, Ann, [Match], [], [TAssocs]}, + {clause, Generated, [Var], [], [?remote(Ann, erlang, error, [Error])]} + ]}, TS}; +translate_struct(Ann, Name, {'%{}', _, Assocs}, S) -> + translate_map(Ann, [{'__struct__', Name}] ++ Assocs, none, S). + +translate_map(Ann, [{'|', Meta, [Update, Assocs]}], S) -> + {TUpdate, SU} = translate(Update, Ann, S), + translate_map(?ann(Meta), Assocs, {ok, TUpdate}, SU); +translate_map(Ann, Assocs, S) -> + translate_map(Ann, Assocs, none, S). + +translate_map(Ann, Assocs, TUpdate, #elixir_erl{extra=Extra} = S) -> + Op = translate_key_val_op(TUpdate, S), + + {TArgs, SA} = lists:mapfoldl(fun({Key, Value}, Acc0) -> + {TKey, Acc1} = translate(Key, Ann, Acc0#elixir_erl{extra=map_key}), + {TValue, Acc2} = translate(Value, Ann, Acc1#elixir_erl{extra=Extra}), + {{Op, Ann, TKey, TValue}, Acc2} + end, S, Assocs), + + build_map(Ann, TUpdate, TArgs, SA). + +translate_key_val_op(_TUpdate, #elixir_erl{extra=map_key}) -> map_field_assoc; +translate_key_val_op(_TUpdate, #elixir_erl{context=match}) -> map_field_exact; +translate_key_val_op(none, _) -> map_field_assoc; +translate_key_val_op(_, _) -> map_field_exact. + +build_map(Ann, {ok, TUpdate}, TArgs, SA) -> {{map, Ann, TUpdate, TArgs}, SA}; +build_map(Ann, none, TArgs, SA) -> {{map, Ann, TArgs}, SA}. + +%% Translate bitstrings + +translate_bitstring(Meta, Args, S) -> + build_bitstr(Args, ?ann(Meta), S, []). + +build_bitstr([{'::', Meta, [H, V]} | T], Ann, S, Acc) -> + {Size, Types} = extract_bit_info(V, Meta, S#elixir_erl{context=nil}), + build_bitstr(T, Ann, S, Acc, H, Size, Types); +build_bitstr([], Ann, S, Acc) -> + {{bin, Ann, lists:reverse(Acc)}, S}. + +build_bitstr(T, Ann, S, Acc, H, default, Types) when is_binary(H) -> + Element = + case requires_utf_conversion(Types) of + false -> + %% See explanation in elixir_erl:elixir_to_erl/1 to + %% know why we can simply convert the binary to a list. + {bin_element, Ann, {string, 0, binary_to_list(H)}, default, default}; + true -> + %% UTF types require conversion. + {bin_element, Ann, {string, 0, elixir_utils:characters_to_list(H)}, default, Types} + end, + build_bitstr(T, Ann, S, [Element | Acc]); + +build_bitstr(T, Ann, S, Acc, H, Size, Types) -> + {Expr, NS} = translate(H, Ann, S), + build_bitstr(T, Ann, NS, [{bin_element, Ann, Expr, Size, Types} | Acc]). + +requires_utf_conversion([bitstring | _]) -> false; +requires_utf_conversion([binary | _]) -> false; +requires_utf_conversion(_) -> true. + +extract_bit_info({'-', _, [L, {size, _, [Size]}]}, Meta, S) -> + {extract_bit_size(Size, Meta, S), extract_bit_type(L, [])}; +extract_bit_info({size, _, [Size]}, Meta, S) -> + {extract_bit_size(Size, Meta, S), []}; +extract_bit_info(L, _Meta, _S) -> + {default, extract_bit_type(L, [])}. + +extract_bit_size(Size, Meta, S) -> + {TSize, _} = translate(Size, ?ann(Meta), S#elixir_erl{context=guard}), + TSize. + +extract_bit_type({'-', _, [L, R]}, Acc) -> + extract_bit_type(L, extract_bit_type(R, Acc)); +extract_bit_type({unit, _, [Arg]}, Acc) -> + [{unit, Arg} | Acc]; +extract_bit_type({Other, _, []}, Acc) -> + [Other | Acc]. + +%% Optimizations that are specific to Erlang and change +%% the format of the AST. + +translate_remote('Elixir.String.Chars', to_string, Meta, [Arg], S) -> + Ann = ?ann(Meta), + {TArg, TS} = translate(Arg, Ann, S), + {VarName, VS} = elixir_erl_var:build('_', TS), + + Generated = erl_anno:set_generated(true, Ann), + Var = {var, Generated, VarName}, + Guard = ?remote(Generated, erlang, is_binary, [Var]), + Slow = ?remote(Generated, 'Elixir.String.Chars', to_string, [Var]), + Fast = Var, + + {{'case', Generated, TArg, [ + {clause, Generated, [Var], [[Guard]], [Fast]}, + {clause, Generated, [Var], [], [Slow]} + ]}, VS}; +translate_remote(maps, put, Meta, [Key, Value, Map], S) -> + Ann = ?ann(Meta), + + case translate_args([Key, Value, Map], Ann, S) of + {[TKey, TValue, {map, _, InnerMap, Pairs}], TS} -> + {{map, Ann, InnerMap, Pairs ++ [{map_field_assoc, Ann, TKey, TValue}]}, TS}; + + {[TKey, TValue, {map, _, Pairs}], TS} -> + {{map, Ann, Pairs ++ [{map_field_assoc, Ann, TKey, TValue}]}, TS}; + + {[TKey, TValue, TMap], TS} -> + {{map, Ann, TMap, [{map_field_assoc, Ann, TKey, TValue}]}, TS} + end; +translate_remote(maps, merge, Meta, [Map1, Map2], S) -> + Ann = ?ann(Meta), + + case translate_args([Map1, Map2], Ann, S) of + {[{map, _, Pairs1}, {map, _, Pairs2}], TS} -> + {{map, Ann, Pairs1 ++ Pairs2}, TS}; + + {[{map, _, InnerMap1, Pairs1}, {map, _, Pairs2}], TS} -> + {{map, Ann, InnerMap1, Pairs1 ++ Pairs2}, TS}; + + {[TMap1, {map, _, Pairs2}], TS} -> + {{map, Ann, TMap1, Pairs2}, TS}; + + {[TMap1, TMap2], TS} -> + {{call, Ann, {remote, Ann, {atom, Ann, maps}, {atom, Ann, merge}}, [TMap1, TMap2]}, TS} + end; +translate_remote(Left, Right, Meta, Args, S) -> + Ann = ?ann(Meta), + {TLeft, SL} = translate(Left, Ann, S), + {TArgs, SA} = translate_args(Args, Ann, SL), + + Arity = length(Args), + TRight = {atom, Ann, Right}, + + %% Rewrite Erlang function calls as operators so they + %% work in guards, matches and so on. + case (Left == erlang) andalso elixir_utils:guard_op(Right, Arity) of + true -> + case TArgs of + [TOne] -> {{op, Ann, Right, TOne}, SA}; + [TOne, TTwo] -> {{op, Ann, Right, TOne, TTwo}, SA} + end; + false -> + {{call, Ann, {remote, Ann, TLeft, TRight}, TArgs}, SA} + end. + +generate_struct_name_guard([{map_field_exact, Ann, {atom, _, '__struct__'} = Key, Var} | Rest], Acc, S0) -> + {ModuleVarName, S1} = elixir_erl_var:build('_', S0), + Generated = erl_anno:set_generated(true, Ann), + ModuleVar = {var, Generated, ModuleVarName}, + Match = {match, Generated, ModuleVar, Var}, + Guard = ?remote(Generated, erlang, is_atom, [ModuleVar]), + S2 = S1#elixir_erl{extra_guards=[Guard | S1#elixir_erl.extra_guards]}, + {lists:reverse(Acc, [{map_field_exact, Ann, Key, Match} | Rest]), S2}; +generate_struct_name_guard([Field | Rest], Acc, S) -> + generate_struct_name_guard(Rest, [Field | Acc], S). diff --git a/lib/elixir/src/elixir_erl_try.erl b/lib/elixir/src/elixir_erl_try.erl new file mode 100644 index 00000000000..24aaabebe3a --- /dev/null +++ b/lib/elixir/src/elixir_erl_try.erl @@ -0,0 +1,241 @@ +-module(elixir_erl_try). +-export([clauses/3]). +-include("elixir.hrl"). +-define(REQUIRES_STACKTRACE, + ['Elixir.FunctionClauseError', 'Elixir.UndefinedFunctionError', + 'Elixir.KeyError', 'Elixir.ArgumentError', 'Elixir.SystemLimitError']). + +clauses(_Ann, Args, S) -> + Catch = elixir_erl_clauses:get_clauses('catch', Args, 'catch'), + Rescue = elixir_erl_clauses:get_clauses(rescue, Args, rescue), + {StackName, SV} = elixir_erl_var:build('__STACKTRACE__', S), + OldStack = SV#elixir_erl.stacktrace, + SS = SV#elixir_erl{stacktrace=StackName}, + reduce_clauses(Rescue ++ Catch, [], OldStack, SS, SS). + +reduce_clauses([H | T], Acc, OldStack, SAcc, S) -> + {TH, TS} = each_clause(H, SAcc), + reduce_clauses(T, [TH | Acc], OldStack, TS, S); +reduce_clauses([], Acc, OldStack, SAcc, _S) -> + {lists:reverse(Acc), SAcc#elixir_erl{stacktrace=OldStack}}. + +each_clause({'catch', Meta, Raw, Expr}, S) -> + {Args, Guards} = elixir_utils:extract_splat_guards(Raw), + + Match = + case Args of + [X] -> [throw, X]; + [X, Y] -> [X, Y] + end, + + {{clause, Line, [TKind, TMatches], TGuards, TBody}, TS} = + elixir_erl_clauses:clause(?ann(Meta), fun elixir_erl_pass:translate_args/3, Match, Expr, Guards, S), + + build_clause(Line, TKind, TMatches, TGuards, TBody, TS); + +each_clause({rescue, Meta, [{in, _, [Left, Right]}], Expr}, S) -> + {TempVar, _, CS} = elixir_erl_var:assign(Meta, S), + {Guards, ErlangAliases} = rescue_guards(Meta, TempVar, Right), + Body = normalize_rescue(Meta, TempVar, Left, Expr, ErlangAliases), + build_rescue(Meta, TempVar, Guards, Body, CS); + +each_clause({rescue, Meta, [{VarName, _, Context} = Left], Expr}, S) when is_atom(VarName), is_atom(Context) -> + {TempVar, _, CS} = elixir_erl_var:assign(Meta, S), + Body = normalize_rescue(Meta, TempVar, Left, Expr, ['Elixir.ErlangError']), + build_rescue(Meta, TempVar, [], Body, CS). + +normalize_rescue(_Meta, _Var, {'_', _, Atom}, Expr, _) when is_atom(Atom) -> + Expr; +normalize_rescue(Meta, Var, Pattern, Expr, []) -> + prepend_to_block(Meta, {'=', Meta, [Pattern, Var]}, Expr); +normalize_rescue(Meta, Var, Pattern, Expr, ErlangAliases) -> + Stacktrace = + case lists:member('Elixir.ErlangError', ErlangAliases) of + true -> + dynamic_normalize(Meta, Var, ?REQUIRES_STACKTRACE); + + false -> + case lists:splitwith(fun is_normalized_with_stacktrace/1, ErlangAliases) of + {[], _} -> []; + {_, []} -> {'__STACKTRACE__', Meta, nil}; + {Some, _} -> dynamic_normalize(Meta, Var, Some) + end + end, + + Normalized = {{'.', Meta, ['Elixir.Exception', normalize]}, Meta, [error, Var, Stacktrace]}, + prepend_to_block(Meta, {'=', Meta, [Pattern, Normalized]}, Expr). + +dynamic_normalize(Meta, Var, [H | T]) -> + Generated = ?generated(Meta), + + Guards = + lists:foldl(fun(Alias, Acc) -> + {'when', Generated, [erl_rescue_stacktrace_for(Generated, Var, Alias), Acc]} + end, erl_rescue_stacktrace_for(Generated, Var, H), T), + + {'case', Generated, [ + Var, + [{do, [ + {'->', Generated, [[{'when', Generated, [{'_', Generated, nil}, Guards]}], {'__STACKTRACE__', Generated, nil}]}, + {'->', Generated, [[{'_', Generated, nil}], []]} + ]}] + ]}. + +erl_rescue_stacktrace_for(_Meta, _Var, 'Elixir.ErlangError') -> + %% ErlangError is a "meta" exception, we should never expand it here. + error(badarg); +erl_rescue_stacktrace_for(Meta, Var, 'Elixir.KeyError') -> + %% Only the two-element tuple requires stacktrace. + erl_and(Meta, erl_tuple_size(Meta, Var, 2), erl_record_compare(Meta, Var, badkey)); +erl_rescue_stacktrace_for(Meta, Var, Module) -> + erl_rescue_guard_for(Meta, Var, Module). + +is_normalized_with_stacktrace(Module) -> + lists:member(Module, ?REQUIRES_STACKTRACE). + +%% Helpers + +build_rescue(Meta, Var, Guards, Body, S) -> + {{clause, Line, [TMatch], TGuards, TBody}, TS} = + elixir_erl_clauses:clause(?ann(Meta), fun elixir_erl_pass:translate_args/3, [Var], Body, Guards, S), + + build_clause(Line, {atom, Line, error}, TMatch, TGuards, TBody, TS). + +%% Convert rescue clauses ("var in [alias1, alias2]") into guards. +rescue_guards(_Meta, _Var, []) -> + {[], []}; +rescue_guards(Meta, Var, Aliases) -> + {ErlangGuards, ErlangAliases} = erl_rescue(Meta, Var, Aliases, [], []), + + ElixirGuards = + [erl_and(Meta, + {erl(Meta, '=='), Meta, [{erl(Meta, map_get), Meta, ['__struct__', Var]}, Alias]}, + {erl(Meta, map_get), Meta, ['__exception__', Var]} + ) || Alias <- Aliases], + + {ElixirGuards ++ ErlangGuards, ErlangAliases}. + +build_clause(Line, Kind, Expr, Guards, Body, #elixir_erl{stacktrace=Var} = TS) -> + Match = {tuple, Line, [Kind, Expr, {var, Line, Var}]}, + {{clause, Line, [Match], Guards, Body}, TS}. + +%% Rescue each atom name considering their Erlang or Elixir matches. +%% Matching of variables is done with Erlang exceptions is done in +%% function for optimization. + +erl_rescue(Meta, Var, [H | T], Guards, Aliases) when is_atom(H) -> + case erl_rescue_guard_for(Meta, Var, H) of + false -> erl_rescue(Meta, Var, T, Guards, Aliases); + Expr -> erl_rescue(Meta, Var, T, [Expr | Guards], [H | Aliases]) + end; +erl_rescue(_, _, [], Guards, Aliases) -> + {Guards, Aliases}. + +%% Handle Erlang rescue matches. + +erl_rescue_guard_for(Meta, Var, 'Elixir.UndefinedFunctionError') -> + {erl(Meta, '=='), Meta, [Var, undef]}; + +erl_rescue_guard_for(Meta, Var, 'Elixir.FunctionClauseError') -> + {erl(Meta, '=='), Meta, [Var, function_clause]}; + +erl_rescue_guard_for(Meta, Var, 'Elixir.SystemLimitError') -> + {erl(Meta, '=='), Meta, [Var, system_limit]}; + +erl_rescue_guard_for(Meta, Var, 'Elixir.ArithmeticError') -> + {erl(Meta, '=='), Meta, [Var, badarith]}; + +erl_rescue_guard_for(Meta, Var, 'Elixir.CondClauseError') -> + {erl(Meta, '=='), Meta, [Var, cond_clause]}; + +erl_rescue_guard_for(Meta, Var, 'Elixir.BadArityError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, badarity)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.BadFunctionError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, badfun)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.MatchError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, badmatch)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.CaseClauseError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, case_clause)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.WithClauseError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, with_clause)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.TryClauseError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, try_clause)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.BadStructError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 3), + erl_record_compare(Meta, Var, badstruct)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.BadMapError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, badmap)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.BadBooleanError') -> + erl_and(Meta, + erl_tuple_size(Meta, Var, 3), + erl_record_compare(Meta, Var, badbool)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.KeyError') -> + erl_and(Meta, + erl_or(Meta, + erl_tuple_size(Meta, Var, 2), + erl_tuple_size(Meta, Var, 3)), + erl_record_compare(Meta, Var, badkey)); + +erl_rescue_guard_for(Meta, Var, 'Elixir.ArgumentError') -> + erl_or(Meta, + {erl(Meta, '=='), Meta, [Var, badarg]}, + erl_and(Meta, + erl_tuple_size(Meta, Var, 2), + erl_record_compare(Meta, Var, badarg))); + +erl_rescue_guard_for(Meta, Var, 'Elixir.ErlangError') -> + Condition = + erl_and( + Meta, + {erl(Meta, is_map), Meta, [Var]}, + {erl(Meta, is_map_key), Meta, ['__exception__', Var]} + ), + {erl(Meta, 'not'), Meta, [Condition]}; + +erl_rescue_guard_for(_, _, _) -> + false. + +%% Helpers + +erl_tuple_size(Meta, Var, Size) -> + {erl(Meta, '=='), Meta, [{erl(Meta, tuple_size), Meta, [Var]}, Size]}. + +erl_record_compare(Meta, Var, Expr) -> + {erl(Meta, '=='), Meta, [ + {erl(Meta, element), Meta, [1, Var]}, + Expr + ]}. + +prepend_to_block(_Meta, Expr, {'__block__', Meta, Args}) -> + {'__block__', Meta, [Expr | Args]}; + +prepend_to_block(Meta, Expr, Args) -> + {'__block__', Meta, [Expr, Args]}. + +erl(Meta, Op) -> {'.', Meta, [erlang, Op]}. +erl_or(Meta, Left, Right) -> {erl(Meta, 'orelse'), Meta, [Left, Right]}. +erl_and(Meta, Left, Right) -> {erl(Meta, 'andalso'), Meta, [Left, Right]}. diff --git a/lib/elixir/src/elixir_erl_var.erl b/lib/elixir/src/elixir_erl_var.erl new file mode 100644 index 00000000000..0bbc06ee52b --- /dev/null +++ b/lib/elixir/src/elixir_erl_var.erl @@ -0,0 +1,90 @@ +%% Convenience functions used to manipulate scope and its variables. +-module(elixir_erl_var). +-export([translate/4, assign/2, build/2, + load_binding/3, dump_binding/3 +]). +-include("elixir.hrl"). + +%% VAR HANDLING + +translate(Meta, '_', _Kind, S) -> + {{var, ?ann(Meta), '_'}, S}; + +translate(Meta, Name, Kind, #elixir_erl{var_names=VarNames} = S) -> + {version, Version} = lists:keyfind(version, 1, Meta), + + case VarNames of + #{Version := ErlName} -> {{var, ?ann(Meta), ErlName}, S}; + #{} when Kind /= nil -> assign(Meta, '_', Version, S); + #{} -> assign(Meta, Name, Version, S) + end. + +assign(Meta, #elixir_erl{var_names=VarNames} = S) -> + Version = -(map_size(VarNames)+1), + ExVar = {var, [{version, Version} | Meta], ?var_context}, + {ErlVar, SV} = assign(Meta, '_', Version, S), + {ExVar, ErlVar, SV}. + +assign(Meta, Name, Version, #elixir_erl{var_names=VarNames} = S) -> + {NewVar, NS} = build(Name, S), + NewVarNames = VarNames#{Version => NewVar}, + {{var, ?ann(Meta), NewVar}, NS#elixir_erl{var_names=NewVarNames}}. + +build(Key, #elixir_erl{counter=Counter} = S) -> + Count = + case Counter of + #{Key := Val} -> Val + 1; + _ -> 1 + end, + {build_name(Key, Count), + S#elixir_erl{counter=Counter#{Key => Count}}}. + +build_name('_', Count) -> list_to_atom("_@" ++ integer_to_list(Count)); +build_name(Name, Count) -> list_to_atom("_" ++ atom_to_list(Name) ++ "@" ++ integer_to_list(Count)). + +%% BINDINGS + +load_binding(Binding, #{versioned_vars := ExVars}, #elixir_erl{var_names=ErlVars}) -> + %% TODO: Remove me once we require Erlang/OTP 24+ + %% Also revisit dump_binding below and remove the vars field for simplicity. + Mod = + case erlang:system_info(otp_release) >= "24" of + true -> maps; + false -> orddict + end, + + KV = + lists:foldl(fun({Key, Value}, Acc) -> + Version = maps:get(Key, ExVars), + Name = maps:get(Version, ErlVars), + [{Name, Value} | Acc] + end, [], Binding), + + Mod:from_list(KV). + +dump_binding(Binding, #elixir_ex{vars={ExVars, _}}, #elixir_erl{var_names=ErlVars}) -> + maps:fold(fun + ({Var, Kind} = Pair, Version, Acc) when is_atom(Kind) -> + Key = case Kind of + nil -> Var; + _ -> Pair + end, + + ErlName = maps:get(Version, ErlVars), + Value = find_binding(ErlName, Binding), + [{Key, Value} | Acc]; + + (_, _, Acc) -> + Acc + end, [], ExVars). + +find_binding(ErlName, Binding = #{}) -> + case Binding of + #{ErlName := V} -> V; + _ -> nil + end; +find_binding(ErlName, Binding) -> + case orddict:find(ErlName, Binding) of + {ok, V} -> V; + error -> nil + end. diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index 84bbb6c7c3b..7b3255f3e02 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -1,217 +1,225 @@ -% A bunch of helpers to help to deal with errors in Elixir source code. -% This is not exposed in the Elixir language. +%% A bunch of helpers to help to deal with errors in Elixir source code. +%% This is not exposed in the Elixir language. +%% +%% Note that this is also called by the Erlang backend, so we also support +%% the line number to be none (as it may happen in some erlang errors). -module(elixir_errors). --export([compile_error/3, compile_error/4, - form_error/4, parse_error/4, warn/2, warn/3, - handle_file_warning/2, handle_file_warning/3, handle_file_error/2]). +-export([compile_error/3, compile_error/4, form_error/4, parse_error/5]). +-export([warning_prefix/0, erl_warn/3, print_warning/3, log_and_print_warning/4, form_warn/4]). -include("elixir.hrl"). +-type location() :: non_neg_integer() | {non_neg_integer(), non_neg_integer()}. + +%% Low-level warning, should be used only from Erlang passes. +-spec erl_warn(location() | none, unicode:chardata(), unicode:chardata()) -> ok. +erl_warn(none, File, Warning) -> + erl_warn(0, File, Warning); +erl_warn(Location, File, Warning) when is_binary(File) -> + send_warning(Location, File, Warning), + print_warning(Location, File, Warning). + +-spec print_warning(location(), unicode:chardata(), unicode:chardata()) -> ok. +print_warning(Location, File, Warning) -> + print_warning([Warning, "\n ", file_format(Location, File), $\n]). + +-spec log_and_print_warning(location(), unicode:chardata() | nil, unicode:chardata(), unicode:chardata()) -> ok. +log_and_print_warning(Location, File, LogMessage, PrintMessage) when is_binary(File) or (File == nil) -> + send_warning(Location, File, LogMessage), + print_warning(PrintMessage). + +-spec warning_prefix() -> binary(). +warning_prefix() -> + case application:get_env(elixir, ansi_enabled) of + {ok, true} -> <<"\e[33mwarning: \e[0m">>; + _ -> <<"warning: ">> + end. --type line_or_meta() :: integer() | list(). - -warn(Warning) -> - CompilerPid = get(elixir_compiler_pid), - if - CompilerPid =/= undefined -> - elixir_code_server:cast({register_warning, CompilerPid}); - true -> false - end, - io:put_chars(standard_error, Warning). - -warn(Caller, Warning) -> - warn([Caller, "warning: ", Warning]). +%% General forms handling. -warn(Line, File, Warning) when is_integer(Line) -> - warn(file_format(Line, File, "warning: " ++ Warning)). +-spec form_error(list(), binary() | #{file := binary(), _ => _}, module(), any()) -> no_return(). +form_error(Meta, #{file := File}, Module, Desc) -> + compile_error(Meta, File, Module:format_error(Desc)); +form_error(Meta, File, Module, Desc) -> + compile_error(Meta, File, Module:format_error(Desc)). + +-spec form_warn(list(), binary() | #{file := binary(), _ => _}, module(), any()) -> ok. +form_warn(Meta, File, Module, Desc) when is_list(Meta), is_binary(File) -> + form_warn(Meta, #{file => File}, Module, Desc); +form_warn(Meta, #{file := File} = E, Module, Desc) when is_list(Meta) -> + % Skip warnings during bootstrap, they will be reported during recompilation + case elixir_config:is_bootstrap() of + true -> ok; + false -> do_form_warn(Meta, File, E, Module:format_error(Desc)) + end. -%% Raised during expansion/translation/compilation. +do_form_warn(Meta, GivenFile, E, Warning) -> + [{file, File}, {line, Line}] = meta_location(Meta, GivenFile), + + Location = + case E of + #{function := {Name, Arity}, module := Module} -> + [file_format(Line, File), ": ", 'Elixir.Exception':format_mfa(Module, Name, Arity)]; + #{module := Module} when Module /= nil -> + [file_format(Line, File), ": ", elixir_aliases:inspect(Module)]; + #{} -> + file_format(Line, File) + end, --spec form_error(line_or_meta(), binary(), module(), any()) -> no_return(). + log_and_print_warning(Line, File, Warning, [Warning, "\n ", Location, $\n]). -form_error(Meta, File, Module, Desc) -> - compile_error(Meta, File, format_error(Module, Desc)). +%% Compilation error. --spec compile_error(line_or_meta(), binary(), iolist()) -> no_return(). --spec compile_error(line_or_meta(), binary(), iolist(), list()) -> no_return(). +-spec compile_error(list(), binary(), binary() | unicode:charlist()) -> no_return(). +-spec compile_error(list(), binary(), string(), list()) -> no_return(). +compile_error(Meta, File, Message) when is_binary(Message) -> + MetaLocation = meta_location(Meta, File), + raise('Elixir.CompileError', Message, MetaLocation); compile_error(Meta, File, Message) when is_list(Message) -> - raise(Meta, File, 'Elixir.CompileError', elixir_utils:characters_to_binary(Message)). + MetaLocation = meta_location(Meta, File), + raise('Elixir.CompileError', elixir_utils:characters_to_binary(Message), MetaLocation). compile_error(Meta, File, Format, Args) when is_list(Format) -> compile_error(Meta, File, io_lib:format(Format, Args)). -%% Raised on tokenizing/parsing +%% Tokenization parsing/errors. +snippet(InputString, Location, StartLine, StartColumn) -> + {line, Line} = lists:keyfind(line, 1, Location), + case lists:keyfind(column, 1, Location) of + {column, Column} -> + Lines = string:split(InputString, "\n", all), + Snippet = (lists:nth(Line - StartLine + 1, Lines)), + Offset = if Line == StartLine -> Column - StartColumn; true -> Column - 1 end, + case string:trim(Snippet, leading) of + [] -> nil; + _ -> #{content => elixir_utils:characters_to_binary(Snippet), offset => Offset} + end; --spec parse_error(line_or_meta(), binary(), binary(), binary()) -> no_return(). + false -> + nil + end. -parse_error(Meta, File, Error, <<>>) -> +-spec parse_error(elixir:keyword(), binary() | {binary(), binary()}, + binary(), binary(), {unicode:charlist(), integer(), integer()}) -> no_return(). +parse_error(Location, File, Error, <<>>, {InputString, StartLine, StartColumn}) when is_list(InputString) -> Message = case Error of <<"syntax error before: ">> -> <<"syntax error: expression is incomplete">>; - _ -> Error + _ -> <> end, - raise(Meta, File, 'Elixir.TokenMissingError', Message); - -%% Show a nicer message for missing end tokens -parse_error(Meta, File, <<"syntax error before: ">>, <<"'end'">>) -> - raise(Meta, File, 'Elixir.SyntaxError', <<"unexpected token: end">>); - -%% Binaries are wrapped in [<<...>>], so we need to unwrap them -parse_error(Meta, File, Error, <<"[", _/binary>> = Full) when is_binary(Error) -> - Rest = - case binary:split(Full, <<"<<">>) of - [Lead, Token] -> - case binary:split(Token, <<">>">>) of - [Part, _] when Lead == <<$[>> -> Part; - _ -> <<$">> - end; - [_] -> - <<$">> - end, - raise(Meta, File, 'Elixir.SyntaxError', <>); - -%% Everything else is fine as is -parse_error(Meta, File, Error, Token) when is_binary(Error), is_binary(Token) -> - Message = <>, - raise(Meta, File, 'Elixir.SyntaxError', Message). - -%% Handle warnings and errors (called during module compilation) - -%% Ignore on bootstrap -handle_file_warning(true, _File, {_Line, sys_core_fold, nomatch_guard}) -> []; -handle_file_warning(true, _File, {_Line, sys_core_fold, {nomatch_shadow, _}}) -> []; - -%% Ignore always -handle_file_warning(_, _File, {_Line, sys_core_fold, useless_building}) -> []; - -%% This is an Erlang bug, it considers {tuple, _}.call to always fail -handle_file_warning(_, _File, {_Line, v3_kernel, bad_call}) -> []; - -%% We handle unused local warnings ourselves -handle_file_warning(_, _File, {_Line, erl_lint, {unused_function, _}}) -> []; - -%% Make no_effect clauses pretty -handle_file_warning(_, File, {Line, sys_core_fold, {no_effect, {erlang, F, A}}}) -> - {Fmt, Args} = case erl_internal:comp_op(F, A) of - true -> {"use of operator ~ts has no effect", [translate_comp_op(F)]}; - false -> - case erl_internal:bif(F, A) of - false -> {"the call to :erlang.~ts/~B has no effect", [F,A]}; - true -> {"the call to ~ts/~B has no effect", [F,A]} - end + Snippet = snippet(InputString, Location, StartLine, StartColumn), + raise(Location, File, 'Elixir.TokenMissingError', Message, Snippet); + +%% Show a nicer message for end of line +parse_error(Location, File, <<"syntax error before: ">>, <<"eol">>, _Input) -> + raise(Location, File, 'Elixir.SyntaxError', + <<"unexpectedly reached end of line. The current expression is invalid or incomplete">>); + + +%% Show a nicer message for keywords pt1 (Erlang keywords show up wrapped in single quotes) +parse_error(Location, File, <<"syntax error before: ">>, Keyword, _Input) + when Keyword == <<"'not'">>; + Keyword == <<"'and'">>; + Keyword == <<"'or'">>; + Keyword == <<"'when'">>; + Keyword == <<"'after'">>; + Keyword == <<"'catch'">>; + Keyword == <<"'end'">> -> + raise_reserved(Location, File, binary_part(Keyword, 1, byte_size(Keyword) - 2)); + +%% Show a nicer message for keywords pt2 (Elixir keywords show up as is) +parse_error(Location, File, <<"syntax error before: ">>, Keyword, _Input) + when Keyword == <<"fn">>; + Keyword == <<"else">>; + Keyword == <<"rescue">>; + Keyword == <<"true">>; + Keyword == <<"false">>; + Keyword == <<"nil">>; + Keyword == <<"in">> -> + raise_reserved(Location, File, Keyword); + +%% Produce a human-readable message for errors before a sigil +parse_error(Location, File, <<"syntax error before: ">>, <<"{sigil,", _Rest/binary>> = Full, _Input) -> + {sigil, _, Sigil, [Content | _], _, _, _} = parse_erl_term(Full), + Content2 = case is_binary(Content) of + true -> Content; + false -> <<>> end, - Message = io_lib:format(Fmt, Args), - warn(Line, File, Message); - -%% Rewrite undefined behaviour to check for protocols -handle_file_warning(_, File, {Line,erl_lint,{undefined_behaviour_func,{Fun,Arity},Module}}) -> - {DefKind, Def, DefArity} = - case atom_to_list(Fun) of - "MACRO-" ++ Rest -> {macro, list_to_atom(Rest), Arity - 1}; - _ -> {function, Fun, Arity} - end, - - Kind = protocol_or_behaviour(Module), - Raw = "undefined ~ts ~ts ~ts/~B (for ~ts ~ts)", - Message = io_lib:format(Raw, [Kind, DefKind, Def, DefArity, Kind, elixir_aliases:inspect(Module)]), - warn(Line, File, Message); - -handle_file_warning(_, File, {Line,erl_lint,{undefined_behaviour,Module}}) -> - case elixir_compiler:get_opt(internal) of - true -> []; - false -> - Message = io_lib:format("behaviour ~ts undefined", [elixir_aliases:inspect(Module)]), - warn(Line, File, Message) - end; - -%% Ignore unused vars at "weird" lines (<= 0) -handle_file_warning(_, _File, {Line,erl_lint,{unused_var,_Var}}) when Line =< 0 -> - []; - -%% Ignore shadowed vars as we guarantee no conflicts ourselves -handle_file_warning(_, _File, {_Line,erl_lint,{shadowed_var,_Var,_Where}}) -> - []; - -%% Properly format other unused vars -handle_file_warning(_, File, {Line,erl_lint,{unused_var,Var}}) -> - Message = format_error(erl_lint, {unused_var, format_var(Var)}), - warn(Line, File, Message); - -%% Default behaviour -handle_file_warning(_, File, {Line,Module,Desc}) -> - Message = format_error(Module, Desc), - warn(Line, File, Message). - -handle_file_warning(File, Desc) -> - handle_file_warning(false, File, Desc). - --spec handle_file_error(file:filename_all(), {non_neg_integer(), module(), any()}) -> no_return(). - -handle_file_error(File, {Line,erl_lint,{unsafe_var,Var,{In,_Where}}}) -> - Translated = case In of - 'orelse' -> 'or'; - 'andalso' -> 'and'; - _ -> In + Message = <<"syntax error before: sigil \~", Sigil, " starting with content '", Content2/binary, "'">>, + raise(Location, File, 'Elixir.SyntaxError', Message); + +%% Binaries (and interpolation) are wrapped in [<<...>>] +parse_error(Location, File, Error, <<"[", _/binary>> = Full, _Input) when is_binary(Error) -> + Term = case parse_erl_term(Full) of + [H | _] when is_binary(H) -> <<$", H/binary, $">>; + _ -> <<$">> end, - Message = io_lib:format("cannot define variable ~ts inside ~ts", [format_var(Var), Translated]), - raise(Line, File, 'Elixir.CompileError', iolist_to_binary(Message)); + raise(Location, File, 'Elixir.SyntaxError', <>); + +%% Given a string prefix and suffix to insert the token inside the error message rather than append it +parse_error(Location, File, {ErrorPrefix, ErrorSuffix}, Token, _Input) when is_binary(ErrorPrefix), is_binary(ErrorSuffix), is_binary(Token) -> + Message = <>, + raise(Location, File, 'Elixir.SyntaxError', Message); + +%% Misplaced char tokens (for example, {char, _, 97}) are translated by Erlang into +%% the char literal (i.e., the token in the previous example becomes $a), +%% because {char, _, _} is a valid Erlang token for an Erlang char literal. We +%% want to represent that token as ?a in the error, according to the Elixir +%% syntax. +parse_error(Location, File, <<"syntax error before: ">>, <<$$, Char/binary>>, _Input) -> + Message = <<"syntax error before: ?", Char/binary>>, + raise(Location, File, 'Elixir.SyntaxError', Message); -handle_file_error(File, {Line,erl_lint,{spec_fun_undefined,{M,F,A}}}) -> - Message = io_lib:format("spec for undefined function ~ts.~ts/~B", [elixir_aliases:inspect(M), F, A]), - raise(Line, File, 'Elixir.CompileError', iolist_to_binary(Message)); - -handle_file_error(File, {Line,Module,Desc}) -> - form_error(Line, File, Module, Desc). +%% Everything else is fine as is +parse_error(Location, File, Error, Token, {InputString, StartLine, StartColumn}) when is_binary(Error), is_binary(Token) -> + Message = <>, + Snippet = snippet(InputString, Location, StartLine, StartColumn), + raise(Location, File, 'Elixir.SyntaxError', Message, Snippet). + +parse_erl_term(Term) -> + {ok, Tokens, _} = erl_scan:string(binary_to_list(Term)), + {ok, Parsed} = erl_parse:parse_term(Tokens ++ [{dot, 1}]), + Parsed. + +raise_reserved(Location, File, Keyword) -> + raise(Location, File, 'Elixir.SyntaxError', + <<"syntax error before: ", Keyword/binary, ". \"", Keyword/binary, "\" is a " + "reserved word in Elixir and therefore its usage is limited. For instance, " + "it can't be used as a variable or be defined nor invoked as a regular function">>). %% Helpers -raise(Meta, File, Kind, Message) when is_list(Meta) -> - raise(?line(Meta), File, Kind, Message); +print_warning(Message) -> + io:put_chars(standard_error, [warning_prefix(), Message, $\n]), + ok. -raise(none, File, Kind, Message) -> - raise(0, File, Kind, Message); - -raise(Line, File, Kind, Message) when is_integer(Line), is_binary(File) -> - %% Populate the stacktrace so we can raise it - try - throw(ok) - catch - ok -> ok +send_warning(Line, File, Message) -> + case get(elixir_compiler_info) of + undefined -> ok; + {CompilerPid, _} -> CompilerPid ! {warning, File, Line, Message} end, - Stacktrace = erlang:get_stacktrace(), - Exception = Kind:exception([{description, Message}, {file, File}, {line, Line}]), - erlang:raise(error, Exception, tl(Stacktrace)). - -file_format(0, File, Message) when is_binary(File) -> - io_lib:format("~ts: ~ts~n", [elixir_utils:relative_to_cwd(File), Message]); - -file_format(Line, File, Message) when is_binary(File) -> - io_lib:format("~ts:~w: ~ts~n", [elixir_utils:relative_to_cwd(File), Line, Message]). - -format_var(Var) -> - list_to_atom(lists:takewhile(fun(X) -> X /= $@ end, atom_to_list(Var))). - -format_error([], Desc) -> - io_lib:format("~p", [Desc]); - -format_error(Module, Desc) -> - Module:format_error(Desc). - -protocol_or_behaviour(Module) -> - case is_protocol(Module) of - true -> protocol; - false -> behaviour + ok. + +file_format({0, _Column}, File) -> + io_lib:format("~ts", [elixir_utils:relative_to_cwd(File)]); +file_format({Line, Column}, File) -> + io_lib:format("~ts:~w:~w", [elixir_utils:relative_to_cwd(File), Line, Column]); +file_format(0, File) -> + io_lib:format("~ts", [elixir_utils:relative_to_cwd(File)]); +file_format(Line, File) -> + io_lib:format("~ts:~w", [elixir_utils:relative_to_cwd(File), Line]). + +meta_location(Meta, File) -> + case elixir_utils:meta_keep(Meta) of + {F, L} -> [{file, F}, {line, L}]; + nil -> [{file, File}, {line, ?line(Meta)}] end. -is_protocol(Module) -> - case code:ensure_loaded(Module) of - {module, _} -> - erlang:function_exported(Module, '__protocol__', 1) andalso - Module:'__protocol__'(name) == Module; - {error, _} -> - false - end. +raise(Location, File, Kind, Message, Snippet) when is_binary(File) -> + raise(Kind, Message, [{file, File}, {snippet, Snippet} | Location]). -translate_comp_op('/=') -> '!='; -translate_comp_op('=<') -> '<='; -translate_comp_op('=:=') -> '==='; -translate_comp_op('=/=') -> '!=='; -translate_comp_op(Other) -> Other. +raise(Location, File, Kind, Message) when is_binary(File) -> + raise(Kind, Message, [{file, File} | Location]). + +raise(Kind, Message, Opts) when is_binary(Message) -> + Stacktrace = try throw(ok) catch _:_:Stack -> Stack end, + Exception = Kind:exception([{description, Message} | Opts]), + erlang:raise(error, Exception, tl(Stacktrace)). diff --git a/lib/elixir/src/elixir_exp.erl b/lib/elixir/src/elixir_exp.erl deleted file mode 100644 index ba786b4b885..00000000000 --- a/lib/elixir/src/elixir_exp.erl +++ /dev/null @@ -1,571 +0,0 @@ --module(elixir_exp). --export([expand/2, expand_args/2, expand_arg/2]). --import(elixir_errors, [compile_error/3, compile_error/4]). --include("elixir.hrl"). - -%% = - -expand({'=', Meta, [Left, Right]}, E) -> - assert_no_guard_scope(Meta, '=', E), - {ERight, ER} = expand(Right, E), - {ELeft, EL} = elixir_exp_clauses:match(fun expand/2, Left, E), - {{'=', Meta, [ELeft, ERight]}, elixir_env:mergev(EL, ER)}; - -%% Literal operators - -expand({'{}', Meta, Args}, E) -> - {EArgs, EA} = expand_args(Args, E), - {{'{}', Meta, EArgs}, EA}; - -expand({'%{}', Meta, Args}, E) -> - elixir_map:expand_map(Meta, Args, E); - -expand({'%', Meta, [Left, Right]}, E) -> - elixir_map:expand_struct(Meta, Left, Right, E); - -expand({'<<>>', Meta, Args}, E) -> - elixir_bitstring:expand(Meta, Args, E); - -%% Other operators - -expand({'__op__', Meta, [_, _] = Args}, E) -> - {EArgs, EA} = expand_args(Args, E), - {{'__op__', Meta, EArgs}, EA}; - -expand({'__op__', Meta, [_, _, _] = Args}, E) -> - {EArgs, EA} = expand_args(Args, E), - {{'__op__', Meta, EArgs}, EA}; - -expand({'->', Meta, _Args}, E) -> - compile_error(Meta, ?m(E, file), "unhandled operator ->"); - -%% __block__ - -expand({'__block__', _Meta, []}, E) -> - {nil, E}; -expand({'__block__', _Meta, [Arg]}, E) -> - expand(Arg, E); -expand({'__block__', Meta, Args}, E) when is_list(Args) -> - {EArgs, EA} = expand_many(Args, E), - {{'__block__', Meta, EArgs}, EA}; - -%% __aliases__ - -expand({'__aliases__', _, _} = Alias, E) -> - case elixir_aliases:expand(Alias, ?m(E, aliases), - ?m(E, macro_aliases), ?m(E, lexical_tracker)) of - Receiver when is_atom(Receiver) -> - elixir_lexical:record_remote(Receiver, ?m(E, lexical_tracker)), - {Receiver, E}; - Aliases -> - {EAliases, EA} = expand_args(Aliases, E), - - case lists:all(fun is_atom/1, EAliases) of - true -> - Receiver = elixir_aliases:concat(EAliases), - elixir_lexical:record_remote(Receiver, ?m(E, lexical_tracker)), - {Receiver, EA}; - false -> - {{{'.', [], [elixir_aliases, concat]}, [], [EAliases]}, EA} - end - end; - -%% alias - -expand({alias, Meta, [Ref]}, E) -> - expand({alias, Meta, [Ref,[]]}, E); -expand({alias, Meta, [Ref, KV]}, E) -> - assert_no_match_or_guard_scope(Meta, alias, E), - {ERef, ER} = expand(Ref, E), - {EKV, ET} = expand_opts(Meta, alias, [as, warn], no_alias_opts(KV), ER), - - if - is_atom(ERef) -> - {{alias, Meta, [ERef, EKV]}, - expand_alias(Meta, true, ERef, EKV, ET)}; - true -> - compile_error(Meta, ?m(E, file), - "invalid argument for alias, expected a compile time atom or alias, got: ~ts", - ['Elixir.Kernel':inspect(ERef)]) - end; - -expand({require, Meta, [Ref]}, E) -> - expand({require, Meta, [Ref, []]}, E); -expand({require, Meta, [Ref, KV]}, E) -> - assert_no_match_or_guard_scope(Meta, require, E), - - {ERef, ER} = expand(Ref, E), - {EKV, ET} = expand_opts(Meta, require, [as, warn], no_alias_opts(KV), ER), - - if - is_atom(ERef) -> - elixir_aliases:ensure_loaded(Meta, ERef, ET), - {{require, Meta, [ERef, EKV]}, - expand_require(Meta, ERef, EKV, ET)}; - true -> - compile_error(Meta, ?m(E, file), - "invalid argument for require, expected a compile time atom or alias, got: ~ts", - ['Elixir.Kernel':inspect(ERef)]) - end; - -expand({import, Meta, [Left]}, E) -> - expand({import, Meta, [Left, []]}, E); - -expand({import, Meta, [Ref, KV]}, E) -> - assert_no_match_or_guard_scope(Meta, import, E), - {ERef, ER} = expand(Ref, E), - {EKV, ET} = expand_opts(Meta, import, [only, except, warn], KV, ER), - - if - is_atom(ERef) -> - elixir_aliases:ensure_loaded(Meta, ERef, ET), - {Functions, Macros} = elixir_import:import(Meta, ERef, EKV, ET), - {{import, Meta, [ERef, EKV]}, - expand_require(Meta, ERef, EKV, ET#{functions := Functions, macros := Macros})}; - true -> - compile_error(Meta, ?m(E, file), - "invalid argument for import, expected a compile time atom or alias, got: ~ts", - ['Elixir.Kernel':inspect(ERef)]) - end; - -%% Pseudo vars - -expand({'__MODULE__', _, Atom}, E) when is_atom(Atom) -> - {?m(E, module), E}; -expand({'__DIR__', _, Atom}, E) when is_atom(Atom) -> - {filename:dirname(?m(E, file)), E}; -expand({'__CALLER__', _, Atom} = Caller, E) when is_atom(Atom) -> - {Caller, E}; -expand({'__ENV__', Meta, Atom}, E) when is_atom(Atom) -> - Env = elixir_env:linify({?line(Meta), E}), - {{'%{}', [], maps:to_list(Env)}, E}; -expand({{'.', DotMeta, [{'__ENV__', Meta, Atom}, Field]}, CallMeta, []}, E) when is_atom(Atom), is_atom(Field) -> - Env = elixir_env:linify({?line(Meta), E}), - case maps:is_key(Field, Env) of - true -> {maps:get(Field, Env), E}; - false -> {{{'.', DotMeta, [{'%{}', [], maps:to_list(Env)}, Field]}, CallMeta, []}, E} - end; - -%% Quote - -expand({Unquote, Meta, [_]}, E) when Unquote == unquote; Unquote == unquote_splicing -> - compile_error(Meta, ?m(E, file), "~p called outside quote", [Unquote]); - -expand({quote, Meta, [Opts]}, E) when is_list(Opts) -> - case lists:keyfind(do, 1, Opts) of - {do, Do} -> - expand({quote, Meta, [lists:keydelete(do, 1, Opts), [{do,Do}]]}, E); - false -> - compile_error(Meta, ?m(E, file), "missing do keyword in quote") - end; - -expand({quote, Meta, [_]}, E) -> - compile_error(Meta, ?m(E, file), "invalid arguments for quote"); - -expand({quote, Meta, [KV, Do]}, E) when is_list(Do) -> - Exprs = - case lists:keyfind(do, 1, Do) of - {do, Expr} -> Expr; - false -> compile_error(Meta, E#elixir_scope.file, "missing do keyword in quote") - end, - - ValidOpts = [context, location, line, unquote, bind_quoted], - {EKV, ET} = expand_opts(Meta, quote, ValidOpts, KV, E), - - Context = case lists:keyfind(context, 1, EKV) of - {context, Ctx} when is_atom(Ctx) and (Ctx /= nil) -> - Ctx; - {context, Ctx} -> - compile_error(Meta, ?m(E, file), "invalid :context for quote, " - "expected non nil compile time atom or alias, got: ~ts", ['Elixir.Kernel':inspect(Ctx)]); - false -> - case ?m(E, module) of - nil -> 'Elixir'; - Mod -> Mod - end - end, - - Keep = lists:keyfind(location, 1, EKV) == {location, keep}, - Line = proplists:get_value(line, EKV, false), - - {Binding, DefaultUnquote} = case lists:keyfind(bind_quoted, 1, EKV) of - {bind_quoted, BQ} -> {BQ, false}; - false -> {nil, true} - end, - - Unquote = case lists:keyfind(unquote, 1, EKV) of - {unquote, Bool} when is_boolean(Bool) -> Bool; - false -> DefaultUnquote - end, - - Q = #elixir_quote{line=Line, keep=Keep, unquote=Unquote, context=Context}, - - {Quoted, _Q} = elixir_quote:quote(Exprs, Binding, Q, ET), - expand(Quoted, ET); - -expand({quote, Meta, [_, _]}, E) -> - compile_error(Meta, ?m(E, file), "invalid arguments for quote"); - -%% Functions - -expand({'&', _, [Arg]} = Original, E) when is_integer(Arg) -> - {Original, E}; -expand({'&', Meta, [Arg]}, E) -> - assert_no_match_or_guard_scope(Meta, '&', E), - case elixir_fn:capture(Meta, Arg, E) of - {local, Fun, Arity} -> - {{'&', Meta, [{'/', [], [{Fun, [], nil}, Arity]}]}, E}; - {expanded, Expr, EE} -> - expand(Expr, EE) - end; - -expand({fn, Meta, Pairs}, E) -> - assert_no_match_or_guard_scope(Meta, fn, E), - elixir_fn:expand(Meta, Pairs, E); - -%% Case/Receive/Try - -expand({'cond', Meta, [KV]}, E) -> - assert_no_match_or_guard_scope(Meta, 'cond', E), - {EClauses, EC} = elixir_exp_clauses:'cond'(Meta, KV, E), - {{'cond', Meta, [EClauses]}, EC}; - -expand({'case', Meta, [Expr, KV]}, E) -> - assert_no_match_or_guard_scope(Meta, 'case', E), - {EExpr, EE} = expand(Expr, E), - {EClauses, EC} = elixir_exp_clauses:'case'(Meta, KV, EE), - FClauses = - case (lists:keyfind(optimize_boolean, 1, Meta) == {optimize_boolean, true}) and - elixir_utils:returns_boolean(EExpr) of - true -> rewrite_case_clauses(EClauses); - false -> EClauses - end, - {{'case', Meta, [EExpr, FClauses]}, EC}; - -expand({'receive', Meta, [KV]}, E) -> - assert_no_match_or_guard_scope(Meta, 'receive', E), - {EClauses, EC} = elixir_exp_clauses:'receive'(Meta, KV, E), - {{'receive', Meta, [EClauses]}, EC}; - -expand({'try', Meta, [KV]}, E) -> - assert_no_match_or_guard_scope(Meta, 'try', E), - {EClauses, EC} = elixir_exp_clauses:'try'(Meta, KV, E), - {{'try', Meta, [EClauses]}, EC}; - -%% Comprehensions - -expand({for, Meta, [_|_] = Args}, E) -> - elixir_for:expand(Meta, Args, E); - -%% Super - -expand({super, Meta, Args}, E) when is_list(Args) -> - assert_no_match_or_guard_scope(Meta, super, E), - {EArgs, EA} = expand_args(Args, E), - {{super, Meta, EArgs}, EA}; - -%% Vars - -expand({'^', Meta, [Arg]}, #{context := match} = E) -> - case expand(Arg, E) of - {{Name, _, Kind} = EArg, EA} when is_atom(Name), is_atom(Kind) -> - {{'^', Meta, [EArg]}, EA}; - _ -> - Msg = "invalid argument for unary operator ^, expected an existing variable, got: ^~ts", - compile_error(Meta, ?m(E, file), Msg, ['Elixir.Macro':to_string(Arg)]) - end; -expand({'^', Meta, [Arg]}, E) -> - compile_error(Meta, ?m(E, file), - "cannot use ^~ts outside of match clauses", ['Elixir.Macro':to_string(Arg)]); - -expand({'_', _, Kind} = Var, E) when is_atom(Kind) -> - {Var, E}; -expand({Name, Meta, Kind} = Var, #{context := match, export_vars := Export} = E) when is_atom(Name), is_atom(Kind) -> - Pair = {Name, var_kind(Meta, Kind)}, - NewVars = ordsets:add_element(Pair, ?m(E, vars)), - NewExport = case (Export /= nil) of - true -> ordsets:add_element(Pair, Export); - false -> Export - end, - {Var, E#{vars := NewVars, export_vars := NewExport}}; -expand({Name, Meta, Kind} = Var, #{vars := Vars} = E) when is_atom(Name), is_atom(Kind) -> - case lists:member({Name, var_kind(Meta, Kind)}, Vars) of - true -> - {Var, E}; - false -> - VarMeta = lists:keyfind(var, 1, Meta), - if - VarMeta == {var, true} -> - Extra = case Kind of - nil -> ""; - _ -> io_lib:format(" (context ~ts)", [elixir_aliases:inspect(Kind)]) - end, - - compile_error(Meta, ?m(E, file), "expected var ~ts~ts to expand to an existing " - "variable or be a part of a match", [Name, Extra]); - true -> - expand({Name, Meta, []}, E) - end - end; - -%% Local calls - -expand({Atom, Meta, Args}, E) when is_atom(Atom), is_list(Meta), is_list(Args) -> - assert_no_ambiguous_op(Atom, Meta, Args, E), - - elixir_dispatch:dispatch_import(Meta, Atom, Args, E, fun() -> - expand_local(Meta, Atom, Args, E) - end); - -%% Remote calls - -expand({{'.', DotMeta, [Left, Right]}, Meta, Args}, E) - when (is_tuple(Left) orelse is_atom(Left)), is_atom(Right), is_list(Meta), is_list(Args) -> - {ELeft, EL} = expand(Left, E), - - elixir_dispatch:dispatch_require(Meta, ELeft, Right, Args, EL, fun(AR, AF, AA) -> - expand_remote(AR, DotMeta, AF, Meta, AA, E, EL) - end); - -%% Anonymous calls - -expand({{'.', DotMeta, [Expr]}, Meta, Args}, E) when is_list(Args) -> - {EExpr, EE} = expand(Expr, E), - if - is_atom(EExpr) -> - compile_error(Meta, ?m(E, file), "invalid function call :~ts.()", [EExpr]); - true -> - {EArgs, EA} = expand_args(Args, elixir_env:mergea(E, EE)), - {{{'.', DotMeta, [EExpr]}, Meta, EArgs}, elixir_env:mergev(EE, EA)} - end; - -%% Invalid calls - -expand({_, Meta, Args} = Invalid, E) when is_list(Meta) and is_list(Args) -> - compile_error(Meta, ?m(E, file), "invalid call ~ts", - ['Elixir.Macro':to_string(Invalid)]); - -expand({_, _, _} = Tuple, E) -> - compile_error([{line,0}], ?m(E, file), "invalid quoted expression: ~ts", - ['Elixir.Kernel':inspect(Tuple, [{records,false}])]); - -%% Literals - -expand({Left, Right}, E) -> - {[ELeft, ERight], EE} = expand_args([Left, Right], E), - {{ELeft, ERight}, EE}; - -expand(List, #{context := match} = E) when is_list(List) -> - expand_list(List, fun expand/2, E, []); - -expand(List, E) when is_list(List) -> - {EArgs, {EC, EV}} = expand_list(List, fun expand_arg/2, {E, E}, []), - {EArgs, elixir_env:mergea(EV, EC)}; - -expand(Function, E) when is_function(Function) -> - case (erlang:fun_info(Function, type) == {type, external}) andalso - (erlang:fun_info(Function, env) == {env, []}) of - true -> - {Function, E}; - false -> - compile_error([{line,0}], ?m(E, file), - "invalid quoted expression: ~ts", ['Elixir.Kernel':inspect(Function)]) - end; - -expand(Other, E) when is_number(Other); is_atom(Other); is_binary(Other); is_pid(Other) -> - {Other, E}; - -expand(Other, E) -> - compile_error([{line,0}], ?m(E, file), - "invalid quoted expression: ~ts", ['Elixir.Kernel':inspect(Other)]). - -%% Helpers - -expand_list([{'|', Meta, [_, _] = Args}], Fun, Acc, List) -> - {EArgs, EAcc} = lists:mapfoldl(Fun, Acc, Args), - expand_list([], Fun, EAcc, [{'|', Meta, EArgs}|List]); -expand_list([H|T], Fun, Acc, List) -> - {EArg, EAcc} = Fun(H, Acc), - expand_list(T, Fun, EAcc, [EArg|List]); -expand_list([], _Fun, Acc, List) -> - {lists:reverse(List), Acc}. - -expand_many(Args, E) -> - lists:mapfoldl(fun expand/2, E, Args). - -%% Variables in arguments are not propagated from one -%% argument to the other. For instance: -%% -%% x = 1 -%% foo(x = x + 2, x) -%% x -%% -%% Should be the same as: -%% -%% foo(3, 1) -%% 3 -%% -%% However, lexical information is. -expand_arg(Arg, Acc) when is_number(Arg); is_atom(Arg); is_binary(Arg); is_pid(Arg) -> - {Arg, Acc}; -expand_arg(Arg, {Acc1, Acc2}) -> - {EArg, EAcc} = expand(Arg, Acc1), - {EArg, {elixir_env:mergea(Acc1, EAcc), elixir_env:mergev(Acc2, EAcc)}}. - -expand_args([Arg], E) -> - {EArg, EE} = expand(Arg, E), - {[EArg], EE}; -expand_args(Args, #{context := match} = E) -> - expand_many(Args, E); -expand_args(Args, E) -> - {EArgs, {EC, EV}} = lists:mapfoldl(fun expand_arg/2, {E, E}, Args), - {EArgs, elixir_env:mergea(EV, EC)}. - -%% Match/var helpers - -var_kind(Meta, Kind) -> - case lists:keyfind(counter, 1, Meta) of - {counter, Counter} -> Counter; - false -> Kind - end. - -%% Locals - -assert_no_ambiguous_op(Name, Meta, [Arg], E) -> - case lists:keyfind(ambiguous_op, 1, Meta) of - {ambiguous_op, Kind} -> - case lists:member({Name, Kind}, ?m(E, vars)) of - true -> - compile_error(Meta, ?m(E, file), "\"~ts ~ts\" looks like a function call but " - "there is a variable named \"~ts\", please use explicit parenthesis or even spaces", - [Name, 'Elixir.Macro':to_string(Arg), Name]); - false -> - ok - end; - _ -> - ok - end; -assert_no_ambiguous_op(_Atom, _Meta, _Args, _E) -> - ok. - -expand_local(Meta, Name, Args, #{local := nil, function := nil} = E) -> - {EArgs, EA} = expand_args(Args, E), - {{Name, Meta, EArgs}, EA}; -expand_local(Meta, Name, Args, #{local := nil, module := Module, function := Function} = E) -> - elixir_locals:record_local({Name, length(Args)}, Module, Function), - {EArgs, EA} = expand_args(Args, E), - {{Name, Meta, EArgs}, EA}; -expand_local(Meta, Name, Args, E) -> - expand({{'.', Meta, [?m(E, local), Name]}, Meta, Args}, E). - -%% Remote - -expand_remote(Receiver, DotMeta, Right, Meta, Args, E, EL) -> - if - is_atom(Receiver) -> elixir_lexical:record_remote(Receiver, ?m(E, lexical_tracker)); - true -> ok - end, - {EArgs, EA} = expand_args(Args, E), - {{{'.', DotMeta, [Receiver, Right]}, Meta, EArgs}, elixir_env:mergev(EL, EA)}. - -%% Lexical helpers - -expand_opts(Meta, Kind, Allowed, Opts, E) -> - {EOpts, EE} = expand(Opts, E), - validate_opts(Meta, Kind, Allowed, EOpts, EE), - {EOpts, EE}. - -validate_opts(Meta, Kind, Allowed, Opts, E) when is_list(Opts) -> - [begin - compile_error(Meta, ?m(E, file), - "unsupported option ~ts given to ~s", ['Elixir.Kernel':inspect(Key), Kind]) - end || {Key, _} <- Opts, not lists:member(Key, Allowed)]; - -validate_opts(Meta, Kind, _Allowed, _Opts, S) -> - compile_error(Meta, S#elixir_scope.file, "invalid options for ~s, expected a keyword list", [Kind]). - -no_alias_opts(KV) when is_list(KV) -> - case lists:keyfind(as, 1, KV) of - {as, As} -> lists:keystore(as, 1, KV, {as, no_alias_expansion(As)}); - false -> KV - end; -no_alias_opts(KV) -> KV. - -no_alias_expansion({'__aliases__', Meta, [H|T]}) when (H /= 'Elixir') and is_atom(H) -> - {'__aliases__', Meta, ['Elixir',H|T]}; -no_alias_expansion(Other) -> - Other. - -expand_require(Meta, Ref, KV, E) -> - RE = E#{requires := ordsets:add_element(Ref, ?m(E, requires))}, - expand_alias(Meta, false, Ref, KV, RE). - -expand_alias(Meta, IncludeByDefault, Ref, KV, #{context_modules := Context} = E) -> - New = expand_as(lists:keyfind(as, 1, KV), Meta, IncludeByDefault, Ref, E), - - %% Add the alias to context_modules if defined is true. - %% This is used by defmodule in order to store the defined - %% module in context modules. - NewContext = - case lists:keyfind(defined, 1, Meta) of - {defined, Mod} when is_atom(Mod) -> [Mod|Context]; - false -> Context - end, - - {Aliases, MacroAliases} = elixir_aliases:store(Meta, New, Ref, KV, ?m(E, aliases), - ?m(E, macro_aliases), ?m(E, lexical_tracker)), - - E#{aliases := Aliases, macro_aliases := MacroAliases, context_modules := NewContext}. - -expand_as({as, true}, _Meta, _IncludeByDefault, Ref, _E) -> - elixir_aliases:last(Ref); -expand_as({as, false}, _Meta, _IncludeByDefault, Ref, _E) -> - Ref; -expand_as({as, Atom}, Meta, _IncludeByDefault, _Ref, E) when is_atom(Atom) -> - case length(string:tokens(atom_to_list(Atom), ".")) of - 1 -> compile_error(Meta, ?m(E, file), - "invalid value for keyword :as, expected an alias, got atom: ~ts", [elixir_aliases:inspect(Atom)]); - 2 -> Atom; - _ -> compile_error(Meta, ?m(E, file), - "invalid value for keyword :as, expected an alias, got nested alias: ~ts", [elixir_aliases:inspect(Atom)]) - end; -expand_as(false, _Meta, IncludeByDefault, Ref, _E) -> - if IncludeByDefault -> elixir_aliases:last(Ref); - true -> Ref - end; -expand_as({as, Other}, Meta, _IncludeByDefault, _Ref, E) -> - compile_error(Meta, ?m(E, file), - "invalid value for keyword :as, expected an alias, got: ~ts", ['Elixir.Macro':to_string(Other)]). - -%% Assertions - -rewrite_case_clauses([{do,[ - {'->', FalseMeta, [ - [{'when', _, [Var, {'__op__', _,[ - 'orelse', - {{'.', _, [erlang, '=:=']}, _, [Var, nil]}, - {{'.', _, [erlang, '=:=']}, _, [Var, false]} - ]}]}], - FalseExpr - ]}, - {'->', TrueMeta, [ - [{'_', _, _}], - TrueExpr - ]} -]}]) -> - [{do, [ - {'->', FalseMeta, [[false], FalseExpr]}, - {'->', TrueMeta, [[true], TrueExpr]} - ]}]; -rewrite_case_clauses(Clauses) -> - Clauses. - -assert_no_match_or_guard_scope(Meta, Kind, E) -> - assert_no_match_scope(Meta, Kind, E), - assert_no_guard_scope(Meta, Kind, E). -assert_no_match_scope(Meta, _Kind, #{context := match, file := File}) -> - compile_error(Meta, File, "invalid expression in match"); -assert_no_match_scope(_Meta, _Kind, _E) -> []. -assert_no_guard_scope(Meta, _Kind, #{context := guard, file := File}) -> - compile_error(Meta, File, "invalid expression in guard"); -assert_no_guard_scope(_Meta, _Kind, _E) -> []. diff --git a/lib/elixir/src/elixir_exp_clauses.erl b/lib/elixir/src/elixir_exp_clauses.erl deleted file mode 100644 index 577e7d6dc8a..00000000000 --- a/lib/elixir/src/elixir_exp_clauses.erl +++ /dev/null @@ -1,214 +0,0 @@ -%% Handle code related to args, guard and -> matching for case, -%% fn, receive and friends. try is handled in elixir_try. --module(elixir_exp_clauses). --export([match/3, clause/5, def/5, head/2, - 'case'/3, 'receive'/3, 'try'/3, 'cond'/3]). --import(elixir_errors, [compile_error/3, compile_error/4]). --include("elixir.hrl"). - -match(Fun, Expr, #{context := Context} = E) -> - {EExpr, EE} = Fun(Expr, E#{context := match}), - {EExpr, EE#{context := Context}}. - -def(Fun, Args, Guards, Body, E) -> - {EArgs, EA} = match(Fun, Args, E), - {EGuards, EG} = guard(Guards, EA#{context := guard}), - {EBody, EB} = elixir_exp:expand(Body, EG#{context := ?m(E, context)}), - {EArgs, EGuards, EBody, EB}. - -clause(Meta, Kind, Fun, {'->', ClauseMeta, [_, _]} = Clause, E) when is_function(Fun, 3) -> - clause(Meta, Kind, fun(X, Acc) -> Fun(ClauseMeta, X, Acc) end, Clause, E); -clause(_Meta, _Kind, Fun, {'->', Meta, [Left, Right]}, E) -> - {ELeft, EL} = Fun(Left, E), - {ERight, ER} = elixir_exp:expand(Right, EL), - {{'->', Meta, [ELeft, ERight]}, ER}; -clause(Meta, Kind, _Fun, _, E) -> - compile_error(Meta, ?m(E, file), "expected -> clauses in ~ts", [Kind]). - -head([{'when', Meta, [_,_|_] = All}], E) -> - {Args, Guard} = elixir_utils:split_last(All), - {EArgs, EA} = match(fun elixir_exp:expand_args/2, Args, E), - {EGuard, EG} = guard(Guard, EA#{context := guard}), - {[{'when', Meta, EArgs ++ [EGuard]}], EG#{context := ?m(E, context)}}; -head(Args, E) -> - match(fun elixir_exp:expand_args/2, Args, E). - -guard({'when', Meta, [Left, Right]}, E) -> - {ELeft, EL} = guard(Left, E), - {ERight, ER} = guard(Right, EL), - {{'when', Meta, [ELeft, ERight]}, ER}; -guard(Other, E) -> - elixir_exp:expand(Other, E). - -%% Case - -'case'(Meta, [], E) -> - compile_error(Meta, ?m(E, file), "missing do keyword in case"); -'case'(Meta, KV, E) when not is_list(KV) -> - compile_error(Meta, ?m(E, file), "invalid arguments for case"); -'case'(Meta, KV, E) -> - EE = E#{export_vars := []}, - {EClauses, EVars} = lists:mapfoldl(fun(X, Acc) -> do_case(Meta, X, Acc, EE) end, [], KV), - {EClauses, elixir_env:mergev(EVars, E)}. - -do_case(Meta, {'do', _} = Do, Acc, E) -> - Fun = expand_one(Meta, 'case', 'do', fun head/2), - expand_with_export(Meta, 'case', Fun, Do, Acc, E); -do_case(Meta, {Key, _}, _Acc, E) -> - compile_error(Meta, ?m(E, file), "unexpected keyword ~ts in case", [Key]). - -%% Cond - -'cond'(Meta, [], E) -> - compile_error(Meta, ?m(E, file), "missing do keyword in cond"); -'cond'(Meta, KV, E) when not is_list(KV) -> - compile_error(Meta, ?m(E, file), "invalid arguments for cond"); -'cond'(Meta, KV, E) -> - EE = E#{export_vars := []}, - {EClauses, EVars} = lists:mapfoldl(fun(X, Acc) -> do_cond(Meta, X, Acc, EE) end, [], KV), - {EClauses, elixir_env:mergev(EVars, E)}. - -do_cond(Meta, {'do', _} = Do, Acc, E) -> - Fun = expand_one(Meta, 'cond', 'do', fun elixir_exp:expand_args/2), - expand_with_export(Meta, 'cond', Fun, Do, Acc, E); -do_cond(Meta, {Key, _}, _Acc, E) -> - compile_error(Meta, ?m(E, file), "unexpected keyword ~ts in cond", [Key]). - -%% Receive - -'receive'(Meta, [], E) -> - compile_error(Meta, ?m(E, file), "missing do or after keyword in receive"); -'receive'(Meta, KV, E) when not is_list(KV) -> - compile_error(Meta, ?m(E, file), "invalid arguments for receive"); -'receive'(Meta, KV, E) -> - EE = E#{export_vars := []}, - {EClauses, EVars} = lists:mapfoldl(fun(X, Acc) -> do_receive(Meta, X, Acc, EE) end, [], KV), - {EClauses, elixir_env:mergev(EVars, E)}. - -do_receive(_Meta, {'do', nil} = Do, Acc, _E) -> - {Do, Acc}; -do_receive(Meta, {'do', _} = Do, Acc, E) -> - Fun = expand_one(Meta, 'receive', 'do', fun head/2), - expand_with_export(Meta, 'receive', Fun, Do, Acc, E); -do_receive(Meta, {'after', [_]} = After, Acc, E) -> - Fun = expand_one(Meta, 'receive', 'after', fun elixir_exp:expand_args/2), - expand_with_export(Meta, 'receive', Fun, After, Acc, E); -do_receive(Meta, {'after', _}, _Acc, E) -> - compile_error(Meta, ?m(E, file), "expected a single -> clause for after in receive"); -do_receive(Meta, {Key, _}, _Acc, E) -> - compile_error(Meta, ?m(E, file), "unexpected keyword ~ts in receive", [Key]). - -%% Try - -'try'(Meta, [], E) -> - compile_error(Meta, ?m(E, file), "missing do keywords in try"); -'try'(Meta, KV, E) when not is_list(KV) -> - elixir_errors:compile_error(Meta, ?m(E, file), "invalid arguments for try"); -'try'(Meta, KV, E) -> - {lists:map(fun(X) -> do_try(Meta, X, E) end, KV), E}. - -do_try(_Meta, {'do', Expr}, E) -> - {EExpr, _} = elixir_exp:expand(Expr, E), - {'do', EExpr}; -do_try(_Meta, {'after', Expr}, E) -> - {EExpr, _} = elixir_exp:expand(Expr, E), - {'after', EExpr}; -do_try(Meta, {'else', _} = Else, E) -> - Fun = expand_one(Meta, 'try', 'else', fun head/2), - expand_without_export(Meta, 'try', Fun, Else, E); -do_try(Meta, {'catch', _} = Catch, E) -> - expand_without_export(Meta, 'try', fun expand_catch/3, Catch, E); -do_try(Meta, {'rescue', _} = Rescue, E) -> - expand_without_export(Meta, 'try', fun expand_rescue/3, Rescue, E); -do_try(Meta, {Key, _}, E) -> - compile_error(Meta, ?m(E, file), "unexpected keyword ~ts in try", [Key]). - -expand_catch(_Meta, [_] = Args, E) -> - head(Args, E); -expand_catch(_Meta, [_, _] = Args, E) -> - head(Args, E); -expand_catch(Meta, _, E) -> - compile_error(Meta, ?m(E, file), "expected one or two args for catch clauses (->) in try"). - -expand_rescue(Meta, [Arg], E) -> - case expand_rescue(Arg, E) of - {EArg, EA} -> - {[EArg], EA}; - false -> - compile_error(Meta, ?m(E, file), "invalid rescue clause. The clause should " - "match on an alias, a variable or be in the `var in [alias]` format") - end; -expand_rescue(Meta, _, E) -> - compile_error(Meta, ?m(E, file), "expected one arg for rescue clauses (->) in try"). - -%% rescue var => var in _ -expand_rescue({Name, _, Atom} = Var, E) when is_atom(Name), is_atom(Atom) -> - expand_rescue({in, [], [Var, {'_', [], ?m(E, module)}]}, E); - -%% rescue var in [Exprs] -expand_rescue({in, Meta, [Left, Right]}, E) -> - {ELeft, EL} = match(fun elixir_exp:expand/2, Left, E), - {ERight, ER} = elixir_exp:expand(Right, EL), - - case ELeft of - {Name, _, Atom} when is_atom(Name), is_atom(Atom) -> - case normalize_rescue(ERight) of - false -> false; - Other -> {{in, Meta, [ELeft, Other]}, ER} - end; - _ -> - false - end; - -%% rescue Error => _ in [Error] -expand_rescue(Arg, E) -> - expand_rescue({in, [], [{'_', [], ?m(E, module)}, Arg]}, E). - -normalize_rescue({'_', _, Atom} = N) when is_atom(Atom) -> N; -normalize_rescue(Atom) when is_atom(Atom) -> [Atom]; -normalize_rescue(Other) -> - is_list(Other) andalso lists:all(fun is_atom/1, Other) andalso Other. - -%% Expansion helpers - -%% Returns a function that expands arguments -%% considering we have at maximum one entry. -expand_one(Meta, Kind, Key, Fun) -> - fun - ([_] = Args, E) -> - Fun(Args, E); - (_, E) -> - compile_error(Meta, ?m(E, file), - "expected one arg for ~ts clauses (->) in ~ts", [Key, Kind]) - end. - -%% Expands all -> pairs in a given key keeping the overall vars. -expand_with_export(Meta, Kind, Fun, {Key, Clauses}, Acc, E) when is_list(Clauses) -> - EFun = - case lists:keyfind(export_head, 1, Meta) of - {export_head, true} -> - Fun; - _ -> - fun(Args, #{export_vars := ExportVars} = EE) -> - {FArgs, FE} = Fun(Args, EE), - {FArgs, FE#{export_vars := ExportVars}} - end - end, - Transformer = fun(Clause, Vars) -> - {EClause, EC} = clause(Meta, Kind, EFun, Clause, E), - {EClause, elixir_env:merge_vars(Vars, ?m(EC, export_vars))} - end, - {EClauses, EVars} = lists:mapfoldl(Transformer, Acc, Clauses), - {{Key, EClauses}, EVars}; -expand_with_export(Meta, Kind, _Fun, {Key, _}, _Acc, E) -> - compile_error(Meta, ?m(E, file), "expected -> clauses for ~ts in ~ts", [Key, Kind]). - -%% Expands all -> pairs in a given key but do not keep the overall vars. -expand_without_export(Meta, Kind, Fun, {Key, Clauses}, E) when is_list(Clauses) -> - Transformer = fun(Clause) -> - {EClause, _} = clause(Meta, Kind, Fun, Clause, E), - EClause - end, - {Key, lists:map(Transformer, Clauses)}; -expand_without_export(Meta, Kind, _Fun, {Key, _}, E) -> - compile_error(Meta, ?m(E, file), "expected -> clauses for ~ts in ~ts", [Key, Kind]). diff --git a/lib/elixir/src/elixir_expand.erl b/lib/elixir/src/elixir_expand.erl new file mode 100644 index 00000000000..1c746505911 --- /dev/null +++ b/lib/elixir/src/elixir_expand.erl @@ -0,0 +1,1341 @@ +-module(elixir_expand). +-export([expand/3, expand_args/3, expand_arg/3, format_error/1]). +-import(elixir_errors, [form_error/4]). +-include("elixir.hrl"). + +%% = + +expand({'=', Meta, [Left, Right]}, S, E) -> + assert_no_guard_scope(Meta, "=", S, E), + {ERight, SR, ER} = expand(Right, S, E), + {ELeft, SL, EL} = elixir_clauses:match(fun expand/3, Left, SR, S, ER), + refute_parallel_bitstring_match(ELeft, ERight, E, ?key(E, context) == match), + {{'=', Meta, [ELeft, ERight]}, SL, EL}; + +%% Literal operators + +expand({'{}', Meta, Args}, S, E) -> + {EArgs, SA, EA} = expand_args(Args, S, E), + {{'{}', Meta, EArgs}, SA, EA}; + +expand({'%{}', Meta, Args}, S, E) -> + elixir_map:expand_map(Meta, Args, S, E); + +expand({'%', Meta, [Left, Right]}, S, E) -> + elixir_map:expand_struct(Meta, Left, Right, S, E); + +expand({'<<>>', Meta, Args}, S, E) -> + elixir_bitstring:expand(Meta, Args, S, E, false); + +expand({'->', Meta, _Args}, _S, E) -> + form_error(Meta, E, ?MODULE, unhandled_arrow_op); + +%% __block__ + +expand({'__block__', _Meta, []}, S, E) -> + {nil, S, E}; +expand({'__block__', _Meta, [Arg]}, S, E) -> + expand(Arg, S, E); +expand({'__block__', Meta, Args}, S, E) when is_list(Args) -> + {EArgs, SA, EA} = expand_block(Args, [], Meta, S, E), + {{'__block__', Meta, EArgs}, SA, EA}; + +%% __aliases__ + +expand({'__aliases__', _, _} = Alias, S, E) -> + expand_aliases(Alias, S, E, true); + +%% alias + +expand({Kind, Meta, [{{'.', _, [Base, '{}']}, _, Refs} | Rest]}, S, E) + when Kind == alias; Kind == require; Kind == import -> + case Rest of + [] -> + expand_multi_alias_call(Kind, Meta, Base, Refs, [], S, E); + [Opts] -> + case lists:keymember(as, 1, Opts) of + true -> + form_error(Meta, E, ?MODULE, as_in_multi_alias_call); + false -> + expand_multi_alias_call(Kind, Meta, Base, Refs, Opts, S, E) + end + end; +expand({alias, Meta, [Ref]}, S, E) -> + expand({alias, Meta, [Ref, []]}, S, E); +expand({alias, Meta, [Ref, Opts]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "alias", S, E), + {ERef, SR, ER} = expand_without_aliases_report(Ref, S, E), + {EOpts, ST, ET} = expand_opts(Meta, alias, [as, warn], no_alias_opts(Opts), SR, ER), + case lists:member(ERef, ['Elixir.True', 'Elixir.False', 'Elixir.Nil']) of + true -> elixir_errors:form_warn(Meta, E, ?MODULE, {commonly_mistaken_alias, Ref}); + false -> ok + end, + if + is_atom(ERef) -> + {ERef, ST, expand_alias(Meta, true, ERef, EOpts, ET)}; + true -> + form_error(Meta, E, ?MODULE, {expected_compile_time_module, alias, Ref}) + end; + +expand({require, Meta, [Ref]}, S, E) -> + expand({require, Meta, [Ref, []]}, S, E); +expand({require, Meta, [Ref, Opts]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "require", S, E), + + {ERef, SR, ER} = expand_without_aliases_report(Ref, S, E), + {EOpts, ST, ET} = expand_opts(Meta, require, [as, warn], no_alias_opts(Opts), SR, ER), + + if + is_atom(ERef) -> + elixir_aliases:ensure_loaded(Meta, ERef, ET), + {ERef, ST, expand_require(Meta, ERef, EOpts, ET)}; + true -> + form_error(Meta, E, ?MODULE, {expected_compile_time_module, require, Ref}) + end; + +expand({import, Meta, [Left]}, S, E) -> + expand({import, Meta, [Left, []]}, S, E); + +expand({import, Meta, [Ref, Opts]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "import", S, E), + {ERef, SR, ER} = expand_without_aliases_report(Ref, S, E), + {EOpts, ST, ET} = expand_opts(Meta, import, [only, except, warn], Opts, SR, ER), + + if + is_atom(ERef) -> + elixir_aliases:ensure_loaded(Meta, ERef, ET), + {Functions, Macros} = elixir_import:import(Meta, ERef, EOpts, ET), + {ERef, ST, expand_require(Meta, ERef, EOpts, ET#{functions := Functions, macros := Macros})}; + true -> + form_error(Meta, E, ?MODULE, {expected_compile_time_module, import, Ref}) + end; + +%% Compilation environment macros + +expand({'__MODULE__', _, Atom}, S, E) when is_atom(Atom) -> + {?key(E, module), S, E}; +expand({'__DIR__', _, Atom}, S, E) when is_atom(Atom) -> + {filename:dirname(?key(E, file)), S, E}; +expand({'__CALLER__', Meta, Atom} = Caller, S, E) when is_atom(Atom) -> + assert_no_match_scope(Meta, "__CALLER__", E), + case S#elixir_ex.caller of + true -> {Caller, S, E}; + false -> form_error(Meta, E, ?MODULE, caller_not_allowed) + end; +expand({'__STACKTRACE__', Meta, Atom} = Stacktrace, S, E) when is_atom(Atom) -> + assert_no_match_scope(Meta, "__STACKTRACE__", E), + case S#elixir_ex.stacktrace of + true -> {Stacktrace, S, E}; + false -> form_error(Meta, E, ?MODULE, stacktrace_not_allowed) + end; +expand({'__ENV__', Meta, Atom}, S, E) when is_atom(Atom) -> + assert_no_match_scope(Meta, "__ENV__", E), + {escape_map(escape_env_entries(Meta, S, E)), S, E}; +expand({{'.', DotMeta, [{'__ENV__', Meta, Atom}, Field]}, CallMeta, []}, S, E) + when is_atom(Atom), is_atom(Field) -> + assert_no_match_scope(Meta, "__ENV__", E), + Env = escape_env_entries(Meta, S, E), + case maps:is_key(Field, Env) of + true -> {maps:get(Field, Env), S, E}; + false -> {{{'.', DotMeta, [escape_map(Env), Field]}, CallMeta, []}, S, E} + end; + +%% Quote + +expand({Unquote, Meta, [_]}, _S, E) when Unquote == unquote; Unquote == unquote_splicing -> + form_error(Meta, E, ?MODULE, {unquote_outside_quote, Unquote}); + +expand({quote, Meta, [Opts]}, S, E) when is_list(Opts) -> + case lists:keyfind(do, 1, Opts) of + {do, Do} -> + expand({quote, Meta, [lists:keydelete(do, 1, Opts), [{do, Do}]]}, S, E); + false -> + form_error(Meta, E, ?MODULE, {missing_option, 'quote', [do]}) + end; + +expand({quote, Meta, [_]}, _S, E) -> + form_error(Meta, E, ?MODULE, {invalid_args, 'quote'}); + +expand({quote, Meta, [Opts, Do]}, S, E) when is_list(Do) -> + Exprs = + case lists:keyfind(do, 1, Do) of + {do, Expr} -> Expr; + false -> form_error(Meta, E, ?MODULE, {missing_option, 'quote', [do]}) + end, + + ValidOpts = [context, location, line, file, unquote, bind_quoted, generated], + {EOpts, ST, ET} = expand_opts(Meta, quote, ValidOpts, Opts, S, E), + + Context = proplists:get_value(context, EOpts, case ?key(E, module) of + nil -> 'Elixir'; + Mod -> Mod + end), + + {File, Line} = case lists:keyfind(location, 1, EOpts) of + {location, keep} -> + {elixir_utils:relative_to_cwd(?key(E, file)), false}; + false -> + {proplists:get_value(file, EOpts, nil), proplists:get_value(line, EOpts, false)} + end, + + {Binding, DefaultUnquote} = case lists:keyfind(bind_quoted, 1, EOpts) of + {bind_quoted, BQ} -> + case is_list(BQ) andalso + lists:all(fun({Key, _}) when is_atom(Key) -> true; (_) -> false end, BQ) of + true -> {BQ, false}; + false -> form_error(Meta, E, ?MODULE, {invalid_bind_quoted_for_quote, BQ}) + end; + false -> + {[], true} + end, + + Unquote = proplists:get_value(unquote, EOpts, DefaultUnquote), + Generated = proplists:get_value(generated, EOpts, false), + + {Q, Prelude} = elixir_quote:build(Meta, Line, File, Context, Unquote, Generated), + Quoted = elixir_quote:quote(Meta, Exprs, Binding, Q, Prelude, ET), + expand(Quoted, ST, ET); + +expand({quote, Meta, [_, _]}, _S, E) -> + form_error(Meta, E, ?MODULE, {invalid_args, 'quote'}); + +%% Functions + +expand({'&', Meta, [{super, SuperMeta, Args} = Expr]}, S, E) when is_list(Args) -> + assert_no_match_or_guard_scope(Meta, "&", S, E), + + case resolve_super(Meta, length(Args), E) of + {Kind, Name, _} when Kind == def; Kind == defp -> + expand_fn_capture(Meta, {Name, SuperMeta, Args}, S, E); + _ -> + expand_fn_capture(Meta, Expr, S, E) + end; + +expand({'&', Meta, [{'/', _, [{super, _, Context}, Arity]} = Expr]}, S, E) when is_atom(Context), is_integer(Arity) -> + assert_no_match_or_guard_scope(Meta, "&", S, E), + + case resolve_super(Meta, Arity, E) of + {Kind, Name, _} when Kind == def; Kind == defp -> + {{'&', Meta, [{'/', [], [{Name, [], Context}, Arity]}]}, S, E}; + _ -> + expand_fn_capture(Meta, Expr, S, E) + end; + +expand({'&', Meta, [Arg]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "&", S, E), + expand_fn_capture(Meta, Arg, S, E); + +expand({fn, Meta, Pairs}, S, E) -> + assert_no_match_or_guard_scope(Meta, "fn", S, E), + elixir_fn:expand(Meta, Pairs, S, E); + +%% Case/Receive/Try + +expand({'cond', Meta, [Opts]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "cond", S, E), + assert_no_underscore_clause_in_cond(Opts, E), + {EClauses, SC, EC} = elixir_clauses:'cond'(Meta, Opts, S, E), + {{'cond', Meta, [EClauses]}, SC, EC}; + +expand({'case', Meta, [Expr, Options]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "case", S, E), + expand_case(Meta, Expr, Options, S, E); + +expand({'receive', Meta, [Opts]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "receive", S, E), + {EClauses, SC, EC} = elixir_clauses:'receive'(Meta, Opts, S, E), + {{'receive', Meta, [EClauses]}, SC, EC}; + +expand({'try', Meta, [Opts]}, S, E) -> + assert_no_match_or_guard_scope(Meta, "try", S, E), + {EClauses, SC, EC} = elixir_clauses:'try'(Meta, Opts, S, E), + {{'try', Meta, [EClauses]}, SC, EC}; + +%% Comprehensions + +expand({for, Meta, [_ | _] = Args}, S, E) -> + assert_no_match_or_guard_scope(Meta, "for", S, E), + + {Cases, Block} = + case elixir_utils:split_last(Args) of + {OuterCases, OuterOpts} when is_list(OuterOpts) -> + case elixir_utils:split_last(OuterCases) of + {InnerCases, InnerOpts} when is_list(InnerOpts) -> + {InnerCases, InnerOpts ++ OuterOpts}; + _ -> + {OuterCases, OuterOpts} + end; + _ -> + {Args, []} + end, + + validate_opts(Meta, for, [do, into, uniq, reduce], Block, E), + + {Expr, Opts} = + case lists:keytake(do, 1, Block) of + {value, {do, Do}, DoOpts} -> + {Do, DoOpts}; + false -> + form_error(Meta, E, ?MODULE, {missing_option, for, [do]}) + end, + + {EOpts, SO, EO} = expand(Opts, elixir_env:reset_unused_vars(S), E), + {ECases, SC, EC} = mapfold(fun expand_for/3, SO, EO, Cases), + assert_generator_start(Meta, ECases, E), + + {EExpr, SE, EE} = + case validate_for_options(EOpts, false, false, false) of + {ok, MaybeReduce} -> expand_for_do_block(Meta, Expr, SC, EC, MaybeReduce); + {error, Error} -> form_error(Meta, E, ?MODULE, Error) + end, + + {{for, Meta, ECases ++ [[{do, EExpr} | EOpts]]}, + elixir_env:merge_and_check_unused_vars(SE, S, EE), + E}; + +%% With + +expand({with, Meta, [_ | _] = Args}, S, E) -> + assert_no_match_or_guard_scope(Meta, "with", S, E), + elixir_clauses:with(Meta, Args, S, E); + +%% Super + +expand({super, Meta, Args}, S, E) when is_list(Args) -> + assert_no_match_or_guard_scope(Meta, "super", S, E), + {Kind, Name, _} = resolve_super(Meta, length(Args), E), + {EArgs, SA, EA} = expand_args(Args, S, E), + {{super, [{super, {Kind, Name}} | Meta], EArgs}, SA, EA}; + +%% Vars + +expand({'^', Meta, [Arg]}, S, #{context := match} = E) -> + #elixir_ex{prematch={Prematch, _}, vars={_Read, Write}} = S, + + %% We need to rollback to a no match context + NoMatchE = E#{context := nil}, + NoMatchS = S#elixir_ex{prematch=pin, vars={Prematch, Write}}, + + case expand(Arg, NoMatchS, NoMatchE) of + {{Name, _, Kind} = Var, #elixir_ex{unused=Unused}, _} when is_atom(Name), is_atom(Kind) -> + {{'^', Meta, [Var]}, S#elixir_ex{unused=Unused}, E}; + + _ -> + form_error(Meta, E, ?MODULE, {invalid_arg_for_pin, Arg}) + end; +expand({'^', Meta, [Arg]}, _S, E) -> + form_error(Meta, E, ?MODULE, {pin_outside_of_match, Arg}); + +expand({'_', _Meta, Kind} = Var, S, #{context := match} = E) when is_atom(Kind) -> + {Var, S, E}; +expand({'_', Meta, Kind}, _S, E) when is_atom(Kind) -> + form_error(Meta, E, ?MODULE, unbound_underscore); + +expand({Name, Meta, Kind}, S, #{context := match} = E) when is_atom(Name), is_atom(Kind) -> + #elixir_ex{ + prematch={_, PrematchVersion}, + unused={Unused, Version}, + vars={Read, Write} + } = S, + + Pair = {Name, elixir_utils:var_context(Meta, Kind)}, + + case Read of + %% Variable was already overridden + #{Pair := VarVersion} when VarVersion >= PrematchVersion -> + maybe_warn_underscored_var_repeat(Meta, Name, Kind, E), + NewUnused = var_used(Pair, VarVersion, Unused), + Var = {Name, [{version, VarVersion} | Meta], Kind}, + {Var, S#elixir_ex{unused={NewUnused, Version}}, E}; + + %% Variable is being overridden now + #{Pair := _} -> + NewUnused = var_unused(Pair, Meta, Version, Unused, true), + NewRead = Read#{Pair => Version}, + NewWrite = (Write /= false) andalso Write#{Pair => Version}, + Var = {Name, [{version, Version} | Meta], Kind}, + {Var, S#elixir_ex{vars={NewRead, NewWrite}, unused={NewUnused, Version + 1}}, E}; + + %% Variable defined for the first time + _ -> + NewUnused = var_unused(Pair, Meta, Version, Unused, false), + NewRead = Read#{Pair => Version}, + NewWrite = (Write /= false) andalso Write#{Pair => Version}, + Var = {Name, [{version, Version} | Meta], Kind}, + {Var, S#elixir_ex{vars={NewRead, NewWrite}, unused={NewUnused, Version + 1}}, E} + end; + +expand({Name, Meta, Kind}, S, E) when is_atom(Name), is_atom(Kind) -> + #elixir_ex{vars={Read, _Write}, unused={Unused, Version}, prematch=Prematch} = S, + Pair = {Name, elixir_utils:var_context(Meta, Kind)}, + + Result = + case Read of + #{Pair := CurrentVersion} -> + case Prematch of + {bitsize, Pre, Original} -> + if + is_map_key(Pair, Pre); not is_map_key(Pair, Original) -> {ok, CurrentVersion}; + true -> raise + end; + + _ -> + {ok, CurrentVersion} + end; + + _ -> + Prematch + end, + + case Result of + {ok, PairVersion} -> + maybe_warn_underscored_var_access(Meta, Name, Kind, E), + Var = {Name, [{version, PairVersion} | Meta], Kind}, + {Var, S#elixir_ex{unused={var_used(Pair, PairVersion, Unused), Version}}, E}; + + Error -> + case lists:keyfind(if_undefined, 1, Meta) of + %% TODO: Remove this clause on v2.0 as we will always raise by default + {if_undefined, raise} -> + form_error(Meta, E, ?MODULE, {undefined_var, Name, Kind}); + + {if_undefined, apply} -> + expand({Name, Meta, []}, S, E); + + %% TODO: Remove this clause on v2.0 + _ when Error == warn -> + elixir_errors:form_warn(Meta, E, ?MODULE, {unknown_variable, Name}), + expand({Name, Meta, []}, S, E); + + _ when Error == pin -> + form_error(Meta, E, ?MODULE, {undefined_var_pin, Name, Kind}); + + _ -> + form_error(Meta, E, ?MODULE, {undefined_var, Name, Kind}) + end + end; + +%% Local calls + +expand({Atom, Meta, Args}, S, E) when is_atom(Atom), is_list(Meta), is_list(Args) -> + assert_no_ambiguous_op(Atom, Meta, Args, S, E), + + elixir_dispatch:dispatch_import(Meta, Atom, Args, S, E, fun() -> + expand_local(Meta, Atom, Args, S, E) + end); + +%% Remote calls + +expand({{'.', DotMeta, [Left, Right]}, Meta, Args}, S, E) + when (is_tuple(Left) orelse is_atom(Left)), is_atom(Right), is_list(Meta), is_list(Args) -> + {ELeft, SL, EL} = expand(Left, elixir_env:prepare_write(S), E), + + elixir_dispatch:dispatch_require(Meta, ELeft, Right, Args, S, EL, fun(AR, AF, AA) -> + expand_remote(AR, DotMeta, AF, Meta, AA, S, SL, EL) + end); + +%% Anonymous calls + +expand({{'.', DotMeta, [Expr]}, Meta, Args}, S, E) when is_list(Args) -> + assert_no_match_or_guard_scope(Meta, "anonymous call", S, E), + + case expand_args([Expr | Args], S, E) of + {[EExpr | _], _, _} when is_atom(EExpr) -> + form_error(Meta, E, ?MODULE, {invalid_function_call, EExpr}); + + {[EExpr | EArgs], SA, EA} -> + {{{'.', DotMeta, [EExpr]}, Meta, EArgs}, SA, EA} + end; + +%% Invalid calls + +expand({_, Meta, Args} = Invalid, _S, E) when is_list(Meta) and is_list(Args) -> + form_error(Meta, E, ?MODULE, {invalid_call, Invalid}); + +%% Literals + +expand({Left, Right}, S, E) -> + {[ELeft, ERight], SE, EE} = expand_args([Left, Right], S, E), + {{ELeft, ERight}, SE, EE}; + +expand(List, S, #{context := match} = E) when is_list(List) -> + expand_list(List, fun expand/3, S, E, []); + +expand(List, S, E) when is_list(List) -> + {EArgs, {SE, _}, EE} = + expand_list(List, fun expand_arg/3, {elixir_env:prepare_write(S), S}, E, []), + + {EArgs, elixir_env:close_write(SE, S), EE}; + +expand(Function, S, E) when is_function(Function) -> + case (erlang:fun_info(Function, type) == {type, external}) andalso + (erlang:fun_info(Function, env) == {env, []}) of + true -> + {elixir_quote:fun_to_quoted(Function), S, E}; + false -> + form_error([{line, 0}], ?key(E, file), ?MODULE, {invalid_quoted_expr, Function}) + end; + +expand(Pid, S, E) when is_pid(Pid) -> + case ?key(E, function) of + nil -> + {Pid, S, E}; + Function -> + %% TODO: Make me an error on v2.0 + elixir_errors:form_warn([], E, ?MODULE, {invalid_pid_in_function, Pid, Function}), + {Pid, E} + end; + +expand(Other, S, E) when is_number(Other); is_atom(Other); is_binary(Other) -> + {Other, S, E}; + +expand(Other, _S, E) -> + form_error([{line, 0}], ?key(E, file), ?MODULE, {invalid_quoted_expr, Other}). + +%% Helpers + +escape_env_entries(Meta, #elixir_ex{vars={Read, _}}, Env0) -> + Env1 = case Env0 of + #{function := nil} -> Env0; + _ -> Env0#{lexical_tracker := nil, tracers := []} + end, + + Env1#{versioned_vars := escape_map(Read), line := ?line(Meta)}. + +escape_map(Map) -> {'%{}', [], maps:to_list(Map)}. + +expand_multi_alias_call(Kind, Meta, Base, Refs, Opts, S, E) -> + {BaseRef, SB, EB} = expand_without_aliases_report(Base, S, E), + + Fun = fun + ({'__aliases__', _, Ref}, SR, ER) -> + expand({Kind, Meta, [elixir_aliases:concat([BaseRef | Ref]), Opts]}, SR, ER); + + (Ref, SR, ER) when is_atom(Ref) -> + expand({Kind, Meta, [elixir_aliases:concat([BaseRef, Ref]), Opts]}, SR, ER); + + (Other, _SR, _ER) -> + form_error(Meta, E, ?MODULE, {expected_compile_time_module, Kind, Other}) + end, + + mapfold(Fun, SB, EB, Refs). + +resolve_super(Meta, Arity, E) -> + Module = assert_module_scope(Meta, super, E), + Function = assert_function_scope(Meta, super, E), + + case Function of + {_, Arity} -> + {Kind, Name, SuperMeta} = elixir_overridable:super(Meta, Module, Function, E), + maybe_warn_deprecated_super_in_gen_server_callback(Meta, Function, SuperMeta, E), + {Kind, Name, SuperMeta}; + + _ -> + form_error(Meta, E, ?MODULE, wrong_number_of_args_for_super) + end. + +expand_fn_capture(Meta, Arg, S, E) -> + case elixir_fn:capture(Meta, Arg, S, E) of + {{remote, Remote, Fun, Arity}, SE, EE} -> + is_atom(Remote) andalso + elixir_env:trace({remote_function, Meta, Remote, Fun, Arity}, E), + AttachedMeta = attach_context_module(Remote, Meta, E), + {{'&', AttachedMeta, [{'/', [], [{{'.', [], [Remote, Fun]}, [], []}, Arity]}]}, SE, EE}; + {{local, Fun, Arity}, _SE, #{function := nil}} -> + form_error(Meta, E, ?MODULE, {undefined_local_capture, Fun, Arity}); + {{local, Fun, Arity}, SE, EE} -> + {{'&', Meta, [{'/', [], [{Fun, [], nil}, Arity]}]}, SE, EE}; + {expand, Expr, SE, EE} -> + expand(Expr, SE, EE) + end. + +expand_list([{'|', Meta, [_, _] = Args}], Fun, S, E, List) -> + {EArgs, SAcc, EAcc} = mapfold(Fun, S, E, Args), + expand_list([], Fun, SAcc, EAcc, [{'|', Meta, EArgs} | List]); +expand_list([H | T], Fun, S, E, List) -> + {EArg, SAcc, EAcc} = Fun(H, S, E), + expand_list(T, Fun, SAcc, EAcc, [EArg | List]); +expand_list([], _Fun, S, E, List) -> + {lists:reverse(List), S, E}. + +expand_block([], Acc, _Meta, S, E) -> + {lists:reverse(Acc), S, E}; +expand_block([H], Acc, Meta, S, E) -> + {EH, SE, EE} = expand(H, S, E), + expand_block([], [EH | Acc], Meta, SE, EE); +expand_block([H | T], Acc, Meta, S, E) -> + {EH, SE, EE} = expand(H, S, E), + + %% Note that checks rely on the code BEFORE expansion + %% instead of relying on Erlang checks. + %% + %% That's because expansion may generate useless + %% terms on their own (think compile time removed + %% logger calls) and we don't want to catch those. + %% + %% Or, similarly, the work is all in the expansion + %% (for example, to register something) and it is + %% simply returning something as replacement. + case is_useless_building(H, EH, Meta) of + {UselessMeta, UselessTerm} -> + elixir_errors:form_warn(UselessMeta, E, ?MODULE, UselessTerm); + + false -> + ok + end, + + expand_block(T, [EH | Acc], Meta, SE, EE). + +%% Note that we don't handle atoms on purpose. They are common +%% when unquoting AST and it is unlikely that we would catch +%% bugs as we don't do binary operations on them like in +%% strings or numbers. +is_useless_building(H, _, Meta) when is_binary(H); is_number(H) -> + {Meta, {useless_literal, H}}; +is_useless_building({'@', Meta, [{Var, _, Ctx}]}, _, _) when is_atom(Ctx); Ctx == [] -> + {Meta, {useless_attr, Var}}; +is_useless_building({Var, Meta, Ctx}, {Var, _, Ctx}, _) when is_atom(Ctx) -> + {Meta, {useless_var, Var}}; +is_useless_building(_, _, _) -> + false. + +%% Variables in arguments are not propagated from one +%% argument to the other. For instance: +%% +%% x = 1 +%% foo(x = x + 2, x) +%% x +%% +%% Should be the same as: +%% +%% foo(3, 1) +%% 3 +%% +%% However, lexical information is. +expand_arg(Arg, Acc, E) when is_number(Arg); is_atom(Arg); is_binary(Arg); is_pid(Arg) -> + {Arg, Acc, E}; +expand_arg(Arg, {Acc, S}, E) -> + {EArg, SAcc, EAcc} = expand(Arg, elixir_env:reset_read(Acc, S), E), + {EArg, {SAcc, S}, EAcc}. + +expand_args([Arg], S, E) -> + {EArg, SE, EE} = expand(Arg, S, E), + {[EArg], SE, EE}; +expand_args(Args, S, #{context := match} = E) -> + mapfold(fun expand/3, S, E, Args); +expand_args(Args, S, E) -> + {EArgs, {SA, _}, EA} = mapfold(fun expand_arg/3, {elixir_env:prepare_write(S), S}, E, Args), + {EArgs, elixir_env:close_write(SA, S), EA}. + +mapfold(Fun, S, E, List) -> + mapfold(Fun, S, E, List, []). + +mapfold(Fun, S, E, [H | T], Acc) -> + {RH, RS, RE} = Fun(H, S, E), + mapfold(Fun, RS, RE, T, [RH | Acc]); +mapfold(_Fun, S, E, [], Acc) -> + {lists:reverse(Acc), S, E}. + +%% Match/var helpers + +var_unused({Name, Kind}, Meta, Version, Unused, Override) -> + case (Kind == nil) andalso should_warn(Meta) of + true -> Unused#{{Name, Version} => {?line(Meta), Override}}; + false -> Unused + end. + +var_used({Name, Kind}, Version, Unused) -> + case Kind of + nil -> Unused#{{Name, Version} => false}; + _ -> Unused + end. + +maybe_warn_underscored_var_repeat(Meta, Name, Kind, E) -> + case should_warn(Meta) andalso atom_to_list(Name) of + "_" ++ _ -> + elixir_errors:form_warn(Meta, E, ?MODULE, {underscored_var_repeat, Name, Kind}); + _ -> + ok + end. + +maybe_warn_underscored_var_access(Meta, Name, Kind, E) -> + case (Kind == nil) andalso should_warn(Meta) andalso atom_to_list(Name) of + "_" ++ _ -> + elixir_errors:form_warn(Meta, E, ?MODULE, {underscored_var_access, Name}); + _ -> + ok + end. + +%% TODO: Remove this on Elixir v2.0 and make all GenServer callbacks optional +maybe_warn_deprecated_super_in_gen_server_callback(Meta, Function, SuperMeta, E) -> + case lists:keyfind(context, 1, SuperMeta) of + {context, 'Elixir.GenServer'} -> + case Function of + {child_spec, 1} -> + ok; + + _ -> + elixir_errors:form_warn(Meta, E, ?MODULE, {super_in_genserver, Function}) + end; + + _ -> + ok + end. + +context_info(Kind) when Kind == nil; is_integer(Kind) -> ""; +context_info(Kind) -> io_lib:format(" (context ~ts)", [elixir_aliases:inspect(Kind)]). + +should_warn(Meta) -> + lists:keyfind(generated, 1, Meta) /= {generated, true}. + +%% Case + +expand_case(Meta, Expr, Opts, S, E) -> + {EExpr, SE, EE} = expand(Expr, S, E), + + ROpts = + case proplists:get_value(optimize_boolean, Meta, false) of + true -> + case elixir_utils:returns_boolean(EExpr) of + true -> rewrite_case_clauses(Opts); + false -> generated_case_clauses(Opts) + end; + + false -> + Opts + end, + + {EOpts, SO, EO} = elixir_clauses:'case'(Meta, ROpts, SE, EE), + {{'case', Meta, [EExpr, EOpts]}, SO, EO}. + +rewrite_case_clauses([{do, [ + {'->', FalseMeta, [ + [{'when', _, [Var, {{'.', _, ['Elixir.Kernel', 'in']}, _, [Var, [false, nil]]}]}], + FalseExpr + ]}, + {'->', TrueMeta, [ + [{'_', _, _}], + TrueExpr + ]} +]}]) -> + rewrite_case_clauses(FalseMeta, FalseExpr, TrueMeta, TrueExpr); + +rewrite_case_clauses([{do, [ + {'->', FalseMeta, [[false], FalseExpr]}, + {'->', TrueMeta, [[true], TrueExpr]} | _ +]}]) -> + rewrite_case_clauses(FalseMeta, FalseExpr, TrueMeta, TrueExpr); + +rewrite_case_clauses(Other) -> + generated_case_clauses(Other). + +rewrite_case_clauses(FalseMeta, FalseExpr, TrueMeta, TrueExpr) -> + [{do, [ + {'->', ?generated(FalseMeta), [[false], FalseExpr]}, + {'->', ?generated(TrueMeta), [[true], TrueExpr]} + ]}]. + +generated_case_clauses([{do, Clauses}]) -> + RClauses = [{'->', ?generated(Meta), Args} || {'->', Meta, Args} <- Clauses], + [{do, RClauses}]. + +%% Comprehensions + +validate_for_options([{into, _} = Pair | Opts], _Into, Uniq, Reduce) -> + validate_for_options(Opts, Pair, Uniq, Reduce); +validate_for_options([{uniq, Boolean} = Pair | Opts], Into, _Uniq, Reduce) when is_boolean(Boolean) -> + validate_for_options(Opts, Into, Pair, Reduce); +validate_for_options([{uniq, Value} | _], _, _, _) -> + {error, {for_invalid_uniq, Value}}; +validate_for_options([{reduce, _} = Pair | Opts], Into, Uniq, _Reduce) -> + validate_for_options(Opts, Into, Uniq, Pair); +validate_for_options([], Into, Uniq, {reduce, _}) when Into /= false; Uniq /= false -> + {error, for_conflicting_reduce_into_uniq}; +validate_for_options([], _Into, _Uniq, Reduce) -> + {ok, Reduce}. + +expand_for_do_block(Meta, [{'->', _, _} | _], _S, E, false) -> + form_error(Meta, E, ?MODULE, for_without_reduce_bad_block); +expand_for_do_block(_Meta, Expr, S, E, false) -> + expand(Expr, S, E); +expand_for_do_block(Meta, [{'->', _, _} | _] = Clauses, S, E, {reduce, _}) -> + Transformer = fun + ({_, _, [[_], _]} = Clause, SA) -> + SReset = elixir_env:reset_unused_vars(SA), + + {EClause, SAcc, EAcc} = + elixir_clauses:clause(Meta, fn, fun elixir_clauses:head/3, Clause, SReset, E), + + {EClause, elixir_env:merge_and_check_unused_vars(SAcc, SA, EAcc)}; + + (_, _) -> + form_error(Meta, E, ?MODULE, for_with_reduce_bad_block) + end, + + {Do, SA} = lists:mapfoldl(Transformer, S, Clauses), + {Do, SA, E}; +expand_for_do_block(Meta, _Expr, _S, E, {reduce, _}) -> + form_error(Meta, E, ?MODULE, for_with_reduce_bad_block). + +%% Locals + +assert_no_ambiguous_op(Name, Meta, [Arg], S, E) -> + case lists:keyfind(ambiguous_op, 1, Meta) of + {ambiguous_op, Kind} -> + Pair = {Name, Kind}, + case S#elixir_ex.vars of + {#{Pair := _}, _} -> + form_error(Meta, E, ?MODULE, {op_ambiguity, Name, Arg}); + _ -> + ok + end; + _ -> + ok + end; +assert_no_ambiguous_op(_Atom, _Meta, _Args, _S, _E) -> + ok. + +assert_no_clauses(_Name, _Meta, [], _E) -> + ok; +assert_no_clauses(Name, Meta, Args, E) -> + assert_arg_with_no_clauses(Name, Meta, lists:last(Args), E). + +assert_arg_with_no_clauses(Name, Meta, [{Key, Value} | Rest], E) when is_atom(Key) -> + case Value of + [{'->', _, _} | _] -> + form_error(Meta, E, ?MODULE, {invalid_clauses, Name}); + _ -> + assert_arg_with_no_clauses(Name, Meta, Rest, E) + end; +assert_arg_with_no_clauses(_Name, _Meta, _Arg, _E) -> + ok. + +expand_local(Meta, Name, Args, S, #{module := Module, function := Function, context := nil} = E) + when Function /= nil -> + assert_no_clauses(Name, Meta, Args, E), + Arity = length(Args), + elixir_env:trace({local_function, Meta, Name, Arity}, E), + elixir_locals:record_local({Name, Arity}, Module, Function, Meta, false), + {EArgs, SA, EA} = expand_args(Args, S, E), + {{Name, Meta, EArgs}, SA, EA}; +expand_local(Meta, Name, Args, _S, E) when Name == '|'; Name == '::' -> + form_error(Meta, E, ?MODULE, {undefined_function, Name, Args}); +expand_local(Meta, Name, Args, _S, #{function := nil} = E) -> + form_error(Meta, E, ?MODULE, {undefined_function, Name, Args}); +expand_local(Meta, Name, Args, _S, #{context := match} = E) -> + form_error(Meta, E, ?MODULE, {invalid_local_invocation, match, {Name, Meta, Args}}); +expand_local(Meta, Name, Args, S, #{context := guard} = E) -> + form_error(Meta, E, ?MODULE, {invalid_local_invocation, guard_context(S), {Name, Meta, Args}}). + +%% Remote + +expand_remote(Receiver, DotMeta, Right, Meta, Args, S, SL, #{context := Context} = E) when is_atom(Receiver) or is_tuple(Receiver) -> + assert_no_clauses(Right, Meta, Args, E), + + case {Context, lists:keyfind(no_parens, 1, Meta)} of + {guard, {no_parens, true}} when is_tuple(Receiver) -> + {{{'.', DotMeta, [Receiver, Right]}, Meta, []}, SL, E}; + + {guard, _} when is_tuple(Receiver) -> + form_error(Meta, E, ?MODULE, {parens_map_lookup, Receiver, Right, guard_context(S)}); + + _ -> + AttachedDotMeta = attach_context_module(Receiver, DotMeta, E), + + is_atom(Receiver) andalso + elixir_env:trace({remote_function, Meta, Receiver, Right, length(Args)}, E), + + {EArgs, {SA, _}, EA} = mapfold(fun expand_arg/3, {SL, S}, E, Args), + + case rewrite(Context, Receiver, AttachedDotMeta, Right, Meta, EArgs, S) of + {ok, Rewritten} -> + maybe_warn_comparison(Rewritten, Args, E), + {Rewritten, elixir_env:close_write(SA, S), EA}; + {error, Error} -> + form_error(Meta, E, elixir_rewrite, Error) + end + end; +expand_remote(Receiver, DotMeta, Right, Meta, Args, _, _, E) -> + Call = {{'.', DotMeta, [Receiver, Right]}, Meta, Args}, + form_error(Meta, E, ?MODULE, {invalid_call, Call}). + +attach_context_module(_Receiver, Meta, #{function := nil}) -> + Meta; +attach_context_module(Receiver, Meta, #{context_modules := [Ctx | _], module := Mod}) + %% If the context is the current module or + %% if the receiver is the current module, + %% then there is no context module. + when Ctx == Mod; Receiver == Mod -> + Meta; +attach_context_module(Receiver, Meta, #{context_modules := ContextModules}) -> + case lists:member(Receiver, ContextModules) of + true -> [{context_module, true} | Meta]; + false -> Meta + end. + +rewrite(match, Receiver, DotMeta, Right, Meta, EArgs, _S) -> + elixir_rewrite:match_rewrite(Receiver, DotMeta, Right, Meta, EArgs); +rewrite(guard, Receiver, DotMeta, Right, Meta, EArgs, S) -> + elixir_rewrite:guard_rewrite(Receiver, DotMeta, Right, Meta, EArgs, guard_context(S)); +rewrite(_, Receiver, DotMeta, Right, Meta, EArgs, _S) -> + {ok, elixir_rewrite:rewrite(Receiver, DotMeta, Right, Meta, EArgs)}. + +maybe_warn_comparison({{'.', _, [erlang, Op]}, Meta, [ELeft, ERight]}, [Left, Right], E) + when Op =:= '>'; Op =:= '<'; Op =:= '=<'; Op =:= '>='; Op =:= min; Op =:= max -> + case is_struct_comparison(ELeft, ERight, Left, Right) of + false -> + case is_nested_comparison(Op, ELeft, ERight, Left, Right) of + false -> ok; + CompExpr -> + elixir_errors:form_warn(Meta, E, ?MODULE, {nested_comparison, CompExpr}) + end; + StructExpr -> + elixir_errors:form_warn(Meta, E, ?MODULE, {struct_comparison, StructExpr}) + end; +maybe_warn_comparison(_, _, _) -> + ok. + +is_struct_comparison(ELeft, ERight, Left, Right) -> + case is_struct_expression(ELeft) of + true -> Left; + false -> + case is_struct_expression(ERight) of + true -> Right; + false -> false + end + end. + +is_struct_expression({'%', _, [Struct, _]}) when is_atom(Struct) -> + true; +is_struct_expression({'%{}', _, KVs}) -> + case lists:keyfind('__struct__', 1, KVs) of + {'__struct__', Struct} when is_atom(Struct) -> true; + false -> false + end; +is_struct_expression(_Other) -> false. + +is_nested_comparison(Op, ELeft, ERight, Left, Right) -> + NestedExpr = {elixir_utils:erlang_comparison_op_to_elixir(Op), [], [Left, Right]}, + case is_comparison_expression(ELeft) of + true -> + NestedExpr; + false -> + case is_comparison_expression(ERight) of + true -> NestedExpr; + false -> false + end + end. +is_comparison_expression({{'.',_,[erlang,Op]},_,_}) + when Op =:= '>'; Op =:= '<'; Op =:= '=<'; Op =:= '>=' -> true; +is_comparison_expression(_Other) -> false. + +%% Lexical helpers + +expand_opts(Meta, Kind, Allowed, Opts, S, E) -> + {EOpts, SE, EE} = expand(Opts, S, E), + validate_opts(Meta, Kind, Allowed, EOpts, EE), + {EOpts, SE, EE}. + +validate_opts(Meta, Kind, Allowed, Opts, E) when is_list(Opts) -> + [begin + form_error(Meta, E, ?MODULE, {unsupported_option, Kind, Key}) + end || {Key, _} <- Opts, not lists:member(Key, Allowed)]; + +validate_opts(Meta, Kind, _Allowed, Opts, E) -> + form_error(Meta, E, ?MODULE, {options_are_not_keyword, Kind, Opts}). + +no_alias_opts(Opts) when is_list(Opts) -> + case lists:keyfind(as, 1, Opts) of + {as, As} -> lists:keystore(as, 1, Opts, {as, no_alias_expansion(As)}); + false -> Opts + end; +no_alias_opts(Opts) -> Opts. + +no_alias_expansion({'__aliases__', _, [H | T]}) when is_atom(H) -> + elixir_aliases:concat([H | T]); +no_alias_expansion(Other) -> + Other. + +expand_require(Meta, Ref, Opts, E) -> + elixir_env:trace({require, Meta, Ref, Opts}, E), + RE = E#{requires := ordsets:add_element(Ref, ?key(E, requires))}, + expand_alias(Meta, false, Ref, Opts, RE). + +expand_alias(Meta, IncludeByDefault, Ref, Opts, #{context_modules := Context} = E) -> + New = expand_as(lists:keyfind(as, 1, Opts), Meta, IncludeByDefault, Ref, E), + + %% Add the alias to context_modules if defined is set. + %% This is used by defmodule in order to store the defined + %% module in context modules. + NewContext = + case lists:keyfind(defined, 1, Meta) of + {defined, Mod} when is_atom(Mod) -> [Mod | Context]; + false -> Context + end, + + {Aliases, MacroAliases} = elixir_aliases:store(Meta, New, Ref, Opts, E), + E#{aliases := Aliases, macro_aliases := MacroAliases, context_modules := NewContext}. + +expand_as({as, nil}, _Meta, _IncludeByDefault, Ref, _E) -> + Ref; +expand_as({as, Atom}, Meta, _IncludeByDefault, _Ref, E) when is_atom(Atom), not is_boolean(Atom) -> + case atom_to_list(Atom) of + "Elixir." ++ ([FirstLetter | _] = Rest) when FirstLetter >= $A, FirstLetter =< $Z -> + case string:tokens(Rest, ".") of + [_] -> + Atom; + _ -> + form_error(Meta, E, ?MODULE, {invalid_alias_for_as, nested_alias, Atom}) + end; + _ -> + form_error(Meta, E, ?MODULE, {invalid_alias_for_as, not_alias, Atom}) + end; +expand_as(false, Meta, IncludeByDefault, Ref, E) -> + if + IncludeByDefault -> + case elixir_aliases:last(Ref) of + {ok, NewRef} -> NewRef; + error -> form_error(Meta, E, ?MODULE, {invalid_alias_module, Ref}) + end; + true -> Ref + end; +expand_as({as, Other}, Meta, _IncludeByDefault, _Ref, E) -> + form_error(Meta, E, ?MODULE, {invalid_alias_for_as, not_alias, Other}). + +%% Aliases + +expand_without_aliases_report({'__aliases__', _, _} = Alias, S, E) -> + expand_aliases(Alias, S, E, false); +expand_without_aliases_report(Other, S, E) -> + expand(Other, S, E). + +expand_aliases({'__aliases__', Meta, _} = Alias, S, E, Report) -> + case elixir_aliases:expand_or_concat(Alias, E) of + Receiver when is_atom(Receiver) -> + Report andalso elixir_env:trace({alias_reference, Meta, Receiver}, E), + {Receiver, S, E}; + + Aliases -> + {EAliases, SA, EA} = expand_args(Aliases, S, E), + + case lists:all(fun is_atom/1, EAliases) of + true -> + Receiver = elixir_aliases:concat(EAliases), + Report andalso elixir_env:trace({alias_reference, Meta, Receiver}, E), + {Receiver, SA, EA}; + + false -> + form_error(Meta, E, ?MODULE, {invalid_alias, Alias}) + end + end. + +%% Comprehensions + +expand_for({'<-', Meta, [Left, Right]}, S, E) -> + {ERight, SR, ER} = expand(Right, S, E), + SM = elixir_env:reset_read(SR, S), + {[ELeft], SL, EL} = elixir_clauses:head([Left], SM, ER), + {{'<-', Meta, [ELeft, ERight]}, SL, EL}; +expand_for({'<<>>', Meta, Args} = X, S, E) when is_list(Args) -> + case elixir_utils:split_last(Args) of + {LeftStart, {'<-', OpMeta, [LeftEnd, Right]}} -> + {ERight, SR, ER} = expand(Right, S, E), + SM = elixir_env:reset_read(SR, S), + {ELeft, SL, EL} = elixir_clauses:match(fun(BArg, BS, BE) -> + elixir_bitstring:expand(Meta, BArg, BS, BE, true) + end, LeftStart ++ [LeftEnd], SM, SM, ER), + {{'<<>>', Meta, [{'<-', OpMeta, [ELeft, ERight]}]}, SL, EL}; + _ -> + expand(X, S, E) + end; +expand_for(X, S, E) -> + expand(X, S, E). + +assert_generator_start(_, [{'<-', _, [_, _]} | _], _) -> + ok; +assert_generator_start(_, [{'<<>>', _, [{'<-', _, [_, _]}]} | _], _) -> + ok; +assert_generator_start(Meta, _, E) -> + elixir_errors:form_error(Meta, E, ?MODULE, for_generator_start). + +%% Assertions + +refute_parallel_bitstring_match({'<<>>', _, _}, {'<<>>', Meta, _} = Arg, E, true) -> + form_error(Meta, E, ?MODULE, {parallel_bitstring_match, Arg}); +refute_parallel_bitstring_match(Left, {'=', _Meta, [MatchLeft, MatchRight]}, E, Parallel) -> + refute_parallel_bitstring_match(Left, MatchLeft, E, true), + refute_parallel_bitstring_match(Left, MatchRight, E, Parallel); +refute_parallel_bitstring_match([_ | _] = Left, [_ | _] = Right, E, Parallel) -> + refute_parallel_bitstring_match_each(Left, Right, E, Parallel); +refute_parallel_bitstring_match({Left1, Left2}, {Right1, Right2}, E, Parallel) -> + refute_parallel_bitstring_match_each([Left1, Left2], [Right1, Right2], E, Parallel); +refute_parallel_bitstring_match({'{}', _, Args1}, {'{}', _, Args2}, E, Parallel) -> + refute_parallel_bitstring_match_each(Args1, Args2, E, Parallel); +refute_parallel_bitstring_match({'%{}', _, Args1}, {'%{}', _, Args2}, E, Parallel) -> + refute_parallel_bitstring_match_map_field(lists:sort(Args1), lists:sort(Args2), E, Parallel); +refute_parallel_bitstring_match({'%', _, [_, Args]}, Right, E, Parallel) -> + refute_parallel_bitstring_match(Args, Right, E, Parallel); +refute_parallel_bitstring_match(Left, {'%', _, [_, Args]}, E, Parallel) -> + refute_parallel_bitstring_match(Left, Args, E, Parallel); +refute_parallel_bitstring_match(_Left, _Right, _E, _Parallel) -> + ok. + +refute_parallel_bitstring_match_each([Arg1 | Rest1], [Arg2 | Rest2], E, Parallel) -> + refute_parallel_bitstring_match(Arg1, Arg2, E, Parallel), + refute_parallel_bitstring_match_each(Rest1, Rest2, E, Parallel); +refute_parallel_bitstring_match_each(_List1, _List2, _E, _Parallel) -> + ok. + +refute_parallel_bitstring_match_map_field([{Key, Val1} | Rest1], [{Key, Val2} | Rest2], E, Parallel) -> + refute_parallel_bitstring_match(Val1, Val2, E, Parallel), + refute_parallel_bitstring_match_map_field(Rest1, Rest2, E, Parallel); +refute_parallel_bitstring_match_map_field([Field1 | Rest1] = Args1, [Field2 | Rest2] = Args2, E, Parallel) -> + case Field1 > Field2 of + true -> + refute_parallel_bitstring_match_map_field(Args1, Rest2, E, Parallel); + false -> + refute_parallel_bitstring_match_map_field(Rest1, Args2, E, Parallel) + end; +refute_parallel_bitstring_match_map_field(_Args1, _Args2, _E, _Parallel) -> + ok. + +assert_module_scope(Meta, Kind, #{module := nil, file := File}) -> + form_error(Meta, File, ?MODULE, {invalid_expr_in_scope, "module", Kind}); +assert_module_scope(_Meta, _Kind, #{module:=Module}) -> Module. + +assert_function_scope(Meta, Kind, #{function := nil, file := File}) -> + form_error(Meta, File, ?MODULE, {invalid_expr_in_scope, "function", Kind}); +assert_function_scope(_Meta, _Kind, #{function := Function}) -> Function. + +assert_no_match_or_guard_scope(Meta, Kind, S, E) -> + assert_no_match_scope(Meta, Kind, E), + assert_no_guard_scope(Meta, Kind, S, E). +assert_no_match_scope(Meta, Kind, #{context := match, file := File}) -> + form_error(Meta, File, ?MODULE, {invalid_pattern_in_match, Kind}); +assert_no_match_scope(_Meta, _Kind, _E) -> []. +assert_no_guard_scope(Meta, Kind, S, #{context := guard, file := File}) -> + Key = + case S#elixir_ex.prematch of + {bitsize, _, _} -> invalid_expr_in_bitsize; + _ -> invalid_expr_in_guard + end, + form_error(Meta, File, ?MODULE, {Key, Kind}); +assert_no_guard_scope(_Meta, _Kind, _S, _E) -> []. + +%% Here we look into the Clauses "optimistically", that is, we don't check for +%% multiple "do"s and similar stuff. After all, the error we're gonna give here +%% is just a friendlier version of the "undefined variable _" error that we +%% would raise if we found a "_ -> ..." clause in a "cond". For this reason, if +%% Clauses has a bad shape, we just do nothing and let future functions catch +%% this. +assert_no_underscore_clause_in_cond([{do, Clauses}], E) when is_list(Clauses) -> + case lists:last(Clauses) of + {'->', Meta, [[{'_', _, Atom}], _]} when is_atom(Atom) -> + form_error(Meta, E, ?MODULE, underscore_in_cond); + _Other -> + ok + end; +assert_no_underscore_clause_in_cond(_Other, _E) -> + ok. + +%% Errors + +guard_context(#elixir_ex{prematch={bitsize, _, _}}) -> "bitstring size specifier"; +guard_context(_) -> "guards". + +format_error({useless_literal, Term}) -> + io_lib:format("code block contains unused literal ~ts " + "(remove the literal or assign it to _ to avoid warnings)", + ['Elixir.Macro':to_string(Term)]); +format_error({useless_var, Var}) -> + io_lib:format("variable ~ts in code block has no effect as it is never returned " + "(remove the variable or assign it to _ to avoid warnings)", + [Var]); +format_error({useless_attr, Attr}) -> + io_lib:format("module attribute @~ts in code block has no effect as it is never returned " + "(remove the attribute or assign it to _ to avoid warnings)", + [Attr]); +format_error({missing_option, Construct, Opts}) when is_list(Opts) -> + StringOpts = lists:map(fun(Opt) -> [$: | atom_to_list(Opt)] end, Opts), + io_lib:format("missing ~ts option in \"~ts\"", [string:join(StringOpts, "/"), Construct]); +format_error({invalid_args, Construct}) -> + io_lib:format("invalid arguments for \"~ts\"", [Construct]); +format_error({for_invalid_uniq, Value}) -> + io_lib:format(":uniq option for comprehensions only accepts a boolean, got: ~ts", ['Elixir.Macro':to_string(Value)]); +format_error(for_conflicting_reduce_into_uniq) -> + "cannot use :reduce alongside :into/:uniq in comprehension"; +format_error(for_with_reduce_bad_block) -> + "when using :reduce with comprehensions, the do block must be written using acc -> expr clauses, where each clause expects the accumulator as a single argument"; +format_error(for_without_reduce_bad_block) -> + "the do block was written using acc -> expr clauses but the :reduce option was not given"; +format_error(for_generator_start) -> + "for comprehensions must start with a generator"; +format_error(unhandled_arrow_op) -> + "unhandled operator ->"; +format_error(as_in_multi_alias_call) -> + ":as option is not supported by multi-alias call"; +format_error({invalid_alias_module, Ref}) -> + io_lib:format("alias cannot be inferred automatically for module: ~ts, please use the :as option. Implicit aliasing is only supported with Elixir modules", + ['Elixir.Macro':to_string(Ref)]); +format_error({commonly_mistaken_alias, Ref}) -> + Module = 'Elixir.Macro':to_string(Ref), + io_lib:format("reserved alias \"~ts\" expands to the atom :\"Elixir.~ts\". Perhaps you meant to write \"~ts\" instead?", [Module, Module, string:casefold(Module)]); +format_error({expected_compile_time_module, Kind, GivenTerm}) -> + io_lib:format("invalid argument for ~ts, expected a compile time atom or alias, got: ~ts", + [Kind, 'Elixir.Macro':to_string(GivenTerm)]); +format_error({unquote_outside_quote, Unquote}) -> + %% Unquote can be "unquote" or "unquote_splicing". + io_lib:format("~p called outside quote", [Unquote]); +format_error({invalid_bind_quoted_for_quote, BQ}) -> + io_lib:format("invalid :bind_quoted for quote, expected a keyword list of variable names, got: ~ts", + ['Elixir.Macro':to_string(BQ)]); +format_error(wrong_number_of_args_for_super) -> + "super must be called with the same number of arguments as the current definition"; +format_error({invalid_arg_for_pin, Arg}) -> + io_lib:format("invalid argument for unary operator ^, expected an existing variable, got: ^~ts", + ['Elixir.Macro':to_string(Arg)]); +format_error({pin_outside_of_match, Arg}) -> + io_lib:format("cannot use ^~ts outside of match clauses", ['Elixir.Macro':to_string(Arg)]); +format_error(unbound_underscore) -> + "invalid use of _. \"_\" represents a value to be ignored in a pattern and cannot be used in expressions"; +format_error({undefined_var, Name, Kind}) -> + io_lib:format("undefined variable \"~ts\"~ts", [Name, context_info(Kind)]); +format_error({undefined_var_pin, Name, Kind}) -> + Message = "undefined variable ^~ts. No variable \"~ts\"~ts has been defined before the current pattern", + io_lib:format(Message, [Name, Name, context_info(Kind)]); +format_error(underscore_in_cond) -> + "invalid use of _ inside \"cond\". If you want the last clause to always match, " + "you probably meant to use: true ->"; +format_error({invalid_pattern_in_match, Kind}) -> + io_lib:format("invalid pattern in match, ~ts is not allowed in matches", [Kind]); +format_error({invalid_expr_in_scope, Scope, Kind}) -> + io_lib:format("cannot invoke ~ts outside ~ts", [Kind, Scope]); +format_error({invalid_expr_in_guard, Kind}) -> + Message = + "invalid expression in guards, ~ts is not allowed in guards. To learn more about " + "guards, visit: https://hexdocs.pm/elixir/patterns-and-guards.html", + io_lib:format(Message, [Kind]); +format_error({invalid_expr_in_bitsize, Kind}) -> + Message = + "~ts is not allowed in bitstring size specifier. The size specifier in matches works like guards. " + "To learn more about guards, visit: https://hexdocs.pm/elixir/patterns-and-guards.html", + io_lib:format(Message, [Kind]); +format_error({invalid_alias, Expr}) -> + Message = + "invalid alias: \"~ts\". If you wanted to define an alias, an alias must expand " + "to an atom at compile time but it did not, you may use Module.concat/2 to build " + "it at runtime. If instead you wanted to invoke a function or access a field, " + "wrap the function or field name in double quotes", + io_lib:format(Message, ['Elixir.Macro':to_string(Expr)]); +format_error({op_ambiguity, Name, Arg}) -> + NameString = atom_to_binary(Name), + ArgString = 'Elixir.Macro':to_string(Arg), + + Message = + "\"~ts ~ts\" looks like a function call but there is a variable named \"~ts\". " + "If you want to perform a function call, use parentheses:\n" + "\n" + " ~ts(~ts)\n" + "\n" + "If you want to perform an operation on the variable ~ts, use spaces " + "around the unary operator", + io_lib:format(Message, [NameString, ArgString, NameString, NameString, ArgString, NameString]); +format_error({invalid_clauses, Name}) -> + Message = + "the function \"~ts\" cannot handle clauses with the -> operator because it is not a macro. " + "Please make sure you are invoking the proper name and that it is a macro", + io_lib:format(Message, [Name]); +format_error({invalid_alias_for_as, Reason, Value}) -> + ExpectedGot = + case Reason of + not_alias -> "expected an alias, got"; + nested_alias -> "expected a simple alias, got nested alias" + end, + io_lib:format("invalid value for option :as, ~ts: ~ts", + [ExpectedGot, 'Elixir.Macro':to_string(Value)]); +format_error({invalid_function_call, Expr}) -> + io_lib:format("invalid function call :~ts.()", [Expr]); +format_error({invalid_call, Call}) -> + io_lib:format("invalid call ~ts", ['Elixir.Macro':to_string(Call)]); +format_error({invalid_quoted_expr, Expr}) -> + Message = + "invalid quoted expression: ~ts\n\n" + "Please make sure your quoted expressions are made of valid AST nodes. " + "If you would like to introduce a value into the AST, such as a four-element " + "tuple or a map, make sure to call Macro.escape/1 before", + io_lib:format(Message, ['Elixir.Kernel':inspect(Expr, [])]); +format_error({invalid_local_invocation, Context, {Name, _, Args} = Call}) -> + Message = + "cannot find or invoke local ~ts/~B inside ~ts. " + "Only macros can be invoked in a ~ts and they must be defined before their invocation. Called as: ~ts", + io_lib:format(Message, [Name, length(Args), Context, Context, 'Elixir.Macro':to_string(Call)]); +format_error({invalid_pid_in_function, Pid, {Name, Arity}}) -> + io_lib:format("cannot compile PID ~ts inside quoted expression for function ~ts/~B", + ['Elixir.Kernel':inspect(Pid, []), Name, Arity]); +format_error({unsupported_option, Kind, Key}) -> + io_lib:format("unsupported option ~ts given to ~s", + ['Elixir.Macro':to_string(Key), Kind]); +format_error({options_are_not_keyword, Kind, Opts}) -> + io_lib:format("invalid options for ~s, expected a keyword list, got: ~ts", + [Kind, 'Elixir.Macro':to_string(Opts)]); +format_error({undefined_function, '|', [_, _]}) -> + "misplaced operator |/2\n\n" + "The | operator is typically used between brackets as the cons operator:\n\n" + " [head | tail]\n\n" + "where head is a single element and the tail is the remaining of a list.\n" + "It is also used to update maps and structs, via the %{map | key: value} notation,\n" + "and in typespecs, such as @type and @spec, to express the union of two types"; +format_error({undefined_function, '::', [_, _]}) -> + "misplaced operator ::/2\n\n" + "The :: operator is typically used in bitstrings to specify types and sizes of segments:\n\n" + " <>\n\n" + "It is also used in typespecs, such as @type and @spec, to describe inputs and outputs"; +format_error({undefined_function, Name, Args}) -> + io_lib:format("undefined function ~ts/~B (there is no such import)", [Name, length(Args)]); +format_error({underscored_var_repeat, Name, Kind}) -> + io_lib:format("the underscored variable \"~ts\"~ts appears more than once in a " + "match. This means the pattern will only match if all \"~ts\" bind " + "to the same value. If this is the intended behaviour, please " + "remove the leading underscore from the variable name, otherwise " + "give the variables different names", [Name, context_info(Kind), Name]); +format_error({underscored_var_access, Name}) -> + io_lib:format("the underscored variable \"~ts\" is used after being set. " + "A leading underscore indicates that the value of the variable " + "should be ignored. If this is intended please rename the " + "variable to remove the underscore", [Name]); +format_error({struct_comparison, StructExpr}) -> + String = 'Elixir.Macro':to_string(StructExpr), + io_lib:format("invalid comparison with struct literal ~ts. Comparison operators " + "(>, <, >=, <=, min, and max) perform structural and not semantic comparison. " + "Comparing with a struct literal is unlikely to give a meaningful result. " + "Struct modules typically define a compare/2 function that can be used for " + "semantic comparison", [String]); +format_error({nested_comparison, CompExpr}) -> + String = 'Elixir.Macro':to_string(CompExpr), + io_lib:format("Elixir does not support nested comparisons. Something like\n\n" + " x < y < z\n\n" + "is equivalent to\n\n" + " (x < y) < z\n\n" + "which ultimately compares z with the boolean result of (x < y). " + "Instead, consider joining together each comparison segment with an \"and\", for example,\n\n" + " x < y and y < z\n\n" + "You wrote: ~ts", [String]); +format_error({undefined_local_capture, Fun, Arity}) -> + io_lib:format("undefined function ~ts/~B (there is no such import)", [Fun, Arity]); +format_error(caller_not_allowed) -> + "__CALLER__ is available only inside defmacro and defmacrop"; +format_error(stacktrace_not_allowed) -> + "__STACKTRACE__ is available only inside catch and rescue clauses of try expressions"; +format_error({unknown_variable, Name}) -> + io_lib:format("variable \"~ts\" does not exist and is being expanded to \"~ts()\"," + " please use parentheses to remove the ambiguity or change the variable name", [Name, Name]); +format_error({parens_map_lookup, Map, Field, Context}) -> + io_lib:format("cannot invoke remote function in ~ts. " + "If you want to do a map lookup instead, please remove parens from ~ts.~ts()", + [Context, 'Elixir.Macro':to_string(Map), Field]); +format_error({super_in_genserver, {Name, Arity}}) -> + io_lib:format("calling super for GenServer callback ~ts/~B is deprecated", [Name, Arity]); +format_error({parallel_bitstring_match, Expr}) -> + Message = + "binary patterns cannot be matched in parallel using \"=\", excess pattern: ~ts", + io_lib:format(Message, ['Elixir.Macro':to_string(Expr)]). diff --git a/lib/elixir/src/elixir_fn.erl b/lib/elixir/src/elixir_fn.erl index d51d5702818..7085cda7e42 100644 --- a/lib/elixir/src/elixir_fn.erl +++ b/lib/elixir/src/elixir_fn.erl @@ -1,164 +1,201 @@ -module(elixir_fn). --export([translate/3, capture/3, expand/3]). --import(elixir_errors, [compile_error/3, compile_error/4]). +-export([capture/4, expand/4, format_error/1]). +-import(elixir_errors, [form_error/4]). -include("elixir.hrl"). -translate(Meta, Clauses, S) -> - Transformer = fun({'->', CMeta, [ArgsWithGuards, Expr]}, Acc) -> - {Args, Guards} = elixir_clauses:extract_splat_guards(ArgsWithGuards), - {TClause, TS } = elixir_clauses:clause(?line(CMeta), fun translate_fn_match/2, - Args, Expr, Guards, true, Acc), - {TClause, elixir_scope:mergef(S, TS)} +%% Anonymous functions + +expand(Meta, Clauses, S, E) when is_list(Clauses) -> + Transformer = fun({_, _, [Left, _Right]} = Clause, SA) -> + case lists:any(fun is_invalid_arg/1, Left) of + true -> + form_error(Meta, E, ?MODULE, defaults_in_args); + false -> + SReset = elixir_env:reset_unused_vars(SA), + + {EClause, SAcc, EAcc} = + elixir_clauses:clause(Meta, fn, fun elixir_clauses:head/3, Clause, SReset, E), + + {EClause, elixir_env:merge_and_check_unused_vars(SAcc, SA, EAcc)} + end end, - {TClauses, NS} = lists:mapfoldl(Transformer, S, Clauses), - Arities = [length(Args) || {clause, _Line, Args, _Guards, _Exprs} <- TClauses], + {EClauses, SE} = lists:mapfoldl(Transformer, S, Clauses), + EArities = [fn_arity(Args) || {'->', _, [Args, _]} <- EClauses], - case lists:usort(Arities) of + case lists:usort(EArities) of [_] -> - {{'fun', ?line(Meta), {clauses, TClauses}}, NS}; + {{fn, Meta, EClauses}, SE, E}; _ -> - compile_error(Meta, S#elixir_scope.file, - "cannot mix clauses with different arities in function definition") + form_error(Meta, E, ?MODULE, clauses_with_different_arities) end. -translate_fn_match(Arg, S) -> - {TArg, TS} = elixir_translator:translate_args(Arg, S#elixir_scope{backup_vars=orddict:new()}), - {TArg, TS#elixir_scope{backup_vars=S#elixir_scope.backup_vars}}. +is_invalid_arg({'\\\\', _, _}) -> true; +is_invalid_arg(_) -> false. -%% Expansion - -expand(Meta, Clauses, E) when is_list(Clauses) -> - Transformer = fun(Clause) -> - {EClause, _} = elixir_exp_clauses:clause(Meta, fn, fun elixir_exp_clauses:head/2, Clause, E), - EClause - end, - {{fn, Meta, lists:map(Transformer, Clauses)}, E}. +fn_arity([{'when', _, Args}]) -> length(Args) - 1; +fn_arity(Args) -> length(Args). %% Capture -capture(Meta, {'/', _, [{{'.', _, [_, F]} = Dot, RequireMeta , []}, A]}, E) when is_atom(F), is_integer(A) -> - Args = [{'&', [], [X]} || X <- lists:seq(1, A)], - capture_require(Meta, {Dot, RequireMeta, Args}, E, true); +capture(Meta, {'/', _, [{{'.', _, [M, F]} = Dot, DotMeta, []}, A]}, S, E) when is_atom(F), is_integer(A) -> + Args = args_from_arity(Meta, A, E), + handle_capture_possible_warning(Meta, DotMeta, M, F, A, E), + capture_require(Meta, {Dot, Meta, Args}, S, E, true); + +capture(Meta, {'/', _, [{F, _, C}, A]}, S, E) when is_atom(F), is_integer(A), is_atom(C) -> + Args = args_from_arity(Meta, A, E), + capture_import(Meta, {F, Meta, Args}, S, E, true); -capture(Meta, {'/', _, [{F, _, C}, A]}, E) when is_atom(F), is_integer(A), is_atom(C) -> - ImportMeta = - case lists:keyfind(import_fa, 1, Meta) of - {import_fa, {Receiver, Context}} -> - lists:keystore(context, 1, - lists:keystore(import, 1, Meta, {import, Receiver}), - {context, Context} - ); - false -> Meta - end, - Args = [{'&', [], [X]} || X <- lists:seq(1, A)], - capture_import(Meta, {F, ImportMeta, Args}, E, true); +capture(Meta, {{'.', _, [_, Fun]}, _, Args} = Expr, S, E) when is_atom(Fun), is_list(Args) -> + capture_require(Meta, Expr, S, E, is_sequential_and_not_empty(Args)); -capture(Meta, {{'.', _, [_, Fun]}, _, Args} = Expr, E) when is_atom(Fun), is_list(Args) -> - capture_require(Meta, Expr, E, is_sequential_and_not_empty(Args)); +capture(Meta, {{'.', _, [_]}, _, Args} = Expr, S, E) when is_list(Args) -> + capture_expr(Meta, Expr, S, E, false); -capture(Meta, {{'.', _, [_]}, _, Args} = Expr, E) when is_list(Args) -> - do_capture(Meta, Expr, E, false); +capture(Meta, {'__block__', _, [Expr]}, S, E) -> + capture(Meta, Expr, S, E); -capture(Meta, {'__block__', _, [Expr]}, E) -> - capture(Meta, Expr, E); +capture(Meta, {'__block__', _, _} = Expr, _S, E) -> + form_error(Meta, E, ?MODULE, {block_expr_in_capture, Expr}); -capture(Meta, {'__block__', _, _} = Expr, E) -> - Message = "invalid args for &, block expressions are not allowed, got: ~ts", - compile_error(Meta, ?m(E, file), Message, ['Elixir.Macro':to_string(Expr)]); +capture(Meta, {Atom, _, Args} = Expr, S, E) when is_atom(Atom), is_list(Args) -> + capture_import(Meta, Expr, S, E, is_sequential_and_not_empty(Args)); -capture(Meta, {Atom, _, Args} = Expr, E) when is_atom(Atom), is_list(Args) -> - capture_import(Meta, Expr, E, is_sequential_and_not_empty(Args)); +capture(Meta, {Left, Right}, S, E) -> + capture(Meta, {'{}', Meta, [Left, Right]}, S, E); -capture(Meta, {Left, Right}, E) -> - capture(Meta, {'{}', Meta, [Left, Right]}, E); +capture(Meta, List, S, E) when is_list(List) -> + capture_expr(Meta, List, S, E, is_sequential_and_not_empty(List)); -capture(Meta, List, E) when is_list(List) -> - do_capture(Meta, List, E, is_sequential_and_not_empty(List)); +capture(Meta, Integer, _S, E) when is_integer(Integer) -> + form_error(Meta, E, ?MODULE, {capture_arg_outside_of_capture, Integer}); -capture(Meta, Arg, E) -> +capture(Meta, Arg, _S, E) -> invalid_capture(Meta, Arg, E). -capture_import(Meta, {Atom, ImportMeta, Args} = Expr, E, Sequential) -> +capture_import(Meta, {Atom, ImportMeta, Args} = Expr, S, E, Sequential) -> Res = Sequential andalso elixir_dispatch:import_function(ImportMeta, Atom, length(Args), E), - handle_capture(Res, Meta, Expr, E, Sequential). - -capture_require(Meta, {{'.', _, [Left, Right]}, RequireMeta, Args} = Expr, E, Sequential) -> - {Mod, EE} = elixir_exp:expand(Left, E), - Res = Sequential andalso case Mod of - {Name, _, Context} when is_atom(Name), is_atom(Context) -> - {remote, Mod, Right, length(Args)}; - _ when is_atom(Mod) -> - elixir_dispatch:require_function(RequireMeta, Mod, Right, length(Args), EE); - _ -> - false - end, - handle_capture(Res, Meta, Expr, EE, Sequential). - -handle_capture({local, Fun, Arity}, _Meta, _Expr, _E, _Sequential) -> - {local, Fun, Arity}; -handle_capture({remote, Receiver, Fun, Arity}, Meta, _Expr, E, _Sequential) -> - Tree = {{'.', [], [erlang, make_fun]}, Meta, [Receiver, Fun, Arity]}, - {expanded, Tree, E}; -handle_capture(false, Meta, Expr, E, Sequential) -> - do_capture(Meta, Expr, E, Sequential). - -do_capture(Meta, Expr, E, Sequential) -> - case do_escape(Expr, elixir_counter:next(), E, []) of + handle_capture(Res, Meta, Expr, S, E, Sequential). + +capture_require(Meta, {{'.', DotMeta, [Left, Right]}, RequireMeta, Args}, S, E, Sequential) -> + case escape(Left, E, []) of + {EscLeft, []} -> + {ELeft, SE, EE} = elixir_expand:expand(EscLeft, S, E), + + Res = Sequential andalso case ELeft of + {Name, _, Context} when is_atom(Name), is_atom(Context) -> + {remote, ELeft, Right, length(Args)}; + _ when is_atom(ELeft) -> + elixir_dispatch:require_function(RequireMeta, ELeft, Right, length(Args), EE); + _ -> + false + end, + + Dot = {{'.', DotMeta, [ELeft, Right]}, RequireMeta, Args}, + handle_capture(Res, Meta, Dot, SE, EE, Sequential); + + {EscLeft, Escaped} -> + Dot = {{'.', DotMeta, [EscLeft, Right]}, RequireMeta, Args}, + capture_expr(Meta, Dot, S, E, Escaped, Sequential) + end. + +handle_capture(false, Meta, Expr, S, E, Sequential) -> + capture_expr(Meta, Expr, S, E, Sequential); +handle_capture(LocalOrRemote, _Meta, _Expr, S, E, _Sequential) -> + {LocalOrRemote, S, E}. + +capture_expr(Meta, Expr, S, E, Sequential) -> + capture_expr(Meta, Expr, S, E, [], Sequential). +capture_expr(Meta, Expr, S, E, Escaped, Sequential) -> + case escape(Expr, E, Escaped) of {_, []} when not Sequential -> invalid_capture(Meta, Expr, E); {EExpr, EDict} -> EVars = validate(Meta, EDict, 1, E), Fn = {fn, Meta, [{'->', Meta, [EVars, EExpr]}]}, - {expanded, Fn, E} + {expand, Fn, S, E} end. invalid_capture(Meta, Arg, E) -> - Message = "invalid args for &, expected an expression in the format of &Mod.fun/arity, " - "&local/arity or a capture containing at least one argument as &1, got: ~ts", - compile_error(Meta, ?m(E, file), Message, ['Elixir.Macro':to_string(Arg)]). - -validate(Meta, [{Pos, Var}|T], Pos, E) -> - [Var|validate(Meta, T, Pos + 1, E)]; - -validate(Meta, [{Pos, _}|_], Expected, E) -> - compile_error(Meta, ?m(E, file), "capture &~B cannot be defined without &~B", [Pos, Expected]); + form_error(Meta, E, ?MODULE, {invalid_args_for_capture, Arg}). +validate(Meta, [{Pos, Var} | T], Pos, E) -> + [Var | validate(Meta, T, Pos + 1, E)]; +validate(Meta, [{Pos, _} | _], Expected, E) -> + form_error(Meta, E, ?MODULE, {capture_arg_without_predecessor, Pos, Expected}); validate(_Meta, [], _Pos, _E) -> []. -do_escape({'&', _, [Pos]}, Counter, _E, Dict) when is_integer(Pos), Pos > 0 -> - Var = {list_to_atom([$x, $@+Pos]), [{counter, Counter}], elixir_fn}, +escape({'&', _, [Pos]}, _E, Dict) when is_integer(Pos), Pos > 0 -> + Var = {list_to_atom([$x | integer_to_list(Pos)]), [], ?var_context}, {Var, orddict:store(Pos, Var, Dict)}; - -do_escape({'&', Meta, [Pos]}, _Counter, E, _Dict) when is_integer(Pos) -> - compile_error(Meta, ?m(E, file), "capture &~B is not allowed", [Pos]); - -do_escape({'&', Meta, _} = Arg, _Counter, E, _Dict) -> - Message = "nested captures via & are not allowed: ~ts", - compile_error(Meta, ?m(E, file), Message, ['Elixir.Macro':to_string(Arg)]); - -do_escape({Left, Meta, Right}, Counter, E, Dict0) -> - {TLeft, Dict1} = do_escape(Left, Counter, E, Dict0), - {TRight, Dict2} = do_escape(Right, Counter, E, Dict1), +escape({'&', Meta, [Pos]}, E, _Dict) when is_integer(Pos) -> + form_error(Meta, E, ?MODULE, {invalid_arity_for_capture, Pos}); +escape({'&', Meta, _} = Arg, E, _Dict) -> + form_error(Meta, E, ?MODULE, {nested_capture, Arg}); +escape({Left, Meta, Right}, E, Dict0) -> + {TLeft, Dict1} = escape(Left, E, Dict0), + {TRight, Dict2} = escape(Right, E, Dict1), {{TLeft, Meta, TRight}, Dict2}; - -do_escape({Left, Right}, Counter, E, Dict0) -> - {TLeft, Dict1} = do_escape(Left, Counter, E, Dict0), - {TRight, Dict2} = do_escape(Right, Counter, E, Dict1), +escape({Left, Right}, E, Dict0) -> + {TLeft, Dict1} = escape(Left, E, Dict0), + {TRight, Dict2} = escape(Right, E, Dict1), {{TLeft, TRight}, Dict2}; - -do_escape(List, Counter, E, Dict) when is_list(List) -> - lists:mapfoldl(fun(X, Acc) -> do_escape(X, Counter, E, Acc) end, Dict, List); - -do_escape(Other, _Counter, _E, Dict) -> +escape(List, E, Dict) when is_list(List) -> + lists:mapfoldl(fun(X, Acc) -> escape(X, E, Acc) end, Dict, List); +escape(Other, _E, Dict) -> {Other, Dict}. +args_from_arity(_Meta, A, _E) when is_integer(A), A >= 0, A =< 255 -> + [{'&', [], [X]} || X <- lists:seq(1, A)]; +args_from_arity(Meta, A, E) -> + form_error(Meta, E, ?MODULE, {invalid_arity_for_capture, A}). + is_sequential_and_not_empty([]) -> false; is_sequential_and_not_empty(List) -> is_sequential(List, 1). -is_sequential([{'&', _, [Int]}|T], Int) -> - is_sequential(T, Int + 1); +is_sequential([{'&', _, [Int]} | T], Int) -> is_sequential(T, Int + 1); is_sequential([], _Int) -> true; is_sequential(_, _Int) -> false. + +handle_capture_possible_warning(Meta, DotMeta, Mod, Fun, Arity, E) -> + case (Arity =:= 0) andalso (lists:keyfind(no_parens, 1, DotMeta) /= {no_parens, true}) of + true -> + elixir_errors:form_warn(Meta, E, ?MODULE, {parens_remote_capture, Mod, Fun}); + + false -> ok + end. + +%% TODO: Raise on Elixir v2.0 +format_error({parens_remote_capture, Mod, Fun}) -> + io_lib:format("extra parentheses on a remote function capture &~ts.~ts()/0 have been " + "deprecated. Please remove the parentheses: &~ts.~ts/0", + ['Elixir.Macro':to_string(Mod), Fun, 'Elixir.Macro':to_string(Mod), Fun]); +format_error(clauses_with_different_arities) -> + "cannot mix clauses with different arities in anonymous functions"; +format_error(defaults_in_args) -> + "anonymous functions cannot have optional arguments"; +format_error({block_expr_in_capture, Expr}) -> + io_lib:format("block expressions are not allowed inside the capture operator &, got: ~ts", + ['Elixir.Macro':to_string(Expr)]); +format_error({nested_capture, Arg}) -> + io_lib:format("nested captures are not allowed. You cannot define a function using " + " the capture operator & inside another function defined via &. Got invalid nested " + "capture: ~ts", ['Elixir.Macro':to_string(Arg)]); +format_error({invalid_arity_for_capture, Arity}) -> + io_lib:format("capture argument &~B must be numbered between 1 and 255", [Arity]); +format_error({capture_arg_outside_of_capture, Integer}) -> + io_lib:format("capture argument &~B must be used within the capture operator &", [Integer]); +format_error({capture_arg_without_predecessor, Pos, Expected}) -> + io_lib:format("capture argument &~B cannot be defined without &~B " + "(you cannot skip arguments, all arguments must be numbered)", [Pos, Expected]); +format_error({invalid_args_for_capture, Arg}) -> + Message = + "invalid args for &, expected one of:\n\n" + " * &Mod.fun/arity to capture a remote function, such as &Enum.map/2\n" + " * &fun/arity to capture a local or imported function, such as &is_atom/1\n" + " * &some_code(&1, ...) containing at least one argument as &1, such as &List.flatten(&1)\n\n" + "Got: ~ts", + io_lib:format(Message, ['Elixir.Macro':to_string(Arg)]). diff --git a/lib/elixir/src/elixir_for.erl b/lib/elixir/src/elixir_for.erl deleted file mode 100644 index d017f6db4d0..00000000000 --- a/lib/elixir/src/elixir_for.erl +++ /dev/null @@ -1,338 +0,0 @@ --module(elixir_for). --export([expand/3, translate/3]). --include("elixir.hrl"). - -%% Expansion - -expand(Meta, Args, E) -> - {Cases, Block} = - case elixir_utils:split_last(Args) of - {OuterCases, OuterOpts} when is_list(OuterOpts) -> - case elixir_utils:split_last(OuterCases) of - {InnerCases, InnerOpts} when is_list(InnerOpts) -> - {InnerCases, InnerOpts ++ OuterOpts}; - _ -> - {OuterCases, OuterOpts} - end; - _ -> - {Args, []} - end, - - {Expr, Opts} = - case lists:keyfind(do, 1, Block) of - {do, Do} -> {Do, lists:keydelete(do, 1, Block)}; - _ -> elixir_errors:compile_error(Meta, ?m(E, file), - "missing do keyword in for comprehension") - end, - - {EOpts, EO} = elixir_exp:expand(Opts, E), - {ECases, EC} = lists:mapfoldl(fun expand/2, EO, Cases), - {EExpr, _} = elixir_exp:expand(Expr, EC), - {{for, Meta, ECases ++ [[{do,EExpr}|EOpts]]}, E}. - -expand({'<-', Meta, [Left, Right]}, E) -> - {ERight, ER} = elixir_exp:expand(Right, E), - {ELeft, EL} = elixir_exp_clauses:match(fun elixir_exp:expand/2, Left, E), - {{'<-', Meta, [ELeft, ERight]}, elixir_env:mergev(EL, ER)}; -expand({'<<>>', Meta, Args} = X, E) when is_list(Args) -> - case elixir_utils:split_last(Args) of - {LeftStart, {'<-', OpMeta, [LeftEnd, Right]}} -> - {ERight, ER} = elixir_exp:expand(Right, E), - Left = {'<<>>', Meta, LeftStart ++ [LeftEnd]}, - {ELeft, EL} = elixir_exp_clauses:match(fun elixir_exp:expand/2, Left, E), - {{'<<>>', [], [ {'<-', OpMeta, [ELeft, ERight]}]}, elixir_env:mergev(EL, ER)}; - _ -> - elixir_exp:expand(X, E) - end; -expand(X, E) -> - elixir_exp:expand(X, E). - -%% Translation - -translate(Meta, Args, #elixir_scope{return=Return} = RS) -> - S = RS#elixir_scope{return=true}, - {AccName, _, SA} = elixir_scope:build_var('_', S), - {VarName, _, SV} = elixir_scope:build_var('_', SA), - - Line = ?line(Meta), - Acc = {var, Line, AccName}, - Var = {var, Line, VarName}, - - {Cases, [{do,Expr}|Opts]} = elixir_utils:split_last(Args), - - {TInto, SI} = - case lists:keyfind(into, 1, Opts) of - {into, Into} -> elixir_translator:translate(Into, SV); - false when Return -> {{nil, Line}, SV}; - false -> {false, SV} - end, - - {TCases, SC} = translate_gen(Meta, Cases, [], SI), - {TExpr, SE} = elixir_translator:translate_block(Expr, Return, SC), - SF = elixir_scope:mergec(SI, SE), - - case comprehension_expr(TInto, TExpr) of - {inline, TIntoExpr} -> - {build_inline(Line, TCases, TIntoExpr, TInto, Var, Acc, SE), SF}; - {into, TIntoExpr} -> - build_into(Line, TCases, TIntoExpr, TInto, Var, Acc, SF) - end. - -translate_gen(ForMeta, [{'<-', Meta, [Left, Right]}|T], Acc, S) -> - {TLeft, TRight, TFilters, TT, TS} = translate_gen(Meta, Left, Right, T, S), - TAcc = [{enum, Meta, TLeft, TRight, TFilters}|Acc], - translate_gen(ForMeta, TT, TAcc, TS); -translate_gen(ForMeta, [{'<<>>', _, [ {'<-', Meta, [Left, Right]} ]}|T], Acc, S) -> - {TLeft, TRight, TFilters, TT, TS} = translate_gen(Meta, Left, Right, T, S), - TAcc = [{bin, Meta, TLeft, TRight, TFilters}|Acc], - case elixir_bitstring:has_size(TLeft) of - true -> translate_gen(ForMeta, TT, TAcc, TS); - false -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, - "bitstring fields without size are not allowed in bitstring generators") - end; -translate_gen(_ForMeta, [], Acc, S) -> - {lists:reverse(Acc), S}; -translate_gen(ForMeta, _, _, S) -> - elixir_errors:compile_error(ForMeta, S#elixir_scope.file, - "for comprehensions must start with a generator"). - -translate_gen(_Meta, Left, Right, T, S) -> - {TRight, SR} = elixir_translator:translate(Right, S), - {TLeft, SL} = elixir_clauses:match(fun elixir_translator:translate/2, Left, SR), - {TT, {TFilters, TS}} = translate_filters(T, SL), - {TLeft, TRight, TFilters, TT, TS}. - -translate_filters(T, S) -> - {Filters, Rest} = collect_filters(T, []), - {Rest, lists:mapfoldr(fun translate_filter/2, S, Filters)}. - -translate_filter(Filter, S) -> - {TFilter, TS} = elixir_translator:translate(Filter, S), - case elixir_utils:returns_boolean(Filter) of - true -> - {{nil, TFilter}, TS}; - false -> - {Name, _, VS} = elixir_scope:build_var('_', TS), - {{{var, 0, Name}, TFilter}, VS} - end. - -collect_filters([{'<-', _, [_, _]}|_] = T, Acc) -> - {Acc, T}; -collect_filters([{'<<>>', _, [{'<-', _, [_, _]}]}|_] = T, Acc) -> - {Acc, T}; -collect_filters([H|T], Acc) -> - collect_filters(T, [H|Acc]); -collect_filters([], Acc) -> - {Acc, []}. - -%% If all we have is one enum generator, we check if it is a list -%% for optimization otherwise fallback to the reduce generator. -build_inline(Line, [{enum, Meta, Left, Right, Filters}] = Orig, Expr, Into, Var, Acc, S) -> - case Right of - {cons, _, _, _} -> - build_comprehension(Line, Orig, Expr, Into); - {Other, _, _} when Other == tuple; Other == map -> - build_reduce(Orig, Expr, Into, Acc, S); - _ -> - Clauses = [{enum, Meta, Left, Var, Filters}], - - {'case', -1, Right, [ - {clause, -1, - [Var], - [[elixir_utils:erl_call(Line, erlang, is_list, [Var])]], - [build_comprehension(Line, Clauses, Expr, Into)]}, - {clause, -1, - [Var], - [], - [build_reduce(Clauses, Expr, Into, Acc, S)]} - ]} - end; - -build_inline(Line, Clauses, Expr, Into, _Var, Acc, S) -> - case lists:all(fun(Clause) -> element(1, Clause) == bin end, Clauses) of - true -> build_comprehension(Line, Clauses, Expr, Into); - false -> build_reduce(Clauses, Expr, Into, Acc, S) - end. - -build_into(Line, Clauses, Expr, Into, Fun, Acc, S) -> - {Kind, SK} = build_var(Line, S), - {Reason, SR} = build_var(Line, SK), - {Stack, ST} = build_var(Line, SR), - {Done, SD} = build_var(Line, ST), - - IntoExpr = {call, Line, Fun, [Acc, pair(Line, cont, Expr)]}, - MatchExpr = {match, Line, - {tuple, Line, [Acc, Fun]}, - elixir_utils:erl_call(Line, 'Elixir.Collectable', into, [Into]) - }, - - TryExpr = - {'try', Line, - [build_reduce_clause(Clauses, IntoExpr, Acc, Acc, SD)], - [{clause, Line, - [Done], - [], - [{call, Line, Fun, [Done, {atom, Line, done}]}]}], - [{clause, Line, - [{tuple, Line, [Kind, Reason, {var, Line, '_'}]}], - [], - [{match, Line, Stack, elixir_utils:erl_call(Line, erlang, get_stacktrace, [])}, - {call, Line, Fun, [Acc, {atom, Line, halt}]}, - elixir_utils:erl_call(Line, erlang, raise, [Kind, Reason, Stack])]}], - []}, - - {{block, Line, [MatchExpr, TryExpr]}, SD}. - -%% Helpers - -build_reduce(Clauses, Expr, false, Acc, S) -> - build_reduce_clause(Clauses, Expr, {nil, 0}, Acc, S); -build_reduce(Clauses, Expr, {nil, Line} = Into, Acc, S) -> - ListExpr = {cons, Line, Expr, Acc}, - elixir_utils:erl_call(Line, lists, reverse, - [build_reduce_clause(Clauses, ListExpr, Into, Acc, S)]); -build_reduce(Clauses, Expr, {bin, _, _} = Into, Acc, S) -> - {bin, Line, Elements} = Expr, - BinExpr = {bin, Line, [{bin_element, Line, Acc, default, [bitstring]}|Elements]}, - build_reduce_clause(Clauses, BinExpr, Into, Acc, S). - -build_reduce_clause([{enum, Meta, Left, Right, Filters}|T], Expr, Arg, Acc, S) -> - Line = ?line(Meta), - Inner = build_reduce_clause(T, Expr, Acc, Acc, S), - - True = pair(Line, cont, Inner), - False = pair(Line, cont, Acc), - - Clauses0 = - case is_var(Left) of - true -> []; - false -> - [{clause, -1, - [{var, Line, '_'}, Acc], [], - [False]}] - end, - - Clauses1 = - [{clause, Line, - [Left, Acc], [], - [join_filters(Line, Filters, True, False)]}|Clauses0], - - Args = [Right, pair(Line, cont, Arg), {'fun', Line, {clauses, Clauses1}}], - Tuple = elixir_utils:erl_call(Line, 'Elixir.Enumerable', reduce, Args), - - %% Use -1 because in case of no returns we don't care about the result - elixir_utils:erl_call(-1, erlang, element, [{integer, Line, 2}, Tuple]); - -build_reduce_clause([{bin, Meta, Left, Right, Filters}|T], Expr, Arg, Acc, S) -> - Line = ?line(Meta), - {Tail, ST} = build_var(Line, S), - {Fun, SF} = build_var(Line, ST), - - True = build_reduce_clause(T, Expr, Acc, Acc, SF), - False = Acc, - - {bin, _, Elements} = Left, - - BinMatch = - {bin, Line, Elements ++ [{bin_element, Line, Tail, default, [bitstring]}]}, - NoVarMatch = - {bin, Line, no_var(Elements) ++ [{bin_element, Line, Tail, default, [bitstring]}]}, - - Clauses = - [{clause, Line, - [BinMatch, Acc], [], - [{call, Line, Fun, [Tail, join_filters(Line, Filters, True, False)]}]}, - {clause, -1, - [NoVarMatch, Acc], [], - [{call, Line, Fun, [Tail, False]}]}, - {clause, -1, - [{bin, Line, []}, Acc], [], - [Acc]}, - {clause, -1, - [Tail, {var, Line, '_'}], [], - [elixir_utils:erl_call(Line, erlang, error, [pair(Line, badarg, Tail)])]}], - - {call, Line, - {named_fun, Line, element(3, Fun), Clauses}, - [Right, Arg]}; - -build_reduce_clause([], Expr, _Arg, _Acc, _S) -> - Expr. - -is_var({var, _, _}) -> true; -is_var(_) -> false. - -pair(Line, Atom, Arg) -> - {tuple, Line, [{atom, Line, Atom}, Arg]}. - -build_var(Line, S) -> - {Name, _, ST} = elixir_scope:build_var('_', S), - {{var, Line, Name}, ST}. - -no_var(Elements) -> - [{bin_element, Line, no_var_expr(Expr), Size, Types} || - {bin_element, Line, Expr, Size, Types} <- Elements]. -no_var_expr({var, Line, _}) -> - {var, Line, '_'}. - -build_comprehension(Line, Clauses, Expr, false) -> - {block, Line, [ - build_comprehension(Line, Clauses, Expr, {nil, Line}), - {nil, Line} - ]}; -build_comprehension(Line, Clauses, Expr, Into) -> - {comprehension_kind(Into), Line, Expr, comprehension_clause(Clauses)}. - -comprehension_clause([{Kind, Meta, Left, Right, Filters}|T]) -> - Line = ?line(Meta), - [{comprehension_generator(Kind), Line, Left, Right}] ++ - comprehension_filter(Line, Filters) ++ - comprehension_clause(T); -comprehension_clause([]) -> - []. - -comprehension_kind({nil, _}) -> lc; -comprehension_kind({bin, _, []}) -> bc. - -comprehension_generator(enum) -> generate; -comprehension_generator(bin) -> b_generate. - -comprehension_expr({bin, _, []}, {bin, _, _} = Expr) -> - {inline, Expr}; -comprehension_expr({bin, Line, []}, Expr) -> - BinExpr = {bin, Line, [{bin_element, Line, Expr, default, [bitstring]}]}, - {inline, BinExpr}; -comprehension_expr({nil, _}, Expr) -> - {inline, Expr}; -comprehension_expr(false, Expr) -> - {inline, Expr}; -comprehension_expr(_, Expr) -> - {into, Expr}. - -comprehension_filter(Line, Filters) -> - [join_filter(Line, Filter, {atom, Line, true}, {atom, Line, false}) || - Filter <- lists:reverse(Filters)]. - -join_filters(_Line, [], True, _False) -> - True; -join_filters(Line, [H|T], True, False) -> - lists:foldl(fun(Filter, Acc) -> - join_filter(Line, Filter, Acc, False) - end, join_filter(Line, H, True, False), T). - -join_filter(Line, {nil, Filter}, True, False) -> - {'case', Line, Filter, [ - {clause, Line, [{atom, Line, true}], [], [True]}, - {clause, Line, [{atom, Line, false}], [], [False]} - ]}; -join_filter(Line, {Var, Filter}, True, False) -> - Guard = - {op, Line, 'orelse', - {op, Line, '==', Var, {atom, Line, false}}, - {op, Line, '==', Var, {atom, Line, nil}}}, - - {'case', Line, Filter, [ - {clause, Line, [Var], [[Guard]], [False]}, - {clause, Line, [{var, Line, '_'}], [], [True]} - ]}. diff --git a/lib/elixir/src/elixir_import.erl b/lib/elixir/src/elixir_import.erl index ece17e12828..243d1e7fb96 100644 --- a/lib/elixir/src/elixir_import.erl +++ b/lib/elixir/src/elixir_import.erl @@ -1,73 +1,114 @@ %% Module responsible for handling imports and conflicts -%% in between local functions and imports. +%% between local functions and imports. %% For imports dispatch, please check elixir_dispatch. -module(elixir_import). -export([import/4, special_form/2, format_error/1]). -include("elixir.hrl"). -%% IMPORT - import(Meta, Ref, Opts, E) -> - Res = + {Functions, Macros, Added} = case keyfind(only, Opts) of {only, functions} -> - {import_functions(Meta, Ref, Opts, E), - ?m(E, macros)}; + {Added1, Funs} = import_functions(Meta, Ref, Opts, E), + {Funs, keydelete(Ref, ?key(E, macros)), Added1}; {only, macros} -> - {?m(E, functions), - import_macros(true, Meta, Ref, Opts, E)}; + {Added2, Macs} = import_macros(true, Meta, Ref, Opts, E), + {keydelete(Ref, ?key(E, functions)), Macs, Added2}; + {only, sigils} -> + {Added1, Funs} = import_sigil_functions(Meta, Ref, Opts, E), + {Added2, Macs} = import_sigil_macros(Meta, Ref, Opts, E), + {Funs, Macs, Added1 or Added2}; {only, List} when is_list(List) -> - {import_functions(Meta, Ref, Opts, E), - import_macros(false, Meta, Ref, Opts, E)}; + {Added1, Funs} = import_functions(Meta, Ref, Opts, E), + {Added2, Macs} = import_macros(false, Meta, Ref, Opts, E), + {Funs, Macs, Added1 or Added2}; + {only, Other} -> + elixir_errors:form_error(Meta, E, ?MODULE, {invalid_option, only, Other}); false -> - {import_functions(Meta, Ref, Opts, E), - import_macros(false, Meta, Ref, Opts, E)} + {Added1, Funs} = import_functions(Meta, Ref, Opts, E), + {Added2, Macs} = import_macros(false, Meta, Ref, Opts, E), + {Funs, Macs, Added1 or Added2} end, - record_warn(Meta, Ref, Opts, E), - Res. + elixir_env:trace({import, [{imported, Added} | Meta], Ref, Opts}, E), + {Functions, Macros}. import_functions(Meta, Ref, Opts, E) -> - calculate(Meta, Ref, Opts, ?m(E, functions), E, fun() -> get_functions(Ref) end). + calculate(Meta, Ref, Opts, ?key(E, functions), ?key(E, file), fun() -> + get_functions(Ref) + end). import_macros(Force, Meta, Ref, Opts, E) -> - calculate(Meta, Ref, Opts, ?m(E, macros), E, fun() -> - case Force of - true -> get_macros(Meta, Ref, E); - false -> get_optional_macros(Ref) + calculate(Meta, Ref, Opts, ?key(E, macros), ?key(E, file), fun() -> + case fetch_macros(Ref) of + {ok, Macros} -> + Macros; + error when Force -> + elixir_errors:form_error(Meta, E, ?MODULE, {no_macros, Ref}); + error -> + [] end end). -record_warn(Meta, Ref, Opts, E) -> - Warn = - case keyfind(warn, Opts) of - {warn, false} -> false; - {warn, true} -> true; - false -> not lists:keymember(context, 1, Meta) - end, - elixir_lexical:record_import(Ref, ?line(Meta), Warn, ?m(E, lexical_tracker)). +import_sigil_functions(Meta, Ref, Opts, E) -> + calculate(Meta, Ref, Opts, ?key(E, functions), ?key(E, file), fun() -> + filter_sigils(get_functions(Ref)) + end). + +import_sigil_macros(Meta, Ref, Opts, E) -> + calculate(Meta, Ref, Opts, ?key(E, macros), ?key(E, file), fun() -> + case fetch_macros(Ref) of + {ok, Macros} -> + filter_sigils(Macros); + error -> + [] + end + end). + +filter_sigils(Key) -> + lists:filter(fun({Atom, _}) -> + case atom_to_list(Atom) of + "sigil_" ++ [L] when L >= $a, L =< $z; L >= $A, L =< $Z -> true; + _ -> false + end + end, Key). %% Calculates the imports based on only and except -calculate(Meta, Key, Opts, Old, E, Existing) -> +calculate(Meta, Key, Opts, Old, File, Existing) -> New = case keyfind(only, Opts) of {only, Only} when is_list(Only) -> + ensure_keyword_list(Meta, File, Only, only), + ensure_no_duplicates(Meta, File, Only, only), + case keyfind(except, Opts) of + false -> + ok; + _ -> + elixir_errors:form_error(Meta, File, ?MODULE, only_and_except_given) + end, case Only -- get_exports(Key) of - [{Name,Arity}|_] -> - Tuple = {invalid_import, {Key, Name, Arity}}, - elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, Tuple); + [{Name, Arity} | _] -> + elixir_errors:form_error(Meta, File, ?MODULE, {invalid_import, {Key, Name, Arity}}); _ -> intersection(Only, Existing()) end; _ -> case keyfind(except, Opts) of - false -> remove_underscored(Existing()); - {except, []} -> remove_underscored(Existing()); + false -> + remove_underscored(Existing()); {except, Except} when is_list(Except) -> + ensure_keyword_list(Meta, File, Except, except), + ensure_no_duplicates(Meta, File, Except, except), + %% We are not checking existence of exports listed in :except option + %% on purpose: to support backwards compatible code. + %% For example, "import String, except: [trim: 1]" + %% should work across all Elixir versions. case keyfind(Key, Old) of false -> remove_underscored(Existing()) -- Except; - {Key,OldImports} -> OldImports -- Except - end + {Key, OldImports} -> OldImports -- Except + end; + {except, Other} -> + elixir_errors:form_error(Meta, File, ?MODULE, {invalid_option, except, Other}) end end, @@ -76,20 +117,17 @@ calculate(Meta, Key, Opts, Old, E, Existing) -> Final = remove_internals(Set), case Final of - [] -> keydelete(Key, Old); + [] -> + {false, keydelete(Key, Old)}; _ -> - ensure_no_special_form_conflict(Meta, ?m(E, file), Key, Final), - [{Key, Final}|keydelete(Key, Old)] + ensure_no_special_form_conflict(Meta, File, Key, Final), + {true, [{Key, Final} | keydelete(Key, Old)]} end. %% Retrieve functions and macros from modules get_exports(Module) -> - try - Module:'__info__'(functions) ++ Module:'__info__'(macros) - catch - error:undef -> Module:module_info(exports) - end. + get_functions(Module) ++ get_macros(Module). get_functions(Module) -> try @@ -98,46 +136,78 @@ get_functions(Module) -> error:undef -> Module:module_info(exports) end. -get_macros(Meta, Module, E) -> - try - Module:'__info__'(macros) - catch - error:undef -> - Tuple = {no_macros, Module}, - elixir_errors:form_error(Meta, ?m(E, file), ?MODULE, Tuple) +get_macros(Module) -> + case fetch_macros(Module) of + {ok, Macros} -> + Macros; + error -> + [] end. -get_optional_macros(Module) -> - case code:ensure_loaded(Module) of - {module, Module} -> - try - Module:'__info__'(macros) - catch - error:undef -> [] - end; - {error, _} -> [] +fetch_macros(Module) -> + try + {ok, Module:'__info__'(macros)} + catch + error:undef -> error end. %% VALIDATION HELPERS -ensure_no_special_form_conflict(Meta, File, Key, [{Name,Arity}|T]) -> +ensure_no_special_form_conflict(Meta, File, Key, [{Name, Arity} | T]) -> case special_form(Name, Arity) of true -> - Tuple = {special_form_conflict, {Key, Name, Arity}}, - elixir_errors:form_error(Meta, File, ?MODULE, Tuple); + elixir_errors:form_error(Meta, File, ?MODULE, {special_form_conflict, {Key, Name, Arity}}); false -> ensure_no_special_form_conflict(Meta, File, Key, T) end; ensure_no_special_form_conflict(_Meta, _File, _Key, []) -> ok. +ensure_keyword_list(_Meta, _File, [], _Kind) -> ok; + +ensure_keyword_list(Meta, File, [{Key, Value} | Rest], Kind) when is_atom(Key), is_integer(Value) -> + ensure_keyword_list(Meta, File, Rest, Kind); + +ensure_keyword_list(Meta, File, _Other, Kind) -> + elixir_errors:form_error(Meta, File, ?MODULE, {invalid_option, Kind}). + +ensure_no_duplicates(Meta, File, Option, Kind) -> + lists:foldl(fun({Name, Arity}, Acc) -> + case lists:member({Name, Arity}, Acc) of + true -> + elixir_errors:form_error(Meta, File, ?MODULE, {duplicated_import, {Kind, Name, Arity}}); + false -> + [{Name, Arity} | Acc] + end + end, [], Option). + %% ERROR HANDLING -format_error({invalid_import,{Receiver, Name, Arity}}) -> - io_lib:format("cannot import ~ts.~ts/~B because it doesn't exist", +format_error(only_and_except_given) -> + ":only and :except can only be given together to import " + "when :only is :functions, :macros, or :sigils"; + +format_error({duplicated_import, {Option, Name, Arity}}) -> + io_lib:format("invalid :~s option for import, ~ts/~B is duplicated", [Option, Name, Arity]); + +format_error({invalid_import, {Receiver, Name, Arity}}) -> + io_lib:format("cannot import ~ts.~ts/~B because it is undefined or private", [elixir_aliases:inspect(Receiver), Name, Arity]); -format_error({special_form_conflict,{Receiver, Name, Arity}}) -> +format_error({invalid_option, Option}) -> + Message = "invalid :~s option for import, expected a keyword list with integer values", + io_lib:format(Message, [Option]); + +format_error({invalid_option, only, Value}) -> + Message = "invalid :only option for import, expected value to be an atom :functions, :macros" + ", or a list literal, got: ~s", + io_lib:format(Message, ['Elixir.Macro':to_string(Value)]); + +format_error({invalid_option, except, Value}) -> + Message = "invalid :except option for import, expected value to be a list literal, got: ~s", + io_lib:format(Message, ['Elixir.Macro':to_string(Value)]); + +format_error({special_form_conflict, {Receiver, Name, Arity}}) -> io_lib:format("cannot import ~ts.~ts/~B because it conflicts with Elixir special forms", [elixir_aliases:inspect(Receiver), Name, Arity]); @@ -152,15 +222,15 @@ keyfind(Key, List) -> keydelete(Key, List) -> lists:keydelete(Key, 1, List). -intersection([H|T], All) -> +intersection([H | T], All) -> case lists:member(H, All) of - true -> [H|intersection(T, All)]; + true -> [H | intersection(T, All)]; false -> intersection(T, All) end; intersection([], _All) -> []. -%% Internal funs that are never imported etc. +%% Internal funs that are never imported, and the like remove_underscored(List) -> lists:filter(fun({Name, _}) -> @@ -180,8 +250,9 @@ special_form('&', 1) -> true; special_form('^', 1) -> true; special_form('=', 2) -> true; special_form('%', 2) -> true; -special_form('__op__', 2) -> true; -special_form('__op__', 3) -> true; +special_form('|', 2) -> true; +special_form('.', 2) -> true; +special_form('::', 2) -> true; special_form('__block__', _) -> true; special_form('->', _) -> true; special_form('<<>>', _) -> true; @@ -195,6 +266,7 @@ special_form('import', 1) -> true; special_form('import', 2) -> true; special_form('__ENV__', 0) -> true; special_form('__CALLER__', 0) -> true; +special_form('__STACKTRACE__', 0) -> true; special_form('__MODULE__', 0) -> true; special_form('__DIR__', 0) -> true; special_form('__aliases__', _) -> true; @@ -205,8 +277,9 @@ special_form('unquote_splicing', 1) -> true; special_form('fn', _) -> true; special_form('super', _) -> true; special_form('for', _) -> true; +special_form('with', _) -> true; special_form('cond', 1) -> true; special_form('case', 2) -> true; -special_form('try', 2) -> true; +special_form('try', 1) -> true; special_form('receive', 1) -> true; special_form(_, _) -> false. diff --git a/lib/elixir/src/elixir_interpolation.erl b/lib/elixir/src/elixir_interpolation.erl index 1221811b7a0..2fea35a62b7 100644 --- a/lib/elixir/src/elixir_interpolation.erl +++ b/lib/elixir/src/elixir_interpolation.erl @@ -1,139 +1,255 @@ % Handle string and string-like interpolations. -module(elixir_interpolation). --export([extract/5, unescape_chars/1, unescape_chars/2, -unescape_tokens/1, unescape_tokens/2, unescape_map/1]). +-export([extract/6, unescape_string/1, unescape_string/2, +unescape_tokens/1, unescape_map/1]). -include("elixir.hrl"). +-include("elixir_tokenizer.hrl"). %% Extract string interpolations -extract(Line, Raw, Interpol, String, Last) -> - %% Ignore whatever is in the scope and enable terminator checking. - Scope = Raw#elixir_tokenizer{terminators=[], check_terminators=true}, - extract(Line, Scope, Interpol, String, [], [], Last). +extract(Line, Column, Scope, Interpol, String, Last) -> + extract(String, [], [], Line, Column, Scope, Interpol, Last). %% Terminators -extract(Line, _Scope, _Interpol, [], Buffer, Output, []) -> - finish_extraction(Line, Buffer, Output, []); +extract([], _Buffer, _Output, Line, Column, #elixir_tokenizer{cursor_completion=false}, _Interpol, Last) -> + {error, {string, Line, Column, io_lib:format("missing terminator: ~ts", [[Last]]), []}}; -extract(Line, _Scope, _Interpol, [], _Buffer, _Output, Last) -> - {error, {string, Line, io_lib:format("missing terminator: ~ts", [[Last]]), []}}; +extract([], Buffer, Output, Line, Column, Scope, _Interpol, _Last) -> + finish_extraction([], Buffer, Output, Line, Column, Scope); -extract(Line, _Scope, _Interpol, [Last|Remaining], Buffer, Output, Last) -> - finish_extraction(Line, Buffer, Output, Remaining); +extract([Last | Rest], Buffer, Output, Line, Column, Scope, _Interpol, Last) -> + finish_extraction(Rest, Buffer, Output, Line, Column + 1, Scope); %% Going through the string -extract(Line, Scope, Interpol, [$\\, $\n|Rest], Buffer, Output, Last) -> - extract(Line+1, Scope, Interpol, Rest, Buffer, Output, Last); +extract([$\\, $\r, $\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol, Last) -> + extract_nl(Rest, [$\n, $\r, $\\ | Buffer], Output, Line, Scope, Interpol, Last); -extract(Line, Scope, Interpol, [$\\, $\r, $\n|Rest], Buffer, Output, Last) -> - extract(Line+1, Scope, Interpol, Rest, Buffer, Output, Last); +extract([$\\, $\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol, Last) -> + extract_nl(Rest, [$\n, $\\ | Buffer], Output, Line, Scope, Interpol, Last); -extract(Line, Scope, Interpol, [$\n|Rest], Buffer, Output, Last) -> - extract(Line+1, Scope, Interpol, Rest, [$\n|Buffer], Output, Last); +extract([$\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol, Last) -> + extract_nl(Rest, [$\n | Buffer], Output, Line, Scope, Interpol, Last); -extract(Line, Scope, Interpol, [$\\, $#, ${|Rest], Buffer, Output, Last) -> - extract(Line, Scope, Interpol, Rest, [${,$#|Buffer], Output, Last); +extract([$\\, Last | Rest], Buffer, Output, Line, Column, Scope, Interpol, Last) -> + extract(Rest, [Last | Buffer], Output, Line, Column+2, Scope, Interpol, Last); -extract(Line, Scope, Interpol, [$\\,Char|Rest], Buffer, Output, Last) -> - extract(Line, Scope, Interpol, Rest, [Char,$\\|Buffer], Output, Last); +extract([$\\, Last, Last, Last | Rest], Buffer, Output, Line, Column, Scope, Interpol, [Last, Last, Last] = All) -> + extract(Rest, [Last, Last, Last | Buffer], Output, Line, Column+4, Scope, Interpol, All); -extract(Line, Scope, true, [$#, ${|Rest], Buffer, Output, Last) -> - Output1 = build_string(Line, Buffer, Output), +extract([$\\, $#, ${ | Rest], Buffer, Output, Line, Column, Scope, true, Last) -> + extract(Rest, [${, $#, $\\ | Buffer], Output, Line, Column+1, Scope, true, Last); - case elixir_tokenizer:tokenize(Rest, Line, Scope) of - {error, {EndLine, _, "}"}, [$}|NewRest], Tokens} -> - Output2 = build_interpol(Line, Tokens, Output1), - extract(EndLine, Scope, true, NewRest, [], Output2, Last); - {error, Reason, _, _} -> +extract([$#, ${ | Rest], Buffer, Output, Line, Column, Scope, true, Last) -> + Output1 = build_string(Buffer, Output), + case elixir_tokenizer:tokenize(Rest, Line, Column + 2, Scope#elixir_tokenizer{terminators=[]}) of + {error, {EndLine, EndColumn, _, "}"}, [$} | NewRest], Warnings, Tokens} -> + NewScope = Scope#elixir_tokenizer{warnings=Warnings}, + Output2 = build_interpol(Line, Column, EndLine, EndColumn, lists:reverse(Tokens), Output1), + extract(NewRest, [], Output2, EndLine, EndColumn + 1, NewScope, true, Last); + {error, Reason, _, _, _} -> {error, Reason}; - {ok, _EndLine, _} -> - {error, {string, Line, "missing interpolation terminator:}", []}} + {ok, EndLine, EndColumn, Warnings, Tokens} when Scope#elixir_tokenizer.cursor_completion /= false -> + NewScope = Scope#elixir_tokenizer{warnings=Warnings, cursor_completion=terminators}, + Output2 = build_interpol(Line, Column, EndLine, EndColumn, Tokens, Output1), + extract([], [], Output2, EndLine, EndColumn, NewScope, true, Last); + {ok, _, _, _, _} -> + {error, {string, Line, Column, "missing interpolation terminator: \"}\"", []}} end; +extract([$\\ | Rest], Buffer, Output, Line, Column, Scope, Interpol, Last) -> + extract_char(Rest, [$\\ | Buffer], Output, Line, Column + 1, Scope, Interpol, Last); + %% Catch all clause -extract(Line, Scope, Interpol, [Char|Rest], Buffer, Output, Last) -> - extract(Line, Scope, Interpol, Rest, [Char|Buffer], Output, Last). +extract([Char1, Char2 | Rest], Buffer, Output, Line, Column, Scope, Interpol, Last) + when Char1 =< 255, Char2 =< 255 -> + extract([Char2 | Rest], [Char1 | Buffer], Output, Line, Column + 1, Scope, Interpol, Last); + +extract(Rest, Buffer, Output, Line, Column, Scope, Interpol, Last) -> + extract_char(Rest, Buffer, Output, Line, Column, Scope, Interpol, Last). + +extract_char(Rest, Buffer, Output, Line, Column, Scope, Interpol, Last) -> + case unicode_util:gc(Rest) of + [Char | _] when ?bidi(Char) -> + Token = io_lib:format("\\u~4.16.0B", [Char]), + Pre = "invalid bidirectional formatting character in string: ", + Pos = io_lib:format(". If you want to use such character, use it in its escaped ~ts form instead", [Token]), + {error, {Line, Column, {Pre, Pos}, Token}}; + + [Char | NewRest] -> + extract(NewRest, [Char | Buffer], Output, Line, Column + 1, Scope, Interpol, Last); + + [] -> + extract([], Buffer, Output, Line, Column, Scope, Interpol, Last) + end. + +%% Handle newlines. Heredocs require special attention + +extract_nl(Rest, Buffer, Output, Line, Scope, Interpol, [H,H,H] = Last) -> + case strip_horizontal_space(Rest, Buffer, 1) of + {[H,H,H|NewRest], _NewBuffer, Column} -> + finish_extraction(NewRest, Buffer, Output, Line + 1, Column + 3, Scope); + {NewRest, NewBuffer, Column} -> + extract(NewRest, NewBuffer, Output, Line + 1, Column, Scope, Interpol, Last) + end; +extract_nl(Rest, Buffer, Output, Line, Scope, Interpol, Last) -> + extract(Rest, Buffer, Output, Line + 1, 1, Scope, Interpol, Last). + +strip_horizontal_space([H | T], Buffer, Counter) when H =:= $\s; H =:= $\t -> + strip_horizontal_space(T, [H | Buffer], Counter + 1); +strip_horizontal_space(T, Buffer, Counter) -> + {T, Buffer, Counter}. %% Unescape a series of tokens as returned by extract. unescape_tokens(Tokens) -> - unescape_tokens(Tokens, fun unescape_map/1). + try [unescape_token(Token, fun unescape_map/1) || Token <- Tokens] of + Unescaped -> {ok, Unescaped} + catch + {error, _Reason, _Token} = Error -> Error + end. -unescape_tokens(Tokens, Map) -> - [unescape_token(Token, Map) || Token <- Tokens]. +unescape_token(Token, Map) when is_list(Token) -> + unescape_chars(elixir_utils:characters_to_binary(Token), Map); +unescape_token(Token, Map) when is_binary(Token) -> + unescape_chars(Token, Map); +unescape_token(Other, _Map) -> + Other. -unescape_token(Token, Map) when is_binary(Token) -> unescape_chars(Token, Map); -unescape_token(Other, _Map) -> Other. +% Unescape string. This is called by Elixir. Wrapped by convenience. -% Unescape chars. For instance, "\" "n" (two chars) needs to be converted to "\n" (one char). +unescape_string(String) -> + unescape_string(String, fun unescape_map/1). + +unescape_string(String, Map) -> + try + unescape_chars(String, Map) + catch + {error, Reason, _} -> + Message = elixir_utils:characters_to_binary(Reason), + error('Elixir.ArgumentError':exception([{message, Message}])) + end. -unescape_chars(String) -> - unescape_chars(String, fun unescape_map/1). +% Unescape chars. For instance, "\" "n" (two chars) needs to be converted to "\n" (one char). unescape_chars(String, Map) -> - Octals = Map($0) /= false, - Hex = Map($x) /= false, - unescape_chars(String, Map, Octals, Hex, <<>>). + unescape_chars(String, Map, <<>>). + +unescape_chars(<<$\\, $x, Rest/binary>>, Map, Acc) -> + case Map(hex) of + true -> unescape_hex(Rest, Map, Acc); + false -> unescape_chars(Rest, Map, <>) + end; + +unescape_chars(<<$\\, $u, Rest/binary>>, Map, Acc) -> + case Map(unicode) of + true -> unescape_unicode(Rest, Map, Acc); + false -> unescape_chars(Rest, Map, <>) + end; -unescape_chars(<<$\\,A,B,C,Rest/binary>>, Map, true, Hex, Acc) when ?is_octal(A), A =< $3, ?is_octal(B), ?is_octal(C) -> - append_escaped(Rest, Map, [A,B,C], true, Hex, Acc, 8); +unescape_chars(<<$\\, $\n, Rest/binary>>, Map, Acc) -> + case Map(newline) of + true -> unescape_chars(Rest, Map, Acc); + false -> unescape_chars(Rest, Map, <>) + end; -unescape_chars(<<$\\,A,B,Rest/binary>>, Map, true, Hex, Acc) when ?is_octal(A), ?is_octal(B) -> - append_escaped(Rest, Map, [A,B], true, Hex, Acc, 8); +unescape_chars(<<$\\, $\r, $\n, Rest/binary>>, Map, Acc) -> + case Map(newline) of + true -> unescape_chars(Rest, Map, Acc); + false -> unescape_chars(Rest, Map, <>) + end; -unescape_chars(<<$\\,A,Rest/binary>>, Map, true, Hex, Acc) when ?is_octal(A) -> - append_escaped(Rest, Map, [A], true, Hex, Acc, 8); +unescape_chars(<<$\\, Escaped, Rest/binary>>, Map, Acc) -> + case Map(Escaped) of + false -> unescape_chars(Rest, Map, <>); + Other -> unescape_chars(Rest, Map, <>) + end; -unescape_chars(<<$\\,P,A,B,Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B) -> - append_escaped(Rest, Map, [A,B], Octal, true, Acc, 16); +unescape_chars(<>, Map, Acc) -> + unescape_chars(Rest, Map, <>); -unescape_chars(<<$\\,P,A,Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A) -> - append_escaped(Rest, Map, [A], Octal, true, Acc, 16); +unescape_chars(<<>>, _Map, Acc) -> Acc. -unescape_chars(<<$\\,P,${,A,$},Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A) -> - append_escaped(Rest, Map, [A], Octal, true, Acc, 16); +% Unescape Helpers -unescape_chars(<<$\\,P,${,A,B,$},Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B) -> - append_escaped(Rest, Map, [A,B], Octal, true, Acc, 16); +unescape_hex(<>, Map, Acc) when ?is_hex(A), ?is_hex(B) -> + Bytes = list_to_integer([A, B], 16), + unescape_chars(Rest, Map, <>); -unescape_chars(<<$\\,P,${,A,B,C,$},Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C) -> - append_escaped(Rest, Map, [A,B,C], Octal, true, Acc, 16); +%% TODO: Remove deprecated sequences on v2.0 -unescape_chars(<<$\\,P,${,A,B,C,D,$},Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> - append_escaped(Rest, Map, [A,B,C,D], Octal, true, Acc, 16); +unescape_hex(<>, Map, Acc) when ?is_hex(A) -> + io:format(standard_error, "warning: \\xH inside strings/sigils/chars is deprecated, please use \\xHH (byte) or \\uHHHH (code point) instead~n", []), + append_codepoint(Rest, Map, [A], Acc, 16); -unescape_chars(<<$\\,P,${,A,B,C,D,E,$},Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E) -> - append_escaped(Rest, Map, [A,B,C,D,E], Octal, true, Acc, 16); +unescape_hex(<<${, A, $}, Rest/binary>>, Map, Acc) when ?is_hex(A) -> + io:format(standard_error, "warning: \\x{H*} inside strings/sigils/chars is deprecated, please use \\xHH (byte) or \\uHHHH (code point) instead~n", []), + append_codepoint(Rest, Map, [A], Acc, 16); -unescape_chars(<<$\\,P,${,A,B,C,D,E,F,$},Rest/binary>>, Map, Octal, true, Acc) when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E), ?is_hex(F) -> - append_escaped(Rest, Map, [A,B,C,D,E,F], Octal, true, Acc, 16); +unescape_hex(<<${, A, B, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B) -> + io:format(standard_error, "warning: \\x{H*} inside strings/sigils/chars is deprecated, please use \\xHH (byte) or \\uHHHH (code point) instead~n", []), + append_codepoint(Rest, Map, [A, B], Acc, 16); -unescape_chars(<<$\\,Escaped,Rest/binary>>, Map, Octals, Hex, Acc) -> - case Map(Escaped) of - false -> unescape_chars(Rest, Map, Octals, Hex, <>); - Other -> unescape_chars(Rest, Map, Octals, Hex, <>) - end; +unescape_hex(<<${, A, B, C, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C) -> + io:format(standard_error, "warning: \\x{H*} inside strings/sigils/chars is deprecated, please use \\xHH (byte) or \\uHHHH (code point) instead~n", []), + append_codepoint(Rest, Map, [A, B, C], Acc, 16); + +unescape_hex(<<${, A, B, C, D, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> + io:format(standard_error, "warning: \\x{H*} inside strings/sigils/chars is deprecated, please use \\xHH (byte) or \\uHHHH (code point) instead~n", []), + append_codepoint(Rest, Map, [A, B, C, D], Acc, 16); + +unescape_hex(<<${, A, B, C, D, E, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E) -> + io:format(standard_error, "warning: \\x{H*} inside strings/sigils/chars is deprecated, please use \\xHH (byte) or \\uHHHH (code point) instead~n", []), + append_codepoint(Rest, Map, [A, B, C, D, E], Acc, 16); + +unescape_hex(<<${, A, B, C, D, E, F, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E), ?is_hex(F) -> + io:format(standard_error, "warning: \\x{H*} inside strings/sigils/chars is deprecated, please use \\xHH (byte) or \\uHHHH (code point) instead~n", []), + append_codepoint(Rest, Map, [A, B, C, D, E, F], Acc, 16); + +unescape_hex(<<_/binary>>, _Map, _Acc) -> + throw({error, "invalid hex escape character, expected \\xHH where H is a hexadecimal digit", "\\x"}). -unescape_chars(<>, Map, Octals, Hex, Acc) -> - unescape_chars(Rest, Map, Octals, Hex, <>); +%% Finish deprecated sequences -unescape_chars(<<>>, _Map, _Octals, _Hex, Acc) -> Acc. +unescape_unicode(<>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> + append_codepoint(Rest, Map, [A, B, C, D], Acc, 16); -append_escaped(Rest, Map, List, Octal, Hex, Acc, Base) -> +unescape_unicode(<<${, A, $}, Rest/binary>>, Map, Acc) when ?is_hex(A) -> + append_codepoint(Rest, Map, [A], Acc, 16); + +unescape_unicode(<<${, A, B, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B) -> + append_codepoint(Rest, Map, [A, B], Acc, 16); + +unescape_unicode(<<${, A, B, C, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C) -> + append_codepoint(Rest, Map, [A, B, C], Acc, 16); + +unescape_unicode(<<${, A, B, C, D, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> + append_codepoint(Rest, Map, [A, B, C, D], Acc, 16); + +unescape_unicode(<<${, A, B, C, D, E, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E) -> + append_codepoint(Rest, Map, [A, B, C, D, E], Acc, 16); + +unescape_unicode(<<${, A, B, C, D, E, F, $}, Rest/binary>>, Map, Acc) when ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E), ?is_hex(F) -> + append_codepoint(Rest, Map, [A, B, C, D, E, F], Acc, 16); + +unescape_unicode(<<_/binary>>, _Map, _Acc) -> + throw({error, "invalid Unicode escape character, expected \\uHHHH or \\u{H*} where H is a hexadecimal digit", "\\u"}). + +append_codepoint(Rest, Map, List, Acc, Base) -> Codepoint = list_to_integer(List, Base), try <> of - Binary -> unescape_chars(Rest, Map, Octal, Hex, Binary) + Binary -> unescape_chars(Rest, Map, Binary) catch error:badarg -> - Msg = <<"invalid or reserved unicode codepoint ", (integer_to_binary(Codepoint))/binary>>, - error('Elixir.ArgumentError':exception([{message,Msg}])) + throw({error, "invalid or reserved Unicode code point \\u{" ++ List ++ "}", "\\u"}) end. -% Unescape Helpers - +unescape_map(newline) -> true; +unescape_map(unicode) -> true; +unescape_map(hex) -> true; +unescape_map($0) -> 0; unescape_map($a) -> 7; unescape_map($b) -> $\b; unescape_map($d) -> $\d; @@ -148,16 +264,16 @@ unescape_map(E) -> E. % Extract Helpers -finish_extraction(Line, Buffer, Output, Remaining) -> - case build_string(Line, Buffer, Output) of - [] -> Final = [<<>>]; - Final -> [] +finish_extraction(Remaining, Buffer, Output, Line, Column, Scope) -> + Final = case build_string(Buffer, Output) of + [] -> [[]]; + F -> F end, - {Line, lists:reverse(Final), Remaining}. -build_string(_Line, [], Output) -> Output; -build_string(_Line, Buffer, Output) -> - [elixir_utils:characters_to_binary(lists:reverse(Buffer))|Output]. + {Line, Column, lists:reverse(Final), Remaining, Scope}. + +build_string([], Output) -> Output; +build_string(Buffer, Output) -> [lists:reverse(Buffer) | Output]. -build_interpol(Line, Buffer, Output) -> - [{Line, lists:reverse(Buffer)}|Output]. +build_interpol(Line, Column, EndLine, EndColumn, Buffer, Output) -> + [{{Line, Column, nil}, {EndLine, EndColumn, nil}, Buffer} | Output]. diff --git a/lib/elixir/src/elixir_lexical.erl b/lib/elixir/src/elixir_lexical.erl index 7f299442deb..c6553337280 100644 --- a/lib/elixir/src/elixir_lexical.erl +++ b/lib/elixir/src/elixir_lexical.erl @@ -1,79 +1,123 @@ %% Module responsible for tracking lexical information. -module(elixir_lexical). --export([run/2, - record_alias/4, record_alias/2, - record_import/4, record_import/2, - record_remote/2, format_error/1 -]). +-export([run/3, with_file/3, trace/2, format_error/1]). -include("elixir.hrl"). -define(tracker, 'Elixir.Kernel.LexicalTracker'). -run(File, Callback) -> - case code:is_loaded(?tracker) of - {file, _} -> - Pid = ?tracker:start_link(), - try - Callback(Pid) +run(#{tracers := Tracers} = E, ExecutionCallback, AfterExecutionCallback) -> + case elixir_config:is_bootstrap() of + false -> + {ok, Pid} = ?tracker:start_link(), + LexEnv = E#{lexical_tracker := Pid, tracers := [?MODULE | Tracers]}, + elixir_env:trace(start, LexEnv), + + try ExecutionCallback(LexEnv) of + Res -> + warn_unused_aliases(Pid, LexEnv), + warn_unused_imports(Pid, LexEnv), + AfterExecutionCallback(LexEnv), + Res after - warn_unused_aliases(File, Pid), - warn_unused_imports(File, Pid), - unlink(Pid), ?tracker:stop(Pid) + elixir_env:trace(stop, LexEnv), + unlink(Pid), + ?tracker:stop(Pid) end; - false -> - Callback(nil) - end. -%% RECORD + true -> + ExecutionCallback(E), + AfterExecutionCallback(E) + end. -record_alias(Module, Line, Warn, Ref) -> - if_tracker(Ref, fun(Pid) -> - ?tracker:add_alias(Pid, Module, Line, Warn), - true - end). +trace({import, Meta, Module, Opts}, #{lexical_tracker := Pid}) -> + {imported, Imported} = lists:keyfind(imported, 1, Meta), -record_import(Module, Line, Warn, Ref) -> - if_tracker(Ref, fun(Pid) -> - ?tracker:add_import(Pid, Module, Line, Warn), - true - end). + Only = + case lists:keyfind(only, 1, Opts) of + {only, List} when is_list(List) -> List; + _ -> [] + end, -record_alias(Module, Ref) -> - if_tracker(Ref, fun(Pid) -> - ?tracker:alias_dispatch(Pid, Module), - true - end). + ?tracker:add_import(Pid, Module, Only, ?line(Meta), Imported and should_warn(Meta, Opts)), + ok; +trace({alias, Meta, _Old, New, Opts}, #{lexical_tracker := Pid}) -> + ?tracker:add_alias(Pid, New, ?line(Meta), should_warn(Meta, Opts)), + ok; +trace({alias_expansion, _Meta, Lookup, _Result}, #{lexical_tracker := Pid}) -> + ?tracker:alias_dispatch(Pid, Lookup), + ok; +trace({require, _Meta, Module, _Opts}, #{lexical_tracker := Pid}) -> + ?tracker:add_require(Pid, Module), + ok; +trace({struct_expansion, _Meta, Module, _Keys}, #{lexical_tracker := Pid}) -> + ?tracker:add_require(Pid, Module), + ok; +trace({alias_reference, _Meta, Module}, #{lexical_tracker := Pid} = E) -> + ?tracker:remote_dispatch(Pid, Module, mode(E)), + ok; +trace({remote_function, _Meta, Module, _Function, _Arity}, #{lexical_tracker := Pid} = E) -> + ?tracker:remote_dispatch(Pid, Module, mode(E)), + ok; +trace({remote_macro, _Meta, Module, _Function, _Arity}, #{lexical_tracker := Pid}) -> + ?tracker:remote_dispatch(Pid, Module, compile), + ok; +trace({imported_function, _Meta, Module, Function, Arity}, #{lexical_tracker := Pid} = E) -> + ?tracker:import_dispatch(Pid, Module, {Function, Arity}, mode(E)), + ok; +trace({imported_macro, _Meta, Module, Function, Arity}, #{lexical_tracker := Pid}) -> + ?tracker:import_dispatch(Pid, Module, {Function, Arity}, compile), + ok; +trace({compile_env, App, Path, Return}, #{lexical_tracker := Pid}) -> + ?tracker:add_compile_env(Pid, App, Path, Return), + ok; +trace(_, _) -> + ok. -record_import(Module, Ref) -> - if_tracker(Ref, fun(Pid) -> - ?tracker:import_dispatch(Pid, Module), - true - end). +mode(#{function := nil}) -> compile; +mode(#{}) -> runtime. -record_remote(Module, Ref) -> - if_tracker(Ref, fun(Pid) -> - ?tracker:remote_dispatch(Pid, Module), - true - end). +should_warn(Meta, Opts) -> + case lists:keyfind(warn, 1, Opts) of + {warn, false} -> false; + {warn, true} -> true; + false -> not lists:keymember(context, 1, Meta) + end. -%% HELPERS +%% EXTERNAL SOURCES -if_tracker(nil, _Callback) -> false; -if_tracker(Pid, Callback) when is_pid(Pid) -> Callback(Pid). +with_file(File, #{lexical_tracker := nil} = E, Callback) -> + Callback(E#{file := File}); +with_file(File, #{lexical_tracker := Pid} = E, Callback) -> + try + ?tracker:set_file(Pid, File), + Callback(E#{file := File}) + after + ?tracker:reset_file(Pid) + end. %% ERROR HANDLING -warn_unused_imports(File, Pid) -> - [ begin - elixir_errors:handle_file_warning(File, {L, ?MODULE, {unused_import, M}}) - end || {M, L} <- ?tracker:collect_unused_imports(Pid)]. +warn_unused_imports(Pid, E) -> + {ModuleImports, MFAImports} = + lists:partition(fun({M, _}) -> is_atom(M) end, ?tracker:collect_unused_imports(Pid)), + + Modules = [M || {M, _L} <- ModuleImports], + MFAImportsFiltered = [T || {{M, _, _}, _} = T <- MFAImports, not lists:member(M, Modules)], + + [begin + elixir_errors:form_warn([{line, L}], ?key(E, file), ?MODULE, {unused_import, M}) + end || {M, L} <- ModuleImports ++ MFAImportsFiltered], + ok. -warn_unused_aliases(File, Pid) -> - [ begin - elixir_errors:handle_file_warning(File, {L, ?MODULE, {unused_alias, M}}) - end || {M, L} <- ?tracker:collect_unused_aliases(Pid)]. +warn_unused_aliases(Pid, E) -> + [begin + elixir_errors:form_warn([{line, L}], ?key(E, file), ?MODULE, {unused_alias, M}) + end || {M, L} <- ?tracker:collect_unused_aliases(Pid)], + ok. format_error({unused_alias, Module}) -> io_lib:format("unused alias ~ts", [elixir_aliases:inspect(Module)]); +format_error({unused_import, {Module, Function, Arity}}) -> + io_lib:format("unused import ~ts.~ts/~w", [elixir_aliases:inspect(Module), Function, Arity]); format_error({unused_import, Module}) -> io_lib:format("unused import ~ts", [elixir_aliases:inspect(Module)]). diff --git a/lib/elixir/src/elixir_locals.erl b/lib/elixir/src/elixir_locals.erl index 92991d772a0..43e6310a483 100644 --- a/lib/elixir/src/elixir_locals.erl +++ b/lib/elixir/src/elixir_locals.erl @@ -1,181 +1,152 @@ %% Module responsible for tracking invocations of module calls. -module(elixir_locals). -export([ - setup/1, cleanup/1, cache_env/1, get_cached_env/1, - record_local/2, record_local/3, record_import/4, - record_definition/3, record_defaults/4, - ensure_no_function_conflict/4, warn_unused_local/3, format_error/1 + setup/1, stop/1, cache_env/1, get_cached_env/1, + record_local/5, record_import/4, record_defaults/5, + yank/2, reattach/6, ensure_no_import_conflict/3, + warn_unused_local/4, ensure_no_undefined_local/3, + format_error/1 ]). --export([macro_for/3, local_for/3, local_for/4]). -include("elixir.hrl"). --define(attr, '__locals_tracker'). +-define(cache, {elixir, cache_env}). +-define(locals, {elixir, locals}). -define(tracker, 'Elixir.Module.LocalsTracker'). -macro_for(Module, Name, Arity) -> - Tuple = {Name, Arity}, - try elixir_def:lookup_definition(Module, Tuple) of - {{Tuple, Kind, Line, _, _, _, _}, [_|_] = Clauses} - when Kind == defmacro; Kind == defmacrop -> - fun() -> get_function(Line, Module, Clauses) end; - _ -> - false - catch - error:badarg -> false - end. - -local_for(Module, Name, Arity) -> - local_for(Module, Name, Arity, nil). -local_for(Module, Name, Arity, Given) -> - Tuple = {Name, Arity}, - case elixir_def:lookup_definition(Module, Tuple) of - {{Tuple, Kind, Line, _, _, _, _}, [_|_] = Clauses} - when Given == nil; Kind == Given -> - get_function(Line, Module, Clauses); - _ -> - [_|T] = erlang:get_stacktrace(), - erlang:raise(error, undef, [{Module,Name,Arity,[]}|T]) - end. +setup({DataSet, _DataBag}) -> + ets:insert(DataSet, {?cache, 0}), -get_function(Line, Module, Clauses) -> - RewrittenClauses = [rewrite_clause(Clause, Module) || Clause <- Clauses], - Fun = {'fun', Line, {clauses, RewrittenClauses}}, - {value, Result, _Binding} = erl_eval:exprs([Fun], []), - Result. - -rewrite_clause({call, Line, {atom, Line, RawName}, Args}, Module) -> - Remote = {remote, Line, - {atom, Line, ?MODULE}, - {atom, Line, local_for} - }, - - %% If we have a macro, its arity in the table is - %% actually one less than in the function call - {Name, Arity} = case atom_to_list(RawName) of - "MACRO-" ++ Rest -> {list_to_atom(Rest), length(Args) - 1}; - _ -> {RawName, length(Args)} + case elixir_config:is_bootstrap() of + false -> ets:insert(DataSet, {?locals, true}); + true -> ok end, - FunCall = {call, Line, Remote, [ - {atom, Line, Module}, {atom, Line, Name}, {integer, Line, Arity} - ]}, - {call, Line, FunCall, Args}; - -rewrite_clause(Tuple, Module) when is_tuple(Tuple) -> - list_to_tuple(rewrite_clause(tuple_to_list(Tuple), Module)); - -rewrite_clause(List, Module) when is_list(List) -> - [rewrite_clause(Item, Module) || Item <- List]; - -rewrite_clause(Else, _) -> Else. + ok. -%% TRACKING +stop({DataSet, _DataBag}) -> + ets:delete(DataSet, ?locals). -setup(Module) -> - case code:is_loaded(?tracker) of - {file, _} -> ets:insert(Module, {?attr, ?tracker:start_link()}); - false -> ok - end. +yank(Tuple, Module) -> + if_tracker(Module, fun(Tracker) -> ?tracker:yank(Tracker, Tuple) end). -cleanup(Module) -> - if_tracker(Module, fun(Pid) -> unlink(Pid), ?tracker:stop(Pid) end). +reattach(Tuple, Kind, Module, Function, Neighbours, Meta) -> + if_tracker(Module, fun(Tracker) -> ?tracker:reattach(Tracker, Tuple, Kind, Function, Neighbours, Meta) end). -record_local(Tuple, Module) when is_atom(Module) -> - if_tracker(Module, fun(Pid) -> - ?tracker:add_local(Pid, Tuple), - true - end). -record_local(Tuple, _Module, Function) - when Function == nil; Function == Tuple -> false; -record_local(Tuple, Module, Function) -> - if_tracker(Module, fun(Pid) -> - ?tracker:add_local(Pid, Function, Tuple), - true - end). +record_local(_Tuple, _Module, nil, _Meta, _IsMacroDispatch) -> + ok; +record_local(Tuple, Module, Function, Meta, IsMacroDispatch) -> + if_tracker(Module, fun(Tracker) -> ?tracker:add_local(Tracker, Function, Tuple, Meta, IsMacroDispatch), ok end). -record_import(_Tuple, Receiver, Module, _Function) - when Module == nil; Module == Receiver -> false; +record_import(_Tuple, Receiver, Module, Function) + when Function == nil; Module == Receiver -> false; record_import(Tuple, Receiver, Module, Function) -> - if_tracker(Module, fun(Pid) -> - ?tracker:add_import(Pid, Function, Receiver, Tuple), - true - end). + if_tracker(Module, fun(Tracker) -> ?tracker:add_import(Tracker, Function, Receiver, Tuple), ok end). -record_definition(Tuple, Kind, Module) -> - if_tracker(Module, fun(Pid) -> - ?tracker:add_definition(Pid, Kind, Tuple), - true - end). - -record_defaults(_Tuple, _Kind, _Module, 0) -> - true; -record_defaults(Tuple, Kind, Module, Defaults) -> - if_tracker(Module, fun(Pid) -> - ?tracker:add_defaults(Pid, Kind, Tuple, Defaults), - true - end). +record_defaults(_Tuple, _Kind, _Module, 0, _Meta) -> + ok; +record_defaults(Tuple, Kind, Module, Defaults, Meta) -> + if_tracker(Module, fun(Tracker) -> ?tracker:add_defaults(Tracker, Kind, Tuple, Defaults, Meta), ok end). if_tracker(Module, Callback) -> - try ets:lookup_element(Module, ?attr, 2) of - Pid -> Callback(Pid) + if_tracker(Module, ok, Callback). + +if_tracker(Module, Default, Callback) -> + try + {DataSet, _} = Tables = elixir_module:data_tables(Module), + {ets:member(DataSet, ?locals), Tables} + of + {true, Tracker} -> Callback(Tracker); + {false, _} -> Default catch - error:badarg -> false + error:badarg -> Default end. %% CACHING -cache_env(#{module := Module} = RE) -> - E = RE#{line := nil,vars := []}, - try ets:lookup_element(Module, ?attr, 2) of - Pid -> - {Pid, ?tracker:cache_env(Pid, E)} - catch - error:badarg -> - {Escaped, _} = elixir_quote:escape(E, false), - Escaped - end. - -get_cached_env({Pid,Ref}) -> ?tracker:get_cached_env(Pid, Ref); -get_cached_env(Env) -> Env. +cache_env(#{line := Line, module := Module} = E) -> + {Set, _} = elixir_module:data_tables(Module), + Cache = elixir_env:reset_vars(E#{line := nil}), + PrevKey = ets:lookup_element(Set, ?cache, 2), + + Pos = + case ets:lookup(Set, {cache_env, PrevKey}) of + [{_, Cache}] -> + PrevKey; + _ -> + NewKey = PrevKey + 1, + ets:insert(Set, [{{cache_env, NewKey}, Cache}, {?cache, NewKey}]), + NewKey + end, + + {Module, {Line, Pos}}. + +get_cached_env({Module, {Line, Pos}}) -> + {Set, _} = elixir_module:data_tables(Module), + (ets:lookup_element(Set, {cache_env, Pos}, 2))#{line := Line}; +get_cached_env(Env) -> + Env. %% ERROR HANDLING -ensure_no_function_conflict(Meta, File, Module, AllDefined) -> - if_tracker(Module, fun(Pid) -> - [ begin - elixir_errors:form_error(Meta, File, ?MODULE, {function_conflict, Error}) - end || Error <- ?tracker:collect_imports_conflicts(Pid, AllDefined) ] - end), - ok. - -warn_unused_local(File, Module, Private) -> - if_tracker(Module, fun(Pid) -> - Args = [ {Fun, Kind, Defaults} || - {Fun, Kind, _Line, true, Defaults} <- Private], +ensure_no_import_conflict(_File, 'Elixir.Kernel', _All) -> + ok; +ensure_no_import_conflict(File, Module, All) -> + if_tracker(Module, ok, fun(Tracker) -> + [elixir_errors:form_error(Meta, File, ?MODULE, {function_conflict, Error}) + || {Meta, Error} <- ?tracker:collect_imports_conflicts(Tracker, All)], + ok + end). - Unused = ?tracker:collect_unused_locals(Pid, Args), +warn_unused_local(File, Module, All, Private) -> + if_tracker(Module, [], fun(Tracker) -> + {Unreachable, Warnings} = ?tracker:collect_unused_locals(Tracker, All, Private), + [elixir_errors:form_warn(Meta, File, ?MODULE, Error) || {Meta, Error} <- Warnings], + Unreachable + end). - [ begin - {_, _, Line, _, _} = lists:keyfind(element(2, Error), 1, Private), - elixir_errors:handle_file_warning(File, {Line, ?MODULE, Error}) - end || Error <- Unused ] +ensure_no_undefined_local(File, Module, All) -> + if_tracker(Module, [], fun(Tracker) -> + case ?tracker:collect_undefined_locals(Tracker, All) of + [] -> ok; + + List -> + [{FirstMeta, FirstTuple, FirstError} | Rest] = lists:sort(List), + [elixir_errors:form_warn(Meta, File, ?MODULE, {Error, Tuple, Module}) || {Meta, Tuple, Error} <- lists:reverse(Rest)], + elixir_errors:form_error(FirstMeta, File, ?MODULE, {FirstError, FirstTuple, Module}), + ok + end end). -format_error({function_conflict,{Receivers, Name, Arity}}) -> +format_error({function_conflict, {Receiver, {Name, Arity}}}) -> io_lib:format("imported ~ts.~ts/~B conflicts with local function", - [elixir_aliases:inspect(hd(Receivers)), Name, Arity]); + [elixir_aliases:inspect(Receiver), Name, Arity]); -format_error({unused_args,{Name, Arity}}) -> - io_lib:format("default arguments in ~ts/~B are never used", [Name, Arity]); +format_error({unused_args, {Name, Arity}}) -> + io_lib:format("default values for the optional arguments in ~ts/~B are never used", [Name, Arity]); -format_error({unused_args,{Name, Arity},1}) -> - io_lib:format("the first default argument in ~ts/~B is never used", [Name, Arity]); +format_error({unused_args, {Name, Arity}, 1}) -> + io_lib:format("the default value for the first optional argument in ~ts/~B is never used", [Name, Arity]); -format_error({unused_args,{Name, Arity},Count}) -> - io_lib:format("the first ~B default arguments in ~ts/~B are never used", [Count, Name, Arity]); +format_error({unused_args, {Name, Arity}, Count}) -> + io_lib:format("the default values for the first ~B optional arguments in ~ts/~B are never used", [Count, Name, Arity]); -format_error({unused_def,{Name, Arity},defp}) -> +format_error({unused_def, {Name, Arity}, defp}) -> io_lib:format("function ~ts/~B is unused", [Name, Arity]); -format_error({unused_def,{Name, Arity},defmacrop}) -> - io_lib:format("macro ~ts/~B is unused", [Name, Arity]). +format_error({unused_def, {Name, Arity}, defmacrop}) -> + io_lib:format("macro ~ts/~B is unused", [Name, Arity]); + +format_error({undefined_function, {F, A}, _}) + when F == '__info__', A == 1; + F == 'behaviour_info', A == 1; + F == 'module_info', A == 1; + F == 'module_info', A == 0 -> + io_lib:format("undefined function ~ts/~B (this function is auto-generated by the compiler " + "and must always be called as a remote, as in __MODULE__.~ts/~B)", [F, A, F, A]); + +format_error({undefined_function, {F, A}, Module}) -> + io_lib:format("undefined function ~ts/~B (expected ~ts to define such a function or " + "for it to be imported, but none are available)", [F, A, elixir_aliases:inspect(Module)]); + +format_error({incorrect_dispatch, {F, A}, _Module}) -> + io_lib:format("cannot invoke macro ~ts/~B before its definition", [F, A]). diff --git a/lib/elixir/src/elixir_map.erl b/lib/elixir/src/elixir_map.erl index 957530e8761..560b7a43706 100644 --- a/lib/elixir/src/elixir_map.erl +++ b/lib/elixir/src/elixir_map.erl @@ -1,175 +1,242 @@ -module(elixir_map). --export([expand_map/3, translate_map/3, expand_struct/4, translate_struct/4]). --import(elixir_errors, [compile_error/4]). +-export([expand_map/4, expand_struct/5, format_error/1, load_struct/5]). +-import(elixir_errors, [form_error/4, form_warn/4]). -include("elixir.hrl"). -expand_map(Meta, [{'|', UpdateMeta, [Left, Right]}], E) -> - {[ELeft|ERight], EA} = elixir_exp:expand_args([Left|Right], E), - {{'%{}', Meta, [{'|', UpdateMeta, [ELeft, ERight]}]}, EA}; -expand_map(Meta, Args, E) -> - {EArgs, EA} = elixir_exp:expand_args(Args, E), - {{'%{}', Meta, EArgs}, EA}. +expand_map(Meta, [{'|', UpdateMeta, [Left, Right]}], S, #{context := nil} = E) -> + {[ELeft | ERight], SE, EE} = elixir_expand:expand_args([Left | Right], S, E), + validate_kv(Meta, ERight, Right, E), + {{'%{}', Meta, [{'|', UpdateMeta, [ELeft, ERight]}]}, SE, EE}; +expand_map(Meta, [{'|', _, [_, _]}] = Args, _S, #{context := Context, file := File}) -> + form_error(Meta, File, ?MODULE, {update_syntax_in_wrong_context, Context, {'%{}', Meta, Args}}); +expand_map(Meta, Args, S, E) -> + {EArgs, SE, EE} = elixir_expand:expand_args(Args, S, E), + validate_kv(Meta, EArgs, Args, E), + {{'%{}', Meta, EArgs}, SE, EE}. + +expand_struct(Meta, Left, {'%{}', MapMeta, MapArgs}, S, #{context := Context} = E) -> + CleanMapArgs = clean_struct_key_from_map_args(Meta, MapArgs, E), + {[ELeft, ERight], SE, EE} = elixir_expand:expand_args([Left, {'%{}', MapMeta, CleanMapArgs}], S, E), + + case validate_struct(ELeft, Context) of + true when is_atom(ELeft) -> + case extract_struct_assocs(Meta, ERight, E) of + {expand, MapMeta, Assocs} when Context /= match -> %% Expand + AssocKeys = [K || {K, _} <- Assocs], + Struct = load_struct(Meta, ELeft, [Assocs], AssocKeys, EE), + Keys = ['__struct__'] ++ AssocKeys, + WithoutKeys = maps:to_list(maps:without(Keys, Struct)), + StructAssocs = elixir_quote:escape(WithoutKeys, none, false), + {{'%', Meta, [ELeft, {'%{}', MapMeta, StructAssocs ++ Assocs}]}, SE, EE}; + + {_, _, Assocs} -> %% Update or match + _ = load_struct(Meta, ELeft, [], [K || {K, _} <- Assocs], EE), + {{'%', Meta, [ELeft, ERight]}, SE, EE} + end; -expand_struct(Meta, Left, Right, E) -> - {[ELeft, ERight], EE} = elixir_exp:expand_args([Left, Right], E), - - case is_atom(ELeft) of - true -> ok; - false -> - compile_error(Meta, ?m(E, file), "expected struct name to be a compile " - "time atom or alias, got: ~ts", ['Elixir.Macro':to_string(ELeft)]) - end, - - EMeta = - case lists:member(ELeft, ?m(E, context_modules)) of - true -> - case (ELeft == ?m(E, module)) and - (?m(E, function) == nil) of - true -> - compile_error(Meta, ?m(E, file), - "cannot access struct ~ts in body of the module that defines it as " - "the struct fields are not yet accessible", - [elixir_aliases:inspect(ELeft)]); - false -> - [{struct, context}|Meta] - end; - false -> - Meta - end, - - case ERight of - {'%{}', _, _} -> ok; - _ -> compile_error(Meta, ?m(E, file), - "expected struct to be followed by a map, got: ~ts", - ['Elixir.Macro':to_string(ERight)]) - end, - - {{'%', EMeta, [ELeft, ERight]}, EE}. - -translate_map(Meta, Args, S) -> - {Assocs, TUpdate, US} = extract_assoc_update(Args, S), - translate_map(Meta, Assocs, TUpdate, US). + true -> + {{'%', Meta, [ELeft, ERight]}, SE, EE}; -translate_struct(Meta, Name, {'%{}', MapMeta, Args}, S) -> - {Assocs, TUpdate, US} = extract_assoc_update(Args, S), - Struct = load_struct(Meta, Name, S), + false when Context == match -> + form_error(Meta, E, ?MODULE, {invalid_struct_name_in_match, ELeft}); - case is_map(Struct) of - true -> - assert_struct_keys(Meta, Name, Struct, Assocs, S); false -> - compile_error(Meta, S#elixir_scope.file, "expected ~ts.__struct__/0 to " - "return a map, got: ~ts", [elixir_aliases:inspect(Name), 'Elixir.Kernel':inspect(Struct)]) - end, - - if - TUpdate /= nil -> - Line = ?line(Meta), - {VarName, _, VS} = elixir_scope:build_var('_', US), - - Var = {var, Line, VarName}, - Map = {map, Line, [{map_field_exact, Line, {atom, Line, '__struct__'}, {atom, Line, Name}}]}, - - Match = {match, Line, Var, Map}, - Error = {tuple, Line, [{atom, Line, badstruct}, {atom, Line, Name}, Var]}, + form_error(Meta, E, ?MODULE, {invalid_struct_name, ELeft}) + end; +expand_struct(Meta, _Left, Right, _S, E) -> + form_error(Meta, E, ?MODULE, {non_map_after_struct, Right}). + +clean_struct_key_from_map_args(Meta, [{'|', PipeMeta, [Left, MapAssocs]}], E) -> + [{'|', PipeMeta, [Left, clean_struct_key_from_map_assocs(Meta, MapAssocs, E)]}]; +clean_struct_key_from_map_args(Meta, MapAssocs, E) -> + clean_struct_key_from_map_assocs(Meta, MapAssocs, E). + +clean_struct_key_from_map_assocs(Meta, Assocs, E) -> + case lists:keytake('__struct__', 1, Assocs) of + {value, _, CleanAssocs} -> + form_warn(Meta, ?key(E, file), ?MODULE, ignored_struct_key_in_struct), + CleanAssocs; + false -> + Assocs + end. - {TMap, TS} = translate_map(MapMeta, Assocs, Var, VS), +validate_match_key(Meta, {Name, _, Context}, E) when is_atom(Name), is_atom(Context) -> + form_error(Meta, E, ?MODULE, {invalid_variable_in_map_key_match, Name}); +validate_match_key(_, {'^', _, [{Name, _, Context}]}, _) when is_atom(Name), is_atom(Context) -> + ok; +validate_match_key(_, {'%{}', _, [_ | _]}, _) -> + ok; +validate_match_key(Meta, {Left, _, Right}, E) -> + validate_match_key(Meta, Left, E), + validate_match_key(Meta, Right, E); +validate_match_key(Meta, {Left, Right}, E) -> + validate_match_key(Meta, Left, E), + validate_match_key(Meta, Right, E); +validate_match_key(Meta, List, E) when is_list(List) -> + [validate_match_key(Meta, Each, E) || Each <- List]; +validate_match_key(_, _, _) -> + ok. + +validate_not_repeated(Meta, Key, Used, E) -> + case is_literal(Key) andalso Used of + #{Key := true} -> + case E of + #{context := match} -> + form_error(Meta, ?key(E, file), ?MODULE, {repeated_key, Key}); + _ -> + form_warn(Meta, ?key(E, file), ?MODULE, {repeated_key, Key}), + Used + end; + + #{} -> + Used#{Key => true}; - {{'case', Line, TUpdate, [ - {clause, Line, [Match], [], [TMap]}, - {clause, Line, [Var], [], [elixir_utils:erl_call(Line, erlang, error, [Error])]} - ]}, TS}; - S#elixir_scope.context == match -> - translate_map(MapMeta, Assocs ++ [{'__struct__', Name}], nil, US); - true -> - Keys = [K || {K,_} <- Assocs], - {StructAssocs, _} = elixir_quote:escape(maps:to_list(maps:without(Keys, Struct)), false), - translate_map(MapMeta, StructAssocs ++ Assocs ++ [{'__struct__', Name}], nil, US) + false -> + Used end. -%% Helpers - -load_struct(Meta, Name, S) -> - Local = - elixir_module:is_open(Name) andalso - (case lists:keyfind(struct, 1, Meta) of - {struct, context} -> true; - _ -> wait_for_struct(Name) - end), +is_literal({_, _, _}) -> false; +is_literal({Left, Right}) -> is_literal(Left) andalso is_literal(Right); +is_literal([_ | _] = List) -> lists:all(fun is_literal/1, List); +is_literal(_) -> true. + +validate_kv(Meta, KV, Original, #{context := Context} = E) -> + lists:foldl(fun + ({K, _V}, {Index, Used}) -> + (Context == match) andalso validate_match_key(Meta, K, E), + NewUsed = validate_not_repeated(Meta, K, Used, E), + {Index + 1, NewUsed}; + (_, {Index, _Used}) -> + form_error(Meta, E, ?MODULE, {not_kv_pair, lists:nth(Index, Original)}) + end, {1, #{}}, KV). + +extract_struct_assocs(_, {'%{}', Meta, [{'|', _, [_, Assocs]}]}, _) -> + {update, Meta, delete_struct_key(Assocs)}; +extract_struct_assocs(_, {'%{}', Meta, Assocs}, _) -> + {expand, Meta, delete_struct_key(Assocs)}; +extract_struct_assocs(Meta, Other, E) -> + form_error(Meta, E, ?MODULE, {non_map_after_struct, Other}). + +delete_struct_key(Assocs) -> + lists:keydelete('__struct__', 1, Assocs). + +validate_struct({'^', _, [{Var, _, Ctx}]}, match) when is_atom(Var), is_atom(Ctx) -> true; +validate_struct({Var, _Meta, Ctx}, match) when is_atom(Var), is_atom(Ctx) -> true; +validate_struct(Atom, _) when is_atom(Atom) -> true; +validate_struct(_, _) -> false. + +load_struct(Meta, Name, Args, Keys, E) -> + %% We also include the current module because it won't be present + %% in context module in case the module name is defined dynamically. + InContext = lists:member(Name, [?key(E, module) | ?key(E, context_modules)]), + + Arity = length(Args), + Local = InContext orelse (not(ensure_loaded(Name)) andalso wait_for_struct(Name)), try - case Local of - true -> + case Local andalso elixir_def:local_for(Name, '__struct__', Arity, [def]) of + false -> + apply(Name, '__struct__', Args); + LocalFun -> + %% There is an inherent race condition when using local_for. + %% By the time we got to execute the function, the ETS table + %% with temporary definitions for the given module may no longer + %% be available, so any function invocation happening inside the + %% local function will fail. In this case, we need to fall back to + %% the regular dispatching since the module will be available if + %% the table has not been deleted (unless compilation of that + %% module failed which should then cause this call to fail too). try - (elixir_locals:local_for(Name, '__struct__', 0, def))() + apply(LocalFun, Args) catch - error:undef -> Name:'__struct__'(); - error:badarg -> Name:'__struct__'() - end; - false -> - Name:'__struct__'() + error:undef -> apply(Name, '__struct__', Args) + end end + of + #{'__struct__' := Name} = Struct -> + assert_struct_keys(Meta, Name, Struct, Keys, E), + elixir_env:trace({struct_expansion, Meta, Name, Keys}, E), + Struct; + + #{'__struct__' := StructName} when is_atom(StructName) -> + form_error(Meta, E, ?MODULE, {struct_name_mismatch, Name, Arity, StructName}); + Other -> + form_error(Meta, E, ?MODULE, {invalid_struct_return_value, Name, Arity, Other}) catch error:undef -> - Inspected = elixir_aliases:inspect(Name), - compile_error(Meta, S#elixir_scope.file, "~ts.__struct__/0 is undefined, " - "cannot expand struct ~ts", [Inspected, Inspected]) + case InContext andalso (?key(E, function) == nil) of + true -> + form_error(Meta, E, ?MODULE, {inaccessible_struct, Name}); + false -> + form_error(Meta, E, ?MODULE, {undefined_struct, Name, Arity}) + end; + + Kind:Reason -> + Info = [{Name, '__struct__', Arity, [{file, "expanding struct"}]}, + elixir_utils:caller(?line(Meta), ?key(E, file), ?key(E, module), ?key(E, function))], + erlang:raise(Kind, Reason, Info) end. +ensure_loaded(Module) -> + code:ensure_loaded(Module) == {module, Module}. + wait_for_struct(Module) -> - case erlang:get(elixir_compiler_pid) of - undefined -> - false; - Pid -> - Ref = erlang:make_ref(), - Pid ! {waiting, struct, self(), Ref, Module}, - receive - {Ref, ready} -> - true; - {Ref, release} -> - 'Elixir.Kernel.ErrorHandler':release(), - false - end - end. + (erlang:get(elixir_compiler_info) /= undefined) andalso + ('Elixir.Kernel.ErrorHandler':ensure_compiled(Module, struct, hard) =:= found). -translate_map(Meta, Assocs, TUpdate, #elixir_scope{extra=Extra} = S) -> - {Op, KeyFun, ValFun} = extract_key_val_op(TUpdate, S), - - Line = ?line(Meta), - - {TArgs, SA} = lists:mapfoldl(fun - ({Key, Value}, Acc) -> - {TKey, Acc1} = KeyFun(Key, Acc), - {TValue, Acc2} = ValFun(Value, Acc1#elixir_scope{extra=Extra}), - {{Op, ?line(Meta), TKey, TValue}, Acc2}; - (Other, _Acc) -> - compile_error(Meta, S#elixir_scope.file, "expected key-value pairs in map, got: ~ts", - ['Elixir.Macro':to_string(Other)]) - end, S, Assocs), - - build_map(Line, TUpdate, TArgs, SA). - -extract_assoc_update([{'|', _Meta, [Update, Args]}], S) -> - {TArg, SA} = elixir_translator:translate_arg(Update, S, S), - {Args, TArg, SA}; -extract_assoc_update(Args, SA) -> {Args, nil, SA}. - -extract_key_val_op(_TUpdate, #elixir_scope{context=match}) -> - {map_field_exact, - fun(X, Acc) -> elixir_translator:translate(X, Acc#elixir_scope{extra=map_key}) end, - fun elixir_translator:translate/2}; -extract_key_val_op(TUpdate, S) -> - KS = S#elixir_scope{extra=map_key}, - Op = if TUpdate == nil -> map_field_assoc; true -> map_field_exact end, - {Op, - fun(X, Acc) -> elixir_translator:translate_arg(X, Acc, KS) end, - fun(X, Acc) -> elixir_translator:translate_arg(X, Acc, S) end}. - -build_map(Line, nil, TArgs, SA) -> {{map, Line, TArgs}, SA}; -build_map(Line, TUpdate, TArgs, SA) -> {{map, Line, TUpdate, TArgs}, SA}. - -assert_struct_keys(Meta, Name, Struct, Assocs, S) -> +assert_struct_keys(Meta, Name, Struct, Keys, E) -> [begin - compile_error(Meta, S#elixir_scope.file, "unknown key ~ts for struct ~ts", - ['Elixir.Kernel':inspect(Key), elixir_aliases:inspect(Name)]) - end || {Key, _} <- Assocs, not maps:is_key(Key, Struct)]. + form_error(Meta, E, ?MODULE, {unknown_key_for_struct, Name, Key}) + end || Key <- Keys, not maps:is_key(Key, Struct)]. + +format_error({update_syntax_in_wrong_context, Context, Expr}) -> + io_lib:format("cannot use map/struct update syntax in ~ts, got: ~ts", + [Context, 'Elixir.Macro':to_string(Expr)]); +format_error({invalid_struct_name_in_match, Expr}) -> + Message = + "expected struct name in a match to be a compile time atom, alias or a " + "variable, got: ~ts", + io_lib:format(Message, ['Elixir.Macro':to_string(Expr)]); +format_error({invalid_struct_name, Expr}) -> + Message = "expected struct name to be a compile time atom or alias, got: ~ts", + io_lib:format(Message, ['Elixir.Macro':to_string(Expr)]); +format_error({invalid_variable_in_map_key_match, Name}) -> + Message = + "cannot use variable ~ts as map key inside a pattern. Map keys in patterns can only be literals " + "(such as atoms, strings, tuples, and the like) or an existing variable matched with the pin operator " + "(such as ^some_var)", + io_lib:format(Message, [Name]); +format_error({repeated_key, Key}) -> + io_lib:format("key ~ts will be overridden in map", ['Elixir.Macro':to_string(Key)]); +format_error({not_kv_pair, Expr}) -> + io_lib:format("expected key-value pairs in a map, got: ~ts", + ['Elixir.Macro':to_string(Expr)]); +format_error({non_map_after_struct, Expr}) -> + io_lib:format("expected struct to be followed by a map, got: ~ts", + ['Elixir.Macro':to_string(Expr)]); +format_error({struct_name_mismatch, Module, Arity, StructName}) -> + Name = elixir_aliases:inspect(Module), + Message = "expected struct name returned by ~ts.__struct__/~p to be ~ts, got: ~ts", + io_lib:format(Message, [Name, Arity, Name, elixir_aliases:inspect(StructName)]); +format_error({invalid_struct_return_value, Module, Arity, Value}) -> + Message = + "expected ~ts.__struct__/~p to return a map with a :__struct__ key that holds the " + "name of the struct (atom), got: ~ts", + io_lib:format(Message, [elixir_aliases:inspect(Module), Arity, 'Elixir.Kernel':inspect(Value)]); +format_error({inaccessible_struct, Module}) -> + Message = + "cannot access struct ~ts, the struct was not yet defined or the struct is " + "being accessed in the same context that defines it", + io_lib:format(Message, [elixir_aliases:inspect(Module)]); +format_error({undefined_struct, Module, Arity}) -> + Name = elixir_aliases:inspect(Module), + io_lib:format( + "~ts.__struct__/~p is undefined, cannot expand struct ~ts. " + "Make sure the struct name is correct. If the struct name exists and is correct " + "but it still cannot be found, you likely have cyclic module usage in your code", + [Name, Arity, Name]); +format_error({unknown_key_for_struct, Module, Key}) -> + io_lib:format("unknown key ~ts for struct ~ts", + ['Elixir.Macro':to_string(Key), elixir_aliases:inspect(Module)]); +format_error(ignored_struct_key_in_struct) -> + "key :__struct__ is ignored when using structs". diff --git a/lib/elixir/src/elixir_module.erl b/lib/elixir/src/elixir_module.erl index 971b50eb472..81155dcab7d 100644 --- a/lib/elixir/src/elixir_module.erl +++ b/lib/elixir/src/elixir_module.erl @@ -1,365 +1,264 @@ -module(elixir_module). --export([compile/4, data_table/1, docs_table/1, is_open/1, - expand_callback/6, add_beam_chunk/3, format_error/1]). +-export([file/1, data_tables/1, is_open/1, mode/1, delete_definition_attributes/6, + compile/4, expand_callback/6, format_error/1, compiler_modules/0, + write_cache/3, read_cache/2, next_counter/1]). -include("elixir.hrl"). +-define(counter_attr, {elixir, counter}). --define(acc_attr, '__acc_attributes'). --define(docs_attr, '__docs_table'). --define(lexical_attr, '__lexical_tracker'). --define(persisted_attr, '__persisted_attributes'). --define(overridable_attr, '__overridable'). --define(location_attr, '__location'). +%% Stores modules currently being defined by the compiler -%% TABLE METHODS - -data_table(Module) -> - Module. - -docs_table(Module) -> - ets:lookup_element(Module, ?docs_attr, 2). - -is_open(Module) -> - Module == ets:info(Module, name). - -%% Compilation hook - -compile(Module, Block, Vars, #{line := Line} = Env) when is_atom(Module) -> - %% In case we are generating a module from inside a function, - %% we get rid of the lexical tracker information as, at this - %% point, the lexical tracker process is long gone. - LexEnv = case ?m(Env, function) of - nil -> Env#{module := Module, local := nil}; - _ -> Env#{lexical_tracker := nil, function := nil, module := Module, local := nil} - end, - - case ?m(LexEnv, lexical_tracker) of - nil -> - elixir_lexical:run(?m(LexEnv, file), fun(Pid) -> - do_compile(Line, Module, Block, Vars, LexEnv#{lexical_tracker := Pid}) - end); - _ -> - do_compile(Line, Module, Block, Vars, LexEnv) - end; - -compile(Module, _Block, _Vars, #{line := Line, file := File}) -> - elixir_errors:form_error(Line, File, ?MODULE, {invalid_module, Module}). - -do_compile(Line, Module, Block, Vars, E) -> - File = ?m(E, file), - check_module_availability(Line, File, Module), - build(Line, File, Module, ?m(E, lexical_tracker)), - - try - {Result, NE} = eval_form(Line, Module, Block, Vars, E), - {Base, Export, Private, Def, Defmacro, Functions} = elixir_def:unwrap_definitions(Module), - - {All, Forms0} = functions_form(Line, File, Module, Base, Export, Def, Defmacro, Functions), - Forms1 = specs_form(Module, Private, Defmacro, Forms0), - Forms2 = types_form(Module, Forms1), - Forms3 = attributes_form(Line, File, Module, Forms2), - - case ets:lookup(data_table(Module), 'on_load') of - [] -> ok; - [{on_load,OnLoad}] -> - [elixir_locals:record_local(Tuple, Module) || Tuple <- OnLoad] - end, - - AllFunctions = Def ++ [T || {T, defp, _, _, _} <- Private], - elixir_locals:ensure_no_function_conflict(Line, File, Module, AllFunctions), - elixir_locals:warn_unused_local(File, Module, Private), - warn_invalid_clauses(Line, File, Module, All), - warn_unused_docs(Line, File, Module), - - Location = {elixir_utils:relative_to_cwd(elixir_utils:characters_to_list(File)), Line}, - - Final = [ - {attribute, Line, file, Location}, - {attribute, Line, module, Module} | Forms3 - ], - - Binary = load_form(Line, Final, compile_opts(Module), NE), - {module, Module, Binary, Result} - after - elixir_locals:cleanup(Module), - elixir_def:cleanup(Module), - ets:delete(docs_table(Module)), - ets:delete(data_table(Module)) +compiler_modules() -> + case erlang:get(elixir_compiler_modules) of + undefined -> []; + M when is_list(M) -> M end. -%% Hook that builds both attribute and functions and set up common hooks. - -build(Line, File, Module, Lexical) -> - %% Table with meta information about the module. - DataTable = data_table(Module), - - OldTable = ets:info(DataTable, name), - case OldTable == DataTable of - true -> - [{OldFile, OldLine}] = ets:lookup_element(OldTable, ?location_attr, 2), - Error = {module_in_definition, Module, OldFile, OldLine}, - elixir_errors:form_error(Line, File, ?MODULE, Error); - false -> - [] - end, - - ets:new(DataTable, [set, named_table, public]), - ets:insert(DataTable, {before_compile, []}), - ets:insert(DataTable, {after_compile, []}), +put_compiler_modules([]) -> + erlang:erase(elixir_compiler_modules); +put_compiler_modules(M) when is_list(M) -> + erlang:put(elixir_compiler_modules, M). - case elixir_compiler:get_opt(docs) of - true -> ets:insert(DataTable, {on_definition, [{'Elixir.Module', compile_doc}]}); - _ -> ets:insert(DataTable, {on_definition, []}) - end, - - Attributes = [behaviour, on_load, spec, type, typep, opaque, callback, compile, external_resource], - ets:insert(DataTable, {?acc_attr, [before_compile, after_compile, on_definition, derive|Attributes]}), - ets:insert(DataTable, {?persisted_attr, [vsn|Attributes]}), - ets:insert(DataTable, {?docs_attr, ets:new(DataTable, [ordered_set, public])}), - ets:insert(DataTable, {?lexical_attr, Lexical}), - ets:insert(DataTable, {?overridable_attr, []}), - ets:insert(DataTable, {?location_attr, [{File, Line}]}), - - %% Setup other modules - elixir_def:setup(Module), - elixir_locals:setup(Module). - -%% Receives the module representation and evaluates it. - -eval_form(Line, Module, Block, Vars, E) -> - {Value, EE} = elixir_compiler:eval_forms(Block, Vars, E), - elixir_def_overridable:store_pending(Module), - EV = elixir_env:linify({Line, EE#{vars := [], export_vars := nil}}), - EC = eval_callbacks(Line, Module, before_compile, [EV], EV), - elixir_def_overridable:store_pending(Module), - {Value, EC}. - -eval_callbacks(Line, Module, Name, Args, E) -> - Callbacks = lists:reverse(ets:lookup_element(data_table(Module), Name, 2)), - - lists:foldl(fun({M,F}, Acc) -> - expand_callback(Line, M, F, Args, Acc#{vars := [], export_vars := nil}, - fun(AM, AF, AA) -> apply(AM, AF, AA) end) - end, E, Callbacks). - -%% Return the form with exports and function declarations. - -functions_form(Line, File, Module, BaseAll, BaseExport, Def, Defmacro, BaseFunctions) -> - {InfoSpec, Info} = add_info_function(Line, File, Module, BaseExport, Def, Defmacro), +%% Table functions - All = [{'__info__', 1}|BaseAll], - Export = [{'__info__', 1}|BaseExport], - Functions = [InfoSpec,Info|BaseFunctions], +file(Module) -> + ets:lookup_element(elixir_modules, Module, 4). - {All, [ - {attribute, Line, export, lists:sort(Export)} | Functions - ]}. +data_tables(Module) -> + ets:lookup_element(elixir_modules, Module, 2). -%% Add attributes handling to the form - -attributes_form(Line, File, Module, Current) -> - Table = data_table(Module), - - AccAttrs = ets:lookup_element(Table, '__acc_attributes', 2), - PersistedAttrs = ets:lookup_element(Table, '__persisted_attributes', 2), - - Transform = fun({Key, Value}, Acc) -> - case lists:member(Key, PersistedAttrs) of - false -> Acc; - true -> - Values = - case lists:member(Key, AccAttrs) of - true -> Value; - false -> [Value] - end, - - lists:foldl(fun(X, Final) -> - [{attribute, Line, Key, X}|Final] - end, Acc, process_attribute(Line, File, Key, Values)) - end - end, - - ets:foldl(Transform, Current, Table). +is_open(Module) -> + ets:member(elixir_modules, Module). -process_attribute(Line, File, external_resource, Values) -> - lists:usort([process_external_resource(Line, File, Value) || Value <- Values]); -process_attribute(_Line, _File, _Key, Values) -> - Values. +mode(Module) -> + try ets:lookup_element(elixir_modules, Module, 5) of + Mode -> Mode + catch + _:badarg -> closed + end. -process_external_resource(_Line, _File, Value) when is_binary(Value) -> - Value; -process_external_resource(Line, File, Value) -> - elixir_errors:handle_file_error(File, - {Line, ?MODULE, {invalid_external_resource, Value}}). +make_readonly(Module) -> + ets:update_element(elixir_modules, Module, {5, readonly}). -%% Types +delete_definition_attributes(#{module := Module}, _, _, _, _, _) -> + {DataSet, _} = data_tables(Module), + ets:delete(DataSet, doc), + ets:delete(DataSet, deprecated), + ets:delete(DataSet, impl). -types_form(Module, Forms0) -> - case code:ensure_loaded('Elixir.Kernel.Typespec') of - {module, 'Elixir.Kernel.Typespec'} -> - Types0 = 'Elixir.Module':get_attribute(Module, type) ++ - 'Elixir.Module':get_attribute(Module, typep) ++ - 'Elixir.Module':get_attribute(Module, opaque), +write_cache(Module, Key, Value) -> + {DataSet, _} = data_tables(Module), + ets:insert(DataSet, {{cache, Key}, Value}). - Types1 = ['Elixir.Kernel.Typespec':translate_type(Kind, Expr, Doc, Caller) || - {Kind, Expr, Doc, Caller} <- Types0], +read_cache(Module, Key) -> + {DataSet, _} = data_tables(Module), + ets:lookup_element(DataSet, {cache, Key}, 2). - 'Elixir.Module':delete_attribute(Module, type), - 'Elixir.Module':delete_attribute(Module, typep), - 'Elixir.Module':delete_attribute(Module, opaque), +next_counter(nil) -> erlang:unique_integer(); +next_counter(Module) -> + try + {DataSet, _} = data_tables(Module), + {Module, ets:update_counter(DataSet, ?counter_attr, 1)} + catch + _:_ -> erlang:unique_integer() + end. - Forms1 = types_attributes(Types1, Forms0), - Forms2 = export_types_attributes(Types1, Forms1), - typedocs_attributes(Types1, Forms2); +%% Compilation hook - {error, _} -> - Forms0 - end. +compile(Module, Block, Vars, Env) when is_atom(Module) -> + #{line := Line, function := Function, versioned_vars := OldVerVars} = Env, -types_attributes(Types, Forms) -> - Fun = fun({{Kind, _NameArity, Expr}, Line, _Export, _Doc}, Acc) -> - [{attribute, Line, Kind, Expr}|Acc] - end, - lists:foldl(Fun, Forms, Types). - -export_types_attributes(Types, Forms) -> - Fun = fun - ({{_Kind, NameArity, _Expr}, Line, true, _Doc}, Acc) -> - [{attribute, Line, export_type, [NameArity]}|Acc]; - ({_Type, _Line, false, _Doc}, Acc) -> - Acc - end, - lists:foldl(Fun, Forms, Types). - -typedocs_attributes(Types, Forms) -> - Fun = fun - ({{_Kind, NameArity, _Expr}, Line, true, Doc}, Acc) when Doc =/= nil -> - [{attribute, Line, typedoc, {NameArity, Doc}}|Acc]; - ({_Type, _Line, _Export, _Doc}, Acc) -> - Acc - end, - lists:foldl(Fun, Forms, Types). + {VerVars, _} = + lists:mapfoldl(fun({Var, _}, I) -> {{Var, I}, I + 1} end, 0, maps:to_list(OldVerVars)), -%% Specs + BaseEnv = Env#{module := Module, versioned_vars := maps:from_list(VerVars)}, -specs_form(Module, Private, Defmacro, Forms) -> - case code:ensure_loaded('Elixir.Kernel.Typespec') of - {module, 'Elixir.Kernel.Typespec'} -> - Defmacrop = [Tuple || {Tuple, defmacrop, _, _, _} <- Private], + MaybeLexEnv = + case Function of + nil -> BaseEnv; + _ -> BaseEnv#{lexical_tracker := nil, tracers := [], function := nil} + end, - Specs0 = 'Elixir.Module':get_attribute(Module, spec) ++ - 'Elixir.Module':get_attribute(Module, callback), + case MaybeLexEnv of + #{lexical_tracker := nil} -> + elixir_lexical:run( + MaybeLexEnv, + fun(LexEnv) -> compile(Line, Module, Block, Vars, LexEnv) end, + fun(_LexEnv) -> ok end + ); + _ -> + compile(Line, Module, Block, Vars, MaybeLexEnv) + end; +compile(Module, _Block, _Vars, #{line := Line, file := File}) -> + elixir_errors:form_error([{line, Line}], File, ?MODULE, {invalid_module, Module}). - Specs1 = ['Elixir.Kernel.Typespec':translate_spec(Kind, Expr, Caller) || - {Kind, Expr, Caller} <- Specs0], - Specs2 = lists:flatmap(fun(Spec) -> - translate_macro_spec(Spec, Defmacro, Defmacrop) - end, Specs1), +compile(Line, Module, Block, Vars, E) -> + File = ?key(E, file), + check_module_availability(Line, File, Module), + ModuleAsCharlist = validate_module_name(Line, File, Module), - 'Elixir.Module':delete_attribute(Module, spec), - 'Elixir.Module':delete_attribute(Module, callback), - specs_attributes(Forms, Specs2); + CompilerModules = compiler_modules(), + {Tables, Ref} = build(Line, File, Module), + {DataSet, DataBag} = Tables, - {error, _} -> - Forms + try + put_compiler_modules([Module | CompilerModules]), + {Result, NE} = eval_form(Line, Module, DataBag, Block, Vars, E), + CheckerInfo = get(elixir_checker_info), + + {Binary, PersistedAttributes, Autoload, CheckerPid} = + elixir_erl_compiler:spawn(fun() -> + PersistedAttributes = ets:lookup_element(DataBag, persisted_attributes, 2), + Attributes = attributes(DataSet, DataBag, PersistedAttributes), + {AllDefinitions, Private} = elixir_def:fetch_definitions(File, Module), + + OnLoadAttribute = lists:keyfind(on_load, 1, Attributes), + NewPrivate = validate_on_load_attribute(OnLoadAttribute, AllDefinitions, Private, File, Line), + + DialyzerAttribute = lists:keyfind(dialyzer, 1, Attributes), + validate_dialyzer_attribute(DialyzerAttribute, AllDefinitions, File, Line), + + Unreachable = elixir_locals:warn_unused_local(File, Module, AllDefinitions, NewPrivate), + elixir_locals:ensure_no_undefined_local(File, Module, AllDefinitions), + elixir_locals:ensure_no_import_conflict(File, Module, AllDefinitions), + + %% We stop tracking locals here to avoid race conditions in case after_load + %% evaluates code in a separate process that may write to locals table. + elixir_locals:stop({DataSet, DataBag}), + make_readonly(Module), + + (not elixir_config:is_bootstrap()) andalso + 'Elixir.Module':check_derive_behaviours_and_impls(E, DataSet, DataBag, AllDefinitions), + + RawCompileOpts = bag_lookup_element(DataBag, {accumulate, compile}, 2), + CompileOpts = validate_compile_opts(RawCompileOpts, AllDefinitions, Unreachable, File, Line), + + ModuleMap = #{ + struct => get_struct(DataSet), + module => Module, + line => Line, + file => File, + relative_file => elixir_utils:relative_to_cwd(File), + attributes => Attributes, + definitions => AllDefinitions, + unreachable => Unreachable, + compile_opts => CompileOpts, + deprecated => get_deprecated(DataBag), + is_behaviour => is_behaviour(DataBag) + }, + + Binary = elixir_erl:compile(ModuleMap), + Autoload = proplists:get_value(autoload, CompileOpts, true), + CheckerPid = spawn_parallel_checker(CheckerInfo, Module, ModuleMap), + {Binary, PersistedAttributes, Autoload, CheckerPid} + end), + + Autoload andalso code:load_binary(Module, beam_location(ModuleAsCharlist), Binary), + eval_callbacks(Line, DataBag, after_compile, [NE, Binary], NE), + elixir_env:trace({on_module, Binary, none}, E), + warn_unused_attributes(File, DataSet, DataBag, PersistedAttributes), + make_module_available(Module, Binary, CheckerPid), + {module, Module, Binary, Result} + catch + error:undef:Stacktrace -> + case Stacktrace of + [{Module, Fun, Args, _Info} | _] = Stack when is_list(Args) -> + compile_undef(Module, Fun, length(Args), Stack); + [{Module, Fun, Arity, _Info} | _] = Stack -> + compile_undef(Module, Fun, Arity, Stack); + Stack -> + erlang:raise(error, undef, Stack) + end + after + put_compiler_modules(CompilerModules), + ets:delete(DataSet), + ets:delete(DataBag), + elixir_code_server:call({undefmodule, Ref}) end. -specs_attributes(Forms, Specs) -> - Dict = lists:foldl(fun({{Kind, NameArity, Spec}, Line}, Acc) -> - dict:append({Kind, NameArity}, {Spec, Line}, Acc) - end, dict:new(), Specs), - dict:fold(fun({Kind, NameArity}, ExprsLines, Acc) -> - {Exprs, Lines} = lists:unzip(ExprsLines), - Line = lists:min(Lines), - [{attribute, Line, Kind, {NameArity, Exprs}}|Acc] - end, Forms, Dict). - -translate_macro_spec({{spec, NameArity, Spec}, Line}, Defmacro, Defmacrop) -> - case ordsets:is_element(NameArity, Defmacrop) of - true -> []; - false -> - case ordsets:is_element(NameArity, Defmacro) of - true -> - {Name, Arity} = NameArity, - [{{spec, {elixir_utils:macro_name(Name), Arity + 1}, spec_for_macro(Spec)}, Line}]; - false -> - [{{spec, NameArity, Spec}, Line}] +validate_compile_opts(Opts, Defs, Unreachable, File, Line) -> + lists:flatmap(fun (Opt) -> validate_compile_opt(Opt, Defs, Unreachable, File, Line) end, Opts). + +%% TODO: Make this an error on v2.0 +validate_compile_opt({parse_transform, Module} = Opt, _Defs, _Unreachable, File, Line) -> + elixir_errors:form_warn([{line, Line}], File, ?MODULE, {parse_transform, Module}), + [Opt]; +validate_compile_opt({inline, Inlines}, Defs, Unreachable, File, Line) -> + case validate_inlines(Inlines, Defs, Unreachable, []) of + {ok, []} -> []; + {ok, FilteredInlines} -> [{inline, FilteredInlines}]; + {error, Def} -> elixir_errors:form_error([{line, Line}], File, ?MODULE, {bad_inline, Def}) + end; +validate_compile_opt(Opt, Defs, Unreachable, File, Line) when is_list(Opt) -> + validate_compile_opts(Opt, Defs, Unreachable, File, Line); +validate_compile_opt(Opt, _Defs, _Unreachable, _File, _Line) -> + [Opt]. + +validate_inlines([Inline | Inlines], Defs, Unreachable, Acc) -> + case lists:keyfind(Inline, 1, Defs) of + false -> {error, Inline}; + _ -> + case lists:member(Inline, Unreachable) of + true -> validate_inlines(Inlines, Defs, Unreachable, Acc); + false -> validate_inlines(Inlines, Defs, Unreachable, [Inline | Acc]) end end; +validate_inlines([], _Defs, _Unreachable, Acc) -> {ok, Acc}. -translate_macro_spec({{callback, NameArity, Spec}, Line}, _Defmacro, _Defmacrop) -> - [{{callback, NameArity, Spec}, Line}]. - -spec_for_macro({type, Line, 'fun', [{type, _, product, Args}|T]}) -> - NewArgs = [{type, Line, term, []}|Args], - {type, Line, 'fun', [{type, Line, product, NewArgs}|T]}; - -spec_for_macro(Else) -> Else. - -%% Loads the form into the code server. +validate_on_load_attribute({on_load, Def}, Defs, Private, File, Line) -> + case lists:keyfind(Def, 1, Defs) of + false -> + elixir_errors:form_error([{line, Line}], File, ?MODULE, {undefined_on_load, Def}); + {_, Kind, _, _} when Kind == def; Kind == defp -> + lists:keydelete(Def, 1, Private); + {_, WrongKind, _, _} -> + elixir_errors:form_error([{line, Line}], File, ?MODULE, {wrong_kind_on_load, Def, WrongKind}) + end; +validate_on_load_attribute(false, _Module, Private, _File, _Line) -> Private. -compile_opts(Module) -> - case ets:lookup(data_table(Module), compile) of - [{compile,Opts}] when is_list(Opts) -> Opts; - [] -> [] +validate_dialyzer_attribute({dialyzer, Dialyzer}, Defs, File, Line) -> + [case lists:keyfind(Fun, 1, Defs) of + false -> + elixir_errors:form_error([{line, Line}], File, ?MODULE, {bad_dialyzer, Key, Fun}); + _ -> + ok + end || {Key, Funs} <- lists:flatten([Dialyzer]), Fun <- lists:flatten([Funs])]; +validate_dialyzer_attribute(false, _Defs, _File, _Line) -> + ok. + +is_behaviour(DataBag) -> + ets:member(DataBag, {accumulate, callback}) orelse ets:member(DataBag, {accumulate, macrocallback}). + +%% An undef error for a function in the module being compiled might result in an +%% exception message suggesting the current module is not loaded. This is +%% misleading so use a custom reason. +compile_undef(Module, Fun, Arity, Stack) -> + case elixir_config:is_bootstrap() of + false -> + Opts = [{module, Module}, {function, Fun}, {arity, Arity}, + {reason, 'function not available'}], + Exception = 'Elixir.UndefinedFunctionError':exception(Opts), + erlang:raise(error, Exception, Stack); + true -> + erlang:raise(error, undef, Stack) end. -load_form(Line, Forms, Opts, #{file := File} = E) -> - elixir_compiler:module(Forms, File, Opts, fun(Module, Binary0) -> - Docs = elixir_compiler:get_opt(docs), - Binary = add_docs_chunk(Binary0, Module, Line, Docs), - eval_callbacks(Line, Module, after_compile, [E, Binary], E), - - case get(elixir_compiled) of - Current when is_list(Current) -> - put(elixir_compiled, [{Module,Binary}|Current]), - - case get(elixir_compiler_pid) of - undefined -> []; - PID -> - Ref = make_ref(), - PID ! {module_available, self(), Ref, File, Module, Binary}, - receive {Ref, ack} -> ok end - end; - _ -> - [] - end, - - Binary - end). - -add_docs_chunk(Bin, Module, Line, true) -> - ChunkData = term_to_binary({elixir_docs_v1, [ - {docs, get_docs(Module)}, - {moduledoc, get_moduledoc(Line, Module)} - ]}), - add_beam_chunk(Bin, "ExDc", ChunkData); - -add_docs_chunk(Bin, _, _, _) -> Bin. - -get_docs(Module) -> - ordsets:from_list( - [{Tuple, Line, Kind, Sig, Doc} || - {Tuple, Line, Kind, Sig, Doc} <- ets:tab2list(docs_table(Module)), - Kind =/= type, Kind =/= opaque]). - -get_moduledoc(Line, Module) -> - {Line, 'Elixir.Module':get_attribute(Module, moduledoc)}. - +%% Handle reserved modules and duplicates. check_module_availability(Line, File, Module) -> - Reserved = ['Elixir.Any', 'Elixir.BitString', 'Elixir.Function', 'Elixir.PID', + Reserved = ['Elixir.True', 'Elixir.False', 'Elixir.Nil', + 'Elixir.Any', 'Elixir.BitString', 'Elixir.PID', 'Elixir.Reference', 'Elixir.Elixir', 'Elixir'], case lists:member(Module, Reserved) of - true -> elixir_errors:handle_file_error(File, {Line, ?MODULE, {module_reserved, Module}}); + true -> elixir_errors:form_error([{line, Line}], File, ?MODULE, {module_reserved, Module}); false -> ok end, - case elixir_compiler:get_opt(ignore_module_conflict) of + case elixir_config:get(ignore_module_conflict) of false -> case code:ensure_loaded(Module) of {module, _} -> - elixir_errors:handle_file_warning(File, {Line, ?MODULE, {module_defined, Module}}); + elixir_errors:form_warn([{line, Line}], File, ?MODULE, {module_defined, Module}); {error, _} -> ok end; @@ -367,132 +266,269 @@ check_module_availability(Line, File, Module) -> ok end. -warn_invalid_clauses(_Line, _File, 'Elixir.Kernel.SpecialForms', _All) -> ok; -warn_invalid_clauses(_Line, File, Module, All) -> - ets:foldl(fun - ({_, _, Kind, _, _}, _) when Kind == type; Kind == opaque -> - ok; - ({Tuple, Line, _, _, _}, _) -> - case lists:member(Tuple, All) of - false -> - elixir_errors:handle_file_warning(File, {Line, ?MODULE, {invalid_clause, Tuple}}); - true -> - ok - end - end, ok, docs_table(Module)). - -warn_unused_docs(Line, File, Module) -> - lists:foreach(fun(Attribute) -> - case ets:member(data_table(Module), Attribute) of - true -> - elixir_errors:handle_file_warning(File, {Line, ?MODULE, {unused_doc, Attribute}}); - _ -> - ok - end - end, [typedoc]). - -% EXTRA FUNCTIONS - -add_info_function(Line, File, Module, Export, Def, Defmacro) -> - Pair = {'__info__', 1}, - case lists:member(Pair, Export) of - true -> - elixir_errors:form_error(Line, File, ?MODULE, {internal_function_overridden, Pair}); +validate_module_name(Line, File, Module) when Module == nil; is_boolean(Module) -> + elixir_errors:form_error([{line, Line}], File, ?MODULE, {invalid_module, Module}); +validate_module_name(Line, File, Module) -> + Charlist = atom_to_list(Module), + case lists:any(fun(Char) -> (Char =:= $/) or (Char =:= $\\) end, Charlist) of + true -> + elixir_errors:form_error([{line, Line}], File, ?MODULE, {invalid_module, Module}); false -> - { - {attribute, Line, spec, {{'__info__', 1}, - [{type, Line, 'fun', [{type, Line, product, [ {type, Line, atom, []}]}, {type, Line, term, []} ]}] - }}, - {function, 0, '__info__', 1, [ - functions_clause(Def), - macros_clause(Defmacro), - module_clause(Module), - else_clause() - ]} - } + Charlist end. -functions_clause(Def) -> - {clause, 0, [{atom, 0, functions}], [], [elixir_utils:elixir_to_erl(Def)]}. +%% Hook that builds both attribute and functions and set up common hooks. -macros_clause(Defmacro) -> - {clause, 0, [{atom, 0, macros}], [], [elixir_utils:elixir_to_erl(Defmacro)]}. +build(Line, File, Module) -> + %% In the set table we store: + %% + %% * {Attribute, Value, AccumulateOrReadOrUnreadline} + %% * {{elixir, ...}, ...} + %% * {{cache, ...}, ...} + %% * {{function, Tuple}, ...}, {{macro, Tuple}, ...} + %% * {{type, Tuple}, ...}, {{opaque, Tuple}, ...} + %% * {{callback, Tuple}, ...}, {{macrocallback, Tuple}, ...} + %% * {{def, Tuple}, ...} (from elixir_def) + %% * {{import, Tuple}, ...} (from elixir_locals) + %% * {{overridable, Tuple}, ...} (from elixir_overridable) + %% + DataSet = ets:new(Module, [set, public]), + + %% In the bag table we store: + %% + %% * {{accumulate, Attribute}, ...} (includes typespecs) + %% * {warn_attributes, ...} + %% * {impls, ...} + %% * {deprecated, ...} + %% * {persisted_attributes, ...} + %% * {defs, ...} (from elixir_def) + %% * {overridables, ...} (from elixir_overridable) + %% * {{default, Name}, ...} (from elixir_def) + %% * {{clauses, Tuple}, ...} (from elixir_def) + %% * {reattach, ...} (from elixir_locals) + %% * {{local, Tuple}, ...} (from elixir_locals) + %% + DataBag = ets:new(Module, [duplicate_bag, public]), + + ets:insert(DataSet, [ + % {Key, Value, ReadOrUnreadLine} + {moduledoc, nil, nil}, + + % {Key, Value, accumulate} + {after_compile, [], accumulate}, + {before_compile, [], accumulate}, + {behaviour, [], accumulate}, + {compile, [], accumulate}, + {derive, [], accumulate}, + {dialyzer, [], accumulate}, + {external_resource, [], accumulate}, + {on_definition, [], accumulate}, + {type, [], accumulate}, + {opaque, [], accumulate}, + {typep, [], accumulate}, + {spec, [], accumulate}, + {callback, [], accumulate}, + {macrocallback, [], accumulate}, + {optional_callbacks, [], accumulate}, + + % Others + {?counter_attr, 0} + ]), + + Persisted = [behaviour, on_load, external_resource, dialyzer, vsn], + ets:insert(DataBag, [{persisted_attributes, Attr} || Attr <- Persisted]), + + OnDefinition = + case elixir_config:is_bootstrap() of + false -> {'Elixir.Module', compile_definition_attributes}; + _ -> {elixir_module, delete_definition_attributes} + end, + ets:insert(DataBag, {{accumulate, on_definition}, OnDefinition}), + + %% Setup definition related modules + Tables = {DataSet, DataBag}, + elixir_def:setup(Tables), + elixir_locals:setup(Tables), + Tuple = {Module, Tables, Line, File, all}, + + Ref = + case elixir_code_server:call({defmodule, Module, self(), Tuple}) of + {ok, ModuleRef} -> + ModuleRef; + {error, {Module, _, OldLine, OldFile, _}} -> + ets:delete(DataSet), + ets:delete(DataBag), + Error = {module_in_definition, Module, OldFile, OldLine}, + elixir_errors:form_error([{line, Line}], File, ?MODULE, Error) + end, -module_clause(Module) -> - {clause, 0, [{atom, 0, module}], [], [{atom, 0, Module}]}. + {Tables, Ref}. -else_clause() -> - Info = {call, 0, {atom, 0, module_info}, [{var, 0, atom}]}, - {clause, 0, [{var, 0, atom}], [], [Info]}. +%% Handles module and callback evaluations. -% HELPERS +eval_form(Line, Module, DataBag, Block, Vars, E) -> + {Value, EE} = elixir_compiler:compile(Block, Vars, E), + elixir_overridable:store_not_overridden(Module), + EV = (elixir_env:reset_vars(EE))#{line := Line}, + EC = eval_callbacks(Line, DataBag, before_compile, [EV], EV), + elixir_overridable:store_not_overridden(Module), + {Value, EC}. -%% Adds custom chunk to a .beam binary -add_beam_chunk(Bin, Id, ChunkData) - when is_binary(Bin), is_list(Id), is_binary(ChunkData) -> - {ok, _, Chunks0} = beam_lib:all_chunks(Bin), - NewChunk = {Id, ChunkData}, - Chunks = [NewChunk|Chunks0], - {ok, NewBin} = beam_lib:build_module(Chunks), - NewBin. +eval_callbacks(Line, DataBag, Name, Args, E) -> + Callbacks = bag_lookup_element(DataBag, {accumulate, Name}, 2), + lists:foldl(fun({M, F}, Acc) -> + expand_callback(Line, M, F, Args, Acc, fun(AM, AF, AA) -> apply(AM, AF, AA) end) + end, E, Callbacks). -%% Expands a callback given by M:F(Args). In case -%% the callback can't be expanded, invokes the given -%% fun passing a possibly expanded AM:AF(Args). -expand_callback(Line, M, F, Args, E, Fun) -> - Meta = [{line,Line},{require,false}], +expand_callback(Line, M, F, Args, Acc, Fun) -> + %% TODO: Remove this reset_vars once we move variables to S + %% as we can expect all variables to have been previously reset + E = elixir_env:reset_vars(Acc), + S = elixir_env:env_to_ex(E), + Meta = [{line, Line}, {required, true}], - {EE, ET} = elixir_dispatch:dispatch_require(Meta, M, F, Args, E, fun(AM, AF, AA) -> - Fun(AM, AF, AA), - {ok, E} - end), + {EE, _S, ET} = + elixir_dispatch:dispatch_require(Meta, M, F, Args, S, E, fun(AM, AF, AA) -> + Fun(AM, AF, AA), + {ok, S, E} + end), if is_atom(EE) -> ET; true -> try - {_Value, _Binding, EF, _S} = elixir:eval_forms(EE, [], ET), + {_Value, _Binding, EF} = elixir:eval_forms(EE, [], ET), EF catch - Kind:Reason -> + Kind:Reason:Stacktrace -> Info = {M, F, length(Args), location(Line, E)}, - erlang:raise(Kind, Reason, prune_stacktrace(Info, erlang:get_stacktrace())) + erlang:raise(Kind, Reason, prune_stacktrace(Info, Stacktrace)) end end. -location(Line, E) -> - [{file, elixir_utils:characters_to_list(?m(E, file))}, {line, Line}]. +%% Add attributes handling to the form + +attributes(DataSet, DataBag, PersistedAttributes) -> + [{Key, Value} || Key <- PersistedAttributes, Value <- lookup_attribute(DataSet, DataBag, Key)]. + +lookup_attribute(DataSet, DataBag, Key) when is_atom(Key) -> + case ets:lookup(DataSet, Key) of + [{_, _, accumulate}] -> bag_lookup_element(DataBag, {accumulate, Key}, 2); + [{_, _, unset}] -> []; + [{_, Value, _}] -> [Value]; + [] -> [] + end. + +warn_unused_attributes(File, DataSet, DataBag, PersistedAttrs) -> + StoredAttrs = bag_lookup_element(DataBag, warn_attributes, 2), + %% This is the same list as in Module.put_attribute + %% without moduledoc which are never warned on. + Attrs = [doc, typedoc, impl, deprecated | StoredAttrs -- PersistedAttrs], + Query = [{{Attr, '_', '$1'}, [{is_integer, '$1'}], [[Attr, '$1']]} || Attr <- Attrs], + [elixir_errors:form_warn([{line, Line}], File, ?MODULE, {unused_attribute, Key}) + || [Key, Line] <- ets:select(DataSet, Query)]. + +get_struct(Set) -> + case ets:lookup(Set, {elixir, struct}) of + [] -> nil; + [{_, Struct}] -> Struct + end. + +get_deprecated(Bag) -> + lists:usort(bag_lookup_element(Bag, deprecated, 2)). + +bag_lookup_element(Table, Name, Pos) -> + try + ets:lookup_element(Table, Name, Pos) + catch + error:badarg -> [] + end. + +beam_location(ModuleAsCharlist) -> + case get(elixir_compiler_dest) of + Dest when is_binary(Dest) -> + filename:join(elixir_utils:characters_to_list(Dest), ModuleAsCharlist ++ ".beam"); + _ -> + "" + end. + +%% Integration with elixir_compiler that makes the module available + +spawn_parallel_checker(undefined, _Module, _ModuleMap) -> + nil; +spawn_parallel_checker(CheckerInfo, Module, ModuleMap) -> + 'Elixir.Module.ParallelChecker':spawn(CheckerInfo, Module, ModuleMap). + +make_module_available(Module, Binary, CheckerPid) -> + case get(elixir_module_binaries) of + Current when is_list(Current) -> + put(elixir_module_binaries, [{{Module, Binary}, CheckerPid} | Current]); + _ -> + ok + end, + + case get(elixir_compiler_info) of + undefined -> + ok; + {PID, _} -> + Ref = make_ref(), + PID ! {module_available, self(), Ref, get(elixir_compiler_file), Module, Binary, CheckerPid}, + receive {Ref, ack} -> ok end + end. + +%% Error handling and helpers. %% We've reached the elixir_module or eval internals, skip it with the rest -prune_stacktrace(Info, [{elixir, eval_forms, _, _}|_]) -> +prune_stacktrace(Info, [{elixir, eval_forms, _, _} | _]) -> [Info]; -prune_stacktrace(Info, [{elixir_module, _, _, _}|_]) -> +prune_stacktrace(Info, [{elixir_module, _, _, _} | _]) -> [Info]; -prune_stacktrace(Info, [H|T]) -> - [H|prune_stacktrace(Info, T)]; +prune_stacktrace(Info, [H | T]) -> + [H | prune_stacktrace(Info, T)]; prune_stacktrace(Info, []) -> [Info]. -% ERROR HANDLING - -format_error({invalid_clause, {Name, Arity}}) -> - io_lib:format("empty clause provided for nonexistent function or macro ~ts/~B", [Name, Arity]); -format_error({invalid_external_resource, Value}) -> - io_lib:format("expected a string value for @external_resource, got: ~p", - ['Elixir.Kernel':inspect(Value)]); -format_error({unused_doc, typedoc}) -> - "@typedoc provided but no type follows it"; -format_error({unused_doc, doc}) -> - "@doc provided but no definition follows it"; -format_error({internal_function_overridden, {Name, Arity}}) -> - io_lib:format("function ~ts/~B is internal and should not be overridden", [Name, Arity]); +location(Line, E) -> + [{file, elixir_utils:characters_to_list(?key(E, file))}, {line, Line}]. + +format_error({unused_attribute, typedoc}) -> + "module attribute @typedoc was set but no type follows it"; +format_error({unused_attribute, doc}) -> + "module attribute @doc was set but no definition follows it"; +format_error({unused_attribute, impl}) -> + "module attribute @impl was set but no definition follows it"; +format_error({unused_attribute, deprecated}) -> + "module attribute @deprecated was set but no definition follows it"; +format_error({unused_attribute, Attr}) -> + io_lib:format("module attribute @~ts was set but never used", [Attr]); format_error({invalid_module, Module}) -> io_lib:format("invalid module name: ~ts", ['Elixir.Kernel':inspect(Module)]); format_error({module_defined, Module}) -> - io_lib:format("redefining module ~ts", [elixir_aliases:inspect(Module)]); + Extra = + case code:which(Module) of + "" -> + " (current version defined in memory)"; + Path when is_list(Path) -> + io_lib:format(" (current version loaded from ~ts)", [elixir_utils:relative_to_cwd(Path)]); + _ -> + "" + end, + io_lib:format("redefining module ~ts~ts", [elixir_aliases:inspect(Module), Extra]); format_error({module_reserved, Module}) -> io_lib:format("module ~ts is reserved and cannot be defined", [elixir_aliases:inspect(Module)]); format_error({module_in_definition, Module, File, Line}) -> io_lib:format("cannot define module ~ts because it is currently being defined in ~ts:~B", - [elixir_aliases:inspect(Module), 'Elixir.Path':relative_to_cwd(File), Line]). + [elixir_aliases:inspect(Module), elixir_utils:relative_to_cwd(File), Line]); +format_error({bad_inline, {Name, Arity}}) -> + io_lib:format("inlined function ~ts/~B undefined", [Name, Arity]); +format_error({bad_dialyzer, Key, {Name, Arity}}) -> + io_lib:format("undefined function ~ts/~B given to @dialyzer :~ts", [Name, Arity, Key]); +format_error({undefined_on_load, {Name, Arity}}) -> + io_lib:format("undefined function ~ts/~B given to @on_load", [Name, Arity]); +format_error({wrong_kind_on_load, {Name, Arity}, WrongKind}) -> + io_lib:format("expected @on_load function ~ts/~B to be a function, got \"~ts\"", + [Name, Arity, WrongKind]); +format_error({parse_transform, Module}) -> + io_lib:format("@compile {:parse_transform, ~ts} is deprecated. Elixir will no longer support " + "Erlang-based transforms in future versions", [elixir_aliases:inspect(Module)]). diff --git a/lib/elixir/src/elixir_overridable.erl b/lib/elixir/src/elixir_overridable.erl new file mode 100644 index 00000000000..071dcf8b24e --- /dev/null +++ b/lib/elixir/src/elixir_overridable.erl @@ -0,0 +1,133 @@ +% Holds the logic responsible for defining overridable functions and handling super. +-module(elixir_overridable). +-export([overridables_for/1, overridable_for/2, + record_overridable/4, super/4, + store_not_overridden/1, format_error/1]). +-include("elixir.hrl"). +-define(overridden_pos, 5). + +overridables_for(Module) -> + {_, Bag} = elixir_module:data_tables(Module), + try + ets:lookup_element(Bag, overridables, 2) + catch + _:_ -> [] + end. + +overridable_for(Module, Tuple) -> + {Set, _} = elixir_module:data_tables(Module), + + case ets:lookup(Set, {overridable, Tuple}) of + [Overridable] -> Overridable; + [] -> not_overridable + end. + +record_overridable(Module, Tuple, Def, Neighbours) -> + {Set, Bag} = elixir_module:data_tables(Module), + + case ets:insert_new(Set, {{overridable, Tuple}, 1, Def, Neighbours, false}) of + true -> + ets:insert(Bag, {overridables, Tuple}); + false -> + [{_, Count, PreviousDef, _, _}] = ets:lookup(Set, {overridable, Tuple}), + {{_, Kind, Meta, File, _, _}, _} = Def, + {{_, PreviousKind, _, _, _, _}, _} = PreviousDef, + + case is_valid_kind(Kind, PreviousKind) of + true -> + ets:insert(Set, {{overridable, Tuple}, Count + 1, Def, Neighbours, false}); + false -> + elixir_errors:form_error(Meta, File, ?MODULE, {bad_kind, Module, Tuple, Kind}) + end + end, + + ok. + +super(Meta, Module, Tuple, E) -> + {Set, _} = elixir_module:data_tables(Module), + + case ets:lookup(Set, {overridable, Tuple}) of + [Overridable] -> + store(Set, Module, Tuple, Overridable, true); + [] -> + elixir_errors:form_error(Meta, E, ?MODULE, {no_super, Module, Tuple}) + end. + +store_not_overridden(Module) -> + {Set, Bag} = elixir_module:data_tables(Module), + + lists:foreach(fun({_, Tuple}) -> + [Overridable] = ets:lookup(Set, {overridable, Tuple}), + + case ets:lookup(Set, {def, Tuple}) of + [] -> + store(Set, Module, Tuple, Overridable, false); + [{_, Kind, Meta, File, _, _}] -> + {{_, OverridableKind, _, _, _, _}, _} = element(3, Overridable), + + case is_valid_kind(Kind, OverridableKind) of + true -> ok; + false -> elixir_errors:form_error(Meta, File, ?MODULE, {bad_kind, Module, Tuple, Kind}) + end + end + end, ets:lookup(Bag, overridables)). + +%% Private + +store(Set, Module, Tuple, {_, Count, Def, Neighbours, Overridden}, Hidden) -> + {{{def, {Name, Arity}}, Kind, Meta, File, _Check, + {Defaults, _HasBody, _LastDefaults}}, Clauses} = Def, + + {FinalKind, FinalName, FinalArity, FinalClauses} = + case Hidden of + false -> + {Kind, Name, Arity, Clauses}; + true when Kind == defmacro; Kind == defmacrop -> + {defmacrop, name(Name, Count), Arity, Clauses}; + true -> + {defp, name(Name, Count), Arity, Clauses} + end, + + case Overridden of + false -> + ets:update_element(Set, {overridable, Tuple}, {?overridden_pos, true}), + elixir_def:store_definition(false, FinalKind, Meta, FinalName, FinalArity, + File, Module, Defaults, FinalClauses), + elixir_locals:reattach({FinalName, FinalArity}, FinalKind, Module, Tuple, Neighbours, Meta); + true -> + ok + end, + + {FinalKind, FinalName, Meta}. + +name(Name, Count) when is_integer(Count) -> + list_to_atom(atom_to_list(Name) ++ " (overridable " ++ integer_to_list(Count) ++ ")"). + +is_valid_kind(NewKind, PreviousKind) -> + is_macro(NewKind) =:= is_macro(PreviousKind). + +is_macro(defmacro) -> true; +is_macro(defmacrop) -> true; +is_macro(_) -> false. + +%% Error handling +format_error({bad_kind, Module, {Name, Arity}, Kind}) -> + case is_macro(Kind) of + true -> + io_lib:format("cannot override function (def, defp) ~ts/~B in module ~ts as a macro (defmacro, defmacrop)", + [Name, Arity, elixir_aliases:inspect(Module)]); + false -> + io_lib:format("cannot override macro (defmacro, defmacrop) ~ts/~B in module ~ts as a function (def, defp)", + [Name, Arity, elixir_aliases:inspect(Module)]) + end; + +format_error({no_super, Module, {Name, Arity}}) -> + Bins = [format_fa(Tuple) || Tuple <- overridables_for(Module)], + Joined = 'Elixir.Enum':join(Bins, <<", ">>), + io_lib:format("no super defined for ~ts/~B in module ~ts. Overridable functions available are: ~ts", + [Name, Arity, elixir_aliases:inspect(Module), Joined]). + +format_fa({Name, Arity}) -> + A = 'Elixir.Macro':inspect_atom(remote_call, Name), + B = integer_to_binary(Arity), + <>. diff --git a/lib/elixir/src/elixir_parser.yrl b/lib/elixir/src/elixir_parser.yrl index e8d4c9f45aa..b18e9576dfa 100644 --- a/lib/elixir/src/elixir_parser.yrl +++ b/lib/elixir/src/elixir_parser.yrl @@ -1,69 +1,83 @@ Nonterminals grammar expr_list - expr container_expr block_expr no_parens_expr no_parens_one_expr access_expr - bracket_expr bracket_at_expr bracket_arg matched_expr unmatched_expr max_expr - op_expr matched_op_expr no_parens_op_expr + expr container_expr block_expr access_expr + no_parens_expr no_parens_zero_expr no_parens_one_expr no_parens_one_ambig_expr + bracket_expr bracket_at_expr bracket_arg matched_expr unmatched_expr + unmatched_op_expr matched_op_expr no_parens_op_expr no_parens_many_expr comp_op_eol at_op_eol unary_op_eol and_op_eol or_op_eol capture_op_eol - add_op_eol mult_op_eol hat_op_eol two_op_eol pipe_op_eol stab_op_eol - arrow_op_eol match_op_eol when_op_eol in_op_eol in_match_op_eol - type_op_eol rel_op_eol - open_paren close_paren empty_paren + dual_op_eol mult_op_eol power_op_eol concat_op_eol xor_op_eol pipe_op_eol + stab_op_eol arrow_op_eol match_op_eol when_op_eol in_op_eol in_match_op_eol + type_op_eol rel_op_eol range_op_eol ternary_op_eol + open_paren close_paren empty_paren eoe list list_args open_bracket close_bracket tuple open_curly close_curly bit_string open_bit close_bit - map map_op map_close map_args map_expr struct_op + map map_op map_close map_args struct_expr struct_op assoc_op_eol assoc_expr assoc_base assoc_update assoc_update_kw assoc - container_args_base container_args call_args_parens_base call_args_parens parens_call - call_args_no_parens_one call_args_no_parens_expr call_args_no_parens_comma_expr - call_args_no_parens_all call_args_no_parens_many call_args_no_parens_many_strict - stab stab_eol stab_expr stab_maybe_expr stab_parens_many - kw_eol kw_base kw call_args_no_parens_kw_expr call_args_no_parens_kw - dot_op dot_alias dot_identifier dot_op_identifier dot_do_identifier - dot_paren_identifier dot_bracket_identifier - do_block fn_eol do_eol end_eol block_eol block_item block_list + container_args_base container_args + call_args_parens_expr call_args_parens_base call_args_parens parens_call + call_args_no_parens_one call_args_no_parens_ambig call_args_no_parens_expr + call_args_no_parens_comma_expr call_args_no_parens_all call_args_no_parens_many + call_args_no_parens_many_strict + stab stab_eoe stab_expr stab_op_eol_and_expr stab_parens_many + kw_eol kw_base kw_data kw_call call_args_no_parens_kw_expr call_args_no_parens_kw + dot_op dot_alias dot_bracket_identifier dot_call_identifier + dot_identifier dot_op_identifier dot_do_identifier dot_paren_identifier + do_block fn_eoe do_eoe end_eoe block_eoe block_item block_list . Terminals identifier kw_identifier kw_identifier_safe kw_identifier_unsafe bracket_identifier - paren_identifier do_identifier block_identifier - fn 'end' aliases - number signed_number atom atom_safe atom_unsafe bin_string list_string sigil - dot_call_op op_identifier + paren_identifier do_identifier block_identifier op_identifier + fn 'end' alias + atom atom_quoted atom_safe atom_unsafe bin_string list_string sigil + bin_heredoc list_heredoc comp_op at_op unary_op and_op or_op arrow_op match_op in_op in_match_op - type_op dual_op add_op mult_op hat_op two_op pipe_op stab_op when_op assoc_op - capture_op rel_op - 'true' 'false' 'nil' 'do' eol ',' '.' + type_op dual_op mult_op power_op concat_op range_op xor_op pipe_op stab_op when_op + assoc_op capture_op rel_op ternary_op dot_call_op + 'true' 'false' 'nil' 'do' eol ';' ',' '.' '(' ')' '[' ']' '{' '}' '<<' '>>' '%{}' '%' + int flt char . Rootsymbol grammar. -%% There are two shift/reduce conflicts coming from call_args_parens. -Expect 2. +%% Two shift/reduce conflicts coming from call_args_parens and +%% one coming from empty_paren on stab. +Expect 3. -%% Changes in ops and precedence should be reflected on lib/elixir/lib/macro.ex -%% Note though the operator => in practice has lower precedence than all others, -%% its entry in the table is only to support the %{user | foo => bar} syntax. +%% Changes in ops and precedence should be reflected on: +%% +%% 1. lib/elixir/lib/code/identifier.ex +%% 2. lib/elixir/pages/operators.md +%% 3. lib/iex/lib/iex/evaluator.ex +%% +%% Note though the operator => in practice has lower precedence +%% than all others, its entry in the table is only to support the +%% %{user | foo => bar} syntax. Left 5 do. Right 10 stab_op_eol. %% -> Left 20 ','. -Nonassoc 30 capture_op_eol. %% & Left 40 in_match_op_eol. %% <-, \\ (allowed in matches along =) Right 50 when_op_eol. %% when Right 60 type_op_eol. %% :: Right 70 pipe_op_eol. %% | Right 80 assoc_op_eol. %% => -Right 90 match_op_eol. %% = -Left 130 or_op_eol. %% ||, |||, or, xor -Left 140 and_op_eol. %% &&, &&&, and -Left 150 comp_op_eol. %% ==, !=, =~, ===, !== -Left 160 rel_op_eol. %% <, >, <=, >= -Left 170 arrow_op_eol. %% < (op), (op) > (e.g |>, <<<, >>>) -Left 180 in_op_eol. %% in -Right 200 two_op_eol. %% ++, --, .., <> -Left 210 add_op_eol. %% + (op), - (op) -Left 220 mult_op_eol. %% * (op), / (op) -Left 250 hat_op_eol. %% ^ (op) (e.g ^^^) +Nonassoc 90 capture_op_eol. %% & +Right 100 match_op_eol. %% = +Left 110 or_op_eol. %% ||, |||, or +Left 130 and_op_eol. %% &&, &&&, and +Left 140 comp_op_eol. %% ==, !=, =~, ===, !== +Left 150 rel_op_eol. %% <, >, <=, >= +Left 160 arrow_op_eol. %% |>, <<<, >>>, <<~, ~>>, <~, ~>, <~>, <|> +Left 170 in_op_eol. %% in, not in +Left 180 xor_op_eol. %% ^^^ +Right 190 ternary_op_eol. %% // +Right 200 concat_op_eol. %% ++, --, +++, ---, <> +Right 200 range_op_eol. %% .. +Left 210 dual_op_eol. %% +, - +Left 220 mult_op_eol. %% *, / +Left 230 power_op_eol. %% ** Nonassoc 300 unary_op_eol. %% +, -, !, ^, not, ~~~ Left 310 dot_call_op. Left 310 dot_op. %% . @@ -72,18 +86,17 @@ Nonassoc 330 dot_identifier. %%% MAIN FLOW OF EXPRESSIONS -grammar -> eol : nil. -grammar -> expr_list : to_block('$1'). -grammar -> eol expr_list : to_block('$2'). -grammar -> expr_list eol : to_block('$1'). -grammar -> eol expr_list eol : to_block('$2'). -grammar -> '$empty' : nil. +grammar -> eoe : {'__block__', meta_from_token('$1'), []}. +grammar -> expr_list : build_block(reverse('$1')). +grammar -> eoe expr_list : build_block(reverse('$2')). +grammar -> expr_list eoe : build_block(reverse('$1')). +grammar -> eoe expr_list eoe : build_block(reverse('$2')). +grammar -> '$empty' : {'__block__', [], []}. % Note expressions are on reverse order expr_list -> expr : ['$1']. -expr_list -> expr_list eol expr : ['$3'|'$1']. +expr_list -> expr_list eoe expr : ['$3' | annotate_eoe('$2', '$1')]. -expr -> empty_paren : nil. expr -> matched_expr : '$1'. expr -> no_parens_expr : '$1'. expr -> unmatched_expr : '$1'. @@ -92,6 +105,23 @@ expr -> unmatched_expr : '$1'. %% without parentheses and with do blocks. They are represented %% in the AST as matched, no_parens and unmatched. %% +%% Calls without parentheses are further divided according to how +%% problematic they are: +%% +%% (a) no_parens_one: a call with one unproblematic argument +%% (for example, `f a` or `f g a` and similar) (includes unary operators) +%% +%% (b) no_parens_many: a call with several arguments (for example, `f a, b`) +%% +%% (c) no_parens_one_ambig: a call with one argument which is +%% itself a no_parens_many or no_parens_one_ambig (for example, `f g a, b`, +%% `f g h a, b` and similar) +%% +%% Note, in particular, that no_parens_one_ambig expressions are +%% ambiguous and are interpreted such that the outer function has +%% arity 1. For instance, `f g a, b` is interpreted as `f(g(a, b))` rather +%% than `f(g(a), b)`. Hence the name, no_parens_one_ambig. +%% %% The distinction is required because we can't, for example, have %% a function call with a do block as argument inside another do %% block call, unless there are parentheses: @@ -104,270 +134,329 @@ expr -> unmatched_expr : '$1'. %% %% foo a, bar b, c #=> invalid %% foo(a, bar b, c) #=> invalid -%% foo a, bar b #=> valid +%% foo bar a, b #=> valid %% foo a, bar(b, c) #=> valid %% %% So the different grammar rules need to take into account %% if calls without parentheses are do blocks in particular %% segments and act accordingly. -matched_expr -> matched_expr matched_op_expr : build_op(element(1, '$2'), '$1', element(2, '$2')). -matched_expr -> matched_expr no_parens_op_expr : build_op(element(1, '$2'), '$1', element(2, '$2')). +matched_expr -> matched_expr matched_op_expr : build_op('$1', '$2'). matched_expr -> unary_op_eol matched_expr : build_unary_op('$1', '$2'). -matched_expr -> unary_op_eol no_parens_expr : build_unary_op('$1', '$2'). matched_expr -> at_op_eol matched_expr : build_unary_op('$1', '$2'). -matched_expr -> at_op_eol no_parens_expr : build_unary_op('$1', '$2'). matched_expr -> capture_op_eol matched_expr : build_unary_op('$1', '$2'). -matched_expr -> capture_op_eol no_parens_expr : build_unary_op('$1', '$2'). matched_expr -> no_parens_one_expr : '$1'. +matched_expr -> no_parens_zero_expr : '$1'. matched_expr -> access_expr : '$1'. - -no_parens_expr -> dot_op_identifier call_args_no_parens_many_strict : build_identifier('$1', '$2'). -no_parens_expr -> dot_identifier call_args_no_parens_many_strict : build_identifier('$1', '$2'). - -unmatched_expr -> empty_paren op_expr : build_op(element(1, '$2'), nil, element(2, '$2')). -unmatched_expr -> matched_expr op_expr : build_op(element(1, '$2'), '$1', element(2, '$2')). -unmatched_expr -> unmatched_expr op_expr : build_op(element(1, '$2'), '$1', element(2, '$2')). +matched_expr -> access_expr kw_identifier : error_invalid_kw_identifier('$2'). + +unmatched_expr -> matched_expr unmatched_op_expr : build_op('$1', '$2'). +unmatched_expr -> unmatched_expr matched_op_expr : build_op('$1', '$2'). +unmatched_expr -> unmatched_expr unmatched_op_expr : build_op('$1', '$2'). +%% TODO: this rule should not be here as it allows [foo do end + foo 1, 2] +%% while it should raise. But the parser raises ambiguity errors if we move +%% it to no_parens_op_expr +unmatched_expr -> unmatched_expr no_parens_op_expr : build_op('$1', '$2'). unmatched_expr -> unary_op_eol expr : build_unary_op('$1', '$2'). unmatched_expr -> at_op_eol expr : build_unary_op('$1', '$2'). unmatched_expr -> capture_op_eol expr : build_unary_op('$1', '$2'). unmatched_expr -> block_expr : '$1'. -block_expr -> parens_call call_args_parens do_block : build_identifier('$1', '$2' ++ '$3'). -block_expr -> parens_call call_args_parens call_args_parens do_block : build_nested_parens('$1', '$2', '$3' ++ '$4'). -block_expr -> dot_do_identifier do_block : build_identifier('$1', '$2'). -block_expr -> dot_identifier call_args_no_parens_all do_block : build_identifier('$1', '$2' ++ '$3'). - -op_expr -> match_op_eol expr : {'$1', '$2'}. -op_expr -> add_op_eol expr : {'$1', '$2'}. -op_expr -> mult_op_eol expr : {'$1', '$2'}. -op_expr -> hat_op_eol expr : {'$1', '$2'}. -op_expr -> two_op_eol expr : {'$1', '$2'}. -op_expr -> and_op_eol expr : {'$1', '$2'}. -op_expr -> or_op_eol expr : {'$1', '$2'}. -op_expr -> in_op_eol expr : {'$1', '$2'}. -op_expr -> in_match_op_eol expr : {'$1', '$2'}. -op_expr -> type_op_eol expr : {'$1', '$2'}. -op_expr -> when_op_eol expr : {'$1', '$2'}. -op_expr -> pipe_op_eol expr : {'$1', '$2'}. -op_expr -> comp_op_eol expr : {'$1', '$2'}. -op_expr -> rel_op_eol expr : {'$1', '$2'}. -op_expr -> arrow_op_eol expr : {'$1', '$2'}. +no_parens_expr -> matched_expr no_parens_op_expr : build_op('$1', '$2'). +no_parens_expr -> unary_op_eol no_parens_expr : build_unary_op('$1', '$2'). +no_parens_expr -> at_op_eol no_parens_expr : build_unary_op('$1', '$2'). +no_parens_expr -> capture_op_eol no_parens_expr : build_unary_op('$1', '$2'). +no_parens_expr -> no_parens_one_ambig_expr : '$1'. +no_parens_expr -> no_parens_many_expr : '$1'. + +block_expr -> dot_call_identifier call_args_parens do_block : build_parens('$1', '$2', '$3'). +block_expr -> dot_call_identifier call_args_parens call_args_parens do_block : build_nested_parens('$1', '$2', '$3', '$4'). +block_expr -> dot_do_identifier do_block : build_no_parens_do_block('$1', [], '$2'). +block_expr -> dot_op_identifier call_args_no_parens_all do_block : build_no_parens_do_block('$1', '$2', '$3'). +block_expr -> dot_identifier call_args_no_parens_all do_block : build_no_parens_do_block('$1', '$2', '$3'). + +matched_op_expr -> match_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> dual_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> mult_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> power_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> concat_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> range_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> ternary_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> xor_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> and_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> or_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> in_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> in_match_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> type_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> when_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> pipe_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> comp_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> rel_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> arrow_op_eol matched_expr : {'$1', '$2'}. +matched_op_expr -> arrow_op_eol no_parens_one_expr : warn_pipe('$1', '$2'), {'$1', '$2'}. + +unmatched_op_expr -> match_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> dual_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> mult_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> power_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> concat_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> range_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> ternary_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> xor_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> and_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> or_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> in_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> in_match_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> type_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> when_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> pipe_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> comp_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> rel_op_eol unmatched_expr : {'$1', '$2'}. +unmatched_op_expr -> arrow_op_eol unmatched_expr : {'$1', '$2'}. no_parens_op_expr -> match_op_eol no_parens_expr : {'$1', '$2'}. -no_parens_op_expr -> add_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> dual_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> mult_op_eol no_parens_expr : {'$1', '$2'}. -no_parens_op_expr -> hat_op_eol no_parens_expr : {'$1', '$2'}. -no_parens_op_expr -> two_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> power_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> concat_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> range_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> ternary_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> xor_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> and_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> or_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> in_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> in_match_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> type_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> when_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> pipe_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> comp_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> rel_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> arrow_op_eol no_parens_expr : {'$1', '$2'}. +no_parens_op_expr -> arrow_op_eol no_parens_one_ambig_expr : warn_pipe('$1', '$2'), {'$1', '$2'}. +no_parens_op_expr -> arrow_op_eol no_parens_many_expr : warn_pipe('$1', '$2'), {'$1', '$2'}. %% Allow when (and only when) with keywords -no_parens_op_expr -> when_op_eol no_parens_expr : {'$1', '$2'}. no_parens_op_expr -> when_op_eol call_args_no_parens_kw : {'$1', '$2'}. -matched_op_expr -> match_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> add_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> mult_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> hat_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> two_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> and_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> or_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> in_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> in_match_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> type_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> when_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> pipe_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> comp_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> rel_op_eol matched_expr : {'$1', '$2'}. -matched_op_expr -> arrow_op_eol matched_expr : {'$1', '$2'}. +no_parens_one_ambig_expr -> dot_op_identifier call_args_no_parens_ambig : build_no_parens('$1', '$2'). +no_parens_one_ambig_expr -> dot_identifier call_args_no_parens_ambig : build_no_parens('$1', '$2'). + +no_parens_many_expr -> dot_op_identifier call_args_no_parens_many_strict : build_no_parens('$1', '$2'). +no_parens_many_expr -> dot_identifier call_args_no_parens_many_strict : build_no_parens('$1', '$2'). -no_parens_one_expr -> dot_op_identifier call_args_no_parens_one : build_identifier('$1', '$2'). -no_parens_one_expr -> dot_identifier call_args_no_parens_one : build_identifier('$1', '$2'). -no_parens_one_expr -> dot_do_identifier : build_identifier('$1', nil). -no_parens_one_expr -> dot_identifier : build_identifier('$1', nil). +no_parens_one_expr -> dot_op_identifier call_args_no_parens_one : build_no_parens('$1', '$2'). +no_parens_one_expr -> dot_identifier call_args_no_parens_one : build_no_parens('$1', '$2'). +no_parens_zero_expr -> dot_do_identifier : build_no_parens('$1', nil). +no_parens_zero_expr -> dot_identifier : build_no_parens('$1', nil). %% From this point on, we just have constructs that can be -%% used with the access syntax. Notice that (dot_)identifier +%% used with the access syntax. Note that (dot_)identifier %% is not included in this list simply because the tokenizer %% marks identifiers followed by brackets as bracket_identifier. access_expr -> bracket_at_expr : '$1'. access_expr -> bracket_expr : '$1'. -access_expr -> at_op_eol number : build_unary_op('$1', ?exprs('$2')). -access_expr -> unary_op_eol number : build_unary_op('$1', ?exprs('$2')). -access_expr -> capture_op_eol number : build_unary_op('$1', ?exprs('$2')). -access_expr -> fn_eol stab end_eol : build_fn('$1', build_stab(reverse('$2'))). -access_expr -> open_paren stab close_paren : build_stab(reverse('$2')). -access_expr -> number : ?exprs('$1'). -access_expr -> signed_number : {element(4, '$1'), meta('$1'), ?exprs('$1')}. +access_expr -> capture_op_eol int : build_unary_op('$1', number_value('$2')). +access_expr -> fn_eoe stab end_eoe : build_fn('$1', '$2', '$3'). +access_expr -> open_paren stab close_paren : build_stab('$1', '$2', '$3'). +access_expr -> open_paren stab ';' close_paren : build_stab('$1', '$2', '$4'). +access_expr -> open_paren ';' stab ';' close_paren : build_stab('$1', '$3', '$5'). +access_expr -> open_paren ';' stab close_paren : build_stab('$1', '$3', '$4'). +access_expr -> open_paren ';' close_paren : build_stab('$1', [], '$3'). +access_expr -> empty_paren : warn_empty_paren('$1'), {'__block__', [], []}. +access_expr -> int : handle_number(number_value('$1'), '$1', ?exprs('$1')). +access_expr -> flt : handle_number(number_value('$1'), '$1', ?exprs('$1')). +access_expr -> char : handle_number(?exprs('$1'), '$1', number_value('$1')). access_expr -> list : element(1, '$1'). access_expr -> map : '$1'. access_expr -> tuple : '$1'. -access_expr -> 'true' : ?id('$1'). -access_expr -> 'false' : ?id('$1'). -access_expr -> 'nil' : ?id('$1'). -access_expr -> bin_string : build_bin_string('$1'). -access_expr -> list_string : build_list_string('$1'). +access_expr -> 'true' : handle_literal(?id('$1'), '$1'). +access_expr -> 'false' : handle_literal(?id('$1'), '$1'). +access_expr -> 'nil' : handle_literal(?id('$1'), '$1'). +access_expr -> bin_string : build_bin_string('$1', delimiter(<<$">>)). +access_expr -> list_string : build_list_string('$1', delimiter(<<$'>>)). +access_expr -> bin_heredoc : build_bin_heredoc('$1'). +access_expr -> list_heredoc : build_list_heredoc('$1'). access_expr -> bit_string : '$1'. access_expr -> sigil : build_sigil('$1'). -access_expr -> max_expr : '$1'. - -%% Aliases and properly formed calls. Used by map_expr. -max_expr -> atom : ?exprs('$1'). -max_expr -> atom_safe : build_quoted_atom('$1', true). -max_expr -> atom_unsafe : build_quoted_atom('$1', false). -max_expr -> parens_call call_args_parens : build_identifier('$1', '$2'). -max_expr -> parens_call call_args_parens call_args_parens : build_nested_parens('$1', '$2', '$3'). -max_expr -> dot_alias : '$1'. - -bracket_arg -> open_bracket ']' : build_list('$1', []). -bracket_arg -> open_bracket kw close_bracket : build_list('$1', '$2'). -bracket_arg -> open_bracket container_expr close_bracket : build_list('$1', '$2'). -bracket_arg -> open_bracket container_expr ',' close_bracket : build_list('$1', '$2'). - -bracket_expr -> dot_bracket_identifier bracket_arg : build_access(build_identifier('$1', nil), '$2'). +access_expr -> atom : handle_literal(?exprs('$1'), '$1'). +access_expr -> atom_quoted : handle_literal(?exprs('$1'), '$1', delimiter(<<$">>)). +access_expr -> atom_safe : build_quoted_atom('$1', true, delimiter(<<$">>)). +access_expr -> atom_unsafe : build_quoted_atom('$1', false, delimiter(<<$">>)). +access_expr -> dot_alias : '$1'. +access_expr -> parens_call : '$1'. +access_expr -> range_op : build_nullary_op('$1'). + +%% Also used by maps and structs +parens_call -> dot_call_identifier call_args_parens : build_parens('$1', '$2', {[], []}). +parens_call -> dot_call_identifier call_args_parens call_args_parens : build_nested_parens('$1', '$2', '$3', {[], []}). + +bracket_arg -> open_bracket kw_data close_bracket : build_access_arg('$1', '$2', '$3'). +bracket_arg -> open_bracket container_expr close_bracket : build_access_arg('$1', '$2', '$3'). +bracket_arg -> open_bracket container_expr ',' close_bracket : build_access_arg('$1', '$2', '$4'). + +bracket_expr -> dot_bracket_identifier bracket_arg : build_access(build_no_parens('$1', nil), '$2'). bracket_expr -> access_expr bracket_arg : build_access('$1', '$2'). bracket_at_expr -> at_op_eol dot_bracket_identifier bracket_arg : - build_access(build_unary_op('$1', build_identifier('$2', nil)), '$3'). + build_access(build_unary_op('$1', build_no_parens('$2', nil)), '$3'). bracket_at_expr -> at_op_eol access_expr bracket_arg : build_access(build_unary_op('$1', '$2'), '$3'). %% Blocks -do_block -> do_eol 'end' : [[{do,nil}]]. -do_block -> do_eol stab end_eol : [[{do, build_stab(reverse('$2'))}]]. -do_block -> do_eol block_list 'end' : [[{do, nil}|'$2']]. -do_block -> do_eol stab_eol block_list 'end' : [[{do, build_stab(reverse('$2'))}|'$3']]. +do_block -> do_eoe 'end' : + {do_end_meta('$1', '$2'), [[{handle_literal(do, '$1'), {'__block__', [], []}}]]}. +do_block -> do_eoe stab end_eoe : + {do_end_meta('$1', '$3'), [[{handle_literal(do, '$1'), build_stab('$2')}]]}. +do_block -> do_eoe block_list 'end' : + {do_end_meta('$1', '$3'), [[{handle_literal(do, '$1'), {'__block__', [], []}} | '$2']]}. +do_block -> do_eoe stab_eoe block_list 'end' : + {do_end_meta('$1', '$4'), [[{handle_literal(do, '$1'), build_stab('$2')} | '$3']]}. -fn_eol -> 'fn' : '$1'. -fn_eol -> 'fn' eol : '$1'. +eoe -> eol : '$1'. +eoe -> ';' : '$1'. +eoe -> eol ';' : '$1'. -do_eol -> 'do' : '$1'. -do_eol -> 'do' eol : '$1'. +fn_eoe -> 'fn' : '$1'. +fn_eoe -> 'fn' eoe : next_is_eol('$1', '$2'). -end_eol -> 'end' : '$1'. -end_eol -> eol 'end' : '$2'. +do_eoe -> 'do' : '$1'. +do_eoe -> 'do' eoe : '$1'. -block_eol -> block_identifier : '$1'. -block_eol -> block_identifier eol : '$1'. - -stab -> stab_expr : ['$1']. -stab -> stab eol stab_expr : ['$3'|'$1']. +end_eoe -> 'end' : '$1'. +end_eoe -> eoe 'end' : '$2'. -stab_eol -> stab : '$1'. -stab_eol -> stab eol : '$1'. +block_eoe -> block_identifier : '$1'. +block_eoe -> block_identifier eoe : '$1'. -stab_expr -> expr : '$1'. -stab_expr -> stab_op_eol stab_maybe_expr : build_op('$1', [], '$2'). -stab_expr -> call_args_no_parens_all stab_op_eol stab_maybe_expr : - build_op('$2', unwrap_when(unwrap_splice('$1')), '$3'). -stab_expr -> stab_parens_many stab_op_eol stab_maybe_expr : - build_op('$2', unwrap_splice('$1'), '$3'). -stab_expr -> stab_parens_many when_op expr stab_op_eol stab_maybe_expr : - build_op('$4', [{'when', meta('$2'), unwrap_splice('$1') ++ ['$3']}], '$5'). - -stab_maybe_expr -> 'expr' : '$1'. -stab_maybe_expr -> '$empty' : nil. - -block_item -> block_eol stab_eol : {?exprs('$1'), build_stab(reverse('$2'))}. -block_item -> block_eol : {?exprs('$1'), nil}. +stab -> stab_expr : ['$1']. +stab -> stab eoe stab_expr : ['$3' | annotate_eoe('$2', '$1')]. + +stab_eoe -> stab : '$1'. +stab_eoe -> stab eoe : '$1'. + +%% Here, `element(1, Token)` is the stab operator, +%% while `element(2, Token)` is the expression. +stab_expr -> expr : + '$1'. +stab_expr -> stab_op_eol_and_expr : + build_op([], '$1'). +stab_expr -> empty_paren stab_op_eol_and_expr : + build_op([], '$2'). +stab_expr -> empty_paren when_op expr stab_op_eol_and_expr : + build_op([{'when', meta_from_token('$2'), ['$3']}], '$4'). +stab_expr -> call_args_no_parens_all stab_op_eol_and_expr : + build_op(unwrap_when(unwrap_splice('$1')), '$2'). +stab_expr -> stab_parens_many stab_op_eol_and_expr : + build_op(unwrap_splice('$1'), '$2'). +stab_expr -> stab_parens_many when_op expr stab_op_eol_and_expr : + build_op([{'when', meta_from_token('$2'), unwrap_splice('$1') ++ ['$3']}], '$4'). + +stab_op_eol_and_expr -> stab_op_eol expr : {'$1', '$2'}. +stab_op_eol_and_expr -> stab_op_eol : warn_empty_stab_clause('$1'), {'$1', handle_literal(nil, '$1')}. + +block_item -> block_eoe stab_eoe : + {handle_literal(?exprs('$1'), '$1'), build_stab('$2')}. +block_item -> block_eoe : + {handle_literal(?exprs('$1'), '$1'), {'__block__', [], []}}. block_list -> block_item : ['$1']. -block_list -> block_item block_list : ['$1'|'$2']. +block_list -> block_item block_list : ['$1' | '$2']. %% Helpers open_paren -> '(' : '$1'. -open_paren -> '(' eol : '$1'. +open_paren -> '(' eol : next_is_eol('$1', '$2'). close_paren -> ')' : '$1'. close_paren -> eol ')' : '$2'. empty_paren -> open_paren ')' : '$1'. -open_bracket -> '[' : '$1'. -open_bracket -> '[' eol : '$1'. -close_bracket -> ']' : '$1'. -close_bracket -> eol ']' : '$2'. +open_bracket -> '[' : '$1'. +open_bracket -> '[' eol : next_is_eol('$1', '$2'). +close_bracket -> ']' : '$1'. +close_bracket -> eol ']' : '$2'. -open_bit -> '<<' : '$1'. -open_bit -> '<<' eol : '$1'. -close_bit -> '>>' : '$1'. -close_bit -> eol '>>' : '$2'. +open_bit -> '<<' : '$1'. +open_bit -> '<<' eol : next_is_eol('$1', '$2'). +close_bit -> '>>' : '$1'. +close_bit -> eol '>>' : '$2'. open_curly -> '{' : '$1'. -open_curly -> '{' eol : '$1'. -close_curly -> '}' : '$1'. -close_curly -> eol '}' : '$2'. +open_curly -> '{' eol : next_is_eol('$1', '$2'). +close_curly -> '}' : '$1'. +close_curly -> eol '}' : '$2'. % Operators -add_op_eol -> add_op : '$1'. -add_op_eol -> add_op eol : '$1'. -add_op_eol -> dual_op : '$1'. -add_op_eol -> dual_op eol : '$1'. +unary_op_eol -> unary_op : '$1'. +unary_op_eol -> unary_op eol : '$1'. +unary_op_eol -> dual_op : '$1'. +unary_op_eol -> dual_op eol : '$1'. +unary_op_eol -> ternary_op : '$1'. +unary_op_eol -> ternary_op eol : '$1'. + +capture_op_eol -> capture_op : '$1'. +capture_op_eol -> capture_op eol : '$1'. + +at_op_eol -> at_op : '$1'. +at_op_eol -> at_op eol : '$1'. + +match_op_eol -> match_op : '$1'. +match_op_eol -> match_op eol : '$1'. + +dual_op_eol -> dual_op : '$1'. +dual_op_eol -> dual_op eol : next_is_eol('$1', '$2'). mult_op_eol -> mult_op : '$1'. -mult_op_eol -> mult_op eol : '$1'. +mult_op_eol -> mult_op eol : next_is_eol('$1', '$2'). -hat_op_eol -> hat_op : '$1'. -hat_op_eol -> hat_op eol : '$1'. +power_op_eol -> power_op : '$1'. +power_op_eol -> power_op eol : next_is_eol('$1', '$2'). -two_op_eol -> two_op : '$1'. -two_op_eol -> two_op eol : '$1'. +concat_op_eol -> concat_op : '$1'. +concat_op_eol -> concat_op eol : next_is_eol('$1', '$2'). -pipe_op_eol -> pipe_op : '$1'. -pipe_op_eol -> pipe_op eol : '$1'. +range_op_eol -> range_op : '$1'. +range_op_eol -> range_op eol : next_is_eol('$1', '$2'). -capture_op_eol -> capture_op : '$1'. -capture_op_eol -> capture_op eol : '$1'. +ternary_op_eol -> ternary_op : '$1'. +ternary_op_eol -> ternary_op eol : next_is_eol('$1', '$2'). -unary_op_eol -> unary_op : '$1'. -unary_op_eol -> unary_op eol : '$1'. -unary_op_eol -> dual_op : '$1'. -unary_op_eol -> dual_op eol : '$1'. +xor_op_eol -> xor_op : '$1'. +xor_op_eol -> xor_op eol : next_is_eol('$1', '$2'). -match_op_eol -> match_op : '$1'. -match_op_eol -> match_op eol : '$1'. +pipe_op_eol -> pipe_op : '$1'. +pipe_op_eol -> pipe_op eol : next_is_eol('$1', '$2'). and_op_eol -> and_op : '$1'. -and_op_eol -> and_op eol : '$1'. +and_op_eol -> and_op eol : next_is_eol('$1', '$2'). or_op_eol -> or_op : '$1'. -or_op_eol -> or_op eol : '$1'. +or_op_eol -> or_op eol : next_is_eol('$1', '$2'). in_op_eol -> in_op : '$1'. -in_op_eol -> in_op eol : '$1'. +in_op_eol -> in_op eol : next_is_eol('$1', '$2'). in_match_op_eol -> in_match_op : '$1'. -in_match_op_eol -> in_match_op eol : '$1'. +in_match_op_eol -> in_match_op eol : next_is_eol('$1', '$2'). type_op_eol -> type_op : '$1'. -type_op_eol -> type_op eol : '$1'. +type_op_eol -> type_op eol : next_is_eol('$1', '$2'). when_op_eol -> when_op : '$1'. -when_op_eol -> when_op eol : '$1'. +when_op_eol -> when_op eol : next_is_eol('$1', '$2'). stab_op_eol -> stab_op : '$1'. -stab_op_eol -> stab_op eol : '$1'. - -at_op_eol -> at_op : '$1'. -at_op_eol -> at_op eol : '$1'. +stab_op_eol -> stab_op eol : next_is_eol('$1', '$2'). comp_op_eol -> comp_op : '$1'. -comp_op_eol -> comp_op eol : '$1'. +comp_op_eol -> comp_op eol : next_is_eol('$1', '$2'). rel_op_eol -> rel_op : '$1'. -rel_op_eol -> rel_op eol : '$1'. +rel_op_eol -> rel_op eol : next_is_eol('$1', '$2'). arrow_op_eol -> arrow_op : '$1'. -arrow_op_eol -> arrow_op eol : '$1'. +arrow_op_eol -> arrow_op eol : next_is_eol('$1', '$2'). % Dot operator @@ -377,8 +466,10 @@ dot_op -> '.' eol : '$1'. dot_identifier -> identifier : '$1'. dot_identifier -> matched_expr dot_op identifier : build_dot('$2', '$1', '$3'). -dot_alias -> aliases : {'__aliases__', meta('$1', 0), ?exprs('$1')}. -dot_alias -> matched_expr dot_op aliases : build_dot_alias('$2', '$1', '$3'). +dot_alias -> alias : build_alias('$1'). +dot_alias -> matched_expr dot_op alias : build_dot_alias('$2', '$1', '$3'). +dot_alias -> matched_expr dot_op open_curly '}' : build_dot_container('$2', '$1', [], []). +dot_alias -> matched_expr dot_op open_curly container_args close_curly : build_dot_container('$2', '$1', '$4', newlines_pair('$3', '$5')). dot_op_identifier -> op_identifier : '$1'. dot_op_identifier -> matched_expr dot_op op_identifier : build_dot('$2', '$1', '$3'). @@ -392,121 +483,138 @@ dot_bracket_identifier -> matched_expr dot_op bracket_identifier : build_dot('$2 dot_paren_identifier -> paren_identifier : '$1'. dot_paren_identifier -> matched_expr dot_op paren_identifier : build_dot('$2', '$1', '$3'). -parens_call -> dot_paren_identifier : '$1'. -parens_call -> matched_expr dot_call_op : {'.', meta('$2'), ['$1']}. % Fun/local calls +dot_call_identifier -> dot_paren_identifier : '$1'. +dot_call_identifier -> matched_expr dot_call_op : {'.', meta_from_token('$2'), ['$1']}. % Fun/local calls % Function calls with no parentheses call_args_no_parens_expr -> matched_expr : '$1'. -call_args_no_parens_expr -> empty_paren : nil. -call_args_no_parens_expr -> no_parens_expr : throw_no_parens_many_strict('$1'). +call_args_no_parens_expr -> no_parens_expr : error_no_parens_many_strict('$1'). call_args_no_parens_comma_expr -> matched_expr ',' call_args_no_parens_expr : ['$3', '$1']. -call_args_no_parens_comma_expr -> call_args_no_parens_comma_expr ',' call_args_no_parens_expr : ['$3'|'$1']. +call_args_no_parens_comma_expr -> call_args_no_parens_comma_expr ',' call_args_no_parens_expr : ['$3' | '$1']. call_args_no_parens_all -> call_args_no_parens_one : '$1'. +call_args_no_parens_all -> call_args_no_parens_ambig : '$1'. call_args_no_parens_all -> call_args_no_parens_many : '$1'. call_args_no_parens_one -> call_args_no_parens_kw : ['$1']. call_args_no_parens_one -> matched_expr : ['$1']. -call_args_no_parens_one -> no_parens_expr : ['$1']. + +call_args_no_parens_ambig -> no_parens_expr : ['$1']. call_args_no_parens_many -> matched_expr ',' call_args_no_parens_kw : ['$1', '$3']. call_args_no_parens_many -> call_args_no_parens_comma_expr : reverse('$1'). -call_args_no_parens_many -> call_args_no_parens_comma_expr ',' call_args_no_parens_kw : reverse(['$3'|'$1']). +call_args_no_parens_many -> call_args_no_parens_comma_expr ',' call_args_no_parens_kw : reverse(['$3' | '$1']). call_args_no_parens_many_strict -> call_args_no_parens_many : '$1'. -call_args_no_parens_many_strict -> empty_paren : throw_no_parens_strict('$1'). -call_args_no_parens_many_strict -> open_paren call_args_no_parens_kw close_paren : throw_no_parens_strict('$1'). -call_args_no_parens_many_strict -> open_paren call_args_no_parens_many close_paren : throw_no_parens_strict('$1'). +call_args_no_parens_many_strict -> open_paren call_args_no_parens_kw close_paren : error_no_parens_strict('$1'). +call_args_no_parens_many_strict -> open_paren call_args_no_parens_many close_paren : error_no_parens_strict('$1'). -stab_parens_many -> empty_paren : []. stab_parens_many -> open_paren call_args_no_parens_kw close_paren : ['$2']. stab_parens_many -> open_paren call_args_no_parens_many close_paren : '$2'. -% Containers and function calls with parentheses +% Containers -container_expr -> empty_paren : nil. container_expr -> matched_expr : '$1'. container_expr -> unmatched_expr : '$1'. -container_expr -> no_parens_expr : throw_no_parens_many_strict('$1'). +container_expr -> no_parens_expr : error_no_parens_container_strict('$1'). container_args_base -> container_expr : ['$1']. -container_args_base -> container_args_base ',' container_expr : ['$3'|'$1']. +container_args_base -> container_args_base ',' container_expr : ['$3' | '$1']. + +container_args -> container_args_base : reverse('$1'). +container_args -> container_args_base ',' : reverse('$1'). +container_args -> container_args_base ',' kw_data : reverse(['$3' | '$1']). + +% Function calls with parentheses -container_args -> container_args_base : lists:reverse('$1'). -container_args -> container_args_base ',' : lists:reverse('$1'). -container_args -> container_args_base ',' kw : lists:reverse(['$3'|'$1']). +call_args_parens_expr -> matched_expr : '$1'. +call_args_parens_expr -> unmatched_expr : '$1'. +call_args_parens_expr -> no_parens_expr : error_no_parens_many_strict('$1'). -call_args_parens_base -> container_expr : ['$1']. -call_args_parens_base -> call_args_parens_base ',' container_expr : ['$3'|'$1']. +call_args_parens_base -> call_args_parens_expr : ['$1']. +call_args_parens_base -> call_args_parens_base ',' call_args_parens_expr : ['$3' | '$1']. -call_args_parens -> empty_paren : []. -call_args_parens -> open_paren no_parens_expr close_paren : ['$2']. -call_args_parens -> open_paren kw close_paren : ['$2']. -call_args_parens -> open_paren call_args_parens_base close_paren : reverse('$2'). -call_args_parens -> open_paren call_args_parens_base ',' kw close_paren : reverse(['$4'|'$2']). +call_args_parens -> open_paren ')' : + {newlines_pair('$1', '$2'), []}. +call_args_parens -> open_paren no_parens_expr close_paren : + {newlines_pair('$1', '$3'), ['$2']}. +call_args_parens -> open_paren kw_call close_paren : + {newlines_pair('$1', '$3'), ['$2']}. +call_args_parens -> open_paren call_args_parens_base close_paren : + {newlines_pair('$1', '$3'), reverse('$2')}. +call_args_parens -> open_paren call_args_parens_base ',' kw_call close_paren : + {newlines_pair('$1', '$5'), reverse(['$4' | '$2'])}. % KV -kw_eol -> kw_identifier : ?exprs('$1'). -kw_eol -> kw_identifier eol : ?exprs('$1'). -kw_eol -> kw_identifier_safe : build_quoted_atom('$1', true). -kw_eol -> kw_identifier_safe eol : build_quoted_atom('$1', true). -kw_eol -> kw_identifier_unsafe : build_quoted_atom('$1', false). -kw_eol -> kw_identifier_unsafe eol : build_quoted_atom('$1', false). +kw_eol -> kw_identifier : handle_literal(?exprs('$1'), '$1', [{format, keyword}]). +kw_eol -> kw_identifier eol : handle_literal(?exprs('$1'), '$1', [{format, keyword}]). +kw_eol -> kw_identifier_safe : build_quoted_atom('$1', true, [{format, keyword}]). +kw_eol -> kw_identifier_safe eol : build_quoted_atom('$1', true, [{format, keyword}]). +kw_eol -> kw_identifier_unsafe : build_quoted_atom('$1', false, [{format, keyword}]). +kw_eol -> kw_identifier_unsafe eol : build_quoted_atom('$1', false, [{format, keyword}]). kw_base -> kw_eol container_expr : [{'$1', '$2'}]. -kw_base -> kw_base ',' kw_eol container_expr : [{'$3', '$4'}|'$1']. +kw_base -> kw_base ',' kw_eol container_expr : [{'$3', '$4'} | '$1']. + +kw_call -> kw_base : reverse('$1'). +kw_call -> kw_base ',' : warn_trailing_comma('$2'), reverse('$1'). +kw_call -> kw_base ',' matched_expr : maybe_bad_keyword_call_follow_up('$2', '$1', '$3'). + +kw_data -> kw_base : reverse('$1'). +kw_data -> kw_base ',' : reverse('$1'). +kw_data -> kw_base ',' matched_expr : maybe_bad_keyword_data_follow_up('$2', '$1', '$3'). -kw -> kw_base : reverse('$1'). -kw -> kw_base ',' : reverse('$1'). +call_args_no_parens_kw_expr -> kw_eol matched_expr : {'$1', '$2'}. +call_args_no_parens_kw_expr -> kw_eol no_parens_expr : {'$1', '$2'}. -call_args_no_parens_kw_expr -> kw_eol call_args_no_parens_expr : {'$1','$2'}. call_args_no_parens_kw -> call_args_no_parens_kw_expr : ['$1']. -call_args_no_parens_kw -> call_args_no_parens_kw_expr ',' call_args_no_parens_kw : ['$1'|'$3']. +call_args_no_parens_kw -> call_args_no_parens_kw_expr ',' call_args_no_parens_kw : ['$1' | '$3']. +call_args_no_parens_kw -> call_args_no_parens_kw_expr ',' matched_expr : maybe_bad_keyword_call_follow_up('$2', ['$1'], '$3'). % Lists -list_args -> kw : '$1'. +list_args -> kw_data : '$1'. list_args -> container_args_base : reverse('$1'). list_args -> container_args_base ',' : reverse('$1'). -list_args -> container_args_base ',' kw : reverse('$1', '$3'). +list_args -> container_args_base ',' kw_data : reverse('$1', '$3'). -list -> open_bracket ']' : build_list('$1', []). -list -> open_bracket list_args close_bracket : build_list('$1', '$2'). +list -> open_bracket ']' : build_list('$1', [], '$2'). +list -> open_bracket list_args close_bracket : build_list('$1', '$2', '$3'). % Tuple -tuple -> open_curly '}' : build_tuple('$1', []). -tuple -> open_curly container_args close_curly : build_tuple('$1', '$2'). +tuple -> open_curly '}' : build_tuple('$1', [], '$2'). +tuple -> open_curly container_args close_curly : build_tuple('$1', '$2', '$3'). % Bitstrings -bit_string -> open_bit '>>' : build_bit('$1', []). -bit_string -> open_bit container_args close_bit : build_bit('$1', '$2'). +bit_string -> open_bit '>>' : build_bit('$1', [], '$2'). +bit_string -> open_bit container_args close_bit : build_bit('$1', '$2', '$3'). % Map and structs -map_expr -> max_expr : '$1'. -map_expr -> dot_identifier : build_identifier('$1', nil). -map_expr -> at_op_eol map_expr : build_unary_op('$1', '$2'). - assoc_op_eol -> assoc_op : '$1'. assoc_op_eol -> assoc_op eol : '$1'. -assoc_expr -> container_expr assoc_op_eol container_expr : {'$1', '$3'}. -assoc_expr -> map_expr : '$1'. +assoc_expr -> matched_expr assoc_op_eol matched_expr : {'$1', '$3'}. +assoc_expr -> unmatched_expr assoc_op_eol unmatched_expr : {'$1', '$3'}. +assoc_expr -> matched_expr assoc_op_eol unmatched_expr : {'$1', '$3'}. +assoc_expr -> unmatched_expr assoc_op_eol matched_expr : {'$1', '$3'}. +assoc_expr -> dot_identifier : build_identifier('$1', nil). +assoc_expr -> no_parens_one_expr : '$1'. +assoc_expr -> parens_call : '$1'. -assoc_update -> matched_expr pipe_op_eol matched_expr assoc_op_eol matched_expr : {'$2', '$1', [{'$3', '$5'}]}. -assoc_update -> unmatched_expr pipe_op_eol expr assoc_op_eol expr : {'$2', '$1', [{'$3', '$5'}]}. -assoc_update -> matched_expr pipe_op_eol map_expr : {'$2', '$1', ['$3']}. +assoc_update -> matched_expr pipe_op_eol assoc_expr : {'$2', '$1', ['$3']}. +assoc_update -> unmatched_expr pipe_op_eol assoc_expr : {'$2', '$1', ['$3']}. -assoc_update_kw -> matched_expr pipe_op_eol kw : {'$2', '$1', '$3'}. -assoc_update_kw -> unmatched_expr pipe_op_eol kw : {'$2', '$1', '$3'}. +assoc_update_kw -> matched_expr pipe_op_eol kw_data : {'$2', '$1', '$3'}. +assoc_update_kw -> unmatched_expr pipe_op_eol kw_data : {'$2', '$1', '$3'}. assoc_base -> assoc_expr : ['$1']. -assoc_base -> assoc_base ',' assoc_expr : ['$3'|'$1']. +assoc_base -> assoc_base ',' assoc_expr : ['$3' | '$1']. assoc -> assoc_base : reverse('$1'). assoc -> assoc_base ',' : reverse('$1'). @@ -514,86 +622,255 @@ assoc -> assoc_base ',' : reverse('$1'). map_op -> '%{}' : '$1'. map_op -> '%{}' eol : '$1'. -map_close -> kw close_curly : '$1'. -map_close -> assoc close_curly : '$1'. -map_close -> assoc_base ',' kw close_curly : reverse('$1', '$3'). +map_close -> kw_data close_curly : {'$1', '$2'}. +map_close -> assoc close_curly : {'$1', '$2'}. +map_close -> assoc_base ',' kw_data close_curly : {reverse('$1', '$3'), '$4'}. -map_args -> open_curly '}' : build_map('$1', []). -map_args -> open_curly map_close : build_map('$1', '$2'). -map_args -> open_curly assoc_update close_curly : build_map_update('$1', '$2', []). -map_args -> open_curly assoc_update ',' close_curly : build_map_update('$1', '$2', []). -map_args -> open_curly assoc_update ',' map_close : build_map_update('$1', '$2', '$4'). -map_args -> open_curly assoc_update_kw close_curly : build_map_update('$1', '$2', []). +map_args -> open_curly '}' : build_map('$1', [], '$2'). +map_args -> open_curly map_close : build_map('$1', element(1, '$2'), element(2, '$2')). +map_args -> open_curly assoc_update close_curly : build_map_update('$1', '$2', '$3', []). +map_args -> open_curly assoc_update ',' close_curly : build_map_update('$1', '$2', '$4', []). +map_args -> open_curly assoc_update ',' map_close : build_map_update('$1', '$2', element(2, '$4'), element(1, '$4')). +map_args -> open_curly assoc_update_kw close_curly : build_map_update('$1', '$2', '$3', []). struct_op -> '%' : '$1'. -struct_op -> '%' eol : '$1'. +struct_expr -> atom : handle_literal(?exprs('$1'), '$1', []). +struct_expr -> atom_quoted : handle_literal(?exprs('$1'), '$1', delimiter(<<$">>)). +struct_expr -> dot_alias : '$1'. +struct_expr -> dot_identifier : build_identifier('$1', nil). +struct_expr -> at_op_eol struct_expr : build_unary_op('$1', '$2'). +struct_expr -> unary_op_eol struct_expr : build_unary_op('$1', '$2'). +struct_expr -> parens_call : '$1'. map -> map_op map_args : '$2'. -map -> struct_op map_expr map_args : {'%', meta('$1'), ['$2', '$3']}. -map -> struct_op map_expr eol map_args : {'%', meta('$1'), ['$2', '$4']}. +map -> struct_op struct_expr map_args : {'%', meta_from_token('$1'), ['$2', '$3']}. +map -> struct_op struct_expr eol map_args : {'%', meta_from_token('$1'), ['$2', '$4']}. Erlang code. --define(id(Node), element(1, Node)). --define(line(Node), element(2, Node)). --define(exprs(Node), element(3, Node)). +-define(columns(), get(elixir_parser_columns)). +-define(token_metadata(), get(elixir_token_metadata)). + +-define(id(Token), element(1, Token)). +-define(location(Token), element(2, Token)). +-define(exprs(Token), element(3, Token)). +-define(meta(Node), element(2, Node)). -define(rearrange_uop(Op), (Op == 'not' orelse Op == '!')). -%% The following directive is needed for (significantly) faster -%% compilation of the generated .erl file by the HiPE compiler --compile([{hipe,[{regalloc,linear_scan}]}]). +-compile({inline, meta_from_token/1, meta_from_location/1, is_eol/1}). -import(lists, [reverse/1, reverse/2]). -meta(Line, Counter) -> [{counter,Counter}|meta(Line)]. -meta(Line) when is_integer(Line) -> [{line,Line}]; -meta(Node) -> meta(?line(Node)). +meta_from_token(Token) -> + meta_from_location(?location(Token)). + +meta_from_location({Line, Column, _}) -> + case ?columns() of + true -> [{line, Line}, {column, Column}]; + false -> [{line, Line}] + end. + +do_end_meta(Do, End) -> + case ?token_metadata() of + true -> + [{do, meta_from_location(?location(Do))}, {'end', meta_from_location(?location(End))}]; + false -> + [] + end. + +meta_from_token_with_closing(Begin, End) -> + case ?token_metadata() of + true -> + [{closing, meta_from_location(?location(End))} | meta_from_token(Begin)]; + false -> + meta_from_token(Begin) + end. + +append_non_empty(Left, []) -> Left; +append_non_empty(Left, Right) -> Left ++ Right. + +%% Handle metadata in literals + +handle_literal(Literal, Token) -> + handle_literal(Literal, Token, []). + +handle_literal(Literal, Token, ExtraMeta) -> + case get(elixir_literal_encoder) of + false -> + Literal; + + Fun -> + Meta = ExtraMeta ++ meta_from_token(Token), + case Fun(Literal, Meta) of + {ok, EncodedLiteral} -> + EncodedLiteral; + {error, Reason} -> + return_error(?location(Token), elixir_utils:characters_to_list(Reason) ++ [": "], "literal") + end + end. + +handle_number(Number, Token, Original) -> + case ?token_metadata() of + true -> handle_literal(Number, Token, [{token, elixir_utils:characters_to_binary(Original)}]); + false -> handle_literal(Number, Token, []) + end. + +number_value({_, {_, _, Value}, _}) -> + Value. %% Operators -build_op({_Kind, Line, 'in'}, {UOp, _, [Left]}, Right) when ?rearrange_uop(UOp) -> - {UOp, meta(Line), [{'in', meta(Line), [Left, Right]}]}; +build_op(Left, {Op, Right}) -> + build_op(Left, Op, Right). + +build_op(AST, {_Kind, Location, '//'}, Right) -> + case AST of + {'..', Meta, [Left, Middle]} -> + {'..//', Meta, [Left, Middle, Right]}; -build_op({_Kind, Line, Op}, Left, Right) -> - {Op, meta(Line), [Left, Right]}. + _ -> + return_error(Location, "the range step operator (//) must immediately follow the range definition operator (..), for example: 1..9//2. If you wanted to define a default argument, use (\\\\) instead. Syntax error before: ", "'//'") + end; -build_unary_op({_Kind, Line, Op}, Expr) -> - {Op, meta(Line), [Expr]}. +build_op({UOp, _, [Left]}, {_Kind, {Line, Column, _} = Location, 'in'}, Right) when ?rearrange_uop(UOp) -> + %% TODO: Remove "not left in right" rearrangement on v2.0 + warn({Line, Column}, "\"not expr1 in expr2\" is deprecated, use \"expr1 not in expr2\" instead"), + Meta = meta_from_location(Location), + {UOp, Meta, [{'in', Meta, [Left, Right]}]}; + +build_op(Left, {_Kind, Location, 'not in'}, Right) -> + Meta = meta_from_location(Location), + {'not', Meta, [{'in', Meta, [Left, Right]}]}; + +build_op(Left, {_Kind, Location, Op}, Right) -> + {Op, newlines_op(Location) ++ meta_from_location(Location), [Left, Right]}. + +build_unary_op({_Kind, {Line, Column, _}, '//'}, Expr) -> + {Outer, Inner} = + case ?columns() of + true -> {[{column, Column+1}], [{column, Column}]}; + false -> {[], []} + end, + {'/', [{line, Line} | Outer], [{'/', [{line, Line} | Inner], nil}, Expr]}; -build_list(Marker, Args) -> - {Args, ?line(Marker)}. +build_unary_op({_Kind, Location, Op}, Expr) -> + {Op, meta_from_location(Location), [Expr]}. -build_tuple(_Marker, [Left, Right]) -> - {Left, Right}; -build_tuple(Marker, Args) -> - {'{}', meta(Marker), Args}. +build_nullary_op({_Kind, Location, Op}) -> + {Op, meta_from_location(Location), []}. -build_bit(Marker, Args) -> - {'<<>>', meta(Marker), Args}. +build_list(Left, Args, Right) -> + {handle_literal(Args, Left, newlines_pair(Left, Right)), ?location(Left)}. -build_map(Marker, Args) -> - {'%{}', meta(Marker), Args}. +build_tuple(Left, [Arg1, Arg2], Right) -> + handle_literal({Arg1, Arg2}, Left, newlines_pair(Left, Right)); +build_tuple(Left, Args, Right) -> + {'{}', newlines_pair(Left, Right) ++ meta_from_token(Left), Args}. -build_map_update(Marker, {Pipe, Left, Right}, Extra) -> - {'%{}', meta(Marker), [build_op(Pipe, Left, Right ++ Extra)]}. +build_bit(Left, Args, Right) -> + {'<<>>', newlines_pair(Left, Right) ++ meta_from_token(Left), Args}. + +build_map(Left, Args, Right) -> + {'%{}', newlines_pair(Left, Right) ++ meta_from_token(Left), Args}. + +build_map_update(Left, {Pipe, Struct, Map}, Right, Extra) -> + Op = build_op(Struct, Pipe, append_non_empty(Map, Extra)), + {'%{}', newlines_pair(Left, Right) ++ meta_from_token(Left), [Op]}. %% Blocks -build_block([{Op,_,[_]}]=Exprs) when ?rearrange_uop(Op) -> {'__block__', [], Exprs}; -build_block([{unquote_splicing,_,Args}]=Exprs) when - length(Args) =< 2 -> {'__block__', [], Exprs}; -build_block([Expr]) -> Expr; -build_block(Exprs) -> {'__block__', [], Exprs}. +build_block([{Op, _, [_]}]=Exprs) when ?rearrange_uop(Op) -> + {'__block__', [], Exprs}; +build_block([{unquote_splicing, _, [_]}]=Exprs) -> + {'__block__', [], Exprs}; +build_block([Expr]) -> + Expr; +build_block(Exprs) -> + {'__block__', [], Exprs}. + +%% Newlines + +newlines_pair(Left, Right) -> + case ?token_metadata() of + true -> + newlines(?location(Left), [{closing, meta_from_location(?location(Right))}]); + false -> + [] + end. + +newlines_op(Location) -> + case ?token_metadata() of + true -> newlines(Location, []); + false -> [] + end. + +next_is_eol(Token, {_, {_, _, Count}}) -> + {Line, Column, _} = ?location(Token), + setelement(2, Token, {Line, Column, Count}). + +newlines({_, _, Count}, Meta) when is_integer(Count) and (Count > 0) -> + [{newlines, Count} | Meta]; +newlines(_, Meta) -> + Meta. + +annotate_eoe(Token, Stack) -> + case ?token_metadata() of + true -> + case {Token, Stack} of + {{_, Location}, [{'->', StabMeta, [StabArgs, {Left, Meta, Right}]} | Rest]} when is_list(Meta) -> + [{'->', StabMeta, [StabArgs, {Left, [{end_of_expression, end_of_expression(Location)} | Meta], Right}]} | Rest]; + + {{_, Location}, [{Left, Meta, Right} | Rest]} when is_list(Meta) -> + [{Left, [{end_of_expression, end_of_expression(Location)} | Meta], Right} | Rest]; + + _ -> + Stack + end; + false -> + Stack + end. + +end_of_expression({_, _, Count} = Location) when is_integer(Count) -> + [{newlines, Count} | meta_from_location(Location)]; +end_of_expression(Location) -> + meta_from_location(Location). %% Dots -build_dot_alias(Dot, {'__aliases__', _, Left}, {'aliases', _, Right}) -> - {'__aliases__', meta(Dot), Left ++ Right}; +build_alias({'alias', Location, Alias}) -> + Meta = meta_from_location(Location), + MetaWithExtra = + case ?token_metadata() of + true -> [{last, meta_from_location(Location)} | Meta]; + false -> Meta + end, + {'__aliases__', MetaWithExtra, [Alias]}. -build_dot_alias(Dot, Other, {'aliases', _, Right}) -> - {'__aliases__', meta(Dot), [Other|Right]}. +build_dot_alias(_Dot, {'__aliases__', Meta, Left}, {'alias', SegmentLocation, Right}) -> + MetaWithExtra = + case ?token_metadata() of + true -> lists:keystore(last, 1, Meta, {last, meta_from_location(SegmentLocation)}); + false -> Meta + end, + {'__aliases__', MetaWithExtra, Left ++ [Right]}; +build_dot_alias(_Dot, Atom, Right) when is_atom(Atom) -> + error_bad_atom(Right); +build_dot_alias(Dot, Expr, {'alias', SegmentLocation, Right}) -> + Meta = meta_from_token(Dot), + MetaWithExtra = + case ?token_metadata() of + true -> [{last, meta_from_location(SegmentLocation)} | Meta]; + false -> Meta + end, + {'__aliases__', MetaWithExtra, [Expr, Right]}. -build_dot(Dot, Left, Right) -> - {'.', meta(Dot), [Left, extract_identifier(Right)]}. +build_dot_container(Dot, Left, Right, Extra) -> + Meta = meta_from_token(Dot), + {{'.', Meta, [Left, '{}']}, Extra ++ Meta, Right}. + +build_dot(Dot, Left, {_, Location, _} = Right) -> + Meta = meta_from_token(Dot), + IdentifierLocation = meta_from_location(Location), + {'.', Meta, IdentifierLocation, [Left, extract_identifier(Right)]}. extract_identifier({Kind, _, Identifier}) when Kind == identifier; Kind == bracket_identifier; Kind == paren_identifier; @@ -602,72 +879,153 @@ extract_identifier({Kind, _, Identifier}) when %% Identifiers -build_nested_parens(Dot, Args1, Args2) -> - Identifier = build_identifier(Dot, Args1), - Meta = element(2, Identifier), - {Identifier, Meta, Args2}. +build_nested_parens(Dot, Args1, {Args2Meta, Args2}, {BlockMeta, Block}) -> + Identifier = build_parens(Dot, Args1, {[], []}), + Meta = BlockMeta ++ Args2Meta ++ ?meta(Identifier), + {Identifier, Meta, append_non_empty(Args2, Block)}. -build_identifier({'.', Meta, _} = Dot, Args) -> - FArgs = case Args of - nil -> []; - _ -> Args - end, - {Dot, Meta, FArgs}; +build_parens(Expr, {ArgsMeta, Args}, {BlockMeta, Block}) -> + {BuiltExpr, BuiltMeta, BuiltArgs} = build_identifier(Expr, append_non_empty(Args, Block)), + {BuiltExpr, BlockMeta ++ ArgsMeta ++ BuiltMeta, BuiltArgs}. + +build_no_parens_do_block(Expr, Args, {BlockMeta, Block}) -> + {BuiltExpr, BuiltMeta, BuiltArgs} = build_no_parens(Expr, Args ++ Block), + {BuiltExpr, BlockMeta ++ BuiltMeta, BuiltArgs}. + +build_no_parens(Expr, Args) -> + build_identifier(Expr, Args). + +build_identifier({'.', Meta, IdentifierLocation, DotArgs}, nil) -> + {{'.', Meta, DotArgs}, [{no_parens, true} | IdentifierLocation], []}; + +build_identifier({'.', Meta, IdentifierLocation, DotArgs}, Args) -> + {{'.', Meta, DotArgs}, IdentifierLocation, Args}; -build_identifier({Keyword, Line}, Args) when Keyword == fn -> - {fn, meta(Line), Args}; +build_identifier({'.', Meta, _} = Dot, nil) -> + {Dot, [{no_parens, true} | Meta], []}; -build_identifier({op_identifier, Line, Identifier}, [Arg]) -> - {Identifier, [{ambiguous_op,nil}|meta(Line)], [Arg]}; +build_identifier({'.', Meta, _} = Dot, Args) -> + {Dot, Meta, Args}; + +build_identifier({op_identifier, Location, Identifier}, [Arg]) -> + {Identifier, [{ambiguous_op, nil} | meta_from_location(Location)], [Arg]}; -build_identifier({_, Line, Identifier}, Args) -> - {Identifier, meta(Line), Args}. +build_identifier({_, Location, Identifier}, Args) -> + {Identifier, meta_from_location(Location), Args}. %% Fn -build_fn(Op, Stab) -> - {fn, meta(Op), Stab}. +build_fn(Fn, Stab, End) -> + case check_stab(Stab, none) of + stab -> + Meta = newlines_op(?location(Fn)) ++ meta_from_token_with_closing(Fn, End), + {fn, Meta, collect_stab(Stab, [], [])}; + block -> + return_error(?location(Fn), "expected anonymous functions to be defined with -> inside: ", "'fn'") + end. %% Access -build_access(Expr, {List, Line}) -> - Meta = meta(Line), +build_access_arg(Left, Args, Right) -> + {Args, newlines_pair(Left, Right) ++ meta_from_token(Left)}. + +build_access(Expr, {List, Meta}) -> {{'.', Meta, ['Elixir.Access', get]}, Meta, [Expr, List]}. %% Interpolation aware -build_sigil({sigil, Line, Sigil, Parts, Modifiers}) -> - Meta = meta(Line), - {list_to_atom("sigil_" ++ [Sigil]), Meta, [ {'<<>>', Meta, string_parts(Parts)}, Modifiers ]}. - -build_bin_string({bin_string, _Line, [H]}) when is_binary(H) -> - H; -build_bin_string({bin_string, Line, Args}) -> - {'<<>>', meta(Line), string_parts(Args)}. - -build_list_string({list_string, _Line, [H]}) when is_binary(H) -> - elixir_utils:characters_to_list(H); -build_list_string({list_string, Line, Args}) -> - Meta = meta(Line), - {{'.', Meta, ['Elixir.String', to_char_list]}, Meta, [{'<<>>', Meta, string_parts(Args)}]}. +build_sigil({sigil, Location, Sigil, Parts, Modifiers, Indentation, Delimiter}) -> + Meta = meta_from_location(Location), + MetaWithDelimiter = [{delimiter, Delimiter} | Meta], + MetaWithIndentation = meta_with_indentation(Meta, Indentation), + {list_to_atom("sigil_" ++ [Sigil]), + MetaWithDelimiter, + [{'<<>>', MetaWithIndentation, string_parts(Parts)}, Modifiers]}. + +meta_with_indentation(Meta, nil) -> + Meta; +meta_with_indentation(Meta, Indentation) -> + [{indentation, Indentation} | Meta]. + +build_bin_heredoc({bin_heredoc, Location, Indentation, Args}) -> + ExtraMeta = + case ?token_metadata() of + true -> [{delimiter, <<$", $", $">>}, {indentation, Indentation}]; + false -> [] + end, + build_bin_string({bin_string, Location, Args}, ExtraMeta). -build_quoted_atom({_, _Line, [H]}, Safe) when is_binary(H) -> - Op = binary_to_atom_op(Safe), erlang:Op(H, utf8); -build_quoted_atom({_, Line, Args}, Safe) -> - Meta = meta(Line), - {{'.', Meta, [erlang, binary_to_atom_op(Safe)]}, Meta, [{'<<>>', Meta, string_parts(Args)}, utf8]}. +build_list_heredoc({list_heredoc, Location, Indentation, Args}) -> + ExtraMeta = + case ?token_metadata() of + true -> [{delimiter, <<$', $', $'>>}, {indentation, Indentation}]; + false -> [] + end, + build_list_string({list_string, Location, Args}, ExtraMeta). + +build_bin_string({bin_string, _Location, [H]} = Token, ExtraMeta) when is_binary(H) -> + handle_literal(H, Token, ExtraMeta); +build_bin_string({bin_string, Location, Args}, ExtraMeta) -> + Meta = + case ?token_metadata() of + true -> ExtraMeta ++ meta_from_location(Location); + false -> meta_from_location(Location) + end, + {'<<>>', Meta, string_parts(Args)}. + +build_list_string({list_string, _Location, [H]} = Token, ExtraMeta) when is_binary(H) -> + handle_literal(elixir_utils:characters_to_list(H), Token, ExtraMeta); +build_list_string({list_string, Location, Args}, ExtraMeta) -> + Meta = meta_from_location(Location), + MetaWithExtra = + case ?token_metadata() of + true -> ExtraMeta ++ Meta; + false -> Meta + end, + {{'.', Meta, ['Elixir.List', to_charlist]}, MetaWithExtra, [charlist_parts(Args)]}. + +build_quoted_atom({_, _Location, [H]} = Token, Safe, ExtraMeta) when is_binary(H) -> + Op = binary_to_atom_op(Safe), + handle_literal(erlang:Op(H, utf8), Token, ExtraMeta); +build_quoted_atom({_, Location, Args}, Safe, ExtraMeta) -> + Meta = meta_from_location(Location), + MetaWithExtra = + case ?token_metadata() of + true -> ExtraMeta ++ Meta; + false -> Meta + end, + {{'.', Meta, [erlang, binary_to_atom_op(Safe)]}, MetaWithExtra, [{'<<>>', Meta, string_parts(Args)}, utf8]}. binary_to_atom_op(true) -> binary_to_existing_atom; binary_to_atom_op(false) -> binary_to_atom. +charlist_parts(Parts) -> + [charlist_part(Part) || Part <- Parts]. +charlist_part(Binary) when is_binary(Binary) -> + Binary; +charlist_part({Begin, End, Tokens}) -> + Form = string_tokens_parse(Tokens), + Meta = meta_from_location(Begin), + MetaWithExtra = + case ?token_metadata() of + true -> [{closing, meta_from_location(End)} | Meta]; + false -> Meta + end, + {{'.', Meta, ['Elixir.Kernel', to_string]}, MetaWithExtra, [Form]}. + string_parts(Parts) -> [string_part(Part) || Part <- Parts]. string_part(Binary) when is_binary(Binary) -> Binary; -string_part({Line, Tokens}) -> +string_part({Begin, End, Tokens}) -> Form = string_tokens_parse(Tokens), - Meta = meta(Line), - {'::', Meta, [{{'.', Meta, ['Elixir.Kernel', to_string]}, Meta, [Form]}, {binary, Meta, nil}]}. + Meta = meta_from_location(Begin), + MetaWithExtra = + case ?token_metadata() of + true -> [{closing, meta_from_location(End)} | meta_from_location(Begin)]; + false -> meta_from_location(Begin) + end, + {'::', Meta, [{{'.', Meta, ['Elixir.Kernel', to_string]}, MetaWithExtra, [Form]}, {binary, Meta, nil}]}. string_tokens_parse(Tokens) -> case parse(Tokens) of @@ -675,36 +1033,56 @@ string_tokens_parse(Tokens) -> {error, _} = Error -> throw(Error) end. +delimiter(Delimiter) -> + case ?token_metadata() of + true -> [{delimiter, Delimiter}]; + false -> [] + end. + %% Keywords -build_stab([{'->', Meta, [Left, Right]}|T]) -> - build_stab(Meta, T, Left, [Right], []); +check_stab([{'->', _, [_, _]}], _) -> stab; +check_stab([], none) -> block; +check_stab([_], none) -> block; +check_stab([_], Meta) -> error_invalid_stab(Meta); +check_stab([{'->', Meta, [_, _]} | T], _) -> check_stab(T, Meta); +check_stab([_ | T], MaybeMeta) -> check_stab(T, MaybeMeta). + +build_stab(Stab) -> + case check_stab(Stab, none) of + block -> build_block(reverse(Stab)); + stab -> collect_stab(Stab, [], []) + end. -build_stab(Else) -> - build_block(Else). +build_stab(Before, Stab, After) -> + case build_stab(Stab) of + {'__block__', Meta, Block} -> + {'__block__', Meta ++ meta_from_token_with_closing(Before, After), Block}; + Other -> + Other + end. -build_stab(Old, [{'->', New, [Left, Right]}|T], Marker, Temp, Acc) -> - H = {'->', Old, [Marker, build_block(reverse(Temp))]}, - build_stab(New, T, Left, [Right], [H|Acc]); +collect_stab([{'->', Meta, [Left, Right]} | T], Exprs, Stabs) -> + Stab = {'->', Meta, [Left, build_block([Right | Exprs])]}, + collect_stab(T, [], [Stab | Stabs]); -build_stab(Meta, [H|T], Marker, Temp, Acc) -> - build_stab(Meta, T, Marker, [H|Temp], Acc); +collect_stab([H | T], Exprs, Stabs) -> + collect_stab(T, [H | Exprs], Stabs); -build_stab(Meta, [], Marker, Temp, Acc) -> - H = {'->', Meta, [Marker, build_block(reverse(Temp))]}, - reverse([H|Acc]). +collect_stab([], [], Stabs) -> + Stabs. %% Every time the parser sees a (unquote_splicing()) %% it assumes that a block is being spliced, wrapping %% the splicing in a __block__. But in the stab clause, -%% we can have (unquote_splicing(1,2,3)) -> :ok, in such +%% we can have (unquote_splicing(1, 2, 3)) -> :ok, in such %% case, we don't actually want the block, since it is %% an arg style call. unwrap_splice unwraps the splice %% from such blocks. -unwrap_splice([{'__block__', [], [{unquote_splicing, _, _}] = Splice}]) -> +unwrap_splice([{'__block__', _, [{unquote_splicing, _, _}] = Splice}]) -> Splice; - -unwrap_splice(Other) -> Other. +unwrap_splice(Other) -> + Other. unwrap_when(Args) -> case elixir_utils:split_last(Args) of @@ -714,25 +1092,124 @@ unwrap_when(Args) -> Args end. -to_block([One]) -> One; -to_block(Other) -> {'__block__', [], reverse(Other)}. - -%% Errors - -throw(Line, Error, Token) -> - throw({error, {Line, ?MODULE, [Error, Token]}}). - -throw_no_parens_strict(Token) -> - throw(?line(Token), "unexpected parenthesis. If you are making a " - "function call, do not insert spaces in between the function name and the " +%% Warnings and errors + +return_error({Line, Column, _}, ErrorMessage, ErrorToken) -> + return_error([{line, Line}, {column, Column}], [ErrorMessage, ErrorToken]). + +%% We should prefer to use return_error as it includes +%% Line and Column but that's not always possible. +return_error_with_meta(Meta, ErrorMessage, ErrorToken) -> + return_error(Meta, [ErrorMessage, ErrorToken]). + +error_invalid_stab(MetaStab) -> + return_error_with_meta(MetaStab, + "unexpected operator ->. If you want to define multiple clauses, the first expression must use ->. " + "Syntax error before: ", "'->'"). + +error_bad_atom(Token) -> + return_error(?location(Token), "atom cannot be followed by an alias. " + "If the '.' was meant to be part of the atom's name, " + "the atom name must be quoted. Syntax error before: ", "'.'"). + +maybe_bad_keyword_call_follow_up(_Token, KW, {'__cursor__', _, []} = Expr) -> + reverse([Expr | KW]); +maybe_bad_keyword_call_follow_up(Token, _KW, _Expr) -> + return_error(?location(Token), + "unexpected expression after keyword list. Keyword lists must always come as the last argument. Therefore, this is not allowed:\n\n" + " function_call(1, some: :option, 2)\n\n" + "Instead, wrap the keyword in brackets:\n\n" + " function_call(1, [some: :option], 2)\n\n" + "Syntax error after: ", "','"). + +maybe_bad_keyword_data_follow_up(_Token, KW, {'__cursor__', _, []} = Expr) -> + reverse([Expr | KW]); +maybe_bad_keyword_data_follow_up(Token, _KW, _Expr) -> + return_error(?location(Token), + "unexpected expression after keyword list. Keyword lists must always come last in lists and maps. Therefore, this is not allowed:\n\n" + " [some: :value, :another]\n" + " %{some: :value, another => value}\n\n" + "Instead, reorder it to be the last entry:\n\n" + " [:another, some: :value]\n" + " %{another => value, some: :value}\n\n" + "Syntax error after: ", "','"). + +error_no_parens_strict(Token) -> + return_error(?location(Token), "unexpected parentheses. If you are making a " + "function call, do not insert spaces between the function name and the " "opening parentheses. Syntax error before: ", "'('"). -throw_no_parens_many_strict(Token) -> - Line = - case lists:keyfind(line, 1, element(2, Token)) of - {line, L} -> L; - false -> 0 - end, - - throw(Line, "unexpected comma. Parentheses are required to solve ambiguity " - "in nested calls. Syntax error before: ", "','"). +error_no_parens_many_strict(Node) -> + return_error_with_meta(?meta(Node), + "unexpected comma. Parentheses are required to solve ambiguity in nested calls.\n\n" + "This error happens when you have nested function calls without parentheses. " + "For example:\n\n" + " one a, two b, c, d\n\n" + "In the example above, we don't know if the parameters \"c\" and \"d\" apply " + "to the function \"one\" or \"two\". You can solve this by explicitly adding " + "parentheses:\n\n" + " one a, two(b, c, d)\n\n" + "Or by adding commas (in case a nested call is not intended):\n\n" + " one, a, two, b, c, d\n\n" + "Elixir cannot compile otherwise. Syntax error before: ", "','"). + +error_no_parens_container_strict(Node) -> + return_error_with_meta(?meta(Node), + "unexpected comma. Parentheses are required to solve ambiguity inside containers.\n\n" + "This error may happen when you forget a comma in a list or other container:\n\n" + " [a, b c, d]\n\n" + "Or when you have ambiguous calls:\n\n" + " [one, two three, four, five]\n\n" + "In the example above, we don't know if the parameters \"four\" and \"five\" " + "belongs to the list or the function \"two\". You can solve this by explicitly " + "adding parentheses:\n\n" + " [one, two(three, four), five]\n\n" + "Elixir cannot compile otherwise. Syntax error before: ", "','"). + +error_invalid_kw_identifier({_, Location, do}) -> + return_error(Location, elixir_tokenizer:invalid_do_error("unexpected keyword: "), "do:"); +error_invalid_kw_identifier({_, Location, KW}) -> + return_error(Location, "syntax error before: ", "'" ++ atom_to_list(KW) ++ ":'"). + +%% TODO: Make this an error on v2.0 +warn_trailing_comma({',', {Line, Column, _}}) -> + warn({Line, Column}, "trailing commas are not allowed inside function/macro call arguments"). + +%% TODO: Make this an error on v2.0 +warn_empty_paren({_, {Line, Column, _}}) -> + warn( + {Line, Column}, + "invalid expression (). " + "If you want to invoke or define a function, make sure there are " + "no spaces between the function name and its arguments. If you wanted " + "to pass an empty block or code, pass a value instead, such as a nil or an atom" + ). + +%% TODO: Make this an error on v2.0 +warn_pipe({arrow_op, {Line, Column, _}, Op}, {_, [_ | _], [_ | _]}) -> + warn( + {Line, Column}, + io_lib:format( + "parentheses are required when piping into a function call. For example:\n\n" + " foo 1 ~ts bar 2 ~ts baz 3\n\n" + "is ambiguous and should be written as\n\n" + " foo(1) ~ts bar(2) ~ts baz(3)\n\n" + "Ambiguous pipe found at:", + [Op, Op, Op, Op] + ) + ); +warn_pipe(_Token, _) -> + ok. + +warn_empty_stab_clause({stab_op, {Line, Column, _}, '->'}) -> + warn( + {Line, Column}, + "an expression is always required on the right side of ->. " + "Please provide a value after ->" + ). + +warn(LineColumn, Message) -> + case get(elixir_parser_warning_file) of + nil -> ok; + File -> elixir_errors:erl_warn(LineColumn, File, Message) + end. \ No newline at end of file diff --git a/lib/elixir/src/elixir_quote.erl b/lib/elixir/src/elixir_quote.erl index 9e8cbc7bd9f..46af801ab42 100644 --- a/lib/elixir/src/elixir_quote.erl +++ b/lib/elixir/src/elixir_quote.erl @@ -1,82 +1,149 @@ -module(elixir_quote). --export([escape/2, linify/2, linify/3, linify_with_context_counter/3, quote/4]). --export([dot/6, tail_list/3, list/2]). %% Quote callbacks +-export([escape/3, linify/3, linify_with_context_counter/3, build/6, quote/6, has_unquotes/1, fun_to_quoted/1]). +-export([dot/5, tail_list/3, list/2, validate_runtime/2]). %% Quote callbacks -include("elixir.hrl"). --define(defs(Kind), Kind == def; Kind == defp; Kind == defmacro; Kind == defmacrop). --define(lexical(Kind), Kind == import; Kind == alias; Kind == '__aliases__'). --compile({inline, [keyfind/2, keystore/3, keydelete/2, keyreplace/3, keynew/3]}). +-define(defs(Kind), Kind == def; Kind == defp; Kind == defmacro; Kind == defmacrop; Kind == '@'). +-define(lexical(Kind), Kind == import; Kind == alias; Kind == require). +-compile({inline, [keyfind/2, keystore/3, keydelete/2, keynew/3, do_tuple_linify/5]}). + +-record(elixir_quote, { + line=false, + file=nil, + context=nil, + vars_hygiene=true, + aliases_hygiene=true, + imports_hygiene=true, + unquote=true, + generated=false +}). + +build(Meta, Line, File, Context, Unquote, Generated) -> + Acc0 = [], + {ELine, Acc1} = validate_compile(Meta, line, Line, Acc0), + {EFile, Acc2} = validate_compile(Meta, file, File, Acc1), + {EContext, Acc3} = validate_compile(Meta, context, Context, Acc2), + validate_runtime(unquote, Unquote), + validate_runtime(generated, Generated), + + Q = #elixir_quote{ + line=ELine, + file=EFile, + unquote=Unquote, + context=EContext, + generated=Generated + }, + + {Q, Acc3}. + +validate_compile(_Meta, line, Value, Acc) when is_boolean(Value) -> + {Value, Acc}; +validate_compile(_Meta, file, nil, Acc) -> + {nil, Acc}; +validate_compile(Meta, Key, Value, Acc) -> + case is_valid(Key, Value) of + true -> + {Value, Acc}; + false -> + Var = {Key, Meta, ?MODULE}, + Call = {{'.', Meta, [?MODULE, validate_runtime]}, Meta, [Key, Value]}, + {Var, [{'=', Meta, [Var, Call]} | Acc]} + end. + +validate_runtime(Key, Value) -> + case is_valid(Key, Value) of + true -> + Value; + + false -> + erlang:error( + 'Elixir.ArgumentError':exception( + <<"invalid runtime value for option :", (erlang:atom_to_binary(Key))/binary, + " in quote, got: ", ('Elixir.Kernel':inspect(Value))/binary>> + ) + ) + end. + +is_valid(line, Line) -> is_integer(Line); +is_valid(file, File) -> is_binary(File); +is_valid(context, Context) -> is_atom(Context) andalso (Context /= nil); +is_valid(generated, Generated) -> is_boolean(Generated); +is_valid(unquote, Unquote) -> is_boolean(Unquote). %% Apply the line from site call on quoted contents. %% Receives a Key to look for the default line as argument. -linify(Line, Exprs) when is_integer(Line) -> - do_linify(Line, line, nil, Exprs). - +linify(0, _Key, Exprs) -> + Exprs; linify(Line, Key, Exprs) when is_integer(Line) -> - do_linify(Line, Key, nil, Exprs). + LinifyMeta = linify_meta(Line, Key), + do_linify(LinifyMeta, Exprs, nil). %% Same as linify but also considers the context counter. linify_with_context_counter(Line, Var, Exprs) when is_integer(Line) -> - do_linify(Line, line, Var, Exprs). + LinifyMeta = linify_meta(Line, line), + do_linify(LinifyMeta, Exprs, Var). + +do_linify(LinifyMeta, {quote, Meta, [_ | _] = Args}, {Receiver, Counter} = Var) + when is_list(Meta) -> + NewMeta = + case keyfind(context, Meta) == {context, Receiver} of + true -> keynew(counter, Meta, Counter); + false -> Meta + end, + do_tuple_linify(LinifyMeta, NewMeta, quote, Args, Var); -do_linify(Line, Key, {Receiver, Counter} = Var, {Left, Meta, Receiver}) +do_linify(LinifyMeta, {Left, Meta, Receiver}, {Receiver, Counter} = Var) when is_atom(Left), is_list(Meta), Left /= '_' -> - do_tuple_linify(Line, Key, Var, keynew(counter, Meta, Counter), Left, Receiver); + do_tuple_linify(LinifyMeta, keynew(counter, Meta, Counter), Left, Receiver, Var); -do_linify(Line, Key, {_, Counter} = Var, {Lexical, [_|_] = Meta, [_|_] = Args}) when ?lexical(Lexical) -> - do_tuple_linify(Line, Key, Var, keynew(counter, Meta, Counter), Lexical, Args); +do_linify(LinifyMeta, {Lexical, Meta, [_ | _] = Args}, {_, Counter} = Var) + when ?lexical(Lexical); Lexical == '__aliases__' -> + do_tuple_linify(LinifyMeta, keynew(counter, Meta, Counter), Lexical, Args, Var); -do_linify(Line, Key, Var, {Left, Meta, Right}) when is_list(Meta) -> - do_tuple_linify(Line, Key, Var, Meta, Left, Right); +do_linify(LinifyMeta, {Left, Meta, Right}, Var) when is_list(Meta) -> + do_tuple_linify(LinifyMeta, Meta, Left, Right, Var); -do_linify(Line, Key, Var, {Left, Right}) -> - {do_linify(Line, Key, Var, Left), do_linify(Line, Key, Var, Right)}; +do_linify(LinifyMeta, {Left, Right}, Var) -> + {do_linify(LinifyMeta, Left, Var), do_linify(LinifyMeta, Right, Var)}; -do_linify(Line, Key, Var, List) when is_list(List) -> - [do_linify(Line, Key, Var, X) || X <- List]; +do_linify(LinifyMeta, List, Var) when is_list(List) -> + [do_linify(LinifyMeta, X, Var) || X <- List]; -do_linify(_, _, _, Else) -> Else. +do_linify(_, Else, _) -> Else. -do_tuple_linify(Line, Key, Var, Meta, Left, Right) -> - {do_linify(Line, Key, Var, Left), - do_linify_meta(Line, Key, Meta), - do_linify(Line, Key, Var, Right)}. +do_tuple_linify(LinifyMeta, Meta, Left, Right, Var) -> + {do_linify(LinifyMeta, Left, Var), LinifyMeta(Meta), do_linify(LinifyMeta, Right, Var)}. -do_linify_meta(0, line, Meta) -> - Meta; -do_linify_meta(Line, line, Meta) -> - case keyfind(line, Meta) of - {line, Int} when is_integer(Int), Int /= 0 -> - Meta; - _ -> - keystore(line, Meta, Line) - end; -do_linify_meta(Line, Key, Meta) -> - case keyfind(Key, Meta) of - {Key, Int} when is_integer(Int), Int /= 0 -> - keyreplace(Key, Meta, {line, Int}); - _ -> - do_linify_meta(Line, line, Meta) +linify_meta(0, line) -> fun(Meta) -> Meta end; +linify_meta(Line, line) -> fun(Meta) -> keynew(line, Meta, Line) end; +linify_meta(Line, keep) -> + fun(Meta) -> + case lists:keytake(keep, 1, Meta) of + {value, {keep, {_, Int}}, MetaNoFile} -> + [{line, Int} | keydelete(line, MetaNoFile)]; + _ -> + keynew(line, Meta, Line) + end end. %% Some expressions cannot be unquoted at compilation time. %% This function is responsible for doing runtime unquoting. -dot(Meta, Left, Right, Args, Context, File) -> - annotate(dot(Meta, Left, Right, Args), Context, File). +dot(Meta, Left, Right, Args, Context) -> + annotate(dot(Meta, Left, Right, Args), Context). dot(Meta, Left, {'__aliases__', _, Args}, nil) -> - {'__aliases__', Meta, [Left|Args]}; + {'__aliases__', Meta, [Left | Args]}; dot(Meta, Left, Right, nil) when is_atom(Right) -> case atom_to_list(Right) of "Elixir." ++ _ -> {'__aliases__', Meta, [Left, Right]}; _ -> - {{'.', Meta, [Left, Right]}, Meta, []} + {{'.', Meta, [Left, Right]}, [{no_parens, true} | Meta], []} end; dot(Meta, Left, {Right, _, Context}, nil) when is_atom(Right), is_atom(Context) -> - {{'.', Meta, [Left, Right]}, Meta, []}; + {{'.', Meta, [Left, Right]}, [{no_parens, true} | Meta], []}; dot(Meta, Left, {Right, _, Args}, nil) when is_atom(Right) -> {{'.', Meta, [Left, Right]}, Meta, Args}; @@ -105,8 +172,8 @@ tail_list(Left, Right, Tail) when is_list(Right), is_list(Tail) -> tail_list(Left, Right, Tail) when is_list(Left) -> validate_list(Left), - [H|T] = lists:reverse(Tail ++ Left), - lists:reverse([{'|', [], [H, Right]}|T]). + [H | T] = lists:reverse(Tail ++ Left), + lists:reverse([{'|', [], [H, Right]} | T]). validate_list(List) when is_list(List) -> ok; @@ -115,80 +182,134 @@ validate_list(List) when not is_list(List) -> ('Elixir.Kernel':inspect(List))/binary>>). argument_error(Message) -> - error('Elixir.ArgumentError':exception([{message,Message}])). + error('Elixir.ArgumentError':exception([{message, Message}])). -%% Annotates the AST with context and other info +%% Annotates the AST with context and other info. +%% +%% Note we need to delete the counter because linify +%% adds the counter recursively, even inside quoted +%% expressions, so we need to clean up the forms to +%% allow them to get a new counter on the next expansion. -annotate({Def, Meta, [{H, M, A}|T]}, Context, File) when ?defs(Def) -> - %% Store the context information in the first element of the - %% definition tuple so we can access it later on. - MM = keystore(context, keystore(file, M, File), Context), - {Def, Meta, [{H, MM, A}|T]}; -annotate({{'.', _, [_, Def]} = Target, Meta, [{H, M, A}|T]}, Context, File) when ?defs(Def) -> - MM = keystore(context, keystore(file, M, File), Context), - {Target, Meta, [{H, MM, A}|T]}; +annotate({Def, Meta, [{H, M, A} | T]}, Context) when ?defs(Def) -> + {Def, Meta, [{H, keystore(context, M, Context), A} | T]}; +annotate({{'.', _, [_, Def]} = Target, Meta, [{H, M, A} | T]}, Context) when ?defs(Def) -> + {Target, Meta, [{H, keystore(context, M, Context), A} | T]}; -annotate({Lexical, Meta, [_|_] = Args}, Context, _File) when Lexical == import; Lexical == alias -> +annotate({Lexical, Meta, [_ | _] = Args}, Context) when ?lexical(Lexical) -> NewMeta = keystore(context, keydelete(counter, Meta), Context), {Lexical, NewMeta, Args}; -annotate(Tree, _Context, _File) -> Tree. +annotate(Tree, _Context) -> Tree. + +has_unquotes({unquote, _, [_]}) -> true; +has_unquotes({unquote_splicing, _, [_]}) -> true; +has_unquotes({{'.', _, [_, unquote]}, _, [_]}) -> true; +has_unquotes({Var, _, Ctx}) when is_atom(Var), is_atom(Ctx) -> false; +has_unquotes({Name, _, Args}) when is_list(Args) -> + has_unquotes(Name) orelse lists:any(fun has_unquotes/1, Args); +has_unquotes({Left, Right}) -> + has_unquotes(Left) orelse has_unquotes(Right); +has_unquotes(List) when is_list(List) -> + lists:any(fun has_unquotes/1, List); +has_unquotes(_Other) -> false. %% Escapes the given expression. It is similar to quote, but %% lines are kept and hygiene mechanisms are disabled. -escape(Expr, Unquote) -> - {Res, Q} = quote(Expr, nil, #elixir_quote{ +escape(Expr, Kind, Unquote) -> + do_quote(Expr, #elixir_quote{ line=true, - keep=false, + file=nil, vars_hygiene=false, aliases_hygiene=false, imports_hygiene=false, - unquote=Unquote, - escape=true - }, nil), - {Res, Q#elixir_quote.unquoted}. + unquote=Unquote + }, Kind). + +%% fun_to_quoted + +fun_to_quoted(Function) -> + Meta = [], + {module, Module} = erlang:fun_info(Function, module), + {name, Name} = erlang:fun_info(Function, name), + {arity, Arity} = erlang:fun_info(Function, arity), + {'&', Meta, [{'/', Meta, [{{'.', Meta, [Module, Name]}, [{no_parens, true} | Meta], []}, Arity]}]}. %% Quotes an expression and return its quoted Elixir AST. -quote(Expr, nil, Q, E) -> - do_quote(Expr, Q, E); +quote(_Meta, {unquote_splicing, _, [_]}, _Binding, #elixir_quote{unquote=true}, _, _) -> + argument_error(<<"unquote_splicing only works inside arguments and block contexts, " + "wrap it in parens if you want it to work with one-liners">>); -quote(Expr, Binding, Q, E) -> +quote(Meta, Expr, Binding, Q, Prelude, E) -> Context = Q#elixir_quote.context, - Vars = [ {'{}', [], - [ '=', [], [ - {'{}', [], [K, [], Context]}, + Vars = [{'{}', [], + ['=', [], [ + {'{}', [], [K, Meta, Context]}, V - ] ] - } || {K, V} <- Binding], + ]] + } || {K, V} <- Binding], - {TExprs, TQ} = do_quote(Expr, Q, E), - {{'{}',[], ['__block__',[], Vars ++ [TExprs] ]}, TQ}. + Quoted = do_quote(Expr, Q, E), + + WithVars = case Vars of + [] -> Quoted; + _ -> {'{}', [], ['__block__', [], Vars ++ [Quoted]]} + end, + + case Prelude of + [] -> WithVars; + _ -> {'__block__', [], Prelude ++ [WithVars]} + end. %% Actual quoting and helpers -do_quote({quote, _, Args} = Tuple, #elixir_quote{unquote=true} = Q, E) when length(Args) == 1; length(Args) == 2 -> - {TTuple, TQ} = do_quote_tuple(Tuple, Q#elixir_quote{unquote=false}, E), - {TTuple, TQ#elixir_quote{unquote=true}}; +do_quote({quote, Meta, [Arg]}, Q, E) -> + TArg = do_quote(Arg, Q#elixir_quote{unquote=false}, E), -do_quote({unquote, _Meta, [Expr]}, #elixir_quote{unquote=true} = Q, _) -> - {Expr, Q#elixir_quote{unquoted=true}}; + NewMeta = case Q of + #elixir_quote{vars_hygiene=true, context=Context} -> keystore(context, Meta, Context); + _ -> Meta + end, -%% Aliases + {'{}', [], [quote, meta(NewMeta, Q), [TArg]]}; + +do_quote({quote, Meta, [Opts, Arg]}, Q, E) -> + TOpts = do_quote(Opts, Q, E), + TArg = do_quote(Arg, Q#elixir_quote{unquote=false}, E), -do_quote({'__aliases__', Meta, [H|T]} = Alias, #elixir_quote{aliases_hygiene=true} = Q, E) when is_atom(H) and (H /= 'Elixir') -> - Annotation = case elixir_aliases:expand(Alias, ?m(E, aliases), - ?m(E, macro_aliases), ?m(E, lexical_tracker)) of - Atom when is_atom(Atom) -> Atom; - Aliases when is_list(Aliases) -> false + NewMeta = case Q of + #elixir_quote{vars_hygiene=true, context=Context} -> keystore(context, Meta, Context); + _ -> Meta end, + + {'{}', [], [quote, meta(NewMeta, Q), [TOpts, TArg]]}; + +do_quote({unquote, _Meta, [Expr]}, #elixir_quote{unquote=true}, _) -> + Expr; + +%% Aliases + +do_quote({'__aliases__', Meta, [H | T]} = Alias, #elixir_quote{aliases_hygiene=true} = Q, E) + when is_atom(H), H /= 'Elixir' -> + Annotation = + case elixir_aliases:expand(Alias, E) of + Atom when is_atom(Atom) -> Atom; + Aliases when is_list(Aliases) -> false + end, AliasMeta = keystore(alias, keydelete(counter, Meta), Annotation), - do_quote_tuple({'__aliases__', AliasMeta, [H|T]}, Q, E); + do_quote_tuple('__aliases__', AliasMeta, [H | T], Q, E); %% Vars -do_quote({Left, Meta, nil}, #elixir_quote{vars_hygiene=true} = Q, E) when is_atom(Left) -> - do_quote_tuple({Left, Meta, Q#elixir_quote.context}, Q, E); +do_quote({Name, Meta, nil}, #elixir_quote{vars_hygiene=true} = Q, E) + when is_atom(Name), is_list(Meta) -> + ImportMeta = if + Q#elixir_quote.imports_hygiene -> import_meta(Meta, Name, 0, Q, E); + true -> Meta + end, + + {'{}', [], [Name, meta(ImportMeta, Q), Q#elixir_quote.context]}; %% Unquote @@ -201,34 +322,29 @@ do_quote({{'.', Meta, [Left, unquote]}, _, [Expr]}, #elixir_quote{unquote=true} %% Imports do_quote({'&', Meta, [{'/', _, [{F, _, C}, A]}] = Args}, - #elixir_quote{imports_hygiene=true} = Q, E) when is_atom(F), is_integer(A), is_atom(C) -> - do_quote_fa('&', Meta, Args, F, A, Q, E); + #elixir_quote{imports_hygiene=true} = Q, E) when is_atom(F), is_integer(A), is_atom(C) -> + NewMeta = + case elixir_dispatch:find_import(Meta, F, A, E) of + false -> + Meta; -do_quote({Name, Meta, ArgsOrAtom}, #elixir_quote{imports_hygiene=true} = Q, E) when is_atom(Name) -> - Arity = case is_atom(ArgsOrAtom) of - true -> 0; - false -> length(ArgsOrAtom) - end, + Receiver -> + keystore(context, keystore(imports, Meta, [{Receiver, A}]), Q#elixir_quote.context) + end, + do_quote_tuple('&', NewMeta, Args, Q, E); - NewMeta = case (keyfind(import, Meta) == false) andalso - elixir_dispatch:find_import(Meta, Name, Arity, E) of - false -> - case (Arity == 1) andalso keyfind(ambiguous_op, Meta) of - {ambiguous_op, nil} -> keystore(ambiguous_op, Meta, Q#elixir_quote.context); - _ -> Meta - end; - Receiver -> - keystore(import, keystore(context, Meta, Q#elixir_quote.context), Receiver) +do_quote({Name, Meta, ArgsOrContext}, #elixir_quote{imports_hygiene=true} = Q, E) + when is_atom(Name), is_list(Meta), is_list(ArgsOrContext) or is_atom(ArgsOrContext) -> + Arity = if + is_atom(ArgsOrContext) -> 0; + true -> length(ArgsOrContext) end, - Annotated = annotate({Name, NewMeta, ArgsOrAtom}, Q#elixir_quote.context, file(E, Q)), + ImportMeta = import_meta(Meta, Name, Arity, Q, E), + Annotated = annotate({Name, ImportMeta, ArgsOrContext}, Q#elixir_quote.context), do_quote_tuple(Annotated, Q, E); -do_quote({_, _, _} = Tuple, #elixir_quote{escape=false} = Q, E) -> - Annotated = annotate(Tuple, Q#elixir_quote.context, file(E, Q)), - do_quote_tuple(Annotated, Q, E); - -%% Literals +%% Two-element tuples do_quote({Left, Right}, #elixir_quote{unquote=true} = Q, E) when is_tuple(Left) andalso (element(1, Left) == unquote_splicing); @@ -236,75 +352,187 @@ do_quote({Left, Right}, #elixir_quote{unquote=true} = Q, E) when do_quote({'{}', [], [Left, Right]}, Q, E); do_quote({Left, Right}, Q, E) -> - {TLeft, LQ} = do_quote(Left, Q, E), - {TRight, RQ} = do_quote(Right, LQ, E), - {{TLeft, TRight}, RQ}; - -do_quote(Map, #elixir_quote{escape=true} = Q, E) when is_map(Map) -> - {TT, TQ} = do_quote(maps:to_list(Map), Q, E), - {{'%{}', [], TT}, TQ}; - -do_quote(Tuple, #elixir_quote{escape=true} = Q, E) when is_tuple(Tuple) -> - {TT, TQ} = do_quote(tuple_to_list(Tuple), Q, E), - {{'{}', [], TT}, TQ}; - -do_quote(List, #elixir_quote{escape=true} = Q, E) when is_list(List) -> - % The improper case is pretty inefficient, but improper lists are are. - case reverse_improper(List) of - {L} -> do_splice(L, Q, E); - {L, R} -> - {TL, QL} = do_splice(L, Q, E, [], []), - {TR, QR} = do_quote(R, QL, E), - {update_last(TL, fun(X) -> {'|', [], [X, TR]} end), QR} + TLeft = do_quote(Left, Q, E), + TRight = do_quote(Right, Q, E), + {TLeft, TRight}; + +%% Everything else + +do_quote(Other, Q, E) when is_atom(E) -> + do_escape(Other, Q, E); + +do_quote({_, _, _} = Tuple, Q, E) -> + Annotated = annotate(Tuple, Q#elixir_quote.context), + do_quote_tuple(Annotated, Q, E); + +do_quote([], _, _) -> + []; + +do_quote([H | T], #elixir_quote{unquote=false} = Q, E) -> + do_quote_simple_list(T, do_quote(H, Q, E), Q, E); + +do_quote([H | T], Q, E) -> + do_quote_tail(lists:reverse(T, [H]), Q, E); + +do_quote(Other, _, _) -> + Other. + +%% do_escape + +do_escape({Left, Meta, Right}, Q, E = prune_metadata) -> + TM = [{K, V} || {K, V} <- Meta, (K == no_parens) orelse (K == line)], + TL = do_quote(Left, Q, E), + TR = do_quote(Right, Q, E), + {'{}', [], [TL, TM, TR]}; + +do_escape(Tuple, Q, E) when is_tuple(Tuple) -> + TT = do_quote(tuple_to_list(Tuple), Q, E), + {'{}', [], TT}; + +do_escape(BitString, _, _) when is_bitstring(BitString) -> + case bit_size(BitString) rem 8 of + 0 -> + BitString; + Size -> + <> = BitString, + {'<<>>', [], [{'::', [], [Bits, {size, [], [Size]}]}, {'::', [], [Bytes, {binary, [], []}]}]} end; -do_quote(List, Q, E) when is_list(List) -> - do_splice(lists:reverse(List), Q, E); -do_quote(Other, Q, _) -> - {Other, Q}. +do_escape(Map, Q, E) when is_map(Map) -> + TT = do_quote(maps:to_list(Map), Q, E), + {'%{}', [], TT}; + +do_escape([], _, _) -> []; + +do_escape([H | T], #elixir_quote{unquote=false} = Q, E) -> + do_quote_simple_list(T, do_quote(H, Q, E), Q, E); + +do_escape([H | T], Q, E) -> + %% The improper case is inefficient, but improper lists are rare. + try lists:reverse(T, [H]) of + L -> do_quote_tail(L, Q, E) + catch + _:_ -> + {L, R} = reverse_improper(T, [H]), + TL = do_quote_splice(L, Q, E, [], []), + TR = do_quote(R, Q, E), + update_last(TL, fun(X) -> {'|', [], [X, TR]} end) + end; -%% Quote helpers +do_escape(Other, _, _) + when is_number(Other); is_pid(Other); is_atom(Other) -> + Other; -do_quote_call(Left, Meta, Expr, Args, Q, E) -> - All = [meta(Meta, Q), Left, {unquote, Meta, [Expr]}, Args, - Q#elixir_quote.context, file(E, Q)], - {TAll, TQ} = lists:mapfoldl(fun(X, Acc) -> do_quote(X, Acc, E) end, Q, All), - {{{'.', Meta, [elixir_quote, dot]}, Meta, TAll}, TQ}. +do_escape(Fun, _, _) when is_function(Fun) -> + case (erlang:fun_info(Fun, env) == {env, []}) andalso + (erlang:fun_info(Fun, type) == {type, external}) of + true -> fun_to_quoted(Fun); + false -> bad_escape(Fun) + end; -do_quote_fa(Target, Meta, Args, F, A, Q, E) -> - NewMeta = - case (keyfind(import_fa, Meta) == false) andalso - elixir_dispatch:find_import(Meta, F, A, E) of - false -> Meta; - Receiver -> keystore(import_fa, Meta, {Receiver, Q#elixir_quote.context}) - end, - do_quote_tuple({Target, NewMeta, Args}, Q, E). +do_escape(Other, _, _) -> + bad_escape(Other). + +bad_escape(Arg) -> + argument_error(<<"cannot escape ", ('Elixir.Kernel':inspect(Arg, []))/binary, ". ", + "The supported values are: lists, tuples, maps, atoms, numbers, bitstrings, ", + "PIDs and remote functions in the format &Mod.fun/arity">>). + +import_meta(Meta, Name, Arity, Q, E) -> + case (keyfind(import, Meta) == false) andalso + elixir_dispatch:find_imports(Meta, Name, Arity, E) of + [] -> + case (Arity == 1) andalso keyfind(ambiguous_op, Meta) of + {ambiguous_op, nil} -> keystore(ambiguous_op, Meta, Q#elixir_quote.context); + _ -> Meta + end; + + Imports -> + keystore(imports, keystore(context, Meta, Q#elixir_quote.context), Imports) + end. + +%% do_quote_* + +do_quote_call(Left, Meta, Expr, Args, Q, E) -> + All = [Left, {unquote, Meta, [Expr]}, Args, Q#elixir_quote.context], + TAll = [do_quote(X, Q, E) || X <- All], + {{'.', Meta, [elixir_quote, dot]}, Meta, [meta(Meta, Q) | TAll]}. do_quote_tuple({Left, Meta, Right}, Q, E) -> - {TLeft, LQ} = do_quote(Left, Q, E), - {TRight, RQ} = do_quote(Right, LQ, E), - {{'{}', [], [TLeft, meta(Meta, Q), TRight]}, RQ}. + do_quote_tuple(Left, Meta, Right, Q, E). + +do_quote_tuple(Left, Meta, Right, Q, E) -> + TLeft = do_quote(Left, Q, E), + TRight = do_quote(Right, Q, E), + {'{}', [], [TLeft, meta(Meta, Q), TRight]}. + +do_quote_simple_list([], Prev, _, _) -> [Prev]; +do_quote_simple_list([H | T], Prev, Q, E) -> + [Prev | do_quote_simple_list(T, do_quote(H, Q, E), Q, E)]; +do_quote_simple_list(Other, Prev, Q, E) -> + [{'|', [], [Prev, do_quote(Other, Q, E)]}]. + +do_quote_tail([{'|', Meta, [{unquote_splicing, _, [Left]}, Right]} | T], #elixir_quote{unquote=true} = Q, E) -> + %% Process the remaining entries on the list. + %% For [1, 2, 3, unquote_splicing(arg) | tail], this will quote + %% 1, 2 and 3, which could even be unquotes. + TT = do_quote_splice(T, Q, E, [], []), + TR = do_quote(Right, Q, E), + do_runtime_list(Meta, tail_list, [Left, TR, TT]); + +do_quote_tail(List, Q, E) -> + do_quote_splice(List, Q, E, [], []). -file(#{file := File}, #elixir_quote{keep=true}) -> File; -file(_, _) -> nil. +do_quote_splice([{unquote_splicing, Meta, [Expr]} | T], #elixir_quote{unquote=true} = Q, E, Buffer, Acc) -> + Runtime = do_runtime_list(Meta, list, [Expr, do_list_concat(Buffer, Acc)]), + do_quote_splice(T, Q, E, [], Runtime); -meta(Meta, #elixir_quote{keep=true}) -> - [case KV of {line, V} -> {keep, V}; _ -> KV end || KV <- Meta]; -meta(Meta, #elixir_quote{line=true}) -> +do_quote_splice([H | T], Q, E, Buffer, Acc) -> + TH = do_quote(H, Q, E), + do_quote_splice(T, Q, E, [TH | Buffer], Acc); + +do_quote_splice([], _Q, _E, Buffer, Acc) -> + do_list_concat(Buffer, Acc). + +do_list_concat(Left, []) -> Left; +do_list_concat([], Right) -> Right; +do_list_concat(Left, Right) -> {{'.', [], [erlang, '++']}, [], [Left, Right]}. + +do_runtime_list(Meta, Fun, Args) -> + {{'.', Meta, [elixir_quote, Fun]}, Meta, Args}. + +%% Helpers + +meta(Meta, Q) -> + generated(keep(Meta, Q), Q). + +generated(Meta, #elixir_quote{generated=true}) -> [{generated, true} | Meta]; +generated(Meta, #elixir_quote{generated=false}) -> Meta. + +keep(Meta, #elixir_quote{file=nil, line=Line}) -> + line(Meta, Line); +keep(Meta, #elixir_quote{file=File}) -> + case lists:keytake(line, 1, Meta) of + {value, {line, Line}, MetaNoLine} -> + [{keep, {File, Line}} | MetaNoLine]; + false -> + [{keep, {File, 0}} | Meta] + end. + +line(Meta, true) -> Meta; -meta(Meta, #elixir_quote{line=false}) -> +line(Meta, false) -> keydelete(line, Meta); -meta(Meta, #elixir_quote{line=Line}) -> +line(Meta, Line) -> keystore(line, Meta, Line). -reverse_improper(L) -> reverse_improper(L, []). -reverse_improper([], Acc) -> {Acc}; -reverse_improper([H|T], Acc) when is_list(T) -> reverse_improper(T, [H|Acc]); -reverse_improper([H|T], Acc) -> {[H|Acc], T}. +reverse_improper([H | T], Acc) -> reverse_improper(T, [H | Acc]); +reverse_improper([], Acc) -> Acc; +reverse_improper(T, Acc) -> {Acc, T}. update_last([], _) -> []; update_last([H], F) -> [F(H)]; -update_last([H|T], F) -> [H|update_last(T,F)]. +update_last([H | T], F) -> [H | update_last(T, F)]. keyfind(Key, Meta) -> lists:keyfind(Key, 1, Meta). @@ -314,42 +542,8 @@ keystore(_Key, Meta, nil) -> Meta; keystore(Key, Meta, Value) -> lists:keystore(Key, 1, Meta, {Key, Value}). -keyreplace(Key, Meta, {Key, _V}) -> - Meta; -keyreplace(Key, Meta, Tuple) -> - lists:keyreplace(Key, 1, Meta, Tuple). keynew(Key, Meta, Value) -> - case keyfind(Key, Meta) of - {Key, _} -> Meta; - _ -> keystore(Key, Meta, Value) + case lists:keymember(Key, 1, Meta) of + true -> Meta; + false -> [{Key, Value} | Meta] end. - -%% Quote splicing - -do_splice([{'|', Meta, [{unquote_splicing, _, [Left]}, Right]}|T], #elixir_quote{unquote=true} = Q, E) -> - %% Process the remaining entries on the list. - %% For [1, 2, 3, unquote_splicing(arg)|tail], this will quote - %% 1, 2 and 3, which could even be unquotes. - {TT, QT} = do_splice(T, Q, E, [], []), - {TR, QR} = do_quote(Right, QT, E), - {do_runtime_list(Meta, tail_list, [Left, TR, TT]), QR#elixir_quote{unquoted=true}}; - -do_splice(List, Q, E) -> - do_splice(List, Q, E, [], []). - -do_splice([{unquote_splicing, Meta, [Expr]}|T], #elixir_quote{unquote=true} = Q, E, Buffer, Acc) -> - do_splice(T, Q#elixir_quote{unquoted=true}, E, [], do_runtime_list(Meta, list, [Expr, do_join(Buffer, Acc)])); - -do_splice([H|T], Q, E, Buffer, Acc) -> - {TH, TQ} = do_quote(H, Q, E), - do_splice(T, TQ, E, [TH|Buffer], Acc); - -do_splice([], Q, _E, Buffer, Acc) -> - {do_join(Buffer, Acc), Q}. - -do_join(Left, []) -> Left; -do_join([], Right) -> Right; -do_join(Left, Right) -> {{'.', [], [erlang, '++']}, [], [Left, Right]}. - -do_runtime_list(Meta, Fun, Args) -> - {{'.', Meta, [elixir_quote, Fun]}, Meta, Args}. diff --git a/lib/elixir/src/elixir_rewrite.erl b/lib/elixir/src/elixir_rewrite.erl new file mode 100644 index 00000000000..138bf8082de --- /dev/null +++ b/lib/elixir/src/elixir_rewrite.erl @@ -0,0 +1,360 @@ +-module(elixir_rewrite). +-compile({inline, [inner_inline/4, inner_rewrite/5]}). +-export([erl_to_ex/3, inline/3, rewrite/5, match_rewrite/5, guard_rewrite/6, format_error/1]). +-include("elixir.hrl"). + +%% Convenience variables + +-define(atom, 'Elixir.Atom'). +-define(bitwise, 'Elixir.Bitwise'). +-define(enum, 'Elixir.Enum'). +-define(function, 'Elixir.Function'). +-define(integer, 'Elixir.Integer'). +-define(io, 'Elixir.IO'). +-define(kernel, 'Elixir.Kernel'). +-define(list, 'Elixir.List'). +-define(map, 'Elixir.Map'). +-define(node, 'Elixir.Node'). +-define(port, 'Elixir.Port'). +-define(process, 'Elixir.Process'). +-define(string, 'Elixir.String'). +-define(string_chars, 'Elixir.String.Chars'). +-define(system, 'Elixir.System'). +-define(tuple, 'Elixir.Tuple'). + +% Macros used to define inline and rewrite rules. +% Defines the rules from Elixir function to Erlang function +% and the reverse, rewrites that are not reversible or have +% complex rules are defined without the macros. +-define( + inline(ExMod, ExFun, Arity, ErlMod, ErlFun), + inner_inline(ex_to_erl, ExMod, ExFun, Arity) -> {ErlMod, ErlFun}; + inner_inline(erl_to_ex, ErlMod, ErlFun, Arity) -> {ExMod, ExFun} +). + +-define( + rewrite(ExMod, ExFun, ExArgs, ErlMod, ErlFun, ErlArgs), + inner_rewrite(ex_to_erl, _Meta, ExMod, ExFun, ExArgs) -> {ErlMod, ErlFun, ErlArgs}; + inner_rewrite(erl_to_ex, _Meta, ErlMod, ErlFun, ErlArgs) -> {ExMod, ExFun, ExArgs} +). + +erl_to_ex(Mod, Fun, Args) -> + case inner_inline(erl_to_ex, Mod, Fun, length(Args)) of + false -> inner_rewrite(erl_to_ex, [], Mod, Fun, Args); + {ExMod, ExFun} -> {ExMod, ExFun, Args} + end. + +%% Inline rules +%% +%% Inline rules are straightforward, they keep the same +%% number and order of arguments and show up on captures. +inline(Mod, Fun, Arity) -> inner_inline(ex_to_erl, Mod, Fun, Arity). + +?inline(?atom, to_charlist, 1, erlang, atom_to_list); +?inline(?atom, to_string, 1, erlang, atom_to_binary); + +?inline(?bitwise, 'bnot', 1, erlang, 'bnot'); +?inline(?bitwise, 'band', 2, erlang, 'band'); +?inline(?bitwise, 'bor', 2, erlang, 'bor'); +?inline(?bitwise, 'bxor', 2, erlang, 'bxor'); +?inline(?bitwise, 'bsl', 2, erlang, 'bsl'); +?inline(?bitwise, 'bsr', 2, erlang, 'bsr'); + +?inline(?function, capture, 3, erlang, make_fun); +?inline(?function, info, 1, erlang, fun_info); +?inline(?function, info, 2, erlang, fun_info); + +?inline(?integer, to_charlist, 1, erlang, integer_to_list); +?inline(?integer, to_charlist, 2, erlang, integer_to_list); +?inline(?integer, to_string, 1, erlang, integer_to_binary); +?inline(?integer, to_string, 2, erlang, integer_to_binary); + +?inline(?io, iodata_length, 1, erlang, iolist_size); +?inline(?io, iodata_to_binary, 1, erlang, iolist_to_binary); + +?inline(?kernel, '!=', 2, erlang, '/='); +?inline(?kernel, '!==', 2, erlang, '=/='); +?inline(?kernel, '*', 2, erlang, '*'); +?inline(?kernel, '+', 1, erlang, '+'); +?inline(?kernel, '+', 2, erlang, '+'); +?inline(?kernel, '++', 2, erlang, '++'); +?inline(?kernel, '-', 1, erlang, '-'); +?inline(?kernel, '-', 2, erlang, '-'); +?inline(?kernel, '--', 2, erlang, '--'); +?inline(?kernel, '/', 2, erlang, '/'); +?inline(?kernel, '<', 2, erlang, '<'); +?inline(?kernel, '<=', 2, erlang, '=<'); +?inline(?kernel, '==', 2, erlang, '=='); +?inline(?kernel, '===', 2, erlang, '=:='); +?inline(?kernel, '>', 2, erlang, '>'); +?inline(?kernel, '>=', 2, erlang, '>='); +?inline(?kernel, abs, 1, erlang, abs); +?inline(?kernel, apply, 2, erlang, apply); +?inline(?kernel, apply, 3, erlang, apply); +?inline(?kernel, binary_part, 3, erlang, binary_part); +?inline(?kernel, bit_size, 1, erlang, bit_size); +?inline(?kernel, byte_size, 1, erlang, byte_size); +?inline(?kernel, ceil, 1, erlang, ceil); +?inline(?kernel, 'div', 2, erlang, 'div'); +?inline(?kernel, exit, 1, erlang, exit); +?inline(?kernel, floor, 1, erlang, floor); +?inline(?kernel, 'function_exported?', 3, erlang, function_exported); +?inline(?kernel, hd, 1, erlang, hd); +?inline(?kernel, is_atom, 1, erlang, is_atom); +?inline(?kernel, is_binary, 1, erlang, is_binary); +?inline(?kernel, is_bitstring, 1, erlang, is_bitstring); +?inline(?kernel, is_boolean, 1, erlang, is_boolean); +?inline(?kernel, is_float, 1, erlang, is_float); +?inline(?kernel, is_function, 1, erlang, is_function); +?inline(?kernel, is_function, 2, erlang, is_function); +?inline(?kernel, is_integer, 1, erlang, is_integer); +?inline(?kernel, is_list, 1, erlang, is_list); +?inline(?kernel, is_map, 1, erlang, is_map); +?inline(?kernel, is_number, 1, erlang, is_number); +?inline(?kernel, is_pid, 1, erlang, is_pid); +?inline(?kernel, is_port, 1, erlang, is_port); +?inline(?kernel, is_reference, 1, erlang, is_reference); +?inline(?kernel, is_tuple, 1, erlang, is_tuple); +?inline(?kernel, length, 1, erlang, length); +?inline(?kernel, make_ref, 0, erlang, make_ref); +?inline(?kernel, map_size, 1, erlang, map_size); +?inline(?kernel, max, 2, erlang, max); +?inline(?kernel, min, 2, erlang, min); +?inline(?kernel, node, 0, erlang, node); +?inline(?kernel, node, 1, erlang, node); +?inline(?kernel, 'not', 1, erlang, 'not'); +?inline(?kernel, 'rem', 2, erlang, 'rem'); +?inline(?kernel, round, 1, erlang, round); +?inline(?kernel, self, 0, erlang, self); +?inline(?kernel, send, 2, erlang, send); +?inline(?kernel, spawn, 1, erlang, spawn); +?inline(?kernel, spawn, 3, erlang, spawn); +?inline(?kernel, spawn_link, 1, erlang, spawn_link); +?inline(?kernel, spawn_link, 3, erlang, spawn_link); +?inline(?kernel, spawn_monitor, 1, erlang, spawn_monitor); +?inline(?kernel, spawn_monitor, 3, erlang, spawn_monitor); +?inline(?kernel, throw, 1, erlang, throw); +?inline(?kernel, tl, 1, erlang, tl); +?inline(?kernel, trunc, 1, erlang, trunc); +?inline(?kernel, tuple_size, 1, erlang, tuple_size); + +?inline(?list, to_atom, 1, erlang, list_to_atom); +?inline(?list, to_existing_atom, 1, erlang, list_to_existing_atom); +?inline(?list, to_float, 1, erlang, list_to_float); +?inline(?list, to_integer, 1, erlang, list_to_integer); +?inline(?list, to_integer, 2, erlang, list_to_integer); +?inline(?list, to_tuple, 1, erlang, list_to_tuple); + +?inline(?map, keys, 1, maps, keys); +?inline(?map, merge, 2, maps, merge); +?inline(?map, to_list, 1, maps, to_list); +?inline(?map, values, 1, maps, values); + +?inline(?node, list, 0, erlang, nodes); +?inline(?node, list, 1, erlang, nodes); +?inline(?node, spawn, 2, erlang, spawn); +?inline(?node, spawn, 3, erlang, spawn_opt); +?inline(?node, spawn, 4, erlang, spawn); +?inline(?node, spawn, 5, erlang, spawn_opt); +?inline(?node, spawn_link, 2, erlang, spawn_link); +?inline(?node, spawn_link, 4, erlang, spawn_link); +?inline(?node, spawn_monitor, 2, erlang, spawn_monitor); +?inline(?node, spawn_monitor, 4, erlang, spawn_monitor); + +?inline(?port, close, 1, erlang, port_close); +?inline(?port, command, 2, erlang, port_command); +?inline(?port, command, 3, erlang, port_command); +?inline(?port, connect, 2, erlang, port_connect); +?inline(?port, list, 0, erlang, ports); +?inline(?port, open, 2, erlang, open_port); + +?inline(?process, 'alive?', 1, erlang, is_process_alive); +?inline(?process, cancel_timer, 1, erlang, cancel_timer); +?inline(?process, cancel_timer, 2, erlang, cancel_timer); +?inline(?process, demonitor, 1, erlang, demonitor); +?inline(?process, demonitor, 2, erlang, demonitor); +?inline(?process, exit, 2, erlang, exit); +?inline(?process, flag, 2, erlang, process_flag); +?inline(?process, flag, 3, erlang, process_flag); +?inline(?process, get, 0, erlang, get); +?inline(?process, get_keys, 0, erlang, get_keys); +?inline(?process, get_keys, 1, erlang, get_keys); +?inline(?process, group_leader, 0, erlang, group_leader); +?inline(?process, hibernate, 3, erlang, hibernate); +?inline(?process, link, 1, erlang, link); +?inline(?process, list, 0, erlang, processes); +?inline(?process, read_timer, 1, erlang, read_timer); +?inline(?process, registered, 0, erlang, registered); +?inline(?process, send, 3, erlang, send); +?inline(?process, spawn, 2, erlang, spawn_opt); +?inline(?process, spawn, 4, erlang, spawn_opt); +?inline(?process, unlink, 1, erlang, unlink); +?inline(?process, unregister, 1, erlang, unregister); + +?inline(?string, duplicate, 2, binary, copy); +?inline(?string, to_float, 1, erlang, binary_to_float); +?inline(?string, to_integer, 1, erlang, binary_to_integer); +?inline(?string, to_integer, 2, erlang, binary_to_integer); + +?inline(?system, monotonic_time, 0, erlang, monotonic_time); +?inline(?system, os_time, 0, os, system_time); +?inline(?system, system_time, 0, erlang, system_time); +?inline(?system, time_offset, 0, erlang, time_offset); +?inline(?system, unique_integer, 0, erlang, unique_integer); +?inline(?system, unique_integer, 1, erlang, unique_integer); + +?inline(?tuple, append, 2, erlang, append_element); +?inline(?tuple, to_list, 1, erlang, tuple_to_list); + +% Defined without macro to avoid conflict with Bitwise named operators +inner_inline(ex_to_erl, ?bitwise, '~~~', 1) -> {erlang, 'bnot'}; +inner_inline(ex_to_erl, ?bitwise, '&&&', 2) -> {erlang, 'band'}; +inner_inline(ex_to_erl, ?bitwise, '|||', 2) -> {erlang, 'bor'}; +inner_inline(ex_to_erl, ?bitwise, '^^^', 2) -> {erlang, 'bxor'}; +inner_inline(ex_to_erl, ?bitwise, '<<<', 2) -> {erlang, 'bsl'}; +inner_inline(ex_to_erl, ?bitwise, '>>>', 2) -> {erlang, 'bsr'}; + +% Defined without macro to avoid conflict with Process.demonitor +inner_inline(ex_to_erl, ?port, demonitor, 1) -> {erlang, demonitor}; +inner_inline(ex_to_erl, ?port, demonitor, 2) -> {erlang, demonitor}; + +inner_inline(_, _, _, _) -> false. + +%% Rewrite rules +%% +%% Rewrite rules are more complex than regular inlining code +%% as they may change the number of arguments. However, they +%% don't add new code (such as case statements), at best they +%% perform dead code removal. +rewrite(?string_chars, DotMeta, to_string, Meta, [Arg]) -> + case is_always_string(Arg) of + true -> Arg; + false -> {{'.', DotMeta, [?string_chars, to_string]}, Meta, [Arg]} + end; +rewrite(Receiver, DotMeta, Right, Meta, Args) -> + {EReceiver, ERight, EArgs} = inner_rewrite(ex_to_erl, DotMeta, Receiver, Right, Args), + {{'.', DotMeta, [EReceiver, ERight]}, Meta, EArgs}. + +?rewrite(?kernel, is_map_key, [Map, Key], erlang, is_map_key, [Key, Map]); +?rewrite(?map, delete, [Map, Key], maps, remove, [Key, Map]); +?rewrite(?map, fetch, [Map, Key], maps, find, [Key, Map]); +?rewrite(?map, 'fetch!', [Map, Key], maps, get, [Key, Map]); +?rewrite(?map, 'has_key?', [Map, Key], maps, is_key, [Key, Map]); +?rewrite(?map, put, [Map, Key, Value], maps, put, [Key, Value, Map]); +?rewrite(?map, 'replace!', [Map, Key, Value], maps, update, [Key, Value, Map]); +?rewrite(?port, monitor, [Arg], erlang, monitor, [port, Arg]); +?rewrite(?process, group_leader, [Pid, Leader], erlang, group_leader, [Leader, Pid]); +?rewrite(?process, monitor, [Arg], erlang, monitor, [process, Arg]); +?rewrite(?process, send_after, [Dest, Msg, Time], erlang, send_after, [Time, Dest, Msg]); +?rewrite(?process, send_after, [Dest, Msg, Time, Opts], erlang, send_after, [Time, Dest, Msg, Opts]); +?rewrite(?string, to_atom, [Arg], erlang, binary_to_atom, [Arg, utf8]); +?rewrite(?string, to_existing_atom, [Arg], erlang, binary_to_existing_atom, [Arg, utf8]); +?rewrite(?tuple, duplicate, [Data, Size], erlang, make_tuple, [Size, Data]); + +inner_rewrite(ex_to_erl, Meta, ?tuple, delete_at, [Tuple, Index]) -> + {erlang, delete_element, [increment(Meta, Index), Tuple]}; +inner_rewrite(ex_to_erl, Meta, ?tuple, insert_at, [Tuple, Index, Term]) -> + {erlang, insert_element, [increment(Meta, Index), Tuple, Term]}; +inner_rewrite(ex_to_erl, Meta, ?kernel, elem, [Tuple, Index]) -> + {erlang, element, [increment(Meta, Index), Tuple]}; +inner_rewrite(ex_to_erl, Meta, ?kernel, put_elem, [Tuple, Index, Value]) -> + {erlang, setelement, [increment(Meta, Index), Tuple, Value]}; + +inner_rewrite(erl_to_ex, _Meta, erlang, delete_element, [Index, Tuple]) when is_number(Index) -> + {?tuple, delete_at, [Tuple, Index - 1]}; +inner_rewrite(erl_to_ex, _Meta, erlang, insert_element, [Index, Tuple, Term]) when is_number(Index) -> + {?tuple, insert_at, [Tuple, Index - 1, Term]}; +inner_rewrite(erl_to_ex, _Meta, erlang, element, [Index, Tuple]) when is_number(Index) -> + {?kernel, elem, [Tuple, Index - 1]}; +inner_rewrite(erl_to_ex, _Meta, erlang, setelement, [Index, Tuple, Value]) when is_number(Index) -> + {?kernel, put_elem, [Tuple, Index - 1, Value]}; + +inner_rewrite(erl_to_ex, _Meta, erlang, delete_element, [{{'.', _, [erlang, '+']}, _, [Index, 1]}, Tuple]) -> + {?tuple, delete_at, [Tuple, Index]}; +inner_rewrite(erl_to_ex, _Meta, erlang, insert_element, [{{'.', _, [erlang, '+']}, _, [Index, 1]}, Tuple, Term]) -> + {?tuple, insert_at, [Tuple, Index, Term]}; +inner_rewrite(erl_to_ex, _Meta, erlang, element, [{{'.', _, [erlang, '+']}, _, [Index, 1]}, Tuple]) -> + {?kernel, elem, [Tuple, Index]}; +inner_rewrite(erl_to_ex, _Meta, erlang, setelement, [{{'.', _, [erlang, '+']}, _, [Index, 1]}, Tuple, Value]) -> + {?kernel, put_elem, [Tuple, Index, Value]}; + +inner_rewrite(erl_to_ex, _Meta, erlang, 'orelse', [_, _] = Args) -> + {?kernel, 'or', Args}; +inner_rewrite(erl_to_ex, _Meta, erlang, 'andalso', [_, _] = Args) -> + {?kernel, 'and', Args}; + +inner_rewrite(_To, _Meta, Mod, Fun, Args) -> {Mod, Fun, Args}. + +increment(_Meta, Number) when is_number(Number) -> + Number + 1; +increment(Meta, Other) -> + {{'.', Meta, [erlang, '+']}, Meta, [Other, 1]}. + +%% Match rewrite +%% +%% Match rewrite is similar to regular rewrite, except +%% it also verifies the rewrite rule applies in a match context. +%% The allowed operations are very limited. +%% The Kernel operators are already inlined by now, we only need to +%% care about Erlang ones. +match_rewrite(erlang, _, '+', _, [Arg]) when is_number(Arg) -> {ok, Arg}; +match_rewrite(erlang, _, '-', _, [Arg]) when is_number(Arg) -> {ok, -Arg}; +match_rewrite(erlang, _, '++', Meta, [Left, Right]) -> + try {ok, static_append(Left, Right, Meta)} + catch impossible -> {error, {invalid_match_append, Left}} + end; +match_rewrite(Receiver, _, Right, _, Args) -> + {error, {invalid_match, Receiver, Right, length(Args)}}. + +static_append([], Right, _Meta) -> Right; +static_append([{'|', InnerMeta, [Head, Tail]}], Right, Meta) when is_list(Tail) -> + [{'|', InnerMeta, [Head, static_append(Tail, Right, Meta)]}]; +static_append([{'|', _, [_, _]}], _, _) -> throw(impossible); +static_append([Last], Right, Meta) -> [{'|', Meta, [Last, Right]}]; +static_append([Head | Tail], Right, Meta) -> [Head | static_append(Tail, Right, Meta)]; +static_append(_, _, _) -> throw(impossible). + +%% Guard rewrite +%% +%% Guard rewrite is similar to regular rewrite, except +%% it also verifies the resulting function is supported in +%% guard context - only certain BIFs and operators are. +guard_rewrite(Receiver, DotMeta, Right, Meta, Args, Context) -> + case inner_rewrite(ex_to_erl, DotMeta, Receiver, Right, Args) of + {erlang, RRight, RArgs} -> + case allowed_guard(RRight, length(RArgs)) of + true -> {ok, {{'.', DotMeta, [erlang, RRight]}, Meta, RArgs}}; + false -> {error, {invalid_guard, Receiver, Right, length(Args), Context}} + end; + _ -> {error, {invalid_guard, Receiver, Right, length(Args), Context}} + end. + +%% erlang:is_record/2-3 are compiler guards in Erlang which we +%% need to explicitly forbid as they are allowed in erl_internal. +allowed_guard(is_record, 2) -> false; +allowed_guard(is_record, 3) -> false; +allowed_guard(Right, Arity) -> + erl_internal:guard_bif(Right, Arity) orelse elixir_utils:guard_op(Right, Arity). + +format_error({invalid_guard, Receiver, Right, Arity, Context}) -> + io_lib:format("cannot invoke remote function ~ts.~ts/~B inside ~ts", + ['Elixir.Macro':to_string(Receiver), Right, Arity, Context]); +format_error({invalid_match, Receiver, Right, Arity}) -> + io_lib:format("cannot invoke remote function ~ts.~ts/~B inside a match", + ['Elixir.Macro':to_string(Receiver), Right, Arity]); +format_error({invalid_match_append, Arg}) -> + io_lib:format("invalid argument for ++ operator inside a match, expected a literal proper list, got: ~ts", + ['Elixir.Macro':to_string(Arg)]). + +is_always_string({{'.', _, [Module, Function]}, _, Args}) -> + is_always_string(Module, Function, length(Args)); +is_always_string(Ast) -> + is_binary(Ast). + +is_always_string('Elixir.Enum', join, _) -> true; +is_always_string('Elixir.Enum', map_join, _) -> true; +is_always_string('Elixir.Kernel', inspect, _) -> true; +is_always_string('Elixir.Macro', to_string, _) -> true; +is_always_string('Elixir.String.Chars', to_string, _) -> true; +is_always_string('Elixir.Path', join, _) -> true; +is_always_string(_Module, _Function, _Args) -> false. diff --git a/lib/elixir/src/elixir_scope.erl b/lib/elixir/src/elixir_scope.erl deleted file mode 100644 index c9f02e3dd0f..00000000000 --- a/lib/elixir/src/elixir_scope.erl +++ /dev/null @@ -1,140 +0,0 @@ -%% Convenience functions used to manipulate scope and its variables. --module(elixir_scope). --export([translate_var/4, build_var/2, - load_binding/2, dump_binding/2, - mergev/2, mergec/2, mergef/2, - merge_vars/2, merge_opt_vars/2 -]). --include("elixir.hrl"). - -%% VAR HANDLING - -translate_var(Meta, Name, Kind, S) when is_atom(Kind); is_integer(Kind) -> - Line = ?line(Meta), - Tuple = {Name, Kind}, - Vars = S#elixir_scope.vars, - - case orddict:find({Name, Kind}, Vars) of - {ok, {Current, _}} -> Exists = true; - error -> Current = nil, Exists = false - end, - - case S#elixir_scope.context of - match -> - MatchVars = S#elixir_scope.match_vars, - - case Exists andalso ordsets:is_element(Tuple, MatchVars) of - true -> - {{var, Line, Current}, S}; - false -> - %% We attempt to give vars a nice name because we - %% still use the unused vars warnings from erl_lint. - %% - %% Once we move the warning to Elixir compiler, we - %% can name vars as _@COUNTER. - {NewVar, Counter, NS} = - if - Kind /= nil -> - build_var('_', S); - true -> - build_var(Name, S) - end, - - FS = NS#elixir_scope{ - vars=orddict:store(Tuple, {NewVar, Counter}, Vars), - match_vars=ordsets:add_element(Tuple, MatchVars), - export_vars=case S#elixir_scope.export_vars of - nil -> nil; - EV -> orddict:store(Tuple, {NewVar, Counter}, EV) - end - }, - - {{var, Line, NewVar}, FS} - end; - _ when Exists -> - {{var, Line, Current}, S} - end. - -build_var(Key, S) -> - New = orddict:update_counter(Key, 1, S#elixir_scope.counter), - Cnt = orddict:fetch(Key, New), - {elixir_utils:atom_concat([Key, "@", Cnt]), Cnt, S#elixir_scope{counter=New}}. - -%% SCOPE MERGING - -%% Receives two scopes and return a new scope based on -%% the second with their variables merged. - -mergev(S1, S2) -> - S2#elixir_scope{ - vars=merge_vars(S1#elixir_scope.vars, S2#elixir_scope.vars), - export_vars=merge_opt_vars(S1#elixir_scope.export_vars, S2#elixir_scope.export_vars) - }. - -%% Receives two scopes and return the first scope with -%% counters and flags from the later. - -mergec(S1, S2) -> - S1#elixir_scope{ - counter=S2#elixir_scope.counter, - super=S2#elixir_scope.super, - caller=S2#elixir_scope.caller - }. - -%% Similar to mergec but does not merge the user vars counter. - -mergef(S1, S2) -> - S1#elixir_scope{ - super=S2#elixir_scope.super, - caller=S2#elixir_scope.caller - }. - -%% Mergers. - -merge_vars(V, V) -> V; -merge_vars(V1, V2) -> - orddict:merge(fun var_merger/3, V1, V2). - -merge_opt_vars(nil, _C2) -> nil; -merge_opt_vars(_C1, nil) -> nil; -merge_opt_vars(C, C) -> C; -merge_opt_vars(C1, C2) -> - orddict:merge(fun var_merger/3, C1, C2). - -var_merger(_Var, {_, V1} = K1, {_, V2}) when V1 > V2 -> K1; -var_merger(_Var, _K1, K2) -> K2. - -%% BINDINGS - -load_binding(Binding, Scope) -> - {NewBinding, NewVars, NewCounter} = load_binding(Binding, [], [], 0), - {NewBinding, Scope#elixir_scope{ - vars=NewVars, - counter=[{'_',NewCounter}] - }}. - -load_binding([{Key,Value}|T], Binding, Vars, Counter) -> - Actual = case Key of - {_Name, _Kind} -> Key; - Name when is_atom(Name) -> {Name, nil} - end, - InternalName = elixir_utils:atom_concat(["_@", Counter]), - load_binding(T, - orddict:store(InternalName, Value, Binding), - orddict:store(Actual, {InternalName, 0}, Vars), Counter + 1); -load_binding([], Binding, Vars, Counter) -> - {Binding, Vars, Counter}. - -dump_binding(Binding, #elixir_scope{vars=Vars}) -> - dump_binding(Vars, Binding, []). - -dump_binding([{{Var, Kind} = Key, {InternalName,_}}|T], Binding, Acc) when is_atom(Kind) -> - Actual = case Kind of - nil -> Var; - _ -> Key - end, - Value = proplists:get_value(InternalName, Binding, nil), - dump_binding(T, Binding, orddict:store(Actual, Value, Acc)); -dump_binding([_|T], Binding, Acc) -> - dump_binding(T, Binding, Acc); -dump_binding([], _Binding, Acc) -> Acc. diff --git a/lib/elixir/src/elixir_sup.erl b/lib/elixir/src/elixir_sup.erl index 80767df73c1..bc111dc5698 100644 --- a/lib/elixir/src/elixir_sup.erl +++ b/lib/elixir/src/elixir_sup.erl @@ -8,24 +8,24 @@ start_link() -> init(ok) -> Workers = [ { - elixir_code_server, - {elixir_code_server, start_link, []}, + elixir_config, + {elixir_config, start_link, []}, permanent, % Restart = permanent | transient | temporary 2000, % Shutdown = brutal_kill | int() >= 0 | infinity worker, % Type = worker | supervisor - [elixir_code_server] % Modules = [Module] | dynamic - }, + [elixir_config] % Modules = [Module] | dynamic + }, { - elixir_counter, - {elixir_counter, start_link, []}, + elixir_code_server, + {elixir_code_server, start_link, []}, permanent, % Restart = permanent | transient | temporary 2000, % Shutdown = brutal_kill | int() >= 0 | infinity worker, % Type = worker | supervisor - [elixir_counter] % Modules = [Module] | dynamic - } + [elixir_code_server] % Modules = [Module] | dynamic + } ], {ok, {{one_for_one, 3, 10}, Workers}}. diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 84310831eb6..590547ea91b 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -1,894 +1,1484 @@ -module(elixir_tokenizer). -include("elixir.hrl"). --export([tokenize/3]). --import(elixir_interpolation, [unescape_tokens/1]). +-include("elixir_tokenizer.hrl"). +-export([tokenize/1, tokenize/3, tokenize/4, invalid_do_error/1]). -define(at_op(T), - T == $@). + T =:= $@). -define(capture_op(T), - T == $&). + T =:= $&). -define(unary_op(T), - T == $!; - T == $^). + T =:= $!; + T =:= $^). --define(unary_op3(T1, T2, T3), - T1 == $~, T2 == $~, T3 == $~). +-define(range_op(T1, T2), + T1 =:= $., T2 =:= $.). + +-define(concat_op(T1, T2), + T1 =:= $+, T2 =:= $+; + T1 =:= $-, T2 =:= $-; + T1 =:= $<, T2 =:= $>). --define(hat_op3(T1, T2, T3), - T1 == $^, T2 == $^, T3 == $^). +-define(concat_op3(T1, T2, T3), + T1 =:= $+, T2 =:= $+, T3 =:= $+; + T1 =:= $-, T2 =:= $-, T3 =:= $-). --define(two_op(T1, T2), - T1 == $+, T2 == $+; - T1 == $-, T2 == $-; - T1 == $<, T2 == $>; - T1 == $., T2 == $.). +-define(power_op(T1, T2), + T1 =:= $*, T2 =:= $*). -define(mult_op(T), - T == $* orelse T == $/). + T =:= $* orelse T =:= $/). -define(dual_op(T), - T == $+ orelse T == $-). + T =:= $+ orelse T =:= $-). -define(arrow_op3(T1, T2, T3), - T1 == $<, T2 == $<, T3 == $<; - T1 == $>, T2 == $>, T3 == $>). + T1 =:= $<, T2 =:= $<, T3 =:= $<; + T1 =:= $>, T2 =:= $>, T3 =:= $>; + T1 =:= $~, T2 =:= $>, T3 =:= $>; + T1 =:= $<, T2 =:= $<, T3 =:= $~; + T1 =:= $<, T2 =:= $~, T3 =:= $>; + T1 =:= $<, T2 =:= $|, T3 =:= $>). -define(arrow_op(T1, T2), - T1 == $|, T2 == $>). + T1 =:= $|, T2 =:= $>; + T1 =:= $~, T2 =:= $>; + T1 =:= $<, T2 =:= $~). -define(rel_op(T), - T == $<; - T == $>). + T =:= $<; + T =:= $>). -define(rel_op2(T1, T2), - T1 == $<, T2 == $=; - T1 == $>, T2 == $=). + T1 =:= $<, T2 =:= $=; + T1 =:= $>, T2 =:= $=). -define(comp_op2(T1, T2), - T1 == $=, T2 == $=; - T1 == $=, T2 == $~; - T1 == $!, T2 == $=). + T1 =:= $=, T2 =:= $=; + T1 =:= $=, T2 =:= $~; + T1 =:= $!, T2 =:= $=). -define(comp_op3(T1, T2, T3), - T1 == $=, T2 == $=, T3 == $=; - T1 == $!, T2 == $=, T3 == $=). + T1 =:= $=, T2 =:= $=, T3 =:= $=; + T1 =:= $!, T2 =:= $=, T3 =:= $=). + +-define(ternary_op(T1, T2), + T1 =:= $/, T2 =:= $/). -define(and_op(T1, T2), - T1 == $&, T2 == $&). + T1 =:= $&, T2 =:= $&). -define(or_op(T1, T2), - T1 == $|, T2 == $|). + T1 =:= $|, T2 =:= $|). -define(and_op3(T1, T2, T3), - T1 == $&, T2 == $&, T3 == $&). + T1 =:= $&, T2 =:= $&, T3 =:= $&). -define(or_op3(T1, T2, T3), - T1 == $|, T2 == $|, T3 == $|). + T1 =:= $|, T2 =:= $|, T3 =:= $|). -define(match_op(T), - T == $=). + T =:= $=). -define(in_match_op(T1, T2), - T1 == $<, T2 == $-; - T1 == $\\, T2 == $\\). + T1 =:= $<, T2 =:= $-; + T1 =:= $\\, T2 =:= $\\). -define(stab_op(T1, T2), - T1 == $-, T2 == $>). + T1 =:= $-, T2 =:= $>). -define(type_op(T1, T2), - T1 == $:, T2 == $:). + T1 =:= $:, T2 =:= $:). --define(pipe_op(T1), - T == $|). +-define(pipe_op(T), + T =:= $|). -tokenize(String, Line, #elixir_tokenizer{} = Scope) -> - tokenize(String, Line, Scope, []); +%% Deprecated operators + +-define(unary_op3(T1, T2, T3), + T1 =:= $~, T2 =:= $~, T3 =:= $~). + +-define(xor_op3(T1, T2, T3), + T1 =:= $^, T2 =:= $^, T3 =:= $^). + +tokenize(String, Line, Column, #elixir_tokenizer{} = Scope) -> + tokenize(String, Line, Column, Scope, []); + +tokenize(String, Line, Column, Opts) -> + IdentifierTokenizer = elixir_config:identifier_tokenizer(), + + Scope = + lists:foldl(fun + ({check_terminators, false}, Acc) -> + Acc#elixir_tokenizer{terminators=none}; + ({cursor_completion, true}, Acc) -> + Acc#elixir_tokenizer{cursor_completion=cursor_and_terminators}; + ({existing_atoms_only, ExistingAtomsOnly}, Acc) when is_boolean(ExistingAtomsOnly) -> + Acc#elixir_tokenizer{existing_atoms_only=ExistingAtomsOnly}; + ({static_atoms_encoder, StaticAtomsEncoder}, Acc) when is_function(StaticAtomsEncoder) -> + Acc#elixir_tokenizer{static_atoms_encoder=StaticAtomsEncoder}; + ({preserve_comments, PreserveComments}, Acc) when is_function(PreserveComments) -> + Acc#elixir_tokenizer{preserve_comments=PreserveComments}; + ({unescape, Unescape}, Acc) when is_boolean(Unescape) -> + Acc#elixir_tokenizer{unescape=Unescape}; + ({warn_on_unnecessary_quotes, Unnecessary}, Acc) when is_boolean(Unnecessary) -> + Acc#elixir_tokenizer{warn_on_unnecessary_quotes=Unnecessary}; + (_, Acc) -> + Acc + end, #elixir_tokenizer{identifier_tokenizer=IdentifierTokenizer}, Opts), + + tokenize(String, Line, Column, Scope, []). tokenize(String, Line, Opts) -> - File = case lists:keyfind(file, 1, Opts) of - {file, V1} -> V1; - false -> <<"nofile">> - end, + tokenize(String, Line, 1, Opts). - Existing = case lists:keyfind(existing_atoms_only, 1, Opts) of - {existing_atoms_only, true} -> true; - false -> false - end, +tokenize([], Line, Column, #elixir_tokenizer{ascii_identifiers_only=Ascii, cursor_completion=Cursor} = Scope, Tokens) when Cursor /= false -> + #elixir_tokenizer{terminators=Terminators, warnings=Warnings} = Scope, - Check = case lists:keyfind(check_terminators, 1, Opts) of - {check_terminators, false} -> false; - false -> true - end, + {CursorColumn, CursorTerminators, CursorTokens} = + add_cursor(Line, Column, Cursor, Terminators, Tokens), + + AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings), + AccTokens = cursor_complete(Line, CursorColumn, CursorTerminators, CursorTokens), + {ok, Line, Column, AllWarnings, AccTokens}; - tokenize(String, Line, #elixir_tokenizer{ - file=File, - existing_atoms_only=Existing, - check_terminators=Check - }). +tokenize([], EndLine, Column, #elixir_tokenizer{terminators=[{Start, StartLine, _} | _]} = Scope, Tokens) -> + End = terminator(Start), + Hint = missing_terminator_hint(Start, End, Scope), + Message = "missing terminator: ~ts (for \"~ts\" starting at line ~B)", + Formatted = io_lib:format(Message, [End, Start, StartLine]), + error({EndLine, Column, [Formatted, Hint], []}, [], Scope, Tokens); + +tokenize([], Line, Column, #elixir_tokenizer{ascii_identifiers_only=Ascii, warnings=Warnings}, Tokens) -> + AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings), + {ok, Line, Column, AllWarnings, lists:reverse(Tokens)}; -tokenize([], Line, #elixir_tokenizer{terminators=[]}, Tokens) -> - {ok, Line, lists:reverse(Tokens)}; +% VC merge conflict -tokenize([], EndLine, #elixir_tokenizer{terminators=[{Start, StartLine}|_]}, Tokens) -> - End = terminator(Start), - Message = io_lib:format("missing terminator: ~ts (for \"~ts\" starting at line ~B)", [End, Start, StartLine]), - {error, {EndLine, Message, []}, [], Tokens}; +tokenize(("<<<<<<<" ++ _) = Original, Line, 1, Scope, Tokens) -> + FirstLine = lists:takewhile(fun(C) -> C =/= $\n andalso C =/= $\r end, Original), + Reason = {Line, 1, "found an unexpected version control marker, please resolve the conflicts: ", FirstLine}, + error(Reason, Original, Scope, Tokens); % Base integers -tokenize([$0,X,H|T], Line, Scope, Tokens) when (X == $x orelse X == $X), ?is_hex(H) -> - {Rest, Number} = tokenize_hex([H|T], []), - tokenize(Rest, Line, Scope, [{number, Line, Number}|Tokens]); +tokenize([$0, $x, H | T], Line, Column, Scope, Tokens) when ?is_hex(H) -> + {Rest, Number, OriginalRepresentation, Length} = tokenize_hex(T, [H], 1), + Token = {int, {Line, Column, Number}, OriginalRepresentation}, + tokenize(Rest, Line, Column + 2 + Length, Scope, [Token | Tokens]); -tokenize([$0,B,H|T], Line, Scope, Tokens) when (B == $b orelse B == $B), ?is_bin(H) -> - {Rest, Number} = tokenize_bin([H|T], []), - tokenize(Rest, Line, Scope, [{number, Line, Number}|Tokens]); +tokenize([$0, $b, H | T], Line, Column, Scope, Tokens) when ?is_bin(H) -> + {Rest, Number, OriginalRepresentation, Length} = tokenize_bin(T, [H], 1), + Token = {int, {Line, Column, Number}, OriginalRepresentation}, + tokenize(Rest, Line, Column + 2 + Length, Scope, [Token | Tokens]); -tokenize([$0,H|T], Line, Scope, Tokens) when ?is_octal(H) -> - {Rest, Number} = tokenize_octal([H|T], []), - tokenize(Rest, Line, Scope, [{number, Line, Number}|Tokens]); +tokenize([$0, $o, H | T], Line, Column, Scope, Tokens) when ?is_octal(H) -> + {Rest, Number, OriginalRepresentation, Length} = tokenize_octal(T, [H], 1), + Token = {int, {Line, Column, Number}, OriginalRepresentation}, + tokenize(Rest, Line, Column + 2 + Length, Scope, [Token | Tokens]); % Comments -tokenize([$#|String], Line, Scope, Tokens) -> - Rest = tokenize_comment(String), - tokenize(Rest, Line, Scope, Tokens); +tokenize([$# | String], Line, Column, Scope, Tokens) -> + case tokenize_comment(String, [$#]) of + {error, Char} -> + error_comment(Char, [$# | String], Line, Column, Scope, Tokens); + {Rest, Comment} -> + preserve_comments(Line, Column, Tokens, Comment, Rest, Scope), + tokenize(Rest, Line, Column, Scope, reset_eol(Tokens)) + end; % Sigils -tokenize([$~,S,H,H,H|T] = Original, Line, Scope, Tokens) when ?is_quote(H), ?is_upcase(S) orelse ?is_downcase(S) -> - case extract_heredoc_with_interpolation(Line, Scope, ?is_downcase(S), T, H) of - {ok, NewLine, Parts, Rest} -> +tokenize([$~, S, H, H, H | T] = Original, Line, Column, Scope, Tokens) when ?is_quote(H), ?is_upcase(S) orelse ?is_downcase(S) -> + case extract_heredoc_with_interpolation(Line, Column, Scope, ?is_downcase(S), T, H) of + {ok, NewLine, NewColumn, Parts, Rest, NewScope} -> {Final, Modifiers} = collect_modifiers(Rest, []), - tokenize(Final, NewLine, Scope, [{sigil, Line, S, Parts, Modifiers}|Tokens]); + Indentation = NewColumn - 4, + Token = {sigil, {Line, Column, nil}, S, Parts, Modifiers, Indentation, <>}, + NewColumnWithModifiers = NewColumn + length(Modifiers), + tokenize(Final, NewLine, NewColumnWithModifiers, NewScope, [Token | Tokens]); + {error, Reason} -> - {error, Reason, Original, Tokens} + error(Reason, Original, Scope, Tokens) end; -tokenize([$~,S,H|T] = Original, Line, Scope, Tokens) when ?is_sigil(H), ?is_upcase(S) orelse ?is_downcase(S) -> - case elixir_interpolation:extract(Line, Scope, ?is_downcase(S), T, sigil_terminator(H)) of - {NewLine, Parts, Rest} -> +tokenize([$~, S, H | T] = Original, Line, Column, Scope, Tokens) when ?is_sigil(H), ?is_upcase(S) orelse ?is_downcase(S) -> + case elixir_interpolation:extract(Line, Column + 3, Scope, ?is_downcase(S), T, sigil_terminator(H)) of + {NewLine, NewColumn, Parts, Rest, NewScope} -> {Final, Modifiers} = collect_modifiers(Rest, []), - tokenize(Final, NewLine, Scope, [{sigil, Line, S, Parts, Modifiers}|Tokens]); + Indentation = nil, + Token = {sigil, {Line, Column, nil}, S, tokens_to_binary(Parts), Modifiers, Indentation, <>}, + NewColumnWithModifiers = NewColumn + length(Modifiers), + tokenize(Final, NewLine, NewColumnWithModifiers, NewScope, [Token | Tokens]); + {error, Reason} -> - Sigil = [$~,S,H], - interpolation_error(Reason, Original, Tokens, " (for sigil ~ts starting at line ~B)", [Sigil, Line]) + Sigil = [$~, S, H], + Message = " (for sigil ~ts starting at line ~B)", + interpolation_error(Reason, Original, Scope, Tokens, Message, [Sigil, Line]) end; +tokenize([$~, S, H | _] = Original, Line, Column, Scope, Tokens) when ?is_upcase(S) orelse ?is_downcase(S) -> + MessageString = + "\"~ts\" (column ~p, code point U+~4.16.0B). The available delimiters are: " + "//, ||, \"\", '', (), [], {}, <>", + Message = io_lib:format(MessageString, [[H], Column + 2, H]), + error({Line, Column, "invalid sigil delimiter: ", Message}, Original, Scope, Tokens); + % Char tokens -tokenize([$?,$\\,P,${,A,B,C,D,E,F,$}|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E), ?is_hex(F) -> - Char = escape_char([$\\,P,${,A,B,C,D,E,F,$}]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,${,A,B,C,D,E,$}|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D), ?is_hex(E) -> - Char = escape_char([$\\,P,${,A,B,C,D,E,$}]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,${,A,B,C,D,$}|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C), ?is_hex(D) -> - Char = escape_char([$\\,P,${,A,B,C,D,$}]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,${,A,B,C,$}|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B), ?is_hex(C) -> - Char = escape_char([$\\,P,${,A,B,C,$}]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,${,A,B,$}|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B) -> - Char = escape_char([$\\,P,${,A,B,$}]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,${,A,$}|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A) -> - Char = escape_char([$\\,P,${,A,$}]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,A,B|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A), ?is_hex(B) -> - Char = escape_char([$\\,P,A,B]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,P,A|T], Line, Scope, Tokens) - when (P == $x orelse P == $X), ?is_hex(A) -> - Char = escape_char([$\\,P,A]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,A,B,C|T], Line, Scope, Tokens) - when ?is_octal(A), A =< $3,?is_octal(B), ?is_octal(C) -> - Char = escape_char([$\\,A,B,C]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,A,B|T], Line, Scope, Tokens) - when ?is_octal(A), ?is_octal(B) -> - Char = escape_char([$\\,A,B]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,A|T], Line, Scope, Tokens) - when ?is_octal(A) -> - Char = escape_char([$\\,A]), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); - -tokenize([$?,$\\,H|T], Line, Scope, Tokens) -> - Char = elixir_interpolation:unescape_map(H), - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); +% We tokenize char literals (?a) as {char, _, CharInt} instead of {number, _, +% CharInt}. This is exactly what Erlang does with Erlang char literals +% ($a). This means we'll have to adjust the error message for char literals in +% elixir_errors.erl as by default {char, _, _} tokens are "hijacked" by Erlang +% and printed with Erlang syntax ($a) in the parser's error messages. -tokenize([$?,Char|T], Line, Scope, Tokens) -> - tokenize(T, Line, Scope, [{number, Line, Char}|Tokens]); +tokenize([$?, $\\, H | T], Line, Column, Scope, Tokens) -> + Char = elixir_interpolation:unescape_map(H), -% Heredocs + NewScope = if + H =:= Char, H =/= $\\ -> + case handle_char(Char) of + {Escape, Name} -> + Msg = io_lib:format("found ?\\ followed by code point 0x~.16B (~ts), please use ?~ts instead", + [Char, Name, Escape]), + prepend_warning(Line, Column, Msg, Scope); -tokenize("\"\"\"" ++ T, Line, Scope, Tokens) -> - handle_heredocs(T, Line, $", Scope, Tokens); + false when ?is_downcase(H); ?is_upcase(H) -> + Msg = io_lib:format("unknown escape sequence ?\\~tc, use ?~tc instead", [H, H]), + prepend_warning(Line, Column, Msg, Scope); -tokenize("'''" ++ T, Line, Scope, Tokens) -> - handle_heredocs(T, Line, $', Scope, Tokens); + false -> + Scope + end; + true -> + Scope + end, -% Strings + Token = {char, {Line, Column, [$?, $\\, H]}, Char}, + tokenize(T, Line, Column + 3, NewScope, [Token | Tokens]); + +tokenize([$?, Char | T], Line, Column, Scope, Tokens) -> + NewScope = case handle_char(Char) of + {Escape, Name} -> + Msg = io_lib:format("found ? followed by code point 0x~.16B (~ts), please use ?~ts instead", + [Char, Name, Escape]), + prepend_warning(Line, Column, Msg, Scope); + false -> + Scope + end, + Token = {char, {Line, Column, [$?, Char]}, Char}, + tokenize(T, Line, Column + 2, NewScope, [Token | Tokens]); -tokenize([$"|T], Line, Scope, Tokens) -> - handle_strings(T, Line, $", Scope, Tokens); -tokenize([$'|T], Line, Scope, Tokens) -> - handle_strings(T, Line, $', Scope, Tokens); +% Heredocs -% Atoms +tokenize("\"\"\"" ++ T, Line, Column, Scope, Tokens) -> + handle_heredocs(T, Line, Column, $", Scope, Tokens); -tokenize([$:,H|T] = Original, Line, Scope, Tokens) when ?is_quote(H) -> - case elixir_interpolation:extract(Line, Scope, true, T, H) of - {NewLine, Parts, Rest} -> - Unescaped = unescape_tokens(Parts), - Key = case Scope#elixir_tokenizer.existing_atoms_only of - true -> atom_safe; - false -> atom_unsafe - end, - tokenize(Rest, NewLine, Scope, [{Key, Line, Unescaped}|Tokens]); - {error, Reason} -> - interpolation_error(Reason, Original, Tokens, " (for atom starting at line ~B)", [Line]) - end; +tokenize("'''" ++ T, Line, Column, Scope, Tokens) -> + handle_heredocs(T, Line, Column, $', Scope, Tokens); -tokenize([$:,T|String] = Original, Line, Scope, Tokens) when ?is_atom_start(T) -> - {Rest, Part} = tokenize_atom([T|String], []), - case unsafe_to_atom(Part, Line, Scope) of - {ok, Atom} -> - tokenize(Rest, Line, Scope, [{atom, Line, Atom}|Tokens]); - {error, Reason} -> - {error, Reason, Original, Tokens} - end; +% Strings -% %% Special atom identifiers / operators - -tokenize(":..." ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, [{atom, Line, '...'}|Tokens]); -tokenize(":<<>>" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, [{atom, Line, '<<>>'}|Tokens]); -tokenize(":%{}" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, [{atom, Line, '%{}'}|Tokens]); -tokenize(":%" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, [{atom, Line, '%'}|Tokens]); -tokenize(":{}" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, [{atom, Line, '{}'}|Tokens]); - -tokenize("...:" ++ Rest, Line, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, '...'}|Tokens]); -tokenize("<<>>:" ++ Rest, Line, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, '<<>>'}|Tokens]); -tokenize("%{}:" ++ Rest, Line, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, '%{}'}|Tokens]); -tokenize("%:" ++ Rest, Line, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, '%'}|Tokens]); -tokenize("{}:" ++ Rest, Line, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, '{}'}|Tokens]); +tokenize([$" | T], Line, Column, Scope, Tokens) -> + handle_strings(T, Line, Column + 1, $", Scope, Tokens); +tokenize([$' | T], Line, Column, Scope, Tokens) -> + handle_strings(T, Line, Column + 1, $', Scope, Tokens); + +% Operator atoms + +tokenize(".:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 2, Scope, [{kw_identifier, {Line, Column, nil}, '.'} | Tokens]); + +tokenize("...:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 4, Scope, [{kw_identifier, {Line, Column, nil}, '...'} | Tokens]); +tokenize("<<>>:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 5, Scope, [{kw_identifier, {Line, Column, nil}, '<<>>'} | Tokens]); +tokenize("%{}:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 4, Scope, [{kw_identifier, {Line, Column, nil}, '%{}'} | Tokens]); +tokenize("%:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 2, Scope, [{kw_identifier, {Line, Column, nil}, '%'} | Tokens]); +tokenize("&:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 2, Scope, [{kw_identifier, {Line, Column, nil}, '&'} | Tokens]); +tokenize("{}:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 3, Scope, [{kw_identifier, {Line, Column, nil}, '{}'} | Tokens]); +tokenize("..//:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> + tokenize(Rest, Line, Column + 5, Scope, [{kw_identifier, {Line, Column, nil}, '..//'} | Tokens]); + +tokenize(":..." ++ Rest, Line, Column, Scope, Tokens) -> + tokenize(Rest, Line, Column + 4, Scope, [{atom, {Line, Column, nil}, '...'} | Tokens]); +tokenize(":<<>>" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize(Rest, Line, Column + 5, Scope, [{atom, {Line, Column, nil}, '<<>>'} | Tokens]); +tokenize(":%{}" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize(Rest, Line, Column + 4, Scope, [{atom, {Line, Column, nil}, '%{}'} | Tokens]); +tokenize(":%" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize(Rest, Line, Column + 2, Scope, [{atom, {Line, Column, nil}, '%'} | Tokens]); +tokenize(":{}" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize(Rest, Line, Column + 3, Scope, [{atom, {Line, Column, nil}, '{}'} | Tokens]); +tokenize(":..//" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize(Rest, Line, Column + 5, Scope, [{atom, {Line, Column, nil}, '..//'} | Tokens]); % ## Three Token Operators -tokenize([$:,T1,T2,T3|Rest], Line, Scope, Tokens) when +tokenize([$:, T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?unary_op3(T1, T2, T3); ?comp_op3(T1, T2, T3); ?and_op3(T1, T2, T3); ?or_op3(T1, T2, T3); - ?arrow_op3(T1, T2, T3); ?hat_op3(T1, T2, T3) -> - tokenize(Rest, Line, Scope, [{atom, Line, list_to_atom([T1,T2,T3])}|Tokens]); + ?arrow_op3(T1, T2, T3); ?xor_op3(T1, T2, T3); ?concat_op3(T1, T2, T3) -> + Token = {atom, {Line, Column, nil}, list_to_atom([T1, T2, T3])}, + tokenize(Rest, Line, Column + 4, Scope, [Token | Tokens]); % ## Two Token Operators -tokenize([$:,T1,T2|Rest], Line, Scope, Tokens) when + +%% TODO: Remove this deprecation on Elixir v2.0 +tokenize([$:, $:, $: | Rest], Line, Column, Scope, Tokens) -> + Message = "atom ::: must be written between quotes, as in :\"::\", to avoid ambiguity", + NewScope = prepend_warning(Line, Column, Message, Scope), + Token = {atom, {Line, Column, nil}, '::'}, + tokenize(Rest, Line, Column + 3, NewScope, [Token | Tokens]); + +tokenize([$:, T1, T2 | Rest], Line, Column, Scope, Tokens) when ?comp_op2(T1, T2); ?rel_op2(T1, T2); ?and_op(T1, T2); ?or_op(T1, T2); - ?arrow_op(T1, T2); ?in_match_op(T1, T2); ?two_op(T1, T2); ?stab_op(T1, T2); - ?type_op(T1, T2) -> - tokenize(Rest, Line, Scope, [{atom, Line, list_to_atom([T1,T2])}|Tokens]); + ?arrow_op(T1, T2); ?in_match_op(T1, T2); ?concat_op(T1, T2); ?power_op(T1, T2); + ?stab_op(T1, T2); ?range_op(T1, T2) -> + Token = {atom, {Line, Column, nil}, list_to_atom([T1, T2])}, + tokenize(Rest, Line, Column + 3, Scope, [Token | Tokens]); % ## Single Token Operators -tokenize([$:,T|Rest], Line, Scope, Tokens) when +tokenize([$:, T | Rest], Line, Column, Scope, Tokens) when ?at_op(T); ?unary_op(T); ?capture_op(T); ?dual_op(T); ?mult_op(T); - ?rel_op(T); ?match_op(T); ?pipe_op(T); T == $. -> - tokenize(Rest, Line, Scope, [{atom, Line, list_to_atom([T])}|Tokens]); + ?rel_op(T); ?match_op(T); ?pipe_op(T); T =:= $. -> + Token = {atom, {Line, Column, nil}, list_to_atom([T])}, + tokenize(Rest, Line, Column + 2, Scope, [Token | Tokens]); + +% ## Stand-alone tokens + +tokenize("..." ++ Rest, Line, Column, Scope, Tokens) -> + NewScope = maybe_warn_too_many_of_same_char("...", Rest, Line, Column, Scope), + Token = check_call_identifier(Line, Column, "...", '...', Rest), + tokenize(Rest, Line, Column + 3, NewScope, [Token | Tokens]); + +tokenize("=>" ++ Rest, Line, Column, Scope, Tokens) -> + Token = {assoc_op, {Line, Column, previous_was_eol(Tokens)}, '=>'}, + tokenize(Rest, Line, Column + 2, Scope, add_token_with_eol(Token, Tokens)); + +tokenize("..//" ++ Rest = String, Line, Column, Scope, Tokens) -> + case strip_horizontal_space(Rest, 0) of + {[$/ | _] = Remaining, Extra} -> + Token = {identifier, {Line, Column, nil}, '..//'}, + tokenize(Remaining, Line, Column + 4 + Extra, Scope, [Token | Tokens]); + {_, _} -> + unexpected_token(String, Line, Column, Scope, Tokens) + end; -% End of line +% ## Ternary operator -tokenize(";" ++ Rest, Line, Scope, []) -> - tokenize(Rest, Line, Scope, eol(Line, ';', [])); +% ## Three token operators +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?unary_op3(T1, T2, T3) -> + handle_unary_op(Rest, Line, Column, unary_op, 3, list_to_atom([T1, T2, T3]), Scope, Tokens); -tokenize(";" ++ Rest, Line, Scope, [Top|Tokens]) when element(1, Top) /= eol -> - tokenize(Rest, Line, Scope, eol(Line, ';', [Top|Tokens])); +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?comp_op3(T1, T2, T3) -> + handle_op(Rest, Line, Column, comp_op, 3, list_to_atom([T1, T2, T3]), Scope, Tokens); -tokenize("\\\n" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line + 1, Scope, Tokens); +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?and_op3(T1, T2, T3) -> + NewScope = maybe_warn_too_many_of_same_char([T1, T2, T3], Rest, Line, Column, Scope), + handle_op(Rest, Line, Column, and_op, 3, list_to_atom([T1, T2, T3]), NewScope, Tokens); -tokenize("\\\r\n" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line + 1, Scope, Tokens); +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?or_op3(T1, T2, T3) -> + NewScope = maybe_warn_too_many_of_same_char([T1, T2, T3], Rest, Line, Column, Scope), + handle_op(Rest, Line, Column, or_op, 3, list_to_atom([T1, T2, T3]), NewScope, Tokens); -tokenize("\n" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line + 1, Scope, eol(Line, newline, Tokens)); +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?xor_op3(T1, T2, T3) -> + NewScope = maybe_warn_too_many_of_same_char([T1, T2, T3], Rest, Line, Column, Scope), + handle_op(Rest, Line, Column, xor_op, 3, list_to_atom([T1, T2, T3]), NewScope, Tokens); -tokenize("\r\n" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line + 1, Scope, eol(Line, newline, Tokens)); +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?concat_op3(T1, T2, T3) -> + NewScope = maybe_warn_too_many_of_same_char([T1, T2, T3], Rest, Line, Column, Scope), + handle_op(Rest, Line, Column, concat_op, 3, list_to_atom([T1, T2, T3]), NewScope, Tokens); -% Stand-alone tokens +tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?arrow_op3(T1, T2, T3) -> + handle_op(Rest, Line, Column, arrow_op, 3, list_to_atom([T1, T2, T3]), Scope, Tokens); -tokenize("..." ++ Rest, Line, Scope, Tokens) -> - Token = check_call_identifier(identifier, Line, '...', Rest), - tokenize(Rest, Line, Scope, [Token|Tokens]); +% ## Containers + punctuation tokens +tokenize([$, | Rest], Line, Column, Scope, Tokens) -> + Token = {',', {Line, Column, 0}}, + tokenize(Rest, Line, Column + 1, Scope, [Token | Tokens]); -tokenize("=>" ++ Rest, Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, add_token_with_nl({assoc_op, Line, '=>'}, Tokens)); +tokenize([$<, $< | Rest], Line, Column, Scope, Tokens) -> + Token = {'<<', {Line, Column, nil}}, + handle_terminator(Rest, Line, Column + 2, Scope, Token, Tokens); -% ## Three token operators -tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?unary_op3(T1, T2, T3) -> - handle_unary_op(Rest, Line, unary_op, list_to_atom([T1,T2,T3]), Scope, Tokens); +tokenize([$>, $> | Rest], Line, Column, Scope, Tokens) -> + Token = {'>>', {Line, Column, previous_was_eol(Tokens)}}, + handle_terminator(Rest, Line, Column + 2, Scope, Token, Tokens); -tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?comp_op3(T1, T2, T3) -> - handle_op(Rest, Line, comp_op, list_to_atom([T1,T2,T3]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when T =:= $(; T =:= ${; T =:= $[ -> + Token = {list_to_atom([T]), {Line, Column, nil}}, + handle_terminator(Rest, Line, Column + 1, Scope, Token, Tokens); -tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?and_op3(T1, T2, T3) -> - handle_op(Rest, Line, and_op, list_to_atom([T1,T2,T3]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when T =:= $); T =:= $}; T =:= $] -> + Token = {list_to_atom([T]), {Line, Column, previous_was_eol(Tokens)}}, + handle_terminator(Rest, Line, Column + 1, Scope, Token, Tokens); -tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?or_op3(T1, T2, T3) -> - handle_op(Rest, Line, or_op, list_to_atom([T1,T2,T3]), Scope, Tokens); +% ## Two Token Operators +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?ternary_op(T1, T2) -> + Op = list_to_atom([T1, T2]), + Token = {ternary_op, {Line, Column, previous_was_eol(Tokens)}, Op}, + tokenize(Rest, Line, Column + 2, Scope, add_token_with_eol(Token, Tokens)); -tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?arrow_op3(T1, T2, T3) -> - handle_op(Rest, Line, arrow_op, list_to_atom([T1,T2,T3]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?power_op(T1, T2) -> + handle_op(Rest, Line, Column, power_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2,T3|Rest], Line, Scope, Tokens) when ?hat_op3(T1, T2, T3) -> - handle_op(Rest, Line, hat_op, list_to_atom([T1,T2,T3]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?range_op(T1, T2) -> + handle_op(Rest, Line, Column, range_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -% ## Containers + punctuation tokens -tokenize([T,T|Rest], Line, Scope, Tokens) when T == $<; T == $> -> - Token = {list_to_atom([T,T]), Line}, - handle_terminator(Rest, Line, Scope, Token, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?concat_op(T1, T2) -> + handle_op(Rest, Line, Column, concat_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when T == $(; - T == ${; T == $}; T == $[; T == $]; T == $); T == $, -> - Token = {list_to_atom([T]), Line}, - handle_terminator(Rest, Line, Scope, Token, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?arrow_op(T1, T2) -> + handle_op(Rest, Line, Column, arrow_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -% ## Two Token Operators -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?two_op(T1, T2) -> - handle_op(Rest, Line, two_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?comp_op2(T1, T2) -> + handle_op(Rest, Line, Column, comp_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?arrow_op(T1, T2) -> - handle_op(Rest, Line, arrow_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?rel_op2(T1, T2) -> + handle_op(Rest, Line, Column, rel_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?comp_op2(T1, T2) -> - handle_op(Rest, Line, comp_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?and_op(T1, T2) -> + handle_op(Rest, Line, Column, and_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?rel_op2(T1, T2) -> - handle_op(Rest, Line, rel_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?or_op(T1, T2) -> + handle_op(Rest, Line, Column, or_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?and_op(T1, T2) -> - handle_op(Rest, Line, and_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?in_match_op(T1, T2) -> + handle_op(Rest, Line, Column, in_match_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?or_op(T1, T2) -> - handle_op(Rest, Line, or_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?type_op(T1, T2) -> + handle_op(Rest, Line, Column, type_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?in_match_op(T1, T2) -> - handle_op(Rest, Line, in_match_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?stab_op(T1, T2) -> + handle_op(Rest, Line, Column, stab_op, 2, list_to_atom([T1, T2]), Scope, Tokens); -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?type_op(T1, T2) -> - handle_op(Rest, Line, type_op, list_to_atom([T1, T2]), Scope, Tokens); +% ## Single Token Operators -tokenize([T1,T2|Rest], Line, Scope, Tokens) when ?stab_op(T1, T2) -> - handle_op(Rest, Line, stab_op, list_to_atom([T1, T2]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?capture_op(T) -> + Kind = + case strip_horizontal_space(Rest, 0) of + {[$/ | NewRest], _} -> + case strip_horizontal_space(NewRest, 0) of + {[$/ | _], _} -> capture_op; + {_, _} -> identifier + end; + + {_, _} -> + capture_op + end, -% ## Single Token Operators + Token = {Kind, {Line, Column, nil}, '&'}, + tokenize(Rest, Line, Column + 1, Scope, [Token | Tokens]); -tokenize([T|Rest], Line, Scope, Tokens) when ?at_op(T) -> - handle_unary_op(Rest, Line, at_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?at_op(T) -> + handle_unary_op(Rest, Line, Column, at_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?capture_op(T) -> - handle_unary_op(Rest, Line, capture_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?unary_op(T) -> + handle_unary_op(Rest, Line, Column, unary_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?unary_op(T) -> - handle_unary_op(Rest, Line, unary_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?rel_op(T) -> + handle_op(Rest, Line, Column, rel_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?rel_op(T) -> - handle_op(Rest, Line, rel_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?dual_op(T) -> + handle_unary_op(Rest, Line, Column, dual_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?dual_op(T) -> - handle_unary_op(Rest, Line, dual_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?mult_op(T) -> + handle_op(Rest, Line, Column, mult_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?mult_op(T) -> - handle_op(Rest, Line, mult_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?match_op(T) -> + handle_op(Rest, Line, Column, match_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?match_op(T) -> - handle_op(Rest, Line, match_op, list_to_atom([T]), Scope, Tokens); +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?pipe_op(T) -> + handle_op(Rest, Line, Column, pipe_op, 1, list_to_atom([T]), Scope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?pipe_op(T) -> - handle_op(Rest, Line, pipe_op, list_to_atom([T]), Scope, Tokens); +% Non-operator Atoms -% Others +tokenize([$:, H | T] = Original, Line, Column, Scope, Tokens) when ?is_quote(H) -> + case elixir_interpolation:extract(Line, Column + 2, Scope, true, T, H) of + {NewLine, NewColumn, Parts, Rest, InterScope} -> + NewScope = case is_unnecessary_quote(Parts, InterScope) of + true -> + WarnMsg = io_lib:format( + "found quoted atom \"~ts\" but the quotes are not required. " + "Atoms made exclusively of ASCII letters, numbers, underscores, " + "beginning with a letter or underscore, and optionally ending with ! or ? " + "do not require quotes", + [hd(Parts)] + ), + prepend_warning(Line, Column, WarnMsg, InterScope); + + false -> + InterScope + end, + + case unescape_tokens(Parts, Line, Column, NewScope) of + {ok, [Part]} when is_binary(Part) -> + case unsafe_to_atom(Part, Line, Column, Scope) of + {ok, Atom} -> + Token = {atom_quoted, {Line, Column, nil}, Atom}, + tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]); + + {error, Reason} -> + error(Reason, Rest, NewScope, Tokens) + end; + + {ok, Unescaped} -> + Key = case Scope#elixir_tokenizer.existing_atoms_only of + true -> atom_safe; + false -> atom_unsafe + end, + Token = {Key, {Line, Column, nil}, Unescaped}, + tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]); + + {error, Reason} -> + error(Reason, Rest, NewScope, Tokens) + end; -tokenize([$%|T], Line, Scope, Tokens) -> - case strip_space(T, 0) of - {[${|_] = Rest, Counter} -> tokenize(Rest, Line + Counter, Scope, [{'%{}', Line}|Tokens]); - {Rest, Counter} -> tokenize(Rest, Line + Counter, Scope, [{'%', Line}|Tokens]) + {error, Reason} -> + Message = " (for atom starting at line ~B)", + interpolation_error(Reason, Original, Scope, Tokens, Message, [Line]) end; -tokenize([$.|T], Line, Scope, Tokens) -> - {Rest, Counter} = strip_space(T, 0), - handle_dot([$.|Rest], Line + Counter, Scope, Tokens); +tokenize([$: | String] = Original, Line, Column, Scope, Tokens) -> + case tokenize_identifier(String, Line, Column, Scope, false) of + {_Kind, Unencoded, Atom, Rest, Length, Ascii, _Special} -> + NewScope = maybe_warn_for_ambiguous_bang_before_equals(atom, Unencoded, Rest, Line, Column, Scope), + TrackedScope = track_ascii(Ascii, NewScope), + Token = {atom, {Line, Column, Unencoded}, Atom}, + tokenize(Rest, Line, Column + 1 + Length, TrackedScope, [Token | Tokens]); + empty when Scope#elixir_tokenizer.cursor_completion == false -> + unexpected_token(Original, Line, Column, Scope, Tokens); + empty -> + tokenize([], Line, Column, Scope, Tokens); + {unexpected_token, Length} -> + unexpected_token(lists:nthtail(Length - 1, String), Line, Column + Length - 1, Scope, Tokens); + {error, Reason} -> + error(Reason, Original, Scope, Tokens) + end; % Integers and floats +% We use int and flt otherwise elixir_parser won't format them +% properly in case of errors. + +tokenize([H | T], Line, Column, Scope, Tokens) when ?is_digit(H) -> + case tokenize_number(T, [H], 1, false) of + {error, Reason, Original} -> + error({Line, Column, Reason, Original}, T, Scope, Tokens); + {[I | Rest], Number, Original, _Length} when ?is_upcase(I); ?is_downcase(I); I == $_ -> + if + Number == 0, (I =:= $x) orelse (I =:= $o) orelse (I =:= $b), Rest == [], + Scope#elixir_tokenizer.cursor_completion /= false -> + tokenize([], Line, Column, Scope, Tokens); + + true -> + Msg = + io_lib:format( + "invalid character \"~ts\" after number ~ts. If you intended to write a number, " + "make sure to separate the number from the character (using comma, space, etc). " + "If you meant to write a function name or a variable, note that identifiers in " + "Elixir cannot start with numbers. Unexpected token: ", + [[I], Original] + ), + + error({Line, Column, Msg, [I]}, T, Scope, Tokens) + end; + {Rest, Number, Original, Length} when is_integer(Number) -> + Token = {int, {Line, Column, Number}, Original}, + tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]); + {Rest, Number, Original, Length} -> + Token = {flt, {Line, Column, Number}, Original}, + tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]) + end; + +% Spaces + +tokenize([T | Rest], Line, Column, Scope, Tokens) when ?is_horizontal_space(T) -> + {Remaining, Stripped} = strip_horizontal_space(Rest, 0), + handle_space_sensitive_tokens(Remaining, Line, Column + 1 + Stripped, Scope, Tokens); + +% End of line + +tokenize(";" ++ Rest, Line, Column, Scope, []) -> + tokenize(Rest, Line, Column + 1, Scope, [{';', {Line, Column, 0}}]); + +tokenize(";" ++ Rest, Line, Column, Scope, [Top | _] = Tokens) when element(1, Top) /= ';' -> + tokenize(Rest, Line, Column + 1, Scope, [{';', {Line, Column, 0}} | Tokens]); + +tokenize("\\" = Original, Line, Column, Scope, Tokens) -> + error({Line, Column, "invalid escape \\ at end of file", []}, Original, Scope, Tokens); + +tokenize("\\\n" = Original, Line, Column, Scope, Tokens) -> + error({Line, Column, "invalid escape \\ at end of file", []}, Original, Scope, Tokens); + +tokenize("\\\r\n" = Original, Line, Column, Scope, Tokens) -> + error({Line, Column, "invalid escape \\ at end of file", []}, Original, Scope, Tokens); + +tokenize("\\\n" ++ Rest, Line, _Column, Scope, Tokens) -> + tokenize_eol(Rest, Line, Scope, Tokens); + +tokenize("\\\r\n" ++ Rest, Line, _Column, Scope, Tokens) -> + tokenize_eol(Rest, Line, Scope, Tokens); + +tokenize("\n" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize_eol(Rest, Line, Scope, eol(Line, Column, Tokens)); + +tokenize("\r\n" ++ Rest, Line, Column, Scope, Tokens) -> + tokenize_eol(Rest, Line, Scope, eol(Line, Column, Tokens)); + +% Others + +tokenize([$%, $( | Rest], Line, Column, Scope, Tokens) -> + Reason = {Line, Column, "expected %{ to define a map, got: ", [$%, $(]}, + error(Reason, Rest, Scope, Tokens); + +tokenize([$%, $[ | Rest], Line, Column, Scope, Tokens) -> + Reason = {Line, Column, "expected %{ to define a map, got: ", [$%, $[]}, + error(Reason, Rest, Scope, Tokens); -tokenize([H|_] = String, Line, Scope, Tokens) when ?is_digit(H) -> - {Rest, Number} = tokenize_number(String, [], false), - tokenize(Rest, Line, Scope, [{number, Line, Number}|Tokens]); +tokenize([$%, ${ | T], Line, Column, Scope, Tokens) -> + tokenize([${ | T], Line, Column + 1, Scope, [{'%{}', {Line, Column, nil}} | Tokens]); -% Aliases +tokenize([$% | T], Line, Column, Scope, Tokens) -> + tokenize(T, Line, Column + 1, Scope, [{'%', {Line, Column, nil}} | Tokens]); + +tokenize([$. | T], Line, Column, Scope, Tokens) -> + tokenize_dot(T, Line, Column + 1, {Line, Column, nil}, Scope, Tokens); + +% Identifiers + +tokenize(String, Line, Column, OriginalScope, Tokens) -> + case tokenize_identifier(String, Line, Column, OriginalScope, not previous_was_dot(Tokens)) of + {Kind, Unencoded, Atom, Rest, Length, Ascii, Special} -> + HasAt = lists:member(at, Special), + Scope = track_ascii(Ascii, OriginalScope), -tokenize([H|_] = Original, Line, Scope, Tokens) when ?is_upcase(H) -> - {Rest, Alias} = tokenize_identifier(Original, []), - case unsafe_to_atom(Alias, Line, Scope) of - {ok, Atom} -> case Rest of - [$:|T] when ?is_space(hd(T)) -> - tokenize(T, Line, Scope, [{kw_identifier, Line, Atom}|Tokens]); + [$: | T] when ?is_space(hd(T)) -> + Token = {kw_identifier, {Line, Column, Unencoded}, Atom}, + tokenize(T, Line, Column + Length + 1, Scope, [Token | Tokens]); + + [$: | T] when hd(T) =/= $: -> + AtomName = atom_to_list(Atom) ++ [$:], + Reason = {Line, Column, "keyword argument must be followed by space after: ", AtomName}, + error(Reason, String, Scope, Tokens); + + _ when HasAt -> + Reason = {Line, Column, invalid_character_error(Kind, $@), atom_to_list(Atom)}, + error(Reason, String, Scope, Tokens); + + _ when Atom == '__aliases__'; Atom == '__block__' -> + error({Line, Column, "reserved token: ", atom_to_list(Atom)}, Rest, Scope, Tokens); + + _ when Kind == alias -> + tokenize_alias(Rest, Line, Column, Unencoded, Atom, Length, Ascii, Special, Scope, Tokens); + + _ when Kind == identifier -> + NewScope = maybe_warn_for_ambiguous_bang_before_equals(identifier, Unencoded, Rest, Line, Column, Scope), + Token = check_call_identifier(Line, Column, Unencoded, Atom, Rest), + tokenize(Rest, Line, Column + Length, NewScope, [Token | Tokens]); + _ -> - tokenize(Rest, Line, Scope, [{aliases, Line, [Atom]}|Tokens]) + unexpected_token(String, Line, Column, Scope, Tokens) end; - {error, Reason} -> - {error, Reason, Original, Tokens} - end; - -% Identifier -tokenize([H|_] = String, Line, Scope, Tokens) when ?is_downcase(H); H == $_ -> - case tokenize_any_identifier(String, Line, Scope, Tokens) of - {keyword, Rest, Check, T} -> - handle_terminator(Rest, Line, Scope, Check, T); - {identifier, Rest, Token} -> - tokenize(Rest, Line, Scope, [Token|Tokens]); - {error, _, _, _} = Error -> - Error - end; + {keyword, Atom, Type, Rest, Length} -> + tokenize_keyword(Type, Rest, Line, Column, Atom, Length, OriginalScope, Tokens); -% Ambiguous unary/binary operators tokens + empty when OriginalScope#elixir_tokenizer.cursor_completion == false -> + unexpected_token(String, Line, Column, OriginalScope, Tokens); -tokenize([Space, Sign, NotMarker|T], Line, Scope, [{Identifier, _, _} = H|Tokens]) when - ?dual_op(Sign), - ?is_horizontal_space(Space), - not(?is_space(NotMarker)), - NotMarker /= $(, NotMarker /= $[, NotMarker /= $<, NotMarker /= ${, %% containers - NotMarker /= $%, NotMarker /= $+, NotMarker /= $-, NotMarker /= $/, NotMarker /= $>, %% operators - Identifier == identifier -> - Rest = [NotMarker|T], - tokenize(Rest, Line, Scope, [{dual_op, Line, list_to_atom([Sign])}, setelement(1, H, op_identifier)|Tokens]); + empty -> + case String of + [$~, L] when ?is_upcase(L); ?is_downcase(L) -> tokenize([], Line, Column, OriginalScope, Tokens); + [$~] -> tokenize([], Line, Column, OriginalScope, Tokens); + _ -> unexpected_token(String, Line, Column, OriginalScope, Tokens) + end; -% Spaces + {unexpected_token, Length} -> + unexpected_token(lists:nthtail(Length - 1, String), Line, Column + Length - 1, OriginalScope, Tokens); -tokenize([T|Rest], Line, Scope, Tokens) when ?is_horizontal_space(T) -> - tokenize(strip_horizontal_space(Rest), Line, Scope, Tokens); -tokenize(T, Line, _Scope, Tokens) -> - {error, {Line, "invalid token: ", until_eol(T)}, T, Tokens}. - -strip_horizontal_space([H|T]) when ?is_horizontal_space(H) -> - strip_horizontal_space(T); -strip_horizontal_space(T) -> - T. - -strip_space(T, Counter) -> - case strip_horizontal_space(T) of - "\r\n" ++ Rest -> strip_space(Rest, Counter + 1); - "\n" ++ Rest -> strip_space(Rest, Counter + 1); - Rest -> {Rest, Counter} + {error, Reason} -> + error(Reason, String, OriginalScope, Tokens) end. -until_eol("\r\n" ++ _) -> []; -until_eol("\n" ++ _) -> []; -until_eol([]) -> []; -until_eol([H|T]) -> [H|until_eol(T)]. +previous_was_dot([{'.', _} | _]) -> true; +previous_was_dot(_) -> false. + +unexpected_token([T | Rest], Line, Column, Scope, Tokens) -> + Message = + case handle_char(T) of + {_Escaped, Explanation} -> + io_lib:format("~ts (column ~p, code point U+~4.16.0B)", [Explanation, Column, T]); + false -> + io_lib:format("\"~ts\" (column ~p, code point U+~4.16.0B)", [[T], Column, T]) + end, + error({Line, Column, "unexpected token: ", Message}, Rest, Scope, Tokens). + +tokenize_eol(Rest, Line, Scope, Tokens) -> + {StrippedRest, Indentation} = strip_horizontal_space(Rest, 0), + IndentedScope = Scope#elixir_tokenizer{indentation=Indentation}, + tokenize(StrippedRest, Line + 1, Indentation + 1, IndentedScope, Tokens). + +strip_horizontal_space([H | T], Counter) when ?is_horizontal_space(H) -> + strip_horizontal_space(T, Counter + 1); +strip_horizontal_space(T, Counter) -> + {T, Counter}. + +tokenize_dot(T, Line, Column, DotInfo, Scope, Tokens) -> + case strip_horizontal_space(T, 0) of + {[$# | R], _} -> + case tokenize_comment(R, [$#]) of + {error, Char} -> + error_comment(Char, [$# | R], Line, Column, Scope, Tokens); + + {Rest, Comment} -> + preserve_comments(Line, Column, Tokens, Comment, Rest, Scope), + tokenize_dot(Rest, Line, 1, DotInfo, Scope, Tokens) + end; + {"\r\n" ++ Rest, _} -> + tokenize_dot(Rest, Line + 1, 1, DotInfo, Scope, Tokens); + {"\n" ++ Rest, _} -> + tokenize_dot(Rest, Line + 1, 1, DotInfo, Scope, Tokens); + {Rest, Length} -> + handle_dot([$. | Rest], Line, Column + Length, DotInfo, Scope, Tokens) + end. -escape_char(List) -> - << Char/utf8 >> = elixir_interpolation:unescape_chars(list_to_binary(List)), - Char. +handle_char(0) -> {"\\0", "null byte"}; +handle_char(7) -> {"\\a", "alert"}; +handle_char($\b) -> {"\\b", "backspace"}; +handle_char($\d) -> {"\\d", "delete"}; +handle_char($\e) -> {"\\e", "escape"}; +handle_char($\f) -> {"\\f", "form feed"}; +handle_char($\n) -> {"\\n", "newline"}; +handle_char($\r) -> {"\\r", "carriage return"}; +handle_char($\s) -> {"\\s", "space"}; +handle_char($\t) -> {"\\t", "tab"}; +handle_char($\v) -> {"\\v", "vertical tab"}; +handle_char(_) -> false. %% Handlers -handle_heredocs(T, Line, H, Scope, Tokens) -> - case extract_heredoc_with_interpolation(Line, Scope, true, T, H) of - {ok, NewLine, Parts, Rest} -> - Token = {string_type(H), Line, unescape_tokens(Parts)}, - tokenize(Rest, NewLine, Scope, [Token|Tokens]); +handle_heredocs(T, Line, Column, H, Scope, Tokens) -> + case extract_heredoc_with_interpolation(Line, Column, Scope, true, T, H) of + {ok, NewLine, NewColumn, Parts, Rest, NewScope} -> + case unescape_tokens(Parts, Line, Column, NewScope) of + {ok, Unescaped} -> + Token = {heredoc_type(H), {Line, Column, nil}, NewColumn - 4, Unescaped}, + tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]); + + {error, Reason} -> + error(Reason, Rest, Scope, Tokens) + end; + {error, Reason} -> - {error, Reason, [H, H, H] ++ T, Tokens} + error(Reason, [H, H, H] ++ T, Scope, Tokens) end. -handle_strings(T, Line, H, Scope, Tokens) -> - case elixir_interpolation:extract(Line, Scope, true, T, H) of +handle_strings(T, Line, Column, H, Scope, Tokens) -> + case elixir_interpolation:extract(Line, Column, Scope, true, T, H) of {error, Reason} -> - interpolation_error(Reason, [H|T], Tokens, " (for string starting at line ~B)", [Line]); - {NewLine, Parts, [$:|Rest]} when ?is_space(hd(Rest)) -> - Unescaped = unescape_tokens(Parts), - Key = case Scope#elixir_tokenizer.existing_atoms_only of - true -> kw_identifier_safe; - false -> kw_identifier_unsafe + interpolation_error(Reason, [H | T], Scope, Tokens, " (for string starting at line ~B)", [Line]); + + {NewLine, NewColumn, Parts, [$: | Rest], InterScope} when ?is_space(hd(Rest)) -> + NewScope = case is_unnecessary_quote(Parts, InterScope) of + true -> + WarnMsg = io_lib:format( + "found quoted keyword \"~ts\" but the quotes are not required. " + "Note that keywords are always atoms, even when quoted. " + "Similar to atoms, keywords made exclusively of ASCII " + "letters, numbers, and underscores and not beginning with a " + "number do not require quotes", + [hd(Parts)] + ), + prepend_warning(Line, Column, WarnMsg, InterScope); + + false -> + InterScope end, - tokenize(Rest, NewLine, Scope, [{Key, Line, Unescaped}|Tokens]); - {NewLine, Parts, Rest} -> - Token = {string_type(H), Line, unescape_tokens(Parts)}, - tokenize(Rest, NewLine, Scope, [Token|Tokens]) - end. -handle_unary_op([$:|Rest], Line, _Kind, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, Op}|Tokens]); + case unescape_tokens(Parts, Line, Column, NewScope) of + {ok, [Part]} when is_binary(Part) -> + case unsafe_to_atom(Part, Line, Column - 1, Scope) of + {ok, Atom} -> + Token = {kw_identifier, {Line, Column - 1, nil}, Atom}, + tokenize(Rest, NewLine, NewColumn + 1, NewScope, [Token | Tokens]); + {error, Reason} -> + {error, Reason, Rest, Tokens} + end; + + {ok, Unescaped} -> + Key = case Scope#elixir_tokenizer.existing_atoms_only of + true -> kw_identifier_safe; + false -> kw_identifier_unsafe + end, + Token = {Key, {Line, Column - 1, nil}, Unescaped}, + tokenize(Rest, NewLine, NewColumn + 1, NewScope, [Token | Tokens]); + + {error, Reason} -> + error(Reason, Rest, NewScope, Tokens) + end; + + {NewLine, NewColumn, Parts, Rest, NewScope} -> + case unescape_tokens(Parts, Line, Column, NewScope) of + {ok, Unescaped} -> + Token = {string_type(H), {Line, Column - 1, nil}, Unescaped}, + tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]); -handle_unary_op(Rest, Line, Kind, Op, Scope, Tokens) -> - case strip_horizontal_space(Rest) of - [$/|_] -> tokenize(Rest, Line, Scope, [{identifier, Line, Op}|Tokens]); - _ -> tokenize(Rest, Line, Scope, [{Kind, Line, Op}|Tokens]) + {error, Reason} -> + error(Reason, Rest, NewScope, Tokens) + end end. -handle_op([$:|Rest], Line, _Kind, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Scope, [{kw_identifier, Line, Op}|Tokens]); +handle_unary_op([$: | Rest], Line, Column, _Kind, Length, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> + Token = {kw_identifier, {Line, Column, nil}, Op}, + tokenize(Rest, Line, Column + Length + 1, Scope, [Token | Tokens]); + +handle_unary_op(Rest, Line, Column, Kind, Length, Op, Scope, Tokens) -> + case strip_horizontal_space(Rest, 0) of + {[$/ | _] = Remaining, Extra} -> + Token = {identifier, {Line, Column, nil}, Op}, + tokenize(Remaining, Line, Column + Length + Extra, Scope, [Token | Tokens]); + {Remaining, Extra} -> + Token = {Kind, {Line, Column, nil}, Op}, + tokenize(Remaining, Line, Column + Length + Extra, Scope, [Token | Tokens]) + end. -handle_op(Rest, Line, Kind, Op, Scope, Tokens) -> - case strip_horizontal_space(Rest) of - [$/|_] -> tokenize(Rest, Line, Scope, [{identifier, Line, Op}|Tokens]); - _ -> tokenize(Rest, Line, Scope, add_token_with_nl({Kind, Line, Op}, Tokens)) +handle_op([$: | Rest], Line, Column, _Kind, Length, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> + Token = {kw_identifier, {Line, Column, nil}, Op}, + tokenize(Rest, Line, Column + Length + 1, Scope, [Token | Tokens]); + +handle_op(Rest, Line, Column, Kind, Length, Op, Scope, Tokens) -> + case strip_horizontal_space(Rest, 0) of + {[$/ | _] = Remaining, Extra} -> + Token = {identifier, {Line, Column, nil}, Op}, + tokenize(Remaining, Line, Column + Length + Extra, Scope, [Token | Tokens]); + {Remaining, Extra} -> + NewScope = + %% TODO: Remove these deprecations on Elixir v2.0 + case Op of + '^^^' -> + Msg = "^^^ is deprecated. It is typically used as xor but it has the wrong precedence, use Bitwise.bxor/2 instead", + prepend_warning(Line, Column, Msg, Scope); + + '~~~' -> + Msg = "~~~ is deprecated. Use Bitwise.bnot/1 instead for clarity", + prepend_warning(Line, Column, Msg, Scope); + + '<|>' -> + Msg = "<|> is deprecated. Use another pipe-like operator", + prepend_warning(Line, Column, Msg, Scope); + + _ -> + Scope + end, + + Token = {Kind, {Line, Column, previous_was_eol(Tokens)}, Op}, + tokenize(Remaining, Line, Column + Length + Extra, NewScope, add_token_with_eol(Token, Tokens)) end. % ## Three Token Operators -handle_dot([$.,T1,T2,T3|Rest], Line, Scope, Tokens) when +handle_dot([$., T1, T2, T3 | Rest], Line, Column, DotInfo, Scope, Tokens) when ?unary_op3(T1, T2, T3); ?comp_op3(T1, T2, T3); ?and_op3(T1, T2, T3); ?or_op3(T1, T2, T3); - ?arrow_op3(T1, T2, T3); ?hat_op3(T1, T2, T3) -> - handle_call_identifier(Rest, Line, list_to_atom([T1, T2, T3]), Scope, Tokens); + ?arrow_op3(T1, T2, T3); ?xor_op3(T1, T2, T3); ?concat_op3(T1, T2, T3) -> + handle_call_identifier(Rest, Line, Column, DotInfo, 3, [T1, T2, T3], Scope, Tokens); % ## Two Token Operators -handle_dot([$.,T1,T2|Rest], Line, Scope, Tokens) when +handle_dot([$., T1, T2 | Rest], Line, Column, DotInfo, Scope, Tokens) when ?comp_op2(T1, T2); ?rel_op2(T1, T2); ?and_op(T1, T2); ?or_op(T1, T2); - ?arrow_op(T1, T2); ?in_match_op(T1, T2); ?two_op(T1, T2); ?stab_op(T1, T2); - ?type_op(T1, T2) -> - handle_call_identifier(Rest, Line, list_to_atom([T1, T2]), Scope, Tokens); + ?arrow_op(T1, T2); ?in_match_op(T1, T2); ?concat_op(T1, T2); ?power_op(T1, T2); + ?type_op(T1, T2); ?range_op(T1, T2) -> + handle_call_identifier(Rest, Line, Column, DotInfo, 2, [T1, T2], Scope, Tokens); % ## Single Token Operators -handle_dot([$.,T|Rest], Line, Scope, Tokens) when +handle_dot([$., T | Rest], Line, Column, DotInfo, Scope, Tokens) when ?at_op(T); ?unary_op(T); ?capture_op(T); ?dual_op(T); ?mult_op(T); - ?rel_op(T); ?match_op(T); ?pipe_op(T); T == $% -> - handle_call_identifier(Rest, Line, list_to_atom([T]), Scope, Tokens); + ?rel_op(T); ?match_op(T); ?pipe_op(T) -> + handle_call_identifier(Rest, Line, Column, DotInfo, 1, [T], Scope, Tokens); % ## Exception for .( as it needs to be treated specially in the parser -handle_dot([$.,$(|Rest], Line, Scope, Tokens) -> - tokenize([$(|Rest], Line, Scope, add_token_with_nl({dot_call_op, Line, '.'}, Tokens)); +handle_dot([$., $( | Rest], Line, Column, DotInfo, Scope, Tokens) -> + TokensSoFar = add_token_with_eol({dot_call_op, DotInfo, '.'}, Tokens), + tokenize([$( | Rest], Line, Column, Scope, TokensSoFar); + +handle_dot([$., H | T] = Original, Line, Column, DotInfo, Scope, Tokens) when ?is_quote(H) -> + case elixir_interpolation:extract(Line, Column + 1, Scope, true, T, H) of + {NewLine, NewColumn, [Part], Rest, InterScope} when is_list(Part) -> + NewScope = case is_unnecessary_quote([Part], InterScope) of + true -> + WarnMsg = io_lib:format( + "found quoted call \"~ts\" but the quotes are not required. " + "Calls made exclusively of Unicode letters, numbers, and underscores " + "and not beginning with a number " + "do not require quotes", + [Part] + ), + prepend_warning(Line, Column, WarnMsg, InterScope); + + false -> + InterScope + end, -handle_dot([$.,H|T] = Original, Line, Scope, Tokens) when ?is_quote(H) -> - case elixir_interpolation:extract(Line, Scope, true, T, H) of - {NewLine, [Part], Rest} when is_binary(Part) -> - case unsafe_to_atom(Part, Line, Scope) of + case unsafe_to_atom(Part, Line, Column, NewScope) of {ok, Atom} -> - Token = check_call_identifier(identifier, Line, Atom, Rest), - tokenize(Rest, NewLine, Scope, [Token|add_token_with_nl({'.', Line}, Tokens)]); + Token = check_call_identifier(Line, Column, Part, Atom, Rest), + TokensSoFar = add_token_with_eol({'.', DotInfo}, Tokens), + tokenize(Rest, NewLine, NewColumn, NewScope, [Token | TokensSoFar]); + {error, Reason} -> - {error, Reason, Original, Tokens} + error(Reason, Original, NewScope, Tokens) end; + {_NewLine, _NewColumn, _Parts, Rest, NewScope} -> + Message = "interpolation is not allowed when calling function/macro. Found interpolation in a call starting with: ", + error({Line, Column, Message, [H]}, Rest, NewScope, Tokens); {error, Reason} -> - interpolation_error(Reason, Original, Tokens, " (for function name starting at line ~B)", [Line]) + interpolation_error(Reason, Original, Scope, Tokens, " (for function name starting at line ~B)", [Line]) end; -handle_dot([$.|Rest], Line, Scope, Tokens) -> - tokenize(Rest, Line, Scope, add_token_with_nl({'.', Line}, Tokens)). +handle_dot([$. | Rest], Line, Column, DotInfo, Scope, Tokens) -> + TokensSoFar = add_token_with_eol({'.', DotInfo}, Tokens), + tokenize(Rest, Line, Column, Scope, TokensSoFar). -handle_call_identifier(Rest, Line, Op, Scope, Tokens) -> - Token = check_call_identifier(identifier, Line, Op, Rest), - tokenize(Rest, Line, Scope, [Token|add_token_with_nl({'.', Line}, Tokens)]). +handle_call_identifier(Rest, Line, Column, DotInfo, Length, UnencodedOp, Scope, Tokens) -> + Token = check_call_identifier(Line, Column, UnencodedOp, list_to_atom(UnencodedOp), Rest), + TokensSoFar = add_token_with_eol({'.', DotInfo}, Tokens), + tokenize(Rest, Line, Column + Length, Scope, [Token | TokensSoFar]). + +% ## Ambiguous unary/binary operators tokens +handle_space_sensitive_tokens([Sign, NotMarker | T], Line, Column, Scope, [{Identifier, _, _} = H | Tokens]) when + ?dual_op(Sign), + not(?is_space(NotMarker)), + NotMarker =/= $(, NotMarker =/= $[, NotMarker =/= $<, NotMarker =/= ${, %% containers + NotMarker =/= $%, NotMarker =/= $+, NotMarker =/= $-, NotMarker =/= $/, NotMarker =/= $>, %% operators + NotMarker =/= $:, %% keywords + Identifier == identifier -> + Rest = [NotMarker | T], + DualOpToken = {dual_op, {Line, Column, nil}, list_to_atom([Sign])}, + tokenize(Rest, Line, Column + 1, Scope, [DualOpToken, setelement(1, H, op_identifier) | Tokens]); + +handle_space_sensitive_tokens([], Line, Column, + #elixir_tokenizer{cursor_completion=Cursor} = Scope, + [{identifier, Info, Identifier} | Tokens]) when Cursor /= false -> + tokenize([$(], Line, Column+1, Scope, [{paren_identifier, Info, Identifier} | Tokens]); + +handle_space_sensitive_tokens(String, Line, Column, Scope, Tokens) -> + tokenize(String, Line, Column, Scope, Tokens). %% Helpers -eol(_Line, _Mod, [{',',_}|_] = Tokens) -> Tokens; -eol(_Line, _Mod, [{eol,_,_}|_] = Tokens) -> Tokens; -eol(Line, Mod, Tokens) -> [{eol,Line,Mod}|Tokens]. +eol(_Line, _Column, [{',', {Line, Column, Count}} | Tokens]) -> + [{',', {Line, Column, Count + 1}} | Tokens]; +eol(_Line, _Column, [{';', {Line, Column, Count}} | Tokens]) -> + [{';', {Line, Column, Count + 1}} | Tokens]; +eol(_Line, _Column, [{eol, {Line, Column, Count}} | Tokens]) -> + [{eol, {Line, Column, Count + 1}} | Tokens]; +eol(Line, Column, Tokens) -> + [{eol, {Line, Column, 1}} | Tokens]. + +is_unnecessary_quote([Part], #elixir_tokenizer{warn_on_unnecessary_quotes=true} = Scope) when is_list(Part) -> + case (Scope#elixir_tokenizer.identifier_tokenizer):tokenize(Part) of + {identifier, _, [], _, true, Special} -> not lists:member(at, Special); + _ -> false + end; + +is_unnecessary_quote(_Parts, _Scope) -> + false. -unsafe_to_atom(Part, Line, #elixir_tokenizer{}) when - is_binary(Part) andalso size(Part) > 255; +unsafe_to_atom(Part, Line, Column, #elixir_tokenizer{}) when + is_binary(Part) andalso byte_size(Part) > 255; is_list(Part) andalso length(Part) > 255 -> - {error, {Line, "atom length must be less than system limit", ":"}}; -unsafe_to_atom(Binary, _Line, #elixir_tokenizer{existing_atoms_only=true}) when is_binary(Binary) -> - {ok, binary_to_existing_atom(Binary, utf8)}; -unsafe_to_atom(Binary, _Line, #elixir_tokenizer{}) when is_binary(Binary) -> + {error, {Line, Column, "atom length must be less than system limit: ", elixir_utils:characters_to_list(Part)}}; +unsafe_to_atom(Part, Line, Column, #elixir_tokenizer{static_atoms_encoder=StaticAtomsEncoder}) when + is_function(StaticAtomsEncoder) -> + Value = elixir_utils:characters_to_binary(Part), + case StaticAtomsEncoder(Value, [{line, Line}, {column, Column}]) of + {ok, Term} -> + {ok, Term}; + {error, Reason} when is_binary(Reason) -> + {error, {Line, Column, elixir_utils:characters_to_list(Reason) ++ ": ", elixir_utils:characters_to_list(Part)}} + end; +unsafe_to_atom(Binary, Line, Column, #elixir_tokenizer{existing_atoms_only=true}) when is_binary(Binary) -> + try + {ok, binary_to_existing_atom(Binary, utf8)} + catch + error:badarg -> {error, {Line, Column, "unsafe atom does not exist: ", elixir_utils:characters_to_list(Binary)}} + end; +unsafe_to_atom(Binary, _Line, _Column, #elixir_tokenizer{}) when is_binary(Binary) -> {ok, binary_to_atom(Binary, utf8)}; -unsafe_to_atom(List, _Line, #elixir_tokenizer{existing_atoms_only=true}) when is_list(List) -> - {ok, list_to_existing_atom(List)}; -unsafe_to_atom(List, _Line, #elixir_tokenizer{}) when is_list(List) -> +unsafe_to_atom(List, Line, Column, #elixir_tokenizer{existing_atoms_only=true}) when is_list(List) -> + try + {ok, list_to_existing_atom(List)} + catch + error:badarg -> {error, {Line, Column, "unsafe atom does not exist: ", List}} + end; +unsafe_to_atom(List, _Line, _Column, #elixir_tokenizer{}) when is_list(List) -> {ok, list_to_atom(List)}. -collect_modifiers([H|T], Buffer) when ?is_downcase(H) -> - collect_modifiers(T, [H|Buffer]); +collect_modifiers([H | T], Buffer) when ?is_downcase(H) or ?is_upcase(H) or ?is_digit(H) -> + collect_modifiers(T, [H | Buffer]); collect_modifiers(Rest, Buffer) -> {Rest, lists:reverse(Buffer)}. %% Heredocs -extract_heredoc_with_interpolation(Line, Scope, Interpol, T, H) -> - case extract_heredoc(Line, T, H) of - {ok, NewLine, Body, Rest} -> - case elixir_interpolation:extract(Line + 1, Scope, Interpol, Body, 0) of - {error, Reason} -> - {error, interpolation_format(Reason, " (for heredoc starting at line ~B)", [Line])}; - {_, Parts, []} -> - {ok, NewLine, Parts, Rest} - end; - {error, _} = Error -> - Error - end. - -extract_heredoc(Line0, Rest0, Marker) -> - case extract_heredoc_header(Rest0) of - {ok, Rest1} -> +extract_heredoc_with_interpolation(Line, Column, Scope, Interpol, T, H) -> + case extract_heredoc_header(T) of + {ok, Headerless} -> %% We prepend a new line so we can transparently remove - %% spaces later. This new line is removed by calling `tl` + %% spaces later. This new line is removed by calling "tl" %% in the final heredoc body three lines below. - case extract_heredoc_body(Line0, Marker, [$\n|Rest1], []) of - {ok, Line1, Body, Rest2, Spaces} -> - {ok, Line1, tl(remove_heredoc_spaces(Body, Spaces)), Rest2}; - {error, ErrorLine} -> - Terminator = [Marker, Marker, Marker], - Message = "missing terminator: ~ts (for heredoc starting at line ~B)", - {error, {ErrorLine, io_lib:format(Message, [Terminator, Line0]), []}} + case elixir_interpolation:extract(Line, Column, Scope, Interpol, [$\n|Headerless], [H,H,H]) of + {NewLine, NewColumn, Parts0, Rest, InterScope} -> + Indent = NewColumn - 4, + Fun = fun(Part, Acc) -> extract_heredoc_indent(Part, Acc, Indent) end, + {Parts1, {ShouldWarn, _}} = lists:mapfoldl(Fun, {false, Line}, Parts0), + Parts2 = extract_heredoc_head(Parts1), + NewScope = maybe_heredoc_warn(ShouldWarn, Column, InterScope, H), + {ok, NewLine, NewColumn, tokens_to_binary(Parts2), Rest, NewScope}; + + {error, Reason} -> + {error, interpolation_format(Reason, " (for heredoc starting at line ~B)", [Line])} end; + error -> - Message = "heredoc start must be followed by a new line after ", - {error, {Line0, io_lib:format(Message, []), [Marker, Marker, Marker]}} + Message = "heredoc allows only zero or more whitespace characters followed by a new line after ", + {error, {Line, Column, io_lib:format(Message, []), [H, H, H]}} end. -%% Remove spaces from heredoc based on the position of the final quotes. - -remove_heredoc_spaces(Body, 0) -> - lists:reverse([0|Body]); -remove_heredoc_spaces(Body, Spaces) -> - remove_heredoc_spaces([0|Body], [], Spaces, Spaces). -remove_heredoc_spaces([H,$\n|T], [Backtrack|Buffer], Spaces, Original) when Spaces > 0, ?is_horizontal_space(H) -> - remove_heredoc_spaces([Backtrack,$\n|T], Buffer, Spaces - 1, Original); -remove_heredoc_spaces([$\n=H|T], Buffer, _Spaces, Original) -> - remove_heredoc_spaces(T, [H|Buffer], Original, Original); -remove_heredoc_spaces([H|T], Buffer, Spaces, Original) -> - remove_heredoc_spaces(T, [H|Buffer], Spaces, Original); -remove_heredoc_spaces([], Buffer, _Spaces, _Original) -> - Buffer. - -%% Extract the heredoc header. - extract_heredoc_header("\r\n" ++ Rest) -> {ok, Rest}; extract_heredoc_header("\n" ++ Rest) -> {ok, Rest}; -extract_heredoc_header([H|T]) when ?is_horizontal_space(H) -> +extract_heredoc_header([H | T]) when ?is_horizontal_space(H) -> extract_heredoc_header(T); extract_heredoc_header(_) -> error. -%% Extract heredoc body. It returns the heredoc body (in reverse order), -%% the remaining of the document and the number of spaces the heredoc -%% is aligned. - -extract_heredoc_body(Line, Marker, Rest, Buffer) -> - case extract_heredoc_line(Marker, Rest, Buffer, 0) of - {ok, NewBuffer, NewRest} -> - extract_heredoc_body(Line + 1, Marker, NewRest, NewBuffer); - {ok, NewBuffer, NewRest, Spaces} -> - {ok, Line, NewBuffer, NewRest, Spaces}; - {error, eof} -> - {error, Line} - end. - -%% Extract a line from the heredoc prepending its contents to a buffer. - -extract_heredoc_line("\r\n" ++ Rest, Buffer) -> - {ok, [$\n|Buffer], Rest}; -extract_heredoc_line("\n" ++ Rest, Buffer) -> - {ok, [$\n|Buffer], Rest}; -extract_heredoc_line([H|T], Buffer) -> - extract_heredoc_line(T, [H|Buffer]); -extract_heredoc_line(_, _) -> - {error, eof}. - -%% Extract each heredoc line trying to find a match according to the marker. +extract_heredoc_indent(Part, {Warned, Line}, Indent) when is_list(Part) -> + extract_heredoc_indent(Part, [], Warned, Line, Indent); +extract_heredoc_indent({_, {EndLine, _, _}, _} = Part, {Warned, _Line}, _Indent) -> + {Part, {Warned, EndLine}}. + +extract_heredoc_indent([$\n | Rest], Acc, Warned, Line, Indent) -> + {Trimmed, ShouldWarn} = trim_space(Rest, Indent), + Warn = if ShouldWarn, not Warned -> Line + 1; true -> Warned end, + extract_heredoc_indent(Trimmed, [$\n | Acc], Warn, Line + 1, Indent); +extract_heredoc_indent([Head | Rest], Acc, Warned, Line, Indent) -> + extract_heredoc_indent(Rest, [Head | Acc], Warned, Line, Indent); +extract_heredoc_indent([], Acc, Warned, Line, _Indent) -> + {lists:reverse(Acc), {Warned, Line}}. + +trim_space(Rest, 0) -> {Rest, false}; +trim_space([$\r, $\n | _] = Rest, _) -> {Rest, false}; +trim_space([$\n | _] = Rest, _) -> {Rest, false}; +trim_space([H | T], Spaces) when ?is_horizontal_space(H) -> trim_space(T, Spaces - 1); +trim_space([], _Spaces) -> {[], false}; +trim_space(Rest, _Spaces) -> {Rest, true}. + +maybe_heredoc_warn(false, _Column, Scope, _Marker) -> + Scope; +maybe_heredoc_warn(Line, Column, Scope, Marker) -> + Msg = io_lib:format("outdented heredoc line. The contents inside the heredoc should be indented " + "at the same level as the closing ~ts. The following is forbidden:~n~n" + " def text do~n" + " \"\"\"~n" + " contents~n" + " \"\"\"~n" + " end~n~n" + "Instead make sure the contents are indented as much as the heredoc closing:~n~n" + " def text do~n" + " \"\"\"~n" + " contents~n" + " \"\"\"~n" + " end~n~n" + "The current heredoc line is indented too little", [[Marker, Marker, Marker]]), + + prepend_warning(Line, Column, Msg, Scope). + +extract_heredoc_head([[$\n|H]|T]) -> [H|T]. + +unescape_tokens(Tokens, Line, Column, #elixir_tokenizer{unescape=true}) -> + case elixir_interpolation:unescape_tokens(Tokens) of + {ok, Result} -> + {ok, Result}; + + {error, Message, Token} -> + {error, {Line, Column, Message ++ ". Syntax error after: ", Token}} + end; +unescape_tokens(Tokens, _Line, _Column, #elixir_tokenizer{unescape=false}) -> + {ok, tokens_to_binary(Tokens)}. -extract_heredoc_line(Marker, [H|T], Buffer, Counter) when ?is_horizontal_space(H) -> - extract_heredoc_line(Marker, T, [H|Buffer], Counter + 1); -extract_heredoc_line(Marker, [Marker,Marker,Marker|T], Buffer, Counter) -> - {ok, Buffer, T, Counter}; -extract_heredoc_line(_Marker, Rest, Buffer, _Counter) -> - extract_heredoc_line(Rest, Buffer). +tokens_to_binary(Tokens) -> + [if is_list(Token) -> elixir_utils:characters_to_binary(Token); true -> Token end + || Token <- Tokens]. %% Integers and floats %% At this point, we are at least sure the first digit is a number. %% Check if we have a point followed by a number; -tokenize_number([$.,H|T], Acc, false) when ?is_digit(H) -> - tokenize_number(T, [H,$.|Acc], true); +tokenize_number([$., H | T], Acc, Length, false) when ?is_digit(H) -> + tokenize_number(T, [H, $. | Acc], Length + 2, true); %% Check if we have an underscore followed by a number; -tokenize_number([$_,H|T], Acc, Bool) when ?is_digit(H) -> - tokenize_number(T, [H|Acc], Bool); +tokenize_number([$_, H | T], Acc, Length, Bool) when ?is_digit(H) -> + tokenize_number(T, [H, $_ | Acc], Length + 2, Bool); %% Check if we have e- followed by numbers (valid only for floats); -tokenize_number([E,S,H|T], Acc, true) - when (E == $E) or (E == $e), ?is_digit(H), S == $+ orelse S == $- -> - tokenize_number(T, [H,S,$e|Acc], true); +tokenize_number([E, S, H | T], Acc, Length, true) + when (E =:= $E) or (E =:= $e), ?is_digit(H), S =:= $+ orelse S =:= $- -> + tokenize_number(T, [H, S, E | Acc], Length + 3, true); %% Check if we have e followed by numbers (valid only for floats); -tokenize_number([E,H|T], Acc, true) - when (E == $E) or (E == $e), ?is_digit(H) -> - tokenize_number(T, [H,$e|Acc], true); +tokenize_number([E, H | T], Acc, Length, true) + when (E =:= $E) or (E =:= $e), ?is_digit(H) -> + tokenize_number(T, [H, E | Acc], Length + 2, true); %% Finally just numbers. -tokenize_number([H|T], Acc, Bool) when ?is_digit(H) -> - tokenize_number(T, [H|Acc], Bool); +tokenize_number([H | T], Acc, Length, Bool) when ?is_digit(H) -> + tokenize_number(T, [H | Acc], Length + 1, Bool); %% Cast to float... -tokenize_number(Rest, Acc, true) -> - {Rest, list_to_float(lists:reverse(Acc))}; +tokenize_number(Rest, Acc, Length, true) -> + try + {Number, Original} = reverse_number(Acc, [], []), + {Rest, list_to_float(Number), Original, Length} + catch + error:badarg -> {error, "invalid float number ", lists:reverse(Acc)} + end; %% Or integer. -tokenize_number(Rest, Acc, false) -> - {Rest, list_to_integer(lists:reverse(Acc))}. - -tokenize_hex([H|T], Acc) when ?is_hex(H) -> tokenize_hex(T, [H|Acc]); -tokenize_hex(Rest, Acc) -> {Rest, list_to_integer(lists:reverse(Acc), 16)}. - -tokenize_octal([H|T], Acc) when ?is_octal(H) -> tokenize_octal(T, [H|Acc]); -tokenize_octal(Rest, Acc) -> {Rest, list_to_integer(lists:reverse(Acc), 8)}. - -tokenize_bin([H|T], Acc) when ?is_bin(H) -> tokenize_bin(T, [H|Acc]); -tokenize_bin(Rest, Acc) -> {Rest, list_to_integer(lists:reverse(Acc), 2)}. +tokenize_number(Rest, Acc, Length, false) -> + {Number, Original} = reverse_number(Acc, [], []), + {Rest, list_to_integer(Number), Original, Length}. + +tokenize_hex([H | T], Acc, Length) when ?is_hex(H) -> + tokenize_hex(T, [H | Acc], Length + 1); +tokenize_hex([$_, H | T], Acc, Length) when ?is_hex(H) -> + tokenize_hex(T, [H, $_ | Acc], Length + 2); +tokenize_hex(Rest, Acc, Length) -> + {Number, Original} = reverse_number(Acc, [], []), + {Rest, list_to_integer(Number, 16), [$0, $x | Original], Length}. + +tokenize_octal([H | T], Acc, Length) when ?is_octal(H) -> + tokenize_octal(T, [H | Acc], Length + 1); +tokenize_octal([$_, H | T], Acc, Length) when ?is_octal(H) -> + tokenize_octal(T, [H, $_ | Acc], Length + 2); +tokenize_octal(Rest, Acc, Length) -> + {Number, Original} = reverse_number(Acc, [], []), + {Rest, list_to_integer(Number, 8), [$0, $o | Original], Length}. + +tokenize_bin([H | T], Acc, Length) when ?is_bin(H) -> + tokenize_bin(T, [H | Acc], Length + 1); +tokenize_bin([$_, H | T], Acc, Length) when ?is_bin(H) -> + tokenize_bin(T, [H, $_ | Acc], Length + 2); +tokenize_bin(Rest, Acc, Length) -> + {Number, Original} = reverse_number(Acc, [], []), + {Rest, list_to_integer(Number, 2), [$0, $b | Original], Length}. + +reverse_number([$_ | T], Number, Original) -> + reverse_number(T, Number, [$_ | Original]); +reverse_number([H | T], Number, Original) -> + reverse_number(T, [H | Number], [H | Original]); +reverse_number([], Number, Original) -> + {Number, Original}. %% Comments -tokenize_comment("\r\n" ++ _ = Rest) -> Rest; -tokenize_comment("\n" ++ _ = Rest) -> Rest; -tokenize_comment([_|Rest]) -> tokenize_comment(Rest); -tokenize_comment([]) -> []. - -%% Atoms -%% Handle atoms specially since they support @ - -tokenize_atom([H|T], Acc) when ?is_atom(H) -> - tokenize_atom(T, [H|Acc]); - -tokenize_atom([H|T], Acc) when H == $?; H == $! -> - {T, lists:reverse([H|Acc])}; - -tokenize_atom(Rest, Acc) -> - {Rest, lists:reverse(Acc)}. +reset_eol([{eol, {Line, Column, _}} | Rest]) -> [{eol, {Line, Column, 0}} | Rest]; +reset_eol(Rest) -> Rest. + +tokenize_comment("\r\n" ++ _ = Rest, Acc) -> + {Rest, lists:reverse(Acc)}; +tokenize_comment("\n" ++ _ = Rest, Acc) -> + {Rest, lists:reverse(Acc)}; +tokenize_comment([H | _Rest], _) when ?bidi(H) -> + {error, H}; +tokenize_comment([H | Rest], Acc) -> + tokenize_comment(Rest, [H | Acc]); +tokenize_comment([], Acc) -> + {[], lists:reverse(Acc)}. + +error_comment(H, Comment, Line, Column, Scope, Tokens) -> + Token = io_lib:format("\\u~4.16.0B", [H]), + Reason = {Line, Column, "invalid bidirectional formatting character in comment: ", Token}, + error(Reason, Comment, Scope, Tokens). + +preserve_comments(Line, Column, Tokens, Comment, Rest, Scope) -> + case Scope#elixir_tokenizer.preserve_comments of + Fun when is_function(Fun) -> + Fun(Line, Column, Tokens, Comment, Rest); + nil -> + ok + end. %% Identifiers -%% At this point, the validity of the first character was already verified. -tokenize_identifier([H|T], Acc) when ?is_identifier(H) -> - tokenize_identifier(T, [H|Acc]); - -tokenize_identifier(Rest, Acc) -> - {Rest, lists:reverse(Acc)}. +tokenize([H | T]) when ?is_upcase(H) -> + {Acc, Rest, Length, Special} = tokenize_continue(T, [H], 1, []), + {alias, lists:reverse(Acc), Rest, Length, true, Special}; +tokenize([H | T]) when ?is_downcase(H); H =:= $_ -> + {Acc, Rest, Length, Special} = tokenize_continue(T, [H], 1, []), + {identifier, lists:reverse(Acc), Rest, Length, true, Special}; +tokenize(_List) -> + {error, empty}. + +tokenize_continue([$@ | T], Acc, Length, Special) -> + tokenize_continue(T, [$@ | Acc], Length + 1, [at | lists:delete(at, Special)]); +tokenize_continue([$! | T], Acc, Length, Special) -> + {[$! | Acc], T, Length + 1, [punctuation | Special]}; +tokenize_continue([$? | T], Acc, Length, Special) -> + {[$? | Acc], T, Length + 1, [punctuation | Special]}; +tokenize_continue([H | T], Acc, Length, Special) when ?is_upcase(H); ?is_downcase(H); ?is_digit(H); H =:= $_ -> + tokenize_continue(T, [H | Acc], Length + 1, Special); +tokenize_continue(Rest, Acc, Length, Special) -> + {Acc, Rest, Length, Special}. + +tokenize_identifier(String, Line, Column, Scope, MaybeKeyword) -> + case (Scope#elixir_tokenizer.identifier_tokenizer):tokenize(String) of + {Kind, Acc, Rest, Length, Ascii, Special} -> + Keyword = MaybeKeyword andalso maybe_keyword(Rest), + + case keyword_or_unsafe_to_atom(Keyword, Acc, Line, Column, Scope) of + {keyword, Atom, Type} -> + {keyword, Atom, Type, Rest, Length}; + {ok, Atom} -> + {Kind, Acc, Atom, Rest, Length, Ascii, Special}; + {error, _Reason} = Error -> + Error + end; -%% Tokenize any identifier, handling kv, punctuated, paren, bracket and do identifiers. + {error, {not_highly_restrictive, Wrong, {Prefix, Suffix}}} -> + WrongColumn = Column + length(Wrong) - 1, + case suggest_simpler_unexpected_token_in_error(Wrong, Line, WrongColumn, Scope) of + no_suggestion -> + %% we append a pointer to more info if we aren't appending a suggestion + MoreInfo = "\nSee https://hexdocs.pm/elixir/unicode-syntax.html for more information.", + {error, {Line, Column, {Prefix, Suffix ++ MoreInfo}, Wrong}}; -tokenize_any_identifier(Original, Line, Scope, Tokens) -> - {Rest, Identifier} = tokenize_identifier(Original, []), + {_, {Line, WrongColumn, _, SuggestionMessage}} = _SuggestionError -> + {error, {Line, WrongColumn, {Prefix, Suffix ++ SuggestionMessage}, Wrong}} + end; - {AllIdentifier, AllRest} = - case Rest of - [H|T] when H == $?; H == $! -> {Identifier ++ [H], T}; - _ -> {Identifier, Rest} - end, + {error, {unexpected_token, Wrong}} -> + WrongColumn = Column + length(Wrong) - 1, + case suggest_simpler_unexpected_token_in_error(Wrong, Line, WrongColumn, Scope) of + no_suggestion -> + [T | _] = lists:reverse(Wrong), + case suggest_simpler_unexpected_token_in_error([T], Line, WrongColumn, Scope) of + no_suggestion -> {unexpected_token, length(Wrong)}; + SuggestionError -> SuggestionError + end; + + SuggestionError -> + SuggestionError + end; - case unsafe_to_atom(AllIdentifier, Line, Scope) of - {ok, Atom} -> - tokenize_kw_or_other(AllRest, identifier, Line, Atom, Tokens); - {error, Reason} -> - {error, Reason, Original, Tokens} + {error, empty} -> + empty end. -tokenize_kw_or_other([$:,H|T], _Kind, Line, Atom, _Tokens) when ?is_space(H) -> - {identifier, [H|T], {kw_identifier, Line, Atom}}; - -tokenize_kw_or_other([$:,H|T], _Kind, Line, Atom, Tokens) when ?is_atom_start(H); ?is_digit(H) -> - Original = atom_to_list(Atom) ++ [$:], - Reason = {Line, "keyword argument must be followed by space after: ", Original}, - {error, Reason, Original ++ [H|T], Tokens}; - -tokenize_kw_or_other(Rest, Kind, Line, Atom, Tokens) -> - case check_keyword(Line, Atom, Tokens) of - nomatch -> - {identifier, Rest, check_call_identifier(Kind, Line, Atom, Rest)}; - {ok, [Check|T]} -> - {keyword, Rest, Check, T}; - {error, Token} -> - {error, {Line, "syntax error before: ", Token}, atom_to_list(Atom) ++ Rest, Tokens} +%% heuristic: try nfkc; try confusability skeleton; try calling this again w/just failed codepoint +suggest_simpler_unexpected_token_in_error(Wrong, Line, WrongColumn, Scope) -> + NFKC = unicode:characters_to_nfkc_list(Wrong), + case (Scope#elixir_tokenizer.identifier_tokenizer):tokenize(NFKC) of + {error, _Reason} -> + ConfusableSkeleton = 'Elixir.String.Tokenizer.Security':confusable_skeleton(Wrong), + case (Scope#elixir_tokenizer.identifier_tokenizer):tokenize(ConfusableSkeleton) of + {_, Simpler, _, _, _, _} -> + Message = suggest_change("Codepoint failed identifier tokenization, but a simpler form was found.", + Wrong, + "You could write the above in a similar way that is accepted by Elixir:", + Simpler, + "See https://hexdocs.pm/elixir/unicode-syntax.html for more information."), + {error, {Line, WrongColumn, "unexpected token: ", Message}}; + _other -> + no_suggestion + end; + {_, _NFKC, _, _, _, _} -> + Message = suggest_change("Elixir expects unquoted Unicode atoms, variables, and calls to use allowed codepoints and to be in NFC form.", + Wrong, + "You could write the above in a compatible format that is accepted by Elixir:", + NFKC, + "See https://hexdocs.pm/elixir/unicode-syntax.html for more information."), + {error, {Line, WrongColumn, "unexpected token: ", Message}} + end. + +suggest_change(Intro, WrongForm, Hint, HintedForm, Ending) -> + WrongCodepoints = list_to_codepoint_hex(WrongForm), + HintedCodepoints = list_to_codepoint_hex(HintedForm), + io_lib:format("~ts\n\nGot:\n\n \"~ts\" (code points~ts)\n\n" + "Hint: ~ts\n\n \"~ts\" (code points~ts)\n\n~ts", + [Intro, WrongForm, WrongCodepoints, Hint, HintedForm, HintedCodepoints, Ending]). + +maybe_keyword([]) -> true; +maybe_keyword([$:, $: | _]) -> true; +maybe_keyword([$: | _]) -> false; +maybe_keyword(_) -> true. + +list_to_codepoint_hex(List) -> + [io_lib:format(" 0x~5.16.0B", [Codepoint]) || Codepoint <- List]. + +tokenize_alias(Rest, Line, Column, Unencoded, Atom, Length, Ascii, Special, Scope, Tokens) -> + if + not Ascii or (Special /= []) -> + Invalid = hd([C || C <- Unencoded, (C < $A) or (C > 127)]), + Reason = {Line, Column, invalid_character_error("alias (only ASCII characters, without punctuation, are allowed)", Invalid), Unencoded}, + error(Reason, Unencoded ++ Rest, Scope, Tokens); + + true -> + AliasesToken = {alias, {Line, Column, Unencoded}, Atom}, + tokenize(Rest, Line, Column + Length, Scope, [AliasesToken | Tokens]) end. %% Check if it is a call identifier (paren | bracket | do) -check_call_identifier(_Kind, Line, Atom, [$(|_]) -> {paren_identifier, Line, Atom}; -check_call_identifier(_Kind, Line, Atom, [$[|_]) -> {bracket_identifier, Line, Atom}; -check_call_identifier(Kind, Line, Atom, _Rest) -> {Kind, Line, Atom}. +check_call_identifier(Line, Column, Unencoded, Atom, [$( | _]) -> + {paren_identifier, {Line, Column, Unencoded}, Atom}; +check_call_identifier(Line, Column, Unencoded, Atom, [$[ | _]) -> + {bracket_identifier, {Line, Column, Unencoded}, Atom}; +check_call_identifier(Line, Column, Unencoded, Atom, _Rest) -> + {identifier, {Line, Column, Unencoded}, Atom}. + +add_token_with_eol({unary_op, _, _} = Left, T) -> [Left | T]; +add_token_with_eol(Left, [{eol, _} | T]) -> [Left | T]; +add_token_with_eol(Left, T) -> [Left | T]. -add_token_with_nl(Left, [{eol,_,newline}|T]) -> [Left|T]; -add_token_with_nl(Left, T) -> [Left|T]. +previous_was_eol([{',', {_, _, Count}} | _]) when Count > 0 -> Count; +previous_was_eol([{';', {_, _, Count}} | _]) when Count > 0 -> Count; +previous_was_eol([{eol, {_, _, Count}} | _]) when Count > 0 -> Count; +previous_was_eol(_) -> nil. %% Error handling -interpolation_error(Reason, Rest, Tokens, Extension, Args) -> - {error, interpolation_format(Reason, Extension, Args), Rest, Tokens}. +interpolation_error(Reason, Rest, Scope, Tokens, Extension, Args) -> + error(interpolation_format(Reason, Extension, Args), Rest, Scope, Tokens). -interpolation_format({string, Line, Message, Token}, Extension, Args) -> - {Line, io_lib:format("~ts" ++ Extension, [Message|Args]), Token}; -interpolation_format({_, _, _} = Reason, _Extension, _Args) -> +interpolation_format({string, Line, Column, Message, Token}, Extension, Args) -> + {Line, Column, [Message, io_lib:format(Extension, Args)], Token}; +interpolation_format({_, _, _, _} = Reason, _Extension, _Args) -> Reason. %% Terminators -handle_terminator(Rest, Line, Scope, Token, Tokens) -> - case handle_terminator(Token, Scope) of +handle_terminator(Rest, Line, Column, Scope, {'(', _}, [{alias, _, Alias} | Tokens]) -> + Reason = + io_lib:format( + "unexpected ( after alias ~ts. Function names and identifiers in Elixir " + "start with lowercase characters or underscore. For example:\n\n" + " hello_world()\n" + " _starting_with_underscore()\n" + " numb3rs_are_allowed()\n" + " may_finish_with_question_mark?()\n" + " may_finish_with_exclamation_mark!()\n\n" + "Unexpected token: ", + [Alias] + ), + + error({Line, Column, Reason, ["("]}, atom_to_list(Alias) ++ [$( | Rest], Scope, Tokens); +handle_terminator(Rest, Line, Column, #elixir_tokenizer{terminators=none} = Scope, Token, Tokens) -> + tokenize(Rest, Line, Column, Scope, [Token | Tokens]); +handle_terminator(Rest, Line, Column, Scope, Token, Tokens) -> + #elixir_tokenizer{terminators=Terminators} = Scope, + + case check_terminator(Token, Terminators, Scope) of {error, Reason} -> - {error, Reason, atom_to_list(element(1, Token)) ++ Rest, Tokens}; - New -> - tokenize(Rest, Line, New, [Token|Tokens]) + error(Reason, atom_to_list(element(1, Token)) ++ Rest, Scope, Tokens); + {ok, New} -> + tokenize(Rest, Line, Column, New, [Token | Tokens]) end. -handle_terminator(_, #elixir_tokenizer{check_terminators=false} = Scope) -> - Scope; -handle_terminator(Token, #elixir_tokenizer{terminators=Terminators} = Scope) -> - case check_terminator(Token, Terminators) of - {error, _} = Error -> Error; - New -> Scope#elixir_tokenizer{terminators=New} - end. +check_terminator({Start, {Line, _, _}}, Terminators, Scope) + when Start == '('; Start == '['; Start == '{'; Start == '<<' -> + Indentation = Scope#elixir_tokenizer.indentation, + {ok, Scope#elixir_tokenizer{terminators=[{Start, Line, Indentation} | Terminators]}}; -check_terminator({S, Line}, Terminators) when S == 'fn' -> - [{fn, Line}|Terminators]; - -check_terminator({S, _} = New, Terminators) when - S == 'do'; - S == '('; - S == '['; - S == '{'; - S == '<<' -> - [New|Terminators]; - -check_terminator({E, _}, [{S, _}|Terminators]) when - S == 'do', E == 'end'; - S == 'fn', E == 'end'; - S == '(', E == ')'; - S == '[', E == ']'; - S == '{', E == '}'; - S == '<<', E == '>>' -> - Terminators; - -check_terminator({E, Line}, [{Start, StartLine}|_]) when - E == 'end'; E == ')'; E == ']'; E == '}'; E == '>>' -> - End = terminator(Start), - Message = io_lib:format("\"~ts\" starting at line ~B is missing terminator \"~ts\". " - "Unexpected token: ", [Start, StartLine, End]), - {error, {Line, Message, atom_to_list(E)}}; +check_terminator({Start, {Line, _, _}}, Terminators, Scope) when Start == 'fn'; Start == 'do' -> + Indentation = Scope#elixir_tokenizer.indentation, + + NewScope = + case Terminators of + %% If the do is indented equally or less than the previous do, it may be a missing end error! + [{Start, _, PreviousIndentation} = Previous | _] when Indentation =< PreviousIndentation -> + Scope#elixir_tokenizer{mismatch_hints=[Previous | Scope#elixir_tokenizer.mismatch_hints]}; + + _ -> + Scope + end, + + {ok, NewScope#elixir_tokenizer{terminators=[{Start, Line, Indentation} | Terminators]}}; + +check_terminator({'end', {EndLine, _, _}}, [{'do', _, Indentation} | Terminators], Scope) -> + NewScope = + %% If the end is more indented than the do, it may be a missing do error! + case Scope#elixir_tokenizer.indentation > Indentation of + true -> + Hint = {'end', EndLine, Scope#elixir_tokenizer.indentation}, + Scope#elixir_tokenizer{mismatch_hints=[Hint | Scope#elixir_tokenizer.mismatch_hints]}; + + false -> + Scope + end, + + {ok, NewScope#elixir_tokenizer{terminators=Terminators}}; + +check_terminator({End, {EndLine, EndColumn, _}}, [{Start, StartLine, _} | Terminators], Scope) + when End == 'end'; End == ')'; End == ']'; End == '}'; End == '>>' -> + case terminator(Start) of + End -> + {ok, Scope#elixir_tokenizer{terminators=Terminators}}; + + ExpectedEnd -> + Context = ". The \"~ts\" at line ~B is missing terminator \"~ts\"", + Suffix = [ + io_lib:format(Context, [Start, StartLine, ExpectedEnd]), + missing_terminator_hint(Start, ExpectedEnd, Scope) + ], + {error, {EndLine, EndColumn, {unexpected_token_or_reserved(End), Suffix}, [atom_to_list(End)]}} + end; + +check_terminator({'end', {Line, Column, _}}, [], #elixir_tokenizer{mismatch_hints=Hints}) -> + Suffix = + case lists:keyfind('end', 1, Hints) of + {'end', HintLine, _Identation} -> + io_lib:format("\n\n HINT: it looks like the \"end\" on line ~B " + "does not have a matching \"do\" defined before it\n", [HintLine]); + false -> + "" + end, + + {error, {Line, Column, {"unexpected reserved word: ", Suffix}, "end"}}; + +check_terminator({End, {Line, Column, _}}, [], _Scope) + when End == ')'; End == ']'; End == '}'; End == '>>' -> + {error, {Line, Column, "unexpected token: ", atom_to_list(End)}}; -check_terminator({E, Line}, []) when - E == 'end'; E == ')'; E == ']'; E == '}'; E == '>>' -> - {error, {Line, "unexpected token: ", atom_to_list(E)}}; +check_terminator(_, _, Scope) -> + {ok, Scope}. -check_terminator(_, Terminators) -> - Terminators. +unexpected_token_or_reserved('end') -> "unexpected reserved word: "; +unexpected_token_or_reserved(_) -> "unexpected token: ". + +missing_terminator_hint(Start, End, #elixir_tokenizer{mismatch_hints=Hints}) -> + case lists:keyfind(Start, 1, Hints) of + {Start, HintLine, _} -> + io_lib:format("\n\n HINT: it looks like the \"~ts\" on line ~B does not have a matching \"~ts\"\n", + [Start, HintLine, End]); + false -> + "" + end. string_type($") -> bin_string; string_type($') -> list_string. +heredoc_type($") -> bin_heredoc; +heredoc_type($') -> list_heredoc. + sigil_terminator($() -> $); sigil_terminator($[) -> $]; sigil_terminator(${) -> $}; @@ -904,55 +1494,246 @@ terminator('<<') -> '>>'. %% Keywords checking -check_keyword(_Line, _Atom, [{'.', _}|_]) -> - nomatch; -check_keyword(DoLine, do, [{Identifier, Line, Atom}|T]) when Identifier == identifier -> - {ok, add_token_with_nl({do, DoLine}, [{do_identifier, Line, Atom}|T])}; -check_keyword(Line, do, Tokens) -> - case do_keyword_valid(Tokens) of - true -> {ok, add_token_with_nl({do, Line}, Tokens)}; - false -> {error, "do"} +keyword_or_unsafe_to_atom(true, "fn", _Line, _Column, _Scope) -> {keyword, 'fn', terminator}; +keyword_or_unsafe_to_atom(true, "do", _Line, _Column, _Scope) -> {keyword, 'do', terminator}; +keyword_or_unsafe_to_atom(true, "end", _Line, _Column, _Scope) -> {keyword, 'end', terminator}; +keyword_or_unsafe_to_atom(true, "true", _Line, _Column, _Scope) -> {keyword, 'true', token}; +keyword_or_unsafe_to_atom(true, "false", _Line, _Column, _Scope) -> {keyword, 'false', token}; +keyword_or_unsafe_to_atom(true, "nil", _Line, _Column, _Scope) -> {keyword, 'nil', token}; + +keyword_or_unsafe_to_atom(true, "not", _Line, _Column, _Scope) -> {keyword, 'not', unary_op}; +keyword_or_unsafe_to_atom(true, "and", _Line, _Column, _Scope) -> {keyword, 'and', and_op}; +keyword_or_unsafe_to_atom(true, "or", _Line, _Column, _Scope) -> {keyword, 'or', or_op}; +keyword_or_unsafe_to_atom(true, "when", _Line, _Column, _Scope) -> {keyword, 'when', when_op}; +keyword_or_unsafe_to_atom(true, "in", _Line, _Column, _Scope) -> {keyword, 'in', in_op}; + +keyword_or_unsafe_to_atom(true, "after", _Line, _Column, _Scope) -> {keyword, 'after', block}; +keyword_or_unsafe_to_atom(true, "else", _Line, _Column, _Scope) -> {keyword, 'else', block}; +keyword_or_unsafe_to_atom(true, "catch", _Line, _Column, _Scope) -> {keyword, 'catch', block}; +keyword_or_unsafe_to_atom(true, "rescue", _Line, _Column, _Scope) -> {keyword, 'rescue', block}; + +keyword_or_unsafe_to_atom(_, Part, Line, Column, Scope) -> + unsafe_to_atom(Part, Line, Column, Scope). + +tokenize_keyword(terminator, Rest, Line, Column, Atom, Length, Scope, Tokens) -> + case tokenize_keyword_terminator(Line, Column, Atom, Tokens) of + {ok, [Check | T]} -> + handle_terminator(Rest, Line, Column + Length, Scope, Check, T); + {error, Message, Token} -> + error({Line, Column, Message, Token}, Token ++ Rest, Scope, Tokens) end; -check_keyword(Line, Atom, Tokens) -> - case keyword(Atom) of - false -> nomatch; - token -> {ok, [{Atom, Line}|Tokens]}; - block -> {ok, [{block_identifier, Line, Atom}|Tokens]}; - unary_op -> {ok, [{unary_op, Line, Atom}|Tokens]}; - Kind -> {ok, add_token_with_nl({Kind, Line, Atom}, Tokens)} - end. -%% do is only valid after the end, true, false and nil keywords -do_keyword_valid([{Atom, _}|_]) -> +tokenize_keyword(token, Rest, Line, Column, Atom, Length, Scope, Tokens) -> + Token = {Atom, {Line, Column, nil}}, + tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]); + +tokenize_keyword(block, Rest, Line, Column, Atom, Length, Scope, Tokens) -> + Token = {block_identifier, {Line, Column, nil}, Atom}, + tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]); + +tokenize_keyword(Kind, Rest, Line, Column, Atom, Length, Scope, Tokens) -> + NewTokens = + case strip_horizontal_space(Rest, 0) of + {[$/ | _], _} -> + [{identifier, {Line, Column, nil}, Atom} | Tokens]; + + _ -> + case {Kind, Tokens} of + {in_op, [{unary_op, NotInfo, 'not'} | T]} -> + add_token_with_eol({in_op, NotInfo, 'not in'}, T); + + {_, _} -> + add_token_with_eol({Kind, {Line, Column, previous_was_eol(Tokens)}, Atom}, Tokens) + end + end, + + tokenize(Rest, Line, Column + Length, Scope, NewTokens). + +%% Fail early on invalid do syntax. For example, after +%% most keywords, after comma and so on. +tokenize_keyword_terminator(DoLine, DoColumn, do, [{identifier, {Line, Column, Meta}, Atom} | T]) -> + {ok, add_token_with_eol({do, {DoLine, DoColumn, nil}}, + [{do_identifier, {Line, Column, Meta}, Atom} | T])}; +tokenize_keyword_terminator(_Line, _Column, do, [{'fn', _} | _]) -> + {error, invalid_do_with_fn_error("unexpected reserved word: "), "do"}; +tokenize_keyword_terminator(Line, Column, do, Tokens) -> + case is_valid_do(Tokens) of + true -> {ok, add_token_with_eol({do, {Line, Column, nil}}, Tokens)}; + false -> {error, invalid_do_error("unexpected reserved word: "), "do"} + end; +tokenize_keyword_terminator(Line, Column, Atom, Tokens) -> + {ok, [{Atom, {Line, Column, nil}} | Tokens]}. + +is_valid_do([{Atom, _} | _]) -> case Atom of - 'end' -> true; - nil -> true; - true -> true; - false -> true; - _ -> keyword(Atom) == false + ',' -> false; + ';' -> false; + 'not' -> false; + 'and' -> false; + 'or' -> false; + 'when' -> false; + 'in' -> false; + 'after' -> false; + 'else' -> false; + 'catch' -> false; + 'rescue' -> false; + _ -> true end; -do_keyword_valid(_) -> +is_valid_do(_) -> true. -% Regular keywords -keyword('fn') -> token; -keyword('end') -> token; -keyword('true') -> token; -keyword('false') -> token; -keyword('nil') -> token; - -% Operators keywords -keyword('not') -> unary_op; -keyword('and') -> and_op; -keyword('or') -> or_op; -keyword('xor') -> or_op; -keyword('when') -> when_op; -keyword('in') -> in_op; - -% Block keywords -keyword('after') -> block; -keyword('else') -> block; -keyword('rescue') -> block; -keyword('catch') -> block; - -keyword(_) -> false. +invalid_character_error(What, Char) -> + io_lib:format("invalid character \"~ts\" (code point U+~4.16.0B) in ~ts: ", [[Char], Char, What]). + +invalid_do_error(Prefix) -> + {Prefix, ". In case you wanted to write a \"do\" expression, " + "you must either use do-blocks or separate the keyword argument with comma. " + "For example, you should either write:\n\n" + " if some_condition? do\n" + " :this\n" + " else\n" + " :that\n" + " end\n\n" + "or the equivalent construct:\n\n" + " if(some_condition?, do: :this, else: :that)\n\n" + "where \"some_condition?\" is the first argument and the second argument is a keyword list.\n\n" + "You may see this error if you forget a trailing comma before the \"do\" in a \"do\" block"}. + +invalid_do_with_fn_error(Prefix) -> + {Prefix, ". Anonymous functions are written as:\n\n" + " fn pattern -> expression end"}. + +% TODO: Turn into an error on v2.0 +maybe_warn_too_many_of_same_char([T | _] = Token, [T | _] = _Rest, Line, Column, Scope) -> + Warning = + case T of + $. -> "please use parens around \"...\" instead"; + _ -> io_lib:format("please use a space between \"~ts\" and the next \"~ts\"", [Token, [T]]) + end, + Message = io_lib:format("found \"~ts\" followed by \"~ts\", ~ts", [Token, [T], Warning]), + prepend_warning(Line, Column, Message, Scope); +maybe_warn_too_many_of_same_char(_Token, _Rest, _Line, _Column, Scope) -> + Scope. + +%% TODO: Turn into an error on v2.0 +maybe_warn_for_ambiguous_bang_before_equals(Kind, Unencoded, [$= | _], Line, Column, Scope) -> + {What, Identifier} = + case Kind of + atom -> {"atom", [$: | Unencoded]}; + identifier -> {"identifier", Unencoded} + end, + + case lists:last(Identifier) of + Last when Last =:= $!; Last =:= $? -> + Msg = io_lib:format("found ~ts \"~ts\", ending with \"~ts\", followed by =. " + "It is unclear if you mean \"~ts ~ts=\" or \"~ts =\". Please add " + "a space before or after ~ts to remove the ambiguity", + [What, Identifier, [Last], lists:droplast(Identifier), [Last], Identifier, [Last]]), + prepend_warning(Line, Column, Msg, Scope); + _ -> + Scope + end; +maybe_warn_for_ambiguous_bang_before_equals(_Kind, _Atom, _Rest, _Line, _Column, Scope) -> + Scope. + +prepend_warning(Line, Column, Msg, #elixir_tokenizer{warnings=Warnings} = Scope) -> + Scope#elixir_tokenizer{warnings = [{{Line, Column}, Msg} | Warnings]}. + +track_ascii(true, Scope) -> Scope; +track_ascii(false, Scope) -> Scope#elixir_tokenizer{ascii_identifiers_only=false}. + +maybe_unicode_lint_warnings(_Ascii=false, Tokens, Warnings) -> + 'Elixir.String.Tokenizer.Security':unicode_lint_warnings(lists:reverse(Tokens)) ++ Warnings; +maybe_unicode_lint_warnings(_Ascii=true, _Tokens, Warnings) -> + Warnings. + +error(Reason, Rest, #elixir_tokenizer{warnings=Warnings}, Tokens) -> + {error, Reason, Rest, Warnings, Tokens}. + +%% Cursor handling + +cursor_complete(Line, Column, Terminators, Tokens) -> + {AccTokens, _} = + lists:foldl( + fun({Start, _, _}, {NewTokens, NewColumn}) -> + End = terminator(Start), + AccTokens = [{End, {Line, NewColumn, nil}} | NewTokens], + AccColumn = NewColumn + length(erlang:atom_to_list(End)), + {AccTokens, AccColumn} + end, + {Tokens, Column}, + Terminators + ), + lists:reverse(AccTokens). + +add_cursor(_Line, Column, terminators, Terminators, Tokens) -> + {Column, Terminators, Tokens}; +add_cursor(Line, Column, cursor_and_terminators, Terminators, Tokens) -> + {PrunedTokens, PrunedTerminators} = prune_tokens(Tokens, [], Terminators), + CursorTokens = [ + {')', {Line, Column + 11, nil}}, + {'(', {Line, Column + 10, nil}}, + {paren_identifier, {Line, Column, nil}, '__cursor__'} + | PrunedTokens + ], + {Column + 12, PrunedTerminators, CursorTokens}. + +%%% Any terminator needs to be closed +prune_tokens([{'end', _} | Tokens], Opener, Terminators) -> + prune_tokens(Tokens, ['end' | Opener], Terminators); +prune_tokens([{')', _} | Tokens], Opener, Terminators) -> + prune_tokens(Tokens, [')' | Opener], Terminators); +prune_tokens([{']', _} | Tokens], Opener, Terminators) -> + prune_tokens(Tokens, [']' | Opener], Terminators); +prune_tokens([{'}', _} | Tokens], Opener, Terminators) -> + prune_tokens(Tokens, ['}' | Opener], Terminators); +prune_tokens([{'>>', _} | Tokens], Opener, Terminators) -> + prune_tokens(Tokens, ['>>' | Opener], Terminators); +%%% Close opened terminators +prune_tokens([{'fn', _} | Tokens], ['end' | Opener], Terminators) -> + prune_tokens(Tokens, Opener, Terminators); +prune_tokens([{'do', _} | Tokens], ['end' | Opener], Terminators) -> + prune_tokens(Tokens, Opener, Terminators); +prune_tokens([{'(', _} | Tokens], [')' | Opener], Terminators) -> + prune_tokens(Tokens, Opener, Terminators); +prune_tokens([{'[', _} | Tokens], [']' | Opener], Terminators) -> + prune_tokens(Tokens, Opener, Terminators); +prune_tokens([{'{', _} | Tokens], ['}' | Opener], Terminators) -> + prune_tokens(Tokens, Opener, Terminators); +prune_tokens([{'<<', _} | Tokens], ['>>' | Opener], Terminators) -> + prune_tokens(Tokens, Opener, Terminators); +%%% Handle anonymous functions +prune_tokens(Tokens, [], [{'fn', _, _} | Terminators]) -> + prune_tokens(drop_including(Tokens, 'fn'), [], Terminators); +prune_tokens([{'(', _}, {capture_op, _, _} | Tokens], [], [{'(', _, _} | Terminators]) -> + prune_tokens(Tokens, [], Terminators); +%%% or it is time to stop... +prune_tokens([{',', _} | _] = Tokens, [], Terminators) -> + {Tokens, Terminators}; +prune_tokens([{'do', _} | _] = Tokens, [], Terminators) -> + {Tokens, Terminators}; +prune_tokens([{'(', _} | _] = Tokens, [], Terminators) -> + {Tokens, Terminators}; +prune_tokens([{'[', _} | _] = Tokens, [], Terminators) -> + {Tokens, Terminators}; +prune_tokens([{'{', _} | _] = Tokens, [], Terminators) -> + {Tokens, Terminators}; +prune_tokens([{'<<', _} | _] = Tokens, [], Terminators) -> + {Tokens, Terminators}; +prune_tokens([{block_identifier, _, _} | _] = Tokens, [], Terminators) -> + {Tokens, Terminators}; +prune_tokens([{kw_identifier, _, _} | _] = Tokens, [], Terminators) -> + {Tokens, Terminators}; +prune_tokens([{kw_identifier_safe, _, _} | _] = Tokens, [], Terminators) -> + {Tokens, Terminators}; +prune_tokens([{kw_identifier_unsafe, _, _} | _] = Tokens, [], Terminators) -> + {Tokens, Terminators}; +%%% or we traverse until the end. +prune_tokens([_ | Tokens], Opener, Terminators) -> + prune_tokens(Tokens, Opener, Terminators); +prune_tokens([], [], Terminators) -> + {[], Terminators}. + +drop_including([{Token, _} | Tokens], Token) -> Tokens; +drop_including([_ | Tokens], Token) -> drop_including(Tokens, Token); +drop_including([], _Token) -> []. diff --git a/lib/elixir/src/elixir_tokenizer.hrl b/lib/elixir/src/elixir_tokenizer.hrl new file mode 100644 index 00000000000..6857db4b834 --- /dev/null +++ b/lib/elixir/src/elixir_tokenizer.hrl @@ -0,0 +1,31 @@ +%% Numbers +-define(is_hex(S), (?is_digit(S) orelse (S >= $A andalso S =< $F) orelse (S >= $a andalso S =< $f))). +-define(is_bin(S), (S >= $0 andalso S =< $1)). +-define(is_octal(S), (S >= $0 andalso S =< $7)). + +%% Digits and letters +-define(is_digit(S), (S >= $0 andalso S =< $9)). +-define(is_upcase(S), (S >= $A andalso S =< $Z)). +-define(is_downcase(S), (S >= $a andalso S =< $z)). + +%% Others +-define(is_quote(S), (S =:= $" orelse S =:= $')). +-define(is_sigil(S), (S =:= $/ orelse S =:= $< orelse S =:= $" orelse S =:= $' orelse + S =:= $[ orelse S =:= $( orelse S =:= ${ orelse S =:= $|)). + +%% Spaces +-define(is_horizontal_space(S), (S =:= $\s orelse S =:= $\t)). +-define(is_vertical_space(S), (S =:= $\r orelse S =:= $\n)). +-define(is_space(S), (?is_horizontal_space(S) orelse ?is_vertical_space(S))). + +%% Bidirectional control +%% Retrieved from https://trojansource.codes/trojan-source.pdf +-define(bidi(C), C =:= 16#202A; + C =:= 16#202B; + C =:= 16#202D; + C =:= 16#202E; + C =:= 16#2066; + C =:= 16#2067; + C =:= 16#2068; + C =:= 16#202C; + C =:= 16#2069). diff --git a/lib/elixir/src/elixir_translator.erl b/lib/elixir/src/elixir_translator.erl deleted file mode 100644 index bc0b1389ee9..00000000000 --- a/lib/elixir/src/elixir_translator.erl +++ /dev/null @@ -1,425 +0,0 @@ -%% Translate Elixir quoted expressions to Erlang Abstract Format. -%% Expects the tree to be expanded. --module(elixir_translator). --export([translate/2, translate_arg/3, translate_args/2, translate_block/3]). --import(elixir_scope, [mergev/2, mergec/2]). --import(elixir_errors, [compile_error/3, compile_error/4]). --include("elixir.hrl"). - -%% = - -translate({'=', Meta, [Left, Right]}, S) -> - Return = case Left of - {'_', _, Atom} when is_atom(Atom) -> false; - _ -> true - end, - - {TRight, SR} = translate_block(Right, Return, S), - {TLeft, SL} = elixir_clauses:match(fun translate/2, Left, SR), - {{match, ?line(Meta), TLeft, TRight}, SL}; - -%% Containers - -translate({'{}', Meta, Args}, S) when is_list(Args) -> - {TArgs, SE} = translate_args(Args, S), - {{tuple, ?line(Meta), TArgs}, SE}; - -translate({'%{}', Meta, Args}, S) when is_list(Args) -> - elixir_map:translate_map(Meta, Args, S); - -translate({'%', Meta, [Left, Right]}, S) -> - elixir_map:translate_struct(Meta, Left, Right, S); - -translate({'<<>>', Meta, Args}, S) when is_list(Args) -> - elixir_bitstring:translate(Meta, Args, S); - -%% Blocks - -translate({'__block__', Meta, Args}, #elixir_scope{return=Return} = S) when is_list(Args) -> - {TArgs, SA} = translate_block(Args, [], Return, S#elixir_scope{return=true}), - {{block, ?line(Meta), TArgs}, SA}; - -%% Erlang op - -translate({'__op__', Meta, [Op, Expr]}, S) when is_atom(Op) -> - {TExpr, NS} = translate(Expr, S), - {{op, ?line(Meta), Op, TExpr}, NS}; - -translate({'__op__', Meta, [Op, Left, Right]}, S) when is_atom(Op) -> - {[TLeft, TRight], NS} = translate_args([Left, Right], S), - {{op, ?line(Meta), Op, TLeft, TRight}, NS}; - -%% Lexical - -translate({Lexical, _, [_, _]}, S) when Lexical == import; Lexical == alias; Lexical == require -> - {{atom, 0, nil}, S}; - -%% Pseudo variables - -translate({'__CALLER__', Meta, Atom}, S) when is_atom(Atom) -> - {{var, ?line(Meta), '__CALLER__'}, S#elixir_scope{caller=true}}; - -%% Functions - -translate({'&', Meta, [{'/', [], [{Fun, [], Atom}, Arity]}]}, S) - when is_atom(Fun), is_atom(Atom), is_integer(Arity) -> - {{'fun', ?line(Meta), {function, Fun, Arity}}, S}; -translate({'&', Meta, [Arg]}, S) when is_integer(Arg) -> - compile_error(Meta, S#elixir_scope.file, "unhandled &~B outside of a capture", [Arg]); - -translate({fn, Meta, Clauses}, S) -> - elixir_fn:translate(Meta, Clauses, S); - -%% Cond - -translate({'cond', _Meta, [[{do, Pairs}]]}, S) -> - [{'->', Meta, [[Condition], Body]}|T] = lists:reverse(Pairs), - Case = - case Condition of - {'_', _, Atom} when is_atom(Atom) -> - compile_error(Meta, S#elixir_scope.file, "unbound variable _ inside cond. " - "If you want the last clause to always match, you probably meant to use: true ->"); - X when is_atom(X) and (X /= false) and (X /= nil) -> - build_cond_clauses(T, Body, Meta); - _ -> - {Truthy, Other} = build_truthy_clause(Meta, Condition, Body), - Error = {{'.', Meta, [erlang, error]}, [], [cond_clause]}, - Falsy = {'->', Meta, [[Other], Error]}, - Acc = {'case', Meta, [Condition, [{do, [Truthy, Falsy]}]]}, - build_cond_clauses(T, Acc, Meta) - end, - translate(Case, S); - -%% Case - -translate({'case', Meta, [Expr, KV]}, #elixir_scope{return=Return} = RS) -> - S = RS#elixir_scope{return=true}, - Clauses = elixir_clauses:get_pairs(do, KV, match), - {TExpr, NS} = translate(Expr, S), - {TClauses, TS} = elixir_clauses:clauses(Meta, Clauses, Return, NS), - {{'case', ?line(Meta), TExpr, TClauses}, TS}; - -%% Try - -translate({'try', Meta, [Clauses]}, #elixir_scope{return=Return} = RS) -> - S = RS#elixir_scope{noname=true, return=true}, - Do = proplists:get_value('do', Clauses, nil), - {TDo, SB} = elixir_translator:translate(Do, S), - - Catch = [Tuple || {X, _} = Tuple <- Clauses, X == 'rescue' orelse X == 'catch'], - {TCatch, SC} = elixir_try:clauses(Meta, Catch, Return, mergec(S, SB)), - - case lists:keyfind('after', 1, Clauses) of - {'after', After} -> - {TBlock, SA} = translate(After, mergec(S, SC)), - TAfter = unblock(TBlock); - false -> - {TAfter, SA} = {[], mergec(S, SC)} - end, - - Else = elixir_clauses:get_pairs(else, Clauses, match), - {TElse, SE} = elixir_clauses:clauses(Meta, Else, Return, mergec(S, SA)), - - SF = (mergec(S, SE))#elixir_scope{noname=RS#elixir_scope.noname}, - {{'try', ?line(Meta), unblock(TDo), TElse, TCatch, TAfter}, SF}; - -%% Receive - -translate({'receive', Meta, [KV]}, #elixir_scope{return=Return} = RS) -> - S = RS#elixir_scope{return=true}, - Do = elixir_clauses:get_pairs(do, KV, match, true), - - case lists:keyfind('after', 1, KV) of - false -> - {TClauses, SC} = elixir_clauses:clauses(Meta, Do, Return, S), - {{'receive', ?line(Meta), TClauses}, SC}; - _ -> - After = elixir_clauses:get_pairs('after', KV, expr), - {TClauses, SC} = elixir_clauses:clauses(Meta, Do ++ After, Return, S), - {FClauses, TAfter} = elixir_utils:split_last(TClauses), - {_, _, [FExpr], _, FAfter} = TAfter, - {{'receive', ?line(Meta), FClauses, FExpr, FAfter}, SC} - end; - -%% Comprehensions - -translate({for, Meta, [_|_] = Args}, S) -> - elixir_for:translate(Meta, Args, S); - -%% Super - -translate({super, Meta, Args}, S) when is_list(Args) -> - Module = assert_module_scope(Meta, super, S), - Function = assert_function_scope(Meta, super, S), - elixir_def_overridable:ensure_defined(Meta, Module, Function, S), - - {_, Arity} = Function, - - {TArgs, TS} = if - length(Args) == Arity -> - translate_args(Args, S); - true -> - compile_error(Meta, S#elixir_scope.file, "super must be called with the same number of " - "arguments as the current function") - end, - - Super = elixir_def_overridable:name(Module, Function), - {{call, ?line(Meta), {atom, ?line(Meta), Super}, TArgs}, TS#elixir_scope{super=true}}; - -%% Variables - -translate({'^', Meta, [{Name, VarMeta, Kind}]}, #elixir_scope{context=match} = S) when is_atom(Name), is_atom(Kind) -> - Tuple = {Name, var_kind(VarMeta, Kind)}, - case orddict:find(Tuple, S#elixir_scope.backup_vars) of - {ok, {Value, _Counter}} -> - {{var, ?line(Meta), Value}, S}; - error -> - compile_error(Meta, S#elixir_scope.file, "unbound variable ^~ts", [Name]) - end; - -translate({'_', Meta, Kind}, #elixir_scope{context=match} = S) when is_atom(Kind) -> - {{var, ?line(Meta), '_'}, S}; - -translate({'_', Meta, Kind}, S) when is_atom(Kind) -> - compile_error(Meta, S#elixir_scope.file, "unbound variable _"); - -translate({Name, Meta, Kind}, #elixir_scope{extra=map_key} = S) when is_atom(Name), is_atom(Kind) -> - compile_error(Meta, S#elixir_scope.file, "illegal use of variable ~ts in map key", [Name]); - -translate({Name, Meta, Kind}, S) when is_atom(Name), is_atom(Kind) -> - elixir_scope:translate_var(Meta, Name, var_kind(Meta, Kind), S); - -%% Local calls - -translate({Name, Meta, Args}, S) when is_atom(Name), is_list(Meta), is_list(Args) -> - if - S#elixir_scope.context == match -> - compile_error(Meta, S#elixir_scope.file, - "cannot invoke function ~ts/~B inside match", [Name, length(Args)]); - S#elixir_scope.context == guard -> - Arity = length(Args), - File = S#elixir_scope.file, - case Arity of - 0 -> compile_error(Meta, File, "unknown variable ~ts or cannot invoke " - "function ~ts/~B inside guard", [Name, Name, Arity]); - _ -> compile_error(Meta, File, "cannot invoke local ~ts/~B inside guard", - [Name, Arity]) - end; - S#elixir_scope.function == nil -> - compile_error(Meta, S#elixir_scope.file, "undefined function ~ts/~B", [Name, length(Args)]); - true -> - Line = ?line(Meta), - {TArgs, NS} = translate_args(Args, S), - {{call, Line, {atom, Line, Name}, TArgs}, NS} - end; - -%% Remote calls - -translate({{'.', _, [Left, Right]}, Meta, Args}, S) - when (is_tuple(Left) orelse is_atom(Left)), is_atom(Right), is_list(Meta), is_list(Args) -> - {TLeft, SL} = translate(Left, S), - {TArgs, SA} = translate_args(Args, mergec(S, SL)), - - Line = ?line(Meta), - Arity = length(Args), - TRight = {atom, Line, Right}, - - %% We need to rewrite erlang function calls as operators - %% because erl_eval chokes on them. We can remove this - %% once a fix is merged into Erlang, keeping only the - %% list operators one (since it is required for inlining - %% [1,2,3] ++ Right in matches). - case (Left == erlang) andalso erl_op(Right, Arity) of - true -> - {list_to_tuple([op, Line, Right] ++ TArgs), mergev(SL, SA)}; - false -> - assert_allowed_in_context(Meta, Left, Right, Arity, S), - SC = mergev(SL, SA), - - case not is_atom(Left) andalso (Arity == 0) of - true -> - {Var, _, SV} = elixir_scope:build_var('_', SC), - TVar = {var, Line, Var}, - TMap = {map, Line, [ - {map_field_assoc, Line, - {atom, Line, '__struct__'}, - {atom, Line, 'Elixir.KeyError'}}, - {map_field_assoc, Line, - {atom, Line, '__exception__'}, - {atom, Line, 'true'}}, - {map_field_assoc, Line, - {atom, Line, key}, - {atom, Line, TRight}}, - {map_field_assoc, Line, - {atom, Line, term}, - TVar}]}, - - %% TODO There is a bug in dialyzer that makes it fail on - %% empty maps. We work around the bug below by using - %% the is_map/1 guard instead of matching on map. Hopefully - %% we can use a match on 17.1. - %% - %% http://erlang.org/pipermail/erlang-bugs/2014-April/004338.html - {{'case', -1, TLeft, [ - {clause, -1, - [{map, Line, [{map_field_exact, Line, TRight, TVar}]}], - [], - [TVar]}, - {clause, -1, - [TVar], - [[elixir_utils:erl_call(Line, erlang, is_map, [TVar])]], - [elixir_utils:erl_call(Line, erlang, error, [TMap])]}, - {clause, -1, - [TVar], - [], - [{call, Line, {remote, Line, TVar, TRight}, []}]} - ]}, SV}; - false -> - {{call, Line, {remote, Line, TLeft, TRight}, TArgs}, SC} - end - end; - -%% Anonymous function calls - -translate({{'.', _, [Expr]}, Meta, Args}, S) when is_list(Args) -> - {TExpr, SE} = translate(Expr, S), - {TArgs, SA} = translate_args(Args, mergec(S, SE)), - {{call, ?line(Meta), TExpr, TArgs}, mergev(SE, SA)}; - -%% Literals - -translate(List, S) when is_list(List) -> - Fun = case S#elixir_scope.context of - match -> fun translate/2; - _ -> fun(X, Acc) -> translate_arg(X, Acc, S) end - end, - translate_list(List, Fun, S, []); - -translate({Left, Right}, S) -> - {TArgs, SE} = translate_args([Left, Right], S), - {{tuple, 0, TArgs}, SE}; - -translate(Other, S) -> - {elixir_utils:elixir_to_erl(Other), S}. - -%% Helpers - -erl_op(Op, Arity) -> - erl_internal:list_op(Op, Arity) orelse - erl_internal:comp_op(Op, Arity) orelse - erl_internal:bool_op(Op, Arity) orelse - erl_internal:arith_op(Op, Arity). - -translate_list([{'|', _, [_, _]=Args}], Fun, Acc, List) -> - {[TLeft,TRight], TAcc} = lists:mapfoldl(Fun, Acc, Args), - {build_list([TLeft|List], TRight), TAcc}; -translate_list([H|T], Fun, Acc, List) -> - {TH, TAcc} = Fun(H, Acc), - translate_list(T, Fun, TAcc, [TH|List]); -translate_list([], _Fun, Acc, List) -> - {build_list(List, {nil, 0}), Acc}. - -build_list([H|T], Acc) -> - build_list(T, {cons, 0, H, Acc}); -build_list([], Acc) -> - Acc. - -var_kind(Meta, Kind) -> - case lists:keyfind(counter, 1, Meta) of - {counter, Counter} -> Counter; - false -> Kind - end. - -%% Pack a list of expressions from a block. -unblock({'block', _, Exprs}) -> Exprs; -unblock(Expr) -> [Expr]. - -%% Translate args - -translate_arg(Arg, Acc, S) when is_number(Arg); is_atom(Arg); is_binary(Arg); is_pid(Arg); is_function(Arg) -> - {TArg, _} = translate(Arg, S), - {TArg, Acc}; -translate_arg(Arg, Acc, S) -> - {TArg, TAcc} = translate(Arg, mergec(S, Acc)), - {TArg, mergev(Acc, TAcc)}. - -translate_args(Args, #elixir_scope{context=match} = S) -> - lists:mapfoldl(fun translate/2, S, Args); - -translate_args(Args, S) -> - lists:mapfoldl(fun(X, Acc) -> translate_arg(X, Acc, S) end, S, Args). - -%% Translate blocks - -translate_block([], Acc, _Return, S) -> - {lists:reverse(Acc), S}; -translate_block([H], Acc, Return, S) -> - {TH, TS} = translate_block(H, Return, S), - translate_block([], [TH|Acc], Return, TS); -translate_block([H|T], Acc, Return, S) -> - {TH, TS} = translate_block(H, false, S), - translate_block(T, [TH|Acc], Return, TS). - -translate_block(Expr, Return, S) -> - case (Return == false) andalso handles_no_return(Expr) of - true -> translate(Expr, S#elixir_scope{return=Return}); - false -> translate(Expr, S) - end. - -%% return is typically true, except when we find one -%% of the expressions below, which may handle return=false -%% but must always return return=true. -handles_no_return({'try', _, [_]}) -> true; -handles_no_return({'cond', _, [_]}) -> true; -handles_no_return({'for', _, [_|_]}) -> true; -handles_no_return({'case', _, [_, _]}) -> true; -handles_no_return({'receive', _, [_]}) -> true; -handles_no_return({'__block__', _, [_|_]}) -> true; -handles_no_return(_) -> false. - -%% Cond - -build_cond_clauses([{'->', NewMeta, [[Condition], Body]}|T], Acc, OldMeta) -> - {Truthy, Other} = build_truthy_clause(NewMeta, Condition, Body), - Falsy = {'->', OldMeta, [[Other], Acc]}, - Case = {'case', NewMeta, [Condition, [{do, [Truthy, Falsy]}]]}, - build_cond_clauses(T, Case, NewMeta); -build_cond_clauses([], Acc, _) -> - Acc. - -build_truthy_clause(Meta, Condition, Body) -> - case elixir_utils:returns_boolean(Condition) of - true -> - {{'->', Meta, [[true], Body]}, false}; - false -> - Var = {'cond', [], 'Elixir'}, - Head = {'when', [], [Var, - {'__op__', [], [ - 'andalso', - {{'.', [], [erlang, '/=']}, [], [Var, nil]}, - {{'.', [], [erlang, '/=']}, [], [Var, false]} - ]} - ]}, - {{'->', Meta, [[Head], Body]}, {'_', [], nil}} - end. - -%% Assertions - -assert_module_scope(Meta, Kind, #elixir_scope{module=nil,file=File}) -> - compile_error(Meta, File, "cannot invoke ~ts outside module", [Kind]); -assert_module_scope(_Meta, _Kind, #elixir_scope{module=Module}) -> Module. - -assert_function_scope(Meta, Kind, #elixir_scope{function=nil,file=File}) -> - compile_error(Meta, File, "cannot invoke ~ts outside function", [Kind]); -assert_function_scope(_Meta, _Kind, #elixir_scope{function=Function}) -> Function. - -assert_allowed_in_context(Meta, Left, Right, Arity, #elixir_scope{context=Context} = S) - when (Context == match) orelse (Context == guard) -> - case (Left == erlang) andalso erl_internal:guard_bif(Right, Arity) of - true -> ok; - false -> - compile_error(Meta, S#elixir_scope.file, "cannot invoke remote function ~ts.~ts/~B inside ~ts", - ['Elixir.Macro':to_string(Left), Right, Arity, Context]) - end; -assert_allowed_in_context(_, _, _, _, _) -> - ok. diff --git a/lib/elixir/src/elixir_try.erl b/lib/elixir/src/elixir_try.erl deleted file mode 100644 index 73729a70ec9..00000000000 --- a/lib/elixir/src/elixir_try.erl +++ /dev/null @@ -1,195 +0,0 @@ --module(elixir_try). --export([clauses/4]). --include("elixir.hrl"). - -clauses(_Meta, Clauses, Return, S) -> - Catch = elixir_clauses:get_pairs('catch', Clauses, 'catch'), - Rescue = elixir_clauses:get_pairs(rescue, Clauses, rescue), - reduce_clauses(Rescue ++ Catch, [], S, Return, S). - -reduce_clauses([H|T], Acc, SAcc, Return, S) -> - {TH, TS} = each_clause(H, Return, SAcc), - reduce_clauses(T, TH ++ Acc, elixir_scope:mergec(S, TS), Return, S); -reduce_clauses([], Acc, SAcc, _Return, _S) -> - {lists:reverse(Acc), SAcc}. - -each_clause({'catch', Meta, Raw, Expr}, Return, S) -> - {Args, Guards} = elixir_clauses:extract_splat_guards(Raw), - - Final = case Args of - [X] -> [throw, X, {'_', Meta, nil}]; - [X,Y] -> [X, Y, {'_', Meta, nil}] - end, - - Condition = [{'{}', Meta, Final}], - {TC, TS} = elixir_clauses:clause(?line(Meta), fun elixir_translator:translate_args/2, - Condition, Expr, Guards, Return, S), - {[TC], TS}; - -each_clause({rescue, Meta, [{in, _, [Left, Right]}], Expr}, Return, S) -> - {VarName, _, CS} = elixir_scope:build_var('_', S), - Var = {VarName, Meta, nil}, - {Parts, Safe, FS} = rescue_guards(Meta, Var, Right, CS), - - Body = - case Left of - {'_', _, Atom} when is_atom(Atom) -> - Expr; - _ -> - Normalized = - case Safe of - true -> Var; - false -> {{'.', Meta, ['Elixir.Exception', normalize]}, Meta, [error, Var]} - end, - prepend_to_block(Meta, {'=', Meta, [Left, Normalized]}, Expr) - end, - - build_rescue(Meta, Parts, Body, Return, FS); - -each_clause({rescue, Meta, _, _}, _Return, S) -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, "invalid arguments for rescue in try"); - -each_clause({Key, Meta, _, _}, _Return, S) -> - elixir_errors:compile_error(Meta, S#elixir_scope.file, "invalid key ~ts in try", [Key]). - -%% Helpers - -build_rescue(Meta, Parts, Body, Return, S) -> - Matches = [Match || {Match, _} <- Parts], - - {{clause, Line, TMatches, _, TBody}, TS} = - elixir_clauses:clause(?line(Meta), fun elixir_translator:translate_args/2, - Matches, Body, [], Return, S), - - TClauses = - [begin - TArgs = [{tuple, Line, [{atom, Line, error}, TMatch, {var, Line, '_'}]}], - TGuards = elixir_clauses:guards(Line, Guards, [], TS), - {clause, Line, TArgs, TGuards, TBody} - end || {TMatch, {_, Guards}} <- lists:zip(TMatches, Parts)], - - {TClauses, TS}. - -%% Convert rescue clauses into guards. -rescue_guards(_, Var, {'_', _, _}, S) -> {[{Var, []}], false, S}; - -rescue_guards(Meta, Var, Aliases, S) -> - {Elixir, Erlang} = rescue_each_ref(Meta, Var, Aliases, [], [], S), - - {ElixirParts, ES} = - case Elixir of - [] -> {[], S}; - _ -> - {VarName, _, CS} = elixir_scope:build_var('_', S), - StructVar = {VarName, Meta, nil}, - Map = {'%{}', Meta, [{'__struct__', StructVar}, {'__exception__', true}]}, - Match = {'=', Meta, [Map, Var]}, - Guards = [{erl(Meta, '=='), Meta, [StructVar, Mod]} || Mod <- Elixir], - {[{Match, Guards}], CS} - end, - - ErlangParts = - case Erlang of - [] -> []; - _ -> [{Var, Erlang}] - end, - - {ElixirParts ++ ErlangParts, ErlangParts == [], ES}. - -%% Rescue each atom name considering their Erlang or Elixir matches. -%% Matching of variables is done with Erlang exceptions is done in -%% function for optimization. - -rescue_each_ref(Meta, Var, [H|T], Elixir, Erlang, S) when is_atom(H) -> - case erl_rescue_guard_for(Meta, Var, H) of - false -> rescue_each_ref(Meta, Var, T, [H|Elixir], Erlang, S); - Expr -> rescue_each_ref(Meta, Var, T, [H|Elixir], [Expr|Erlang], S) - end; - -rescue_each_ref(_, _, [], Elixir, Erlang, _) -> - {Elixir, Erlang}. - -%% Handle erlang rescue matches. - -erl_rescue_guard_for(Meta, Var, 'Elixir.UndefinedFunctionError') -> - {erl(Meta, '=='), Meta, [Var, undef]}; - -erl_rescue_guard_for(Meta, Var, 'Elixir.FunctionClauseError') -> - {erl(Meta, '=='), Meta, [Var, function_clause]}; - -erl_rescue_guard_for(Meta, Var, 'Elixir.SystemLimitError') -> - {erl(Meta, '=='), Meta, [Var, system_limit]}; - -erl_rescue_guard_for(Meta, Var, 'Elixir.ArithmeticError') -> - {erl(Meta, '=='), Meta, [Var, badarith]}; - -erl_rescue_guard_for(Meta, Var, 'Elixir.CondClauseError') -> - {erl(Meta, '=='), Meta, [Var, cond_clause]}; - -erl_rescue_guard_for(Meta, Var, 'Elixir.BadArityError') -> - erl_and(Meta, - erl_tuple_size(Meta, Var, 2), - erl_record_compare(Meta, Var, badarity)); - -erl_rescue_guard_for(Meta, Var, 'Elixir.BadFunctionError') -> - erl_and(Meta, - erl_tuple_size(Meta, Var, 2), - erl_record_compare(Meta, Var, badfun)); - -erl_rescue_guard_for(Meta, Var, 'Elixir.MatchError') -> - erl_and(Meta, - erl_tuple_size(Meta, Var, 2), - erl_record_compare(Meta, Var, badmatch)); - -erl_rescue_guard_for(Meta, Var, 'Elixir.CaseClauseError') -> - erl_and(Meta, - erl_tuple_size(Meta, Var, 2), - erl_record_compare(Meta, Var, case_clause)); - -erl_rescue_guard_for(Meta, Var, 'Elixir.TryClauseError') -> - erl_and(Meta, - erl_tuple_size(Meta, Var, 2), - erl_record_compare(Meta, Var, try_clause)); - -erl_rescue_guard_for(Meta, Var, 'Elixir.BadStructError') -> - erl_and(Meta, - erl_tuple_size(Meta, Var, 3), - erl_record_compare(Meta, Var, badstruct)); - -erl_rescue_guard_for(Meta, Var, 'Elixir.ArgumentError') -> - erl_or(Meta, - {erl(Meta, '=='), Meta, [Var, badarg]}, - erl_and(Meta, - erl_tuple_size(Meta, Var, 2), - erl_record_compare(Meta, Var, badarg))); - -erl_rescue_guard_for(Meta, Var, 'Elixir.ErlangError') -> - IsNotTuple = {erl(Meta, 'not'), Meta, [{erl(Meta, is_tuple), Meta, [Var]}]}, - IsException = {erl(Meta, '/='), Meta, [ - {erl(Meta, element), Meta, [2, Var]}, '__exception__' - ]}, - erl_or(Meta, IsNotTuple, IsException); - -erl_rescue_guard_for(_, _, _) -> - false. - -%% Helpers - -erl_tuple_size(Meta, Var, Size) -> - {erl(Meta, '=='), Meta, [{erl(Meta, tuple_size), Meta, [Var]}, Size]}. - -erl_record_compare(Meta, Var, Expr) -> - {erl(Meta, '=='), Meta, [ - {erl(Meta, element), Meta, [1, Var]}, - Expr - ]}. - -prepend_to_block(_Meta, Expr, {'__block__', Meta, Args}) -> - {'__block__', Meta, [Expr|Args]}; - -prepend_to_block(Meta, Expr, Args) -> - {'__block__', Meta, [Expr, Args]}. - -erl(Meta, Op) -> {'.', Meta, [erlang, Op]}. -erl_or(Meta, Left, Right) -> {'__op__', Meta, ['orelse', Left, Right]}. -erl_and(Meta, Left, Right) -> {'__op__', Meta, ['andalso', Left, Right]}. diff --git a/lib/elixir/src/elixir_utils.erl b/lib/elixir/src/elixir_utils.erl index 1268bd6b185..e05017be3e9 100644 --- a/lib/elixir/src/elixir_utils.erl +++ b/lib/elixir/src/elixir_utils.erl @@ -1,38 +1,83 @@ %% Convenience functions used throughout elixir source code %% for ast manipulation and querying. -module(elixir_utils). --export([elixir_to_erl/1, get_line/1, split_last/1, - characters_to_list/1, characters_to_binary/1, macro_name/1, - convert_to_boolean/4, returns_boolean/1, atom_concat/1, - read_file_type/1, read_link_type/1, relative_to_cwd/1, erl_call/4]). +-export([get_line/1, split_last/1, noop/0, var_context/2, + characters_to_list/1, characters_to_binary/1, relative_to_cwd/1, + macro_name/1, returns_boolean/1, caller/4, meta_keep/1, + read_file_type/1, read_file_type/2, read_link_type/1, read_posix_mtime_and_size/1, + change_posix_time/2, change_universal_time/2, + guard_op/2, extract_splat_guards/1, extract_guards/1, + erlang_comparison_op_to_elixir/1, erl_fa_to_elixir_fa/2]). -include("elixir.hrl"). -include_lib("kernel/include/file.hrl"). macro_name(Macro) -> - list_to_atom(lists:concat(['MACRO-',Macro])). + list_to_atom("MACRO-" ++ atom_to_list(Macro)). -atom_concat(Atoms) -> - list_to_atom(lists:concat(Atoms)). +erl_fa_to_elixir_fa(Name, Arity) -> + case atom_to_list(Name) of + "MACRO-" ++ Rest -> {list_to_atom(Rest), Arity - 1}; + _ -> {Name, Arity} + end. -erl_call(Line, Module, Function, Args) -> - {call, Line, - {remote, Line, {atom, Line, Module}, {atom, Line, Function}}, - Args - }. +guard_op('andalso', 2) -> + true; +guard_op('orelse', 2) -> + true; +guard_op(Op, Arity) -> + try erl_internal:op_type(Op, Arity) of + arith -> true; + comp -> true; + bool -> true; + list -> false; + send -> false + catch + _:_ -> false + end. -get_line(Opts) when is_list(Opts) -> - case lists:keyfind(line, 1, Opts) of - {line, Line} when is_integer(Line) -> Line; - false -> 0 +erlang_comparison_op_to_elixir('/=') -> '!='; +erlang_comparison_op_to_elixir('=<') -> '<='; +erlang_comparison_op_to_elixir('=:=') -> '==='; +erlang_comparison_op_to_elixir('=/=') -> '!=='; +erlang_comparison_op_to_elixir(Other) -> Other. + +var_context(Meta, Kind) -> + case lists:keyfind(counter, 1, Meta) of + {counter, Counter} -> Counter; + false -> Kind end. -split_last([]) -> {[], []}; -split_last(List) -> split_last(List, []). -split_last([H], Acc) -> {lists:reverse(Acc), H}; -split_last([H|T], Acc) -> split_last(T, [H|Acc]). +% Extract guards + +extract_guards({'when', _, [Left, Right]}) -> {Left, extract_or_guards(Right)}; +extract_guards(Else) -> {Else, []}. + +extract_or_guards({'when', _, [Left, Right]}) -> [Left | extract_or_guards(Right)]; +extract_or_guards(Term) -> [Term]. + +% Extract guards when multiple left side args are allowed. + +extract_splat_guards([{'when', _, [_ | _] = Args}]) -> + {Left, Right} = split_last(Args), + {Left, extract_or_guards(Right)}; +extract_splat_guards(Else) -> + {Else, []}. + +%% No-op function that can be used for stuff like preventing tail-call +%% optimization to kick in. +noop() -> + ok. + +split_last([]) -> {[], []}; +split_last(List) -> split_last(List, []). +split_last([H], Acc) -> {lists:reverse(Acc), H}; +split_last([H | T], Acc) -> split_last(T, [H | Acc]). read_file_type(File) -> - case file:read_file_info(File) of + read_file_type(File, []). + +read_file_type(File, Opts) -> + case file:read_file_info(File, [{time, posix} | Opts]) of {ok, #file_info{type=Type}} -> {ok, Type}; {error, _} = Error -> Error end. @@ -43,89 +88,81 @@ read_link_type(File) -> {error, _} = Error -> Error end. +read_posix_mtime_and_size(File) -> + case file:read_file_info(File, [raw, {time, posix}]) of + {ok, #file_info{mtime=Mtime, size=Size}} -> {ok, Mtime, Size}; + {error, _} = Error -> Error + end. + +change_posix_time(Name, Time) when is_integer(Time) -> + file:write_file_info(Name, #file_info{mtime=Time}, [raw, {time, posix}]). + +change_universal_time(Name, {{Y, M, D}, {H, Min, Sec}}=Time) + when is_integer(Y), is_integer(M), is_integer(D), + is_integer(H), is_integer(Min), is_integer(Sec) -> + file:write_file_info(Name, #file_info{mtime=Time}, [{time, universal}]). + relative_to_cwd(Path) -> - case elixir_compiler:get_opt(internal) of - true -> Path; - false -> 'Elixir.String':to_char_list('Elixir.Path':relative_to_cwd(Path)) + try elixir_config:get(relative_paths) of + true -> 'Elixir.Path':relative_to_cwd(Path); + false -> Path + catch + _:_ -> Path end. characters_to_list(Data) when is_list(Data) -> Data; characters_to_list(Data) -> - case elixir_compiler:get_opt(internal) of - true -> unicode:characters_to_list(Data); - false -> 'Elixir.String':to_char_list(Data) + case unicode:characters_to_list(Data) of + Result when is_list(Result) -> Result; + {error, Encoded, Rest} -> conversion_error(invalid, Encoded, Rest); + {incomplete, Encoded, Rest} -> conversion_error(incomplete, Encoded, Rest) end. characters_to_binary(Data) when is_binary(Data) -> Data; characters_to_binary(Data) -> - case elixir_compiler:get_opt(internal) of - true -> unicode:characters_to_binary(Data); - false -> 'Elixir.List':to_string(Data) + case unicode:characters_to_binary(Data) of + Result when is_binary(Result) -> Result; + {error, Encoded, Rest} -> conversion_error(invalid, Encoded, Rest); + {incomplete, Encoded, Rest} -> conversion_error(incomplete, Encoded, Rest) end. -%% elixir to erl. Handles only valid quoted expressions, -%% that's why things like maps and references are not in the list. +conversion_error(Kind, Encoded, Rest) -> + error('Elixir.UnicodeConversionError':exception([{encoded, Encoded}, {rest, Rest}, {kind, Kind}])). -elixir_to_erl(Tree) when is_tuple(Tree) -> - {tuple, 0, [elixir_to_erl(X) || X <- tuple_to_list(Tree)]}; +%% Returns the caller as a stacktrace entry. +caller(Line, File, nil, _) -> + {elixir_compiler_0, '__FILE__', 1, stack_location(Line, File)}; +caller(Line, File, Module, nil) -> + {Module, '__MODULE__', 0, stack_location(Line, File)}; +caller(Line, File, Module, {Name, Arity}) -> + {Module, Name, Arity, stack_location(Line, File)}. -elixir_to_erl([]) -> - {nil, 0}; +stack_location(Line, File) -> + [{file, elixir_utils:characters_to_list(elixir_utils:relative_to_cwd(File))}, + {line, Line}]. -elixir_to_erl(<<>>) -> - {bin, 0, []}; - -elixir_to_erl(Tree) when is_list(Tree) -> - elixir_to_erl_cons_1(Tree, []); - -elixir_to_erl(Tree) when is_atom(Tree) -> - {atom, 0, Tree}; - -elixir_to_erl(Tree) when is_integer(Tree) -> - {integer, 0, Tree}; - -elixir_to_erl(Tree) when is_float(Tree) -> - {float, 0, Tree}; - -elixir_to_erl(Tree) when is_binary(Tree) -> - %% Note that our binaries are utf-8 encoded and we are converting - %% to a list using binary_to_list. The reason for this is that Erlang - %% considers a string in a binary to be encoded in latin1, so the bytes - %% are not changed in any fashion. - {bin, 0, [{bin_element, 0, {string, 0, binary_to_list(Tree)}, default, default}]}; - -elixir_to_erl(Function) when is_function(Function) -> - case (erlang:fun_info(Function, type) == {type, external}) andalso - (erlang:fun_info(Function, env) == {env, []}) of - true -> - {module, Module} = erlang:fun_info(Function, module), - {name, Name} = erlang:fun_info(Function, name), - {arity, Arity} = erlang:fun_info(Function, arity), - - {'fun', 0, {function, - {atom, 0, Module}, - {atom, 0, Name}, - {integer, 0, Arity}}}; - false -> - error(badarg) - end; - -elixir_to_erl(Pid) when is_pid(Pid) -> - elixir_utils:erl_call(0, erlang, binary_to_term, - [elixir_utils:elixir_to_erl(term_to_binary(Pid))]); - -elixir_to_erl(_Other) -> - error(badarg). - -elixir_to_erl_cons_1([H|T], Acc) -> elixir_to_erl_cons_1(T, [H|Acc]); -elixir_to_erl_cons_1(Other, Acc) -> elixir_to_erl_cons_2(Acc, elixir_to_erl(Other)). +get_line(Opts) when is_list(Opts) -> + case lists:keyfind(line, 1, Opts) of + {line, Line} when is_integer(Line) -> Line; + _ -> 0 + end. -elixir_to_erl_cons_2([H|T], Acc) -> - elixir_to_erl_cons_2(T, {cons, 0, elixir_to_erl(H), Acc}); -elixir_to_erl_cons_2([], Acc) -> - Acc. +%% Meta location. +%% +%% Macros add a file pair on location keep which we +%% should take into account for error reporting. +%% +%% Returns {binary, integer} on location keep or nil. + +meta_keep(Meta) -> + case lists:keyfind(keep, 1, Meta) of + {keep, {File, Line} = Pair} when is_binary(File), is_integer(Line) -> + Pair; + _ -> + nil + end. %% Boolean checks @@ -138,61 +175,33 @@ returns_boolean({{'.', _, [erlang, Op]}, _, [_, _]}) when Op == '=='; Op == '/='; Op == '=<'; Op == '>='; Op == '<'; Op == '>'; Op == '=:='; Op == '=/=' -> true; -returns_boolean({'__op__', _, [Op, _, Right]}) when Op == 'andalso'; Op == 'orelse' -> +returns_boolean({{'.', _, [erlang, Op]}, _, [_, Right]}) when + Op == 'andalso'; Op == 'orelse' -> returns_boolean(Right); returns_boolean({{'.', _, [erlang, Fun]}, _, [_]}) when Fun == is_atom; Fun == is_binary; Fun == is_bitstring; Fun == is_boolean; Fun == is_float; Fun == is_function; Fun == is_integer; Fun == is_list; Fun == is_number; Fun == is_pid; Fun == is_port; Fun == is_reference; - Fun == is_tuple -> true; + Fun == is_tuple; Fun == is_map; Fun == is_process_alive -> true; returns_boolean({{'.', _, [erlang, Fun]}, _, [_, _]}) when - Fun == is_function -> true; + Fun == is_map_key; Fun == is_function; Fun == is_record -> true; returns_boolean({{'.', _, [erlang, Fun]}, _, [_, _, _]}) when - Fun == function_exported -> true; + Fun == function_exported; Fun == is_record -> true; returns_boolean({'case', _, [_, [{do, Clauses}]]}) -> lists:all(fun - ({'->',_,[_, Expr]}) -> returns_boolean(Expr) + ({'->', _, [_, Expr]}) -> returns_boolean(Expr) end, Clauses); returns_boolean({'cond', _, [[{do, Clauses}]]}) -> lists:all(fun - ({'->',_,[_, Expr]}) -> returns_boolean(Expr) + ({'->', _, [_, Expr]}) -> returns_boolean(Expr) end, Clauses); -returns_boolean({'__block__', [], Exprs}) -> +returns_boolean({'__block__', _, Exprs}) -> returns_boolean(lists:last(Exprs)); returns_boolean(_) -> false. - -convert_to_boolean(Line, Expr, Bool, S) when is_integer(Line) -> - case {returns_boolean(Expr), Bool} of - {true, true} -> {Expr, S}; - {true, false} -> {{op, Line, 'not', Expr}, S}; - _ -> do_convert_to_boolean(Line, Expr, Bool, S) - end. - -%% Notice we use a temporary var and bundle nil -%% and false checks in the same clause since -%% it makes dialyzer happy. -do_convert_to_boolean(Line, Expr, Bool, S) -> - {Name, _, TS} = elixir_scope:build_var('_', S), - Var = {var, Line, Name}, - Any = {var, Line, '_'}, - OrElse = do_guarded_convert_to_boolean(Line, Var, 'orelse', '=='), - - FalseResult = {atom,Line,not Bool}, - TrueResult = {atom,Line,Bool}, - - {{'case', Line, Expr, [ - {clause, Line, [Var], [[OrElse]], [FalseResult]}, - {clause, Line, [Any], [], [TrueResult]} - ]}, TS}. - -do_guarded_convert_to_boolean(Line, Expr, Op, Comp) -> - Left = {op, Line, Comp, Expr, {atom, Line, false}}, - Right = {op, Line, Comp, Expr, {atom, Line, nil}}, - {op, Line, Op, Left, Right}. diff --git a/lib/elixir/test/doc_test.exs b/lib/elixir/test/doc_test.exs deleted file mode 100644 index 3713782a0b1..00000000000 --- a/lib/elixir/test/doc_test.exs +++ /dev/null @@ -1,42 +0,0 @@ -ExUnit.start [] - -defmodule KernelTest do - use ExUnit.Case, async: true - - doctest Access - doctest Atom - doctest Base - doctest Bitwise - doctest Code - doctest Collectable - doctest Enum - doctest Exception - doctest Float - doctest Inspect - doctest Inspect.Algebra - doctest Integer - doctest IO - doctest IO.ANSI - doctest Kernel - doctest Kernel.SpecialForms - doctest Keyword - doctest List - doctest Macro - doctest Map - doctest Module - doctest Node - doctest OptionParser - doctest Path - doctest Process - doctest Protocol - doctest Range - doctest Record - doctest Regex - doctest Stream - doctest String - doctest String.Chars - doctest StringIO - doctest Tuple - doctest URI - doctest Version -end diff --git a/lib/elixir/test/elixir/access_test.exs b/lib/elixir/test/elixir/access_test.exs index b28993addf3..04b7ade8377 100644 --- a/lib/elixir/test/elixir/access_test.exs +++ b/lib/elixir/test/elixir/access_test.exs @@ -1,8 +1,10 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule AccessTest do use ExUnit.Case, async: true + doctest Access + # Test nil at compilation time does not fail # and that @config[:foo] has proper precedence. @config nil @@ -19,8 +21,12 @@ defmodule AccessTest do test "for nil" do assert nil[:foo] == nil + assert Access.fetch(nil, :foo) == :error assert Access.get(nil, :foo) == nil - assert Access.get_and_update(nil, :foo, fn nil -> {:ok, :bar} end) == {:ok, :bar} + + assert_raise ArgumentError, "could not put/update key :foo on a nil value", fn -> + Access.get_and_update(nil, :foo, fn nil -> {:ok, :bar} end) + end end test "for keywords" do @@ -28,9 +34,23 @@ defmodule AccessTest do assert [foo: [bar: :baz]][:foo][:bar] == :baz assert [foo: [bar: :baz]][:fuu][:bar] == nil + assert Access.fetch([foo: :bar], :foo) == {:ok, :bar} + assert Access.fetch([foo: :bar], :bar) == :error + + msg = ~r/the Access calls for keywords expect the key to be an atom/ + + assert_raise ArgumentError, msg, fn -> + Access.fetch([], "foo") + end + assert Access.get([foo: :bar], :foo) == :bar assert Access.get_and_update([], :foo, fn nil -> {:ok, :baz} end) == {:ok, [foo: :baz]} - assert Access.get_and_update([foo: :bar], :foo, fn :bar -> {:ok, :baz} end) == {:ok, [foo: :baz]} + + assert Access.get_and_update([foo: :bar], :foo, fn :bar -> {:ok, :baz} end) == + {:ok, [foo: :baz]} + + assert Access.pop([foo: :bar], :foo) == {:bar, []} + assert Access.pop([], :foo) == {nil, []} end test "for maps" do @@ -39,18 +59,153 @@ defmodule AccessTest do assert %{1.0 => 1.0}[1.0] == 1.0 assert %{1 => 1}[1.0] == nil + assert Access.fetch(%{foo: :bar}, :foo) == {:ok, :bar} + assert Access.fetch(%{foo: :bar}, :bar) == :error + assert Access.get(%{foo: :bar}, :foo) == :bar assert Access.get_and_update(%{}, :foo, fn nil -> {:ok, :baz} end) == {:ok, %{foo: :baz}} - assert Access.get_and_update(%{foo: :bar}, :foo, fn :bar -> {:ok, :baz} end) == {:ok, %{foo: :baz}} + + assert Access.get_and_update(%{foo: :bar}, :foo, fn :bar -> {:ok, :baz} end) == + {:ok, %{foo: :baz}} + + assert Access.pop(%{foo: :bar}, :foo) == {:bar, %{}} + assert Access.pop(%{}, :foo) == {nil, %{}} end - test "for atoms" do - assert_raise Protocol.UndefinedError, ~r"protocol Access not implemented for :foo", fn -> - Access.get(:foo, :bar) + test "for struct" do + defmodule Sample do + defstruct [:name] + end + + message = + ~r"function AccessTest.Sample.fetch/2 is undefined \(AccessTest.Sample does not implement the Access behaviour" + + assert_raise UndefinedFunctionError, message, fn -> + Access.fetch(struct(Sample, []), :name) + end + + message = + ~r"function AccessTest.Sample.get_and_update/3 is undefined \(AccessTest.Sample does not implement the Access behaviour" + + assert_raise UndefinedFunctionError, message, fn -> + Access.get_and_update(struct(Sample, []), :name, fn nil -> {:ok, :baz} end) + end + + message = + ~r"function AccessTest.Sample.pop/2 is undefined \(AccessTest.Sample does not implement the Access behaviour" + + assert_raise UndefinedFunctionError, message, fn -> + Access.pop(struct(Sample, []), :name) + end + end + + describe "fetch!/2" do + assert Access.fetch!(%{foo: :bar}, :foo) == :bar + + assert_raise ArgumentError, + ~r/the Access calls for keywords expect the key to be an atom/, + fn -> Access.fetch!([], "foo") end + + assert_raise KeyError, + ~r/key \"foo\" not found/, + fn -> Access.fetch!(nil, "foo") end + end + + describe "filter/1" do + @test_list [1, 2, 3, 4, 5, 6] + + test "filters in get_in" do + assert get_in(@test_list, [Access.filter(&(&1 > 3))]) == [4, 5, 6] + end + + test "retains order in get_and_update_in" do + assert get_and_update_in(@test_list, [Access.filter(&(&1 == 3 || &1 == 2))], &{&1 * 2, &1}) == + {[4, 6], [1, 2, 3, 4, 5, 6]} + end + + test "retains order in pop_in" do + assert pop_in(@test_list, [Access.filter(&(&1 == 3 || &1 == 2))]) == {[2, 3], [1, 4, 5, 6]} + end + + test "chains with other access functions" do + mixed_map_and_list = %{foo: Enum.map(@test_list, &%{value: &1})} + + assert get_in(mixed_map_and_list, [:foo, Access.filter(&(&1.value <= 3)), :value]) == + [1, 2, 3] + end + end + + describe "at/1" do + @test_list [1, 2, 3, 4, 5, 6] + + test "returns element from the end if index is negative" do + assert get_in(@test_list, [Access.at(-2)]) == 5 + end + + test "returns nil if index is out of bounds counting from the end" do + assert get_in(@test_list, [Access.at(-10)]) == nil + end + + test "updates the element counting from the end if index is negative" do + assert get_and_update_in(@test_list, [Access.at(-2)], fn prev -> + {prev, :foo} + end) == {5, [1, 2, 3, 4, :foo, 6]} + end + + test "returns nil and does not update if index is out of bounds" do + assert get_and_update_in(@test_list, [Access.at(-10)], fn prev -> + {prev, :foo} + end) == {nil, [1, 2, 3, 4, 5, 6]} + end + end + + describe "at!/1" do + @test_list [1, 2, 3, 4, 5, 6] + + test "returns a list element when the index is within bounds, with get_in" do + assert get_in(@test_list, [Access.at!(5)]) == 6 + assert get_in(@test_list, [Access.at!(-6)]) == 1 + end + + test "updates a list element when the index is within bounds, with get_and_update_in" do + assert get_and_update_in(@test_list, [Access.at!(5)], fn prev -> + {prev, :foo} + end) == {6, [1, 2, 3, 4, 5, :foo]} + + assert get_and_update_in(@test_list, [Access.at!(-6)], fn prev -> + {prev, :foo} + end) == {1, [:foo, 2, 3, 4, 5, 6]} + end + + test "raises OutOfBoundsError when out of bounds, with get_in" do + assert_raise Enum.OutOfBoundsError, fn -> + get_in(@test_list, [Access.at!(6)]) + end + + assert_raise Enum.OutOfBoundsError, fn -> + get_in(@test_list, [Access.at!(-7)]) + end + end + + test "raises OutOfBoundsError when out of bounds, with get_and_update_in" do + assert_raise Enum.OutOfBoundsError, fn -> + get_and_update_in(@test_list, [Access.at!(6)], fn prev -> {prev, :foo} end) + end + + assert_raise Enum.OutOfBoundsError, fn -> + get_and_update_in(@test_list, [Access.at!(-7)], fn prev -> {prev, :foo} end) + end + end + + test "raises when not given a list" do + assert_raise RuntimeError, "Access.at!/1 expected a list, got: %{}", fn -> + get_in(%{}, [Access.at!(0)]) + end end - assert_raise Protocol.UndefinedError, ~r"protocol Access not implemented for :foo", fn -> - Access.get_and_update(:foo, :bar, fn _ -> {:ok, :baz} end) + test "chains" do + input = %{list: [%{greeting: "hi"}]} + assert get_in(input, [:list, Access.at!(0), :greeting]) == "hi" end end end diff --git a/lib/elixir/test/elixir/agent_test.exs b/lib/elixir/test/elixir/agent_test.exs index d4d58548806..b1d4d3ec6ed 100644 --- a/lib/elixir/test/elixir/agent_test.exs +++ b/lib/elixir/test/elixir/agent_test.exs @@ -1,37 +1,77 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule AgentTest do use ExUnit.Case, async: true - test "start_link/2 workflow with unregistered name" do - {:ok, pid} = Agent.start_link(fn -> %{} end) + doctest Agent + + def identity(state) do + state + end + + test "can be supervised directly" do + assert {:ok, _} = Supervisor.start_link([{Agent, fn -> :ok end}], strategy: :one_for_one) + end + + test "generates child_spec/1" do + defmodule MyAgent do + use Agent + end + + assert MyAgent.child_spec([:hello]) == %{ + id: MyAgent, + start: {MyAgent, :start_link, [[:hello]]} + } + + defmodule CustomAgent do + use Agent, id: :id, restart: :temporary, shutdown: :infinity, start: {:foo, :bar, []} + end + + assert CustomAgent.child_spec([:hello]) == %{ + id: :id, + restart: :temporary, + shutdown: :infinity, + start: {:foo, :bar, []} + } + end - {:links, links} = Process.info(self, :links) + test "start_link/2 workflow with unregistered name and anonymous functions" do + {:ok, pid} = Agent.start_link(&Map.new/0) + + {:links, links} = Process.info(self(), :links) assert pid in links + assert :proc_lib.translate_initial_call(pid) == {Map, :new, 0} + assert Agent.update(pid, &Map.put(&1, :hello, :world)) == :ok assert Agent.get(pid, &Map.get(&1, :hello), 3000) == :world assert Agent.get_and_update(pid, &Map.pop(&1, :hello), 3000) == :world - assert Agent.get(pid, &(&1)) == %{} + assert Agent.get(pid, & &1) == %{} assert Agent.stop(pid) == :ok wait_until_dead(pid) end - test "start/2 workflow with registered name" do - {:ok, pid} = Agent.start(fn -> %{} end, name: :agent) + test "start_link/2 with spawn_opt" do + {:ok, pid} = Agent.start_link(fn -> 0 end, spawn_opt: [priority: :high]) + assert Process.info(pid, :priority) == {:priority, :high} + end + + test "start/2 workflow with registered name and module functions" do + {:ok, pid} = Agent.start(Map, :new, [], name: :agent) assert Process.info(pid, :registered_name) == {:registered_name, :agent} - assert Agent.cast(:agent, &Map.put(&1, :hello, :world)) == :ok - assert Agent.get(:agent, &Map.get(&1, :hello)) == :world - assert Agent.get_and_update(:agent, &Map.pop(&1, :hello)) == :world - assert Agent.get(:agent, &(&1)) == %{} + assert :proc_lib.translate_initial_call(pid) == {Map, :new, 0} + assert Agent.cast(:agent, Map, :put, [:hello, :world]) == :ok + assert Agent.get(:agent, Map, :get, [:hello]) == :world + assert Agent.get_and_update(:agent, Map, :pop, [:hello]) == :world + assert Agent.get(:agent, AgentTest, :identity, []) == %{} assert Agent.stop(:agent) == :ok assert Process.info(pid, :registered_name) == nil end test ":sys.change_code/4 with mfa" do - { :ok, pid } = Agent.start_link(fn -> %{} end) + {:ok, pid} = Agent.start_link(fn -> %{} end) :ok = :sys.suspend(pid) - mfa = { Map, :put, [:hello, :world] } + mfa = {Map, :put, [:hello, :world]} assert :sys.change_code(pid, __MODULE__, "vsn", mfa) == :ok :ok = :sys.resume(pid) assert Agent.get(pid, &Map.get(&1, :hello)) == :world @@ -39,12 +79,12 @@ defmodule AgentTest do end test ":sys.change_code/4 with raising mfa" do - { :ok, pid } = Agent.start_link(fn -> %{} end) + {:ok, pid} = Agent.start_link(fn -> %{} end) :ok = :sys.suspend(pid) - mfa = { :erlang, :error, [] } - assert match?({ :error, _ }, :sys.change_code(pid, __MODULE__, "vsn", mfa)) + mfa = {:erlang, :error, []} + assert match?({:error, _}, :sys.change_code(pid, __MODULE__, "vsn", mfa)) :ok = :sys.resume(pid) - assert Agent.get(pid, &(&1)) == %{} + assert Agent.get(pid, & &1) == %{} assert Agent.stop(pid) == :ok end diff --git a/lib/elixir/test/elixir/application_test.exs b/lib/elixir/test/elixir/application_test.exs index 20258320fb2..4bdd37c3afd 100644 --- a/lib/elixir/test/elixir/application_test.exs +++ b/lib/elixir/test/elixir/application_test.exs @@ -1,31 +1,197 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule ApplicationTest do use ExUnit.Case, async: true + import PathHelpers + import ExUnit.CaptureIO + + @app :elixir + test "application environment" do + assert_raise ArgumentError, ~r/because the application was not loaded nor configured/, fn -> + Application.fetch_env!(:unknown, :unknown) + end + + assert_raise ArgumentError, ~r/because configuration at :unknown was not set/, fn -> + Application.fetch_env!(:elixir, :unknown) + end + assert Application.get_env(:elixir, :unknown) == nil assert Application.get_env(:elixir, :unknown, :default) == :default assert Application.fetch_env(:elixir, :unknown) == :error assert Application.put_env(:elixir, :unknown, :known) == :ok assert Application.fetch_env(:elixir, :unknown) == {:ok, :known} + assert Application.fetch_env!(:elixir, :unknown) == :known assert Application.get_env(:elixir, :unknown, :default) == :known assert {:unknown, :known} in Application.get_all_env(:elixir) assert Application.delete_env(:elixir, :unknown) == :ok assert Application.get_env(:elixir, :unknown, :default) == :default + after + Application.delete_env(:elixir, :unknown) + end + + test "deprecated non-atom keys" do + assert_deprecated(fn -> + Application.put_env(:elixir, [:a, :b], :c) + end) + + assert_deprecated(fn -> + assert Application.get_env(:elixir, [:a, :b]) == :c + end) + + assert_deprecated(fn -> + assert Application.fetch_env!(:elixir, [:a, :b]) == :c + end) + after + assert_deprecated(fn -> + Application.delete_env(:elixir, [:a, :b]) + end) + end + + defp assert_deprecated(fun) do + assert capture_io(:stderr, fun) =~ ~r/passing non-atom as application env key is deprecated/ + end + + describe "compile environment" do + test "invoked at compile time" do + assert_raise ArgumentError, ~r/because the application was not loaded nor configured/, fn -> + compile_env!(:unknown, :unknown) + end + + assert_received {:compile_env, :unknown, [:unknown], :error} + + assert_raise ArgumentError, ~r/because configuration at :unknown was not set/, fn -> + compile_env!(:elixir, :unknown) + end + + assert_received {:compile_env, :elixir, [:unknown], :error} + + assert compile_env(:elixir, :unknown) == nil + assert_received {:compile_env, :elixir, [:unknown], :error} + + assert compile_env(:elixir, :unknown, :default) == :default + assert_received {:compile_env, :elixir, [:unknown], :error} + + assert Application.put_env(:elixir, :unknown, nested: [key: :value]) == :ok + + assert compile_env(@app, :unknown, :default) == [nested: [key: :value]] + assert compile_env(:elixir, :unknown, :default) == [nested: [key: :value]] + assert_received {:compile_env, :elixir, [:unknown], {:ok, [nested: [key: :value]]}} + + assert compile_env(:elixir, :unknown) == [nested: [key: :value]] + assert_received {:compile_env, :elixir, [:unknown], {:ok, [nested: [key: :value]]}} + + assert compile_env!(@app, :unknown) == [nested: [key: :value]] + assert compile_env!(:elixir, :unknown) == [nested: [key: :value]] + assert_received {:compile_env, :elixir, [:unknown], {:ok, [nested: [key: :value]]}} + + assert compile_env(:elixir, [:unknown, :nested]) == [key: :value] + assert_received {:compile_env, :elixir, [:unknown, :nested], {:ok, [key: :value]}} + + assert compile_env!(:elixir, [:unknown, :nested]) == [key: :value] + assert_received {:compile_env, :elixir, [:unknown, :nested], {:ok, [key: :value]}} + + assert compile_env(:elixir, [:unknown, :nested, :key]) == :value + assert_received {:compile_env, :elixir, [:unknown, :nested, :key], {:ok, :value}} + + assert compile_env!(:elixir, [:unknown, :nested, :key]) == :value + assert_received {:compile_env, :elixir, [:unknown, :nested, :key], {:ok, :value}} + + assert compile_env(:elixir, [:unknown, :unknown, :key], :default) == :default + assert_received {:compile_env, :elixir, [:unknown, :unknown, :key], :error} + + assert compile_env(:elixir, [:unknown, :nested, :unkown], :default) == :default + assert_received {:compile_env, :elixir, [:unknown, :nested, :unkown], :error} + after + Application.delete_env(:elixir, :unknown) + end + + def trace({:compile_env, _, _, _} = msg, %Macro.Env{}) do + send(self(), msg) + :ok + end + + def trace(_, _), do: :ok + + defp compile_env(app, key, default \\ nil) do + code = + quote do + require Application + Application.compile_env(unquote(app), unquote(key), unquote(default)) + end + + {result, _binding} = Code.eval_quoted(code, [], tracers: [__MODULE__]) + result + end + + defp compile_env!(app, key) do + code = + quote do + require Application + Application.compile_env!(unquote(app), unquote(key)) + end + + {result, _binding} = Code.eval_quoted(code, [], tracers: [__MODULE__]) + result + end + end + + test "loaded and started applications" do + started = Application.started_applications() + assert is_list(started) + assert {:elixir, 'elixir', _} = List.keyfind(started, :elixir, 0) + + started_timeout = Application.started_applications(7000) + assert is_list(started_timeout) + assert {:elixir, 'elixir', _} = List.keyfind(started_timeout, :elixir, 0) + + loaded = Application.loaded_applications() + assert is_list(loaded) + assert {:elixir, 'elixir', _} = List.keyfind(loaded, :elixir, 0) + end + + test "application specification" do + assert is_list(Application.spec(:elixir)) + assert Application.spec(:unknown) == nil + assert Application.spec(:unknown, :description) == nil + + assert Application.spec(:elixir, :description) == 'elixir' + assert_raise FunctionClauseError, fn -> Application.spec(:elixir, :unknown) end + end + + test "application module" do + assert Application.get_application(String) == :elixir + assert Application.get_application(__MODULE__) == nil + assert Application.get_application(__MODULE__.Unknown) == nil end test "application directory" do root = Path.expand("../../../..", __DIR__) - assert Application.app_dir(:elixir) == - Path.join(root, "bin/../lib/elixir") - assert Application.app_dir(:elixir, "priv") == - Path.join(root, "bin/../lib/elixir/priv") + + assert normalize_app_dir(Application.app_dir(:elixir)) == + normalize_app_dir(Path.join(root, "bin/../lib/elixir")) + + assert normalize_app_dir(Application.app_dir(:elixir, "priv")) == + normalize_app_dir(Path.join(root, "bin/../lib/elixir/priv")) + + assert normalize_app_dir(Application.app_dir(:elixir, ["priv", "foo"])) == + normalize_app_dir(Path.join(root, "bin/../lib/elixir/priv/foo")) assert_raise ArgumentError, fn -> Application.app_dir(:unknown) end end + + if windows?() do + defp normalize_app_dir(path) do + path |> String.downcase() |> Path.expand() + end + else + defp normalize_app_dir(path) do + path |> String.downcase() + end + end end diff --git a/lib/elixir/test/elixir/atom_test.exs b/lib/elixir/test/elixir/atom_test.exs index 7d6d5bcb7ff..0f07a033c14 100644 --- a/lib/elixir/test/elixir/atom_test.exs +++ b/lib/elixir/test/elixir/atom_test.exs @@ -1,13 +1,15 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule AtomTest do use ExUnit.Case, async: true + doctest Atom + test "to_string/1" do - assert Atom.to_string(:"héllo") == "héllo" + assert "héllo" |> String.to_atom() |> Atom.to_string() == "héllo" end - test "to_char_list/1" do - assert Atom.to_char_list(:"héllo") == 'héllo' + test "to_charlist/1" do + assert "héllo" |> String.to_atom() |> Atom.to_charlist() == 'héllo' end end diff --git a/lib/elixir/test/elixir/base_test.exs b/lib/elixir/test/elixir/base_test.exs index d976bc14173..eec524b6ddc 100644 --- a/lib/elixir/test/elixir/base_test.exs +++ b/lib/elixir/test/elixir/base_test.exs @@ -1,10 +1,12 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule BaseTest do use ExUnit.Case, async: true + + doctest Base import Base - test "encode16" do + test "encode16/1" do assert "" == encode16("") assert "66" == encode16("f") assert "666F" == encode16("fo") @@ -14,10 +16,11 @@ defmodule BaseTest do assert "666F6F626172" == encode16("foobar") assert "A1B2C3D4E5F67891" == encode16(<<161, 178, 195, 212, 229, 246, 120, 145>>) - assert "a1b2c3d4e5f67891" == encode16(<<161, 178, 195, 212, 229, 246, 120, 145>>, case: :lower) + assert "a1b2c3d4e5f67891" == + encode16(<<161, 178, 195, 212, 229, 246, 120, 145>>, case: :lower) end - test "decode16" do + test "decode16/1" do assert {:ok, ""} == decode16("") assert {:ok, "f"} == decode16("66") assert {:ok, "fo"} == decode16("666F") @@ -25,13 +28,16 @@ defmodule BaseTest do assert {:ok, "foob"} == decode16("666F6F62") assert {:ok, "fooba"} == decode16("666F6F6261") assert {:ok, "foobar"} == decode16("666F6F626172") - assert {:ok, <<161, 178, 195, 212, 229, 246, 120, 145>>} == decode16("A1B2C3D4E5F67891") + assert {:ok, <<161, 178, 195, 212, 229, 246, 120, 145>>} == decode16("A1B2C3D4E5F67891") + + assert {:ok, <<161, 178, 195, 212, 229, 246, 120, 145>>} == + decode16("a1b2c3d4e5f67891", case: :lower) - assert {:ok, <<161, 178, 195, 212, 229, 246, 120, 145>>} == decode16("a1b2c3d4e5f67891", case: :lower) - assert {:ok, <<161, 178, 195, 212, 229, 246, 120, 145>>} == decode16("a1B2c3D4e5F67891", case: :mixed) + assert {:ok, <<161, 178, 195, 212, 229, 246, 120, 145>>} == + decode16("a1B2c3D4e5F67891", case: :mixed) end - test "decode16!" do + test "decode16!/1" do assert "" == decode16!("") assert "f" == decode16!("66") assert "fo" == decode16!("666F") @@ -41,424 +47,776 @@ defmodule BaseTest do assert "foobar" == decode16!("666F6F626172") assert <<161, 178, 195, 212, 229, 246, 120, 145>> == decode16!("A1B2C3D4E5F67891") - assert <<161, 178, 195, 212, 229, 246, 120, 145>> == decode16!("a1b2c3d4e5f67891", case: :lower) - assert <<161, 178, 195, 212, 229, 246, 120, 145>> == decode16!("a1B2c3D4e5F67891", case: :mixed) + assert <<161, 178, 195, 212, 229, 246, 120, 145>> == + decode16!("a1b2c3d4e5f67891", case: :lower) + + assert <<161, 178, 195, 212, 229, 246, 120, 145>> == + decode16!("a1B2c3D4e5F67891", case: :mixed) end - test "decode16 non-alphabet digit" do + test "decode16/1 errors on non-alphabet digit" do assert :error == decode16("66KF") assert :error == decode16("66ff") assert :error == decode16("66FF", case: :lower) end - test "decode16! non-alphabet digit" do - assert_raise ArgumentError, "non-alphabet digit found: K", fn -> + test "decode16!/1 errors on non-alphabet digit" do + assert_raise ArgumentError, "non-alphabet digit found: \"K\" (byte 75)", fn -> decode16!("66KF") end - assert_raise ArgumentError, "non-alphabet digit found: f", fn -> + + assert_raise ArgumentError, "non-alphabet digit found: \"f\" (byte 102)", fn -> decode16!("66ff") end - assert_raise ArgumentError, "non-alphabet digit found: F", fn -> + + assert_raise ArgumentError, "non-alphabet digit found: \"F\" (byte 70)", fn -> decode16!("66FF", case: :lower) end end - test "decode16 odd-length string" do + test "decode16/1 errors on odd-length string" do assert :error == decode16("666") end - test "decode16! odd-length string" do - assert_raise ArgumentError, "odd-length string", fn -> + test "decode16!/1 errors odd-length string" do + assert_raise ArgumentError, ~r/string given to decode has wrong length/, fn -> decode16!("666") end end - test "encode64 empty" do + test "encode64/1 can deal with empty strings" do assert "" == encode64("") end - test "encode64 two pads" do + test "encode64/1 with two pads" do assert "QWxhZGRpbjpvcGVuIHNlc2FtZQ==" == encode64("Aladdin:open sesame") end - test "encode64 one pad" do + test "encode64/1 with one pad" do assert "SGVsbG8gV29ybGQ=" == encode64("Hello World") end - test "encode64 no pad" do + test "encode64/1 with no pad" do assert "QWxhZGRpbjpvcGVuIHNlc2Ft" == encode64("Aladdin:open sesam") - assert "MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9" == encode64(<<"0123456789!@#0^&*();:<>,. []{}">>) + + assert "MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9" == + encode64(<<"0123456789!@#0^&*();:<>,. []{}">>) end - test "decode64 empty" do + test "encode64/1 with one pad and ignoring padding" do + assert "SGVsbG8gV29ybGQ" == encode64("Hello World", padding: false) + end + + test "encode64/1 with two pads and ignoring padding" do + assert "QWxhZGRpbjpvcGVuIHNlc2FtZQ" == encode64("Aladdin:open sesame", padding: false) + end + + test "encode64/1 with no pads and ignoring padding" do + assert "QWxhZGRpbjpvcGVuIHNlc2Ft" == encode64("Aladdin:open sesam", padding: false) + end + + test "decode64/1 can deal with empty strings" do assert {:ok, ""} == decode64("") end - test "decode64! empty" do + test "decode64!/1 can deal with empty strings" do assert "" == decode64!("") end - test "decode64 two pads" do + test "decode64/1 with two pads" do assert {:ok, "Aladdin:open sesame"} == decode64("QWxhZGRpbjpvcGVuIHNlc2FtZQ==") end - test "decode64! two pads" do + test "decode64!/1 with two pads" do assert "Aladdin:open sesame" == decode64!("QWxhZGRpbjpvcGVuIHNlc2FtZQ==") end - test "decode64 one pad" do + test "decode64/1 with one pad" do assert {:ok, "Hello World"} == decode64("SGVsbG8gV29ybGQ=") end - test "decode64! one pad" do + test "decode64!/1 with one pad" do assert "Hello World" == decode64!("SGVsbG8gV29ybGQ=") end - test "decode64 no pad" do + test "decode64/1 with no pad" do assert {:ok, "Aladdin:open sesam"} == decode64("QWxhZGRpbjpvcGVuIHNlc2Ft") end - test "decode64! no pad" do + test "decode64!/1 with no pad" do assert "Aladdin:open sesam" == decode64!("QWxhZGRpbjpvcGVuIHNlc2Ft") end - test "decode64 non-alphabet digit" do + test "decode64/1 errors on non-alphabet digit" do assert :error == decode64("Zm9)") end - test "decode64! non-alphabet digit" do - assert_raise ArgumentError, "non-alphabet digit found: )", fn -> + test "decode64!/1 errors on non-alphabet digit" do + assert_raise ArgumentError, "non-alphabet digit found: \")\" (byte 41)", fn -> decode64!("Zm9)") end end - test "decode64 incorrect padding" do + test "decode64/1 errors on whitespace unless there's ignore: :whitespace" do + assert :error == decode64("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t") + + assert {:ok, "Aladdin:open sesam"} == + decode64("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t", ignore: :whitespace) + end + + test "decode64!/1 errors on whitespace unless there's ignore: :whitespace" do + assert_raise ArgumentError, "non-alphabet digit found: \"\\n\" (byte 10)", fn -> + decode64!("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t") + end + + assert "Aladdin:open sesam" == + decode64!("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t", ignore: :whitespace) + end + + test "decode64/1 errors on incorrect padding" do assert :error == decode64("SGVsbG8gV29ybGQ") end - test "decode64! incorrect padding" do + test "decode64!/1 errors on incorrect padding" do assert_raise ArgumentError, "incorrect padding", fn -> decode64!("SGVsbG8gV29ybGQ") end end - test "url_encode64 empty" do + test "decode64/2 with two pads and ignoring padding" do + assert {:ok, "Aladdin:open sesame"} == decode64("QWxhZGRpbjpvcGVuIHNlc2FtZQ", padding: false) + end + + test "decode64!/2 with two pads and ignoring padding" do + assert "Aladdin:open sesame" == decode64!("QWxhZGRpbjpvcGVuIHNlc2FtZQ", padding: false) + end + + test "decode64/2 with one pad and ignoring padding" do + assert {:ok, "Hello World"} == decode64("SGVsbG8gV29ybGQ", padding: false) + end + + test "decode64!/2 with one pad and ignoring padding" do + assert "Hello World" == decode64!("SGVsbG8gV29ybGQ", padding: false) + end + + test "decode64/2 with no pad and ignoring padding" do + assert {:ok, "Aladdin:open sesam"} == decode64("QWxhZGRpbjpvcGVuIHNlc2Ft", padding: false) + end + + test "decode64!/2 with no pad and ignoring padding" do + assert "Aladdin:open sesam" == decode64!("QWxhZGRpbjpvcGVuIHNlc2Ft", padding: false) + end + + test "decode64/2 with incorrect padding and ignoring padding" do + assert {:ok, "Hello World"} == decode64("SGVsbG8gV29ybGQ", padding: false) + end + + test "decode64!/2 with incorrect padding and ignoring padding" do + assert "Hello World" == decode64!("SGVsbG8gV29ybGQ", padding: false) + end + + test "url_encode64/1 can deal with empty strings" do assert "" == url_encode64("") end - test "url_encode64 two pads" do + test "url_encode64/1 with two pads" do assert "QWxhZGRpbjpvcGVuIHNlc2FtZQ==" == url_encode64("Aladdin:open sesame") end - test "url_encode64 one pad" do + test "url_encode64/1 with one pad" do assert "SGVsbG8gV29ybGQ=" == url_encode64("Hello World") end - test "url_encode64 no pad" do + test "url_encode64/1 with no pad" do assert "QWxhZGRpbjpvcGVuIHNlc2Ft" == url_encode64("Aladdin:open sesam") - assert "MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9" == url_encode64(<<"0123456789!@#0^&*();:<>,. []{}">>) + + assert "MDEyMzQ1Njc4OSFAIzBeJiooKTs6PD4sLiBbXXt9" == + url_encode64(<<"0123456789!@#0^&*();:<>,. []{}">>) + end + + test "url_encode64/2 with two pads and ignoring padding" do + assert "QWxhZGRpbjpvcGVuIHNlc2FtZQ" == url_encode64("Aladdin:open sesame", padding: false) + end + + test "url_encode64/2 with one pad and ignoring padding" do + assert "SGVsbG8gV29ybGQ" == url_encode64("Hello World", padding: false) + end + + test "url_encode64/2 with no pad and ignoring padding" do + assert "QWxhZGRpbjpvcGVuIHNlc2Ft" == url_encode64("Aladdin:open sesam", padding: false) end - test "url_encode64 no URL unsafe characters" do - refute "/3/+/A==" == url_encode64(<<255,127,254,252>>) - assert "_3_-_A==" == url_encode64(<<255,127,254,252>>) + test "url_encode64/1 doesn't produce URL-unsafe characters" do + refute "/3/+/A==" == url_encode64(<<255, 127, 254, 252>>) + assert "_3_-_A==" == url_encode64(<<255, 127, 254, 252>>) end - test "url_decode64 empty" do + test "url_decode64/1 can deal with empty strings" do assert {:ok, ""} == url_decode64("") end - test "url_decode64! empty" do + test "url_decode64!/1 can deal with empty strings" do assert "" == url_decode64!("") end - test "url_decode64 two pads" do + test "url_decode64/1 with two pads" do assert {:ok, "Aladdin:open sesame"} == url_decode64("QWxhZGRpbjpvcGVuIHNlc2FtZQ==") end - test "url_decode64! two pads" do + test "url_decode64!/1 with two pads" do assert "Aladdin:open sesame" == url_decode64!("QWxhZGRpbjpvcGVuIHNlc2FtZQ==") end - test "url_decode64 one pad" do + test "url_decode64/1 with one pad" do assert {:ok, "Hello World"} == url_decode64("SGVsbG8gV29ybGQ=") end - test "url_decode64! one pad" do + test "url_decode64!/1 with one pad" do assert "Hello World" == url_decode64!("SGVsbG8gV29ybGQ=") end - test "url_decode64 no pad" do + test "url_decode64/1 with no pad" do assert {:ok, "Aladdin:open sesam"} == url_decode64("QWxhZGRpbjpvcGVuIHNlc2Ft") end - test "url_decode64! no pad" do + test "url_decode64!/1 with no pad" do assert "Aladdin:open sesam" == url_decode64!("QWxhZGRpbjpvcGVuIHNlc2Ft") end - test "url_decode64 non-alphabet digit" do + test "url_decode64/1,2 error on whitespace unless there's ignore: :whitespace" do + assert :error == url_decode64("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t") + + assert {:ok, "Aladdin:open sesam"} == + url_decode64("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t", ignore: :whitespace) + end + + test "url_decode64!/1,2 error on whitespace unless there's ignore: :whitespace" do + assert_raise ArgumentError, "non-alphabet digit found: \"\\n\" (byte 10)", fn -> + url_decode64!("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t") + end + + assert "Aladdin:open sesam" == + url_decode64!("\nQWxhZGRp bjpvcGVu\sIHNlc2Ft\t", ignore: :whitespace) + end + + test "url_decode64/1 errors on non-alphabet digit" do assert :error == url_decode64("Zm9)") end - test "url_decode64! non-alphabet digit" do - assert_raise ArgumentError, "non-alphabet digit found: )", fn -> + test "url_decode64!/1 errors on non-alphabet digit" do + assert_raise ArgumentError, "non-alphabet digit found: \")\" (byte 41)", fn -> url_decode64!("Zm9)") end end - test "url_decode64 incorrect padding" do + test "url_decode64/1 errors on incorrect padding" do assert :error == url_decode64("SGVsbG8gV29ybGQ") end - test "url_decode64! incorrect padding" do + test "url_decode64!/1 errors on incorrect padding" do assert_raise ArgumentError, "incorrect padding", fn -> url_decode64!("SGVsbG8gV29ybGQ") end end - test "encode32 empty" do + test "url_decode64/2 with two pads and ignoring padding" do + assert {:ok, "Aladdin:open sesame"} == + url_decode64("QWxhZGRpbjpvcGVuIHNlc2FtZQ", padding: false) + end + + test "url_decode64!/2 with two pads and ignoring padding" do + assert "Aladdin:open sesame" == url_decode64!("QWxhZGRpbjpvcGVuIHNlc2FtZQ", padding: false) + end + + test "url_decode64/2 with one pad and ignoring padding" do + assert {:ok, "Hello World"} == url_decode64("SGVsbG8gV29ybGQ", padding: false) + end + + test "url_decode64!/2 with one pad and ignoring padding" do + assert "Hello World" == url_decode64!("SGVsbG8gV29ybGQ", padding: false) + end + + test "url_decode64/2 with no pad and ignoring padding" do + assert {:ok, "Aladdin:open sesam"} == url_decode64("QWxhZGRpbjpvcGVuIHNlc2Ft", padding: false) + end + + test "url_decode64!/2 with no pad and ignoring padding" do + assert "Aladdin:open sesam" == url_decode64!("QWxhZGRpbjpvcGVuIHNlc2Ft", padding: false) + end + + test "url_decode64/2 ignores incorrect padding when :padding is false" do + assert {:ok, "Hello World"} == url_decode64("SGVsbG8gV29ybGQ", padding: false) + end + + test "url_decode64!/2 ignores incorrect padding when :padding is false" do + assert "Hello World" == url_decode64!("SGVsbG8gV29ybGQ", padding: false) + end + + test "encode32/1 can deal with empty strings" do assert "" == encode32("") end - test "encode32 one pad" do + test "encode32/1 with one pad" do assert "MZXW6YQ=" == encode32("foob") end - test "encode32 three pads" do + test "encode32/1 with three pads" do assert "MZXW6===" == encode32("foo") end - test "encode32 four pads" do + test "encode32/1 with four pads" do assert "MZXQ====" == encode32("fo") end - test "encode32 six pads" do + test "encode32/1 with six pads" do assert "MZXW6YTBOI======" == encode32("foobar") assert "MY======" == encode32("f") end - test "encode32 no pads" do + test "encode32/1 with no pads" do assert "MZXW6YTB" == encode32("fooba") end - test "encode32 lowercase" do + test "encode32/2 with one pad and ignoring padding" do + assert "MZXW6YQ" == encode32("foob", padding: false) + end + + test "encode32/2 with three pads and ignoring padding" do + assert "MZXW6" == encode32("foo", padding: false) + end + + test "encode32/2 with four pads and ignoring padding" do + assert "MZXQ" == encode32("fo", padding: false) + end + + test "encode32/2 with six pads and ignoring padding" do + assert "MZXW6YTBOI" == encode32("foobar", padding: false) + end + + test "encode32/2 with no pads and ignoring padding" do + assert "MZXW6YTB" == encode32("fooba", padding: false) + end + + test "encode32/2 with lowercase" do assert "mzxw6ytb" == encode32("fooba", case: :lower) end - test "decode32 empty" do + test "decode32/1 can deal with empty strings" do assert {:ok, ""} == decode32("") end - test "decode32! empty" do + test "decode32!/2 can deal with empty strings" do assert "" == decode32!("") end - test "decode32 one pad" do + test "decode32/1 with one pad" do assert {:ok, "foob"} == decode32("MZXW6YQ=") end - test "decode32! one pad" do + test "decode32!/1 with one pad" do assert "foob" == decode32!("MZXW6YQ=") end - test "decode32 three pads" do + test "decode32/1 with three pads" do assert {:ok, "foo"} == decode32("MZXW6===") end - test "decode32! three pads" do + test "decode32!/1 with three pads" do assert "foo" == decode32!("MZXW6===") end - test "decode32 four pads" do + test "decode32/1 with four pads" do assert {:ok, "fo"} == decode32("MZXQ====") end - test "decode32! four pads" do + test "decode32!/1 with four pads" do assert "fo" == decode32!("MZXQ====") end - test "decode32 lowercase" do + test "decode32/2 with lowercase" do assert {:ok, "fo"} == decode32("mzxq====", case: :lower) end - test "decode32! lowercase" do + test "decode32!/2 with lowercase" do assert "fo" == decode32!("mzxq====", case: :lower) end - test "decode32 mixed case" do + test "decode32/2 with mixed case" do assert {:ok, "fo"} == decode32("mZXq====", case: :mixed) end - test "decode32! mixed case" do + test "decode32!/2 with mixed case" do assert "fo" == decode32!("mZXq====", case: :mixed) end - test "decode32 six pads" do + test "decode32/1 with six pads" do assert {:ok, "foobar"} == decode32("MZXW6YTBOI======") assert {:ok, "f"} == decode32("MY======") end - test "decode32! six pads" do + test "decode32!/1 with six pads" do assert "foobar" == decode32!("MZXW6YTBOI======") assert "f" == decode32!("MY======") end - test "decode32 no pads" do + test "decode32/1 with no pads" do assert {:ok, "fooba"} == decode32("MZXW6YTB") end - test "decode32! no pads" do + test "decode32!/1 with no pads" do assert "fooba" == decode32!("MZXW6YTB") end - test "decode32 non-alphabet digit" do + test "decode32/1,2 error on non-alphabet digit" do assert :error == decode32("MZX)6YTB") assert :error == decode32("66ff") assert :error == decode32("66FF", case: :lower) end - test "decode32! non-alphabet digit" do - assert_raise ArgumentError, "non-alphabet digit found: )", fn -> + test "decode32!/1,2 argument error on non-alphabet digit" do + assert_raise ArgumentError, "non-alphabet digit found: \")\" (byte 41)", fn -> decode32!("MZX)6YTB") end - assert_raise ArgumentError, "non-alphabet digit found: m", fn -> + + assert_raise ArgumentError, "non-alphabet digit found: \"m\" (byte 109)", fn -> decode32!("mzxw6ytboi======") end - assert_raise ArgumentError, "non-alphabet digit found: M", fn -> + + assert_raise ArgumentError, "non-alphabet digit found: \"M\" (byte 77)", fn -> decode32!("MZXW6YTBOI======", case: :lower) end + + assert_raise ArgumentError, "non-alphabet digit found: \"0\" (byte 48)", fn -> + decode32!("0ZXW6YTB0I======", case: :mixed) + end end - test "decode32 incorrect padding" do + test "decode32/1 errors on incorrect padding" do assert :error == decode32("MZXW6YQ") end - test "decode32! incorrect padding" do + test "decode32!/1 errors on incorrect padding" do assert_raise ArgumentError, "incorrect padding", fn -> decode32!("MZXW6YQ") end end - test "hex_encode32 empty" do + test "decode32/2 with one pad and :padding to false" do + assert {:ok, "foob"} == decode32("MZXW6YQ", padding: false) + end + + test "decode32!/2 with one pad and :padding to false" do + assert "foob" == decode32!("MZXW6YQ", padding: false) + end + + test "decode32/2 with three pads and ignoring padding" do + assert {:ok, "foo"} == decode32("MZXW6", padding: false) + end + + test "decode32!/2 with three pads and ignoring padding" do + assert "foo" == decode32!("MZXW6", padding: false) + end + + test "decode32/2 with four pads and ignoring padding" do + assert {:ok, "fo"} == decode32("MZXQ", padding: false) + end + + test "decode32!/2 with four pads and ignoring padding" do + assert "fo" == decode32!("MZXQ", padding: false) + end + + test "decode32/2 with :lower case and ignoring padding" do + assert {:ok, "fo"} == decode32("mzxq", case: :lower, padding: false) + end + + test "decode32!/2 with :lower case and ignoring padding" do + assert "fo" == decode32!("mzxq", case: :lower, padding: false) + end + + test "decode32/2 with :mixed case and ignoring padding" do + assert {:ok, "fo"} == decode32("mZXq", case: :mixed, padding: false) + end + + test "decode32!/2 with :mixed case and ignoring padding" do + assert "fo" == decode32!("mZXq", case: :mixed, padding: false) + end + + test "decode32/2 with six pads and ignoring padding" do + assert {:ok, "foobar"} == decode32("MZXW6YTBOI", padding: false) + end + + test "decode32!/2 with six pads and ignoring padding" do + assert "foobar" == decode32!("MZXW6YTBOI", padding: false) + end + + test "decode32/2 with no pads and ignoring padding" do + assert {:ok, "fooba"} == decode32("MZXW6YTB", padding: false) + end + + test "decode32!/2 with no pads and ignoring padding" do + assert "fooba" == decode32!("MZXW6YTB", padding: false) + end + + test "decode32/2 ignores incorrect padding when :padding is false" do + assert {:ok, "foob"} == decode32("MZXW6YQ", padding: false) + end + + test "decode32!/2 ignores incorrect padding when :padding is false" do + "foob" = decode32!("MZXW6YQ", padding: false) + end + + test "hex_encode32/1 can deal with empty strings" do assert "" == hex_encode32("") end - test "hex_encode32 one pad" do + test "hex_encode32/1 with one pad" do assert "CPNMUOG=" == hex_encode32("foob") end - test "hex_encode32 three pads" do + test "hex_encode32/1 with three pads" do assert "CPNMU===" == hex_encode32("foo") end - test "hex_encode32 four pads" do + test "hex_encode32/1 with four pads" do assert "CPNG====" == hex_encode32("fo") end - test "hex_encode32 six pads" do + test "hex_encode32/1 with six pads" do assert "CPNMUOJ1E8======" == hex_encode32("foobar") assert "CO======" == hex_encode32("f") end - test "hex_encode32 no pads" do + test "hex_encode32/1 with no pads" do assert "CPNMUOJ1" == hex_encode32("fooba") end - test "hex_encode32 lowercase" do + test "hex_encode32/2 with one pad and ignoring padding" do + assert "CPNMUOG" == hex_encode32("foob", padding: false) + end + + test "hex_encode32/2 with three pads and ignoring padding" do + assert "CPNMU" == hex_encode32("foo", padding: false) + end + + test "hex_encode32/2 with four pads and ignoring padding" do + assert "CPNG" == hex_encode32("fo", padding: false) + end + + test "hex_encode32/2 with six pads and ignoring padding" do + assert "CPNMUOJ1E8" == hex_encode32("foobar", padding: false) + end + + test "hex_encode32/2 with no pads and ignoring padding" do + assert "CPNMUOJ1" == hex_encode32("fooba", padding: false) + end + + test "hex_encode32/2 with lowercase" do assert "cpnmuoj1" == hex_encode32("fooba", case: :lower) end - test "hex_decode32 empty" do + test "hex_decode32/1 can deal with empty strings" do assert {:ok, ""} == hex_decode32("") end - test "hex_decode32! empty" do + test "hex_decode32!/1 can deal with empty strings" do assert "" == hex_decode32!("") end - test "hex_decode32 one pad" do + test "hex_decode32/1 with one pad" do assert {:ok, "foob"} == hex_decode32("CPNMUOG=") end - test "hex_decode32! one pad" do + test "hex_decode32!/1 with one pad" do assert "foob" == hex_decode32!("CPNMUOG=") end - test "hex_decode32 three pads" do + test "hex_decode32/1 with three pads" do assert {:ok, "foo"} == hex_decode32("CPNMU===") end - test "hex_decode32! three pads" do + test "hex_decode32!/1 with three pads" do assert "foo" == hex_decode32!("CPNMU===") end - test "hex_decode32 four pads" do + test "hex_decode32/1 with four pads" do assert {:ok, "fo"} == hex_decode32("CPNG====") end - test "hex_decode32! four pads" do + test "hex_decode32!/1 with four pads" do assert "fo" == hex_decode32!("CPNG====") end - test "hex_decode32 six pads" do + test "hex_decode32/1 with six pads" do assert {:ok, "foobar"} == hex_decode32("CPNMUOJ1E8======") assert {:ok, "f"} == hex_decode32("CO======") end - test "hex_decode32! six pads" do + test "hex_decode32!/1 with six pads" do assert "foobar" == hex_decode32!("CPNMUOJ1E8======") assert "f" == hex_decode32!("CO======") end - test "hex_decode32 no pads" do + test "hex_decode32/1 with no pads" do assert {:ok, "fooba"} == hex_decode32("CPNMUOJ1") end - test "hex_decode32! no pads" do + test "hex_decode32!/1 with no pads" do assert "fooba" == hex_decode32!("CPNMUOJ1") end - test "hex_decode32 non-alphabet digit" do + test "hex_decode32/1,2 error on non-alphabet digit" do assert :error == hex_decode32("CPN)UOJ1") assert :error == hex_decode32("66f") assert :error == hex_decode32("66F", case: :lower) end - test "hex_decode32! non-alphabet digit" do - assert_raise ArgumentError, "non-alphabet digit found: )", fn -> + test "hex_decode32!/1,2 error non-alphabet digit" do + assert_raise ArgumentError, "non-alphabet digit found: \")\" (byte 41)", fn -> hex_decode32!("CPN)UOJ1") end - assert_raise ArgumentError, "non-alphabet digit found: c", fn -> + + assert_raise ArgumentError, "non-alphabet digit found: \"c\" (byte 99)", fn -> hex_decode32!("cpnmuoj1e8======") end - assert_raise ArgumentError, "non-alphabet digit found: C", fn -> + + assert_raise ArgumentError, "non-alphabet digit found: \"C\" (byte 67)", fn -> hex_decode32!("CPNMUOJ1E8======", case: :lower) end end - test "hex_decode32 incorrect padding" do + test "hex_decode32/1 errors on incorrect padding" do assert :error == hex_decode32("CPNMUOG") end - test "hex_decode32! incorrect padding" do + test "hex_decode32!/1 errors on incorrect padding" do assert_raise ArgumentError, "incorrect padding", fn -> hex_decode32!("CPNMUOG") end end - test "hex_decode32 lowercase" do + test "hex_decode32/2 with lowercase" do assert {:ok, "fo"} == hex_decode32("cpng====", case: :lower) end - test "hex_decode32! lowercase" do + test "hex_decode32!/2 with lowercase" do assert "fo" == hex_decode32!("cpng====", case: :lower) end - test "hex_decode32 mixed case" do + test "hex_decode32/2 with mixed case" do assert {:ok, "fo"} == hex_decode32("cPNg====", case: :mixed) end - test "hex_decode32! mixed case" do + test "hex_decode32!/2 with mixed case" do assert "fo" == hex_decode32!("cPNg====", case: :mixed) end + + test "decode16!/1 errors on non-UTF-8 char" do + assert_raise ArgumentError, "non-alphabet digit found: \"\\0\" (byte 0)", fn -> + decode16!("012" <> <<0>>) + end + end + + test "hex_decode32/2 with one pad and ignoring padding" do + assert {:ok, "foob"} == hex_decode32("CPNMUOG", padding: false) + end + + test "hex_decode32!/2 with one pad and ignoring padding" do + assert "foob" == hex_decode32!("CPNMUOG", padding: false) + end + + test "hex_decode32/2 with three pads and ignoring padding" do + assert {:ok, "foo"} == hex_decode32("CPNMU", padding: false) + end + + test "hex_decode32!/2 with three pads and ignoring padding" do + assert "foo" == hex_decode32!("CPNMU", padding: false) + end + + test "hex_decode32/2 with four pads and ignoring padding" do + assert {:ok, "fo"} == hex_decode32("CPNG", padding: false) + end + + test "hex_decode32!/2 with four pads and ignoring padding" do + assert "fo" == hex_decode32!("CPNG", padding: false) + end + + test "hex_decode32/2 with six pads and ignoring padding" do + assert {:ok, "foobar"} == hex_decode32("CPNMUOJ1E8", padding: false) + end + + test "hex_decode32!/2 with six pads and ignoring padding" do + assert "foobar" == hex_decode32!("CPNMUOJ1E8", padding: false) + end + + test "hex_decode32/2 with no pads and ignoring padding" do + assert {:ok, "fooba"} == hex_decode32("CPNMUOJ1", padding: false) + end + + test "hex_decode32!/2 with no pads and ignoring padding" do + assert "fooba" == hex_decode32!("CPNMUOJ1", padding: false) + end + + test "hex_decode32/2 ignores incorrect padding when :padding is false" do + assert {:ok, "foob"} == hex_decode32("CPNMUOG", padding: false) + end + + test "hex_decode32!/2 ignores incorrect padding when :padding is false" do + "foob" = hex_decode32!("CPNMUOG", padding: false) + end + + test "hex_decode32/2 with :lower case and ignoring padding" do + assert {:ok, "fo"} == hex_decode32("cpng", case: :lower, padding: false) + end + + test "hex_decode32!/2 with :lower case and ignoring padding" do + assert "fo" == hex_decode32!("cpng", case: :lower, padding: false) + end + + test "hex_decode32/2 with :mixed case and ignoring padding" do + assert {:ok, "fo"} == hex_decode32("cPNg====", case: :mixed, padding: false) + end + + test "hex_decode32!/2 with :mixed case and ignoring padding" do + assert "fo" == hex_decode32!("cPNg", case: :mixed, padding: false) + end + + test "encode then decode is identity" do + for {encode, decode} <- [ + {&encode16/2, &decode16!/2}, + {&encode32/2, &decode32!/2}, + {&hex_encode32/2, &hex_decode32!/2}, + {&encode64/2, &decode64!/2}, + {&url_encode64/2, &url_decode64!/2} + ], + encode_case <- [:upper, :lower], + decode_case <- [:upper, :lower, :mixed], + encode_case == decode_case or decode_case == :mixed, + pad? <- [true, false], + len <- 0..256 do + data = + 0 + |> :lists.seq(len - 1) + |> Enum.shuffle() + |> IO.iodata_to_binary() + + allowed_opts = + encode + |> Function.info() + |> Keyword.fetch!(:name) + |> case do + :encode16 -> [:case] + :encode64 -> [:padding] + :url_encode64 -> [:padding] + _ -> [:case, :padding] + end + + expected = + data + |> encode.(Keyword.take([case: encode_case, padding: pad?], allowed_opts)) + |> decode.(Keyword.take([case: decode_case, padding: pad?], allowed_opts)) + + assert data == expected, + "identity did not match for #{inspect(data)} when #{inspect(encode)} (#{encode_case})" + end + end end diff --git a/lib/elixir/test/elixir/behaviour_test.exs b/lib/elixir/test/elixir/behaviour_test.exs deleted file mode 100644 index 3ff575302d5..00000000000 --- a/lib/elixir/test/elixir/behaviour_test.exs +++ /dev/null @@ -1,64 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -defmodule BehaviourTest do - use ExUnit.Case, async: true - - defmodule Sample do - use Behaviour - - @doc "I should be first." - defcallback first(integer) :: integer - - @doc "Foo" - defcallback foo(atom(), binary) :: binary - - @doc "Bar" - defcallback bar(External.hello, my_var :: binary) :: binary - - defcallback guarded(my_var) :: my_var when my_var: binary - - defcallback orr(atom | integer) :: atom - - defcallback literal(123, {atom}, :atom, [integer], true) :: atom - - @doc "I should be last." - defmacrocallback last(integer) :: Macro.t - end - - test :docs do - docs = Sample.__behaviour__(:docs) - assert [ - {{:first, 1}, 10, :def, "I should be first."}, - {{:foo, 2}, 13, :def, "Foo"}, - {{:bar, 2}, 16, :def, "Bar"}, - {{:guarded, 1}, 18, :def, nil}, - {{:orr, 1}, 20, :def, nil}, - {{:literal, 5}, 22, :def, nil}, - {{:last, 1}, 25, :defmacro, "I should be last."} - ] = docs - end - - test :callbacks do - assert Sample.__behaviour__(:callbacks) == [first: 1, guarded: 1, "MACRO-last": 2, literal: 5, orr: 1, foo: 2, bar: 2] - end - - test :specs do - assert length(Keyword.get_values(Sample.module_info[:attributes], :callback)) == 7 - end - - test :default_is_not_supported do - assert_raise ArgumentError, fn -> - defmodule WithDefault do - use Behaviour - defcallback hello(num \\ 0 :: integer) :: integer - end - end - - assert_raise ArgumentError, fn -> - defmodule WithDefault do - use Behaviour - defcallback hello(num :: integer \\ 0) :: integer - end - end - end -end diff --git a/lib/elixir/test/elixir/bitwise_test.exs b/lib/elixir/test/elixir/bitwise_test.exs index 17872979d26..1ccc2cd5e4d 100644 --- a/lib/elixir/test/elixir/bitwise_test.exs +++ b/lib/elixir/test/elixir/bitwise_test.exs @@ -1,59 +1,48 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) -defmodule Bitwise.FunctionsTest do +defmodule BitwiseTest do use ExUnit.Case, async: true - use Bitwise, skip_operators: true - test :bnot do + import Bitwise + doctest Bitwise + + test "bnot/1" do assert bnot(1) == -2 end - test :band do + test "band/2" do assert band(1, 1) == 1 end - test :bor do + test "bor/2" do assert bor(0, 1) == 1 end - test :bxor do + test "bxor/2" do assert bxor(1, 1) == 0 end - test :bsl do + test "bsl/2" do assert bsl(1, 1) == 2 end - test :bsr do + test "bsr/2" do assert bsr(1, 1) == 0 end -end - -defmodule Bitwise.OperatorsTest do - use ExUnit.Case, async: true - use Bitwise, only_operators: true - - test :bnot do - assert ~~~1 == -2 - end - test :band do + test "band (&&&)" do assert (1 &&& 1) == 1 end - test :bor do + test "bor (|||)" do assert (0 ||| 1) == 1 end - test :bxor do - assert 1 ^^^ 1 == 0 - end - - test :bsl do - assert (1 <<< 1) == 2 + test "bsl (<<<)" do + assert 1 <<< 1 == 2 end - test :bsr do - assert (1 >>> 1) == 0 + test "bsr (>>>)" do + assert 1 >>> 1 == 0 end end diff --git a/lib/elixir/test/elixir/calendar/date_range_test.exs b/lib/elixir/test/elixir/calendar/date_range_test.exs new file mode 100644 index 00000000000..b8641341732 --- /dev/null +++ b/lib/elixir/test/elixir/calendar/date_range_test.exs @@ -0,0 +1,213 @@ +Code.require_file("../test_helper.exs", __DIR__) +Code.require_file("holocene.exs", __DIR__) + +defmodule Date.RangeTest do + use ExUnit.Case, async: true + + @asc_range Date.range(~D[2000-01-01], ~D[2001-01-01]) + @asc_range_2 Date.range(~D[2000-01-01], ~D[2001-01-01], 2) + @desc_range Date.range(~D[2001-01-01], ~D[2000-01-01]) + @desc_range_2 Date.range(~D[2001-01-01], ~D[2000-01-01], -2) + @empty_range Date.range(~D[2001-01-01], ~D[2000-01-01], 1) + + describe "Enum.member?/2" do + test "for ascending range" do + assert Enum.member?(@asc_range, ~D[2000-02-22]) + assert Enum.member?(@asc_range, ~D[2000-01-01]) + assert Enum.member?(@asc_range, ~D[2001-01-01]) + refute Enum.member?(@asc_range, ~D[2002-01-01]) + refute Enum.member?(@asc_range, Calendar.Holocene.date(12000, 1, 1)) + + assert Enum.member?(@asc_range_2, ~D[2000-01-03]) + refute Enum.member?(@asc_range_2, ~D[2000-01-02]) + end + + test "for descending range" do + assert Enum.member?(@desc_range, ~D[2000-02-22]) + assert Enum.member?(@desc_range, ~D[2000-01-01]) + assert Enum.member?(@desc_range, ~D[2001-01-01]) + refute Enum.member?(@desc_range, ~D[1999-01-01]) + refute Enum.member?(@desc_range, Calendar.Holocene.date(12000, 1, 1)) + + assert Enum.member?(@desc_range_2, ~D[2000-12-30]) + refute Enum.member?(@desc_range_2, ~D[2000-12-29]) + end + + test "empty range" do + refute Enum.member?(@empty_range, @empty_range.first) + end + end + + describe "Enum.count/1" do + test "for ascending range" do + assert Enum.count(@asc_range) == 367 + assert Enum.count(@asc_range_2) == 184 + end + + test "for descending range" do + assert Enum.count(@desc_range) == 367 + assert Enum.count(@desc_range_2) == 184 + end + + test "for empty range" do + assert Enum.count(@empty_range) == 0 + end + end + + describe "Enum.slice/3" do + test "for ascending range" do + assert Enum.slice(@asc_range, 3, 3) == [~D[2000-01-04], ~D[2000-01-05], ~D[2000-01-06]] + assert Enum.slice(@asc_range, -3, 3) == [~D[2000-12-30], ~D[2000-12-31], ~D[2001-01-01]] + + assert Enum.slice(@asc_range_2, 3, 3) == [~D[2000-01-07], ~D[2000-01-09], ~D[2000-01-11]] + assert Enum.slice(@asc_range_2, -3, 3) == [~D[2000-12-28], ~D[2000-12-30], ~D[2001-01-01]] + end + + test "for descending range" do + assert Enum.slice(@desc_range, 3, 3) == [~D[2000-12-29], ~D[2000-12-28], ~D[2000-12-27]] + assert Enum.slice(@desc_range, -3, 3) == [~D[2000-01-03], ~D[2000-01-02], ~D[2000-01-01]] + + assert Enum.slice(@desc_range_2, 3, 3) == [~D[2000-12-26], ~D[2000-12-24], ~D[2000-12-22]] + assert Enum.slice(@desc_range_2, -3, 3) == [~D[2000-01-05], ~D[2000-01-03], ~D[2000-01-01]] + end + + test "for empty range" do + assert Enum.slice(@empty_range, 1, 3) == [] + assert Enum.slice(@empty_range, 3, 3) == [] + assert Enum.slice(@empty_range, -1, 3) == [] + assert Enum.slice(@empty_range, -3, 3) == [] + end + end + + describe "Enum.reduce/3" do + test "for ascending range" do + assert Enum.take(@asc_range, 3) == [~D[2000-01-01], ~D[2000-01-02], ~D[2000-01-03]] + + assert Enum.take(@asc_range_2, 3) == [~D[2000-01-01], ~D[2000-01-03], ~D[2000-01-05]] + end + + test "for descending range" do + assert Enum.take(@desc_range, 3) == [~D[2001-01-01], ~D[2000-12-31], ~D[2000-12-30]] + + assert Enum.take(@desc_range_2, 3) == [~D[2001-01-01], ~D[2000-12-30], ~D[2000-12-28]] + end + + test "for empty range" do + assert Enum.take(@empty_range, 3) == [] + end + end + + test "works with date-like structs" do + range = Date.range(~N[2000-01-01 09:00:00], ~U[2000-01-02 09:00:00Z]) + assert range.first == ~D[2000-01-01] + assert range.last == ~D[2000-01-02] + assert Enum.to_list(range) == [~D[2000-01-01], ~D[2000-01-02]] + + range = Date.range(~N[2000-01-01 09:00:00], ~U[2000-01-03 09:00:00Z], 2) + assert range.first == ~D[2000-01-01] + assert range.last == ~D[2000-01-03] + assert Enum.to_list(range) == [~D[2000-01-01], ~D[2000-01-03]] + end + + test "both dates must have matching calendars" do + first = ~D[2000-01-01] + last = Calendar.Holocene.date(12001, 1, 1) + + assert_raise ArgumentError, "both dates must have matching calendars", fn -> + Date.range(first, last) + end + end + + test "accepts equal but non Calendar.ISO calendars" do + first = Calendar.Holocene.date(12000, 1, 1) + last = Calendar.Holocene.date(12001, 1, 1) + range = Date.range(first, last) + assert range + assert first in range + assert last in range + assert Enum.count(range) == 367 + end + + test "step is a non-zero integer" do + step = 1.0 + message = ~r"the step must be a non-zero integer" + + assert_raise ArgumentError, message, fn -> + Date.range(~D[2000-01-01], ~D[2000-01-31], step) + end + + step = 0 + message = ~r"the step must be a non-zero integer" + + assert_raise ArgumentError, message, fn -> + Date.range(~D[2000-01-01], ~D[2000-01-31], step) + end + end + + describe "old date ranges" do + test "inspect" do + asc = %{ + __struct__: Date.Range, + first: ~D[2021-07-14], + first_in_iso_days: 738_350, + last: ~D[2021-07-17], + last_in_iso_days: 738_353 + } + + desc = %{ + __struct__: Date.Range, + first: ~D[2021-07-17], + first_in_iso_days: 738_353, + last: ~D[2021-07-14], + last_in_iso_days: 738_350 + } + + assert inspect(asc) == "Date.range(~D[2021-07-14], ~D[2021-07-17])" + assert inspect(desc) == "Date.range(~D[2021-07-17], ~D[2021-07-14], -1)" + end + + test "enumerable" do + asc = %{ + __struct__: Date.Range, + first: ~D[2021-07-14], + first_in_iso_days: 738_350, + last: ~D[2021-07-17], + last_in_iso_days: 738_353 + } + + desc = %{ + __struct__: Date.Range, + first: ~D[2021-07-17], + first_in_iso_days: 738_353, + last: ~D[2021-07-14], + last_in_iso_days: 738_350 + } + + # member? implementations tests also empty? + assert Enumerable.member?(asc, ~D[2021-07-15]) + assert {:ok, 4, _} = Enumerable.slice(asc) + + assert Enum.reduce(asc, [], fn x, acc -> [x | acc] end) == [ + ~D[2021-07-17], + ~D[2021-07-16], + ~D[2021-07-15], + ~D[2021-07-14] + ] + + assert Enum.count(asc) == 4 + + # member? implementations tests also empty? + assert Enumerable.member?(desc, ~D[2021-07-15]) + assert {:ok, 4, _} = Enumerable.slice(desc) + + assert Enum.reduce(desc, [], fn x, acc -> [x | acc] end) == [ + ~D[2021-07-14], + ~D[2021-07-15], + ~D[2021-07-16], + ~D[2021-07-17] + ] + + assert Enum.count(desc) == 4 + end + end +end diff --git a/lib/elixir/test/elixir/calendar/date_test.exs b/lib/elixir/test/elixir/calendar/date_test.exs new file mode 100644 index 00000000000..763c11a6a25 --- /dev/null +++ b/lib/elixir/test/elixir/calendar/date_test.exs @@ -0,0 +1,171 @@ +Code.require_file("../test_helper.exs", __DIR__) +Code.require_file("holocene.exs", __DIR__) +Code.require_file("fakes.exs", __DIR__) + +defmodule DateTest do + use ExUnit.Case, async: true + doctest Date + + test "sigil_D" do + assert ~D[2000-01-01] == + %Date{calendar: Calendar.ISO, year: 2000, month: 1, day: 1} + + assert ~D[20001-01-01 Calendar.Holocene] == + %Date{calendar: Calendar.Holocene, year: 20001, month: 1, day: 1} + + assert_raise ArgumentError, + ~s/cannot parse "2000-50-50" as Date for Calendar.ISO, reason: :invalid_date/, + fn -> Code.eval_string("~D[2000-50-50]") end + + assert_raise ArgumentError, + ~s/cannot parse "2000-04-15 notalias" as Date for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~D[2000-04-15 notalias]") end + + assert_raise ArgumentError, + ~s/cannot parse "20010415" as Date for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string(~s{~D[20010415]}) end + + assert_raise ArgumentError, + ~s/cannot parse "20001-50-50" as Date for Calendar.Holocene, reason: :invalid_date/, + fn -> Code.eval_string("~D[20001-50-50 Calendar.Holocene]") end + + assert_raise UndefinedFunctionError, fn -> + Code.eval_string("~D[2000-01-01 UnknownCalendar]") + end + end + + test "to_string/1" do + date = ~D[2000-01-01] + assert to_string(date) == "2000-01-01" + assert Date.to_string(date) == "2000-01-01" + assert Date.to_string(Map.from_struct(date)) == "2000-01-01" + + assert to_string(%{date | calendar: FakeCalendar}) == "1/1/2000" + assert Date.to_string(%{date | calendar: FakeCalendar}) == "1/1/2000" + end + + test "inspect/1" do + assert inspect(~D[2000-01-01]) == "~D[2000-01-01]" + assert inspect(~D[-0100-12-31]) == "~D[-0100-12-31]" + + date = %{~D[2000-01-01] | calendar: FakeCalendar} + assert inspect(date) == "~D[1/1/2000 FakeCalendar]" + end + + test "compare/2" do + date1 = ~D[-0001-12-30] + date2 = ~D[-0001-12-31] + date3 = ~D[0001-01-01] + assert Date.compare(date1, date1) == :eq + assert Date.compare(date1, date2) == :lt + assert Date.compare(date2, date1) == :gt + assert Date.compare(date3, date3) == :eq + assert Date.compare(date2, date3) == :lt + assert Date.compare(date3, date2) == :gt + end + + test "compare/2 across calendars" do + date1 = ~D[2000-01-01] + date2 = Calendar.Holocene.date(12000, 01, 01) + assert Date.compare(date1, date2) == :eq + + date2 = Calendar.Holocene.date(12001, 01, 01) + assert Date.compare(date1, date2) == :lt + assert Date.compare(date2, date1) == :gt + end + + test "day_of_week/1" do + assert Date.day_of_week(Calendar.Holocene.date(2016, 10, 31)) == 1 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 01)) == 2 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 02)) == 3 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 03)) == 4 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 04)) == 5 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 05)) == 6 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 06)) == 7 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 07)) == 1 + + assert Date.day_of_week(Calendar.Holocene.date(2016, 10, 30), :sunday) == 1 + assert Date.day_of_week(Calendar.Holocene.date(2016, 10, 31), :sunday) == 2 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 01), :sunday) == 3 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 02), :sunday) == 4 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 03), :sunday) == 5 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 04), :sunday) == 6 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 05), :sunday) == 7 + assert Date.day_of_week(Calendar.Holocene.date(2016, 11, 06), :sunday) == 1 + end + + test "beginning_of_week" do + assert Date.beginning_of_week(Calendar.Holocene.date(2020, 07, 11)) == + Calendar.Holocene.date(2020, 07, 06) + + assert Date.beginning_of_week(Calendar.Holocene.date(2020, 07, 06)) == + Calendar.Holocene.date(2020, 07, 06) + + assert Date.beginning_of_week(Calendar.Holocene.date(2020, 07, 11), :sunday) == + Calendar.Holocene.date(2020, 07, 05) + + assert Date.beginning_of_week(Calendar.Holocene.date(2020, 07, 11), :saturday) == + Calendar.Holocene.date(2020, 07, 11) + end + + test "end_of_week" do + assert Date.end_of_week(Calendar.Holocene.date(2020, 07, 11)) == + Calendar.Holocene.date(2020, 07, 12) + + assert Date.end_of_week(Calendar.Holocene.date(2020, 07, 05)) == + Calendar.Holocene.date(2020, 07, 05) + + assert Date.end_of_week(Calendar.Holocene.date(2020, 07, 05), :sunday) == + Calendar.Holocene.date(2020, 07, 11) + + assert Date.end_of_week(Calendar.Holocene.date(2020, 07, 05), :saturday) == + Calendar.Holocene.date(2020, 07, 10) + end + + test "convert/2" do + assert Date.convert(~D[2000-01-01], Calendar.Holocene) == + {:ok, Calendar.Holocene.date(12000, 01, 01)} + + assert ~D[2000-01-01] + |> Date.convert!(Calendar.Holocene) + |> Date.convert!(Calendar.ISO) == ~D[2000-01-01] + + assert Date.convert(~D[2016-02-03], FakeCalendar) == {:error, :incompatible_calendars} + + assert Date.convert(~N[2000-01-01 00:00:00], Calendar.Holocene) == + {:ok, Calendar.Holocene.date(12000, 01, 01)} + end + + test "add/2" do + assert Date.add(~D[0000-01-01], 3_652_424) == ~D[9999-12-31] + + assert_raise FunctionClauseError, fn -> + Date.add(~D[0000-01-01], 3_652_425) + end + + assert Date.add(~D[0000-01-01], -1) == ~D[-0001-12-31] + assert Date.add(~D[0000-01-01], -365) == ~D[-0001-01-01] + assert Date.add(~D[0000-01-01], -366) == ~D[-0002-12-31] + assert Date.add(~D[0000-01-01], -(365 * 4)) == ~D[-0004-01-02] + assert Date.add(~D[0000-01-01], -(365 * 5)) == ~D[-0005-01-02] + assert Date.add(~D[0000-01-01], -(365 * 100)) == ~D[-0100-01-25] + assert Date.add(~D[0000-01-01], -3_652_059) == ~D[-9999-01-01] + + assert_raise FunctionClauseError, fn -> + Date.add(~D[0000-01-01], -3_652_060) + end + end + + test "diff/2" do + assert Date.diff(~D[2000-01-31], ~D[2000-01-01]) == 30 + assert Date.diff(~D[2000-01-01], ~D[2000-01-31]) == -30 + + assert Date.diff(~D[0000-01-01], ~D[-0001-01-01]) == 365 + assert Date.diff(~D[-0003-01-01], ~D[-0004-01-01]) == 366 + + date1 = ~D[2000-01-01] + date2 = Calendar.Holocene.date(12000, 01, 14) + assert Date.diff(date1, date2) == -13 + assert Date.diff(date2, date1) == 13 + end +end diff --git a/lib/elixir/test/elixir/calendar/datetime_test.exs b/lib/elixir/test/elixir/calendar/datetime_test.exs new file mode 100644 index 00000000000..d517f4599df --- /dev/null +++ b/lib/elixir/test/elixir/calendar/datetime_test.exs @@ -0,0 +1,1043 @@ +Code.require_file("../test_helper.exs", __DIR__) +Code.require_file("holocene.exs", __DIR__) +Code.require_file("fakes.exs", __DIR__) + +defmodule DateTimeTest do + use ExUnit.Case + doctest DateTime + + test "sigil_U" do + assert ~U[2000-01-01T12:34:56Z] == + %DateTime{ + calendar: Calendar.ISO, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56, + std_offset: 0, + utc_offset: 0, + time_zone: "Etc/UTC", + zone_abbr: "UTC" + } + + assert ~U[2000-01-01T12:34:56+00:00 Calendar.Holocene] == + %DateTime{ + calendar: Calendar.Holocene, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56, + std_offset: 0, + utc_offset: 0, + time_zone: "Etc/UTC", + zone_abbr: "UTC" + } + + assert ~U[2000-01-01 12:34:56+00:00] == + %DateTime{ + calendar: Calendar.ISO, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56, + std_offset: 0, + utc_offset: 0, + time_zone: "Etc/UTC", + zone_abbr: "UTC" + } + + assert ~U[2000-01-01 12:34:56Z Calendar.Holocene] == + %DateTime{ + calendar: Calendar.Holocene, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56, + std_offset: 0, + utc_offset: 0, + time_zone: "Etc/UTC", + zone_abbr: "UTC" + } + + assert_raise ArgumentError, + ~s/cannot parse "2001-50-50T12:34:56Z" as UTC DateTime for Calendar.ISO, reason: :invalid_date/, + fn -> Code.eval_string("~U[2001-50-50T12:34:56Z]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:65Z" as UTC DateTime for Calendar.ISO, reason: :invalid_time/, + fn -> Code.eval_string("~U[2001-01-01T12:34:65Z]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:56\+01:00" as UTC DateTime for Calendar.ISO, reason: :non_utc_offset/, + fn -> Code.eval_string("~U[2001-01-01T12:34:56+01:00]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01 12:34:56Z notalias" as UTC DateTime for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~U[2001-01-01 12:34:56Z notalias]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:56Z notalias" as UTC DateTime for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~U[2001-01-01T12:34:56Z notalias]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-50-50T12:34:56Z" as UTC DateTime for Calendar.Holocene, reason: :invalid_date/, + fn -> Code.eval_string("~U[2001-50-50T12:34:56Z Calendar.Holocene]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:65Z" as UTC DateTime for Calendar.Holocene, reason: :invalid_time/, + fn -> Code.eval_string("~U[2001-01-01T12:34:65Z Calendar.Holocene]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:56+01:00 Calendar.Holocene" as UTC DateTime for Calendar.Holocene, reason: :non_utc_offset/, + fn -> Code.eval_string("~U[2001-01-01T12:34:56+01:00 Calendar.Holocene]") end + + assert_raise UndefinedFunctionError, fn -> + Code.eval_string("~U[2001-01-01 12:34:56 UnknownCalendar]") + end + + assert_raise UndefinedFunctionError, fn -> + Code.eval_string("~U[2001-01-01T12:34:56 UnknownCalendar]") + end + end + + test "to_string/1" do + datetime = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "BRM", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: -12600, + std_offset: 3600, + time_zone: "Brazil/Manaus" + } + + assert to_string(datetime) == "2000-02-29 23:00:07-02:30 BRM Brazil/Manaus" + assert DateTime.to_string(datetime) == "2000-02-29 23:00:07-02:30 BRM Brazil/Manaus" + + assert DateTime.to_string(Map.from_struct(datetime)) == + "2000-02-29 23:00:07-02:30 BRM Brazil/Manaus" + + assert to_string(%{datetime | calendar: FakeCalendar}) == + "29/2/2000F23::0::7 Brazil/Manaus BRM -12600 3600" + + assert DateTime.to_string(%{datetime | calendar: FakeCalendar}) == + "29/2/2000F23::0::7 Brazil/Manaus BRM -12600 3600" + end + + test "inspect/1" do + utc_datetime = ~U[2000-01-01 23:00:07.005Z] + assert inspect(utc_datetime) == "~U[2000-01-01 23:00:07.005Z]" + + assert inspect(%{utc_datetime | calendar: FakeCalendar}) == + "~U[1/1/2000F23::0::7 Etc/UTC UTC 0 0 FakeCalendar]" + + datetime = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "BRM", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: -12600, + std_offset: 3600, + time_zone: "Brazil/Manaus" + } + + assert inspect(datetime) == "#DateTime<2000-02-29 23:00:07-02:30 BRM Brazil/Manaus>" + + assert inspect(%{datetime | calendar: FakeCalendar}) == + "#DateTime<29/2/2000F23::0::7 Brazil/Manaus BRM -12600 3600 FakeCalendar>" + end + + test "from_iso8601/1 handles positive and negative offsets" do + assert DateTime.from_iso8601("2015-01-24T09:50:07-10:00") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 1, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 2015, + zone_abbr: "UTC", + day: 24, + hour: 19, + minute: 50, + second: 7 + } + + assert DateTime.from_iso8601("2015-01-24T09:50:07+10:00") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 1, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 2015, + zone_abbr: "UTC", + day: 23, + hour: 23, + minute: 50, + second: 7 + } + + assert DateTime.from_iso8601("0000-01-01T01:22:07+10:30") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 12, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: -1, + zone_abbr: "UTC", + day: 31, + hour: 14, + minute: 52, + second: 7 + } + end + + test "from_iso8601/1 handles negative dates" do + assert DateTime.from_iso8601("-2015-01-24T09:50:07-10:00") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 1, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: -2015, + zone_abbr: "UTC", + day: 24, + hour: 19, + minute: 50, + second: 7 + } + + assert DateTime.from_iso8601("-2015-01-24T09:50:07+10:00") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 1, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: -2015, + zone_abbr: "UTC", + day: 23, + hour: 23, + minute: 50, + second: 7 + } + + assert DateTime.from_iso8601("-0001-01-01T01:22:07+10:30") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 12, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: -2, + zone_abbr: "UTC", + day: 31, + hour: 14, + minute: 52, + second: 7 + } + + assert DateTime.from_iso8601("-0001-01-01T01:22:07-10:30") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 1, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: -1, + zone_abbr: "UTC", + day: 1, + hour: 11, + minute: 52, + second: 7 + } + + assert DateTime.from_iso8601("-0001-12-31T23:22:07-10:30") |> elem(1) == + %DateTime{ + microsecond: {0, 0}, + month: 1, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 0, + zone_abbr: "UTC", + day: 1, + hour: 9, + minute: 52, + second: 7 + } + end + + test "from_iso8601/3 with basic format handles positive and negative offsets" do + assert DateTime.from_iso8601("20150124T095007-1000", Calendar.ISO, :basic) == + DateTime.from_iso8601("2015-01-24T09:50:07-10:00", Calendar.ISO) + + assert DateTime.from_iso8601("20150124T095007+1000", Calendar.ISO, :basic) == + DateTime.from_iso8601("2015-01-24T09:50:07+10:00", Calendar.ISO) + + assert DateTime.from_iso8601("00000101T012207+1030", Calendar.ISO, :basic) == + DateTime.from_iso8601("0000-01-01T01:22:07+10:30", Calendar.ISO) + end + + test "from_iso8601/3 with basic format handles negative dates" do + assert DateTime.from_iso8601("-20150124T095007-1000", Calendar.ISO, :basic) == + DateTime.from_iso8601("-2015-01-24T09:50:07-10:00", Calendar.ISO) + + assert DateTime.from_iso8601("-20150124T095007+1000", Calendar.ISO, :basic) == + DateTime.from_iso8601("-2015-01-24T09:50:07+10:00", Calendar.ISO) + + assert DateTime.from_iso8601("-00010101T012207+1030", Calendar.ISO, :basic) == + DateTime.from_iso8601("-0001-01-01T01:22:07+10:30", Calendar.ISO) + + assert DateTime.from_iso8601("-00010101T012207-1030", Calendar.ISO, :basic) == + DateTime.from_iso8601("-0001-01-01T01:22:07-10:30", Calendar.ISO) + + assert DateTime.from_iso8601("-00011231T232207-1030", Calendar.ISO, :basic) == + DateTime.from_iso8601("-0001-12-31T23:22:07-10:30", Calendar.ISO) + end + + test "from_iso8601/2 handles either a calendar or a format as the second parameter" do + assert DateTime.from_iso8601("20150124T095007-1000", :basic) == + DateTime.from_iso8601("2015-01-24T09:50:07-10:00", Calendar.ISO) + end + + test "from_iso8601 handles invalid date, time, formats correctly" do + assert DateTime.from_iso8601("2015-01-23T23:50:07") == {:error, :missing_offset} + assert DateTime.from_iso8601("2015-01-23 23:50:61") == {:error, :invalid_time} + assert DateTime.from_iso8601("2015-01-32 23:50:07") == {:error, :invalid_date} + assert DateTime.from_iso8601("2015-01-23 23:50:07A") == {:error, :invalid_format} + assert DateTime.from_iso8601("2015-01-23T23:50:07.123-00:60") == {:error, :invalid_format} + + assert DateTime.from_iso8601("20150123T235007", Calendar.ISO, :basic) == + {:error, :missing_offset} + + assert DateTime.from_iso8601("20150123 235061", Calendar.ISO, :basic) == + {:error, :invalid_time} + + assert DateTime.from_iso8601("20150132 235007", Calendar.ISO, :basic) == + {:error, :invalid_date} + + assert DateTime.from_iso8601("20150123 235007A", Calendar.ISO, :basic) == + {:error, :invalid_format} + + assert DateTime.from_iso8601("2015-01-24T09:50:07-10:00", Calendar.ISO, :basic) == + {:error, :invalid_format} + + assert DateTime.from_iso8601("20150123T235007.123-0060", Calendar.ISO, :basic) == + {:error, :invalid_format} + end + + test "from_unix/2" do + min_datetime = %DateTime{ + calendar: Calendar.ISO, + day: 1, + hour: 0, + microsecond: {0, 0}, + minute: 0, + month: 1, + second: 0, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: -9999, + zone_abbr: "UTC" + } + + assert DateTime.from_unix(-377_705_116_800) == {:ok, min_datetime} + + assert DateTime.from_unix(-377_705_116_800_000_001, :microsecond) == + {:error, :invalid_unix_time} + + assert DateTime.from_unix(143_256_036_886_856, 1024) == + {:ok, + %DateTime{ + calendar: Calendar.ISO, + day: 17, + hour: 7, + microsecond: {320_312, 6}, + minute: 5, + month: 3, + second: 22, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 6403, + zone_abbr: "UTC" + }} + + max_datetime = %DateTime{ + calendar: Calendar.ISO, + day: 31, + hour: 23, + microsecond: {999_999, 6}, + minute: 59, + month: 12, + second: 59, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 9999, + zone_abbr: "UTC" + } + + assert DateTime.from_unix(253_402_300_799_999_999, :microsecond) == {:ok, max_datetime} + + assert DateTime.from_unix(253_402_300_800) == {:error, :invalid_unix_time} + + minus_datetime = %DateTime{ + calendar: Calendar.ISO, + day: 31, + hour: 23, + microsecond: {999_999, 6}, + minute: 59, + month: 12, + second: 59, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 1969, + zone_abbr: "UTC" + } + + assert DateTime.from_unix(-1, :microsecond) == {:ok, minus_datetime} + + assert_raise ArgumentError, fn -> + DateTime.from_unix(0, :unknown_atom) + end + + assert_raise ArgumentError, fn -> + DateTime.from_unix(0, "invalid type") + end + end + + test "from_unix!/2" do + # with Unix times back to 0 Gregorian seconds + datetime = %DateTime{ + calendar: Calendar.ISO, + day: 1, + hour: 0, + microsecond: {0, 0}, + minute: 0, + month: 1, + second: 0, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 0, + zone_abbr: "UTC" + } + + assert DateTime.from_unix!(-62_167_219_200) == datetime + + assert_raise ArgumentError, fn -> + DateTime.from_unix!(-377_705_116_801) + end + + assert_raise ArgumentError, fn -> + DateTime.from_unix!(0, :unknown_atom) + end + + assert_raise ArgumentError, fn -> + DateTime.from_unix!(0, "invalid type") + end + end + + test "to_unix/2 works with Unix times back to 0 Gregorian seconds" do + # with Unix times back to 0 Gregorian seconds + gregorian_0 = %DateTime{ + calendar: Calendar.ISO, + day: 1, + hour: 0, + microsecond: {0, 0}, + minute: 0, + month: 1, + second: 0, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 0, + zone_abbr: "UTC" + } + + assert DateTime.to_unix(gregorian_0) == -62_167_219_200 + assert DateTime.to_unix(Map.from_struct(gregorian_0)) == -62_167_219_200 + + min_datetime = %DateTime{gregorian_0 | year: -9999} + + assert DateTime.to_unix(min_datetime) == -377_705_116_800 + end + + test "compare/2" do + datetime1 = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + datetime2 = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "AMT", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: -14400, + std_offset: 0, + time_zone: "America/Manaus" + } + + datetime3 = %DateTime{ + year: -99, + month: 2, + day: 28, + zone_abbr: "AMT", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: -14400, + std_offset: 0, + time_zone: "America/Manaus" + } + + assert DateTime.compare(datetime1, datetime1) == :eq + assert DateTime.compare(datetime1, datetime2) == :lt + assert DateTime.compare(datetime2, datetime1) == :gt + assert DateTime.compare(datetime3, datetime3) == :eq + assert DateTime.compare(datetime2, datetime3) == :gt + assert DateTime.compare(datetime3, datetime1) == :lt + assert DateTime.compare(Map.from_struct(datetime3), Map.from_struct(datetime1)) == :lt + end + + test "convert/2" do + datetime_iso = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + datetime_hol = %DateTime{ + year: 12000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw", + calendar: Calendar.Holocene + } + + assert DateTime.convert(datetime_iso, Calendar.Holocene) == {:ok, datetime_hol} + + assert datetime_iso + |> DateTime.convert!(Calendar.Holocene) + |> DateTime.convert!(Calendar.ISO) == datetime_iso + + assert %{datetime_iso | microsecond: {123, 6}} + |> DateTime.convert!(Calendar.Holocene) + |> DateTime.convert!(Calendar.ISO) == %{datetime_iso | microsecond: {123, 6}} + + assert DateTime.convert(datetime_iso, FakeCalendar) == {:error, :incompatible_calendars} + + # Test passing non-struct map when converting to different calendar returns DateTime struct + assert DateTime.convert(Map.from_struct(datetime_iso), Calendar.Holocene) == + {:ok, datetime_hol} + + # Test passing non-struct map when converting to same calendar returns DateTime struct + assert DateTime.convert(Map.from_struct(datetime_iso), Calendar.ISO) == + {:ok, datetime_iso} + end + + test "from_iso8601/1 with tz offsets" do + assert DateTime.from_iso8601("2017-06-02T14:00:00+01:00") + |> elem(1) == + %DateTime{ + year: 2017, + month: 6, + day: 2, + zone_abbr: "UTC", + hour: 13, + minute: 0, + second: 0, + microsecond: {0, 0}, + utc_offset: 0, + std_offset: 0, + time_zone: "Etc/UTC" + } + + assert DateTime.from_iso8601("2017-06-02T14:00:00-04:00") + |> elem(1) == + %DateTime{ + year: 2017, + month: 6, + day: 2, + zone_abbr: "UTC", + hour: 18, + minute: 0, + second: 0, + microsecond: {0, 0}, + utc_offset: 0, + std_offset: 0, + time_zone: "Etc/UTC" + } + + assert DateTime.from_iso8601("2017-06-02T14:00:00+0100") + |> elem(1) == + %DateTime{ + year: 2017, + month: 6, + day: 2, + zone_abbr: "UTC", + hour: 13, + minute: 0, + second: 0, + microsecond: {0, 0}, + utc_offset: 0, + std_offset: 0, + time_zone: "Etc/UTC" + } + + assert DateTime.from_iso8601("2017-06-02T14:00:00-0400") + |> elem(1) == + %DateTime{ + year: 2017, + month: 6, + day: 2, + zone_abbr: "UTC", + hour: 18, + minute: 0, + second: 0, + microsecond: {0, 0}, + utc_offset: 0, + std_offset: 0, + time_zone: "Etc/UTC" + } + + assert DateTime.from_iso8601("2017-06-02T14:00:00+01") + |> elem(1) == + %DateTime{ + year: 2017, + month: 6, + day: 2, + zone_abbr: "UTC", + hour: 13, + minute: 0, + second: 0, + microsecond: {0, 0}, + utc_offset: 0, + std_offset: 0, + time_zone: "Etc/UTC" + } + + assert DateTime.from_iso8601("2017-06-02T14:00:00-04") + |> elem(1) == + %DateTime{ + year: 2017, + month: 6, + day: 2, + zone_abbr: "UTC", + hour: 18, + minute: 0, + second: 0, + microsecond: {0, 0}, + utc_offset: 0, + std_offset: 0, + time_zone: "Etc/UTC" + } + end + + test "from_iso8601/3 with basic format with tz offsets" do + assert DateTime.from_iso8601("20170602T140000+0100", Calendar.ISO, :basic) == + DateTime.from_iso8601("2017-06-02T14:00:00+01:00", Calendar.ISO) + + assert DateTime.from_iso8601("20170602T140000-0400", Calendar.ISO, :basic) == + DateTime.from_iso8601("2017-06-02T14:00:00-04:00") + + assert DateTime.from_iso8601("20170602T140000+01", Calendar.ISO, :basic) == + DateTime.from_iso8601("2017-06-02T14:00:00+01") + + assert DateTime.from_iso8601("20170602T140000-04", Calendar.ISO, :basic) == + DateTime.from_iso8601("2017-06-02T14:00:00-04") + end + + test "truncate/2" do + datetime = %DateTime{ + year: 2017, + month: 11, + day: 6, + zone_abbr: "CET", + hour: 0, + minute: 6, + second: 23, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Paris" + } + + datetime_map = Map.from_struct(datetime) + + assert DateTime.truncate(%{datetime | microsecond: {123_456, 6}}, :microsecond) == + %{datetime | microsecond: {123_456, 6}} + + # A struct should be returned when passing a map. + assert DateTime.truncate(%{datetime_map | microsecond: {123_456, 6}}, :microsecond) == + %{datetime | microsecond: {123_456, 6}} + + assert DateTime.truncate(%{datetime | microsecond: {0, 0}}, :millisecond) == + %{datetime | microsecond: {0, 0}} + + assert DateTime.truncate(%{datetime | microsecond: {000_100, 6}}, :millisecond) == + %{datetime | microsecond: {0, 3}} + + assert DateTime.truncate(%{datetime | microsecond: {000_999, 6}}, :millisecond) == + %{datetime | microsecond: {0, 3}} + + assert DateTime.truncate(%{datetime | microsecond: {001_000, 6}}, :millisecond) == + %{datetime | microsecond: {1000, 3}} + + assert DateTime.truncate(%{datetime | microsecond: {001_200, 6}}, :millisecond) == + %{datetime | microsecond: {1000, 3}} + + assert DateTime.truncate(%{datetime | microsecond: {123_456, 6}}, :millisecond) == + %{datetime | microsecond: {123_000, 3}} + + assert DateTime.truncate(%{datetime | microsecond: {123_456, 6}}, :second) == + %{datetime | microsecond: {0, 0}} + end + + test "diff/2" do + dt1 = %DateTime{ + year: 100, + month: 2, + day: 28, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + dt2 = %DateTime{ + year: -0004, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + assert DateTime.diff(dt1, dt2) == 3_281_904_000 + + # Test with a non-struct map conforming to Calendar.datetime + assert DateTime.diff(Map.from_struct(dt1), Map.from_struct(dt2)) == 3_281_904_000 + end + + describe "from_naive" do + test "uses default time zone database from config" do + Calendar.put_time_zone_database(FakeTimeZoneDatabase) + + assert DateTime.from_naive( + ~N[2018-07-01 12:34:25.123456], + "Europe/Copenhagen", + FakeTimeZoneDatabase + ) == + {:ok, + %DateTime{ + day: 1, + hour: 12, + microsecond: {123_456, 6}, + minute: 34, + month: 7, + second: 25, + std_offset: 3600, + time_zone: "Europe/Copenhagen", + utc_offset: 3600, + year: 2018, + zone_abbr: "CEST" + }} + after + Calendar.put_time_zone_database(Calendar.UTCOnlyTimeZoneDatabase) + end + + test "with compatible calendar on unambiguous wall clock" do + holocene_ndt = %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 12018, + month: 7, + day: 1, + hour: 12, + minute: 34, + second: 25, + microsecond: {123_456, 6} + } + + assert DateTime.from_naive(holocene_ndt, "Europe/Copenhagen", FakeTimeZoneDatabase) == + {:ok, + %DateTime{ + calendar: Calendar.Holocene, + day: 1, + hour: 12, + microsecond: {123_456, 6}, + minute: 34, + month: 7, + second: 25, + std_offset: 3600, + time_zone: "Europe/Copenhagen", + utc_offset: 3600, + year: 12018, + zone_abbr: "CEST" + }} + end + + test "with compatible calendar on ambiguous wall clock" do + holocene_ndt = %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 12018, + month: 10, + day: 28, + hour: 02, + minute: 30, + second: 00, + microsecond: {123_456, 6} + } + + assert {:ambiguous, first_dt, second_dt} = + DateTime.from_naive(holocene_ndt, "Europe/Copenhagen", FakeTimeZoneDatabase) + + assert %DateTime{calendar: Calendar.Holocene, zone_abbr: "CEST"} = first_dt + assert %DateTime{calendar: Calendar.Holocene, zone_abbr: "CET"} = second_dt + end + + test "with compatible calendar on gap" do + holocene_ndt = %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 12019, + month: 03, + day: 31, + hour: 02, + minute: 30, + second: 00, + microsecond: {123_456, 6} + } + + assert {:gap, first_dt, second_dt} = + DateTime.from_naive(holocene_ndt, "Europe/Copenhagen", FakeTimeZoneDatabase) + + assert %DateTime{calendar: Calendar.Holocene, zone_abbr: "CET"} = first_dt + assert %DateTime{calendar: Calendar.Holocene, zone_abbr: "CEST"} = second_dt + end + + test "with incompatible calendar" do + ndt = %{~N[2018-07-20 00:00:00] | calendar: FakeCalendar} + + assert DateTime.from_naive(ndt, "Europe/Copenhagen", FakeTimeZoneDatabase) == + {:error, :incompatible_calendars} + end + end + + describe "from_naive!" do + test "raises on ambiguous wall clock" do + assert_raise ArgumentError, ~r"ambiguous", fn -> + DateTime.from_naive!(~N[2018-10-28 02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + end + end + + test "raises on gap" do + assert_raise ArgumentError, ~r"gap", fn -> + DateTime.from_naive!(~N[2019-03-31 02:30:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + end + end + end + + describe "shift_zone" do + test "with compatible calendar" do + holocene_ndt = %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 12018, + month: 7, + day: 1, + hour: 12, + minute: 34, + second: 25, + microsecond: {123_456, 6} + } + + {:ok, holocene_dt} = + DateTime.from_naive(holocene_ndt, "Europe/Copenhagen", FakeTimeZoneDatabase) + + {:ok, dt} = DateTime.shift_zone(holocene_dt, "America/Los_Angeles", FakeTimeZoneDatabase) + + assert dt == %DateTime{ + calendar: Calendar.Holocene, + day: 1, + hour: 3, + microsecond: {123_456, 6}, + minute: 34, + month: 7, + second: 25, + std_offset: 3600, + time_zone: "America/Los_Angeles", + utc_offset: -28800, + year: 12018, + zone_abbr: "PDT" + } + end + + test "uses default time zone database from config" do + Calendar.put_time_zone_database(FakeTimeZoneDatabase) + + {:ok, dt} = DateTime.from_naive(~N[2018-07-01 12:34:25.123456], "Europe/Copenhagen") + {:ok, dt} = DateTime.shift_zone(dt, "America/Los_Angeles") + + assert dt == %DateTime{ + day: 1, + hour: 3, + microsecond: {123_456, 6}, + minute: 34, + month: 7, + second: 25, + std_offset: 3600, + time_zone: "America/Los_Angeles", + utc_offset: -28800, + year: 2018, + zone_abbr: "PDT" + } + after + Calendar.put_time_zone_database(Calendar.UTCOnlyTimeZoneDatabase) + end + end + + describe "add" do + test "add with non-struct map that conforms to Calendar.datetime" do + dt_map = DateTime.from_naive!(~N[2018-08-28 00:00:00], "Etc/UTC") |> Map.from_struct() + + assert DateTime.add(dt_map, 1, :second) == %DateTime{ + calendar: Calendar.ISO, + year: 2018, + month: 8, + day: 28, + hour: 0, + minute: 0, + second: 1, + std_offset: 0, + time_zone: "Etc/UTC", + zone_abbr: "UTC", + utc_offset: 0, + microsecond: {0, 0} + } + end + + test "error with UTC only database and non UTC datetime" do + dt = + DateTime.from_naive!(~N[2018-08-28 00:00:00], "Europe/Copenhagen", FakeTimeZoneDatabase) + + assert_raise ArgumentError, fn -> + DateTime.add(dt, 1, :second) + end + end + + test "add/2 with other calendars" do + assert ~N[2000-01-01 12:34:15.123456] + |> NaiveDateTime.convert!(Calendar.Holocene) + |> DateTime.from_naive!("Etc/UTC") + |> DateTime.add(10, :second) == + %DateTime{ + calendar: Calendar.Holocene, + year: 12000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 25, + std_offset: 0, + time_zone: "Etc/UTC", + zone_abbr: "UTC", + utc_offset: 0, + microsecond: {123_456, 6} + } + end + end + + describe "to_iso8601" do + test "to_iso8601/2 with a normal DateTime struct" do + datetime = DateTime.from_naive!(~N[2018-07-01 12:34:25.123456], "Etc/UTC") + + assert DateTime.to_iso8601(datetime) == "2018-07-01T12:34:25.123456Z" + end + + test "to_iso8601/2 with a non-struct map conforming to the Calendar.datetime type" do + datetime_map = + DateTime.from_naive!(~N[2018-07-01 12:34:25.123456], "Etc/UTC") |> Map.from_struct() + + assert DateTime.to_iso8601(datetime_map) == "2018-07-01T12:34:25.123456Z" + end + end + + describe "to_date/1" do + test "upcasting" do + assert catch_error(DateTime.to_date(~N[2000-02-29 12:23:34])) + end + end + + describe "to_time/1" do + test "upcasting" do + assert catch_error(DateTime.to_time(~N[2000-02-29 12:23:34])) + end + end + + describe "to_naive/1" do + test "upcasting" do + assert catch_error(DateTime.to_naive(~N[2000-02-29 12:23:34])) + end + end +end diff --git a/lib/elixir/test/elixir/calendar/fakes.exs b/lib/elixir/test/elixir/calendar/fakes.exs new file mode 100644 index 00000000000..3da151e0ac4 --- /dev/null +++ b/lib/elixir/test/elixir/calendar/fakes.exs @@ -0,0 +1,184 @@ +defmodule FakeCalendar do + def time_to_string(hour, minute, second, _), do: "#{hour}::#{minute}::#{second}" + def date_to_string(year, month, day), do: "#{day}/#{month}/#{year}" + + def naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) do + date_to_string(year, month, day) <> "F" <> time_to_string(hour, minute, second, microsecond) + end + + def datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + time_zone, + abbr, + utc_offset, + std_offset + ) do + naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) <> + " #{time_zone} #{abbr} #{utc_offset} #{std_offset}" + end + + def day_rollover_relative_to_midnight_utc, do: {123_456, 123_457} +end + +defmodule FakeTimeZoneDatabase do + @behaviour Calendar.TimeZoneDatabase + + @time_zone_period_cph_summer_2018 %{ + std_offset: 3600, + utc_offset: 3600, + zone_abbr: "CEST", + from_wall: ~N[2018-03-25 03:00:00], + until_wall: ~N[2018-10-28 03:00:00] + } + + @time_zone_period_cph_winter_2018_2019 %{ + std_offset: 0, + utc_offset: 3600, + zone_abbr: "CET", + from_wall: ~N[2018-10-28 02:00:00], + until_wall: ~N[2019-03-31 02:00:00] + } + + @time_zone_period_cph_summer_2019 %{ + std_offset: 3600, + utc_offset: 3600, + zone_abbr: "CEST", + from_wall: ~N[2019-03-31 03:00:00], + until_wall: ~N[2019-10-27 03:00:00] + } + + @spec time_zone_period_from_utc_iso_days(Calendar.iso_days(), Calendar.time_zone()) :: + {:ok, TimeZoneDatabase.time_zone_period()} | {:error, :time_zone_not_found} + @impl true + def time_zone_period_from_utc_iso_days(iso_days, time_zone) do + {:ok, ndt} = naive_datetime_from_iso_days(iso_days) + time_zone_periods_from_utc(time_zone, NaiveDateTime.to_erl(ndt)) + end + + @spec time_zone_periods_from_wall_datetime(Calendar.naive_datetime(), Calendar.time_zone()) :: + {:ok, TimeZoneDatabase.time_zone_period()} + | {:ambiguous, TimeZoneDatabase.time_zone_period(), TimeZoneDatabase.time_zone_period()} + | {:gap, + {TimeZoneDatabase.time_zone_period(), TimeZoneDatabase.time_zone_period_limit()}, + {TimeZoneDatabase.time_zone_period(), TimeZoneDatabase.time_zone_period_limit()}} + | {:error, :time_zone_not_found} + @impl true + def time_zone_periods_from_wall_datetime(naive_datetime, time_zone) do + time_zone_periods_from_wall(time_zone, NaiveDateTime.to_erl(naive_datetime)) + end + + defp time_zone_periods_from_utc("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2018, 3, 25}, {1, 0, 0}} and + erl_datetime < {{2018, 10, 28}, {3, 0, 0}} do + {:ok, @time_zone_period_cph_summer_2018} + end + + defp time_zone_periods_from_utc("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2018, 10, 28}, {2, 0, 0}} and + erl_datetime < {{2019, 3, 31}, {1, 0, 0}} do + {:ok, @time_zone_period_cph_winter_2018_2019} + end + + defp time_zone_periods_from_utc("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2019, 3, 31}, {1, 0, 0}} and + erl_datetime < {{2019, 10, 28}, {3, 0, 0}} do + {:ok, @time_zone_period_cph_summer_2019} + end + + defp time_zone_periods_from_utc("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2015, 3, 29}, {1, 0, 0}} and + erl_datetime < {{2015, 10, 25}, {1, 0, 0}} do + {:ok, + %{ + std_offset: 3600, + utc_offset: 3600, + zone_abbr: "CEST" + }} + end + + defp time_zone_periods_from_utc("America/Los_Angeles", erl_datetime) + when erl_datetime >= {{2018, 3, 11}, {10, 0, 0}} and + erl_datetime < {{2018, 11, 4}, {9, 0, 0}} do + {:ok, + %{ + std_offset: 3600, + utc_offset: -28800, + zone_abbr: "PDT" + }} + end + + defp time_zone_periods_from_utc(time_zone, _) when time_zone != "Europe/Copenhagen" do + {:error, :time_zone_not_found} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2019, 3, 31}, {2, 0, 0}} and + erl_datetime < {{2019, 3, 31}, {3, 0, 0}} do + {:gap, + {@time_zone_period_cph_winter_2018_2019, @time_zone_period_cph_winter_2018_2019.until_wall}, + {@time_zone_period_cph_summer_2019, @time_zone_period_cph_summer_2019.from_wall}} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime < {{2018, 10, 28}, {3, 0, 0}} and + erl_datetime >= {{2018, 10, 28}, {2, 0, 0}} do + {:ambiguous, @time_zone_period_cph_summer_2018, @time_zone_period_cph_winter_2018_2019} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2018, 3, 25}, {3, 0, 0}} and + erl_datetime < {{2018, 10, 28}, {3, 0, 0}} do + {:ok, @time_zone_period_cph_summer_2018} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2018, 10, 28}, {2, 0, 0}} and + erl_datetime < {{2019, 3, 31}, {2, 0, 0}} do + {:ok, @time_zone_period_cph_winter_2018_2019} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2019, 3, 31}, {3, 0, 0}} and + erl_datetime < {{2019, 10, 27}, {3, 0, 0}} do + {:ok, @time_zone_period_cph_summer_2019} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2015, 3, 29}, {3, 0, 0}} and + erl_datetime < {{2015, 10, 25}, {3, 0, 0}} do + {:ok, + %{ + std_offset: 3600, + utc_offset: 3600, + zone_abbr: "CEST" + }} + end + + defp time_zone_periods_from_wall("Europe/Copenhagen", erl_datetime) + when erl_datetime >= {{2090, 3, 26}, {3, 0, 0}} and + erl_datetime < {{2090, 10, 29}, {3, 0, 0}} do + {:ok, + %{ + std_offset: 3600, + utc_offset: 3600, + zone_abbr: "CEST" + }} + end + + defp time_zone_periods_from_wall(time_zone, _) when time_zone != "Europe/Copenhagen" do + {:error, :time_zone_not_found} + end + + defp naive_datetime_from_iso_days(iso_days) do + {year, month, day, hour, minute, second, microsecond} = + Calendar.ISO.naive_datetime_from_iso_days(iso_days) + + NaiveDateTime.new(year, month, day, hour, minute, second, microsecond) + end +end diff --git a/lib/elixir/test/elixir/calendar/holocene.exs b/lib/elixir/test/elixir/calendar/holocene.exs new file mode 100644 index 00000000000..66ae03176ef --- /dev/null +++ b/lib/elixir/test/elixir/calendar/holocene.exs @@ -0,0 +1,151 @@ +defmodule Calendar.Holocene do + # This calendar is used to test conversions between calendars. + # It implements the Holocene calendar, which is based on the + # Proleptic Gregorian calendar with every year + 10000. + + @behaviour Calendar + + def date(year, month, day) do + %Date{year: year, month: month, day: day, calendar: __MODULE__} + end + + def naive_datetime(year, month, day, hour, minute, second, microsecond \\ {0, 0}) do + %NaiveDateTime{ + year: year, + month: month, + day: day, + hour: hour, + minute: minute, + second: second, + microsecond: microsecond, + calendar: __MODULE__ + } + end + + @impl true + def date_to_string(year, month, day) do + "#{year}-#{zero_pad(month, 2)}-#{zero_pad(day, 2)}" + end + + @impl true + def naive_datetime_to_string(year, month, day, hour, minute, second, microsecond) do + "#{year}-#{zero_pad(month, 2)}-#{zero_pad(day, 2)}" <> + Calendar.ISO.time_to_string(hour, minute, second, microsecond) + end + + @impl true + def datetime_to_string( + year, + month, + day, + hour, + minute, + second, + microsecond, + _time_zone, + zone_abbr, + _utc_offset, + _std_offset + ) do + "#{year}-#{zero_pad(month, 2)}-#{zero_pad(day, 2)}" <> + Calendar.ISO.time_to_string(hour, minute, second, microsecond) <> + " #{zone_abbr}" + end + + @impl true + defdelegate time_to_string(hour, minute, second, microsecond), to: Calendar.ISO + + @impl true + def day_rollover_relative_to_midnight_utc(), do: {0, 1} + + @impl true + def naive_datetime_from_iso_days(entry) do + {year, month, day, hour, minute, second, microsecond} = + Calendar.ISO.naive_datetime_from_iso_days(entry) + + {year + 10000, month, day, hour, minute, second, microsecond} + end + + @impl true + def naive_datetime_to_iso_days(year, month, day, hour, minute, second, microsecond) do + Calendar.ISO.naive_datetime_to_iso_days( + year - 10000, + month, + day, + hour, + minute, + second, + microsecond + ) + end + + defp zero_pad(val, count) when val >= 0 do + String.pad_leading("#{val}", count, ["0"]) + end + + defp zero_pad(val, count) do + "-" <> zero_pad(-val, count) + end + + @impl true + def parse_date(string) do + {year, month, day} = + string + |> String.split("-") + |> Enum.map(&String.to_integer/1) + |> List.to_tuple() + + if valid_date?(year, month, day) do + {:ok, {year, month, day}} + else + {:error, :invalid_date} + end + end + + @impl true + def valid_date?(year, month, day) do + :calendar.valid_date(year, month, day) + end + + @impl true + defdelegate parse_time(string), to: Calendar.ISO + + @impl true + defdelegate parse_naive_datetime(string), to: Calendar.ISO + + @impl true + defdelegate parse_utc_datetime(string), to: Calendar.ISO + + @impl true + defdelegate time_from_day_fraction(day_fraction), to: Calendar.ISO + + @impl true + defdelegate time_to_day_fraction(hour, minute, second, microsecond), to: Calendar.ISO + + @impl true + defdelegate leap_year?(year), to: Calendar.ISO + + @impl true + defdelegate days_in_month(year, month), to: Calendar.ISO + + @impl true + defdelegate months_in_year(year), to: Calendar.ISO + + @impl true + defdelegate day_of_week(year, month, day, starting_on), to: Calendar.ISO + + @impl true + defdelegate day_of_year(year, month, day), to: Calendar.ISO + + @impl true + defdelegate quarter_of_year(year, month, day), to: Calendar.ISO + + @impl true + defdelegate year_of_era(year, month, day), to: Calendar.ISO + + @impl true + defdelegate day_of_era(year, month, day), to: Calendar.ISO + + @impl true + defdelegate valid_time?(hour, minute, second, microsecond), to: Calendar.ISO +end diff --git a/lib/elixir/test/elixir/calendar/iso_test.exs b/lib/elixir/test/elixir/calendar/iso_test.exs new file mode 100644 index 00000000000..6cbd0a00df8 --- /dev/null +++ b/lib/elixir/test/elixir/calendar/iso_test.exs @@ -0,0 +1,431 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Calendar.ISOTest do + use ExUnit.Case, async: true + doctest Calendar.ISO + + describe "date_from_iso_days" do + test "with positive dates" do + assert {0, 1, 1} == iso_day_roundtrip(0, 1, 1) + assert {0, 12, 31} == iso_day_roundtrip(0, 12, 31) + assert {1, 12, 31} == iso_day_roundtrip(1, 12, 31) + assert {4, 1, 1} == iso_day_roundtrip(4, 1, 1) + assert {4, 12, 31} == iso_day_roundtrip(4, 12, 31) + assert {9999, 12, 31} == iso_day_roundtrip(9999, 12, 31) + assert {9999, 1, 1} == iso_day_roundtrip(9999, 1, 1) + assert {9996, 12, 31} == iso_day_roundtrip(9996, 12, 31) + assert {9996, 1, 1} == iso_day_roundtrip(9996, 1, 1) + end + + test "with negative dates" do + assert {-1, 1, 1} == iso_day_roundtrip(-1, 1, 1) + assert {-1, 12, 31} == iso_day_roundtrip(-1, 12, 31) + assert {-1, 12, 31} == iso_day_roundtrip(-1, 12, 31) + assert {-2, 1, 1} == iso_day_roundtrip(-2, 1, 1) + assert {-5, 12, 31} == iso_day_roundtrip(-5, 12, 31) + + assert {-4, 1, 1} == iso_day_roundtrip(-4, 1, 1) + assert {-4, 12, 31} == iso_day_roundtrip(-4, 12, 31) + + assert {-9999, 12, 31} == iso_day_roundtrip(-9999, 12, 31) + assert {-9996, 12, 31} == iso_day_roundtrip(-9996, 12, 31) + + assert {-9996, 12, 31} == iso_day_roundtrip(-9996, 12, 31) + assert {-9996, 1, 1} == iso_day_roundtrip(-9996, 1, 1) + end + end + + describe "date_to_string/4" do + test "regular use" do + assert Calendar.ISO.date_to_string(1000, 1, 1, :basic) == "10000101" + assert Calendar.ISO.date_to_string(1000, 1, 1, :extended) == "1000-01-01" + + assert Calendar.ISO.date_to_string(-123, 1, 1, :basic) == "-01230101" + assert Calendar.ISO.date_to_string(-123, 1, 1, :extended) == "-0123-01-01" + end + + test "handles years > 9999" do + assert Calendar.ISO.date_to_string(10000, 1, 1, :basic) == "100000101" + assert Calendar.ISO.date_to_string(10000, 1, 1, :extended) == "10000-01-01" + end + end + + describe "naive_datetime_to_iso_days/7" do + test "raises with invalid dates" do + assert_raise ArgumentError, "invalid date: 2018-02-30", fn -> + Calendar.ISO.naive_datetime_to_iso_days(2018, 2, 30, 0, 0, 0, 0) + end + + assert_raise ArgumentError, "invalid date: 2017-11--03", fn -> + Calendar.ISO.naive_datetime_to_iso_days(2017, 11, -3, 0, 0, 0, 0) + end + end + end + + describe "day_of_week/3" do + test "raises with invalid dates" do + assert_raise ArgumentError, "invalid date: 2018-02-30", fn -> + Calendar.ISO.day_of_week(2018, 2, 30) + end + + assert_raise ArgumentError, "invalid date: 2017-11-00", fn -> + Calendar.ISO.day_of_week(2017, 11, 0) + end + end + end + + describe "day_of_era/3" do + test "raises with invalid dates" do + assert_raise ArgumentError, "invalid date: 2018-02-30", fn -> + Calendar.ISO.day_of_era(2018, 2, 30) + end + + assert_raise ArgumentError, "invalid date: 2017-11-00", fn -> + Calendar.ISO.day_of_era(2017, 11, 0) + end + end + end + + describe "day_of_year/3" do + test "raises with invalid dates" do + assert_raise ArgumentError, "invalid date: 2018-02-30", fn -> + Calendar.ISO.day_of_year(2018, 2, 30) + end + + assert_raise ArgumentError, "invalid date: 2017-11-00", fn -> + Calendar.ISO.day_of_year(2017, 11, 0) + end + end + end + + test "year_of_era/3" do + # Compatibility tests for year_of_era/1 + assert Calendar.ISO.year_of_era(-9999) == {10000, 0} + assert Calendar.ISO.year_of_era(-1) == {2, 0} + assert Calendar.ISO.year_of_era(0) == {1, 0} + assert Calendar.ISO.year_of_era(1) == {1, 1} + assert Calendar.ISO.year_of_era(1984) == {1984, 1} + + assert Calendar.ISO.year_of_era(-9999, 1, 1) == {10000, 0} + assert Calendar.ISO.year_of_era(-1, 1, 1) == {2, 0} + assert Calendar.ISO.year_of_era(0, 12, 1) == {1, 0} + assert Calendar.ISO.year_of_era(1, 12, 1) == {1, 1} + assert Calendar.ISO.year_of_era(1984, 12, 1) == {1984, 1} + + random_positive_year = Enum.random(1..9999) + assert Calendar.ISO.year_of_era(random_positive_year, 1, 1) == {random_positive_year, 1} + + assert_raise FunctionClauseError, fn -> + Calendar.ISO.year_of_era(10000, 1, 1) + end + + assert_raise FunctionClauseError, fn -> + Calendar.ISO.year_of_era(-10000, 12, 1) + end + end + + defp iso_day_roundtrip(year, month, day) do + iso_days = Calendar.ISO.date_to_iso_days(year, month, day) + Calendar.ISO.date_from_iso_days(iso_days) + end + + describe "parse_date/1" do + test "supports both only extended format by default" do + assert Calendar.ISO.parse_date("20150123") == {:error, :invalid_format} + assert Calendar.ISO.parse_date("2015-01-23") == {:ok, {2015, 1, 23}} + end + end + + describe "parse_date/2" do + test "allows enforcing basic formats" do + assert Calendar.ISO.parse_date("20150123", :basic) == {:ok, {2015, 1, 23}} + assert Calendar.ISO.parse_date("2015-01-23", :basic) == {:error, :invalid_format} + end + + test "allows enforcing extended formats" do + assert Calendar.ISO.parse_date("20150123", :extended) == {:error, :invalid_format} + assert Calendar.ISO.parse_date("2015-01-23", :extended) == {:ok, {2015, 1, 23}} + end + + test "errors on other format names" do + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_date("20150123", :other) + end + + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_date("2015-01-23", :other) + end + end + end + + describe "parse_time/1" do + test "supports only extended format by default" do + assert Calendar.ISO.parse_time("235007") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("23:50:07") == {:ok, {23, 50, 7, {0, 0}}} + end + + test "ignores offset data but requires valid ones" do + assert Calendar.ISO.parse_time("23:50:07Z") == {:ok, {23, 50, 7, {0, 0}}} + assert Calendar.ISO.parse_time("23:50:07+01:00") == {:ok, {23, 50, 7, {0, 0}}} + + assert Calendar.ISO.parse_time("2015-01-23 23:50-00:00") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015-01-23 23:50-00:60") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015-01-23 23:50-24:00") == {:error, :invalid_format} + end + + test "supports either comma or period millisecond delimiters" do + assert Calendar.ISO.parse_time("23:50:07,012345") == {:ok, {23, 50, 7, {12345, 6}}} + assert Calendar.ISO.parse_time("23:50:07.012345") == {:ok, {23, 50, 7, {12345, 6}}} + end + + test "only supports reduced precision for milliseconds" do + assert Calendar.ISO.parse_time("23:50:07.012345") == {:ok, {23, 50, 7, {12345, 6}}} + assert Calendar.ISO.parse_time("23:50:07") == {:ok, {23, 50, 7, {0, 0}}} + assert Calendar.ISO.parse_time("23:50") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("23") == {:error, :invalid_format} + end + + test "supports various millisecond precisions" do + assert Calendar.ISO.parse_time("23:50:07.012345") == {:ok, {23, 50, 7, {12345, 6}}} + assert Calendar.ISO.parse_time("23:50:07.0123") == {:ok, {23, 50, 7, {12300, 4}}} + assert Calendar.ISO.parse_time("23:50:07.01") == {:ok, {23, 50, 7, {10000, 2}}} + assert Calendar.ISO.parse_time("23:50:07.0") == {:ok, {23, 50, 7, {0, 1}}} + assert Calendar.ISO.parse_time("23:50:07") == {:ok, {23, 50, 7, {0, 0}}} + end + + test "truncates extra millisecond precision" do + assert Calendar.ISO.parse_time("23:50:07.012345") == {:ok, {23, 50, 7, {12345, 6}}} + assert Calendar.ISO.parse_time("23:50:07.0123456") == {:ok, {23, 50, 7, {12345, 6}}} + end + + test "rejects strings with formatting errors" do + assert Calendar.ISO.parse_time("23:50:07A") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("23:50:07.") == {:error, :invalid_format} + end + + test "refuses to parse the wrong thing" do + assert Calendar.ISO.parse_time("2015:01:23 23-50-07") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015-01-23 23:50:07") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015:01:23T23-50-07") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015-01-23T23:50:07") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015:01:23") == {:error, :invalid_format} + assert Calendar.ISO.parse_time("2015-01-23") == {:error, :invalid_format} + end + + test "recognizes invalid times" do + assert Calendar.ISO.parse_time("23:59:61") == {:error, :invalid_time} + assert Calendar.ISO.parse_time("23:61:59") == {:error, :invalid_time} + assert Calendar.ISO.parse_time("25:59:59") == {:error, :invalid_time} + end + end + + describe "parse_time/2" do + test "allows enforcing basic formats" do + assert Calendar.ISO.parse_time("235007", :basic) == {:ok, {23, 50, 7, {0, 0}}} + assert Calendar.ISO.parse_time("23:50:07", :basic) == {:error, :invalid_format} + end + + test "allows enforcing extended formats" do + assert Calendar.ISO.parse_time("235007", :extended) == {:error, :invalid_format} + assert Calendar.ISO.parse_time("23:50:07", :extended) == {:ok, {23, 50, 7, {0, 0}}} + end + + test "errors on other format names" do + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_time("235007", :other) + end + + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_time("23:50:07", :other) + end + end + end + + describe "parse_naive_datetime/1" do + test "rejects strings with formatting errors" do + assert Calendar.ISO.parse_naive_datetime("2015:01:23 23-50-07") == {:error, :invalid_format} + assert Calendar.ISO.parse_naive_datetime("2015-01-23P23:50:07") == {:error, :invalid_format} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07A") == + {:error, :invalid_format} + end + + test "recognizes invalid dates and times" do + assert Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:61") == {:error, :invalid_time} + assert Calendar.ISO.parse_naive_datetime("2015-01-32 23:50:07") == {:error, :invalid_date} + end + + test "ignores offset data but requires valid ones" do + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.123+02:30") == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.123+00:00") == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.123-02:30") == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.123-00:00") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.123-00:60") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07.123-24:00") == + {:error, :invalid_format} + end + + test "supports both spaces and 'T' as datetime separators" do + assert Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07") == + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23T23:50:07") == + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}} + end + + test "supports only extended format by default" do + assert Calendar.ISO.parse_naive_datetime("20150123 235007.123") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.123") == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}} + end + + test "errors on mixed basic and extended formats" do + assert Calendar.ISO.parse_naive_datetime("20150123 23:50:07.123") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23 235007.123") == + {:error, :invalid_format} + end + end + + describe "parse_naive_datetime/2" do + test "allows enforcing basic formats" do + assert Calendar.ISO.parse_naive_datetime("20150123 235007.123", :basic) == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.123", :basic) == + {:error, :invalid_format} + end + + test "allows enforcing extended formats" do + assert Calendar.ISO.parse_naive_datetime("20150123 235007.123", :extended) == + {:error, :invalid_format} + + assert Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.123", :extended) == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}} + end + + test "errors on other format names" do + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_naive_datetime("20150123 235007.123", :other) + end + + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.123", :other) + end + end + end + + describe "parse_utc_datetime/1" do + test "rejects strings with formatting errors" do + assert Calendar.ISO.parse_utc_datetime("2015:01:23 23-50-07Z") == {:error, :invalid_format} + assert Calendar.ISO.parse_utc_datetime("2015-01-23P23:50:07Z") == {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07A") == {:error, :invalid_format} + end + + test "recognizes invalid dates, times, and offsets" do + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07") == {:error, :missing_offset} + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:61Z") == {:error, :invalid_time} + assert Calendar.ISO.parse_utc_datetime("2015-01-32 23:50:07Z") == {:error, :invalid_date} + end + + test "interprets offset data" do + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.123+02:30") == + {:ok, {2015, 1, 23, 21, 20, 7, {123_000, 3}}, 9000} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.123+00:00") == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}, 0} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.123-02:30") == + {:ok, {2015, 1, 24, 2, 20, 7, {123_000, 3}}, -9000} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.123-00:00") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.123-00:60") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07.123-24:00") == + {:error, :invalid_format} + end + + test "supports both spaces and 'T' as datetime separators" do + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07Z") == + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}, 0} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23T23:50:07Z") == + {:ok, {2015, 1, 23, 23, 50, 7, {0, 0}}, 0} + end + + test "supports only extended format by default" do + assert Calendar.ISO.parse_utc_datetime("20150123 235007.123Z") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07.123Z") == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}, 0} + end + + test "errors on mixed basic and extended formats" do + assert Calendar.ISO.parse_utc_datetime("20150123 23:50:07.123Z") == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 235007.123Z") == + {:error, :invalid_format} + end + end + + describe "parse_utc_datetime/2" do + test "allows enforcing basic formats" do + assert Calendar.ISO.parse_utc_datetime("20150123 235007.123Z", :basic) == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}, 0} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07.123Z", :basic) == + {:error, :invalid_format} + end + + test "allows enforcing extended formats" do + assert Calendar.ISO.parse_utc_datetime("20150123 235007.123Z", :extended) == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 23:50:07.123Z", :extended) == + {:ok, {2015, 1, 23, 23, 50, 7, {123_000, 3}}, 0} + end + + test "errors on other format names" do + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_naive_datetime("20150123 235007.123Z", :other) + end + + assert_raise FunctionClauseError, fn -> + Calendar.ISO.parse_naive_datetime("2015-01-23 23:50:07.123Z", :other) + end + end + + test "errors on mixed basic and extended formats" do + assert Calendar.ISO.parse_utc_datetime("20150123 23:50:07.123Z", :basic) == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("20150123 23:50:07.123Z", :extended) == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 235007.123Z", :basic) == + {:error, :invalid_format} + + assert Calendar.ISO.parse_utc_datetime("2015-01-23 235007.123Z", :extended) == + {:error, :invalid_format} + end + end +end diff --git a/lib/elixir/test/elixir/calendar/naive_datetime_test.exs b/lib/elixir/test/elixir/calendar/naive_datetime_test.exs new file mode 100644 index 00000000000..b119d9f16ef --- /dev/null +++ b/lib/elixir/test/elixir/calendar/naive_datetime_test.exs @@ -0,0 +1,340 @@ +Code.require_file("../test_helper.exs", __DIR__) +Code.require_file("holocene.exs", __DIR__) +Code.require_file("fakes.exs", __DIR__) + +defmodule NaiveDateTimeTest do + use ExUnit.Case, async: true + doctest NaiveDateTime + + test "sigil_N" do + assert ~N[2000-01-01T12:34:56] == + %NaiveDateTime{ + calendar: Calendar.ISO, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56 + } + + assert ~N[2000-01-01T12:34:56 Calendar.Holocene] == + %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56 + } + + assert ~N[2000-01-01 12:34:56] == + %NaiveDateTime{ + calendar: Calendar.ISO, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56 + } + + assert ~N[2000-01-01 12:34:56 Calendar.Holocene] == + %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 2000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 56 + } + + assert_raise ArgumentError, + ~s/cannot parse "2001-50-50T12:34:56" as NaiveDateTime for Calendar.ISO, reason: :invalid_date/, + fn -> Code.eval_string("~N[2001-50-50T12:34:56]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:65" as NaiveDateTime for Calendar.ISO, reason: :invalid_time/, + fn -> Code.eval_string("~N[2001-01-01T12:34:65]") end + + assert_raise ArgumentError, + ~s/cannot parse "20010101 123456" as NaiveDateTime for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string(~s{~N[20010101 123456]}) end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01 12:34:56 notalias" as NaiveDateTime for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~N[2001-01-01 12:34:56 notalias]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:56 notalias" as NaiveDateTime for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~N[2001-01-01T12:34:56 notalias]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-50-50T12:34:56" as NaiveDateTime for Calendar.Holocene, reason: :invalid_date/, + fn -> Code.eval_string("~N[2001-50-50T12:34:56 Calendar.Holocene]") end + + assert_raise ArgumentError, + ~s/cannot parse "2001-01-01T12:34:65" as NaiveDateTime for Calendar.Holocene, reason: :invalid_time/, + fn -> Code.eval_string("~N[2001-01-01T12:34:65 Calendar.Holocene]") end + + assert_raise UndefinedFunctionError, fn -> + Code.eval_string("~N[2001-01-01 12:34:56 UnknownCalendar]") + end + + assert_raise UndefinedFunctionError, fn -> + Code.eval_string("~N[2001-01-01T12:34:56 UnknownCalendar]") + end + end + + test "to_string/1" do + assert to_string(~N[2000-01-01 23:00:07.005]) == "2000-01-01 23:00:07.005" + assert NaiveDateTime.to_string(~N[2000-01-01 23:00:07.005]) == "2000-01-01 23:00:07.005" + + ndt = %{~N[2000-01-01 23:00:07.005] | calendar: FakeCalendar} + assert to_string(ndt) == "1/1/2000F23::0::7" + end + + test "inspect/1" do + assert inspect(~N[2000-01-01 23:00:07.005]) == "~N[2000-01-01 23:00:07.005]" + assert inspect(~N[-0100-12-31 23:00:07.005]) == "~N[-0100-12-31 23:00:07.005]" + + ndt = %{~N[2000-01-01 23:00:07.005] | calendar: FakeCalendar} + assert inspect(ndt) == "~N[1/1/2000F23::0::7 FakeCalendar]" + end + + test "compare/2" do + ndt1 = ~N[2000-04-16 13:30:15.0049] + ndt2 = ~N[2000-04-16 13:30:15.0050] + ndt3 = ~N[2001-04-16 13:30:15.0050] + ndt4 = ~N[-0001-04-16 13:30:15.004] + assert NaiveDateTime.compare(ndt1, ndt1) == :eq + assert NaiveDateTime.compare(ndt1, ndt2) == :lt + assert NaiveDateTime.compare(ndt2, ndt1) == :gt + assert NaiveDateTime.compare(ndt3, ndt1) == :gt + assert NaiveDateTime.compare(ndt3, ndt2) == :gt + assert NaiveDateTime.compare(ndt4, ndt4) == :eq + assert NaiveDateTime.compare(ndt1, ndt4) == :gt + assert NaiveDateTime.compare(ndt4, ndt3) == :lt + end + + test "to_iso8601/1" do + ndt = ~N[2000-04-16 12:34:15.1234] + ndt = put_in(ndt.calendar, FakeCalendar) + + message = + "cannot convert #{inspect(ndt)} to target calendar Calendar.ISO, " <> + "reason: #{inspect(ndt.calendar)} and Calendar.ISO have different day rollover moments, " <> + "making this conversion ambiguous" + + assert_raise ArgumentError, message, fn -> + NaiveDateTime.to_iso8601(ndt) + end + end + + test "add/2 with other calendars" do + assert ~N[2000-01-01 12:34:15.123456] + |> NaiveDateTime.convert!(Calendar.Holocene) + |> NaiveDateTime.add(10, :second) == + %NaiveDateTime{ + calendar: Calendar.Holocene, + year: 12000, + month: 1, + day: 1, + hour: 12, + minute: 34, + second: 25, + microsecond: {123_456, 6} + } + end + + test "add/2 with datetime" do + dt = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + assert NaiveDateTime.add(dt, 21, :second) == ~N[2000-02-29 23:00:28] + end + + test "diff/2 with other calendars" do + assert ~N[2000-01-01 12:34:15.123456] + |> NaiveDateTime.convert!(Calendar.Holocene) + |> NaiveDateTime.add(10, :second) + |> NaiveDateTime.diff(~N[2000-01-01 12:34:15.123456]) == 10 + end + + test "diff/2 with datetime" do + dt = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {0, 0}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + assert NaiveDateTime.diff(%{dt | second: 57}, dt, :second) == 50 + end + + test "convert/2" do + assert NaiveDateTime.convert(~N[2000-01-01 12:34:15.123400], Calendar.Holocene) == + {:ok, Calendar.Holocene.naive_datetime(12000, 1, 1, 12, 34, 15, {123_400, 6})} + + assert ~N[2000-01-01 12:34:15] + |> NaiveDateTime.convert!(Calendar.Holocene) + |> NaiveDateTime.convert!(Calendar.ISO) == ~N[2000-01-01 12:34:15] + + assert ~N[2000-01-01 12:34:15.123456] + |> NaiveDateTime.convert!(Calendar.Holocene) + |> NaiveDateTime.convert!(Calendar.ISO) == ~N[2000-01-01 12:34:15.123456] + + assert NaiveDateTime.convert(~N[2016-02-03 00:00:01], FakeCalendar) == + {:error, :incompatible_calendars} + + assert NaiveDateTime.convert(~N[1970-01-01 00:00:00], Calendar.Holocene) == + {:ok, Calendar.Holocene.naive_datetime(11970, 1, 1, 0, 0, 0, {0, 0})} + + assert NaiveDateTime.convert(DateTime.from_unix!(0, :second), Calendar.Holocene) == + {:ok, Calendar.Holocene.naive_datetime(11970, 1, 1, 0, 0, 0, {0, 0})} + end + + test "truncate/2" do + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.123456], :microsecond) == + ~N[2017-11-06 00:23:51.123456] + + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.0], :millisecond) == + ~N[2017-11-06 00:23:51.0] + + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.999], :millisecond) == + ~N[2017-11-06 00:23:51.999] + + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.1009], :millisecond) == + ~N[2017-11-06 00:23:51.100] + + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.123456], :millisecond) == + ~N[2017-11-06 00:23:51.123] + + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.000456], :millisecond) == + ~N[2017-11-06 00:23:51.000] + + assert NaiveDateTime.truncate(~N[2017-11-06 00:23:51.123456], :second) == + ~N[2017-11-06 00:23:51] + end + + test "truncate/2 with datetime" do + dt = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {3000, 6}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + assert NaiveDateTime.truncate(dt, :millisecond) == ~N[2000-02-29 23:00:07.003] + assert catch_error(NaiveDateTime.truncate(~T[00:00:00.000000], :millisecond)) + end + + describe "utc_now/1" do + test "utc_now/1 with default calendar (ISO)" do + naive_datetime = NaiveDateTime.utc_now() + assert naive_datetime.year >= 2019 + end + + test "utc_now/1 with alternative calendar" do + naive_datetime = NaiveDateTime.utc_now(Calendar.Holocene) + assert naive_datetime.calendar == Calendar.Holocene + assert naive_datetime.year >= 12019 + end + end + + describe "local_now/1" do + test "local_now/1 with default calendar (ISO)" do + naive_datetime = NaiveDateTime.local_now() + assert naive_datetime.year >= 2018 + end + + test "local_now/1 alternative calendar" do + naive_datetime = NaiveDateTime.local_now(Calendar.Holocene) + assert naive_datetime.calendar == Calendar.Holocene + assert naive_datetime.year >= 12018 + end + + test "local_now/1 incompatible calendar" do + assert_raise ArgumentError, + ~s(cannot get "local now" in target calendar FakeCalendar, reason: cannot convert from Calendar.ISO to FakeCalendar.), + fn -> + NaiveDateTime.local_now(FakeCalendar) + end + end + end + + describe "to_date/2" do + test "downcasting" do + dt = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {3000, 6}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + assert NaiveDateTime.to_date(dt) == ~D[2000-02-29] + end + + test "upcasting" do + assert catch_error(NaiveDateTime.to_date(~D[2000-02-29])) + end + end + + describe "to_time/2" do + test "downcasting" do + dt = %DateTime{ + year: 2000, + month: 2, + day: 29, + zone_abbr: "CET", + hour: 23, + minute: 0, + second: 7, + microsecond: {3000, 6}, + utc_offset: 3600, + std_offset: 0, + time_zone: "Europe/Warsaw" + } + + assert NaiveDateTime.to_time(dt) == ~T[23:00:07.003000] + end + + test "upcasting" do + assert catch_error(NaiveDateTime.to_time(~T[00:00:00.000000])) + end + end +end diff --git a/lib/elixir/test/elixir/calendar/time_test.exs b/lib/elixir/test/elixir/calendar/time_test.exs new file mode 100644 index 00000000000..1285f7df515 --- /dev/null +++ b/lib/elixir/test/elixir/calendar/time_test.exs @@ -0,0 +1,82 @@ +Code.require_file("../test_helper.exs", __DIR__) +Code.require_file("holocene.exs", __DIR__) +Code.require_file("fakes.exs", __DIR__) + +defmodule TimeTest do + use ExUnit.Case, async: true + doctest Time + + test "sigil_T" do + assert ~T[12:34:56] == + %Time{calendar: Calendar.ISO, hour: 12, minute: 34, second: 56} + + assert ~T[12:34:56 Calendar.Holocene] == + %Time{calendar: Calendar.Holocene, hour: 12, minute: 34, second: 56} + + assert_raise ArgumentError, + ~s/cannot parse "12:34:65" as Time for Calendar.ISO, reason: :invalid_time/, + fn -> Code.eval_string("~T[12:34:65]") end + + assert_raise ArgumentError, + ~s/cannot parse "123456" as Time for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~T[123456]") end + + assert_raise ArgumentError, + ~s/cannot parse "12:34:56 notalias" as Time for Calendar.ISO, reason: :invalid_format/, + fn -> Code.eval_string("~T[12:34:56 notalias]") end + + assert_raise ArgumentError, + ~s/cannot parse "12:34:65" as Time for Calendar.Holocene, reason: :invalid_time/, + fn -> Code.eval_string("~T[12:34:65 Calendar.Holocene]") end + + assert_raise UndefinedFunctionError, fn -> + Code.eval_string("~T[12:34:56 UnknownCalendar]") + end + end + + test "to_string/1" do + time = ~T[23:00:07.005] + assert to_string(time) == "23:00:07.005" + assert Time.to_string(time) == "23:00:07.005" + assert Time.to_string(Map.from_struct(time)) == "23:00:07.005" + + assert to_string(%{time | calendar: FakeCalendar}) == "23::0::7" + assert Time.to_string(%{time | calendar: FakeCalendar}) == "23::0::7" + end + + test "inspect/1" do + assert inspect(~T[23:00:07.005]) == "~T[23:00:07.005]" + + time = %{~T[23:00:07.005] | calendar: FakeCalendar} + assert inspect(time) == "~T[23::0::7 FakeCalendar]" + end + + test "compare/2" do + time0 = ~T[01:01:01.0] + time1 = ~T[01:01:01.005] + time2 = ~T[01:01:01.0050] + time3 = ~T[23:01:01.0050] + assert Time.compare(time0, time1) == :lt + assert Time.compare(time1, time1) == :eq + assert Time.compare(time1, time2) == :eq + assert Time.compare(time1, time3) == :lt + assert Time.compare(time3, time2) == :gt + end + + test "truncate/2" do + assert Time.truncate(~T[01:01:01.123456], :microsecond) == ~T[01:01:01.123456] + + assert Time.truncate(~T[01:01:01.0], :millisecond) == ~T[01:01:01.0] + assert Time.truncate(~T[01:01:01.00], :millisecond) == ~T[01:01:01.00] + assert Time.truncate(~T[01:01:01.1], :millisecond) == ~T[01:01:01.1] + assert Time.truncate(~T[01:01:01.100], :millisecond) == ~T[01:01:01.100] + assert Time.truncate(~T[01:01:01.999], :millisecond) == ~T[01:01:01.999] + assert Time.truncate(~T[01:01:01.1000], :millisecond) == ~T[01:01:01.100] + assert Time.truncate(~T[01:01:01.1001], :millisecond) == ~T[01:01:01.100] + assert Time.truncate(~T[01:01:01.123456], :millisecond) == ~T[01:01:01.123] + assert Time.truncate(~T[01:01:01.000123], :millisecond) == ~T[01:01:01.000] + assert Time.truncate(~T[01:01:01.00012], :millisecond) == ~T[01:01:01.000] + + assert Time.truncate(~T[01:01:01.123456], :second) == ~T[01:01:01] + end +end diff --git a/lib/elixir/test/elixir/calendar_test.exs b/lib/elixir/test/elixir/calendar_test.exs new file mode 100644 index 00000000000..3da165cfc3f --- /dev/null +++ b/lib/elixir/test/elixir/calendar_test.exs @@ -0,0 +1,368 @@ +Code.require_file("test_helper.exs", __DIR__) + +defmodule CalendarTest do + use ExUnit.Case, async: true + doctest Calendar + + describe "strftime/3" do + test "returns received string if there is no datetime formatting to be found in it" do + assert Calendar.strftime(~N[2019-08-20 15:47:34.001], "same string") == "same string" + end + + test "formats all time zones blank when receiving a NaiveDateTime" do + assert Calendar.strftime(~N[2019-08-15 17:07:57.001], "%z%Z") == "" + end + + test "raises error when trying to format a date with a map that has no date fields" do + time_without_date = %{hour: 15, minute: 47, second: 34, microsecond: {0, 0}} + + assert_raise KeyError, fn -> Calendar.strftime(time_without_date, "%x") end + end + + test "raises error when trying to format a time with a map that has no time fields" do + date_without_time = %{year: 2019, month: 8, day: 20} + + assert_raise KeyError, fn -> Calendar.strftime(date_without_time, "%X") end + end + + test "raises error when the format is invalid" do + assert_raise ArgumentError, "invalid strftime format: %-", fn -> + Calendar.strftime(~N[2019-08-20 15:47:34.001], "%-2-ç") + end + + assert_raise ArgumentError, "invalid strftime format: %", fn -> + Calendar.strftime(~N[2019-08-20 15:47:34.001], "%") + end + end + + test "raises error when the preferred_datetime calls itself" do + assert_raise ArgumentError, fn -> + Calendar.strftime(~N[2019-08-20 15:47:34.001], "%c", preferred_datetime: "%c") + end + end + + test "raises error when the preferred_date calls itself" do + assert_raise ArgumentError, fn -> + Calendar.strftime(~N[2019-08-20 15:47:34.001], "%x", preferred_date: "%x") + end + end + + test "raises error when the preferred_time calls itself" do + assert_raise ArgumentError, fn -> + Calendar.strftime(~N[2019-08-20 15:47:34.001], "%X", preferred_time: "%X") + end + end + + test "raises error when the preferred formats creates a circular chain" do + assert_raise ArgumentError, fn -> + Calendar.strftime(~N[2019-08-20 15:47:34.001], "%c", + preferred_datetime: "%x", + preferred_date: "%X", + preferred_time: "%c" + ) + end + end + + test "with preferred formats are included multiple times on the same string" do + assert Calendar.strftime(~N[2019-08-15 17:07:57.001], "%c %c %x %x %X %X") == + "2019-08-15 17:07:57 2019-08-15 17:07:57 2019-08-15 2019-08-15 17:07:57 17:07:57" + end + + test "`-` removes padding" do + assert Calendar.strftime(~D[2019-01-01], "%-j") == "1" + assert Calendar.strftime(~T[17:07:57.001], "%-999M") == "7" + end + + test "formats time zones correctly when receiving a DateTime" do + datetime_with_zone = %DateTime{ + year: 2019, + month: 8, + day: 15, + zone_abbr: "EEST", + hour: 17, + minute: 7, + second: 57, + microsecond: {0, 0}, + utc_offset: 7200, + std_offset: 3600, + time_zone: "UK" + } + + assert Calendar.strftime(datetime_with_zone, "%z %Z") == "+0300 EEST" + end + + test "formats AM and PM correctly on the %P and %p options" do + am_time_almost_pm = ~U[2019-08-26 11:59:59.001Z] + pm_time = ~U[2019-08-26 12:00:57.001Z] + pm_time_almost_am = ~U[2019-08-26 23:59:57.001Z] + am_time = ~U[2019-08-26 00:00:01.001Z] + + assert Calendar.strftime(am_time_almost_pm, "%P %p") == "am AM" + assert Calendar.strftime(pm_time, "%P %p") == "pm PM" + assert Calendar.strftime(pm_time_almost_am, "%P %p") == "pm PM" + assert Calendar.strftime(am_time, "%P %p") == "am AM" + end + + test "formats all weekdays correctly with %A and %a formats" do + sunday = ~U[2019-08-25 11:59:59.001Z] + monday = ~U[2019-08-26 11:59:59.001Z] + tuesday = ~U[2019-08-27 11:59:59.001Z] + wednesday = ~U[2019-08-28 11:59:59.001Z] + thursday = ~U[2019-08-29 11:59:59.001Z] + friday = ~U[2019-08-30 11:59:59.001Z] + saturday = ~U[2019-08-31 11:59:59.001Z] + + assert Calendar.strftime(sunday, "%A %a") == "Sunday Sun" + assert Calendar.strftime(monday, "%A %a") == "Monday Mon" + assert Calendar.strftime(tuesday, "%A %a") == "Tuesday Tue" + assert Calendar.strftime(wednesday, "%A %a") == "Wednesday Wed" + assert Calendar.strftime(thursday, "%A %a") == "Thursday Thu" + assert Calendar.strftime(friday, "%A %a") == "Friday Fri" + assert Calendar.strftime(saturday, "%A %a") == "Saturday Sat" + end + + test "formats all months correctly with the %B and %b formats" do + assert Calendar.strftime(%{month: 1}, "%B %b") == "January Jan" + assert Calendar.strftime(%{month: 2}, "%B %b") == "February Feb" + assert Calendar.strftime(%{month: 3}, "%B %b") == "March Mar" + assert Calendar.strftime(%{month: 4}, "%B %b") == "April Apr" + assert Calendar.strftime(%{month: 5}, "%B %b") == "May May" + assert Calendar.strftime(%{month: 6}, "%B %b") == "June Jun" + assert Calendar.strftime(%{month: 7}, "%B %b") == "July Jul" + assert Calendar.strftime(%{month: 8}, "%B %b") == "August Aug" + assert Calendar.strftime(%{month: 9}, "%B %b") == "September Sep" + assert Calendar.strftime(%{month: 10}, "%B %b") == "October Oct" + assert Calendar.strftime(%{month: 11}, "%B %b") == "November Nov" + assert Calendar.strftime(%{month: 12}, "%B %b") == "December Dec" + end + + test "formats all weekdays correctly on %A with day_of_week_names option" do + sunday = ~U[2019-08-25 11:59:59.001Z] + monday = ~U[2019-08-26 11:59:59.001Z] + tuesday = ~U[2019-08-27 11:59:59.001Z] + wednesday = ~U[2019-08-28 11:59:59.001Z] + thursday = ~U[2019-08-29 11:59:59.001Z] + friday = ~U[2019-08-30 11:59:59.001Z] + saturday = ~U[2019-08-31 11:59:59.001Z] + + day_of_week_names = fn day_of_week -> + {"segunda-feira", "terça-feira", "quarta-feira", "quinta-feira", "sexta-feira", "sábado", + "domingo"} + |> elem(day_of_week - 1) + end + + assert Calendar.strftime(sunday, "%A", day_of_week_names: day_of_week_names) == + "domingo" + + assert Calendar.strftime(monday, "%A", day_of_week_names: day_of_week_names) == + "segunda-feira" + + assert Calendar.strftime(tuesday, "%A", day_of_week_names: day_of_week_names) == + "terça-feira" + + assert Calendar.strftime(wednesday, "%A", day_of_week_names: day_of_week_names) == + "quarta-feira" + + assert Calendar.strftime(thursday, "%A", day_of_week_names: day_of_week_names) == + "quinta-feira" + + assert Calendar.strftime(friday, "%A", day_of_week_names: day_of_week_names) == + "sexta-feira" + + assert Calendar.strftime(saturday, "%A", day_of_week_names: day_of_week_names) == + "sábado" + end + + test "formats all months correctly on the %B with month_names option" do + month_names = fn month -> + {"январь", "февраль", "март", "апрель", "май", "июнь", "июль", "август", "сентябрь", + "октябрь", "ноябрь", "декабрь"} + |> elem(month - 1) + end + + assert Calendar.strftime(%{month: 1}, "%B", month_names: month_names) == "январь" + assert Calendar.strftime(%{month: 2}, "%B", month_names: month_names) == "февраль" + assert Calendar.strftime(%{month: 3}, "%B", month_names: month_names) == "март" + assert Calendar.strftime(%{month: 4}, "%B", month_names: month_names) == "апрель" + assert Calendar.strftime(%{month: 5}, "%B", month_names: month_names) == "май" + assert Calendar.strftime(%{month: 6}, "%B", month_names: month_names) == "июнь" + assert Calendar.strftime(%{month: 7}, "%B", month_names: month_names) == "июль" + assert Calendar.strftime(%{month: 8}, "%B", month_names: month_names) == "август" + assert Calendar.strftime(%{month: 9}, "%B", month_names: month_names) == "сентябрь" + assert Calendar.strftime(%{month: 10}, "%B", month_names: month_names) == "октябрь" + assert Calendar.strftime(%{month: 11}, "%B", month_names: month_names) == "ноябрь" + assert Calendar.strftime(%{month: 12}, "%B", month_names: month_names) == "декабрь" + end + + test "formats all weekdays correctly on the %a format with abbreviated_day_of_week_names option" do + sunday = ~U[2019-08-25 11:59:59.001Z] + monday = ~U[2019-08-26 11:59:59.001Z] + tuesday = ~U[2019-08-27 11:59:59.001Z] + wednesday = ~U[2019-08-28 11:59:59.001Z] + thursday = ~U[2019-08-29 11:59:59.001Z] + friday = ~U[2019-08-30 11:59:59.001Z] + saturday = ~U[2019-08-31 11:59:59.001Z] + + abbreviated_day_of_week_names = fn day_of_week -> + {"seg", "ter", "qua", "qui", "sex", "sáb", "dom"} + |> elem(day_of_week - 1) + end + + assert Calendar.strftime(sunday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "dom" + + assert Calendar.strftime(monday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "seg" + + assert Calendar.strftime(tuesday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "ter" + + assert Calendar.strftime(wednesday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "qua" + + assert Calendar.strftime(thursday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "qui" + + assert Calendar.strftime(friday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "sex" + + assert Calendar.strftime(saturday, "%a", + abbreviated_day_of_week_names: abbreviated_day_of_week_names + ) == "sáb" + end + + test "formats all months correctly on the %b format with abbreviated_month_names option" do + abbreviated_month_names = fn month -> + {"янв", "февр", "март", "апр", "май", "июнь", "июль", "авг", "сент", "окт", "нояб", "дек"} + |> elem(month - 1) + end + + assert Calendar.strftime(%{month: 1}, "%b", abbreviated_month_names: abbreviated_month_names) == + "янв" + + assert Calendar.strftime(%{month: 2}, "%b", abbreviated_month_names: abbreviated_month_names) == + "февр" + + assert Calendar.strftime(%{month: 3}, "%b", abbreviated_month_names: abbreviated_month_names) == + "март" + + assert Calendar.strftime(%{month: 4}, "%b", abbreviated_month_names: abbreviated_month_names) == + "апр" + + assert Calendar.strftime(%{month: 5}, "%b", abbreviated_month_names: abbreviated_month_names) == + "май" + + assert Calendar.strftime(%{month: 6}, "%b", abbreviated_month_names: abbreviated_month_names) == + "июнь" + + assert Calendar.strftime(%{month: 7}, "%b", abbreviated_month_names: abbreviated_month_names) == + "июль" + + assert Calendar.strftime(%{month: 8}, "%b", abbreviated_month_names: abbreviated_month_names) == + "авг" + + assert Calendar.strftime(%{month: 9}, "%b", abbreviated_month_names: abbreviated_month_names) == + "сент" + + assert Calendar.strftime(%{month: 10}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == "окт" + + assert Calendar.strftime(%{month: 11}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == "нояб" + + assert Calendar.strftime(%{month: 12}, "%b", + abbreviated_month_names: abbreviated_month_names + ) == "дек" + end + + test "formats ignores padding and width options on microseconds" do + datetime = ~U[2019-08-15 17:07:57.001234Z] + assert Calendar.strftime(datetime, "%f") == "001234" + assert Calendar.strftime(datetime, "%f") == Calendar.strftime(datetime, "%_20f") + assert Calendar.strftime(datetime, "%f") == Calendar.strftime(datetime, "%020f") + assert Calendar.strftime(datetime, "%f") == Calendar.strftime(datetime, "%-f") + end + + test "formats properly dates with different microsecond precisions" do + assert Calendar.strftime(~U[2019-08-15 17:07:57.5Z], "%f") == "5" + assert Calendar.strftime(~U[2019-08-15 17:07:57.45Z], "%f") == "45" + assert Calendar.strftime(~U[2019-08-15 17:07:57.345Z], "%f") == "345" + assert Calendar.strftime(~U[2019-08-15 17:07:57.2345Z], "%f") == "2345" + assert Calendar.strftime(~U[2019-08-15 17:07:57.12345Z], "%f") == "12345" + assert Calendar.strftime(~U[2019-08-15 17:07:57.012345Z], "%f") == "012345" + end + + test "formats properly different microsecond precisions of zero" do + assert Calendar.strftime(~N[2019-08-15 17:07:57.0], "%f") == "0" + assert Calendar.strftime(~N[2019-08-15 17:07:57.00], "%f") == "00" + assert Calendar.strftime(~N[2019-08-15 17:07:57.000], "%f") == "000" + assert Calendar.strftime(~N[2019-08-15 17:07:57.0000], "%f") == "0000" + assert Calendar.strftime(~N[2019-08-15 17:07:57.00000], "%f") == "00000" + assert Calendar.strftime(~N[2019-08-15 17:07:57.000000], "%f") == "000000" + end + + test "returns a single zero if there's no microseconds precision" do + assert Calendar.strftime(~N[2019-08-15 17:07:57], "%f") == "0" + end + + test "handles `0` both as padding and as part of a width" do + assert Calendar.strftime(~N[2019-08-15 17:07:57], "%10A") == " Thursday" + assert Calendar.strftime(~N[2019-08-15 17:07:57], "%010A") == "00Thursday" + end + + test "formats datetime with all options and modifiers" do + assert Calendar.strftime( + ~U[2019-08-15 17:07:57.001Z], + "%04% %a %A %b %B %-3c %d %f %H %I %j %m %_5M %p %P %q %S %u %x %X %y %Y %z %Z" + ) == + "000% Thu Thursday Aug August 2019-08-15 17:07:57 15 001 17 05 227 08 7 PM pm 3 57 4 2019-08-15 17:07:57 19 2019 +0000 UTC" + end + + test "formats according to custom configs" do + assert Calendar.strftime( + ~U[2019-08-15 17:07:57.001Z], + "%A %a %p %B %b %c %x %X", + am_pm_names: fn + :am -> "a" + :pm -> "p" + end, + month_names: fn month -> + {"Janeiro", "Fevereiro", "Março", "Abril", "Maio", "Junho", "Julho", "Agosto", + "Setembro", "Outubro", "Novembro", "Dezembro"} + |> elem(month - 1) + end, + day_of_week_names: fn day_of_week -> + {"понедельник", "вторник", "среда", "четверг", "пятница", "суббота", + "воскресенье"} + |> elem(day_of_week - 1) + end, + abbreviated_month_names: fn month -> + {"Jan", "Fev", "Mar", "Abr", "Mai", "Jun", "Jul", "Ago", "Set", "Out", "Nov", + "Dez"} + |> elem(month - 1) + end, + abbreviated_day_of_week_names: fn day_of_week -> + {"ПНД", "ВТР", "СРД", "ЧТВ", "ПТН", "СБТ", "ВСК"} + |> elem(day_of_week - 1) + end, + preferred_date: "%05Y-%m-%d", + preferred_time: "%M:%_3H%S", + preferred_datetime: "%%" + ) == "четверг ЧТВ P Agosto Ago % 02019-08-15 07: 1757" + end + + test "raises on unknown option according to custom configs" do + assert_raise ArgumentError, "unknown option :unknown given to Calendar.strftime/3", fn -> + Calendar.strftime(~D[2019-08-15], "%D", unknown: "option") + end + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/calls_test.exs b/lib/elixir/test/elixir/code_formatter/calls_test.exs new file mode 100644 index 00000000000..cd07a0431c7 --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/calls_test.exs @@ -0,0 +1,1223 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.CallsTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + @medium_length [line_length: 20] + + describe "next break fits" do + test "does not apply to function calls" do + bad = "foo(very_long_call(bar))" + + good = """ + foo( + very_long_call( + bar + ) + ) + """ + + assert_format bad, good, @short_length + end + + test "does not apply to strings" do + bad = "foo(\"very long string\")" + + good = """ + foo( + "very long string" + ) + """ + + assert_format bad, good, @short_length + end + + test "for functions" do + assert_same """ + foo(fn x -> y end) + """ + + assert_same """ + foo(fn + a1 -> :ok + b2 -> :error + end) + """ + + assert_same """ + foo(bar, fn + a1 -> :ok + b2 -> :error + end) + """ + + assert_same """ + foo(fn x -> + :really_long_atom + end) + """, + @medium_length + + assert_same """ + foo(bar, fn + a1 -> + :ok + + b2 -> + :really_long_error + end) + """, + @medium_length + end + + test "for heredocs" do + assert_same """ + foo(''' + bar + ''') + """ + + assert_same to_string(''' + foo(""" + bar + """) + ''') + + assert_same """ + foo(~S''' + bar + ''') + """ + + assert_same """ + foo(~S''' + very long line does trigger another break + ''') + """, + @short_length + end + + test "for lists" do + bad = "foo([1, 2, 3, 4])" + + good = """ + foo([ + 1, + 2, + 3, + 4 + ]) + """ + + assert_format bad, good, @short_length + end + + test "for {} calls" do + bad = """ + alias Foo.{ + Bar, Baz + } + """ + + good = """ + alias Foo.{ + Bar, + Baz + } + """ + + assert_format bad, good, @medium_length + end + + test "for binaries only on eol" do + bad = "foo(<<1, 2, 3, 4>>)" + + good = """ + foo( + <<1, 2, + 3, 4>> + ) + """ + + assert_format bad, good, @short_length + + bad = """ + foo(<< + # foo + 1, + 2, + 3, + 4>>) + """ + + good = """ + foo(<< + # foo + 1, + 2, + 3, + 4 + >>) + """ + + assert_format bad, good, @short_length + end + end + + describe "local calls" do + test "without arguments" do + assert_format "foo( )", "foo()" + end + + test "without arguments doesn't split on line limit" do + assert_same "very_long_function_name()", @short_length + end + + test "removes outer parens except for unquote_splicing/1" do + assert_format "(foo())", "foo()" + assert_same "(unquote_splicing(123))" + end + + test "with arguments" do + assert_format "foo( :one ,:two,\n :three)", "foo(:one, :two, :three)" + end + + test "with arguments splits on line limit" do + bad = """ + fun(x, y, z) + """ + + good = """ + fun( + x, + y, + z + ) + """ + + assert_format bad, good, @short_length + end + + test "with arguments on comma limit" do + bad = """ + import(foo(abc, cde), :next) + """ + + good = """ + import( + foo(abc, cde), + :next + ) + """ + + assert_format bad, good, @medium_length + end + + test "with keyword lists" do + assert_same "foo(foo: 1, bar: 2)" + assert_same "foo(:hello, foo: 1, bar: 2)" + + bad = """ + foo(:hello, foo: 1, bar: 2) + """ + + good = """ + foo( + :hello, + foo: 1, + bar: 2 + ) + """ + + assert_format bad, good, @short_length + + bad = """ + foo(:hello, foo: 1, + bar: 2, baz: 3) + """ + + assert_format bad, """ + foo(:hello, foo: 1, bar: 2, baz: 3) + """ + end + + test "with lists maybe rewritten as keyword lists" do + assert_format "foo([foo: 1, bar: 2])", "foo(foo: 1, bar: 2)" + assert_format "foo(:arg, [foo: 1, bar: 2])", "foo(:arg, foo: 1, bar: 2)" + assert_same "foo(:arg, [:elem, foo: 1, bar: 2])" + end + + test "without parens" do + assert_same "import :foo, :bar" + assert_same "bar = if foo, do: bar, else: baz" + + assert_same """ + for :one, + :two, + :three, + fn -> + :ok + end + """ + + assert_same """ + for :one, fn -> + :ok + end + """ + end + + test "without parens on line limit" do + bad = "import :long_atom, :other_arg" + + good = """ + import :long_atom, + :other_arg + """ + + assert_format bad, good, @short_length + end + + test "without parens on comma limit" do + bad = """ + import foo(abc, cde), :next + """ + + good = """ + import foo( + abc, + cde + ), + :next + """ + + assert_format bad, good, @medium_length + end + + test "without parens and with keyword lists preserves multiline" do + assert_same """ + defstruct foo: 1, + bar: 2 + """ + + assert_same """ + config :app, + foo: 1 + """ + + assert_same """ + config :app, + foo: 1, + bar: 2 + """ + + assert_same """ + config :app, :key, + foo: 1, + bar: 2 + """ + + assert_same """ + config :app, + :key, + foo: 1, + bar: 2 + """ + + bad = """ + config :app, foo: 1, + bar: 2 + """ + + assert_format bad, """ + config :app, + foo: 1, + bar: 2 + """ + end + + test "without parens and with keyword lists on comma limit" do + bad = """ + import foo(abc, cde), opts: :next + """ + + good = """ + import foo( + abc, + cde + ), + opts: :next + """ + + assert_format bad, good, @medium_length + end + + test "without parens and with keyword lists on line limit" do + assert_same "import :atom, opts: [foo: :bar]" + + bad = "import :atom, opts: [foo: :bar]" + + good = """ + import :atom, + opts: [foo: :bar] + """ + + assert_format bad, good, @medium_length + + bad = "import :atom, really_long_key: [foo: :bar]" + + good = """ + import :atom, + really_long_key: [ + foo: :bar + ] + """ + + assert_format bad, good, @medium_length + + assert_same """ + import :foo, + one: two, + three: four, + five: [6, 7, 8, 9] + """, + @medium_length + + assert_same """ + import :really_long_atom_but_no_breaks, + one: two, + three: four + """, + @medium_length + + bad = "with :really_long_atom1, :really_long_atom2, opts: [foo: :bar]" + + good = """ + with :really_long_atom1, + :really_long_atom2, + opts: [ + foo: :bar + ] + """ + + assert_format bad, good, @medium_length + end + + test "without parens from option" do + assert_format "foo bar", "foo(bar)" + assert_same "foo bar", locals_without_parens: [foo: 1] + assert_same "foo(bar)", locals_without_parens: [foo: 1] + assert_same "foo bar", locals_without_parens: [foo: :*] + assert_same "foo(bar)", locals_without_parens: [foo: :*] + end + + test "without parens on unique argument" do + assert_same "foo(for 1, 2, 3)" + assert_same "foo(bar, for(1, 2, 3))" + assert_same "assert for 1, 2, 3" + assert_same "assert foo, for(1, 2, 3)" + + assert_same """ + assert for 1, 2, 3 do + :ok + end + """ + + assert_same """ + assert foo, for(1, 2, 3) do + :ok + end + """ + + assert_same """ + assert for(1, 2, 3) do + :ok + end + """ + + assert_same """ + assert (for 1, 2, 3 do + :ok + end) + """ + end + + test "call on call" do + assert_same "unquote(call)()" + assert_same "unquote(call)(one, two)" + + assert_same """ + unquote(call)() do + :ok + end + """ + + assert_same """ + unquote(call)(one, two) do + :ok + end + """ + end + + test "call on call on line limit" do + bad = "foo(bar)(one, two, three)" + + good = """ + foo(bar)( + one, + two, + three + ) + """ + + assert_format bad, good, @short_length + end + + test "with generators" do + assert_same "foo(bar <- baz, is_bat(bar))" + assert_same "for bar <- baz, is_bat(bar)" + + assert_same """ + foo( + bar <- baz, + is_bat(bar), + bat <- bar + ) + """ + + assert_same """ + for bar <- baz, + is_bat(bar), + bat <- bar + """ + + assert_same """ + for bar <- baz, + is_bat(bar), + bat <- bar do + :ok + end + """ + + assert_same """ + for bar <- baz, + is_bat(bar), + bat <- bar, + into: %{} + """ + end + + test "preserves user choice on parens even when it fits" do + assert_same """ + call( + :hello, + :foo, + :bar + ) + """ + + assert_same """ + call( + :hello, + :foo, + :bar + ) do + 1 + 2 + end + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "call(foo, bar, baz\n)", "call(foo, bar, baz)" + + # Doesn't preserve because there are no args + bad = """ + call() do + 1 + 2 + end + """ + + assert_format bad, """ + call do + 1 + 2 + end + """ + + # Doesn't preserve because we have a single argument with next break fits + bad = """ + call( + %{ + key: :value + } + ) + """ + + assert_format bad, """ + call(%{ + key: :value + }) + """ + end + end + + describe "remote calls" do + test "with no arguments" do + assert_format "Foo . Bar . baz", "Foo.Bar.baz()" + assert_format ":erlang.\nget_stacktrace", ":erlang.get_stacktrace()" + assert_format "@foo.bar", "@foo.bar" + assert_format "@foo.bar()", "@foo.bar()" + assert_format "(@foo).bar()", "@foo.bar()" + assert_format "__MODULE__.start_link", "__MODULE__.start_link()" + assert_format "Foo.bar.baz.bong", "Foo.bar().baz.bong" + assert_format "(1 + 2).foo", "(1 + 2).foo" + assert_format "(1 + 2).foo()", "(1 + 2).foo()" + end + + test "with arguments" do + assert_format "Foo . Bar. baz(1, 2, 3)", "Foo.Bar.baz(1, 2, 3)" + assert_format ":erlang.\nget(\n:some_key)", ":erlang.get(:some_key)" + assert_format ":erlang.\nget(:some_key\n)", ":erlang.get(:some_key)" + assert_same "@foo.bar(1, 2, 3)" + assert_same "__MODULE__.start_link(1, 2, 3)" + assert_same "foo.bar(1).baz(2, 3)" + end + + test "inspects function names correctly" do + assert_same ~S[MyModule."my function"(1, 2)] + assert_same ~S[MyModule."Foo.Bar"(1, 2)] + assert_same ~S[Kernel.+(1, 2)] + assert_same ~S[:erlang.+(1, 2)] + assert_same ~S[foo."bar baz"(1, 2)] + end + + test "splits on arguments and dot on line limit" do + bad = """ + MyModule.Foo.bar(:one, :two, :three) + """ + + good = """ + MyModule.Foo.bar( + :one, + :two, + :three + ) + """ + + assert_format bad, good, @medium_length + + bad = """ + My_function.foo().bar(2, 3).baz(4, 5) + """ + + good = """ + My_function.foo().bar( + 2, + 3 + ).baz(4, 5) + """ + + assert_format bad, good, @medium_length + end + + test "doesn't split on parens on empty arguments" do + assert_same "Mod.func()", @short_length + end + + test "with keyword lists" do + assert_same "mod.foo(foo: 1, bar: 2)" + + assert_same "mod.foo(:hello, foo: 1, bar: 2)" + + assert_same """ + mod.really_long_function_name( + :hello, + foo: 1, + bar: 2 + ) + """, + @short_length + + assert_same """ + really_long_module_name.foo( + :hello, + foo: 1, + bar: 2 + ) + """, + @short_length + end + + test "wraps left side in parens if it is an anonymous function" do + assert_same "(fn -> :ok end).foo" + end + + test "wraps left side in parens if it is a do-end block" do + assert_same """ + (if true do + :ok + end).foo + """ + end + + test "wraps left side in parens if it is a do-end block as an argument" do + assert_same """ + import (if true do + :ok + end).foo + """ + end + + test "call on call" do + assert_same "foo.bar(call)()" + assert_same "foo.bar(call)(one, two)" + + assert_same """ + foo.bar(call)() do + end + """ + + assert_same """ + foo.bar(call)(one, two) do + :ok + end + """ + end + + test "call on call on line limit" do + bad = "a.b(foo)(one, two, three)" + + good = """ + a.b(foo)( + one, + two, + three + ) + """ + + assert_format bad, good, @short_length + end + + test "on vars" do + assert_same "foo.bar" + assert_same "foo.bar()" + end + + test "on vars before blocks" do + assert_same """ + if var.field do + raise "oops" + end + """ + end + + test "on vars before brackets" do + assert_same """ + exception.opts[:foo] + """ + end + + test "preserves user choice on parens even when it fits" do + assert_same """ + Remote.call( + :hello, + :foo, + :bar + ) + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "Remote.call(foo, bar, baz\n)", "Remote.call(foo, bar, baz)" + + assert_same """ + Remote.call( + :hello, + :foo, + fn -> :bar end + ) + """ + end + end + + describe "anonymous function calls" do + test "without arguments" do + assert_format "foo . ()", "foo.()" + assert_format "(foo.()).().()", "foo.().().()" + assert_same "@foo.()" + assert_same "(1 + 1).()" + assert_same ":foo.()" + end + + test "with arguments" do + assert_format "foo . (1, 2 , 3 )", "foo.(1, 2, 3)" + assert_format "foo . (1, 2 ).(3,4)", "foo.(1, 2).(3, 4)" + assert_same "@foo.(:one, :two)" + assert_same "foo.(1 + 1).(hello)" + end + + test "does not split on dot on line limit" do + assert_same "my_function.()", @short_length + end + + test "splits on arguments on line limit" do + bad = """ + my_function.(1, 2, 3) + """ + + good = """ + my_function.( + 1, + 2, + 3 + ) + """ + + assert_format bad, good, @short_length + + bad = """ + my_function.(1, 2).f(3, 4).(5, 6) + """ + + good = """ + my_function.( + 1, + 2 + ).f(3, 4).( + 5, + 6 + ) + """ + + assert_format bad, good, @short_length + end + + test "with keyword lists" do + assert_same "foo.(foo: 1, bar: 2)" + + assert_same "foo.(:hello, foo: 1, bar: 2)" + + assert_same """ + foo.( + :hello, + foo: 1, + bar: 2 + ) + """, + @short_length + end + + test "wraps left side in parens if it is an anonymous function" do + assert_same "(fn -> :ok end).()" + end + + test "wraps left side in parens if it is a do-end block" do + assert_same """ + (if true do + :ok + end).() + """ + end + + test "wraps left side in parens if it is a do-end block as an argument" do + assert_same """ + import (if true do + :ok + end).() + """ + end + + test "preserves user choice on parens even when it fits" do + assert_same """ + call.( + :hello, + :foo, + :bar + ) + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "call.(foo, bar, baz\n)", "call.(foo, bar, baz)" + end + end + + describe "do-end blocks" do + test "with non-block keywords" do + assert_same "foo(do: nil)" + end + + test "with forced block keywords" do + good = """ + foo do + nil + end + """ + + assert_format "foo(do: nil)", good, force_do_end_blocks: true + + # Avoid false positives + assert_same "foo(do: 1, do: 2)", force_do_end_blocks: true + assert_same "foo(do: 1, another: 2)", force_do_end_blocks: true + end + + test "with multiple keywords" do + assert_same """ + foo do + :do + rescue + :rescue + catch + :catch + else + :else + after + :after + end + """ + end + + test "with multiple keywords and arrows" do + assert_same """ + foo do + a1 -> a2 + b1 -> b2 + rescue + a1 -> a2 + b1 -> b2 + catch + a1 -> a2 + b1 -> b2 + else + a1 -> a2 + b1 -> b2 + after + a1 -> a2 + b1 -> b2 + end + """ + end + + test "with no extra arguments" do + assert_same """ + foo do + :ok + end + """ + end + + test "with no extra arguments and line breaks" do + assert_same """ + foo do + a1 -> + really_long_line + + b1 -> + b2 + rescue + c1 + catch + d1 -> d1 + e1 -> e1 + else + f2 + after + g1 -> + really_long_line + + h1 -> + h2 + end + """, + @medium_length + end + + test "with extra arguments" do + assert_same """ + foo bar, baz do + :ok + end + """ + end + + test "with extra arguments and line breaks" do + assert_same """ + foo bar do + a1 -> + really_long_line + + b1 -> + b2 + rescue + c1 + catch + d1 -> d1 + e1 -> e1 + else + f2 + after + g1 -> + really_long_line + + h1 -> + h2 + end + """, + @medium_length + + assert_same """ + foo really, + long, + list, + of, + arguments do + a1 -> + really_long_line + + b1 -> + b2 + rescue + c1 + catch + d1 -> d1 + e1 -> e1 + else + f2 + after + g1 -> + really_long_line + + h1 -> + h2 + end + """, + @medium_length + end + + test "when empty" do + assert_same """ + foo do + end + """ + + assert_same """ + foo do + rescue + catch + else + after + end + """ + end + + test "inside call" do + bad = "foo (bar do :ok end)" + + good = """ + foo( + bar do + :ok + end + ) + """ + + assert_format bad, good + + bad = "import (bar do :ok end)" + + good = """ + import (bar do + :ok + end) + """ + + assert_format bad, good + end + + test "inside operator" do + bad = "foo + bar do :ok end" + + good = """ + foo + + bar do + :ok + end + """ + + assert_format bad, good + end + + test "inside operator inside argument" do + bad = "fun foo + (bar do :ok end)" + + good = """ + fun( + foo + + bar do + :ok + end + ) + """ + + assert_format bad, good + + bad = "if foo + (bar do :ok end) do :ok end" + + good = """ + if foo + + (bar do + :ok + end) do + :ok + end + """ + + assert_format bad, good + end + + test "inside operator inside argument with remote call" do + bad = "if foo + (Bar.baz do :ok end) do :ok end" + + good = """ + if foo + + (Bar.baz do + :ok + end) do + :ok + end + """ + + assert_format bad, good + end + + test "keeps repeated keys" do + assert_same """ + receive do + :ok + after + 0 -> 1 + after + 2 -> 3 + end + """ + end + + test "preserves user choice even when it fits" do + assert_same """ + case do + 1 -> + :ok + + 2 -> + :ok + end + """ + + assert_same """ + case do + 1 -> + :ok + + 2 -> + :ok + + 3 -> + :ok + end + """ + end + end + + describe "tuple calls" do + test "without arguments" do + assert_format "foo . {}", "foo.{}" + end + + test "with arguments" do + assert_format "foo.{bar,baz,bat,}", "foo.{bar, baz, bat}" + end + + test "with arguments on line limit" do + bad = "foo.{bar,baz,bat,}" + + good = """ + foo.{ + bar, + baz, + bat + } + """ + + assert_format bad, good, @short_length + + bad = "really_long_expression.{bar,baz,bat,}" + + good = """ + really_long_expression.{ + bar, + baz, + bat + } + """ + + assert_format bad, good, @short_length + end + + test "with keywords" do + assert_same "expr.{:hello, foo: bar, baz: bat}" + end + + test "preserves user choice on parens even when it fits" do + assert_same """ + call.{ + :hello, + :foo, + :bar + } + """ + + assert_format "call.{foo, bar, baz\n}", "call.{foo, bar, baz}" + end + end + + describe "access" do + test "with one argument" do + assert_format "foo[ bar ]", "foo[bar]" + end + + test "with arguments on line limit" do + bad = "foo[really_long_argument()]" + + good = """ + foo[ + really_long_argument() + ] + """ + + assert_format bad, good, @short_length + + bad = "really_long_expression[really_long_argument()]" + + good = """ + really_long_expression[ + really_long_argument() + ] + """ + + assert_format bad, good, @short_length + end + + test "with do-end blocks" do + assert_same """ + (if true do + false + end)[key] + """ + end + + test "with keywords" do + assert_format "expr[[]]", "expr[[]]" + assert_format "expr[foo: bar, baz: bat]", "expr[foo: bar, baz: bat]" + assert_format "expr[[foo: bar, baz: bat]]", "expr[[foo: bar, baz: bat]]" + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/comments_test.exs b/lib/elixir/test/elixir/code_formatter/comments_test.exs new file mode 100644 index 00000000000..09c79b46f2f --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/comments_test.exs @@ -0,0 +1,1549 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.CommentsTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + + describe "at the root" do + test "for empty documents" do + assert_same "# hello world" + end + + test "are reformatted" do + assert_format "#oops", "# oops" + assert_format "##oops", "## oops" + assert_same "# ## oops" + end + + test "recognizes hashbangs" do + assert_format "#! /usr/bin/env elixir", "#! /usr/bin/env elixir" + assert_format "#!/usr/bin/env elixir", "#!/usr/bin/env elixir" + assert_same "#!" + end + + test "before and after expressions" do + assert_same """ + # before comment + :hello + """ + + assert_same """ + :hello + # after comment + """ + + assert_same """ + # before comment + :hello + # after comment + """ + end + + test "on expressions" do + bad = """ + :hello # this is hello + :world # this is world + """ + + good = """ + # this is hello + :hello + # this is world + :world + """ + + assert_format bad, good + + bad = """ + foo # this is foo + ++ bar # this is bar + ++ baz # this is baz + """ + + good = """ + # this is foo + # this is bar + # this is baz + foo ++ + bar ++ + baz + """ + + assert_format bad, good, @short_length + end + + test "empty comment" do + assert_same """ + # + :foo + """ + end + + test "before and after expressions with newlines" do + assert_same """ + # before comment + # second line + + :hello + + # middle comment 1 + + # + + # middle comment 2 + + :world + + # after comment + # second line + """ + end + end + + describe "modules attributes" do + test "with comments around" do + assert_same """ + defmodule Sample do + # Comment 0 + @moduledoc false + # Comment 1 + + # Comment 2 + @attr1 1 + # Comment 3 + + # Comment 4 + @doc "Doc" + # Comment 5 + @attr2 2 + # Comment 6 + def sample, do: :sample + end + """ + end + + test "with comments only after" do + assert_same """ + @moduledoc false + # Comment 1 + + @attr 1 + """ + end + + test "with too many new lines" do + bad = """ + defmodule Sample do + + # Comment 0 + + + @moduledoc false + + + # Comment 1 + + + # Comment 2 + + + @attr1 1 + + + # Comment 3 + + + # Comment 4 + + + @doc "Doc" + + + # Comment 5 + + + @attr2 2 + + + # Comment 6 + + + def sample, do: :sample + end + """ + + assert_format bad, """ + defmodule Sample do + # Comment 0 + + @moduledoc false + + # Comment 1 + + # Comment 2 + + @attr1 1 + + # Comment 3 + + # Comment 4 + + @doc "Doc" + + # Comment 5 + + @attr2 2 + + # Comment 6 + + def sample, do: :sample + end + """ + end + end + + describe "interpolation" do + test "with comment outside before, during and after" do + assert_same ~S""" + # comment + IO.puts("Hello #{world}") + """ + + assert_same ~S""" + IO.puts("Hello #{world}") + # comment + """ + end + + test "with trailing comments" do + # This is trailing so we move the comment out + trailing = ~S""" + IO.puts("Hello #{world}") # comment + """ + + assert_format trailing, ~S""" + # comment + IO.puts("Hello #{world}") + """ + + # This is ambiguous so we move the comment out + ambiguous = ~S""" + IO.puts("Hello #{world # comment + }") + """ + + assert_format ambiguous, ~S""" + # comment + IO.puts("Hello #{world}") + """ + end + + test "with comment inside before and after" do + bad = ~S""" + IO.puts( + "Hello #{ + # comment + world + }" + ) + """ + + good = ~S""" + IO.puts( + # comment + "Hello #{world}" + ) + """ + + assert_format bad, good + + bad = ~S""" + IO.puts( + "Hello #{ + world + # comment + }" + ) + """ + + good = ~S""" + IO.puts( + "Hello #{world}" + # comment + ) + """ + + assert_format bad, good + + bad = ~S""" + IO.puts("Hello #{hello + world}") + """ + + good = ~S""" + IO.puts( + "Hello #{hello + world}" + ) + """ + + assert_format bad, good + end + end + + describe "parens blocks" do + test "with comment outside before and after" do + assert_same ~S""" + # comment + assert ( + hello + world + ) + """ + + assert_same ~S""" + assert ( + hello + world + ) + + # comment + """ + end + + test "with trailing comments" do + # This is ambiguous so we move the comment out + ambiguous = ~S""" + assert ( # comment + hello + world + ) + """ + + assert_format ambiguous, ~S""" + # comment + assert ( + hello + world + ) + """ + + # This is ambiguous so we move the comment out + ambiguous = ~S""" + assert ( + hello + world + ) # comment + """ + + assert_format ambiguous, ~S""" + assert ( + hello + world + ) + + # comment + """ + end + + test "with comment inside before and after" do + assert_same ~S""" + assert ( + # comment + hello + world + ) + """ + + assert_same ~S""" + assert ( + hello + world + # comment + ) + """ + end + end + + describe "access" do + test "before and after single arg" do + assert_same ~S""" + foo[ + # bar + baz + # bat + ] + """ + end + + test "before and after keywords" do + assert_same ~S""" + foo[ + # bar + one: :two, + # baz + three: :four + # bat + ] + """ + end + end + + describe "calls" do + test "local with parens inside before and after" do + assert_same ~S""" + call( + # before + hello, + # middle + world + # after + ) + """ + + assert_same ~S""" + call( + # command + ) + """ + end + + test "remote with parens inside before and after" do + assert_same ~S""" + Remote.call( + # before + hello, + # middle + world + # after + ) + """ + + assert_same ~S""" + Remote.call( + # command + ) + """ + end + + test "local with parens and keywords inside before and after" do + assert_same ~S""" + call( + # before + hello, + # middle + world, + # key before + key: hello, + # key middle + key: world + # key after + ) + """ + end + + test "remote with parens and keywords inside before and after" do + assert_same ~S""" + call( + # before + hello, + # middle + world, + # key before + key: hello, + # key middle + key: world + # key after + ) + """ + end + + test "local with no parens inside before and after" do + bad = ~S""" + # before + assert hello, + # middle + world + # after + """ + + assert_format bad, ~S""" + # before + assert hello, + # middle + world + + # after + """ + end + + test "local with no parens and keywords inside before and after" do + bad = ~S""" + config hello, world, + # key before + key: hello, + # key middle + key: world + # key after + """ + + assert_format bad, ~S""" + config hello, world, + # key before + key: hello, + # key middle + key: world + + # key after + """ + + bad = ~S""" + # before + config hello, + # middle + world, + # key before + key: hello, + # key middle + key: world + # key after + """ + + assert_format bad, ~S""" + # before + config hello, + # middle + world, + # key before + key: hello, + # key middle + key: world + + # key after + """ + end + end + + describe "anonymous functions" do + test "with one clause and no args" do + assert_same ~S""" + fn -> + # comment + hello + world + end + """ + + assert_same ~S""" + fn -> + hello + world + # comment + end + """ + end + + test "with one clause and no args and trailing comments" do + bad = ~S""" + fn # comment + -> + hello + world + end + """ + + assert_format bad, ~S""" + # comment + fn -> + hello + world + end + """ + + bad = ~S""" + fn + # comment + -> + hello + world + end + """ + + assert_format bad, ~S""" + # comment + fn -> + hello + world + end + """ + end + + test "with one clause and args" do + assert_same ~S""" + fn hello -> + # before + hello + # middle + world + # after + end + """ + end + + test "with one clause and args and trailing comments" do + bad = ~S""" + fn # fn + # before head + hello # middle head + # after head + -> + # before body + world # middle body + # after body + end + """ + + assert_format bad, ~S""" + # fn + fn + # before head + # middle head + hello -> + # after head + # before body + # middle body + world + # after body + end + """ + end + + test "with multiple clauses and args" do + bad = ~S""" + fn # fn + # before one + one, # middle one + # after one / before two + two # middle two + # after two + -> + # before hello + hello # middle hello + # after hello + + # before three + three # middle three + # after three + -> + # before world + world # middle world + # after world + end + """ + + assert_format bad, ~S""" + # fn + fn + # before one + # middle one + # after one / before two + # middle two + one, two -> + # after two + # before hello + # middle hello + hello + + # after hello + + # before three + # middle three + three -> + # after three + # before world + # middle world + world + # after world + end + """ + end + + test "with commented out clause" do + assert_same """ + fn + arg1 -> + body1 + + # arg2 -> + # body 2 + + arg3 -> + body3 + end + """ + end + end + + describe "do-end blocks" do + test "with comment outside before and after" do + assert_same ~S""" + # comment + assert do + hello + world + end + """ + + assert_same ~S""" + assert do + hello + world + end + + # comment + """ + end + + test "with trailing comments" do + # This is ambiguous so we move the comment out + ambiguous = ~S""" + assert do # comment + hello + world + end + """ + + assert_format ambiguous, ~S""" + # comment + assert do + hello + world + end + """ + + # This is ambiguous so we move the comment out + ambiguous = ~S""" + assert do + hello + world + end # comment + """ + + assert_format ambiguous, ~S""" + assert do + hello + world + end + + # comment + """ + end + + test "with comment inside before and after" do + assert_same ~S""" + assert do + # comment + hello + world + end + """ + + assert_same ~S""" + assert do + hello + world + # comment + end + """ + end + + test "with comment inside before and after and multiple keywords" do + assert_same ~S""" + assert do + # before + hello + world + # after + rescue + # before + hello + world + # after + after + # before + hello + world + # after + catch + # before + hello + world + # after + else + # before + hello + world + # after + end + """ + end + + test "when empty" do + assert_same ~S""" + assert do + # comment + end + """ + + assert_same ~S""" + assert do + # comment + rescue + # comment + after + # comment + catch + # comment + else + # comment + end + """ + end + + test "with one-line clauses" do + bad = ~S""" + assert do # do + # before + one -> two + end + """ + + assert_format bad, ~S""" + # do + assert do + # before + one -> two + end + """ + + bad = ~S""" + assert do # do + # before + one -> two + # after + three -> four + end + """ + + assert_format bad, ~S""" + # do + assert do + # before + one -> two + # after + three -> four + end + """ + end + + test "with multiple clauses and args" do + bad = ~S""" + assert do # do + # before one + one, # middle one + # after one / before two + two # middle two + # after two + -> + # before hello + hello # middle hello + # after hello + + # before three + three # middle three + # after three + -> + # before world + world # middle world + # after world + end + """ + + assert_format bad, ~S""" + # do + assert do + # before one + # middle one + # after one / before two + # middle two + one, two -> + # after two + # before hello + # middle hello + hello + + # after hello + + # before three + # middle three + three -> + # after three + # before world + # middle world + world + # after world + end + """ + end + end + + describe "operators" do + test "with comment before, during and after uniform pipelines" do + assert_same """ + foo + # |> bar + # |> baz + |> bat + """ + + bad = """ + # before + foo # this is foo + |> bar # this is bar + |> baz # this is baz + # after + """ + + good = """ + # before + # this is foo + foo + # this is bar + |> bar + # this is baz + |> baz + + # after + """ + + assert_format bad, good, @short_length + end + + test "with comment before, during and after mixed pipelines" do + assert_same """ + foo + # |> bar + # |> baz + ~> bat + """ + + bad = """ + # before + foo # this is foo + ~> bar # this is bar + <~> baz # this is baz + # after + """ + + good = """ + # before + # this is foo + foo + # this is bar + ~> bar + # this is baz + <~> baz + + # after + """ + + assert_format bad, good, @short_length + end + + test "with comment before, during and after uniform right" do + assert_same """ + foo + # | bar + # | baz + | bat + """ + + bad = """ + # before + foo # this is foo + | bar # this is bar + | baz # this is baz + # after + """ + + good = """ + # before + # this is foo + foo + # this is bar + | bar + # this is baz + | baz + + # after + """ + + assert_format bad, good, @short_length + end + + test "with comment before, during and after mixed right" do + assert_same """ + one + # when two + # when three + when four + # | five + | six + """ + end + + test "handles nodes without meta info" do + assert_same "(a -> b) |> (c -> d)" + assert_same "(a -> b) when c: d" + assert_same "(a -> b) when (c -> d)" + end + end + + describe "containers" do + test "with comment outside before, during and after" do + assert_same ~S""" + # comment + [one, two, three] + """ + + assert_same ~S""" + [one, two, three] + # comment + """ + end + + test "with trailing comments" do + # This is trailing so we move the comment out + trailing = ~S""" + [one, two, three] # comment + """ + + assert_format trailing, ~S""" + # comment + [one, two, three] + """ + + # This is ambiguous so we move the comment out + ambiguous = ~S""" + [# comment + one, two, three] + """ + + assert_format ambiguous, ~S""" + # comment + [ + one, + two, + three + ] + """ + end + + test "when empty" do + assert_same ~S""" + [ + # comment + ] + """ + end + + test "with block" do + assert_same ~S""" + [ + ( + # before + multi + line + # after + ) + ] + """ + end + + test "with comments inside lists before and after" do + bad = ~S""" + [ + # 1. one + + # 1. two + # 1. three + one, + after_one, + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + three # final + + # 4. one + + # 4. two + # 4. three + # four + ] + """ + + good = ~S""" + [ + # 1. one + + # 1. two + # 1. three + one, + after_one, + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + # final + three + + # 4. one + + # 4. two + # 4. three + # four + ] + """ + + assert_format bad, good + end + + test "with comments inside tuples before and after" do + bad = ~S""" + { + # 1. one + + # 1. two + # 1. three + one, + after_one, + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + three # final + + # 4. one + + # 4. two + # 4. three + # four + } + """ + + good = ~S""" + { + # 1. one + + # 1. two + # 1. three + one, + after_one, + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + # final + three + + # 4. one + + # 4. two + # 4. three + # four + } + """ + + assert_format bad, good + end + + test "with comments inside bitstrings before and after" do + bad = ~S""" + << + # 1. one + + # 1. two + # 1. three + one, + after_one, + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + three # final + + # 4. one + + # 4. two + # 4. three + # four + >> + """ + + good = ~S""" + << + # 1. one + + # 1. two + # 1. three + one, + after_one, + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + # final + three + + # 4. one + + # 4. two + # 4. three + # four + >> + """ + + assert_format bad, good + end + + test "with comments inside maps before and after" do + bad = ~S""" + %{ + # 1. one + + # 1. two + # 1. three + one: one, + one: after_one, + one: after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + three: three # final + + # 4. one + + # 4. two + # 4. three + # four + } + """ + + good = ~S""" + %{ + # 1. one + + # 1. two + # 1. three + one: one, + one: after_one, + one: + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + # final + three: three + + # 4. one + + # 4. two + # 4. three + # four + } + """ + + assert_format bad, good + end + + test "with comments inside structs before and after" do + bad = ~S""" + %Foo{bar | + # 1. one + + # 1. two + # 1. three + one: one, + one: after_one, + one: after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + three: three # final + + # 4. one + + # 4. two + # 4. three + # four + } + """ + + good = ~S""" + %Foo{ + bar + | # 1. one + + # 1. two + # 1. three + one: one, + one: after_one, + one: + after_one do + :ok + end, + + # 2. one + + # 2. two + # 2. three + # two, + + # 3. one + + # 3. two + # 3. three + # final + three: three + + # 4. one + + # 4. two + # 4. three + # four + } + """ + + assert_format bad, good + end + end + + describe "defstruct" do + test "with first field comments" do + bad = ~S""" + defmodule Foo do + # defstruct + defstruct [ # foo + # 1. one + one: 1, # 2. one + # 1. two + # 2. two + two: 2 + ] + end + """ + + good = ~S""" + defmodule Foo do + # defstruct + # foo + defstruct [ + # 1. one + # 2. one + one: 1, + # 1. two + # 2. two + two: 2 + ] + end + """ + + assert_format bad, good + end + + test "with first field comments and defstruct has the parens" do + bad = ~S""" + defmodule Foo do + # defstruct + defstruct([ # foo + # 1. one + one: 1, # 2. one + # 1. two + # 2. two + two: 2 + ]) + end + """ + + good = ~S""" + defmodule Foo do + # defstruct + # foo + defstruct( + # 1. one + # 2. one + one: 1, + # 1. two + # 2. two + two: 2 + ) + end + """ + + assert_format bad, good + end + + test "without first field comments" do + bad = ~S""" + defmodule Foo do + # defstruct + defstruct [ + one: 1, + # 1. two + two: 2 # 2. two + ] + end + """ + + good = ~S""" + defmodule Foo do + # defstruct + defstruct one: 1, + # 1. two + # 2. two + two: 2 + end + """ + + assert_format bad, good + end + + test "without field comments" do + bad = ~S""" + defmodule Foo do + # defstruct + defstruct [ + one: 1, + two: 2 + ] + end + """ + + good = ~S""" + defmodule Foo do + # defstruct + defstruct one: 1, + two: 2 + end + """ + + assert_format bad, good + end + + test "without square brackets" do + assert_same ~S""" + defmodule Foo do + # defstruct + defstruct one: 1, + # 1. two + # 2. two + two: 2 + end + """ + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/containers_test.exs b/lib/elixir/test/elixir/code_formatter/containers_test.exs new file mode 100644 index 00000000000..66959edaa95 --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/containers_test.exs @@ -0,0 +1,673 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.ContainersTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + @medium_length [line_length: 20] + + describe "tuples" do + test "without arguments" do + assert_format "{ }", "{}" + end + + test "with arguments" do + assert_format "{1,2}", "{1, 2}" + assert_format "{1,2,3}", "{1, 2, 3}" + end + + test "is flex on line limits" do + bad = "{1, 2, 3, 4}" + + good = """ + {1, 2, 3, + 4} + """ + + assert_format bad, good, @short_length + end + + test "removes trailing comma" do + assert_format "{1,}", "{1}" + assert_format "{1, 2, 3,}", "{1, 2, 3}" + end + + test "with keyword lists" do + # The one below is not valid syntax + # assert_same "{foo: 1, bar: 2}" + assert_same "{:hello, foo: 1, bar: 2}" + + tuple = """ + { + :hello, + foo: 1, + bar: 2 + } + """ + + assert_same tuple, @short_length + end + + test "preserves user choice even when it fits" do + assert_same """ + { + :hello, + :foo, + :bar + } + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "{foo, bar, baz\n}", "{foo, bar, baz}" + end + + test "preserves user choice even when it fits with trailing comma" do + bad = """ + { + :hello, + :foo, + :bar, + } + """ + + assert_format bad, """ + { + :hello, + :foo, + :bar + } + """ + end + end + + describe "lists" do + test "empty" do + assert_format "[ ]", "[]" + assert_format "[\n]", "[]" + end + + test "with elements" do + assert_format "[ 1 , 2,3, 4 ]", "[1, 2, 3, 4]" + end + + test "with tail" do + assert_format "[1,2,3|4]", "[1, 2, 3 | 4]" + end + + test "are strict on line limit" do + bad = """ + [11, 22, 33, 44] + """ + + good = """ + [ + 11, + 22, + 33, + 44 + ] + """ + + assert_format bad, good, @short_length + + bad = """ + [11, 22, 33 | 44] + """ + + good = """ + [ + 11, + 22, + 33 | 44 + ] + """ + + assert_format bad, good, @short_length + + bad = """ + [1, 2, 3 | 4] + """ + + good = """ + [ + 1, + 2, + 3 | 4 + ] + """ + + assert_format bad, good, @short_length + + bad = """ + [1, 2, 3 | really_long_expression()] + """ + + good = """ + [ + 1, + 2, + 3 + | really_long_expression() + ] + """ + + assert_format bad, good, @short_length + end + + test "removes trailing comma" do + assert_format "[1,]", "[1]" + assert_format "[1, 2, 3,]", "[1, 2, 3]" + end + + test "with keyword lists" do + assert_same "[foo: 1, bar: 2]" + assert_same "[:hello, foo: 1, bar: 2]" + + # Pseudo keyword lists are kept as is + assert_same "[{:foo, 1}, {:bar, 2}]" + + keyword = """ + [ + foo: 1, + bar: 2 + ] + """ + + assert_same keyword, @short_length + end + + test "with keyword lists on comma line limit" do + bad = """ + [ + foooo: 1, + barrr: 2 + ] + """ + + good = """ + [ + foooo: + 1, + barrr: 2 + ] + """ + + assert_format bad, good, @short_length + end + + test "with quoted keyword lists" do + assert_same ~S(["with spaces": 1]) + assert_same ~S(["one #{two} three": 1]) + assert_same ~S(["\w": 1, "\\w": 2]) + assert_same ~S(["Elixir.Foo": 1, "Elixir.Bar": 2]) + assert_format ~S(["Foo": 1, "Bar": 2]), ~S([Foo: 1, Bar: 2]) + end + + test "with operators keyword lists" do + assert_same ~S([.: :.]) + assert_same ~S([..: :..]) + assert_same ~S([...: :...]) + end + + test "preserves user choice even when it fits" do + assert_same """ + [ + :hello, + :foo, + :bar + ] + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "[foo, bar, baz\n]", "[foo, bar, baz]" + end + + test "preserves user choice even when it fits with trailing comma" do + bad = """ + [ + :hello, + :foo, + :bar, + ] + """ + + assert_format bad, """ + [ + :hello, + :foo, + :bar + ] + """ + end + end + + describe "bitstrings" do + test "without arguments" do + assert_format "<< >>", "<<>>" + assert_format "<<\n>>", "<<>>" + end + + test "with arguments" do + assert_format "<<1,2,3>>", "<<1, 2, 3>>" + end + + test "add parens on first and last in case of binary ambiguity" do + assert_format "<< <<>> >>", "<<(<<>>)>>" + assert_format "<< <<>> + <<>> >>", "<<(<<>> + <<>>)>>" + assert_format "<< 1 + <<>> >>", "<<(1 + <<>>)>>" + assert_format "<< <<>> + 1 >>", "<<(<<>> + 1)>>" + assert_format "<< <<>>, <<>>, <<>> >>", "<<(<<>>), <<>>, (<<>>)>>" + assert_format "<< <<>>::1, <<>>::2, <<>>::3 >>", "<<(<<>>)::1, <<>>::2, <<>>::3>>" + assert_format "<< <<>>::<<>> >>", "<<(<<>>)::(<<>>)>>" + end + + test "add parens on first in case of operator ambiguity" do + assert_format "<< ~~~1::8 >>", "<<(~~~1)::8>>" + assert_format "<< ~s[foo]::binary >>", "<<(~s[foo])::binary>>" + end + + test "with modifiers" do + assert_format "<< 1 :: 1 >>", "<<1::1>>" + assert_format "<< 1 :: 2 + 3 >>", "<<1::(2 + 3)>>" + assert_format "<< 1 :: 2 - integer >>", "<<1::2-integer>>" + assert_format "<< 1 :: 2 - unit(3) >>", "<<1::2-unit(3)>>" + assert_format "<< 1 :: 2 * 3 - unit(4) >>", "<<1::2*3-unit(4)>>" + assert_format "<< 1 :: 2 - unit(3) - 4 / 5 >>", "<<1::2-unit(3)-(4 / 5)>>" + end + + test "in comprehensions" do + assert_format "<< 0, 1 :: 1 <- x >>", "<<0, 1::1 <- x>>" + assert_format "<< 0, 1 :: 2 + 3 <- x >>", "<<0, 1::(2 + 3) <- x>>" + assert_format "<< 0, 1 :: 2 - integer <- x >>", "<<0, 1::2-integer <- x>>" + assert_format "<< 0, 1 :: 2 - unit(3) <- x >>", "<<0, 1::2-unit(3) <- x>>" + assert_format "<< 0, 1 :: 2 * 3 - unit(4) <- x >>", "<<0, 1::2*3-unit(4) <- x>>" + assert_format "<< 0, 1 :: 2 - unit(3) - 4 / 5 <- x >>", "<<0, 1::2-unit(3)-(4 / 5) <- x>>" + + assert_same "<<(<> <- <>)>>" + assert_same "<<(y <- <>)>>" + assert_same "<<(<> <- x)>>" + end + + test "normalizes bitstring modifiers by default" do + assert_format "<>", "<>" + assert_same "<>" + + assert_format "<>", "<>" + assert_same "<>" + + assert_format "<>", "<>" + assert_same "<>" + + assert_format "<<0, 1::2-integer() <- x>>", "<<0, 1::2-integer <- x>>" + assert_same "<<0, 1::2-integer <- x>>" + end + + test "keeps parentheses when normalize_bitstring_modifiers is false" do + opts = [normalize_bitstring_modifiers: false] + + assert_same "<>", opts + assert_same "<>", opts + + assert_same "<>", opts + assert_same "<>", opts + + assert_same "<>", opts + assert_same "<<0, 1::2-integer() <- x>>", opts + end + + test "is flex on line limits" do + bad = "<<1, 2, 3, 4>>" + + good = """ + <<1, 2, 3, + 4>> + """ + + assert_format bad, good, @short_length + end + + test "preserves user choice even when it fits" do + assert_same """ + << + :hello, + :foo, + :bar + >> + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "<>", "<>" + end + + test "preserves user choice even when it fits with trailing comma" do + bad = """ + << + :hello, + :foo, + :bar, + >> + """ + + assert_format bad, """ + << + :hello, + :foo, + :bar + >> + """ + end + end + + describe "maps" do + test "without arguments" do + assert_format "%{ }", "%{}" + end + + test "with arguments" do + assert_format "%{1 => 2,3 => 4}", "%{1 => 2, 3 => 4}" + end + + test "is strict on line limits" do + bad = "%{1 => 2, 3 => 4}" + + good = """ + %{ + 1 => 2, + 3 => 4 + } + """ + + assert_format bad, good, @short_length + + map = """ + %{ + a(1, 2) => b, + c(3, 4) => d + } + """ + + assert_same map, @medium_length + + map = """ + %{ + a => fn x -> + y + end, + b => fn y -> + z + end + } + """ + + assert_same map, @medium_length + + map = """ + %{ + a => + for( + y <- x, + z <- y, + do: 123 + ) + } + """ + + assert_same map, @medium_length + + map = """ + %{ + a => + for do + :ok + end + } + """ + + assert_same map, @short_length + end + + test "removes trailing comma" do + assert_format "%{1 => 2,}", "%{1 => 2}" + end + + test "with keyword lists" do + assert_same "%{:foo => :bar, baz: :bat}" + + map = """ + %{ + :foo => :bar, + baz: :bat + } + """ + + assert_same map, @medium_length + end + + test "preserves user choice in regards to =>" do + assert_same "%{:hello => 1, :world => 2}" + assert_format "%{:true => 1, :false => 2}", "%{true => 1, false => 2}" + end + + test "preserves user choice even when it fits" do + assert_same """ + %{ + :hello => 1, + :foo => 2, + :bar => 3 + } + """ + + # Doesn't preserve this because only the ending has a newline + assert_format "%{foo: 1, bar: 2\n}", "%{foo: 1, bar: 2}" + end + + test "preserves user choice even when it fits with trailing comma" do + bad = """ + %{ + hello, + foo, + bar, + } + """ + + assert_format bad, """ + %{ + hello, + foo, + bar + } + """ + end + + test "preserves user choice when a newline is used after keyword" do + good = """ + %{ + hello: + {:ok, :world} + } + """ + + assert_same good, @medium_length + end + + test "preserves user choice when a newline is used after assoc" do + good = """ + %{ + hello => + {:ok, :world} + } + """ + + assert_same good, @medium_length + end + end + + describe "maps with update" do + test "with arguments" do + assert_format "%{foo | 1 => 2,3 => 4}", "%{foo | 1 => 2, 3 => 4}" + end + + test "is strict on line limits" do + bad = "%{foo | 1 => 2, 3 => 4}" + + good = """ + %{ + foo + | 1 => 2, + 3 => 4 + } + """ + + assert_format bad, good, line_length: 11 + end + + test "removes trailing comma" do + assert_format "%{foo | 1 => 2,}", "%{foo | 1 => 2}" + end + + test "with keyword lists" do + assert_same "%{foo | :foo => :bar, baz: :bat}" + + map = """ + %{ + foo + | :foo => :bar, + baz: :bat + } + """ + + assert_same map, @medium_length + end + + test "preserves user choice even when it fits" do + assert_same """ + %{ + foo + | :hello => 1, + :foo => 2, + :bar => 3 + } + """ + end + + test "wraps operators in parens" do + assert_format "%{foo && bar | baz: :bat}", "%{(foo && bar) | baz: :bat}" + assert_same "%{@foo | baz: :bat}" + end + end + + describe "structs" do + test "without arguments" do + assert_format "%struct{ }", "%struct{}" + end + + test "with arguments" do + assert_format "%struct{1 => 2,3 => 4}", "%struct{1 => 2, 3 => 4}" + end + + test "is strict on line limits" do + bad = "%struct{1 => 2, 3 => 4}" + + good = """ + %struct{ + 1 => 2, + 3 => 4 + } + """ + + assert_format bad, good, @short_length + end + + test "removes trailing comma" do + assert_format "%struct{1 => 2,}", "%struct{1 => 2}" + end + + test "with keyword lists" do + assert_same "%struct{:foo => :bar, baz: :bat}" + + struct = """ + %struct{ + :foo => :bar, + baz: :bat + } + """ + + assert_same struct, @medium_length + end + + test "preserves user choice even when it fits" do + assert_same """ + %Foo{ + :hello => 1, + :foo => 2, + :bar => 3 + } + """ + end + end + + describe "struct with update" do + test "with arguments" do + assert_format "%struct{foo | 1 => 2,3 => 4}", "%struct{foo | 1 => 2, 3 => 4}" + end + + test "is strict on line limits" do + bad = "%struct{foo | 1 => 2, 3 => 4}" + + good = """ + %struct{ + foo + | 1 => 2, + 3 => 4 + } + """ + + assert_format bad, good, line_length: 11 + end + + test "removes trailing comma" do + assert_format "%struct{foo | 1 => 2,}", "%struct{foo | 1 => 2}" + end + + test "with keyword lists" do + assert_same "%struct{foo | :foo => :bar, baz: :bat}" + + struct = """ + %struct{ + foo + | :foo => :bar, + baz: :bat + } + """ + + assert_same struct, @medium_length + end + + test "preserves user choice even when it fits" do + assert_same """ + %Foo{ + foo + | :hello => 1, + :foo => 2, + :bar => 3 + } + """ + end + + test "converges" do + bad = "hello_world(%struct{foo | 1 => 2, 3 => 4})" + + good = """ + hello_world(%struct{ + foo + | 1 => 2, + 3 => 4 + }) + """ + + assert_format bad, good, line_length: 30 + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/general_test.exs b/lib/elixir/test/elixir/code_formatter/general_test.exs new file mode 100644 index 00000000000..50c44a3df04 --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/general_test.exs @@ -0,0 +1,895 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.GeneralTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + + test "does not emit warnings" do + assert_format "fn -> end", "fn -> nil end" + end + + describe "unicode normalization" do + test "with nfc normalizations" do + assert_format "ç", "ç" + end + + test "with custom normalizations" do + assert_format "µs", "μs" + end + end + + describe "aliases" do + test "with atom-only parts" do + assert_same "Elixir" + assert_same "Elixir.Foo" + assert_same "Foo.Bar.Baz" + end + + test "removes spaces between aliases" do + assert_format "Foo . Bar . Baz", "Foo.Bar.Baz" + end + + test "starting with expression" do + assert_same "__MODULE__.Foo.Bar" + # Syntactically valid, semantically invalid + assert_same "'Foo'.Bar.Baz" + end + + test "wraps the head in parens if it has an operator" do + assert_format "+(Foo . Bar . Baz)", "+Foo.Bar.Baz" + assert_format "(+Foo) . Bar . Baz", "(+Foo).Bar.Baz" + end + end + + describe "sigils" do + test "without interpolation" do + assert_same ~S[~s(foo)] + assert_same ~S[~s{foo bar}] + assert_same ~S[~r/Bar Baz/] + assert_same ~S[~w<>] + assert_same ~S[~W()] + end + + test "with escapes" do + assert_same ~S[~s(foo \) bar)] + assert_same ~S[~s(f\a\b\ro)] + + assert_same ~S""" + ~S(foo\ + bar) + """ + end + + test "with nested new lines" do + assert_same ~S""" + foo do + ~S(foo\ + bar) + end + """ + + assert_same ~S""" + foo do + ~s(#{bar} + ) + end + """ + end + + test "with interpolation" do + assert_same ~S[~s(one #{2} three)] + end + + test "with modifiers" do + assert_same ~S[~w(one two three)a] + assert_same ~S[~z(one two three)foo] + end + + test "with interpolation on line limit" do + assert_same ~S""" + ~s(one #{"two"} three) + """, + @short_length + end + + test "with heredoc syntax" do + assert_same ~S""" + ~s''' + one\a + #{:two}\r + three\0 + ''' + """ + + assert_same ~S''' + ~s""" + one\a + #{:two}\r + three\0 + """ + ''' + end + + test "with heredoc syntax and modifier" do + assert_same ~S""" + ~s''' + foo + '''rsa + """ + end + + test "with heredoc syntax and interpolation on line limit" do + assert_same ~S""" + ~s''' + one #{"two two"} three + ''' + """, + @short_length + end + + test "with custom formatting" do + bad = """ + ~W/foo bar baz/ + """ + + good = """ + ~W/foo bar baz/ + """ + + formatter = fn content, opts -> + assert opts == [file: nil, line: 1, sigil: :W, modifiers: [], opening_delimiter: "/"] + content |> String.split(~r/ +/) |> Enum.join(" ") + end + + assert_format bad, good, sigils: [W: formatter] + + bad = """ + var = ~Wabc + """ + + good = """ + var = ~Wabc + """ + + formatter = fn content, opts -> + assert opts == [file: nil, line: 1, sigil: :W, modifiers: 'abc', opening_delimiter: "<"] + content |> String.split(~r/ +/) |> Enum.intersperse(" ") + end + + assert_format bad, good, sigils: [W: formatter] + end + + test "with custom formatting on heredocs" do + bad = """ + ~W''' + foo bar baz + ''' + """ + + good = """ + ~W''' + foo bar baz + ''' + """ + + formatter = fn content, opts -> + assert opts == [file: nil, line: 1, sigil: :W, modifiers: [], opening_delimiter: "'''"] + content |> String.split(~r/ +/) |> Enum.join(" ") + end + + assert_format bad, good, sigils: [W: formatter] + + bad = ~S''' + if true do + ~W""" + foo + bar + baz + """abc + end + ''' + + good = ~S''' + if true do + ~W""" + foo + bar + baz + """abc + end + ''' + + formatter = fn content, opts -> + assert opts == [ + file: nil, + line: 2, + sigil: :W, + modifiers: 'abc', + opening_delimiter: ~S/"""/ + ] + + content |> String.split(~r/ +/) |> Enum.join("\n") + end + + assert_format bad, good, sigils: [W: formatter] + end + end + + describe "anonymous functions" do + test "with a single clause and no arguments" do + assert_format "fn ->:ok end", "fn -> :ok end" + + bad = "fn -> :foo end" + + good = """ + fn -> + :foo + end + """ + + assert_format bad, good, @short_length + + assert_same "fn () when node() == :nonode@nohost -> true end" + end + + test "with a single clause and arguments" do + assert_format "fn x ,y-> x + y end", "fn x, y -> x + y end" + + bad = "fn x -> foo(x) end" + + good = """ + fn x -> + foo(x) + end + """ + + assert_format bad, good, @short_length + + bad = "fn one, two, three -> foo(x) end" + + good = """ + fn one, + two, + three -> + foo(x) + end + """ + + assert_format bad, good, @short_length + end + + test "with a single clause and when" do + code = """ + fn arg + when guard -> + :ok + end + """ + + assert_same code, @short_length + end + + test "with a single clause, followed by a newline, and can fit in one line" do + assert_same """ + fn + hello -> world + end + """ + end + + test "with a single clause, followed by a newline, and can not fit in one line" do + assert_same """ + SomeModule.long_function_name_that_approaches_max_columns(argument, acc, fn + %SomeStruct{key: key}, acc -> more_code(key, acc) + end) + """ + end + + test "with multiple clauses" do + code = """ + fn + 1 -> :ok + 2 -> :ok + end + """ + + assert_same code, @short_length + + code = """ + fn + 1 -> + :ok + + 2 -> + :error + end + """ + + assert_same code, @short_length + + code = """ + fn + arg11, + arg12 -> + body1 + + arg21, + arg22 -> + body2 + end + """ + + assert_same code, @short_length + + code = """ + fn + arg11, + arg12 -> + body1 + + arg21, + arg22 -> + body2 + + arg31, + arg32 -> + body3 + end + """ + + assert_same code, @short_length + end + + test "with heredocs" do + assert_same """ + fn + arg1 -> + ''' + foo + ''' + + arg2 -> + ''' + bar + ''' + end + """ + end + + test "with multiple empty clauses" do + assert_same """ + fn + () -> :ok1 + () -> :ok2 + end + """ + end + + test "with when in clauses" do + assert_same """ + fn + a1 when a + b -> :ok + b1 when c + d -> :ok + end + """ + + long = """ + fn + a1, a2 when a + b -> :ok + b1, b2 when c + d -> :ok + end + """ + + assert_same long + + good = """ + fn + a1, a2 + when a + + b -> + :ok + + b1, b2 + when c + + d -> + :ok + end + """ + + assert_format long, good, @short_length + end + + test "uses block context for the body of each clause" do + assert_same "fn -> @foo bar end" + end + + test "preserves user choice even when it fits" do + assert_same """ + fn + 1 -> + :ok + + 2 -> + :ok + end + """ + + assert_same """ + fn + 1 -> + :ok + + 2 -> + :ok + + 3 -> + :ok + end + """ + end + + test "with -> on line limit" do + bad = """ + fn ab, cd -> + ab + cd + end + """ + + good = """ + fn ab, + cd -> + ab + cd + end + """ + + assert_format bad, good, @short_length + + bad = """ + fn + ab, cd -> + 1 + xy, zw -> + 2 + end + """ + + good = """ + fn + ab, + cd -> + 1 + + xy, + zw -> + 2 + end + """ + + assert_format bad, good, @short_length + end + end + + describe "anonymous functions types" do + test "with a single clause and no arguments" do + assert_format "(->:ok)", "(() -> :ok)" + assert_same "(() -> :really_long_atom)", @short_length + assert_same "(() when node() == :nonode@nohost -> true)" + end + + test "with a single clause and arguments" do + assert_format "( x ,y-> x + y )", "(x, y -> x + y)" + + bad = "(x -> :really_long_atom)" + + good = """ + (x -> + :really_long_atom) + """ + + assert_format bad, good, @short_length + + bad = "(one, two, three -> foo(x))" + + good = """ + (one, + two, + three -> + foo(x)) + """ + + assert_format bad, good, @short_length + end + + test "with multiple clauses" do + code = """ + ( + 1 -> :ok + 2 -> :ok + ) + """ + + assert_same code, @short_length + + code = """ + ( + 1 -> + :ok + + 2 -> + :error + ) + """ + + assert_same code, @short_length + + code = """ + ( + arg11, + arg12 -> + body1 + + arg21, + arg22 -> + body2 + ) + """ + + assert_same code, @short_length + + code = """ + ( + arg11, + arg12 -> + body1 + + arg21, + arg22 -> + body2 + + arg31, + arg32 -> + body2 + ) + """ + + assert_same code, @short_length + end + + test "with heredocs" do + assert_same """ + ( + arg1 -> + ''' + foo + ''' + + arg2 -> + ''' + bar + ''' + ) + """ + end + + test "with multiple empty clauses" do + assert_same """ + ( + () -> :ok1 + () -> :ok2 + ) + """ + end + + test "preserves user choice even when it fits" do + assert_same """ + ( + 1 -> + :ok + + 2 -> + :ok + ) + """ + + assert_same """ + ( + 1 -> + :ok + + 2 -> + :ok + + 3 -> + :ok + ) + """ + end + end + + describe "blocks" do + test "empty" do + assert_format "(;)", "" + assert_format "quote do: (;)", "quote do: nil" + assert_format "quote do end", "quote do\nend" + assert_format "quote do ; end", "quote do\nend" + end + + test "with multiple lines" do + assert_same """ + foo = bar + baz = bat + """ + end + + test "with multiple lines with line limit" do + code = """ + foo = + bar(one) + + baz = + bat(two) + + a(b) + """ + + assert_same code, @short_length + + code = """ + foo = + bar(one) + + a(b) + + baz = + bat(two) + """ + + assert_same code, @short_length + + code = """ + a(b) + + foo = + bar(one) + + baz = + bat(two) + """ + + assert_same code, @short_length + + code = """ + foo = + bar(one) + + one = + two(ghi) + + baz = + bat(two) + """ + + assert_same code, @short_length + end + + test "with multiple lines with line limit inside block" do + code = """ + block do + a = + b(foo) + + c = + d(bar) + + e = + f(baz) + end + """ + + assert_same code, @short_length + end + + test "with multiple lines with cancel expressions" do + code = """ + foo(%{ + key: 1 + }) + + bar(%{ + key: 1 + }) + + baz(%{ + key: 1 + }) + """ + + assert_same code, @short_length + end + + test "with heredoc" do + assert_same """ + block do + ''' + a + + b + + c + ''' + end + """ + end + + test "keeps user newlines" do + assert_same """ + defmodule Mod do + field(:foo) + field(:bar) + field(:baz) + belongs_to(:one) + belongs_to(:two) + timestamp() + lock() + has_many(:three) + has_many(:four) + :ok + has_one(:five) + has_one(:six) + foo = 1 + bar = 2 + :before + baz = 3 + :after + end + """ + + bad = """ + defmodule Mod do + field(:foo) + + field(:bar) + + field(:baz) + + + belongs_to(:one) + belongs_to(:two) + + + timestamp() + + lock() + + + has_many(:three) + has_many(:four) + + + :ok + + + has_one(:five) + has_one(:six) + + + foo = 1 + bar = 2 + + + :before + baz = 3 + :after + end + """ + + good = """ + defmodule Mod do + field(:foo) + + field(:bar) + + field(:baz) + + belongs_to(:one) + belongs_to(:two) + + timestamp() + + lock() + + has_many(:three) + has_many(:four) + + :ok + + has_one(:five) + has_one(:six) + + foo = 1 + bar = 2 + + :before + baz = 3 + :after + end + """ + + assert_format bad, good + end + + test "with multiple defs" do + assert_same """ + def foo(:one), do: 1 + def foo(:two), do: 2 + def foo(:three), do: 3 + """ + end + + test "with module attributes" do + assert_same """ + defmodule Foo do + @constant 1 + @constant 2 + + @doc ''' + foo + ''' + def foo do + :ok + end + + @spec bar :: 1 + @spec bar :: 2 + def bar do + :ok + end + + @other_constant 3 + + @spec baz :: 4 + @doc ''' + baz + ''' + def baz do + :ok + end + + @another_constant 5 + @another_constant 5 + + @doc ''' + baz + ''' + @spec baz :: 6 + def baz do + :ok + end + end + """ + end + + test "as function arguments" do + assert_same """ + fun( + ( + foo + bar + ) + ) + """ + + assert_same """ + assert true, + do: + ( + foo + bar + ) + """ + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/integration_test.exs b/lib/elixir/test/elixir/code_formatter/integration_test.exs new file mode 100644 index 00000000000..4e584708e0a --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/integration_test.exs @@ -0,0 +1,681 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.IntegrationTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + test "empty documents" do + assert_format " ", "" + assert_format "\n", "" + assert_format ";", "" + end + + test "function with multiple calls and case" do + assert_same """ + def equivalent(string1, string2) when is_binary(string1) and is_binary(string2) do + quoted1 = Code.string_to_quoted!(string1) + quoted2 = Code.string_to_quoted!(string2) + + case not_equivalent(quoted1, quoted2) do + {left, right} -> {:error, left, right} + nil -> :ok + end + end + """ + end + + test "function with long pipeline" do + assert_same ~S""" + def to_algebra!(string, opts \\ []) when is_binary(string) and is_list(opts) do + string + |> Code.string_to_quoted!(wrap_literals_in_blocks: true, unescape: false) + |> block_to_algebra(state(opts)) + |> elem(0) + end + """ + end + + test "case with multiple multi-line arrows" do + assert_same ~S""" + case meta[:format] do + :list_heredoc -> + string = list |> List.to_string() |> escape_string(:heredoc) + {@single_heredoc |> line(string) |> concat(@single_heredoc) |> force_unfit(), state} + + :charlist -> + string = list |> List.to_string() |> escape_string(@single_quote) + {@single_quote |> concat(string) |> concat(@single_quote), state} + + _other -> + list_to_algebra(list, state) + end + """ + end + + test "function with long guards" do + assert_same """ + defp module_attribute_read?({:@, _, [{var, _, var_context}]}) + when is_atom(var) and is_atom(var_context) do + Macro.classify_atom(var) == :identifier + end + """ + end + + test "anonymous function with single clause and blocks" do + assert_same """ + {args_doc, state} = + Enum.reduce(args, {[], state}, fn quoted, {acc, state} -> + {doc, state} = quoted_to_algebra(quoted, :block, state) + doc = doc |> concat(nest(break(""), :reset)) |> group() + {[doc | acc], state} + end) + """ + end + + test "anonymous function with long single clause and blocks" do + assert_same """ + {function_count, call_count, total_time} = + Enum.reduce(call_results, {0, 0, 0}, fn {_, {count, time}}, + {function_count, call_count, total_time} -> + {function_count + 1, call_count + count, total_time + time} + end) + """ + end + + test "cond with long clause args" do + assert_same """ + cond do + parent_prec == prec and parent_assoc == side -> + binary_op_to_algebra(op, op_string, left, right, context, state, op_info, nesting) + + parent_op in @required_parens_on_binary_operands or parent_prec > prec or + (parent_prec == prec and parent_assoc != side) -> + {operand, state} = + binary_op_to_algebra(op, op_string, left, right, context, state, op_info, 2) + + {concat(concat("(", nest(operand, 1)), ")"), state} + + true -> + binary_op_to_algebra(op, op_string, left, right, context, state, op_info, 2) + end + """ + end + + test "type with multiple |" do + assert_same """ + @type t :: + binary + | :doc_nil + | :doc_line + | doc_string + | doc_cons + | doc_nest + | doc_break + | doc_group + | doc_color + | doc_force + | doc_cancel + """ + end + + test "spec with when keywords and |" do + assert_same """ + @spec send(dest, msg, [option]) :: :ok | :noconnect | :nosuspend + when dest: pid | port | atom | {atom, node}, + msg: any, + option: :noconnect | :nosuspend + """ + + assert_same """ + @spec send(dest, msg, [option]) :: :ok | :noconnect | :nosuspend + when dest: + pid + | port + | atom + | {atom, node} + | and_a_really_long_type_to_force_a_line_break + | followed_by_another_really_long_type + """ + + assert_same """ + @callback get_and_update(data, key, (value -> {get_value, value} | :pop)) :: {get_value, data} + when get_value: var, data: container + """ + end + + test "spec with multiple keys on type" do + assert_same """ + @spec foo(%{(String.t() | atom) => any}) :: any + """ + end + + test "multiple whens with new lines" do + assert_same """ + def sleep(timeout) + when is_integer(timeout) and timeout >= 0 + when timeout == :infinity do + receive after: (timeout -> :ok) + end + """ + end + + test "function with operator and pipeline" do + assert_same """ + defp apply_next_break_fits?({fun, meta, args}) when is_atom(fun) and is_list(args) do + meta[:terminator] in [@double_heredoc, @single_heredoc] and + fun |> Atom.to_string() |> String.starts_with?("sigil_") + end + """ + end + + test "mixed parens and no parens calls with anonymous function" do + assert_same ~S""" + node interface do + resolve_type(fn + %{__struct__: str}, _ -> + str |> Model.Node.model_to_node_type() + + value, _ -> + Logger.warn("Could not extract node type from value: #{inspect(value)}") + nil + end) + end + """ + end + + test "long defstruct definition" do + assert_same """ + defstruct name: nil, + module: nil, + schema: nil, + alias: nil, + base_module: nil, + web_module: nil, + basename: nil, + file: nil, + test_file: nil + """ + end + + test "mix of operators and arguments" do + assert_same """ + def count(%{path: path, line_or_bytes: bytes}) do + case File.stat(path) do + {:ok, %{size: 0}} -> {:error, __MODULE__} + {:ok, %{size: size}} -> {:ok, div(size, bytes) + if(rem(size, bytes) == 0, do: 0, else: 1)} + {:error, reason} -> raise File.Error, reason: reason, action: "stream", path: path + end + end + """ + end + + test "mix of left and right operands" do + assert_same """ + defp server_get_modules(handlers) do + for(handler(module: module) <- handlers, do: module) + |> :ordsets.from_list() + |> :ordsets.to_list() + end + """ + + assert_same """ + neighbours = for({_, _} = t <- neighbours, do: t) |> :sets.from_list() + """ + end + + test "long expression with single line anonymous function" do + assert_same """ + for_many(uniq_list_of(integer(1..10000)), fn list -> + assert Enum.uniq(list) == list + end) + """ + end + + test "long comprehension" do + assert_same """ + for %{app: app, opts: opts, top_level: true} <- Mix.Dep.cached(), + Keyword.get(opts, :app, true), + Keyword.get(opts, :runtime, true), + not Keyword.get(opts, :optional, false), + app not in included_applications, + app not in included_applications, + do: app + """ + end + + test "short comprehensions" do + assert_same """ + for {protocol, :protocol, _beam} <- removed_metadata, + remove_consolidated(protocol, output), + do: {protocol, true}, + into: %{} + """ + end + + test "comprehensions with when" do + assert_same """ + for {key, value} when is_atom(key) <- Map.to_list(map), + key = Atom.to_string(key), + String.starts_with?(key, hint) do + %{kind: :map_key, name: key, value_is_map: is_map(value)} + end + """ + + assert_same """ + with {_, doc} when unquote(doc_attr?) <- + Module.get_attribute(__MODULE__, unquote(name), unquote(escaped)), + do: doc + """ + end + + test "next break fits followed by inline tuple" do + assert_same """ + assert ExUnit.Filters.eval([line: "1"], [:line], %{line: 3, describe_line: 2}, tests) == + {:error, "due to line filter"} + """ + end + + test "try/catch with clause comment" do + assert_same """ + def format_error(reason) do + try do + do_format_error(reason) + catch + # A user could create an error that looks like a built-in one + # causing an error. + :error, _ -> + inspect(reason) + end + end + """ + end + + test "case with when and clause comment" do + assert_same """ + case decomposition do + # Decomposition + <> when h != ?< -> + decomposition = + decomposition + |> :binary.split(" ", [:global]) + |> Enum.map(&String.to_integer(&1, 16)) + + Map.put(dacc, String.to_integer(codepoint, 16), decomposition) + + _ -> + dacc + end + """ + end + + test "do-end inside binary" do + assert_same """ + <> + """ + end + + test "anonymous function with parens around integer argument" do + bad = """ + fn (1) -> "hello" end + """ + + assert_format bad, """ + fn 1 -> "hello" end + """ + end + + test "no parens keywords at the end of the line" do + bad = """ + defmodule Mod do + def token_list_downcase(<>, acc) when is_whitespace(char) or is_comma(char), do: token_list_downcase(rest, acc) + def token_list_downcase(some_really_long_arg11, some_really_long_arg22, some_really_long_arg33), do: token_list_downcase(rest, acc) + end + """ + + assert_format bad, """ + defmodule Mod do + def token_list_downcase(<>, acc) when is_whitespace(char) or is_comma(char), + do: token_list_downcase(rest, acc) + + def token_list_downcase(some_really_long_arg11, some_really_long_arg22, some_really_long_arg33), + do: token_list_downcase(rest, acc) + end + """ + end + + test "do at the end of the line" do + bad = """ + foo bar, baz, quux do + :ok + end + """ + + good = """ + foo bar, + baz, + quux do + :ok + end + """ + + assert_format bad, good, line_length: 18 + end + + test "keyword lists in last line" do + assert_same """ + content = + config(VeryLongModuleNameThatWillCauseBreak, "new.html", + conn: conn, + changeset: changeset, + categories: categories + ) + """ + + assert_same """ + content = + config VeryLongModuleNameThatWillCauseBreak, "new.html", + conn: conn, + changeset: changeset, + categories: categories + """ + end + + test "do at the end of the line with single argument" do + bad = """ + defmodule Location do + def new(line, column) when is_integer(line) and line >= 0 and is_integer(column) and column >= 0 do + %{column: column, line: line} + end + end + """ + + assert_format bad, """ + defmodule Location do + def new(line, column) + when is_integer(line) and line >= 0 and is_integer(column) and column >= 0 do + %{column: column, line: line} + end + end + """ + end + + test "tuples as trees" do + bad = """ + @document Parser.parse( + {"html", [], [ + {"head", [], []}, + {"body", [], [ + {"div", [], [ + {"p", [], ["1"]}, + {"p", [], ["2"]}, + {"div", [], [ + {"p", [], ["3"]}, + {"p", [], ["4"]}]}, + {"p", [], ["5"]}]}]}]}) + """ + + assert_format bad, """ + @document Parser.parse( + {"html", [], + [ + {"head", [], []}, + {"body", [], + [ + {"div", [], + [ + {"p", [], ["1"]}, + {"p", [], ["2"]}, + {"div", [], + [ + {"p", [], ["3"]}, + {"p", [], ["4"]} + ]}, + {"p", [], ["5"]} + ]} + ]} + ]} + ) + """ + end + + test "first argument in a call without parens with comments" do + assert_same """ + with bar :: + :ok + | :invalid + # | :unknown + | :other + """ + + assert_same """ + @spec bar :: + :ok + | :invalid + # | :unknown + | :other + """ + end + + test "when with keywords inside call" do + assert_same """ + quote((bar(foo(1)) when bat: foo(1)), []) + """ + + assert_same """ + quote(do: (bar(foo(1)) when bat: foo(1)), line: 1) + """ + + assert_same """ + typespec(quote(do: (bar(foo(1)) when bat: foo(1))), [foo: 1], []) + """ + end + + test "false positive sigil" do + assert_same """ + def sigil_d(<>, calendar) do + ymd(year, month, day, calendar) + end + """ + end + + test "newline after stab" do + assert_same """ + capture_io(":erl. mof*,,l", fn -> + assert :io.scan_erl_form('>') == {:ok, [{:":", 1}, {:atom, 1, :erl}, {:dot, 1}], 1} + + expected_tokens = [{:atom, 1, :mof}, {:*, 1}, {:",", 1}, {:",", 1}, {:atom, 1, :l}] + assert :io.scan_erl_form('>') == {:ok, expected_tokens, 1} + + assert :io.scan_erl_form('>') == {:eof, 1} + end) + """ + end + + test "capture with operators" do + assert_same """ + "this works" |> (&String.upcase/1) |> (&String.downcase/1) + """ + + assert_same """ + "this works" || (&String.upcase/1) || (&String.downcase/1) + """ + + assert_same """ + "this works" == (&String.upcase/1) == (&String.downcase/1) + """ + + bad = """ + "this works" = (&String.upcase/1) = (&String.downcase/1) + """ + + assert_format bad, """ + "this works" = (&String.upcase/1) = &String.downcase/1 + """ + + bad = """ + "this works" ++ (&String.upcase/1) ++ (&String.downcase/1) + """ + + assert_format bad, """ + "this works" ++ (&String.upcase/1) ++ &String.downcase/1 + """ + + bad = """ + "this works" +++ (&String.upcase/1) +++ (&String.downcase/1) + """ + + assert_format bad, """ + "this works" +++ (&String.upcase/1) +++ &String.downcase/1 + """ + + bad = """ + "this works" | (&String.upcase/1) | (&String.downcase/1) + """ + + assert_format bad, """ + "this works" | (&String.upcase/1) | &String.downcase/1 + """ + + bad = ~S""" + "this works" \\ (&String.upcase/1) \\ (&String.downcase/1) + """ + + assert_format bad, ~S""" + "this works" \\ &String.upcase/1 \\ &String.downcase/1 + """ + end + + test "multiline expression inside interpolation" do + bad = ~S""" + Logger.info("Example: #{ + inspect(%{ + a: 1, + b: 2 + }) + }") + """ + + assert_format bad, ~S""" + Logger.info("Example: #{inspect(%{a: 1, b: 2})}") + """ + end + + test "comment inside operator with when" do + bad = """ + raise function(x) :: + # Comment + any + """ + + assert_format bad, """ + # Comment + raise function(x) :: + any + """ + + bad = """ + raise function(x) :: + # Comment + any + when x: any + """ + + assert_format bad, """ + raise function(x) :: + any + # Comment + when x: any + """ + + bad = """ + @spec function(x) :: + # Comment + any + when x: any + """ + + assert_format bad, """ + @spec function(x) :: + any + # Comment + when x: any + """ + + bad = """ + @spec function(x) :: + # Comment + any + when x + when y + """ + + assert_format bad, """ + @spec function(x) :: + any + # Comment + when x + when y + """ + end + + test "nested heredocs with multi-line string in interpolation" do + bad = ~S''' + def foo do + """ + #{(feature_flag(:feature_x) && " + new_field + " || "")} + """ + end + ''' + + good = ~S''' + def foo do + """ + #{(feature_flag(:feature_x) && " + new_field + ") || ""} + """ + end + ''' + + assert_format bad, good + end + + test "functions with infinity line length" do + assert_same ~S""" + x = fn -> + {:ok, pid} = Repl.start_link({self(), opts}) + assert Exception.message(error) =~ msg + end + """, + line_length: :infinity + + assert_same ~S""" + capture_log(fn x -> + {:ok, pid} = Repl.start_link({self(), opts}) + assert Exception.message(error) =~ msg + end) =~ msg + """, + line_length: :infinity + + assert_same ~S""" + capture_log(fn -> + {:ok, pid} = Repl.start_link({self(), opts}) + assert Exception.message(error) =~ msg + end) =~ msg + """, + line_length: :infinity + + assert_same ~S""" + capture_log(fn x -> + {:ok, pid} = Repl.start_link({self(), opts}) + assert Exception.message(error) =~ msg + end) =~ msg + """, + line_length: :infinity + end +end diff --git a/lib/elixir/test/elixir/code_formatter/literals_test.exs b/lib/elixir/test/elixir/code_formatter/literals_test.exs new file mode 100644 index 00000000000..909d5f4faad --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/literals_test.exs @@ -0,0 +1,420 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.LiteralsTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + + describe "integers" do + test "in decimal base" do + assert_same "0" + assert_same "100" + assert_same "007" + assert_same "10000" + assert_same "100_00" + assert_format "100000", "100_000" + assert_format "1000000", "1_000_000" + end + + test "in binary base" do + assert_same "0b0" + assert_same "0b1" + assert_same "0b101" + assert_same "0b01" + assert_same "0b111_111" + end + + test "in octal base" do + assert_same "0o77" + assert_same "0o0" + assert_same "0o01" + assert_same "0o777_777" + end + + test "in hex base" do + assert_same "0x1" + assert_format "0xabcdef", "0xABCDEF" + assert_same "0x01" + assert_format "0xfff_fff", "0xFFF_FFF" + end + + test "as chars" do + assert_same "?a" + assert_same "?1" + assert_same "?è" + assert_same "??" + assert_same "?\\\\" + assert_same "?\\s" + assert_same "?🎾" + end + end + + describe "floats" do + test "with normal notation" do + assert_same "0.0" + assert_same "1.0" + assert_same "123.456" + assert_same "0.0000001" + assert_same "001.100" + assert_same "0_10000_0.000_000" + assert_format "0100000.000000", "0_100_000.000000" + end + + test "with scientific notation" do + assert_same "1.0e1" + assert_same "1.0e-1" + assert_same "1.0e01" + assert_same "1.0e-01" + assert_same "001.100e-010" + assert_same "0_100_0000.100e-010" + assert_format "0100000.0e-5", "0_100_000.0e-5" + + assert_format "1.0E01", "1.0e01" + assert_format "1.0E-01", "1.0e-01" + end + end + + describe "atoms" do + test "true, false, nil" do + assert_same "nil" + assert_same "true" + assert_same "false" + end + + test "without escapes" do + assert_same ~S[:foo] + assert_same ~S[:\\] + end + + test "with escapes" do + assert_same ~S[:"f\a\b\ro"] + assert_format ~S[:'f\a\b\ro'], ~S[:"f\a\b\ro"] + assert_format ~S[:'single \' quote'], ~S[:"single ' quote"] + assert_format ~S[:"double \" quote"], ~S[:"double \" quote"] + assert_same ~S[:"\\"] + end + + test "with unicode" do + assert_same ~S[:ólá] + end + + test "does not reformat aliases" do + assert_same ~S[:"Elixir.String"] + end + + test "removes quotes when they are not necessary" do + assert_format ~S[:"foo"], ~S[:foo] + assert_format ~S[:"++"], ~S[:++] + end + + test "quoted operators" do + assert_same ~S[:"::"] + assert_same ~S[:"..//"] + assert_format ~S[:..//], ~S[:"..//"] + assert_format ~S{[..//: 1]}, ~S{["..//": 1]} + assert_same ~S{["..//": 1]} + end + + test "uses double quotes even when single quotes are used" do + assert_format ~S[:'foo bar'], ~S[:"foo bar"] + end + + test "with interpolation" do + assert_same ~S[:"one #{2} three"] + end + + test "with escapes and interpolation" do + assert_same ~S[:"one\n\"#{2}\"\nthree"] + end + + test "with interpolation on line limit" do + assert_same ~S""" + :"one #{"two"} three" + """, + @short_length + end + end + + describe "strings" do + test "without escapes" do + assert_same ~S["foo"] + end + + test "with escapes" do + assert_same ~S["f\a\b\ro"] + assert_same ~S["double \" quote"] + end + + test "keeps literal new lines" do + assert_same """ + "fo + o" + """ + end + + test "with interpolation" do + assert_same ~S["one #{} three"] + assert_same ~S["one #{2} three"] + end + + test "with interpolation uses block content" do + assert_format ~S["one #{@two(three)}"], ~S["one #{@two three}"] + end + + test "with interpolation on line limit" do + assert_same ~S""" + "one #{"two"} three" + """, + @short_length + end + + test "with escaped interpolation" do + assert_same ~S["one\#{two}three"] + end + + test "with escapes and interpolation" do + assert_same ~S["one\n\"#{2}\"\nthree"] + end + + test "is measured in graphemes" do + assert_same ~S""" + "áá#{0}áá" + """, + @short_length + end + + test "literal new lines don't count towards line limit" do + assert_same ~S""" + "one + #{"two"} + three" + """, + @short_length + end + end + + describe "charlists" do + test "without escapes" do + assert_same ~S[''] + assert_same ~S[' '] + assert_same ~S['foo'] + end + + test "with escapes" do + assert_same ~S['f\a\b\ro'] + assert_same ~S['single \' quote'] + end + + test "keeps literal new lines" do + assert_same """ + 'fo + o' + """ + end + + test "with interpolation" do + assert_same ~S['one #{2} three'] + end + + test "with escape and interpolation" do + assert_same ~S['one\n\'#{2}\'\nthree'] + end + + test "with interpolation on line limit" do + assert_same ~S""" + 'one #{"two"} three' + """, + @short_length + end + + test "literal new lines don't count towards line limit" do + assert_same ~S""" + 'one + #{"two"} + three' + """, + @short_length + end + end + + describe "string heredocs" do + test "without escapes" do + assert_same ~S''' + """ + hello + """ + ''' + end + + test "with escapes" do + assert_same ~S''' + """ + f\a\b\ro + """ + ''' + + assert_same ~S''' + """ + multiple "\"" quotes + """ + ''' + end + + test "with interpolation" do + assert_same ~S''' + """ + one + #{2} + three + """ + ''' + + assert_same ~S''' + """ + one + " + #{2} + " + three + """ + ''' + end + + test "with interpolation on line limit" do + assert_same ~S''' + """ + one #{"two two"} three + """ + ''', + @short_length + end + + test "nested with empty lines" do + assert_same ~S''' + nested do + """ + + foo + + + bar + + """ + end + ''' + end + + test "nested with empty lines and interpolation" do + assert_same ~S''' + nested do + """ + + #{foo} + + + #{bar} + + """ + end + ''' + + assert_same ~S''' + nested do + """ + #{foo} + + + #{bar} + """ + end + ''' + end + + test "literal new lines don't count towards line limit" do + assert_same ~S''' + """ + one + #{"two"} + three + """ + ''', + @short_length + end + + test "with escaped new lines" do + assert_same ~S''' + """ + one\ + #{"two"}\ + three\ + """ + ''' + end + end + + describe "charlist heredocs" do + test "without escapes" do + assert_same ~S""" + ''' + hello + ''' + """ + end + + test "with escapes" do + assert_same ~S""" + ''' + f\a\b\ro + ''' + """ + + assert_same ~S""" + ''' + multiple "\"" quotes + ''' + """ + end + + test "with interpolation" do + assert_same ~S""" + ''' + one + #{2} + three + ''' + """ + + assert_same ~S""" + ''' + one + " + #{2} + " + three + ''' + """ + end + + test "with interpolation on line limit" do + assert_same ~S""" + ''' + one #{"two two"} three + ''' + """, + @short_length + end + + test "literal new lines don't count towards line limit" do + assert_same ~S""" + ''' + one + #{"two"} + three + ''' + """, + @short_length + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/operators_test.exs b/lib/elixir/test/elixir/code_formatter/operators_test.exs new file mode 100644 index 00000000000..168746e40f0 --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/operators_test.exs @@ -0,0 +1,967 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.OperatorsTest do + use ExUnit.Case, async: true + + import CodeFormatterHelpers + + @short_length [line_length: 10] + @medium_length [line_length: 20] + + describe "nullary" do + test "formats symbol operators" do + assert_same ".." + end + + test "combines with unary and binary operators" do + assert_same "not .." + assert_same "left = .." + assert_same ".. = right" + end + + test "is wrapped in parentheses on ambiguous calls" do + assert_same "require (..)" + assert_same "require foo, (..)" + assert_same "require (..), bar" + assert_same "require(..)" + assert_same "require(foo, ..)" + assert_same "require(.., bar)" + + assert_same "assert [.., :ok]" + assert_same "assert {.., :ok}" + assert_same "assert (..) == 0..-1//1" + assert_same "assert 0..-1//1 == (..)" + + assert_same """ + defmacro (..) do + :ok + end\ + """ + + assert_format "Range.range? (..)", "Range.range?(..)" + end + end + + describe "unary" do + test "formats symbol operators without spaces" do + assert_format "+ 1", "+1" + assert_format "- 1", "-1" + assert_format "! 1", "!1" + assert_format "^ 1", "^1" + end + + test "formats word operators with spaces" do + assert_same "not 1" + assert_same "not true" + end + + test "wraps operand if it is a unary or binary operator" do + assert_format "!+1", "!(+1)" + assert_format "+ +1", "+(+1)" + assert_format "not +1", "not (+1)" + assert_format "!not 1", "!(not 1)" + assert_format "not !1", "not (!1)" + assert_format "not(!1)", "not (!1)" + assert_format "not(1 + 1)", "not (1 + 1)" + end + + test "does not wrap operand if it is a nestable operator" do + assert_format "! ! var", "!!var" + assert_same "not not var" + end + + test "nests operand" do + bad = "+foo(bar, baz, bat)" + + good = """ + +foo( + bar, + baz, + bat + ) + """ + + assert_format bad, good, @short_length + + operator = """ + +assert foo, + bar + """ + + assert_same operator, @short_length + end + + test "does not nest operand" do + bad = "not foo(bar, baz, bat)" + + good = """ + not foo( + bar, + baz, + bat + ) + """ + + assert_format bad, good, @short_length + + operator = """ + not assert foo, + bar + """ + + assert_same operator, @short_length + end + + test "inside do-end block" do + assert_same """ + if +value do + true + end + """ + end + end + + describe "binary without space" do + test "formats without spaces" do + assert_format "1 .. 2", "1..2" + end + + test "never breaks" do + assert_same "123_456_789..987_654_321", @short_length + end + end + + describe "ternary without space" do + test "formats without spaces" do + assert_format "1 .. 2 // 3", "1..2//3" + assert_same "(1..2//3).step" + end + + test "never breaks" do + assert_same "123_456_789..987_654_321//147_268_369", @short_length + end + end + + describe "binary without newline" do + test "formats without spaces" do + assert_same "1 in 2" + assert_format "1\\\\2", "1 \\\\ 2" + end + + test "never breaks" do + assert_same "123_456_789 in 987_654_321", @short_length + end + + test "not in" do + assert_format "not(foo in bar)", "foo not in bar" + + assert_same "foo not in bar" + assert_same "(not foo) in bar" + assert_same "(!foo) in bar" + end + + test "bitwise precedence" do + assert_format "(crc >>> 8) ||| byte", "crc >>> 8 ||| byte" + assert_same "crc >>> (8 ||| byte)" + end + end + + describe "binary operators with preceding new line" do + test "formats with spaces" do + assert_format "1|>2", "1 |> 2" + end + + test "breaks into new line" do + bad = "123_456_789 |> 987_654_321" + + good = """ + 123_456_789 + |> 987_654_321 + """ + + assert_format bad, good, @short_length + + bad = "123 |> foo(bar, baz)" + + good = """ + 123 + |> foo( + bar, + baz + ) + """ + + assert_format bad, good, @short_length + + bad = "123 |> foo(bar) |> bar(bat)" + + good = """ + 123 + |> foo( + bar + ) + |> bar( + bat + ) + """ + + assert_format bad, good, @short_length + + bad = "foo(bar, 123 |> bar(baz))" + + good = """ + foo( + bar, + 123 + |> bar( + baz + ) + ) + """ + + assert_format bad, good, @short_length + + bad = "foo(bar, baz) |> 123" + + good = """ + foo( + bar, + baz + ) + |> 123 + """ + + assert_format bad, good, @short_length + + bad = "foo(bar, baz) |> 123 |> 456" + + good = """ + foo( + bar, + baz + ) + |> 123 + |> 456 + """ + + assert_format bad, good, @short_length + + bad = "123 |> foo(bar, baz) |> 456" + + good = """ + 123 + |> foo( + bar, + baz + ) + |> 456 + """ + + assert_format bad, good, @short_length + end + + test "with multiple of the different entry and same precedence" do + assert_same "foo <~> bar ~> baz" + + bad = "foo <~> bar ~> baz" + + good = """ + foo + <~> bar + ~> baz + """ + + assert_format bad, good, @short_length + end + + test "with multiple of the different entry, same precedence and right associative" do + assert_format "foo ++ bar ++ baz -- bat", "foo ++ bar ++ (baz -- bat)" + + assert_format "foo +++ bar +++ baz --- bat", "foo +++ bar +++ (baz --- bat)" + end + + test "preserves user choice even when it fits" do + assert_same """ + foo + |> bar + """ + + assert_same """ + foo = + one + |> two() + |> three() + """ + + bad = """ + foo |> + bar + """ + + good = """ + foo + |> bar + """ + + assert_format bad, good + end + end + + describe "binary with following new line" do + test "formats with spaces" do + assert_format "1++2", "1 ++ 2" + + assert_format "1+++2", "1 +++ 2" + end + + test "breaks into new line" do + bad = "123_456_789 ++ 987_654_321" + + good = """ + 123_456_789 ++ + 987_654_321 + """ + + assert_format bad, good, @short_length + + bad = "123 ++ foo(bar)" + + good = """ + 123 ++ + foo(bar) + """ + + assert_format bad, good, @short_length + + bad = "123 ++ foo(bar, baz)" + + good = """ + 123 ++ + foo( + bar, + baz + ) + """ + + assert_format bad, good, @short_length + + bad = "foo(bar, 123 ++ bar(baz))" + + good = """ + foo( + bar, + 123 ++ + bar( + baz + ) + ) + """ + + assert_format bad, good, @short_length + + bad = "foo(bar, baz) ++ 123" + + good = """ + foo( + bar, + baz + ) ++ 123 + """ + + assert_format bad, good, @short_length + end + + test "with multiple of the same entry and left associative" do + assert_same "foo == bar == baz" + + bad = "a == b == c" + + good = """ + a == b == + c + """ + + assert_format bad, good, @short_length + + bad = "(a == (b == c))" + + good = """ + a == + (b == c) + """ + + assert_format bad, good, @short_length + + bad = "foo == bar == baz" + + good = """ + foo == bar == + baz + """ + + assert_format bad, good, @short_length + + bad = "(foo == (bar == baz))" + + good = """ + foo == + (bar == + baz) + """ + + assert_format bad, good, @short_length + end + + test "with multiple of the same entry and right associative" do + assert_same "foo ++ bar ++ baz" + + bad = "a ++ b ++ c" + + good = """ + a ++ + b ++ c + """ + + assert_format bad, good, @short_length + + bad = "((a ++ b) ++ c)" + + good = """ + (a ++ b) ++ + c + """ + + assert_format bad, good, @short_length + + bad = "foo ++ bar ++ baz" + + good = """ + foo ++ + bar ++ + baz + """ + + assert_format bad, good, @short_length + + bad = "((foo ++ bar) ++ baz)" + + good = """ + (foo ++ + bar) ++ + baz + """ + + assert_format bad, good, @short_length + end + + test "with precedence" do + assert_format "(a + b) == (c + d)", "a + b == c + d" + assert_format "a + (b == c) + d", "a + (b == c) + d" + + bad = "(a + b) == (c + d)" + + good = """ + a + b == + c + d + """ + + assert_format bad, good, @short_length + + bad = "a * (b + c) * d" + + good = """ + a * + (b + c) * + d + """ + + assert_format bad, good, @short_length + + bad = "(one + two) == (three + four)" + + good = """ + one + two == + three + four + """ + + assert_format bad, good, @medium_length + + bad = "one * (two + three) * four" + + good = """ + one * (two + three) * + four + """ + + assert_format bad, good, @medium_length + + bad = "one * (two + three + four) * five" + + good = """ + one * + (two + three + + four) * five + """ + + assert_format bad, good, @medium_length + + bad = "var = one * (two + three + four) * five" + + good = """ + var = + one * + (two + three + + four) * five + """ + + assert_format bad, good, @medium_length + end + + test "with required parens" do + assert_same "(a |> b) ++ (c |> d)" + assert_format "a + b |> c + d", "(a + b) |> (c + d)" + assert_format "a ++ b |> c ++ d", "(a ++ b) |> (c ++ d)" + assert_format "a |> b ++ c |> d", "a |> (b ++ c) |> d" + end + + test "with required parens skips on no parens" do + assert_same "1..2 |> 3..4" + end + + test "with logical operators" do + assert_same "a or b or c" + assert_format "a or b and c", "a or (b and c)" + assert_format "a and b or c", "(a and b) or c" + end + + test "mixed before and after lines" do + bad = "var :: a | b and c | d" + + good = """ + var :: + a + | b and + c + | d + """ + + assert_format bad, good, @short_length + + bad = "var :: a | b and c + d + e + f | g" + + good = """ + var :: + a + | b and + c + d + e + f + | g + """ + + assert_format bad, good, @medium_length + + assert_same """ + var :: + { + :one, + :two + } + | :three + """ + end + + test "preserves user choice even when it fits and left associative" do + assert_same """ + foo + bar + + baz + bat + """ + + assert_same """ + foo + + bar + + baz + + bat + """ + end + + test "preserves user choice even when it fits and right associative" do + bad = """ + foo ++ bar ++ + baz ++ bat + """ + + assert_format bad, """ + foo ++ + bar ++ + baz ++ bat + """ + + assert_same """ + foo ++ + bar ++ + baz ++ + bat + """ + end + end + + # Theoretically it fits under binary operators + # but the goal of this section is to test common idioms. + describe "match" do + test "with calls" do + bad = "var = fun(one, two, three)" + + good = """ + var = + fun( + one, + two, + three + ) + """ + + assert_format bad, good, @short_length + + bad = "fun(one, two, three) = var" + + good = """ + fun( + one, + two, + three + ) = var + """ + + assert_format bad, good, @short_length + + bad = "fun(foo, bar) = fun(baz, bat)" + + good = """ + fun( + foo, + bar + ) = + fun( + baz, + bat + ) + """ + + assert_format bad, good, @short_length + + bad = "fun(foo, bar) = fun(baz, bat)" + + good = """ + fun(foo, bar) = + fun(baz, bat) + """ + + assert_format bad, good, @medium_length + end + + test "with containers" do + bad = "var = [one, two, three]" + + good = """ + var = [ + one, + two, + three + ] + """ + + assert_format bad, good, @short_length + + bad = """ + var = + [one, two, three] + """ + + good = """ + var = [ + one, + two, + three + ] + """ + + assert_format bad, good, @short_length + + bad = "[one, two, three] = var" + + good = """ + [ + one, + two, + three + ] = var + """ + + assert_format bad, good, @short_length + + bad = "[one, two, three] = foo(bar, baz)" + + good = """ + [one, two, three] = + foo(bar, baz) + """ + + assert_format bad, good, @medium_length + end + + test "with heredoc" do + heredoc = ~S""" + var = ''' + one + ''' + """ + + assert_same heredoc, @short_length + + heredoc = ~S""" + var = ''' + #{one} + ''' + """ + + assert_same heredoc, @short_length + end + + test "with anonymous functions" do + bad = "var = fn arg1 -> body1; arg2 -> body2 end" + + good = """ + var = fn + arg1 -> + body1 + + arg2 -> + body2 + end + """ + + assert_format bad, good, @short_length + + good = """ + var = fn + arg1 -> body1 + arg2 -> body2 + end + """ + + assert_format bad, good, @medium_length + end + + test "with do-end blocks" do + assert_same """ + var = + case true do + foo -> bar + baz -> bat + end + """ + end + end + + describe "module attributes" do + test "when reading" do + assert_format "@ my_attribute", "@my_attribute" + end + + test "when setting" do + assert_format "@ my_attribute(:some_value)", "@my_attribute :some_value" + end + + test "doesn't split when reading on line limit" do + assert_same "@my_long_attribute", @short_length + end + + test "doesn't split when setting on line limit" do + assert_same "@my_long_attribute :some_value", @short_length + end + + test "with do-end block" do + assert_same """ + @attr (for x <- y do + z + end) + """ + end + + test "is parenthesized when setting inside a call" do + assert_same "my_fun(@foo(bar), baz)" + end + + test "fall back to @ as an operator when needed" do + assert_same "@(1 + 1)" + assert_same "@:foo" + assert_same "+@foo" + assert_same "@@foo" + assert_same "@(+foo)" + assert_same "!(@(1 + 1))" + assert_same "(@Foo).Baz" + assert_same "@bar(1, 2)" + + assert_format "@+1", "@(+1)" + assert_format "@Foo.Baz", "(@Foo).Baz" + assert_format "@(Foo.Bar).Baz", "(@(Foo.Bar)).Baz" + end + + test "with next break fits" do + attribute = """ + @doc ''' + foo + ''' + """ + + assert_same attribute + + attribute = """ + @doc foo: ''' + bar + ''' + """ + + assert_same attribute + end + + test "without next break fits" do + bad = "@really_long_expr foo + bar" + + good = """ + @really_long_expr foo + + bar + """ + + assert_format bad, good, @short_length + end + + test "with do-end blocks" do + attribute = """ + @doc do + :ok + end + """ + + assert_same attribute, @short_length + + attribute = """ + use (@doc do + :end + end) + """ + + assert_same attribute, @short_length + end + + test "do not rewrite lists to keyword lists" do + assert_same """ + @foo [ + bar: baz + ] + """ + end + end + + describe "capture" do + test "with integers" do + assert_same "&1" + assert_format "&(&1)", "& &1" + assert_format "&(&1.foo)", "& &1.foo" + end + + test "with operators inside" do + assert_format "& +1", "&(+1)" + assert_format "& not &1", "&(not &1)" + assert_format "& a ++ b", "&(a ++ b)" + assert_format "& &1 && &2", "&(&1 && &2)" + assert_same "&(&1 | &2)" + end + + test "with operators outside" do + assert_same "(& &1) == (& &2)" + assert_same "(& &1) and (& &2)" + assert_same "(&foo/1) and (&bar/1)" + assert_same "[(&IO.puts/1) | &IO.puts/2]" + end + + test "with call expressions" do + assert_format "& local(&1, &2)", "&local(&1, &2)" + assert_format "&-local(&1, &2)", "&(-local(&1, &2))" + end + + test "with blocks" do + bad = "&(1; 2)" + + good = """ + &( + 1 + 2 + ) + """ + + assert_format bad, good + end + + test "with no parens" do + capture = """ + &assert foo, + bar + """ + + assert_same capture, @short_length + end + + test "precedence when combined with calls" do + assert_same "(&Foo).Bar" + assert_format "&(Foo).Bar", "&Foo.Bar" + assert_format "&(Foo.Bar).Baz", "&Foo.Bar.Baz" + end + + test "local/arity" do + assert_format "&(foo/1)", "&foo/1" + assert_format "&(foo/bar)", "&(foo / bar)" + end + + test "operator/arity" do + assert_same "&+/2" + assert_same "&and/2" + assert_same "& &&/2" + assert_same "& &/1" + assert_same "&//2" + end + + test "Module.remote/arity" do + assert_format "&(Mod.foo/1)", "&Mod.foo/1" + assert_format "&(Mod.++/1)", "&Mod.++/1" + assert_format ~s[&(Mod."foo bar"/1)], ~s[&Mod."foo bar"/1] + + # Invalid + assert_format "& Mod.foo/bar", "&(Mod.foo() / bar)" + + # This is "invalid" as a special form but we don't + # have enough knowledge to know that, so let's just + # make sure we format it properly with proper wrapping. + assert_same "&(1 + 2).foo/1" + + assert_same "&my_function.foo.bar/3", @short_length + end + end + + describe "when" do + test "with keywords" do + assert_same "foo when bar: :baz" + end + + test "with keywords on line breaks" do + bad = "foo when one: :two, three: :four" + + good = """ + foo + when one: :two, + three: :four + """ + + assert_format bad, good, @medium_length + end + end +end diff --git a/lib/elixir/test/elixir/code_fragment_test.exs b/lib/elixir/test/elixir/code_fragment_test.exs new file mode 100644 index 00000000000..1a944985915 --- /dev/null +++ b/lib/elixir/test/elixir/code_fragment_test.exs @@ -0,0 +1,937 @@ +Code.require_file("test_helper.exs", __DIR__) + +defmodule CodeFragmentTest do + use ExUnit.Case, async: true + + doctest Code.Fragment + alias Code.Fragment, as: CF + + describe "cursor_context/2" do + test "expressions" do + assert CF.cursor_context([]) == :expr + assert CF.cursor_context(",") == :expr + assert CF.cursor_context("[") == :expr + assert CF.cursor_context("<<") == :expr + assert CF.cursor_context("=>") == :expr + assert CF.cursor_context("->") == :expr + assert CF.cursor_context("foo(<<") == :expr + assert CF.cursor_context("hello: ") == :expr + assert CF.cursor_context("\n") == :expr + assert CF.cursor_context('\n') == :expr + assert CF.cursor_context("\n\n") == :expr + assert CF.cursor_context('\n\n') == :expr + assert CF.cursor_context("\r\n") == :expr + assert CF.cursor_context('\r\n') == :expr + assert CF.cursor_context("\r\n\r\n") == :expr + assert CF.cursor_context('\r\n\r\n') == :expr + end + + test "local_or_var" do + assert CF.cursor_context("hello_wo") == {:local_or_var, 'hello_wo'} + assert CF.cursor_context("hello_world?") == {:local_or_var, 'hello_world?'} + assert CF.cursor_context("hello_world!") == {:local_or_var, 'hello_world!'} + assert CF.cursor_context("hello/wor") == {:local_or_var, 'wor'} + assert CF.cursor_context("hello..wor") == {:local_or_var, 'wor'} + assert CF.cursor_context("hello::wor") == {:local_or_var, 'wor'} + assert CF.cursor_context("[hello_wo") == {:local_or_var, 'hello_wo'} + assert CF.cursor_context("'hello_wo") == {:local_or_var, 'hello_wo'} + assert CF.cursor_context("hellò_wó") == {:local_or_var, 'hellò_wó'} + assert CF.cursor_context("hello? world") == {:local_or_var, 'world'} + assert CF.cursor_context("hello! world") == {:local_or_var, 'world'} + assert CF.cursor_context("hello: world") == {:local_or_var, 'world'} + end + + test "dot" do + assert CF.cursor_context("hello.") == {:dot, {:var, 'hello'}, ''} + assert CF.cursor_context(":hello.") == {:dot, {:unquoted_atom, 'hello'}, ''} + assert CF.cursor_context("nested.map.") == {:dot, {:dot, {:var, 'nested'}, 'map'}, ''} + + assert CF.cursor_context("Hello.") == {:dot, {:alias, 'Hello'}, ''} + assert CF.cursor_context("Hello.World.") == {:dot, {:alias, 'Hello.World'}, ''} + assert CF.cursor_context("Hello.wor") == {:dot, {:alias, 'Hello'}, 'wor'} + assert CF.cursor_context("hello.wor") == {:dot, {:var, 'hello'}, 'wor'} + assert CF.cursor_context("Hello.++") == {:dot, {:alias, 'Hello'}, '++'} + assert CF.cursor_context(":hello.wor") == {:dot, {:unquoted_atom, 'hello'}, 'wor'} + assert CF.cursor_context(":hell@o.wor") == {:dot, {:unquoted_atom, 'hell@o'}, 'wor'} + assert CF.cursor_context(":he@ll@o.wor") == {:dot, {:unquoted_atom, 'he@ll@o'}, 'wor'} + assert CF.cursor_context(":hell@@o.wor") == {:dot, {:unquoted_atom, 'hell@@o'}, 'wor'} + assert CF.cursor_context("@hello.wor") == {:dot, {:module_attribute, 'hello'}, 'wor'} + + assert CF.cursor_context("nested.map.wor") == + {:dot, {:dot, {:var, 'nested'}, 'map'}, 'wor'} + end + + test "local_arity" do + assert CF.cursor_context("hello/") == {:local_arity, 'hello'} + end + + test "local_call" do + assert CF.cursor_context("hello\s") == {:local_call, 'hello'} + assert CF.cursor_context("hello\t") == {:local_call, 'hello'} + assert CF.cursor_context("hello(") == {:local_call, 'hello'} + assert CF.cursor_context("hello(\s") == {:local_call, 'hello'} + assert CF.cursor_context("hello(\t") == {:local_call, 'hello'} + end + + test "dot_arity" do + assert CF.cursor_context("Foo.hello/") == {:dot_arity, {:alias, 'Foo'}, 'hello'} + assert CF.cursor_context("Foo.+/") == {:dot_arity, {:alias, 'Foo'}, '+'} + assert CF.cursor_context("Foo . hello /") == {:dot_arity, {:alias, 'Foo'}, 'hello'} + assert CF.cursor_context("Foo . + /") == {:dot_arity, {:alias, 'Foo'}, '+'} + assert CF.cursor_context("foo.hello/") == {:dot_arity, {:var, 'foo'}, 'hello'} + assert CF.cursor_context(":foo.hello/") == {:dot_arity, {:unquoted_atom, 'foo'}, 'hello'} + assert CF.cursor_context("@f.hello/") == {:dot_arity, {:module_attribute, 'f'}, 'hello'} + end + + test "dot_call" do + assert CF.cursor_context("Foo.hello\s") == {:dot_call, {:alias, 'Foo'}, 'hello'} + assert CF.cursor_context("Foo.hello\t") == {:dot_call, {:alias, 'Foo'}, 'hello'} + assert CF.cursor_context("Foo.hello(") == {:dot_call, {:alias, 'Foo'}, 'hello'} + assert CF.cursor_context("Foo.hello(\s") == {:dot_call, {:alias, 'Foo'}, 'hello'} + assert CF.cursor_context("Foo.hello(\t") == {:dot_call, {:alias, 'Foo'}, 'hello'} + assert CF.cursor_context("Foo . hello (") == {:dot_call, {:alias, 'Foo'}, 'hello'} + assert CF.cursor_context("Foo . hello (\s") == {:dot_call, {:alias, 'Foo'}, 'hello'} + assert CF.cursor_context("Foo . hello (\t") == {:dot_call, {:alias, 'Foo'}, 'hello'} + + assert CF.cursor_context(":foo.hello\s") == {:dot_call, {:unquoted_atom, 'foo'}, 'hello'} + assert CF.cursor_context(":foo.hello\t") == {:dot_call, {:unquoted_atom, 'foo'}, 'hello'} + assert CF.cursor_context(":foo.hello(") == {:dot_call, {:unquoted_atom, 'foo'}, 'hello'} + assert CF.cursor_context(":foo.hello(\s") == {:dot_call, {:unquoted_atom, 'foo'}, 'hello'} + assert CF.cursor_context(":foo.hello(\t") == {:dot_call, {:unquoted_atom, 'foo'}, 'hello'} + assert CF.cursor_context(":foo.hello\s") == {:dot_call, {:unquoted_atom, 'foo'}, 'hello'} + + assert CF.cursor_context("foo.hello\s") == {:dot_call, {:var, 'foo'}, 'hello'} + assert CF.cursor_context("foo.hello\t") == {:dot_call, {:var, 'foo'}, 'hello'} + assert CF.cursor_context("foo.hello(") == {:dot_call, {:var, 'foo'}, 'hello'} + assert CF.cursor_context("foo.hello(\s") == {:dot_call, {:var, 'foo'}, 'hello'} + assert CF.cursor_context("foo.hello(\t") == {:dot_call, {:var, 'foo'}, 'hello'} + + assert CF.cursor_context("@f.hello\s") == {:dot_call, {:module_attribute, 'f'}, 'hello'} + assert CF.cursor_context("@f.hello\t") == {:dot_call, {:module_attribute, 'f'}, 'hello'} + assert CF.cursor_context("@f.hello(") == {:dot_call, {:module_attribute, 'f'}, 'hello'} + assert CF.cursor_context("@f.hello(\s") == {:dot_call, {:module_attribute, 'f'}, 'hello'} + assert CF.cursor_context("@f.hello(\t") == {:dot_call, {:module_attribute, 'f'}, 'hello'} + + assert CF.cursor_context("Foo.+\s") == {:dot_call, {:alias, 'Foo'}, '+'} + assert CF.cursor_context("Foo.+\t") == {:dot_call, {:alias, 'Foo'}, '+'} + assert CF.cursor_context("Foo.+(") == {:dot_call, {:alias, 'Foo'}, '+'} + assert CF.cursor_context("Foo.+(\s") == {:dot_call, {:alias, 'Foo'}, '+'} + assert CF.cursor_context("Foo.+(\t") == {:dot_call, {:alias, 'Foo'}, '+'} + assert CF.cursor_context("Foo . + (") == {:dot_call, {:alias, 'Foo'}, '+'} + assert CF.cursor_context("Foo . + (\s") == {:dot_call, {:alias, 'Foo'}, '+'} + assert CF.cursor_context("Foo . + (\t") == {:dot_call, {:alias, 'Foo'}, '+'} + end + + test "alias" do + assert CF.cursor_context("HelloWor") == {:alias, 'HelloWor'} + assert CF.cursor_context("Hello.Wor") == {:alias, 'Hello.Wor'} + assert CF.cursor_context("Hello . Wor") == {:alias, 'Hello.Wor'} + assert CF.cursor_context("Hello::Wor") == {:alias, 'Wor'} + assert CF.cursor_context("Hello..Wor") == {:alias, 'Wor'} + end + + test "structs" do + assert CF.cursor_context("%") == {:struct, ''} + assert CF.cursor_context(":%") == {:unquoted_atom, '%'} + assert CF.cursor_context("::%") == {:struct, ''} + + assert CF.cursor_context("%HelloWor") == {:struct, 'HelloWor'} + assert CF.cursor_context("%Hello.") == {:struct, 'Hello.'} + assert CF.cursor_context("%Hello.Wor") == {:struct, 'Hello.Wor'} + assert CF.cursor_context("% Hello . Wor") == {:struct, 'Hello.Wor'} + end + + test "unquoted atom" do + assert CF.cursor_context(":") == {:unquoted_atom, ''} + assert CF.cursor_context(":HelloWor") == {:unquoted_atom, 'HelloWor'} + assert CF.cursor_context(":HelloWór") == {:unquoted_atom, 'HelloWór'} + assert CF.cursor_context(":hello_wor") == {:unquoted_atom, 'hello_wor'} + assert CF.cursor_context(":Óla_mundo") == {:unquoted_atom, 'Óla_mundo'} + assert CF.cursor_context(":Ol@_mundo") == {:unquoted_atom, 'Ol@_mundo'} + assert CF.cursor_context(":Ol@") == {:unquoted_atom, 'Ol@'} + assert CF.cursor_context("foo:hello_wor") == {:unquoted_atom, 'hello_wor'} + + # Operators from atoms + assert CF.cursor_context(":+") == {:unquoted_atom, '+'} + assert CF.cursor_context(":or") == {:unquoted_atom, 'or'} + assert CF.cursor_context(":<") == {:unquoted_atom, '<'} + assert CF.cursor_context(":.") == {:unquoted_atom, '.'} + assert CF.cursor_context(":..") == {:unquoted_atom, '..'} + assert CF.cursor_context(":->") == {:unquoted_atom, '->'} + assert CF.cursor_context(":%") == {:unquoted_atom, '%'} + end + + test "operators" do + assert CF.cursor_context("+") == {:operator, '+'} + assert CF.cursor_context("++") == {:operator, '++'} + assert CF.cursor_context("!") == {:operator, '!'} + assert CF.cursor_context("<") == {:operator, '<'} + assert CF.cursor_context("<<<") == {:operator, '<<<'} + assert CF.cursor_context("..") == {:operator, '..'} + assert CF.cursor_context("<~") == {:operator, '<~'} + assert CF.cursor_context("=~") == {:operator, '=~'} + assert CF.cursor_context("<~>") == {:operator, '<~>'} + assert CF.cursor_context("::") == {:operator, '::'} + + assert CF.cursor_context("+ ") == {:operator_call, '+'} + assert CF.cursor_context("++ ") == {:operator_call, '++'} + assert CF.cursor_context("! ") == {:operator_call, '!'} + assert CF.cursor_context("< ") == {:operator_call, '<'} + assert CF.cursor_context("<<< ") == {:operator_call, '<<<'} + assert CF.cursor_context(".. ") == {:operator_call, '..'} + assert CF.cursor_context("<~ ") == {:operator_call, '<~'} + assert CF.cursor_context("=~ ") == {:operator_call, '=~'} + assert CF.cursor_context("<~> ") == {:operator_call, '<~>'} + assert CF.cursor_context(":: ") == {:operator_call, '::'} + + assert CF.cursor_context("+/") == {:operator_arity, '+'} + assert CF.cursor_context("++/") == {:operator_arity, '++'} + assert CF.cursor_context("!/") == {:operator_arity, '!'} + assert CF.cursor_context("/") == {:operator_arity, '<~>'} + assert CF.cursor_context("::/") == {:operator_arity, '::'} + + # Unknown operators altogether + assert CF.cursor_context("***") == :none + + # Textual operators are shown as local_or_var UNLESS there is space + assert CF.cursor_context("when") == {:local_or_var, 'when'} + assert CF.cursor_context("when ") == {:operator_call, 'when'} + assert CF.cursor_context("when.") == :none + + assert CF.cursor_context("not") == {:local_or_var, 'not'} + assert CF.cursor_context("not ") == {:operator_call, 'not'} + assert CF.cursor_context("not.") == :none + end + + test "sigil" do + assert CF.cursor_context("~") == {:sigil, ''} + assert CF.cursor_context("~ ") == :none + + assert CF.cursor_context("~r") == {:sigil, 'r'} + assert CF.cursor_context("~r/") == :none + assert CF.cursor_context("~r<") == :none + + assert CF.cursor_context("~R") == {:sigil, 'R'} + assert CF.cursor_context("~R/") == :none + assert CF.cursor_context("~R<") == :none + + assert CF.cursor_context("Foo.~") == :none + assert CF.cursor_context("Foo.~ ") == :none + end + + test "module attribute" do + assert CF.cursor_context("@") == {:module_attribute, ''} + assert CF.cursor_context("@hello_wo") == {:module_attribute, 'hello_wo'} + end + + test "none" do + # Punctuation + assert CF.cursor_context(")") == :none + assert CF.cursor_context("}") == :none + assert CF.cursor_context(">>") == :none + assert CF.cursor_context("'") == :none + assert CF.cursor_context("\"") == :none + + # Numbers + assert CF.cursor_context("123") == :none + assert CF.cursor_context("123?") == :none + assert CF.cursor_context("123!") == :none + assert CF.cursor_context("123var?") == :none + assert CF.cursor_context("0x") == :none + + # Codepoints + assert CF.cursor_context("?") == :none + assert CF.cursor_context("?a") == :none + assert CF.cursor_context("?foo") == :none + + # Dots + assert CF.cursor_context(".") == :none + assert CF.cursor_context("Mundo.Óla") == :none + assert CF.cursor_context(":hello.World") == :none + + # Aliases + assert CF.cursor_context("Hello::Wór") == :none + assert CF.cursor_context("ÓlaMundo") == :none + assert CF.cursor_context("HelloWór") == :none + assert CF.cursor_context("@Hello") == :none + assert CF.cursor_context("Hello(") == :none + assert CF.cursor_context("Hello ") == :none + assert CF.cursor_context("hello.World") == :none + + # Identifier + assert CF.cursor_context("foo@bar") == :none + assert CF.cursor_context("@foo@bar") == :none + end + + test "newlines" do + assert CF.cursor_context("this+does-not*matter\nHello.") == {:dot, {:alias, 'Hello'}, ''} + assert CF.cursor_context('this+does-not*matter\nHello.') == {:dot, {:alias, 'Hello'}, ''} + assert CF.cursor_context("this+does-not*matter\r\nHello.") == {:dot, {:alias, 'Hello'}, ''} + assert CF.cursor_context('this+does-not*matter\r\nHello.') == {:dot, {:alias, 'Hello'}, ''} + end + end + + describe "surround_context/2" do + test "newlines" do + for i <- 1..8 do + assert CF.surround_context("\n\nhello_wo\n", {3, i}) == %{ + context: {:local_or_var, 'hello_wo'}, + begin: {3, 1}, + end: {3, 9} + } + + assert CF.surround_context("\r\n\r\nhello_wo\r\n", {3, i}) == %{ + context: {:local_or_var, 'hello_wo'}, + begin: {3, 1}, + end: {3, 9} + } + + assert CF.surround_context('\r\n\r\nhello_wo\r\n', {3, i}) == %{ + context: {:local_or_var, 'hello_wo'}, + begin: {3, 1}, + end: {3, 9} + } + end + end + + test "column out of range" do + assert CF.surround_context("hello", {1, 20}) == :none + end + + test "local_or_var" do + for i <- 1..8 do + assert CF.surround_context("hello_wo", {1, i}) == %{ + context: {:local_or_var, 'hello_wo'}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("hello_wo", {1, 9}) == :none + + for i <- 2..9 do + assert CF.surround_context(" hello_wo", {1, i}) == %{ + context: {:local_or_var, 'hello_wo'}, + begin: {1, 2}, + end: {1, 10} + } + end + + assert CF.surround_context(" hello_wo", {1, 10}) == :none + + for i <- 1..6 do + assert CF.surround_context("hello!", {1, i}) == %{ + context: {:local_or_var, 'hello!'}, + begin: {1, 1}, + end: {1, 7} + } + end + + assert CF.surround_context("hello!", {1, 7}) == :none + + for i <- 1..5 do + assert CF.surround_context("안녕_세상", {1, i}) == %{ + context: {:local_or_var, '안녕_세상'}, + begin: {1, 1}, + end: {1, 6} + } + end + + assert CF.surround_context("안녕_세상", {1, 6}) == :none + + # Keywords are not local or var + for keyword <- ~w(do end after catch else rescue fn true false nil)c do + keyword_length = length(keyword) + 1 + + assert %{ + context: {:keyword, ^keyword}, + begin: {1, 1}, + end: {1, ^keyword_length} + } = CF.surround_context(keyword, {1, 1}) + end + end + + test "local call" do + for i <- 1..8 do + assert CF.surround_context("hello_wo(", {1, i}) == %{ + context: {:local_call, 'hello_wo'}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("hello_wo(", {1, 9}) == :none + + for i <- 1..8 do + assert CF.surround_context("hello_wo (", {1, i}) == %{ + context: {:local_call, 'hello_wo'}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("hello_wo (", {1, 9}) == :none + + for i <- 1..6 do + assert CF.surround_context("hello!(", {1, i}) == %{ + context: {:local_call, 'hello!'}, + begin: {1, 1}, + end: {1, 7} + } + end + + assert CF.surround_context("hello!(", {1, 7}) == :none + + for i <- 1..5 do + assert CF.surround_context("안녕_세상(", {1, i}) == %{ + context: {:local_call, '안녕_세상'}, + begin: {1, 1}, + end: {1, 6} + } + end + + assert CF.surround_context("안녕_세상(", {1, 6}) == :none + end + + test "local arity" do + for i <- 1..8 do + assert CF.surround_context("hello_wo/", {1, i}) == %{ + context: {:local_arity, 'hello_wo'}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("hello_wo/", {1, 9}) == :none + + for i <- 1..8 do + assert CF.surround_context("hello_wo /", {1, i}) == %{ + context: {:local_arity, 'hello_wo'}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("hello_wo /", {1, 9}) == :none + + for i <- 1..6 do + assert CF.surround_context("hello!/", {1, i}) == %{ + context: {:local_arity, 'hello!'}, + begin: {1, 1}, + end: {1, 7} + } + end + + assert CF.surround_context("hello!/", {1, 7}) == :none + + for i <- 1..5 do + assert CF.surround_context("안녕_세상/", {1, i}) == %{ + context: {:local_arity, '안녕_세상'}, + begin: {1, 1}, + end: {1, 6} + } + end + + assert CF.surround_context("안녕_세상/", {1, 6}) == :none + end + + test "textual operators" do + for op <- ~w(when not or and in), i <- 1..byte_size(op) do + assert CF.surround_context("#{op}", {1, i}) == %{ + context: {:operator, String.to_charlist(op)}, + begin: {1, 1}, + end: {1, byte_size(op) + 1} + } + end + end + + test "dot" do + for i <- 1..5 do + assert CF.surround_context("Hello.wor", {1, i}) == %{ + context: {:alias, 'Hello'}, + begin: {1, 1}, + end: {1, 6} + } + end + + for i <- 6..9 do + assert CF.surround_context("Hello.wor", {1, i}) == %{ + context: {:dot, {:alias, 'Hello'}, 'wor'}, + begin: {1, 1}, + end: {1, 10} + } + end + + assert CF.surround_context("Hello.", {1, 6}) == :none + + for i <- 1..5 do + assert CF.surround_context("Hello . wor", {1, i}) == %{ + context: {:alias, 'Hello'}, + begin: {1, 1}, + end: {1, 6} + } + end + + for i <- 6..11 do + assert CF.surround_context("Hello . wor", {1, i}) == %{ + context: {:dot, {:alias, 'Hello'}, 'wor'}, + begin: {1, 1}, + end: {1, 12} + } + end + + assert CF.surround_context("Hello .", {1, 6}) == :none + + for i <- 1..5 do + assert CF.surround_context("hello.wor", {1, i}) == %{ + context: {:local_or_var, 'hello'}, + begin: {1, 1}, + end: {1, 6} + } + end + + for i <- 6..9 do + assert CF.surround_context("hello.wor", {1, i}) == %{ + context: {:dot, {:var, 'hello'}, 'wor'}, + begin: {1, 1}, + end: {1, 10} + } + end + end + + test "alias" do + for i <- 1..8 do + assert CF.surround_context("HelloWor", {1, i}) == %{ + context: {:alias, 'HelloWor'}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("HelloWor", {1, 9}) == :none + + for i <- 2..9 do + assert CF.surround_context(" HelloWor", {1, i}) == %{ + context: {:alias, 'HelloWor'}, + begin: {1, 2}, + end: {1, 10} + } + end + + assert CF.surround_context(" HelloWor", {1, 10}) == :none + + for i <- 1..9 do + assert CF.surround_context("Hello.Wor", {1, i}) == %{ + context: {:alias, 'Hello.Wor'}, + begin: {1, 1}, + end: {1, 10} + } + end + + assert CF.surround_context("Hello.Wor", {1, 10}) == :none + + for i <- 1..11 do + assert CF.surround_context("Hello . Wor", {1, i}) == %{ + context: {:alias, 'Hello.Wor'}, + begin: {1, 1}, + end: {1, 12} + } + end + + assert CF.surround_context("Hello . Wor", {1, 12}) == :none + + for i <- 1..15 do + assert CF.surround_context("Foo . Bar . Baz", {1, i}) == %{ + context: {:alias, 'Foo.Bar.Baz'}, + begin: {1, 1}, + end: {1, 16} + } + end + end + + test "struct" do + assert CF.surround_context("%", {1, 1}) == :none + assert CF.surround_context("::%", {1, 1}) == :none + assert CF.surround_context("::%", {1, 2}) == :none + assert CF.surround_context("::%Hello", {1, 1}) == :none + assert CF.surround_context("::%Hello", {1, 2}) == :none + + assert CF.surround_context("::%Hello", {1, 3}) == %{ + context: {:struct, 'Hello'}, + begin: {1, 3}, + end: {1, 9} + } + + assert CF.surround_context("::% Hello", {1, 3}) == %{ + context: {:struct, 'Hello'}, + begin: {1, 3}, + end: {1, 10} + } + + assert CF.surround_context("::% Hello", {1, 4}) == %{ + context: {:struct, 'Hello'}, + begin: {1, 3}, + end: {1, 10} + } + + # Alias + assert CF.surround_context("%HelloWor", {1, 1}) == %{ + context: {:struct, 'HelloWor'}, + begin: {1, 1}, + end: {1, 10} + } + + for i <- 2..9 do + assert CF.surround_context("%HelloWor", {1, i}) == %{ + context: {:struct, 'HelloWor'}, + begin: {1, 1}, + end: {1, 10} + } + end + + assert CF.surround_context("%HelloWor", {1, 10}) == :none + + # With dot + assert CF.surround_context("%Hello.Wor", {1, 1}) == %{ + context: {:struct, 'Hello.Wor'}, + begin: {1, 1}, + end: {1, 11} + } + + for i <- 2..10 do + assert CF.surround_context("%Hello.Wor", {1, i}) == %{ + context: {:struct, 'Hello.Wor'}, + begin: {1, 1}, + end: {1, 11} + } + end + + assert CF.surround_context("%Hello.Wor", {1, 11}) == :none + + # With spaces + assert CF.surround_context("% Hello . Wor", {1, 1}) == %{ + context: {:struct, 'Hello.Wor'}, + begin: {1, 1}, + end: {1, 14} + } + + for i <- 2..13 do + assert CF.surround_context("% Hello . Wor", {1, i}) == %{ + context: {:struct, 'Hello.Wor'}, + begin: {1, 1}, + end: {1, 14} + } + end + + assert CF.surround_context("% Hello . Wor", {1, 14}) == :none + end + + test "module attributes" do + for i <- 1..10 do + assert CF.surround_context("@hello_wor", {1, i}) == %{ + context: {:module_attribute, 'hello_wor'}, + begin: {1, 1}, + end: {1, 11} + } + end + + assert CF.surround_context("@Hello", {1, 1}) == :none + end + + test "operators" do + for i <- 2..4 do + assert CF.surround_context("1<<<3", {1, i}) == %{ + context: {:operator, '<<<'}, + begin: {1, 2}, + end: {1, 5} + } + end + + for i <- 3..5 do + assert CF.surround_context("1 <<< 3", {1, i}) == %{ + context: {:operator, '<<<'}, + begin: {1, 3}, + end: {1, 6} + } + end + + for i <- 2..3 do + assert CF.surround_context("1::3", {1, i}) == %{ + context: {:operator, '::'}, + begin: {1, 2}, + end: {1, 4} + } + end + + for i <- 3..4 do + assert CF.surround_context("1 :: 3", {1, i}) == %{ + context: {:operator, '::'}, + begin: {1, 3}, + end: {1, 5} + } + end + + for i <- 2..3 do + assert CF.surround_context("x..y", {1, i}) == %{ + context: {:operator, '..'}, + begin: {1, 2}, + end: {1, 4} + } + end + + for i <- 3..4 do + assert CF.surround_context("x .. y", {1, i}) == %{ + context: {:operator, '..'}, + begin: {1, 3}, + end: {1, 5} + } + end + + assert CF.surround_context("@", {1, 1}) == %{ + context: {:operator, '@'}, + begin: {1, 1}, + end: {1, 2} + } + + assert CF.surround_context("!", {1, 1}) == %{ + context: {:operator, '!'}, + begin: {1, 1}, + end: {1, 2} + } + + assert CF.surround_context("!foo", {1, 1}) == %{ + context: {:operator, '!'}, + begin: {1, 1}, + end: {1, 2} + } + + assert CF.surround_context("foo !bar", {1, 5}) == %{ + context: {:operator, '!'}, + begin: {1, 5}, + end: {1, 6} + } + end + + test "sigil" do + assert CF.surround_context("~", {1, 1}) == :none + assert CF.surround_context("~~r", {1, 1}) == :none + assert CF.surround_context("~~r", {1, 2}) == :none + + assert CF.surround_context("~r/foo/", {1, 1}) == %{ + begin: {1, 1}, + context: {:sigil, 'r'}, + end: {1, 3} + } + + assert CF.surround_context("~r/foo/", {1, 2}) == %{ + begin: {1, 1}, + context: {:sigil, 'r'}, + end: {1, 3} + } + + assert CF.surround_context("~r/foo/", {1, 3}) == :none + + assert CF.surround_context("~R", {1, 1}) == %{ + begin: {1, 1}, + context: {:sigil, 'R'}, + end: {1, 3} + } + + assert CF.surround_context("~R", {1, 2}) == %{ + begin: {1, 1}, + context: {:sigil, 'R'}, + end: {1, 3} + } + + assert CF.surround_context("~R", {1, 3}) == :none + end + + test "dot operator" do + for i <- 4..7 do + assert CF.surround_context("Foo.<<<", {1, i}) == %{ + context: {:dot, {:alias, 'Foo'}, '<<<'}, + begin: {1, 1}, + end: {1, 8} + } + end + + for i <- 4..9 do + assert CF.surround_context("Foo . <<<", {1, i}) == %{ + context: {:dot, {:alias, 'Foo'}, '<<<'}, + begin: {1, 1}, + end: {1, 10} + } + end + + for i <- 4..6 do + assert CF.surround_context("Foo.::", {1, i}) == %{ + context: {:dot, {:alias, 'Foo'}, '::'}, + begin: {1, 1}, + end: {1, 7} + } + end + + for i <- 4..8 do + assert CF.surround_context("Foo . ::", {1, i}) == %{ + context: {:dot, {:alias, 'Foo'}, '::'}, + begin: {1, 1}, + end: {1, 9} + } + end + end + + test "unquoted atom" do + for i <- 1..10 do + assert CF.surround_context(":hello_wor", {1, i}) == %{ + context: {:unquoted_atom, 'hello_wor'}, + begin: {1, 1}, + end: {1, 11} + } + end + + for i <- 1..10 do + assert CF.surround_context(":Hello@Wor", {1, i}) == %{ + context: {:unquoted_atom, 'Hello@Wor'}, + begin: {1, 1}, + end: {1, 11} + } + end + + assert CF.surround_context(":", {1, 1}) == :none + end + end + + describe "container_cursor_to_quoted/2" do + def s2q(arg, opts \\ []), do: Code.string_to_quoted(arg, opts) + def cc2q(arg, opts \\ []), do: CF.container_cursor_to_quoted(arg, opts) + + test "completes terminators" do + assert cc2q("(") == s2q("(__cursor__())") + assert cc2q("[") == s2q("[__cursor__()]") + assert cc2q("{") == s2q("{__cursor__()}") + assert cc2q("<<") == s2q("<<__cursor__()>>") + assert cc2q("foo do") == s2q("foo do __cursor__() end") + assert cc2q("foo do true else") == s2q("foo do true else __cursor__() end") + end + + test "inside interpolation" do + assert cc2q(~S|"foo #{(|) == s2q(~S|"foo #{(__cursor__())}"|) + assert cc2q(~S|"foo #{"bar #{{|) == s2q(~S|"foo #{"bar #{{__cursor__()}}"}"|) + end + + test "do-end blocks" do + assert cc2q("foo(bar do baz") == s2q("foo(bar do __cursor__() end)") + assert cc2q("foo(bar do baz ") == s2q("foo(bar do baz(__cursor__()) end)") + assert cc2q("foo(bar do baz(") == s2q("foo(bar do baz(__cursor__()) end)") + assert cc2q("foo(bar do baz bat,") == s2q("foo(bar do baz(bat, __cursor__()) end)") + + assert {:error, {_, "syntax error before: ", "'end'"}} = cc2q("foo(bar do baz, bat") + end + + test "keyword lists" do + assert cc2q("[bar: ") == s2q("[bar: __cursor__()]") + assert cc2q("[bar: baz,") == s2q("[bar: baz, __cursor__()]") + assert cc2q("[arg, bar: baz,") == s2q("[arg, bar: baz, __cursor__()]") + assert cc2q("[arg: val, bar: baz,") == s2q("[arg: val, bar: baz, __cursor__()]") + + assert cc2q("{arg, bar: ") == s2q("{arg, bar: __cursor__()}") + assert cc2q("{arg, bar: baz,") == s2q("{arg, bar: baz, __cursor__()}") + assert cc2q("{arg: val, bar: baz,") == s2q("{arg: val, bar: baz, __cursor__()}") + + assert cc2q("foo(bar: ") == s2q("foo(bar: __cursor__())") + assert cc2q("foo(bar: baz,") == s2q("foo([bar: baz, __cursor__()])") + assert cc2q("foo(arg, bar: ") == s2q("foo(arg, bar: __cursor__())") + assert cc2q("foo(arg, bar: baz,") == s2q("foo(arg, [bar: baz, __cursor__()])") + assert cc2q("foo(arg: val, bar: ") == s2q("foo(arg: val, bar: __cursor__())") + assert cc2q("foo(arg: val, bar: baz,") == s2q("foo([arg: val, bar: baz, __cursor__()])") + + assert cc2q("foo bar: ") == s2q("foo(bar: __cursor__())") + assert cc2q("foo bar: baz,") == s2q("foo([bar: baz, __cursor__()])") + assert cc2q("foo arg, bar: ") == s2q("foo(arg, bar: __cursor__())") + assert cc2q("foo arg, bar: baz,") == s2q("foo(arg, [bar: baz, __cursor__()])") + assert cc2q("foo arg: val, bar: ") == s2q("foo(arg: val, bar: __cursor__())") + assert cc2q("foo arg: val, bar: baz,") == s2q("foo([arg: val, bar: baz, __cursor__()])") + end + + test "maps and structs" do + assert cc2q("%") == s2q("__cursor__()") + assert cc2q("%{") == s2q("%{__cursor__()}") + assert cc2q("%{bar:") == s2q("%{__cursor__()}") + assert cc2q("%{bar: ") == s2q("%{bar: __cursor__()}") + assert cc2q("%{bar: baz,") == s2q("%{bar: baz, __cursor__()}") + + assert cc2q("%Foo") == s2q("__cursor__()") + assert cc2q("%Foo{") == s2q("%Foo{__cursor__()}") + assert cc2q("%Foo{bar: ") == s2q("%Foo{bar: __cursor__()}") + assert cc2q("%Foo{bar: baz,") == s2q("%Foo{bar: baz, __cursor__()}") + end + + test "removes tokens until opening" do + assert cc2q("(123") == s2q("(__cursor__())") + assert cc2q("[foo") == s2q("[__cursor__()]") + assert cc2q("{'foo'") == s2q("{__cursor__()}") + assert cc2q("<<1+2") == s2q("<<__cursor__()>>") + assert cc2q("foo do :atom") == s2q("foo do __cursor__() end") + assert cc2q("foo(:atom") == s2q("foo(__cursor__())") + end + + test "removes tokens until comma" do + assert cc2q("[bar, 123") == s2q("[bar, __cursor__()]") + assert cc2q("{bar, 'foo'") == s2q("{bar, __cursor__()}") + assert cc2q("<>") + assert cc2q("foo(bar, :atom") == s2q("foo(bar, __cursor__())") + assert cc2q("foo bar, :atom") == s2q("foo(bar, __cursor__())") + end + + test "removes functions" do + assert cc2q("(fn") == s2q("(__cursor__())") + assert cc2q("(fn x") == s2q("(__cursor__())") + assert cc2q("(fn x ->") == s2q("(__cursor__())") + assert cc2q("(fn x -> x") == s2q("(__cursor__())") + assert cc2q("(fn x, y -> x + y") == s2q("(__cursor__())") + assert cc2q("(fn x, y -> x + y end") == s2q("(__cursor__())") + end + + test "removes captures" do + assert cc2q("[& &1") == s2q("[__cursor__()]") + assert cc2q("[&(&1") == s2q("[__cursor__()]") + end + + test "removes closed terminators" do + assert cc2q("foo([1, 2, 3] |>") == s2q("foo(__cursor__())") + assert cc2q("foo({1, 2, 3} |>") == s2q("foo(__cursor__())") + assert cc2q("foo((1, 2, 3) |>") == s2q("foo(__cursor__())") + assert cc2q("foo(<<1, 2, 3>> |>") == s2q("foo(__cursor__())") + assert cc2q("foo(bar do :done end |>") == s2q("foo(__cursor__())") + end + + test "incomplete expressions" do + assert cc2q("foo(123, :") == s2q("foo(123, __cursor__())") + assert cc2q("foo(123, %") == s2q("foo(123, __cursor__())") + assert cc2q("foo(123, 0x") == s2q("foo(123, __cursor__())") + assert cc2q("foo(123, ~") == s2q("foo(123, __cursor__())") + assert cc2q("foo(123, ~r") == s2q("foo(123, __cursor__())") + assert cc2q("foo(123, ~r/") == s2q("foo(123, __cursor__())") + end + + test "no warnings" do + assert cc2q(~s"?\\ ") == s2q("__cursor__()") + assert cc2q(~s"{fn -> end, ") == s2q("{fn -> nil end, __cursor__()}") + end + + test "options" do + opts = [columns: true] + assert cc2q("foo(", opts) == s2q("foo(__cursor__())", opts) + assert cc2q("foo(123,", opts) == s2q("foo(123,__cursor__())", opts) + + opts = [token_metadata: true] + assert cc2q("foo(", opts) == s2q("foo(__cursor__())", opts) + assert cc2q("foo(123,", opts) == s2q("foo(123,__cursor__())", opts) + end + end +end diff --git a/lib/elixir/test/elixir/code_identifier_test.exs b/lib/elixir/test/elixir/code_identifier_test.exs new file mode 100644 index 00000000000..4ac5ceb26b7 --- /dev/null +++ b/lib/elixir/test/elixir/code_identifier_test.exs @@ -0,0 +1,6 @@ +Code.require_file("test_helper.exs", __DIR__) + +defmodule Code.IdentifierTest do + use ExUnit.Case, async: true + doctest Code.Identifier +end diff --git a/lib/elixir/test/elixir/code_normalizer/formatted_ast_test.exs b/lib/elixir/test/elixir/code_normalizer/formatted_ast_test.exs new file mode 100644 index 00000000000..0a6b6904b3b --- /dev/null +++ b/lib/elixir/test/elixir/code_normalizer/formatted_ast_test.exs @@ -0,0 +1,572 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Normalizer.FormatterASTTest do + use ExUnit.Case, async: true + + defmacro assert_same(good, opts \\ []) do + quote bind_quoted: [good: good, opts: opts], location: :keep do + assert IO.iodata_to_binary(Code.format_string!(good, opts)) == + string_to_string(good, opts) + end + end + + def string_to_string(good, opts) do + line_length = Keyword.get(opts, :line_length, 98) + good = String.trim(good) + + to_quoted_opts = + Keyword.merge( + [ + literal_encoder: &{:ok, {:__block__, &2, [&1]}}, + token_metadata: true, + unescape: false + ], + opts + ) + + {quoted, comments} = Code.string_to_quoted_with_comments!(good, to_quoted_opts) + + to_algebra_opts = [comments: comments, escape: false] ++ opts + + quoted + |> Code.quoted_to_algebra(to_algebra_opts) + |> Inspect.Algebra.format(line_length) + |> IO.iodata_to_binary() + end + + describe "integers" do + test "in decimal base" do + assert_same "0" + assert_same "100" + assert_same "007" + assert_same "10000" + assert_same "100_00" + end + + test "in binary base" do + assert_same "0b0" + assert_same "0b1" + assert_same "0b101" + assert_same "0b01" + assert_same "0b111_111" + end + + test "in octal base" do + assert_same "0o77" + assert_same "0o0" + assert_same "0o01" + assert_same "0o777_777" + end + + test "in hex base" do + assert_same "0x1" + assert_same "0x01" + end + + test "as chars" do + assert_same "?a" + assert_same "?1" + assert_same "?è" + assert_same "??" + assert_same "?\\\\" + assert_same "?\\s" + assert_same "?🎾" + end + end + + describe "floats" do + test "with normal notation" do + assert_same "0.0" + assert_same "1.0" + assert_same "123.456" + assert_same "0.0000001" + assert_same "001.100" + assert_same "0_10000_0.000_000" + end + + test "with scientific notation" do + assert_same "1.0e1" + assert_same "1.0e-1" + assert_same "1.0e01" + assert_same "1.0e-01" + assert_same "001.100e-010" + assert_same "0_100_0000.100e-010" + end + end + + describe "atoms" do + test "true, false, nil" do + assert_same "nil" + assert_same "true" + assert_same "false" + end + + test "without escapes" do + assert_same ~S[:foo] + end + + test "with escapes" do + assert_same ~S[:"f\a\b\ro"] + end + + test "with unicode" do + assert_same ~S[:ólá] + end + + test "does not reformat aliases" do + assert_same ~S[:"Elixir.String"] + assert_same ~S[:"Elixir"] + end + + test "quoted operators" do + assert_same ~S[:"::"] + assert_same ~S[:"..//"] + assert_same ~S{["..//": 1]} + end + + test "with interpolation" do + assert_same ~S[:"one #{2} three"] + end + + test "with escapes and interpolation" do + assert_same ~S[:"one\n\"#{2}\"\nthree"] + end + end + + describe "strings" do + test "without escapes" do + assert_same ~S["foo"] + end + + test "with escapes" do + assert_same ~S["\x0A"] + assert_same ~S["f\a\b\ro"] + assert_same ~S["double \" quote"] + end + + test "keeps literal new lines" do + assert_same """ + "fo + o" + """ + end + + test "with interpolation" do + assert_same ~S["one #{} three"] + assert_same ~S["one #{2} three"] + end + + test "with escaped interpolation" do + assert_same ~S["one\#{two}three"] + end + + test "with escapes and interpolation" do + assert_same ~S["one\n\"#{2}\"\nthree"] + end + end + + describe "lists" do + test "on module attribute" do + assert_same ~S"@foo [1]" + end + end + + describe "charlists" do + test "without escapes" do + assert_same ~S[''] + assert_same ~S[' '] + assert_same ~S['foo'] + end + + test "with escapes" do + assert_same ~S['f\a\b\ro'] + assert_same ~S['single \' quote'] + end + + test "keeps literal new lines" do + assert_same """ + 'fo + o' + """ + end + + test "with interpolation" do + assert_same ~S['one #{2} three'] + end + + test "with escape and interpolation" do + assert_same ~S['one\n\'#{2}\'\nthree'] + end + end + + describe "string heredocs" do + test "without escapes" do + assert_same ~S''' + """ + hello + """ + ''' + end + + test "with escapes" do + assert_same ~S''' + """ + f\a\b\ro + """ + ''' + + assert_same ~S''' + """ + multiple "\"" quotes + """ + ''' + end + + test "with interpolation" do + assert_same ~S''' + """ + one + #{2} + three + """ + ''' + + assert_same ~S''' + """ + one + " + #{2} + " + three + """ + ''' + end + + test "nested with empty lines" do + assert_same ~S''' + nested do + """ + + foo + + + bar + + """ + end + ''' + end + + test "nested with empty lines and interpolation" do + assert_same ~S''' + nested do + """ + + #{foo} + + + #{bar} + + """ + end + ''' + + assert_same ~S''' + nested do + """ + #{foo} + + + #{bar} + """ + end + ''' + end + + test "with escaped new lines" do + assert_same ~S''' + """ + one\ + #{"two"}\ + three\ + """ + ''' + end + end + + describe "charlist heredocs" do + test "without escapes" do + assert_same ~S""" + ''' + hello + ''' + """ + end + + test "with escapes" do + assert_same ~S""" + ''' + f\a\b\ro + ''' + """ + + assert_same ~S""" + ''' + multiple "\"" quotes + ''' + """ + end + + test "with interpolation" do + assert_same ~S""" + ''' + one + #{2} + three + ''' + """ + + assert_same ~S""" + ''' + one + " + #{2} + " + three + ''' + """ + end + end + + describe "keyword list" do + test "blocks" do + assert_same ~S""" + defmodule Example do + def sample, do: :ok + end + """ + end + + test "omitting brackets" do + assert_same ~S""" + @type foo :: a when b: :c + """ + end + + test "last tuple element as keyword list keeps its format" do + assert_same ~S"{:wrapped, [opt1: true, opt2: false]}" + assert_same ~S"{:unwrapped, opt1: true, opt2: false}" + assert_same ~S"{:wrapped, 1, [opt1: true, opt2: false]}" + assert_same ~S"{:unwrapped, 1, opt1: true, opt2: false}" + end + + test "on module attribute" do + assert_same ~S""" + @foo a: b, + c: d + """ + + assert_same ~S"@foo [ + a: b, + c: d + ]" + end + end + + describe "preserves user choice on parenthesis" do + test "in functions with do blocks" do + assert_same(~S""" + foo Bar do + :ok + end + """) + + assert_same(~S""" + foo(Bar) do + :ok + end + """) + end + end + + describe "preserves formatting for sigils" do + test "without interpolation" do + assert_same ~S[~s(foo)] + assert_same ~S[~s{foo bar}] + assert_same ~S[~r/Bar Baz/] + assert_same ~S[~w<>] + assert_same ~S[~W()] + end + + test "with escapes" do + assert_same ~S[~s(foo \) bar)] + assert_same ~S[~s(f\a\b\ro)] + + assert_same ~S""" + ~S(foo\ + bar) + """ + end + + test "with nested new lines" do + assert_same ~S""" + foo do + ~S(foo\ + bar) + end + """ + + assert_same ~S""" + foo do + ~s(#{bar} + ) + end + """ + end + + test "with interpolation" do + assert_same ~S[~s(one #{2} three)] + end + + test "with modifiers" do + assert_same ~S[~w(one two three)a] + assert_same ~S[~z(one two three)foo] + end + + test "with heredoc syntax" do + assert_same ~S""" + ~s''' + one\a + #{:two}\r + three\0 + ''' + """ + + assert_same ~S''' + ~s""" + one\a + #{:two}\r + three\0 + """ + ''' + end + + test "with heredoc syntax and modifier" do + assert_same ~S""" + ~s''' + foo + '''rsa + """ + end + end + + describe "preserves comments formatting" do + test "before and after expressions" do + assert_same """ + # before comment + :hello + """ + + assert_same """ + :hello + # after comment + """ + + assert_same """ + # before comment + :hello + # after comment + """ + end + + test "empty comment" do + assert_same """ + # + :foo + """ + end + + test "handles comments with unescaped literal" do + assert_same """ + # before + Mix.install([:foo]) + # after + """, + literal_encoder: fn literal, _ -> {:ok, literal} end + + assert_same """ + # before + Mix.install([1 + 2, :foo]) + # after + """, + literal_encoder: fn literal, _ -> {:ok, literal} end + + assert_same """ + # before + Mix.install([:foo, 1 + 2]) + # after + """, + literal_encoder: fn literal, _ -> {:ok, literal} end + end + + test "before and after expressions with newlines" do + assert_same """ + # before comment + # second line + + :hello + + # middle comment 1 + + # + + # middle comment 2 + + :world + + # after comment + # second line + """ + end + + test "interpolation with comment outside before and after" do + assert_same ~S""" + # comment + IO.puts("Hello #{world}") + """ + + assert_same ~S""" + IO.puts("Hello #{world}") + # comment + """ + end + + test "blocks with keyword list" do + assert_same ~S""" + defp sample do + [ + # comment + {:a, "~> 1.2"} + ] + end + """ + + assert_same ~S""" + defp sample do + [ + # comment + {:a, "~> 1.2"}, + {:b, "~> 1.2"} + ] + end + """ + end + + test "keyword literals with variable values" do + assert_same(~S""" + foo = foo() + [foo: foo] + """) + end + end +end diff --git a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs new file mode 100644 index 00000000000..6fc1d4bfec9 --- /dev/null +++ b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs @@ -0,0 +1,680 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Normalizer.QuotedASTTest do + use ExUnit.Case, async: true + + describe "quoted_to_algebra/2" do + test "variable" do + assert quoted_to_string(quote(do: foo)) == "foo" + assert quoted_to_string({:{}, [], nil}) == "{}" + end + + test "local call" do + assert quoted_to_string(quote(do: foo(1, 2, 3))) == "foo(1, 2, 3)" + assert quoted_to_string(quote(do: foo([1, 2, 3]))) == "foo([1, 2, 3])" + + assert quoted_to_string(quote(do: foo(1, 2, 3)), locals_without_parens: [foo: 3]) == + "foo 1, 2, 3" + + # Mixing literals and non-literals + assert quoted_to_string(quote(do: foo(a, 2))) == "foo(a, 2)" + assert quoted_to_string(quote(do: foo(1, b))) == "foo(1, b)" + + # Mixing literals and non-literals with line + assert quoted_to_string(quote(line: __ENV__.line, do: foo(a, 2))) == "foo(a, 2)" + assert quoted_to_string(quote(line: __ENV__.line, do: foo(1, b))) == "foo(1, b)" + end + + test "local call no parens" do + assert quoted_to_string({:def, [], [1, 2]}) == "def 1, 2" + assert quoted_to_string({:def, [closing: []], [1, 2]}) == "def(1, 2)" + end + + test "remote call" do + assert quoted_to_string(quote(do: foo.bar(1, 2, 3))) == "foo.bar(1, 2, 3)" + assert quoted_to_string(quote(do: foo.bar([1, 2, 3]))) == "foo.bar([1, 2, 3])" + + quoted = + quote do + (foo do + :ok + end).bar([1, 2, 3]) + end + + assert quoted_to_string(quoted) == "(foo do\n :ok\n end).bar([1, 2, 3])" + end + + test "nullary remote call" do + assert quoted_to_string(quote do: foo.bar) == "foo.bar" + assert quoted_to_string(quote do: foo.bar()) == "foo.bar()" + end + + test "atom remote call" do + assert quoted_to_string(quote(do: :foo.bar(1, 2, 3))) == ":foo.bar(1, 2, 3)" + end + + test "remote and fun call" do + assert quoted_to_string(quote(do: foo.bar().(1, 2, 3))) == "foo.bar().(1, 2, 3)" + assert quoted_to_string(quote(do: foo.bar().([1, 2, 3]))) == "foo.bar().([1, 2, 3])" + end + + test "unusual remote atom fun call" do + assert quoted_to_string(quote(do: Foo."42"())) == ~s/Foo."42"()/ + assert quoted_to_string(quote(do: Foo."Bar"())) == ~s/Foo."Bar"()/ + assert quoted_to_string(quote(do: Foo."bar baz"().""())) == ~s/Foo."bar baz"().""()/ + assert quoted_to_string(quote(do: Foo."%{}"())) == ~s/Foo."%{}"()/ + assert quoted_to_string(quote(do: Foo."..."())) == ~s/Foo."..."()/ + end + + test "atom fun call" do + assert quoted_to_string(quote(do: :foo.(1, 2, 3))) == ":foo.(1, 2, 3)" + end + + test "aliases call" do + assert quoted_to_string(quote(do: Foo.Bar.baz(1, 2, 3))) == "Foo.Bar.baz(1, 2, 3)" + assert quoted_to_string(quote(do: Foo.Bar.baz([1, 2, 3]))) == "Foo.Bar.baz([1, 2, 3])" + assert quoted_to_string(quote(do: ?0.Bar.baz([1, 2, 3]))) == "48.Bar.baz([1, 2, 3])" + assert quoted_to_string(quote(do: Foo.bar(<<>>, []))) == "Foo.bar(<<>>, [])" + end + + test "keyword call" do + assert quoted_to_string(quote(do: Foo.bar(foo: :bar))) == "Foo.bar(foo: :bar)" + assert quoted_to_string(quote(do: Foo.bar("Elixir.Foo": :bar))) == "Foo.bar([{Foo, :bar}])" + end + + test "sigil call" do + assert quoted_to_string(quote(do: ~r"123")) == ~S/~r"123"/ + assert quoted_to_string(quote(do: ~r"\n123")) == ~S/~r"\n123"/ + assert quoted_to_string(quote(do: ~r"12\"3")) == ~S/~r"12\"3"/ + assert quoted_to_string(quote(do: ~r/12\/3/u)) == ~S"~r/12\/3/u" + assert quoted_to_string(quote(do: ~r{\n123})) == ~S/~r{\n123}/ + assert quoted_to_string(quote(do: ~r((1\)(2\)3))) == ~S/~r((1\)(2\)3)/ + assert quoted_to_string(quote(do: ~r{\n1{1\}23})) == ~S/~r{\n1{1\}23}/ + assert quoted_to_string(quote(do: ~r|12\|3|)) == ~S"~r|12\|3|" + + assert quoted_to_string(quote(do: ~r[1#{two}3])) == ~S/~r[1#{two}3]/ + assert quoted_to_string(quote(do: ~r|1[#{two}]3|)) == ~S/~r|1[#{two}]3|/ + assert quoted_to_string(quote(do: ~r'1#{two}3'u)) == ~S/~r'1#{two}3'u/ + + assert quoted_to_string(quote(do: ~R"123")) == ~S/~R"123"/ + assert quoted_to_string(quote(do: ~R"123"u)) == ~S/~R"123"u/ + assert quoted_to_string(quote(do: ~R"\n123")) == ~S/~R"\n123"/ + + assert quoted_to_string(quote(do: ~S["'(123)'"])) == ~S/~S["'(123)'"]/ + assert quoted_to_string(quote(do: ~s"#{"foo"}")) == ~S/~s"#{"foo"}"/ + + assert quoted_to_string(quote(do: ~S["'(123)'"]) |> strip_metadata()) == ~S/~S"\"'(123)'\""/ + assert quoted_to_string(quote(do: ~s"#{"foo"}") |> strip_metadata()) == ~S/~s"#{"foo"}"/ + + assert quoted_to_string( + quote do + ~s""" + "\""foo"\"" + """ + end + ) == ~s[~s"""\n"\\""foo"\\""\n"""] + + assert quoted_to_string( + quote do + ~s''' + '\''foo'\'' + ''' + end + ) == ~s[~s'''\n'\\''foo'\\''\n'''] + + assert quoted_to_string( + quote do + ~s""" + "\"foo\"" + """ + end + ) == ~s[~s"""\n"\\"foo\\""\n"""] + + assert quoted_to_string( + quote do + ~s''' + '\"foo\"' + ''' + end + ) == ~s[~s'''\n'\\"foo\\"'\n'''] + + assert quoted_to_string( + quote do + ~S""" + "123" + """ + end + ) == ~s[~S"""\n"123"\n"""] + end + + test "tuple" do + assert quoted_to_string(quote do: {1, 2}) == "{1, 2}" + assert quoted_to_string(quote do: {1}) == "{1}" + assert quoted_to_string(quote do: {1, 2, 3}) == "{1, 2, 3}" + assert quoted_to_string(quote do: {1, 2, 3, foo: :bar}) == "{1, 2, 3, foo: :bar}" + end + + test "tuple call" do + assert quoted_to_string(quote(do: alias(Foo.{Bar, Baz, Bong}))) == + "alias Foo.{Bar, Baz, Bong}" + + assert quoted_to_string(quote(do: foo(Foo.{}))) == "foo(Foo.{})" + end + + test "arrow" do + assert quoted_to_string(quote(do: foo(1, (2 -> 3)))) == "foo(1, (2 -> 3))" + end + + test "block" do + quoted = + quote do + 1 + 2 + + ( + :foo + :bar + ) + + 3 + end + + expected = """ + 1 + 2 + + ( + :foo + :bar + ) + + 3 + """ + + assert quoted_to_string(quoted) <> "\n" == expected + end + + test "not in" do + assert quoted_to_string(quote(do: false not in [])) == "false not in []" + end + + test "if else" do + expected = """ + if foo do + bar + else + baz + end + """ + + assert quoted_to_string(quote(do: if(foo, do: bar, else: baz))) <> "\n" == expected + end + + test "case" do + quoted = + quote do + case foo do + true -> + 0 + + false -> + 1 + 2 + end + end + + expected = """ + case foo do + true -> + 0 + + false -> + 1 + 2 + end + """ + + assert quoted_to_string(quoted) <> "\n" == expected + end + + test "case if else" do + expected = """ + case (if foo do + bar + else + baz + end) do + end + """ + + assert quoted_to_string( + quote( + do: + case if(foo, do: bar, else: baz) do + end + ) + ) <> "\n" == expected + end + + test "try" do + quoted = + quote do + try do + foo + catch + _, _ -> + 2 + rescue + ArgumentError -> + 1 + after + 4 + else + _ -> + 3 + end + end + + expected = """ + try do + foo + catch + _, _ -> 2 + rescue + ArgumentError -> 1 + after + 4 + else + _ -> 3 + end + """ + + assert quoted_to_string(quoted) <> "\n" == expected + end + + test "fn" do + assert quoted_to_string(quote(do: fn -> 1 + 2 end)) == "fn -> 1 + 2 end" + assert quoted_to_string(quote(do: fn x -> x + 1 end)) == "fn x -> x + 1 end" + + quoted = + quote do + fn x -> + y = x + 1 + y + end + end + + expected = """ + fn x -> + y = x + 1 + y + end + """ + + assert quoted_to_string(quoted) <> "\n" == expected + + quoted = + quote do + fn + x -> + y = x + 1 + y + + z -> + z + end + end + + expected = """ + fn + x -> + y = x + 1 + y + + z -> + z + end + """ + + assert quoted_to_string(quoted) <> "\n" == expected + + assert quoted_to_string(quote(do: (fn x -> x end).(1))) == "(fn x -> x end).(1)" + + quoted = + quote do + (fn + %{} -> :map + _ -> :other + end).(1) + end + + expected = """ + (fn + %{} -> :map + _ -> :other + end).(1) + """ + + assert quoted_to_string(quoted) <> "\n" == expected + end + + test "range" do + assert quoted_to_string(quote(do: unquote(-1..+2))) == "-1..2" + assert quoted_to_string(quote(do: Foo.integer()..3)) == "Foo.integer()..3" + assert quoted_to_string(quote(do: unquote(-1..+2//-3))) == "-1..2//-3" + + assert quoted_to_string(quote(do: Foo.integer()..3//Bar.bat())) == + "Foo.integer()..3//Bar.bat()" + end + + test "when" do + assert quoted_to_string(quote(do: (() -> x))) == "(() -> x)" + assert quoted_to_string(quote(do: (x when y -> z))) == "(x when y -> z)" + assert quoted_to_string(quote(do: (x, y when z -> w))) == "(x, y when z -> w)" + assert quoted_to_string(quote(do: (x, y when z -> w))) == "(x, y when z -> w)" + assert quoted_to_string(quote(do: (x, y when z -> w))) == "(x, y when z -> w)" + assert quoted_to_string(quote(do: (x when y: z))) == "x when y: z" + assert quoted_to_string(quote(do: (x when y: z, z: w))) == "x when y: z, z: w" + end + + test "nested" do + quoted = + quote do + defmodule Foo do + def foo do + 1 + 1 + end + end + end + + expected = """ + defmodule Foo do + def foo do + 1 + 1 + end + end + """ + + assert quoted_to_string(quoted) <> "\n" == expected + end + + test "operator precedence" do + assert quoted_to_string(quote(do: (1 + 2) * (3 - 4))) == "(1 + 2) * (3 - 4)" + assert quoted_to_string(quote(do: (1 + 2) * 3 - 4)) == "(1 + 2) * 3 - 4" + assert quoted_to_string(quote(do: 1 + 2 + 3)) == "1 + 2 + 3" + assert quoted_to_string(quote(do: 1 + 2 - 3)) == "1 + 2 - 3" + end + + test "capture operator" do + assert quoted_to_string(quote(do: &foo/0)) == "&foo/0" + assert quoted_to_string(quote(do: &Foo.foo/0)) == "&Foo.foo/0" + assert quoted_to_string(quote(do: &(&1 + &2))) == "&(&1 + &2)" + assert quoted_to_string(quote(do: & &1)) == "& &1" + assert quoted_to_string(quote(do: & &1.(:x))) == "& &1.(:x)" + assert quoted_to_string(quote(do: (& &1).(:x))) == "(& &1).(:x)" + end + + test "operators" do + assert quoted_to_string(quote(do: foo |> {1, 2})) == "foo |> {1, 2}" + assert quoted_to_string(quote(do: foo |> {:map, arg})) == "foo |> {:map, arg}" + end + + test "containers" do + assert quoted_to_string(quote(do: {})) == "{}" + assert quoted_to_string(quote(do: [])) == "[]" + assert quoted_to_string(quote(do: {1, 2, 3})) == "{1, 2, 3}" + assert quoted_to_string(quote(do: [1, 2, 3])) == "[1, 2, 3]" + assert quoted_to_string(quote(do: ["Elixir.Foo": :bar])) == "[{Foo, :bar}]" + assert quoted_to_string(quote(do: %{})) == "%{}" + assert quoted_to_string(quote(do: %{:foo => :bar})) == "%{foo: :bar}" + assert quoted_to_string(quote(do: %{:"Elixir.Foo" => :bar})) == "%{Foo => :bar}" + assert quoted_to_string(quote(do: %{{1, 2} => [1, 2, 3]})) == "%{{1, 2} => [1, 2, 3]}" + assert quoted_to_string(quote(do: %{map | "a" => "b"})) == "%{map | \"a\" => \"b\"}" + assert quoted_to_string(quote(do: [1, 2, 3])) == "[1, 2, 3]" + end + + test "false positive containers" do + assert quoted_to_string({:%{}, [], nil}) == "%{}" + end + + test "struct" do + assert quoted_to_string(quote(do: %Test{})) == "%Test{}" + assert quoted_to_string(quote(do: %Test{foo: 1, bar: 1})) == "%Test{foo: 1, bar: 1}" + assert quoted_to_string(quote(do: %Test{struct | foo: 2})) == "%Test{struct | foo: 2}" + assert quoted_to_string(quote(do: %Test{} + 1)) == "%Test{} + 1" + assert quoted_to_string(quote(do: %Test{foo(1)} + 2)) == "%Test{foo(1)} + 2" + end + + test "binary operators" do + assert quoted_to_string(quote(do: 1 + 2)) == "1 + 2" + assert quoted_to_string(quote(do: [1, 2 | 3])) == "[1, 2 | 3]" + assert quoted_to_string(quote(do: [h | t] = [1, 2, 3])) == "[h | t] = [1, 2, 3]" + assert quoted_to_string(quote(do: (x ++ y) ++ z)) == "(x ++ y) ++ z" + assert quoted_to_string(quote(do: (x +++ y) +++ z)) == "(x +++ y) +++ z" + end + + test "unary operators" do + assert quoted_to_string(quote(do: not 1)) == "not 1" + assert quoted_to_string(quote(do: not foo)) == "not foo" + assert quoted_to_string(quote(do: -1)) == "-1" + assert quoted_to_string(quote(do: +(+1))) == "+(+1)" + assert quoted_to_string(quote(do: !(foo > bar))) == "!(foo > bar)" + assert quoted_to_string(quote(do: @foo(bar))) == "@foo bar" + assert quoted_to_string(quote(do: identity(&1))) == "identity(&1)" + end + + test "access" do + assert quoted_to_string(quote(do: a[b])) == "a[b]" + assert quoted_to_string(quote(do: a[1 + 2])) == "a[1 + 2]" + assert quoted_to_string(quote(do: (a || [a: 1])[:a])) == "(a || [a: 1])[:a]" + assert quoted_to_string(quote(do: Map.put(%{}, :a, 1)[:a])) == "Map.put(%{}, :a, 1)[:a]" + end + + test "keyword list" do + assert quoted_to_string(quote(do: [a: a, b: b])) == "[a: a, b: b]" + assert quoted_to_string(quote(do: [a: 1, b: 1 + 2])) == "[a: 1, b: 1 + 2]" + assert quoted_to_string(quote(do: ["a.b": 1, c: 1 + 2])) == "[\"a.b\": 1, c: 1 + 2]" + + tuple = {{:__block__, [format: :keyword], [:a]}, {:b, [], nil}} + assert quoted_to_string([tuple, :foo, tuple]) == "[{:a, b}, :foo, a: b]" + assert quoted_to_string([tuple, :foo, {:c, :d}, tuple]) == "[{:a, b}, :foo, c: :d, a: b]" + + # Not keyword lists + assert quoted_to_string(quote(do: [{binary(), integer()}])) == "[{binary(), integer()}]" + end + + test "interpolation" do + assert quoted_to_string(quote(do: "foo#{bar}baz")) == ~S["foo#{bar}baz"] + end + + test "bit syntax" do + ast = quote(do: <<1::8*4>>) + assert quoted_to_string(ast) == "<<1::8*4>>" + + ast = quote(do: @type(foo :: <<_::8, _::_*4>>)) + assert quoted_to_string(ast) == "@type foo :: <<_::8, _::_*4>>" + + ast = quote(do: <<69 - 4::bits-size(8 - 4)-unit(1), 65>>) + assert quoted_to_string(ast) == "<<69 - 4::bits-size(8 - 4)-unit(1), 65>>" + + ast = quote(do: <<(<<65>>), 65>>) + assert quoted_to_string(ast) == "<<(<<65>>), 65>>" + + ast = quote(do: <<65, (<<65>>)>>) + assert quoted_to_string(ast) == "<<65, (<<65>>)>>" + + ast = quote(do: for(<<(a::4 <- <<1, 2>>)>>, do: a)) + assert quoted_to_string(ast) == "for <<(a::4 <- <<1, 2>>)>> do\n a\nend" + end + + test "integer/float" do + assert quoted_to_string(1) == "1" + assert quoted_to_string({:__block__, [], [1]}) == "1" + end + + test "charlist" do + assert quoted_to_string(quote(do: [])) == "[]" + assert quoted_to_string(quote(do: 'abc')) == "'abc'" + end + + test "string" do + assert quoted_to_string(quote(do: "")) == ~S/""/ + assert quoted_to_string(quote(do: "abc")) == ~S/"abc"/ + assert quoted_to_string(quote(do: "#{"abc"}")) == ~S/"#{"abc"}"/ + end + + test "catch-all" do + assert quoted_to_string(quote do: {unquote(self())}) == "{#{inspect(self())}}" + assert quoted_to_string(quote do: foo(unquote(self()))) == "foo(#{inspect(self())})" + end + + test "last arg keyword list" do + assert quoted_to_string(quote(do: foo([]))) == "foo([])" + assert quoted_to_string(quote(do: foo(x: y))) == "foo(x: y)" + assert quoted_to_string(quote(do: foo(x: 1 + 2))) == "foo(x: 1 + 2)" + assert quoted_to_string(quote(do: foo(x: y, p: q))) == "foo(x: y, p: q)" + assert quoted_to_string(quote(do: foo(a, x: y, p: q))) == "foo(a, x: y, p: q)" + + assert quoted_to_string(quote(do: {[]})) == "{[]}" + assert quoted_to_string(quote(do: {[a: b]})) == "{[a: b]}" + assert quoted_to_string(quote(do: {x, a: b})) == "{x, a: b}" + assert quoted_to_string(quote(do: foo(else: a))) == "foo(else: a)" + assert quoted_to_string(quote(do: foo(catch: a))) == "foo(catch: a)" + assert quoted_to_string(quote(do: foo |> [bar: :baz])) == "foo |> [bar: :baz]" + end + + test "list in module attribute" do + assert quoted_to_string( + quote do + @foo [1] + end + ) == "@foo [1]" + + assert quoted_to_string( + quote do + @foo [foo: :bar] + end + ) == "@foo foo: :bar" + + assert quoted_to_string( + quote do + @foo [1, foo: :bar] + end + ) == "@foo [1, foo: :bar]" + end + end + + describe "quoted_to_algebra/2 escapes" do + test "strings with slash escapes" do + assert quoted_to_string(quote(do: "\a\b\d\e\f\n\r\t\v"), escape: false) == + ~s/"\a\b\d\e\f\n\r\t\v"/ + + assert quoted_to_string(quote(do: "\a\b\d\e\f\n\r\t\v")) == + ~s/"\\a\\b\\d\\e\\f\\n\\r\\t\\v"/ + + assert quoted_to_string({:__block__, [], ["\a\b\d\e\f\n\r\t\v"]}, escape: false) == + ~s/"\a\b\d\e\f\n\r\t\v"/ + + assert quoted_to_string({:__block__, [], ["\a\b\d\e\f\n\r\t\v"]}) == + ~s/"\\a\\b\\d\\e\\f\\n\\r\\t\\v"/ + end + + test "strings with non printable characters" do + assert quoted_to_string(quote(do: "\x00\x01\x10"), escape: false) == ~s/"\x00\x01\x10"/ + assert quoted_to_string(quote(do: "\x00\x01\x10")) == ~S/"\0\x01\x10"/ + end + + test "charlists with slash escapes" do + assert quoted_to_string(quote(do: '\a\b\e\n\r\t\v'), escape: false) == + ~s/'\a\b\e\n\r\t\v'/ + + assert quoted_to_string(quote(do: '\a\b\e\n\r\t\v')) == + ~s/'\\a\\b\\e\\n\\r\\t\\v'/ + + assert quoted_to_string({:__block__, [], ['\a\b\e\n\r\t\v']}, escape: false) == + ~s/'\a\b\e\n\r\t\v'/ + + assert quoted_to_string({:__block__, [], ['\a\b\e\n\r\t\v']}) == + ~s/'\\a\\b\\e\\n\\r\\t\\v'/ + end + + test "charlists with non printable characters" do + assert quoted_to_string(quote(do: '\x00\x01\x10'), escape: false) == ~S/[0, 1, 16]/ + assert quoted_to_string(quote(do: '\x00\x01\x10')) == ~S/[0, 1, 16]/ + end + + test "charlists with interpolations" do + assert quoted_to_string(quote(do: 'one #{2} three'), escape: false) == ~S/'one #{2} three'/ + assert quoted_to_string(quote(do: 'one #{2} three')) == ~S/'one #{2} three'/ + + assert quoted_to_string(quote(do: 'one\n\'#{2}\'\nthree'), escape: false) == + ~s['one\n\\'\#{2}\\'\nthree'] + + assert quoted_to_string(quote(do: 'one\n\'#{2}\'\nthree')) == ~S['one\n\'#{2}\'\nthree'] + end + + test "atoms" do + assert quoted_to_string(quote(do: :"a\nb\tc"), escape: false) == ~s/:"a\nb\tc"/ + assert quoted_to_string(quote(do: :"a\nb\tc")) == ~S/:"a\nb\tc"/ + + assert quoted_to_string({:__block__, [], [:"a\nb\tc"]}, escape: false) == ~s/:"a\nb\tc"/ + assert quoted_to_string({:__block__, [], [:"a\nb\tc"]}) == ~S/:"a\nb\tc"/ + + assert quoted_to_string(quote(do: :"Elixir")) == "Elixir" + assert quoted_to_string(quote(do: :"Elixir.Foo")) == "Foo" + assert quoted_to_string(quote(do: :"Elixir.Foo.Bar")) == "Foo.Bar" + assert quoted_to_string(quote(do: :"Elixir.foobar")) == ~S/:"Elixir.foobar"/ + end + + test "atoms with non printable characters" do + assert quoted_to_string(quote(do: :"\x00\x01\x10"), escape: false) == ~s/:"\0\x01\x10"/ + assert quoted_to_string(quote(do: :"\x00\x01\x10")) == ~S/:"\0\x01\x10"/ + end + + test "atoms with interpolations" do + assert quoted_to_string(quote(do: :"foo\n#{bar}\tbaz"), escape: false) == + ~s[:"foo\n\#{bar}\tbaz"] + + assert quoted_to_string(quote(do: :"foo\n#{bar}\tbaz")) == ~S[:"foo\n#{bar}\tbaz"] + + assert quoted_to_string(quote(do: :"foo\"bar"), escape: false) == ~S[:"foo\"bar"] + assert quoted_to_string(quote(do: :"foo\"bar")) == ~S[:"foo\"bar"] + + assert quoted_to_string(quote(do: :"foo#{~s/\n/}bar"), escape: false) == + ~S[:"foo#{~s/\n/}bar"] + + assert quoted_to_string(quote(do: :"foo#{~s/\n/}bar")) == ~S[:"foo#{~s/\n/}bar"] + + assert quoted_to_string(quote(do: :"one\n\"#{2}\"\nthree"), escape: false) == + ~s[:"one\n\\"\#{2}\\"\nthree"] + + assert quoted_to_string(quote(do: :"one\n\"#{2}\"\nthree")) == ~S[:"one\n\"#{2}\"\nthree"] + end + end + + describe "quoted_to_algebra/2 does not escape" do + test "sigils" do + assert quoted_to_string(quote(do: ~s/a\nb\tc/), escape: false) == ~S"~s/a\nb\tc/" + assert quoted_to_string(quote(do: ~s/a\nb\tc/)) == ~S"~s/a\nb\tc/" + + assert quoted_to_string(quote(do: ~s/\a\b\d\e\f\n\r\t\v/), escape: false) == + ~S"~s/\a\b\d\e\f\n\r\t\v/" + + assert quoted_to_string(quote(do: ~s/\a\b\d\e\f\n\r\t\v/)) == ~S"~s/\a\b\d\e\f\n\r\t\v/" + + assert quoted_to_string(quote(do: ~s/\x00\x01\x10/), escape: false) == ~S"~s/\x00\x01\x10/" + assert quoted_to_string(quote(do: ~s/\x00\x01\x10/)) == ~S"~s/\x00\x01\x10/" + end + end + + defp strip_metadata(ast) do + Macro.prewalk(ast, &Macro.update_meta(&1, fn _ -> [] end)) + end + + defp quoted_to_string(quoted, opts \\ []) do + doc = Code.quoted_to_algebra(quoted, opts) + + Inspect.Algebra.format(doc, 98) + |> IO.iodata_to_binary() + end +end diff --git a/lib/elixir/test/elixir/code_test.exs b/lib/elixir/test/elixir/code_test.exs index 645f784548c..d0b2884fe8d 100644 --- a/lib/elixir/test/elixir/code_test.exs +++ b/lib/elixir/test/elixir/code_test.exs @@ -1,74 +1,148 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule CodeTest do use ExUnit.Case, async: true + + doctest Code import PathHelpers - def one, do: 1 def genmodule(name) do defmodule name do - Kernel.LexicalTracker.remotes(__MODULE__) + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) end end - contents = quote do - defmodule CodeTest.Sample do - def eval_quoted_info, do: {__MODULE__, __ENV__.file, __ENV__.line} + contents = + quote do + defmodule CodeTest.Sample do + def eval_quoted_info, do: {__MODULE__, __ENV__.file, __ENV__.line} + end end - end - Code.eval_quoted contents, [], file: "sample.ex", line: 13 + Code.eval_quoted(contents, [], file: "sample.ex", line: 13) - test :eval_string do - assert Code.eval_string("1 + 2") == {3, []} - assert {3, _} = Code.eval_string("a + b", [a: 1, b: 2], Macro.Env.location(__ENV__)) - end + describe "eval_string/1,2,3" do + test "correctly evaluates a string of code" do + assert Code.eval_string("1 + 2") == {3, []} + assert Code.eval_string("two = 1 + 1") == {2, [two: 2]} + end - test :eval_string_with_other_context do - assert Code.eval_string("var!(a, Sample) = 1") == {1, [{{:a,Sample},1}]} - end + test "keeps bindings on optimized evals" do + assert Code.eval_string("import Enum", x: 1) == {Enum, [x: 1]} + end - test :eval_with_unnamed_scopes do - assert {%RuntimeError{}, [a: %RuntimeError{}]} = - Code.eval_string("a = (try do (raise \"hello\") rescue e -> e end)") - end + test "supports a %Macro.Env{} struct as the third argument" do + assert {3, _} = Code.eval_string("a + b", [a: 1, b: 2], __ENV__) + end - test :eval_with_scope do - assert Code.eval_string("one", [], delegate_locals_to: __MODULE__) == {1, []} - end + test "supports unnamed scopes" do + assert {%RuntimeError{}, [a: %RuntimeError{}]} = + Code.eval_string("a = (try do (raise \"hello\") rescue e -> e end)") + end - test :eval_options do - assert Code.eval_string("is_atom(:foo) and K.is_list([])", [], - functions: [{Kernel, [is_atom: 1]}], - macros: [{Kernel, [..: 2, and: 2]}], - aliases: [{K, Kernel}], - requires: [Kernel]) == {true, []} - end + test "supports the :requires option" do + assert Code.eval_string("Kernel.if true, do: :ok", [], requires: [Z, Kernel]) == {:ok, []} + end - test :eval_stacktrace do - try do - Code.eval_string("<>", a: :a, b: :b) - rescue - _ -> - assert System.stacktrace |> Enum.any?(&(elem(&1, 0) == __MODULE__)) + test "returns bindings from a different context" do + assert Code.eval_string("var!(a, Sample) = 1") == {1, [{{:a, Sample}, 1}]} end - end - test :eval_with_requires do - assert Code.eval_string("Kernel.if true, do: :ok", [], requires: [Z, Kernel]) == {:ok, []} + test "does not raise on duplicate bindings" do + # The order of which values win is not guaranteed, but it should evaluate successfully. + assert Code.eval_string("b = String.Chars.to_string(a)", a: 0, a: 1) == + {"1", [{:b, "1"}, {:a, 1}]} + + assert Code.eval_string("b = String.Chars.to_string(a)", a: 0, a: 1, c: 2) == + {"1", [{:c, 2}, {:b, "1"}, {:a, 1}]} + end + + test "with many options" do + options = [ + functions: [{Kernel, [is_atom: 1]}], + macros: [{Kernel, [and: 2]}], + aliases: [{K, Kernel}], + requires: [Kernel] + ] + + code = "is_atom(:foo) and K.is_list([])" + assert Code.eval_string(code, [], options) == {true, []} + end + + test "keeps caller in stacktrace" do + try do + Code.eval_string("<>", [a: :a, b: :b], file: "myfile") + rescue + _ -> + assert Enum.any?(__STACKTRACE__, &(elem(&1, 0) == __MODULE__)) + end + end + + if System.otp_release() >= "25" do + test "includes eval file in stacktrace" do + try do + Code.eval_string("<>", [a: :a, b: :b], file: "myfile") + rescue + _ -> + assert Exception.format_stacktrace(__STACKTRACE__) =~ "myfile:1" + end + + try do + Code.eval_string( + "Enum.map([a: :a, b: :b], fn {a, b} -> <> end)", + [], + file: "myfile" + ) + rescue + _ -> + assert Exception.format_stacktrace(__STACKTRACE__) =~ "myfile:1" + end + end + end + + test "warns when lexical tracker process is dead" do + {pid, ref} = spawn_monitor(fn -> :ok end) + assert_receive {:DOWN, ^ref, _, _, _} + env = %{__ENV__ | lexical_tracker: pid} + + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert Code.eval_string("1 + 2", [], env) == {3, []} + end) =~ "an __ENV__ with outdated compilation information was given to eval" + end + + test "emits checker warnings" do + output = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + Code.eval_string(File.read!(fixture_path("checker_warning.exs")), []) + end) + + assert output =~ "incompatible types" + end end - test :eval_quoted do + test "eval_quoted/1" do assert Code.eval_quoted(quote(do: 1 + 2)) == {3, []} assert CodeTest.Sample.eval_quoted_info() == {CodeTest.Sample, "sample.ex", 13} end - test :eval_quoted_with_env do + test "eval_quoted/2 with %Macro.Env{} at runtime" do alias :lists, as: MyList - assert Code.eval_quoted(quote(do: MyList.flatten [[1, 2, 3]]), [], __ENV__) == {[1, 2, 3],[]} + quoted = quote(do: MyList.flatten([[1, 2, 3]])) + + assert Code.eval_quoted(quoted, [], __ENV__) == {[1, 2, 3], []} + + # Let's check it discards tracers since the lexical tracker is explicitly nil + assert Code.eval_quoted(quoted, [], %{__ENV__ | tracers: [:bad]}) == {[1, 2, 3], []} end - test :eval_file do + test "eval_quoted/2 with %Macro.Env{} at compile time" do + defmodule CompileTimeEnv do + alias String.Chars + {"foo", []} = Code.eval_string("Chars.to_string(:foo)", [], __ENV__) + end + end + + test "eval_file/1" do assert Code.eval_file(fixture_path("code_sample.exs")) == {3, [var: 3]} assert_raise Code.LoadError, fn -> @@ -76,94 +150,215 @@ defmodule CodeTest do end end - test :require do - Code.require_file fixture_path("code_sample.exs") - assert fixture_path("code_sample.exs") in Code.loaded_files - assert Code.require_file(fixture_path("code_sample.exs")) == nil + test "eval_quoted_with_env/3" do + alias :lists, as: MyList + quoted = quote(do: MyList.flatten([[1, 2, 3]])) + env = Code.env_for_eval(__ENV__) + assert Code.eval_quoted_with_env(quoted, [], env) == {[1, 2, 3], [], env} - Code.unload_files [fixture_path("code_sample.exs")] - refute fixture_path("code_sample.exs") in Code.loaded_files - assert Code.require_file(fixture_path("code_sample.exs")) != nil + quoted = quote(do: alias(:dict, as: MyDict)) + {:dict, [], env} = Code.eval_quoted_with_env(quoted, [], env) + assert Macro.Env.fetch_alias(env, :MyDict) == {:ok, :dict} end - test :string_to_quoted do - assert Code.string_to_quoted("1 + 2") == {:ok, {:+, [line: 1], [1, 2]}} - assert Code.string_to_quoted!("1 + 2") == {:+, [line: 1], [1, 2]} + test "eval_quoted_with_env/3 with vars" do + env = Code.env_for_eval(__ENV__) + {1, [x: 1], env} = Code.eval_quoted_with_env(quote(do: var!(x) = 1), [], env) + assert Macro.Env.vars(env) == [{:x, nil}] + end - assert Code.string_to_quoted("a.1") == - {:error, {1, "syntax error before: ", "1"}} + test "compile_file/1" do + assert Code.compile_file(fixture_path("code_sample.exs")) == [] + refute fixture_path("code_sample.exs") in Code.required_files() + end - assert_raise SyntaxError, fn -> - Code.string_to_quoted!("a.1") - end + test "compile_file/1 also emits checker warnings" do + output = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + Code.compile_file(fixture_path("checker_warning.exs")) + end) + + assert output =~ "incompatible types" end - test :string_to_quoted_existing_atoms_only do - assert :badarg = catch_error(Code.string_to_quoted!(":thereisnosuchatom", existing_atoms_only: true)) + test "require_file/1" do + assert Code.require_file(fixture_path("code_sample.exs")) == [] + assert fixture_path("code_sample.exs") in Code.required_files() + assert Code.require_file(fixture_path("code_sample.exs")) == nil + + Code.unrequire_files([fixture_path("code_sample.exs")]) + refute fixture_path("code_sample.exs") in Code.required_files() + assert Code.require_file(fixture_path("code_sample.exs")) != nil + after + Code.unrequire_files([fixture_path("code_sample.exs")]) end - test :string_to_quoted! do - assert Code.string_to_quoted!("1 + 2") == {:+, [line: 1], [1, 2]} + test "string_to_quoted!/2 errors take lines and columns into account" do + message = "nofile:1:5: syntax error before: '*'\n |\n 1 | 1 + * 3\n | ^" + + assert_raise SyntaxError, message, fn -> + Code.string_to_quoted!("1 + * 3") + end + + message = "nofile:10:5: syntax error before: '*'\n |\n 10 | 1 + * 3\n | ^" - assert_raise SyntaxError, fn -> - Code.string_to_quoted!("a.1") + assert_raise SyntaxError, message, fn -> + Code.string_to_quoted!("1 + * 3", line: 10) end - assert_raise TokenMissingError, fn -> - Code.string_to_quoted!("1 +") + message = "nofile:10:7: syntax error before: '*'\n |\n 10 | 1 + * 3\n | ^" + + assert_raise SyntaxError, message, fn -> + Code.string_to_quoted!("1 + * 3", line: 10, column: 3) + end + + message = "nofile:11:5: syntax error before: '*'\n |\n 11 | 1 + * 3\n | ^" + + assert_raise SyntaxError, message, fn -> + Code.string_to_quoted!(":ok\n1 + * 3", line: 10, column: 3) end end - test :compile_source do - assert __MODULE__.__info__(:compile)[:source] == String.to_char_list(__ENV__.file) + test "string_to_quoted only requires the List.Chars protocol implementation to work" do + assert {:ok, 1.23} = Code.string_to_quoted(1.23) + assert 1.23 = Code.string_to_quoted!(1.23) + assert {:ok, 1.23, []} = Code.string_to_quoted_with_comments(1.23) + assert {1.23, []} = Code.string_to_quoted_with_comments!(1.23) + end + + test "string_to_quoted returns error on incomplete escaped string" do + assert Code.string_to_quoted("\"\\") == + {:error, + {[line: 1, column: 3], "missing terminator: \" (for string starting at line 1)", ""}} end - test :compile_info_returned_with_source_accessible_through_keyword_module do + test "compile source" do + assert __MODULE__.__info__(:compile)[:source] == String.to_charlist(__ENV__.file) + end + + test "compile info returned with source accessible through keyword module" do compile = __MODULE__.__info__(:compile) assert Keyword.get(compile, :source) != nil end - test :compile_string_works_accross_lexical_scopes do - assert [{CompileCrossSample, _}] = Code.compile_string("CodeTest.genmodule CompileCrossSample") - after - :code.purge CompileCrossSample - :code.delete CompileCrossSample - end + describe "compile_string/1" do + test "compiles the given string" do + assert [{CompileStringSample, _}] = + Code.compile_string("defmodule CompileStringSample, do: :ok") + after + :code.purge(CompileSimpleSample) + :code.delete(CompileSimpleSample) + end - test :compile_string do - assert [{CompileStringSample, _}] = Code.compile_string("defmodule CompileStringSample, do: :ok") - after - :code.purge CompileSimpleSample - :code.delete CompileSimpleSample + test "emits checker warnings" do + output = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + Code.compile_string(File.read!(fixture_path("checker_warning.exs"))) + end) + + assert output =~ "incompatible types" + end + + test "works across lexical scopes" do + assert [{CompileCrossSample, _}] = + Code.compile_string("CodeTest.genmodule CompileCrossSample") + after + :code.purge(CompileCrossSample) + :code.delete(CompileCrossSample) + end end - test :compile_quoted do - assert [{CompileQuotedSample, _}] = Code.compile_string("defmodule CompileQuotedSample, do: :ok") - after - :code.purge CompileQuotedSample - :code.delete CompileQuotedSample + test "format_string/2 returns empty iodata for empty string" do + assert Code.format_string!("") == [] end - test :ensure_loaded? do + test "ensure_loaded?/1" do assert Code.ensure_loaded?(__MODULE__) - refute Code.ensure_loaded?(Unknown.Module) + refute Code.ensure_loaded?(Code.NoFile) + end + + test "ensure_loaded!/1" do + assert Code.ensure_loaded!(__MODULE__) == __MODULE__ + + assert_raise ArgumentError, "could not load module Code.NoFile due to reason :nofile", fn -> + Code.ensure_loaded!(Code.NoFile) + end end - test :ensure_compiled? do - assert Code.ensure_compiled?(__MODULE__) - refute Code.ensure_compiled?(Unknown.Module) + test "ensure_compiled/1" do + assert Code.ensure_compiled(__MODULE__) == {:module, __MODULE__} + assert Code.ensure_compiled(Code.NoFile) == {:error, :nofile} + end + + test "ensure_compiled!/1" do + assert Code.ensure_compiled!(__MODULE__) == __MODULE__ + + assert_raise ArgumentError, "could not load module Code.NoFile due to reason :nofile", fn -> + Code.ensure_compiled!(Code.NoFile) + end + end + + test "put_compiler_option/2 validates options" do + message = "unknown compiler option: :not_a_valid_option" + + assert_raise RuntimeError, message, fn -> + Code.put_compiler_option(:not_a_valid_option, :foo) + end + + message = "compiler option :debug_info should be a boolean, got: :not_a_boolean" + + assert_raise RuntimeError, message, fn -> + Code.put_compiler_option(:debug_info, :not_a_boolean) + end end end defmodule Code.SyncTest do use ExUnit.Case - test :path_manipulation do + import PathHelpers + + test "path manipulation" do path = Path.join(__DIR__, "fixtures") - Code.prepend_path path - assert to_char_list(path) in :code.get_path + Code.prepend_path(path) + assert to_charlist(path) in :code.get_path() + + Code.delete_path(path) + refute to_charlist(path) in :code.get_path() + end - Code.delete_path path - refute to_char_list(path) in :code.get_path + test "purges compiler modules" do + quoted = quote(do: Agent.start_link(fn -> :ok end)) + Code.compile_quoted(quoted) + + {:ok, claimed} = Code.purge_compiler_modules() + assert claimed > 0 + + {:ok, claimed} = Code.purge_compiler_modules() + assert claimed == 0 + end + + test "returns previous options when setting compiler options" do + Code.compiler_options(debug_info: false) + assert Code.compiler_options(debug_info: true) == %{debug_info: false} + after + Code.compiler_options(debug_info: true) + end + + test "compile_file/1 return value" do + assert [{CompileSample, binary}] = Code.compile_file(fixture_path("compile_sample.ex")) + assert is_binary(binary) + after + :code.purge(CompileSample) + :code.delete(CompileSample) + end + + test "require_file/1 return value" do + assert [{CompileSample, binary}] = Code.require_file(fixture_path("compile_sample.ex")) + assert is_binary(binary) + after + Code.unrequire_files([fixture_path("compile_sample.ex")]) + :code.purge(CompileSample) + :code.delete(CompileSample) end end diff --git a/lib/elixir/test/elixir/collectable_test.exs b/lib/elixir/test/elixir/collectable_test.exs new file mode 100644 index 00000000000..50cad162af8 --- /dev/null +++ b/lib/elixir/test/elixir/collectable_test.exs @@ -0,0 +1,7 @@ +Code.require_file("test_helper.exs", __DIR__) + +defmodule CollectableTest do + use ExUnit.Case, async: true + + doctest Collectable +end diff --git a/lib/elixir/test/elixir/config/provider_test.exs b/lib/elixir/test/elixir/config/provider_test.exs new file mode 100644 index 00000000000..0c533227ce7 --- /dev/null +++ b/lib/elixir/test/elixir/config/provider_test.exs @@ -0,0 +1,182 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Config.ProviderTest do + use ExUnit.Case + + doctest Config.Provider + alias Config.Provider + import PathHelpers + import ExUnit.CaptureIO + + @tmp_path tmp_path("config_provider") + @env_var "ELIXIR_CONFIG_PROVIDER_BOOTED" + @sys_config Path.join(@tmp_path, "sys.config") + + setup context do + File.rm_rf(@tmp_path) + File.mkdir_p!(@tmp_path) + write_sys_config!(context[:sys_config] || []) + + on_exit(fn -> + Application.delete_env(:elixir, :config_provider_init) + Application.delete_env(:elixir, :config_provider_booted) + System.delete_env(@env_var) + end) + end + + test "validate_compile_env" do + Config.Provider.validate_compile_env([{:elixir, [:unknown], :error}]) + + Application.put_env(:elixir, :unknown, nested: [key: :value]) + Config.Provider.validate_compile_env([{:elixir, [:unknown], {:ok, [nested: [key: :value]]}}]) + Config.Provider.validate_compile_env([{:elixir, [:unknown, :nested], {:ok, [key: :value]}}]) + Config.Provider.validate_compile_env([{:elixir, [:unknown, :nested, :key], {:ok, :value}}]) + Config.Provider.validate_compile_env([{:elixir, [:unknown, :nested, :unknown], :error}]) + + assert capture_abort(fn -> + Config.Provider.validate_compile_env([{:elixir, [:unknown, :nested], :error}]) + end) =~ "Compile time value was not set" + + assert capture_abort(fn -> + Config.Provider.validate_compile_env([ + {:elixir, [:unknown, :nested], {:ok, :another}} + ]) + end) =~ "Compile time value was set to: :another" + + assert capture_abort(fn -> + keys = [:unknown, :nested, :key, :too_deep] + Config.Provider.validate_compile_env([{:elixir, keys, :error}]) + end) =~ + "application :elixir failed reading its compile environment for path [:nested, :key, :too_deep] inside key :unknown" + after + Application.delete_env(:elixir, :unknown) + end + + describe "config_path" do + test "validate!" do + assert Provider.validate_config_path!("/foo") == :ok + assert Provider.validate_config_path!({:system, "foo", "bar"}) == :ok + + assert_raise ArgumentError, fn -> Provider.validate_config_path!({:system, 1, 2}) end + assert_raise ArgumentError, fn -> Provider.validate_config_path!('baz') end + end + + test "resolve!" do + env_var = "ELIXIR_CONFIG_PROVIDER_PATH" + + try do + System.put_env(env_var, @tmp_path) + assert Provider.resolve_config_path!("/foo") == "/foo" + assert Provider.resolve_config_path!({:system, env_var, "/bar"}) == @tmp_path <> "/bar" + after + System.delete_env(env_var) + end + end + end + + describe "boot" do + test "runs providers" do + init_and_assert_boot() + config = consult(@sys_config) + assert config[:my_app] == [key: :value] + assert config[:elixir] == [config_provider_booted: {:booted, nil}] + end + + @tag sys_config: [my_app: [encoding: {:_μ, :"£", "£", '£'}]] + test "writes sys_config with encoding" do + init_and_assert_boot() + config = consult(@sys_config) + assert config[:my_app][:encoding] == {:_μ, :"£", "£", '£'} + end + + @tag sys_config: [my_app: [key: :old_value, sys_key: :sys_value, extra_config: :old_value]] + test "writes extra config with overrides" do + init_and_assert_boot(extra_config: [my_app: [key: :old_extra_value, extra_config: :value]]) + + assert consult(@sys_config)[:my_app] == + [sys_key: :sys_value, extra_config: :value, key: :value] + end + + test "returns :booted if already booted and keeps config file" do + init_and_assert_boot() + Application.put_all_env(Keyword.take(consult(@sys_config), [:elixir])) + assert boot() == :booted + refute_received :restart + assert File.exists?(@sys_config) + end + + test "returns :booted if already booted and prunes config file" do + init_and_assert_boot(prune_runtime_sys_config_after_boot: true) + Application.put_all_env(Keyword.take(consult(@sys_config), [:elixir])) + assert boot() == :booted + refute_received :restart + refute File.exists?(@sys_config) + end + + test "returns :booted if already booted and runs validate_compile_env" do + init_and_assert_boot( + prune_runtime_sys_config_after_boot: true, + validate_compile_env: [{:elixir, [:unknown], {:ok, :value}}] + ) + + Application.put_all_env(Keyword.take(consult(@sys_config), [:elixir])) + + assert capture_abort(fn -> boot() end) =~ + "the application :elixir has a different value set for key :unknown" + end + + test "returns without rebooting" do + reader = {Config.Reader, fixture_path("configs/kernel.exs")} + init = Config.Provider.init([reader], @sys_config, reboot_system_after_config: false) + Application.put_all_env(init) + + assert capture_abort(fn -> + Provider.boot(fn -> + raise "should not be called" + end) + end) =~ + "Cannot configure :kernel because :reboot_system_after_config has been set to false" + + # Make sure values before and after match + write_sys_config!(kernel: [elixir_reboot: true]) + Application.put_all_env(init) + System.delete_env(@env_var) + + Provider.boot(fn -> raise "should not be called" end) + assert Application.get_env(:kernel, :elixir_reboot) == true + assert Application.get_env(:elixir_reboot, :key) == :value + end + end + + defp init(opts) do + reader = {Config.Reader, fixture_path("configs/good_config.exs")} + init = Config.Provider.init([reader], Keyword.get(opts, :path, @sys_config), opts) + Application.put_all_env(init) + init + end + + defp boot do + Provider.boot(fn -> send(self(), :restart) end) + end + + defp init_and_assert_boot(opts \\ []) do + init(opts ++ [reboot_system_after_config: true]) + boot() + assert_received :restart + end + + defp consult(file) do + {:ok, [config]} = :file.consult(file) + config + end + + defp capture_abort(fun) do + capture_io(fn -> + assert_raise ErlangError, fun + end) + end + + defp write_sys_config!(data) do + File.write!(@sys_config, IO.chardata_to_string(:io_lib.format("~tw.~n", [data]))) + end +end diff --git a/lib/elixir/test/elixir/config/reader_test.exs b/lib/elixir/test/elixir/config/reader_test.exs new file mode 100644 index 00000000000..4237997becb --- /dev/null +++ b/lib/elixir/test/elixir/config/reader_test.exs @@ -0,0 +1,89 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Config.ReaderTest do + use ExUnit.Case, async: true + + doctest Config.Reader + import PathHelpers + + test "read_imports!/2" do + assert Config.Reader.read_imports!(fixture_path("configs/good_kw.exs")) == + {[my_app: [key: :value]], [fixture_path("configs/good_kw.exs")]} + + assert Config.Reader.read_imports!(fixture_path("configs/good_config.exs")) == + {[my_app: [key: :value]], [fixture_path("configs/good_config.exs")]} + + assert Config.Reader.read_imports!(fixture_path("configs/good_import.exs")) == + {[my_app: [key: :value]], + [fixture_path("configs/good_config.exs"), fixture_path("configs/good_import.exs")]} + + assert_raise ArgumentError, + ":imports must be a list of paths", + fn -> Config.Reader.read_imports!("config", imports: :disabled) end + + assert_raise File.Error, + fn -> Config.Reader.read_imports!(fixture_path("configs/bad_root.exs")) end + + assert_raise File.Error, + fn -> Config.Reader.read_imports!(fixture_path("configs/bad_import.exs")) end + end + + test "read!/2" do + assert Config.Reader.read!(fixture_path("configs/good_kw.exs")) == + [my_app: [key: :value]] + + assert Config.Reader.read!(fixture_path("configs/good_config.exs")) == + [my_app: [key: :value]] + + assert Config.Reader.read!(fixture_path("configs/good_import.exs")) == + [my_app: [key: :value]] + + assert Config.Reader.read!(fixture_path("configs/env.exs"), env: :dev, target: :host) == + [my_app: [env: :dev, target: :host]] + + assert Config.Reader.read!(fixture_path("configs/env.exs"), env: :prod, target: :embedded) == + [my_app: [env: :prod, target: :embedded]] + + assert_raise ArgumentError, + ~r"expected config for app :sample in .*/bad_app.exs to return keyword list", + fn -> Config.Reader.read!(fixture_path("configs/bad_app.exs")) end + + assert_raise RuntimeError, "no :env key was given to this configuration file", fn -> + Config.Reader.read!(fixture_path("configs/env.exs")) + end + + assert_raise RuntimeError, "no :target key was given to this configuration file", fn -> + Config.Reader.read!(fixture_path("configs/env.exs"), env: :prod) + end + + assert_raise RuntimeError, + ~r"import_config/1 is not enabled for this configuration file", + fn -> + Config.Reader.read!(fixture_path("configs/good_import.exs"), imports: :disabled) + end + end + + test "eval!/3" do + files = ["configs/good_kw.exs", "configs/good_config.exs", "configs/good_import.exs"] + + for file <- files do + file = fixture_path(file) + assert Config.Reader.read!(file) == Config.Reader.eval!(file, File.read!(file)) + end + + file = fixture_path("configs/env.exs") + + assert Config.Reader.read!(file, env: :dev, target: :host) == + Config.Reader.eval!(file, File.read!(file), env: :dev, target: :host) + end + + test "as a provider" do + state = Config.Reader.init(fixture_path("configs/good_config.exs")) + assert Config.Reader.load([my_app: [key: :old_value]], state) == [my_app: [key: :value]] + + state = Config.Reader.init(path: fixture_path("configs/env.exs"), env: :prod, target: :host) + + assert Config.Reader.load([my_app: [env: :dev]], state) == + [my_app: [env: :prod, target: :host]] + end +end diff --git a/lib/elixir/test/elixir/config_test.exs b/lib/elixir/test/elixir/config_test.exs new file mode 100644 index 00000000000..cd6f30c5e5b --- /dev/null +++ b/lib/elixir/test/elixir/config_test.exs @@ -0,0 +1,119 @@ +Code.require_file("test_helper.exs", __DIR__) + +defmodule ConfigTest do + use ExUnit.Case, async: true + + doctest Config + import Config + import PathHelpers + + setup config do + Process.put({Config, :opts}, {config[:env], config[:target]}) + Process.put({Config, :config}, []) + Process.put({Config, :imports}, config[:imports] || []) + :ok + end + + defp config do + Process.get({Config, :config}) + end + + defp files do + Process.get({Config, :imports}) + end + + test "config/2" do + assert config() == [] + + config :lager, key: :value + assert config() == [lager: [key: :value]] + + config :lager, other: :value + assert config() == [lager: [key: :value, other: :value]] + + config :lager, key: :other + assert config() == [lager: [other: :value, key: :other]] + + # Works inside functions too... + f = fn -> config(:lager, key: :fn) end + f.() + assert config() == [lager: [other: :value, key: :fn]] + + # ...and in for comprehensions. + for _ <- 0..0, do: config(:lager, key: :for) + assert config() == [lager: [other: :value, key: :for]] + end + + test "config/3" do + config :app, Repo, key: :value + assert config() == [app: [{Repo, key: :value}]] + + config :app, Repo, other: :value + assert config() == [app: [{Repo, key: :value, other: :value}]] + + config :app, Repo, key: :other + assert config() == [app: [{Repo, other: :value, key: :other}]] + + config :app, Repo, key: [nested: false] + assert config() == [app: [{Repo, other: :value, key: [nested: false]}]] + + config :app, Repo, key: [nested: true] + assert config() == [app: [{Repo, other: :value, key: [nested: true]}]] + + config :app, Repo, key: :other + assert config() == [app: [{Repo, other: :value, key: :other}]] + end + + @tag env: :dev + test "config_env/0" do + assert config_env() == :dev + end + + test "config_env/0 raises if no env is set" do + assert_raise RuntimeError, "no :env key was given to this configuration file", fn -> + config_env() + end + end + + @tag target: :host + test "config_target/0" do + assert config_target() == :host + end + + test "config_target/0 raises if no env is set" do + assert_raise RuntimeError, "no :target key was given to this configuration file", fn -> + config_target() + end + end + + test "import_config/1" do + import_config fixture_path("configs/good_config.exs") + assert config() == [my_app: [key: :value]] + assert files() == [fixture_path("configs/good_config.exs")] + end + + @tag imports: :disabled + test "import_config/1 raises when disabled" do + assert_raise RuntimeError, + ~r"import_config/1 is not enabled for this configuration file", + fn -> import_config fixture_path("configs/good_config.exs") end + end + + test "import_config/1 raises for recursive import" do + assert_raise ArgumentError, + ~r"attempting to load configuration .*/imports_recursive.exs recursively", + fn -> import_config fixture_path("configs/imports_recursive.exs") end + end + + test "import_config/1 with nested" do + config :app, Repo, key: [nested: false, other: true] + import_config fixture_path("configs/nested.exs") + assert config() == [app: [{Repo, key: [other: true, nested: true]}]] + end + + test "import_config/1 with bad path" do + assert_raise File.Error, ~r"could not read file .*/configs/unknown.exs", fn -> + import_config fixture_path("configs/unknown.exs") + end + end +end diff --git a/lib/elixir/test/elixir/dict_test.exs b/lib/elixir/test/elixir/dict_test.exs deleted file mode 100644 index 31e15b447c0..00000000000 --- a/lib/elixir/test/elixir/dict_test.exs +++ /dev/null @@ -1,415 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -# A TestDict implementation used only for testing. -defmodule TestDict do - defstruct list: [] - - def new(list \\ []) when is_list(list) do - %TestDict{list: list} - end - - def size(%TestDict{list: list}) do - length(list) - end - - def update(%TestDict{list: list} = map, key, initial, fun) do - %{map | list: update(list, key, initial, fun)} - end - - def update([{key, value}|list], key, _initial, fun) do - [{key, fun.(value)}|list] - end - - def update([{_, _} = e|list], key, initial, fun) do - [e|update(list, key, initial, fun)] - end - - def update([], key, initial, _fun) do - [{key, initial}] - end - - defimpl Enumerable do - def reduce(%{list: list}, acc, fun), do: Enumerable.List.reduce(list, acc, fun) - def member?(%{list: list}, other), do: Enumerable.List.member(list, other) - def count(%{list: list}), do: Enumerable.List.count(list) - end -end - -defmodule DictTest.Common do - defmacro __using__(_) do - quote location: :keep do - import Enum, only: [sort: 1] - - defp new_dict(list \\ [{"first_key", 1}, {"second_key", 2}]) do - Enum.into list, dict_impl.new - end - - defp new_dict(list, transform) do - Enum.into list, dict_impl.new, transform - end - - defp int_dict do - Enum.into [{1,1}], dict_impl.new - end - - test "access" do - dict = new_dict() - assert dict["first_key"] == 1 - assert dict["other_key"] == nil - end - - test "access uses match operation" do - dict = int_dict() - assert dict[1] == 1 - assert dict[1.0] == nil - end - - test "get/2 and get/3" do - dict = new_dict() - assert Dict.get(dict, "first_key") == 1 - assert Dict.get(dict, "second_key") == 2 - assert Dict.get(dict, "other_key") == nil - assert Dict.get(dict, "other_key", 3) == 3 - end - - test "get/2 with match" do - assert Dict.get(int_dict, 1) == 1 - assert Dict.get(int_dict, 1.0) == nil - end - - test "fetch/2" do - dict = new_dict() - assert Dict.fetch(dict, "first_key") == {:ok, 1} - assert Dict.fetch(dict, "second_key") == {:ok, 2} - assert Dict.fetch(dict, "other_key") == :error - end - - test "fetch/2 with match" do - assert Dict.fetch(int_dict, 1) == {:ok, 1} - assert Dict.fetch(int_dict, 1.0) == :error - end - - test "fetch!/2" do - dict = new_dict() - assert Dict.fetch!(dict, "first_key") == 1 - assert Dict.fetch!(dict, "second_key") == 2 - assert_raise KeyError, fn -> - Dict.fetch!(dict, "other_key") - end - end - - test "put/3" do - dict = new_dict() |> Dict.put("first_key", {1}) - assert Dict.get(dict, "first_key") == {1} - assert Dict.get(dict, "second_key") == 2 - end - - test "put/3 with_match" do - dict = int_dict() - assert Dict.get(Dict.put(dict, 1, :other), 1) == :other - assert Dict.get(Dict.put(dict, 1.0, :other), 1) == 1 - assert Dict.get(Dict.put(dict, 1, :other), 1.0) == nil - assert Dict.get(Dict.put(dict, 1.0, :other), 1.0) == :other - end - - test "put_new/3" do - dict = Dict.put_new(new_dict(), "first_key", {1}) - assert Dict.get(dict, "first_key") == 1 - end - - test "put_new/3 with_match" do - assert Dict.get(Dict.put_new(int_dict, 1, :other), 1) == 1 - assert Dict.get(Dict.put_new(int_dict, 1.0, :other), 1) == 1 - assert Dict.get(Dict.put_new(int_dict, 1, :other), 1.0) == nil - assert Dict.get(Dict.put_new(int_dict, 1.0, :other), 1.0) == :other - end - - test "keys/1" do - assert Enum.sort(Dict.keys(new_dict())) == ["first_key", "second_key"] - assert Dict.keys(new_dict([])) == [] - end - - test "values/1" do - assert Enum.sort(Dict.values(new_dict())) == [1, 2] - assert Dict.values(new_dict([])) == [] - end - - test "delete/2" do - dict = Dict.delete(new_dict(), "second_key") - assert Dict.size(dict) == 1 - assert Dict.has_key?(dict, "first_key") - refute Dict.has_key?(dict, "second_key") - - dict = Dict.delete(new_dict(), "other_key") - assert dict == new_dict() - assert Dict.size(dict) == 2 - end - - test "delete/2 with match" do - assert Dict.get(Dict.delete(int_dict, 1), 1) == nil - assert Dict.get(Dict.delete(int_dict, 1.0), 1) == 1 - end - - test "merge/2" do - dict = new_dict() - assert Dict.merge(new_dict([]), dict) == dict - assert Dict.merge(dict, new_dict([])) == dict - assert Dict.merge(dict, dict) == dict - assert Dict.merge(new_dict([]), new_dict([])) == new_dict([]) - - dict1 = new_dict [{"a", 1}, {"b", 2}, {"c", 3}] - dict2 = new_dict [{"a", 3}, {"c", :a}, {"d", 0}] - assert Dict.merge(dict1, dict2) |> Enum.sort == - [{"a", 3}, {"b", 2}, {"c", :a}, {"d", 0}] - end - - test "merge/2 with other dict" do - dict1 = new_dict [{"a", 1}, {"b", 2}, {"c", 3}] - dict2 = TestDict.new [{"a",3}, {"c",:a}, {"d",0}] - actual = Dict.merge(dict1, dict2) - assert Dict.merge(dict1, dict2) |> Enum.sort == - [{"a", 3}, {"b", 2}, {"c", :a}, {"d", 0}] - assert Dict.merge(dict2, dict1) |> Enum.sort == - [{"a", 1}, {"b", 2}, {"c", 3}, {"d", 0}] - end - - test "merge/3" do - dict1 = new_dict [{"a", 1}, {"b", 2}] - dict2 = new_dict [{"a", 3}, {"d", 4}] - actual = Dict.merge dict1, dict2, fn _k, v1, v2 -> v1 + v2 end - assert Enum.sort(actual) == [{"a", 4}, {"b", 2}, {"d", 4}] - end - - test "has_key?/2" do - dict = new_dict() - assert Dict.has_key?(dict, "first_key") - refute Dict.has_key?(dict, "other_key") - end - - test "has_key?/2 with match" do - assert Dict.has_key?(int_dict, 1) - refute Dict.has_key?(int_dict, 1.0) - end - - test "size/1" do - assert Dict.size(new_dict()) == 2 - assert Dict.size(new_dict([])) == 0 - end - - test "update!/3" do - dict = Dict.update!(new_dict(), "first_key", fn val -> -val end) - assert Dict.get(dict, "first_key") == -1 - - assert_raise KeyError, fn -> - Dict.update!(new_dict(), "non-existent", fn val -> -val end) - end - end - - test "update!/3 with match" do - assert Dict.get(Dict.update!(int_dict(), 1, &(&1 + 1)), 1) == 2 - end - - test "update/4" do - dict = Dict.update(new_dict(), "first_key", 0, fn val -> -val end) - assert Dict.get(dict, "first_key") == -1 - - dict = Dict.update(new_dict(), "non-existent", "...", fn val -> -val end) - assert Dict.get(dict, "non-existent") == "..." - end - - test "update/4 with match" do - dict = int_dict() - assert Dict.get(Dict.update(dict, 1.0, 2, &(&1 + 1)), 1) == 1 - assert Dict.get(Dict.update(dict, 1.0, 2, &(&1 + 1)), 1.0) == 2 - end - - test "pop/2 and pop/3" do - dict = new_dict() - - {v, actual} = Dict.pop(dict, "first_key") - assert v == 1 - assert actual == new_dict([{"second_key", 2}]) - - {v, actual} = Dict.pop(dict, "other_key") - assert v == nil - assert dict == actual - - {v, actual} = Dict.pop(dict, "other_key", "default") - assert v == "default" - assert dict == actual - end - - test "pop/2 and pop/3 with match" do - dict = int_dict() - - {v, actual} = Dict.pop(dict, 1) - assert v == 1 - assert Enum.sort(actual) == [] - - {v, actual} = Dict.pop(dict, 1.0) - assert v == nil - assert actual == dict - end - - test "split/2" do - dict = new_dict() - - {take, drop} = Dict.split(dict, []) - assert take == new_dict([]) - assert drop == dict - - {take, drop} = Dict.split(dict, ["unknown_key"]) - assert take == new_dict([]) - assert drop == dict - - split_keys = ["first_key", "second_key", "unknown_key"] - {take, drop} = Dict.split(dict, split_keys) - - take_expected = new_dict([]) - |> Dict.put("first_key", 1) - |> Dict.put("second_key", 2) - - drop_expected = new_dict([]) - |> Dict.delete("first_key") - |> Dict.delete("second_key") - - assert Enum.sort(take) == Enum.sort(take_expected) - assert Enum.sort(drop) == Enum.sort(drop_expected) - end - - test "split/2 with match" do - dict = int_dict() - {take, drop} = Dict.split(dict, [1]) - assert take == dict - assert drop == new_dict([]) - - {take, drop} = Dict.split(dict, [1.0]) - assert take == new_dict([]) - assert drop == dict - end - - test "split/2 with enum" do - dict = int_dict() - {take, drop} = Dict.split(dict, 1..3) - assert take == dict - assert drop == new_dict([]) - end - - test "take/2" do - dict = new_dict() - take = Dict.take(dict, ["unknown_key"]) - assert take == new_dict([]) - - take = Dict.take(dict, ["first_key"]) - assert take == new_dict([{"first_key", 1}]) - end - - test "take/2 with match" do - dict = int_dict() - assert Dict.take(dict, [1]) == dict - assert Dict.take(dict, [1.0]) == new_dict([]) - end - - test "take/2 with enum" do - dict = int_dict() - assert Dict.take(dict, 1..3) == dict - end - - test "drop/2" do - dict = new_dict() - drop = Dict.drop(dict, ["unknown_key"]) - assert drop == dict - - drop = Dict.drop(dict, ["first_key"]) - assert drop == new_dict([{"second_key", 2}]) - end - - test "drop/2 with match" do - dict = int_dict() - assert Dict.drop(dict, [1]) == new_dict([]) - assert Dict.drop(dict, [1.0]) == dict - end - - test "drop/2 with enum" do - dict = int_dict() - assert Dict.drop(dict, 1..3) == new_dict([]) - end - - test "equal?/2" do - dict1 = new_dict(a: 2, b: 3, f: 5, c: 123) - dict2 = new_dict(a: 2, b: 3, f: 5, c: 123) - assert dict_impl.equal?(dict1, dict2) - assert Dict.equal?(dict1, dict2) - - dict2 = Dict.put(dict2, :a, 3) - refute dict_impl.equal?(dict1, dict2) - refute Dict.equal?(dict1, dict2) - - dict3 = [a: 2, b: 3, f: 5, c: 123, z: 666] - refute Dict.equal?(dict1, dict3) - refute Dict.equal?(dict3, dict1) - end - - test "equal?/2 with match" do - dict1 = new_dict([{1,1}]) - dict2 = new_dict([{1.0,1}]) - assert Dict.equal?(dict1, dict1) - refute Dict.equal?(dict1, dict2) - end - - test "equal?/2 with other dict" do - dict = new_dict([{1,1}]) - assert Dict.equal?(dict, TestDict.new([{1,1}])) - refute Dict.equal?(dict, TestDict.new([{1.0,1}])) - end - - test "is enumerable" do - dict = new_dict() - assert Enum.empty?(new_dict([])) - refute Enum.empty?(dict) - assert Enum.member?(dict, {"first_key", 1}) - refute Enum.member?(dict, {"first_key", 2}) - assert Enum.count(dict) == 2 - assert Enum.reduce(dict, 0, fn({k, v}, acc) -> v + acc end) == 3 - end - - test "is collectable" do - dict = new_dict() - assert Dict.size(dict) == 2 - assert Enum.sort(dict) == [{"first_key", 1}, {"second_key", 2}] - - dict = new_dict([{1}, {2}, {3}], fn {x} -> {<>, x} end) - assert Dict.size(dict) == 3 - assert Enum.sort(dict) == [{"A", 1}, {"B", 2}, {"C", 3}] - - assert Collectable.empty(new_dict) == new_dict([]) - end - - test "is zippable" do - dict = new_dict() - list = Dict.to_list(dict) - assert Enum.zip(list, list) == Enum.zip(dict, dict) - - dict = new_dict(1..120, fn i -> {i, i} end) - list = Dict.to_list(dict) - assert Enum.zip(list, list) == Enum.zip(dict, dict) - end - end - end -end - -defmodule Dict.HashDictTest do - use ExUnit.Case, async: true - use DictTest.Common - - doctest Dict - defp dict_impl, do: HashDict -end - -defmodule Dict.MapDictTest do - use ExUnit.Case, async: true - use DictTest.Common - - doctest Dict - defp dict_impl, do: Map -end diff --git a/lib/elixir/test/elixir/dynamic_supervisor_test.exs b/lib/elixir/test/elixir/dynamic_supervisor_test.exs new file mode 100644 index 00000000000..ae55c014bcf --- /dev/null +++ b/lib/elixir/test/elixir/dynamic_supervisor_test.exs @@ -0,0 +1,736 @@ +Code.require_file("test_helper.exs", __DIR__) + +defmodule DynamicSupervisorTest do + use ExUnit.Case, async: true + + defmodule Simple do + use DynamicSupervisor + + def init(args), do: args + end + + test "can be supervised directly" do + children = [{DynamicSupervisor, name: :dyn_sup_spec_test}] + assert {:ok, _} = Supervisor.start_link(children, strategy: :one_for_one) + assert DynamicSupervisor.which_children(:dyn_sup_spec_test) == [] + end + + test "multiple supervisors can be supervised and identified with simple child spec" do + {:ok, _} = Registry.start_link(keys: :unique, name: DynSup.Registry) + + children = [ + {DynamicSupervisor, name: :simple_name}, + {DynamicSupervisor, name: {:global, :global_name}}, + {DynamicSupervisor, name: {:via, Registry, {DynSup.Registry, "via_name"}}} + ] + + assert {:ok, supsup} = Supervisor.start_link(children, strategy: :one_for_one) + + assert {:ok, no_name_dynsup} = + Supervisor.start_child(supsup, {DynamicSupervisor, strategy: :one_for_one}) + + assert DynamicSupervisor.which_children(:simple_name) == [] + assert DynamicSupervisor.which_children({:global, :global_name}) == [] + assert DynamicSupervisor.which_children({:via, Registry, {DynSup.Registry, "via_name"}}) == [] + assert DynamicSupervisor.which_children(no_name_dynsup) == [] + + assert Supervisor.start_child(supsup, {DynamicSupervisor, strategy: :one_for_one}) == + {:error, {:already_started, no_name_dynsup}} + end + + describe "use/2" do + test "generates child_spec/1" do + assert Simple.child_spec([:hello]) == %{ + id: Simple, + start: {Simple, :start_link, [[:hello]]}, + type: :supervisor + } + + defmodule Custom do + use DynamicSupervisor, + id: :id, + restart: :temporary, + shutdown: :infinity, + start: {:foo, :bar, []} + + def init(arg), do: {:producer, arg} + end + + assert Custom.child_spec([:hello]) == %{ + id: :id, + restart: :temporary, + shutdown: :infinity, + start: {:foo, :bar, []}, + type: :supervisor + } + end + end + + describe "init/1" do + test "set default options" do + assert DynamicSupervisor.init([]) == + {:ok, + %{ + strategy: :one_for_one, + intensity: 3, + period: 5, + max_children: :infinity, + extra_arguments: [] + }} + end + end + + describe "start_link/3" do + test "with non-ok init" do + Process.flag(:trap_exit, true) + + assert DynamicSupervisor.start_link(Simple, {:ok, %{strategy: :unknown}}) == + {:error, {:supervisor_data, {:invalid_strategy, :unknown}}} + + assert DynamicSupervisor.start_link(Simple, {:ok, %{intensity: -1}}) == + {:error, {:supervisor_data, {:invalid_intensity, -1}}} + + assert DynamicSupervisor.start_link(Simple, {:ok, %{period: 0}}) == + {:error, {:supervisor_data, {:invalid_period, 0}}} + + assert DynamicSupervisor.start_link(Simple, {:ok, %{max_children: -1}}) == + {:error, {:supervisor_data, {:invalid_max_children, -1}}} + + assert DynamicSupervisor.start_link(Simple, {:ok, %{extra_arguments: -1}}) == + {:error, {:supervisor_data, {:invalid_extra_arguments, -1}}} + + assert DynamicSupervisor.start_link(Simple, :unknown) == + {:error, {:bad_return, {Simple, :init, :unknown}}} + + assert DynamicSupervisor.start_link(Simple, :ignore) == :ignore + end + + test "with registered process" do + {:ok, pid} = DynamicSupervisor.start_link(Simple, {:ok, %{}}, name: __MODULE__) + + # Sets up a link + {:links, links} = Process.info(self(), :links) + assert pid in links + + # A name + assert Process.whereis(__MODULE__) == pid + + # And the initial call + assert {:supervisor, DynamicSupervisorTest.Simple, 1} = + :proc_lib.translate_initial_call(pid) + + # And shuts down + assert DynamicSupervisor.stop(__MODULE__) == :ok + end + + test "with spawn_opt" do + {:ok, pid} = + DynamicSupervisor.start_link(strategy: :one_for_one, spawn_opt: [priority: :high]) + + assert Process.info(pid, :priority) == {:priority, :high} + end + + test "sets initial call to the same as a regular supervisor" do + {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) + assert :proc_lib.initial_call(pid) == {:supervisor, Supervisor.Default, [:Argument__1]} + + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + assert :proc_lib.initial_call(pid) == {:supervisor, Supervisor.Default, [:Argument__1]} + end + + test "returns the callback module" do + {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) + assert :supervisor.get_callback_module(pid) == Supervisor.Default + + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + assert :supervisor.get_callback_module(pid) == Supervisor.Default + end + end + + ## Code change + + describe "code_change/3" do + test "with non-ok init" do + {:ok, pid} = DynamicSupervisor.start_link(Simple, {:ok, %{}}) + + assert fake_upgrade(pid, {:ok, %{strategy: :unknown}}) == + {:error, {:error, {:supervisor_data, {:invalid_strategy, :unknown}}}} + + assert fake_upgrade(pid, {:ok, %{intensity: -1}}) == + {:error, {:error, {:supervisor_data, {:invalid_intensity, -1}}}} + + assert fake_upgrade(pid, {:ok, %{period: 0}}) == + {:error, {:error, {:supervisor_data, {:invalid_period, 0}}}} + + assert fake_upgrade(pid, {:ok, %{max_children: -1}}) == + {:error, {:error, {:supervisor_data, {:invalid_max_children, -1}}}} + + assert fake_upgrade(pid, :unknown) == {:error, :unknown} + assert fake_upgrade(pid, :ignore) == :ok + end + + test "with ok init" do + {:ok, pid} = DynamicSupervisor.start_link(Simple, {:ok, %{}}) + {:ok, _} = DynamicSupervisor.start_child(pid, sleepy_worker()) + assert %{active: 1} = DynamicSupervisor.count_children(pid) + + assert fake_upgrade(pid, {:ok, %{max_children: 1}}) == :ok + assert %{active: 1} = DynamicSupervisor.count_children(pid) + assert DynamicSupervisor.start_child(pid, {Task, fn -> :ok end}) == {:error, :max_children} + end + + defp fake_upgrade(pid, init_arg) do + :ok = :sys.suspend(pid) + :sys.replace_state(pid, fn state -> %{state | args: init_arg} end) + res = :sys.change_code(pid, :gen_server, 123, :extra) + :ok = :sys.resume(pid) + res + end + end + + describe "start_child/2" do + test "supports old child spec" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + child = {Task, {Task, :start_link, [fn -> :ok end]}, :temporary, 5000, :worker, [Task]} + assert {:ok, pid} = DynamicSupervisor.start_child(pid, child) + assert is_pid(pid) + end + + test "supports new child spec as tuple" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + child = %{id: Task, restart: :temporary, start: {Task, :start_link, [fn -> :ok end]}} + assert {:ok, pid} = DynamicSupervisor.start_child(pid, child) + assert is_pid(pid) + end + + test "supports new child spec" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + child = {Task, fn -> Process.sleep(:infinity) end} + assert {:ok, pid} = DynamicSupervisor.start_child(pid, child) + assert is_pid(pid) + end + + test "supports extra arguments" do + parent = self() + fun = fn -> send(parent, :from_child) end + + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, extra_arguments: [fun]) + child = %{id: Task, restart: :temporary, start: {Task, :start_link, []}} + assert {:ok, pid} = DynamicSupervisor.start_child(pid, child) + assert is_pid(pid) + assert_receive :from_child + end + + test "with invalid child spec" do + assert DynamicSupervisor.start_child(:not_used, %{}) == {:error, {:invalid_child_spec, %{}}} + + assert DynamicSupervisor.start_child(:not_used, {1, 2, 3, 4, 5, 6}) == + {:error, {:invalid_mfa, 2}} + + assert DynamicSupervisor.start_child(:not_used, %{id: 1, start: {Task, :foo, :bar}}) == + {:error, {:invalid_mfa, {Task, :foo, :bar}}} + end + + test "with different returns" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + + assert {:ok, _, :extra} = DynamicSupervisor.start_child(pid, current_module_worker([:ok3])) + assert {:ok, _} = DynamicSupervisor.start_child(pid, current_module_worker([:ok2])) + assert :ignore = DynamicSupervisor.start_child(pid, current_module_worker([:ignore])) + + assert {:error, :found} = + DynamicSupervisor.start_child(pid, current_module_worker([:error])) + + assert {:error, :unknown} = + DynamicSupervisor.start_child(pid, current_module_worker([:unknown])) + end + + test "with throw/error/exit" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + + assert {:error, {{:nocatch, :oops}, [_ | _]}} = + DynamicSupervisor.start_child(pid, current_module_worker([:non_local, :throw])) + + assert {:error, {%RuntimeError{}, [_ | _]}} = + DynamicSupervisor.start_child(pid, current_module_worker([:non_local, :error])) + + assert {:error, :oops} = + DynamicSupervisor.start_child(pid, current_module_worker([:non_local, :exit])) + end + + test "with max_children" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, max_children: 0) + + assert {:error, :max_children} = + DynamicSupervisor.start_child(pid, current_module_worker([:ok2])) + end + + test "temporary child is not restarted regardless of reason" do + child = current_module_worker([:ok2], restart: :temporary) + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(pid) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :whatever) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(pid) + end + + test "transient child is restarted unless normal/shutdown/{shutdown, _}" do + child = current_module_worker([:ok2], restart: :transient) + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(pid) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, {:shutdown, :signal}) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(pid) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :whatever) + assert %{workers: 1, active: 1} = DynamicSupervisor.count_children(pid) + end + + test "permanent child is restarted regardless of reason" do + child = current_module_worker([:ok2], restart: :permanent) + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, max_restarts: 100_000) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert %{workers: 1, active: 1} = DynamicSupervisor.count_children(pid) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, {:shutdown, :signal}) + assert %{workers: 2, active: 2} = DynamicSupervisor.count_children(pid) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :whatever) + assert %{workers: 3, active: 3} = DynamicSupervisor.count_children(pid) + end + + test "child is restarted with different values" do + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, max_restarts: 100_000) + + assert {:ok, child1} = + DynamicSupervisor.start_child(pid, current_module_worker([:restart, :ok2])) + + assert [{:undefined, ^child1, :worker, [DynamicSupervisorTest]}] = + DynamicSupervisor.which_children(pid) + + assert_kill(child1, :shutdown) + assert %{workers: 1, active: 1} = DynamicSupervisor.count_children(pid) + + assert {:ok, child2} = + DynamicSupervisor.start_child(pid, current_module_worker([:restart, :ok3])) + + assert [ + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, ^child2, :worker, [DynamicSupervisorTest]} + ] = DynamicSupervisor.which_children(pid) + + assert_kill(child2, :shutdown) + assert %{workers: 2, active: 2} = DynamicSupervisor.count_children(pid) + + assert {:ok, child3} = + DynamicSupervisor.start_child(pid, current_module_worker([:restart, :ignore])) + + assert [ + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, _, :worker, [DynamicSupervisorTest]} + ] = DynamicSupervisor.which_children(pid) + + assert_kill(child3, :shutdown) + assert %{workers: 2, active: 2} = DynamicSupervisor.count_children(pid) + + assert {:ok, child4} = + DynamicSupervisor.start_child(pid, current_module_worker([:restart, :error])) + + assert [ + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, _, :worker, [DynamicSupervisorTest]} + ] = DynamicSupervisor.which_children(pid) + + assert_kill(child4, :shutdown) + assert %{workers: 3, active: 2} = DynamicSupervisor.count_children(pid) + + assert {:ok, child5} = + DynamicSupervisor.start_child(pid, current_module_worker([:restart, :unknown])) + + assert [ + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, _, :worker, [DynamicSupervisorTest]}, + {:undefined, :restarting, :worker, [DynamicSupervisorTest]}, + {:undefined, _, :worker, [DynamicSupervisorTest]} + ] = DynamicSupervisor.which_children(pid) + + assert_kill(child5, :shutdown) + assert %{workers: 4, active: 2} = DynamicSupervisor.count_children(pid) + end + + test "restarting on init children counted in max_children" do + child = current_module_worker([:restart, :error], restart: :permanent) + opts = [strategy: :one_for_one, max_children: 1, max_restarts: 100_000] + {:ok, pid} = DynamicSupervisor.start_link(opts) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert %{workers: 1, active: 0} = DynamicSupervisor.count_children(pid) + + child = current_module_worker([:restart, :ok2], restart: :permanent) + assert {:error, :max_children} = DynamicSupervisor.start_child(pid, child) + end + + test "restarting on exit children counted in max_children" do + child = current_module_worker([:ok2], restart: :permanent) + opts = [strategy: :one_for_one, max_children: 1, max_restarts: 100_000] + {:ok, pid} = DynamicSupervisor.start_link(opts) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert %{workers: 1, active: 1} = DynamicSupervisor.count_children(pid) + + child = current_module_worker([:ok2], restart: :permanent) + assert {:error, :max_children} = DynamicSupervisor.start_child(pid, child) + end + + test "restarting a child with extra_arguments successfully restarts child" do + parent = self() + + fun = fn -> + send(parent, :from_child) + Process.sleep(:infinity) + end + + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one, extra_arguments: [fun]) + child = %{id: Task, restart: :transient, start: {Task, :start_link, []}} + + assert {:ok, child} = DynamicSupervisor.start_child(sup, child) + assert is_pid(child) + assert_receive :from_child + assert %{active: 1, workers: 1} = DynamicSupervisor.count_children(sup) + assert_kill(child, :oops) + assert_receive :from_child + assert %{workers: 1, active: 1} = DynamicSupervisor.count_children(sup) + end + + test "child is restarted when trying again" do + child = current_module_worker([:try_again, self()], restart: :permanent) + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, max_restarts: 2) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_received {:try_again, true} + assert_kill(child_pid, :shutdown) + assert_receive {:try_again, false} + assert_receive {:try_again, true} + assert %{workers: 1, active: 1} = DynamicSupervisor.count_children(pid) + end + + test "child triggers maximum restarts" do + Process.flag(:trap_exit, true) + child = current_module_worker([:restart, :error], restart: :permanent) + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, max_restarts: 1) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert_receive {:EXIT, ^pid, :shutdown} + end + + test "child triggers maximum intensity when trying again" do + Process.flag(:trap_exit, true) + child = current_module_worker([:restart, :error], restart: :permanent) + {:ok, pid} = DynamicSupervisor.start_link(strategy: :one_for_one, max_restarts: 10) + + assert {:ok, child_pid} = DynamicSupervisor.start_child(pid, child) + assert_kill(child_pid, :shutdown) + assert_receive {:EXIT, ^pid, :shutdown} + end + + def start_link(:ok3), do: {:ok, spawn_link(fn -> Process.sleep(:infinity) end), :extra} + def start_link(:ok2), do: {:ok, spawn_link(fn -> Process.sleep(:infinity) end)} + def start_link(:error), do: {:error, :found} + def start_link(:ignore), do: :ignore + def start_link(:unknown), do: :unknown + + def start_link(:non_local, :throw), do: throw(:oops) + def start_link(:non_local, :error), do: raise("oops") + def start_link(:non_local, :exit), do: exit(:oops) + + def start_link(:try_again, notify) do + if Process.get(:try_again) do + Process.put(:try_again, false) + send(notify, {:try_again, false}) + {:error, :try_again} + else + Process.put(:try_again, true) + send(notify, {:try_again, true}) + start_link(:ok2) + end + end + + def start_link(:restart, value) do + if Process.get({:restart, value}) do + start_link(value) + else + Process.put({:restart, value}, true) + start_link(:ok2) + end + end + end + + describe "terminate/2" do + test "terminates children with brutal kill" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + + child = sleepy_worker(shutdown: :brutal_kill) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child) + + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + assert_receive {:DOWN, _, :process, ^child1, :killed} + assert_receive {:DOWN, _, :process, ^child2, :killed} + assert_receive {:DOWN, _, :process, ^child3, :killed} + end + + test "terminates children with infinity shutdown" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + + child = sleepy_worker(shutdown: :infinity) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child) + + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + assert_receive {:DOWN, _, :process, ^child1, :shutdown} + assert_receive {:DOWN, _, :process, ^child2, :shutdown} + assert_receive {:DOWN, _, :process, ^child3, :shutdown} + end + + test "terminates children with infinity shutdown and abnormal reason" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + parent = self() + + fun = fn -> + Process.flag(:trap_exit, true) + send(parent, :ready) + receive(do: (_ -> exit({:shutdown, :oops}))) + end + + child = Supervisor.child_spec({Task, fun}, shutdown: :infinity) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child) + + assert_receive :ready + assert_receive :ready + assert_receive :ready + + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + + assert_receive {:DOWN, _, :process, ^child1, {:shutdown, :oops}} + assert_receive {:DOWN, _, :process, ^child2, {:shutdown, :oops}} + assert_receive {:DOWN, _, :process, ^child3, {:shutdown, :oops}} + end + + test "terminates children with integer shutdown" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + + child = sleepy_worker(shutdown: 1000) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child) + + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + assert_receive {:DOWN, _, :process, ^child1, :shutdown} + assert_receive {:DOWN, _, :process, ^child2, :shutdown} + assert_receive {:DOWN, _, :process, ^child3, :shutdown} + end + + test "terminates children with integer shutdown and abnormal reason" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + parent = self() + + fun = fn -> + Process.flag(:trap_exit, true) + send(parent, :ready) + receive(do: (_ -> exit({:shutdown, :oops}))) + end + + child = Supervisor.child_spec({Task, fun}, shutdown: 1000) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child) + + assert_receive :ready + assert_receive :ready + assert_receive :ready + + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + + assert_receive {:DOWN, _, :process, ^child1, {:shutdown, :oops}} + assert_receive {:DOWN, _, :process, ^child2, {:shutdown, :oops}} + assert_receive {:DOWN, _, :process, ^child3, {:shutdown, :oops}} + end + + test "terminates children with expired integer shutdown" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + parent = self() + + fun = fn -> + Process.sleep(:infinity) + end + + tmt = fn -> + Process.flag(:trap_exit, true) + send(parent, :ready) + Process.sleep(:infinity) + end + + child_fun = Supervisor.child_spec({Task, fun}, shutdown: 1) + child_tmt = Supervisor.child_spec({Task, tmt}, shutdown: 1) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child_fun) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child_tmt) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child_fun) + + assert_receive :ready + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + + assert_receive {:DOWN, _, :process, ^child1, :shutdown} + assert_receive {:DOWN, _, :process, ^child2, :killed} + assert_receive {:DOWN, _, :process, ^child3, :shutdown} + end + + test "terminates children with permanent restart and normal reason" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + parent = self() + + fun = fn -> + Process.flag(:trap_exit, true) + send(parent, :ready) + receive(do: (_ -> exit(:normal))) + end + + child = Supervisor.child_spec({Task, fun}, shutdown: :infinity, restart: :permanent) + assert {:ok, child1} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child2} = DynamicSupervisor.start_child(sup, child) + assert {:ok, child3} = DynamicSupervisor.start_child(sup, child) + + assert_receive :ready + assert_receive :ready + assert_receive :ready + + Process.monitor(child1) + Process.monitor(child2) + Process.monitor(child3) + assert_kill(sup, :shutdown) + assert_receive {:DOWN, _, :process, ^child1, :normal} + assert_receive {:DOWN, _, :process, ^child2, :normal} + assert_receive {:DOWN, _, :process, ^child3, :normal} + end + + test "terminates with mixed children" do + Process.flag(:trap_exit, true) + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + + assert {:ok, child1} = + DynamicSupervisor.start_child(sup, sleepy_worker(shutdown: :infinity)) + + assert {:ok, child2} = + DynamicSupervisor.start_child(sup, sleepy_worker(shutdown: :brutal_kill)) + + Process.monitor(child1) + Process.monitor(child2) + assert_kill(sup, :shutdown) + assert_receive {:DOWN, _, :process, ^child1, :shutdown} + assert_receive {:DOWN, _, :process, ^child2, :killed} + end + end + + describe "terminate_child/2" do + test "terminates child with brutal kill" do + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + + child = sleepy_worker(shutdown: :brutal_kill) + assert {:ok, child_pid} = DynamicSupervisor.start_child(sup, child) + + Process.monitor(child_pid) + assert :ok = DynamicSupervisor.terminate_child(sup, child_pid) + assert_receive {:DOWN, _, :process, ^child_pid, :killed} + + assert {:error, :not_found} = DynamicSupervisor.terminate_child(sup, child_pid) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(sup) + end + + test "terminates child with integer shutdown" do + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one) + + child = sleepy_worker(shutdown: 1000) + assert {:ok, child_pid} = DynamicSupervisor.start_child(sup, child) + + Process.monitor(child_pid) + assert :ok = DynamicSupervisor.terminate_child(sup, child_pid) + assert_receive {:DOWN, _, :process, ^child_pid, :shutdown} + + assert {:error, :not_found} = DynamicSupervisor.terminate_child(sup, child_pid) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(sup) + end + + test "terminates restarting child" do + {:ok, sup} = DynamicSupervisor.start_link(strategy: :one_for_one, max_restarts: 100_000) + + child = current_module_worker([:restart, :error], restart: :permanent) + assert {:ok, child_pid} = DynamicSupervisor.start_child(sup, child) + assert_kill(child_pid, :shutdown) + assert :ok = DynamicSupervisor.terminate_child(sup, child_pid) + + assert {:error, :not_found} = DynamicSupervisor.terminate_child(sup, child_pid) + assert %{workers: 0, active: 0} = DynamicSupervisor.count_children(sup) + end + end + + defp sleepy_worker(opts \\ []) do + mfa = {Task, :start_link, [Process, :sleep, [:infinity]]} + Supervisor.child_spec(%{id: Task, start: mfa}, opts) + end + + defp current_module_worker(args, opts \\ []) do + Supervisor.child_spec(%{id: __MODULE__, start: {__MODULE__, :start_link, args}}, opts) + end + + defp assert_kill(pid, reason) do + ref = Process.monitor(pid) + Process.exit(pid, reason) + assert_receive {:DOWN, ^ref, _, _, _} + end +end diff --git a/lib/elixir/test/elixir/enum_test.exs b/lib/elixir/test/elixir/enum_test.exs index 779d73605fa..1e74ce0d56e 100644 --- a/lib/elixir/test/elixir/enum_test.exs +++ b/lib/elixir/test/elixir/enum_test.exs @@ -1,53 +1,90 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) -defmodule EnumTest.List do +defmodule EnumTest do use ExUnit.Case, async: true + doctest Enum - test :empty? do - assert Enum.empty?([]) - refute Enum.empty?([1, 2, 3]) - refute Enum.empty?(1..3) - end + defp assert_runs_enumeration_only_once(enum_fun) do + enumerator = + Stream.map([:element], fn element -> + send(self(), element) + element + end) - test :member? do - assert Enum.member?([1, 2, 3], 2) - refute Enum.member?([], 0) - refute Enum.member?([1, 2, 3], 0) - assert Enum.member?(1..3, 2) - refute Enum.member?(1..3, 0) + enum_fun.(enumerator) + assert_received :element + refute_received :element end - test :count do - assert Enum.count([1, 2, 3]) == 3 - assert Enum.count([]) == 0 - end + describe "zip_reduce/4" do + test "two non lists" do + left = %{a: 1} + right = %{b: 2} + reducer = fn {_, x}, {_, y}, acc -> [x + y | acc] end + assert Enum.zip_reduce(left, right, [], reducer) == [3] + + # Empty Left + assert Enum.zip_reduce(%{}, right, [], reducer) == [] + + # Empty Right + assert Enum.zip_reduce(left, %{}, [], reducer) == [] + end - test :count_fun do - assert Enum.count([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) == 1 - assert Enum.count([], fn(x) -> rem(x, 2) == 0 end) == 0 + test "lists" do + assert Enum.zip_reduce([1, 2], [3, 4], 0, fn x, y, acc -> x + y + acc end) == 10 + assert Enum.zip_reduce([1, 2], [3, 4], [], fn x, y, acc -> [x + y | acc] end) == [6, 4] + end + + test "when left empty" do + assert Enum.zip_reduce([], [1, 2], 0, fn x, y, acc -> x + y + acc end) == 0 + end + + test "when right empty" do + assert Enum.zip_reduce([1, 2], [], 0, fn x, y, acc -> x + y + acc end) == 0 + end end - test :all? do - assert Enum.all?([2, 4, 6], fn(x) -> rem(x, 2) == 0 end) - refute Enum.all?([2, 3, 4], fn(x) -> rem(x, 2) == 0 end) + describe "zip_reduce/3" do + test "when enums empty" do + assert Enum.zip_reduce([], 0, fn _, acc -> acc end) == 0 + end + + test "lists work" do + enums = [[1, 1], [2, 2], [3, 3]] + result = Enum.zip_reduce(enums, [], fn elements, acc -> [List.to_tuple(elements) | acc] end) + assert result == [{1, 2, 3}, {1, 2, 3}] + end + + test "mix and match" do + enums = [[1, 2], %{a: 3, b: 4}, [5, 6]] + result = Enum.zip_reduce(enums, [], fn elements, acc -> [List.to_tuple(elements) | acc] end) + assert result == [{2, {:b, 4}, 6}, {1, {:a, 3}, 5}] + end + end + test "all?/2" do assert Enum.all?([2, 4, 6]) refute Enum.all?([2, nil, 4]) - assert Enum.all?([]) + + assert Enum.all?([2, 4, 6], fn x -> rem(x, 2) == 0 end) + refute Enum.all?([2, 3, 4], fn x -> rem(x, 2) == 0 end) end - test :any? do - refute Enum.any?([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) - assert Enum.any?([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) + test "any?/2" do + refute Enum.any?([2, 4, 6], fn x -> rem(x, 2) == 1 end) + assert Enum.any?([2, 3, 4], fn x -> rem(x, 2) == 1 end) refute Enum.any?([false, false, false]) assert Enum.any?([false, true, false]) + assert Enum.any?([:foo, false, false]) + refute Enum.any?([false, nil, false]) + refute Enum.any?([]) end - test :at do + test "at/3" do assert Enum.at([2, 4, 6], 0) == 2 assert Enum.at([2, 4, 6], 2) == 6 assert Enum.at([2, 4, 6], 4) == nil @@ -56,42 +93,163 @@ defmodule EnumTest.List do assert Enum.at([2, 4, 6], -4) == nil end - test :concat_1 do + test "chunk/3" do + enum = Enum + assert enum.chunk(1..5, 2, 1) == Enum.chunk_every(1..5, 2, 1, :discard) + end + + test "chunk/4" do + enum = Enum + assert enum.chunk(1..5, 2, 1, nil) == Enum.chunk_every(1..5, 2, 1, :discard) + end + + test "chunk_every/2" do + assert Enum.chunk_every([1, 2, 3, 4, 5], 2) == [[1, 2], [3, 4], [5]] + end + + test "chunk_every/4" do + assert Enum.chunk_every([1, 2, 3, 4, 5], 2, 2, [6]) == [[1, 2], [3, 4], [5, 6]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, :discard) == [[1, 2, 3], [3, 4, 5]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6], 2, 3, :discard) == [[1, 2], [4, 5]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, []) == [[1, 2, 3], [3, 4, 5], [5, 6]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6], 3, 3, []) == [[1, 2, 3], [4, 5, 6]] + assert Enum.chunk_every([1, 2, 3, 4, 5], 4, 4, 6..10) == [[1, 2, 3, 4], [5, 6, 7, 8]] + assert Enum.chunk_every([1, 2, 3, 4, 5], 2, 3, []) == [[1, 2], [4, 5]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6], 2, 3, []) == [[1, 2], [4, 5]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6, 7], 2, 3, []) == [[1, 2], [4, 5], [7]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6, 7], 2, 3, [8]) == [[1, 2], [4, 5], [7, 8]] + assert Enum.chunk_every([1, 2, 3, 4, 5, 6, 7], 2, 4, []) == [[1, 2], [5, 6]] + end + + test "chunk_by/2" do + assert Enum.chunk_by([1, 2, 2, 3, 4, 4, 6, 7, 7], &(rem(&1, 2) == 1)) == + [[1], [2, 2], [3], [4, 4, 6], [7, 7]] + + assert Enum.chunk_by([1, 2, 3, 4], fn _ -> true end) == [[1, 2, 3, 4]] + assert Enum.chunk_by([], fn _ -> true end) == [] + assert Enum.chunk_by([1], fn _ -> true end) == [[1]] + end + + test "chunk_while/4" do + chunk_fun = fn i, acc -> + cond do + i > 10 -> + {:halt, acc} + + rem(i, 2) == 0 -> + {:cont, Enum.reverse([i | acc]), []} + + true -> + {:cont, [i | acc]} + end + end + + after_fun = fn + [] -> {:cont, []} + acc -> {:cont, Enum.reverse(acc), []} + end + + assert Enum.chunk_while([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [], chunk_fun, after_fun) == + [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + assert Enum.chunk_while(0..9, [], chunk_fun, after_fun) == + [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9]] + + assert Enum.chunk_while(0..10, [], chunk_fun, after_fun) == + [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + assert Enum.chunk_while(0..11, [], chunk_fun, after_fun) == + [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + assert Enum.chunk_while([5, 7, 9, 11], [], chunk_fun, after_fun) == [[5, 7, 9]] + + assert Enum.chunk_while([1, 2, 3, 5, 7], [], chunk_fun, after_fun) == [[1, 2], [3, 5, 7]] + + chunk_fn2 = fn + -1, acc -> {:cont, acc, 0} + i, acc -> {:cont, acc + i} + end + + after_fn2 = fn acc -> {:cont, acc, 0} end + + assert Enum.chunk_while([1, -1, 2, 3, -1, 4, 5, 6], 0, chunk_fn2, after_fn2) == [1, 5, 15] + end + + test "concat/1" do assert Enum.concat([[1, [2], 3], [4], [5, 6]]) == [1, [2], 3, 4, 5, 6] - assert Enum.concat(1..3, []) == [1,2,3] assert Enum.concat([[], []]) == [] - assert Enum.concat([[]]) == [] - assert Enum.concat([]) == [] - - assert Enum.concat([1..5, fn acc, _ -> acc end, [1]]) == [1,2,3,4,5,1] + assert Enum.concat([[]]) == [] + assert Enum.concat([]) == [] end - test :concat_2 do + test "concat/2" do assert Enum.concat([], [1]) == [1] assert Enum.concat([1, [2], 3], [4, 5]) == [1, [2], 3, 4, 5] - assert Enum.concat(1..3, []) == [1,2,3] + + assert Enum.concat([1, 2], 3..5) == [1, 2, 3, 4, 5] assert Enum.concat([], []) == [] + assert Enum.concat([], 1..3) == [1, 2, 3] assert Enum.concat(fn acc, _ -> acc end, [1]) == [1] end - test :fetch! do - assert Enum.fetch!([2, 4, 6], 0) == 2 - assert Enum.fetch!([2, 4, 6], 2) == 6 - assert Enum.fetch!([2, 4, 6], -2) == 4 + test "count/1" do + assert Enum.count([1, 2, 3]) == 3 + assert Enum.count([]) == 0 + assert Enum.count([1, true, false, nil]) == 4 + end - assert_raise Enum.OutOfBoundsError, fn -> - Enum.fetch!([2, 4, 6], 4) - end + test "count/2" do + assert Enum.count([1, 2, 3], fn x -> rem(x, 2) == 0 end) == 1 + assert Enum.count([], fn x -> rem(x, 2) == 0 end) == 0 + assert Enum.count([1, true, false, nil], & &1) == 2 + end - assert_raise Enum.OutOfBoundsError, fn -> - Enum.fetch!([2, 4, 6], -4) - end + test "count_until/2" do + assert Enum.count_until([1, 2, 3], 2) == 2 + assert Enum.count_until([], 2) == 0 + assert Enum.count_until([1, 2], 2) == 2 + end + + test "count_until/3" do + assert Enum.count_until([1, 2, 3, 4, 5, 6], fn x -> rem(x, 2) == 0 end, 2) == 2 + assert Enum.count_until([1, 2], fn x -> rem(x, 2) == 0 end, 2) == 1 + assert Enum.count_until([1, 2, 3, 4], fn x -> rem(x, 2) == 0 end, 2) == 2 + assert Enum.count_until([], fn x -> rem(x, 2) == 0 end, 2) == 0 + end + + test "dedup/1" do + assert Enum.dedup([1, 1, 2, 1, 1, 2, 1]) == [1, 2, 1, 2, 1] + assert Enum.dedup([2, 1, 1, 2, 1]) == [2, 1, 2, 1] + assert Enum.dedup([1, 2, 3, 4]) == [1, 2, 3, 4] + assert Enum.dedup([1, 1.0, 2.0, 2]) == [1, 1.0, 2.0, 2] + assert Enum.dedup([]) == [] + assert Enum.dedup([nil, nil, true, {:value, true}]) == [nil, true, {:value, true}] + assert Enum.dedup([nil]) == [nil] + end + + test "dedup/1 with streams" do + dedup_stream = fn list -> list |> Stream.map(& &1) |> Enum.dedup() end + + assert dedup_stream.([1, 1, 2, 1, 1, 2, 1]) == [1, 2, 1, 2, 1] + assert dedup_stream.([2, 1, 1, 2, 1]) == [2, 1, 2, 1] + assert dedup_stream.([1, 2, 3, 4]) == [1, 2, 3, 4] + assert dedup_stream.([1, 1.0, 2.0, 2]) == [1, 1.0, 2.0, 2] + assert dedup_stream.([]) == [] + assert dedup_stream.([nil, nil, true, {:value, true}]) == [nil, true, {:value, true}] + assert dedup_stream.([nil]) == [nil] + end + + test "dedup_by/2" do + assert Enum.dedup_by([{1, :x}, {2, :y}, {2, :z}, {1, :x}], fn {x, _} -> x end) == + [{1, :x}, {2, :y}, {1, :x}] + + assert Enum.dedup_by([5, 1, 2, 3, 2, 1], fn x -> x > 2 end) == [5, 1, 3, 2] end - test :drop do + test "drop/2" do assert Enum.drop([1, 2, 3], 0) == [1, 2, 3] assert Enum.drop([1, 2, 3], 1) == [2, 3] assert Enum.drop([1, 2, 3], 2) == [3] @@ -101,186 +259,979 @@ defmodule EnumTest.List do assert Enum.drop([1, 2, 3], -2) == [1] assert Enum.drop([1, 2, 3], -4) == [] assert Enum.drop([], 3) == [] - end - test :drop_while do - assert Enum.drop_while([1, 2, 3, 4, 3, 2, 1], fn(x) -> x <= 3 end) == [4, 3, 2, 1] - assert Enum.drop_while([1, 2, 3], fn(_) -> false end) == [1, 2, 3] - assert Enum.drop_while([1, 2, 3], fn(x) -> x <= 3 end) == [] - assert Enum.drop_while([], fn(_) -> false end) == [] + assert_raise FunctionClauseError, fn -> + Enum.drop([1, 2, 3], 0.0) + end end - test :find do - assert Enum.find([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) == nil - assert Enum.find([2, 4, 6], 0, fn(x) -> rem(x, 2) == 1 end) == 0 - assert Enum.find([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) == 3 + test "drop_every/2" do + assert Enum.drop_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 2) == [2, 4, 6, 8, 10] + assert Enum.drop_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 3) == [2, 3, 5, 6, 8, 9] + assert Enum.drop_every([], 2) == [] + assert Enum.drop_every([1, 2], 2) == [2] + assert Enum.drop_every([1, 2, 3], 0) == [1, 2, 3] + + assert_raise FunctionClauseError, fn -> + Enum.drop_every([1, 2, 3], -1) + end end - test :find_value do - assert Enum.find_value([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) == nil - assert Enum.find_value([2, 4, 6], 0, fn(x) -> rem(x, 2) == 1 end) == 0 - assert Enum.find_value([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) + test "drop_while/2" do + assert Enum.drop_while([1, 2, 3, 4, 3, 2, 1], fn x -> x <= 3 end) == [4, 3, 2, 1] + assert Enum.drop_while([1, 2, 3], fn _ -> false end) == [1, 2, 3] + assert Enum.drop_while([1, 2, 3], fn x -> x <= 3 end) == [] + assert Enum.drop_while([], fn _ -> false end) == [] end - test :find_index do - assert Enum.find_index([2, 4, 6], fn(x) -> rem(x, 2) == 1 end) == nil - assert Enum.find_index([2, 3, 4], fn(x) -> rem(x, 2) == 1 end) == 1 + test "each/2" do + try do + assert Enum.each([], fn x -> x end) == :ok + assert Enum.each([1, 2, 3], fn x -> Process.put(:enum_test_each, x * 2) end) == :ok + assert Process.get(:enum_test_each) == 6 + after + Process.delete(:enum_test_each) + end end - test :each do - assert Enum.each([], fn(x) -> x end) == :ok - assert Enum.each([1, 2, 3], fn(x) -> Process.put(:enum_test_each, x * 2) end) == :ok - assert Process.get(:enum_test_each) == 6 - after - Process.delete(:enum_test_each) + test "empty?/1" do + assert Enum.empty?([]) + assert Enum.empty?(%{}) + refute Enum.empty?([1, 2, 3]) + refute Enum.empty?(%{one: 1}) + refute Enum.empty?(1..3) end - test :fetch do + test "fetch/2" do + assert Enum.fetch([66], 0) == {:ok, 66} + assert Enum.fetch([66], -1) == {:ok, 66} + assert Enum.fetch([66], 1) == :error + assert Enum.fetch([66], -2) == :error + assert Enum.fetch([2, 4, 6], 0) == {:ok, 2} + assert Enum.fetch([2, 4, 6], -1) == {:ok, 6} assert Enum.fetch([2, 4, 6], 2) == {:ok, 6} assert Enum.fetch([2, 4, 6], 4) == :error assert Enum.fetch([2, 4, 6], -2) == {:ok, 4} assert Enum.fetch([2, 4, 6], -4) == :error + + assert Enum.fetch([], 0) == :error + assert Enum.fetch([], 1) == :error end - test :filter do - assert Enum.filter([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) == [2] - assert Enum.filter([2, 4, 6], fn(x) -> rem(x, 2) == 0 end) == [2, 4, 6] + test "fetch!/2" do + assert Enum.fetch!([2, 4, 6], 0) == 2 + assert Enum.fetch!([2, 4, 6], 2) == 6 + assert Enum.fetch!([2, 4, 6], -2) == 4 + + assert_raise Enum.OutOfBoundsError, fn -> + Enum.fetch!([2, 4, 6], 4) + end + + assert_raise Enum.OutOfBoundsError, fn -> + Enum.fetch!([2, 4, 6], -4) + end end - test :filter_with_match do + test "filter/2" do + assert Enum.filter([1, 2, 3], fn x -> rem(x, 2) == 0 end) == [2] + assert Enum.filter([2, 4, 6], fn x -> rem(x, 2) == 0 end) == [2, 4, 6] + + assert Enum.filter([1, 2, false, 3, nil], & &1) == [1, 2, 3] assert Enum.filter([1, 2, 3], &match?(1, &1)) == [1] assert Enum.filter([1, 2, 3], &match?(x when x < 3, &1)) == [1, 2] - assert Enum.filter([1, 2, 3], &match?(_, &1)) == [1, 2, 3] + assert Enum.filter([1, 2, 3], fn _ -> true end) == [1, 2, 3] end - test :filter_map do - assert Enum.filter_map([1, 2, 3], fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) == [4] - assert Enum.filter_map([2, 4, 6], fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) == [4, 8, 12] + test "find/3" do + assert Enum.find([2, 4, 6], fn x -> rem(x, 2) == 1 end) == nil + assert Enum.find([2, 4, 6], 0, fn x -> rem(x, 2) == 1 end) == 0 + assert Enum.find([2, 3, 4], fn x -> rem(x, 2) == 1 end) == 3 end - test :flat_map do - assert Enum.flat_map([], fn(x) -> [x, x] end) == [] - assert Enum.flat_map([1, 2, 3], fn(x) -> [x, x] end) == [1, 1, 2, 2, 3, 3] - assert Enum.flat_map([1, 2, 3], fn(x) -> x..x+1 end) == [1, 2, 2, 3, 3, 4] + test "find_index/2" do + assert Enum.find_index([2, 4, 6], fn x -> rem(x, 2) == 1 end) == nil + assert Enum.find_index([2, 3, 4], fn x -> rem(x, 2) == 1 end) == 1 + assert Stream.take(1..3, 3) |> Enum.find_index(fn _ -> false end) == nil + assert Stream.take(1..6, 6) |> Enum.find_index(fn x -> x == 5 end) == 4 end - test :flat_map_reduce do - assert Enum.flat_map_reduce([1, 2, 3], 0, &{[&1, &2], &1 + &2}) == - {[1, 0, 2, 1, 3, 3], 6} + test "find_value/2" do + assert Enum.find_value([2, 4, 6], fn x -> rem(x, 2) == 1 end) == nil + assert Enum.find_value([2, 4, 6], 0, fn x -> rem(x, 2) == 1 end) == 0 + assert Enum.find_value([2, 3, 4], fn x -> rem(x, 2) == 1 end) + end - assert Enum.flat_map_reduce(1..100, 0, fn i, acc -> - if acc < 3, do: {[i], acc + 1}, else: {:halt, acc} - end) == {[1,2,3], 3} + test "flat_map/2" do + assert Enum.flat_map([], fn x -> [x, x] end) == [] + assert Enum.flat_map([1, 2, 3], fn x -> [x, x] end) == [1, 1, 2, 2, 3, 3] + assert Enum.flat_map([1, 2, 3], fn x -> x..(x + 1) end) == [1, 2, 2, 3, 3, 4] + end + + test "flat_map_reduce/3" do + assert Enum.flat_map_reduce([1, 2, 3], 0, &{[&1, &2], &1 + &2}) == {[1, 0, 2, 1, 3, 3], 6} + end + + test "frequencies/1" do + assert Enum.frequencies([]) == %{} + assert Enum.frequencies(~w{a c a a c b}) == %{"a" => 3, "b" => 1, "c" => 2} end - test :group_by do - assert Enum.group_by([], fn -> nil end) == %{} - assert Enum.group_by(1..6, &rem(&1, 3)) == - %{0 => [6, 3], 1 => [4, 1], 2 => [5, 2]} + test "frequencies_by/2" do + assert Enum.frequencies_by([], fn _ -> raise "oops" end) == %{} + assert Enum.frequencies_by([12, 7, 6, 5, 1], &Integer.mod(&1, 2)) == %{0 => 2, 1 => 3} + end - result = Enum.group_by(1..6, %{3 => :default}, &rem(&1, 3)) - assert result[0] == [6, 3] - assert result[3] == :default + test "group_by/3" do + assert Enum.group_by([], fn _ -> raise "oops" end) == %{} + assert Enum.group_by([1, 2, 3], &rem(&1, 2)) == %{0 => [2], 1 => [1, 3]} end - test :into do + test "intersperse/2" do + assert Enum.intersperse([], true) == [] + assert Enum.intersperse([1], true) == [1] + assert Enum.intersperse([1, 2, 3], true) == [1, true, 2, true, 3] + end + + test "into/2" do assert Enum.into([a: 1, b: 2], %{}) == %{a: 1, b: 2} assert Enum.into([a: 1, b: 2], %{c: 3}) == %{a: 1, b: 2, c: 3} + assert Enum.into(MapSet.new(a: 1, b: 2), %{}) == %{a: 1, b: 2} + assert Enum.into(MapSet.new(a: 1, b: 2), %{c: 3}) == %{a: 1, b: 2, c: 3} assert Enum.into(%{a: 1, b: 2}, []) == [a: 1, b: 2] - assert Enum.into([1, 2, 3], "numbers: ", &to_string/1) == "numbers: 123" - assert Enum.into([1, 2, 3], fn - func, {:cont, x} when is_function(func) -> [x] - list, {:cont, x} -> [x|list] - list, _ -> list - end) == [3, 2, 1] + assert Enum.into(1..3, []) == [1, 2, 3] + assert Enum.into(["H", "i"], "") == "Hi" end - test :intersperse do - assert Enum.intersperse([], true) == [] - assert Enum.intersperse([1], true) == [1] - assert Enum.intersperse([1,2,3], true) == [1, true, 2, true, 3] + test "into/3" do + assert Enum.into([1, 2, 3], [], fn x -> x * 2 end) == [2, 4, 6] + assert Enum.into([1, 2, 3], "numbers: ", &to_string/1) == "numbers: 123" + + assert_raise MatchError, fn -> + Enum.into([2, 3], %{a: 1}, & &1) + end end - test :join do + test "join/2" do assert Enum.join([], " = ") == "" assert Enum.join([1, 2, 3], " = ") == "1 = 2 = 3" assert Enum.join([1, "2", 3], " = ") == "1 = 2 = 3" assert Enum.join([1, 2, 3]) == "123" assert Enum.join(["", "", 1, 2, "", 3, "", "\n"], ";") == ";;1;2;;3;;\n" assert Enum.join([""]) == "" + + assert Enum.join(fn acc, _ -> acc end, ".") == "" + end + + test "map/2" do + assert Enum.map([], fn x -> x * 2 end) == [] + assert Enum.map([1, 2, 3], fn x -> x * 2 end) == [2, 4, 6] + end + + test "map_every/3" do + assert Enum.map_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 2, fn x -> x * 2 end) == + [2, 2, 6, 4, 10, 6, 14, 8, 18, 10] + + assert Enum.map_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 3, fn x -> x * 2 end) == + [2, 2, 3, 8, 5, 6, 14, 8, 9, 20] + + assert Enum.map_every([], 2, fn x -> x * 2 end) == [] + assert Enum.map_every([1, 2], 2, fn x -> x * 2 end) == [2, 2] + + assert Enum.map_every([1, 2, 3], 0, fn _x -> raise "should not be invoked" end) == + [1, 2, 3] + + assert Enum.map_every(1..3, 1, fn x -> x * 2 end) == [2, 4, 6] + + assert_raise FunctionClauseError, fn -> + Enum.map_every([1, 2, 3], -1, fn x -> x * 2 end) + end + + assert_raise FunctionClauseError, fn -> + Enum.map_every(1..10, 3.33, fn x -> x * 2 end) + end + + assert Enum.map_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 9, fn x -> x + 1000 end) == + [1001, 2, 3, 4, 5, 6, 7, 8, 9, 1010] + + assert Enum.map_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 10, fn x -> x + 1000 end) == + [1001, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + assert Enum.map_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 100, fn x -> x + 1000 end) == + [1001, 2, 3, 4, 5, 6, 7, 8, 9, 10] end - test :map_join do + test "map_intersperse/3" do + assert Enum.map_intersperse([], :a, &(&1 * 2)) == [] + assert Enum.map_intersperse([1], :a, &(&1 * 2)) == [2] + assert Enum.map_intersperse([1, 2, 3], :a, &(&1 * 2)) == [2, :a, 4, :a, 6] + end + + test "map_join/3" do assert Enum.map_join([], " = ", &(&1 * 2)) == "" assert Enum.map_join([1, 2, 3], " = ", &(&1 * 2)) == "2 = 4 = 6" assert Enum.map_join([1, 2, 3], &(&1 * 2)) == "246" - assert Enum.map_join(["", "", 1, 2, "", 3, "", "\n"], ";", &(&1)) == ";;1;2;;3;;\n" - assert Enum.map_join([""], "", &(&1)) == "" + assert Enum.map_join(["", "", 1, 2, "", 3, "", "\n"], ";", & &1) == ";;1;2;;3;;\n" + assert Enum.map_join([""], "", & &1) == "" + assert Enum.map_join(fn acc, _ -> acc end, ".", &(&1 + 0)) == "" end - test :join_empty do - fun = fn (acc, _) -> acc end - assert Enum.join(fun, ".") == "" - assert Enum.map_join(fun, ".", &(&1 + 0)) == "" + test "map_reduce/3" do + assert Enum.map_reduce([], 1, fn x, acc -> {x * 2, x + acc} end) == {[], 1} + assert Enum.map_reduce([1, 2, 3], 1, fn x, acc -> {x * 2, x + acc} end) == {[2, 4, 6], 7} end - test :map do - assert Enum.map([], fn x -> x * 2 end) == [] - assert Enum.map([1, 2, 3], fn x -> x * 2 end) == [2, 4, 6] + test "max/1" do + assert Enum.max([1]) == 1 + assert Enum.max([1, 2, 3]) == 3 + assert Enum.max([1, [], :a, {}]) == [] + + assert Enum.max([1, 1.0]) === 1 + assert Enum.max([1.0, 1]) === 1.0 + + assert_raise Enum.EmptyError, fn -> + Enum.max([]) + end end - test :map_reduce do - assert Enum.map_reduce([], 1, fn(x, acc) -> {x * 2, x + acc} end) == {[], 1} - assert Enum.map_reduce([1, 2, 3], 1, fn(x, acc) -> {x * 2, x + acc} end) == {[2, 4, 6], 7} + test "max/2 with stable sorting" do + assert Enum.max([1, 1.0], &>=/2) === 1 + assert Enum.max([1.0, 1], &>=/2) === 1.0 + assert Enum.max([1, 1.0], &>/2) === 1.0 + assert Enum.max([1.0, 1], &>/2) === 1 end - test :partition do - assert Enum.partition([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) == {[2], [1, 3]} - assert Enum.partition([2, 4, 6], fn(x) -> rem(x, 2) == 0 end) == {[2, 4, 6], []} + test "max/2 with module" do + assert Enum.max([~D[2019-01-01], ~D[2020-01-01]], Date) === ~D[2020-01-01] end - test :reduce do - assert Enum.reduce([], 1, fn(x, acc) -> x + acc end) == 1 - assert Enum.reduce([1, 2, 3], 1, fn(x, acc) -> x + acc end) == 7 + test "max/3" do + assert Enum.max([1], &>=/2, fn -> nil end) == 1 + assert Enum.max([1, 2, 3], &>=/2, fn -> nil end) == 3 + assert Enum.max([1, [], :a, {}], &>=/2, fn -> nil end) == [] + assert Enum.max([], &>=/2, fn -> :empty_value end) == :empty_value + assert Enum.max(%{}, &>=/2, fn -> :empty_value end) == :empty_value + assert_runs_enumeration_only_once(&Enum.max(&1, fn a, b -> a >= b end, fn -> nil end)) + end + + test "max_by/2" do + assert Enum.max_by(["a", "aa", "aaa"], fn x -> String.length(x) end) == "aaa" + + assert Enum.max_by([1, 1.0], & &1) === 1 + assert Enum.max_by([1.0, 1], & &1) === 1.0 - assert Enum.reduce([1, 2, 3], fn(x, acc) -> x + acc end) == 6 assert_raise Enum.EmptyError, fn -> - Enum.reduce([], fn(x, acc) -> x + acc end) + Enum.max_by([], fn x -> String.length(x) end) + end + + assert_raise Enum.EmptyError, fn -> + Enum.max_by(%{}, & &1) end end - test :reject do - assert Enum.reject([1, 2, 3], fn(x) -> rem(x, 2) == 0 end) == [1, 3] - assert Enum.reject([2, 4, 6], fn(x) -> rem(x, 2) == 0 end) == [] + test "max_by/3 with stable sorting" do + assert Enum.max_by([1, 1.0], & &1, &>=/2) === 1 + assert Enum.max_by([1.0, 1], & &1, &>=/2) === 1.0 + assert Enum.max_by([1, 1.0], & &1, &>/2) === 1.0 + assert Enum.max_by([1.0, 1], & &1, &>/2) === 1 + end + + test "max_by/3 with module" do + users = [%{id: 1, date: ~D[2019-01-01]}, %{id: 2, date: ~D[2020-01-01]}] + assert Enum.max_by(users, & &1.date, Date).id == 2 + + users = [%{id: 1, date: ~D[2020-01-01]}, %{id: 2, date: ~D[2020-01-01]}] + assert Enum.max_by(users, & &1.date, Date).id == 1 + end + + test "max_by/4" do + assert Enum.max_by(["a", "aa", "aaa"], fn x -> String.length(x) end, &>=/2, fn -> nil end) == + "aaa" + + assert Enum.max_by([], fn x -> String.length(x) end, &>=/2, fn -> :empty_value end) == + :empty_value + + assert Enum.max_by(%{}, & &1, &>=/2, fn -> :empty_value end) == :empty_value + assert Enum.max_by(%{}, & &1, &>=/2, fn -> {:a, :tuple} end) == {:a, :tuple} + + assert_runs_enumeration_only_once( + &Enum.max_by(&1, fn e -> e end, fn a, b -> a >= b end, fn -> nil end) + ) + end + + test "member?/2" do + assert Enum.member?([1, 2, 3], 2) + refute Enum.member?([], 0) + refute Enum.member?([1, 2, 3], 0) + end + + test "min/1" do + assert Enum.min([1]) == 1 + assert Enum.min([1, 2, 3]) == 1 + assert Enum.min([[], :a, {}]) == :a + + assert Enum.min([1, 1.0]) === 1 + assert Enum.min([1.0, 1]) === 1.0 + + assert_raise Enum.EmptyError, fn -> + Enum.min([]) + end + end + + test "min/2 with stable sorting" do + assert Enum.min([1, 1.0], &<=/2) === 1 + assert Enum.min([1.0, 1], &<=/2) === 1.0 + assert Enum.min([1, 1.0], & nil end) == 1 + assert Enum.min([1, 2, 3], &<=/2, fn -> nil end) == 1 + assert Enum.min([[], :a, {}], &<=/2, fn -> nil end) == :a + assert Enum.min([], &<=/2, fn -> :empty_value end) == :empty_value + assert Enum.min(%{}, &<=/2, fn -> :empty_value end) == :empty_value + assert_runs_enumeration_only_once(&Enum.min(&1, fn a, b -> a <= b end, fn -> nil end)) + end + + test "min_by/2" do + assert Enum.min_by(["a", "aa", "aaa"], fn x -> String.length(x) end) == "a" + + assert Enum.min_by([1, 1.0], & &1) === 1 + assert Enum.min_by([1.0, 1], & &1) === 1.0 + + assert_raise Enum.EmptyError, fn -> + Enum.min_by([], fn x -> String.length(x) end) + end + + assert_raise Enum.EmptyError, fn -> + Enum.min_by(%{}, & &1) + end end - test :reverse do + test "min_by/3 with stable sorting" do + assert Enum.min_by([1, 1.0], & &1, &<=/2) === 1 + assert Enum.min_by([1.0, 1], & &1, &<=/2) === 1.0 + assert Enum.min_by([1, 1.0], & &1, & String.length(x) end, &<=/2, fn -> nil end) == + "a" + + assert Enum.min_by([], fn x -> String.length(x) end, &<=/2, fn -> :empty_value end) == + :empty_value + + assert Enum.min_by(%{}, & &1, &<=/2, fn -> :empty_value end) == :empty_value + assert Enum.min_by(%{}, & &1, &<=/2, fn -> {:a, :tuple} end) == {:a, :tuple} + + assert_runs_enumeration_only_once( + &Enum.min_by(&1, fn e -> e end, fn a, b -> a <= b end, fn -> nil end) + ) + end + + test "min_max/1" do + assert Enum.min_max([1]) == {1, 1} + assert Enum.min_max([2, 3, 1]) == {1, 3} + assert Enum.min_max([[], :a, {}]) == {:a, []} + + assert Enum.min_max([1, 1.0]) === {1, 1} + assert Enum.min_max([1.0, 1]) === {1.0, 1.0} + + assert_raise Enum.EmptyError, fn -> + Enum.min_max([]) + end + end + + test "min_max/2" do + assert Enum.min_max([1], fn -> nil end) == {1, 1} + assert Enum.min_max([2, 3, 1], fn -> nil end) == {1, 3} + assert Enum.min_max([[], :a, {}], fn -> nil end) == {:a, []} + assert Enum.min_max([], fn -> {:empty_min, :empty_max} end) == {:empty_min, :empty_max} + assert Enum.min_max(%{}, fn -> {:empty_min, :empty_max} end) == {:empty_min, :empty_max} + assert_runs_enumeration_only_once(&Enum.min_max(&1, fn -> nil end)) + end + + test "min_max_by/2" do + assert Enum.min_max_by(["aaa", "a", "aa"], fn x -> String.length(x) end) == {"a", "aaa"} + + assert Enum.min_max_by([1, 1.0], & &1) === {1, 1} + assert Enum.min_max_by([1.0, 1], & &1) === {1.0, 1.0} + + assert_raise Enum.EmptyError, fn -> + Enum.min_max_by([], fn x -> String.length(x) end) + end + end + + test "min_max_by/3" do + assert Enum.min_max_by(["aaa", "a", "aa"], fn x -> String.length(x) end, fn -> nil end) == + {"a", "aaa"} + + assert Enum.min_max_by([], fn x -> String.length(x) end, fn -> {:no_min, :no_max} end) == + {:no_min, :no_max} + + assert Enum.min_max_by(%{}, fn x -> String.length(x) end, fn -> {:no_min, :no_max} end) == + {:no_min, :no_max} + + assert Enum.min_max_by(["aaa", "a", "aa"], fn x -> String.length(x) end, &>/2) == {"aaa", "a"} + + assert_runs_enumeration_only_once(&Enum.min_max_by(&1, fn x -> x end, fn -> nil end)) + end + + test "min_max_by/4" do + users = [%{id: 1, date: ~D[2019-01-01]}, %{id: 2, date: ~D[2020-01-01]}] + + assert Enum.min_max_by(users, & &1.date, Date) == + {%{id: 1, date: ~D[2019-01-01]}, %{id: 2, date: ~D[2020-01-01]}} + + assert Enum.min_max_by(["aaa", "a", "aa"], fn x -> String.length(x) end, &>/2, fn -> nil end) == + {"aaa", "a"} + + assert Enum.min_max_by([], fn x -> String.length(x) end, &>/2, fn -> {:no_min, :no_max} end) == + {:no_min, :no_max} + + assert Enum.min_max_by(%{}, fn x -> String.length(x) end, &>/2, fn -> {:no_min, :no_max} end) == + {:no_min, :no_max} + + assert_runs_enumeration_only_once( + &Enum.min_max_by(&1, fn x -> x end, fn a, b -> a > b end, fn -> nil end) + ) + end + + test "split_with/2" do + assert Enum.split_with([], fn x -> rem(x, 2) == 0 end) == {[], []} + assert Enum.split_with([1, 2, 3], fn x -> rem(x, 2) == 0 end) == {[2], [1, 3]} + assert Enum.split_with([2, 4, 6], fn x -> rem(x, 2) == 0 end) == {[2, 4, 6], []} + + assert Enum.split_with(1..5, fn x -> rem(x, 2) == 0 end) == {[2, 4], [1, 3, 5]} + assert Enum.split_with(-3..0, fn x -> x > 0 end) == {[], [-3, -2, -1, 0]} + + assert Enum.split_with(%{}, fn x -> rem(x, 2) == 0 end) == {[], []} + + assert Enum.split_with(%{a: 1, b: 2, c: 3}, fn {_k, v} -> rem(v, 2) == 0 end) == + {[b: 2], [a: 1, c: 3]} + + assert Enum.split_with(%{b: 2, d: 4, f: 6}, fn {_k, v} -> rem(v, 2) == 0 end) == + {[b: 2, d: 4, f: 6], []} + end + + test "random/1" do + # corner cases, independent of the seed + assert_raise Enum.EmptyError, fn -> Enum.random([]) end + assert Enum.random([1]) == 1 + + # set a fixed seed so the test can be deterministic + # please note the order of following assertions is important + seed1 = {1406, 407_414, 139_258} + seed2 = {1306, 421_106, 567_597} + :rand.seed(:exsss, seed1) + assert Enum.random([1, 2]) == 1 + assert Enum.random([1, 2]) == 2 + :rand.seed(:exsss, seed1) + assert Enum.random([1, 2]) == 1 + assert Enum.random([1, 2, 3]) == 1 + assert Enum.random([1, 2, 3, 4]) == 2 + assert Enum.random([1, 2, 3, 4, 5]) == 3 + :rand.seed(:exsss, seed2) + assert Enum.random([1, 2]) == 1 + assert Enum.random([1, 2, 3]) == 2 + assert Enum.random([1, 2, 3, 4]) == 4 + assert Enum.random([1, 2, 3, 4, 5]) == 3 + end + + test "reduce/2" do + assert Enum.reduce([1, 2, 3], fn x, acc -> x + acc end) == 6 + + assert_raise Enum.EmptyError, fn -> + Enum.reduce([], fn x, acc -> x + acc end) + end + + assert_raise Enum.EmptyError, fn -> + Enum.reduce(%{}, fn _, acc -> acc end) + end + end + + test "reduce/3" do + assert Enum.reduce([], 1, fn x, acc -> x + acc end) == 1 + assert Enum.reduce([1, 2, 3], 1, fn x, acc -> x + acc end) == 7 + end + + test "reduce_while/3" do + assert Enum.reduce_while([1, 2, 3], 1, fn i, acc -> {:cont, acc + i} end) == 7 + assert Enum.reduce_while([1, 2, 3], 1, fn _i, acc -> {:halt, acc} end) == 1 + assert Enum.reduce_while([], 0, fn _i, acc -> {:cont, acc} end) == 0 + end + + test "reject/2" do + assert Enum.reject([1, 2, 3], fn x -> rem(x, 2) == 0 end) == [1, 3] + assert Enum.reject([2, 4, 6], fn x -> rem(x, 2) == 0 end) == [] + assert Enum.reject([1, true, nil, false, 2], & &1) == [nil, false] + end + + test "reverse/1" do assert Enum.reverse([]) == [] assert Enum.reverse([1, 2, 3]) == [3, 2, 1] + assert Enum.reverse([5..5]) == [5..5] + end + + test "reverse/2" do assert Enum.reverse([1, 2, 3], [4, 5, 6]) == [3, 2, 1, 4, 5, 6] + assert Enum.reverse([1, 2, 3], []) == [3, 2, 1] + assert Enum.reverse([5..5], [5]) == [5..5, 5] + end + + test "reverse_slice/3" do + assert Enum.reverse_slice([], 1, 2) == [] + assert Enum.reverse_slice([1, 2, 3], 0, 0) == [1, 2, 3] + assert Enum.reverse_slice([1, 2, 3], 0, 1) == [1, 2, 3] + assert Enum.reverse_slice([1, 2, 3], 0, 2) == [2, 1, 3] + assert Enum.reverse_slice([1, 2, 3], 0, 20_000_000) == [3, 2, 1] + assert Enum.reverse_slice([1, 2, 3], 100, 2) == [1, 2, 3] + assert Enum.reverse_slice([1, 2, 3], 10, 10) == [1, 2, 3] + end + + describe "slide/3" do + test "on an empty enum produces an empty list" do + for enum <- [[], %{}, 0..-1//1, MapSet.new()] do + assert Enum.slide(enum, 0..0, 0) == [] + end + end + + test "on a single-element enumerable is the same as transforming to list" do + for enum <- [["foo"], [1], [%{foo: "bar"}], %{foo: :bar}, MapSet.new(["foo"]), 1..1] do + assert Enum.slide(enum, 0..0, 0) == Enum.to_list(enum) + end + end + + test "moves a single element" do + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + expected_numbers = Enum.flat_map([0..7, [14], 8..13, 15..20], &Enum.to_list/1) + assert Enum.slide(zero_to_20, 14..14, 8) == expected_numbers + end + + assert Enum.slide([:a, :b, :c, :d, :e, :f], 3..3, 2) == [:a, :b, :d, :c, :e, :f] + assert Enum.slide([:a, :b, :c, :d, :e, :f], 3, 3) == [:a, :b, :c, :d, :e, :f] + end + + test "on a subsection of a list reorders the range correctly" do + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + expected_numbers = Enum.flat_map([0..7, 14..18, 8..13, 19..20], &Enum.to_list/1) + assert Enum.slide(zero_to_20, 14..18, 8) == expected_numbers + end + + assert Enum.slide([:a, :b, :c, :d, :e, :f], 3..4, 2) == [:a, :b, :d, :e, :c, :f] + end + + test "handles negative indices" do + make_negative_range = fn first..last, length -> + (first - length)..(last - length)//1 + end + + test_specs = [ + {[], 0..0, 0}, + {[1], 0..0, 0}, + {[-2, 1], 1..1, 1}, + {[4, -3, 2, -1], 3..3, 2}, + {[-5, -3, 4, 4, 5], 0..2, 3}, + {[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 4..7, 9}, + {[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 4..7, 0} + ] + + for {list, range, insertion_point} <- test_specs do + negative_range = make_negative_range.(range, length(list)) + + assert Enum.slide(list, negative_range, insertion_point) == + Enum.slide(list, range, insertion_point) + end + end + + test "handles mixed positive and negative indices" do + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + assert Enum.slide(zero_to_20, -6..-1, 8) == + Enum.slide(zero_to_20, 15..20, 8) + + assert Enum.slide(zero_to_20, 15..-1//1, 8) == + Enum.slide(zero_to_20, 15..20, 8) + + assert Enum.slide(zero_to_20, -6..20, 8) == + Enum.slide(zero_to_20, 15..20, 8) + + assert Enum.slide(zero_to_20, -100..5, 8) == + Enum.slide(zero_to_20, 0..5, 8) + end + end + + test "raises an error when the step is not exactly 1" do + slide_ranges_that_should_fail = [2..10//2, 8..-1, 10..2//-1, 10..4//-2, -1..-8//-1] + + for zero_to_20 <- [0..20, Enum.to_list(0..20)], + range_that_should_fail <- slide_ranges_that_should_fail do + assert_raise(ArgumentError, fn -> + Enum.slide(zero_to_20, range_that_should_fail, 1) + end) + end + end + + test "doesn't change the order when the first and middle indices match" do + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + assert Enum.slide(zero_to_20, 8..18, 8) == Enum.to_list(0..20) + end + + assert Enum.slide([:a, :b, :c, :d, :e, :f], 1..3, 1) == [:a, :b, :c, :d, :e, :f] + end + + test "on the whole of an enumerable reorders it correctly" do + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + expected_numbers = Enum.flat_map([10..20, 0..9], &Enum.to_list/1) + assert Enum.slide(zero_to_20, 10..20, 0) == expected_numbers + end + + assert Enum.slide([:a, :b, :c, :d, :e, :f], 4..5, 0) == [:e, :f, :a, :b, :c, :d] + end + + test "raises when the insertion point is inside the range" do + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + assert_raise ArgumentError, fn -> + Enum.slide(zero_to_20, 10..18, 14) + end + end + end + + test "accepts range starts that are off the end of the enum, returning the input list" do + assert Enum.slide([], 1..5, 0) == [] + + for zero_to_20 <- [0..20, Enum.to_list(0..20)] do + assert Enum.slide(zero_to_20, 21..25, 3) == Enum.to_list(0..20) + end + end + + test "accepts range ends that are off the end of the enum, truncating the moved range" do + for zero_to_10 <- [0..10, Enum.to_list(0..10)] do + assert Enum.slide(zero_to_10, 8..15, 4) == Enum.slide(zero_to_10, 8..10, 4) + end + end + + test "matches behavior for lists vs. ranges" do + range = 0..20 + list = Enum.to_list(range) + # Below 32 elements, the map implementation currently sticks values in order. + # If ever the MapSet implementation changes, this will fail (not affecting the correctness + # of slide). I figured it'd be worth testing this for the time being just to have + # another enumerable (aside from range) testing the generic implementation. + set = MapSet.new(list) + + test_specs = [ + {0..0, 0}, + {0..0, 20}, + {11..11, 14}, + {11..11, 3}, + {4..8, 19}, + {4..8, 0}, + {4..8, 2}, + {10..20, 0} + ] + + for {slide_range, insertion_point} <- test_specs do + slide = &Enum.slide(&1, slide_range, insertion_point) + assert slide.(list) == slide.(set) + assert slide.(list) == slide.(range) + end + end + + test "inserts at negative indices" do + for zero_to_5 <- [0..5, Enum.to_list(0..5)] do + assert Enum.slide(zero_to_5, 0, -1) == [1, 2, 3, 4, 5, 0] + assert Enum.slide(zero_to_5, 1, -1) == [0, 2, 3, 4, 5, 1] + assert Enum.slide(zero_to_5, 1..2, -2) == [0, 3, 4, 1, 2, 5] + assert Enum.slide(zero_to_5, -5..-4//1, -2) == [0, 3, 4, 1, 2, 5] + end + + assert Enum.slide([:a, :b, :c, :d, :e, :f], -5..-3//1, -2) == + Enum.slide([:a, :b, :c, :d, :e, :f], 1..3, 4) + end + + test "raises when insertion index would fall inside the range" do + for zero_to_5 <- [0..5, Enum.to_list(0..5)] do + assert_raise ArgumentError, fn -> + Enum.slide(zero_to_5, 2..3, -3) + end + end + + for zero_to_10 <- [0..10, Enum.to_list(0..10)], + insertion_idx <- 3..5 do + assert_raise ArgumentError, fn -> + assert Enum.slide(zero_to_10, 2..5, insertion_idx) + end + end + end + end + + test "scan/2" do + assert Enum.scan([1, 2, 3, 4, 5], &(&1 + &2)) == [1, 3, 6, 10, 15] + assert Enum.scan([], &(&1 + &2)) == [] + end + + test "scan/3" do + assert Enum.scan([1, 2, 3, 4, 5], 0, &(&1 + &2)) == [1, 3, 6, 10, 15] + assert Enum.scan([], 0, &(&1 + &2)) == [] + end + + test "shuffle/1" do + # set a fixed seed so the test can be deterministic + :rand.seed(:exsss, {1374, 347_975, 449_264}) + assert Enum.shuffle([1, 2, 3, 4, 5]) == [1, 3, 4, 5, 2] + end + + test "slice/2" do + list = [1, 2, 3, 4, 5] + assert Enum.slice(list, 0..0) == [1] + assert Enum.slice(list, 0..1) == [1, 2] + assert Enum.slice(list, 0..2) == [1, 2, 3] + + assert Enum.slice(list, 0..10//2) == [1, 3, 5] + assert Enum.slice(list, 0..10//3) == [1, 4] + assert Enum.slice(list, 0..10//4) == [1, 5] + assert Enum.slice(list, 0..10//5) == [1] + assert Enum.slice(list, 0..10//6) == [1] + + assert Enum.slice(list, 0..2//2) == [1, 3] + assert Enum.slice(list, 0..2//3) == [1] + + assert Enum.slice(list, 0..-1//2) == [1, 3, 5] + assert Enum.slice(list, 0..-1//3) == [1, 4] + assert Enum.slice(list, 0..-1//4) == [1, 5] + assert Enum.slice(list, 0..-1//5) == [1] + assert Enum.slice(list, 0..-1//6) == [1] + + assert Enum.slice(list, 1..-1//2) == [2, 4] + assert Enum.slice(list, 1..-1//3) == [2, 5] + assert Enum.slice(list, 1..-1//4) == [2] + assert Enum.slice(list, 1..-1//5) == [2] + + assert Enum.slice(list, -4..-1//2) == [2, 4] + assert Enum.slice(list, -4..-1//3) == [2, 5] + assert Enum.slice(list, -4..-1//4) == [2] + assert Enum.slice(list, -4..-1//5) == [2] + end + + test "slice/3" do + list = [1, 2, 3, 4, 5] + assert Enum.slice(list, 0, 0) == [] + assert Enum.slice(list, 0, 1) == [1] + assert Enum.slice(list, 0, 2) == [1, 2] + assert Enum.slice(list, 1, 2) == [2, 3] + assert Enum.slice(list, 1, 0) == [] + assert Enum.slice(list, 2, 5) == [3, 4, 5] + assert Enum.slice(list, 2, 6) == [3, 4, 5] + assert Enum.slice(list, 5, 5) == [] + assert Enum.slice(list, 6, 5) == [] + assert Enum.slice(list, 6, 0) == [] + assert Enum.slice(list, -6, 0) == [] + assert Enum.slice(list, -6, 5) == [1, 2, 3, 4, 5] + assert Enum.slice(list, -2, 5) == [4, 5] + assert Enum.slice(list, -3, 1) == [3] + + assert_raise FunctionClauseError, fn -> + Enum.slice(list, 0, -1) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(list, 0.99, 0) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(list, 0, 0.99) + end + end + + test "slice on infinite streams" do + assert [1, 2, 3] |> Stream.cycle() |> Enum.slice(0, 2) == [1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Enum.slice(0, 5) == [1, 2, 3, 1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Enum.slice(0..1) == [1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Enum.slice(0..4) == [1, 2, 3, 1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Enum.slice(0..4//2) == [1, 3, 2] end - test :scan do - assert Enum.scan([1,2,3,4,5], &(&1 + &2)) == [1,3,6,10,15] - assert Enum.scan([], &(&1 + &2)) == [] + test "slice on pruned infinite streams" do + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(0, 2) == [1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(0, 5) == [1, 2, 3, 1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(0..1) == [1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(0..4) == [1, 2, 3, 1, 2] + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(0..4//2) == [1, 3, 2] - assert Enum.scan([1,2,3,4,5], 0, &(&1 + &2)) == [1,3,6,10,15] - assert Enum.scan([], 0, &(&1 + &2)) == [] - end + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(-10..-9//1) == + [1, 2] - test :shuffle do - # set a fixed seed so the test can be deterministic - :random.seed(1374, 347975, 449264) - assert Enum.shuffle([1, 2, 3, 4, 5]) == [2, 4, 1, 5, 3] + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(-10..-6//1) == + [1, 2, 3, 1, 2] + + assert [1, 2, 3] |> Stream.cycle() |> Stream.take(10) |> Enum.slice(-10..-6//2) == + [1, 3, 2] end - test :sort do + test "sort/1" do assert Enum.sort([5, 3, 2, 4, 1]) == [1, 2, 3, 4, 5] - assert Enum.sort([5, 3, 2, 4, 1], &(&1 > &2)) == [5, 4, 3, 2, 1] end - test :split do + test "sort/2" do + assert Enum.sort([5, 3, 2, 4, 1], &(&1 >= &2)) == [5, 4, 3, 2, 1] + assert Enum.sort([5, 3, 2, 4, 1], :asc) == [1, 2, 3, 4, 5] + assert Enum.sort([5, 3, 2, 4, 1], :desc) == [5, 4, 3, 2, 1] + end + + test "sort/2 with module" do + assert Enum.sort([~D[2020-01-01], ~D[2018-01-01], ~D[2019-01-01]], Date) == + [~D[2018-01-01], ~D[2019-01-01], ~D[2020-01-01]] + + assert Enum.sort([~D[2020-01-01], ~D[2018-01-01], ~D[2019-01-01]], {:asc, Date}) == + [~D[2018-01-01], ~D[2019-01-01], ~D[2020-01-01]] + + assert Enum.sort([~D[2020-01-01], ~D[2018-01-01], ~D[2019-01-01]], {:desc, Date}) == + [~D[2020-01-01], ~D[2019-01-01], ~D[2018-01-01]] + end + + test "sort_by/3" do + collection = [ + [sorted_data: 4], + [sorted_data: 5], + [sorted_data: 2], + [sorted_data: 1], + [sorted_data: 3] + ] + + asc = [ + [sorted_data: 1], + [sorted_data: 2], + [sorted_data: 3], + [sorted_data: 4], + [sorted_data: 5] + ] + + desc = [ + [sorted_data: 5], + [sorted_data: 4], + [sorted_data: 3], + [sorted_data: 2], + [sorted_data: 1] + ] + + assert Enum.sort_by(collection, & &1[:sorted_data]) == asc + assert Enum.sort_by(collection, & &1[:sorted_data], :asc) == asc + assert Enum.sort_by(collection, & &1[:sorted_data], &>=/2) == desc + assert Enum.sort_by(collection, & &1[:sorted_data], :desc) == desc + end + + test "sort_by/3 with stable sorting" do + collection = [ + [other_data: 2, sorted_data: 4], + [other_data: 1, sorted_data: 5], + [other_data: 2, sorted_data: 2], + [other_data: 3, sorted_data: 1], + [other_data: 4, sorted_data: 3] + ] + + # Stable sorting + assert Enum.sort_by(collection, & &1[:other_data]) == [ + [other_data: 1, sorted_data: 5], + [other_data: 2, sorted_data: 4], + [other_data: 2, sorted_data: 2], + [other_data: 3, sorted_data: 1], + [other_data: 4, sorted_data: 3] + ] + + assert Enum.sort_by(collection, & &1[:other_data]) == + Enum.sort_by(collection, & &1[:other_data], :asc) + + assert Enum.sort_by(collection, & &1[:other_data], & + Enum.split([1, 2, 3], 0.0) + end end - test :split_while do - assert Enum.split_while([1, 2, 3], fn(_) -> false end) == {[], [1, 2, 3]} - assert Enum.split_while([1, 2, 3], fn(_) -> true end) == {[1, 2, 3], []} - assert Enum.split_while([1, 2, 3], fn(x) -> x > 2 end) == {[], [1, 2, 3]} - assert Enum.split_while([1, 2, 3], fn(x) -> x > 3 end) == {[], [1, 2, 3]} - assert Enum.split_while([1, 2, 3], fn(x) -> x < 3 end) == {[1, 2], [3]} - assert Enum.split_while([], fn(_) -> true end) == {[], []} + test "split_while/2" do + assert Enum.split_while([1, 2, 3], fn _ -> false end) == {[], [1, 2, 3]} + assert Enum.split_while([1, 2, 3], fn _ -> true end) == {[1, 2, 3], []} + assert Enum.split_while([1, 2, 3], fn x -> x > 2 end) == {[], [1, 2, 3]} + assert Enum.split_while([1, 2, 3], fn x -> x > 3 end) == {[], [1, 2, 3]} + assert Enum.split_while([1, 2, 3], fn x -> x < 3 end) == {[1, 2], [3]} + assert Enum.split_while([], fn _ -> true end) == {[], []} end - test :sum do + test "sum/1" do assert Enum.sum([]) == 0 assert Enum.sum([1]) == 1 assert Enum.sum([1, 2, 3]) == 6 assert Enum.sum([1.1, 2.2, 3.3]) == 6.6 + assert Enum.sum([-3, -2, -1, 0, 1, 2, 3]) == 0 + assert Enum.sum(42..42) == 42 + assert Enum.sum(11..17) == 98 + assert Enum.sum(17..11) == 98 + assert Enum.sum(11..-17) == Enum.sum(-17..11) + assert_raise ArithmeticError, fn -> Enum.sum([{}]) end + + assert_raise ArithmeticError, fn -> + Enum.sum([1, {}]) + end + end + + test "product/1" do + assert Enum.product([]) == 1 + assert Enum.product([1]) == 1 + assert Enum.product([1, 2, 3, 4, 5]) == 120 + assert Enum.product([1, -2, 3, 4, 5]) == -120 + assert Enum.product(1..5) == 120 + assert Enum.product(11..-17) == Enum.product(-17..11) + + assert_raise ArithmeticError, fn -> + Enum.product([{}]) + end + + assert_raise ArithmeticError, fn -> + Enum.product([1, {}]) + end + assert_raise ArithmeticError, fn -> - Enum.sum([1,{}]) + Enum.product(%{a: 1, b: 2}) end end - test :take do + test "take/2" do assert Enum.take([1, 2, 3], 0) == [] assert Enum.take([1, 2, 3], 1) == [1] assert Enum.take([1, 2, 3], 2) == [1, 2] @@ -325,165 +1308,466 @@ defmodule EnumTest.List do assert Enum.take([1, 2, 3], -2) == [2, 3] assert Enum.take([1, 2, 3], -4) == [1, 2, 3] assert Enum.take([], 3) == [] + + assert_raise FunctionClauseError, fn -> + Enum.take([1, 2, 3], 0.0) + end end - test :take_every do + test "take_every/2" do assert Enum.take_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 2) == [1, 3, 5, 7, 9] + assert Enum.take_every([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 3) == [1, 4, 7, 10] assert Enum.take_every([], 2) == [] assert Enum.take_every([1, 2], 2) == [1] assert Enum.take_every([1, 2, 3], 0) == [] + assert Enum.take_every(1..3, 1) == [1, 2, 3] + + assert_raise FunctionClauseError, fn -> + Enum.take_every([1, 2, 3], -1) + end + + assert_raise FunctionClauseError, fn -> + Enum.take_every(1..10, 3.33) + end + end + + test "take_random/2" do + assert Enum.take_random(-42..-42, 1) == [-42] + + # corner cases, independent of the seed + assert_raise FunctionClauseError, fn -> Enum.take_random([1, 2], -1) end + assert Enum.take_random([], 0) == [] + assert Enum.take_random([], 3) == [] + assert Enum.take_random([1], 0) == [] + assert Enum.take_random([1], 2) == [1] + assert Enum.take_random([1, 2], 0) == [] + + # set a fixed seed so the test can be deterministic + # please note the order of following assertions is important + seed1 = {1406, 407_414, 139_258} + seed2 = {1406, 421_106, 567_597} + :rand.seed(:exsss, seed1) + assert Enum.take_random([1, 2, 3], 1) == [3] + assert Enum.take_random([1, 2, 3], 2) == [3, 2] + assert Enum.take_random([1, 2, 3], 3) == [3, 1, 2] + assert Enum.take_random([1, 2, 3], 4) == [1, 3, 2] + :rand.seed(:exsss, seed2) + assert Enum.take_random([1, 2, 3], 1) == [1] + assert Enum.take_random([1, 2, 3], 2) == [3, 1] + assert Enum.take_random([1, 2, 3], 3) == [3, 1, 2] + assert Enum.take_random([1, 2, 3], 4) == [2, 1, 3] + assert Enum.take_random([1, 2, 3], 129) == [2, 3, 1] + + # assert that every item in the sample comes from the input list + list = for _ <- 1..100, do: make_ref() + + for x <- Enum.take_random(list, 50) do + assert x in list + end + + assert_raise FunctionClauseError, fn -> + Enum.take_random(1..10, -1) + end + + assert_raise FunctionClauseError, fn -> + Enum.take_random(1..10, 10.0) + end + + assert_raise FunctionClauseError, fn -> + Enum.take_random(1..10, 128.1) + end end - test :take_while do - assert Enum.take_while([1, 2, 3], fn(x) -> x > 3 end) == [] - assert Enum.take_while([1, 2, 3], fn(x) -> x <= 1 end) == [1] - assert Enum.take_while([1, 2, 3], fn(x) -> x <= 3 end) == [1, 2, 3] - assert Enum.take_while([], fn(_) -> true end) == [] + test "take_while/2" do + assert Enum.take_while([1, 2, 3], fn x -> x > 3 end) == [] + assert Enum.take_while([1, 2, 3], fn x -> x <= 1 end) == [1] + assert Enum.take_while([1, 2, 3], fn x -> x <= 3 end) == [1, 2, 3] + assert Enum.take_while([], fn _ -> true end) == [] end - test :to_list do + test "to_list/1" do assert Enum.to_list([]) == [] - assert Enum.to_list(1 .. 3) == [1, 2, 3] end - test :traverse do - assert Enum.traverse([1, 2, 3], &(&1 * &1)) == [1, 4, 9] - assert Enum.traverse(%{a: 1, b: 2}, fn {k, v} -> {k, v*2} end) == %{a: 2, b: 4} + test "uniq/1" do + assert Enum.uniq([5, 1, 2, 3, 2, 1]) == [5, 1, 2, 3] + end + + test "uniq_by/2" do + assert Enum.uniq_by([1, 2, 3, 2, 1], fn x -> x end) == [1, 2, 3] end - test :uniq do - assert Enum.uniq([1, 2, 3, 2, 1]) == [1, 2, 3] - assert Enum.uniq([1, 2, 3, 2, 1], fn x -> x end) == [1, 2, 3] + test "unzip/1" do + assert Enum.unzip([{:a, 1}, {:b, 2}, {:c, 3}]) == {[:a, :b, :c], [1, 2, 3]} + assert Enum.unzip([]) == {[], []} + assert Enum.unzip(%{a: 1, b: 2}) == {[:a, :b], [1, 2]} + assert Enum.unzip(foo: "a", bar: "b") == {[:foo, :bar], ["a", "b"]} + + assert_raise FunctionClauseError, fn -> Enum.unzip([{:a, 1}, {:b, 2, "foo"}]) end + assert_raise FunctionClauseError, fn -> Enum.unzip([{1, 2, {3, {4, 5}}}]) end + assert_raise FunctionClauseError, fn -> Enum.unzip([1, 2, 3]) end + end + + test "with_index/2" do + assert Enum.with_index([]) == [] + assert Enum.with_index([1, 2, 3]) == [{1, 0}, {2, 1}, {3, 2}] + assert Enum.with_index([1, 2, 3], 10) == [{1, 10}, {2, 11}, {3, 12}] + + assert Enum.with_index([1, 2, 3], fn element, index -> {index, element} end) == + [{0, 1}, {1, 2}, {2, 3}] + + assert Enum.with_index(1..0//1) == [] + assert Enum.with_index(1..3) == [{1, 0}, {2, 1}, {3, 2}] + assert Enum.with_index(1..3, 10) == [{1, 10}, {2, 11}, {3, 12}] + + assert Enum.with_index(1..3, fn element, index -> {index, element} end) == + [{0, 1}, {1, 2}, {2, 3}] end - test :zip do + test "zip/2" do assert Enum.zip([:a, :b], [1, 2]) == [{:a, 1}, {:b, 2}] assert Enum.zip([:a, :b], [1, 2, 3, 4]) == [{:a, 1}, {:b, 2}] assert Enum.zip([:a, :b, :c, :d], [1, 2]) == [{:a, 1}, {:b, 2}] + assert Enum.zip([], [1]) == [] assert Enum.zip([1], []) == [] - assert Enum.zip([], []) == [] + assert Enum.zip([], []) == [] end - test :with_index do - assert Enum.with_index([]) == [] - assert Enum.with_index([1,2,3]) == [{1,0},{2,1},{3,2}] - end + test "zip/1" do + assert Enum.zip([[:a, :b], [1, 2], ["foo", "bar"]]) == [{:a, 1, "foo"}, {:b, 2, "bar"}] - test :max do - assert Enum.max([1]) == 1 - assert Enum.max([1, 2, 3]) == 3 - assert Enum.max([1, [], :a, {}]) == [] - assert_raise Enum.EmptyError, fn -> - Enum.max([]) - end - end + assert Enum.zip([[:a, :b], [1, 2, 3, 4], ["foo", "bar", "baz", "qux"]]) == + [{:a, 1, "foo"}, {:b, 2, "bar"}] - test :max_by do - assert Enum.max_by(["a", "aa", "aaa"], fn(x) -> String.length(x) end) == "aaa" - assert_raise Enum.EmptyError, fn -> - Enum.max_by([], fn(x) -> String.length(x) end) - end - end + assert Enum.zip([[:a, :b, :c, :d], [1, 2], ["foo", "bar", "baz", "qux"]]) == + [{:a, 1, "foo"}, {:b, 2, "bar"}] - test :min do - assert Enum.min([1]) == 1 - assert Enum.min([1, 2, 3]) == 1 - assert Enum.min([[], :a, {}]) == :a - assert_raise Enum.EmptyError, fn -> - Enum.min([]) - end - end + assert Enum.zip([[:a, :b, :c, :d], [1, 2, 3, 4], ["foo", "bar"]]) == + [{:a, 1, "foo"}, {:b, 2, "bar"}] - test :min_by do - assert Enum.min_by(["a", "aa", "aaa"], fn(x) -> String.length(x) end) == "a" - assert_raise Enum.EmptyError, fn -> - Enum.min_by([], fn(x) -> String.length(x) end) - end - end + assert Enum.zip([1..10, ["foo", "bar"]]) == [{1, "foo"}, {2, "bar"}] - test :chunk do - assert Enum.chunk([1, 2, 3, 4, 5], 2) == [[1, 2], [3, 4]] - assert Enum.chunk([1, 2, 3, 4, 5], 2, 2, [6]) == [[1, 2], [3, 4], [5, 6]] - assert Enum.chunk([1, 2, 3, 4, 5, 6], 3, 2) == [[1, 2, 3], [3, 4, 5]] - assert Enum.chunk([1, 2, 3, 4, 5, 6], 2, 3) == [[1, 2], [4, 5]] - assert Enum.chunk([1, 2, 3, 4, 5, 6], 3, 2, []) == [[1, 2, 3], [3, 4, 5], [5, 6]] - assert Enum.chunk([1, 2, 3, 4, 5, 6], 3, 3, []) == [[1, 2, 3], [4, 5, 6]] - assert Enum.chunk([1, 2, 3, 4, 5], 4, 4, 6..10) == [[1, 2, 3, 4], [5, 6, 7, 8]] + assert Enum.zip([]) == [] + assert Enum.zip([[]]) == [] + assert Enum.zip([[1]]) == [{1}] + + assert Enum.zip([[], [], [], []]) == [] + assert Enum.zip(%{}) == [] end - test :chunk_by do - assert Enum.chunk_by([1, 2, 2, 3, 4, 4, 6, 7, 7], &(rem(&1, 2) == 1)) == [[1], [2, 2], [3], [4, 4, 6], [7, 7]] - assert Enum.chunk_by([1, 2, 3, 4], fn _ -> true end) == [[1, 2, 3, 4]] - assert Enum.chunk_by([], fn _ -> true end) == [] - assert Enum.chunk_by([1], fn _ -> true end) == [[1]] + test "zip_with/3" do + assert Enum.zip_with([1, 2], [3, 4], fn a, b -> a * b end) == [3, 8] + assert Enum.zip_with([:a, :b], [1, 2], &{&1, &2}) == [{:a, 1}, {:b, 2}] + assert Enum.zip_with([:a, :b], [1, 2, 3, 4], &{&1, &2}) == [{:a, 1}, {:b, 2}] + assert Enum.zip_with([:a, :b, :c, :d], [1, 2], &{&1, &2}) == [{:a, 1}, {:b, 2}] + assert Enum.zip_with([], [1], &{&1, &2}) == [] + assert Enum.zip_with([1], [], &{&1, &2}) == [] + assert Enum.zip_with([], [], &{&1, &2}) == [] + + # Ranges + assert Enum.zip_with(1..6, 3..4, fn a, b -> a + b end) == [4, 6] + assert Enum.zip_with([1, 2, 5, 6], 3..4, fn a, b -> a + b end) == [4, 6] + assert Enum.zip_with(fn _, _ -> {:cont, [1, 2]} end, 3..4, fn a, b -> a + b end) == [4, 6] + assert Enum.zip_with(1..1, 0..0, fn a, b -> a + b end) == [1] + + # Date.range + week_1 = Date.range(~D[2020-10-12], ~D[2020-10-16]) + week_2 = Date.range(~D[2020-10-19], ~D[2020-10-23]) + + result = + Enum.zip_with(week_1, week_2, fn a, b -> + Date.day_of_week(a) + Date.day_of_week(b) + end) + + assert result == [2, 4, 6, 8, 10] + + # Maps + result = Enum.zip_with(%{a: 7, c: 9}, 3..4, fn {key, value}, b -> {key, value + b} end) + assert result == [a: 10, c: 13] + + colour_1 = %{r: 176, g: 175, b: 255} + colour_2 = %{r: 12, g: 176, b: 176} + + result = Enum.zip_with(colour_1, colour_2, fn {k, left}, {k, right} -> {k, left + right} end) + assert result == [b: 431, g: 351, r: 188] end - test :slice do - assert Enum.slice([1,2,3,4,5], 0, 0) == [] - assert Enum.slice([1,2,3,4,5], 0, 1) == [1] - assert Enum.slice([1,2,3,4,5], 0, 2) == [1, 2] - assert Enum.slice([1,2,3,4,5], 1, 2) == [2, 3] - assert Enum.slice([1,2,3,4,5], 1, 0) == [] - assert Enum.slice([1,2,3,4,5], 2, 5) == [3, 4, 5] - assert Enum.slice([1,2,3,4,5], 2, 6) == [3, 4, 5] - assert Enum.slice([1,2,3,4,5], 5, 5) == [] - assert Enum.slice([1,2,3,4,5], 6, 5) == [] - assert Enum.slice([1,2,3,4,5], 6, 0) == [] - assert Enum.slice([1,2,3,4,5], -6, 0) == [] - assert Enum.slice([1,2,3,4,5], -6, 5) == [] - assert Enum.slice([1,2,3,4,5], -2, 5) == [4, 5] - assert Enum.slice([1,2,3,4,5], -3, 1) == [3] - end - - test :slice_range do - assert Enum.slice([1,2,3,4,5], 0..0) == [1] - assert Enum.slice([1,2,3,4,5], 0..1) == [1, 2] - assert Enum.slice([1,2,3,4,5], 0..2) == [1, 2, 3] - assert Enum.slice([1,2,3,4,5], 1..2) == [2, 3] - assert Enum.slice([1,2,3,4,5], 1..0) == [] - assert Enum.slice([1,2,3,4,5], 2..5) == [3, 4, 5] - assert Enum.slice([1,2,3,4,5], 2..6) == [3, 4, 5] - assert Enum.slice([1,2,3,4,5], 4..4) == [5] - assert Enum.slice([1,2,3,4,5], 5..5) == [] - assert Enum.slice([1,2,3,4,5], 6..5) == [] - assert Enum.slice([1,2,3,4,5], 6..0) == [] - assert Enum.slice([1,2,3,4,5], -6..0) == [] - assert Enum.slice([1,2,3,4,5], -6..5) == [] - assert Enum.slice([1,2,3,4,5], -5..-1) == [1, 2, 3, 4, 5] - assert Enum.slice([1,2,3,4,5], -5..-3) == [1, 2, 3] - assert Enum.slice([1,2,3,4,5], -6..-1) == [] - assert Enum.slice([1,2,3,4,5], -6..-3) == [] + test "zip_with/2" do + zip_fun = fn items -> List.to_tuple(items) end + result = Enum.zip_with([[:a, :b], [1, 2], ["foo", "bar"]], zip_fun) + assert result == [{:a, 1, "foo"}, {:b, 2, "bar"}] + + lots = Enum.zip_with([[:a, :b], [1, 2], ["foo", "bar"], %{a: :b, c: :d}], zip_fun) + assert lots == [{:a, 1, "foo", {:a, :b}}, {:b, 2, "bar", {:c, :d}}] + + assert Enum.zip_with([[:a, :b], [1, 2, 3, 4], ["foo", "bar", "baz", "qux"]], zip_fun) == + [{:a, 1, "foo"}, {:b, 2, "bar"}] + + assert Enum.zip_with([[:a, :b, :c, :d], [1, 2], ["foo", "bar", "baz", "qux"]], zip_fun) == + [{:a, 1, "foo"}, {:b, 2, "bar"}] + + assert Enum.zip_with([[:a, :b, :c, :d], [1, 2, 3, 4], ["foo", "bar"]], zip_fun) == + [{:a, 1, "foo"}, {:b, 2, "bar"}] + + assert Enum.zip_with([1..10, ["foo", "bar"]], zip_fun) == [{1, "foo"}, {2, "bar"}] + assert Enum.zip_with([], zip_fun) == [] + assert Enum.zip_with([[]], zip_fun) == [] + assert Enum.zip_with([[1]], zip_fun) == [{1}] + assert Enum.zip_with([[], [], [], []], zip_fun) == [] + assert Enum.zip_with(%{}, zip_fun) == [] + assert Enum.zip_with([[1, 2, 5, 6], 3..4], fn [x, y] -> x + y end) == [4, 6] + + # Ranges + assert Enum.zip_with([1..6, 3..4], fn [a, b] -> a + b end) == [4, 6] + assert Enum.zip_with([[1, 2, 5, 6], 3..4], fn [a, b] -> a + b end) == [4, 6] + assert Enum.zip_with([fn _, _ -> {:cont, [1, 2]} end, 3..4], fn [a, b] -> a + b end) == [4, 6] + assert Enum.zip_with([1..1, 0..0], fn [a, b] -> a + b end) == [1] + + # Date.range + week_1 = Date.range(~D[2020-10-12], ~D[2020-10-16]) + week_2 = Date.range(~D[2020-10-19], ~D[2020-10-23]) + + result = + Enum.zip_with([week_1, week_2], fn [a, b] -> + Date.day_of_week(a) + Date.day_of_week(b) + end) + + assert result == [2, 4, 6, 8, 10] + + # Maps + result = Enum.zip_with([%{a: 7, c: 9}, 3..4], fn [{key, value}, b] -> {key, value + b} end) + assert result == [a: 10, c: 13] + + colour_1 = %{r: 176, g: 175, b: 255} + colour_2 = %{r: 12, g: 176, b: 176} + + result = + Enum.zip_with([colour_1, colour_2], fn [{k, left}, {k, right}] -> {k, left + right} end) + + assert result == [b: 431, g: 351, r: 188] + + assert Enum.zip_with([%{a: :b, c: :d}, %{e: :f, g: :h}], & &1) == [ + [a: :b, e: :f], + [c: :d, g: :h] + ] end end defmodule EnumTest.Range do + # Ranges use custom callbacks for protocols in many operations. use ExUnit.Case, async: true - test :all? do - range = 0..5 - refute Enum.all?(range, fn(x) -> rem(x, 2) == 0 end) + test "all?/2" do + assert Enum.all?(0..1) + assert Enum.all?(1..0) + refute Enum.all?(0..5, fn x -> rem(x, 2) == 0 end) + assert Enum.all?(0..1, fn x -> x < 2 end) + + assert Enum.all?(0..1//-1) + assert Enum.all?(0..5//2, fn x -> rem(x, 2) == 0 end) + refute Enum.all?(1..5//2, fn x -> rem(x, 2) == 0 end) + end + + test "any?/2" do + assert Enum.any?(1..0) + refute Enum.any?(0..5, &(&1 > 10)) + assert Enum.any?(0..5, &(&1 > 3)) + + refute Enum.any?(0..1//-1) + assert Enum.any?(0..5//2, fn x -> rem(x, 2) == 0 end) + refute Enum.any?(1..5//2, fn x -> rem(x, 2) == 0 end) + end + + test "at/3" do + assert Enum.at(2..6, 0) == 2 + assert Enum.at(2..6, 4) == 6 + assert Enum.at(2..6, 6) == nil + assert Enum.at(2..6, 6, :none) == :none + assert Enum.at(2..6, -2) == 5 + assert Enum.at(2..6, -8) == nil + + assert Enum.at(0..1//-1, 0) == nil + assert Enum.at(1..1//5, 0) == 1 + assert Enum.at(1..3//2, 0) == 1 + assert Enum.at(1..3//2, 1) == 3 + assert Enum.at(1..3//2, 2) == nil + assert Enum.at(1..3//2, -1) == 3 + assert Enum.at(1..3//2, -2) == 1 + assert Enum.at(1..3//2, -3) == nil + end + + test "chunk_every/2" do + assert Enum.chunk_every(1..5, 2) == [[1, 2], [3, 4], [5]] + assert Enum.chunk_every(1..10//2, 2) == [[1, 3], [5, 7], [9]] + end - range = 0..1 - assert Enum.all?(range, fn(x) -> x < 2 end) - assert Enum.all?(range) + test "chunk_every/4" do + assert Enum.chunk_every(1..5, 2, 2) == [[1, 2], [3, 4], [5]] + assert Enum.chunk_every(1..6, 3, 2, :discard) == [[1, 2, 3], [3, 4, 5]] + assert Enum.chunk_every(1..6, 2, 3, :discard) == [[1, 2], [4, 5]] + assert Enum.chunk_every(1..6, 3, 2, []) == [[1, 2, 3], [3, 4, 5], [5, 6]] + assert Enum.chunk_every(1..5, 4, 4, 6..10) == [[1, 2, 3, 4], [5, 6, 7, 8]] + assert Enum.chunk_every(1..10//2, 4, 4, 11..20) == [[1, 3, 5, 7], [9, 11, 12, 13]] + end - range = 1..0 - assert Enum.all?(range) + test "chunk_by/2" do + assert Enum.chunk_by(1..4, fn _ -> true end) == [[1, 2, 3, 4]] + assert Enum.chunk_by(1..4, &(rem(&1, 2) == 1)) == [[1], [2], [3], [4]] + assert Enum.chunk_by(1..20//3, &(rem(&1, 2) == 1)) == [[1], [4], [7], [10], [13], [16], [19]] + end + + test "concat/1" do + assert Enum.concat([1..2, 4..6]) == [1, 2, 4, 5, 6] + assert Enum.concat([1..5, fn acc, _ -> acc end, [1]]) == [1, 2, 3, 4, 5, 1] + assert Enum.concat([1..5, 6..10//2]) == [1, 2, 3, 4, 5, 6, 8, 10] + end + + test "concat/2" do + assert Enum.concat(1..3, 4..5) == [1, 2, 3, 4, 5] + assert Enum.concat(1..3, [4, 5]) == [1, 2, 3, 4, 5] + assert Enum.concat(1..3, []) == [1, 2, 3] + assert Enum.concat(1..3, 0..0) == [1, 2, 3, 0] + assert Enum.concat(1..5, 6..10//2) == [1, 2, 3, 4, 5, 6, 8, 10] + assert Enum.concat(1..5, 0..1//-1) == [1, 2, 3, 4, 5] + assert Enum.concat(1..5, 1..0//1) == [1, 2, 3, 4, 5] + end + + test "count/1" do + assert Enum.count(1..5) == 5 + assert Enum.count(1..1) == 1 + assert Enum.count(1..9//2) == 5 + assert Enum.count(1..10//2) == 5 + assert Enum.count(1..11//2) == 6 + assert Enum.count(1..11//-2) == 0 + assert Enum.count(11..1//-2) == 6 + assert Enum.count(10..1//-2) == 5 + assert Enum.count(9..1//-2) == 5 + assert Enum.count(9..1//2) == 0 + end + + test "count/2" do + assert Enum.count(1..5, fn x -> rem(x, 2) == 0 end) == 2 + assert Enum.count(1..1, fn x -> rem(x, 2) == 0 end) == 0 + assert Enum.count(0..5//2, fn x -> rem(x, 2) == 0 end) == 3 + assert Enum.count(1..5//2, fn x -> rem(x, 2) == 0 end) == 0 + end + + test "dedup/1" do + assert Enum.dedup(1..3) == [1, 2, 3] + assert Enum.dedup(1..3//2) == [1, 3] + end + + test "dedup_by/2" do + assert Enum.dedup_by(1..3, fn _ -> 1 end) == [1] + assert Enum.dedup_by(1..3//2, fn _ -> 1 end) == [1] + end + + test "drop/2" do + assert Enum.drop(1..3, 0) == [1, 2, 3] + assert Enum.drop(1..3, 1) == [2, 3] + assert Enum.drop(1..3, 2) == [3] + assert Enum.drop(1..3, 3) == [] + assert Enum.drop(1..3, 4) == [] + assert Enum.drop(1..3, -1) == [1, 2] + assert Enum.drop(1..3, -2) == [1] + assert Enum.drop(1..3, -4) == [] + assert Enum.drop(1..0, 3) == [] + + assert Enum.drop(1..9//2, 2) == [5, 7, 9] + assert Enum.drop(1..9//2, -2) == [1, 3, 5] + assert Enum.drop(9..1//-2, 2) == [5, 3, 1] + assert Enum.drop(9..1//-2, -2) == [9, 7, 5] + end + + test "drop_every/2" do + assert Enum.drop_every(1..10, 2) == [2, 4, 6, 8, 10] + assert Enum.drop_every(1..10, 3) == [2, 3, 5, 6, 8, 9] + assert Enum.drop_every(0..0, 2) == [] + assert Enum.drop_every(1..2, 2) == [2] + assert Enum.drop_every(1..3, 0) == [1, 2, 3] + assert Enum.drop_every(1..3, 1) == [] + + assert Enum.drop_every(1..5//2, 0) == [1, 3, 5] + assert Enum.drop_every(1..5//2, 1) == [] + assert Enum.drop_every(1..5//2, 2) == [3] + + assert_raise FunctionClauseError, fn -> + Enum.drop_every(1..10, 3.33) + end end - test :any? do - range = 0..5 - refute Enum.any?(range, &(&1 > 10)) + test "drop_while/2" do + assert Enum.drop_while(0..6, fn x -> x <= 3 end) == [4, 5, 6] + assert Enum.drop_while(0..6, fn _ -> false end) == [0, 1, 2, 3, 4, 5, 6] + assert Enum.drop_while(0..3, fn x -> x <= 3 end) == [] + assert Enum.drop_while(1..0, fn _ -> nil end) == [1, 0] + end - range = 0..5 - assert Enum.any?(range, &(&1 > 3)) + test "each/2" do + try do + assert Enum.each(1..0, fn x -> x end) == :ok + assert Enum.each(1..3, fn x -> Process.put(:enum_test_each, x * 2) end) == :ok + assert Process.get(:enum_test_each) == 6 + after + Process.delete(:enum_test_each) + end - range = 1..0 - assert Enum.any?(range) + try do + assert Enum.each(-1..-3, fn x -> Process.put(:enum_test_each, x * 2) end) == :ok + assert Process.get(:enum_test_each) == -6 + after + Process.delete(:enum_test_each) + end end - test :fetch! do + test "empty?/1" do + refute Enum.empty?(1..0) + refute Enum.empty?(1..2) + refute Enum.empty?(1..2//2) + assert Enum.empty?(1..2//-2) + end + + test "fetch/2" do + # ascending order + assert Enum.fetch(-10..20, 4) == {:ok, -6} + assert Enum.fetch(-10..20, -4) == {:ok, 17} + # ascending order, first + assert Enum.fetch(-10..20, 0) == {:ok, -10} + assert Enum.fetch(-10..20, -31) == {:ok, -10} + # ascending order, last + assert Enum.fetch(-10..20, -1) == {:ok, 20} + assert Enum.fetch(-10..20, 30) == {:ok, 20} + # ascending order, out of bound + assert Enum.fetch(-10..20, 31) == :error + assert Enum.fetch(-10..20, -32) == :error + + # descending order + assert Enum.fetch(20..-10, 4) == {:ok, 16} + assert Enum.fetch(20..-10, -4) == {:ok, -7} + # descending order, first + assert Enum.fetch(20..-10, 0) == {:ok, 20} + assert Enum.fetch(20..-10, -31) == {:ok, 20} + # descending order, last + assert Enum.fetch(20..-10, -1) == {:ok, -10} + assert Enum.fetch(20..-10, 30) == {:ok, -10} + # descending order, out of bound + assert Enum.fetch(20..-10, 31) == :error + assert Enum.fetch(20..-10, -32) == :error + + # edge cases + assert Enum.fetch(42..42, 0) == {:ok, 42} + assert Enum.fetch(42..42, -1) == {:ok, 42} + assert Enum.fetch(42..42, 2) == :error + assert Enum.fetch(42..42, -2) == :error + + assert Enum.fetch(42..42//2, 0) == {:ok, 42} + assert Enum.fetch(42..42//2, -1) == {:ok, 42} + assert Enum.fetch(42..42//2, 2) == :error + assert Enum.fetch(42..42//2, -2) == :error + end + + test "fetch!/2" do assert Enum.fetch!(2..6, 0) == 2 assert Enum.fetch!(2..6, 4) == 6 assert Enum.fetch!(2..6, -1) == 6 @@ -504,274 +1788,272 @@ defmodule EnumTest.Range do end end - test :count do - range = 1..5 - assert Enum.count(range) == 5 - range = 1..1 - assert Enum.count(range) == 1 - end + test "filter/2" do + assert Enum.filter(1..3, fn x -> rem(x, 2) == 0 end) == [2] + assert Enum.filter(1..6, fn x -> rem(x, 2) == 0 end) == [2, 4, 6] - test :count_fun do - range = 1..5 - assert Enum.count(range, fn(x) -> rem(x, 2) == 0 end) == 2 - range = 1..1 - assert Enum.count(range, fn(x) -> rem(x, 2) == 0 end) == 0 + assert Enum.filter(1..3, &match?(1, &1)) == [1] + assert Enum.filter(1..3, &match?(x when x < 3, &1)) == [1, 2] + assert Enum.filter(1..3, fn _ -> true end) == [1, 2, 3] end - test :chunk do - assert Enum.chunk(1..5, 2) == [[1, 2], [3, 4]] - assert Enum.chunk(1..5, 2, 2, [6]) == [[1, 2], [3, 4], [5, 6]] - assert Enum.chunk(1..6, 3, 2) == [[1, 2, 3], [3, 4, 5]] - assert Enum.chunk(1..6, 2, 3) == [[1, 2], [4, 5]] - assert Enum.chunk(1..6, 3, 2, []) == [[1, 2, 3], [3, 4, 5], [5, 6]] - assert Enum.chunk(1..5, 4, 4, 6..10) == [[1, 2, 3, 4], [5, 6, 7, 8]] + test "find/3" do + assert Enum.find(2..6, fn x -> rem(x, 2) == 0 end) == 2 + assert Enum.find(2..6, fn x -> rem(x, 2) == 1 end) == 3 + assert Enum.find(2..6, fn _ -> false end) == nil + assert Enum.find(2..6, 0, fn _ -> false end) == 0 end - test :chunk_by do - assert Enum.chunk_by(1..4, fn _ -> true end) == [[1, 2, 3, 4]] - assert Enum.chunk_by(1..4, &(rem(&1, 2) == 1)) == [[1], [2], [3], [4]] + test "find_index/2" do + assert Enum.find_index(2..6, fn x -> rem(x, 2) == 1 end) == 1 end - test :drop do - range = 1..3 - assert Enum.drop(range, 0) == [1, 2, 3] - assert Enum.drop(range, 1) == [2, 3] - assert Enum.drop(range, 2) == [3] - assert Enum.drop(range, 3) == [] - assert Enum.drop(range, 4) == [] - assert Enum.drop(range, -1) == [1, 2] - assert Enum.drop(range, -2) == [1] - assert Enum.drop(range, -4) == [] + test "find_value/3" do + assert Enum.find_value(2..6, fn x -> rem(x, 2) == 1 end) + end - range = 1..0 - assert Enum.drop(range, 3) == [] + test "flat_map/2" do + assert Enum.flat_map(1..3, fn x -> [x, x] end) == [1, 1, 2, 2, 3, 3] end - test :drop_while do - range = 0..6 - assert Enum.drop_while(range, fn(x) -> x <= 3 end) == [4, 5, 6] - assert Enum.drop_while(range, fn(_) -> false end) == [0, 1, 2, 3, 4, 5, 6] + test "flat_map_reduce/3" do + assert Enum.flat_map_reduce(1..100, 0, fn i, acc -> + if acc < 3, do: {[i], acc + 1}, else: {:halt, acc} + end) == {[1, 2, 3], 3} + end - range = 0..3 - assert Enum.drop_while(range, fn(x) -> x <= 3 end) == [] + test "group_by/3" do + assert Enum.group_by(1..6, &rem(&1, 3)) == %{0 => [3, 6], 1 => [1, 4], 2 => [2, 5]} - range = 1..0 - assert Enum.drop_while(range, fn(_) -> false end) == [1, 0] + assert Enum.group_by(1..6, &rem(&1, 3), &(&1 * 2)) == + %{0 => [6, 12], 1 => [2, 8], 2 => [4, 10]} end - test :find do - range = 2..6 - assert Enum.find(range, fn(x) -> rem(x, 2) == 0 end) == 2 - assert Enum.find(range, fn(x) -> rem(x, 2) == 1 end) == 3 - assert Enum.find(range, fn _ -> false end) == nil - assert Enum.find(range, 0, fn _ -> false end) == 0 + test "intersperse/2" do + assert Enum.intersperse(1..0, true) == [1, true, 0] + assert Enum.intersperse(1..3, false) == [1, false, 2, false, 3] end - test :find_value do - range = 2..6 - assert Enum.find_value(range, fn(x) -> rem(x, 2) == 1 end) + test "into/2" do + assert Enum.into(1..5, []) == [1, 2, 3, 4, 5] end - test :find_index do - range = 2..6 - assert Enum.find_index(range, fn(x) -> rem(x, 2) == 1 end) == 1 + test "into/3" do + assert Enum.into(1..5, [], fn x -> x * 2 end) == [2, 4, 6, 8, 10] + assert Enum.into(1..3, "numbers: ", &to_string/1) == "numbers: 123" end - test :empty? do - range = 1..0 - refute Enum.empty?(range) + test "join/2" do + assert Enum.join(1..0, " = ") == "1 = 0" + assert Enum.join(1..3, " = ") == "1 = 2 = 3" + assert Enum.join(1..3) == "123" + end - range = 1..2 - refute Enum.empty?(range) + test "map/2" do + assert Enum.map(1..3, fn x -> x * 2 end) == [2, 4, 6] + assert Enum.map(-1..-3, fn x -> x * 2 end) == [-2, -4, -6] end - test :each do - try do - range = 1..0 - assert Enum.each(range, fn(x) -> x end) == :ok + test "map_every/3" do + assert Enum.map_every(1..10, 2, fn x -> x * 2 end) == [2, 2, 6, 4, 10, 6, 14, 8, 18, 10] - range = 1..3 - assert Enum.each(range, fn(x) -> Process.put(:enum_test_each, x * 2) end) == :ok - assert Process.get(:enum_test_each) == 6 - after - Process.delete(:enum_test_each) - end + assert Enum.map_every(-1..-10, 2, fn x -> x * 2 end) == + [-2, -2, -6, -4, -10, -6, -14, -8, -18, -10] - try do - range = -1..-3 - assert Enum.each(range, fn(x) -> Process.put(:enum_test_each, x * 2) end) == :ok - assert Process.get(:enum_test_each) == -6 - after - Process.delete(:enum_test_each) + assert Enum.map_every(1..2, 2, fn x -> x * 2 end) == [2, 2] + assert Enum.map_every(1..3, 0, fn x -> x * 2 end) == [1, 2, 3] + + assert_raise FunctionClauseError, fn -> + Enum.map_every(1..3, -1, fn x -> x * 2 end) end end - test :filter do - range = 1..3 - assert Enum.filter(range, fn(x) -> rem(x, 2) == 0 end) == [2] + test "map_intersperse/3" do + assert Enum.map_intersperse(1..1, :a, &(&1 * 2)) == [2] + assert Enum.map_intersperse(1..3, :a, &(&1 * 2)) == [2, :a, 4, :a, 6] + end - range = 1..6 - assert Enum.filter(range, fn(x) -> rem(x, 2) == 0 end) == [2, 4, 6] + test "map_join/3" do + assert Enum.map_join(1..0, " = ", &(&1 * 2)) == "2 = 0" + assert Enum.map_join(1..3, " = ", &(&1 * 2)) == "2 = 4 = 6" + assert Enum.map_join(1..3, &(&1 * 2)) == "246" end - test :filter_with_match do - range = 1..3 - assert Enum.filter(range, &match?(1, &1)) == [1] - assert Enum.filter(range, &match?(x when x < 3, &1)) == [1, 2] - assert Enum.filter(range, &match?(_, &1)) == [1, 2, 3] + test "map_reduce/3" do + assert Enum.map_reduce(1..0, 1, fn x, acc -> {x * 2, x + acc} end) == {[2, 0], 2} + assert Enum.map_reduce(1..3, 1, fn x, acc -> {x * 2, x + acc} end) == {[2, 4, 6], 7} end - test :filter_map do - range = 1..3 - assert Enum.filter_map(range, fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) == [4] + test "max/1" do + assert Enum.max(1..1) == 1 + assert Enum.max(1..3) == 3 + assert Enum.max(3..1) == 3 - range = 2..6 - assert Enum.filter_map(range, fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) == [4, 8, 12] - end + assert Enum.max(1..9//2) == 9 + assert Enum.max(1..10//2) == 9 + assert Enum.max(-1..-9//-2) == -1 - test :flat_map do - range = 1..3 - assert Enum.flat_map(range, fn(x) -> [x, x] end) == [1, 1, 2, 2, 3, 3] + assert_raise Enum.EmptyError, fn -> Enum.max(1..0//1) end end - test :intersperse do - range = 1..0 - assert Enum.intersperse(range, true) == [1, true, 0] + test "max_by/2" do + assert Enum.max_by(1..1, fn x -> :math.pow(-2, x) end) == 1 + assert Enum.max_by(1..3, fn x -> :math.pow(-2, x) end) == 2 - range = 1..3 - assert Enum.intersperse(range, false) == [1, false, 2, false, 3] + assert Enum.max_by(1..8//3, fn x -> :math.pow(-2, x) end) == 4 + assert_raise Enum.EmptyError, fn -> Enum.max_by(1..0//1, & &1) end end - test :into do - assert Enum.into([a: 1, b: 2], %{}) == %{a: 1, b: 2} - assert Enum.into(%{a: 1, b: 2}, []) == [a: 1, b: 2] - assert Enum.into(3..5, [1, 2]) == [1, 2, 3, 4, 5] - assert Enum.into(1..5, []) == [1, 2, 3, 4, 5] - assert Enum.into(1..5, [], fn x -> x * 2 end) == [2, 4, 6, 8, 10] - assert Enum.into(1..3, "numbers: ", &to_string/1) == "numbers: 123" - end + test "member?/2" do + assert Enum.member?(1..3, 2) + refute Enum.member?(1..3, 0) + + assert Enum.member?(1..9//2, 1) + assert Enum.member?(1..9//2, 9) + refute Enum.member?(1..9//2, 10) + refute Enum.member?(1..10//2, 10) + assert Enum.member?(1..2//2, 1) + refute Enum.member?(1..2//2, 2) - test :join do - range = 1..0 - assert Enum.join(range, " = ") == "1 = 0" + assert Enum.member?(-1..-9//-2, -1) + assert Enum.member?(-1..-9//-2, -9) + refute Enum.member?(-1..-9//-2, -8) - range = 1..3 - assert Enum.join(range, " = ") == "1 = 2 = 3" - assert Enum.join(range) == "123" + refute Enum.member?(1..0//1, 1) + refute Enum.member?(0..1//-1, 1) end - test :map_join do - range = 1..0 - assert Enum.map_join(range, " = ", &(&1 * 2)) == "2 = 0" + test "min/1" do + assert Enum.min(1..1) == 1 + assert Enum.min(1..3) == 1 - range = 1..3 - assert Enum.map_join(range, " = ", &(&1 * 2)) == "2 = 4 = 6" - assert Enum.map_join(range, &(&1 * 2)) == "246" + assert Enum.min(1..9//2) == 1 + assert Enum.min(1..10//2) == 1 + assert Enum.min(-1..-9//-2) == -9 + + assert_raise Enum.EmptyError, fn -> Enum.min(1..0//1) end end - test :map do - range = 1..3 - assert Enum.map(range, fn x -> x * 2 end) == [2, 4, 6] + test "min_by/2" do + assert Enum.min_by(1..1, fn x -> :math.pow(-2, x) end) == 1 + assert Enum.min_by(1..3, fn x -> :math.pow(-2, x) end) == 3 - range = -1..-3 - assert Enum.map(range, fn x -> x * 2 end) == [-2, -4, -6] + assert Enum.min_by(1..8//3, fn x -> :math.pow(-2, x) end) == 7 + assert_raise Enum.EmptyError, fn -> Enum.min_by(1..0//1, & &1) end end - test :map_reduce do - range = 1..0 - assert Enum.map_reduce(range, 1, fn(x, acc) -> {x * 2, x + acc} end) == {[2, 0], 2} + test "min_max/1" do + assert Enum.min_max(1..1) == {1, 1} + assert Enum.min_max(1..3) == {1, 3} + assert Enum.min_max(3..1) == {1, 3} - range = 1..3 - assert Enum.map_reduce(range, 1, fn(x, acc) -> {x * 2, x + acc} end) == {[2, 4, 6], 7} - end + assert Enum.min_max(1..9//2) == {1, 9} + assert Enum.min_max(1..10//2) == {1, 9} + assert Enum.min_max(-1..-9//-2) == {-9, -1} - test :max do - assert Enum.max(1..1) == 1 - assert Enum.max(1..3) == 3 - assert Enum.max(3..1) == 3 + assert_raise Enum.EmptyError, fn -> Enum.min_max(1..0//1) end end - test :max_by do - assert Enum.max_by(1..1, fn(x) -> :math.pow(-2, x) end) == 1 - assert Enum.max_by(1..3, fn(x) -> :math.pow(-2, x) end) == 2 - end + test "min_max_by/2" do + assert Enum.min_max_by(1..1, fn x -> x end) == {1, 1} + assert Enum.min_max_by(1..3, fn x -> x end) == {1, 3} - test :min do - assert Enum.min([1]) == 1 - assert Enum.min([1, 2, 3]) == 1 - assert Enum.min([[], :a, {}]) == :a + assert Enum.min_max_by(1..8//3, fn x -> :math.pow(-2, x) end) == {7, 4} + assert_raise Enum.EmptyError, fn -> Enum.min_max_by(1..0//1, & &1) end end - test :min_by do - assert Enum.min_by(1..1, fn(x) -> :math.pow(-2, x) end) == 1 - assert Enum.min_by(1..3, fn(x) -> :math.pow(-2, x) end) == 3 + test "split_with/2" do + assert Enum.split_with(1..3, fn x -> rem(x, 2) == 0 end) == {[2], [1, 3]} end - test :partition do - range = 1..3 - assert Enum.partition(range, fn(x) -> rem(x, 2) == 0 end) == {[2], [1, 3]} - end + test "random/1" do + # corner cases, independent of the seed + assert Enum.random(1..1) == 1 + + # set a fixed seed so the test can be deterministic + # please note the order of following assertions is important + seed1 = {1406, 407_414, 139_258} + seed2 = {1306, 421_106, 567_597} + :rand.seed(:exsss, seed1) + assert Enum.random(1..2) == 1 + assert Enum.random(1..3) == 1 + assert Enum.random(3..1) == 2 - test :reduce do - range = 1..0 - assert Enum.reduce(range, 1, fn(x, acc) -> x + acc end) == 2 + :rand.seed(:exsss, seed2) + assert Enum.random(1..2) == 1 + assert Enum.random(1..3) == 2 - range = 1..3 - assert Enum.reduce(range, 1, fn(x, acc) -> x + acc end) == 7 + assert Enum.random(1..10//2) == 7 + assert Enum.random(1..10//2) == 5 + end - range = 1..3 - assert Enum.reduce(range, fn(x, acc) -> x + acc end) == 6 + test "reduce/2" do + assert Enum.reduce(1..3, fn x, acc -> x + acc end) == 6 + assert Enum.reduce(1..10//2, fn x, acc -> x + acc end) == 25 + assert_raise Enum.EmptyError, fn -> Enum.reduce(0..1//-1, &+/2) end end - test :reject do - range = 1..3 - assert Enum.reject(range, fn(x) -> rem(x, 2) == 0 end) == [1, 3] + test "reduce/3" do + assert Enum.reduce(1..0, 1, fn x, acc -> x + acc end) == 2 + assert Enum.reduce(1..3, 1, fn x, acc -> x + acc end) == 7 + assert Enum.reduce(1..10//2, 1, fn x, acc -> x + acc end) == 26 + assert Enum.reduce(0..1//-1, 1, fn x, acc -> x + acc end) == 1 + end - range = 1..6 - assert Enum.reject(range, fn(x) -> rem(x, 2) == 0 end) == [1, 3, 5] + test "reduce_while/3" do + assert Enum.reduce_while(1..100, 0, fn i, acc -> + if i <= 3, do: {:cont, acc + i}, else: {:halt, acc} + end) == 6 end - test :reverse do - assert Enum.reverse([]) == [] - assert Enum.reverse([1, 2, 3]) == [3, 2, 1] - assert Enum.reverse([1, 2, 3], [4, 5, 6]) == [3, 2, 1, 4, 5, 6] + test "reject/2" do + assert Enum.reject(1..3, fn x -> rem(x, 2) == 0 end) == [1, 3] + assert Enum.reject(1..6, fn x -> rem(x, 2) == 0 end) == [1, 3, 5] + end + test "reverse/1" do assert Enum.reverse(0..0) == [0] assert Enum.reverse(1..3) == [3, 2, 1] + assert Enum.reverse(-3..5) == [5, 4, 3, 2, 1, 0, -1, -2, -3] + assert Enum.reverse(5..5) == [5] + + assert Enum.reverse(0..1//-1) == [] + assert Enum.reverse(1..10//2) == [9, 7, 5, 3, 1] + end + + test "reverse/2" do assert Enum.reverse(1..3, 4..6) == [3, 2, 1, 4, 5, 6] assert Enum.reverse([1, 2, 3], 4..6) == [3, 2, 1, 4, 5, 6] assert Enum.reverse(1..3, [4, 5, 6]) == [3, 2, 1, 4, 5, 6] + assert Enum.reverse(-3..5, MapSet.new([-3, -2])) == [5, 4, 3, 2, 1, 0, -1, -2, -3, -3, -2] + assert Enum.reverse(5..5, [5]) == [5, 5] end - test :scan do - assert Enum.scan(1..5, &(&1 + &2)) == [1,3,6,10,15] - assert Enum.scan(1..5, 0, &(&1 + &2)) == [1,3,6,10,15] + test "reverse_slice/3" do + assert Enum.reverse_slice(1..6, 2, 0) == [1, 2, 3, 4, 5, 6] + assert Enum.reverse_slice(1..6, 2, 2) == [1, 2, 4, 3, 5, 6] + assert Enum.reverse_slice(1..6, 2, 4) == [1, 2, 6, 5, 4, 3] + assert Enum.reverse_slice(1..6, 2, 10_000_000) == [1, 2, 6, 5, 4, 3] + assert Enum.reverse_slice(1..6, 10_000_000, 4) == [1, 2, 3, 4, 5, 6] + assert Enum.reverse_slice(1..6, 50, 50) == [1, 2, 3, 4, 5, 6] end - test :shuffle do - # set a fixed seed so the test can be deterministic - :random.seed(1374, 347975, 449264) - assert Enum.shuffle(1..5) == [2, 4, 1, 5, 3] + test "scan/2" do + assert Enum.scan(1..5, &(&1 + &2)) == [1, 3, 6, 10, 15] end - test :slice do - assert Enum.slice(1..5, 0, 0) == [] - assert Enum.slice(1..5, 0, 1) == [1] - assert Enum.slice(1..5, 0, 2) == [1, 2] - assert Enum.slice(1..5, 1, 2) == [2, 3] - assert Enum.slice(1..5, 1, 0) == [] - assert Enum.slice(1..5, 2, 5) == [3, 4, 5] - assert Enum.slice(1..5, 2, 6) == [3, 4, 5] - assert Enum.slice(1..5, 5, 5) == [] - assert Enum.slice(1..5, 6, 5) == [] - assert Enum.slice(1..5, 6, 0) == [] - assert Enum.slice(1..5, -6, 0) == [] - assert Enum.slice(1..5, -6, 5) == [] - assert Enum.slice(1..5, -2, 5) == [4, 5] - assert Enum.slice(1..5, -3, 1) == [3] + test "scan/3" do + assert Enum.scan(1..5, 0, &(&1 + &2)) == [1, 3, 6, 10, 15] + end + + test "shuffle/1" do + # set a fixed seed so the test can be deterministic + :rand.seed(:exsss, {1374, 347_975, 449_264}) + assert Enum.shuffle(1..5) == [1, 3, 4, 5, 2] + assert Enum.shuffle(1..10//2) == [3, 9, 7, 1, 5] end - test :slice_range do + test "slice/2" do assert Enum.slice(1..5, 0..0) == [1] assert Enum.slice(1..5, 0..1) == [1, 2] assert Enum.slice(1..5, 0..2) == [1, 2, 3] @@ -783,92 +2065,293 @@ defmodule EnumTest.Range do assert Enum.slice(1..5, 5..5) == [] assert Enum.slice(1..5, 6..5) == [] assert Enum.slice(1..5, 6..0) == [] - assert Enum.slice(1..5, -6..0) == [] - assert Enum.slice(1..5, -6..5) == [] + assert Enum.slice(1..5, -3..0) == [] + assert Enum.slice(1..5, -3..1) == [] + assert Enum.slice(1..5, -6..0) == [1] + assert Enum.slice(1..5, -6..5) == [1, 2, 3, 4, 5] + assert Enum.slice(1..5, -6..-1) == [1, 2, 3, 4, 5] assert Enum.slice(1..5, -5..-1) == [1, 2, 3, 4, 5] assert Enum.slice(1..5, -5..-3) == [1, 2, 3] - assert Enum.slice(1..5, -6..-1) == [] - assert Enum.slice(1..5, -6..-3) == [] - end - test :sort do + assert Enum.slice(1..5, 0..10//2) == [1, 3, 5] + assert Enum.slice(1..5, 0..10//3) == [1, 4] + assert Enum.slice(1..5, 0..10//4) == [1, 5] + assert Enum.slice(1..5, 0..10//5) == [1] + assert Enum.slice(1..5, 0..10//6) == [1] + + assert Enum.slice(1..5, 0..2//2) == [1, 3] + assert Enum.slice(1..5, 0..2//3) == [1] + + assert Enum.slice(1..5, 0..-1//2) == [1, 3, 5] + assert Enum.slice(1..5, 0..-1//3) == [1, 4] + assert Enum.slice(1..5, 0..-1//4) == [1, 5] + assert Enum.slice(1..5, 0..-1//5) == [1] + assert Enum.slice(1..5, 0..-1//6) == [1] + + assert Enum.slice(1..5, 1..-1//2) == [2, 4] + assert Enum.slice(1..5, 1..-1//3) == [2, 5] + assert Enum.slice(1..5, 1..-1//4) == [2] + assert Enum.slice(1..5, 1..-1//5) == [2] + + assert Enum.slice(1..5, -4..-1//2) == [2, 4] + assert Enum.slice(1..5, -4..-1//3) == [2, 5] + assert Enum.slice(1..5, -4..-1//4) == [2] + assert Enum.slice(1..5, -4..-1//5) == [2] + + assert Enum.slice(5..1, 0..0) == [5] + assert Enum.slice(5..1, 0..1) == [5, 4] + assert Enum.slice(5..1, 0..2) == [5, 4, 3] + assert Enum.slice(5..1, 1..2) == [4, 3] + assert Enum.slice(5..1, 1..0) == [] + assert Enum.slice(5..1, 2..5) == [3, 2, 1] + assert Enum.slice(5..1, 2..6) == [3, 2, 1] + assert Enum.slice(5..1, 4..4) == [1] + assert Enum.slice(5..1, 5..5) == [] + assert Enum.slice(5..1, 6..5) == [] + assert Enum.slice(5..1, 6..0) == [] + assert Enum.slice(5..1, -6..0) == [5] + assert Enum.slice(5..1, -6..5) == [5, 4, 3, 2, 1] + assert Enum.slice(5..1, -6..-1) == [5, 4, 3, 2, 1] + assert Enum.slice(5..1, -5..-1) == [5, 4, 3, 2, 1] + assert Enum.slice(5..1, -5..-3) == [5, 4, 3] + + assert Enum.slice(1..10//2, 0..0) == [1] + assert Enum.slice(1..10//2, 0..1) == [1, 3] + assert Enum.slice(1..10//2, 0..2) == [1, 3, 5] + assert Enum.slice(1..10//2, 1..2) == [3, 5] + assert Enum.slice(1..10//2, 1..0) == [] + assert Enum.slice(1..10//2, 2..5) == [5, 7, 9] + assert Enum.slice(1..10//2, 2..6) == [5, 7, 9] + assert Enum.slice(1..10//2, 4..4) == [9] + assert Enum.slice(1..10//2, 5..5) == [] + assert Enum.slice(1..10//2, 6..5) == [] + assert Enum.slice(1..10//2, 6..0) == [] + assert Enum.slice(1..10//2, -3..0) == [] + assert Enum.slice(1..10//2, -3..1) == [] + assert Enum.slice(1..10//2, -6..0) == [1] + assert Enum.slice(1..10//2, -6..5) == [1, 3, 5, 7, 9] + assert Enum.slice(1..10//2, -6..-1) == [1, 3, 5, 7, 9] + assert Enum.slice(1..10//2, -5..-1) == [1, 3, 5, 7, 9] + assert Enum.slice(1..10//2, -5..-3) == [1, 3, 5] + + assert_raise ArgumentError, + "Enum.slice/2 does not accept ranges with negative steps, got: 1..3//-2", + fn -> Enum.slice(1..5, 1..3//-2) end + end + + test "slice/3" do + assert Enum.slice(1..5, 0, 0) == [] + assert Enum.slice(1..5, 0, 1) == [1] + assert Enum.slice(1..5, 0, 2) == [1, 2] + assert Enum.slice(1..5, 1, 2) == [2, 3] + assert Enum.slice(1..5, 1, 0) == [] + assert Enum.slice(1..5, 2, 3) == [3, 4, 5] + assert Enum.slice(1..5, 2, 6) == [3, 4, 5] + assert Enum.slice(1..5, 5, 5) == [] + assert Enum.slice(1..5, 6, 5) == [] + assert Enum.slice(1..5, 6, 0) == [] + assert Enum.slice(1..5, -6, 0) == [] + assert Enum.slice(1..5, -6, 5) == [1, 2, 3, 4, 5] + assert Enum.slice(1..5, -2, 5) == [4, 5] + assert Enum.slice(1..5, -3, 1) == [3] + + assert_raise FunctionClauseError, fn -> + Enum.slice(1..5, 0, -1) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(1..5, 0.99, 0) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(1..5, 0, 0.99) + end + + assert Enum.slice(5..1, 0, 0) == [] + assert Enum.slice(5..1, 0, 1) == [5] + assert Enum.slice(5..1, 0, 2) == [5, 4] + assert Enum.slice(5..1, 1, 2) == [4, 3] + assert Enum.slice(5..1, 1, 0) == [] + assert Enum.slice(5..1, 2, 3) == [3, 2, 1] + assert Enum.slice(5..1, 2, 6) == [3, 2, 1] + assert Enum.slice(5..1, 4, 4) == [1] + assert Enum.slice(5..1, 5, 5) == [] + assert Enum.slice(5..1, 6, 5) == [] + assert Enum.slice(5..1, 6, 0) == [] + assert Enum.slice(5..1, -6, 0) == [] + assert Enum.slice(5..1, -6, 5) == [5, 4, 3, 2, 1] + + assert Enum.slice(1..10//2, 0, 0) == [] + assert Enum.slice(1..10//2, 0, 1) == [1] + assert Enum.slice(1..10//2, 0, 2) == [1, 3] + assert Enum.slice(1..10//2, 1, 2) == [3, 5] + assert Enum.slice(1..10//2, 1, 0) == [] + assert Enum.slice(1..10//2, 2, 3) == [5, 7, 9] + assert Enum.slice(1..10//2, 2, 6) == [5, 7, 9] + assert Enum.slice(1..10//2, 5, 5) == [] + assert Enum.slice(1..10//2, 6, 5) == [] + assert Enum.slice(1..10//2, 6, 0) == [] + assert Enum.slice(1..10//2, -6, 0) == [] + assert Enum.slice(1..10//2, -6, 5) == [1, 3, 5, 7, 9] + assert Enum.slice(1..10//2, -2, 5) == [7, 9] + assert Enum.slice(1..10//2, -3, 1) == [5] + end + + test "sort/1" do assert Enum.sort(3..1) == [1, 2, 3] assert Enum.sort(2..1) == [1, 2] assert Enum.sort(1..1) == [1] + end + test "sort/2" do assert Enum.sort(3..1, &(&1 > &2)) == [3, 2, 1] assert Enum.sort(2..1, &(&1 > &2)) == [2, 1] assert Enum.sort(1..1, &(&1 > &2)) == [1] - end - test :split do - range = 1..3 - assert Enum.split(range, 0) == {[], [1, 2, 3]} - assert Enum.split(range, 1) == {[1], [2, 3]} - assert Enum.split(range, 2) == {[1, 2], [3]} - assert Enum.split(range, 3) == {[1, 2, 3], []} - assert Enum.split(range, 4) == {[1, 2, 3], []} - assert Enum.split(range, -1) == {[1, 2], [3]} - assert Enum.split(range, -2) == {[1], [2, 3]} - assert Enum.split(range, -3) == {[], [1, 2, 3]} - assert Enum.split(range, -10) == {[], [1, 2, 3]} + assert Enum.sort(3..1, :asc) == [1, 2, 3] + assert Enum.sort(3..1, :desc) == [3, 2, 1] + end - range = 1..0 - assert Enum.split(range, 3) == {[1, 0], []} + test "sort_by/2" do + assert Enum.sort_by(3..1, & &1) == [1, 2, 3] + assert Enum.sort_by(3..1, & &1, :asc) == [1, 2, 3] + assert Enum.sort_by(3..1, & &1, :desc) == [3, 2, 1] end - test :split_while do - range = 1..3 - assert Enum.split_while(range, fn(_) -> false end) == {[], [1, 2, 3]} - assert Enum.split_while(range, fn(_) -> true end) == {[1, 2, 3], []} - assert Enum.split_while(range, fn(x) -> x > 2 end) == {[], [1, 2, 3]} - assert Enum.split_while(range, fn(x) -> x > 3 end) == {[], [1, 2, 3]} - assert Enum.split_while(range, fn(x) -> x < 3 end) == {[1, 2], [3]} + test "split/2" do + assert Enum.split(1..3, 0) == {[], [1, 2, 3]} + assert Enum.split(1..3, 1) == {[1], [2, 3]} + assert Enum.split(1..3, 2) == {[1, 2], [3]} + assert Enum.split(1..3, 3) == {[1, 2, 3], []} + assert Enum.split(1..3, 4) == {[1, 2, 3], []} + assert Enum.split(1..3, -1) == {[1, 2], [3]} + assert Enum.split(1..3, -2) == {[1], [2, 3]} + assert Enum.split(1..3, -3) == {[], [1, 2, 3]} + assert Enum.split(1..3, -10) == {[], [1, 2, 3]} + assert Enum.split(1..0, 3) == {[1, 0], []} + end - range = 1..0 - assert Enum.split_while(range, fn(_) -> true end) == {[1, 0], []} + test "split_while/2" do + assert Enum.split_while(1..3, fn _ -> false end) == {[], [1, 2, 3]} + assert Enum.split_while(1..3, fn _ -> nil end) == {[], [1, 2, 3]} + assert Enum.split_while(1..3, fn _ -> true end) == {[1, 2, 3], []} + assert Enum.split_while(1..3, fn x -> x > 2 end) == {[], [1, 2, 3]} + assert Enum.split_while(1..3, fn x -> x > 3 end) == {[], [1, 2, 3]} + assert Enum.split_while(1..3, fn x -> x < 3 end) == {[1, 2], [3]} + assert Enum.split_while(1..3, fn x -> x end) == {[1, 2, 3], []} + assert Enum.split_while(1..0, fn _ -> true end) == {[1, 0], []} end - test :sum do + test "sum/1" do + assert Enum.sum(0..0) == 0 assert Enum.sum(1..1) == 1 assert Enum.sum(1..3) == 6 + assert Enum.sum(0..100) == 5050 + assert Enum.sum(10..100) == 5005 + assert Enum.sum(100..10) == 5005 + assert Enum.sum(-10..-20) == -165 + assert Enum.sum(-10..2) == -52 + + assert Enum.sum(0..1//-1) == 0 + assert Enum.sum(1..9//2) == 25 + assert Enum.sum(1..10//2) == 25 + assert Enum.sum(9..1//-2) == 25 + end + + test "take/2" do + assert Enum.take(1..3, 0) == [] + assert Enum.take(1..3, 1) == [1] + assert Enum.take(1..3, 2) == [1, 2] + assert Enum.take(1..3, 3) == [1, 2, 3] + assert Enum.take(1..3, 4) == [1, 2, 3] + assert Enum.take(1..3, -1) == [3] + assert Enum.take(1..3, -2) == [2, 3] + assert Enum.take(1..3, -4) == [1, 2, 3] + assert Enum.take(1..0, 3) == [1, 0] + end + + test "take_every/2" do + assert Enum.take_every(1..10, 2) == [1, 3, 5, 7, 9] + assert Enum.take_every(1..2, 2) == [1] + assert Enum.take_every(1..3, 0) == [] + + assert_raise FunctionClauseError, fn -> + Enum.take_every(1..3, -1) + end end - test :take do - range = 1..3 - assert Enum.take(range, 0) == [] - assert Enum.take(range, 1) == [1] - assert Enum.take(range, 2) == [1, 2] - assert Enum.take(range, 3) == [1, 2, 3] - assert Enum.take(range, 4) == [1, 2, 3] - assert Enum.take(range, -1) == [3] - assert Enum.take(range, -2) == [2, 3] - assert Enum.take(range, -4) == [1, 2, 3] + test "take_random/2" do + # corner cases, independent of the seed + assert_raise FunctionClauseError, fn -> Enum.take_random(1..2, -1) end + assert Enum.take_random(1..1, 0) == [] + assert Enum.take_random(1..1, 1) == [1] + assert Enum.take_random(1..1, 2) == [1] + assert Enum.take_random(1..2, 0) == [] - range = 1..0 - assert Enum.take(range, 3) == [1, 0] + # set a fixed seed so the test can be deterministic + # please note the order of following assertions is important + seed1 = {1406, 407_414, 139_258} + seed2 = {1406, 421_106, 567_597} + :rand.seed(:exsss, seed1) + assert Enum.take_random(1..3, 1) == [3] + :rand.seed(:exsss, seed1) + assert Enum.take_random(1..3, 2) == [3, 1] + :rand.seed(:exsss, seed1) + assert Enum.take_random(1..3, 3) == [3, 1, 2] + :rand.seed(:exsss, seed1) + assert Enum.take_random(1..3, 4) == [3, 1, 2] + :rand.seed(:exsss, seed1) + assert Enum.take_random(3..1, 1) == [1] + :rand.seed(:exsss, seed2) + assert Enum.take_random(1..3, 1) == [1] + :rand.seed(:exsss, seed2) + assert Enum.take_random(1..3, 2) == [1, 3] + :rand.seed(:exsss, seed2) + assert Enum.take_random(1..3, 3) == [1, 3, 2] + :rand.seed(:exsss, seed2) + assert Enum.take_random(1..3, 4) == [1, 3, 2] + + # make sure optimizations don't change fixed seeded tests + :rand.seed(:exsss, {101, 102, 103}) + one = Enum.take_random(1..100, 1) + :rand.seed(:exsss, {101, 102, 103}) + two = Enum.take_random(1..100, 2) + assert hd(one) == hd(two) + end + + test "take_while/2" do + assert Enum.take_while(1..3, fn x -> x > 3 end) == [] + assert Enum.take_while(1..3, fn x -> x <= 1 end) == [1] + assert Enum.take_while(1..3, fn x -> x <= 3 end) == [1, 2, 3] + assert Enum.take_while(1..3, fn x -> x end) == [1, 2, 3] + assert Enum.take_while(1..3, fn _ -> nil end) == [] + end + + test "to_list/1" do + assert Enum.to_list(1..3) == [1, 2, 3] + assert Enum.to_list(1..3//2) == [1, 3] + assert Enum.to_list(3..1//-2) == [3, 1] + assert Enum.to_list(0..1//-1) == [] + end + + test "uniq/1" do + assert Enum.uniq(1..3) == [1, 2, 3] end - test :take_every do - assert Enum.take_every(1..10, 2) == [1, 3, 5, 7, 9] - assert Enum.take_every(1..2, 2) == [1] - assert Enum.take_every(1..3, 0) == [] + test "uniq_by/2" do + assert Enum.uniq_by(1..3, fn x -> x end) == [1, 2, 3] end - test :take_while do - range = 1..3 - assert Enum.take_while(range, fn(x) -> x > 3 end) == [] - assert Enum.take_while(range, fn(x) -> x <= 1 end) == [1] - assert Enum.take_while(range, fn(x) -> x <= 3 end) == [1, 2, 3] - assert Enum.take_while([], fn(_) -> true end) == [] + test "unzip/1" do + assert_raise FunctionClauseError, fn -> Enum.unzip(1..3) end end - test :uniq do - assert Enum.uniq(1..3) == [1, 2, 3] - assert Enum.uniq(1..3, fn x -> x end) == [1, 2, 3] + test "with_index/2" do + assert Enum.with_index(1..3) == [{1, 0}, {2, 1}, {3, 2}] + assert Enum.with_index(1..3, 3) == [{1, 3}, {2, 4}, {3, 5}] end - test :zip do + test "zip/2" do assert Enum.zip([:a, :b], 1..2) == [{:a, 1}, {:b, 2}] assert Enum.zip([:a, :b], 1..4) == [{:a, 1}, {:b, 2}] assert Enum.zip([:a, :b, :c, :d], 1..2) == [{:a, 1}, {:b, 2}] @@ -880,10 +2363,155 @@ defmodule EnumTest.Range do assert Enum.zip(1..2, 1..2) == [{1, 1}, {2, 2}] assert Enum.zip(1..4, 1..2) == [{1, 1}, {2, 2}] assert Enum.zip(1..2, 1..4) == [{1, 1}, {2, 2}] + + assert Enum.zip(1..10//2, 1..10//3) == [{1, 1}, {3, 4}, {5, 7}, {7, 10}] + assert Enum.zip(0..1//-1, 1..10//3) == [] + end +end + +defmodule EnumTest.Map do + # Maps use different protocols path than lists and ranges in the cases below. + use ExUnit.Case, async: true + + test "random/1" do + map = %{a: 1, b: 2, c: 3} + seed1 = {1406, 407_414, 139_258} + seed2 = {1406, 421_106, 567_597} + :rand.seed(:exsss, seed1) + assert Enum.random(map) == {:c, 3} + assert Enum.random(map) == {:a, 1} + assert Enum.random(map) == {:b, 2} + + :rand.seed(:exsss, seed2) + assert Enum.random(map) == {:c, 3} + assert Enum.random(map) == {:b, 2} end - test :with_index do - assert Enum.with_index(1..3) == [{1,0},{2,1},{3,2}] + test "take_random/2" do + # corner cases, independent of the seed + assert_raise FunctionClauseError, fn -> Enum.take_random(1..2, -1) end + assert Enum.take_random(%{a: 1}, 0) == [] + assert Enum.take_random(%{a: 1}, 2) == [a: 1] + assert Enum.take_random(%{a: 1, b: 2}, 0) == [] + + # set a fixed seed so the test can be deterministic + # please note the order of following assertions is important + map = %{a: 1, b: 2, c: 3} + seed1 = {1406, 407_414, 139_258} + seed2 = {1406, 421_106, 567_597} + :rand.seed(:exsss, seed1) + assert Enum.take_random(map, 1) == [c: 3] + :rand.seed(:exsss, seed1) + assert Enum.take_random(map, 2) == [c: 3, a: 1] + :rand.seed(:exsss, seed1) + assert Enum.take_random(map, 3) == [c: 3, a: 1, b: 2] + :rand.seed(:exsss, seed1) + assert Enum.take_random(map, 4) == [c: 3, a: 1, b: 2] + :rand.seed(:exsss, seed2) + assert Enum.take_random(map, 1) == [a: 1] + :rand.seed(:exsss, seed2) + assert Enum.take_random(map, 2) == [a: 1, c: 3] + :rand.seed(:exsss, seed2) + assert Enum.take_random(map, 3) == [a: 1, c: 3, b: 2] + :rand.seed(:exsss, seed2) + assert Enum.take_random(map, 4) == [a: 1, c: 3, b: 2] + end + + test "reverse/1" do + assert Enum.reverse(%{}) == [] + assert Enum.reverse(MapSet.new()) == [] + assert Enum.reverse(%{a: 1, b: 2, c: 3}) == [c: 3, b: 2, a: 1] + end + + test "reverse/2" do + assert Enum.reverse([a: 1, b: 2, c: 3, a: 1], %{x: 1, y: 2, z: 3}) == + [a: 1, c: 3, b: 2, a: 1, x: 1, y: 2, z: 3] + + assert Enum.reverse([], %{a: 1}) == [a: 1] + assert Enum.reverse([], %{}) == [] + assert Enum.reverse(%{a: 1}, []) == [a: 1] + assert Enum.reverse(MapSet.new(), %{}) == [] + end + + test "fetch/2" do + map = %{a: 1, b: 2, c: 3, d: 4, e: 5} + assert Enum.fetch(map, 0) == {:ok, {:a, 1}} + assert Enum.fetch(map, -2) == {:ok, {:d, 4}} + assert Enum.fetch(map, -6) == :error + assert Enum.fetch(map, 5) == :error + assert Enum.fetch(%{}, 0) == :error + + assert Stream.take(map, 3) |> Enum.fetch(3) == :error + assert Stream.take(map, 5) |> Enum.fetch(4) == {:ok, {:e, 5}} + end + + test "map_intersperse/3" do + assert Enum.map_intersperse(%{}, :a, & &1) == [] + assert Enum.map_intersperse(%{foo: :bar}, :a, & &1) == [{:foo, :bar}] + + assert Enum.map_intersperse(%{foo: :bar, baz: :bat}, :a, & &1) == + [{:baz, :bat}, :a, {:foo, :bar}] + end + + test "slice/2" do + map = %{a: 1, b: 2, c: 3, d: 4, e: 5} + assert Enum.slice(map, 0..0) == [a: 1] + assert Enum.slice(map, 0..1) == [a: 1, b: 2] + assert Enum.slice(map, 0..2) == [a: 1, b: 2, c: 3] + end + + test "slice/3" do + map = %{a: 1, b: 2, c: 3, d: 4, e: 5} + assert Enum.slice(map, 1, 2) == [b: 2, c: 3] + assert Enum.slice(map, 1, 0) == [] + assert Enum.slice(map, 2, 5) == [c: 3, d: 4, e: 5] + assert Enum.slice(map, 2, 6) == [c: 3, d: 4, e: 5] + assert Enum.slice(map, 5, 5) == [] + assert Enum.slice(map, 6, 5) == [] + assert Enum.slice(map, 6, 0) == [] + assert Enum.slice(map, -6, 0) == [] + assert Enum.slice(map, -6, 5) == [a: 1, b: 2, c: 3, d: 4, e: 5] + assert Enum.slice(map, -2, 5) == [d: 4, e: 5] + assert Enum.slice(map, -3, 1) == [c: 3] + + assert_raise FunctionClauseError, fn -> + Enum.slice(map, 0, -1) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(map, 0.99, 0) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(map, 0, 0.99) + end + + assert Enum.slice(map, 0, 0) == [] + assert Enum.slice(map, 0, 1) == [a: 1] + assert Enum.slice(map, 0, 2) == [a: 1, b: 2] + assert Enum.slice(map, 1, 2) == [b: 2, c: 3] + assert Enum.slice(map, 1, 0) == [] + assert Enum.slice(map, 2, 5) == [c: 3, d: 4, e: 5] + assert Enum.slice(map, 2, 6) == [c: 3, d: 4, e: 5] + assert Enum.slice(map, 5, 5) == [] + assert Enum.slice(map, 6, 5) == [] + assert Enum.slice(map, 6, 0) == [] + assert Enum.slice(map, -6, 0) == [] + assert Enum.slice(map, -6, 5) == [a: 1, b: 2, c: 3, d: 4, e: 5] + assert Enum.slice(map, -2, 5) == [d: 4, e: 5] + assert Enum.slice(map, -3, 1) == [c: 3] + + assert_raise FunctionClauseError, fn -> + Enum.slice(map, 0, -1) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(map, 0.99, 0) + end + + assert_raise FunctionClauseError, fn -> + Enum.slice(map, 0, 0.99) + end end end @@ -891,17 +2519,22 @@ defmodule EnumTest.SideEffects do use ExUnit.Case, async: true import ExUnit.CaptureIO - import PathHelpers - test "take with side effects" do - stream = Stream.unfold(1, fn x -> IO.puts x; {x, x + 1} end) + test "take/2 with side effects" do + stream = + Stream.unfold(1, fn x -> + IO.puts(x) + {x, x + 1} + end) + assert capture_io(fn -> - Enum.take(stream, 1) - end) == "1\n" + Enum.take(stream, 1) + end) == "1\n" end - test "take does not consume next without a need" do - path = tmp_path("oneliner.txt") + @tag :tmp_dir + test "take/2 does not consume next without a need", config do + path = Path.join(config.tmp_dir, "oneliner.txt") File.mkdir(Path.dirname(path)) try do @@ -917,8 +2550,8 @@ defmodule EnumTest.SideEffects do end end - test "take with no item works as no-op" do - iterator = File.stream!(fixture_path("unknown.txt")) + test "take/2 with no elements works as no-op" do + iterator = File.stream!(PathHelpers.fixture_path("unknown.txt")) assert Enum.take(iterator, 0) == [] assert Enum.take(iterator, 0) == [] @@ -926,3 +2559,13 @@ defmodule EnumTest.SideEffects do assert Enum.take(iterator, 0) == [] end end + +defmodule EnumTest.Function do + use ExUnit.Case, async: true + + test "raises Protocol.UndefinedError for funs of wrong arity" do + assert_raise Protocol.UndefinedError, fn -> + Enum.to_list(fn -> nil end) + end + end +end diff --git a/lib/elixir/test/elixir/exception_test.exs b/lib/elixir/test/elixir/exception_test.exs index d474028aae7..fd720df874a 100644 --- a/lib/elixir/test/elixir/exception_test.exs +++ b/lib/elixir/test/elixir/exception_test.exs @@ -1,328 +1,917 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) -defmodule Kernel.ExceptionTest do +defmodule ExceptionTest do use ExUnit.Case, async: true - test "raise preserves the stacktrace" do - stacktrace = - try do - raise "a" - rescue _ -> - [top|_] = System.stacktrace - top - end - file = __ENV__.file |> Path.relative_to_cwd |> String.to_char_list - assert {Kernel.ExceptionTest, :"test raise preserves the stacktrace", _, - [file: ^file, line: 9]} = stacktrace + defp capture_err(fun) do + ExUnit.CaptureIO.capture_io(:stderr, fun) end - test "exception?" do - assert Exception.exception?(%RuntimeError{}) - refute Exception.exception?(%Regex{}) - refute Exception.exception?({}) - end + doctest Exception - test "message" do + test "message/1" do defmodule BadException do - def message(_) do - raise "oops" + def message(exception) do + if exception.raise do + raise "oops" + end end end - message = ~r/Got RuntimeError with message \"oops\" while retrieving message for/ + assert Exception.message(%{__struct__: BadException, __exception__: true, raise: true}) =~ + "got RuntimeError with message \"oops\" while retrieving Exception.message/1 " <> + "for %{__exception__: true, __struct__: ExceptionTest.BadException, raise: true}" - assert_raise ArgumentError, message, fn -> - Exception.message(%{__struct__: BadException, __exception__: true}) - end + assert Exception.message(%{__struct__: BadException, __exception__: true, raise: false}) =~ + "got nil while retrieving Exception.message/1 " <> + "for %{__exception__: true, __struct__: ExceptionTest.BadException, raise: false}" end - require Record - - test "normalize" do - assert Exception.normalize(:throw, :badarg) == :badarg - assert Exception.normalize(:exit, :badarg) == :badarg - assert Exception.normalize({:EXIT, self}, :badarg) == :badarg - assert Exception.normalize(:error, :badarg).__struct__ == ArgumentError - assert Exception.normalize(:error, %ArgumentError{}).__struct__ == ArgumentError + test "normalize/2" do + assert Exception.normalize(:throw, :badarg, []) == :badarg + assert Exception.normalize(:exit, :badarg, []) == :badarg + assert Exception.normalize({:EXIT, self()}, :badarg, []) == :badarg + assert Exception.normalize(:error, :badarg, []).__struct__ == ArgumentError + assert Exception.normalize(:error, %ArgumentError{}, []).__struct__ == ArgumentError + + assert %ErlangError{original: :no_translation, reason: ": foo"} = + Exception.normalize(:error, :no_translation, [ + {:io, :put_chars, [self(), <<222>>], + [error_info: %{module: __MODULE__, function: :dummy_error_extras}]} + ]) end - test "format_banner" do - assert Exception.format_banner(:error, :badarg) == "** (ArgumentError) argument error" - assert Exception.format_banner(:throw, :badarg) == "** (throw) :badarg" - assert Exception.format_banner(:exit, :badarg) == "** (exit) :badarg" - assert Exception.format_banner({:EXIT, self}, :badarg) == "** (EXIT from #{inspect self}) :badarg" - end + test "format/2 without stacktrace" do + stacktrace = + try do + throw(:stack) + catch + :stack -> __STACKTRACE__ + end - test "format without stacktrace" do - stacktrace = try do throw(:stack) catch :stack -> System.stacktrace() end - assert Exception.format(:error, :badarg) == "** (ArgumentError) argument error" <> - "\n" <> Exception.format_stacktrace(stacktrace) + assert Exception.format(:error, :badarg, stacktrace) == + "** (ArgumentError) argument error\n" <> Exception.format_stacktrace(stacktrace) end - test "format with empty stacktrace" do + test "format/2 with empty stacktrace" do assert Exception.format(:error, :badarg, []) == "** (ArgumentError) argument error" end - test "format with EXIT has no stacktrace" do - try do throw(:stack) catch :stack -> System.stacktrace() end - assert Exception.format({:EXIT, self}, :badarg) == "** (EXIT from #{inspect self}) :badarg" + test "format/2 with EXIT (has no stacktrace)" do + assert Exception.format({:EXIT, self()}, :badarg, []) == + "** (EXIT from #{inspect(self())}) :badarg" end - test "format_exit" do - assert Exception.format_exit(:bye) == ":bye" - assert Exception.format_exit(:noconnection) == "no connection" - assert Exception.format_exit({:nodedown, :"node@host"}) == "no connection to node@host" - assert Exception.format_exit(:timeout) == "time out" - assert Exception.format_exit(:noproc) == "no process" - assert Exception.format_exit(:killed) == "killed" - assert Exception.format_exit(:normal) == "normal" - assert Exception.format_exit(:shutdown) == "shutdown" - assert Exception.format_exit({:shutdown, :bye}) == "shutdown: :bye" - assert Exception.format_exit({:badarg,[{:not_a_real_module, :function, 0, []}]}) == - "an exception was raised:\n ** (ArgumentError) argument error\n :not_a_real_module.function/0" - assert Exception.format_exit({:bad_call, :request}) == "bad call: :request" - assert Exception.format_exit({:bad_cast, :request}) == "bad cast: :request" - assert Exception.format_exit({:start_spec, :unexpected}) == - "bad start spec: :unexpected" - assert Exception.format_exit({:supervisor_data, :unexpected}) == - "bad supervisor data: :unexpected" - end - - defmodule Sup do - def start_link(fun), do: :supervisor.start_link(__MODULE__, fun) - - def init(fun), do: fun.() - end + test "format_banner/2" do + assert Exception.format_banner(:error, :badarg) == "** (ArgumentError) argument error" + assert Exception.format_banner(:throw, :badarg) == "** (throw) :badarg" + assert Exception.format_banner(:exit, :badarg) == "** (exit) :badarg" - test "format_exit with supervisor errors" do - trap = Process.flag(:trap_exit, true) - - {:error, reason} = __MODULE__.Sup.start_link(fn() -> :foo end) - assert Exception.format_exit(reason) == - "#{inspect(__MODULE__.Sup)}.init/1 returned a bad value: :foo" - - return = {:ok, {:foo, []}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad supervisor data: invalid type: :foo" - - return = {:ok, {{:foo, 1, 1}, []}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad supervisor data: invalid strategy: :foo" - - return = {:ok, {{:one_for_one, :foo, 1}, []}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad supervisor data: invalid intensity: :foo" - - return = {:ok, {{:one_for_one, 1, :foo}, []}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad supervisor data: invalid period: :foo" - - return = {:ok, {{:simple_one_for_one, 1, 1}, :foo}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid children: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, [:foo]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid child spec: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, :foo, :temporary, 1, :worker, []}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid mfa: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {:m, :f, []}, :foo, 1, :worker, []}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid restart type: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {:m, :f, []}, :temporary, :foo, :worker, []}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid shutdown: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {:m, :f, []}, :temporary, 1, :foo, []}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid child type: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {:m, :f, []}, :temporary, 1, :worker, :foo}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid modules: :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {:m, :f, []}, :temporary, 1, :worker, [{:foo}]}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "bad start spec: invalid module: {:foo}" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {Kernel, :exit, [:foo]}, :temporary, 1, :worker, []}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "shutdown: failed to start child: :child\n ** (EXIT) :foo" - - return = {:ok, {{:one_for_one, 1, 1}, - [{:child, {Kernel, :apply, [fn() -> {:error, :foo} end, []]}, :temporary, 1, :worker, []}]}} - {:error, reason} = __MODULE__.Sup.start_link(fn() -> return end) - assert Exception.format_exit(reason) == - "shutdown: failed to start child: :child\n ** (EXIT) :foo" - - Process.flag(:trap_exit, trap) + assert Exception.format_banner({:EXIT, self()}, :badarg) == + "** (EXIT from #{inspect(self())}) :badarg" end - test "format_exit with call" do - reason = try do - :gen_server.call(:does_not_exist, :hello) - catch - :exit, reason -> reason + test "format_stacktrace/1 from file" do + try do + Code.eval_string("def foo do end", [], file: "my_file") + rescue + ArgumentError -> + assert Exception.format_stacktrace(__STACKTRACE__) =~ "my_file:1: (file)" + else + _ -> flunk("expected failure") end - - assert Exception.format_exit(reason) == - "exited in: :gen_server.call(:does_not_exist, :hello)\n ** (EXIT) no process" end - test "format_exit with call with exception" do - # Fake reason to prevent error_logger printing to stdout - fsm_reason = {%ArgumentError{}, [{:not_a_real_module, :function, 0, []}]} - reason = try do - :gen_fsm.sync_send_event(spawn(fn() -> - :timer.sleep(200) ; exit(fsm_reason) - end), :hello) - catch - :exit, reason -> reason - end - - formatted = Exception.format_exit(reason) - assert formatted =~ ~r"exited in: :gen_fsm\.sync_send_event\(#PID<\d+\.\d+\.\d+>, :hello\)" - assert formatted =~ ~r"\s{4}\*\* \(EXIT\) an exception was raised:\n" - assert formatted =~ ~r"\s{8}\*\* \(ArgumentError\) argument error\n" - assert formatted =~ ~r"\s{12}:not_a_real_module\.function/0" - end - - test "format_exit with nested calls" do - # Fake reason to prevent error_logger printing to stdout - event_fun = fn() -> :timer.sleep(200) ; exit(:normal) end - server_pid = spawn(fn()-> :gen_event.call(spawn(event_fun), :handler, :hello) end) - reason = try do - :gen_server.call(server_pid, :hi) - catch - :exit, reason -> reason + test "format_stacktrace/1 from module" do + try do + Code.eval_string( + "defmodule FmtStack do raise ArgumentError, ~s(oops) end", + [], + file: "my_file" + ) + rescue + ArgumentError -> + assert Exception.format_stacktrace(__STACKTRACE__) =~ "my_file:1: (module)" + else + _ -> flunk("expected failure") end - - formatted = Exception.format_exit(reason) - assert formatted =~ ~r"exited in: :gen_server\.call\(#PID<\d+\.\d+\.\d+>, :hi\)\n" - assert formatted =~ ~r"\s{4}\*\* \(EXIT\) exited in: :gen_event\.call\(#PID<\d+\.\d+\.\d+>, :handler, :hello\)\n" - assert formatted =~ ~r"\s{8}\*\* \(EXIT\) normal" - end - - test "format_exit with nested calls and exception" do - # Fake reason to prevent error_logger printing to stdout - event_reason = {%ArgumentError{}, [{:not_a_real_module, :function, 0, []}]} - event_fun = fn() -> :timer.sleep(200) ; exit(event_reason) end - server_pid = spawn(fn()-> :gen_event.call(spawn(event_fun), :handler, :hello) end) - reason = try do - :gen_server.call(server_pid, :hi) - catch - :exit, reason -> reason - end - - formatted = Exception.format_exit(reason) - assert formatted =~ ~r"exited in: :gen_server\.call\(#PID<\d+\.\d+\.\d+>, :hi\)\n" - assert formatted =~ ~r"\s{4}\*\* \(EXIT\) exited in: :gen_event\.call\(#PID<\d+\.\d+\.\d+>, :handler, :hello\)\n" - assert formatted =~ ~r"\s{8}\*\* \(EXIT\) an exception was raised:\n" - assert formatted =~ ~r"\s{12}\*\* \(ArgumentError\) argument error\n" - assert formatted =~ ~r"\s{16}:not_a_real_module\.function/0" end - test "format_stacktrace_entry with no file or line" do + test "format_stacktrace_entry/1 with no file or line" do assert Exception.format_stacktrace_entry({Foo, :bar, [1, 2, 3], []}) == "Foo.bar(1, 2, 3)" assert Exception.format_stacktrace_entry({Foo, :bar, [], []}) == "Foo.bar()" assert Exception.format_stacktrace_entry({Foo, :bar, 1, []}) == "Foo.bar/1" end - test "format_stacktrace_entry with file and line" do - assert Exception.format_stacktrace_entry({Foo, :bar, [], [file: 'file.ex', line: 10]}) == "file.ex:10: Foo.bar()" - assert Exception.format_stacktrace_entry({Foo, :bar, [1, 2, 3], [file: 'file.ex', line: 10]}) == "file.ex:10: Foo.bar(1, 2, 3)" - assert Exception.format_stacktrace_entry({Foo, :bar, 1, [file: 'file.ex', line: 10]}) == "file.ex:10: Foo.bar/1" + test "format_stacktrace_entry/1 with file and line" do + assert Exception.format_stacktrace_entry({Foo, :bar, [], [file: 'file.ex', line: 10]}) == + "file.ex:10: Foo.bar()" + + assert Exception.format_stacktrace_entry({Foo, :bar, [1, 2, 3], [file: 'file.ex', line: 10]}) == + "file.ex:10: Foo.bar(1, 2, 3)" + + assert Exception.format_stacktrace_entry({Foo, :bar, 1, [file: 'file.ex', line: 10]}) == + "file.ex:10: Foo.bar/1" end - test "format_stacktrace_entry with file no line" do - assert Exception.format_stacktrace_entry({Foo, :bar, [], [file: 'file.ex']}) == "file.ex: Foo.bar()" - assert Exception.format_stacktrace_entry({Foo, :bar, [], [file: 'file.ex', line: 0]}) == "file.ex: Foo.bar()" - assert Exception.format_stacktrace_entry({Foo, :bar, [1, 2, 3], [file: 'file.ex']}) == "file.ex: Foo.bar(1, 2, 3)" - assert Exception.format_stacktrace_entry({Foo, :bar, 1, [file: 'file.ex']}) == "file.ex: Foo.bar/1" + test "format_stacktrace_entry/1 with file no line" do + assert Exception.format_stacktrace_entry({Foo, :bar, [], [file: 'file.ex']}) == + "file.ex: Foo.bar()" + + assert Exception.format_stacktrace_entry({Foo, :bar, [], [file: 'file.ex', line: 0]}) == + "file.ex: Foo.bar()" + + assert Exception.format_stacktrace_entry({Foo, :bar, [1, 2, 3], [file: 'file.ex']}) == + "file.ex: Foo.bar(1, 2, 3)" + + assert Exception.format_stacktrace_entry({Foo, :bar, 1, [file: 'file.ex']}) == + "file.ex: Foo.bar/1" end - test "format_stacktrace_entry with application" do + test "format_stacktrace_entry/1 with application" do assert Exception.format_stacktrace_entry({Exception, :bar, [], [file: 'file.ex']}) == - "(elixir) file.ex: Exception.bar()" + "(elixir #{System.version()}) file.ex: Exception.bar()" + assert Exception.format_stacktrace_entry({Exception, :bar, [], [file: 'file.ex', line: 10]}) == - "(elixir) file.ex:10: Exception.bar()" - assert Exception.format_stacktrace_entry({:lists, :bar, [1, 2, 3], []}) == - "(stdlib) :lists.bar(1, 2, 3)" + "(elixir #{System.version()}) file.ex:10: Exception.bar()" end - test "format_stacktrace_entry with fun" do - assert Exception.format_stacktrace_entry({fn(x) -> x end, [1], []}) =~ ~r/#Function<.+>\(1\)/ - assert Exception.format_stacktrace_entry({fn(x, y) -> {x, y} end, 2, []}) =~ ~r"#Function<.+>/2" + test "format_stacktrace_entry/1 with fun" do + assert Exception.format_stacktrace_entry({fn x -> x end, [1], []}) =~ ~r/#Function<.+>\(1\)/ + + assert Exception.format_stacktrace_entry({fn x, y -> {x, y} end, 2, []}) =~ + ~r"#Function<.+>/2" end - test "format_mfa" do + test "format_mfa/3" do + # Let's create this atom so that String.to_existing_atom/1 inside + # format_mfa/3 doesn't raise. + _ = :"some function" + assert Exception.format_mfa(Foo, nil, 1) == "Foo.nil/1" assert Exception.format_mfa(Foo, :bar, 1) == "Foo.bar/1" assert Exception.format_mfa(Foo, :bar, []) == "Foo.bar()" assert Exception.format_mfa(nil, :bar, []) == "nil.bar()" assert Exception.format_mfa(:foo, :bar, [1, 2]) == ":foo.bar(1, 2)" + assert Exception.format_mfa(Foo, :b@r, 1) == "Foo.\"b@r\"/1" assert Exception.format_mfa(Foo, :"bar baz", 1) == "Foo.\"bar baz\"/1" assert Exception.format_mfa(Foo, :"-func/2-fun-0-", 4) == "anonymous fn/4 in Foo.func/2" + + assert Exception.format_mfa(Foo, :"-some function/2-fun-0-", 4) == + "anonymous fn/4 in Foo.\"some function\"/2" + + assert Exception.format_mfa(Foo, :"42", 1) == "Foo.\"42\"/1" + assert Exception.format_mfa(Foo, :Bar, [1, 2]) == "Foo.\"Bar\"(1, 2)" + assert Exception.format_mfa(Foo, :%{}, [1, 2]) == "Foo.\"%{}\"(1, 2)" + assert Exception.format_mfa(Foo, :..., 1) == "Foo.\"...\"/1" + end + + test "format_mfa/3 with Unicode" do + assert Exception.format_mfa(Foo, :olá, [1, 2]) == "Foo.olá(1, 2)" + assert Exception.format_mfa(Foo, :Olá, [1, 2]) == "Foo.\"Olá\"(1, 2)" + assert Exception.format_mfa(Foo, :Ólá, [1, 2]) == "Foo.\"Ólá\"(1, 2)" + assert Exception.format_mfa(Foo, :こんにちは世界, [1, 2]) == "Foo.こんにちは世界(1, 2)" + + nfd = :unicode.characters_to_nfd_binary("olá") + assert Exception.format_mfa(Foo, String.to_atom(nfd), [1, 2]) == "Foo.\"#{nfd}\"(1, 2)" + end + + test "format_fa/2" do + assert Exception.format_fa(fn -> nil end, 1) =~ + ~r"#Function<\d+\.\d+/0 in ExceptionTest\.\"test format_fa/2\"/1>/1" + end + + describe "format_exit/1" do + test "with atom/tuples" do + assert Exception.format_exit(:bye) == ":bye" + assert Exception.format_exit(:noconnection) == "no connection" + assert Exception.format_exit({:nodedown, :node@host}) == "no connection to node@host" + assert Exception.format_exit(:timeout) == "time out" + assert Exception.format_exit(:noproc) |> String.starts_with?("no process:") + assert Exception.format_exit(:killed) == "killed" + assert Exception.format_exit(:normal) == "normal" + assert Exception.format_exit(:shutdown) == "shutdown" + assert Exception.format_exit(:calling_self) == "process attempted to call itself" + assert Exception.format_exit({:shutdown, :bye}) == "shutdown: :bye" + + assert Exception.format_exit({:badarg, [{:not_a_real_module, :function, 0, []}]}) == + "an exception was raised:\n ** (ArgumentError) argument error\n :not_a_real_module.function/0" + + assert Exception.format_exit({:bad_call, :request}) == "bad call: :request" + assert Exception.format_exit({:bad_cast, :request}) == "bad cast: :request" + + assert Exception.format_exit({:start_spec, :unexpected}) == + "bad child specification, got: :unexpected" + + assert Exception.format_exit({:supervisor_data, :unexpected}) == + "bad supervisor configuration, got: :unexpected" + end + + defmodule Sup do + def start_link(fun), do: :supervisor.start_link(__MODULE__, fun) + + def init(fun), do: fun.() + end + + test "with supervisor errors" do + Process.flag(:trap_exit, true) + + {:error, reason} = __MODULE__.Sup.start_link(fn -> :foo end) + + assert Exception.format_exit(reason) == + "#{inspect(__MODULE__.Sup)}.init/1 returned a bad value: :foo" + + return = {:ok, {:foo, []}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) == "bad supervisor configuration, invalid type: :foo" + + return = {:ok, {{:foo, 1, 1}, []}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) == + "bad supervisor configuration, invalid strategy: :foo" + + return = {:ok, {{:one_for_one, :foo, 1}, []}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) == + "bad supervisor configuration, invalid max_restarts (intensity): :foo" + + return = {:ok, {{:one_for_one, 1, :foo}, []}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) == + "bad supervisor configuration, invalid max_seconds (period): :foo" + + return = {:ok, {{:simple_one_for_one, 1, 1}, :foo}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) == "bad child specification, invalid children: :foo" + + return = {:ok, {{:one_for_one, 1, 1}, [:foo]}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) == + "bad child specification, invalid child specification: :foo" + + return = {:ok, {{:one_for_one, 1, 1}, [{:child, :foo, :temporary, 1, :worker, []}]}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) == "bad child specification, invalid mfa: :foo" + + return = {:ok, {{:one_for_one, 1, 1}, [{:child, {:m, :f, []}, :foo, 1, :worker, []}]}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) =~ + "bad child specification, invalid restart type: :foo" + + return = { + :ok, + {{:one_for_one, 1, 1}, [{:child, {:m, :f, []}, :temporary, :foo, :worker, []}]} + } + + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) =~ "bad child specification, invalid shutdown: :foo" + + return = {:ok, {{:one_for_one, 1, 1}, [{:child, {:m, :f, []}, :temporary, 1, :foo, []}]}} + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) =~ "bad child specification, invalid child type: :foo" + + return = + {:ok, {{:one_for_one, 1, 1}, [{:child, {:m, :f, []}, :temporary, 1, :worker, :foo}]}} + + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) =~ "bad child specification, invalid modules: :foo" + + return = { + :ok, + {{:one_for_one, 1, 1}, [{:child, {:m, :f, []}, :temporary, 1, :worker, [{:foo}]}]} + } + + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + assert Exception.format_exit(reason) =~ "bad child specification, invalid module: {:foo}" + + return = { + :ok, + { + {:one_for_one, 1, 1}, + [ + {:child, {:m, :f, []}, :permanent, 1, :worker, []}, + {:child, {:m, :f, []}, :permanent, 1, :worker, []} + ] + } + } + + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) =~ + "bad child specification, more than one child specification has the id: :child" + + return = { + :ok, + {{:one_for_one, 1, 1}, [{:child, {Kernel, :exit, [:foo]}, :temporary, 1, :worker, []}]} + } + + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) == + "shutdown: failed to start child: :child\n ** (EXIT) :foo" + + return = { + :ok, + { + {:one_for_one, 1, 1}, + [{:child, {Kernel, :apply, [fn -> {:error, :foo} end, []]}, :temporary, 1, :worker, []}] + } + } + + {:error, reason} = __MODULE__.Sup.start_link(fn -> return end) + + assert Exception.format_exit(reason) == + "shutdown: failed to start child: :child\n ** (EXIT) :foo" + end + + test "with call" do + reason = + try do + :gen_server.call(:does_not_exist, :hello) + catch + :exit, reason -> reason + end + + expected_to_start_with = + "exited in: :gen_server.call(:does_not_exist, :hello)\n ** (EXIT) no process:" + + assert Exception.format_exit(reason) |> String.starts_with?(expected_to_start_with) + end + + test "with nested calls" do + Process.flag(:trap_exit, true) + # Fake reason to prevent error_logger printing to stdout + exit_fun = fn -> receive do: (_ -> exit(:normal)) end + + outer_pid = + spawn_link(fn -> + Process.flag(:trap_exit, true) + + receive do + _ -> + :gen_event.call(spawn_link(exit_fun), :handler, :hello) + end + end) + + reason = + try do + :gen_server.call(outer_pid, :hi) + catch + :exit, reason -> reason + end + + formatted = Exception.format_exit(reason) + assert formatted =~ ~r"exited in: :gen_server\.call\(#PID<\d+\.\d+\.\d+>, :hi\)\n" + + assert formatted =~ + ~r"\s{4}\*\* \(EXIT\) exited in: :gen_event\.call\(#PID<\d+\.\d+\.\d+>, :handler, :hello\)\n" + + assert formatted =~ ~r"\s{8}\*\* \(EXIT\) normal" + end + + test "format_exit/1 with nested calls and exception" do + Process.flag(:trap_exit, true) + # Fake reason to prevent error_logger printing to stdout + exit_reason = {%ArgumentError{}, [{:not_a_real_module, :function, 0, []}]} + exit_fun = fn -> receive do: (_ -> exit(exit_reason)) end + + outer_pid = + spawn_link(fn -> + Process.flag(:trap_exit, true) + :gen_event.call(spawn_link(exit_fun), :handler, :hello) + end) + + reason = + try do + :gen_server.call(outer_pid, :hi) + catch + :exit, reason -> reason + end + + formatted = Exception.format_exit(reason) + assert formatted =~ ~r"exited in: :gen_server\.call\(#PID<\d+\.\d+\.\d+>, :hi\)\n" + + assert formatted =~ + ~r"\s{4}\*\* \(EXIT\) exited in: :gen_event\.call\(#PID<\d+\.\d+\.\d+>, :handler, :hello\)\n" + + assert formatted =~ ~r"\s{8}\*\* \(EXIT\) an exception was raised:\n" + assert formatted =~ ~r"\s{12}\*\* \(ArgumentError\) argument error\n" + assert formatted =~ ~r"\s{16}:not_a_real_module\.function/0" + end end - test "format_fa" do - assert Exception.format_fa(fn -> end, 1) =~ - ~r"#Function<\d\.\d+/0 in Kernel\.ExceptionTest\.test format_fa/1>/1" + describe "blaming" do + test "does not annotate throws/exits" do + stack = [{Keyword, :pop, [%{}, :key, nil], [line: 13]}] + assert Exception.blame(:throw, :function_clause, stack) == {:function_clause, stack} + assert Exception.blame(:exit, :function_clause, stack) == {:function_clause, stack} + end + + test "reverts is_struct macro on guards for blaming" do + import PathHelpers + + write_beam( + defmodule Req do + def get!(url) + when is_binary(url) or (is_struct(url) and is_struct(url, URI) and false) do + url + end + + def get!(url, url_module) + when is_binary(url) or (is_struct(url) and is_struct(url, url_module) and false) do + url + end + end + ) + + :code.delete(Req) + :code.purge(Req) + + assert blame_message(Req, & &1.get!(url: "https://elixir-lang.org")) =~ """ + no function clause matching in ExceptionTest.Req.get!/1 + + The following arguments were given to ExceptionTest.Req.get!/1: + + # 1 + [url: "https://elixir-lang.org"] + + Attempted function clauses (showing 1 out of 1): + + def get!(url) when -is_binary(url)- or -is_struct(url)- and -is_struct(url, URI)- and -false- + """ + + elixir_uri = %URI{} = URI.parse("https://elixir-lang.org") + + assert blame_message(Req, & &1.get!(elixir_uri, URI)) =~ """ + no function clause matching in ExceptionTest.Req.get!/2 + + The following arguments were given to ExceptionTest.Req.get!/2: + + # 1 + %URI{scheme: \"https\", authority: \"elixir-lang.org\", userinfo: nil, host: \"elixir-lang.org\", port: 443, path: nil, query: nil, fragment: nil} + + # 2 + URI + + Attempted function clauses (showing 1 out of 1): + + def get!(url, url_module) when -is_binary(url)- or is_struct(url) and is_struct(url, url_module) and -false- + """ + + assert blame_message(Req, & &1.get!(elixir_uri)) =~ """ + no function clause matching in ExceptionTest.Req.get!/1 + + The following arguments were given to ExceptionTest.Req.get!/1: + + # 1 + %URI{scheme: \"https\", authority: \"elixir-lang.org\", userinfo: nil, host: \"elixir-lang.org\", port: 443, path: nil, query: nil, fragment: nil} + + Attempted function clauses (showing 1 out of 1): + + def get!(url) when -is_binary(url)- or is_struct(url) and is_struct(url, URI) and -false- + """ + end + + test "annotates badarg on apply" do + assert blame_message([], & &1.foo) == + "you attempted to apply a function named :foo on []. If you are using Kernel.apply/3, make sure " <> + "the module is an atom. If you are using the dot syntax, such as " <> + "map.field or module.function(), make sure the left side of the dot is an atom or a map" + + assert blame_message([], &apply(&1, :foo, [])) == + "you attempted to apply a function named :foo on []. If you are using Kernel.apply/3, make sure " <> + "the module is an atom. If you are using the dot syntax, such as " <> + "map.field or module.function(), make sure the left side of the dot is an atom or a map" + + assert blame_message([], &apply(Kernel, &1, [1, 2])) == + "you attempted to apply a function named [] on module Kernel. However [] is not a valid function name. " <> + "Function names (the second argument of apply) must always be an atom" + + assert blame_message(123, &apply(Kernel, :+, &1)) == + "you attempted to apply a function named :+ on module Kernel with arguments 123. " <> + "Arguments (the third argument of apply) must always be a proper list" + + assert blame_message(123, &apply(Kernel, :+, [&1 | 456])) == + "you attempted to apply a function named :+ on module Kernel with arguments [123 | 456]. " <> + "Arguments (the third argument of apply) must always be a proper list" + end + + test "annotates function clause errors" do + assert blame_message(Access, & &1.fetch(:foo, :bar)) =~ """ + no function clause matching in Access.fetch/2 + + The following arguments were given to Access.fetch/2: + + # 1 + :foo + + # 2 + :bar + + Attempted function clauses (showing 5 out of 5): + + def fetch(-%module{} = container-, key) + """ + end + + test "annotates undefined function error with suggestions" do + assert blame_message(Enum, & &1.map(:ok)) == """ + function Enum.map/1 is undefined or private. Did you mean: + + * map/2 + """ + + assert blame_message(Enum, & &1.man(:ok)) == """ + function Enum.man/1 is undefined or private. Did you mean: + + * map/2 + * max/1 + * max/2 + * max/3 + * min/1 + """ + + message = blame_message(:erlang, & &1.gt_cookie()) + assert message =~ "function :erlang.gt_cookie/0 is undefined or private. Did you mean:" + assert message =~ "* get_cookie/0" + assert message =~ "* set_cookie/2" + end + + test "annotates undefined function clause error with macro hints" do + assert blame_message(Integer, & &1.is_odd(1)) == + "function Integer.is_odd/1 is undefined or private. However there is " <> + "a macro with the same name and arity. Be sure to require Integer if " <> + "you intend to invoke this macro" + end + + test "annotates undefined function clause error with callback hints" do + capture_err(fn -> + Code.eval_string(""" + defmodule Behaviour do + @callback callback() :: :ok + end + + defmodule Implementation do + @behaviour Behaviour + end + """) + end) + + assert blame_message(Implementation, & &1.callback()) == + "function Implementation.callback/0 is undefined or private" <> + ", but the behaviour Behaviour expects it to be present" + end + + test "does not annotate undefined function clause error with callback hints when callback is optional" do + defmodule BehaviourWithOptional do + @callback callback() :: :ok + @callback optional() :: :ok + @optional_callbacks callback: 0, optional: 0 + end + + defmodule ImplementationWithOptional do + @behaviour BehaviourWithOptional + def callback(), do: :ok + end + + assert blame_message(ImplementationWithOptional, & &1.optional()) == + "function ExceptionTest.ImplementationWithOptional.optional/0 is undefined or private" + end + + test "annotates undefined function clause error with otp obsolete hints" do + assert blame_message(:erlang, & &1.hash(1, 2)) == + "function :erlang.hash/2 is undefined or private, use erlang:phash2/2 instead" + end + + test "annotates undefined function clause error with nil hints" do + assert blame_message(nil, & &1.foo) == + "function nil.foo/0 is undefined. If you are using the dot syntax, " <> + "such as map.field or module.function(), make sure the left side of the dot is an atom or a map" + end + + test "annotates key error with suggestions if keys are atoms" do + message = blame_message(%{first: nil, second: nil}, fn map -> map.firts end) + + assert message == """ + key :firts not found in: %{first: nil, second: nil}. Did you mean: + + * :first + """ + + message = blame_message(%{"first" => nil, "second" => nil}, fn map -> map.firts end) + + assert message == "key :firts not found in: %{\"first\" => nil, \"second\" => nil}" + + message = + blame_message(%{"first" => nil, "second" => nil}, fn map -> Map.fetch!(map, "firts") end) + + assert message == "key \"firts\" not found in: %{\"first\" => nil, \"second\" => nil}" + + message = + blame_message([first: nil, second: nil], fn kwlist -> Keyword.fetch!(kwlist, :firts) end) + + assert message == """ + key :firts not found in: [first: nil, second: nil]. Did you mean: + + * :first + """ + end + + test "annotates key error with suggestions for structs" do + message = blame_message(%URI{}, fn map -> map.schema end) + assert message =~ "key :schema not found in: %URI{" + assert message =~ "Did you mean:" + assert message =~ "* :scheme" + end + + test "annotates +/1 arithmetic errors" do + assert blame_message(:foo, &(+&1)) == "bad argument in arithmetic expression: +(:foo)" + end + + test "annotates -/1 arithmetic errors" do + assert blame_message(:foo, &(-&1)) == "bad argument in arithmetic expression: -(:foo)" + end + + test "annotates div arithmetic errors" do + assert blame_message(0, &div(10, &1)) == + "bad argument in arithmetic expression: div(10, 0)" + end + + test "annotates rem arithmetic errors" do + assert blame_message(0, &rem(10, &1)) == + "bad argument in arithmetic expression: rem(10, 0)" + end + + test "annotates band arithmetic errors" do + import Bitwise + + assert blame_message(:foo, &band(&1, 10)) == + "bad argument in arithmetic expression: Bitwise.band(:foo, 10)" + + assert blame_message(:foo, &(&1 &&& 10)) == + "bad argument in arithmetic expression: Bitwise.band(:foo, 10)" + end + + test "annotates bor arithmetic errors" do + import Bitwise + + assert blame_message(:foo, &bor(&1, 10)) == + "bad argument in arithmetic expression: Bitwise.bor(:foo, 10)" + + assert blame_message(:foo, &(&1 ||| 10)) == + "bad argument in arithmetic expression: Bitwise.bor(:foo, 10)" + end + + test "annotates bxor arithmetic errors" do + import Bitwise + + assert blame_message(:foo, &bxor(&1, 10)) == + "bad argument in arithmetic expression: Bitwise.bxor(:foo, 10)" + end + + test "annotates bsl arithmetic errors" do + import Bitwise + + assert blame_message(:foo, &bsl(10, &1)) == + "bad argument in arithmetic expression: Bitwise.bsl(10, :foo)" + + assert blame_message(:foo, &(10 <<< &1)) == + "bad argument in arithmetic expression: Bitwise.bsl(10, :foo)" + end + + test "annotates bsr arithmetic errors" do + import Bitwise + + assert blame_message(:foo, &bsr(10, &1)) == + "bad argument in arithmetic expression: Bitwise.bsr(10, :foo)" + + assert blame_message(:foo, &(10 >>> &1)) == + "bad argument in arithmetic expression: Bitwise.bsr(10, :foo)" + end + + test "annotates bnot arithmetic errors" do + import Bitwise + + assert blame_message(:foo, &bnot(&1)) == + "bad argument in arithmetic expression: Bitwise.bnot(:foo)" + end + + defp blame_message(arg, fun) do + try do + fun.(arg) + rescue + e -> + Exception.blame(:error, e, __STACKTRACE__) |> elem(0) |> Exception.message() + end + end end - import Exception, only: [message: 1] + describe "blaming unit tests" do + test "annotates clauses errors" do + args = [%{}, :key, nil] + + {exception, stack} = + Exception.blame(:error, :function_clause, [{Keyword, :pop, args, [line: 13]}]) + + assert %FunctionClauseError{kind: :def, args: ^args, clauses: [_]} = exception + assert stack == [{Keyword, :pop, 3, [line: 13]}] + end + + test "annotates args and clauses from mfa" do + import PathHelpers + + write_beam( + defmodule Blaming do + def with_elem(x, y) when elem(x, 1) == 0 and elem(x, y) == 1 do + {x, y} + end + + def fetch(%module{} = container, key), do: {module, container, key} + def fetch(map, key) when is_map(map), do: {map, key} + def fetch(list, key) when is_list(list) and is_atom(key), do: {list, key} + def fetch(nil, _key), do: nil + + require Integer + def even_and_odd(foo, bar) when Integer.is_even(foo) and Integer.is_odd(bar), do: :ok + end + ) + + :code.delete(Blaming) + :code.purge(Blaming) + + {:ok, :def, clauses} = Exception.blame_mfa(Blaming, :with_elem, [1, 2]) + + assert annotated_clauses_to_string(clauses) == [ + "{[+x+, +y+], [-elem(x, 1) == 0- and -elem(x, y) == 1-]}" + ] + + {:ok, :def, clauses} = Exception.blame_mfa(Blaming, :fetch, [self(), "oops"]) + + assert annotated_clauses_to_string(clauses) == [ + "{[-%module{} = container-, +key+], []}", + "{[+map+, +key+], [-is_map(map)-]}", + "{[+list+, +key+], [-is_list(list)- and -is_atom(key)-]}", + "{[-nil-, +_key+], []}" + ] + + {:ok, :def, clauses} = Exception.blame_mfa(Blaming, :even_and_odd, [1, 1]) + + assert annotated_clauses_to_string(clauses) == [ + "{[+foo+, +bar+], [+is_integer(foo)+ and -Bitwise.band(foo, 1) == 0- and +is_integer(bar)+ and +Bitwise.band(bar, 1) == 1+]}" + ] + + {:ok, :defmacro, clauses} = Exception.blame_mfa(Kernel, :!, [true]) - test "runtime error message" do - assert %RuntimeError{} |> message == "runtime error" - assert %RuntimeError{message: "exception"} |> message == "exception" + assert annotated_clauses_to_string(clauses) == [ + "{[-{:!, _, [value]}-], []}", + "{[+value+], []}" + ] + end + + defp annotated_clauses_to_string(clauses) do + Enum.map(clauses, fn {args, clauses} -> + args = Enum.map_join(args, ", ", &arg_to_string/1) + clauses = Enum.map_join(clauses, ", ", &clause_to_string/1) + "{[#{args}], [#{clauses}]}" + end) + end + + defp arg_to_string(%{match?: true, node: node}), do: "+" <> Macro.to_string(node) <> "+" + defp arg_to_string(%{match?: false, node: node}), do: "-" <> Macro.to_string(node) <> "-" + + defp clause_to_string({op, _, [left, right]}), + do: clause_to_string(left) <> " #{op} " <> clause_to_string(right) + + defp clause_to_string(other), + do: arg_to_string(other) end - test "argument error message" do - assert %ArgumentError{} |> message == "argument error" - assert %ArgumentError{message: "exception"} |> message == "exception" + describe "exception messages" do + import Exception, only: [message: 1] + + test "RuntimeError" do + assert %RuntimeError{} |> message == "runtime error" + assert %RuntimeError{message: "unexpected roquefort"} |> message == "unexpected roquefort" + end + + test "ArithmeticError" do + assert %ArithmeticError{} |> message == "bad argument in arithmetic expression" + + assert %ArithmeticError{message: "unexpected camembert"} + |> message == "unexpected camembert" + end + + test "ArgumentError" do + assert %ArgumentError{} |> message == "argument error" + assert %ArgumentError{message: "unexpected comté"} |> message == "unexpected comté" + end + + test "KeyError" do + assert %KeyError{} |> message == "key nil not found" + assert %KeyError{message: "key missed"} |> message == "key missed" + end + + test "Enum.OutOfBoundsError" do + assert %Enum.OutOfBoundsError{} |> message == "out of bounds error" + + assert %Enum.OutOfBoundsError{message: "the brie is not on the table"} + |> message == "the brie is not on the table" + end + + test "Enum.EmptyError" do + assert %Enum.EmptyError{} |> message == "empty error" + + assert %Enum.EmptyError{message: "there is no saint-nectaire left!"} + |> message == "there is no saint-nectaire left!" + end + + test "UndefinedFunctionError" do + assert %UndefinedFunctionError{} |> message == "undefined function" + + assert %UndefinedFunctionError{module: Kernel, function: :bar, arity: 1} + |> message == "function Kernel.bar/1 is undefined or private" + + assert %UndefinedFunctionError{module: Foo, function: :bar, arity: 1} + |> message == "function Foo.bar/1 is undefined (module Foo is not available)" + + assert %UndefinedFunctionError{module: nil, function: :bar, arity: 3} + |> message == "function nil.bar/3 is undefined" + + assert %UndefinedFunctionError{module: nil, function: :bar, arity: 0} + |> message == "function nil.bar/0 is undefined" + end + + test "FunctionClauseError" do + assert %FunctionClauseError{} |> message == "no function clause matches" + + assert %FunctionClauseError{module: Foo, function: :bar, arity: 1} + |> message == "no function clause matching in Foo.bar/1" + end + + test "ErlangError" do + assert %ErlangError{original: :sample} |> message == "Erlang error: :sample" + end end - test "undefined function message" do - assert %UndefinedFunctionError{} |> message == "undefined function" - assert %UndefinedFunctionError{module: Foo, function: :bar, arity: 1} |> message == - "undefined function: Foo.bar/1" - assert %UndefinedFunctionError{module: nil, function: :bar, arity: 0} |> message == - "undefined function: nil.bar/0" + if System.otp_release() >= "24" do + describe "error_info" do + test "badarg on erlang" do + assert message(:erlang, & &1.element("foo", "bar")) == """ + errors were found at the given arguments: + + * 1st argument: not an integer + * 2nd argument: not a tuple + """ + end + + test "badarg on ets" do + ets = :ets.new(:foo, []) + :ets.delete(ets) + + assert message(:ets, & &1.insert(ets, 1)) == """ + errors were found at the given arguments: + + * 1st argument: the table identifier does not refer to an existing ETS table + * 2nd argument: not a tuple + """ + end + + test "system_limit on counters" do + assert message(:counters, & &1.new(123_456_789_123_456_789_123_456_789, [])) == """ + a system limit has been reached due to errors at the given arguments: + + * 1st argument: counters array size reached a system limit + """ + end + end end - test "function clause message" do - assert %FunctionClauseError{} |> message == - "no function clause matches" - assert %FunctionClauseError{module: Foo, function: :bar, arity: 1} |> message == - "no function clause matching in Foo.bar/1" + if System.otp_release() >= "25" do + describe "binary constructor error info" do + defp concat(a, b), do: a <> b + + test "on binary concatenation" do + assert message(123, &concat(&1, "bar")) == + "construction of binary failed: segment 1 of type 'binary': expected a binary but got: 123" + + assert message(~D[0001-02-03], &concat(&1, "bar")) == + "construction of binary failed: segment 1 of type 'binary': expected a binary but got: ~D[0001-02-03]" + end + end end - test "erlang error message" do - assert %ErlangError{original: :sample} |> message == - "erlang error: :sample" + defp message(arg, fun) do + try do + fun.(arg) + rescue + e -> Exception.message(e) + end end + + def dummy_error_extras(_exception, _stacktrace), do: %{general: "foo"} end diff --git a/lib/elixir/test/elixir/file_test.exs b/lib/elixir/test/elixir/file_test.exs index 3c43287e693..7fa48363d2f 100644 --- a/lib/elixir/test/elixir/file_test.exs +++ b/lib/elixir/test/elixir/file_test.exs @@ -1,4 +1,4 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule Elixir.FileCase do use ExUnit.CaseTemplate @@ -11,21 +11,371 @@ defmodule Elixir.FileCase do end setup do - File.mkdir_p!(tmp_path) - on_exit(fn -> File.rm_rf(tmp_path) end) + File.mkdir_p!(tmp_path()) + on_exit(fn -> File.rm_rf(tmp_path()) end) :ok end end defmodule FileTest do use Elixir.FileCase - import Regex, only: [escape: 1] + + defmodule Rename do + # Following Erlang's underlying implementation + # + # Renaming files + # :ok -> rename file to existing file default behaviour + # {:error, :eisdir} -> rename file to existing empty dir + # {:error, :eisdir} -> rename file to existing non-empty dir + # :ok -> rename file to non-existing location + # {:error, :eexist} -> rename file to existing file + # :ok -> rename file to itself + + # Renaming dirs + # {:error, :enotdir} -> rename dir to existing file + # :ok -> rename dir to non-existing leaf location + # {:error, ??} -> rename dir to non-existing parent location + # :ok -> rename dir to itself + # :ok -> rename dir to existing empty dir default behaviour + # {:error, :eexist} -> rename dir to existing empty dir + # {:error, :einval} -> rename parent dir to existing sub dir + # {:error, :einval} -> rename parent dir to non-existing sub dir + # {:error, :eexist} -> rename dir to existing non-empty dir + + # other tests + # {:error, :enoent} -> rename unknown source + # :ok -> rename preserves mode + use Elixir.FileCase + + test "rename file to existing file default behaviour" do + src = tmp_fixture_path("file.txt") + dest = tmp_path("tmp.file") + + File.write!(dest, "hello") + + try do + assert File.exists?(dest) + assert File.rename(src, dest) == :ok + refute File.exists?(src) + assert File.read!(dest) == "FOO\n" + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename file to existing empty dir" do + src = tmp_fixture_path("file.txt") + dest = tmp_path("tmp") + + try do + File.mkdir(dest) + assert File.rename(src, dest) == {:error, :eisdir} + assert File.exists?(src) + refute File.exists?(tmp_path("tmp/file.txt")) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename file to existing non-empty dir" do + src = tmp_fixture_path("file.txt") + dest = tmp_path("tmp") + + try do + File.mkdir_p(Path.join(dest, "a")) + assert File.rename(src, dest) in [{:error, :eisdir}, {:error, :eexist}] + assert File.exists?(src) + refute File.exists?(Path.join(dest, "file.txt")) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename file to non-existing location" do + src = tmp_fixture_path("file.txt") + dest = tmp_path("tmp.file") + + try do + refute File.exists?(dest) + assert File.rename(src, dest) == :ok + assert File.exists?(dest) + refute File.exists?(src) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename file to existing file" do + src = tmp_fixture_path("file.txt") + dest = tmp_path("tmp.file") + + File.write!(dest, "hello") + + try do + assert File.exists?(dest) + assert File.rename(src, dest) == :ok + refute File.exists?(src) + assert File.read!(dest) == "FOO\n" + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename file to itself" do + src = tmp_fixture_path("file.txt") + dest = src + + try do + assert File.exists?(src) + assert File.rename(src, dest) == :ok + assert File.exists?(src) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename! file to existing file default behaviour" do + src = tmp_fixture_path("file.txt") + dest = tmp_path("tmp.file") + + File.write!(dest, "hello") + + try do + assert File.exists?(dest) + assert File.rename!(src, dest) == :ok + refute File.exists?(src) + assert File.read!(dest) == "FOO\n" + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename! with invalid file" do + src = tmp_fixture_path("invalid.txt") + dest = tmp_path("tmp.file") + + message = + "could not rename from #{inspect(src)} to #{inspect(dest)}: no such file or directory" + + assert_raise File.RenameError, message, fn -> + File.rename!(src, dest) + end + end + + test "rename dir to existing file" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("tmp.file") + + try do + File.touch(dest) + assert File.rename(src, dest) == {:error, :enotdir} + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename dir to non-existing leaf location" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("tmp") + + try do + refute File.exists?(tmp_path("tmp/a/1.txt")) + refute File.exists?(tmp_path("tmp/a/a/2.txt")) + refute File.exists?(tmp_path("tmp/b/3.txt")) + + assert File.rename(src, dest) == :ok + {:ok, files} = File.ls(dest) + assert length(files) == 2 + assert "a" in files + + {:ok, files} = File.ls(tmp_path("tmp/a")) + assert length(files) == 2 + assert "1.txt" in files + + assert File.exists?(tmp_path("tmp/a/1.txt")) + assert File.exists?(tmp_path("tmp/a/a/2.txt")) + assert File.exists?(tmp_path("tmp/b/3.txt")) + + refute File.exists?(src) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename dir to non-existing parent location" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("tmp/a/b") + + try do + assert File.rename(src, dest) == {:error, :enoent} + assert File.exists?(src) + refute File.exists?(dest) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename dir to itself" do + src = tmp_fixture_path("cp_r") + dest = src + + try do + assert File.exists?(src) + assert File.rename(src, dest) == :ok + assert File.exists?(src) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename parent dir to existing sub dir" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("cp_r/a") + + try do + assert File.exists?(src) + assert File.rename(src, dest) in [{:error, :einval}, {:error, :eexist}] + assert File.exists?(src) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename parent dir to non-existing sub dir" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("cp_r/x") + + try do + assert File.exists?(src) + assert File.rename(src, dest) == {:error, :einval} + assert File.exists?(src) + refute File.exists?(dest) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename dir to existing empty dir default behaviour" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("tmp") + + File.mkdir(dest) + + try do + refute File.exists?(tmp_path("tmp/a")) + + assert File.rename(src, dest) == :ok + {:ok, files} = File.ls(dest) + assert length(files) == 2 + assert "a" in files + + {:ok, files} = File.ls(tmp_path("tmp/a")) + assert length(files) == 2 + assert "1.txt" in files + + assert File.exists?(tmp_path("tmp/a/1.txt")) + assert File.exists?(tmp_path("tmp/a/a/2.txt")) + assert File.exists?(tmp_path("tmp/b/3.txt")) + + refute File.exists?(src) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename dir to existing empty dir" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("tmp") + + File.mkdir(dest) + + try do + assert File.exists?(dest) + assert File.rename(src, dest) == :ok + refute File.exists?(src) + assert File.exists?(tmp_path("tmp/a")) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename dir to existing non-empty dir" do + src = tmp_fixture_path("cp_r") + dest = tmp_path("tmp") + + File.mkdir_p(tmp_path("tmp/x")) + + try do + assert File.exists?(tmp_path("tmp/x")) + assert File.exists?(src) + refute File.exists?(tmp_path("tmp/a")) + + assert File.rename(src, dest) == {:error, :eexist} + + assert File.exists?(tmp_path("tmp/x")) + assert File.exists?(src) + refute File.exists?(tmp_path("tmp/a")) + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + test "rename unknown source" do + src = fixture_path("unknown") + dest = tmp_path("tmp") + + try do + assert File.rename(src, dest) == {:error, :enoent} + after + File.rm_rf(dest) + end + end + + test "rename preserves mode" do + File.mkdir_p!(tmp_path("tmp")) + src = tmp_fixture_path("cp_mode") + dest = tmp_path("tmp/cp_mode") + + try do + %File.Stat{mode: src_mode} = File.stat!(src) + File.rename(src, dest) + %File.Stat{mode: dest_mode} = File.stat!(dest) + assert src_mode == dest_mode + after + File.rm_rf(src) + File.rm_rf(dest) + end + end + + def tmp_fixture_path(extra) do + src = fixture_path(extra) + dest = tmp_path(extra) + File.cp_r(src, dest) + dest + end + end defmodule Cp do use Elixir.FileCase - test :cp_with_src_file_and_dest_file do - src = fixture_path("file.txt") + test "cp with src file and dest file" do + src = fixture_path("file.txt") dest = tmp_path("sample.txt") File.touch(dest) @@ -39,8 +389,8 @@ defmodule FileTest do end end - test :cp_with_src_file_and_dest_dir do - src = fixture_path("file.txt") + test "cp with src file and dest dir" do + src = fixture_path("file.txt") dest = tmp_path("tmp") File.mkdir(dest) @@ -48,32 +398,32 @@ defmodule FileTest do try do assert File.cp(src, dest) == {:error, :eisdir} after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_with_src_file_and_dest_unknown do - src = fixture_path("file.txt") - dest = tmp_path("tmp.file") + test "cp with src file and dest unknown" do + src = fixture_path("file.txt") + dest = tmp_path("tmp.file") try do refute File.exists?(dest) assert File.cp(src, dest) == :ok assert File.exists?(dest) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_with_src_dir do - src = fixture_path("cp_r") - dest = tmp_path("tmp.file") + test "cp with src dir" do + src = fixture_path("cp_r") + dest = tmp_path("tmp.file") assert File.cp(src, dest) == {:error, :eisdir} end - test :cp_with_conflict do - src = fixture_path("file.txt") - dest = tmp_path("tmp.file") + test "cp with conflict" do + src = fixture_path("file.txt") + dest = tmp_path("tmp.file") File.write!(dest, "hello") @@ -82,31 +432,33 @@ defmodule FileTest do assert File.cp(src, dest) == :ok assert File.read!(dest) == "FOO\n" after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_with_conflict_with_function do - src = fixture_path("file.txt") - dest = tmp_path("tmp.file") + test "cp with conflict with function" do + src = fixture_path("file.txt") + dest = tmp_path("tmp.file") File.write!(dest, "hello") try do assert File.exists?(dest) - assert File.cp(src, dest, fn(src_file, dest_file) -> - assert src_file == src - assert dest_file == dest - false - end) == :ok + + assert File.cp(src, dest, fn src_file, dest_file -> + assert src_file == src + assert dest_file == dest + false + end) == :ok + assert File.read!(dest) == "hello" after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_with_src_file_and_dest_file! do - src = fixture_path("file.txt") + test "cp! with src file and dest file" do + src = fixture_path("file.txt") dest = tmp_path("sample.txt") File.touch(dest) @@ -120,17 +472,39 @@ defmodule FileTest do end end - test :cp_with_src_dir! do - src = fixture_path("cp_r") - dest = tmp_path("tmp.file") - assert_raise File.CopyError, "could not copy recursively from #{src} to #{dest}: " <> - "illegal operation on a directory", fn -> + test "cp! with src dir" do + src = fixture_path("cp_r") + dest = tmp_path("tmp.file") + + message = + "could not copy from #{inspect(src)} to #{inspect(dest)}: illegal operation on a directory" + + assert_raise File.CopyError, message, fn -> File.cp!(src, dest) end end - test :cp_r_with_src_file_and_dest_file do - src = fixture_path("file.txt") + test "copy file to itself" do + src = dest = tmp_path("tmp.file") + + File.write!(src, "here") + + try do + assert File.cp(src, dest) == :ok + assert File.read!(dest) == "here" + assert File.cp_r(src, dest) == {:ok, []} + after + File.rm(dest) + end + end + + test "cp_r raises on path with null byte" do + assert_raise ArgumentError, ~r/null byte/, fn -> File.cp_r("source", "foo\0bar") end + assert_raise ArgumentError, ~r/null byte/, fn -> File.cp_r("foo\0bar", "dest") end + end + + test "cp_r with src file and dest file" do + src = fixture_path("file.txt") dest = tmp_path("sample.txt") File.touch(dest) @@ -144,34 +518,34 @@ defmodule FileTest do end end - test :cp_r_with_src_file_and_dest_dir do - src = fixture_path("file.txt") - dest = tmp_path("tmp") + test "cp_r with src file and dest dir" do + src = fixture_path("file.txt") + dest = tmp_path("tmp") File.mkdir(dest) try do - assert io_error? File.cp_r(src, dest) + assert io_error?(File.cp_r(src, dest)) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_file_and_dest_unknown do - src = fixture_path("file.txt") - dest = tmp_path("tmp.file") + test "cp_r with src file and dest unknown" do + src = fixture_path("file.txt") + dest = tmp_path("tmp.file") try do refute File.exists?(dest) assert File.cp_r(src, dest) == {:ok, [dest]} assert File.exists?(dest) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_dir_and_dest_dir do - src = fixture_path("cp_r") + test "cp_r with src dir and dest dir" do + src = fixture_path("cp_r") dest = tmp_path("tmp") File.mkdir(dest) @@ -190,24 +564,24 @@ defmodule FileTest do assert File.exists?(tmp_path("tmp/a/a/2.txt")) assert File.exists?(tmp_path("tmp/b/3.txt")) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_dir_and_dest_file do - src = fixture_path("cp_r") + test "cp_r with src dir and dest file" do + src = fixture_path("cp_r") dest = tmp_path("tmp.file") try do File.touch!(dest) - assert (File.cp_r(src, dest) |> io_error?) + assert File.cp_r(src, dest) |> io_error? after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_dir_and_dest_unknown do - src = fixture_path("cp_r") + test "cp_r with src dir and dest unknown" do + src = fixture_path("cp_r") dest = tmp_path("tmp") try do @@ -222,32 +596,32 @@ defmodule FileTest do assert File.exists?(tmp_path("tmp/a/a/2.txt")) assert File.exists?(tmp_path("tmp/b/3.txt")) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_unknown do - src = fixture_path("unknown") + test "cp_r with src unknown" do + src = fixture_path("unknown") dest = tmp_path("tmp") assert File.cp_r(src, dest) == {:error, :enoent, src} end - test :cp_r_with_dir_and_file_conflict do - src = fixture_path("cp_r") + test "cp_r with dir and file conflict" do + src = fixture_path("cp_r") dest = tmp_path("tmp") try do File.mkdir(dest) File.write!(Path.join(dest, "a"), "hello") - assert io_error? File.cp_r(src, dest) + assert io_error?(File.cp_r(src, dest)) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_dir_and_dest_dir_using_lists do - src = fixture_path("cp_r") |> to_char_list - dest = tmp_path("tmp") |> to_char_list + test "cp_r with src dir and dest dir using lists" do + src = fixture_path("cp_r") |> to_charlist + dest = tmp_path("tmp") |> to_charlist File.mkdir(dest) @@ -264,48 +638,50 @@ defmodule FileTest do assert File.exists?(tmp_path("tmp/a/a/2.txt")) assert File.exists?(tmp_path("tmp/b/3.txt")) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_with_file_conflict do - src = fixture_path("cp_r") + test "cp_r with src with file conflict" do + src = fixture_path("cp_r") dest = tmp_path("tmp") - File.mkdir_p tmp_path("tmp/a") - File.write! tmp_path("tmp/a/1.txt"), "hello" + File.mkdir_p(tmp_path("tmp/a")) + File.write!(tmp_path("tmp/a/1.txt"), "hello") try do assert File.exists?(tmp_path("tmp/a/1.txt")) File.cp_r(src, dest) assert File.read!(tmp_path("tmp/a/1.txt")) == "" after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_with_file_conflict_callback do - src = fixture_path("cp_r") + test "cp_r with src with file conflict callback" do + src = fixture_path("cp_r") dest = tmp_path("tmp") - File.mkdir_p tmp_path("tmp/a") - File.write! tmp_path("tmp/a/1.txt"), "hello" + File.mkdir_p(tmp_path("tmp/a")) + File.write!(tmp_path("tmp/a/1.txt"), "hello") try do assert File.exists?(tmp_path("tmp/a/1.txt")) - File.cp_r(src, dest, fn(src_file, dest_file) -> + + File.cp_r(src, dest, fn src_file, dest_file -> assert src_file == fixture_path("cp_r/a/1.txt") assert dest_file == tmp_path("tmp/a/1.txt") false end) + assert File.read!(tmp_path("tmp/a/1.txt")) == "hello" after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r! do - src = fixture_path("cp_r") + test "cp_r!" do + src = fixture_path("cp_r") dest = tmp_path("tmp") File.mkdir(dest) @@ -321,33 +697,37 @@ defmodule FileTest do assert File.exists?(tmp_path("tmp/a/a/2.txt")) assert File.exists?(tmp_path("tmp/b/3.txt")) after - File.rm_rf dest + File.rm_rf(dest) end end - test :cp_r_with_src_unknown! do - src = fixture_path("unknown") + test "cp_r with src_unknown!" do + src = fixture_path("unknown") dest = tmp_path("tmp") - assert_raise File.CopyError, "could not copy recursively from #{src} to #{dest}. #{src}: no such file or directory", fn -> + + message = + "could not copy recursively from #{inspect(src)} to #{inspect(dest)}. #{src}: no such file or directory" + + assert_raise File.CopyError, message, fn -> File.cp_r!(src, dest) end end - test :cp_preserves_mode do - File.mkdir_p!(tmp_path("tmp")) - src = fixture_path("cp_mode") - dest = tmp_path("tmp/cp_mode") + test "cp preserves mode" do + File.mkdir_p!(tmp_path("tmp")) + src = fixture_path("cp_mode") + dest = tmp_path("tmp/cp_mode") - File.cp!(src, dest) - %File.Stat{mode: src_mode} = File.stat! src - %File.Stat{mode: dest_mode} = File.stat! dest - assert src_mode == dest_mode + File.cp!(src, dest) + %File.Stat{mode: src_mode} = File.stat!(src) + %File.Stat{mode: dest_mode} = File.stat!(dest) + assert src_mode == dest_mode - # On overwrite - File.cp! src, dest, fn(_, _) -> true end - %File.Stat{mode: src_mode} = File.stat! src - %File.Stat{mode: dest_mode} = File.stat! dest - assert src_mode == dest_mode + # On overwrite + File.cp!(src, dest, fn _, _ -> true end) + %File.Stat{mode: src_mode} = File.stat!(src) + %File.Stat{mode: dest_mode} = File.stat!(dest) + assert src_mode == dest_mode end defp io_error?(result) do @@ -358,32 +738,44 @@ defmodule FileTest do defmodule Queries do use ExUnit.Case - test :regular do + test "regular" do assert File.regular?(__ENV__.file) - assert File.regular?(String.to_char_list(__ENV__.file)) + assert File.regular?(String.to_charlist(__ENV__.file)) refute File.regular?("#{__ENV__.file}.unknown") end - test :exists do + test "exists" do assert File.exists?(__ENV__.file) - assert File.exists?(fixture_path) + assert File.exists?(fixture_path()) assert File.exists?(fixture_path("file.txt")) refute File.exists?(fixture_path("missing.txt")) refute File.exists?("_missing.txt") end + + test "exists with dangling symlink" do + invalid_file = tmp_path("invalid_file") + dest = tmp_path("dangling_symlink") + File.ln_s(invalid_file, dest) + + try do + refute File.exists?(dest) + after + File.rm(dest) + end + end end - test :ls do - {:ok, value} = File.ls(fixture_path) + test "ls" do + {:ok, value} = File.ls(fixture_path()) assert "code_sample.exs" in value assert "file.txt" in value {:error, :enoent} = File.ls(fixture_path("non-existent-subdirectory")) end - test :ls! do - value = File.ls!(fixture_path) + test "ls!" do + value = File.ls!(fixture_path()) assert "code_sample.exs" in value assert "file.txt" in value @@ -395,31 +787,32 @@ defmodule FileTest do defmodule OpenReadWrite do use Elixir.FileCase - test :read_with_binary do + test "read with binary" do assert {:ok, "FOO\n"} = File.read(fixture_path("file.txt")) assert {:error, :enoent} = File.read(fixture_path("missing.txt")) end - test :read_with_list do + test "read with list" do assert {:ok, "FOO\n"} = File.read(Path.expand('fixtures/file.txt', __DIR__)) assert {:error, :enoent} = File.read(Path.expand('fixtures/missing.txt', __DIR__)) end - test :read_with_utf8 do + test "read with UTF-8" do assert {:ok, "Русский\n日\n"} = File.read(Path.expand('fixtures/utf8.txt', __DIR__)) end - test :read! do + test "read!" do assert File.read!(fixture_path("file.txt")) == "FOO\n" - expected_message = "could not read file fixtures/missing.txt: no such file or directory" + expected_message = "could not read file \"fixtures/missing.txt\": no such file or directory" assert_raise File.Error, expected_message, fn -> File.read!("fixtures/missing.txt") end end - test :write_ascii_content do + test "write ASCII content" do fixture = tmp_path("tmp_test.txt") + try do refute File.exists?(fixture) assert File.write(fixture, 'test text') == :ok @@ -429,8 +822,9 @@ defmodule FileTest do end end - test :write_utf8 do + test "write UTF-8" do fixture = tmp_path("tmp_test.txt") + try do refute File.exists?(fixture) assert File.write(fixture, "Русский\n日\n") == :ok @@ -440,8 +834,9 @@ defmodule FileTest do end end - test :write_with_options do + test "write with options" do fixture = tmp_path("tmp_test.txt") + try do refute File.exists?(fixture) assert File.write(fixture, "Русский\n日\n") == :ok @@ -452,32 +847,33 @@ defmodule FileTest do end end - test :open_file_without_modes do + test "open file without modes" do {:ok, file} = File.open(fixture_path("file.txt")) assert IO.gets(file, "") == "FOO\n" assert File.close(file) == :ok end - test :open_file_with_char_list do - {:ok, file} = File.open(fixture_path("file.txt"), [:char_list]) + test "open file with charlist" do + {:ok, file} = File.open(fixture_path("file.txt"), [:charlist]) assert IO.gets(file, "") == 'FOO\n' assert File.close(file) == :ok end - test :open_utf8_by_default do + test "open UTF-8 by default" do {:ok, file} = File.open(fixture_path("utf8.txt"), [:utf8]) assert IO.gets(file, "") == "Русский\n" assert File.close(file) == :ok end - test :open_readonly_by_default do + test "open readonly by default" do {:ok, file} = File.open(fixture_path("file.txt")) assert_raise ArgumentError, fn -> IO.write(file, "foo") end assert File.close(file) == :ok end - test :open_with_write_permission do + test "open with write permission" do fixture = tmp_path("tmp_text.txt") + try do {:ok, file} = File.open(fixture, [:write]) assert IO.write(file, "foo") == :ok @@ -488,8 +884,9 @@ defmodule FileTest do end end - test :open_with_binwrite_permission do + test "open with binwrite permission" do fixture = tmp_path("tmp_text.txt") + try do {:ok, file} = File.open(fixture, [:write]) assert IO.binwrite(file, "Русский") == :ok @@ -500,35 +897,41 @@ defmodule FileTest do end end - test :open_utf8_and_charlist do - {:ok, file} = File.open(fixture_path("utf8.txt"), [:char_list, :utf8]) + test "open UTF-8 and charlist" do + {:ok, file} = File.open(fixture_path("utf8.txt"), [:charlist, :utf8]) assert IO.gets(file, "") == [1056, 1091, 1089, 1089, 1082, 1080, 1081, 10] assert File.close(file) == :ok end - test :open_respects_encoding do + test "open respects encoding" do {:ok, file} = File.open(fixture_path("utf8.txt"), [{:encoding, :latin1}]) - assert IO.gets(file, "") == <<195, 144, 194, 160, 195, 145, 194, 131, 195, 145, 194, 129, 195, 145, 194, 129, 195, 144, 194, 186, 195, 144, 194, 184, 195, 144, 194, 185, 10>> + + data = + <<195, 144, 194, 160, 195, 145, 194, 131, 195, 145, 194, 129, 195, 145>> <> + <<194, 129, 195, 144, 194, 186, 195, 144, 194, 184, 195, 144, 194, 185, 10>> + + assert IO.gets(file, "") == data assert File.close(file) == :ok end - test :open_a_missing_file do + test "open a missing file" do assert File.open('missing.txt') == {:error, :enoent} end - test :open_a_file_with_function do + test "open a file with function" do file = fixture_path("file.txt") assert File.open(file, &IO.read(&1, :line)) == {:ok, "FOO\n"} end - test :open_a_missing_file! do - message = "could not open missing.txt: no such file or directory" + test "open! a missing file" do + message = "could not open \"missing.txt\": no such file or directory" + assert_raise File.Error, message, fn -> File.open!('missing.txt') end end - test :open_a_file_with_function! do + test "open! a file with function" do file = fixture_path("file.txt") assert File.open!(file, &IO.read(&1, :line)) == "FOO\n" end @@ -537,69 +940,77 @@ defmodule FileTest do defmodule Mkdir do use Elixir.FileCase - test :mkdir_with_binary do + test "mkdir with binary" do fixture = tmp_path("tmp_test") + try do refute File.exists?(fixture) assert File.mkdir(fixture) == :ok assert File.exists?(fixture) after - File.rmdir fixture + File.rmdir(fixture) end end - test :mkdir_with_list do - fixture = tmp_path("tmp_test") |> to_char_list + test "mkdir with list" do + fixture = tmp_path("tmp_test") |> to_charlist + try do refute File.exists?(fixture) assert File.mkdir(fixture) == :ok assert File.exists?(fixture) after - File.rmdir fixture + File.rmdir(fixture) end end - test :mkdir_with_invalid_path do + test "mkdir with invalid path" do fixture = fixture_path("file.txt") - invalid = Path.join fixture, "test" + invalid = Path.join(fixture, "test") assert File.exists?(fixture) - assert io_error? File.mkdir(invalid) + assert io_error?(File.mkdir(invalid)) refute File.exists?(invalid) end - test :mkdir! do + test "mkdir!" do fixture = tmp_path("tmp_test") + try do refute File.exists?(fixture) assert File.mkdir!(fixture) == :ok assert File.exists?(fixture) after - File.rmdir fixture + File.rmdir(fixture) end end - test :mkdir_with_invalid_path! do + test "mkdir! with invalid path" do fixture = fixture_path("file.txt") - invalid = Path.join fixture, "test" + invalid = Path.join(fixture, "test") assert File.exists?(fixture) - assert_raise File.Error, ~r"^could not make directory #{escape invalid}: (not a directory|no such file or directory)", fn -> + + message = + ~r"\Acould not make directory #{inspect(invalid)}: (not a directory|no such file or directory)" + + assert_raise File.Error, message, fn -> File.mkdir!(invalid) end end - test :mkdir_p_with_one_directory do + test "mkdir_p with one directory" do fixture = tmp_path("tmp_test") + try do refute File.exists?(fixture) assert File.mkdir_p(fixture) == :ok assert File.exists?(fixture) after - File.rm_rf fixture + File.rm_rf(fixture) end end - test :mkdir_p_with_nested_directory_and_binary do - base = tmp_path("tmp_test") + test "mkdir_p with nested directory and binary" do + base = tmp_path("tmp_test") fixture = Path.join(base, "test") refute File.exists?(base) @@ -608,12 +1019,12 @@ defmodule FileTest do assert File.exists?(base) assert File.exists?(fixture) after - File.rm_rf base + File.rm_rf(base) end end - test :mkdir_p_with_nested_directory_and_list do - base = tmp_path("tmp_test") |> to_char_list + test "mkdir_p with nested directory and list" do + base = tmp_path("tmp_test") |> to_charlist fixture = Path.join(base, "test") refute File.exists?(base) @@ -622,12 +1033,12 @@ defmodule FileTest do assert File.exists?(base) assert File.exists?(fixture) after - File.rm_rf base + File.rm_rf(base) end end - test :mkdir_p_with_nested_directory_and_existing_parent do - base = tmp_path("tmp_test") + test "mkdir_p with nested directory and existing parent" do + base = tmp_path("tmp_test") fixture = Path.join(base, "test") File.mkdir(base) @@ -637,33 +1048,38 @@ defmodule FileTest do assert File.exists?(base) assert File.exists?(fixture) after - File.rm_rf base + File.rm_rf(base) end end - test :mkdir_p_with_invalid_path do + test "mkdir_p with invalid path" do assert File.exists?(fixture_path("file.txt")) - invalid = Path.join fixture_path("file.txt"), "test/foo" - assert io_error? File.mkdir(invalid) + invalid = Path.join(fixture_path("file.txt"), "test/foo") + assert io_error?(File.mkdir(invalid)) refute File.exists?(invalid) end - test :mkdir_p! do + test "mkdir_p!" do fixture = tmp_path("tmp_test") + try do refute File.exists?(fixture) assert File.mkdir_p!(fixture) == :ok assert File.exists?(fixture) after - File.rm_rf fixture + File.rm_rf(fixture) end end - test :mkdir_p_with_invalid_path! do + test "mkdir_p! with invalid path" do fixture = fixture_path("file.txt") - invalid = Path.join fixture, "test" + invalid = Path.join(fixture, "test") assert File.exists?(fixture) - assert_raise File.Error, ~r"^could not make directory \(with -p\) #{escape invalid}: (not a directory|no such file or directory)", fn -> + + message = + ~r"\Acould not make directory \(with -p\) #{inspect(invalid)}: (not a directory|no such file or directory)" + + assert_raise File.Error, message, fn -> File.mkdir_p!(invalid) end end @@ -677,32 +1093,32 @@ defmodule FileTest do defmodule Rm do use Elixir.FileCase - test :rm_file do + test "rm file" do fixture = tmp_path("tmp_test.txt") File.write(fixture, "test") assert File.exists?(fixture) assert File.rm(fixture) == :ok refute File.exists?(fixture) end - - test :rm_read_only_file do + + test "rm read only file" do fixture = tmp_path("tmp_test.txt") File.write(fixture, "test") assert File.exists?(fixture) - File.chmod(fixture, 0100444) + File.chmod(fixture, 0o100444) assert File.rm(fixture) == :ok refute File.exists?(fixture) end - test :rm_file_with_dir do - assert File.rm(fixture_path) == {:error, :eperm} + test "rm file with dir" do + assert File.rm(fixture_path()) == {:error, :eperm} end - test :rm_nonexistent_file do + test "rm nonexistent file" do assert File.rm('missing.txt') == {:error, :enoent} end - test :rm! do + test "rm!" do fixture = tmp_path("tmp_test.txt") File.write(fixture, "test") assert File.exists?(fixture) @@ -710,13 +1126,15 @@ defmodule FileTest do refute File.exists?(fixture) end - test :rm_with_invalid_file! do - assert_raise File.Error, "could not remove file missing.file: no such file or directory", fn -> + test "rm! with invalid file" do + message = "could not remove file \"missing.file\": no such file or directory" + + assert_raise File.Error, message, fn -> File.rm!("missing.file") end end - test :rmdir do + test "rmdir" do fixture = tmp_path("tmp_test") File.mkdir_p(fixture) assert File.dir?(fixture) @@ -724,11 +1142,11 @@ defmodule FileTest do refute File.exists?(fixture) end - test :rmdir_with_file do - assert io_error? File.rmdir(fixture_path("file.txt")) + test "rmdir with file" do + assert io_error?(File.rmdir(fixture_path("file.txt"))) end - test :rmdir! do + test "rmdir!" do fixture = tmp_path("tmp_test") File.mkdir_p(fixture) assert File.dir?(fixture) @@ -736,14 +1154,42 @@ defmodule FileTest do refute File.exists?(fixture) end - test :rmdir_with_file! do + test "rmdir! with file" do fixture = fixture_path("file.txt") - assert_raise File.Error, ~r"^could not remove directory #{escape fixture}: (not a directory|I/O error)", fn -> + message = ~r"\Acould not remove directory #{inspect(fixture)}: (not a directory|I/O error)" + + assert_raise File.Error, message, fn -> File.rmdir!(fixture) end end - test :rm_rf do + test "rmdir! error messages" do + fixture = tmp_path("tmp_test") + File.mkdir_p(fixture) + File.touch(fixture <> "/file") + + # directory is not empty + dir_not_empty_message = + "could not remove directory #{inspect(fixture)}: directory is not empty" + + assert_raise File.Error, dir_not_empty_message, fn -> + File.rmdir!(fixture) + end + + # directory does not exist + non_existent_dir = fixture <> "/non_existent_dir" + + non_existent_dir_message = + ~r"\Acould not remove directory #{inspect(non_existent_dir)}: (not a directory|no such file or directory)" + + assert_raise File.Error, non_existent_dir_message, fn -> + File.rmdir!(non_existent_dir) + end + + File.rm_rf(fixture) + end + + test "rm_rf" do fixture = tmp_path("tmp") File.mkdir(fixture) File.cp_r!(fixture_path("cp_r"), fixture) @@ -763,15 +1209,19 @@ defmodule FileTest do refute File.exists?(fixture) end - test :rm_rf_with_symlink do + test "rm_rf raises on path with null byte" do + assert_raise ArgumentError, ~r/null byte/, fn -> File.rm_rf("foo\0bar") end + end + + test "rm_rf with symlink" do from = tmp_path("tmp/from") - to = tmp_path("tmp/to") + to = tmp_path("tmp/to") File.mkdir_p!(to) File.write!(Path.join(to, "hello"), "world") :file.make_symlink(to, from) - if File.exists?(from) or not is_win? do + if File.exists?(from) or not windows?() do assert File.exists?(from) {:ok, files} = File.rm_rf(from) @@ -784,8 +1234,8 @@ defmodule FileTest do File.rm(tmp_path("tmp/from")) end - test :rm_rf_with_char_list do - fixture = tmp_path("tmp") |> to_char_list + test "rm_rf with charlist" do + fixture = tmp_path("tmp") |> to_charlist File.mkdir(fixture) File.cp_r!(fixture_path("cp_r"), fixture) @@ -804,23 +1254,23 @@ defmodule FileTest do refute File.exists?(fixture) end - test :rm_rf_with_file do + test "rm_rf with file" do fixture = tmp_path("tmp") File.write(fixture, "hello") assert File.rm_rf(fixture) == {:ok, [fixture]} end - test :rm_rf_with_unknown do + test "rm_rf with unknown" do fixture = tmp_path("tmp.unknown") assert File.rm_rf(fixture) == {:ok, []} end - test :rm_rf_with_invalid do - fixture = fixture_path "file.txt/path" + test "rm_rf with invalid" do + fixture = fixture_path("file.txt/path") assert File.rm_rf(fixture) == {:ok, []} end - test :rm_rf! do + test "rm_rf!" do fixture = tmp_path("tmp") File.mkdir(fixture) File.cp_r!(fixture_path("cp_r"), fixture) @@ -840,8 +1290,8 @@ defmodule FileTest do refute File.exists?(fixture) end - test :rm_rf_with_invalid! do - fixture = fixture_path "file.txt/path" + test "rm_rf! with invalid path" do + fixture = fixture_path("file.txt/path") assert File.rm_rf!(fixture) == [] end @@ -850,182 +1300,432 @@ defmodule FileTest do end end - test :stat do + test "stat" do {:ok, info} = File.stat(__ENV__.file) assert info.mtime end - test :stat! do + test "stat!" do assert File.stat!(__ENV__.file).mtime end - test :stat_with_invalid_file do + test "stat with invalid file" do assert {:error, _} = File.stat("./invalid_file") end - test :stat_with_invalid_file! do + test "stat! with invalid_file" do assert_raise File.Error, fn -> File.stat!("./invalid_file") end end - test :io_stream_utf8 do - src = File.open! fixture_path("file.txt"), [:utf8] - dest = tmp_path("tmp_test.txt") + test "lstat" do + {:ok, info} = File.lstat(__ENV__.file) + assert info.mtime + end - try do - stream = IO.stream(src, :line) - File.open dest, [:write], fn(target) -> - Enum.into stream, IO.stream(target, :line), &String.replace(&1, "O", "A") - end - assert File.read(dest) == {:ok, "FAA\n"} - after - File.rm(dest) + test "lstat!" do + assert File.lstat!(__ENV__.file).mtime + end + + test "lstat with invalid file" do + invalid_file = tmp_path("invalid_file") + assert {:error, _} = File.lstat(invalid_file) + end + + test "lstat! with invalid file" do + invalid_file = tmp_path("invalid_file") + + assert_raise File.Error, fn -> + File.lstat!(invalid_file) end end - test :io_stream do - src = File.open! fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") + test "lstat with dangling symlink" do + invalid_file = tmp_path("invalid_file") + dest = tmp_path("dangling_symlink") + File.ln_s(invalid_file, dest) try do - stream = IO.binstream(src, :line) - File.open dest, [:write], fn(target) -> - Enum.into stream, IO.binstream(target, :line), &String.replace(&1, "O", "A") - end - assert File.read(dest) == {:ok, "FAA\n"} + assert {:ok, info} = File.lstat(dest) + assert info.type == :symlink after File.rm(dest) end end - test :stream_map do - src = fixture_path("file.txt") - stream = File.stream!(src) - assert %File.Stream{} = stream - assert stream.modes == [:raw, :read_ahead, :binary] - assert stream.raw - assert stream.line_or_bytes == :line - - src = fixture_path("file.txt") - stream = File.stream!(src, [:utf8], 10) - assert %File.Stream{} = stream - assert stream.modes == [{:encoding, :utf8}, :binary] - refute stream.raw - assert stream.line_or_bytes == 10 - end - - test :stream_line_utf8 do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") + test "lstat! with dangling symlink" do + invalid_file = tmp_path("invalid_file") + dest = tmp_path("dangling_symlink") + File.ln_s(invalid_file, dest) try do - stream = File.stream!(src) - File.open dest, [:write, :utf8], fn(target) -> - Enum.each stream, fn(line) -> - IO.write target, String.replace(line, "O", "A") - end - end - assert File.read(dest) == {:ok, "FAA\n"} + assert File.lstat!(dest).type == :symlink after File.rm(dest) end end - test :stream_bytes_utf8 do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") + test "read_link with regular file" do + dest = tmp_path("symlink") + File.touch(dest) try do - stream = File.stream!(src, [:utf8], 1) - File.open dest, [:write], fn(target) -> - Enum.each stream, fn(line) -> - IO.write target, String.replace(line, "OO", "AA") - end - end - assert File.read(dest) == {:ok, "FOO\n"} + assert File.read_link(dest) == {:error, :einval} after File.rm(dest) end end - test :stream_line do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") + test "read_link with nonexistent file" do + dest = tmp_path("does_not_exist") + assert File.read_link(dest) == {:error, :enoent} + end - try do - stream = File.stream!(src) - File.open dest, [:write], fn(target) -> - Enum.each stream, fn(line) -> - IO.write target, String.replace(line, "O", "A") - end + test "read_link! with nonexistent file" do + dest = tmp_path("does_not_exist") + assert_raise File.Error, fn -> File.read_link!(dest) end + end + + unless windows?() do + test "read_link with symlink" do + target = tmp_path("does_not_need_to_exist") + dest = tmp_path("symlink") + File.ln_s(target, dest) + + try do + assert File.read_link(dest) == {:ok, target} + after + File.rm(dest) + end + end + + test "read_link! with symlink" do + target = tmp_path("does_not_need_to_exist") + dest = tmp_path("symlink") + File.ln_s(target, dest) + + try do + assert File.read_link!(dest) == target + after + File.rm(dest) end - assert File.read(dest) == {:ok, "FAA\n"} - after - File.rm(dest) end end - test :stream_bytes do - src = fixture_path("file.txt") + test "IO stream UTF-8" do + src = File.open!(fixture_path("file.txt"), [:utf8]) dest = tmp_path("tmp_test.txt") try do - stream = File.stream!(src, [], 1) - File.open dest, [:write], fn(target) -> - Enum.each stream, fn(line) -> - IO.write target, String.replace(line, "OO", "AA") - end - end - assert File.read(dest) == {:ok, "FOO\n"} + stream = IO.stream(src, :line) + + File.open(dest, [:write], fn target -> + Enum.into(stream, IO.stream(target, :line), &String.replace(&1, "O", "A")) + end) + + assert File.read(dest) == {:ok, "FAA\n"} after File.rm(dest) end end - test :stream_into do - src = fixture_path("file.txt") + test "IO stream" do + src = File.open!(fixture_path("file.txt")) dest = tmp_path("tmp_test.txt") try do - refute File.exists?(dest) + stream = IO.binstream(src, :line) - original = File.stream!(dest) - stream = File.stream!(src) - |> Stream.map(&String.replace(&1, "O", "A")) - |> Enum.into(original) + File.open(dest, [:write], fn target -> + Enum.into(stream, IO.binstream(target, :line), &String.replace(&1, "O", "A")) + end) - assert stream == original assert File.read(dest) == {:ok, "FAA\n"} after File.rm(dest) end end - test :stream_into_append do - src = fixture_path("file.txt") - dest = tmp_path("tmp_test.txt") + describe "file stream" do + test "returns a struct" do + src = fixture_path("file.txt") + stream = File.stream!(src) + assert %File.Stream{} = stream + assert stream.modes == [:raw, :read_ahead, :binary] + assert stream.raw + assert stream.line_or_bytes == :line + + stream = File.stream!(src, read_ahead: false) + assert %File.Stream{} = stream + assert stream.modes == [:raw, :binary] + assert stream.raw + + stream = File.stream!(src, read_ahead: 5000) + assert %File.Stream{} = stream + assert stream.modes == [:raw, {:read_ahead, 5000}, :binary] + assert stream.raw + + stream = File.stream!(src, [:utf8], 10) + assert %File.Stream{} = stream + assert stream.modes == [{:encoding, :utf8}, :binary] + refute stream.raw + assert stream.line_or_bytes == 10 + end + + test "counts bytes/characters" do + src = fixture_path("file.txt") + stream = File.stream!(src) + assert Enum.count(stream) == 1 - try do - refute File.exists?(dest) - original = File.stream!(dest, [:append]) + stream = File.stream!(src, [:utf8]) + assert Enum.count(stream) == 1 + + stream = File.stream!(src, [], 2) + assert Enum.count(stream) == 2 + end - File.stream!(src, [:append]) - |> Stream.map(&String.replace(&1, "O", "A")) - |> Enum.into(original) + test "reads and writes lines" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + stream = File.stream!(src) + + File.open(dest, [:write], fn target -> + Enum.each(stream, fn line -> + IO.write(target, String.replace(line, "O", "A")) + end) + end) + + assert File.read(dest) == {:ok, "FAA\n"} + after + File.rm(dest) + end + end + + test "reads and writes bytes" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + stream = File.stream!(src, [], 1) + + File.open(dest, [:write], fn target -> + Enum.each(stream, fn <> -> + IO.write(target, <>) + end) + end) + + assert File.read(dest) == {:ok, "GPP\v"} + after + File.rm(dest) + end + end + + test "is collectable" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + refute File.exists?(dest) + original = File.stream!(dest) - File.stream!(src, [:append]) - |> Enum.into(original) + stream = + File.stream!(src) + |> Stream.map(&String.replace(&1, "O", "A")) + |> Enum.into(original) + + assert stream == original + assert File.read(dest) == {:ok, "FAA\n"} + after + File.rm(dest) + end + end + + test "is collectable with append" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + refute File.exists?(dest) + original = File.stream!(dest, [:append]) + + File.stream!(src, [:append]) + |> Stream.map(&String.replace(&1, "O", "A")) + |> Enum.into(original) + + File.stream!(src, [:append]) + |> Enum.into(original) + + assert File.read(dest) == {:ok, "FAA\nFOO\n"} + after + File.rm(dest) + end + end + + test "keeps BOM when raw" do + src = fixture_path("utf8_bom.txt") + + assert src + |> File.stream!([]) + |> Enum.take(1) == [<<239, 187, 191>> <> "Русский\n"] + + assert src + |> File.stream!([], 1) + |> Enum.take(5) == [<<239>>, <<187>>, <<191>>, <<208>>, <<160>>] + + assert src |> File.stream!([]) |> Enum.count() == 2 + assert src |> File.stream!([], 1) |> Enum.count() == 22 + end + + test "trims BOM via option when raw" do + src = fixture_path("utf8_bom.txt") + + assert src + |> File.stream!([:trim_bom]) + |> Enum.take(1) == ["Русский\n"] + + assert src + |> File.stream!([:trim_bom], 1) + |> Enum.take(5) == [<<208>>, <<160>>, <<209>>, <<131>>, <<209>>] + + assert src |> File.stream!([:trim_bom]) |> Enum.count() == 2 + assert src |> File.stream!([:trim_bom], 1) |> Enum.count() == 19 + end + + test "keeps BOM with utf8 encoding" do + src = fixture_path("utf8_bom.txt") + + assert src + |> File.stream!([{:encoding, :utf8}]) + |> Enum.take(1) == [<<239, 187, 191>> <> "Русский\n"] + + assert src + |> File.stream!([{:encoding, :utf8}], 1) + |> Enum.take(9) == ["\uFEFF", "Р", "у", "с", "с", "к", "и", "й", "\n"] + end - assert File.read(dest) == {:ok, "FAA\nFOO\n"} + test "trims BOM via option with utf8 encoding" do + src = fixture_path("utf8_bom.txt") + + assert src + |> File.stream!([{:encoding, :utf8}, :trim_bom]) + |> Enum.take(1) == ["Русский\n"] + + assert src + |> File.stream!([{:encoding, :utf8}, :trim_bom], 1) + |> Enum.take(8) == ["Р", "у", "с", "с", "к", "и", "й", "\n"] + end + + test "keeps BOM with UTF16 BE" do + src = fixture_path("utf16_be_bom.txt") + + assert src + |> File.stream!([{:encoding, {:utf16, :big}}]) + |> Enum.take(1) == ["\uFEFFРусский\n"] + end + + test "keeps BOM with UTF16 LE" do + src = fixture_path("utf16_le_bom.txt") + + assert src + |> File.stream!([{:encoding, {:utf16, :little}}]) + |> Enum.take(1) == ["\uFEFFРусский\n"] + end + + test "trims BOM via option with utf16 BE encoding" do + src = fixture_path("utf16_be_bom.txt") + + assert src + |> File.stream!([{:encoding, {:utf16, :big}}, :trim_bom]) + |> Enum.take(1) == ["Русский\n"] + + assert src + |> File.stream!([{:encoding, {:utf16, :big}}, :trim_bom], 1) + |> Enum.take(8) == ["Р", "у", "с", "с", "к", "и", "й", "\n"] + end + + test "trims BOM via option with utf16 LE encoding" do + src = fixture_path("utf16_le_bom.txt") + + assert src + |> File.stream!([{:encoding, {:utf16, :little}}, :trim_bom]) + |> Enum.take(1) == ["Русский\n"] + + assert src + |> File.stream!([{:encoding, {:utf16, :little}}, :trim_bom], 1) + |> Enum.take(8) == ["Р", "у", "с", "с", "к", "и", "й", "\n"] + end + + test "reads and writes line by line in UTF-8" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + stream = File.stream!(src) + + File.open(dest, [:write, :utf8], fn target -> + Enum.each(stream, fn line -> + IO.write(target, String.replace(line, "O", "A")) + end) + end) + + assert File.read(dest) == {:ok, "FAA\n"} + after + File.rm(dest) + end + end + + test "reads and writes character in UTF-8" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + stream = File.stream!(src, [:utf8], 1) + + File.open(dest, [:write], fn target -> + Enum.each(stream, fn <> -> + IO.write(target, <>) + end) + end) + + assert File.read(dest) == {:ok, "GPP\v"} + after + File.rm(dest) + end + end + end + + test "ln" do + existing = fixture_path("file.txt") + new = tmp_path("tmp_test.txt") + + try do + refute File.exists?(new) + assert File.ln(existing, new) == :ok + assert File.read(new) == {:ok, "FOO\n"} after - File.rm(dest) + File.rm(new) end end - test :ln_s do - existing = fixture_path("file.txt") + test "ln with existing destination" do + existing = fixture_path("file.txt") + assert File.ln(existing, existing) == {:error, :eexist} + end + + test "ln! with existing destination" do + assert_raise File.LinkError, fn -> + existing = fixture_path("file.txt") + File.ln!(existing, existing) + end + end + + test "ln_s" do + existing = fixture_path("file.txt") new = tmp_path("tmp_test.txt") + try do refute File.exists?(new) assert File.ln_s(existing, new) == :ok @@ -1035,26 +1735,78 @@ defmodule FileTest do end end - test :ln_s_with_existing_destination do - existing = fixture_path("file.txt") + test "ln_s with existing destination" do + existing = fixture_path("file.txt") assert File.ln_s(existing, existing) == {:error, :eexist} end - test :copy do - src = fixture_path("file.txt") + test "ln_s! with existing destination" do + existing = fixture_path("file.txt") + + assert_raise File.LinkError, fn -> + File.ln_s!(existing, existing) + end + end + + test "copy" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + + try do + refute File.exists?(dest) + assert File.copy(src, dest) == {:ok, 4} + assert File.read(dest) == {:ok, "FOO\n"} + after + File.rm(dest) + end + end + + test "copy with an io_device" do + {:ok, src} = File.open(fixture_path("file.txt")) dest = tmp_path("tmp_test.txt") + try do refute File.exists?(dest) assert File.copy(src, dest) == {:ok, 4} assert File.read(dest) == {:ok, "FOO\n"} after + File.close(src) File.rm(dest) end end - test :copy_with_bytes_count do - src = fixture_path("file.txt") + test "copy with raw io_device" do + {:ok, src} = File.open(fixture_path("file.txt"), [:raw]) dest = tmp_path("tmp_test.txt") + + try do + refute File.exists?(dest) + assert File.copy(src, dest) == {:ok, 4} + assert File.read(dest) == {:ok, "FOO\n"} + after + File.close(src) + File.rm(dest) + end + end + + test "copy with ram io_device" do + {:ok, src} = File.open("FOO\n", [:ram]) + dest = tmp_path("tmp_test.txt") + + try do + refute File.exists?(dest) + assert File.copy(src, dest) == {:ok, 4} + assert File.read(dest) == {:ok, "FOO\n"} + after + File.close(src) + File.rm(dest) + end + end + + test "copy with bytes count" do + src = fixture_path("file.txt") + dest = tmp_path("tmp_test.txt") + try do refute File.exists?(dest) assert File.copy(src, dest, 2) == {:ok, 2} @@ -1064,15 +1816,16 @@ defmodule FileTest do end end - test :copy_with_invalid_file do - src = fixture_path("invalid.txt") + test "copy with invalid file" do + src = fixture_path("invalid.txt") dest = tmp_path("tmp_test.txt") assert File.copy(src, dest, 2) == {:error, :enoent} end - test :copy! do - src = fixture_path("file.txt") + test "copy!" do + src = fixture_path("file.txt") dest = tmp_path("tmp_test.txt") + try do refute File.exists?(dest) assert File.copy!(src, dest) == 4 @@ -1082,9 +1835,10 @@ defmodule FileTest do end end - test :copy_with_bytes_count! do - src = fixture_path("file.txt") + test "copy! with bytes count" do + src = fixture_path("file.txt") dest = tmp_path("tmp_test.txt") + try do refute File.exists?(dest) assert File.copy!(src, dest, 2) == 2 @@ -1094,55 +1848,60 @@ defmodule FileTest do end end - test :copy_with_invalid_file! do - src = fixture_path("invalid.txt") + test "copy! with invalid file" do + src = fixture_path("invalid.txt") dest = tmp_path("tmp_test.txt") - assert_raise File.CopyError, "could not copy from #{src} to #{dest}: no such file or directory", fn -> + message = "could not copy from #{inspect(src)} to #{inspect(dest)}: no such file or directory" + + assert_raise File.CopyError, message, fn -> File.copy!(src, dest, 2) end end - test :cwd_and_cd do - {:ok, current} = File.cwd + test "cwd and cd" do + {:ok, current} = File.cwd() + try do - assert File.cd(fixture_path) == :ok + assert File.cd(fixture_path()) == :ok assert File.exists?("file.txt") after File.cd!(current) end end - if :file.native_name_encoding == :utf8 do - test :cwd_and_cd_with_utf8 do + if :file.native_name_encoding() == :utf8 do + test "cwd and cd with UTF-8" do File.mkdir_p(tmp_path("héllò")) File.cd!(tmp_path("héllò"), fn -> - assert Path.basename(File.cwd!) == "héllò" + assert Path.basename(File.cwd!()) == "héllò" end) after - File.rm_rf tmp_path("héllò") + File.rm_rf(tmp_path("héllò")) end end - test :invalid_cd do - assert io_error? File.cd(fixture_path("file.txt")) + test "invalid cd" do + assert io_error?(File.cd(fixture_path("file.txt"))) end - test :invalid_cd! do - message = ~r"^could not set current working directory to #{escape fixture_path("file.txt")}: (not a directory|no such file or directory)" + test "invalid_cd!" do + message = + ~r"\Acould not set current working directory to #{inspect(fixture_path("file.txt"))}: (not a directory|no such file or directory|I/O error)" + assert_raise File.Error, message, fn -> File.cd!(fixture_path("file.txt")) end end - test :cd_with_function do - assert File.cd!(fixture_path, fn -> - assert File.exists?("file.txt") - :cd_result - end) == :cd_result + test "cd with function" do + assert File.cd!(fixture_path(), fn -> + assert File.exists?("file.txt") + :cd_result + end) == :cd_result end - test :touch_with_no_file do + test "touch with no file" do fixture = tmp_path("tmp_test.txt") time = {{2010, 4, 17}, {14, 0, 0}} @@ -1156,135 +1915,150 @@ defmodule FileTest do end end - test :touch_with_timestamp do - fixture = tmp_path("tmp_test.txt") + test "touch with Erlang timestamp" do + fixture = tmp_path("tmp_erlang_touch.txt") try do - assert File.touch!(fixture) == :ok + assert File.touch!(fixture, :erlang.universaltime()) == :ok stat = File.stat!(fixture) - assert File.touch!(fixture, last_year) == :ok + assert File.touch!(fixture, last_year()) == :ok assert stat.mtime > File.stat!(fixture).mtime after File.rm(fixture) end end - test :touch_with_dir do - assert File.touch(fixture_path) == :ok + test "touch with posix timestamp" do + fixture = tmp_path("tmp_posix_touch.txt") + + try do + assert File.touch!(fixture, System.os_time(:second)) == :ok + stat = File.stat!(fixture) + + assert File.touch!(fixture, last_year()) == :ok + assert stat.mtime > File.stat!(fixture).mtime + after + File.rm(fixture) + end end - test :touch_with_failure do - fixture = fixture_path("file.txt/bar") - assert io_error? File.touch(fixture) + test "touch with dir" do + assert File.touch(fixture_path()) == :ok end - test :touch_with_success! do - assert File.touch!(fixture_path) == :ok + test "touch with failure" do + fixture = fixture_path("file.txt/bar") + assert io_error?(File.touch(fixture)) end - test :touch_with_failure! do + test "touch! raises" do fixture = fixture_path("file.txt/bar") - assert_raise File.Error, ~r"could not touch #{escape fixture}: (not a directory|no such file or directory)", fn -> + + message = + ~r"\Acould not touch #{inspect(fixture)}: (not a directory|no such file or directory)" + + assert_raise File.Error, message, fn -> File.touch!(fixture) end end - test :chmod_with_success do + test "chmod with success" do fixture = tmp_path("tmp_test.txt") File.touch(fixture) + try do - assert File.chmod(fixture, 0100666) == :ok + assert File.chmod(fixture, 0o100666) == :ok stat = File.stat!(fixture) - assert stat.mode == 0100666 + assert stat.mode == 0o100666 - unless is_win? do - assert File.chmod(fixture, 0100777) == :ok + unless windows?() do + assert File.chmod(fixture, 0o100777) == :ok stat = File.stat!(fixture) - assert stat.mode == 0100777 + assert stat.mode == 0o100777 end after File.rm(fixture) end end - test :chmod_with_success! do + test "chmod! with success" do fixture = tmp_path("tmp_test.txt") File.touch(fixture) + try do - assert File.chmod!(fixture, 0100666) == :ok + assert File.chmod!(fixture, 0o100666) == :ok stat = File.stat!(fixture) - assert stat.mode == 0100666 + assert stat.mode == 0o100666 - unless is_win? do - assert File.chmod!(fixture, 0100777) == :ok + unless windows?() do + assert File.chmod!(fixture, 0o100777) == :ok stat = File.stat!(fixture) - assert stat.mode == 0100777 + assert stat.mode == 0o100777 end after File.rm(fixture) end end - test :chmod_with_failure do + test "chmod with failure" do fixture = tmp_path("tmp_test.txt") File.rm(fixture) - assert File.chmod(fixture, 0100777) == {:error,:enoent} + assert File.chmod(fixture, 0o100777) == {:error, :enoent} end - test :chmod_with_failure! do + test "chmod! with failure" do fixture = tmp_path("tmp_test.txt") File.rm(fixture) - message = ~r"could not change mode for #{escape fixture}: no such file or directory" + message = ~r"could not change mode for #{inspect(fixture)}: no such file or directory" + assert_raise File.Error, message, fn -> - File.chmod!(fixture, 0100777) + File.chmod!(fixture, 0o100777) end end - test :chgrp_with_failure do + test "chgrp with failure" do fixture = tmp_path("tmp_test.txt") File.rm(fixture) - assert File.chgrp(fixture, 1) == {:error,:enoent} + assert File.chgrp(fixture, 1) == {:error, :enoent} end - test :chgrp_with_failure! do + test "chgrp! with failure" do fixture = tmp_path("tmp_test.txt") File.rm(fixture) - message = ~r"could not change group for #{escape fixture}: no such file or directory" + message = ~r"could not change group for #{inspect(fixture)}: no such file or directory" + assert_raise File.Error, message, fn -> File.chgrp!(fixture, 1) end end - test :chown_with_failure do + test "chown with failure" do fixture = tmp_path("tmp_test.txt") File.rm(fixture) - assert File.chown(fixture, 1) == {:error,:enoent} + assert File.chown(fixture, 1) == {:error, :enoent} end - test :chown_with_failure! do + test "chown! with failure" do fixture = tmp_path("tmp_test.txt") File.rm(fixture) - message = ~r"could not change owner for #{escape fixture}: no such file or directory" + message = ~r"could not change owner for #{inspect(fixture)}: no such file or directory" + assert_raise File.Error, message, fn -> File.chown!(fixture, 1) end end defp last_year do - last_year :calendar.local_time - end - - defp last_year({{year, month, day}, time}) do - {{year - 1, month, day}, time} + System.os_time(:second) - 365 * 24 * 60 * 60 end defp io_error?(result) do diff --git a/lib/elixir/test/elixir/fixtures/at_exit.exs b/lib/elixir/test/elixir/fixtures/at_exit.exs index 78460b222ba..f0150a4333b 100644 --- a/lib/elixir/test/elixir/fixtures/at_exit.exs +++ b/lib/elixir/test/elixir/fixtures/at_exit.exs @@ -1,8 +1,9 @@ defmodule AtExit do def at_exit(str) do - System.at_exit fn(_) -> IO.write(str) end + System.at_exit(fn _ -> IO.write(str) end) end end -System.at_exit fn(status) -> IO.puts "cruel world with status #{status}" end + +System.at_exit(fn status -> IO.puts("cruel world with status #{status}") end) AtExit.at_exit("goodbye ") -exit(0) \ No newline at end of file +exit({:shutdown, 1}) diff --git a/lib/elixir/test/elixir/fixtures/checker_warning.exs b/lib/elixir/test/elixir/fixtures/checker_warning.exs new file mode 100644 index 00000000000..a9c0cab9fcc --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/checker_warning.exs @@ -0,0 +1,3 @@ +defmodule CodeTest.CheckerWarning do + def foo(x) when is_atom(x) and is_list(x), do: x +end diff --git a/lib/elixir/test/elixir/fixtures/code_sample.exs b/lib/elixir/test/elixir/fixtures/code_sample.exs index f1895089889..511623e3a7c 100644 --- a/lib/elixir/test/elixir/fixtures/code_sample.exs +++ b/lib/elixir/test/elixir/fixtures/code_sample.exs @@ -1,3 +1,3 @@ # Some Comments var = 1 + 2 -var \ No newline at end of file +var diff --git a/lib/elixir/test/elixir/fixtures/compile_sample.ex b/lib/elixir/test/elixir/fixtures/compile_sample.ex index 7cbd92b5a29..78761880f11 100644 --- a/lib/elixir/test/elixir/fixtures/compile_sample.ex +++ b/lib/elixir/test/elixir/fixtures/compile_sample.ex @@ -1 +1 @@ -defmodule CompileSample, do: nil \ No newline at end of file +defmodule(CompileSample, do: nil) diff --git a/lib/mix/test/fixtures/configs/bad_app.exs b/lib/elixir/test/elixir/fixtures/configs/bad_app.exs similarity index 100% rename from lib/mix/test/fixtures/configs/bad_app.exs rename to lib/elixir/test/elixir/fixtures/configs/bad_app.exs diff --git a/lib/elixir/test/elixir/fixtures/configs/bad_import.exs b/lib/elixir/test/elixir/fixtures/configs/bad_import.exs new file mode 100644 index 00000000000..d03906ea1e9 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/bad_import.exs @@ -0,0 +1,2 @@ +import Config +import_config "bad_root.exs" diff --git a/lib/elixir/test/elixir/fixtures/configs/env.exs b/lib/elixir/test/elixir/fixtures/configs/env.exs new file mode 100644 index 00000000000..739e6236941 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/env.exs @@ -0,0 +1,2 @@ +import Config +config :my_app, env: config_env(), target: config_target() diff --git a/lib/elixir/test/elixir/fixtures/configs/good_config.exs b/lib/elixir/test/elixir/fixtures/configs/good_config.exs new file mode 100644 index 00000000000..dd2bb471195 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/good_config.exs @@ -0,0 +1,2 @@ +import Config +config :my_app, :key, :value diff --git a/lib/mix/test/fixtures/configs/good_import.exs b/lib/elixir/test/elixir/fixtures/configs/good_import.exs similarity index 71% rename from lib/mix/test/fixtures/configs/good_import.exs rename to lib/elixir/test/elixir/fixtures/configs/good_import.exs index 759edb0108a..549312a2db6 100644 --- a/lib/mix/test/fixtures/configs/good_import.exs +++ b/lib/elixir/test/elixir/fixtures/configs/good_import.exs @@ -1,3 +1,3 @@ -use Mix.Config +import Config import_config "good_config.exs" :done diff --git a/lib/elixir/test/elixir/fixtures/configs/good_kw.exs b/lib/elixir/test/elixir/fixtures/configs/good_kw.exs new file mode 100644 index 00000000000..0c13b63b986 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/good_kw.exs @@ -0,0 +1 @@ +[my_app: [key: :value]] diff --git a/lib/elixir/test/elixir/fixtures/configs/imports_recursive.exs b/lib/elixir/test/elixir/fixtures/configs/imports_recursive.exs new file mode 100644 index 00000000000..c71c4418ec6 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/imports_recursive.exs @@ -0,0 +1,2 @@ +import Config +import_config "recursive.exs" diff --git a/lib/elixir/test/elixir/fixtures/configs/kernel.exs b/lib/elixir/test/elixir/fixtures/configs/kernel.exs new file mode 100644 index 00000000000..5020372a5d8 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/kernel.exs @@ -0,0 +1,3 @@ +import Config +config :kernel, :elixir_reboot, true +config :elixir_reboot, :key, :value diff --git a/lib/elixir/test/elixir/fixtures/configs/nested.exs b/lib/elixir/test/elixir/fixtures/configs/nested.exs new file mode 100644 index 00000000000..5570ef976d9 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/nested.exs @@ -0,0 +1,2 @@ +import Config +config :app, Repo, key: [nested: true] diff --git a/lib/elixir/test/elixir/fixtures/configs/recursive.exs b/lib/elixir/test/elixir/fixtures/configs/recursive.exs new file mode 100644 index 00000000000..eb25bb4076d --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/configs/recursive.exs @@ -0,0 +1,2 @@ +import Config +import_config "imports_recursive.exs" diff --git a/lib/elixir/test/elixir/fixtures/consolidation/sample.ex b/lib/elixir/test/elixir/fixtures/consolidation/sample.ex new file mode 100644 index 00000000000..ee7edea3fbc --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/consolidation/sample.ex @@ -0,0 +1,7 @@ +defprotocol Protocol.ConsolidationTest.Sample do + @type t :: any + @doc "Ok" + @deprecated "Reason" + @spec ok(t) :: boolean + def ok(term) +end diff --git a/lib/elixir/test/elixir/fixtures/consolidation/with_any.ex b/lib/elixir/test/elixir/fixtures/consolidation/with_any.ex new file mode 100644 index 00000000000..42912e5633c --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/consolidation/with_any.ex @@ -0,0 +1,5 @@ +defprotocol Protocol.ConsolidationTest.WithAny do + @fallback_to_any true + @doc "Ok" + def ok(term) +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/boolean_check.ex b/lib/elixir/test/elixir/fixtures/dialyzer/boolean_check.ex new file mode 100644 index 00000000000..e8123315370 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/boolean_check.ex @@ -0,0 +1,9 @@ +defmodule Dialyzer.BooleanCheck do + def and_check(arg) when is_boolean(arg) do + arg and arg + end + + def or_check(arg) when is_boolean(arg) do + arg or arg + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/callback.ex b/lib/elixir/test/elixir/fixtures/dialyzer/callback.ex new file mode 100644 index 00000000000..0eeff5f5c3d --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/callback.ex @@ -0,0 +1,14 @@ +defmodule Dialyzer.Callback do + @callback required(atom) :: atom + @callback required(list) :: list +end + +defmodule Dialyzer.Callback.ImplAtom do + @behaviour Dialyzer.Callback + def required(:ok), do: :ok +end + +defmodule Dialyzer.Callback.ImplList do + @behaviour Dialyzer.Callback + def required([a, b]), do: [b, a] +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/cond.ex b/lib/elixir/test/elixir/fixtures/dialyzer/cond.ex new file mode 100644 index 00000000000..9419b9b8e36 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/cond.ex @@ -0,0 +1,27 @@ +defmodule Dialyzer.Cond do + def one_boolean do + cond do + true -> :ok + end + end + + def two_boolean do + cond do + List.flatten([]) == [] -> :ok + true -> :ok + end + end + + def one_otherwise do + cond do + :otherwise -> :ok + end + end + + def two_otherwise do + cond do + List.flatten([]) == [] -> :ok + :otherwise -> :ok + end + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/defmacrop.ex b/lib/elixir/test/elixir/fixtures/dialyzer/defmacrop.ex new file mode 100644 index 00000000000..e8d0a42ce7d --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/defmacrop.ex @@ -0,0 +1,11 @@ +defmodule Dialyzer.Defmacrop do + defmacrop good_macro(id) do + quote do + {:good, {:good_macro, unquote(id)}} + end + end + + def run() do + good_macro("Not So Bad") + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/for_bitstring.ex b/lib/elixir/test/elixir/fixtures/dialyzer/for_bitstring.ex new file mode 100644 index 00000000000..1d22de458e6 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/for_bitstring.ex @@ -0,0 +1,5 @@ +defmodule Dialyzer.ForBitstring do + def foo() do + for a <- 1..3, into: "", do: <> + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/for_boolean_check.ex b/lib/elixir/test/elixir/fixtures/dialyzer/for_boolean_check.ex new file mode 100644 index 00000000000..968efe7873e --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/for_boolean_check.ex @@ -0,0 +1,7 @@ +defmodule Dialyzer.ForBooleanCheck do + def foo(enum, potential) when is_binary(potential) do + for element <- enum, string = Atom.to_string(element), string == potential do + element + end + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/is_struct.ex b/lib/elixir/test/elixir/fixtures/dialyzer/is_struct.ex new file mode 100644 index 00000000000..c61a6a1b1b3 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/is_struct.ex @@ -0,0 +1,9 @@ +defmodule Dialyzer.IsStruct do + def map_literal_atom_literal() do + is_struct(%Macro.Env{}, Macro.Env) + end + + def arg_atom_literal(arg) do + is_struct(arg, Macro.Env) + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/macrocallback.ex b/lib/elixir/test/elixir/fixtures/dialyzer/macrocallback.ex new file mode 100644 index 00000000000..42bd2bc9bc7 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/macrocallback.ex @@ -0,0 +1,11 @@ +defmodule Dialyzer.Macrocallback do + @macrocallback required(atom) :: Macro.t() + @macrocallback optional(atom) :: Macro.t() + @optional_callbacks [optional: 1] +end + +defmodule Dialyzer.Macrocallback.Impl do + @behaviour Dialyzer.Macrocallback + defmacro required(var), do: Macro.expand(var, __CALLER__) + defmacro optional(var), do: Macro.expand(var, __CALLER__) +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/protocol_opaque.ex b/lib/elixir/test/elixir/fixtures/dialyzer/protocol_opaque.ex new file mode 100644 index 00000000000..d6a00f31229 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/protocol_opaque.ex @@ -0,0 +1,29 @@ +defmodule Dialyzer.ProtocolOpaque do + def circus() do + duck = Dialyzer.ProtocolOpaque.Duck.new() + Dialyzer.ProtocolOpaque.Entity.speak(duck) + end +end + +defprotocol Dialyzer.ProtocolOpaque.Entity do + @fallback_to_any true + def speak(entity) +end + +defmodule Dialyzer.ProtocolOpaque.Duck do + @opaque t :: %__MODULE__{feathers: :white_and_grey} + defstruct feathers: :white_and_grey + + @spec new :: t + def new(), do: %__MODULE__{} + + defimpl Dialyzer.ProtocolOpaque.Entity do + def speak(%Dialyzer.ProtocolOpaque.Duck{}), do: "Quack!" + end +end + +defimpl Dialyzer.ProtocolOpaque.Entity, for: Any do + def speak(_any) do + "I can be anything" + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/raise.ex b/lib/elixir/test/elixir/fixtures/dialyzer/raise.ex new file mode 100644 index 00000000000..808ba3b0233 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/raise.ex @@ -0,0 +1,21 @@ +defmodule Dialyzer.Raise do + defexception [:message] + + def exception_var() do + ex = %Dialyzer.Raise{} + raise ex + end + + def exception_var(ex = %Dialyzer.Raise{}) do + raise ex + end + + def string_var() do + string = "hello" + raise string + end + + def string_var(string) when is_binary(string) do + raise string + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/remote_call.ex b/lib/elixir/test/elixir/fixtures/dialyzer/remote_call.ex new file mode 100644 index 00000000000..e93d128759b --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/remote_call.ex @@ -0,0 +1,29 @@ +defmodule Dialyzer.RemoteCall do + _ = Application.load(:dialyzer) + + case Application.spec(:dialyzer, :vsn) do + ~c(2.) ++ _ -> + @dialyzer {:no_fail_call, [map_var: 0]} + + three when three < ~c(3.0.2) -> + # regression introduced in 3.0 for map warnings fixed in 3.0.2 + @dialyzer {:no_match, [map_var: 0, mod_var: 0]} + + _ -> + :ok + end + + def map_var() do + map = %{key: 1} + map.key + end + + def map_var(map) when is_map(map) do + map.key + end + + def mod_var() do + module = Hello + module.fun() + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/rewrite.ex b/lib/elixir/test/elixir/fixtures/dialyzer/rewrite.ex new file mode 100644 index 00000000000..129f0bbea68 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/rewrite.ex @@ -0,0 +1,9 @@ +defmodule Dialyzer.Rewrite do + def interpolation do + "foo #{:a}" + end + + def reverse do + Enum.reverse(1..3) + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/struct_update.ex b/lib/elixir/test/elixir/fixtures/dialyzer/struct_update.ex new file mode 100644 index 00000000000..fbacdab6778 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/struct_update.ex @@ -0,0 +1,7 @@ +defmodule Dialyzer.StructUpdate do + defstruct [:foo] + + def update(%__MODULE__{} = struct) do + %__MODULE__{struct | foo: :bar} + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/try.ex b/lib/elixir/test/elixir/fixtures/dialyzer/try.ex new file mode 100644 index 00000000000..69f9ca4b45f --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/try.ex @@ -0,0 +1,9 @@ +defmodule Dialyzer.Try do + def rescue_error do + try do + :erlang.error(:badarg) + rescue + e in ErlangError -> {:ok, e} + end + end +end diff --git a/lib/elixir/test/elixir/fixtures/dialyzer/with.ex b/lib/elixir/test/elixir/fixtures/dialyzer/with.ex new file mode 100644 index 00000000000..e4a9a0eb87b --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/dialyzer/with.ex @@ -0,0 +1,45 @@ +defmodule Dialyzer.With do + def with_else do + with :ok <- ok_or_error(), + :ok <- ok_or_other_error(), + :ok <- ok_or_tuple_error(), + :ok <- ok_or_tuple_list_error() do + :ok + else + :error -> + :error + + :other_error -> + :other_error + + {:error, msg} when is_list(msg) or is_tuple(msg) -> + :error + + {:error, msg} when is_list(msg) when is_tuple(msg) -> + :error + + {:error, _msg} -> + :error + end + end + + @spec ok_or_error() :: :ok | :error + defp ok_or_error do + Enum.random([:ok, :error]) + end + + @spec ok_or_other_error() :: :ok | :other_error + defp ok_or_other_error do + Enum.random([:ok, :other_error]) + end + + @spec ok_or_tuple_error() :: :ok | {:error, :err} + defp ok_or_tuple_error do + Enum.random([:ok, {:error, :err}]) + end + + @spec ok_or_tuple_list_error() :: :ok | {:error, [:err]} + defp ok_or_tuple_list_error do + Enum.random([:ok, {:error, [:err]}]) + end +end diff --git a/lib/elixir/test/elixir/fixtures/file.bin b/lib/elixir/test/elixir/fixtures/file.bin new file mode 100644 index 00000000000..491c0980ddb --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/file.bin @@ -0,0 +1,4 @@ +LF +CR CRLF +LFCR + \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/init_sample.exs b/lib/elixir/test/elixir/fixtures/init_sample.exs deleted file mode 100644 index 4a1775c5027..00000000000 --- a/lib/elixir/test/elixir/fixtures/init_sample.exs +++ /dev/null @@ -1 +0,0 @@ -IO.puts to_string(1 + 2) \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/multiline_file.txt b/lib/elixir/test/elixir/fixtures/multiline_file.txt new file mode 100644 index 00000000000..9899a767b6c --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/multiline_file.txt @@ -0,0 +1,2 @@ +this is the first line +this is the second line diff --git a/lib/elixir/test/elixir/fixtures/parallel_compiler/bar.ex b/lib/elixir/test/elixir/fixtures/parallel_compiler/bar.ex deleted file mode 100644 index d6134b4c825..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_compiler/bar.ex +++ /dev/null @@ -1,5 +0,0 @@ -defmodule Bar do -end - -require Foo -IO.puts Foo.message \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/parallel_compiler/bat.ex b/lib/elixir/test/elixir/fixtures/parallel_compiler/bat.ex deleted file mode 100644 index 87cfccc56c9..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_compiler/bat.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Bat do - ThisModuleWillNeverBeAvailable[] -end \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/parallel_compiler/foo.ex b/lib/elixir/test/elixir/fixtures/parallel_compiler/foo.ex deleted file mode 100644 index ea9a7cd6d50..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_compiler/foo.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Foo do - def message, do: "message_from_foo" -end \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/parallel_deadlock/bar.ex b/lib/elixir/test/elixir/fixtures/parallel_deadlock/bar.ex deleted file mode 100644 index 4cb9fca2c7b..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_deadlock/bar.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Bar do - Foo.__info__(:macros) -end \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/parallel_deadlock/foo.ex b/lib/elixir/test/elixir/fixtures/parallel_deadlock/foo.ex deleted file mode 100644 index d50072d11af..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_deadlock/foo.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Foo do - Bar.__info__(:macros) -end \ No newline at end of file diff --git a/lib/elixir/test/elixir/fixtures/parallel_struct/bar.ex b/lib/elixir/test/elixir/fixtures/parallel_struct/bar.ex deleted file mode 100644 index 6e697731403..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_struct/bar.ex +++ /dev/null @@ -1,4 +0,0 @@ -defmodule Bar do - defstruct name: "" - def foo?(%Foo{}), do: true -end diff --git a/lib/elixir/test/elixir/fixtures/parallel_struct/foo.ex b/lib/elixir/test/elixir/fixtures/parallel_struct/foo.ex deleted file mode 100644 index 0e7153e01de..00000000000 --- a/lib/elixir/test/elixir/fixtures/parallel_struct/foo.ex +++ /dev/null @@ -1,4 +0,0 @@ -defmodule Foo do - defstruct name: "" - def bar?(%Bar{}), do: true -end diff --git a/lib/elixir/test/elixir/fixtures/utf16_be_bom.txt b/lib/elixir/test/elixir/fixtures/utf16_be_bom.txt new file mode 100644 index 00000000000..2499c9df2b1 Binary files /dev/null and b/lib/elixir/test/elixir/fixtures/utf16_be_bom.txt differ diff --git a/lib/elixir/test/elixir/fixtures/utf16_le_bom.txt b/lib/elixir/test/elixir/fixtures/utf16_le_bom.txt new file mode 100644 index 00000000000..5e16b0d385a Binary files /dev/null and b/lib/elixir/test/elixir/fixtures/utf16_le_bom.txt differ diff --git a/lib/elixir/test/elixir/fixtures/utf8_bom.txt b/lib/elixir/test/elixir/fixtures/utf8_bom.txt new file mode 100644 index 00000000000..bed73fc4ca1 --- /dev/null +++ b/lib/elixir/test/elixir/fixtures/utf8_bom.txt @@ -0,0 +1,2 @@ +Русский +日 diff --git a/lib/elixir/test/elixir/fixtures/warnings_sample.ex b/lib/elixir/test/elixir/fixtures/warnings_sample.ex deleted file mode 100644 index 4a0bfef0ffd..00000000000 --- a/lib/elixir/test/elixir/fixtures/warnings_sample.ex +++ /dev/null @@ -1,4 +0,0 @@ -defmodule WarningsSample do - def hello(a), do: a - def hello(b), do: b -end diff --git a/lib/elixir/test/elixir/float_test.exs b/lib/elixir/test/elixir/float_test.exs index a1a1471095e..3545e4e7fe4 100644 --- a/lib/elixir/test/elixir/float_test.exs +++ b/lib/elixir/test/elixir/float_test.exs @@ -1,17 +1,21 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule FloatTest do use ExUnit.Case, async: true - test :parse do + doctest Float + + test "parse/1" do assert Float.parse("12") === {12.0, ""} assert Float.parse("-12") === {-12.0, ""} assert Float.parse("-0.1") === {-0.1, ""} - assert Float.parse("123456789") === {123456789.0, ""} + assert Float.parse("123456789") === {123_456_789.0, ""} assert Float.parse("12.5") === {12.5, ""} assert Float.parse("12.524235") === {12.524235, ""} assert Float.parse("-12.5") === {-12.5, ""} assert Float.parse("-12.524235") === {-12.524235, ""} + assert Float.parse("0.3534091") === {0.3534091, ""} + assert Float.parse("0.3534091elixir") === {0.3534091, "elixir"} assert Float.parse("7.5e3") === {7.5e3, ""} assert Float.parse("7.5e-3") === {7.5e-3, ""} assert Float.parse("12x") === {12.0, "x"} @@ -22,49 +26,180 @@ defmodule FloatTest do assert Float.parse("1.32453e-10") === {1.32453e-10, ""} assert Float.parse("1.32.45") === {1.32, ".45"} assert Float.parse("1.o") === {1.0, ".o"} + assert Float.parse("+12.3E+4") === {1.23e5, ""} + assert Float.parse("+12.3E-4x") === {0.00123, "x"} + assert Float.parse("-1.23e-0xFF") === {-1.23, "xFF"} + assert Float.parse("-1.e2") === {-1.0, ".e2"} + assert Float.parse(".12") === :error assert Float.parse("--1.2") === :error assert Float.parse("++1.2") === :error assert Float.parse("pi") === :error + assert Float.parse("1.7976931348623157e308") === {1.7976931348623157e308, ""} + + assert_raise ArgumentError, fn -> + Float.parse("1.7976931348623159e308") + end + end + + test "floor/1" do + assert Float.floor(12.524235) === 12.0 + assert Float.floor(-12.5) === -13.0 + assert Float.floor(-12.524235) === -13.0 + assert Float.floor(7.5e3) === 7500.0 + assert Float.floor(7.5432e3) === 7543.0 + assert Float.floor(7.5e-3) === 0.0 + assert Float.floor(-12.32453e4) === -123_246.0 + assert Float.floor(-12.32453e-10) === -1.0 + assert Float.floor(0.32453e-10) === 0.0 + assert Float.floor(-0.32453e-10) === -1.0 + assert Float.floor(1.32453e-10) === 0.0 + end + + describe "floor/2" do + test "with 0.0" do + for precision <- 0..15 do + assert Float.floor(0.0, precision) === 0.0 + assert Float.floor(-0.0, precision) === -0.0 + end + end + + test "floor/2 with precision" do + assert Float.floor(12.524235, 0) === 12.0 + assert Float.floor(-12.524235, 0) === -13.0 + + assert Float.floor(12.52, 2) === 12.51 + assert Float.floor(-12.52, 2) === -12.52 + + assert Float.floor(12.524235, 2) === 12.52 + assert Float.floor(-12.524235, 3) === -12.525 + + assert Float.floor(12.32453e-20, 2) === 0.0 + assert Float.floor(-12.32453e-20, 2) === -0.01 + + assert_raise ArgumentError, "precision 16 is out of valid range of 0..15", fn -> + Float.floor(1.1, 16) + end + end + + test "with subnormal floats" do + assert Float.floor(-5.0e-324, 0) === -1.0 + assert Float.floor(-5.0e-324, 1) === -0.1 + assert Float.floor(-5.0e-324, 2) === -0.01 + assert Float.floor(-5.0e-324, 15) === -0.000000000000001 + + for precision <- 0..15 do + assert Float.floor(5.0e-324, precision) === 0.0 + end + end + end + + test "ceil/1" do + assert Float.ceil(12.524235) === 13.0 + assert Float.ceil(-12.5) === -12.0 + assert Float.ceil(-12.524235) === -12.0 + assert Float.ceil(7.5e3) === 7500.0 + assert Float.ceil(7.5432e3) === 7544.0 + assert Float.ceil(7.5e-3) === 1.0 + assert Float.ceil(-12.32453e4) === -123_245.0 + assert Float.ceil(-12.32453e-10) === 0.0 + assert Float.ceil(0.32453e-10) === 1.0 + assert Float.ceil(-0.32453e-10) === 0.0 + assert Float.ceil(1.32453e-10) === 1.0 + assert Float.ceil(0.0) === 0.0 end - test :floor do - assert Float.floor(12) === 12 - assert Float.floor(-12) === -12 - assert Float.floor(12.524235) === 12 - assert Float.floor(-12.5) === -13 - assert Float.floor(-12.524235) === -13 - assert Float.floor(7.5e3) === 7500 - assert Float.floor(7.5432e3) === 7543 - assert Float.floor(7.5e-3) === 0 - assert Float.floor(-12.32453e4) === -123246 - assert Float.floor(-12.32453e-10) === -1 - assert Float.floor(0.32453e-10) === 0 - assert Float.floor(-0.32453e-10) === -1 - assert Float.floor(1.32453e-10) === 0 + describe "ceil/2" do + test "with 0.0" do + for precision <- 0..15 do + assert Float.ceil(0.0, precision) === 0.0 + assert Float.ceil(-0.0, precision) === -0.0 + end + end + + test "with regular floats" do + assert Float.ceil(12.524235, 0) === 13.0 + assert Float.ceil(-12.524235, 0) === -12.0 + + assert Float.ceil(12.52, 2) === 12.52 + assert Float.ceil(-12.52, 2) === -12.51 + + assert Float.ceil(12.524235, 2) === 12.53 + assert Float.ceil(-12.524235, 3) === -12.524 + + assert Float.ceil(12.32453e-20, 2) === 0.01 + assert Float.ceil(-12.32453e-20, 2) === 0.0 + + assert Float.ceil(0.0, 2) === 0.0 + + assert_raise ArgumentError, "precision 16 is out of valid range of 0..15", fn -> + Float.ceil(1.1, 16) + end + end + + test "with subnormal floats" do + assert Float.ceil(5.0e-324, 0) === 1.0 + assert Float.ceil(5.0e-324, 1) === 0.1 + assert Float.ceil(5.0e-324, 2) === 0.01 + assert Float.ceil(5.0e-324, 15) === 0.000000000000001 + + for precision <- 0..15 do + assert Float.ceil(-5.0e-324, precision) === -0.0 + end + end end - test :ceil do - assert Float.ceil(12) === 12 - assert Float.ceil(-12) === -12 - assert Float.ceil(12.524235) === 13 - assert Float.ceil(-12.5) === -12 - assert Float.ceil(-12.524235) === -12 - assert Float.ceil(7.5e3) === 7500 - assert Float.ceil(7.5432e3) === 7544 - assert Float.ceil(7.5e-3) === 1 - assert Float.ceil(-12.32453e4) === -123245 - assert Float.ceil(-12.32453e-10) === 0 - assert Float.ceil(0.32453e-10) === 1 - assert Float.ceil(-0.32453e-10) === 0 - assert Float.ceil(1.32453e-10) === 1 + describe "round/2" do + test "with 0.0" do + for precision <- 0..15 do + assert Float.round(0.0, precision) === 0.0 + assert Float.round(-0.0, precision) === -0.0 + end + end + + test "with regular floats" do + assert Float.round(5.5675, 3) === 5.567 + assert Float.round(-5.5674, 3) === -5.567 + assert Float.round(5.5, 3) === 5.5 + assert Float.round(5.5e-10, 10) === 5.0e-10 + assert Float.round(5.5e-10, 8) === 0.0 + assert Float.round(5.0, 0) === 5.0 + + assert_raise ArgumentError, "precision 16 is out of valid range of 0..15", fn -> + Float.round(1.1, 16) + end + end + + test "with subnormal floats" do + for precision <- 0..15 do + assert Float.round(5.0e-324, precision) === 0.0 + assert Float.round(-5.0e-324, precision) === -0.0 + end + end end - test :round do - assert Float.round(5.5675, 3) === 5.568 - assert Float.round(-5.5674, 3) === -5.567 - assert Float.round(5.5, 3) === 5.5 - assert Float.round(5.5e-10, 10) === 6.0e-10 - assert Float.round(5.5e-10, 8) === 0.0 - assert Float.round(5.0, 0) === 5.0 + describe "ratio/1" do + test "with 0.0" do + assert Float.ratio(0.0) == {0, 1} + end + + test "with regular floats" do + assert Float.ratio(3.14) == {7_070_651_414_971_679, 2_251_799_813_685_248} + assert Float.ratio(-3.14) == {-7_070_651_414_971_679, 2_251_799_813_685_248} + assert Float.ratio(1.5) == {3, 2} + end + + test "with subnormal floats" do + assert Float.ratio(5.0e-324) == + {1, + 202_402_253_307_310_618_352_495_346_718_917_307_049_556_649_764_142_118_356_901_358_027_430_339_567_995_346_891_960_383_701_437_124_495_187_077_864_316_811_911_389_808_737_385_793_476_867_013_399_940_738_509_921_517_424_276_566_361_364_466_907_742_093_216_341_239_767_678_472_745_068_562_007_483_424_692_698_618_103_355_649_159_556_340_810_056_512_358_769_552_333_414_615_230_502_532_186_327_508_646_006_263_307_707_741_093_494_784} + + assert Float.ratio(1.0e-323) == + {1, + 101_201_126_653_655_309_176_247_673_359_458_653_524_778_324_882_071_059_178_450_679_013_715_169_783_997_673_445_980_191_850_718_562_247_593_538_932_158_405_955_694_904_368_692_896_738_433_506_699_970_369_254_960_758_712_138_283_180_682_233_453_871_046_608_170_619_883_839_236_372_534_281_003_741_712_346_349_309_051_677_824_579_778_170_405_028_256_179_384_776_166_707_307_615_251_266_093_163_754_323_003_131_653_853_870_546_747_392} + + assert Float.ratio(2.225073858507201e-308) == + {4_503_599_627_370_495, + 202_402_253_307_310_618_352_495_346_718_917_307_049_556_649_764_142_118_356_901_358_027_430_339_567_995_346_891_960_383_701_437_124_495_187_077_864_316_811_911_389_808_737_385_793_476_867_013_399_940_738_509_921_517_424_276_566_361_364_466_907_742_093_216_341_239_767_678_472_745_068_562_007_483_424_692_698_618_103_355_649_159_556_340_810_056_512_358_769_552_333_414_615_230_502_532_186_327_508_646_006_263_307_707_741_093_494_784} + end end end diff --git a/lib/elixir/test/elixir/function_test.exs b/lib/elixir/test/elixir/function_test.exs new file mode 100644 index 00000000000..813d9462a48 --- /dev/null +++ b/lib/elixir/test/elixir/function_test.exs @@ -0,0 +1,83 @@ +Code.require_file("test_helper.exs", __DIR__) + +defmodule DummyFunction do + def function_with_arity_0 do + true + end + + def zero?(0), do: true + def zero?(_), do: false +end + +defmodule FunctionTest do + use ExUnit.Case, async: true + + doctest Function + import Function + + @information_keys_for_named [:type, :module, :arity, :name, :env] + @information_keys_for_anonymous @information_keys_for_named ++ + [:pid, :index, :new_index, :new_uniq, :uniq] + + describe "capture/3" do + test "captures module functions with arity 0" do + f = capture(DummyFunction, :function_with_arity_0, 0) + + assert is_function(f) + end + + test "captures module functions with any arity" do + f = capture(DummyFunction, :zero?, 1) + + assert is_function(f) + assert f.(0) + end + end + + describe "info/1" do + test "returns info for named captured functions" do + f = &DummyFunction.zero?/1 + expected = [module: DummyFunction, name: :zero?, arity: 1, env: [], type: :external] + + result = info(f) + + assert expected == result + end + + test "returns info for anonymous functions" do + f = fn x -> x end + + result = info(f) + + for {key, _value} <- result do + assert key in @information_keys_for_anonymous + end + end + end + + describe "info/2" do + test "returns info for every possible information key for named functions" do + f = &DummyFunction.zero?/1 + + for x <- @information_keys_for_named do + assert {^x, _} = info(f, x) + end + end + + test "returns info for every possible information key for anonymous functions" do + f = &DummyFunction.zero?/1 + + for x <- @information_keys_for_anonymous do + assert {^x, _} = info(f, x) + end + + assert {:arity, 1} = info(f, :arity) + end + end + + describe "identity/1" do + test "returns whatever it gets passed" do + assert :hello = Function.identity(:hello) + end + end +end diff --git a/lib/elixir/test/elixir/gen_event_test.exs b/lib/elixir/test/elixir/gen_event_test.exs deleted file mode 100644 index 005cb715b21..00000000000 --- a/lib/elixir/test/elixir/gen_event_test.exs +++ /dev/null @@ -1,392 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -defmodule GenEventTest do - use ExUnit.Case, async: true - - defmodule LoggerHandler do - use GenEvent - - def handle_event({:log, x}, messages) do - {:ok, [x|messages]} - end - - def handle_call(:messages, messages) do - {:ok, Enum.reverse(messages), []} - end - - def handle_call(call, state) do - super(call, state) - end - end - - defmodule SlowHandler do - use GenEvent - - def handle_event(_event, _state) do - :timer.sleep(100) - :remove_handler - end - end - - @receive_timeout 1000 - - test "start_link/2 and handler workflow" do - {:ok, pid} = GenEvent.start_link() - - {:links, links} = Process.info(self, :links) - assert pid in links - - assert GenEvent.notify(pid, {:log, 0}) == :ok - assert GenEvent.add_handler(pid, LoggerHandler, []) == :ok - assert GenEvent.notify(pid, {:log, 1}) == :ok - assert GenEvent.notify(pid, {:log, 2}) == :ok - - assert GenEvent.call(pid, LoggerHandler, :messages) == [1, 2] - assert GenEvent.call(pid, LoggerHandler, :messages) == [] - - assert GenEvent.call(pid, LoggerHandler, :whatever) == {:error, :bad_call} - assert GenEvent.call(pid, UnknownHandler, :messages) == {:error, :bad_module} - - assert GenEvent.remove_handler(pid, LoggerHandler, []) == :ok - assert GenEvent.stop(pid) == :ok - end - - test "start/2 with linked handler" do - {:ok, pid} = GenEvent.start() - - {:links, links} = Process.info(self, :links) - refute pid in links - - assert GenEvent.add_handler(pid, LoggerHandler, [], link: true) == :ok - - {:links, links} = Process.info(self, :links) - assert pid in links - - assert GenEvent.notify(pid, {:log, 1}) == :ok - assert GenEvent.sync_notify(pid, {:log, 2}) == :ok - - assert GenEvent.call(pid, LoggerHandler, :messages) == [1, 2] - assert GenEvent.stop(pid) == :ok - end - - test "start/2 with linked swap" do - {:ok, pid} = GenEvent.start() - - assert GenEvent.add_handler(pid, LoggerHandler, []) == :ok - - {:links, links} = Process.info(self, :links) - refute pid in links - - assert GenEvent.swap_handler(pid, LoggerHandler, [], LoggerHandler, [], link: true) == :ok - - {:links, links} = Process.info(self, :links) - assert pid in links - - assert GenEvent.stop(pid) == :ok - end - - test "start/2 with registered name" do - {:ok, _} = GenEvent.start(name: :logger) - assert GenEvent.stop(:logger) == :ok - end - - test "sync stream/2" do - {:ok, pid} = GenEvent.start_link() - parent = self() - - spawn_link fn -> - send parent, Enum.take(GenEvent.stream(pid, mode: :sync), 3) - end - - wait_for_handlers(pid, 1) - - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - - # Receive one of the results - assert_receive [1, 2, 3], @receive_timeout - wait_for_handlers(pid, 0) - - spawn_link fn -> - Enum.each(GenEvent.stream(pid, mode: :sync), fn _ -> - :timer.sleep(:infinity) - end) - end - - wait_for_handlers(pid, 1) - - for i <- 1..6 do - GenEvent.notify(pid, i) - end - - wait_for_queue_length(pid, 5) - end - - test "async stream/2" do - {:ok, pid} = GenEvent.start_link() - parent = self() - - spawn_link fn -> - Enum.each(GenEvent.stream(pid, mode: :async), fn _ -> - :timer.sleep(:infinity) - end) - end - - spawn_link fn -> - send parent, Enum.take(GenEvent.stream(pid, mode: :async), 3) - end - - wait_for_handlers(pid, 2) - - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - - # Receive one of the results - assert_receive [1, 2, 3], @receive_timeout - - # One of the subscriptions are gone - wait_for_handlers(pid, 1) - end - - Enum.each [:sync, :async], fn mode -> - test "#{mode} stream/2 with parallel use (and first finishing first)" do - {:ok, pid} = GenEvent.start_link() - stream = GenEvent.stream(pid, duration: 200, mode: unquote(mode)) - - parent = self() - spawn_link fn -> send parent, {:take, Enum.take(stream, 3)} end - wait_for_handlers(pid, 1) - spawn_link fn -> send parent, {:to_list, Enum.to_list(stream)} end - wait_for_handlers(pid, 2) - - # Notify the events for both handlers - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - assert_receive {:take, [1, 2, 3]}, @receive_timeout - - # Notify the events for to_list stream handler - for i <- 4..5 do - GenEvent.sync_notify(pid, i) - end - - assert_receive {:to_list, [1, 2, 3, 4, 5]}, @receive_timeout - end - - test "#{mode} stream/2 with timeout" do - # Start a manager - {:ok, pid} = GenEvent.start_link() - Process.flag(:trap_exit, true) - - pid = spawn_link fn -> - Enum.take(GenEvent.stream(pid, timeout: 50, mode: unquote(mode)), 5) - end - - assert_receive {:EXIT, ^pid, - {:timeout, {Enumerable.GenEvent, :next, [_, _]}}}, @receive_timeout - end - - test "#{mode} stream/2 with error/timeout on subscription" do - # Start a manager - {:ok, pid} = GenEvent.start_link() - - # Start a subscriber with timeout - child = spawn fn -> Enum.to_list(GenEvent.stream(pid, mode: unquote(mode))) end - wait_for_handlers(pid, 1) - - # Kill and wait until we have 0 handlers - Process.exit(child, :kill) - wait_for_handlers(pid, 0) - GenEvent.stop(pid) - end - - test "#{mode} stream/2 with manager stop" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link() - - parent = self() - stream_pid = spawn_link fn -> - send parent, Enum.take(GenEvent.stream(pid, mode: unquote(mode)), 5) - end - wait_for_handlers(pid, 1) - - # Notify the events - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - - Process.flag(:trap_exit, true) - GenEvent.stop(pid) - assert_receive {:EXIT, ^stream_pid, - {:shutdown, {Enumerable.GenEvent, :next, [_, _]}}}, @receive_timeout - end - - test "#{mode} stream/2 with cancel streams" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link() - stream = GenEvent.stream(pid, id: make_ref(), mode: unquote(mode)) - - parent = self() - spawn_link fn -> send parent, Enum.take(stream, 5) end - wait_for_handlers(pid, 1) - - # Notify the events - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - - GenEvent.cancel_streams(stream) - assert_receive [1, 2, 3], @receive_timeout - GenEvent.stop(pid) - end - - test "#{mode} stream/2 with swap_handler" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link() - stream = GenEvent.stream(pid, id: make_ref(), mode: unquote(mode)) - - parent = self() - stream_pid = spawn_link fn -> send parent, Enum.take(stream, 5) end - wait_for_handlers(pid, 1) - - # Notify the events - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - - [handler] = GenEvent.which_handlers(pid) - Process.flag(:trap_exit, true) - GenEvent.swap_handler(pid, handler, :swap_handler, LogHandler, []) - assert_receive {:EXIT, ^stream_pid, - {{:swapped, LogHandler, _}, - {Enumerable.GenEvent, :next, [_, _]}}}, @receive_timeout - end - - test "#{mode} stream/2 with duration" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link() - stream = GenEvent.stream(pid, duration: 200, mode: unquote(mode)) - - parent = self() - spawn_link fn -> send parent, {:duration, Enum.take(stream, 10)} end - wait_for_handlers(pid, 1) - - # Notify the events - for i <- 1..5 do - GenEvent.sync_notify(pid, i) - end - - # Wait until the handler is gone - wait_for_handlers(pid, 0) - - # The stream is not complete but terminated anyway due to duration - assert_receive {:duration, [1, 2, 3, 4, 5]}, @receive_timeout - - GenEvent.stop(pid) - end - - test "#{mode} stream/2 with manager killed and trap_exit" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link() - stream = GenEvent.stream(pid, mode: unquote(mode)) - - parent = self() - stream_pid = spawn_link fn -> - send parent, Enum.to_list(stream) - end - wait_for_handlers(pid, 1) - - Process.flag(:trap_exit, true) - Process.exit(pid, :kill) - assert_receive {:EXIT, ^pid, :killed}, @receive_timeout - assert_receive {:EXIT, ^stream_pid, - {:killed, {Enumerable.GenEvent, :next, [_,_]}}}, @receive_timeout - end - - test "#{mode} stream/2 with manager not alive" do - # Start a manager and subscribers - stream = GenEvent.stream(:does_not_exit, mode: unquote(mode)) - - parent = self() - stream_pid = spawn_link fn -> - send parent, Enum.to_list(stream) - end - - Process.flag(:trap_exit, true) - assert_receive {:EXIT, ^stream_pid, - {:noproc, {Enumerable.GenEvent, :start, [_]}}}, @receive_timeout - end - - test "#{mode} stream/2 with manager unregistered" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link(name: :unreg) - stream = GenEvent.stream(:unreg, mode: unquote(mode)) - - parent = self() - spawn_link fn -> - send parent, Enum.take(stream, 5) - end - wait_for_handlers(pid, 1) - - # Notify the events - for i <- 1..3 do - GenEvent.sync_notify(pid, i) - end - - # Unregister the process - Process.unregister(:unreg) - - # Notify the remaining events - for i <- 4..5 do - GenEvent.sync_notify(pid, i) - end - - # We should have gotten the message and all handlers were removed - assert_receive [1, 2, 3, 4, 5], @receive_timeout - wait_for_handlers(pid, 0) - end - - test "#{mode} stream/2 flushes events on abort" do - # Start a manager and subscribers - {:ok, pid} = GenEvent.start_link() - - spawn_link fn -> - wait_for_handlers(pid, 2) - GenEvent.notify(pid, 1) - GenEvent.notify(pid, 2) - GenEvent.notify(pid, 3) - end - - GenEvent.add_handler(pid, SlowHandler, []) - stream = GenEvent.stream(pid, mode: unquote(mode)) - - try do - Enum.each stream, fn _ -> throw :done end - catch - :done -> :ok - end - - # Wait for the slow handler to be removed - # so all events have been handled - wait_for_handlers(pid, 0) - - # Check no messages leaked. - refute_received _any - end - end - - defp wait_for_handlers(pid, count) do - unless length(GenEvent.which_handlers(pid)) == count do - wait_for_handlers(pid, count) - end - end - - defp wait_for_queue_length(pid, count) do - {:message_queue_len, n} = Process.info(pid, :message_queue_len) - unless n == count do - wait_for_queue_length(pid, count) - end - end -end diff --git a/lib/elixir/test/elixir/gen_server_test.exs b/lib/elixir/test/elixir/gen_server_test.exs index 880d7d0f864..e50972619a8 100644 --- a/lib/elixir/test/elixir/gen_server_test.exs +++ b/lib/elixir/test/elixir/gen_server_test.exs @@ -1,4 +1,4 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule GenServerTest do use ExUnit.Case, async: true @@ -6,49 +6,143 @@ defmodule GenServerTest do defmodule Stack do use GenServer - def handle_call(:stop, from, state) do - GenServer.reply(from, :ok) - {:stop, :normal, state} + def init(args) do + {:ok, args} end - def handle_call(:pop, _from, [h|t]) do + def handle_call(:pop, _from, [h | t]) do {:reply, h, t} end - def handle_call(request, from, state) do - super(request, from, state) + def handle_call(:noreply, _from, h) do + {:noreply, h} end - def handle_cast({:push, item}, state) do - {:noreply, [item|state]} + def handle_call(:stop_self, _from, state) do + reason = catch_exit(GenServer.stop(self())) + {:reply, reason, state} end - def handle_cast(request, state) do - super(request, state) + def handle_cast({:push, element}, state) do + {:noreply, [element | state]} end def terminate(_reason, _state) do # There is a race condition if the agent is # restarted too fast and it is registered. try do - self |> Process.info(:registered_name) |> elem(1) |> Process.unregister + self() |> Process.info(:registered_name) |> elem(1) |> Process.unregister() rescue _ -> :ok end + :ok end end + test "generates child_spec/1" do + assert Stack.child_spec([:hello]) == %{ + id: Stack, + start: {Stack, :start_link, [[:hello]]} + } + + defmodule CustomStack do + use GenServer, id: :id, restart: :temporary, shutdown: :infinity, start: {:foo, :bar, []} + + def init(args) do + {:ok, args} + end + end + + assert CustomStack.child_spec([:hello]) == %{ + id: :id, + restart: :temporary, + shutdown: :infinity, + start: {:foo, :bar, []} + } + end + + test "start_link/3" do + assert_raise ArgumentError, ~r"expected :name option to be one of the following:", fn -> + GenServer.start_link(Stack, [:hello], name: "my_gen_server_name") + end + + assert_raise ArgumentError, ~r"expected :name option to be one of the following:", fn -> + GenServer.start_link(Stack, [:hello], name: {:invalid_tuple, "my_gen_server_name"}) + end + + assert_raise ArgumentError, ~r"expected :name option to be one of the following:", fn -> + GenServer.start_link(Stack, [:hello], name: {:via, "Via", "my_gen_server_name"}) + end + + assert_raise ArgumentError, ~r/Got: "my_gen_server_name"/, fn -> + GenServer.start_link(Stack, [:hello], name: "my_gen_server_name") + end + end + + test "start_link/3 with via" do + GenServer.start_link(Stack, [:hello], name: {:via, :global, :via_stack}) + assert GenServer.call({:via, :global, :via_stack}, :pop) == :hello + end + + test "start_link/3 with global" do + GenServer.start_link(Stack, [:hello], name: {:global, :global_stack}) + assert GenServer.call({:global, :global_stack}, :pop) == :hello + end + + test "start_link/3 with local" do + GenServer.start_link(Stack, [:hello], name: :stack) + assert GenServer.call(:stack, :pop) == :hello + end + test "start_link/2, call/2 and cast/2" do {:ok, pid} = GenServer.start_link(Stack, [:hello]) - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) assert pid in links assert GenServer.call(pid, :pop) == :hello assert GenServer.cast(pid, {:push, :world}) == :ok assert GenServer.call(pid, :pop) == :world - assert GenServer.call(pid, :stop) == :ok + assert GenServer.stop(pid) == :ok + + assert GenServer.cast({:global, :foo}, {:push, :world}) == :ok + assert GenServer.cast({:via, :foo, :bar}, {:push, :world}) == :ok + assert GenServer.cast(:foo, {:push, :world}) == :ok + end + + @tag capture_log: true + test "call/3 exit messages" do + name = :self + Process.register(self(), name) + :global.register_name(name, self()) + {:ok, pid} = GenServer.start_link(Stack, [:hello]) + {:ok, stopped_pid} = GenServer.start(Stack, [:hello]) + GenServer.stop(stopped_pid) + + assert catch_exit(GenServer.call(name, :pop, 5000)) == + {:calling_self, {GenServer, :call, [name, :pop, 5000]}} + + assert catch_exit(GenServer.call({:global, name}, :pop, 5000)) == + {:calling_self, {GenServer, :call, [{:global, name}, :pop, 5000]}} + + assert catch_exit(GenServer.call({:via, :global, name}, :pop, 5000)) == + {:calling_self, {GenServer, :call, [{:via, :global, name}, :pop, 5000]}} + + assert catch_exit(GenServer.call(self(), :pop, 5000)) == + {:calling_self, {GenServer, :call, [self(), :pop, 5000]}} + + assert catch_exit(GenServer.call(pid, :noreply, 1)) == + {:timeout, {GenServer, :call, [pid, :noreply, 1]}} + + assert catch_exit(GenServer.call(nil, :pop, 5000)) == + {:noproc, {GenServer, :call, [nil, :pop, 5000]}} + + assert catch_exit(GenServer.call(stopped_pid, :pop, 5000)) == + {:noproc, {GenServer, :call, [stopped_pid, :pop, 5000]}} + + assert catch_exit(GenServer.call({:stack, :bogus_node}, :pop, 5000)) == + {{:nodedown, :bogus_node}, {GenServer, :call, [{:stack, :bogus_node}, :pop, 5000]}} end test "nil name" do @@ -58,31 +152,65 @@ defmodule GenServerTest do test "start/2" do {:ok, pid} = GenServer.start(Stack, [:hello]) - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) refute pid in links - GenServer.call(pid, :stop) + GenServer.stop(pid) end - test "abcast/3" do - {:ok, _} = GenServer.start_link(Stack, [], name: :stack) + test "abcast/3", %{test: name} do + {:ok, _} = GenServer.start_link(Stack, [], name: name) + + assert GenServer.abcast(name, {:push, :hello}) == :abcast + assert GenServer.call({name, node()}, :pop) == :hello - assert GenServer.abcast(:stack, {:push, :hello}) == :abcast - assert GenServer.call({:stack, node()}, :pop) == :hello + assert GenServer.abcast([node(), :foo@bar], name, {:push, :world}) == :abcast + assert GenServer.call(name, :pop) == :world + end + + test "multi_call/4", %{test: name} do + {:ok, _} = GenServer.start_link(Stack, [:hello, :world], name: name) - assert GenServer.abcast([node, :foo@bar], :stack, {:push, :world}) == :abcast - assert GenServer.call(:stack, :pop) == :world + assert GenServer.multi_call(name, :pop) == {[{node(), :hello}], []} - GenServer.call(:stack, :stop) + assert GenServer.multi_call([node(), :foo@bar], name, :pop) == + {[{node(), :world}], [:foo@bar]} end - test "multi_call/4" do - {:ok, _} = GenServer.start_link(Stack, [:hello, :world], name: :stack) + test "whereis/1" do + name = :whereis_server + + {:ok, pid} = GenServer.start_link(Stack, [], name: name) + assert GenServer.whereis(name) == pid + assert GenServer.whereis({name, node()}) == pid + assert GenServer.whereis({name, :another_node}) == {name, :another_node} + assert GenServer.whereis(pid) == pid + assert GenServer.whereis(:whereis_bad_server) == nil + + {:ok, pid} = GenServer.start_link(Stack, [], name: {:global, name}) + assert GenServer.whereis({:global, name}) == pid + assert GenServer.whereis({:global, :whereis_bad_server}) == nil + assert GenServer.whereis({:via, :global, name}) == pid + assert GenServer.whereis({:via, :global, :whereis_bad_server}) == nil + end + + test "stop/3", %{test: name} do + {:ok, pid} = GenServer.start(Stack, []) + assert GenServer.stop(pid, :normal) == :ok + + stopped_pid = pid + + assert catch_exit(GenServer.stop(stopped_pid)) == + {:noproc, {GenServer, :stop, [stopped_pid, :normal, :infinity]}} + + assert catch_exit(GenServer.stop(nil)) == + {:noproc, {GenServer, :stop, [nil, :normal, :infinity]}} + + {:ok, pid} = GenServer.start(Stack, []) - assert GenServer.multi_call(:stack, :pop) == - {[{node(), :hello}], []} - assert GenServer.multi_call([node, :foo@bar], :stack, :pop) == - {[{node, :world}], [:foo@bar]} + assert GenServer.call(pid, :stop_self) == + {:calling_self, {GenServer, :stop, [pid, :normal, :infinity]}} - GenServer.call(:stack, :stop) + {:ok, _} = GenServer.start(Stack, [], name: name) + assert GenServer.stop(name, :normal) == :ok end end diff --git a/lib/elixir/test/elixir/hash_dict_test.exs b/lib/elixir/test/elixir/hash_dict_test.exs deleted file mode 100644 index 3a64f3e818b..00000000000 --- a/lib/elixir/test/elixir/hash_dict_test.exs +++ /dev/null @@ -1,91 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -defmodule HashDictTest do - use ExUnit.Case, async: true - - @dict Enum.into([foo: :bar], HashDict.new) - - test "access" do - dict = Enum.into([foo: :baz], HashDict.new) - assert Access.get(@dict, :foo) == :bar - assert Access.get_and_update(@dict, :foo, fn :bar -> {:ok, :baz} end) == {:ok, dict} - assert Access.get_and_update(HashDict.new, :foo, fn nil -> {:ok, :baz} end) == {:ok, dict} - end - - test "is serializable as attribute" do - assert @dict == Enum.into([foo: :bar], HashDict.new) - end - - test "is accessible as attribute" do - assert @dict[:foo] == :bar - end - - test "small dict smoke test" do - smoke_test(1..8) - smoke_test(8..1) - end - - test "medium dict smoke test" do - smoke_test(1..80) - smoke_test(80..1) - end - - test "large dict smoke test" do - smoke_test(1..1200) - smoke_test(1200..1) - end - - test "reduce/3 (via to_list)" do - dict = filled_dict(8) - list = dict |> HashDict.to_list - assert length(list) == 8 - assert {1, 1} in list - assert list == Enum.to_list(dict) - - dict = filled_dict(20) - list = dict |> HashDict.to_list - assert length(list) == 20 - assert {1, 1} in list - assert list == Enum.to_list(dict) - - dict = filled_dict(120) - list = dict |> HashDict.to_list - assert length(list) == 120 - assert {1, 1} in list - assert list == Enum.to_list(dict) - end - - test "comparison when subsets" do - d1 = Enum.into [a: 0], HashDict.new - d2 = Enum.into [a: 0, b: 1], HashDict.new - - refute HashDict.equal?(d1, d2) - refute HashDict.equal?(d2, d1) - end - - defp smoke_test(range) do - {dict, _} = Enum.reduce range, {HashDict.new, 1}, fn(x, {acc, i}) -> - acc = HashDict.put(acc, x, x) - assert HashDict.size(acc) == i - {acc, i + 1} - end - - Enum.each range, fn(x) -> - assert HashDict.get(dict, x) == x - end - - {dict, _} = Enum.reduce range, {dict, Enum.count(range)}, fn(x, {acc, i}) -> - assert HashDict.size(acc) == i - acc = HashDict.delete(acc, x) - assert HashDict.size(acc) == i - 1 - assert HashDict.get(acc, x) == nil - {acc, i - 1} - end - - assert dict == HashDict.new - end - - defp filled_dict(range) do - Enum.reduce 1..range, HashDict.new, &HashDict.put(&2, &1, &1) - end -end diff --git a/lib/elixir/test/elixir/hash_set_test.exs b/lib/elixir/test/elixir/hash_set_test.exs deleted file mode 100644 index cb231a1a2a6..00000000000 --- a/lib/elixir/test/elixir/hash_set_test.exs +++ /dev/null @@ -1,55 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -defmodule HashSetTest do - use ExUnit.Case, async: true - - test "union" do - assert HashSet.union(filled_set(21), filled_set(22)) == filled_set(22) - assert HashSet.union(filled_set(121), filled_set(120)) == filled_set(121) - end - - test "intersection" do - assert HashSet.intersection(filled_set(21), filled_set(20)) == filled_set(20) - assert HashSet.equal?(HashSet.intersection(filled_set(120), filled_set(121)), filled_set(120)) - end - - test "difference" do - assert HashSet.equal?(HashSet.difference(filled_set(20), filled_set(21)), HashSet.new) - - diff = HashSet.difference(filled_set(9000), filled_set(9000)) - assert HashSet.equal?(diff, HashSet.new) - assert HashSet.size(diff) == 0 - end - - test "subset?" do - assert HashSet.subset?(HashSet.new, HashSet.new) - assert HashSet.subset?(filled_set(6), filled_set(10)) - assert HashSet.subset?(filled_set(6), filled_set(120)) - refute HashSet.subset?(filled_set(120), filled_set(6)) - end - - test "equal?" do - assert HashSet.equal?(HashSet.new, HashSet.new) - assert HashSet.equal?(filled_set(20), HashSet.delete(filled_set(21), 21)) - assert HashSet.equal?(filled_set(120), filled_set(120)) - end - - test "to_list" do - set = filled_set(20) - list = HashSet.to_list(set) - assert length(list) == 20 - assert 1 in list - assert Enum.sort(list) == Enum.sort(1..20) - - set = filled_set(120) - list = HashSet.to_list(set) - assert length(list) == 120 - assert 1 in list - assert Enum.sort(list) == Enum.sort(1..120) - end - - defp filled_set(range) do - Enum.into 1..range, HashSet.new - end -end - diff --git a/lib/elixir/test/elixir/inspect/algebra_test.exs b/lib/elixir/test/elixir/inspect/algebra_test.exs index 3b4b46a5263..e5cb8b70111 100644 --- a/lib/elixir/test/elixir/inspect/algebra_test.exs +++ b/lib/elixir/test/elixir/inspect/algebra_test.exs @@ -1,125 +1,319 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Inspect.OptsTest do + use ExUnit.Case + + test "new" do + opts = Inspect.Opts.new(limit: 13, pretty: true) + assert opts.limit == 13 + assert opts.pretty + end + + test "default_inspect_fun" do + assert Inspect.Opts.default_inspect_fun() == (&Inspect.inspect/2) + + assert Inspect.Opts.default_inspect_fun(fn + :rewrite_atom, _ -> "rewritten_atom" + value, opts -> Inspect.inspect(value, opts) + end) == :ok + + assert inspect(:rewrite_atom) == "rewritten_atom" + after + Inspect.Opts.default_inspect_fun(&Inspect.inspect/2) + end +end defmodule Inspect.AlgebraTest do use ExUnit.Case, async: true + doctest Inspect.Algebra + import Inspect.Algebra - def helloabcd do - concat( - glue( - glue( - glue("hello", "a"), - "b"), - "c"), - "d") + defp render(doc, limit) do + doc |> group() |> format(limit) |> IO.iodata_to_binary() end - def factor(doc, w), do: format(w, 0, [{0, :flat, group(doc)}]) - test "empty doc" do - # Consistence with definitions - assert empty == :doc_nil - - # Consistence of corresponding sdoc - assert factor(empty, 80) == [] + # Consistent with definitions + assert empty() == :doc_nil # Consistent formatting - assert pretty(empty, 80) == "" + assert render(empty(), 80) == "" end - test "break doc" do - # Consistence with definitions - assert break("break") == {:doc_break, "break"} - assert break("") == {:doc_break, ""} + test "strict break doc" do + # Consistent with definitions + assert break("break") == {:doc_break, "break", :strict} + assert break("") == {:doc_break, "", :strict} # Wrong argument type assert_raise FunctionClauseError, fn -> break(42) end - # Consistence of corresponding sdoc - assert factor(break("_"), 80) == ["_"] + # Consistent formatting + assert render(break("_"), 80) == "_" + assert render(glue("foo", " ", glue("bar", " ", "baz")), 10) == "foo\nbar\nbaz" + end + + test "flex break doc" do + # Consistent with definitions + assert flex_break("break") == {:doc_break, "break", :flex} + assert flex_break("") == {:doc_break, "", :flex} + + # Wrong argument type + assert_raise FunctionClauseError, fn -> flex_break(42) end # Consistent formatting - assert pretty(break("_"), 80) == "_" + assert render(flex_break("_"), 80) == "_" + assert render(flex_glue("foo", " ", flex_glue("bar", " ", "baz")), 10) == "foo bar\nbaz" end test "glue doc" do - # Consistence with definitions - assert glue("a", "->", "b") == {:doc_cons, - "a", {:doc_cons, {:doc_break, "->"}, "b"} - } + # Consistent with definitions + assert glue("a", "->", "b") == {:doc_cons, "a", {:doc_cons, {:doc_break, "->", :strict}, "b"}} assert glue("a", "b") == glue("a", " ", "b") # Wrong argument type assert_raise FunctionClauseError, fn -> glue("a", 42, "b") end end - test "text doc" do - # Consistence of corresponding sdoc - assert factor("_", 80) == ["_"] + test "flex glue doc" do + # Consistent with definitions + assert flex_glue("a", "->", "b") == + {:doc_cons, "a", {:doc_cons, {:doc_break, "->", :flex}, "b"}} - # Consistent formatting - assert pretty("_", 80) == "_" + assert flex_glue("a", "b") == flex_glue("a", " ", "b") + + # Wrong argument type + assert_raise FunctionClauseError, fn -> flex_glue("a", 42, "b") end + end + + test "binary doc" do + assert render("_", 80) == "_" + end + + test "string doc" do + # Consistent with definitions + assert string("ólá") == {:doc_string, "ólá", 3} + + # Counts graphemes + doc = glue(string("olá"), " ", string("mundo")) + assert render(doc, 9) == "olá mundo" end test "space doc" do - # Consistency with definitions - assert space("a", "b") == {:doc_cons, - "a", {:doc_cons, " ", "b"} - } + # Consistent with definitions + assert space("a", "b") == {:doc_cons, "a", {:doc_cons, " ", "b"}} end - test "nest doc" do - # Consistence with definitions - assert nest(empty, 1) == {:doc_nest, empty, 1} - assert nest(empty, 0) == :doc_nil + test "always nest doc" do + # Consistent with definitions + assert nest(empty(), 1) == {:doc_nest, empty(), 1, :always} + assert nest(empty(), 0) == :doc_nil # Wrong argument type - assert_raise FunctionClauseError, fn -> nest("foo", empty) end + assert_raise FunctionClauseError, fn -> nest("foo", empty()) end + + # Consistent formatting + assert render(nest("a", 1), 80) == "a" + assert render(nest(glue("a", "b"), 1), 2) == "a\n b" + assert render(nest(line("a", "b"), 1), 20) == "a\n b" + end - # Consistence of corresponding sdoc - assert factor(nest("a", 1), 80) == ["a"] - assert format(2, 0, [{0, :break, nest(glue("a", "b"), 1)}]) == ["a", "\n ", "b"] + test "break nest doc" do + # Consistent with definitions + assert nest(empty(), 1, :break) == {:doc_nest, empty(), 1, :break} + assert nest(empty(), 0, :break) == :doc_nil + + # Wrong argument type + assert_raise FunctionClauseError, fn -> nest("foo", empty(), :break) end # Consistent formatting - assert pretty(nest("a", 1), 80) == "a" - assert render(format 2, 0, [{0, :break, nest(glue("a", "b"), 1)}]) == "a\n b" + assert render(nest("a", 1, :break), 80) == "a" + assert render(nest(glue("a", "b"), 1, :break), 2) == "a\n b" + assert render(nest(line("a", "b"), 1, :break), 20) == "a\nb" end - test "line doc" do - # Consistency with definitions - assert line("a", "b") == - {:doc_cons, "a", {:doc_cons, :doc_line, "b"}} + test "cursor nest doc" do + # Consistent with definitions + assert nest(empty(), :cursor) == {:doc_nest, empty(), :cursor, :always} - # Consistence of corresponding sdoc - assert factor(line("a", "b"), 1) == ["a", "\n", "b"] - assert factor(line("a", "b"), 9) == ["a", "\n", "b"] + # Consistent formatting + assert render(nest("a", :cursor), 80) == "a" + assert render(concat("prefix ", nest(glue("a", "b"), :cursor)), 2) == "prefix a\n b" + assert render(concat("prefix ", nest(line("a", "b"), :cursor)), 2) == "prefix a\n b" + end + + test "reset nest doc" do + # Consistent with definitions + assert nest(empty(), :reset) == {:doc_nest, empty(), :reset, :always} # Consistent formatting - assert pretty(line(glue("aaa", "bbb"), glue("ccc", "ddd")), 10) == - "aaa bbb\nccc ddd" + assert render(nest("a", :reset), 80) == "a" + assert render(nest(nest(glue("a", "b"), :reset), 10), 2) == "a\nb" + assert render(nest(nest(line("a", "b"), :reset), 10), 2) == "a\nb" + end + + test "color doc" do + # Consistent with definitions + opts = %Inspect.Opts{} + assert color(empty(), :atom, opts) == empty() + + opts = %Inspect.Opts{syntax_colors: [regex: :red]} + assert color(empty(), :atom, opts) == empty() + + opts = %Inspect.Opts{syntax_colors: [atom: :red]} + doc1 = {:doc_color, "Hi", :red} + doc2 = {:doc_color, empty(), :reset} + assert color("Hi", :atom, opts) == concat(doc1, doc2) + + opts = %Inspect.Opts{syntax_colors: [reset: :red]} + assert color(empty(), :atom, opts) == empty() + + opts = %Inspect.Opts{syntax_colors: [number: :cyan, reset: :red]} + doc1 = {:doc_color, "123", :cyan} + doc2 = {:doc_color, empty(), :red} + assert color("123", :number, opts) == concat(doc1, doc2) + + # Consistent formatting + opts = %Inspect.Opts{syntax_colors: [atom: :cyan]} + assert render(glue(color("AA", :atom, opts), "BB"), 5) == "\e[36mAA\e[0m BB" + assert render(glue(color("AA", :atom, opts), "BB"), 3) == "\e[36mAA\e[0m\nBB" + assert render(glue("AA", color("BB", :atom, opts)), 6) == "AA \e[36mBB\e[0m" + end + + test "line doc" do + # Consistent with definitions + assert line("a", "b") == {:doc_cons, "a", {:doc_cons, :doc_line, "b"}} + + # Consistent formatting + assert render(line(glue("aaa", "bbb"), glue("ccc", "ddd")), 10) == "aaa bbb\nccc ddd" end test "group doc" do - # Consistency with definitions - assert group(glue("a", "b")) == - {:doc_group, {:doc_cons, "a", concat(break, "b")}} - assert group(empty) == {:doc_group, empty} + # Consistent with definitions + assert group("ab") == {:doc_group, "ab", :self} + assert group(empty()) == {:doc_group, empty(), :self} + + # Consistent formatting + doc = concat(glue(glue(glue("hello", "a"), "b"), "c"), "d") + assert render(group(doc), 5) == "hello\na\nb\ncd" + end + + test "group doc with inherit" do + # Consistent with definitions + assert group("ab", :inherit) == {:doc_group, "ab", :inherit} + assert group(empty(), :inherit) == {:doc_group, empty(), :inherit} + + # Consistent formatting + doc = concat(glue(glue(group(glue("a", "b"), :self), "c"), "d"), "hello") + assert render(group(doc), 5) == "a b\nc\ndhello" + + doc = concat(glue(glue(group(glue("a", "b"), :inherit), "c"), "d"), "hello") + assert render(group(doc), 5) == "a\nb\nc\ndhello" + end + + test "no limit doc" do + doc = no_limit(group(glue(glue("hello", "a"), "b"))) + assert render(doc, 5) == "hello a b" + assert render(doc, :infinity) == "hello a b" + end + + test "collapse lines" do + # Consistent with definitions + assert collapse_lines(3) == {:doc_collapse, 3} + + # Wrong argument type + assert_raise FunctionClauseError, fn -> collapse_lines(0) end + assert_raise FunctionClauseError, fn -> collapse_lines(empty()) end + + # Consistent formatting + doc = concat([collapse_lines(2), line(), line(), line()]) + assert render(doc, 10) == "\n\n" + assert render(nest(doc, 2), 10) == "\n\n " + + doc = concat([collapse_lines(2), line(), line()]) + assert render(doc, 10) == "\n\n" + assert render(nest(doc, 2), 10) == "\n\n " + + doc = concat([collapse_lines(2), line()]) + assert render(doc, 10) == "\n" + assert render(nest(doc, 2), 10) == "\n " - # Consistence of corresponding sdoc - assert factor(glue("a", "b"), 1) == ["a", " ", "b"] - assert factor(glue("a", "b"), 9) == ["a", " ", "b"] + doc = concat([collapse_lines(2), line(), "", line(), "", line()]) + assert render(doc, 10) == "\n\n" + assert render(nest(doc, 2), 10) == "\n\n " + + doc = concat([collapse_lines(2), line(), "foo", line(), "bar", line()]) + assert render(doc, 10) == "\nfoo\nbar\n" + assert render(nest(doc, 2), 10) == "\n foo\n bar\n " + end + + test "force doc and cancel doc" do + # Consistent with definitions + assert force_unfit("ab") == {:doc_force, "ab"} + assert force_unfit(empty()) == {:doc_force, empty()} + + # Consistent with definitions + assert next_break_fits("ab") == {:doc_fits, "ab", :enabled} + assert next_break_fits(empty()) == {:doc_fits, empty(), :enabled} + assert next_break_fits("ab", :disabled) == {:doc_fits, "ab", :disabled} + assert next_break_fits(empty(), :disabled) == {:doc_fits, empty(), :disabled} # Consistent formatting - assert pretty(helloabcd, 5) == "hello\na b\ncd" - assert pretty(helloabcd, 80) == "hello a b cd" + doc = force_unfit(concat(glue(glue(glue("hello", "a"), "b"), "c"), "d")) + assert render(doc, 20) == "hello\na\nb\ncd" + assert render(next_break_fits(doc, :enabled), 20) == "hello a b cd" + + assert render(next_break_fits(next_break_fits(doc, :enabled), :disabled), 20) == + "hello\na\nb\ncd" + end + + test "formatting groups with lines" do + doc = line(glue("a", "b"), glue("hello", "world")) + assert render(group(doc), 5) == "a\nb\nhello\nworld" + assert render(group(doc), 100) == "a b\nhello world" end test "formatting with infinity" do - s = String.duplicate "x", 50 - g = ";" - doc = group(glue(s, g, s) |> glue(g, s) |> glue(g, s) |> glue(g, s)) + str = String.duplicate("x", 50) + colon = ";" + + doc = + str + |> glue(colon, str) + |> glue(colon, str) + |> glue(colon, str) + |> glue(colon, str) + |> group() + + assert render(doc, :infinity) == + str <> colon <> str <> colon <> str <> colon <> str <> colon <> str + end + + test "formatting container_doc with empty" do + sm = &container_doc("[", &1, "]", %Inspect.Opts{}, fn d, _ -> d end, separator: ",") + + assert sm.([]) |> render(80) == "[]" + assert sm.([empty()]) |> render(80) == "[]" + assert sm.([empty(), empty()]) |> render(80) == "[]" + assert sm.(["a"]) |> render(80) == "[a]" + assert sm.(["a", empty()]) |> render(80) == "[a]" + assert sm.([empty(), "a"]) |> render(80) == "[a]" + assert sm.(["a", empty(), "b"]) |> render(80) == "[a, b]" + assert sm.([empty(), "a", "b"]) |> render(80) == "[a, b]" + assert sm.(["a", "b", empty()]) |> render(80) == "[a, b]" + assert sm.(["a", "b" | "c"]) |> render(80) == "[a, b | c]" + assert sm.(["a" | "b"]) |> render(80) == "[a | b]" + assert sm.(["a" | empty()]) |> render(80) == "[a]" + assert sm.([empty() | "b"]) |> render(80) == "[b]" + end + + test "formatting container_doc with empty and limit" do + opts = %Inspect.Opts{limit: 2} + value = ["a", empty(), "b"] - assert pretty(doc, :infinity) == s <> g <> s <> g <> s <> g <> s <> g <> s + assert container_doc("[", value, "]", opts, fn d, _ -> d end, separator: ",") |> render(80) == + "[a, b]" end end diff --git a/lib/elixir/test/elixir/inspect_test.exs b/lib/elixir/test/elixir/inspect_test.exs index 307801d8efc..27f76ced173 100644 --- a/lib/elixir/test/elixir/inspect_test.exs +++ b/lib/elixir/test/elixir/inspect_test.exs @@ -1,307 +1,812 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) + +# This is to temporarily test some inconsistencies in +# the error ArgumentError messages +# https://github.com/erlang/otp/issues/5440 +# TODO: once fixed in OTP and that minimum version is required, +# please remove MyArgumentError and replace the calls to: +# - MyArgumentError with ArgumentError +# - MyArgumentError.culprit() with Atom.to_string("Foo") +defmodule MyArgumentError do + defexception message: "argument error" + + @impl true + def message(_) do + """ + errors were found at the given arguments: + + * 1st argument: not an atom + """ + end + + def culprit() do + raise = fn -> raise(MyArgumentError) end + raise.() + end +end defmodule Inspect.AtomTest do use ExUnit.Case, async: true - test :basic do + doctest Inspect + + test "basic" do assert inspect(:foo) == ":foo" end - test :empty do + test "empty" do assert inspect(:"") == ":\"\"" end - test :true_false_nil do + test "true, false, nil" do assert inspect(false) == "false" assert inspect(true) == "true" assert inspect(nil) == "nil" end - test :with_uppercase do + test "with uppercase letters" do assert inspect(:fOO) == ":fOO" assert inspect(:FOO) == ":FOO" end - test :alias_atom do + test "aliases" do assert inspect(Foo) == "Foo" assert inspect(Foo.Bar) == "Foo.Bar" assert inspect(Elixir) == "Elixir" + assert inspect(Elixir.Foo) == "Foo" assert inspect(Elixir.Elixir) == "Elixir.Elixir" + assert inspect(Elixir.Elixir.Foo) == "Elixir.Elixir.Foo" end - test :with_integers do - assert inspect(User1) == "User1" + test "with integers" do + assert inspect(User1) == "User1" assert inspect(:user1) == ":user1" end - test :with_punctuation do + test "with trailing ? or !" do assert inspect(:foo?) == ":foo?" assert inspect(:bar!) == ":bar!" + assert inspect(:Foo?) == ":Foo?" end - test :op do - assert inspect(:+) == ":+" + test "operators" do + assert inspect(:+) == ":+" + assert inspect(:<~) == ":<~" + assert inspect(:~>) == ":~>" assert inspect(:&&&) == ":&&&" - assert inspect(:~~~) == ":~~~" + assert inspect(:"~~~") == ":\"~~~\"" + assert inspect(:<<~) == ":<<~" + assert inspect(:~>>) == ":~>>" + assert inspect(:<~>) == ":<~>" + assert inspect(:+++) == ":+++" + assert inspect(:---) == ":---" end - test :... do - assert inspect(:...) == ":..." + test "::" do + assert inspect(:"::") == ~s[:"::"] end - test :@ do + test "with @" do assert inspect(:@) == ":@" assert inspect(:foo@bar) == ":foo@bar" assert inspect(:foo@bar@) == ":foo@bar@" assert inspect(:foo@bar@baz) == ":foo@bar@baz" end - test :others do + test "others" do + assert inspect(:...) == ":..." assert inspect(:<<>>) == ":<<>>" - assert inspect(:{}) == ":{}" - assert inspect(:%{}) == ":%{}" - assert inspect(:%) == ":%" + assert inspect(:{}) == ":{}" + assert inspect(:%{}) == ":%{}" + assert inspect(:%) == ":%" + assert inspect(:->) == ":->" + end + + test "escaping" do + assert inspect(:"hy-phen") == ~s(:"hy-phen") + assert inspect(:"@hello") == ~s(:"@hello") + assert inspect(:"Wat!?") == ~s(:"Wat!?") + assert inspect(:"'quotes' and \"double quotes\"") == ~S(:"'quotes' and \"double quotes\"") + end + + test "colors" do + opts = [syntax_colors: [atom: :red]] + assert inspect(:hello, opts) == "\e[31m:hello\e[0m" + opts = [syntax_colors: [reset: :cyan]] + assert inspect(:hello, opts) == ":hello" + end + + test "Unicode" do + assert inspect(:olá) == ":olá" + assert inspect(:Olá) == ":Olá" + assert inspect(:Ólá) == ":Ólá" + assert inspect(:こんにちは世界) == ":こんにちは世界" + + nfd = :unicode.characters_to_nfd_binary("olá") + assert inspect(String.to_atom(nfd)) == ":\"#{nfd}\"" end end defmodule Inspect.BitStringTest do use ExUnit.Case, async: true - test :bitstring do - assert inspect(<<1 :: [size(12), integer, signed]>>) == "<<0, 1::size(4)>>" + test "bitstring" do + assert inspect(<<1::12-integer-signed>>) == "<<0, 1::size(4)>>" + + assert inspect(<<1::12-integer-signed>>, syntax_colors: [number: :blue]) == + "<<\e[34m0\e[0m, \e[34m1\e[0m::size(4)>>" + + assert inspect(<<1, 2, 3, 4, 5>>, pretty: true, width: 10) == "<<1, 2, 3,\n 4, 5>>" end - test :binary do + test "binary" do assert inspect("foo") == "\"foo\"" assert inspect(<>) == "\"abc\"" end - test :escape do + test "escaping" do assert inspect("f\no") == "\"f\\no\"" assert inspect("f\\o") == "\"f\\\\o\"" assert inspect("f\ao") == "\"f\\ao\"" + + assert inspect("\a\b\d\e\f\n\r\s\t\v") == "\"\\a\\b\\d\\e\\f\\n\\r \\t\\v\"" end - test :utf8 do + test "UTF-8" do assert inspect(" ゆんゆん") == "\" ゆんゆん\"" + # BOM + assert inspect("\uFEFFhello world") == "\"\\uFEFFhello world\"" end - test :all_escapes do - assert inspect("\a\b\d\e\f\n\r\s\t\v") == - "\"\\a\\b\\d\\e\\f\\n\\r \\t\\v\"" - end + test "infer" do + assert inspect(<<"john", 193, "doe">>, binaries: :infer) == + ~s(<<106, 111, 104, 110, 193, 100, 111, 101>>) - test :opt_infer do - assert inspect(<<"eric", 193, "mj">>, binaries: :infer) == ~s(<<101, 114, 105, 99, 193, 109, 106>>) - assert inspect(<<"eric">>, binaries: :infer) == ~s("eric") + assert inspect(<<"john">>, binaries: :infer) == ~s("john") assert inspect(<<193>>, binaries: :infer) == ~s(<<193>>) end - test :opt_as_strings do - assert inspect(<<"eric", 193, "mj">>, binaries: :as_strings) == ~s("eric\\301mj") - assert inspect(<<"eric">>, binaries: :as_strings) == ~s("eric") - assert inspect(<<193>>, binaries: :as_strings) == ~s("\\301") + test "as strings" do + assert inspect(<<"john", 193, "doe">>, binaries: :as_strings) == ~s("john\\xC1doe") + assert inspect(<<"john">>, binaries: :as_strings) == ~s("john") + assert inspect(<<193>>, binaries: :as_strings) == ~s("\\xC1") end - test :opt_as_binaries do - assert inspect(<<"eric", 193, "mj">>, binaries: :as_binaries) == "<<101, 114, 105, 99, 193, 109, 106>>" - assert inspect(<<"eric">>, binaries: :as_binaries) == "<<101, 114, 105, 99>>" + test "as binaries" do + assert inspect(<<"john", 193, "doe">>, binaries: :as_binaries) == + "<<106, 111, 104, 110, 193, 100, 111, 101>>" + + assert inspect(<<"john">>, binaries: :as_binaries) == "<<106, 111, 104, 110>>" assert inspect(<<193>>, binaries: :as_binaries) == "<<193>>" + + # Any base other than :decimal implies "binaries: :as_binaries" + assert inspect("abc", base: :hex) == "<<0x61, 0x62, 0x63>>" + assert inspect("abc", base: :octal) == "<<0o141, 0o142, 0o143>>" + + # Size is still represented as decimal + assert inspect(<<10, 11, 12::4>>, base: :hex) == "<<0xA, 0xB, 0xC::size(4)>>" end - test :unprintable_with_opts do + test "unprintable with limit" do assert inspect(<<193, 193, 193, 193>>, limit: 3) == "<<193, 193, 193, ...>>" end + + test "printable limit" do + assert inspect("hello world", printable_limit: 4) == ~s("hell" <> ...) + + # Non-printable characters after the limit don't matter + assert inspect("hello world" <> <<0>>, printable_limit: 4) == ~s("hell" <> ...) + + # Non printable strings aren't affected by printable limit + assert inspect(<<0, 1, 2, 3, 4>>, printable_limit: 3) == ~s(<<0, 1, 2, 3, 4>>) + end end defmodule Inspect.NumberTest do use ExUnit.Case, async: true - test :integer do + test "integer" do assert inspect(100) == "100" end - test :float do + test "decimal" do + assert inspect(100, base: :decimal) == "100" + end + + test "hex" do + assert inspect(100, base: :hex) == "0x64" + assert inspect(-100, base: :hex) == "-0x64" + end + + test "octal" do + assert inspect(100, base: :octal) == "0o144" + assert inspect(-100, base: :octal) == "-0o144" + end + + test "binary" do + assert inspect(86, base: :binary) == "0b1010110" + assert inspect(-86, base: :binary) == "-0b1010110" + end + + test "float" do assert inspect(1.0) == "1.0" - assert inspect(1.0E10) == "1.0e10" - assert inspect(1.0e10) == "1.0e10" + assert inspect(1.0e10) == "10000000000.0" assert inspect(1.0e-10) == "1.0e-10" end + + test "integer colors" do + opts = [syntax_colors: [number: :red]] + assert inspect(123, opts) == "\e[31m123\e[0m" + opts = [syntax_colors: [reset: :cyan]] + assert inspect(123, opts) == "123" + end + + test "float colors" do + opts = [syntax_colors: [number: :red]] + assert inspect(1.3, opts) == "\e[31m1.3\e[0m" + opts = [syntax_colors: [reset: :cyan]] + assert inspect(1.3, opts) == "1.3" + end end defmodule Inspect.TupleTest do - use ExUnit.Case + use ExUnit.Case, async: true - test :basic do + test "basic" do assert inspect({1, "b", 3}) == "{1, \"b\", 3}" - assert inspect({1, "b", 3}, [pretty: true, width: 1]) == "{1,\n \"b\",\n 3}" + assert inspect({1, "b", 3}, pretty: true, width: 1) == "{1,\n \"b\",\n 3}" + assert inspect({1, "b", 3}, pretty: true, width: 10) == "{1, \"b\",\n 3}" end - test :empty do + test "empty" do assert inspect({}) == "{}" end - test :with_limit do + test "with limit" do assert inspect({1, 2, 3, 4}, limit: 3) == "{1, 2, 3, ...}" end + + test "colors" do + opts = [syntax_colors: []] + assert inspect({}, opts) == "{}" + + opts = [syntax_colors: [reset: :cyan]] + assert inspect({}, opts) == "{}" + assert inspect({:x, :y}, opts) == "{:x, :y}" + + opts = [syntax_colors: [reset: :cyan, atom: :red]] + assert inspect({}, opts) == "{}" + assert inspect({:x, :y}, opts) == "{\e[31m:x\e[36m, \e[31m:y\e[36m}" + + opts = [syntax_colors: [tuple: :green, reset: :cyan, atom: :red]] + assert inspect({}, opts) == "\e[32m{\e[36m\e[32m}\e[36m" + + assert inspect({:x, :y}, opts) == + "\e[32m{\e[36m\e[31m:x\e[36m\e[32m,\e[36m \e[31m:y\e[36m\e[32m}\e[36m" + end end defmodule Inspect.ListTest do use ExUnit.Case, async: true - test :basic do - assert inspect([ 1, "b", 3 ]) == "[1, \"b\", 3]" - assert inspect([ 1, "b", 3 ], [pretty: true, width: 1]) == "[1,\n \"b\",\n 3]" + test "basic" do + assert inspect([1, "b", 3]) == "[1, \"b\", 3]" + assert inspect([1, "b", 3], pretty: true, width: 1) == "[1,\n \"b\",\n 3]" end - test :printable do + test "printable" do assert inspect('abc') == "'abc'" end - test :keyword do - assert inspect([a: 1]) == "[a: 1]" - assert inspect([a: 1, b: 2]) == "[a: 1, b: 2]" - assert inspect([a: 1, a: 2, b: 2]) == "[a: 1, a: 2, b: 2]" - assert inspect(["123": 1]) == ~s(["123": 1]) + test "printable limit" do + assert inspect('hello world', printable_limit: 4) == ~s('hell' ++ ...) + # Non printable characters after the limit don't matter + assert inspect('hello world' ++ [0], printable_limit: 4) == ~s('hell' ++ ...) + # Non printable strings aren't affected by printable limit + assert inspect([0, 1, 2, 3, 4], printable_limit: 3) == ~s([0, 1, 2, 3, 4]) + end + + test "keyword" do + assert inspect(a: 1) == "[a: 1]" + assert inspect(a: 1, b: 2) == "[a: 1, b: 2]" + assert inspect(a: 1, a: 2, b: 2) == "[a: 1, a: 2, b: 2]" + assert inspect("123": 1) == ~s(["123": 1]) - assert inspect([foo: [1,2,3,:bar], bazzz: :bat], [pretty: true, width: 30]) == - "[foo: [1, 2, 3, :bar],\n bazzz: :bat]" + assert inspect([foo: [1, 2, 3], baz: [4, 5, 6]], pretty: true, width: 20) == + "[\n foo: [1, 2, 3],\n baz: [4, 5, 6]\n]" end - test :opt_infer do - assert inspect('eric' ++ [0] ++ 'mj', char_lists: :infer) == "[101, 114, 105, 99, 0, 109, 106]" - assert inspect('eric', char_lists: :infer) == "'eric'" - assert inspect([0], char_lists: :infer) == "[0]" + test "keyword operators" do + assert inspect("::": 1, +: 2) == ~s(["::": 1, +: 2]) end - test :opt_as_strings do - assert inspect('eric' ++ [0] ++ 'mj', char_lists: :as_char_lists) == "'eric\\000mj'" - assert inspect('eric', char_lists: :as_char_lists) == "'eric'" - assert inspect([0], char_lists: :as_char_lists) == "'\\000'" + test "opt infer" do + assert inspect('john' ++ [0] ++ 'doe', charlists: :infer) == + "[106, 111, 104, 110, 0, 100, 111, 101]" + + assert inspect('john', charlists: :infer) == "'john'" + assert inspect([0], charlists: :infer) == "[0]" + end + + test "opt as strings" do + assert inspect('john' ++ [0] ++ 'doe', charlists: :as_charlists) == "'john\\0doe'" + assert inspect('john', charlists: :as_charlists) == "'john'" + assert inspect([0], charlists: :as_charlists) == "'\\0'" end - test :opt_as_lists do - assert inspect('eric' ++ [0] ++ 'mj', char_lists: :as_lists) == "[101, 114, 105, 99, 0, 109, 106]" - assert inspect('eric', char_lists: :as_lists) == "[101, 114, 105, 99]" - assert inspect([0], char_lists: :as_lists) == "[0]" + test "opt as lists" do + assert inspect('john' ++ [0] ++ 'doe', charlists: :as_lists) == + "[106, 111, 104, 110, 0, 100, 111, 101]" + + assert inspect('john', charlists: :as_lists) == "[106, 111, 104, 110]" + assert inspect([0], charlists: :as_lists) == "[0]" end - test :non_printable do + test "non printable" do assert inspect([{:b, 1}, {:a, 1}]) == "[b: 1, a: 1]" end - test :unproper do + test "improper" do assert inspect([:foo | :bar]) == "[:foo | :bar]" - assert inspect([1,2,3,4,5|42], [pretty: true, width: 1]) == "[1,\n 2,\n 3,\n 4,\n 5 |\n 42]" + assert inspect([1, 2, 3, 4, 5 | 42], pretty: true, width: 1) == + "[1,\n 2,\n 3,\n 4,\n 5 |\n 42]" end - test :codepoints do + test "nested" do + assert inspect(Enum.reduce(1..100, [0], &[&2, Integer.to_string(&1)]), limit: 5) == + "[[[[[[...], ...], \"97\"], \"98\"], \"99\"], \"100\"]" + + assert inspect(Enum.reduce(1..100, [0], &[&2 | Integer.to_string(&1)]), limit: 5) == + "[[[[[[...] | \"96\"] | \"97\"] | \"98\"] | \"99\"] | \"100\"]" + end + + test "codepoints" do assert inspect('é') == "[233]" end - test :empty do + test "empty" do assert inspect([]) == "[]" end - test :with_limit do - assert inspect([ 1, 2, 3, 4 ], limit: 3) == "[1, 2, 3, ...]" + test "with limit" do + assert inspect([1, 2, 3, 4], limit: 3) == "[1, 2, 3, ...]" + end + + test "colors" do + opts = [syntax_colors: []] + assert inspect([], opts) == "[]" + + opts = [syntax_colors: [reset: :cyan]] + assert inspect([], opts) == "[]" + assert inspect([:x, :y], opts) == "[:x, :y]" + + opts = [syntax_colors: [reset: :cyan, atom: :red]] + assert inspect([], opts) == "[]" + assert inspect([:x, :y], opts) == "[\e[31m:x\e[36m, \e[31m:y\e[36m]" + + opts = [syntax_colors: [reset: :cyan, atom: :red, list: :green]] + assert inspect([], opts) == "\e[32m[]\e[36m" + + assert inspect([:x, :y], opts) == + "\e[32m[\e[36m\e[31m:x\e[36m\e[32m,\e[36m \e[31m:y\e[36m\e[32m]\e[36m" + end + + test "keyword with colors" do + opts = [syntax_colors: [reset: :cyan, list: :green, number: :blue]] + assert inspect([], opts) == "\e[32m[]\e[36m" + + assert inspect([a: 9999], opts) == "\e[32m[\e[36ma: \e[34m9999\e[36m\e[32m]\e[36m" + + opts = [syntax_colors: [reset: :cyan, atom: :red, list: :green, number: :blue]] + assert inspect([], opts) == "\e[32m[]\e[36m" + + assert inspect([a: 9999], opts) == "\e[32m[\e[36m\e[31ma:\e[36m \e[34m9999\e[36m\e[32m]\e[36m" + end + + test "limit with colors" do + opts = [limit: 1, syntax_colors: [reset: :cyan, list: :green, atom: :red]] + assert inspect([], opts) == "\e[32m[]\e[36m" + + assert inspect([:x, :y], opts) == "\e[32m[\e[36m\e[31m:x\e[36m\e[32m,\e[36m ...\e[32m]\e[36m" end end defmodule Inspect.MapTest do - use ExUnit.Case + use ExUnit.Case, async: true - test :basic do + test "basic" do assert inspect(%{1 => "b"}) == "%{1 => \"b\"}" - assert inspect(%{1 => "b", 2 => "c"}, [pretty: true, width: 1]) == "%{1 => \"b\",\n 2 => \"c\"}" + + assert inspect(%{1 => "b", 2 => "c"}, pretty: true, width: 1) == + "%{\n 1 => \"b\",\n 2 => \"c\"\n}" end - test :keyword do + test "keyword" do assert inspect(%{a: 1}) == "%{a: 1}" assert inspect(%{a: 1, b: 2}) == "%{a: 1, b: 2}" assert inspect(%{a: 1, b: 2, c: 3}) == "%{a: 1, b: 2, c: 3}" end - test :with_limit do - assert inspect(%{1 => 1, 2 => 2, 3 => 3, 4 => 4}, limit: 3) == "%{1 => 1, 2 => 2, 3 => 3, ...}" + test "with limit" do + assert inspect(%{1 => 1, 2 => 2, 3 => 3, 4 => 4}, limit: 3) == + "%{1 => 1, 2 => 2, 3 => 3, ...}" end defmodule Public do - def __struct__ do - %{key: 0, __struct__: Public} - end + defstruct key: 0 end defmodule Private do end - test :public_struct do + test "public struct" do assert inspect(%Public{key: 1}) == "%Inspect.MapTest.Public{key: 1}" end - test :public_modified_struct do + test "public modified struct" do public = %Public{key: 1} + assert inspect(Map.put(public, :foo, :bar)) == - "%{__struct__: Inspect.MapTest.Public, foo: :bar, key: 1}" + "%{__struct__: Inspect.MapTest.Public, foo: :bar, key: 1}" end - test :private_struct do - assert inspect(%{__struct__: Private, key: 1}) == "%{__struct__: Inspect.MapTest.Private, key: 1}" + test "private struct" do + assert inspect(%{__struct__: Private, key: 1}) == + "%{__struct__: Inspect.MapTest.Private, key: 1}" end defmodule Failing do - def __struct__ do - %{key: 0} - end + @enforce_keys [:name] + defstruct @enforce_keys defimpl Inspect do - def inspect(_, _) do - raise "failing" + def inspect(%Failing{name: _name}, _) do + MyArgumentError.culprit() end end end - test :bad_implementation do - msg = "Got RuntimeError with message \"failing\" " <> - "while inspecting %{__struct__: Inspect.MapTest.Failing, key: 0}" + test "safely inspect bad implementation" do + assert_raise MyArgumentError, ~r/errors were found at the given arguments:/, fn -> + raise(MyArgumentError) + end + + message = ~s''' + #Inspect.Error< + got MyArgumentError with message: + + """ + errors were found at the given arguments: + + * 1st argument: not an atom + """ + + while inspecting: + + %{__struct__: Inspect.MapTest.Failing, name: "Foo"} + ''' + + assert inspect(%Failing{name: "Foo"}) =~ message + end + + test "safely inspect bad implementation disables colors" do + message = ~s''' + #Inspect.Error< + got MyArgumentError with message: + + """ + errors were found at the given arguments: + + * 1st argument: not an atom + """ + + while inspecting: + + %{__struct__: Inspect.MapTest.Failing, name: "Foo"} + ''' + + assert inspect(%Failing{name: "Foo"}, syntax_colors: [atom: [:green]]) =~ message + end + + test "unsafely inspect bad implementation" do + exception_message = ~s''' + got MyArgumentError with message: + + """ + errors were found at the given arguments: + + * 1st argument: not an atom + """ + + while inspecting: + + %{__struct__: Inspect.MapTest.Failing, name: "Foo"} + ''' + + try do + inspect(%Failing{name: "Foo"}, safe: false) + rescue + exception in Inspect.Error -> + assert Exception.message(exception) =~ exception_message + assert [{MyArgumentError, fun_name, 0, [{:file, _}, {:line, _} | _]} | _] = __STACKTRACE__ + + assert fun_name in [:"-culprit/0-fun-0-", :culprit] + assert Exception.message(exception) =~ exception_message + else + _ -> flunk("expected failure") + end + end + + test "raise when trying to inspect with a bad implementation from inside another exception that is being raised" do + # Inspect.Error is raised here when we tried to print the error message + # called by another exception (Protocol.UndefinedError in this case) + exception_message = ~s''' + protocol Enumerable not implemented for #Inspect.Error< + got MyArgumentError with message: + + """ + errors were found at the given arguments: + + * 1st argument: not an atom + """ + + while inspecting: - assert_raise ArgumentError, msg, fn -> - inspect(%Failing{}) + %{__struct__: Inspect.MapTest.Failing, name: "Foo"} + ''' + + try do + Enum.to_list(%Failing{name: "Foo"}) + rescue + exception in Protocol.UndefinedError -> + assert Exception.message(exception) =~ exception_message + + assert [ + {Enumerable, :impl_for!, 1, _} | _ + ] = __STACKTRACE__ + + # The culprit + assert Enum.any?(__STACKTRACE__, fn + {Enum, :to_list, 1, _} -> true + _ -> false + end) + + # The line calling the culprit + assert Enum.any?(__STACKTRACE__, fn + {Inspect.MapTest, _test_name, 1, file: file, line: _line_number} -> + String.ends_with?(List.to_string(file), "test/elixir/inspect_test.exs") + + _ -> + false + end) + else + _ -> flunk("expected failure") end end - test :exception do + test "Exception.message/1 with bad implementation" do + message = ~s''' + #Inspect.Error< + got MyArgumentError with message: + + """ + errors were found at the given arguments: + + * 1st argument: not an atom + """ + + while inspecting: + + %{__struct__: Inspect.MapTest.Failing, name: "Foo"} + ''' + + {my_argument_error, stacktrace} = + try do + MyArgumentError.culprit() + rescue + e -> + {e, __STACKTRACE__} + end + + inspected = + inspect( + Inspect.Error.exception( + exception: my_argument_error, + stacktrace: stacktrace, + inspected_struct: "%{__struct__: Inspect.MapTest.Failing, name: \"Foo\"}" + ) + ) + + assert inspect(%Failing{name: "Foo"}) =~ message + assert inspected =~ message + end + + test "exception" do assert inspect(%RuntimeError{message: "runtime error"}) == - "%RuntimeError{message: \"runtime error\"}" + "%RuntimeError{message: \"runtime error\"}" + end + + test "colors" do + opts = [syntax_colors: [reset: :cyan, atom: :red, number: :magenta]] + assert inspect(%{1 => 2}, opts) == "%{\e[35m1\e[36m => \e[35m2\e[36m}" + + assert inspect(%{a: 1}, opts) == "%{\e[31ma:\e[36m \e[35m1\e[36m}" + + assert inspect(%Public{key: 1}, opts) == + "%Inspect.MapTest.Public{\e[31mkey:\e[36m \e[35m1\e[36m}" + + opts = [syntax_colors: [reset: :cyan, atom: :red, map: :green, number: :blue]] + + assert inspect(%{a: 9999}, opts) == + "\e[32m%{\e[36m" <> "\e[31ma:\e[36m " <> "\e[34m9999\e[36m" <> "\e[32m}\e[36m" + end + + defmodule StructWithoutOptions do + @derive Inspect + defstruct [:a, :b, :c, :d] + end + + test "struct without options" do + struct = %StructWithoutOptions{a: 1, b: 2, c: 3, d: 4} + assert inspect(struct) == "%Inspect.MapTest.StructWithoutOptions{a: 1, b: 2, c: 3, d: 4}" + + assert inspect(struct, pretty: true, width: 1) == + "%Inspect.MapTest.StructWithoutOptions{\n a: 1,\n b: 2,\n c: 3,\n d: 4\n}" + end + + defmodule StructWithOnlyOption do + @derive {Inspect, only: [:b, :c]} + defstruct [:a, :b, :c, :d] + end + + test "struct with :only option" do + struct = %StructWithOnlyOption{a: 1, b: 2, c: 3, d: 4} + assert inspect(struct) == "#Inspect.MapTest.StructWithOnlyOption" + + assert inspect(struct, pretty: true, width: 1) == + "#Inspect.MapTest.StructWithOnlyOption<\n b: 2,\n c: 3,\n ...\n>" + + struct = %{struct | c: [1, 2, 3, 4]} + assert inspect(struct) == "#Inspect.MapTest.StructWithOnlyOption" + end + + defmodule StructWithEmptyOnlyOption do + @derive {Inspect, only: []} + defstruct [:a, :b, :c, :d] + end + + test "struct with empty :only option" do + struct = %StructWithEmptyOnlyOption{a: 1, b: 2, c: 3, d: 4} + assert inspect(struct) == "#Inspect.MapTest.StructWithEmptyOnlyOption<...>" + end + + defmodule StructWithAllFieldsInOnlyOption do + @derive {Inspect, only: [:a, :b]} + defstruct [:a, :b] + end + + test "struct with all fields in the :only option" do + struct = %StructWithAllFieldsInOnlyOption{a: 1, b: 2} + assert inspect(struct) == "%Inspect.MapTest.StructWithAllFieldsInOnlyOption{a: 1, b: 2}" + + assert inspect(struct, pretty: true, width: 1) == + "%Inspect.MapTest.StructWithAllFieldsInOnlyOption{\n a: 1,\n b: 2\n}" + end + + test "struct missing fields in the :only option" do + assert_raise ArgumentError, + "unknown fields [:c] in :only when deriving the Inspect protocol for Inspect.MapTest.StructMissingFieldsInOnlyOption", + fn -> + defmodule StructMissingFieldsInOnlyOption do + @derive {Inspect, only: [:c]} + defstruct [:a, :b] + end + end + end + + test "struct missing fields in the :except option" do + assert_raise ArgumentError, + "unknown fields [:c, :d] in :except when deriving the Inspect protocol for Inspect.MapTest.StructMissingFieldsInExceptOption", + fn -> + defmodule StructMissingFieldsInExceptOption do + @derive {Inspect, except: [:c, :d]} + defstruct [:a, :b] + end + end + end + + defmodule StructWithExceptOption do + @derive {Inspect, except: [:b, :c]} + defstruct [:a, :b, :c, :d] + end + + test "struct with :except option" do + struct = %StructWithExceptOption{a: 1, b: 2, c: 3, d: 4} + assert inspect(struct) == "#Inspect.MapTest.StructWithExceptOption" + + assert inspect(struct, pretty: true, width: 1) == + "#Inspect.MapTest.StructWithExceptOption<\n a: 1,\n d: 4,\n ...\n>" + end + + defmodule StructWithBothOnlyAndExceptOptions do + @derive {Inspect, only: [:a, :b], except: [:b, :c]} + defstruct [:a, :b, :c, :d] + end + + test "struct with both :only and :except options" do + struct = %StructWithBothOnlyAndExceptOptions{a: 1, b: 2, c: 3, d: 4} + assert inspect(struct) == "#Inspect.MapTest.StructWithBothOnlyAndExceptOptions" + + assert inspect(struct, pretty: true, width: 1) == + "#Inspect.MapTest.StructWithBothOnlyAndExceptOptions<\n a: 1,\n ...\n>" + end + + defmodule StructWithOptionalAndOrder do + @derive {Inspect, optional: [:b, :c]} + defstruct [:c, :d, :a, :b] + end + + test "struct with both :order and :optional options" do + struct = %StructWithOptionalAndOrder{a: 1, b: 2, c: 3, d: 4} + + assert inspect(struct) == + "%Inspect.MapTest.StructWithOptionalAndOrder{c: 3, d: 4, a: 1, b: 2}" + + struct = %StructWithOptionalAndOrder{} + assert inspect(struct) == "%Inspect.MapTest.StructWithOptionalAndOrder{d: nil, a: nil}" + end + + defmodule StructWithExceptOptionalAndOrder do + @derive {Inspect, optional: [:b, :c], except: [:e]} + defstruct [:c, :d, :e, :a, :b] + end + + test "struct with :except, :order, and :optional options" do + struct = %StructWithExceptOptionalAndOrder{a: 1, b: 2, c: 3, d: 4} + + assert inspect(struct) == + "#Inspect.MapTest.StructWithExceptOptionalAndOrder" + + struct = %StructWithExceptOptionalAndOrder{} + + assert inspect(struct) == + "#Inspect.MapTest.StructWithExceptOptionalAndOrder" end end defmodule Inspect.OthersTest do use ExUnit.Case, async: true - def f do - fn() -> :ok end + def fun() do + fn -> :ok end end - test :external_elixir_funs do + def unquote(:"weirdly named/fun-")() do + fn -> :ok end + end + + test "external Elixir funs" do bin = inspect(&Enum.map/2) assert bin == "&Enum.map/2" + + assert inspect(&__MODULE__."weirdly named/fun-"/0) == + ~s(&Inspect.OthersTest."weirdly named/fun-"/0) end - test :external_erlang_funs do + test "external Erlang funs" do bin = inspect(&:lists.map/2) assert bin == "&:lists.map/2" end - test :outdated_functions do + test "outdated functions" do defmodule V do def fun do fn -> 1 end end end - Application.put_env(:elixir, :anony, V.fun) + Application.put_env(:elixir, :anony, V.fun()) Application.put_env(:elixir, :named, &V.fun/0) :code.delete(V) @@ -310,34 +815,157 @@ defmodule Inspect.OthersTest do anony = Application.get_env(:elixir, :anony) named = Application.get_env(:elixir, :named) - assert inspect(anony) =~ ~r"#Function<0.\d+/0 in Inspect.OthersTest.V>" + assert inspect(anony) =~ ~r"#Function<0.\d+/0" assert inspect(named) =~ ~r"&Inspect.OthersTest.V.fun/0" after Application.delete_env(:elixir, :anony) Application.delete_env(:elixir, :named) end - test :other_funs do - assert "#Function<" <> _ = inspect(fn(x) -> x + 1 end) - assert "#Function<" <> _ = inspect(f) + test "other funs" do + assert "#Function<" <> _ = inspect(fn x -> x + 1 end) + assert "#Function<" <> _ = inspect(fun()) + opts = [syntax_colors: []] + assert "#Function<" <> _ = inspect(fun(), opts) + opts = [syntax_colors: [reset: :red]] + assert "#Function<" <> rest = inspect(fun(), opts) + assert String.ends_with?(rest, ">") + + inspected = inspect(__MODULE__."weirdly named/fun-"()) + assert inspected =~ ~r(#Function<\d+\.\d+/0 in Inspect\.OthersTest\."weirdly named/fun-"/0>) + end + + test "map set" do + assert "MapSet.new(" <> _ = inspect(MapSet.new()) + end + + test "PIDs" do + assert "#PID<" <> _ = inspect(self()) + opts = [syntax_colors: []] + assert "#PID<" <> _ = inspect(self(), opts) + opts = [syntax_colors: [reset: :cyan]] + assert "#PID<" <> rest = inspect(self(), opts) + assert String.ends_with?(rest, ">") + end + + test "references" do + assert "#Reference<" <> _ = inspect(make_ref()) + end + + test "regex" do + assert inspect(~r(foo)m) == "~r/foo/m" + + assert inspect(Regex.compile!("a\\/b")) == "~r/a\\/b/" + + assert inspect(Regex.compile!("\a\b\d\e\f\n\r\s\t\v/")) == + "~r/\\a\\x08\\x7F\\x1B\\f\\n\\r \\t\\v\\//" + + assert inspect(~r<\a\b\d\e\f\n\r\s\t\v/>) == "~r/\\a\\b\\d\\e\\f\\n\\r\\s\\t\\v\\//" + assert inspect(~r" \\/ ") == "~r/ \\\\\\/ /" + assert inspect(~r/hi/, syntax_colors: [regex: :red]) == "\e[31m~r/hi/\e[0m" + end + + test "inspect_fun" do + fun = fn + integer, _opts when is_integer(integer) -> + "<#{integer}>" + + %URI{} = uri, _opts -> + "#URI<#{uri}>" + + term, opts -> + Inspect.inspect(term, opts) + end + + opts = [inspect_fun: fun] + + assert inspect(1000, opts) == "<1000>" + assert inspect([1000], opts) == "[<1000>]" + + uri = URI.parse("https://elixir-lang.org") + assert inspect(uri, opts) == "#URI" + assert inspect([uri], opts) == "[#URI]" + end + + defmodule Nested do + defstruct nested: nil + + defimpl Inspect do + import Inspect.Algebra + + def inspect(%Nested{nested: nested}, opts) do + indent = Keyword.get(opts.custom_options, :indent, 2) + level = Keyword.get(opts.custom_options, :level, 1) + + nested_str = + Kernel.inspect(nested, custom_options: [level: level + 1, indent: indent + 2]) + + concat( + nest(line("#Nested[##{level}/#{indent}]<", nested_str), indent), + nest(line("", ">"), indent - 2) + ) + end + end end - test :hash_dict_set do - assert "#HashDict<" <> _ = inspect(HashDict.new) - assert "#HashSet<" <> _ = inspect(HashSet.new) + test "custom_options" do + assert inspect(%Nested{nested: %Nested{nested: 42}}) == + "#Nested[#1/2]<\n #Nested[#2/4]<\n 42\n >\n>" end +end - test :pids do - assert "#PID<" <> _ = inspect(self) +defmodule Inspect.CustomProtocolTest do + use ExUnit.Case, async: true + + defprotocol CustomInspect do + def inspect(term, opts) end - test :references do - assert "#Reference<" <> _ = inspect(make_ref) + defmodule MissingImplementation do + defstruct [] end - test :regex do - "~r/foo/m" = inspect(~r(foo)m) - "~r/\\a\\010\\177\\033\\f\\n\\r \\t\\v\\//" = inspect(Regex.compile!("\a\b\d\e\f\n\r\s\t\v/")) - "~r/\\a\\b\\d\\e\\f\\n\\r\\s\\t\\v\\//" = inspect(~r<\a\b\d\e\f\n\r\s\t\v/>) + test "unsafely inspect missing implementation" do + msg = ~S''' + got Protocol.UndefinedError with message: + + """ + protocol Inspect.CustomProtocolTest.CustomInspect not implemented for %Inspect.CustomProtocolTest.MissingImplementation{} of type Inspect.CustomProtocolTest.MissingImplementation (a struct) + """ + + while inspecting: + + %{__struct__: Inspect.CustomProtocolTest.MissingImplementation} + ''' + + opts = [safe: false, inspect_fun: &CustomInspect.inspect/2] + + try do + inspect(%MissingImplementation{}, opts) + rescue + e in Inspect.Error -> + assert Exception.message(e) =~ msg + assert [{Inspect.CustomProtocolTest.CustomInspect, :impl_for!, 1, _} | _] = __STACKTRACE__ + else + _ -> flunk("expected failure") + end + end + + test "safely inspect missing implementation" do + msg = ~S''' + #Inspect.Error< + got Protocol.UndefinedError with message: + + """ + protocol Inspect.CustomProtocolTest.CustomInspect not implemented for %Inspect.CustomProtocolTest.MissingImplementation{} of type Inspect.CustomProtocolTest.MissingImplementation (a struct) + """ + + while inspecting: + + %{__struct__: Inspect.CustomProtocolTest.MissingImplementation} + ''' + + opts = [inspect_fun: &CustomInspect.inspect/2] + assert inspect(%MissingImplementation{}, opts) =~ msg end end diff --git a/lib/elixir/test/elixir/integer_test.exs b/lib/elixir/test/elixir/integer_test.exs index 547c71567a0..28c2eaf5201 100644 --- a/lib/elixir/test/elixir/integer_test.exs +++ b/lib/elixir/test/elixir/integer_test.exs @@ -1,33 +1,130 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule IntegerTest do use ExUnit.Case, async: true + + doctest Integer + require Integer - test :odd? do - assert Integer.odd?(0) == false - assert Integer.odd?(1) == true - assert Integer.odd?(2) == false - assert Integer.odd?(3) == true - assert Integer.odd?(-1) == true - assert Integer.odd?(-2) == false - assert Integer.odd?(-3) == true + def test_is_odd_in_guards(number) when Integer.is_odd(number), do: number + def test_is_odd_in_guards(_number), do: false + + def test_is_even_in_guards(number) when Integer.is_even(number), do: number + def test_is_even_in_guards(_number), do: false + + test "is_odd/1" do + assert Integer.is_odd(0) == false + assert Integer.is_odd(1) == true + assert Integer.is_odd(2) == false + assert Integer.is_odd(3) == true + assert Integer.is_odd(-1) == true + assert Integer.is_odd(-2) == false + assert Integer.is_odd(-3) == true + assert test_is_odd_in_guards(10) == false + assert test_is_odd_in_guards(11) == 11 + assert test_is_odd_in_guards(:not_integer) == false + end + + test "is_even/1" do + assert Integer.is_even(0) == true + assert Integer.is_even(1) == false + assert Integer.is_even(2) == true + assert Integer.is_even(3) == false + assert Integer.is_even(-1) == false + assert Integer.is_even(-2) == true + assert Integer.is_even(-3) == false + assert test_is_even_in_guards(10) == 10 + assert test_is_even_in_guards(11) == false + assert test_is_even_in_guards(:not_integer) == false + end + + test "mod/2" do + assert Integer.mod(3, 2) == 1 + assert Integer.mod(0, 10) == 0 + assert Integer.mod(30000, 2001) == 1986 + assert Integer.mod(-20, 11) == 2 + end + + test "mod/2 raises ArithmeticError when divisor is 0" do + assert_raise ArithmeticError, fn -> Integer.mod(3, 0) end + assert_raise ArithmeticError, fn -> Integer.mod(-50, 0) end + end + + test "mod/2 raises ArithmeticError when non-integers used as arguments" do + assert_raise ArithmeticError, fn -> Integer.mod(3.0, 2) end + assert_raise ArithmeticError, fn -> Integer.mod(20, 1.2) end end - test :even? do - assert Integer.even?(0) == true - assert Integer.even?(1) == false - assert Integer.even?(2) == true - assert Integer.even?(3) == false - assert Integer.even?(-1) == false - assert Integer.even?(-2) == true - assert Integer.even?(-3) == false + test "floor_div/2" do + assert Integer.floor_div(3, 2) == 1 + assert Integer.floor_div(0, 10) == 0 + assert Integer.floor_div(30000, 2001) == 14 + assert Integer.floor_div(-20, 11) == -2 end - test :parse do + test "floor_div/2 raises ArithmeticError when divisor is 0" do + assert_raise ArithmeticError, fn -> Integer.floor_div(3, 0) end + assert_raise ArithmeticError, fn -> Integer.floor_div(-50, 0) end + end + + test "floor_div/2 raises ArithmeticError when non-integers used as arguments" do + assert_raise ArithmeticError, fn -> Integer.floor_div(3.0, 2) end + assert_raise ArithmeticError, fn -> Integer.floor_div(20, 1.2) end + end + + test "digits/2" do + assert Integer.digits(0) == [0] + assert Integer.digits(0, 2) == [0] + assert Integer.digits(1) == [1] + assert Integer.digits(-1) == [-1] + assert Integer.digits(123, 123) == [1, 0] + assert Integer.digits(-123, 123) == [-1, 0] + assert Integer.digits(456, 1000) == [456] + assert Integer.digits(-456, 1000) == [-456] + assert Integer.digits(123) == [1, 2, 3] + assert Integer.digits(-123) == [-1, -2, -3] + assert Integer.digits(58127, 2) == [1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1] + assert Integer.digits(-58127, 2) == [-1, -1, -1, 0, 0, 0, -1, -1, 0, 0, 0, 0, -1, -1, -1, -1] + + for n <- Enum.to_list(-1..1) do + assert_raise FunctionClauseError, fn -> + Integer.digits(10, n) + Integer.digits(-10, n) + end + end + end + + test "undigits/2" do + assert Integer.undigits([]) == 0 + assert Integer.undigits([0]) == 0 + assert Integer.undigits([1]) == 1 + assert Integer.undigits([1, 0, 1]) == 101 + assert Integer.undigits([1, 4], 16) == 0x14 + assert Integer.undigits([1, 4], 8) == 0o14 + assert Integer.undigits([1, 1], 2) == 0b11 + assert Integer.undigits([1, 2, 3, 4, 5]) == 12345 + assert Integer.undigits([1, 0, -5]) == 95 + assert Integer.undigits([-1, -1, -5]) == -115 + assert Integer.undigits([0, 0, 0, -1, -1, -5]) == -115 + + for n <- Enum.to_list(-1..1) do + assert_raise FunctionClauseError, fn -> + Integer.undigits([1, 0, 1], n) + end + end + + assert_raise ArgumentError, "invalid digit 17 in base 16", fn -> + Integer.undigits([1, 2, 17], 16) + end + end + + test "parse/2" do assert Integer.parse("12") === {12, ""} + assert Integer.parse("012") === {12, ""} + assert Integer.parse("+12") === {12, ""} assert Integer.parse("-12") === {-12, ""} - assert Integer.parse("123456789") === {123456789, ""} + assert Integer.parse("123456789") === {123_456_789, ""} assert Integer.parse("12.5") === {12, ".5"} assert Integer.parse("7.5e-3") === {7, ".5e-3"} assert Integer.parse("12x") === {12, "x"} @@ -35,5 +132,128 @@ defmodule IntegerTest do assert Integer.parse("--1") === :error assert Integer.parse("+-1") === :error assert Integer.parse("three") === :error + + assert Integer.parse("12", 10) === {12, ""} + assert Integer.parse("-12", 12) === {-14, ""} + assert Integer.parse("12345678", 9) === {6_053_444, ""} + assert Integer.parse("3.14", 4) === {3, ".14"} + assert Integer.parse("64eb", 16) === {25835, ""} + assert Integer.parse("64eb", 10) === {64, "eb"} + assert Integer.parse("10", 2) === {2, ""} + assert Integer.parse("++4", 10) === :error + + # Base should be in range 2..36 + assert_raise ArgumentError, "invalid base 1", fn -> Integer.parse("2", 1) end + assert_raise ArgumentError, "invalid base 37", fn -> Integer.parse("2", 37) end + + # Base should be an integer + assert_raise ArgumentError, "invalid base 10.2", fn -> Integer.parse("2", 10.2) end + + assert_raise ArgumentError, "invalid base nil", fn -> Integer.parse("2", nil) end + end + + test "to_string/2" do + assert Integer.to_string(42) == "42" + assert Integer.to_string(+42) == "42" + assert Integer.to_string(-42) == "-42" + assert Integer.to_string(-0001) == "-1" + + for n <- [42.0, :forty_two, '42', "42"] do + assert_raise ArgumentError, fn -> + Integer.to_string(n) + end + end + + assert Integer.to_string(42, 2) == "101010" + assert Integer.to_string(42, 10) == "42" + assert Integer.to_string(42, 16) == "2A" + assert Integer.to_string(+42, 16) == "2A" + assert Integer.to_string(-42, 16) == "-2A" + assert Integer.to_string(-042, 16) == "-2A" + + for n <- [42.0, :forty_two, '42', "42"] do + assert_raise ArgumentError, fn -> + Integer.to_string(n, 42) + end + end + + for n <- [-1, 0, 1, 37] do + assert_raise ArgumentError, fn -> + Integer.to_string(42, n) + end + + assert_raise ArgumentError, fn -> + Integer.to_string(n, n) + end + end + end + + test "to_charlist/2" do + module = Integer + + assert Integer.to_charlist(42) == '42' + assert Integer.to_charlist(+42) == '42' + assert Integer.to_charlist(-42) == '-42' + assert Integer.to_charlist(-0001) == '-1' + + for n <- [42.0, :forty_two, '42', "42"] do + assert_raise ArgumentError, fn -> + Integer.to_charlist(n) + end + end + + assert module.to_char_list(42) == '42' + assert module.to_char_list(42, 2) == '101010' + + assert Integer.to_charlist(42, 2) == '101010' + assert Integer.to_charlist(42, 10) == '42' + assert Integer.to_charlist(42, 16) == '2A' + assert Integer.to_charlist(+42, 16) == '2A' + assert Integer.to_charlist(-42, 16) == '-2A' + assert Integer.to_charlist(-042, 16) == '-2A' + + for n <- [42.0, :forty_two, '42', "42"] do + assert_raise ArgumentError, fn -> + Integer.to_charlist(n, 42) + end + end + + for n <- [-1, 0, 1, 37] do + assert_raise ArgumentError, fn -> + Integer.to_charlist(42, n) + end + + assert_raise ArgumentError, fn -> + Integer.to_charlist(n, n) + end + end + end + + test "gcd/2" do + assert Integer.gcd(1, 5) == 1 + assert Integer.gcd(2, 3) == 1 + assert Integer.gcd(8, 12) == 4 + assert Integer.gcd(-8, 12) == 4 + assert Integer.gcd(8, -12) == 4 + assert Integer.gcd(-8, -12) == 4 + assert Integer.gcd(27, 27) == 27 + assert Integer.gcd(-27, -27) == 27 + assert Integer.gcd(-27, 27) == 27 + assert Integer.gcd(0, 3) == 3 + assert Integer.gcd(0, -3) == 3 + assert Integer.gcd(3, 0) == 3 + assert Integer.gcd(-3, 0) == 3 + assert Integer.gcd(0, 0) == 0 + end + + test "extended_gcd" do + # Poor's man properby based testing + for _ <- 1..100 do + left = :rand.uniform(1000) + right = :rand.uniform(1000) + {gcd, m, n} = Integer.extended_gcd(left, right) + assert Integer.gcd(left, right) == gcd + assert m * left + n * right == gcd + end end end diff --git a/lib/elixir/test/elixir/io/ansi/docs_test.exs b/lib/elixir/test/elixir/io/ansi/docs_test.exs index 66173bc3b5c..167d0964ad7 100644 --- a/lib/elixir/test/elixir/io/ansi/docs_test.exs +++ b/lib/elixir/test/elixir/io/ansi/docs_test.exs @@ -1,219 +1,808 @@ -Code.require_file "../../test_helper.exs", __DIR__ +Code.require_file("../../test_helper.exs", __DIR__) defmodule IO.ANSI.DocsTest do use ExUnit.Case, async: true import ExUnit.CaptureIO - def format_heading(str) do - capture_io(fn -> IO.ANSI.Docs.print_heading(str, []) end) |> String.strip + def format_headings(list) do + capture_io(fn -> IO.ANSI.Docs.print_headings(list, []) end) |> String.trim_trailing() end - def format(str) do - capture_io(fn -> IO.ANSI.Docs.print(str, []) end) |> String.strip + def format_metadata(map) do + capture_io(fn -> IO.ANSI.Docs.print_metadata(map, []) end) end - test "heading is formatted" do - result = format_heading("wibble") - assert String.starts_with?(result, "\e[0m\n\e[7m\e[33m\e[1m") - assert String.ends_with?(result, "\e[0m\n\e[0m") - assert String.contains?(result, " wibble ") - end + def format_markdown(str, opts \\ []) do + capture_io(fn -> IO.ANSI.Docs.print(str, "text/markdown", opts) end) + |> String.trim_trailing() + end + + def format_erlang(str, opts \\ []) do + capture_io(fn -> IO.ANSI.Docs.print(str, "application/erlang+html", opts) end) + end + + describe "heading" do + test "is formatted" do + result = format_headings(["foo"]) + assert String.starts_with?(result, "\e[0m\n\e[7m\e[33m") + assert String.ends_with?(result, "\e[0m\n\e[0m") + assert String.contains?(result, " foo ") + end + + test "multiple entries formatted" do + result = format_headings(["foo", "bar"]) + assert :binary.matches(result, "\e[0m\n\e[7m\e[33m") |> length == 2 + assert String.starts_with?(result, "\e[0m\n\e[7m\e[33m") + assert String.ends_with?(result, "\e[0m\n\e[0m") + assert String.contains?(result, " foo ") + assert String.contains?(result, " bar ") + end + end + + describe "metadata" do + test "is formatted" do + result = + format_metadata(%{ + since: "1.2.3", + deprecated: "Use that other one", + author: "Alice", + delegate_to: {Foo, :bar, 3} + }) + + assert result == """ + \e[33mdelegate_to:\e[0m Foo.bar/3 + \e[33mdeprecated:\e[0m Use that other one + \e[33msince:\e[0m 1.2.3 + + """ + + assert format_metadata(%{author: "Alice"}) == "" + end + end + + describe "markdown" do + test "first level heading is converted" do + result = format_markdown("# wibble\n\ntext\n") + assert result == "\e[33m# wibble\e[0m\n\e[0m\ntext\n\e[0m" + end + + test "second level heading is converted" do + result = format_markdown("## wibble\n\ntext\n") + assert result == "\e[33m## wibble\e[0m\n\e[0m\ntext\n\e[0m" + end + + test "third level heading is converted" do + result = format_markdown("### wibble\n\ntext\n") + assert result == "\e[33m### wibble\e[0m\n\e[0m\ntext\n\e[0m" + end + + test "short single-line quote block is converted into single-line quote" do + result = + format_markdown(""" + line + + > normal *italics* `code` + + line2 + """) + + assert result == + """ + line + \e[0m + \e[90m> \e[0mnormal \e[4mitalics\e[0m \e[36mcode\e[0m + \e[0m + line2 + \e[0m\ + """ + end + + test "short multi-line quote block is converted into single-line quote" do + result = + format_markdown(""" + line + + > normal + > *italics* + > `code` + + line2 + """) + + assert result == + """ + line + \e[0m + \e[90m> \e[0mnormal \e[4mitalics\e[0m \e[36mcode\e[0m + \e[0m + line2 + \e[0m\ + """ + end + + test "long multi-line quote block is converted into wrapped multi-line quote" do + result = + format_markdown(""" + line + + > normal + > *italics* + > `code` + > some-extremely-long-word-which-can-not-possibly-fit-into-the-previous-line + + line2 + """) + + assert result == + """ + line + \e[0m + \e[90m> \e[0mnormal \e[4mitalics\e[0m \e[36mcode\e[0m + \e[90m> \e[0msome-extremely-long-word-which-can-not-possibly-fit-into-the-previous-line + \e[0m + line2 + \e[0m\ + """ + end + + test "multi-line quote block containing empty lines is converted into wrapped multi-line quote" do + result = + format_markdown(""" + line + + > normal + > *italics* + > + > `code` + > some-extremely-long-word-which-can-not-possibly-fit-into-the-previous-line + + line2 + """) + + assert result == + """ + line + \e[0m + \e[90m> \e[0mnormal \e[4mitalics\e[0m + \e[90m> \e[0m + \e[90m> \e[0m\e[36mcode\e[0m + \e[90m> \e[0msome-extremely-long-word-which-can-not-possibly-fit-into-the-previous-line + \e[0m + line2 + \e[0m\ + """ + end + + test "code block is converted" do + result = format_markdown("line\n\n code\n code2\n\nline2\n") + assert result == "line\n\e[0m\n\e[36m code\n code2\e[0m\n\e[0m\nline2\n\e[0m" + end + + test "fenced code block is converted" do + result = format_markdown("line\n```\ncode\ncode2\n```\nline2\n") + assert result == "line\n\e[0m\n\e[36m code\n code2\e[0m\n\e[0m\nline2\n\e[0m" + result = format_markdown("line\n```elixir\ncode\ncode2\n```\nline2\n") + assert result == "line\n\e[0m\n\e[36m code\n code2\e[0m\n\e[0m\nline2\n\e[0m" + result = format_markdown("line\n~~~elixir\ncode\n```\n~~~\nline2\n") + assert result == "line\n\e[0m\n\e[36m code\n ```\e[0m\n\e[0m\nline2\n\e[0m" + end + + test "* list is converted" do + result = format_markdown("* one\n* two\n* three\n") + assert result == " • one\n • two\n • three\n\e[0m" + end + + test "* list is converted without ansi" do + result = format_markdown("* one\n* two\n* three\n", enabled: false) + assert result == " * one\n * two\n * three" + end + + test "* list surrounded by text is converted" do + result = format_markdown("Count:\n\n* one\n* two\n* three\n\nDone") + assert result == "Count:\n\e[0m\n • one\n • two\n • three\n\e[0m\nDone\n\e[0m" + end + + test "* list with continuation is converted" do + result = format_markdown("* one\ntwo\n\n three\nfour\n* five") + assert result == " • one two\n three four\n\e[0m\n • five\n\e[0m" + end + + test "* nested lists are converted" do + result = format_markdown("* one\n * one.one\n * one.two\n* two") + assert result == " • one\n • one.one\n • one.two\n\e[0m\n • two\n\e[0m" + end + + test "* deep nested lists are converted" do + result = + format_markdown(""" + * level 1 + * level 2a + * level 2b + * level 3 + * level 4a + * level 4b + * level 5 + * level 6 + """) + + assert result == + " • level 1\n • level 2a\n • level 2b\n • level 3\n • level 4a\n • level 4b\n • level 5\n • level 6\n\e[0m\n\e[0m\n\e[0m\n\e[0m\n\e[0m\n\e[0m" + end + + test "* lists with spaces are converted" do + result = format_markdown(" * one\n * two\n * three") + assert result == " • one\n • two\n • three\n\e[0m" + end + + test "* lists with code" do + result = format_markdown(" * one\n two three") + assert result == " • one\n\e[36m two three\e[0m\n\e[0m\n\e[0m" + end + + test "- list is converted" do + result = format_markdown("- one\n- two\n- three\n") + assert result == " • one\n • two\n • three\n\e[0m" + end + + test "+ list is converted" do + result = format_markdown("+ one\n+ two\n+ three\n") + assert result == " • one\n • two\n • three\n\e[0m" + end + + test "+ and - nested lists are converted" do + result = format_markdown("- one\n + one.one\n + one.two\n- two") + assert result == " • one\n • one.one\n • one.two\n\e[0m\n • two\n\e[0m" + end + + test "paragraphs are split" do + result = format_markdown("para1\n\npara2") + assert result == "para1\n\e[0m\npara2\n\e[0m" + end + + test "extra whitespace is ignored between paras" do + result = format_markdown("para1\n \npara2") + assert result == "para1\n\e[0m\npara2\n\e[0m" + end + + test "extra whitespace doesn't mess up a following list" do + result = format_markdown("para1\n \n* one\n* two") + assert result == "para1\n\e[0m\n • one\n • two\n\e[0m" + end + + test "star/underscore/backtick works" do + result = format_markdown("*world*") + assert result == "\e[4mworld\e[0m\n\e[0m" + + result = format_markdown("*world*.") + assert result == "\e[4mworld\e[0m.\n\e[0m" + + result = format_markdown("**world**") + assert result == "\e[1mworld\e[0m\n\e[0m" + + result = format_markdown("_world_") + assert result == "\e[4mworld\e[0m\n\e[0m" + + result = format_markdown("__world__") + assert result == "\e[1mworld\e[0m\n\e[0m" + + result = format_markdown("`world`") + assert result == "\e[36mworld\e[0m\n\e[0m" + end + + test "star/underscore/backtick works across words" do + result = format_markdown("*hello world*") + assert result == "\e[4mhello world\e[0m\n\e[0m" + + result = format_markdown("**hello world**") + assert result == "\e[1mhello world\e[0m\n\e[0m" + + result = format_markdown("_hello world_") + assert result == "\e[4mhello world\e[0m\n\e[0m" + + result = format_markdown("__hello world__") + assert result == "\e[1mhello world\e[0m\n\e[0m" + + result = format_markdown("`hello world`") + assert result == "\e[36mhello world\e[0m\n\e[0m" + end + + test "star/underscore/backtick works across words with ansi disabled" do + result = format_markdown("*hello world*", enabled: false) + assert result == "*hello world*" + + result = format_markdown("**hello world**", enabled: false) + assert result == "**hello world**" + + result = format_markdown("_hello world_", enabled: false) + assert result == "_hello world_" + + result = format_markdown("__hello world__", enabled: false) + assert result == "__hello world__" - test "first level heading is converted" do - result = format("# wibble\n\ntext\n") - assert result == "\e[33m\e[1mWIBBLE\e[0m\n\e[0m\ntext\n\e[0m" - end + result = format_markdown("`hello world`", enabled: false) + assert result == "`hello world`" + end - test "second level heading is converted" do - result = format("## wibble\n\ntext\n") - assert result == "\e[33m\e[1mwibble\e[0m\n\e[0m\ntext\n\e[0m" - end + test "multiple stars/underscores/backticks work" do + result = format_markdown("*hello world* *hello world*") + assert result == "\e[4mhello world\e[0m \e[4mhello world\e[0m\n\e[0m" - test "third level heading is converted" do - result = format("## wibble\n\ntext\n") - assert result == "\e[33m\e[1mwibble\e[0m\n\e[0m\ntext\n\e[0m" - end + result = format_markdown("_hello world_ _hello world_") + assert result == "\e[4mhello world\e[0m \e[4mhello world\e[0m\n\e[0m" - test "code block is converted" do - result = format("line\n\n code\n code2\n\nline2\n") - assert result == "line\n\e[0m\n\e[36m\e[1m┃ code\n┃ code2\e[0m\n\e[0m\nline2\n\e[0m" - end + result = format_markdown("`hello world` `hello world`") + assert result == "\e[36mhello world\e[0m \e[36mhello world\e[0m\n\e[0m" + end - test "* list is converted" do - result = format("* one\n* two\n* three\n") - assert result == "• one\n• two\n• three\n\e[0m" - end + test "multiple stars/underscores/backticks work when separated by other words" do + result = format_markdown("*hello world* unit test *hello world*") + assert result == "\e[4mhello world\e[0m unit test \e[4mhello world\e[0m\n\e[0m" - test "* list surrounded by text is converted" do - result = format("Count:\n\n* one\n* two\n* three\n\nDone") - assert result == "Count:\n\e[0m\n• one\n• two\n• three\n\e[0m\nDone\n\e[0m" - end + result = format_markdown("_hello world_ unit test _hello world_") + assert result == "\e[4mhello world\e[0m unit test \e[4mhello world\e[0m\n\e[0m" - test "* list with continuation is converted" do - result = format("* one\n two\n three\n* four") - assert result == "• one two three\n• four" - end + result = format_markdown("`hello world` unit test `hello world`") + assert result == "\e[36mhello world\e[0m unit test \e[36mhello world\e[0m\n\e[0m" + end - test "* nested lists are converted" do - result = format("* one\n * one.one\n * one.two\n* two") - assert result == "• one\n • one.one\n • one.two\n• two" - end + test "star/underscore preceded by space doesn't get interpreted" do + result = format_markdown("_unit _size") + assert result == "_unit _size\n\e[0m" - test "* lists with spaces are converted" do - result = format(" * one\n * two\n * three") - assert result == "• one\n• two\n• three" - end + result = format_markdown("**unit **size") + assert result == "**unit **size\n\e[0m" - test "- list is converted" do - result = format("- one\n- two\n- three\n") - assert result == "• one\n• two\n• three\n\e[0m" - end + result = format_markdown("*unit *size") + assert result == "*unit *size\n\e[0m" + end - test "- list surrounded by text is converted" do - result = format("Count:\n\n- one\n- two\n- three\n\nDone") - assert result == "Count:\n\e[0m\n• one\n• two\n• three\n\e[0m\nDone\n\e[0m" - end + test "star/underscore/backtick preceded by non-space delimiters gets interpreted" do + result = format_markdown("(`hello world`)") + assert result == "(\e[36mhello world\e[0m)\n\e[0m" + result = format_markdown("<`hello world`>") + assert result == "<\e[36mhello world\e[0m>\n\e[0m" - test "- list with continuation is converted" do - result = format("- one\n two\n three\n- four") - assert result == "• one two three\n• four" - end + result = format_markdown("(*hello world*)") + assert result == "(\e[4mhello world\e[0m)\n\e[0m" + result = format_markdown("@*hello world*@") + assert result == "@\e[4mhello world\e[0m@\n\e[0m" - test "+ list is converted" do - result = format("+ one\n+ two\n+ three\n") - assert result == "• one\n• two\n• three\n\e[0m" - end + result = format_markdown("(_hello world_)") + assert result == "(\e[4mhello world\e[0m)\n\e[0m" + result = format_markdown("'_hello world_'") + assert result == "'\e[4mhello world\e[0m'\n\e[0m" + end - test "+ and - nested lists are converted" do - result = format("- one\n + one.one\n + one.two\n- two") - assert result == "• one\n • one.one\n • one.two\n• two" - end + test "star/underscore/backtick starts/ends within a word doesn't get interpreted" do + result = format_markdown("foo_bar, foo_bar_baz!") + assert result == "foo_bar, foo_bar_baz!\n\e[0m" - test "paragraphs are split" do - result = format("para1\n\npara2") - assert result == "para1\n\e[0m\npara2\n\e[0m" - end + result = format_markdown("_foo_bar") + assert result == "_foo_bar\n\e[0m" - test "extra whitespace is ignored between paras" do - result = format("para1\n \npara2") - assert result == "para1\n\e[0m\npara2\n\e[0m" - end + result = format_markdown("foo_bar_") + assert result == "foo_bar_\n\e[0m" - test "extra whitespace doesn't mess up a following list" do - result = format("para1\n \n* one\n* two") - assert result == "para1\n\e[0m\n• one\n• two" - end + result = format_markdown("foo*bar, foo*bar*baz!") + assert result == "foo*bar, foo*bar*baz!\n\e[0m" - test "star/underscore/backtick works" do - result = format("*world*") - assert result == "\e[1mworld\e[0m\n\e[0m" + result = format_markdown("*foo*bar") + assert result == "*foo*bar\n\e[0m" - result = format("**world**") - assert result == "\e[1mworld\e[0m\n\e[0m" + result = format_markdown("foo*bar*") + assert result == "foo*bar*\n\e[0m" + end - result = format("_world_") - assert result == "\e[4mworld\e[0m\n\e[0m" + test "backtick preceded by space gets interpreted" do + result = format_markdown("`unit `size") + assert result == "\e[36munit \e[0msize\n\e[0m" + end - result = format("`world`") - assert result == "\e[36mworld\e[0m\n\e[0m" - end + test "backtick does not escape characters" do + result = format_markdown("`Ctrl+\\ `") + assert result == "\e[36mCtrl+\\ \e[0m\n\e[0m" + end - test "star/underscore/backtick works accross words" do - result = format("*hello world*") - assert result == "\e[1mhello world\e[0m\n\e[0m" + test "star/underscore/backtick with leading escape" do + result = format_markdown("\\_unit_") + assert result == "_unit_\n\e[0m" - result = format("**hello world**") - assert result == "\e[1mhello world\e[0m\n\e[0m" + result = format_markdown("\\*unit*") + assert result == "*unit*\n\e[0m" - result = format("_hello world_") - assert result == "\e[4mhello world\e[0m\n\e[0m" + result = format_markdown("\\`unit`") + assert result == "`unit`\n\e[0m" + end - result = format("`hello world`") - assert result == "\e[36mhello world\e[0m\n\e[0m" - end + test "star/underscore/backtick with closing escape" do + result = format_markdown("_unit\\_") + assert result == "_unit_\n\e[0m" - test "star/underscore preceeded by space doesn't get interpreted" do - result = format("_unit _size") - assert result == "_unit _size\n\e[0m" + result = format_markdown("*unit\\*") + assert result == "*unit*\n\e[0m" - result = format("**unit **size") - assert result == "**unit **size\n\e[0m" + result = format_markdown("`unit\\`") + assert result == "\e[36munit\\\e[0m\n\e[0m" + end - result = format("*unit *size") - assert result == "*unit *size\n\e[0m" - end + test "star/underscore/backtick with double escape" do + result = format_markdown("\\\\*world*") + assert result == "\\\e[4mworld\e[0m\n\e[0m" + + result = format_markdown("\\\\_world_") + assert result == "\\\e[4mworld\e[0m\n\e[0m" + + result = format_markdown("\\\\`world`") + assert result == "\\\e[36mworld\e[0m\n\e[0m" + end + + test "star/underscore/backtick when incomplete" do + result = format_markdown("unit_size") + assert result == "unit_size\n\e[0m" + + result = format_markdown("unit`size") + assert result == "unit`size\n\e[0m" + + result = format_markdown("unit*size") + assert result == "unit*size\n\e[0m" + + result = format_markdown("unit**size") + assert result == "unit**size\n\e[0m" + end + + test "backtick with escape" do + result = format_markdown("`\\`") + assert result == "\e[36m\\\e[0m\n\e[0m" + end + + test "backtick close to underscores gets interpreted as code" do + result = format_markdown("`__world__`") + assert result == "\e[36m__world__\e[0m\n\e[0m" + end + + test "escaping of underlines within links" do + result = format_markdown("(https://en.wikipedia.org/wiki/ANSI_escape_code)") + assert result == "(https://en.wikipedia.org/wiki/ANSI_escape_code)\n\e[0m" + + result = + format_markdown("[ANSI escape code](https://en.wikipedia.org/wiki/ANSI_escape_code)") + + assert result == "ANSI escape code (https://en.wikipedia.org/wiki/ANSI_escape_code)\n\e[0m" + + result = format_markdown("(ftp://example.com/ANSI_escape_code.zip)") + assert result == "(ftp://example.com/ANSI_escape_code.zip)\n\e[0m" + end + + test "escaping of underlines within links does not escape surrounding text" do + result = + format_markdown( + "_emphasis_ (https://en.wikipedia.org/wiki/ANSI_escape_code) more _emphasis_" + ) + + assert result == + "\e[4memphasis\e[0m (https://en.wikipedia.org/wiki/ANSI_escape_code) more \e[4memphasis\e[0m\n\e[0m" + end + + test "escaping of underlines within links avoids false positives" do + assert format_markdown("`https_proxy`") == "\e[36mhttps_proxy\e[0m\n\e[0m" + end + + test "escaping of several Markdown links in one line" do + assert format_markdown("[List](`List`) (`[1, 2, 3]`), [Map](`Map`)") == + "List (\e[36mList\e[0m) (\e[36m[1, 2, 3]\e[0m), Map (\e[36mMap\e[0m)\n\e[0m" + end + + test "one reference link label per line" do + assert format_markdown(" [id]: //example.com\n [Elixir]: https://elixir-lang.org") == + " [id]: //example.com\n [Elixir]: https://elixir-lang.org" + end + end + + describe "markdown tables" do + test "lone thing that looks like a table line isn't" do + assert format_markdown("one\n2 | 3\ntwo\n") == "one 2 | 3 two\n\e[0m" + end + + test "lone table line at end of input isn't" do + assert format_markdown("one\n2 | 3") == "one 2 | 3\n\e[0m" + end + + test "two successive table lines are a table" do + # note spacing + assert format_markdown("a | b\none | two\n") == "a | b \none | two\n\e[0m" + end + + test "table with heading" do + assert format_markdown("column 1 | and 2\n-- | --\na | b\none | two\n") == + "\e[7mcolumn 1 | and 2\e[0m\na | b \none | two \n\e[0m" + end + + test "table with heading alignment" do + table = """ + column 1 | 2 | and three + -------: | :------: | :----- + a | even | c\none | odd | three + """ + + expected = + """ + \e[7mcolumn 1 | 2 | and three\e[0m + a | even | c\s\s\s\s\s\s\s\s + one | odd | three\s\s\s\s + \e[0m + """ + |> String.trim_trailing() + + assert format_markdown(table) == expected + end + + test "table with heading alignment and no space around \"|\"" do + table = """ + | Value | Encoding | Value | Encoding | + |------:|:---------|------:|:---------| + | 0 | A | 17 | R | + | 1 | B | 18 | S | + """ + + expected = + "\e[7m" <> + "Value | Encoding | Value | Encoding\e[0m\n" <> + " 0 | A | 17 | R \n" <> + " 1 | B | 18 | S \n\e[0m" + + assert format_markdown(table) == expected + end + + test "table with formatting in cells" do + assert format_markdown("`a` | _b_\nc | d") == "\e[36ma\e[0m | \e[4mb\e[0m\nc | d\n\e[0m" + + assert format_markdown("`abc` | d \n`e` | f") == + "\e[36mabc\e[0m | d\n\e[36me\e[0m | f\n\e[0m" + end + + test "table with variable number of columns" do + assert format_markdown("a | b | c\nd | e") == "a | b | c\nd | e | \n\e[0m" + end + + test "table with escaped \"|\" inside cell" do + table = "a | smth\\|smth_else | c\nd | e | f" + + expected = + """ + a | smth|smth_else | c + d | e | f + \e[0m + """ + |> String.trim_trailing() + + assert format_markdown(table) == expected + end + + test "table with last two columns empty" do + table = """ + AAA | | | + BBB | CCC | | + GGG | HHH | III | + JJJ | KKK | LLL | MMM + """ + + expected = + """ + AAA | | |\s\s\s\s + BBB | CCC | |\s\s\s\s + GGG | HHH | III |\s\s\s\s + JJJ | KKK | LLL | MMM + \e[0m + """ + |> String.trim_trailing() + + assert format_markdown(table) == expected + end + end + + describe "erlang" do + @hello_world [{:p, [], ["Hello"]}, {:p, [], ["World"]}] + + test "text" do + assert format_erlang("Hello world") == "Hello world" + end + + test "skips line breaks" do + assert format_erlang([{:p, [], ["Hello"]}, {:br, [], []}, {:p, [], ["World"]}]) == + "Hello\n\nWorld\n\n" + end + + test "paragraphs" do + assert format_erlang(@hello_world) == "Hello\n\nWorld\n\n" + end + + test "code chunks" do + code = """ + def foo do + :bar + end\ + """ + + assert format_erlang({:pre, [], [{:code, [], [code]}]}) == """ + def foo do + :bar + end + + """ + end + + test "unordered lists" do + assert format_erlang([{:ul, [], [{:li, [], ["Hello"]}, {:li, [], ["World"]}]}]) == + " • Hello\n\n • World\n\n" + + assert format_erlang([{:ul, [], [{:li, [], [@hello_world]}]}]) == + " • Hello\n\n World\n\n" + + assert format_erlang([ + {:ul, [], [{:li, [], [{:p, [], ["Hello"]}]}, {:li, [], [{:p, [], ["World"]}]}]} + ]) == + " • Hello\n\n • World\n\n" + end + + test "ordered lists" do + assert format_erlang([{:ol, [], [{:li, [], ["Hello"]}, {:li, [], ["World"]}]}]) == + " 1. Hello\n\n 2. World\n\n" + + assert format_erlang([ + {:ol, [], [{:li, [], [{:p, [], ["Hello"]}]}, {:li, [], [{:p, [], ["World"]}]}]} + ]) == + " 1. Hello\n\n 2. World\n\n" + end + + test "admonition blocks" do + assert format_erlang([{:div, [class: "warning"], @hello_world}]) == """ + \e[90m> \e[0mWARNING + \e[90m> \e[0m + \e[90m> \e[0mHello + \e[90m> \e[0m + \e[90m> \e[0mWorld + + """ + end + + test "headers" do + assert format_erlang([{:h1, [], ["Hello"]}]) == + "\e[33m# Hello\e[0m\n\n" + + assert format_erlang([{:h2, [], ["Hello"]}]) == + "\e[33m## Hello\e[0m\n\n" + + assert format_erlang([{:h3, [], ["Hello"]}]) == + "\e[33m### Hello\e[0m\n\n" - test "backtick preceeded by space gets interpreted" do - result = format("`unit `size") - assert result == "\e[36munit \e[0msize\n\e[0m" - end + assert format_erlang([{:h4, [], ["Hello"]}]) == + "\e[33m#### Hello\e[0m\n\n" - test "star/underscore/backtick with leading escape" do - result = format("\\_unit_") - assert result == "_unit_\n\e[0m" + assert format_erlang([{:h5, [], ["Hello"]}]) == + "\e[33m##### Hello\e[0m\n\n" - result = format("\\*unit*") - assert result == "*unit*\n\e[0m" + assert format_erlang([{:h6, [], ["Hello"]}]) == + "\e[33m###### Hello\e[0m\n\n" - result = format("\\`unit`") - assert result == "`unit`\n\e[0m" - end + assert format_erlang([{:h1, [], [{:code, [], ["Hello"]}]}]) == + "\e[33m# \e[36mHello\e[0m\e[0m\n\n" - test "star/underscore/backtick with closing escape" do - result = format("_unit\\_") - assert result == "_unit_\n\e[0m" + assert format_erlang([{:h2, [], [{:code, [], ["Hello"]}]}]) == + "\e[33m## \e[36mHello\e[0m\e[0m\n\n" - result = format("*unit\\*") - assert result == "*unit*\n\e[0m" + assert format_erlang([{:h3, [], [{:code, [], ["Hello"]}]}]) == + "\e[33m### \e[36mHello\e[0m\e[0m\n\n" - result = format("`unit\\`") - assert result == "\e[36munit\\\e[0m\n\e[0m" - end + assert format_erlang([{:h4, [], [{:code, [], ["Hello"]}]}]) == + "\e[33m#### \e[36mHello\e[0m\e[0m\n\n" - test "star/underscore/backtick with double escape" do - result = format("\\\\*world*") - assert result == "\\\e[1mworld\e[0m\n\e[0m" + assert format_erlang([{:h5, [], [{:code, [], ["Hello"]}]}]) == + "\e[33m##### \e[36mHello\e[0m\e[0m\n\n" - result = format("\\\\_world_") - assert result == "\\\e[4mworld\e[0m\n\e[0m" + assert format_erlang([{:h6, [], [{:code, [], ["Hello"]}]}]) == + "\e[33m###### \e[36mHello\e[0m\e[0m\n\n" + end - result = format("\\\\`world`") - assert result == "\\\e[36mworld\e[0m\n\e[0m" - end + test "inline tags" do + assert format_erlang([{:i, [], ["Hello"]}]) == "\e[4mHello\e[0m" + assert format_erlang([{:i, [], ["Hello"]}], enabled: false) == "_Hello_" - test "star/underscore/backtick when incomplete" do - result = format("unit_size") - assert result == "unit_size\n\e[0m" + assert format_erlang([{:em, [], ["Hello"]}]) == "\e[4mHello\e[0m" + assert format_erlang([{:em, [], ["Hello"]}], enabled: false) == "*Hello*" - result = format("unit`size") - assert result == "unit`size\n\e[0m" + assert format_erlang([{:b, [], ["Hello"]}]) == "\e[1mHello\e[0m" + assert format_erlang([{:b, [], ["Hello"]}], enabled: false) == "**Hello**" + assert format_erlang([{:strong, [], ["Hello"]}]) == "\e[1mHello\e[0m" + assert format_erlang([{:strong, [], ["Hello"]}], enabled: false) == "**Hello**" - result = format("unit*size") - assert result == "unit*size\n\e[0m" + assert format_erlang([{:code, [], ["Hello"]}]) == "\e[36mHello\e[0m" + assert format_erlang([{:code, [], ["Hello"]}], enabled: false) == "`Hello`" + end - result = format("unit**size") - assert result == "unit**size\n\e[0m" - end + test "inline tags within paragraphs" do + assert format_erlang([{:p, [], [[{:em, [], ["Hello"]}, {:code, [], ["World"]}]]}]) == + "\e[4mHello\e[0m\e[36mWorld\e[0m" + end - test "backtick with escape" do - result = format("`\\`") - assert result == "\e[36m\\\e[0m\n\e[0m" - end + test "inline tags within list item" do + assert format_erlang([ + {:ul, [], [{:li, [], [{:em, [], ["Hello"]}, {:code, [], ["World"]}]}]} + ]) == + " • \e[4mHello\e[0m\e[36mWorld\e[0m\n\n" + end - test "backtick close to underscores gets interpreted as code" do - result = format("`__world__`") - assert result == "\e[36m__world__\e[0m\n\e[0m" - end + test "links" do + assert format_erlang([{:a, [], ["Hello"]}]) == "Hello" + assert format_erlang([{:a, [href: "foo/bar"], ["Hello"]}]) == "Hello (foo/bar)" + end + + test "definition lists" do + assert format_erlang([{:dl, [], [[{:dt, [], ["Hello"]}, {:dd, [], ["World"]}]]}]) == """ + • Hello + + World + + """ + end + + test "typespecs" do + assert format_erlang([{:ul, [class: "types"], [{:li, [], []}]}]) == "" + + assert format_erlang([{:ul, [class: "types"], [{:li, [], ["Hello"]}, {:li, [], ["World"]}]}]) == + """ + Typespecs: + + Hello + World + + """ + + assert format_erlang([ + {:ul, [class: "types"], [{:li, [], ["Hello", {:code, [], ["World"]}]}]} + ]) == + """ + Typespecs: + + Hello + \e[36mWorld\e[0m + + """ + end + + test "extra markup" do + assert format_erlang([{:p, [], ["Hello"]}, {:unknown, [], ["Unknown"]}, {:p, [], ["World"]}]) == + """ + Hello + + + Unknown + + + World + + """ + + assert format_erlang([ + {:p, [], ["Hello"]}, + {:unknown, [], [{:p, [], ["Unknown"]}]}, + {:p, [], ["World"]} + ]) == """ + Hello + + + Unknown + + + World - test "backtick works inside parenthesis" do - result = format("(`hello world`)") - assert result == "(\e[36mhello world\e[0m)\n\e[0m" + """ + end end - test "escaping of underlines within links" do - result = format("(http://en.wikipedia.org/wiki/ANSI_escape_code)") - assert result == "(http://en.wikipedia.org/wiki/ANSI_escape_code)\n\e[0m" - result = format("[ANSI escape code](http://en.wikipedia.org/wiki/ANSI_escape_code)") - assert result == "ANSI escape code (http://en.wikipedia.org/wiki/ANSI_escape_code)\n\e[0m" + describe "invalid format" do + test "prints message" do + assert capture_io(fn -> IO.ANSI.Docs.print("hello", "text/unknown", []) end) == + "\nUnknown documentation format \"text/unknown\"\n\n" + end end end diff --git a/lib/elixir/test/elixir/io/ansi_test.exs b/lib/elixir/test/elixir/io/ansi_test.exs index f05a0679985..81d5bfc88c1 100644 --- a/lib/elixir/test/elixir/io/ansi_test.exs +++ b/lib/elixir/test/elixir/io/ansi_test.exs @@ -1,54 +1,223 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule IO.ANSITest do use ExUnit.Case, async: true - test :escape_single do - assert IO.ANSI.escape("Hello, %{red}world!", true) == - "Hello, #{IO.ANSI.red}world!#{IO.ANSI.reset}" - assert IO.ANSI.escape("Hello, %{red}world!", true) == - "Hello, #{IO.ANSI.red}world!#{IO.ANSI.reset}" + doctest IO.ANSI + + test "format ansicode" do + assert IO.chardata_to_string(IO.ANSI.format(:green, true)) == + "#{IO.ANSI.green()}#{IO.ANSI.reset()}" + + assert IO.chardata_to_string(IO.ANSI.format(:green, false)) == "" + end + + test "format binary" do + assert IO.chardata_to_string(IO.ANSI.format("Hello, world!", true)) == "Hello, world!" + + assert IO.chardata_to_string(IO.ANSI.format("A map: %{foo: :bar}", false)) == + "A map: %{foo: :bar}" + end + + test "format empty list" do + assert IO.chardata_to_string(IO.ANSI.format([], true)) == "" + assert IO.chardata_to_string(IO.ANSI.format([], false)) == "" + end + + test "format ansicode list" do + assert IO.chardata_to_string(IO.ANSI.format([:red, :bright], true)) == + "#{IO.ANSI.red()}#{IO.ANSI.bright()}#{IO.ANSI.reset()}" + + assert IO.chardata_to_string(IO.ANSI.format([:red, :bright], false)) == "" + end + + test "format binary list" do + assert IO.chardata_to_string(IO.ANSI.format(["Hello, ", "world!"], true)) == "Hello, world!" + assert IO.chardata_to_string(IO.ANSI.format(["Hello, ", "world!"], false)) == "Hello, world!" + end + + test "format charlist" do + assert IO.chardata_to_string(IO.ANSI.format('Hello, world!', true)) == "Hello, world!" + assert IO.chardata_to_string(IO.ANSI.format('Hello, world!', false)) == "Hello, world!" + end + + test "format mixed list" do + data = ["Hello", ?,, 32, :red, "world!"] + + assert IO.chardata_to_string(IO.ANSI.format(data, true)) == + "Hello, #{IO.ANSI.red()}world!#{IO.ANSI.reset()}" + + assert IO.chardata_to_string(IO.ANSI.format(data, false)) == "Hello, world!" + end + + test "format nested list" do + data = ["Hello, ", ["nested", 32, :red, "world!"]] + + assert IO.chardata_to_string(IO.ANSI.format(data, true)) == + "Hello, nested #{IO.ANSI.red()}world!#{IO.ANSI.reset()}" + + assert IO.chardata_to_string(IO.ANSI.format(data, false)) == "Hello, nested world!" + end + + test "format improper list" do + data = ["Hello, ", :red, "world" | "!"] + + assert IO.chardata_to_string(IO.ANSI.format(data, true)) == + "Hello, #{IO.ANSI.red()}world!#{IO.ANSI.reset()}" + + assert IO.chardata_to_string(IO.ANSI.format(data, false)) == "Hello, world!" end - test :escape_non_attribute do - assert IO.ANSI.escape("Hello %{clear}world!", true) == - "Hello #{IO.ANSI.clear}world!#{IO.ANSI.reset}" - assert IO.ANSI.escape("Hello %{home}world!", true) == - "Hello #{IO.ANSI.home}world!#{IO.ANSI.reset}" + test "format nested improper list" do + data = [["Hello, " | :red], "world!" | :green] + + assert IO.chardata_to_string(IO.ANSI.format(data, true)) == + "Hello, #{IO.ANSI.red()}world!#{IO.ANSI.green()}#{IO.ANSI.reset()}" + + assert IO.chardata_to_string(IO.ANSI.format(data, false)) == "Hello, world!" end - test :escape_multiple do - assert IO.ANSI.escape("Hello, %{red,bright}world!", true) == - "Hello, #{IO.ANSI.red}#{IO.ANSI.bright}world!#{IO.ANSI.reset}" - assert IO.ANSI.escape("Hello, %{red, bright}world!", true) == - "Hello, #{IO.ANSI.red}#{IO.ANSI.bright}world!#{IO.ANSI.reset}" - assert IO.ANSI.escape("Hello, %{red , bright}world!", true) == - "Hello, #{IO.ANSI.red}#{IO.ANSI.bright}world!#{IO.ANSI.reset}" + test "format fragment" do + assert IO.chardata_to_string(IO.ANSI.format_fragment([:red, "Hello!"], true)) == + "#{IO.ANSI.red()}Hello!" end - test :no_emit do - assert IO.ANSI.escape("Hello, %{}world!", false) == - "Hello, world!" + test "format invalid sequence" do + assert_raise ArgumentError, "invalid ANSI sequence specification: :brigh", fn -> + IO.ANSI.format([:brigh, "Hello!"], true) + end - assert IO.ANSI.escape("Hello, %{red,bright}world!", false) == - "Hello, world!" + assert_raise ArgumentError, "invalid ANSI sequence specification: nil", fn -> + IO.ANSI.format(["Hello!", nil], true) + end end - test :fragment do - assert IO.ANSI.escape("%{red}", true) == "#{IO.ANSI.red}#{IO.ANSI.reset}" - assert IO.ANSI.escape_fragment("", true) == "" + test "colors" do + assert IO.ANSI.red() == "\e[31m" + assert IO.ANSI.light_red() == "\e[91m" + + assert IO.ANSI.red_background() == "\e[41m" + assert IO.ANSI.light_red_background() == "\e[101m" end - test :noop do - assert IO.ANSI.escape("") == "" + test "color/1" do + assert IO.ANSI.color(0) == "\e[38;5;0m" + assert IO.ANSI.color(42) == "\e[38;5;42m" + assert IO.ANSI.color(255) == "\e[38;5;255m" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color(-1) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color(256) + end end - test :invalid do - assert_raise ArgumentError, "invalid ANSI sequence specification: brigh", fn -> - IO.ANSI.escape("%{brigh}, yes") + test "color/3" do + assert IO.ANSI.color(0, 4, 2) == "\e[38;5;42m" + assert IO.ANSI.color(1, 1, 1) == "\e[38;5;59m" + assert IO.ANSI.color(5, 5, 5) == "\e[38;5;231m" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color(0, 6, 1) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color(5, -1, 1) end - assert_raise ArgumentError, "invalid ANSI sequence specification: brigh", fn -> - IO.ANSI.escape("%{brigh,red}, yes") + end + + test "color_background/1" do + assert IO.ANSI.color_background(0) == "\e[48;5;0m" + assert IO.ANSI.color_background(42) == "\e[48;5;42m" + assert IO.ANSI.color_background(255) == "\e[48;5;255m" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color_background(-1) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color_background(256) + end + end + + test "color_background/3" do + assert IO.ANSI.color_background(0, 4, 2) == "\e[48;5;42m" + assert IO.ANSI.color_background(1, 1, 1) == "\e[48;5;59m" + assert IO.ANSI.color_background(5, 5, 5) == "\e[48;5;231m" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color_background(0, 6, 1) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.color_background(5, -1, 1) + end + end + + test "cursor/2" do + assert IO.ANSI.cursor(0, 0) == "\e[0;0H" + assert IO.ANSI.cursor(11, 12) == "\e[11;12H" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor(-1, 5) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor(5, -1) + end + end + + test "cursor_up/1" do + assert IO.ANSI.cursor_up() == "\e[1A" + assert IO.ANSI.cursor_up(12) == "\e[12A" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_up(0) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_up(-1) + end + end + + test "cursor_down/1" do + assert IO.ANSI.cursor_down() == "\e[1B" + assert IO.ANSI.cursor_down(2) == "\e[2B" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_right(0) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_down(-1) + end + end + + test "cursor_left/1" do + assert IO.ANSI.cursor_left() == "\e[1D" + assert IO.ANSI.cursor_left(3) == "\e[3D" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_left(0) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_left(-1) + end + end + + test "cursor_right/1" do + assert IO.ANSI.cursor_right() == "\e[1C" + assert IO.ANSI.cursor_right(4) == "\e[4C" + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_right(0) + end + + assert_raise FunctionClauseError, fn -> + IO.ANSI.cursor_right(-1) end end end diff --git a/lib/elixir/test/elixir/io_test.exs b/lib/elixir/test/elixir/io_test.exs index 7d46dbbf563..2244ed7cb52 100644 --- a/lib/elixir/test/elixir/io_test.exs +++ b/lib/elixir/test/elixir/io_test.exs @@ -1,28 +1,53 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule IOTest do use ExUnit.Case, async: true + + doctest IO + import ExUnit.CaptureIO - test :read_with_count do - {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__), [:char_list]) + test "read with count" do + {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__), [:charlist]) assert 'FOO' == IO.read(file, 3) assert File.close(file) == :ok end - test :read_with_utf8_and_binary do + test "read with UTF-8 and binary" do {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__), [:utf8]) assert "Русский" == IO.read(file, 7) assert File.close(file) == :ok end - test :binread do + test "read all charlist" do + {:ok, file} = File.open(Path.expand('fixtures/multiline_file.txt', __DIR__), [:charlist]) + assert 'this is the first line\nthis is the second line\n' == IO.read(file, :all) + assert File.close(file) == :ok + end + + test "read empty file" do + {:ok, file} = File.open(Path.expand('fixtures/cp_mode', __DIR__), []) + assert IO.read(file, :all) == "" + assert File.close(file) == :ok + + {:ok, file} = File.open(Path.expand('fixtures/cp_mode', __DIR__), [:charlist]) + assert IO.read(file, :all) == '' + assert File.close(file) == :ok + end + + test "binread" do {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__)) assert "Русский" == IO.binread(file, 14) assert File.close(file) == :ok end - test :getn do + test "binread all" do + {:ok, file} = File.open(Path.expand('fixtures/file.bin', __DIR__)) + assert "LF\nCR\rCRLF\r\nLFCR\n\r" == IO.binread(file, :all) + assert File.close(file) == :ok + end + + test "getn" do {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__)) assert "F" == IO.getn(file, "") assert "O" == IO.getn(file, "") @@ -32,78 +57,241 @@ defmodule IOTest do assert File.close(file) == :ok end - test :getn_with_count do - {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__), [:char_list]) - assert 'FOO' == IO.getn(file, "", 3) + test "getn with count" do + {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__), [:charlist]) + assert 'F' == IO.getn(file, "λ") + assert 'OO' == IO.getn(file, "", 2) + assert '\n' == IO.getn(file, "λ", 99) + assert :eof == IO.getn(file, "λ", 1) + assert File.close(file) == :ok + end + + test "getn with eof" do + {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__), [:charlist]) + assert 'F' == IO.getn(file, "λ") + assert 'OO\n' == IO.getn(file, "", :eof) + assert :eof == IO.getn(file, "", :eof) assert File.close(file) == :ok end - test :getn_with_utf8_and_binary do + test "getn with UTF-8 and binary" do {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__), [:utf8]) assert "Русский" == IO.getn(file, "", 7) + assert "\n日\n" == IO.getn(file, "", :eof) + assert :eof == IO.getn(file, "", :eof) assert File.close(file) == :ok end - test :gets do - {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__), [:char_list]) + test "gets" do + {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__), [:charlist]) assert 'FOO\n' == IO.gets(file, "") assert :eof == IO.gets(file, "") assert File.close(file) == :ok end - test :gets_with_utf8_and_binary do + test "gets with UTF-8 and binary" do {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__), [:utf8]) assert "Русский\n" == IO.gets(file, "") assert "日\n" == IO.gets(file, "") assert File.close(file) == :ok end - test :readline do + test "read with all" do + {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__)) + assert "FOO\n" == IO.read(file, :all) + assert "" == IO.read(file, :all) + assert File.close(file) == :ok + end + + test "read with eof" do + {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__)) + assert "FOO\n" == IO.read(file, :eof) + assert :eof == IO.read(file, :eof) + assert File.close(file) == :ok + end + + test "read with eof and UTF-8 and binary" do + {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__), [:utf8]) + assert "Русский\n日\n" == IO.read(file, :eof) + assert :eof == IO.read(file, :eof) + assert File.close(file) == :ok + end + + test "readline" do {:ok, file} = File.open(Path.expand('fixtures/file.txt', __DIR__)) assert "FOO\n" == IO.read(file, :line) assert :eof == IO.read(file, :line) assert File.close(file) == :ok end - test :readline_with_utf8_and_binary do + test "readline with UTF-8 and binary" do {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__), [:utf8]) assert "Русский\n" == IO.read(file, :line) assert "日\n" == IO.read(file, :line) assert File.close(file) == :ok end - test :binreadline do + test "binread with all" do + {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__)) + assert "Русский\n日\n" == IO.binread(file, :all) + assert "" == IO.binread(file, :all) + assert File.close(file) == :ok + end + + test "binread with eof" do + {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__)) + assert "Русский\n日\n" == IO.binread(file, :eof) + assert :eof == IO.binread(file, :eof) + assert File.close(file) == :ok + end + + test "binread with line" do {:ok, file} = File.open(Path.expand('fixtures/utf8.txt', __DIR__)) assert "Русский\n" == IO.binread(file, :line) assert "日\n" == IO.binread(file, :line) assert File.close(file) == :ok end - test :puts_with_chardata do + test "puts with chardata" do assert capture_io(fn -> IO.puts("hello") end) == "hello\n" assert capture_io(fn -> IO.puts('hello') end) == "hello\n" assert capture_io(fn -> IO.puts(:hello) end) == "hello\n" assert capture_io(fn -> IO.puts(13) end) == "13\n" end - test :write_with_chardata do + describe "warn" do + test "with chardata" do + assert capture_io(:stderr, fn -> IO.warn("hello") end) =~ + "hello\n (ex_unit #{System.version()}) lib/ex_unit" + + assert capture_io(:stderr, fn -> IO.warn('hello') end) =~ + "hello\n (ex_unit #{System.version()}) lib/ex_unit" + + assert capture_io(:stderr, fn -> IO.warn(:hello) end) =~ + "hello\n (ex_unit #{System.version()}) lib/ex_unit" + + assert capture_io(:stderr, fn -> IO.warn(13) end) =~ + "13\n (ex_unit #{System.version()}) lib/ex_unit" + end + + test "no stacktrace" do + assert capture_io(:stderr, fn -> IO.warn("hello", []) end) =~ "hello\n\n" + end + + test "with stacktrace" do + stacktrace = [{IEx.Evaluator, :eval, 4, [file: 'lib/iex/evaluator.ex', line: 108]}] + + assert capture_io(:stderr, fn -> IO.warn("hello", stacktrace) end) =~ """ + hello + lib/iex/evaluator.ex:108: IEx.Evaluator.eval/4 + """ + end + + test "with env" do + assert capture_io(:stderr, fn -> IO.warn("hello", __ENV__) end) =~ ~r""" + hello + (lib/elixir/)?test/elixir/io_test.exs:#{__ENV__.line - 2}: IOTest.\"test warn with env\"/1 + """ + end + + test "with options" do + assert capture_io(:stderr, fn -> + IO.warn("hello", line: 13, file: "lib/foo.ex", module: Foo, function: {:bar, 1}) + end) =~ """ + hello + lib/foo.ex:13: Foo.bar/1 + """ + + assert capture_io(:stderr, fn -> + IO.warn("hello", file: "lib/foo.ex", module: Foo, function: {:bar, 1}) + end) =~ """ + hello + lib/foo.ex: Foo.bar/1 + """ + + assert capture_io(:stderr, fn -> IO.warn("hello", file: "lib/foo.ex", module: Foo) end) =~ + """ + hello + lib/foo.ex: Foo (module) + """ + + assert capture_io(:stderr, fn -> IO.warn("hello", file: "lib/foo.ex") end) =~ """ + hello + lib/foo.ex: (file) + """ + + assert capture_io(:stderr, fn -> + IO.warn("hello", file: "lib/foo.ex", function: {:bar, 1}) + end) =~ """ + hello + lib/foo.ex: (file) + """ + + assert capture_io(:stderr, fn -> + IO.warn("hello", line: 13, module: Foo, function: {:bar, 1}) + end) =~ """ + hello + + """ + end + end + + test "write with chardata" do assert capture_io(fn -> IO.write("hello") end) == "hello" assert capture_io(fn -> IO.write('hello') end) == "hello" assert capture_io(fn -> IO.write(:hello) end) == "hello" assert capture_io(fn -> IO.write(13) end) == "13" end - test :gets_with_chardata do + test "gets with chardata" do assert capture_io("foo\n", fn -> IO.gets("hello") end) == "hello" assert capture_io("foo\n", fn -> IO.gets('hello') end) == "hello" assert capture_io("foo\n", fn -> IO.gets(:hello) end) == "hello" assert capture_io("foo\n", fn -> IO.gets(13) end) == "13" end - test :getn_with_chardata do + test "getn with chardata" do assert capture_io("foo\n", fn -> IO.getn("hello", 3) end) == "hello" assert capture_io("foo\n", fn -> IO.getn('hello', 3) end) == "hello" assert capture_io("foo\n", fn -> IO.getn(:hello, 3) end) == "hello" assert capture_io("foo\n", fn -> IO.getn(13, 3) end) == "13" end + + test "getn with different arities" do + assert capture_io("hello", fn -> + input = IO.getn(">") + IO.write(input) + end) == ">h" + + assert capture_io("hello", fn -> + input = IO.getn(">", 3) + IO.write(input) + end) == ">hel" + + assert capture_io("hello", fn -> + input = IO.getn(Process.group_leader(), ">") + IO.write(input) + end) == ">h" + + assert capture_io("hello", fn -> + input = IO.getn(Process.group_leader(), ">") + IO.write(input) + end) == ">h" + + assert capture_io("hello", fn -> + input = IO.getn(Process.group_leader(), ">", 99) + IO.write(input) + end) == ">hello" + end + + test "inspect" do + assert capture_io(fn -> IO.inspect(1) end) == "1\n" + assert capture_io(fn -> IO.inspect(1, label: "foo") end) == "foo: 1\n" + assert capture_io(fn -> IO.inspect(1, label: :foo) end) == "foo: 1\n" + end + + test "stream" do + assert IO.stream() == IO.stream(:stdio, :line) + assert IO.binstream() == IO.binstream(:stdio, :line) + end end diff --git a/lib/elixir/test/elixir/kernel/alias_test.exs b/lib/elixir/test/elixir/kernel/alias_test.exs index be552061d10..9284fc68e19 100644 --- a/lib/elixir/test/elixir/kernel/alias_test.exs +++ b/lib/elixir/test/elixir/kernel/alias_test.exs @@ -1,4 +1,4 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) alias Kernel.AliasTest.Nested, as: Nested @@ -9,26 +9,26 @@ end defmodule Kernel.AliasTest do use ExUnit.Case, async: true - test :alias_erlang do + test "alias Erlang" do alias :lists, as: MyList assert MyList.flatten([1, [2], 3]) == [1, 2, 3] assert Elixir.MyList.Bar == :"Elixir.MyList.Bar" assert MyList.Bar == :"Elixir.lists.Bar" end - test :double_alias do + test "double alias" do alias Kernel.AliasTest.Nested, as: Nested2 - assert Nested.value == 1 - assert Nested2.value == 1 + assert Nested.value() == 1 + assert Nested2.value() == 1 end - test :overwriten_alias do - alias List, as: Nested + test "overwritten alias" do + assert alias(List, as: Nested) == List assert Nested.flatten([[13]]) == [13] end - test :lexical do - if true do + test "lexical" do + if true_fun() do alias OMG, as: List, warn: false else alias ABC, as: List, warn: false @@ -37,12 +37,29 @@ defmodule Kernel.AliasTest do assert List.flatten([1, [2], 3]) == [1, 2, 3] end + defp true_fun(), do: true + defmodule Elixir do def sample, do: 1 end - test :nested_elixir_alias do - assert Kernel.AliasTest.Elixir.sample == 1 + test "nested Elixir alias" do + assert Kernel.AliasTest.Elixir.sample() == 1 + end + + test "multi-call" do + result = alias unquote(Inspect).{Opts, Algebra} + assert result == [Inspect.Opts, Inspect.Algebra] + assert %Opts{} == %Inspect.Opts{} + assert Algebra.empty() == :doc_nil + end + + test "alias removal" do + alias __MODULE__.Foo + assert Foo == __MODULE__.Foo + alias Elixir.Foo + assert Foo == Elixir.Foo + alias Elixir.Bar end end @@ -54,7 +71,7 @@ defmodule Kernel.AliasNestingGenerator do end defmodule Parent.Child do - def b, do: Parent.a + def b, do: Parent.a() end end end @@ -64,19 +81,19 @@ defmodule Kernel.AliasNestingTest do use ExUnit.Case, async: true require Kernel.AliasNestingGenerator - Kernel.AliasNestingGenerator.create + Kernel.AliasNestingGenerator.create() - test :aliases_nesting do - assert Parent.a == :a - assert Parent.Child.b == :a + test "aliases nesting" do + assert Parent.a() == :a + assert Parent.Child.b() == :a end defmodule Nested do def value, do: 2 end - test :aliases_nesting_with_previous_alias do - assert Nested.value == 2 + test "aliases nesting with previous alias" do + assert Nested.value() == 2 end end @@ -96,6 +113,7 @@ defmodule Macro.AliasTest.Definer do defmodule First do defstruct foo: :bar end + defmodule Second do defstruct baz: %First{} end @@ -118,7 +136,7 @@ defmodule Macro.AliasTest.User do use Macro.AliasTest.Aliaser test "has a struct defined from after compile" do - assert is_map struct(Macro.AliasTest.User.First, []) - assert is_map struct(Macro.AliasTest.User.Second, []).baz + assert is_map(struct(Macro.AliasTest.User.First, [])) + assert is_map(struct(Macro.AliasTest.User.Second, []).baz) end end diff --git a/lib/elixir/test/elixir/kernel/binary_test.exs b/lib/elixir/test/elixir/kernel/binary_test.exs index e6e7934dd16..884153f70d8 100644 --- a/lib/elixir/test/elixir/kernel/binary_test.exs +++ b/lib/elixir/test/elixir/kernel/binary_test.exs @@ -1,56 +1,75 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.BinaryTest do use ExUnit.Case, async: true - test :heredoc do + test "heredoc" do assert 7 == __ENV__.line + assert "foo\nbar\n" == """ -foo -bar -""" + foo + bar + """ + + assert 14 == __ENV__.line - assert 13 == __ENV__.line assert "foo\nbar \"\"\"\n" == """ -foo -bar """ -""" + foo + bar \""" + """ end - test :aligned_heredoc do + test "aligned heredoc" do assert "foo\nbar\n" == """ - foo - bar - """ + foo + bar + """ end - test :heredoc_with_interpolation do - assert "29\n" == """ - #{__ENV__.line} - """ + test "heredoc with interpolation" do + assert "31\n" == """ + #{__ENV__.line} + """ - assert "\n34\n" == """ + assert "\n36\n" == """ - #{__ENV__.line} - """ + #{__ENV__.line} + """ end - test :heredoc_in_call do - assert "foo\nbar" == Kernel.<>(""" - foo - """, "bar") + test "heredoc in call" do + assert "foo\nbar" == + Kernel.<>( + """ + foo + """, + "bar" + ) end - test :utf8 do + test "heredoc with heredoc inside interpolation" do + assert """ + 1 + #{""" + 2 + """} + """ == "1\n2\n\n" + end + + test "UTF-8" do assert byte_size(" ゆんゆん") == 13 end - test :utf8_char do + test "UTF-8 char" do assert ?ゆ == 12422 - assert ?\ゆ == 12422 end - test :string_concatenation_as_match do + test "size outside match" do + x = 16 + assert <<0::size(x)>> == <<0, 0>> + end + + test "string concatenation as match" do "foo" <> x = "foobar" assert x == "bar" @@ -60,142 +79,212 @@ bar """ <<"f", "oo">> <> x = "foobar" assert x == "bar" - <> <> _ = "foobar" + <> <> _ = "foobar" assert x == "foo" size = 3 - <> <> _ = "foobar" + <> <> _ = "foobar" + assert x == "foo" + + <> <> _ = "foobar" + assert x == "foo" + + <> <> _ = "foobar" assert x == "foo" - <> <> _ = "foobar" + <> <> _ = "foobar" assert x == "foo" - assert_raise ErlangError, fn -> - Code.eval_string(~s{<> <> _ = "foobar"}) + <> <> _ = "foobar" + assert x == ?f + end + + test "string concatenation outside match" do + x = "bar" + assert "foobar" = "foo" <> x + assert "barfoo" = x <> "foo" + end + + test "invalid string concatenation arguments" do + assert_raise ArgumentError, ~r"expected binary argument in <> operator but got: :bar", fn -> + Code.eval_string(~s["foo" <> :bar]) end - assert_raise ErlangError, fn -> - Code.eval_string(~s{<> <> _ = "foobar"}) + assert_raise ArgumentError, ~r"expected binary argument in <> operator but got: 1", fn -> + Code.eval_string(~s["foo" <> 1]) + end + + message = ~r"left argument of <> operator inside a match" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(~s[a <> "b" = "ab"]) + end + + assert_raise ArgumentError, message, fn -> + Code.eval_string(~s["a" <> b <> "c" = "abc"]) end - end - test :octals do - assert "\1" == <<1>> - assert "\12" == "\n" - assert "\123" == "S" - assert "\123" == "S" - assert "\377" == "ÿ" - assert "\128" == "\n8" - assert "\18" == <<1, ?8>> + assert_raise ArgumentError, message, fn -> + Code.eval_string(~s[ + a = "a" + ^a <> "b" = "ab" + ]) + end + + assert_raise ArgumentError, message, fn -> + Code.eval_string(~s[ + b = "b" + "a" <> ^b <> "c" = "abc" + ]) + end end - test :hex do - assert "\xa" == "\n" - assert "\xE9" == "é" - assert "\xFF" == "ÿ" - assert "\x{A}"== "\n" - assert "\x{E9}"== "é" - assert "\x{10F}" == <<196, 143>> - assert "\x{10FF}" == <<225, 131, 191>> - assert "\x{10FFF}" == <<240, 144, 191, 191>> - assert "\x{10FFFF}" == <<244, 143, 191, 191>> + test "hex" do + assert "\x76" == "v" + assert "\u00FF" == "ÿ" + assert "\u{A}" == "\n" + assert "\u{E9}" == "é" + assert "\u{10F}" == <<196, 143>> + assert "\u{10FF}" == <<225, 131, 191>> + assert "\u{10FFF}" == <<240, 144, 191, 191>> + assert "\u{10FFFF}" == <<244, 143, 191, 191>> end - test :match do - assert match?(<< ?a, _ :: binary >>, "ab") - refute match?(<< ?a, _ :: binary >>, "cd") - assert match?(<< _ :: utf8 >> <> _, "éf") + test "match" do + assert match?(<>, "ab") + refute match?(<>, "cd") + assert match?(<<_::utf8>> <> _, "éf") end - test :interpolation do + test "interpolation" do res = "hello \\abc" assert "hello #{"\\abc"}" == res assert "hello #{"\\abc" <> ""}" == res end - test :pattern_match do + test "pattern match" do s = 16 - assert <<_a, _b :: size(s)>> = "foo" + assert <<_a, _b::size(s)>> = "foo" end - test :pattern_match_with_splice do - assert << 1, <<2, 3, 4>>, 5 >> = <<1, 2, 3, 4, 5>> + test "pattern match with splice" do + assert <<1, <<2, 3, 4>>, 5>> = <<1, 2, 3, 4, 5>> end - test :partial_application do - assert (&<< &1, 2 >>).(1) == << 1, 2 >> - assert (&<< &1, &2 >>).(1, 2) == << 1, 2 >> - assert (&<< &2, &1 >>).(2, 1) == << 1, 2 >> + test "partial application" do + assert (&<<&1, 2>>).(1) == <<1, 2>> + assert (&<<&1, &2>>).(1, 2) == <<1, 2>> + assert (&<<&2, &1>>).(2, 1) == <<1, 2>> end - test :literal do - assert <<106,111,115,195,169>> == << "josé" :: binary >> - assert <<106,111,115,195,169>> == << "josé" :: bits >> - assert <<106,111,115,195,169>> == << "josé" :: bitstring >> - assert <<106,111,115,195,169>> == << "josé" :: bytes >> - - assert <<106,111,115,195,169>> == << "josé" :: utf8 >> - assert <<0,106,0,111,0,115,0,233>> == << "josé" :: utf16 >> - assert <<106,0,111,0,115,0,233,0>> == << "josé" :: [utf16, little] >> - assert <<0,0,0,106,0,0,0,111,0,0,0,115,0,0,0,233>> == << "josé" :: utf32 >> + test "literal" do + assert <<106, 111, 115, 195, 169>> == <<"josé">> + assert <<106, 111, 115, 195, 169>> == <<"#{:josé}">> + assert <<106, 111, 115, 195, 169>> == <<"josé"::binary>> + assert <<106, 111, 115, 195, 169>> == <<"josé"::bits>> + assert <<106, 111, 115, 195, 169>> == <<"josé"::bitstring>> + assert <<106, 111, 115, 195, 169>> == <<"josé"::bytes>> + + assert <<106, 111, 115, 195, 169>> == <<"josé"::utf8>> + assert <<0, 106, 0, 111, 0, 115, 0, 233>> == <<"josé"::utf16>> + assert <<106, 0, 111, 0, 115, 0, 233, 0>> == <<"josé"::little-utf16>> + assert <<0, 0, 0, 106, 0, 0, 0, 111, 0, 0, 0, 115, 0, 0, 0, 233>> == <<"josé"::utf32>> end - test :literal_errors do - assert_raise CompileError, fn -> - Code.eval_string(~s[<< "foo" :: integer >>]) - end + test "literal errors" do + message = ~r"conflicting type specification for bit field" - assert_raise CompileError, fn -> - Code.eval_string(~s[<< "foo" :: float >>]) + assert_raise CompileError, message, fn -> + Code.eval_string(~s[<<"foo"::integer>>]) end - assert_raise CompileError, fn -> - Code.eval_string(~s[<< 'foo' :: binary >>]) + assert_raise CompileError, message, fn -> + Code.eval_string(~s[<<"foo"::float>>]) end - assert_raise ArgumentError, fn -> - Code.eval_string(~s[<<1::size(4)>> <> "foo"]) + message = ~r"invalid literal 'foo'" + + assert_raise CompileError, message, fn -> + Code.eval_string(~s[<<'foo'::binary>>]) end end + @bitstring <<"foo", 16::4>> + + test "bitstring attribute" do + assert @bitstring == <<"foo", 16::4>> + end + @binary "new " - test :bitsyntax_with_expansion do + test "bitsyntax expansion" do assert <<@binary, "world">> == "new world" end - test :bitsyntax_translation do + test "bitsyntax translation" do refb = "sample" sec_data = "another" - << byte_size(refb) :: [size(1), big, signed, integer, unit(8)], - refb :: binary, - byte_size(sec_data) :: [size(1), big, signed, integer, unit(16)], - sec_data :: binary >> + + << + byte_size(refb)::size(1)-big-signed-integer-unit(8), + refb::binary, + byte_size(sec_data)::1*16-big-signed-integer, + sec_data::binary + >> end - test :bitsyntax_size_shorcut do - assert << 1 :: 3 >> == << 1 :: size(3) >> - assert << 1 :: [unit(8), 3] >> == << 1 :: [unit(8), size(3)] >> + test "bitsyntax size shortcut" do + assert <<1::3>> == <<1::size(3)>> + assert <<1::3*8>> == <<1::size(3)-unit(8)>> + end + + test "bitsyntax variable size" do + x = 8 + assert <<_, _::size(x)>> = <> + assert (fn <<_, _::size(x)>> -> true end).(<>) + end + + test "bitsyntax size using expressions" do + x = 8 + assert <<1::size(x - 5)>> + + foo = %{bar: 5} + assert <<1::size(foo.bar)>> + assert <<1::size(length('abcd'))>> + assert <<255::size(hd(List.flatten([3])))>> + end + + test "bitsyntax size using guard expressions in match context" do + x = 8 + assert <<1::size(x - 5)>> = <<1::3>> + assert <<1::size(x - 5)-unit(8)>> = <<1::3*8>> + assert <<1::size(length('abcd'))>> = <<1::4>> + + foo = %{bar: 5} + assert <<1::size(foo.bar)>> = <<1::5>> end defmacrop signed_16 do quote do - [big, signed, integer, unit(16)] + big - signed - integer - unit(16) end end defmacrop refb_spec do quote do - [size(1), big, signed, integer, unit(8)] + 1 * 8 - big - signed - integer end end - test :bitsyntax_macro do + test "bitsyntax macro" do refb = "sample" sec_data = "another" - << byte_size(refb) :: refb_spec, - refb :: binary, - byte_size(sec_data) :: [size(1), signed_16], - sec_data :: binary >> + + << + byte_size(refb)::refb_spec, + refb::binary, + byte_size(sec_data)::size(1)-signed_16, + sec_data::binary + >> end end diff --git a/lib/elixir/test/elixir/kernel/case_test.exs b/lib/elixir/test/elixir/kernel/case_test.exs deleted file mode 100644 index 9c9755c5564..00000000000 --- a/lib/elixir/test/elixir/kernel/case_test.exs +++ /dev/null @@ -1,66 +0,0 @@ -Code.require_file "../test_helper.exs", __DIR__ - -defmodule Kernel.CaseTest do - use ExUnit.Case, async: true - - test :inline_case do - assert (case 1, do: (1 -> :ok; 2 -> :wrong)) == :ok - end - - test :nested_variables do - assert vars_case(400, 1) == {400, 1} - assert vars_case(401, 1) == {400, -1} - assert vars_case(0, -1) == {0, -1} - assert vars_case(-1, -1) == {0, 1} - end - - test :nested_vars_match do - x = {:error, {:ok, :done}} - assert (case x do - {:ok, right} -> - right - {_left, right} -> - case right do - {:ok, right} -> right - end - end) == :done - end - - test :in_operator_outside_case do - x = 1 - y = 4 - assert x in [1, 2, 3], "in assertion" - assert not y in [1, 2, 3], "not in assertion" - end - - test :in_with_match do - refute 1.0 in [1, 2, 3], "not in assertion" - end - - test :in_cond_clause do - assert (cond do - format() && (f = format()) -> - f - true -> - :text - end) == :html - end - - defp format, do: :html - - defp vars_case(x, vx) do - case x > 400 do - true -> - x = 400 - vx = -vx - _ -> - case x < 0 do - true -> - x = 0 - vx = -vx - _ -> nil - end - end - {x, vx} - end -end diff --git a/lib/elixir/test/elixir/kernel/char_list_test.exs b/lib/elixir/test/elixir/kernel/char_list_test.exs deleted file mode 100644 index 7bbc381f2b2..00000000000 --- a/lib/elixir/test/elixir/kernel/char_list_test.exs +++ /dev/null @@ -1,45 +0,0 @@ -Code.require_file "../test_helper.exs", __DIR__ - -defmodule CharListTest do - use ExUnit.Case, async: true - - test :heredoc do - assert __ENV__.line == 7 - assert 'foo\nbar\n' == ''' -foo -bar -''' - - assert __ENV__.line == 13 - assert 'foo\nbar \'\'\'\n' == ''' -foo -bar ''' -''' - end - - test :utf8 do - assert length(' ゆんゆん') == 5 - end - - test :octals do - assert '\1' == [1] - assert '\12' == '\n' - assert '\123' == 'S' - assert '\123' == 'S' - assert '\377' == 'ÿ' - assert '\128' == '\n8' - assert '\18' == [1, ?8] - end - - test :hex do - assert '\xa' == '\n' - assert '\xE9' == 'é' - assert '\xfF' == 'ÿ' - assert '\x{A}' == '\n' - assert '\x{e9}' == 'é' - assert '\x{10F}' == [271] - assert '\x{10FF}' == [4351] - assert '\x{10FFF}' == [69631] - assert '\x{10FFFF}' == [1114111] - end -end diff --git a/lib/elixir/test/elixir/kernel/charlist_test.exs b/lib/elixir/test/elixir/kernel/charlist_test.exs new file mode 100644 index 00000000000..796459fbe2f --- /dev/null +++ b/lib/elixir/test/elixir/kernel/charlist_test.exs @@ -0,0 +1,36 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule CharlistTest do + use ExUnit.Case, async: true + + test "heredoc" do + assert __ENV__.line == 7 + + assert 'foo\nbar\n' == ''' + foo + bar + ''' + + assert __ENV__.line == 14 + + assert 'foo\nbar \'\'\'\n' == ''' + foo + bar \'\'\' + ''' + end + + test "UTF-8" do + assert length(' ゆんゆん') == 5 + end + + test "hex" do + assert '\x76' == 'v' + assert '\u00fF' == 'ÿ' + assert '\u{A}' == '\n' + assert '\u{e9}' == 'é' + assert '\u{10F}' == [271] + assert '\u{10FF}' == [4351] + assert '\u{10FFF}' == [69631] + assert '\u{10FFFF}' == [1_114_111] + end +end diff --git a/lib/elixir/test/elixir/kernel/cli_test.exs b/lib/elixir/test/elixir/kernel/cli_test.exs index 65369f4d78e..65018340099 100644 --- a/lib/elixir/test/elixir/kernel/cli_test.exs +++ b/lib/elixir/test/elixir/kernel/cli_test.exs @@ -1,11 +1,42 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) import PathHelpers -defmodule Kernel.CLI.ARGVTest do +defmodule Retry do + # Tests that write to stderr fail on Windows due to late writes, + # so we do a simple retry already them. + defmacro stderr_test(msg, context \\ quote(do: _), do: block) do + if windows?() do + quote do + test unquote(msg), unquote(context) do + unquote(__MODULE__).retry(fn -> unquote(block) end, 3) + end + end + else + quote do + test(unquote(msg), unquote(context), do: unquote(block)) + end + end + end + + def retry(fun, 1) do + fun.() + end + + def retry(fun, n) do + try do + fun.() + rescue + _ -> retry(fun, n - 1) + end + end +end + +defmodule Kernel.CLITest do use ExUnit.Case, async: true import ExUnit.CaptureIO + import Retry defp run(argv) do {config, argv} = Kernel.CLI.parse_argv(argv) @@ -15,152 +46,234 @@ defmodule Kernel.CLI.ARGVTest do test "argv handling" do assert capture_io(fn -> - assert run(["-e", "IO.puts :ok", "sample.exs", "-o", "1", "2"]) == - ["sample.exs", "-o", "1", "2"] - end) == "ok\n" + assert run(["-e", "IO.puts :ok", "sample.exs", "-o", "1", "2"]) == + ["sample.exs", "-o", "1", "2"] + end) == "ok\n" assert capture_io(fn -> - assert run(["-e", "IO.puts :ok", "--", "sample.exs", "-o", "1", "2"]) == - ["sample.exs", "-o", "1", "2"] - end) == "ok\n" + assert run(["-e", "IO.puts :ok", "--", "sample.exs", "-o", "1", "2"]) == + ["sample.exs", "-o", "1", "2"] + end) == "ok\n" assert capture_io(fn -> - assert run(["-e", "IO.puts :ok", "--hidden", "sample.exs", "-o", "1", "2"]) == - ["sample.exs", "-o", "1", "2"] - end) == "ok\n" + assert run(["-e", "", "--", "sample.exs", "-o", "1", "2"]) == + ["sample.exs", "-o", "1", "2"] + end) + end - assert capture_io(fn -> - assert run(["-e", "IO.puts :ok", "--", "--hidden", "sample.exs", "-o", "1", "2"]) == - ["--hidden", "sample.exs", "-o", "1", "2"] - end) == "ok\n" + stderr_test "--help smoke test" do + output = elixir('--help') + assert output =~ "Usage: elixir" end -end -defmodule Kernel.CLI.OptionParsingTest do - use ExUnit.Case, async: true + test "--version smoke test" do + output = elixir('--version') + assert output =~ "Erlang/OTP #{System.otp_release()}" + assert output =~ "Elixir #{System.version()}" + + output = iex('--version') + assert output =~ "Erlang/OTP #{System.otp_release()}" + assert output =~ "IEx #{System.version()}" + + output = elixir('--version -e "IO.puts(:test_output)"') + assert output =~ "Erlang/OTP #{System.otp_release()}" + assert output =~ "Elixir #{System.version()}" + assert output =~ "Standalone options can't be combined with other options" + end + + test "--short-version smoke test" do + output = elixir('--short-version') + assert output =~ System.version() + refute output =~ "Erlang" + end + + stderr_test "combining --help results in error" do + output = elixir('-e 1 --help') + assert output =~ "--help : Standalone options can't be combined with other options" + + output = elixir('--help -e 1') + assert output =~ "--help : Standalone options can't be combined with other options" + end + + stderr_test "combining --short-version results in error" do + output = elixir('--short-version -e 1') + assert output =~ "--short-version : Standalone options can't be combined with other options" + + output = elixir('-e 1 --short-version') + assert output =~ "--short-version : Standalone options can't be combined with other options" + end test "properly parses paths" do - root = fixture_path("../../..") |> to_char_list - list = elixir('-pa "#{root}/*" -pz "#{root}/lib/*" -e "IO.inspect(:code.get_path, limit: :infinity)"') - {path, _} = Code.eval_string list, [] + root = fixture_path("../../..") |> to_charlist + + args = + '-pa "#{root}/*" -pz "#{root}/lib/*" -e "IO.inspect(:code.get_path(), limit: :infinity)"' + + list = elixir(args) + {path, _} = Code.eval_string(list, []) # pa - assert to_char_list(Path.expand('ebin', root)) in path - assert to_char_list(Path.expand('lib', root)) in path - assert to_char_list(Path.expand('src', root)) in path + assert to_charlist(Path.expand('ebin', root)) in path + assert to_charlist(Path.expand('lib', root)) in path + assert to_charlist(Path.expand('src', root)) in path # pz - assert to_char_list(Path.expand('lib/list', root)) in path + assert to_charlist(Path.expand('lib/list', root)) in path end -end -defmodule Kernel.CLI.AtExitTest do - use ExUnit.Case, async: true + stderr_test "properly formats errors" do + assert String.starts_with?(elixir('-e ":erlang.throw 1"'), "** (throw) 1") + assert String.starts_with?(elixir('-e ":erlang.error 1"'), "** (ErlangError) Erlang error: 1") + assert String.starts_with?(elixir('-e "1 +"'), "** (TokenMissingError)") - test "invokes at_exit callbacks" do - assert elixir(fixture_path("at_exit.exs") |> to_char_list) == - 'goodbye cruel world with status 0\n' + assert elixir('-e "Task.async(fn -> raise ArgumentError end) |> Task.await"') =~ + "an exception was raised:\n ** (ArgumentError) argument error" + + assert elixir('-e "IO.puts(Process.flag(:trap_exit, false)); exit({:shutdown, 1})"') == + "false\n" + end + + stderr_test "blames exceptions" do + error = elixir('-e "Access.fetch :foo, :bar"') + assert error =~ "** (FunctionClauseError) no function clause matching in Access.fetch/2" + assert error =~ "The following arguments were given to Access.fetch/2" + assert error =~ ":foo" + assert error =~ "def fetch(-%module{} = container-, +key+)" + assert error =~ ~r"\(elixir #{System.version()}\) lib/access\.ex:\d+: Access\.fetch/2" end end -defmodule Kernel.CLI.ErrorTest do +defmodule Kernel.CLI.RPCTest do use ExUnit.Case, async: true - test "properly format errors" do - assert :string.str('** (throw) 1', elixir('-e "throw 1"')) == 0 - assert :string.str('** (ErlangError) erlang error: 1', elixir('-e "error 1"')) == 0 + import Retry - # It does not catch exits with integers nor strings... - assert elixir('-e "exit 1"') == '' + defp rpc_eval(command) do + node = "cli-rpc#{System.unique_integer()}@127.0.0.1" + elixir('--name #{node} --rpc-eval #{node} "#{command}"') end -end -defmodule Kernel.CLI.CompileTest do - use ExUnit.Case, async: true + test "invokes command on remote node" do + assert rpc_eval("IO.puts :ok") == "ok\n" + end - test "compiles code" do - fixture = fixture_path "compile_sample.ex" - assert elixirc('#{fixture} -o #{tmp_path}') == '' - assert File.regular?(tmp_path "Elixir.CompileSample.beam") - after - File.rm(tmp_path("Elixir.CompileSample.beam")) + test "invokes command on remote node without host and --name after --rpc-eval" do + node = "cli-rpc#{System.unique_integer()}" + assert elixir('--rpc-eval #{node} "IO.puts :ok" --name #{node}@127.0.0.1 ') == "ok\n" end - test "compiles code with verbose mode" do - fixture = fixture_path "compile_sample.ex" - assert elixirc('#{fixture} -o #{tmp_path} --verbose') == - 'Compiled #{fixture}\n' - assert File.regular?(tmp_path "Elixir.CompileSample.beam") - after - File.rm(tmp_path("Elixir.CompileSample.beam")) + test "can be invoked multiple times" do + node = "cli-rpc#{System.unique_integer()}" + + assert elixir( + '--name #{node}@127.0.0.1 --rpc-eval #{node} "IO.puts :foo" --rpc-eval #{node} "IO.puts :bar"' + ) == "foo\nbar\n" end - test "fails on missing patterns" do - fixture = fixture_path "compile_sample.ex" - output = elixirc('#{fixture} non_existing.ex -o #{tmp_path}') - assert :string.str(output, 'non_existing.ex') > 0, "expected non_existing.ex to be mentionned" - assert :string.str(output, 'compile_sample.ex') == 0, "expected compile_sample.ex to not be mentionned" - refute File.exists?(tmp_path("Elixir.CompileSample.beam")) , "expected the sample to not be compiled" + # Windows does not provide an easy to check for missing args + @tag :unix + test "fails on wrong arguments" do + node = "cli-rpc#{System.unique_integer()}" + + assert elixir('--name #{node}@127.0.0.1 --rpc-eval') == + "--rpc-eval : wrong number of arguments\n" + + assert elixir('--name #{node}@127.0.0.1 --rpc-eval #{node}') == + "--rpc-eval : wrong number of arguments\n" + end + + stderr_test "properly formats errors" do + assert String.starts_with?(rpc_eval(":erlang.throw 1"), "** (throw) 1") + assert String.starts_with?(rpc_eval(":erlang.error 1"), "** (ErlangError) Erlang error: 1") + assert String.starts_with?(rpc_eval("1 +"), "** (TokenMissingError)") + + assert rpc_eval("Task.async(fn -> raise ArgumentError end) |> Task.await") =~ + "an exception was raised:\n ** (ArgumentError) argument error" + + assert rpc_eval("IO.puts(Process.flag(:trap_exit, false)); exit({:shutdown, 1})") == + "false\n" end end -defmodule Kernel.CLI.ParallelCompilerTest do - use ExUnit.Case - import ExUnit.CaptureIO +defmodule Kernel.CLI.AtExitTest do + use ExUnit.Case, async: true - test "compiles files solving dependencies" do - fixtures = [fixture_path("parallel_compiler/bar.ex"), fixture_path("parallel_compiler/foo.ex")] - assert capture_io(fn -> - assert [Bar, Foo] = Kernel.ParallelCompiler.files fixtures - end) =~ "message_from_foo" - after - Enum.map [Foo, Bar], fn mod -> - :code.purge(mod) - :code.delete(mod) - end + test "invokes at_exit callbacks" do + assert elixir(fixture_path("at_exit.exs") |> to_charlist) == + "goodbye cruel world with status 1\n" end +end - test "compiles files with structs solving dependencies" do - fixtures = [fixture_path("parallel_struct/bar.ex"), fixture_path("parallel_struct/foo.ex")] - assert [Bar, Foo] = Kernel.ParallelCompiler.files(fixtures) |> Enum.sort +defmodule Kernel.CLI.CompileTest do + use ExUnit.Case, async: true + + import Retry + @moduletag :tmp_dir + + setup context do + beam_file_path = Path.join([context.tmp_dir, "Elixir.CompileSample.beam"]) + fixture = fixture_path("compile_sample.ex") + {:ok, [beam_file_path: beam_file_path, fixture: fixture]} + end + + test "compiles code", context do + assert elixirc('#{context.fixture} -o #{context.tmp_dir}') == "" + assert File.regular?(context.beam_file_path) + + # Assert that the module is loaded into memory with the proper destination for the BEAM file. + Code.append_path(context.tmp_dir) + assert :code.which(CompileSample) |> List.to_string() == Path.expand(context.beam_file_path) after - Enum.map [Foo, Bar], fn mod -> - :code.purge(mod) - :code.delete(mod) + :code.purge(CompileSample) + :code.delete(CompileSample) + Code.delete_path(context.tmp_dir) + end + + @tag :windows + stderr_test "compiles code with Windows paths", context do + try do + fixture = String.replace(context.fixture, "/", "\\") + tmp_dir_path = String.replace(context.tmp_dir, "/", "\\") + assert elixirc('#{fixture} -o #{tmp_dir_path}') == "" + assert File.regular?(context[:beam_file_path]) + + # Assert that the module is loaded into memory with the proper destination for the BEAM file. + Code.append_path(context.tmp_dir) + + assert :code.which(CompileSample) |> List.to_string() == + Path.expand(context[:beam_file_path]) + after + :code.purge(CompileSample) + :code.delete(CompileSample) + Code.delete_path(context.tmp_dir) end end - test "does not hang on missing dependencies" do - fixtures = [fixture_path("parallel_compiler/bat.ex")] - assert capture_io(fn -> - assert catch_exit(Kernel.ParallelCompiler.files(fixtures)) == 1 - end) =~ "Compilation error" + stderr_test "fails on missing patterns", context do + output = elixirc('#{context.fixture} non_existing.ex -o #{context.tmp_dir}') + assert output =~ "non_existing.ex" + refute output =~ "compile_sample.ex" + refute File.exists?(context.beam_file_path) end - test "handles possible deadlocks" do - fixtures = [fixture_path("parallel_deadlock/foo.ex"), - fixture_path("parallel_deadlock/bar.ex")] + stderr_test "fails on missing write access to .beam file", context do + compilation_args = '#{context.fixture} -o #{context.tmp_dir}' - msg = capture_io(fn -> - assert catch_exit(Kernel.ParallelCompiler.files fixtures) == 1 - end) + assert elixirc(compilation_args) == "" + assert File.regular?(context.beam_file_path) - assert msg =~ ~r"== Compilation error on file .+parallel_deadlock/foo\.ex ==" - assert msg =~ ~r"== Compilation error on file .+parallel_deadlock/bar\.ex ==" - end + # Set the .beam file to read-only + File.chmod!(context.beam_file_path, 4) + {:ok, %{access: access}} = File.stat(context.beam_file_path) - test "warnings as errors" do - warnings_as_errors = Code.compiler_options[:warnings_as_errors] - fixtures = [fixture_path("warnings_sample.ex")] + # Can only assert when read-only applies to the user + if access != :read_write do + output = elixirc(compilation_args) - try do - Code.compiler_options(warnings_as_errors: true) + expected = + "(File.Error) could not write to file #{inspect(context.beam_file_path)}: permission denied" - capture_io :stderr, fn -> - assert catch_exit(Kernel.ParallelCompiler.files fixtures) == 1 - end - after - Code.compiler_options(warnings_as_errors: warnings_as_errors) + assert output =~ expected end end end diff --git a/lib/elixir/test/elixir/kernel/comprehension_test.exs b/lib/elixir/test/elixir/kernel/comprehension_test.exs index 1a731460407..7c1ebf7615d 100644 --- a/lib/elixir/test/elixir/kernel/comprehension_test.exs +++ b/lib/elixir/test/elixir/kernel/comprehension_test.exs @@ -1,4 +1,4 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.ComprehensionTest do use ExUnit.Case, async: true @@ -6,8 +6,24 @@ defmodule Kernel.ComprehensionTest do import ExUnit.CaptureIO require Integer + defmodule Pdict do + defstruct [] + + defimpl Collectable do + def into(struct) do + fun = fn + _, {:cont, x} -> Process.put(:into_cont, [x | Process.get(:into_cont)]) + _, :done -> Process.put(:into_done, true) + _, :halt -> Process.put(:into_halt, true) + end + + {struct, fun} + end + end + end + defp to_bin(x) do - << x >> + <> end defp nilly, do: nil @@ -20,46 +36,131 @@ defmodule Kernel.ComprehensionTest do end test "for comprehensions with matching" do - assert for({_,x} <- 1..3, do: x * 2) == [] + assert for({_, x} <- 1..3, do: x * 2) == [] + end + + test "for comprehensions with pin matching" do + maps = [x: 1, y: 2, x: 3] + assert for({:x, v} <- maps, do: v * 2) == [2, 6] + x = :x + assert for({^x, v} <- maps, do: v * 2) == [2, 6] + end + + test "for comprehensions with guards" do + assert for(x when x < 4 <- 1..10, do: x) == [1, 2, 3] + assert for(x when x == 3 when x == 7 <- 1..10, do: x) == [3, 7] + end + + test "for comprehensions with guards and filters" do + assert for( + {var, _} + when is_atom(var) <- [{:foo, 1}, {2, :bar}], + var = Atom.to_string(var), + do: var + ) == ["foo"] + end + + test "for comprehensions with map key matching" do + maps = [%{x: 1}, %{y: 2}, %{x: 3}] + assert for(%{x: v} <- maps, do: v * 2) == [2, 6] + x = :x + assert for(%{^x => v} <- maps, do: v * 2) == [2, 6] end test "for comprehensions with filters" do assert for(x <- 1..3, x > 1, x < 3, do: x * 2) == [4] end + test "for comprehensions with unique values" do + list = [1, 1, 2, 3] + assert for(x <- list, uniq: true, do: x * 2) == [2, 4, 6] + assert for(x <- list, uniq: true, into: [], do: x * 2) == [2, 4, 6] + assert for(x <- list, uniq: true, into: %{}, do: {x, 1}) == %{1 => 1, 2 => 1, 3 => 1} + assert for(x <- list, uniq: true, into: "", do: to_bin(x * 2)) == <<2, 4, 6>> + assert for(<>, uniq: true, into: "", do: to_bin(x)) == "abc" + + Process.put(:into_cont, []) + Process.put(:into_done, false) + Process.put(:into_halt, false) + + for x <- list, uniq: true, into: %Pdict{} do + x * 2 + end + + assert Process.get(:into_cont) == [6, 4, 2] + assert Process.get(:into_done) + refute Process.get(:into_halt) + + assert_raise RuntimeError, "oops", fn -> + for _ <- [1, 2, 3], uniq: true, into: %Pdict{}, do: raise("oops") + end + + assert Process.get(:into_halt) + end + + test "nested for comprehensions with unique values" do + assert for(x <- [1, 1, 2], uniq: true, do: for(y <- [3, 3], uniq: true, do: x * y)) == [ + [3], + [6] + ] + + assert for(<>, + uniq: true, + into: "", + do: for(<>, uniq: true, into: "", do: to_bin(x) <> to_bin(y)) + ) == "azbzcz" + end + test "for comprehensions with nilly filters" do - assert for(x <- 1..3, nilly, do: x * 2) == [] + assert for(x <- 1..3, nilly(), do: x * 2) == [] end test "for comprehensions with errors on filters" do assert_raise ArgumentError, fn -> - for(x <- 1..3, hd(x), do: x * 2) + for x <- 1..3, hd(x), do: x * 2 end end test "for comprehensions with variables in filters" do - assert for(x <- 1..3, y = x + 1, y > 2, z = y, do: x * z) == - [6, 12] + assert for(x <- 1..3, y = x + 1, y > 2, z = y, do: x * z) == [6, 12] end test "for comprehensions with two enum generators" do - assert (for x <- [1, 2, 3], y <- [4, 5, 6], do: x * y) == - [4, 5, 6, 8, 10, 12, 12, 15, 18] + assert for( + x <- [1, 2, 3], + y <- [4, 5, 6], + do: x * y + ) == [4, 5, 6, 8, 10, 12, 12, 15, 18] end test "for comprehensions with two enum generators and filters" do - assert (for x <- [1, 2, 3], y <- [4, 5, 6], y / 2 == x, do: x * y) == - [8, 18] + assert for( + x <- [1, 2, 3], + y <- [4, 5, 6], + y / 2 == x, + do: x * y + ) == [8, 18] end test "for comprehensions generators precedence" do - assert (for {_, _} = x <- [foo: :bar], do: x) == - [foo: :bar] + assert for({_, _} = x <- [foo: :bar], do: x) == [foo: :bar] + end + + test "for comprehensions with shadowing" do + assert for( + a <- + ( + b = 1 + _ = b + [1] + ), + b <- [2], + do: a + b + ) == [3] end test "for comprehensions with binary, enum generators and filters" do - assert (for x <- [1, 2, 3], << y <- <<4, 5, 6>> >>, y / 2 == x, do: x * y) == - [8, 18] + assert for(x <- [1, 2, 3], <<(y <- <<4, 5, 6>>)>>, y / 2 == x, do: x * y) == [8, 18] end test "for comprehensions into list" do @@ -68,17 +169,47 @@ defmodule Kernel.ComprehensionTest do end test "for comprehensions into binary" do - enum = 1..3 - assert for(x <- enum, into: "", do: to_bin(x * 2)) == <<2, 4, 6>> + enum = 0..3 + + assert (for x <- enum, into: "" do + to_bin(x * 2) + end) == <<0, 2, 4, 6>> + + assert (for x <- enum, into: "" do + if Integer.is_even(x), do: <>, else: <> + end) == <<0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + end + + test "for comprehensions into dynamic binary" do + enum = 0..3 + into = "" + + assert (for x <- enum, into: into do + to_bin(x * 2) + end) == <<0, 2, 4, 6>> + + assert (for x <- enum, into: into do + if Integer.is_even(x), do: <>, else: <> + end) == <<0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + + into = <<7::size(1)>> + + assert (for x <- enum, into: into do + to_bin(x * 2) + end) == <<7::size(1), 0, 2, 4, 6>> + + assert (for x <- enum, into: into do + if Integer.is_even(x), do: <>, else: <> + end) == <<7::size(1), 0::size(2), 1::size(1), 2::size(2), 3::size(1)>> end test "for comprehensions where value is not used" do enum = 1..3 assert capture_io(fn -> - for(x <- enum, do: IO.puts x) - nil - end) == "1\n2\n3\n" + for x <- enum, do: IO.puts(x) + nil + end) == "1\n2\n3\n" end test "for comprehensions with into" do @@ -86,7 +217,7 @@ defmodule Kernel.ComprehensionTest do Process.put(:into_done, false) Process.put(:into_halt, false) - for x <- 1..3, into: collectable_pdict do + for x <- 1..3, into: %Pdict{} do x * 2 end @@ -101,7 +232,7 @@ defmodule Kernel.ComprehensionTest do Process.put(:into_halt, false) catch_error( - for x <- 1..3, into: collectable_pdict do + for x <- 1..3, into: %Pdict{} do if x > 2, do: raise("oops"), else: x end ) @@ -114,19 +245,38 @@ defmodule Kernel.ComprehensionTest do test "for comprehension with into, generators and filters" do Process.put(:into_cont, []) - for x <- 1..3, Integer.odd?(x), << y <- "hello" >>, into: collectable_pdict do + for x <- 1..3, Integer.is_odd(x), <>, into: %Pdict{} do x + y end assert IO.iodata_to_binary(Process.get(:into_cont)) == "roohkpmmfi" end - defp collectable_pdict do - fn - _, {:cont, x} -> Process.put(:into_cont, [x|Process.get(:into_cont)]) - _, :done -> Process.put(:into_done, true) - _, :halt -> Process.put(:into_halt, true) - end + test "for comprehensions of map into map" do + enum = %{a: 2, b: 3} + assert for({k, v} <- enum, into: %{}, do: {k, v * v}) == %{a: 4, b: 9} + end + + test "for comprehensions with reduce, generators and filters" do + acc = + for x <- 1..3, Integer.is_odd(x), <>, reduce: %{} do + acc -> Map.update(acc, x, [y], &[y | &1]) + end + + assert acc == %{1 => 'olleh', 3 => 'olleh'} + end + + test "for comprehensions with matched reduce" do + acc = + for entry <- [1, 2, 3], reduce: {:ok, nil} do + {:ok, _} -> + {:ok, entry} + + {:error, _} = error -> + error + end + + assert acc == {:ok, 3} end ## List generators (inlined by the compiler) @@ -137,26 +287,33 @@ defmodule Kernel.ComprehensionTest do end test "list for comprehensions with matching" do - assert for({_,x} <- [1, 2, a: 3, b: 4, c: 5], do: x * 2) == [6, 8, 10] + assert for({_, x} <- [1, 2, a: 3, b: 4, c: 5], do: x * 2) == [6, 8, 10] + end + + test "list for comprehension matched to '_' on last line of block" do + assert (if true_fun() do + _ = for x <- [1, 2, 3], do: x * 2 + end) == [2, 4, 6] end + defp true_fun(), do: true + test "list for comprehensions with filters" do assert for(x <- [1, 2, 3], x > 1, x < 3, do: x * 2) == [4] end test "list for comprehensions with nilly filters" do - assert for(x <- [1, 2, 3], nilly, do: x * 2) == [] + assert for(x <- [1, 2, 3], nilly(), do: x * 2) == [] end test "list for comprehensions with errors on filters" do assert_raise ArgumentError, fn -> - for(x <- [1, 2, 3], hd(x), do: x * 2) + for x <- [1, 2, 3], hd(x), do: x * 2 end end test "list for comprehensions with variables in filters" do - assert for(x <- [1, 2, 3], y = x + 1, y > 2, z = y, do: x * z) == - [6, 12] + assert for(x <- [1, 2, 3], y = x + 1, y > 2, z = y, do: x * z) == [6, 12] end test "list for comprehensions into list" do @@ -164,53 +321,181 @@ defmodule Kernel.ComprehensionTest do assert for(x <- enum, into: [], do: x * 2) == [2, 4, 6] end - test "list for comprehensions into binaries" do - enum = [1, 2, 3] - assert for(x <- enum, into: "", do: to_bin(x * 2)) == <<2, 4, 6>> + test "list for comprehensions into binary" do + enum = [0, 1, 2, 3] + + assert (for x <- enum, into: "" do + to_bin(x * 2) + end) == <<0, 2, 4, 6>> + + assert (for x <- enum, into: "" do + if Integer.is_even(x), do: <>, else: <> + end) == <<0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + end + + test "list for comprehensions into dynamic binary" do + enum = [0, 1, 2, 3] + into = "" + + assert (for x <- enum, into: into do + to_bin(x * 2) + end) == <<0, 2, 4, 6>> + + assert (for x <- enum, into: into do + if Integer.is_even(x), do: <>, else: <> + end) == <<0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + + into = <<7::size(1)>> + + assert (for x <- enum, into: into do + to_bin(x * 2) + end) == <<7::size(1), 0, 2, 4, 6>> + + assert (for x <- enum, into: into do + if Integer.is_even(x), do: <>, else: <> + end) == <<7::size(1), 0::size(2), 1::size(1), 2::size(2), 3::size(1)>> end test "list for comprehensions where value is not used" do - enum = [1,2,3] + enum = [1, 2, 3] assert capture_io(fn -> - for(x <- enum, do: IO.puts x) - nil - end) == "1\n2\n3\n" + for x <- enum, do: IO.puts(x) + nil + end) == "1\n2\n3\n" + end + + test "list for comprehensions with reduce, generators and filters" do + acc = + for x <- [1, 2, 3], Integer.is_odd(x), <>, reduce: %{} do + acc -> Map.update(acc, x, [y], &[y | &1]) + end + + assert acc == %{1 => 'olleh', 3 => 'olleh'} end ## Binary generators (inlined by the compiler) test "binary for comprehensions" do bin = <<1, 2, 3>> - assert for(<< x <- bin >>, do: x * 2) == [2, 4, 6] + assert for(<>, do: x * 2) == [2, 4, 6] end test "binary for comprehensions with inner binary" do bin = <<1, 2, 3>> - assert for(<< <> <- bin >>, do: x * 2) == [2, 4, 6] + assert for(<<(<> <- bin)>>, do: x * 2) == [2, 4, 6] end test "binary for comprehensions with two generators" do - assert (for << x <- <<1, 2, 3>> >>, << y <- <<4, 5, 6>> >>, y / 2 == x, do: x * y) == - [8, 18] + assert for(<<(x <- <<1, 2, 3>>)>>, <<(y <- <<4, 5, 6>>)>>, y / 2 == x, do: x * y) == [8, 18] end test "binary for comprehensions into list" do bin = <<1, 2, 3>> - assert for(<< x <- bin >>, into: [], do: x * 2) == [2, 4, 6] + assert for(<>, into: [], do: x * 2) == [2, 4, 6] end - test "binary for comprehensions into binaries" do - bin = <<1, 2, 3>> - assert for(<< x <- bin >>, into: "", do: to_bin(x * 2)) == <<2, 4, 6>> + test "binary for comprehensions into binary" do + bin = <<0, 1, 2, 3>> + + assert (for <>, into: "" do + to_bin(x * 2) + end) == <<0, 2, 4, 6>> + + assert (for <>, into: "" do + if Integer.is_even(x), do: <>, else: <> + end) == <<0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + end + + test "binary for comprehensions into dynamic binary" do + bin = <<0, 1, 2, 3>> + into = "" + + assert (for <>, into: into do + to_bin(x * 2) + end) == <<0, 2, 4, 6>> + + assert (for <>, into: into do + if Integer.is_even(x), do: <>, else: <> + end) == <<0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + + into = <<7::size(1)>> + + assert (for <>, into: into do + to_bin(x * 2) + end) == <<7::size(1), 0, 2, 4, 6>> + + assert (for <>, into: into do + if Integer.is_even(x), do: <>, else: <> + end) == <<7::size(1), 0::size(2), 1::size(1), 2::size(2), 3::size(1)>> + end + + test "binary for comprehensions with literal matches" do + # Integers + bin = <<1, 2, 1, 3, 1, 4>> + assert for(<<1, x <- bin>>, into: "", do: to_bin(x)) == <<2, 3, 4>> + assert for(<<1, x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2, 3 => 3, 4 => 4} + + bin = <<1, 2, 3, 1, 4>> + assert for(<<1, x <- bin>>, into: "", do: to_bin(x)) == <<2>> + assert for(<<1, x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2} + + # Floats + bin = <<1.0, 2, 1.0, 3, 1.0, 4>> + assert for(<<1.0, x <- bin>>, into: "", do: to_bin(x)) == <<2, 3, 4>> + assert for(<<1.0, x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2, 3 => 3, 4 => 4} + + bin = <<1.0, 2, 3, 1.0, 4>> + assert for(<<1.0, x <- bin>>, into: "", do: to_bin(x)) == <<2>> + assert for(<<1.0, x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2} + + # Binaries + bin = <<"foo", 2, "foo", 3, "foo", 4>> + assert for(<<"foo", x <- bin>>, into: "", do: to_bin(x)) == <<2, 3, 4>> + assert for(<<"foo", x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2, 3 => 3, 4 => 4} + + bin = <<"foo", 2, 3, "foo", 4>> + assert for(<<"foo", x <- bin>>, into: "", do: to_bin(x)) == <<2>> + assert for(<<"foo", x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2} + + bin = <<"foo", 2, 3, 4, "foo", 5>> + assert for(<<"foo", x <- bin>>, into: "", do: to_bin(x)) == <<2>> + assert for(<<"foo", x <- bin>>, into: %{}, do: {x, x}) == %{2 => 2} + end + + test "binary for comprehensions with variable size" do + s = 16 + bin = <<1, 2, 3, 4, 5, 6>> + assert for(<>, into: "", do: to_bin(div(x, 2))) == <<129, 130, 131>> + + # Aligned + bin = <<8, 1, 16, 2, 3>> + assert for(<>, into: "", do: <>) == <<1, 2, 3>> + assert for(<>, into: %{}, do: {s, x}) == %{8 => 1, 16 => 515} + + # Unaligned + bin = <<8, 1, 32, 2, 3>> + assert for(<>, into: "", do: <>) == <<1>> + assert for(<>, into: %{}, do: {s, x}) == %{8 => 1} end test "binary for comprehensions where value is not used" do bin = <<1, 2, 3>> assert capture_io(fn -> - for(<>, do: IO.puts x) - nil - end) == "1\n2\n3\n" + for <>, do: IO.puts(x) + nil + end) == "1\n2\n3\n" + end + + test "binary for comprehensions with reduce, generators and filters" do + bin = <<1, 2, 3>> + + acc = + for <>, Integer.is_odd(x), <>, reduce: %{} do + acc -> Map.update(acc, x, [y], &[y | &1]) + end + + assert acc == %{1 => 'olleh', 3 => 'olleh'} end end diff --git a/lib/elixir/test/elixir/kernel/defaults_test.exs b/lib/elixir/test/elixir/kernel/defaults_test.exs new file mode 100644 index 00000000000..d8502658fb7 --- /dev/null +++ b/lib/elixir/test/elixir/kernel/defaults_test.exs @@ -0,0 +1,99 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.DefaultsTest do + use ExUnit.Case, async: true + + import ExUnit.CaptureIO + + def fun_with_block_defaults( + x, + y \\ ( + default = "y" + default + ), + z \\ ( + default = "z" + default + ) + ) do + {x, y, z} + end + + test "with block defaults" do + assert {1, "y", "z"} = fun_with_block_defaults(1) + assert {1, 2, "z"} = fun_with_block_defaults(1, 2) + assert {1, 2, 3} = fun_with_block_defaults(1, 2, 3) + end + + test "errors on accessing variable from default block" do + message = "variable \"default\" does not exist" + + assert capture_io(:stderr, fn -> + assert_raise CompileError, ~r/undefined function default\/0/, fn -> + defmodule VarDefaultScope do + def test(_ \\ default = 1), + do: default + end + end + end) =~ message + end + + test "errors on multiple defaults" do + message = ~r"def hello/1 defines defaults multiple times" + + assert_raise CompileError, message, fn -> + defmodule Kernel.ErrorsTest.ClauseWithDefaults do + def hello(_arg \\ 0) + def hello(_arg \\ 1) + end + end + + assert_raise CompileError, message, fn -> + defmodule Kernel.ErrorsTest.ClauseWithDefaults do + def hello(_arg \\ 0), do: nil + def hello(_arg \\ 1), do: nil + end + end + + assert_raise CompileError, message, fn -> + defmodule Kernel.ErrorsTest.ClauseWithDefaults do + def hello(_arg \\ 0) + def hello(_arg \\ 1), do: nil + end + end + + assert_raise CompileError, message, fn -> + defmodule Kernel.ErrorsTest.ClauseWithDefaults do + def hello(_arg \\ 0), do: nil + def hello(_arg \\ 1) + end + end + + assert capture_io(:stderr, fn -> + assert_raise CompileError, ~r"undefined function foo/0", fn -> + defmodule Kernel.ErrorsTest.ClauseWithDefaults5 do + def hello(foo, bar \\ foo) + def hello(foo, bar), do: foo + bar + end + end + end) =~ + "variable \"foo\" does not exist and is being expanded to \"foo()\", " <> + "please use parentheses to remove the ambiguity or change the variable name" + end + + test "errors on conflicting defaults" do + assert_raise CompileError, ~r"def hello/3 defaults conflicts with hello/2", fn -> + defmodule Kernel.ErrorsTest.DifferentDefsWithDefaults1 do + def hello(a, b \\ nil), do: a + b + def hello(a, b \\ nil, c \\ nil), do: a + b + c + end + end + + assert_raise CompileError, ~r"def hello/2 conflicts with defaults from hello/3", fn -> + defmodule Kernel.ErrorsTest.DifferentDefsWithDefaults2 do + def hello(a, b \\ nil, c \\ nil), do: a + b + c + def hello(a, b \\ nil), do: a + b + end + end + end +end diff --git a/lib/elixir/test/elixir/kernel/deprecated_test.exs b/lib/elixir/test/elixir/kernel/deprecated_test.exs new file mode 100644 index 00000000000..726c896f438 --- /dev/null +++ b/lib/elixir/test/elixir/kernel/deprecated_test.exs @@ -0,0 +1,45 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.DeprecatedTest do + use ExUnit.Case + + import PathHelpers + + test "raises on invalid @deprecated" do + assert_raise ArgumentError, ~r"should be a string with the reason", fn -> + defmodule InvalidDeprecated do + @deprecated 1.2 + def foo, do: :bar + end + end + end + + test "takes into account deprecated from defaults" do + defmodule DefaultDeprecated do + @deprecated "reason" + def foo(x \\ true), do: x + end + + assert DefaultDeprecated.__info__(:deprecated) == [ + {{:foo, 0}, "reason"}, + {{:foo, 1}, "reason"} + ] + end + + test "add deprecated to __info__" do + write_beam( + defmodule SampleDeprecated do + @deprecated "Use SampleDeprecated.bar/0 instead" + def foo, do: true + + def bar, do: false + end + ) + + deprecated = [ + {{:foo, 0}, "Use SampleDeprecated.bar/0 instead"} + ] + + assert SampleDeprecated.__info__(:deprecated) == deprecated + end +end diff --git a/lib/elixir/test/elixir/kernel/dialyzer_test.exs b/lib/elixir/test/elixir/kernel/dialyzer_test.exs new file mode 100644 index 00000000000..0796a584506 --- /dev/null +++ b/lib/elixir/test/elixir/kernel/dialyzer_test.exs @@ -0,0 +1,179 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.DialyzerTest do + use ExUnit.Case, async: true + + @moduletag :dialyzer + import PathHelpers + + setup_all do + dir = tmp_path("dialyzer") + File.rm_rf!(dir) + File.mkdir_p!(dir) + + plt = + dir + |> Path.join("base_plt") + |> String.to_charlist() + + # Some OSs (like Windows) do not provide the HOME environment variable. + unless System.get_env("HOME") do + System.put_env("HOME", System.user_home()) + end + + # Add a few key Elixir modules for types and macro functions + mods = [:elixir, :elixir_env, Atom, Enum, Exception, Kernel, Macro, Macro.Env, String] + files = Enum.map(mods, &:code.which/1) + dialyzer_run(analysis_type: :plt_build, output_plt: plt, apps: [:erts], files: files) + + # Compile Dialyzer fixtures + source_files = Path.wildcard(Path.join(fixture_path("dialyzer"), "*")) + {:ok, _, _} = Kernel.ParallelCompiler.compile_to_path(source_files, dir) + + {:ok, [base_dir: dir, base_plt: plt]} + end + + setup context do + dir = String.to_charlist(context.tmp_dir) + + plt = + dir + |> Path.join("plt") + |> String.to_charlist() + + File.cp!(context.base_plt, plt) + warnings = Map.get(context, :warnings, []) + + dialyzer = [ + analysis_type: :succ_typings, + check_plt: false, + files_rec: [dir], + plts: [plt], + warnings: warnings + ] + + {:ok, [outdir: dir, dialyzer: dialyzer]} + end + + @moduletag :tmp_dir + + @tag warnings: [:specdiffs] + test "no warnings on specdiffs", context do + copy_beam!(context, Dialyzer.RemoteCall) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on valid remote calls", context do + copy_beam!(context, Dialyzer.RemoteCall) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on rewrites", context do + copy_beam!(context, Dialyzer.Rewrite) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on raise", context do + copy_beam!(context, Dialyzer.Raise) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on macrocallback", context do + copy_beam!(context, Dialyzer.Macrocallback) + copy_beam!(context, Dialyzer.Macrocallback.Impl) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on callback", context do + copy_beam!(context, Dialyzer.Callback) + copy_beam!(context, Dialyzer.Callback.ImplAtom) + copy_beam!(context, Dialyzer.Callback.ImplList) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on struct update", context do + copy_beam!(context, Dialyzer.StructUpdate) + assert_dialyze_no_warnings!(context) + end + + @tag warnings: [:specdiffs] + test "no warnings on protocol calls with opaque types", context do + alias Dialyzer.ProtocolOpaque + + copy_beam!(context, ProtocolOpaque) + copy_beam!(context, ProtocolOpaque.Entity) + copy_beam!(context, ProtocolOpaque.Duck) + assert_dialyze_no_warnings!(context) + + # Also ensure no warnings after consolidation. + Code.prepend_path(context.base_dir) + {:ok, binary} = Protocol.consolidate(ProtocolOpaque.Entity, [ProtocolOpaque.Duck, Any]) + File.write!(Path.join(context.outdir, "#{ProtocolOpaque.Entity}.beam"), binary) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on and/2 and or/2", context do + copy_beam!(context, Dialyzer.BooleanCheck) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on cond", context do + copy_beam!(context, Dialyzer.Cond) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on for comprehensions with bitstrings", context do + copy_beam!(context, Dialyzer.ForBitstring) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on for falsy check that always boolean", context do + copy_beam!(context, Dialyzer.ForBooleanCheck) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on with/else", context do + copy_beam!(context, Dialyzer.With) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on defmacrop", context do + copy_beam!(context, Dialyzer.Defmacrop) + assert_dialyze_no_warnings!(context) + end + + test "no warnings on try", context do + copy_beam!(context, Dialyzer.Try) + assert_dialyze_no_warnings!(context) + end + + test "no warning on is_struct/2", context do + copy_beam!(context, Dialyzer.IsStruct) + assert_dialyze_no_warnings!(context) + end + + defp copy_beam!(context, module) do + name = "#{module}.beam" + File.cp!(Path.join(context.base_dir, name), Path.join(context.outdir, name)) + end + + defp assert_dialyze_no_warnings!(context) do + case dialyzer_run(context.dialyzer) do + [] -> + :ok + + warnings -> + formatted = for warn <- warnings, do: [:dialyzer.format_warning(warn), ?\n] + formatted |> IO.chardata_to_string() |> flunk() + end + end + + defp dialyzer_run(opts) do + try do + :dialyzer.run(opts) + catch + :throw, {:dialyzer_error, chardata} -> + raise "dialyzer error: " <> IO.chardata_to_string(chardata) + end + end +end diff --git a/lib/elixir/test/elixir/kernel/docs_test.exs b/lib/elixir/test/elixir/kernel/docs_test.exs index 1b0d55a4986..576613aa001 100644 --- a/lib/elixir/test/elixir/kernel/docs_test.exs +++ b/lib/elixir/test/elixir/kernel/docs_test.exs @@ -1,60 +1,384 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.DocsTest do use ExUnit.Case - test "compiled with docs" do - deftestmodule(SampleDocs) - docs = Code.get_docs(SampleDocs, :all) + import PathHelpers - assert [{{:fun, 2}, _, :def, [{:x, [], nil}, {:y, [], nil}], "This is fun!\n"}, - {{:nofun, 0}, _, :def, [], nil}, - {{:sneaky, 1}, _, :def, [{:bool1, [], Elixir}], false}] = docs[:docs] - assert {_, "Hello, I am a module"} = docs[:moduledoc] + defmacro wrong_doc_baz do + quote do + @doc "Wrong doc" + @doc since: "1.2" + def baz(_arg) + def baz(arg), do: arg + 1 + end + end + + test "attributes format" do + defmodule DocAttributes do + @moduledoc "Module doc" + assert @moduledoc == "Module doc" + assert Module.get_attribute(__MODULE__, :moduledoc) == {__ENV__.line - 2, "Module doc"} + + @typedoc "Type doc" + assert @typedoc == "Type doc" + assert Module.get_attribute(__MODULE__, :typedoc) == {__ENV__.line - 2, "Type doc"} + @type foobar :: any + + @doc "Function doc" + assert @doc == "Function doc" + assert Module.get_attribute(__MODULE__, :doc) == {__ENV__.line - 2, "Function doc"} + + def foobar() do + :ok + end + end end test "compiled without docs" do Code.compiler_options(docs: false) - deftestmodule(SampleNoDocs) + write_beam( + defmodule WithoutDocs do + @moduledoc "Module doc" + + @doc "Some doc" + def foobar(arg), do: arg + end + ) - assert Code.get_docs(SampleNoDocs, :docs) == nil - assert Code.get_docs(SampleNoDocs, :moduledoc) == nil + assert Code.fetch_docs(WithoutDocs) == {:error, :chunk_not_found} after Code.compiler_options(docs: true) end test "compiled in memory does not have accessible docs" do - defmodule NoDocs do - @moduledoc "moduledoc" + defmodule InMemoryDocs do + @moduledoc "Module doc" + + @doc "Some doc" + def foobar(arg), do: arg + end + + assert Code.fetch_docs(InMemoryDocs) == {:error, :module_not_found} + end + + test "non-existent beam file" do + assert {:error, :module_not_found} = Code.fetch_docs("bad.beam") + end + + test "raises on invalid @doc since: ..." do + assert_raise ArgumentError, ~r"should be a string representing the version", fn -> + defmodule InvalidSince do + @doc since: 1.2 + def foo, do: :bar + end + end + end + + test "raises on invalid @doc" do + assert_raise ArgumentError, ~r/When set dynamically, it should be {line, doc}/, fn -> + defmodule DocAttributesFormat do + Module.put_attribute(__MODULE__, :moduledoc, "Other") + end + end + + message = ~r/should be either false, nil, a string, or a keyword list/ - @doc "Some example" - def example(var), do: var + assert_raise ArgumentError, message, fn -> + defmodule AtSyntaxDocAttributesFormat do + @moduledoc :not_a_binary + end + end + + assert_raise ArgumentError, message, fn -> + defmodule AtSyntaxDocAttributesFormat do + @moduledoc true + end + end + end + + describe "compiled with docs" do + test "infers signatures" do + write_beam( + defmodule SignatureDocs do + def arg_names([], [], %{}, [], %{}), do: false + + @year 2015 + def with_defaults(@year, arg \\ 0, year \\ @year, fun \\ &>=/2) do + {fun, arg + year} + end + + def with_map_and_default(%{key: value} \\ %{key: :default}), do: value + def with_struct(%URI{}), do: :ok + + def with_underscore({_, _} = _two_tuple), do: :ok + def with_underscore(_), do: :error + + def only_underscore(_), do: :ok + + def two_good_names(first, :ok), do: first + def two_good_names(second, :error), do: second + + def really_long_signature( + really_long_var_named_one, + really_long_var_named_two, + really_long_var_named_three + ) do + {really_long_var_named_one, really_long_var_named_two, really_long_var_named_three} + end + end + ) + + assert {:docs_v1, _, :elixir, _, _, _, docs} = Code.fetch_docs(SignatureDocs) + signatures = for {{:function, n, a}, _, signature, _, %{}} <- docs, do: {{n, a}, signature} + + assert [ + arg_names, + only_underscore, + really_long_signature, + two_good_names, + with_defaults, + with_map_and_default, + with_struct, + with_underscore + ] = Enum.sort(signatures) + + # arg_names/5 + assert {{:arg_names, 5}, ["arg_names(list1, list2, map1, list3, map2)"]} = arg_names + + # only_underscore/1 + assert {{:only_underscore, 1}, ["only_underscore(_)"]} = only_underscore + + # really_long_signature/3 + assert {{:really_long_signature, 3}, + [ + "really_long_signature(really_long_var_named_one, really_long_var_named_two, really_long_var_named_three)" + ]} = really_long_signature + + # two_good_names/2 + assert {{:two_good_names, 2}, ["two_good_names(first, atom)"]} = two_good_names + + # with_defaults/4 + assert {{:with_defaults, 4}, + ["with_defaults(int, arg \\\\ 0, year \\\\ 2015, fun \\\\ &>=/2)"]} = with_defaults + + # with_map_and_default/1 + assert {{:with_map_and_default, 1}, ["with_map_and_default(map \\\\ %{key: :default})"]} = + with_map_and_default + + # with_struct/1 + assert {{:with_struct, 1}, ["with_struct(uri)"]} = with_struct + + # with_underscore/1 + assert {{:with_underscore, 1}, ["with_underscore(two_tuple)"]} = with_underscore end - assert Code.get_docs(NoDocs, :docs) == nil - assert Code.get_docs(NoDocs, :moduledoc) == nil + test "includes docs for functions, modules, types and callbacks" do + write_beam( + defmodule SampleDocs do + @moduledoc "Module doc" + @moduledoc authors: "Elixir Contributors", purpose: :test + + @doc "My struct" + defstruct [:sample] + + @typedoc "Type doc" + @typedoc since: "1.2.3", color: :red + @type foo(any) :: any + + @typedoc "Opaque type doc" + @opaque bar(any) :: any + + @doc "Callback doc" + @doc since: "1.2.3", color: :red, deprecated: "use baz/2 instead" + @doc color: :blue, stable: true + @callback foo(any) :: any + + @doc false + @doc since: "1.2.3" + @callback bar() :: term + @callback baz(any, term) :: any + + @doc "Callback with multiple clauses" + @callback callback_multi(integer) :: integer + @callback callback_multi(atom) :: atom + + @doc "Macrocallback doc" + @macrocallback qux(any) :: any + + @doc "Macrocallback with multiple clauses" + @macrocallback macrocallback_multi(integer) :: integer + @macrocallback macrocallback_multi(atom) :: atom + + @doc "Function doc" + @doc since: "1.2.3", color: :red + @doc color: :blue, stable: true + @deprecated "use baz/2 instead" + def foo(arg \\ 0), do: arg + 1 + + @doc "Multiple function head doc" + @deprecated "something else" + def bar(_arg) + def bar(arg), do: arg + 1 + + require Kernel.DocsTest + Kernel.DocsTest.wrong_doc_baz() + + @doc "Multiple function head and docs" + @doc since: "1.2.3" + def baz(_arg) + + @doc false + def qux(true), do: false + + @doc "A guard" + defguard is_zero(v) when v == 0 + + # We do this to avoid the deprecation warning. + module = Module + module.add_doc(__MODULE__, __ENV__.line, :def, {:nullary, 0}, [], "add_doc") + def nullary, do: 0 + end + ) + + assert {:docs_v1, _, :elixir, "text/markdown", %{"en" => module_doc}, module_doc_meta, docs} = + Code.fetch_docs(SampleDocs) + + assert module_doc == "Module doc" + + assert %{authors: "Elixir Contributors", purpose: :test} = module_doc_meta + + [ + callback_bar, + callback_baz, + callback_multi, + callback_foo, + function_struct_0, + function_struct_1, + function_bar, + function_baz, + function_foo, + function_nullary, + function_qux, + guard_is_zero, + macrocallback_multi, + macrocallback_qux, + type_bar, + type_foo + ] = Enum.sort(docs) + + assert {{:callback, :bar, 0}, _, [], :hidden, %{}} = callback_bar + assert {{:callback, :baz, 2}, _, [], :none, %{}} = callback_baz + + assert {{:callback, :foo, 1}, _, [], %{"en" => "Callback doc"}, + %{since: "1.2.3", deprecated: "use baz/2 instead", color: :blue, stable: true}} = + callback_foo + + assert {{:callback, :callback_multi, 1}, _, [], %{"en" => "Callback with multiple clauses"}, + %{}} = callback_multi + + assert {{:function, :__struct__, 0}, _, ["%Kernel.DocsTest.SampleDocs{}"], + %{"en" => "My struct"}, %{}} = function_struct_0 + + assert {{:function, :__struct__, 1}, _, ["__struct__(kv)"], :hidden, %{}} = + function_struct_1 + + assert {{:function, :bar, 1}, _, ["bar(arg)"], %{"en" => "Multiple function head doc"}, + %{deprecated: "something else"}} = function_bar + + assert {{:function, :baz, 1}, _, ["baz(arg)"], %{"en" => "Multiple function head and docs"}, + %{since: "1.2.3"}} = function_baz + + assert {{:function, :foo, 1}, _, ["foo(arg \\\\ 0)"], %{"en" => "Function doc"}, + %{ + since: "1.2.3", + deprecated: "use baz/2 instead", + color: :blue, + stable: true, + defaults: 1 + }} = function_foo + + assert {{:function, :nullary, 0}, _, ["nullary()"], %{"en" => "add_doc"}, %{}} = + function_nullary + + assert {{:function, :qux, 1}, _, ["qux(bool)"], :hidden, %{}} = function_qux + + assert {{:macro, :is_zero, 1}, _, ["is_zero(v)"], %{"en" => "A guard"}, %{guard: true}} = + guard_is_zero + + assert {{:macrocallback, :macrocallback_multi, 1}, _, [], + %{"en" => "Macrocallback with multiple clauses"}, %{}} = macrocallback_multi + + assert {{:macrocallback, :qux, 1}, _, [], %{"en" => "Macrocallback doc"}, %{}} = + macrocallback_qux + + assert {{:type, :bar, 1}, _, [], %{"en" => "Opaque type doc"}, %{opaque: true}} = type_bar + + assert {{:type, :foo, 1}, _, [], %{"en" => "Type doc"}, %{since: "1.2.3", color: :red}} = + type_foo + end end - defp deftestmodule(name) do - import PathHelpers + test "fetch docs chunk from doc/chunks" do + Code.compiler_options(docs: false) - write_beam(defmodule name do - @moduledoc "Hello, I am a module" + doc_chunks_path = Path.join([tmp_path(), "doc", "chunks"]) + File.rm_rf!(doc_chunks_path) + File.mkdir_p!(doc_chunks_path) - @doc """ - This is fun! - """ - def fun(x, y) do - {x, y} + write_beam( + defmodule ExternalDocs do end + ) + + assert Code.fetch_docs(ExternalDocs) == {:error, :chunk_not_found} - @doc false - def sneaky(true), do: false + path = Path.join([doc_chunks_path, "#{ExternalDocs}.chunk"]) + chunk = {:docs_v1, 1, :elixir, "text/markdown", %{"en" => "Some docs"}, %{}} + File.write!(path, :erlang.term_to_binary(chunk)) - def nofun() do - 'not fun at all' + assert Code.fetch_docs(ExternalDocs) == chunk + after + Code.compiler_options(docs: true) + end + + test "@impl true doesn't set @doc false if previous implementation has docs" do + write_beam( + defmodule Docs do + defmodule SampleBehaviour do + @callback foo(any()) :: any() + @callback bar() :: any() + @callback baz() :: any() + end + + @behaviour SampleBehaviour + + @doc "Foo docs" + def foo(nil), do: nil + + @impl true + def foo(_), do: false + + @impl true + def bar(), do: true + + @doc "Baz docs" + @impl true + def baz(), do: true + + def fuz(), do: true end - end) + ) + + {:docs_v1, _, _, _, _, _, docs} = Code.fetch_docs(Docs) + function_docs = for {{:function, name, arity}, _, _, doc, _} <- docs, do: {{name, arity}, doc} + + assert [ + {{:bar, 0}, :hidden}, + {{:baz, 0}, %{"en" => "Baz docs"}}, + {{:foo, 1}, %{"en" => "Foo docs"}}, + {{:fuz, 0}, :none} + ] = Enum.sort(function_docs) end end diff --git a/lib/elixir/test/elixir/kernel/errors_test.exs b/lib/elixir/test/elixir/kernel/errors_test.exs index 5e5fa194a7a..d339db0ddb0 100644 --- a/lib/elixir/test/elixir/kernel/errors_test.exs +++ b/lib/elixir/test/elixir/kernel/errors_test.exs @@ -1,701 +1,869 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.ErrorsTest do use ExUnit.Case, async: true - import CompileAssertion - defmodule UnproperMacro do - defmacro unproper(args), do: args - defmacro exit(args), do: args - end - - test :invalid_token do - assert_compile_fail SyntaxError, - "nofile:1: invalid token: \end", - '\end\nlol\nbarbecue' - end - - test :invalid_quoted_token do - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: \"world\"", - '"hello" "world"' - - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: foo", - 'Foo.:foo' - - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: \"foo\"", - 'Foo.:"foo\#{:bar}"' - - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: \"", - 'Foo.:"\#{:bar}"' - end - - test :invalid_or_reserved_codepoint do - assert_compile_fail ArgumentError, - "invalid or reserved unicode codepoint 55296", - '?\\x{D800}' - end - - test :sigil_terminator do - assert_compile_fail TokenMissingError, - "nofile:3: missing terminator: \" (for sigil ~r\" starting at line 1)", - '~r"foo\n\n' - - assert_compile_fail TokenMissingError, - "nofile:3: missing terminator: } (for sigil ~r{ starting at line 1)", - '~r{foo\n\n' - end - - test :dot_terminator do - assert_compile_fail TokenMissingError, - "nofile:1: missing terminator: \" (for function name starting at line 1)", - 'foo."bar' - end - - test :string_terminator do - assert_compile_fail TokenMissingError, - "nofile:1: missing terminator: \" (for string starting at line 1)", - '"bar' - end + import ExUnit.CaptureIO - test :heredoc_start do - assert_compile_fail SyntaxError, - "nofile:1: heredoc start must be followed by a new line after \"\"\"", - '"""bar\n"""' - end - - test :heredoc_terminator do - assert_compile_fail TokenMissingError, - "nofile:2: missing terminator: \"\"\" (for heredoc starting at line 1)", - '"""\nbar' + defmacro hello do + quote location: :keep do + def hello, do: :world + end end - test :unexpected_end do - assert_compile_fail SyntaxError, - "nofile:1: unexpected token: end", - '1 end' + test "no default arguments in fn" do + assert_eval_raise CompileError, + "nofile:1: anonymous functions cannot have optional arguments", + 'fn x \\\\ 1 -> x end' + + assert_eval_raise CompileError, + "nofile:1: anonymous functions cannot have optional arguments", + 'fn x, y \\\\ 1 -> x + y end' + end + + test "invalid __CALLER__" do + assert_eval_raise CompileError, + "nofile:1: __CALLER__ is available only inside defmacro and defmacrop", + 'defmodule Sample do def hello do __CALLER__ end end' + end + + test "invalid __STACKTRACE__" do + assert_eval_raise CompileError, + "nofile:1: __STACKTRACE__ is available only inside catch and rescue clauses of try expressions", + 'defmodule Sample do def hello do __STACKTRACE__ end end' + + assert_eval_raise CompileError, + "nofile:1: __STACKTRACE__ is available only inside catch and rescue clauses of try expressions", + 'defmodule Sample do try do raise "oops" rescue _ -> def hello do __STACKTRACE__ end end end' + end + + test "undefined function" do + assert_eval_raise CompileError, + "hello.ex:4: undefined function bar/0 (expected Kernel.ErrorsTest.BadForm to define such a function or for it to be imported, but none are available)", + ''' + defmodule Kernel.ErrorsTest.BadForm do + @file "hello.ex" + def foo do + bar() + end + end + ''' + + assert_eval_raise CompileError, + "nofile:2: undefined function module_info/0 (this function is auto-generated by the compiler and must always be called as a remote, as in __MODULE__.module_info/0)", + ''' + defmodule Kernel.ErrorsTest.Info do + def foo, do: module_info() + end + ''' + + assert_eval_raise CompileError, + "nofile:3: undefined function behaviour_info/1 (this function is auto-generated by the compiler and must always be called as a remote, as in __MODULE__.behaviour_info/1)", + ''' + defmodule Kernel.ErrorsTest.BehaviourInfo do + @callback dummy() :: :ok + def foo, do: behaviour_info(:callbacks) + end + ''' + + assert capture_io(:stderr, fn -> + assert_eval_raise CompileError, + "nofile:3: undefined function bar/1 (expected Kernel.ErrorsTest.BadForm to define such a function or for it to be imported, but none are available)", + ''' + defmodule Kernel.ErrorsTest.BadForm do + def foo do + bar( + baz(1, 2) + ) + end + end + ''' + end) =~ "undefined function baz/2" + + assert_eval_raise CompileError, + "nofile:8: undefined function baz/0 (expected Sample to define such a function or for it to be imported, but none are available)", + ''' + defmodule Sample do + def foo do + bar() + end + + defoverridable [foo: 0] + def foo do + baz() + end + end + ''' + end + + test "undefined non-local function" do + assert_eval_raise CompileError, + "nofile:1: undefined function call/2 (there is no such import)", + 'call foo, do: :foo' + end + + test "function without definition" do + assert_eval_raise CompileError, + "nofile:2: implementation not provided for predefined def foo/0", + ''' + defmodule Kernel.ErrorsTest.FunctionWithoutDefition do + def foo + end + ''' + + assert_eval_raise CompileError, + "nofile:10: implementation not provided for predefined def example/2", + ''' + defmodule Kernel.ErrorsTest.FunctionTemplate do + defmacro __using__(_) do + quote do + def example(foo, bar \\\\ []) + end + end + end + + defmodule Kernel.ErrorsTest.UseFunctionTemplate do + use Kernel.ErrorsTest.FunctionTemplate + end + ''' + end + + test "guard without definition" do + assert_eval_raise CompileError, + "nofile:2: implementation not provided for predefined defmacro foo/1", + ''' + defmodule Kernel.ErrorsTest.GuardWithoutDefition do + defguard foo(bar) + end + ''' + end + + test "literal on map and struct" do + assert_eval_raise CompileError, + "nofile:1: expected key-value pairs in a map, got: put_in(foo.bar.baz, nil)", + 'foo = 1; %{put_in(foo.bar.baz, nil), foo}' + end + + test "struct fields on defstruct" do + assert_eval_raise ArgumentError, "struct field names must be atoms, got: 1", ''' + defmodule Kernel.ErrorsTest.StructFieldsOnDefstruct do + defstruct [1, 2, 3] + end + ''' end - test :syntax_error do - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: '.'", - '+.foo' + test "struct access on body" do + assert_eval_raise CompileError, + "nofile:3: cannot access struct Kernel.ErrorsTest.StructAccessOnBody, " <> + "the struct was not yet defined or the struct " <> + "is being accessed in the same context that defines it", + ''' + defmodule Kernel.ErrorsTest.StructAccessOnBody do + defstruct %{name: "Brasilia"} + %Kernel.ErrorsTest.StructAccessOnBody{} + end + ''' end - test :compile_error_on_op_ambiguity do - msg = "nofile:1: \"a -1\" looks like a function call but there is a variable named \"a\", " <> - "please use explicit parenthesis or even spaces" - assert_compile_fail CompileError, msg, 'a = 1; a -1' - - max = 1 - assert max == 1 - assert (max 1, 2) == 2 - end + describe "struct errors" do + test "bad errors" do + assert_eval_raise CompileError, + ~r"nofile:1: BadStruct.__struct__/1 is undefined, cannot expand struct BadStruct", + '%BadStruct{}' - test :syntax_error_on_parens_call do - msg = "nofile:1: unexpected parenthesis. If you are making a function call, do not " <> - "insert spaces in between the function name and the opening parentheses. " <> - "Syntax error before: '('" + assert_eval_raise CompileError, + ~r"nofile:1: BadStruct.__struct__/0 is undefined, cannot expand struct BadStruct", + '%BadStruct{} = %{}' - assert_compile_fail SyntaxError, msg, 'foo (hello, world)' - assert_compile_fail SyntaxError, msg, 'foo ()' - assert_compile_fail SyntaxError, msg, 'foo (), 1' - end + bad_struct_type_error = + ~r"expected Kernel.ErrorsTest.BadStructType.__struct__/(0|1) to return a map.*, got: :invalid" - test :syntax_error_on_nested_no_parens_call do - msg = "nofile:1: unexpected comma. Parentheses are required to solve ambiguity in " <> - "nested calls. Syntax error before: ','" + defmodule BadStructType do + def __struct__, do: :invalid + def __struct__(_), do: :invalid - assert_compile_fail SyntaxError, msg, '[foo 1, 2]' - assert_compile_fail SyntaxError, msg, '[do: foo 1, 2]' - assert_compile_fail SyntaxError, msg, 'foo(do: bar 1, 2)' - assert_compile_fail SyntaxError, msg, '{foo 1, 2}' - assert_compile_fail SyntaxError, msg, 'foo 1, foo 2, 3' - assert_compile_fail SyntaxError, msg, 'foo(1, foo 2, 3)' + assert_raise CompileError, bad_struct_type_error, fn -> + Macro.struct!(__MODULE__, __ENV__) + end + end - assert is_list List.flatten [1] - assert is_list Enum.reverse [3, 2, 1], [4, 5, 6] - assert is_list(Enum.reverse [3, 2, 1], [4, 5, 6]) - end + assert_eval_raise CompileError, + bad_struct_type_error, + '%#{BadStructType}{} = %{}' - test :syntax_error_with_no_token do - assert_compile_fail TokenMissingError, - "nofile:1: missing terminator: ) (for \"(\" starting at line 1)", - 'case 1 (' - end + assert_eval_raise CompileError, + bad_struct_type_error, + '%#{BadStructType}{}' - test :clause_with_defaults do - assert_compile_fail CompileError, - "nofile:3: def hello/1 has default values and multiple clauses, " <> - "define a function head with the defaults", - ~C''' - defmodule ErrorsTest do - def hello(arg \\ 0), do: nil - def hello(arg \\ 1), do: nil + assert_raise ArgumentError, bad_struct_type_error, fn -> + struct(BadStructType) end - ''' - end - test :invalid_match_pattern do - assert_compile_fail CompileError, - "nofile:2: invalid expression in match", - ''' - case true do - true && true -> true + assert_raise ArgumentError, bad_struct_type_error, fn -> + struct(BadStructType, foo: 1) + end end - ''' - end - test :different_defs_with_defaults do - assert_compile_fail CompileError, - "nofile:3: def hello/3 defaults conflicts with def hello/2", - ~C''' - defmodule ErrorsTest do - def hello(a, b \\ nil), do: a + b - def hello(a, b \\ nil, c \\ nil), do: a + b + c - end - ''' - - assert_compile_fail CompileError, - "nofile:3: def hello/2 conflicts with defaults from def hello/3", - ~C''' - defmodule ErrorsTest do - def hello(a, b \\ nil, c \\ nil), do: a + b + c - def hello(a, b \\ nil), do: a + b - end - ''' - end + test "missing struct key" do + missing_struct_key_error = + ~r"expected Kernel.ErrorsTest.MissingStructKey.__struct__/(0|1) to return a map.*, got: %\{\}" - test :bad_form do - assert_compile_fail CompileError, - "nofile:2: function bar/0 undefined", - ''' - defmodule ErrorsTest do - def foo, do: bar + defmodule MissingStructKey do + def __struct__, do: %{} + def __struct__(_), do: %{} + + assert_raise CompileError, missing_struct_key_error, fn -> + Macro.struct!(__MODULE__, __ENV__) + end end - ''' - end - test :unbound_var do - assert_compile_fail CompileError, - "nofile:1: unbound variable ^x", - '^x = 1' - end + assert_eval_raise CompileError, + missing_struct_key_error, + '%#{MissingStructKey}{} = %{}' - test :unbound_not_match do - assert_compile_fail CompileError, - "nofile:1: cannot use ^x outside of match clauses", - '^x' - end + assert_eval_raise CompileError, + missing_struct_key_error, + '%#{MissingStructKey}{}' - test :unbound_expr do - assert_compile_fail CompileError, - "nofile:1: invalid argument for unary operator ^, expected an existing variable, got: ^x(1)", - '^x(1) = 1' - end + assert_raise ArgumentError, missing_struct_key_error, fn -> + struct(MissingStructKey) + end - test :literal_on_map_and_struct do - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: '}'", - '%{{:a, :b}}' + assert_raise ArgumentError, missing_struct_key_error, fn -> + struct(MissingStructKey, foo: 1) + end - assert_compile_fail SyntaxError, - "nofile:1: syntax error before: '{'", - '%{:a, :b}{a: :b}' - end + invalid_struct_key_error = + ~r"expected Kernel.ErrorsTest.InvalidStructKey.__struct__/(0|1) to return a map.*, got: %\{__struct__: 1\}" - test :struct_fields_on_defstruct do - assert_compile_fail ArgumentError, - "struct field names must be atoms, got: 1", - ''' - defmodule TZ do - defstruct [1, 2, 3] - end - ''' - end + defmodule InvalidStructKey do + def __struct__, do: %{__struct__: 1} + def __struct__(_), do: %{__struct__: 1} - test :struct_access_on_body do - assert_compile_fail CompileError, - "nofile:3: cannot access struct TZ in body of the module that defines it " <> - "as the struct fields are not yet accessible", - ''' - defmodule TZ do - defstruct %{name: "Brasilia"} - %TZ{} + assert_raise CompileError, invalid_struct_key_error, fn -> + Macro.struct!(__MODULE__, __ENV__) + end end - ''' - end - - test :unbound_map_key_var do - assert_compile_fail CompileError, - "nofile:1: illegal use of variable x in map key", - '%{x => 1} = %{}' - assert_compile_fail CompileError, - "nofile:1: illegal use of variable x in map key", - '%{x = 1 => 1}' - end + assert_eval_raise CompileError, + invalid_struct_key_error, + '%#{InvalidStructKey}{} = %{}' - test :struct_errors do - assert_compile_fail CompileError, - "nofile:1: BadStruct.__struct__/0 is undefined, cannot expand struct BadStruct", - '%BadStruct{}' + assert_eval_raise CompileError, + invalid_struct_key_error, + '%#{InvalidStructKey}{}' - defmodule BadStruct do - def __struct__ do - [] + assert_raise ArgumentError, invalid_struct_key_error, fn -> + struct(InvalidStructKey) end - end - assert_compile_fail CompileError, - "nofile:1: expected Kernel.ErrorsTest.BadStruct.__struct__/0 to return a map, got: []", - '%#{BadStruct}{}' - - defmodule GoodStruct do - def __struct__ do - %{name: "josé"} + assert_raise ArgumentError, invalid_struct_key_error, fn -> + struct(InvalidStructKey, foo: 1) end end - assert_compile_fail CompileError, - "nofile:1: unknown key :age for struct Kernel.ErrorsTest.GoodStruct", - '%#{GoodStruct}{age: 27}' - end - - test :name_for_defmodule do - assert_compile_fail CompileError, - "nofile:1: invalid module name: 3", - 'defmodule 1 + 2, do: 3' - end - - test :invalid_unquote do - assert_compile_fail CompileError, - "nofile:1: unquote called outside quote", - 'unquote 1' - end - - test :invalid_quote_args do - assert_compile_fail CompileError, - "nofile:1: invalid arguments for quote", - 'quote 1' - end - - test :invalid_calls do - assert_compile_fail CompileError, - "nofile:1: invalid call foo(1)(2)", - 'foo(1)(2)' + test "invalid struct" do + invalid_struct_name_error = + ~r"expected struct name returned by Kernel.ErrorsTest.InvalidStructName.__struct__/(0|1) to be Kernel.ErrorsTest.InvalidStructName, got: InvalidName" - assert_compile_fail CompileError, - "nofile:1: invalid call 1.foo()", - '1.foo' - end + defmodule InvalidStructName do + def __struct__, do: %{__struct__: InvalidName} + def __struct__(_), do: %{__struct__: InvalidName} - test :unhandled_stab do - assert_compile_fail CompileError, - "nofile:3: unhandled operator ->", - ''' - defmodule Mod do - def fun do - casea foo, do: (bar -> baz) + assert_raise CompileError, invalid_struct_name_error, fn -> + Macro.struct!(__MODULE__, __ENV__) end end - ''' - end - test :undefined_non_local_function do - assert_compile_fail CompileError, - "nofile:1: undefined function casea/2", - 'casea foo, do: 1' - end + assert_eval_raise CompileError, + invalid_struct_name_error, + '%#{InvalidStructName}{} = %{}' - test :invalid_fn_args do - assert_compile_fail TokenMissingError, - "nofile:1: missing terminator: end (for \"fn\" starting at line 1)", - 'fn 1' - end + assert_eval_raise CompileError, + invalid_struct_name_error, + '%#{InvalidStructName}{}' - test :function_local_conflict do - assert_compile_fail CompileError, - "nofile:1: imported Kernel.&&/2 conflicts with local function", - ''' - defmodule ErrorsTest do - def other, do: 1 && 2 - def _ && _, do: :error + assert_raise ArgumentError, invalid_struct_name_error, fn -> + struct(InvalidStructName) end - ''' - end - test :macro_local_conflict do - assert_compile_fail CompileError, - "nofile:6: call to local macro &&/2 conflicts with imported Kernel.&&/2, " <> - "please rename the local macro or remove the conflicting import", - ''' - defmodule ErrorsTest do - def hello, do: 1 || 2 - defmacro _ || _, do: :ok - - defmacro _ && _, do: :error - def world, do: 1 && 2 + assert_raise ArgumentError, invalid_struct_name_error, fn -> + struct(InvalidStructName, foo: 1) end - ''' - end + end - test :macro_with_undefined_local do - assert_compile_fail UndefinedFunctionError, - "undefined function: ErrorsTest.unknown/1", - ''' - defmodule ErrorsTest do - defmacrop bar, do: unknown(1) - def baz, do: bar() + test "good struct" do + defmodule GoodStruct do + defstruct name: "john" end - ''' - end - test :private_macro do - assert_compile_fail UndefinedFunctionError, - "undefined function: ErrorsTest.foo/0", - ''' - defmodule ErrorsTest do - defmacrop foo, do: 1 - defmacro bar, do: __MODULE__.foo - defmacro baz, do: bar - end - ''' - end + assert_eval_raise KeyError, + "key :age not found", + '%#{GoodStruct}{age: 27}' - test :function_definition_with_alias do - assert_compile_fail CompileError, - "nofile:2: function names should start with lowercase characters or underscore, invalid name Bar", - ''' - defmodule ErrorsTest do - def Bar do - :baz - end - end - ''' - end + assert_eval_raise CompileError, + "nofile:1: unknown key :age for struct Kernel.ErrorsTest.GoodStruct", + '%#{GoodStruct}{age: 27} = %{}' + end - test :function_import_conflict do - assert_compile_fail CompileError, - "nofile:3: function exit/1 imported from both :erlang and Kernel, call is ambiguous", - ''' - defmodule ErrorsTest do - import :erlang, warn: false - def foo, do: exit(:test) + test "enforce @enforce_keys" do + defmodule EnforceKeys do + @enforce_keys [:foo] + defstruct(foo: nil) end - ''' - end - - test :import_invalid_macro do - assert_compile_fail CompileError, - "nofile:1: cannot import Kernel.invalid/1 because it doesn't exist", - 'import Kernel, only: [invalid: 1]' - end - test :unrequired_macro do - assert_compile_fail SyntaxError, - "nofile:2: you must require Kernel.ErrorsTest.UnproperMacro before invoking " <> - "the macro Kernel.ErrorsTest.UnproperMacro.unproper/1 " - ''' - defmodule ErrorsTest do - Kernel.ErrorsTest.UnproperMacro.unproper([]) - end - ''' + assert_raise ArgumentError, + "@enforce_keys required keys ([:fo, :bar]) that are not defined in defstruct: [foo: nil]", + fn -> + defmodule EnforceKeysError do + @enforce_keys [:foo, :fo, :bar] + defstruct(foo: nil) + end + end + end end - test :def_defmacro_clause_change do - assert_compile_fail CompileError, - "nofile:3: defmacro foo/1 already defined as def", - ''' - defmodule ErrorsTest do - def foo(1), do: 1 - defmacro foo(x), do: x - end - ''' + test "invalid unquote" do + assert_eval_raise CompileError, "nofile:1: unquote called outside quote", 'unquote 1' end - test :internal_function_overridden do - assert_compile_fail CompileError, - "nofile:1: function __info__/1 is internal and should not be overridden", - ''' - defmodule ErrorsTest do - def __info__(_), do: [] - end - ''' - end + test "invalid unquote splicing in one-liners" do + assert_eval_raise ArgumentError, + "unquote_splicing only works inside arguments and block contexts, " <> + "wrap it in parens if you want it to work with one-liners", + ''' + defmodule Kernel.ErrorsTest.InvalidUnquoteSplicingInOneliners do + defmacro oneliner2 do + quote do: unquote_splicing 1 + end - test :no_macros do - assert_compile_fail CompileError, - "nofile:2: could not load macros from module :lists", - ''' - defmodule ErrorsTest do - import :lists, only: :macros - end - ''' + def callme do + oneliner2 + end + end + ''' end - test :invalid_macro do - assert_compile_fail CompileError, - "nofile: invalid quoted expression: {:foo, :bar, :baz, :bat}", - ''' - defmodule ErrorsTest do - defmacrop oops do - {:foo, :bar, :baz, :bat} - end + test "invalid attribute" do + msg = ~r"cannot inject attribute @foo into function/macro because cannot escape " - def test, do: oops + assert_raise ArgumentError, msg, fn -> + defmodule InvalidAttribute do + @foo fn -> nil end + def bar, do: @foo end - ''' + end end - test :unloaded_module do - assert_compile_fail CompileError, - "nofile:1: module Certainly.Doesnt.Exist is not loaded and could not be found", - 'import Certainly.Doesnt.Exist' - end + test "typespec attributes set via Module.put_attribute/4" do + message = + "attributes type, typep, opaque, spec, callback, and macrocallback " <> + "must be set directly via the @ notation" - test :scheduled_module do - assert_compile_fail CompileError, - "nofile:4: module ErrorsTest.Hygiene is not loaded but was defined. " <> - "This happens because you are trying to use a module in the same context it is defined. " <> - "Try defining the module outside the context that requires it.", - ''' - defmodule ErrorsTest do - defmodule Hygiene do - end - import ErrorsTest.Hygiene - end - ''' + for kind <- [:type, :typep, :opaque, :spec, :callback, :macrocallback] do + assert_eval_raise ArgumentError, + message, + """ + defmodule PutTypespecAttribute do + Module.put_attribute(__MODULE__, #{inspect(kind)}, {}) + end + """ + end end - test :already_compiled_module do - assert_compile_fail ArgumentError, - "could not call eval_quoted on module Record " <> - "because it was already compiled", - 'Module.eval_quoted Record, quote(do: 1), [], file: __ENV__.file' - end + test "invalid struct field value" do + msg = ~r"invalid value for struct field baz, cannot escape " - test :interpolation_error do - assert_compile_fail SyntaxError, - "nofile:1: \"do\" starting at line 1 is missing terminator \"end\". Unexpected token: )", - '"foo\#{case 1 do )}bar"' + assert_raise ArgumentError, msg, fn -> + defmodule InvalidStructFieldValue do + defstruct baz: fn -> nil end + end + end end - test :in_definition_module do - assert_compile_fail CompileError, - "nofile:1: cannot define module ErrorsTest because it is currently being defined in nofile:1", - 'defmodule ErrorsTest, do: (defmodule Elixir.ErrorsTest, do: true)' + test "invalid case clauses" do + assert_eval_raise CompileError, + "nofile:1: expected one argument for :do clauses (->) in \"case\"", + 'case nil do 0, z when not is_nil(z) -> z end' + end + + test "invalid fn args" do + assert_eval_raise TokenMissingError, + ~r/nofile:1:5: missing terminator: end \(for "fn" starting at line 1\).*/, + 'fn 1' + end + + test "invalid escape" do + assert_eval_raise TokenMissingError, ~r/nofile:1:3: invalid escape \\ at end of file/, '1 \\' + end + + test "show snippet on missing tokens" do + assert_eval_raise TokenMissingError, + "nofile:1:25: missing terminator: end (for \"do\" starting at line 1)\n" <> + " |\n" <> + " 1 | defmodule ShowSnippet do\n" <> + " | ^", + 'defmodule ShowSnippet do' + end + + test "don't show snippet when error line is empty" do + assert_eval_raise TokenMissingError, + "nofile:3:1: missing terminator: end (for \"do\" starting at line 1)", + 'defmodule ShowSnippet do\n\n' + end + + test "function local conflict" do + assert_eval_raise CompileError, + "nofile:3: imported Kernel.&&/2 conflicts with local function", + ''' + defmodule Kernel.ErrorsTest.FunctionLocalConflict do + def other, do: 1 && 2 + def _ && _, do: :error + end + ''' + end + + test "macro local conflict" do + assert_eval_raise CompileError, + "nofile:6: call to local macro &&/2 conflicts with imported Kernel.&&/2, " <> + "please rename the local macro or remove the conflicting import", + ''' + defmodule Kernel.ErrorsTest.MacroLocalConflict do + def hello, do: 1 || 2 + defmacro _ || _, do: :ok + + defmacro _ && _, do: :error + def world, do: 1 && 2 + end + ''' + end + + test "macro with undefined local" do + assert_eval_raise UndefinedFunctionError, + "function Kernel.ErrorsTest.MacroWithUndefinedLocal.unknown/1" <> + " is undefined (function not available)", + ''' + defmodule Kernel.ErrorsTest.MacroWithUndefinedLocal do + defmacrop bar, do: unknown(1) + def baz, do: bar() + end + ''' + end + + test "private macro" do + assert_eval_raise UndefinedFunctionError, + "function Kernel.ErrorsTest.PrivateMacro.foo/0 is undefined (function not available)", + ''' + defmodule Kernel.ErrorsTest.PrivateMacro do + defmacrop foo, do: 1 + defmacro bar, do: __MODULE__.foo() + defmacro baz, do: bar() + end + ''' + end + + test "macro invoked before its definition" do + assert_eval_raise CompileError, + ~r"nofile:2: cannot invoke macro bar/0 before its definition", + ''' + defmodule Kernel.ErrorsTest.IncorrectMacroDispatch do + def foo, do: bar() + defmacro bar, do: :bar + end + ''' + + assert_eval_raise CompileError, + ~r"nofile:2: cannot invoke macro bar/0 before its definition", + ''' + defmodule Kernel.ErrorsTest.IncorrectMacropDispatch do + def foo, do: bar() + defmacrop bar, do: :ok + end + ''' + + assert_eval_raise CompileError, + ~r"nofile:2: cannot invoke macro bar/1 before its definition", + ''' + defmodule Kernel.ErrorsTest.IncorrectMacroDispatch do + defmacro bar(a) when is_atom(a), do: bar([a]) + end + ''' + end + + test "macro captured before its definition" do + assert_eval_raise CompileError, + ~r"nofile:3: cannot invoke macro is_ok/1 before its definition", + ''' + defmodule Kernel.ErrorsTest.IncorrectMacroDispatch.Capture do + def foo do + predicate = &is_ok/1 + Enum.any?([:ok, :error, :foo], predicate) + end + + defmacro is_ok(atom), do: atom == :ok + end + ''' + end + + test "function definition with alias" do + assert_eval_raise CompileError, + "nofile:2: function names should start with lowercase characters or underscore, invalid name Bar", + ''' + defmodule Kernel.ErrorsTest.FunctionDefinitionWithAlias do + def Bar do + :baz + end + end + ''' + end + + test "function import conflict" do + assert_eval_raise CompileError, + "nofile:3: function exit/1 imported from both :erlang and Kernel, call is ambiguous", + ''' + defmodule Kernel.ErrorsTest.FunctionImportConflict do + import :erlang, only: [exit: 1], warn: false + def foo, do: exit(:test) + end + ''' + end + + test "duplicated function on import options" do + assert_eval_raise CompileError, + "nofile:2: invalid :only option for import, flatten/1 is duplicated", + ''' + defmodule Kernel.ErrorsTest.DuplicatedFunctionOnImportOnly do + import List, only: [flatten: 1, keyfind: 4, flatten: 1] + end + ''' + + assert_eval_raise CompileError, + "nofile:2: invalid :except option for import, flatten/1 is duplicated", + ''' + defmodule Kernel.ErrorsTest.DuplicatedFunctionOnImportExcept do + import List, except: [flatten: 1, keyfind: 4, flatten: 1] + end + ''' + end + + test "ensure valid import :only option" do + assert_eval_raise CompileError, + "nofile:3: invalid :only option for import, expected value to be an atom " <> + ":functions, :macros, or a list literal, got: x", + ''' + defmodule Kernel.ErrorsTest.Only do + x = [flatten: 1] + import List, only: x + end + ''' + end + + test "ensure valid import :except option" do + assert_eval_raise CompileError, + "nofile:3: invalid :except option for import, expected value to be a list " <> + "literal, got: Module.__get_attribute__(Kernel.ErrorsTest.Only, :x, 3)", + ''' + defmodule Kernel.ErrorsTest.Only do + @x [flatten: 1] + import List, except: @x + end + ''' + end + + test "def defmacro clause change" do + assert_eval_raise CompileError, + "nofile:3: defmacro foo/1 already defined as def in nofile:2", + ''' + defmodule Kernel.ErrorsTest.DefDefmacroClauseChange do + def foo(1), do: 1 + defmacro foo(x), do: x + end + ''' + end + + test "def defp clause change from another file" do + assert_eval_raise CompileError, ~r"nofile:4: def hello/0 already defined as defp", ''' + defmodule Kernel.ErrorsTest.DefDefmacroClauseChange do + require Kernel.ErrorsTest + defp hello, do: :world + Kernel.ErrorsTest.hello() + end + ''' end - test :invalid_definition do - assert_compile_fail CompileError, - "nofile:1: invalid syntax in def 1.(hello)", - 'defmodule ErrorsTest, do: (def 1.(hello), do: true)' + test "internal function overridden" do + assert_eval_raise CompileError, + "nofile:2: cannot define def __info__/1 as it is automatically defined by Elixir", + ''' + defmodule Kernel.ErrorsTest.InternalFunctionOverridden do + def __info__(_), do: [] + end + ''' end - test :duplicated_bitstring_size do - assert_compile_fail CompileError, - "nofile:1: duplicated size definition in bitstring", - '<<1 :: [size(12), size(13)]>>' + test "no macros" do + assert_eval_raise CompileError, "nofile:2: could not load macros from module :lists", ''' + defmodule Kernel.ErrorsTest.NoMacros do + import :lists, only: :macros + end + ''' end - test :invalid_bitstring_specified do - assert_compile_fail CompileError, - "nofile:1: unknown bitstring specifier :atom", - '<<1 :: :atom>>' - - assert_compile_fail CompileError, - "nofile:1: unknown bitstring specifier unknown()", - '<<1 :: unknown>>' - - assert_compile_fail CompileError, - "nofile:1: unknown bitstring specifier another(12)", - '<<1 :: another(12)>>' + test "invalid macro" do + assert_eval_raise CompileError, + ~r"nofile: invalid quoted expression: {:foo, :bar, :baz, :bat}", + ''' + defmodule Kernel.ErrorsTest.InvalidMacro do + defmacrop oops do + {:foo, :bar, :baz, :bat} + end - assert_compile_fail CompileError, - "nofile:1: size in bitstring expects an integer or a variable as argument, got: :a", - '<<1 :: size(:a)>>' - - assert_compile_fail CompileError, - "nofile:1: unit in bitstring expects an integer as argument, got: :x", - '<<1 :: unit(:x)>>' + def test, do: oops() + end + ''' end - test :invalid_var! do - assert_compile_fail CompileError, - "nofile:1: expected var x to expand to an existing variable or be a part of a match", - 'var!(x)' + test "unloaded module" do + assert_eval_raise CompileError, + "nofile:1: module Certainly.Doesnt.Exist is not loaded and could not be found", + 'import Certainly.Doesnt.Exist' end - test :invalid_alias do - assert_compile_fail CompileError, - "nofile:1: invalid value for keyword :as, expected an alias, got nested alias: Sample.Lists", - 'alias :lists, as: Sample.Lists' + test "module imported from the context it was defined in" do + assert_eval_raise CompileError, + ~r"nofile:4: module Kernel.ErrorsTest.ScheduledModule.Hygiene is not loaded but was defined.", + ''' + defmodule Kernel.ErrorsTest.ScheduledModule do + defmodule Hygiene do + end + import Kernel.ErrorsTest.ScheduledModule.Hygiene + end + ''' end - test :invalid_import_option do - assert_compile_fail CompileError, - "nofile:1: unsupported option :ops given to import", - 'import :lists, [ops: 1]' + test "module imported from the same module" do + assert_eval_raise CompileError, + ~r"nofile:3: you are trying to use the module Kernel.ErrorsTest.ScheduledModule.Hygiene which is currently being defined", + ''' + defmodule Kernel.ErrorsTest.ScheduledModule do + defmodule Hygiene do + import Kernel.ErrorsTest.ScheduledModule.Hygiene + end + end + ''' end - test :invalid_rescue_clause do - assert_compile_fail CompileError, - "nofile:4: invalid rescue clause. The clause should match on an alias, a variable or be in the `var in [alias]` format", - 'try do\n1\nrescue\n%UndefinedFunctionError{arity: 1} -> false\nend' + test "already compiled module" do + assert_eval_raise ArgumentError, + "could not call Module.eval_quoted/4 because the module Record is already compiled", + 'Module.eval_quoted Record, quote(do: 1), [], file: __ENV__.file' end - test :invalid_for_without_generators do - assert_compile_fail CompileError, - "nofile:1: for comprehensions must start with a generator", - 'for x, do: x' + test "@compile inline with undefined function" do + assert_eval_raise CompileError, + "nofile:1: inlined function foo/1 undefined", + 'defmodule Test do @compile {:inline, foo: 1} end' end - test :invalid_for_bit_generator do - assert_compile_fail CompileError, - "nofile:1: bitstring fields without size are not allowed in bitstring generators", - 'for << x :: binary <- "123" >>, do: x' - end + test "invalid @dialyzer options" do + assert_eval_raise CompileError, + "nofile:1: undefined function foo/1 given to @dialyzer :nowarn_function", + 'defmodule Test do @dialyzer {:nowarn_function, {:foo, 1}} end' - test :unbound_cond do - assert_compile_fail CompileError, - "nofile:1: unbound variable _ inside cond. If you want the last clause to always match, " <> - "you probably meant to use: true ->", - 'cond do _ -> true end' - end + assert_eval_raise CompileError, + "nofile:1: undefined function foo/1 given to @dialyzer :no_opaque", + 'defmodule Test do @dialyzer {:no_opaque, {:foo, 1}} end' - test :fun_different_arities do - assert_compile_fail CompileError, - "nofile:1: cannot mix clauses with different arities in function definition", - 'fn x -> x; x, y -> x + y end' + assert_eval_raise ArgumentError, + "invalid value for @dialyzer attribute: :not_an_option", + 'defmodule Test do @dialyzer :not_an_option end' end - test :new_line_error do - assert_compile_fail SyntaxError, - "nofile:3: syntax error before: newline", - 'if true do\n foo = [],\n baz\nend' + test "@on_load attribute format" do + assert_raise ArgumentError, ~r/should be an atom or an {atom, 0} tuple/, fn -> + defmodule BadOnLoadAttribute do + Module.put_attribute(__MODULE__, :on_load, "not an atom") + end + end end - test :invalid_var_or_function_on_guard do - assert_compile_fail CompileError, - "nofile:2: unknown variable something_that_does_not_exist or " <> - "cannot invoke function something_that_does_not_exist/0 inside guard", - ''' - case [] do - [] when something_that_does_not_exist == [] -> :ok + test "duplicated @on_load attribute" do + assert_raise ArgumentError, "the @on_load attribute can only be set once per module", fn -> + defmodule DuplicatedOnLoadAttribute do + @on_load :foo + @on_load :bar end - ''' + end end - test :bodyless_function_with_guard do - assert_compile_fail CompileError, - "nofile:2: missing do keyword in def", - ''' - defmodule ErrorsTest do - def foo(n) when is_number(n) - end - ''' + test "@on_load attribute with undefined function" do + assert_eval_raise CompileError, + "nofile:1: undefined function foo/0 given to @on_load", + 'defmodule UndefinedOnLoadFunction do @on_load :foo end' end - test :invalid_args_for_bodyless_clause do - assert_compile_fail CompileError, - "nofile:2: can use only variables and \\\\ as arguments of bodyless clause", - ''' - defmodule ErrorsTest do - def foo(arg // nil) - def foo(_), do: :ok - end - ''' + test "wrong kind for @on_load attribute" do + assert_eval_raise CompileError, + "nofile:1: expected @on_load function foo/0 to be a function, " <> + "got \"defmacro\"", + ''' + defmodule PrivateOnLoadFunction do + @on_load :foo + defmacro foo, do: :ok + end + ''' end - test :invalid_function_on_match do - assert_compile_fail CompileError, - "nofile:1: cannot invoke function something_that_does_not_exist/0 inside match", - 'case [] do; something_that_does_not_exist() -> :ok; end' + test "in definition module" do + assert_eval_raise CompileError, + "nofile:2: cannot define module Kernel.ErrorsTest.InDefinitionModule " <> + "because it is currently being defined in nofile:1", + ''' + defmodule Kernel.ErrorsTest.InDefinitionModule do + defmodule Elixir.Kernel.ErrorsTest.InDefinitionModule, do: true + end + ''' end - test :invalid_remote_on_match do - assert_compile_fail CompileError, - "nofile:1: cannot invoke remote function Hello.something_that_does_not_exist/0 inside match", - 'case [] do; Hello.something_that_does_not_exist() -> :ok; end' + test "invalid definition" do + assert_eval_raise CompileError, + "nofile:1: invalid syntax in def 1.(hello)", + 'defmodule Kernel.ErrorsTest.InvalidDefinition, do: (def 1.(hello), do: true)' end - test :invalid_remote_on_guard do - assert_compile_fail CompileError, - "nofile:1: cannot invoke remote function Hello.something_that_does_not_exist/0 inside guard", - 'case [] do; [] when Hello.something_that_does_not_exist == [] -> :ok; end' + test "invalid size in bitstrings" do + assert_eval_raise CompileError, + "nofile:1: cannot use ^x outside of match clauses", + 'x = 8; <> = <>' end - test :typespec_errors do - assert_compile_fail CompileError, - "nofile:2: type foo() undefined", - ''' - defmodule ErrorsTest do - @type omg :: foo - end - ''' + test "function head with guard" do + assert_eval_raise CompileError, "nofile:2: missing :do option in \"def\"", ''' + defmodule Kernel.ErrorsTest.BodyessFunctionWithGuard do + def foo(n) when is_number(n) + end + ''' - assert_compile_fail CompileError, - "nofile:2: spec for undefined function ErrorsTest.omg/0", - ''' - defmodule ErrorsTest do - @spec omg :: atom - end - ''' + assert_eval_raise CompileError, "nofile:2: missing :do option in \"def\"", ''' + defmodule Kernel.ErrorsTest.BodyessFunctionWithGuard do + def foo(n) when is_number(n), true + end + ''' end - test :bad_unquoting do - assert_compile_fail CompileError, - "nofile: invalid quoted expression: {:foo, 0, 1}", - ''' - defmodule ErrorsTest do - def range(unquote({:foo, 0, 1})), do: :ok - end - ''' + test "invalid args for function head" do + assert_eval_raise CompileError, + ~r"nofile:2: only variables and \\\\ are allowed as arguments in function head.", + ''' + defmodule Kernel.ErrorsTest.InvalidArgsForBodylessClause do + def foo(nil) + def foo(_), do: :ok + end + ''' + end + + test "bad multi-call" do + assert_eval_raise CompileError, + "nofile:1: invalid argument for alias, expected a compile time atom or alias, got: 42", + 'alias IO.{ANSI, 42}' + + assert_eval_raise CompileError, + "nofile:1: :as option is not supported by multi-alias call", + 'alias Elixir.{Map}, as: Dict' + + assert_eval_raise UndefinedFunctionError, + "function List.\"{}\"/1 is undefined or private", + '[List.{Chars}, "one"]' + end + + test "macros error stacktrace" do + assert [ + {:erlang, :+, [1, :foo], _}, + {Kernel.ErrorsTest.MacrosErrorStacktrace, :sample, 1, _} | _ + ] = + rescue_stacktrace(""" + defmodule Kernel.ErrorsTest.MacrosErrorStacktrace do + defmacro sample(num), do: num + :foo + def other, do: sample(1) + end + """) + end + + test "macros function clause stacktrace" do + assert [{__MODULE__, :sample, 1, _} | _] = + rescue_stacktrace(""" + defmodule Kernel.ErrorsTest.MacrosFunctionClauseStacktrace do + import Kernel.ErrorsTest + sample(1) + end + """) + end + + test "macros interpreted function clause stacktrace" do + assert [{Kernel.ErrorsTest.MacrosInterpretedFunctionClauseStacktrace, :sample, 1, _} | _] = + rescue_stacktrace(""" + defmodule Kernel.ErrorsTest.MacrosInterpretedFunctionClauseStacktrace do + defmacro sample(0), do: 0 + def other, do: sample(1) + end + """) + end + + test "macros compiled callback" do + assert [{Kernel.ErrorsTest, :__before_compile__, [env], _} | _] = + rescue_stacktrace(""" + defmodule Kernel.ErrorsTest.MacrosCompiledCallback do + Module.put_attribute(__MODULE__, :before_compile, Kernel.ErrorsTest) + end + """) + + assert %Macro.Env{module: Kernel.ErrorsTest.MacrosCompiledCallback} = env + end + + test "failed remote call stacktrace includes file/line info" do + try do + bad_remote_call(1) + rescue + ArgumentError -> + assert [ + {:erlang, :apply, [1, :foo, []], _}, + {__MODULE__, :bad_remote_call, 1, [file: _, line: _]} | _ + ] = __STACKTRACE__ + end end - test :macros_error_stacktrace do - assert [{:erlang, :+, [1, :foo], _}, {ErrorsTest, :sample, 1, _}|_] = - rescue_stacktrace(""" - defmodule ErrorsTest do - defmacro sample(num), do: num + :foo - def other, do: sample(1) + test "def fails when rescue, else or catch don't have clauses" do + assert_eval_raise CompileError, ~r"expected -> clauses for :rescue in \"def\"", """ + defmodule Example do + def foo do + bar() + rescue + baz() end - """) + end + """ end - test :macros_function_clause_stacktrace do - assert [{__MODULE__, :sample, 1, _}|_] = - rescue_stacktrace(""" - defmodule ErrorsTest do - import Kernel.ErrorsTest - sample(1) - end - """) - end + test "duplicate map keys" do + assert_eval_raise CompileError, "nofile:1: key :a will be overridden in map", """ + %{a: :b, a: :c} = %{a: :c} + """ - test :macros_interpreted_function_clause_stacktrace do - assert [{ErrorsTest, :sample, 1, _}|_] = - rescue_stacktrace(""" - defmodule ErrorsTest do - defmacro sample(0), do: 0 - def other, do: sample(1) - end - """) + assert_eval_raise CompileError, "nofile:1: key :a will be overridden in map", """ + %{a: :b, a: :c, a: :d} = %{a: :c} + """ end - test :macros_compiled_callback do - assert [{Kernel.ErrorsTest, :__before_compile__, [%Macro.Env{module: ErrorsTest}], _}|_] = - rescue_stacktrace(""" - defmodule ErrorsTest do - Module.put_attribute(__MODULE__, :before_compile, Kernel.ErrorsTest) - end - """) + test "| outside of cons" do + assert_eval_raise CompileError, ~r"nofile:1: misplaced operator |/2", "1 | 2" + + assert_eval_raise CompileError, + ~r"nofile:1: misplaced operator |/2", + "defmodule MisplacedOperator, do: (def bar(1 | 2), do: :ok)" end + defp bad_remote_call(x), do: x.foo + defmacro sample(0), do: 0 defmacro before_compile(_) do @@ -704,14 +872,20 @@ defmodule Kernel.ErrorsTest do ## Helpers - defp rescue_stacktrace(expr) do - result = try do - :elixir.eval(to_char_list(expr), []) + defp assert_eval_raise(given_exception, given_message, string) do + assert_raise given_exception, given_message, fn -> + Code.eval_string(string) + end + end + + defp rescue_stacktrace(string) do + try do + Code.eval_string(string) nil rescue - _ -> System.stacktrace + _ -> __STACKTRACE__ + else + _ -> flunk("Expected expression to fail") end - - result || raise(ExUnit.AssertionError, message: "Expected function given to rescue_stacktrace to fail") end end diff --git a/lib/elixir/test/elixir/kernel/expansion_test.exs b/lib/elixir/test/elixir/kernel/expansion_test.exs index ca6048bd386..300ea617d4f 100644 --- a/lib/elixir/test/elixir/kernel/expansion_test.exs +++ b/lib/elixir/test/elixir/kernel/expansion_test.exs @@ -1,474 +1,2903 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.ExpansionTarget do defmacro seventeen, do: 17 + defmacro bar, do: "bar" + + defmacro message_hello(arg) do + send(self(), :hello) + arg + end end defmodule Kernel.ExpansionTest do use ExUnit.Case, async: true - ## __block__ - - test "__block__: expands to nil when empty" do - assert expand(quote do: __block__()) == nil + defmacrop var_ver(var, version) do + quote do + {unquote(var), [version: unquote(version)], __MODULE__} + end end - test "__block__: expands to argument when arity is 1" do - assert expand(quote do: __block__(1)) == 1 + test "tracks variable version" do + assert {:__block__, _, [{:=, _, [var_ver(:x, 0), 0]}, {:=, _, [_, var_ver(:x, 0)]}]} = + expand_with_version( + quote do + x = 0 + _ = x + end + ) + + assert {:__block__, _, + [ + {:=, _, [var_ver(:x, 0), 0]}, + {:=, _, [_, var_ver(:x, 0)]}, + {:=, _, [var_ver(:x, 1), 1]}, + {:=, _, [_, var_ver(:x, 1)]} + ]} = + expand_with_version( + quote do + x = 0 + _ = x + x = 1 + _ = x + end + ) + + assert {:__block__, _, + [ + {:=, _, [var_ver(:x, 0), 0]}, + {:fn, _, [{:->, _, [[var_ver(:x, 1)], {:=, _, [var_ver(:x, 2), 2]}]}]}, + {:=, _, [_, var_ver(:x, 0)]}, + {:=, _, [var_ver(:x, 3), 3]} + ]} = + expand_with_version( + quote do + x = 0 + fn x -> x = 2 end + _ = x + x = 3 + end + ) + + assert {:__block__, _, + [ + {:=, _, [var_ver(:x, 0), 0]}, + {:case, _, [:foo, [do: [{:->, _, [[var_ver(:x, 1)], var_ver(:x, 1)]}]]]}, + {:=, _, [_, var_ver(:x, 0)]}, + {:=, _, [var_ver(:x, 2), 2]} + ]} = + expand_with_version( + quote do + x = 0 + case(:foo, do: (x -> x)) + _ = x + x = 2 + end + ) + end + + defp expand_with_version(expr) do + env = :elixir_env.reset_vars(__ENV__) + {expr, _, _} = :elixir_expand.expand(expr, :elixir_env.env_to_ex(env), env) + expr end - test "__block__: is recursive to argument when arity is 1" do - assert expand(quote do: __block__(1, __block__(2))) == quote do: __block__(1, 2) - end + describe "__block__" do + test "expands to nil when empty" do + assert expand(quote(do: unquote(:__block__)())) == nil + end - test "__block__: accumulates vars" do - assert expand(quote(do: (a = 1; a))) == quote do: (a = 1; a) - end + test "expands to argument when arity is 1" do + assert expand(quote(do: unquote(:__block__)(1))) == 1 + end + + test "is recursive to argument when arity is 1" do + expanded = + quote do + _ = 1 + 2 + end - ## alias + assert expand(quote(do: unquote(:__block__)(_ = 1, unquote(:__block__)(2)))) == expanded + end - test "alias: expand args, defines alias and returns itself" do - alias true, as: True + test "accumulates vars" do + before_expansion = + quote do + a = 1 + a + end - input = quote do: (alias :hello, as: World, warn: True) - {output, env} = expand_env(input, __ENV__) + after_expansion = + quote do + a = 1 + a + end - assert output == quote do: (alias :hello, as: :"Elixir.World", warn: true) - assert env.aliases == [{:"Elixir.True", true}, {:"Elixir.World", :hello}] + assert expand(before_expansion) == after_expansion + end end - ## __aliases__ + describe "alias" do + test "expand args, defines alias and returns itself" do + alias true, as: True - test "__aliases__: expands even if no alias" do - assert expand(quote do: World) == :"Elixir.World" - assert expand(quote do: Elixir.World) == :"Elixir.World" - end + input = quote(do: alias(:hello, as: World, warn: True)) + {output, env} = expand_env(input, __ENV__) - test "__aliases__: expands with alias" do - alias Hello, as: World - assert expand_env(quote(do: World), __ENV__) |> elem(0) == :"Elixir.Hello" - end + assert output == :hello + assert env.aliases == [{:"Elixir.True", true}, {:"Elixir.World", :hello}] + end - test "__aliases__: expands with alias is recursive" do - alias Source, as: Hello - alias Hello, as: World - assert expand_env(quote(do: World), __ENV__) |> elem(0) == :"Elixir.Source" - end + test "invalid alias" do + message = + ~r"invalid value for option :as, expected a simple alias, got nested alias: Sample.Lists" - test "__aliases__: expands to elixir_aliases on runtime" do - assert expand(quote do: hello.World) == - quote do: :elixir_aliases.concat([hello(), :World]) - end + assert_raise CompileError, message, fn -> + expand(quote(do: alias(:lists, as: Sample.Lists))) + end - ## = + message = ~r"invalid argument for alias, expected a compile time atom or alias, got: 1 \+ 2" - test "=: sets context to match" do - assert expand(quote do: __ENV__.context = :match) == quote do: :match = :match - end + assert_raise CompileError, message, fn -> + expand(quote(do: alias(1 + 2))) + end - test "=: defines vars" do - {output, env} = expand_env(quote(do: a = 1), __ENV__) - assert output == quote(do: a = 1) - assert {:a, __MODULE__} in env.vars - end + message = ~r"invalid value for option :as, expected an alias, got: :foobar" - test "=: does not carry rhs imports" do - assert expand(quote(do: flatten([1,2,3]) = import List)) == - quote(do: flatten([1,2,3]) = import :"Elixir.List", []) - end + assert_raise CompileError, message, fn -> + expand(quote(do: alias(:lists, as: :foobar))) + end - test "=: does not define _" do - {output, env} = expand_env(quote(do: _ = 1), __ENV__) - assert output == quote(do: _ = 1) - assert env.vars == [] - end + message = ~r"invalid value for option :as, expected an alias, got: :\"Elixir.foobar\"" - ## Pseudo vars + assert_raise CompileError, message, fn -> + expand(quote(do: alias(:lists, as: :"Elixir.foobar"))) + end - test "__MODULE__" do - assert expand(quote do: __MODULE__) == __MODULE__ - end + message = + ~r"alias cannot be inferred automatically for module: :lists, please use the :as option" - test "__DIR__" do - assert expand(quote do: __DIR__) == __DIR__ - end + assert_raise CompileError, message, fn -> + expand(quote(do: alias(:lists))) + end + end - test "__CALLER__" do - assert expand(quote do: __CALLER__) == quote do: __CALLER__ - end + test "invalid expansion" do + assert_raise CompileError, ~r"invalid alias: \"foo\.Foo\"", fn -> + code = + quote do + foo = :foo + foo.Foo + end - test "__ENV__" do - env = %{__ENV__ | line: 0} - assert expand_env(quote(do: __ENV__), env) == - {{:%{}, [], Map.to_list(env)}, env} - end + expand(code) + end + end + + test "raises if :as is passed to multi-alias aliases" do + assert_raise CompileError, ~r":as option is not supported by multi-alias call", fn -> + expand(quote(do: alias(Foo.{Bar, Baz}, as: BarBaz))) + end + end - test "__ENV__.accessor" do - env = %{__ENV__ | line: 0} - assert expand_env(quote(do: __ENV__.file), env) == {__ENV__.file, env} - assert expand_env(quote(do: __ENV__.unknown), env) == - {quote(do: unquote({:%{}, [], Map.to_list(env)}).unknown), env} + test "invalid options" do + assert_raise CompileError, ~r"unsupported option :ops given to alias", fn -> + expand(quote(do: alias(Foo, ops: 1))) + end + end end - ## Super + describe "__aliases__" do + test "expands even if no alias" do + assert expand(quote(do: World)) == :"Elixir.World" + assert expand(quote(do: Elixir.World)) == :"Elixir.World" + end + + test "expands with alias" do + alias Hello, as: World + assert expand_env(quote(do: World), __ENV__) |> elem(0) == :"Elixir.Hello" + end - test "super: expand args" do - assert expand(quote do: super(a, b)) == quote do: super(a(), b()) + test "expands with alias is recursive" do + alias Source, as: Hello + alias Hello, as: World + assert expand_env(quote(do: World), __ENV__) |> elem(0) == :"Elixir.Source" + end end - ## Vars + describe "import" do + test "raises on invalid macro" do + message = ~r"cannot import Kernel.invalid/1 because it is undefined or private" - test "vars: expand to local call" do - {output, env} = expand_env(quote(do: a), __ENV__) - assert output == quote(do: a()) - assert env.vars == [] - end + assert_raise CompileError, message, fn -> + expand(quote(do: import(Kernel, only: [invalid: 1]))) + end + end - test "vars: forces variable to exist" do - assert expand(quote do: (var!(a) = 1; var!(a))) + test "raises on invalid options" do + message = ~r"invalid :only option for import, expected a keyword list with integer values" - message = ~r"expected var a to expand to an existing variable or be a part of a match" - assert_raise CompileError, message, fn -> expand(quote do: var!(a)) end + assert_raise CompileError, message, fn -> + expand(quote(do: import(Kernel, only: [invalid: nil]))) + end - message = ~r"expected var a \(context Unknown\) to expand to an existing variable or be a part of a match" - assert_raise CompileError, message, fn -> expand(quote do: var!(a, Unknown)) end - end + message = ~r"invalid :except option for import, expected a keyword list with integer values" - test "^: expands args" do - assert expand(quote do: ^a = 1) == quote do: ^a = 1 - end + assert_raise CompileError, message, fn -> + expand(quote(do: import(Kernel, except: [invalid: nil]))) + end - test "^: raises outside match" do - assert_raise CompileError, ~r"cannot use \^a outside of match clauses", fn -> - expand(quote do: ^a) + message = ~r/invalid options for import, expected a keyword list, got: "invalid_options"/ + + assert_raise CompileError, message, fn -> + expand(quote(do: import(Kernel, "invalid_options"))) + end end - end - test "^: raises without var" do - assert_raise CompileError, ~r"invalid argument for unary operator \^, expected an existing variable, got: \^1", fn -> - expand(quote do: ^1 = 1) + test "raises on conflicting options" do + message = + ~r":only and :except can only be given together to import when :only is :functions, :macros, or :sigils" + + assert_raise CompileError, message, fn -> + expand(quote(do: import(Kernel, only: [], except: []))) + end end - end - ## Locals + test "invalid import option" do + assert_raise CompileError, ~r"unsupported option :ops given to import", fn -> + expand(quote(do: import(:lists, ops: 1))) + end + end - test "locals: expands to remote calls" do - assert {{:., _, [Kernel, :=~]}, _, [{:a, _, []}, {:b, _, []}]} = - expand(quote do: a =~ b) + test "raises for non-compile-time module" do + assert_raise CompileError, ~r"invalid argument for import, .*, got: {:a, :tuple}", fn -> + expand(quote(do: import({:a, :tuple}))) + end + end end - test "locals: expands to configured local" do - assert expand_env(quote(do: a), %{__ENV__ | local: Hello}) |> elem(0) == - quote(do: :"Elixir.Hello".a()) - end + describe "require" do + test "raises for non-compile-time module" do + assert_raise CompileError, ~r"invalid argument for require, .*, got: {:a, :tuple}", fn -> + expand(quote(do: require({:a, :tuple}))) + end + end - test "locals: in guards" do - assert expand(quote(do: fn pid when :erlang.==(pid, self) -> pid end)) == - quote(do: fn pid when :erlang.==(pid, :erlang.self()) -> pid end) + test "invalid options" do + assert_raise CompileError, ~r"unsupported option :ops given to require", fn -> + expand(quote(do: require(Foo, ops: 1))) + end + end end - test "locals: custom imports" do - assert expand(quote do: (import Kernel.ExpansionTarget; seventeen)) == - quote do: (import :"Elixir.Kernel.ExpansionTarget", []; 17) + describe "=" do + test "defines vars" do + {output, env} = expand_env(quote(do: a = 1), __ENV__) + assert output == quote(do: a = 1) + assert Macro.Env.has_var?(env, {:a, __MODULE__}) + end + + test "does not define _" do + {output, env} = expand_env(quote(do: _ = 1), __ENV__) + assert output == quote(do: _ = 1) + assert Macro.Env.vars(env) == [] + end end - ## Tuples + describe "environment macros" do + test "__MODULE__" do + assert expand(quote(do: __MODULE__)) == __MODULE__ + end - test "tuples: expanded as arguments" do - assert expand(quote(do: {a = 1, a})) == quote do: {a = 1, a()} - assert expand(quote(do: {b, a = 1, a})) == quote do: {b(), a = 1, a()} - end + test "__DIR__" do + assert expand(quote(do: __DIR__)) == __DIR__ + end - ## Maps & structs + test "__ENV__" do + env = %{__ENV__ | line: 0} + assert expand_env(quote(do: __ENV__), env) == {Macro.escape(env), env} + assert %{lexical_tracker: nil, tracers: []} = __ENV__ + end - test "maps: expanded as arguments" do - assert expand(quote(do: %{a: a = 1, b: a})) == quote do: %{a: a = 1, b: a()} - end + test "__ENV__.accessor" do + env = %{__ENV__ | line: 0} + assert expand_env(quote(do: __ENV__.file), env) == {__ENV__.file, env} - test "structs: expanded as arguments" do - assert expand(quote(do: %:elixir{a: a = 1, b: a})) == - quote do: %:elixir{a: a = 1, b: a()} + assert expand_env(quote(do: __ENV__.unknown), env) == + {quote(do: unquote(Macro.escape(env)).unknown), env} - assert expand(quote(do: %:"Elixir.Kernel"{a: a = 1, b: a})) == - quote do: %:"Elixir.Kernel"{a: a = 1, b: a()} - end + assert __ENV__.lexical_tracker == nil + assert __ENV__.tracers == [] + end + + test "on match" do + assert_raise CompileError, + ~r"invalid pattern in match, __ENV__ is not allowed in matches", + fn -> expand(quote(do: __ENV__ = :ok)) end - test "structs: expects atoms" do - assert_raise CompileError, ~r"expected struct name to be a compile time atom or alias", fn -> - expand(quote do: %unknown{a: 1}) + assert_raise CompileError, + ~r"invalid pattern in match, __CALLER__ is not allowed in matches", + fn -> expand(quote(do: __CALLER__ = :ok)) end + + assert_raise CompileError, + ~r"invalid pattern in match, __STACKTRACE__ is not allowed in matches", + fn -> expand(quote(do: __STACKTRACE__ = :ok)) end end end - ## quote + describe "vars" do + test "expand to local call" do + {output, env} = expand_env(quote(do: a), __ENV__) + assert output == quote(do: a()) + assert Macro.Env.vars(env) == [] + end + + test "forces variable to exist" do + code = + quote do + var!(a) = 1 + var!(a) + end - test "quote: expanded to raw forms" do - assert expand(quote do: (quote do: hello)) == {:{}, [], [:hello, [], __MODULE__]} - end + assert expand(code) - ## Anonymous calls + message = ~r"undefined variable \"a\"" - test "anonymous calls: expands base and args" do - assert expand(quote do: a.(b)) == quote do: a().(b()) - end + assert_raise CompileError, message, fn -> + expand(quote(do: var!(a))) + end + + message = ~r"undefined variable \"a\" \(context Unknown\)" - test "anonymous calls: raises on atom base" do - assert_raise CompileError, ~r"invalid function call :foo.()", fn -> - expand(quote do: :foo.(a)) + assert_raise CompileError, message, fn -> + expand(quote(do: var!(a, Unknown))) + end + end + + test "raises for _ used outside of a match" do + assert_raise CompileError, ~r"invalid use of _", fn -> + expand(quote(do: {1, 2, _})) + end end end - ## Remote calls + describe "^" do + test "expands args" do + before_expansion = + quote do + after_expansion = 1 + ^after_expansion = 1 + end - test "remote calls: expands to erlang" do - assert expand(quote do: Kernel.is_atom(a)) == quote do: :erlang.is_atom(a()) - end + after_expansion = + quote do + after_expansion = 1 + ^after_expansion = 1 + end - test "remote calls: expands macros" do - assert expand(quote do: Kernel.ExpansionTest.thirteen) == 13 - end + assert expand(before_expansion) == after_expansion + end - test "remote calls: expands receiver and args" do - assert expand(quote do: a.is_atom(b)) == quote do: a().is_atom(b()) - assert expand(quote do: (a = :foo).is_atom(a)) == quote do: (a = :foo).is_atom(a()) - end + test "raises outside match" do + assert_raise CompileError, ~r"cannot use \^a outside of match clauses", fn -> + expand(quote(do: ^a)) + end + end - test "remote calls: modules must be required for macros" do - assert expand(quote do: (require Kernel.ExpansionTarget; Kernel.ExpansionTarget.seventeen)) == - quote do: (require :"Elixir.Kernel.ExpansionTarget", []; 17) - end + test "raises without var" do + message = + ~r"invalid argument for unary operator \^, expected an existing variable, got: \^1" - test "remote calls: raises when not required" do - msg = ~r"you must require Kernel\.ExpansionTarget before invoking the macro Kernel\.ExpansionTarget\.seventeen/0" - assert_raise CompileError, msg, fn -> - expand(quote do: Kernel.ExpansionTarget.seventeen) + assert_raise CompileError, message, fn -> + expand(quote(do: ^1 = 1)) + end + end + + test "raises when the var is undefined" do + assert_raise CompileError, ~r"undefined variable \^foo", fn -> + expand(quote(do: ^foo = :foo)) + end end end - ## Comprehensions + describe "locals" do + test "expands to remote calls" do + assert {{:., _, [Kernel, :=~]}, _, [{:a, _, []}, {:b, _, []}]} = expand(quote(do: a =~ b)) + end - test "variables inside comprehensions do not leak with enums" do - assert expand(quote do: (for(a <- b, do: c = 1); c)) == - quote do: (for(a <- b(), do: c = 1); c()) - end + test "in matches" do + message = ~r"cannot find or invoke local foo/1 inside match. .+ Called as: foo\(:bar\)" - test "variables inside comprehensions do not leak with binaries" do - assert expand(quote do: (for(<>, do: c = 1); c)) == - quote do: (for(<< <> <- b() >>, do: c = 1); c()) - end + assert_raise CompileError, message, fn -> + expand(quote(do: foo(:bar) = :bar)) + end + end - test "variables inside filters are available in blocks" do - assert expand(quote do: for(a <- b, c = a, do: c)) == - quote do: (for(a <- b(), c = a, do: c)) - end + test "in guards" do + code = quote(do: fn pid when :erlang.==(pid, self) -> pid end) + expanded_code = quote(do: fn pid when :erlang.==(pid, :erlang.self()) -> pid end) + assert clean_meta(expand(code), [:imports, :context]) == expanded_code - test "variables inside comprehensions options do not leak" do - assert expand(quote do: (for(a <- c = b, into: [], do: 1); c)) == - quote do: (for(a <- c = b(), do: 1, into: []); c()) + message = ~r"cannot find or invoke local foo/1" - assert expand(quote do: (for(a <- b, into: c = [], do: 1); c)) == - quote do: (for(a <- b(), do: 1, into: c = []); c()) - end + assert_raise CompileError, message, fn -> + expand(quote(do: fn arg when foo(arg) -> arg end)) + end + end + + test "custom imports" do + before_expansion = + quote do + import Kernel.ExpansionTarget + seventeen() + end - ## Capture + after_expansion = + quote do + :"Elixir.Kernel.ExpansionTarget" + 17 + end - test "&: keeps locals" do - assert expand(quote do: &unknown/2) == - {:&, [], [{:/, [], [{:unknown,[],nil}, 2]}]} - assert expand(quote do: &unknown(&1, &2)) == - {:&, [], [{:/, [], [{:unknown,[],nil}, 2]}]} + assert expand(before_expansion) == after_expansion + end end - test "&: expands remotes" do - assert expand(quote do: &List.flatten/2) == - quote do: :erlang.make_fun(:"Elixir.List", :flatten, 2) + describe "tuples" do + test "expanded as arguments" do + assert expand(quote(do: {after_expansion = 1, a})) == quote(do: {after_expansion = 1, a()}) - assert expand(quote do: &Kernel.is_atom/1) == - quote do: :erlang.make_fun(:erlang, :is_atom, 1) + assert expand(quote(do: {b, after_expansion = 1, a})) == + quote(do: {b(), after_expansion = 1, a()}) + end end - test "&: expands macros" do + describe "maps" do + test "expanded as arguments" do + assert expand(quote(do: %{a: after_expansion = 1, b: a})) == + quote(do: %{a: after_expansion = 1, b: a()}) + end - assert expand(quote do: (require Kernel.ExpansionTarget; &Kernel.ExpansionTarget.seventeen/0)) == - quote do: (require :"Elixir.Kernel.ExpansionTarget", []; fn -> 17 end) - end + test "with variables on keys" do + ast = + quote do + %{(x = 1) => 1} + end - ## fn + assert expand(ast) == ast - test "fn: expands each clause" do - assert expand(quote do: fn x -> x; _ -> x end) == - quote do: fn x -> x; _ -> x() end - end + ast = + quote do + x = 1 + %{%{^x => 1} => 2} = y() + end - test "fn: does not share lexical in between clauses" do - assert expand(quote do: fn 1 -> import List; 2 -> flatten([1,2,3]) end) == - quote do: fn 1 -> import :"Elixir.List", []; 2 -> flatten([1,2,3]) end - end + assert expand(ast) == ast - test "fn: expands guards" do - assert expand(quote do: fn x when x when __ENV__.context -> true end) == - quote do: fn x when x when :guard -> true end - end + ast = + quote do + x = 1 + %{{^x} => 1} = %{{1} => 1} + end - test "fn: does not leak vars" do - assert expand(quote do: (fn x -> x end; x)) == - quote do: (fn x -> x end; x()) - end + assert expand(ast) == ast - ## Cond + assert_raise CompileError, ~r"cannot use variable x as map key inside a pattern", fn -> + expand(quote(do: %{x => 1} = %{})) + end - test "cond: expands each clause" do - assert expand_and_clean(quote do: (cond do x = 1 -> x; _ -> x end)) == - quote do: (cond do x = 1 -> x; _ -> x() end) - end + assert_raise CompileError, ~r"undefined variable \^x", fn -> + expand(quote(do: {x, %{^x => 1}} = %{})) + end + end - test "cond: does not share lexical in between clauses" do - assert expand_and_clean(quote do: (cond do 1 -> import List; 2 -> flatten([1,2,3]) end)) == - quote do: (cond do 1 -> import :"Elixir.List", []; 2 -> flatten([1,2,3]) end) + test "expects key-value pairs" do + assert_raise CompileError, ~r"expected key-value pairs in a map, got: :foo", fn -> + expand(quote(do: unquote({:%{}, [], [:foo]}))) + end + end end - test "cond: does not leaks vars on head" do - assert expand_and_clean(quote do: (cond do x = 1 -> x; y = 2 -> y end; :erlang.+(x, y))) == - quote do: (cond do x = 1 -> x; y = 2 -> y end; :erlang.+(x(), y())) + defmodule User do + defstruct name: "", age: 0 end - test "cond: leaks vars" do - assert expand_and_clean(quote do: (cond do 1 -> x = 1; 2 -> y = 2 end; :erlang.+(x, y))) == - quote do: (cond do 1 -> x = 1; 2 -> y = 2 end; :erlang.+(x, y)) - end + describe "structs" do + test "expanded as arguments" do + assert expand(quote(do: %User{})) == + quote(do: %:"Elixir.Kernel.ExpansionTest.User"{age: 0, name: ""}) - ## Case + assert expand(quote(do: %User{name: "john doe"})) == + quote(do: %:"Elixir.Kernel.ExpansionTest.User"{age: 0, name: "john doe"}) + end - test "case: expands each clause" do - assert expand_and_clean(quote do: (case w do x -> x; _ -> x end)) == - quote do: (case w() do x -> x; _ -> x() end) - end + test "expects atoms" do + expand(quote(do: %unknown{a: 1} = x)) - test "case: does not share lexical in between clauses" do - assert expand_and_clean(quote do: (case w do 1 -> import List; 2 -> flatten([1,2,3]) end)) == - quote do: (case w() do 1 -> import :"Elixir.List", []; 2 -> flatten([1,2,3]) end) - end + message = ~r"expected struct name to be a compile time atom or alias" - test "case: expands guards" do - assert expand_and_clean(quote do: (case w do x when x when __ENV__.context -> true end)) == - quote do: (case w() do x when x when :guard -> true end) - end + assert_raise CompileError, message, fn -> + expand(quote(do: %unknown{a: 1})) + end - test "case: does not leaks vars on head" do - assert expand_and_clean(quote do: (case w do x -> x; y -> y end; :erlang.+(x, y))) == - quote do: (case w() do x -> x; y -> y end; :erlang.+(x(), y())) - end + message = ~r"expected struct name to be a compile time atom or alias" - test "case: leaks vars" do - assert expand_and_clean(quote do: (case w do x -> x = x; y -> y = y end; :erlang.+(x, y))) == - quote do: (case w() do x -> x = x; y -> y = y end; :erlang.+(x, y)) - end + assert_raise CompileError, message, fn -> + expand(quote(do: %unquote(1){a: 1})) + end - ## Receive + message = ~r"expected struct name in a match to be a compile time atom, alias or a variable" - test "receive: expands each clause" do - assert expand_and_clean(quote do: (receive do x -> x; _ -> x end)) == - quote do: (receive do x -> x; _ -> x() end) - end + assert_raise CompileError, message, fn -> + expand(quote(do: %unquote(1){a: 1} = x)) + end + end - test "receive: does not share lexical in between clauses" do - assert expand_and_clean(quote do: (receive do 1 -> import List; 2 -> flatten([1,2,3]) end)) == - quote do: (receive do 1 -> import :"Elixir.List", []; 2 -> flatten([1,2,3]) end) - end + test "update syntax" do + expand(quote(do: %{%{a: 0} | a: 1})) - test "receive: expands guards" do - assert expand_and_clean(quote do: (receive do x when x when __ENV__.context -> true end)) == - quote do: (receive do x when x when :guard -> true end) - end + assert_raise CompileError, ~r"cannot use map/struct update syntax in match", fn -> + expand(quote(do: %{%{a: 0} | a: 1} = %{})) + end + end - test "receive: does not leaks clause vars" do - assert expand_and_clean(quote do: (receive do x -> x; y -> y end; :erlang.+(x, y))) == - quote do: (receive do x -> x; y -> y end; :erlang.+(x(), y())) - end + test "dynamic syntax expands to itself" do + assert expand(quote(do: %x{} = 1)) == quote(do: %x{} = 1) + end - test "receive: leaks vars" do - assert expand_and_clean(quote do: (receive do x -> x = x; y -> y = y end; :erlang.+(x, y))) == - quote do: (receive do x -> x = x; y -> y = y end; :erlang.+(x, y)) - end + test "unknown ^keys in structs" do + message = ~r"unknown key \^my_key for struct Kernel\.ExpansionTest\.User" - test "receive: leaks vars on after" do - assert expand_and_clean(quote do: (receive do x -> x = x after y -> y; w = y end; :erlang.+(x, w))) == - quote do: (receive do x -> x = x after y() -> y(); w = y() end; :erlang.+(x, w)) + assert_raise CompileError, message, fn -> + code = + quote do + my_key = :my_key + %User{^my_key => :my_value} = %{} + end + + expand(code) + end + end end - ## Try + describe "quote" do + test "expanded to raw forms" do + assert expand(quote(do: quote(do: hello))) == {:{}, [], [:hello, [], __MODULE__]} + end - test "try: expands do" do - assert expand(quote do: (try do x = y end; x)) == - quote do: (try do x = y() end; x()) - end + test "raises if the :bind_quoted option is invalid" do + assert_raise CompileError, ~r"invalid :bind_quoted for quote", fn -> + expand(quote(do: quote(bind_quoted: self(), do: :ok))) + end - test "try: expands catch" do - assert expand(quote do: (try do x catch x, y -> z = :erlang.+(x, y) end; z)) == - quote do: (try do x() catch x, y -> z = :erlang.+(x, y) end; z()) - end + assert_raise CompileError, ~r"invalid :bind_quoted for quote", fn -> + expand(quote(do: quote(bind_quoted: [{1, 2}], do: :ok))) + end + end - test "try: expands after" do - assert expand(quote do: (try do x after z = y end; z)) == - quote do: (try do x() after z = y() end; z()) - end + test "raises for missing do" do + assert_raise CompileError, ~r"missing :do option in \"quote\"", fn -> + expand(quote(do: quote(context: Foo))) + end + end - test "try: expands else" do - assert expand(quote do: (try do x else z -> z end; z)) == - quote do: (try do x() else z -> z end; z()) + test "raises for invalid arguments" do + assert_raise CompileError, ~r"invalid arguments for \"quote\"", fn -> + expand(quote(do: quote(1 + 1))) + end + end + + test "raises unless its options are a keyword list" do + assert_raise CompileError, ~r"invalid options for quote, expected a keyword list", fn -> + expand(quote(do: quote(:foo, do: :foo))) + end + end end - test "try: expands rescue" do - assert expand(quote do: (try do x rescue x -> x; Error -> x end; x)) == - quote do: (try do x() rescue unquote(:in)(x, _) -> x; unquote(:in)(_, [:"Elixir.Error"]) -> x() end; x()) + describe "anonymous calls" do + test "expands base and args" do + assert expand(quote(do: a.(b))) == quote(do: a().(b())) + end + + test "raises on atom base" do + assert_raise CompileError, ~r"invalid function call :foo.()", fn -> + expand(quote(do: :foo.(a))) + end + end end - ## Binaries + describe "remotes" do + test "expands to Erlang" do + assert expand(quote(do: Kernel.is_atom(a))) == quote(do: :erlang.is_atom(a())) + end + + test "expands macros" do + assert expand(quote(do: Kernel.ExpansionTest.thirteen())) == 13 + end - test "bitstrings: expands modifiers" do - assert expand(quote do: (import Kernel.ExpansionTarget; << x :: seventeen >>)) == - quote do: (import :"Elixir.Kernel.ExpansionTarget", []; << x() :: [unquote(:size)(17)] >>) + test "expands receiver and args" do + assert expand(quote(do: a.is_atom(b))) == quote(do: a().is_atom(b())) - assert expand(quote do: (import Kernel.ExpansionTarget; << seventeen :: seventeen, x :: size(seventeen) >> = 1)) == - quote do: (import :"Elixir.Kernel.ExpansionTarget", []; - << seventeen :: [unquote(:size)(17)], x :: [unquote(:size)(seventeen)] >> = 1) - end + assert expand(quote(do: (after_expansion = :foo).is_atom(a))) == + quote(do: (after_expansion = :foo).is_atom(a())) + end + + test "modules must be required for macros" do + before_expansion = + quote do + require Kernel.ExpansionTarget + Kernel.ExpansionTarget.seventeen() + end - test "bitstrings: expands modifiers args" do - assert expand(quote do: (require Kernel.ExpansionTarget; << x :: size(Kernel.ExpansionTarget.seventeen) >>)) == - quote do: (require :"Elixir.Kernel.ExpansionTarget", []; << x() :: [unquote(:size)(17)] >>) + after_expansion = + quote do + :"Elixir.Kernel.ExpansionTarget" + 17 + end + + assert expand(before_expansion) == after_expansion + end + + test "in matches" do + message = ~r"cannot invoke remote function Hello.fun_that_does_not_exist/0 inside a match" + + assert_raise CompileError, message, fn -> + expand(quote(do: Hello.fun_that_does_not_exist() = :foo)) + end + + message = ~r"cannot invoke remote function :erlang.make_ref/0 inside a match" + assert_raise CompileError, message, fn -> expand(quote(do: make_ref() = :foo)) end + + message = ~r"invalid argument for \+\+ operator inside a match" + + assert_raise CompileError, message, fn -> + expand(quote(do: "a" ++ "b" = "ab")) + end + + assert_raise CompileError, message, fn -> + expand(quote(do: [1 | 2] ++ [3] = [1, 2, 3])) + end + + assert_raise CompileError, message, fn -> + expand(quote(do: [1] ++ 2 ++ [3] = [1, 2, 3])) + end + + assert {:=, _, [-1, {{:., _, [:erlang, :-]}, _, [1]}]} = expand(quote(do: -1 = -1)) + assert {:=, _, [1, {{:., _, [:erlang, :+]}, _, [1]}]} = expand(quote(do: +1 = +1)) + + assert {:=, _, [[{:|, _, [1, [{:|, _, [2, 3]}]]}], [1, 2, 3]]} = + expand(quote(do: [1] ++ [2] ++ 3 = [1, 2, 3])) + end + + test "in guards" do + message = + ~r"cannot invoke remote function Hello.something_that_does_not_exist/1 inside guard" + + assert_raise CompileError, message, fn -> + expand(quote(do: fn arg when Hello.something_that_does_not_exist(arg) -> arg end)) + end + + message = ~r"cannot invoke remote function :erlang.make_ref/0 inside guard" + + assert_raise CompileError, message, fn -> + expand(quote(do: fn arg when make_ref() -> arg end)) + end + end + + test "in guards with bitstrings" do + message = ~r"cannot invoke remote function String.Chars.to_string/1 inside guards" + + assert_raise CompileError, message, fn -> + expand(quote(do: fn arg when "#{arg}foo" == "argfoo" -> arg end)) + end + + assert_raise CompileError, message, fn -> + expand( + quote do + fn arg when <<:"Elixir.Kernel".to_string(arg)::binary, "foo">> == "argfoo" -> + arg + end + end + ) + end + end end - ## Invalid + describe "comprehensions" do + test "variables do not leak with enums" do + before_expansion = + quote do + for(a <- b, do: c = 1) + c + end - test "handles invalid expressions" do - assert_raise CompileError, ~r"invalid quoted expression: {1, 2, 3}", fn -> - expand(quote do: unquote({1, 2, 3})) + after_expansion = + quote do + for(a <- b(), do: c = 1) + c() + end + + assert expand(before_expansion) == after_expansion + end + + test "variables do not leak with binaries" do + before_expansion = + quote do + for(<>, do: c = 1) + c + end + + after_expansion = + quote do + for(<<(<> <- b())>>, do: c = 1) + c() + end + + assert expand(before_expansion) |> clean_meta([:alignment]) == after_expansion + end + + test "variables inside generator args do not leak" do + before_expansion = + quote do + for( + b <- + ( + a = 1 + [2] + ), + do: {a, b} + ) + + a + end + + after_expansion = + quote do + for( + b <- + ( + a = 1 + [2] + ), + do: {a(), b} + ) + + a() + end + + assert expand(before_expansion) == after_expansion + + before_expansion = + quote do + for( + b <- + ( + a = 1 + [2] + ), + d <- + ( + c = 3 + [4] + ), + do: {a, b, c, d} + ) + end + + after_expansion = + quote do + for( + b <- + ( + a = 1 + [2] + ), + d <- + ( + c = 3 + [4] + ), + do: {a(), b, c(), d} + ) + end + + assert expand(before_expansion) == after_expansion + end + + test "variables inside filters are available in blocks" do + assert expand(quote(do: for(a <- b, c = a, do: c))) == + quote(do: for(a <- b(), c = a, do: c)) + end + + test "variables inside options do not leak" do + before_expansion = + quote do + for(a <- c = b, into: [], do: 1) + c + end + + after_expansion = + quote do + for(a <- c = b(), do: 1, into: []) + c() + end + + assert expand(before_expansion) == after_expansion + + before_expansion = + quote do + for(a <- b, into: c = [], do: 1) + c + end + + after_expansion = + quote do + for(a <- b(), do: 1, into: c = []) + c() + end + + assert expand(before_expansion) == after_expansion end - assert_raise CompileError, ~r"invalid quoted expression: #Function<", fn -> - expand(quote do: unquote({:sample, fn -> end})) + test "must start with generators" do + assert_raise CompileError, ~r"for comprehensions must start with a generator", fn -> + expand(quote(do: for(is_atom(:foo), do: :foo))) + end + + assert_raise CompileError, ~r"for comprehensions must start with a generator", fn -> + expand(quote(do: for(do: :foo))) + end + end + + test "requires size on binary generators" do + message = ~r"a binary field without size is only allowed at the end of a binary pattern" + + assert_raise CompileError, message, fn -> + expand(quote(do: for(<>, do: x))) + end + end + + test "require do option" do + assert_raise CompileError, ~r"missing :do option in \"for\"", fn -> + expand(quote(do: for(_ <- 1..2))) + end + end + + test "uniq option is boolean" do + message = ~r":uniq option for comprehensions only accepts a boolean, got: x" + + assert_raise CompileError, message, fn -> + expand(quote(do: for(x <- 1..2, uniq: x, do: x))) + end + end + + test "raise error on invalid reduce" do + assert_raise CompileError, + ~r"cannot use :reduce alongside :into/:uniq in comprehension", + fn -> + expand(quote(do: for(x <- 1..3, reduce: %{}, into: %{}, do: (acc -> acc)))) + end + + assert_raise CompileError, + ~r"the do block was written using acc -> expr clauses but the :reduce option was not given", + fn -> expand(quote(do: for(x <- 1..3, do: (acc -> acc)))) end + + assert_raise CompileError, + ~r"when using :reduce with comprehensions, the do block must be written using acc -> expr clauses", + fn -> expand(quote(do: for(x <- 1..3, reduce: %{}, do: x))) end + + assert_raise CompileError, + ~r"when using :reduce with comprehensions, the do block must be written using acc -> expr clauses", + fn -> expand(quote(do: for(x <- 1..3, reduce: %{}, do: (acc, x -> x)))) end + end + + test "raise error for unknown options" do + assert_raise CompileError, ~r"unsupported option :else given to for", fn -> + expand(quote(do: for(_ <- 1..2, do: 1, else: 1))) + end + + assert_raise CompileError, ~r"unsupported option :other given to for", fn -> + expand(quote(do: for(_ <- 1..2, do: 1, other: 1))) + end end end - ## Helpers + describe "with" do + test "variables do not leak" do + before_expansion = + quote do + with({foo} <- {bar}, do: baz = :ok) + baz + end - defmacro thirteen do - 13 + after_expansion = + quote do + with({foo} <- {bar()}, do: baz = :ok) + baz() + end + + assert expand(before_expansion) == after_expansion + end + + test "variables inside args expression do not leak" do + before_expansion = + quote do + with( + b <- + ( + a = 1 + 2 + ), + do: {a, b} + ) + + a + end + + after_expansion = + quote do + with( + b <- + ( + a = 1 + 2 + ), + do: {a(), b} + ) + + a() + end + + assert expand(before_expansion) == after_expansion + + before_expansion = + quote do + with( + b <- + ( + a = 1 + 2 + ), + d <- + ( + c = 3 + 4 + ), + do: {a, b, c, d} + ) + end + + after_expansion = + quote do + with( + b <- + ( + a = 1 + 2 + ), + d <- + ( + c = 3 + 4 + ), + do: {a(), b, c(), d} + ) + end + + assert expand(before_expansion) == after_expansion + end + + test "variables are available in do option" do + before_expansion = + quote do + with({foo} <- {bar}, do: baz = foo) + baz + end + + after_expansion = + quote do + with({foo} <- {bar()}, do: baz = foo) + baz() + end + + assert expand(before_expansion) == after_expansion + end + + test "variables inside else do not leak" do + before_expansion = + quote do + with({foo} <- {bar}, do: :ok, else: (baz -> baz)) + baz + end + + after_expansion = + quote do + with({foo} <- {bar()}, do: :ok, else: (baz -> baz)) + baz() + end + + assert expand(before_expansion) == after_expansion + end + + test "fails if \"do\" is missing" do + assert_raise CompileError, ~r"missing :do option in \"with\"", fn -> + expand(quote(do: with(_ <- true, []))) + end + end + + test "fails on invalid else option" do + assert_raise CompileError, ~r"expected -> clauses for :else in \"with\"", fn -> + expand(quote(do: with(_ <- true, do: :ok, else: [:error]))) + end + + assert_raise CompileError, ~r"expected -> clauses for :else in \"with\"", fn -> + expand(quote(do: with(_ <- true, do: :ok, else: :error))) + end + + assert_raise CompileError, ~r"expected -> clauses for :else in \"with\"", fn -> + expand(quote(do: with(_ <- true, do: :ok, else: []))) + end + end + + test "fails for invalid options" do + # Only the required "do" is present alongside the unexpected option. + assert_raise CompileError, ~r"unexpected option :foo in \"with\"", fn -> + expand(quote(do: with(_ <- true, foo: :bar, do: :ok))) + end + + # More options are present alongside the unexpected option. + assert_raise CompileError, ~r"unexpected option :foo in \"with\"", fn -> + expand(quote(do: with(_ <- true, do: :ok, else: (_ -> :ok), foo: :bar))) + end + end end - defp expand_and_clean(expr) do - cleaner = &Keyword.drop(&1, [:export]) - expr - |> expand_env(__ENV__) - |> elem(0) - |> Macro.prewalk(&Macro.update_meta(&1, cleaner)) + describe "&" do + test "keeps locals" do + assert expand(quote(do: &unknown/2)) == {:&, [], [{:/, [], [{:unknown, [], nil}, 2]}]} + assert expand(quote(do: &unknown(&1, &2))) == {:&, [], [{:/, [], [{:unknown, [], nil}, 2]}]} + end + + test "expands remotes" do + assert expand(quote(do: &List.flatten/2)) == + quote(do: &:"Elixir.List".flatten/2) + |> clean_meta([:imports, :context, :no_parens]) + + assert expand(quote(do: &Kernel.is_atom/1)) == + quote(do: &:erlang.is_atom/1) |> clean_meta([:imports, :context, :no_parens]) + end + + test "expands macros" do + before_expansion = + quote do + require Kernel.ExpansionTarget + &Kernel.ExpansionTarget.seventeen/0 + end + + after_expansion = + quote do + :"Elixir.Kernel.ExpansionTarget" + fn -> 17 end + end + + assert expand(before_expansion) == after_expansion + end + + test "fails on non-continuous" do + assert_raise CompileError, ~r"capture argument &0 must be numbered between 1 and 255", fn -> + expand(quote(do: &foo(&0))) + end + + assert_raise CompileError, ~r"capture argument &2 cannot be defined without &1", fn -> + expand(quote(do: & &2)) + end + + assert_raise CompileError, ~r"capture argument &255 cannot be defined without &1", fn -> + expand(quote(do: & &255)) + end + end + + test "fails on block" do + message = ~r"block expressions are not allowed inside the capture operator &, got: 1\n2" + + assert_raise CompileError, message, fn -> + code = + quote do + &( + 1 + 2 + ) + end + + expand(code) + end + end + + test "fails on other types" do + assert_raise CompileError, ~r"invalid args for &, expected one of:", fn -> + expand(quote(do: &:foo)) + end + end + + test "fails on invalid arity" do + message = ~r"capture argument &256 must be numbered between 1 and 255" + + assert_raise CompileError, message, fn -> + expand(quote(do: &Mod.fun/256)) + end + end + + test "fails when no captures" do + assert_raise CompileError, ~r"invalid args for &, expected one of:", fn -> + expand(quote(do: &foo())) + end + end + + test "fails on nested capture" do + assert_raise CompileError, ~r"nested captures are not allowed", fn -> + expand(quote(do: &(& &1))) + end + end + + test "fails on integers" do + assert_raise CompileError, + ~r"capture argument &1 must be used within the capture operator &", + fn -> expand(quote(do: &1)) end + end end - defp expand(expr) do - expand_env(expr, __ENV__) |> elem(0) + describe "fn" do + test "expands each clause" do + before_expansion = + quote do + fn + x -> x + _ -> x + end + end + + after_expansion = + quote do + fn + x -> x + _ -> x() + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not share lexical scope between clauses" do + before_expansion = + quote do + fn + 1 -> import List + 2 -> flatten([1, 2, 3]) + end + end + + after_expansion = + quote do + fn + 1 -> :"Elixir.List" + 2 -> flatten([1, 2, 3]) + end + end + + assert expand(before_expansion) == after_expansion + end + + test "expands guards" do + assert expand(quote(do: fn x when x when __ENV__.context -> true end)) == + quote(do: fn x when x when :guard -> true end) + end + + test "does not leak vars" do + before_expansion = + quote do + fn x -> x end + x + end + + after_expansion = + quote do + fn x -> x end + x() + end + + assert expand(before_expansion) == after_expansion + end + + test "raises on mixed arities" do + message = ~r"cannot mix clauses with different arities in anonymous functions" + + assert_raise CompileError, message, fn -> + code = + quote do + fn + x -> x + x, y -> x + y + end + end + + expand(code) + end + end end - defp expand_env(expr, env) do - :elixir_exp.expand(expr, env) + describe "cond" do + test "expands each clause" do + before_expansion = + quote do + cond do + x = 1 -> x + true -> x + end + end + + after_expansion = + quote do + cond do + x = 1 -> x + true -> x() + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not share lexical scope between clauses" do + before_expansion = + quote do + cond do + 1 -> import List + 2 -> flatten([1, 2, 3]) + end + end + + after_expansion = + quote do + cond do + 1 -> :"Elixir.List" + 2 -> flatten([1, 2, 3]) + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leaks vars on head" do + before_expansion = + quote do + cond do + x = 1 -> x + y = 2 -> y + end + + :erlang.+(x, y) + end + + after_expansion = + quote do + cond do + x = 1 -> x + y = 2 -> y + end + + :erlang.+(x(), y()) + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leak vars" do + before_expansion = + quote do + cond do + 1 -> x = 1 + 2 -> y = 2 + end + + :erlang.+(x, y) + end + + after_expansion = + quote do + cond do + 1 -> x = 1 + 2 -> y = 2 + end + + :erlang.+(x(), y()) + end + + assert expand(before_expansion) == after_expansion + end + + test "expects exactly one do" do + assert_raise CompileError, ~r"missing :do option in \"cond\"", fn -> + expand(quote(do: cond([]))) + end + + assert_raise CompileError, ~r"duplicated :do clauses given for \"cond\"", fn -> + expand(quote(do: cond(do: (x -> x), do: (y -> y)))) + end + end + + test "expects clauses" do + assert_raise CompileError, ~r"expected -> clauses for :do in \"cond\"", fn -> + expand(quote(do: cond(do: :ok))) + end + + assert_raise CompileError, ~r"expected -> clauses for :do in \"cond\"", fn -> + expand(quote(do: cond(do: [:not, :clauses]))) + end + end + + test "expects one argument in clauses" do + assert_raise CompileError, + ~r"expected one argument for :do clauses \(->\) in \"cond\"", + fn -> + code = + quote do + cond do + _, _ -> :ok + end + end + + expand(code) + end + end + + test "raises for invalid arguments" do + assert_raise CompileError, ~r"invalid arguments for \"cond\"", fn -> + expand(quote(do: cond(:foo))) + end + end + + test "raises with invalid options" do + assert_raise CompileError, ~r"unexpected option :foo in \"cond\"", fn -> + expand(quote(do: cond(do: (1 -> 1), foo: :bar))) + end + end + + test "raises for _ in clauses" do + message = ~r"invalid use of _ inside \"cond\"\. If you want the last clause" + + assert_raise CompileError, message, fn -> + code = + quote do + cond do + x -> x + _ -> :raise + end + end + + expand(code) + end + end + end + + describe "case" do + test "expands each clause" do + before_expansion = + quote do + case w do + x -> x + _ -> x + end + end + + after_expansion = + quote do + case w() do + x -> x + _ -> x() + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not share lexical scope between clauses" do + before_expansion = + quote do + case w do + 1 -> import List + 2 -> flatten([1, 2, 3]) + end + end + + after_expansion = + quote do + case w() do + 1 -> :"Elixir.List" + 2 -> flatten([1, 2, 3]) + end + end + + assert expand(before_expansion) == after_expansion + end + + test "expands guards" do + before_expansion = + quote do + case w do + x when x when __ENV__.context -> true + end + end + + after_expansion = + quote do + case w() do + x when x when :guard -> true + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leaks vars on head" do + before_expansion = + quote do + case w do + x -> x + y -> y + end + + :erlang.+(x, y) + end + + after_expansion = + quote do + case w() do + x -> x + y -> y + end + + :erlang.+(x(), y()) + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leak vars" do + before_expansion = + quote do + case w do + x -> x = x + y -> y = y + end + + :erlang.+(x, y) + end + + after_expansion = + quote do + case w() do + x -> x = x + y -> y = y + end + + :erlang.+(x(), y()) + end + + assert expand(before_expansion) == after_expansion + end + + test "expects exactly one do" do + assert_raise CompileError, ~r"missing :do option in \"case\"", fn -> + expand(quote(do: case(e, []))) + end + + assert_raise CompileError, ~r"duplicated :do clauses given for \"case\"", fn -> + expand(quote(do: case(e, do: (x -> x), do: (y -> y)))) + end + end + + test "expects clauses" do + assert_raise CompileError, ~r"expected -> clauses for :do in \"case\"", fn -> + code = + quote do + case e do + x + end + end + + expand(code) + end + + assert_raise CompileError, ~r"expected -> clauses for :do in \"case\"", fn -> + code = + quote do + case e do + [:not, :clauses] + end + end + + expand(code) + end + end + + test "expects exactly one argument in clauses" do + assert_raise CompileError, + ~r"expected one argument for :do clauses \(->\) in \"case\"", + fn -> + code = + quote do + case e do + _, _ -> :ok + end + end + + expand(code) + end + end + + test "fails with invalid arguments" do + assert_raise CompileError, ~r"invalid arguments for \"case\"", fn -> + expand(quote(do: case(:foo, :bar))) + end + end + + test "fails for invalid options" do + assert_raise CompileError, ~r"unexpected option :foo in \"case\"", fn -> + expand(quote(do: case(e, do: (x -> x), foo: :bar))) + end + end + end + + describe "receive" do + test "expands each clause" do + before_expansion = + quote do + receive do + x -> x + _ -> x + end + end + + after_expansion = + quote do + receive do + x -> x + _ -> x() + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not share lexical scope between clauses" do + before_expansion = + quote do + receive do + 1 -> import List + 2 -> flatten([1, 2, 3]) + end + end + + after_expansion = + quote do + receive do + 1 -> :"Elixir.List" + 2 -> flatten([1, 2, 3]) + end + end + + assert expand(before_expansion) == after_expansion + end + + test "expands guards" do + before_expansion = + quote do + receive do + x when x when __ENV__.context -> true + end + end + + after_expansion = + quote do + receive do + x when x when :guard -> true + end + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leaks clause vars" do + before_expansion = + quote do + receive do + x -> x + y -> y + end + + :erlang.+(x, y) + end + + after_expansion = + quote do + receive do + x -> x + y -> y + end + + :erlang.+(x(), y()) + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leak vars" do + before_expansion = + quote do + receive do + x -> x = x + y -> y = y + end + + :erlang.+(x, y) + end + + after_expansion = + quote do + receive do + x -> x = x + y -> y = y + end + + :erlang.+(x(), y()) + end + + assert expand(before_expansion) == after_expansion + end + + test "does not leak vars on after" do + before_expansion = + quote do + receive do + x -> x = x + after + y -> + y + w = y + end + + :erlang.+(x, w) + end + + after_expansion = + quote do + receive do + x -> x = x + after + y() -> + y() + w = y() + end + + :erlang.+(x(), w()) + end + + assert expand(before_expansion) == after_expansion + end + + test "expects exactly one do or after" do + assert_raise CompileError, ~r"missing :do/:after option in \"receive\"", fn -> + expand(quote(do: receive([]))) + end + + assert_raise CompileError, ~r"duplicated :do clauses given for \"receive\"", fn -> + expand(quote(do: receive(do: (x -> x), do: (y -> y)))) + end + + assert_raise CompileError, ~r"duplicated :after clauses given for \"receive\"", fn -> + code = + quote do + receive do + x -> x + after + y -> y + after + z -> z + end + end + + expand(code) + end + end + + test "expects clauses" do + assert_raise CompileError, ~r"expected -> clauses for :do in \"receive\"", fn -> + code = + quote do + receive do + x + end + end + + expand(code) + end + + assert_raise CompileError, ~r"expected -> clauses for :do in \"receive\"", fn -> + code = + quote do + receive do + [:not, :clauses] + end + end + + expand(code) + end + end + + test "expects on argument for do/after clauses" do + assert_raise CompileError, + ~r"expected one argument for :do clauses \(->\) in \"receive\"", + fn -> + code = + quote do + receive do + _, _ -> :ok + end + end + + expand(code) + end + + message = ~r"expected one argument for :after clauses \(->\) in \"receive\"" + + assert_raise CompileError, message, fn -> + code = + quote do + receive do + x -> x + after + _, _ -> :ok + end + end + + expand(code) + end + end + + test "expects a single clause for \"after\"" do + assert_raise CompileError, ~r"expected a single -> clause for :after in \"receive\"", fn -> + code = + quote do + receive do + x -> x + after + 1 -> y + 2 -> z + end + end + + expand(code) + end + end + + test "raises for invalid arguments" do + assert_raise CompileError, ~r"invalid arguments for \"receive\"", fn -> + expand(quote(do: receive(:foo))) + end + end + + test "raises with invalid options" do + assert_raise CompileError, ~r"unexpected option :foo in \"receive\"", fn -> + expand(quote(do: receive(do: (x -> x), foo: :bar))) + end + end + end + + describe "try" do + test "expands catch" do + before_expansion = + quote do + try do + x + catch + x, y -> z = :erlang.+(x, y) + end + + z + end + + after_expansion = + quote do + try do + x() + catch + x, y -> z = :erlang.+(x, y) + end + + z() + end + + assert expand(before_expansion) == after_expansion + end + + test "expands after" do + before_expansion = + quote do + try do + x + after + z = y + end + + z + end + + after_expansion = + quote do + try do + x() + after + z = y() + end + + z() + end + + assert expand(before_expansion) == after_expansion + end + + test "expands else" do + before_expansion = + quote do + try do + x + catch + _, _ -> :ok + else + z -> z + end + + z + end + + after_expansion = + quote do + try do + x() + catch + _, _ -> :ok + else + z -> z + end + + z() + end + + assert expand(before_expansion) == after_expansion + end + + test "expands rescue" do + before_expansion = + quote do + try do + x + rescue + x -> x + Error -> x + end + + x + end + + after_expansion = + quote do + try do + x() + rescue + x -> x + unquote(:in)(_, [:"Elixir.Error"]) -> x() + end + + x() + end + + assert expand(before_expansion) == after_expansion + end + + test "expects more than do" do + assert_raise CompileError, ~r"missing :catch/:rescue/:after option in \"try\"", fn -> + code = + quote do + try do + x = y + end + + x + end + + expand(code) + end + end + + test "raises if do is missing" do + assert_raise CompileError, ~r"missing :do option in \"try\"", fn -> + expand(quote(do: try([]))) + end + end + + test "expects at most one clause" do + assert_raise CompileError, ~r"duplicated :do clauses given for \"try\"", fn -> + expand(quote(do: try(do: e, do: f))) + end + + assert_raise CompileError, ~r"duplicated :rescue clauses given for \"try\"", fn -> + code = + quote do + try do + e + rescue + x -> x + rescue + y -> y + end + end + + expand(code) + end + + assert_raise CompileError, ~r"duplicated :after clauses given for \"try\"", fn -> + code = + quote do + try do + e + after + x = y + after + x = y + end + end + + expand(code) + end + + assert_raise CompileError, ~r"duplicated :else clauses given for \"try\"", fn -> + code = + quote do + try do + e + else + x -> x + else + y -> y + end + end + + expand(code) + end + + assert_raise CompileError, ~r"duplicated :catch clauses given for \"try\"", fn -> + code = + quote do + try do + e + catch + x -> x + catch + y -> y + end + end + + expand(code) + end + end + + test "raises with invalid arguments" do + assert_raise CompileError, ~r"invalid arguments for \"try\"", fn -> + expand(quote(do: try(:foo))) + end + end + + test "raises with invalid options" do + assert_raise CompileError, ~r"unexpected option :foo in \"try\"", fn -> + expand(quote(do: try(do: x, foo: :bar))) + end + end + + test "expects exactly one argument in rescue clauses" do + assert_raise CompileError, + ~r"expected one argument for :rescue clauses \(->\) in \"try\"", + fn -> + code = + quote do + try do + x + rescue + _, _ -> :ok + end + end + + expand(code) + end + end + + test "expects an alias, a variable, or \"var in [alias]\" as the argument of rescue clauses" do + assert_raise CompileError, ~r"invalid \"rescue\" clause\. The clause should match", fn -> + code = + quote do + try do + x + rescue + function(:call) -> :ok + end + end + + expand(code) + end + end + + test "expects one or two args for catch clauses" do + message = ~r"expected one or two args for :catch clauses \(->\) in \"try\"" + + assert_raise CompileError, message, fn -> + code = + quote do + try do + x + catch + _, _, _ -> :ok + end + end + + expand(code) + end + end + + test "expects clauses for rescue, else, catch" do + assert_raise CompileError, ~r"expected -> clauses for :rescue in \"try\"", fn -> + code = + quote do + try do + e + rescue + x + end + end + + expand(code) + end + + assert_raise CompileError, ~r"expected -> clauses for :rescue in \"try\"", fn -> + code = + quote do + try do + e + rescue + [:not, :clauses] + end + end + + expand(code) + end + + assert_raise CompileError, ~r"expected -> clauses for :rescue in \"try\"", fn -> + code = + quote do + try do + e + rescue + [] + end + end + + expand(code) + end + + assert_raise CompileError, ~r"expected -> clauses for :catch in \"try\"", fn -> + code = + quote do + try do + e + catch + x + end + end + + expand(code) + end + + assert_raise CompileError, ~r"expected -> clauses for :catch in \"try\"", fn -> + code = + quote do + try do + e + catch + [:not, :clauses] + end + end + + expand(code) + end + + assert_raise CompileError, ~r"expected -> clauses for :catch in \"try\"", fn -> + code = + quote do + try do + e + catch + [] + end + end + + expand(code) + end + + assert_raise CompileError, ~r"expected -> clauses for :else in \"try\"", fn -> + code = + quote do + try do + e + catch + _ -> :ok + else + x + end + end + + expand(code) + end + + assert_raise CompileError, ~r"expected -> clauses for :else in \"try\"", fn -> + code = + quote do + try do + e + catch + _ -> :ok + else + [:not, :clauses] + end + end + + expand(code) + end + + assert_raise CompileError, ~r"expected -> clauses for :else in \"try\"", fn -> + code = + quote do + try do + e + catch + _ -> :ok + else + [] + end + end + + expand(code) + end + end + end + + describe "bitstrings" do + test "parallel match" do + assert expand(quote(do: <> = <>)) |> clean_meta([:alignment]) == + quote(do: <> = <>) + + assert expand(quote(do: <> = baz = <>)) |> clean_meta([:alignment]) == + quote(do: <> = baz = <>) + + assert expand(quote(do: <> = {<>} = bar())) |> clean_meta([:alignment]) == + quote(do: <> = {<>} = bar()) + + message = ~r"binary patterns cannot be matched in parallel using \"=\"" + + assert_raise CompileError, message, fn -> + expand(quote(do: <> = <> = bar())) + end + + assert_raise CompileError, message, fn -> + expand(quote(do: <> = qux = <> = bar())) + end + + assert_raise CompileError, message, fn -> + expand(quote(do: {<>} = {qux} = {<>} = bar())) + end + + assert expand(quote(do: {:foo, <>} = {<>, :baz} = bar())) + + # two-element tuples are special cased + assert_raise CompileError, message, fn -> + expand(quote(do: {:foo, <>} = {:foo, <>} = bar())) + end + + assert_raise CompileError, message, fn -> + expand(quote(do: %{foo: <>} = %{baz: <>, foo: <>} = bar())) + end + + assert expand(quote(do: %{foo: <>} = %{baz: <>} = bar())) + + assert_raise CompileError, message, fn -> + expand(quote(do: %_{foo: <>} = %_{foo: <>} = bar())) + end + + assert expand(quote(do: %_{foo: <>} = %_{baz: <>} = bar())) + + assert_raise CompileError, message, fn -> + expand(quote(do: %_{foo: <>} = %{foo: <>} = bar())) + end + + assert expand(quote(do: %_{foo: <>} = %{baz: <>} = bar())) + + assert_raise CompileError, message, fn -> + code = + quote do + case bar() do + <> = <> -> nil + end + end + + expand(code) + end + + assert_raise CompileError, message, fn -> + code = + quote do + case bar() do + <> = qux = <> -> nil + end + end + + expand(code) + end + + assert_raise CompileError, message, fn -> + code = + quote do + case bar() do + [<>] = [<>] -> nil + end + end + + expand(code) + end + end + + test "nested match" do + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <> = rest()::binary>>)) + |> clean_meta([:alignment]) == + quote(do: <<45::integer(), <<_::integer(), _::binary()>> = rest()::binary()>>) + + message = ~r"cannot pattern match inside a bitstring that is already in match" + + assert_raise CompileError, message, fn -> + expand(quote(do: <> = foo())) + end + + assert_raise CompileError, message, fn -> + expand(quote(do: <> = rest::binary>> = foo())) + end + end + + test "inlines binaries inside interpolation" do + import Kernel.ExpansionTarget + + # Check expansion happens only once + assert expand(quote(do: "foo#{message_hello("bar")}")) |> clean_meta([:alignment]) == + quote(do: <<"foo"::binary(), "bar"::binary()>>) + + assert_received :hello + refute_received :hello + + # And it also works in match + assert expand(quote(do: "foo#{bar()}" = "foobar")) |> clean_meta([:alignment]) == + quote(do: <<"foo"::binary(), "bar"::binary()>> = "foobar") + end + + test "inlines binaries inside interpolation is isomorphic after manual expansion" do + import Kernel.ExpansionTarget + + quoted = Macro.prewalk(quote(do: "foo#{bar()}" = "foobar"), &Macro.expand(&1, __ENV__)) + + assert expand(quoted) |> clean_meta([:alignment]) == + quote(do: <<"foo"::binary(), "bar"::binary()>> = "foobar") + end + + test "expands size * unit" do + import Kernel, except: [-: 2] + import Kernel.ExpansionTarget + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + end + + test "expands binary/bitstring specifiers" do + import Kernel, except: [-: 2] + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + message = ~r"signed and unsigned specifiers are supported only on integer and float type" + + assert_raise CompileError, message, fn -> + expand(quote(do: <>)) + end + end + + test "expands utf* specifiers" do + import Kernel, except: [-: 2] + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + message = ~r"signed and unsigned specifiers are supported only on integer and float type" + + assert_raise CompileError, message, fn -> + expand(quote(do: <>)) + end + + assert_raise CompileError, ~r"size and unit are not supported on utf types", fn -> + expand(quote(do: <>)) + end + end + + test "expands numbers specifiers" do + import Kernel, except: [-: 2] + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + message = + ~r"integer and float types require a size specifier if the unit specifier is given" + + assert_raise CompileError, message, fn -> + expand(quote(do: <>)) + end + end + + test "expands macro specifiers" do + import Kernel, except: [-: 2] + import Kernel.ExpansionTarget + + assert expand(quote(do: <>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <> = 1)) + |> clean_meta([:alignment]) == + quote(do: <> = 1) + end + + test "expands macro in args" do + import Kernel, except: [-: 2] + + before_expansion = + quote do + require Kernel.ExpansionTarget + <> + end + + after_expansion = + quote do + :"Elixir.Kernel.ExpansionTarget" + <> + end + + assert expand(before_expansion) |> clean_meta([:alignment]) == after_expansion + end + + test "supports dynamic size" do + import Kernel, except: [-: 2] + + before_expansion = + quote do + var = 1 + <> + end + + after_expansion = + quote do + var = 1 + <> + end + + assert expand(before_expansion) |> clean_meta([:alignment]) == after_expansion + end + + defmacro offset(size, binary) do + quote do + offset = unquote(size) + <<_::size(offset)>> = unquote(binary) + end + end + + test "supports size from counters" do + assert offset(8, <<0>>) + end + + test "merges bitstrings" do + import Kernel, except: [-: 2] + + assert expand(quote(do: <>, z>>)) |> clean_meta([:alignment]) == + quote(do: <>) + + assert expand(quote(do: <>::bitstring, z>>)) + |> clean_meta([:alignment]) == + quote(do: <>) + end + + test "merges binaries" do + import Kernel, except: [-: 2] + + assert expand(quote(do: "foo" <> x)) |> clean_meta([:alignment]) == + quote(do: <<"foo"::binary(), x()::binary()>>) + + assert expand(quote(do: "foo" <> <>)) |> clean_meta([:alignment]) == + quote(do: <<"foo"::binary(), x()::integer()-size(4), y()::integer()-size(4)>>) + + assert expand(quote(do: <<"foo", <>::binary>>)) + |> clean_meta([:alignment]) == + quote(do: <<"foo"::binary(), x()::integer()-size(4), y()::integer()-size(4)>>) + end + + test "guard expressions on size" do + import Kernel, except: [-: 2, +: 2, length: 1] + + # Arithmetic operations with literals and variables are valid expressions + # for bitstring size in OTP 23+ + + before_expansion = + quote do + var = 1 + <> + end + + after_expansion = + quote do + var = 1 + <> + end + + assert expand(before_expansion) |> clean_meta([:alignment]) == after_expansion + + # Other valid guard expressions are also legal for bitstring size in OTP 23+ + + before_expansion = quote(do: <>) + after_expansion = quote(do: <>) + + assert expand(before_expansion) |> clean_meta([:alignment]) == after_expansion + end + + test "map lookup on size" do + import Kernel, except: [-: 2] + + before_expansion = + quote do + var = %{foo: 3} + <> + end + + after_expansion = + quote do + var = %{foo: 3} + <> + end + + assert expand(before_expansion) |> clean_meta([:alignment]) == after_expansion + end + + test "raises on unaligned binaries in match" do + message = ~r"its number of bits is not divisible by 8" + + assert_raise CompileError, message, fn -> + expand(quote(do: <> <> _ = "foo")) + end + + assert_raise CompileError, message, fn -> + expand(quote(do: <<1::4>> <> "foo")) + end + end + + test "raises on size or unit for literal bitstrings" do + message = ~r"literal <<>> in bitstring supports only type specifiers" + + assert_raise CompileError, message, fn -> + expand(quote(do: <<(<<"foo">>)::32>>)) + end + end + + test "raises on size or unit for literal strings" do + message = ~r"literal string in bitstring supports only endianness and type specifiers" + + assert_raise CompileError, message, fn -> + expand(quote(do: <<"foo"::32>>)) + end + end + + # TODO: Simplify when we require Erlang/OTP 24 + if System.otp_release() >= "24" do + test "16-bit floats" do + import Kernel, except: [-: 2] + + assert expand(quote(do: <<12.3::float-16>>)) |> clean_meta([:alignment]) == + quote(do: <<12.3::float()-size(16)>>) + end + + test "raises for invalid size * unit for floats" do + message = ~r"float requires size\*unit to be 16, 32, or 64 \(default\), got: 128" + + assert_raise CompileError, message, fn -> + expand(quote(do: <<12.3::32*4>>)) + end + + message = ~r"float requires size\*unit to be 16, 32, or 64 \(default\), got: 256" + + assert_raise CompileError, message, fn -> + expand(quote(do: <<12.3::256>>)) + end + end + else + test "16-bit floats" do + message = ~r"float requires size\*unit to be 32 or 64 \(default\), got: 16" + + assert_raise CompileError, message, fn -> + expand(quote(do: <<12.3::16>>)) + end + end + + test "raises for invalid size * unit for floats" do + message = ~r"float requires size\*unit to be 32 or 64 \(default\), got: 128" + + assert_raise CompileError, message, fn -> + expand(quote(do: <<12.3::32*4>>)) + end + + message = ~r"float requires size\*unit to be 32 or 64 \(default\), got: 256" + + assert_raise CompileError, message, fn -> + expand(quote(do: <<12.3::256>>)) + end + end + end + + test "raises for invalid size" do + assert_raise CompileError, ~r/undefined variable "foo"/, fn -> + code = + quote do + fn <<_::size(foo)>> -> :ok end + end + + expand(code) + end + + assert_raise CompileError, ~r/undefined variable "foo"/, fn -> + code = + quote do + fn <<_::size(foo), foo::size(8)>> -> :ok end + end + + expand(code) + end + + assert_raise CompileError, ~r/undefined variable "foo"/, fn -> + code = + quote do + fn foo, <<_::size(foo)>> -> :ok end + end + + expand(code) + end + + assert_raise CompileError, ~r/undefined variable "foo"/, fn -> + code = + quote do + fn foo, <<_::size(foo + 1)>> -> :ok end + end + + expand(code) + end + + message = ~r"cannot find or invoke local foo/0 inside bitstring size specifier" + + assert_raise CompileError, message, fn -> + code = + quote do + fn <<_::size(foo())>> -> :ok end + end + + expand(code) + end + + message = ~r"anonymous call is not allowed in bitstring size specifier" + + assert_raise CompileError, message, fn -> + code = + quote do + fn <<_::size(foo.())>> -> :ok end + end + + expand(code) + end + + message = ~r"cannot invoke remote function in bitstring size specifier" + + assert_raise CompileError, message, fn -> + code = + quote do + foo = %{bar: true} + fn <<_::size(foo.bar())>> -> :ok end + end + + expand(code) + end + + message = ~r"cannot invoke remote function Foo.bar/0 inside bitstring size specifier" + + assert_raise CompileError, message, fn -> + code = + quote do + fn <<_::size(Foo.bar())>> -> :ok end + end + + expand(code) + end + end + + test "raises for invalid unit" do + message = ~r"unit in bitstring expects an integer as argument, got: :oops" + + assert_raise CompileError, message, fn -> + expand(quote(do: <<"foo"::size(8)-unit(:oops)>>)) + end + end + + test "raises for unknown specifier" do + assert_raise CompileError, ~r"unknown bitstring specifier: unknown()", fn -> + expand(quote(do: <<1::unknown>>)) + end + end + + test "raises for conflicting specifiers" do + assert_raise CompileError, ~r"conflicting endianness specification for bit field", fn -> + expand(quote(do: <<1::little-big>>)) + end + + assert_raise CompileError, ~r"conflicting unit specification for bit field", fn -> + expand(quote(do: <>)) + end + end + + test "raises for invalid literals" do + assert_raise CompileError, ~r"invalid literal :foo in <<>>", fn -> + expand(quote(do: <<:foo>>)) + end + + assert_raise CompileError, ~r"invalid literal \[\] in <<>>", fn -> + expand(quote(do: <<[]::size(8)>>)) + end + end + + test "raises on binary fields with size in matches" do + assert expand(quote(do: <> = "foobar")) + + message = ~r"a binary field without size is only allowed at the end of a binary pattern" + + assert_raise CompileError, message, fn -> + expand(quote(do: <> = "foobar")) + end + + assert_raise CompileError, message, fn -> + expand(quote(do: <<(<>), y::binary>> = "foobar")) + end + + assert_raise CompileError, message, fn -> + expand(quote(do: <<(<>), y::bitstring>> = "foobar")) + end + + assert_raise CompileError, message, fn -> + expand(quote(do: <<(<>)::bitstring, y::bitstring>> = "foobar")) + end + end + end + + describe "op ambiguity" do + test "raises when a call is ambiguous" do + # We use string_to_quoted! here to avoid the formatter adding parentheses + message = ~r["a -1" looks like a function call but there is a variable named "a"] + + assert_raise CompileError, message, fn -> + code = + Code.string_to_quoted!(""" + a = 1 + a -1 + """) + + expand(code) + end + + message = + ~r["a -1\.\.\(a \+ 1\)" looks like a function call but there is a variable named "a"] + + assert_raise CompileError, message, fn -> + code = + Code.string_to_quoted!(""" + a = 1 + a -1 .. a + 1 + """) + + expand(code) + end + end + end + + test "handles invalid expressions" do + assert_raise CompileError, ~r"invalid quoted expression: {1, 2, 3}", fn -> + expand(quote(do: unquote({1, 2, 3}))) + end + + assert_raise CompileError, ~r"invalid quoted expression: #Function\<", fn -> + expand(quote(do: unquote({:sample, fn -> nil end}))) + end + + assert_raise CompileError, ~r"invalid pattern in match", fn -> + code = + quote do + x = & &1 + + case true do + x.(false) -> true + end + end + + expand(code) + end + + assert_raise CompileError, ~r"anonymous call is not allowed in guards", fn -> + code = + quote do + x = & &1 + + case true do + true when x.(true) -> true + end + end + + expand(code) + end + + assert_raise CompileError, ~r"invalid call foo\(1\)\(2\)", fn -> + expand(quote(do: foo(1)(2))) + end + + assert_raise CompileError, ~r"invalid call 1\.foo", fn -> + expand(quote(do: 1.foo)) + end + + assert_raise CompileError, ~r"invalid call 0\.foo", fn -> + expand(quote(do: __ENV__.line.foo)) + end + + assert_raise CompileError, ~r"unhandled operator ->", fn -> + expand(quote(do: (foo -> bar))) + end + + message = ~r/"wrong_fun" cannot handle clauses with the ->/ + + assert_raise CompileError, message, fn -> + code = + quote do + wrong_fun do + _ -> :ok + end + end + + expand(code) + end + + assert_raise CompileError, message, fn -> + code = + quote do + wrong_fun do + foo -> bar + after + :ok + end + end + + expand(code) + end + + assert_raise CompileError, ~r/"length" cannot handle clauses with the ->/, fn -> + code = + quote do + length do + _ -> :ok + end + end + + expand(code) + end + end + + ## Helpers + + defmacro thirteen do + 13 + end + + defp clean_meta(expr, vars) do + cleaner = &Keyword.drop(&1, vars) + Macro.prewalk(expr, &Macro.update_meta(&1, cleaner)) + end + + defp expand(expr) do + expand_env(expr, __ENV__) |> elem(0) + end + + defp expand_env(expr, env) do + ExUnit.CaptureIO.capture_io(:stderr, fn -> + send(self(), {:expand_env, :elixir_expand.expand(expr, :elixir_env.env_to_ex(env), env)}) + end) + + receive do + {:expand_env, {expr, scope, env}} -> + env = :elixir_env.to_caller({env.line, scope, env}) + {clean_meta(expr, [:version, :inferred_bitstring_spec]), env} + end end end diff --git a/lib/elixir/test/elixir/kernel/fn_test.exs b/lib/elixir/test/elixir/kernel/fn_test.exs index 5e934d4ca9d..2a158230a46 100644 --- a/lib/elixir/test/elixir/kernel/fn_test.exs +++ b/lib/elixir/test/elixir/kernel/fn_test.exs @@ -1,47 +1,77 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.FnTest do use ExUnit.Case, async: true - import CompileAssertion test "arithmetic constants on match" do - assert (fn 1 + 2 -> :ok end).(3) == :ok - assert (fn 1 - 2 -> :ok end).(-1) == :ok - assert (fn -1 -> :ok end).(-1) == :ok - assert (fn +1 -> :ok end).(1) == :ok + assert (fn -1 -> true end).(-1) + assert (fn +1 -> true end).(1) + end + + defp fun_match(x) do + fn + ^x -> true + _ -> false + end + end + + test "pin operator on match" do + refute fun_match(1).(0) + assert fun_match(1).(1) + refute fun_match(1).(1.0) + end + + test "guards with no args" do + fun = fn () when node() == :nonode@nohost -> true end + assert is_function(fun, 0) + end + + test "case function hoisting does not affect anonymous fns" do + result = + if atom?(0) do + user = :defined + user + else + (fn -> + user = :undefined + user + end).() + end + + assert result == :undefined end test "capture with access" do - assert (&(&1[:hello])).([hello: :world]) == :world + assert (& &1[:hello]).(hello: :world) == :world end test "capture remote" do assert (&:erlang.atom_to_list/1).(:a) == 'a' - assert (&Atom.to_char_list/1).(:a) == 'a' + assert (&Atom.to_charlist/1).(:a) == 'a' assert (&List.flatten/1).([[0]]) == [0] - assert (&(List.flatten/1)).([[0]]) == [0] + assert (&List.flatten/1).([[0]]) == [0] assert (&List.flatten(&1)).([[0]]) == [0] assert (&List.flatten(&1)) == (&List.flatten/1) end test "capture local" do assert (&atl/1).(:a) == 'a' - assert (&(atl/1)).(:a) == 'a' + assert (&atl/1).(:a) == 'a' assert (&atl(&1)).(:a) == 'a' end test "capture local with question mark" do - assert (&is_a?/2).(:atom, :a) - assert (&(is_a?/2)).(:atom, :a) - assert (&is_a?(&1, &2)).(:atom, :a) + assert (&atom?/1).(:a) + assert (&atom?/1).(:a) + assert (&atom?(&1)).(:a) end test "capture imported" do assert (&is_atom/1).(:a) - assert (&(is_atom/1)).(:a) + assert (&is_atom/1).(:a) assert (&is_atom(&1)).(:a) - assert (&is_atom(&1)) == &is_atom/1 + assert (&is_atom(&1)) == (&is_atom/1) end test "capture macro" do @@ -52,16 +82,21 @@ defmodule Kernel.FnTest do end test "capture operator" do - assert is_function &+/2 - assert is_function &(&&/2) - assert is_function & &1 + &2, 2 + assert is_function(&+/2) + assert is_function(& &&/2) + assert is_function(&(&1 + &2), 2) + assert is_function(&and/2) + end + + test "capture precedence in cons" do + assert [(&IO.puts/1) | &IO.puts/2] == [(&IO.puts/1) | &IO.puts/2] end test "capture with variable module" do mod = List assert (&mod.flatten(&1)).([1, [2], 3]) == [1, 2, 3] assert (&mod.flatten/1).([1, [2], 3]) == [1, 2, 3] - assert (&mod.flatten/1) == &List.flatten/1 + assert (&mod.flatten/1) == (&List.flatten/1) end test "local partial application" do @@ -71,7 +106,7 @@ defmodule Kernel.FnTest do test "imported partial application" do import Record - assert (&record?(&1, :sample)).({:sample, 1}) + assert (&is_record(&1, :sample)).({:sample, 1}) end test "remote partial application" do @@ -88,17 +123,17 @@ defmodule Kernel.FnTest do end test "capture and partially apply lists" do - assert (&[ &1, &2 ]).(1, 2) == [ 1, 2 ] - assert (&[ &1, &2, &3 ]).(1, 2, 3) == [ 1, 2, 3 ] + assert (&[&1, &2]).(1, 2) == [1, 2] + assert (&[&1, &2, &3]).(1, 2, 3) == [1, 2, 3] - assert (&[ 1, &1 ]).(2) == [ 1, 2 ] - assert (&[ 1, &1, &2 ]).(2, 3) == [ 1, 2, 3 ] + assert (&[1, &1]).(2) == [1, 2] + assert (&[1, &1, &2]).(2, 3) == [1, 2, 3] - assert (&[&1|&2]).(1, 2) == [1|2] + assert (&[&1 | &2]).(1, 2) == [1 | 2] end test "capture and partially apply on call" do - assert (&(&1.module)).(__ENV__) == __MODULE__ + assert (& &1.module).(__ENV__) == __MODULE__ end test "capture block like" do @@ -112,38 +147,8 @@ defmodule Kernel.FnTest do assert (&fun.(&1, 2)).(1) == 3 end - test "failure on non-continuous" do - assert_compile_fail CompileError, "nofile:1: capture &2 cannot be defined without &1", "&(&2)" - end - - test "failure on integers" do - assert_compile_fail CompileError, "nofile:1: unhandled &1 outside of a capture", "&1" - assert_compile_fail CompileError, "nofile:1: capture &0 is not allowed", "&foo(&0)" - end - - test "failure on block" do - assert_compile_fail CompileError, - "nofile:1: invalid args for &, block expressions " <> - "are not allowed, got: (\n 1\n 2\n)", - "&(1;2)" - end - - test "failure on other types" do - assert_compile_fail CompileError, - "nofile:1: invalid args for &, expected an expression in the format of &Mod.fun/arity, " <> - "&local/arity or a capture containing at least one argument as &1, got: :foo", - "&:foo" - end - - test "failure when no captures" do - assert_compile_fail CompileError, - "nofile:1: invalid args for &, expected an expression in the format of &Mod.fun/arity, " <> - "&local/arity or a capture containing at least one argument as &1, got: foo()", - "&foo()" - end - - defp is_a?(:atom, atom) when is_atom(atom), do: true - defp is_a?(_, _), do: false + defp atom?(atom) when is_atom(atom), do: true + defp atom?(_), do: false defp atl(arg) do :erlang.atom_to_list(arg) diff --git a/lib/elixir/test/elixir/kernel/guard_test.exs b/lib/elixir/test/elixir/kernel/guard_test.exs new file mode 100644 index 00000000000..be9f50e04d9 --- /dev/null +++ b/lib/elixir/test/elixir/kernel/guard_test.exs @@ -0,0 +1,457 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.GuardTest do + use ExUnit.Case, async: true + + describe "defguard(p) usage" do + defmodule GuardsInMacros do + defguard is_foo(atom) when atom == :foo + + defmacro is_compile_time_foo(atom) when is_foo(atom) do + quote do: unquote(__MODULE__).is_foo(unquote(atom)) + end + end + + test "guards can be used in other macros in the same module" do + require GuardsInMacros + assert GuardsInMacros.is_foo(:foo) + refute GuardsInMacros.is_foo(:baz) + assert GuardsInMacros.is_compile_time_foo(:foo) + end + + defmodule GuardsInFuns do + defguard is_foo(atom) when atom == :foo + defguard is_equal(foo, bar) when foo == bar + + def is_foobar(atom) when is_foo(atom) do + is_foo(atom) + end + end + + test "guards can be used in other funs in the same module" do + require GuardsInFuns + assert GuardsInFuns.is_foo(:foo) + refute GuardsInFuns.is_foo(:bar) + end + + test "guards do not change code evaluation semantics" do + require GuardsInFuns + x = 1 + assert GuardsInFuns.is_equal(x = 2, x) == false + assert x == 2 + end + + defmodule MacrosInGuards do + defmacro is_foo(atom) do + quote do + unquote(atom) == :foo + end + end + + defguard is_foobar(atom) when is_foo(atom) or atom == :bar + end + + test "macros can be used in other guards in the same module" do + require MacrosInGuards + assert MacrosInGuards.is_foobar(:foo) + assert MacrosInGuards.is_foobar(:bar) + refute MacrosInGuards.is_foobar(:baz) + end + + defmodule GuardsInGuards do + defguard is_foo(atom) when atom == :foo + defguard is_foobar(atom) when is_foo(atom) or atom == :bar + end + + test "guards can be used in other guards in the same module" do + require GuardsInGuards + assert GuardsInGuards.is_foobar(:foo) + assert GuardsInGuards.is_foobar(:bar) + refute GuardsInGuards.is_foobar(:baz) + end + + defmodule DefaultArgs do + defguard is_divisible(value, remainder \\ 2) + when is_integer(value) and rem(value, remainder) == 0 + end + + test "permits default values in args" do + require DefaultArgs + assert DefaultArgs.is_divisible(2) + refute DefaultArgs.is_divisible(1) + assert DefaultArgs.is_divisible(3, 3) + refute DefaultArgs.is_divisible(3, 4) + end + + test "doesn't allow matching in args" do + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule Integer.Args do + defguard foo(value, 1) when is_integer(value) + end + end + + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule String.Args do + defguard foo(value, "string") when is_integer(value) + end + end + + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule Atom.Args do + defguard foo(value, :atom) when is_integer(value) + end + end + + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule Tuple.Args do + defguard foo(value, {foo, bar}) when is_integer(value) + end + end + end + + defmodule GuardFromMacro do + defmacro __using__(_) do + quote do + defguard is_even(value) when is_integer(value) and rem(value, 2) == 0 + end + end + end + + test "defguard defines a guard from inside another macro" do + defmodule UseGuardFromMacro do + use GuardFromMacro + + def assert! do + assert is_even(0) + refute is_even(1) + end + end + + UseGuardFromMacro.assert!() + end + + defmodule IntegerPrivateGuards do + defguardp is_even(value) when is_integer(value) and rem(value, 2) == 0 + + def is_even_and_large?(value) when is_even(value) and value > 100, do: true + def is_even_and_large?(_), do: false + + def is_even_and_small?(value) do + if is_even(value) and value <= 100, do: true, else: false + end + end + + test "defguardp defines private guards that work inside and outside guard clauses" do + assert IntegerPrivateGuards.is_even_and_large?(102) + refute IntegerPrivateGuards.is_even_and_large?(98) + refute IntegerPrivateGuards.is_even_and_large?(99) + refute IntegerPrivateGuards.is_even_and_large?(103) + + assert IntegerPrivateGuards.is_even_and_small?(98) + refute IntegerPrivateGuards.is_even_and_small?(99) + refute IntegerPrivateGuards.is_even_and_small?(102) + refute IntegerPrivateGuards.is_even_and_small?(103) + + assert_raise CompileError, ~r"cannot find or invoke local is_even/1", fn -> + defmodule IntegerPrivateGuardUtils do + import IntegerPrivateGuards + + def is_even_and_large?(value) when is_even(value) and value > 100, do: true + def is_even_and_large?(_), do: false + end + end + + assert_raise CompileError, + ~r"undefined function is_even/1", + fn -> + defmodule IntegerPrivateFunctionUtils do + import IntegerPrivateGuards + + def is_even_and_small?(value) do + if is_even(value) and value <= 100, do: true, else: false + end + end + end + end + + test "requires a proper macro name" do + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule(LiteralUsage, do: defguard("literal is bad")) + end + + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule(RemoteUsage, do: defguard(Remote.call(is_bad))) + end + end + + test "handles overriding appropriately" do + assert_raise CompileError, ~r"defmacro (.*?) already defined as def", fn -> + defmodule OverriddenFunUsage do + def foo(bar), do: bar + defguard foo(bar) when bar + end + end + + assert_raise CompileError, ~r"defmacro (.*?) already defined as defp", fn -> + defmodule OverriddenPrivateFunUsage do + defp foo(bar), do: bar + defguard foo(bar) when bar + end + end + + assert_raise CompileError, ~r"defmacro (.*?) already defined as defmacrop", fn -> + defmodule OverriddenPrivateFunUsage do + defmacrop foo(bar), do: bar + defguard foo(bar) when bar + end + end + + assert_raise CompileError, ~r"defmacrop (.*?) already defined as def", fn -> + defmodule OverriddenFunUsage do + def foo(bar), do: bar + defguardp foo(bar) when bar + end + end + + assert_raise CompileError, ~r"defmacrop (.*?) already defined as defp", fn -> + defmodule OverriddenPrivateFunUsage do + defp foo(bar), do: bar + defguardp foo(bar) when bar + end + end + + assert_raise CompileError, ~r"defmacrop (.*?) already defined as defmacro", fn -> + defmodule OverriddenPrivateFunUsage do + defmacro foo(bar), do: bar + defguardp foo(bar) when bar + end + end + end + + test "does not allow multiple guard clauses" do + assert_raise ArgumentError, ~r"invalid syntax in defguard", fn -> + defmodule MultiGuardUsage do + defguardp foo(bar, baz) when bar == 1 when baz == 2 + end + end + end + + test "does not accept a block" do + assert_raise CompileError, ~r"undefined function defguard/2", fn -> + defmodule OnelinerBlockUsage do + defguard(foo(bar), do: one_liner) + end + end + + assert_raise CompileError, ~r"undefined function defguard/2", fn -> + defmodule MultilineBlockUsage do + defguard foo(bar) do + multi + liner + end + end + end + + assert_raise CompileError, ~r"undefined function defguard/2", fn -> + defmodule ImplAndBlockUsage do + defguard(foo(bar) when both_given, do: error) + end + end + end + end + + describe "defguard(p) compilation" do + test "refuses to compile nonsensical code" do + assert_raise CompileError, ~r"cannot find or invoke local undefined/1", fn -> + defmodule UndefinedUsage do + defguard foo(function) when undefined(function) + end + end + end + + test "fails on expressions not allowed in guards" do + # Slightly unique errors + + assert_raise ArgumentError, ~r{invalid right argument for operator "in"}, fn -> + defmodule RuntimeListUsage do + defguard foo(bar, baz) when bar in baz + end + end + + assert_raise CompileError, ~r"cannot invoke remote function", fn -> + defmodule BadErlangFunctionUsage do + defguard foo(bar) when :erlang.binary_to_atom("foo") + end + end + + assert_raise CompileError, ~r"cannot invoke remote function", fn -> + defmodule SendUsage do + defguard foo(bar) when send(self(), :baz) + end + end + + # Consistent errors + + assert_raise ArgumentError, ~r"invalid expression in guard, ! is not allowed", fn -> + defmodule SoftNegationLogicUsage do + defguard foo(logic) when !logic + end + end + + assert_raise ArgumentError, ~r"invalid expression in guard, && is not allowed", fn -> + defmodule SoftAndLogicUsage do + defguard foo(soft, logic) when soft && logic + end + end + + assert_raise ArgumentError, ~r"invalid expression in guard, || is not allowed", fn -> + defmodule SoftOrLogicUsage do + defguard foo(soft, logic) when soft || logic + end + end + + assert_raise CompileError, + ~r"cannot invoke remote function :erlang\.is_record/2 inside guards", + fn -> + defmodule IsRecord2Usage do + defguard foo(rec) when :erlang.is_record(rec, :tag) + end + end + + assert_raise CompileError, + ~r"cannot invoke remote function :erlang\.is_record/3 inside guards", + fn -> + defmodule IsRecord3Usage do + defguard foo(rec) when :erlang.is_record(rec, :tag, 7) + end + end + + assert_raise CompileError, + ~r"cannot invoke remote function :erlang\.\+\+/2 inside guards", + fn -> + defmodule ListSubtractionUsage do + defguard foo(list) when list ++ [] + end + end + + assert_raise CompileError, + ~r"cannot invoke remote function :erlang\.\-\-/2 inside guards", + fn -> + defmodule ListSubtractionUsage do + defguard foo(list) when list -- [] + end + end + + assert_raise CompileError, ~r"invalid expression in guard", fn -> + defmodule LocalCallUsage do + defguard foo(local, call) when local.(call) + end + end + + assert_raise CompileError, ~r"invalid expression in guard", fn -> + defmodule ComprehensionUsage do + defguard foo(bar) when for(x <- [1, 2, 3], do: x * bar) + end + end + + assert_raise CompileError, ~r"invalid expression in guard", fn -> + defmodule AliasUsage do + defguard foo(bar) when alias(bar) + end + end + + assert_raise CompileError, ~r"invalid expression in guard", fn -> + defmodule ImportUsage do + defguard foo(bar) when import(bar) + end + end + + assert_raise CompileError, ~r"invalid expression in guard", fn -> + defmodule RequireUsage do + defguard foo(bar) when require(bar) + end + end + + assert_raise CompileError, ~r"invalid expression in guard", fn -> + defmodule SuperUsage do + defguard foo(bar) when super(bar) + end + end + + assert_raise CompileError, ~r"invalid expression in guard", fn -> + defmodule SpawnUsage do + defguard foo(bar) when spawn(& &1) + end + end + + assert_raise CompileError, ~r"invalid expression in guard", fn -> + defmodule ReceiveUsage do + defguard foo(bar) when receive(do: (baz -> baz)) + end + end + + assert_raise CompileError, ~r"invalid expression in guard", fn -> + defmodule CaseUsage do + defguard foo(bar) when case(bar, do: (baz -> :baz)) + end + end + + assert_raise CompileError, ~r"invalid expression in guard", fn -> + defmodule CondUsage do + defguard foo(bar) when cond(do: (bar -> :baz)) + end + end + + assert_raise CompileError, ~r"invalid expression in guard", fn -> + defmodule TryUsage do + defguard foo(bar) when try(do: (baz -> baz)) + end + end + + assert_raise CompileError, ~r"invalid expression in guard", fn -> + defmodule WithUsage do + defguard foo(bar) when with(do: (baz -> baz)) + end + end + end + end + + describe "defguard(p) expansion" do + defguard with_unused_vars(foo, bar, _baz) when foo + bar + + test "doesn't obscure unused variables" do + args = quote(do: [1 + 1, 2 + 2, 3 + 3]) + + assert expand_defguard_to_string(:with_unused_vars, args, :guard) == """ + :erlang.+(1 + 1, 2 + 2) + """ + + assert expand_defguard_to_string(:with_unused_vars, args, nil) == """ + {arg1, arg2} = {1 + 1, 2 + 2} + :erlang.+(arg1, arg2) + """ + end + + defguard with_reused_vars(foo, bar, baz) when foo + foo + bar + baz + + test "handles re-used variables" do + args = quote(do: [1 + 1, 2 + 2, 3 + 3]) + + assert expand_defguard_to_string(:with_reused_vars, args, :guard) == """ + :erlang.+(:erlang.+(:erlang.+(1 + 1, 1 + 1), 2 + 2), 3 + 3) + """ + + assert expand_defguard_to_string(:with_reused_vars, args, nil) == """ + {arg1, arg2, arg3} = {1 + 1, 2 + 2, 3 + 3} + :erlang.+(:erlang.+(:erlang.+(arg1, arg1), arg2), arg3) + """ + end + + defp expand_defguard_to_string(fun, args, context) do + {{:., [], [__MODULE__, fun]}, [], args} + |> Macro.expand(%{__ENV__ | context: context}) + |> Macro.to_string() + |> Kernel.<>("\n") + end + end +end diff --git a/lib/elixir/test/elixir/kernel/impl_test.exs b/lib/elixir/test/elixir/kernel/impl_test.exs new file mode 100644 index 00000000000..2ad09fac2f6 --- /dev/null +++ b/lib/elixir/test/elixir/kernel/impl_test.exs @@ -0,0 +1,585 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.ImplTest do + use ExUnit.Case, async: true + + defp capture_err(fun) do + ExUnit.CaptureIO.capture_io(:stderr, fun) + end + + defp purge(module) do + :code.purge(module) + :code.delete(module) + end + + setup do + on_exit(fn -> purge(Kernel.ImplTest.ImplAttributes) end) + end + + defprotocol AProtocol do + def foo(term) + def bar(term) + end + + defmodule Behaviour do + @callback foo() :: any + end + + defmodule BehaviourWithArgument do + @callback foo(any) :: any + end + + defmodule BehaviourWithThreeArguments do + @callback foo(any, any, any) :: any + end + + defmodule UseBehaviourWithoutImpl do + @callback foo_without_impl() :: any + @callback bar_without_impl() :: any + @callback baz_without_impl() :: any + + defmacro __using__(_opts) do + quote do + @behaviour Kernel.ImplTest.UseBehaviourWithoutImpl + def foo_without_impl(), do: :auto_generated + end + end + end + + defmodule UseBehaviourWithImpl do + @callback foo_with_impl() :: any + @callback bar_with_impl() :: any + @callback baz_with_impl() :: any + + defmacro __using__(_opts) do + quote do + @behaviour Kernel.ImplTest.UseBehaviourWithImpl + @impl true + def foo_with_impl(), do: :auto_generated + def bar_with_impl(), do: :auto_generated + end + end + end + + defmodule MacroBehaviour do + @macrocallback bar :: any + end + + defmodule ManualBehaviour do + def behaviour_info(:callbacks), do: [foo: 0] + def behaviour_info(:optional_callbacks), do: :undefined + end + + test "sets @impl to boolean" do + defmodule ImplAttributes do + @behaviour Behaviour + + @impl true + def foo(), do: :ok + + @impl false + def foo(term) do + term + end + end + end + + test "sets @impl to nil" do + assert_raise ArgumentError, ~r/should be a module or a boolean/, fn -> + defmodule ImplAttributes do + @behaviour Behaviour + @impl nil + def foo(), do: :ok + end + end + end + + test "sets @impl to behaviour" do + defmodule ImplAttributes do + @behaviour Behaviour + @impl Behaviour + def foo(), do: :ok + end + end + + test "does not set @impl" do + defmodule ImplAttributes do + @behaviour Behaviour + def foo(), do: :ok + end + end + + test "sets @impl to boolean on manual behaviour" do + defmodule ImplAttributes do + @behaviour ManualBehaviour + + @impl true + def foo(), do: :ok + end + end + + test "warns for undefined value" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour :abc + + @impl :abc + def foo(), do: :ok + end + """) + end) =~ + "got \"@impl :abc\" for function foo/0 but this behaviour does not specify such callback. There are no known callbacks" + end + + test "warns for callbacks without impl and @impl has been set before" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @behaviour Kernel.ImplTest.MacroBehaviour + + @impl true + def foo(), do: :ok + + defmacro bar(), do: :ok + end + """) + end) =~ + "module attribute @impl was not set for macro bar/0 callback (specified in Kernel.ImplTest.MacroBehaviour)" + end + + test "warns for callbacks without impl and @impl has been set after" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @behaviour Kernel.ImplTest.MacroBehaviour + + defmacro bar(), do: :ok + + @impl true + def foo(), do: :ok + end + """) + end) =~ + "module attribute @impl was not set for macro bar/0 callback (specified in Kernel.ImplTest.MacroBehaviour)" + end + + test "warns when @impl is set on private function" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl true + defp foo(), do: :ok + end + """) + end) =~ + "function foo/0 is private, @impl attribute is always discarded for private functions/macros" + end + + test "warns when @impl is set and no function" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl true + end + """) + end) =~ "module attribute @impl was set but no definition follows it" + end + + test "warns for @impl true and no behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @impl true + def foo(), do: :ok + end + """) + end) =~ "got \"@impl true\" for function foo/0 but no behaviour was declared" + end + + test "warns for @impl true with callback name not in behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl true + def bar(), do: :ok + end + """) + end) =~ + "got \"@impl true\" for function bar/0 but no behaviour specifies such callback" + end + + test "warns for @impl true with macro callback name not in behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.MacroBehaviour + @impl true + defmacro foo(), do: :ok + end + """) + end) =~ "got \"@impl true\" for macro foo/0 but no behaviour specifies such callback" + end + + test "warns for @impl true with callback kind not in behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.MacroBehaviour + @impl true + def foo(), do: :ok + end + """) + end) =~ + "got \"@impl true\" for function foo/0 but no behaviour specifies such callback" + end + + test "warns for @impl true with wrong arity" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl true + def foo(arg), do: arg + end + """) + end) =~ + "got \"@impl true\" for function foo/1 but no behaviour specifies such callback" + end + + test "warns for @impl false and there are no callbacks" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @impl false + def baz(term), do: term + end + """) + end) =~ "got \"@impl false\" for function baz/1 but no behaviour was declared" + end + + test "warns for @impl false and it is a callback" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl false + def foo(), do: :ok + end + """) + end) =~ + "got \"@impl false\" for function foo/0 but it is a callback specified in Kernel.ImplTest.Behaviour" + end + + test "warns for @impl module and no behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @impl Kernel.ImplTest.Behaviour + def foo(), do: :ok + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.Behaviour\" for function foo/0 but no behaviour was declared" + end + + test "warns for @impl module with callback name not in behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl Kernel.ImplTest.Behaviour + def bar(), do: :ok + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.Behaviour\" for function bar/0 but this behaviour does not specify such callback" + end + + test "warns for @impl module with macro callback name not in behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.MacroBehaviour + @impl Kernel.ImplTest.MacroBehaviour + defmacro foo(), do: :ok + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.MacroBehaviour\" for macro foo/0 but this behaviour does not specify such callback" + end + + test "warns for @impl module with macro callback kind not in behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.MacroBehaviour + @impl Kernel.ImplTest.MacroBehaviour + def foo(), do: :ok + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.MacroBehaviour\" for function foo/0 but this behaviour does not specify such callback" + end + + test "warns for @impl module and callback belongs to another known module" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @behaviour Kernel.ImplTest.MacroBehaviour + @impl Kernel.ImplTest.Behaviour + def bar(), do: :ok + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.Behaviour\" for function bar/0 but this behaviour does not specify such callback" + end + + test "warns for @impl module and callback belongs to another unknown module" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @impl Kernel.ImplTest.MacroBehaviour + def bar(), do: :ok + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.MacroBehaviour\" for function bar/0 but this behaviour was not declared with @behaviour" + end + + test "does not warn for @impl when the function with default conforms with several typespecs" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @behaviour Kernel.ImplTest.BehaviourWithArgument + + @impl true + def foo(args \\ []), do: args + end + """) + end + + test "does not warn for @impl when the function conforms to behaviour but has default value for arg" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.BehaviourWithArgument + + @impl true + def foo(args \\ []), do: args + end + """) + end + + test "does not warn for @impl when the function conforms to behaviour but has additional trailing default args" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.BehaviourWithArgument + + @impl true + def foo(arg_1, _args \\ []), do: arg_1 + end + """) + end + + test "does not warn for @impl when the function conforms to behaviour but has additional leading default args" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.BehaviourWithArgument + + @impl true + def foo(_defaulted_arg \\ [], args), do: args + end + """) + end + + test "does not warn for @impl when the function has more args than callback, but they're all defaulted" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.BehaviourWithArgument + + @impl true + def foo(args \\ [], _bar \\ []), do: args + end + """) + end + + test "does not warn for @impl with defaults when the same function is defined multiple times" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.BehaviourWithArgument + @behaviour Kernel.ImplTest.BehaviourWithThreeArguments + + @impl Kernel.ImplTest.BehaviourWithArgument + def foo(_foo \\ [], _bar \\ []), do: :ok + + @impl Kernel.ImplTest.BehaviourWithThreeArguments + def foo(_foo, _bar, _baz, _qux \\ []), do: :ok + end + """) + end + + test "does not warn for no @impl when overriding callback" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + + def foo(), do: :overridable + + defoverridable Kernel.ImplTest.Behaviour + + def foo(), do: :overridden + end + """) + end + + test "does not warn for overridable function missing @impl" do + Code.eval_string(~S""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + + def foo(), do: :overridable + + defoverridable Kernel.ImplTest.Behaviour + + @impl Kernel.ImplTest.Behaviour + def foo(), do: :overridden + end + """) + end + + test "warns correctly for missing @impl only for end-user implemented function" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @behaviour Kernel.ImplTest.MacroBehaviour + + def foo(), do: :overridable + + defoverridable Kernel.ImplTest.Behaviour + + def foo(), do: :overridden + + @impl true + defmacro bar(), do: :overridden + end + """) + end) =~ + "module attribute @impl was not set for function foo/0 callback (specified in Kernel.ImplTest.Behaviour)" + end + + test "warns correctly for incorrect @impl in overridable callback" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + @behaviour Kernel.ImplTest.MacroBehaviour + + @impl Kernel.ImplTest.MacroBehaviour + def foo(), do: :overridable + + defoverridable Kernel.ImplTest.Behaviour + + @impl Kernel.ImplTest.Behaviour + def foo(), do: :overridden + end + """) + end) =~ + "got \"@impl Kernel.ImplTest.MacroBehaviour\" for function foo/0 but this behaviour does not specify such callback" + end + + test "warns only for non-generated functions in non-generated @impl" do + message = + capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + @behaviour Kernel.ImplTest.Behaviour + use Kernel.ImplTest.UseBehaviourWithoutImpl + + @impl true + def bar_without_impl(), do: :overridden + def baz_without_impl(), do: :overridden + + defdelegate foo(), to: __MODULE__, as: :baz + def baz(), do: :ok + end + """) + end) + + assert message =~ + "module attribute @impl was not set for function baz_without_impl/0 callback" + + assert message =~ + "module attribute @impl was not set for function foo/0 callback" + + refute message =~ "foo_without_impl/0" + end + + test "warns only for non-generated functions in non-generated @impl in protocols" do + message = + capture_err(fn -> + Code.eval_string(""" + defimpl Kernel.ImplTest.AProtocol, for: List do + @impl true + def foo(_list), do: :ok + + defdelegate bar(list), to: __MODULE__, as: :baz + def baz(_list), do: :ok + end + """) + end) + + assert message =~ + "module attribute @impl was not set for function bar/1 callback" + end + + test "warns only for generated functions in generated @impl" do + message = + capture_err(fn -> + Code.eval_string(""" + defmodule Kernel.ImplTest.ImplAttributes do + use Kernel.ImplTest.UseBehaviourWithImpl + def baz_with_impl(), do: :overridden + end + """) + end) + + assert message =~ "module attribute @impl was not set for function bar_with_impl/0 callback" + refute message =~ "foo_with_impl/0" + end + + test "does not warn for overridable callback when using __before_compile__/1 hook" do + Code.eval_string(~S""" + defmodule BeforeCompile do + defmacro __before_compile__(_) do + quote do + @behaviour Kernel.ImplTest.Behaviour + + def foo(), do: :overridable + + defoverridable Kernel.ImplTest.Behaviour + end + end + end + + defmodule Kernel.ImplTest.ImplAttributes do + @before_compile BeforeCompile + @behaviour Kernel.ImplTest.MacroBehaviour + + defmacro bar(), do: :overridable + + defoverridable Kernel.ImplTest.MacroBehaviour + + @impl Kernel.ImplTest.MacroBehaviour + defmacro bar(), do: :overridden + end + """) + end +end diff --git a/lib/elixir/test/elixir/kernel/import_test.exs b/lib/elixir/test/elixir/kernel/import_test.exs index 2b107cdd83f..d5145a92db9 100644 --- a/lib/elixir/test/elixir/kernel/import_test.exs +++ b/lib/elixir/test/elixir/kernel/import_test.exs @@ -1,16 +1,37 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.ImportTest do use ExUnit.Case, async: true + # This should not warn due to the empty only + import URI, only: [] + defmodule ImportAvailable do defmacro flatten do [flatten: 1] end end + test "multi-call" do + assert [List, String] = import(Elixir.{List, unquote(:String)}) + assert keymember?([a: 1], :a, 0) + assert valid?("ø") + end + + test "blank multi-call" do + assert [] = import(List.{}) + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] + end + + test "multi-call with options" do + assert [List] = import(Elixir.{List}, only: []) + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] + end + test "import all" do - import :lists + assert :lists = import(:lists) assert flatten([1, [2], 3]) == [1, 2, 3] end @@ -20,13 +41,15 @@ defmodule Kernel.ImportTest do end test "import except one" do - import :lists, except: [each: 2] + import :lists, except: [duplicate: 2] assert flatten([1, [2], 3]) == [1, 2, 3] + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] end test "import only via macro" do require ImportAvailable - import :lists, only: ImportAvailable.flatten + import :lists, only: ImportAvailable.flatten() assert flatten([1, [2], 3]) == [1, 2, 3] end @@ -35,7 +58,7 @@ defmodule Kernel.ImportTest do end test "import with options via macro" do - import :lists, dynamic_opts + import :lists, dynamic_opts() assert flatten([1, [2], 3]) == [1, 2, 3] end @@ -47,8 +70,24 @@ defmodule Kernel.ImportTest do assert duplicate([1], 2) == [1] end + test "import except none respects previous import with except" do + import :lists, except: [duplicate: 2] + import :lists, except: [] + assert append([1], [2, 3]) == [1, 2, 3] + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] + end + + test "import except none respects previous import with only" do + import :lists, only: [append: 2] + import :lists, except: [] + assert append([1], [2, 3]) == [1, 2, 3] + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] + end + defmodule Underscored do - def hello(x), do: x + def hello(x), do: x def __underscore__(x), do: x end @@ -61,7 +100,7 @@ defmodule Kernel.ImportTest do assert __underscore__(3) == 3 end - test "import non underscored" do + test "import non-underscored" do import ExplicitUnderscored, only: [__underscore__: 1] import Underscored assert hello(2) == 2 @@ -69,14 +108,14 @@ defmodule Kernel.ImportTest do end defmodule MessedBitwise do - defmacro bnot(x), do: x + defmacro bnot(x), do: x defmacro bor(x, _), do: x end - import Bitwise, only: :macros + import Bitwise, only: :functions - test "conflicing imports with only and except" do - import Bitwise, only: :macros, except: [bnot: 1] + test "conflicting imports with only and except" do + import Bitwise, only: :functions, except: [bnot: 1] import MessedBitwise, only: [bnot: 1] assert bnot(0) == 0 assert bor(0, 1) == 1 @@ -100,15 +139,59 @@ defmodule Kernel.ImportTest do test "import many" do [import(List), import(String)] - assert capitalize("foo") == "Foo" + assert capitalize("foo") == "Foo" assert flatten([1, [2], 3]) == [1, 2, 3] end + defmodule ModuleWithSigils do + def sigil_i(string, []), do: String.to_integer(string) + + defmacro sigil_I(string, []) do + quote do + String.to_integer(unquote(string)) + end + end + + def sigil_w(_string, []), do: [] + + def bnot(x), do: x + defmacro bor(x, _), do: x + end + + test "import only sigils" do + import Kernel, except: [sigil_w: 2] + import ModuleWithSigils, only: :sigils + + # Ensure that both function and macro sigils are imported + assert ~i'10' == 10 + assert ~I'10' == 10 + assert ~w(abc def) == [] + + # Ensure that non-sigil functions and macros from ModuleWithSigils were not loaded + assert bnot(0) == -1 + assert bor(0, 1) == 1 + end + + test "import only sigils with except" do + import ModuleWithSigils, only: :sigils, except: [sigil_w: 2] + + assert ~i'10' == 10 + assert ~I'10' == 10 + assert ~w(abc def) == ["abc", "def"] + end + + test "import only removes the non-import part" do + import List + import List, only: :macros + # Buggy local duplicate is used because we asked only for macros + assert duplicate([1], 2) == [1] + end + test "import lexical on if" do if false do - import :lists + import List flatten([1, [2], 3]) - flunk + flunk() else # Buggy local duplicate is untouched assert duplicate([1], 2) == [1] @@ -118,20 +201,43 @@ defmodule Kernel.ImportTest do test "import lexical on case" do case true do false -> - import :lists + import List flatten([1, [2], 3]) - flunk + flunk() + true -> # Buggy local duplicate is untouched assert duplicate([1], 2) == [1] end end + test "import lexical on for" do + for x <- [1, 2, 3], x > 10 do + import List + flatten([1, [2], 3]) + flunk() + end + + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] + end + + test "import lexical on with" do + with true <- false do + import List + flatten([1, [2], 3]) + flunk() + end + + # Buggy local duplicate is untouched + assert duplicate([1], 2) == [1] + end + test "import lexical on try" do try do - import :lists + import List flatten([1, [2], 3]) - flunk + flunk() catch _, _ -> # Buggy local duplicate is untouched diff --git a/lib/elixir/test/elixir/kernel/lexical_tracker_test.exs b/lib/elixir/test/elixir/kernel/lexical_tracker_test.exs index da2a9cf4931..c69a841a6ee 100644 --- a/lib/elixir/test/elixir/kernel/lexical_tracker_test.exs +++ b/lib/elixir/test/elixir/kernel/lexical_tracker_test.exs @@ -1,4 +1,4 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.LexicalTrackerTest do use ExUnit.Case, async: true @@ -6,43 +6,95 @@ defmodule Kernel.LexicalTrackerTest do alias Kernel.LexicalTracker, as: D setup do - {:ok, [pid: D.start_link]} + {:ok, pid} = D.start_link() + {:ok, [pid: pid]} end - test "can add remote dispatches", config do - D.remote_dispatch(config[:pid], String) - assert D.remotes(config[:pid]) == [String] + test "can add remote dispatch", config do + D.remote_dispatch(config[:pid], String, :runtime) + assert D.references(config[:pid]) == {[], [], [String], []} + + D.remote_dispatch(config[:pid], String, :compile) + assert D.references(config[:pid]) == {[String], [], [], []} + + D.remote_dispatch(config[:pid], String, :runtime) + assert D.references(config[:pid]) == {[String], [], [], []} + end + + test "can add requires", config do + D.add_require(config[:pid], URI) + assert D.references(config[:pid]) == {[], [URI], [], []} + + D.remote_dispatch(config[:pid], URI, :runtime) + assert D.references(config[:pid]) == {[], [URI], [URI], []} + + D.remote_dispatch(config[:pid], URI, :compile) + assert D.references(config[:pid]) == {[URI], [URI], [], []} end - test "can add imports", config do - D.add_import(config[:pid], String, 1, true) - assert D.remotes(config[:pid]) == [String] + test "can add module imports", config do + D.add_require(config[:pid], String) + D.add_import(config[:pid], String, [], 1, true) + + D.import_dispatch(config[:pid], String, {:upcase, 1}, :runtime) + assert D.references(config[:pid]) == {[], [String], [String], []} + + D.import_dispatch(config[:pid], String, {:upcase, 1}, :compile) + assert D.references(config[:pid]) == {[String], [String], [], []} + end + + test "can add module with {function, arity} imports", config do + D.add_require(config[:pid], String) + D.add_import(config[:pid], String, [upcase: 1], 1, true) + + D.import_dispatch(config[:pid], String, {:upcase, 1}, :compile) + assert D.references(config[:pid]) == {[String], [String], [], []} end test "can add aliases", config do D.add_alias(config[:pid], String, 1, true) - assert D.remotes(config[:pid]) == [String] + D.alias_dispatch(config[:pid], String) + assert D.references(config[:pid]) == {[], [], [], []} end - test "unused imports", config do - D.add_import(config[:pid], String, 1, true) - assert D.collect_unused_imports(config[:pid]) == [{String,1}] + test "unused module imports", config do + D.add_import(config[:pid], String, [], 1, true) + assert D.collect_unused_imports(config[:pid]) == [{String, 1}] end - test "used imports are not unused", config do - D.add_import(config[:pid], String, 1, true) - D.import_dispatch(config[:pid], String) + test "used module imports are not unused", config do + D.add_import(config[:pid], String, [], 1, true) + D.import_dispatch(config[:pid], String, {:upcase, 1}, :compile) + assert D.collect_unused_imports(config[:pid]) == [] + end + + test "unused {module, function, arity} imports", config do + D.add_import(config[:pid], String, [upcase: 1], 1, true) + assert D.collect_unused_imports(config[:pid]) == [{String, 1}, {{String, :upcase, 1}, 1}] + end + + test "used {module, function, arity} imports are not unused", config do + D.add_import(config[:pid], String, [upcase: 1], 1, true) + D.add_import(config[:pid], String, [downcase: 1], 1, true) + D.import_dispatch(config[:pid], String, {:upcase, 1}, :compile) + assert D.collect_unused_imports(config[:pid]) == [{{String, :downcase, 1}, 1}] + end + + test "overwriting {module, function, arity} import with module import", config do + D.add_import(config[:pid], String, [upcase: 1], 1, true) + D.add_import(config[:pid], String, [], 1, true) + D.import_dispatch(config[:pid], String, {:downcase, 1}, :compile) assert D.collect_unused_imports(config[:pid]) == [] end test "imports with no warn are not unused", config do - D.add_import(config[:pid], String, 1, false) + D.add_import(config[:pid], String, [], 1, false) assert D.collect_unused_imports(config[:pid]) == [] end test "unused aliases", config do D.add_alias(config[:pid], String, 1, true) - assert D.collect_unused_aliases(config[:pid]) == [{String,1}] + assert D.collect_unused_aliases(config[:pid]) == [{String, 1}] end test "used aliases are not unused", config do @@ -55,4 +107,180 @@ defmodule Kernel.LexicalTrackerTest do D.add_alias(config[:pid], String, 1, false) assert D.collect_unused_aliases(config[:pid]) == [] end + + describe "references" do + test "typespecs do not tag aliases nor types" do + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.AliasTypespecs do + alias Foo.Bar, as: Bar, warn: false + @type bar :: Foo.Bar | Foo.Bar.t + @opaque bar2 :: Foo.Bar.t + @typep bar3 :: Foo.Bar.t + @callback foo :: Foo.Bar.t + @macrocallback foo2(Foo.Bar.t) :: Foo.Bar.t + @spec foo(bar3) :: Foo.Bar.t + def foo(_), do: :ok + + # References from specs are processed only late + @after_compile __MODULE__ + def __after_compile__(env, _) do + send(self(), {:references, Kernel.LexicalTracker.references(env.lexical_tracker)}) + end + end + """) + + assert_received {:references, {compile, _exports, runtime, _}} + + refute Elixir.Bar in runtime + refute Elixir.Bar in compile + + refute Foo.Bar in runtime + refute Foo.Bar in compile + end + + test "typespecs track structs as exports" do + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.StructTypespecs do + @type uri :: %URI{} + + # References from specs are processed only late + @after_compile __MODULE__ + def __after_compile__(env, _) do + send(self(), {:references, Kernel.LexicalTracker.references(env.lexical_tracker)}) + end + end + """) + + assert_received {:references, {compile, exports, runtime, _}} + + assert URI in runtime + assert URI in exports + refute URI in compile + end + + test "@compile adds a runtime dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Compile do + @compile {:no_warn_undefined, String} + @compile {:no_warn_undefined, {Enum, :concat, 1}} + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute String in compile + refute String in exports + assert String in runtime + + refute Enum in compile + refute Enum in exports + assert Enum in runtime + end + + test "defdelegate with literal adds runtime dependency" do + {{compile, _exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Defdelegate do + defdelegate decode_query(query), to: URI + + opts = [to: Enum] + defdelegate concat(enum), opts + + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute URI in compile + assert Enum in compile + assert URI in runtime + end + + test "imports adds an export dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Imports do + import String, warn: false + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute String in compile + assert String in exports + refute String in runtime + end + + test "structs are exports or compile time" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.StructRuntime do + def expand, do: %URI{} + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute URI in compile + assert URI in exports + assert URI in runtime + + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.StructCompile do + _ = %URI{} + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + assert URI in compile + assert URI in exports + refute URI in runtime + end + + test "Macro.struct! adds an export dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.MacroStruct do + # We do not use the alias because it would be a compile time + # dependency. The alias may happen in practice, which is the + # mechanism to make this expansion become a compile-time one. + # However, in some cases, such as typespecs, we don't necessarily + # want the compile-time dependency to happen. + Macro.struct!(:"Elixir.URI", __ENV__) + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute URI in compile + assert URI in exports + refute URI in runtime + end + + test "compile_env! does not add a compile dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.CompileEnvStruct do + require Application + Application.compile_env(:elixir, URI) + Application.compile_env(:elixir, [:foo, URI, :bar]) + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute URI in compile + refute URI in exports + assert URI in runtime + end + + test "defmodule does not add a compile dependency" do + {{compile, exports, runtime, _}, _binding} = + Code.eval_string(""" + defmodule Kernel.LexicalTrackerTest.Defmodule do + Kernel.LexicalTracker.references(__ENV__.lexical_tracker) + end |> elem(3) + """) + + refute Kernel.LexicalTrackerTest.Defmodule in compile + refute Kernel.LexicalTrackerTest.Defmodule in exports + refute Kernel.LexicalTrackerTest.Defmodule in runtime + end + end end diff --git a/lib/elixir/test/elixir/kernel/macros_test.exs b/lib/elixir/test/elixir/kernel/macros_test.exs index fd2f7511e7c..96cc0e0ebad 100644 --- a/lib/elixir/test/elixir/kernel/macros_test.exs +++ b/lib/elixir/test/elixir/kernel/macros_test.exs @@ -1,4 +1,4 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.MacrosTest.Nested do defmacro value, do: 1 @@ -9,46 +9,73 @@ defmodule Kernel.MacrosTest.Nested do end defmodule Kernel.MacrosTest do - require Kernel.MacrosTest.Nested, as: Nested - use ExUnit.Case, async: true + Kernel.MacrosTest.Nested = require Kernel.MacrosTest.Nested, as: Nested + + @spec my_macro :: Macro.t() defmacro my_macro do - quote do: 1 + 1 + quote(do: 1 + 1) end + @spec my_private_macro :: Macro.t() defmacrop my_private_macro do - quote do: 1 + 3 + quote(do: 1 + 3) end defmacro my_macro_with_default(value \\ 5) do - quote do: 1 + unquote(value) + quote(do: 1 + unquote(value)) + end + + defp by_two(x), do: x * 2 + + defmacro my_macro_with_local(value) do + value = by_two(by_two(value)) + quote(do: 1 + unquote(value)) + end + + defmacro my_macro_with_capture(value) do + Enum.map(value, &by_two/1) end - test :require do - assert Kernel.MacrosTest.Nested.value == 1 + test "require" do + assert Kernel.MacrosTest.Nested.value() == 1 end - test :require_with_alias do - assert Nested.value == 1 + test "require with alias" do + assert Nested.value() == 1 end - test :local_but_private_macro do - assert my_private_macro == 4 + test "local with private macro" do + assert my_private_macro() == 4 end - test :local_with_defaults_macro do - assert my_macro_with_default == 6 + test "local with defaults macro" do + assert my_macro_with_default() == 6 end - test :macros_cannot_be_called_dynamically do + test "local with local call" do + assert my_macro_with_local(4) == 17 + end + + test "local with capture" do + assert my_macro_with_capture([1, 2, 3]) == [2, 4, 6] + end + + test "macros cannot be called dynamically" do x = Nested - assert_raise UndefinedFunctionError, fn -> x.value end + assert_raise UndefinedFunctionError, fn -> x.func() end end - test :bang_do_block do + test "macros with bang and do block have proper precedence" do import Kernel.MacrosTest.Nested - assert (do_identity! do 1 end) == 1 - assert (Kernel.MacrosTest.Nested.do_identity! do 1 end) == 1 + + assert (do_identity! do + 1 + end) == 1 + + assert (Kernel.MacrosTest.Nested.do_identity! do + 1 + end) == 1 end -end \ No newline at end of file +end diff --git a/lib/elixir/test/elixir/kernel/overridable_test.exs b/lib/elixir/test/elixir/kernel/overridable_test.exs index 1397e53d793..f7a6eaa3aa9 100644 --- a/lib/elixir/test/elixir/kernel/overridable_test.exs +++ b/lib/elixir/test/elixir/kernel/overridable_test.exs @@ -1,10 +1,6 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.Overridable do - defmacrop super? do - Module.overridable?(__CALLER__.module, __CALLER__.function) - end - def sample do 1 end @@ -17,50 +13,54 @@ defmodule Kernel.Overridable do 1 end - def explicit_nested_super do - {super?, 2} + def super_with_multiple_args(x, y) do + x + y end - false = Module.overridable? __MODULE__, {:explicit_nested_super, 0} - - defoverridable [sample: 0, with_super: 0, without_super: 0, explicit_nested_super: 0] - - true = Module.overridable? __MODULE__, {:explicit_nested_super, 0} - - def explicit_nested_super do - {super, super?, 1} + def capture_super(x) do + x end - true = Module.overridable? __MODULE__, {:explicit_nested_super, 0} - - defoverridable [explicit_nested_super: 0] - - true = Module.overridable? __MODULE__, {:explicit_nested_super, 0} - - def implicit_nested_super do - {super?, 1} + defmacro capture_super_macro(x) do + x end - defoverridable [implicit_nested_super: 0] + def many_clauses(0) do + 11 + end - def implicit_nested_super do - {super, super?, 0} + def many_clauses(1) do + 13 end - def super_with_explicit_args(x, y) do - x + y + def locals do + undefined_function() end - def many_clauses(0) do - 11 + def multiple_overrides do + [1] end - def many_clauses(1) do - 13 + def public_to_private do + :public end - defoverridable [implicit_nested_super: 0, - super_with_explicit_args: 2, many_clauses: 1] + defoverridable sample: 0, + with_super: 0, + without_super: 0, + super_with_multiple_args: 2, + capture_super: 1, + capture_super_macro: 1, + many_clauses: 1, + locals: 0, + multiple_overrides: 0, + public_to_private: 0 + + true = Module.overridable?(__MODULE__, {:without_super, 0}) + true = Module.overridable?(__MODULE__, {:with_super, 0}) + + true = {:with_super, 0} in Module.overridables_in(__MODULE__) + true = {:without_super, 0} in Module.overridables_in(__MODULE__) def without_super do :without_super @@ -70,16 +70,22 @@ defmodule Kernel.Overridable do super() + 2 end - def no_overridable do - {:no_overridable, super?} + true = Module.overridable?(__MODULE__, {:without_super, 0}) + true = Module.overridable?(__MODULE__, {:with_super, 0}) + + true = {:with_super, 0} in Module.overridables_in(__MODULE__) + true = {:without_super, 0} in Module.overridables_in(__MODULE__) + + def super_with_multiple_args(x, y) do + super(x, y * 2) end - def explicit_nested_super do - {super, super?, 0} + def capture_super(x) do + Enum.map(1..x, &super(&1)) ++ Enum.map(1..x, &super/1) end - def super_with_explicit_args(x, y) do - super x, y * 2 + defmacro capture_super_macro(x) do + Enum.map(1..x, &super(&1)) ++ Enum.map(1..x, &super/1) end def many_clauses(2) do @@ -93,38 +99,173 @@ defmodule Kernel.Overridable do def many_clauses(x) do super(x) end + + def locals do + :ok + end + + def multiple_overrides do + [2 | super()] + end + + defp public_to_private do + :private + end + + def test_public_to_private do + public_to_private() + end + + defoverridable multiple_overrides: 0 + + def multiple_overrides do + [3 | super()] + end + + ## Macros + + defmacro overridable_macro(x) do + quote do + unquote(x) + 100 + end + end + + defoverridable overridable_macro: 1 + + defmacro overridable_macro(x) do + quote do + unquote(super(x)) + 1000 + end + end + + defmacrop private_macro(x \\ raise("never called")) + + defmacrop private_macro(x) do + quote do + unquote(x) + 100 + end + end + + defoverridable private_macro: 1 + + defmacrop private_macro(x) do + quote do + unquote(super(x)) + 1000 + end + end + + def private_macro_call(val \\ 11) do + private_macro(val) + end +end + +defmodule Kernel.OverridableExampleBehaviour do + @callback required_callback :: any + @callback optional_callback :: any + @macrocallback required_macro_callback(arg :: any) :: Macro.t() + @macrocallback optional_macro_callback(arg :: any, arg2 :: any) :: Macro.t() + @optional_callbacks optional_callback: 0, optional_macro_callback: 2 end defmodule Kernel.OverridableTest do require Kernel.Overridable, as: Overridable - use ExUnit.Case, async: true + use ExUnit.Case + + defp purge(module) do + :code.purge(module) + :code.delete(module) + end + + test "overridable keeps function ordering" do + defmodule OverridableOrder do + def not_private(str) do + process_url(str) + end + + def process_url(_str) do + :first + end + + # There was a bug where the order in which we removed + # overridable expressions lead to errors. This module + # aims to guarantee removing process_url/1 before we + # remove the function that depends on it does not cause + # errors. If it compiles, it works! + defoverridable process_url: 1, not_private: 1 + + def process_url(_str) do + :second + end + end + end + + test "overridable works with defaults" do + defmodule OverridableDefault do + def fun(value, opt \\ :from_parent) do + {value, opt} + end + + defmacro macro(value, opt \\ :from_parent) do + {{value, opt}, Macro.escape(__CALLER__)} + end + + # There was a bug where the default function would + # attempt to call its overridable name instead of + # func/1. If it compiles, it works! + defoverridable fun: 1, fun: 2, macro: 1, macro: 2 + + def fun(value) do + {value, super(value)} + end + + defmacro macro(value) do + {{value, super(value)}, Macro.escape(__CALLER__)} + end + end + + defmodule OverridableCall do + require OverridableDefault + OverridableDefault.fun(:foo) + OverridableDefault.macro(:bar) + end + end test "overridable is made concrete if no other is defined" do - assert Overridable.sample == 1 + assert Overridable.sample() == 1 end test "overridable overridden with super" do - assert Overridable.with_super == 3 + assert Overridable.with_super() == 3 end test "overridable overridden without super" do - assert Overridable.without_super == :without_super + assert Overridable.without_super() == :without_super + end + + test "public overridable overridden as private function" do + assert Overridable.test_public_to_private() == :private + refute {:public_to_private, 0} in Overridable.module_info(:exports) + end + + test "overridable locals are ignored without super" do + assert Overridable.locals() == :ok end - test "overridable overridden with nested super" do - assert Overridable.explicit_nested_super == {{{false, 2}, true, 1}, true, 0} + test "calling super with multiple args" do + assert Overridable.super_with_multiple_args(1, 2) == 5 end - test "overridable node overridden with nested super" do - assert Overridable.implicit_nested_super == {{false, 1}, true, 0} + test "calling super using function captures" do + assert Overridable.capture_super(5) == [1, 2, 3, 4, 5, 1, 2, 3, 4, 5] end - test "calling super with explicit args" do - assert Overridable.super_with_explicit_args(1, 2) == 5 + test "calling super of an overridable macro using function captures" do + assert Overridable.capture_super_macro(5) == [1, 2, 3, 4, 5, 1, 2, 3, 4, 5] end - test "function without overridable returns false for super?" do - assert Overridable.no_overridable == {:no_overridable, false} + test "super as a variable" do + super = :ok + assert super == :ok end test "overridable with many clauses" do @@ -135,18 +276,262 @@ defmodule Kernel.OverridableTest do end test "overridable definitions are private" do - refute {:"with_super (overridable 0)", 0} in Overridable.__info__(:exports) + refute {:"with_super (overridable 0)", 0} in Overridable.module_info(:exports) + refute {:"with_super (overridable 1)", 0} in Overridable.module_info(:exports) + end + + test "multiple overrides" do + assert Overridable.multiple_overrides() == [3, 2, 1] + end + + test "overridable macros" do + a = 11 + assert Overridable.overridable_macro(a) == 1111 + assert Overridable.private_macro_call() == 1111 end test "invalid super call" do - try do - :elixir.eval 'defmodule Foo.Forwarding do\ndef bar, do: 1\ndefoverridable [bar: 0]\ndef foo, do: super\nend', [] - flunk "expected eval to fail" - rescue - error -> - assert Exception.message(error) == - "nofile:4: no super defined for foo/0 in module Foo.Forwarding. " <> - "Overridable functions available are: bar/0" + message = + "nofile:4: no super defined for foo/0 in module Kernel.OverridableOrder.Forwarding. " <> + "Overridable functions available are: bar/0" + + assert_raise CompileError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableOrder.Forwarding do + def bar(), do: 1 + defoverridable bar: 0 + def foo(), do: super() + end + """) + end + + purge(Kernel.OverridableOrder.Forwarding) + end + + test "invalid super call with different arity" do + message = + "nofile:4: super must be called with the same number of arguments as the current definition" + + assert_raise CompileError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableSuper.DifferentArities do + def bar(a), do: a + defoverridable bar: 1 + def bar(_), do: super() + end + """) + end + end + + test "invalid super capture with different arity" do + message = + "nofile:4: super must be called with the same number of arguments as the current definition" + + assert_raise CompileError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableSuperCapture.DifferentArities do + def bar(a), do: a + defoverridable bar: 1 + def bar(_), do: (&super/0).() + end + """) + end + end + + test "does not allow to override a macro as a function" do + message = + "nofile:4: cannot override macro (defmacro, defmacrop) foo/0 in module " <> + "Kernel.OverridableMacro.FunctionOverride as a function (def, defp)" + + assert_raise CompileError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableMacro.FunctionOverride do + defmacro foo(), do: :ok + defoverridable foo: 0 + def foo(), do: :invalid + end + """) end + + purge(Kernel.OverridableMacro.FunctionOverride) + + assert_raise CompileError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableMacro.FunctionOverride do + defmacro foo(), do: :ok + defoverridable foo: 0 + def foo(), do: :invalid + defoverridable foo: 0 + def foo(), do: :invalid + end + """) + end + + purge(Kernel.OverridableMacro.FunctionOverride) + + assert_raise CompileError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableMacro.FunctionOverride do + defmacro foo(), do: :ok + defoverridable foo: 0 + def foo(), do: super() + end + """) + end + + purge(Kernel.OverridableMacro.FunctionOverride) + end + + test "does not allow to override a function as a macro" do + message = + "nofile:4: cannot override function (def, defp) foo/0 in module " <> + "Kernel.OverridableFunction.MacroOverride as a macro (defmacro, defmacrop)" + + assert_raise CompileError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableFunction.MacroOverride do + def foo(), do: :ok + defoverridable foo: 0 + defmacro foo(), do: :invalid + end + """) + end + + purge(Kernel.OverridableFunction.MacroOverride) + + assert_raise CompileError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableFunction.MacroOverride do + def foo(), do: :ok + defoverridable foo: 0 + defmacro foo(), do: :invalid + defoverridable foo: 0 + defmacro foo(), do: :invalid + end + """) + end + + purge(Kernel.OverridableFunction.MacroOverride) + + assert_raise CompileError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableFunction.MacroOverride do + def foo(), do: :ok + defoverridable foo: 0 + defmacro foo(), do: super() + end + """) + end + + purge(Kernel.OverridableFunction.MacroOverride) + end + + test "undefined functions can't be marked as overridable" do + message = "cannot make function foo/2 overridable because it was not defined" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableOrder.Foo do + defoverridable foo: 2 + end + """) + end + + purge(Kernel.OverridableOrder.Foo) + end + + test "overrides with behaviour" do + defmodule OverridableWithBehaviour do + @behaviour Elixir.Kernel.OverridableExampleBehaviour + + def required_callback(), do: "original" + + def optional_callback(), do: "original" + + def not_a_behaviour_callback(), do: "original" + + defmacro required_macro_callback(boolean) do + quote do + if unquote(boolean) do + "original" + end + end + end + + defoverridable Elixir.Kernel.OverridableExampleBehaviour + + defmacro optional_macro_callback(arg1, arg2), do: {arg1, arg2} + + assert Module.overridable?(__MODULE__, {:required_callback, 0}) + assert Module.overridable?(__MODULE__, {:optional_callback, 0}) + assert Module.overridable?(__MODULE__, {:required_macro_callback, 1}) + refute Module.overridable?(__MODULE__, {:optional_macro_callback, 1}) + refute Module.overridable?(__MODULE__, {:not_a_behaviour_callback, 1}) + end + end + + test "undefined module can't be passed as argument to defoverridable" do + message = + "cannot pass module Kernel.OverridableTest.Bar as argument to defoverridable/1 because it was not defined" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableTest.Foo do + defoverridable Kernel.OverridableTest.Bar + end + """) + end + + purge(Kernel.OverridableTest.Foo) + end + + test "module without @behaviour can't be passed as argument to defoverridable" do + message = + "cannot pass module Kernel.OverridableExampleBehaviour as argument to defoverridable/1" <> + " because its corresponding behaviour is missing. Did you forget to add " <> + "@behaviour Kernel.OverridableExampleBehaviour?" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableTest.Foo do + defoverridable Kernel.OverridableExampleBehaviour + end + """) + end + + purge(Kernel.OverridableTest.Foo) + end + + test "module with no callbacks can't be passed as argument to defoverridable" do + message = + "cannot pass module Kernel.OverridableTest.Bar as argument to defoverridable/1 because it does not define any callbacks" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableTest.Bar do + end + defmodule Kernel.OverridableTest.Foo do + @behaviour Kernel.OverridableTest.Bar + defoverridable Kernel.OverridableTest.Bar + end + """) + end + + purge(Kernel.OverridableTest.Bar) + purge(Kernel.OverridableTest.Foo) + end + + test "atom which is not a module can't be passed as argument to defoverridable" do + message = "cannot pass module :abc as argument to defoverridable/1 because it was not defined" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule Kernel.OverridableTest.Foo do + defoverridable :abc + end + """) + end + + purge(Kernel.OverridableTest.Foo) end end diff --git a/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs b/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs new file mode 100644 index 00000000000..dd083a5dd49 --- /dev/null +++ b/lib/elixir/test/elixir/kernel/parallel_compiler_test.exs @@ -0,0 +1,549 @@ +Code.require_file("../test_helper.exs", __DIR__) + +import PathHelpers + +defmodule Kernel.ParallelCompilerTest do + use ExUnit.Case + import ExUnit.CaptureIO + + defp purge(modules) do + Enum.map(modules, fn mod -> + :code.purge(mod) + :code.delete(mod) + end) + end + + defp write_tmp(context, kv) do + dir = tmp_path(context) + File.rm_rf!(dir) + File.mkdir_p!(dir) + + for {key, contents} <- kv do + path = Path.join(dir, "#{key}.ex") + File.write!(path, contents) + path + end + end + + describe "compile" do + test "with profiling" do + fixtures = + write_tmp( + "profile_time", + bar: """ + defmodule HelloWorld do + end + """ + ) + + profile = + capture_io(:stderr, fn -> + assert {:ok, modules, []} = Kernel.ParallelCompiler.compile(fixtures, profile: :time) + + assert HelloWorld in modules + end) + + assert profile =~ + ~r"\[profile\] [\s\d]{6}ms compiling \+ 0ms waiting for .*tmp/profile_time/bar.ex" + + assert profile =~ ~r"\[profile\] Finished compilation cycle of 1 modules in \d+ms" + assert profile =~ ~r"\[profile\] Finished group pass check of 1 modules in \d+ms" + after + purge([HelloWorld]) + end + + test "solves dependencies between modules" do + fixtures = + write_tmp( + "parallel_compiler", + bar: """ + defmodule BarParallel do + end + + require FooParallel + IO.puts(FooParallel.message()) + """, + foo: """ + defmodule FooParallel do + # We use this ensure_compiled clause so both Foo and + # Bar block. Foo depends on Unknown and Bar depends on + # Foo. The compiler will see this dependency and first + # release Foo and then Bar, compiling with success. + {:error, _} = Code.ensure_compiled(Unknown) + def message, do: "message_from_foo" + end + """ + ) + + assert capture_io(fn -> + assert {:ok, modules, []} = Kernel.ParallelCompiler.compile(fixtures) + assert BarParallel in modules + assert FooParallel in modules + end) =~ "message_from_foo" + after + purge([FooParallel, BarParallel]) + end + + test "solves dependencies between structs" do + fixtures = + write_tmp( + "parallel_struct", + bar: """ + defmodule BarStruct do + defstruct name: "", foo: %FooStruct{} + end + """, + foo: """ + defmodule FooStruct do + defstruct name: "" + def bar?(%BarStruct{}), do: true + end + """ + ) + + assert {:ok, modules, []} = Kernel.ParallelCompiler.compile(fixtures) + assert [BarStruct, FooStruct] = Enum.sort(modules) + after + purge([FooStruct, BarStruct]) + end + + test "solves dependencies between structs in typespecs" do + fixtures = + write_tmp( + "parallel_typespec_struct", + bar: """ + defmodule BarStruct do + defstruct name: "" + @type t :: %FooStruct{} + end + """, + foo: """ + defmodule FooStruct do + defstruct name: "" + @type t :: %BarStruct{} + end + """ + ) + + assert {:ok, modules, []} = Kernel.ParallelCompiler.compile(fixtures) + assert [BarStruct, FooStruct] = Enum.sort(modules) + after + purge([FooStruct, BarStruct]) + end + + test "returns struct undefined error when local struct is undefined" do + [fixture] = + write_tmp( + "compile_struct", + undef: """ + defmodule Undef do + def undef() do + %__MODULE__{} + end + end + """ + ) + + expected_msg = "Undef.__struct__/1 is undefined, cannot expand struct Undef" + + assert capture_io(fn -> + assert {:error, [{^fixture, 3, msg}], []} = + Kernel.ParallelCompiler.compile([fixture]) + + assert msg =~ expected_msg + end) =~ expected_msg + end + + test "returns error when fails to expand struct" do + [fixture] = + write_tmp( + "compile_struct_invalid_key", + undef: """ + defmodule InvalidStructKey do + def invalid_struct_key() do + %Date{invalid_key: 2020} + end + end + """ + ) + + expected_msg = "** (KeyError) key :invalid_key not found" + + assert capture_io(fn -> + assert {:error, [{^fixture, 3, msg}], []} = + Kernel.ParallelCompiler.compile([fixture]) + + assert msg =~ expected_msg + end) =~ expected_msg + end + + test "does not hang on missing dependencies" do + [fixture] = + write_tmp( + "compile_does_not_hang", + with_behaviour_and_struct: """ + # We need to ensure it won't block even after multiple calls. + # So we use both behaviour and struct expansion below. + defmodule WithBehaviourAndStruct do + # @behaviour will call ensure_compiled(). + @behaviour :unknown + # Struct expansion calls it as well. + %ThisModuleWillNeverBeAvailable{} + end + """ + ) + + expected_msg = + "ThisModuleWillNeverBeAvailable.__struct__/1 is undefined, cannot expand struct ThisModuleWillNeverBeAvailable" + + assert capture_io(fn -> + assert {:error, [{^fixture, 7, msg}], []} = + Kernel.ParallelCompiler.compile([fixture]) + + assert msg =~ expected_msg + end) =~ "== Compilation error" + end + + test "does not deadlock on missing dependencies" do + [missing_struct, depends_on] = + write_tmp( + "does_not_deadlock", + missing_struct: """ + defmodule MissingStruct do + %ThisModuleWillNeverBeAvailable{} + def hello, do: :ok + end + """, + depends_on_missing_struct: """ + MissingStruct.hello() + """ + ) + + expected_msg = + "ThisModuleWillNeverBeAvailable.__struct__/1 is undefined, cannot expand struct ThisModuleWillNeverBeAvailable" + + assert capture_io(fn -> + assert {:error, [{^missing_struct, 2, msg}], []} = + Kernel.ParallelCompiler.compile([missing_struct, depends_on]) + + assert msg =~ expected_msg + end) =~ "== Compilation error" + end + + test "does not deadlock on missing import/struct dependencies" do + [missing_import, depends_on] = + write_tmp( + "import_and_structs", + missing_import: """ + defmodule MissingStruct do + import Unknown.Module + end + """, + depends_on_missing_struct: """ + %MissingStruct{} + """ + ) + + assert capture_io(fn -> + assert {:error, [{^missing_import, 2, msg}], []} = + Kernel.ParallelCompiler.compile([missing_import, depends_on]) + + assert msg =~ "module Unknown.Module is not loaded and could not be found" + end) =~ "== Compilation error" + end + + test "handles deadlocks" do + [foo, bar] = + write_tmp( + "parallel_deadlock", + foo: """ + defmodule FooDeadlock do + BarDeadlock.__info__(:module) + end + """, + bar: """ + defmodule BarDeadlock do + FooDeadlock.__info__(:module) + end + """ + ) + + msg = + capture_io(fn -> + fixtures = [foo, bar] + assert {:error, [bar_error, foo_error], []} = Kernel.ParallelCompiler.compile(fixtures) + assert bar_error == {bar, nil, "deadlocked waiting on module FooDeadlock"} + assert foo_error == {foo, nil, "deadlocked waiting on module BarDeadlock"} + end) + + assert msg =~ "Compilation failed because of a deadlock between files." + assert msg =~ "parallel_deadlock/foo.ex => BarDeadlock" + assert msg =~ "parallel_deadlock/bar.ex => FooDeadlock" + assert msg =~ ~r"== Compilation error in file .+parallel_deadlock/foo\.ex ==" + assert msg =~ "** (CompileError) deadlocked waiting on module BarDeadlock" + assert msg =~ ~r"== Compilation error in file .+parallel_deadlock/bar\.ex ==" + assert msg =~ "** (CompileError) deadlocked waiting on module FooDeadlock" + end + + test "does not deadlock from Code.ensure_compiled" do + [foo, bar] = + write_tmp( + "parallel_ensure_nodeadlock", + foo: """ + defmodule FooCircular do + {:error, :unavailable} = Code.ensure_compiled(BarCircular) + end + """, + bar: """ + defmodule BarCircular do + {:error, :unavailable} = Code.ensure_compiled(FooCircular) + end + """ + ) + + assert {:ok, _modules, []} = Kernel.ParallelCompiler.compile([foo, bar]) + assert Enum.sort([FooCircular, BarCircular]) == [BarCircular, FooCircular] + after + purge([FooCircular, BarCircular]) + end + + test "handles async compilation" do + [foo, bar] = + write_tmp( + "async_compile", + foo: """ + defmodule FooAsync do + true = Code.can_await_module_compilation?() + + Kernel.ParallelCompiler.async(fn -> + true = Code.can_await_module_compilation?() + BarAsync.__info__(:module) + end) + end + """, + bar: """ + defmodule BarAsync do + true = Code.can_await_module_compilation?() + end + """ + ) + + capture_io(fn -> + fixtures = [foo, bar] + assert assert {:ok, modules, []} = Kernel.ParallelCompiler.compile(fixtures) + assert FooAsync in modules + assert BarAsync in modules + end) + after + purge([FooAsync, BarAsync]) + end + + test "handles async deadlocks" do + [foo, bar] = + write_tmp( + "async_deadlock", + foo: """ + defmodule FooAsyncDeadlock do + Kernel.ParallelCompiler.async(fn -> + BarAsyncDeadlock.__info__(:module) + end) + + BarAsyncDeadlock.__info__(:module) + end + """, + bar: """ + defmodule BarAsyncDeadlock do + FooAsyncDeadlock.__info__(:module) + end + """ + ) + + capture_io(fn -> + fixtures = [foo, bar] + assert {:error, [bar_error, foo_error], []} = Kernel.ParallelCompiler.compile(fixtures) + assert bar_error == {bar, nil, "deadlocked waiting on module FooAsyncDeadlock"} + assert foo_error == {foo, nil, "deadlocked waiting on module BarAsyncDeadlock"} + end) + end + + test "supports warnings as errors" do + warnings_as_errors = Code.get_compiler_option(:warnings_as_errors) + + [fixture] = + write_tmp( + "warnings_as_errors", + warnings_as_errors: """ + defmodule WarningsSample do + def hello(a), do: a + def hello(b), do: b + end + """ + ) + + output = tmp_path("not_to_be_used") + + try do + Code.compiler_options(warnings_as_errors: true) + + msg = + capture_io(:stderr, fn -> + assert {:error, [error], []} = + Kernel.ParallelCompiler.compile_to_path([fixture], output) + + assert {^fixture, 3, "this clause " <> _} = error + end) + + assert msg =~ + "Compilation failed due to warnings while using the --warnings-as-errors option\n" + after + Code.compiler_options(warnings_as_errors: warnings_as_errors) + purge([WarningsSample]) + end + + refute File.exists?(output) + end + + test "does not use incorrect line number when error originates in another file" do + File.mkdir_p!(tmp_path()) + + [a, b] = + write_tmp( + "error_line", + a: """ + defmodule A do + def fun(arg), do: arg / 2 + end + """, + b: """ + defmodule B do + def fun(arg) do + A.fun(arg) + :ok + end + end + B.fun(:not_a_number) + """ + ) + + capture_io(fn -> + assert {:error, [{^b, 0, _}], _} = Kernel.ParallelCompiler.compile([a, b]) + end) + end + + test "gets correct line number for UndefinedFunctionError" do + File.mkdir_p!(tmp_path()) + + [fixture] = + write_tmp("undef", + undef: """ + defmodule UndefErrorLine do + Bogus.fun() + end + """ + ) + + capture_io(fn -> + assert {:error, [{^fixture, 2, _}], _} = Kernel.ParallelCompiler.compile([fixture]) + end) + end + + test "gets proper beam destinations from dynamic modules" do + fixtures = + write_tmp( + "dynamic", + dynamic: """ + Module.create(Dynamic, quote(do: :ok), file: "dynamic.ex") + [_ | _] = :code.which(Dynamic) + """ + ) + + assert {:ok, [Dynamic], []} = Kernel.ParallelCompiler.compile(fixtures, dest: "sample") + after + purge([Dynamic]) + end + end + + describe "require" do + test "returns struct undefined error when local struct is undefined" do + [fixture] = + write_tmp( + "require_struct", + undef: """ + defmodule Undef do + def undef() do + %__MODULE__{} + end + end + """ + ) + + expected_msg = "Undef.__struct__/1 is undefined, cannot expand struct Undef" + + assert capture_io(fn -> + assert {:error, [{^fixture, 3, msg}], []} = + Kernel.ParallelCompiler.require([fixture]) + + assert msg =~ expected_msg + end) =~ expected_msg + end + + test "does not hang on missing dependencies" do + [fixture] = + write_tmp( + "require_does_not_hang", + with_behaviour_and_struct: """ + # We need to ensure it won't block even after multiple calls. + # So we use both behaviour and struct expansion below. + defmodule WithBehaviourAndStruct do + # @behaviour will call ensure_compiled(). + @behaviour :unknown + # Struct expansion calls it as well. + %ThisModuleWillNeverBeAvailable{} + end + """ + ) + + expected_msg = + "ThisModuleWillNeverBeAvailable.__struct__/1 is undefined, cannot expand struct ThisModuleWillNeverBeAvailable" + + assert capture_io(fn -> + assert {:error, [{^fixture, 7, msg}], []} = + Kernel.ParallelCompiler.require([fixture]) + + assert msg =~ expected_msg + end) =~ "== Compilation error" + end + + test "supports warnings as errors" do + warnings_as_errors = Code.get_compiler_option(:warnings_as_errors) + + [fixture] = + write_tmp( + "warnings_as_errors", + warnings_as_errors: """ + defmodule WarningsSample do + def hello(a), do: a + def hello(b), do: b + end + """ + ) + + try do + Code.compiler_options(warnings_as_errors: true) + + msg = + capture_io(:stderr, fn -> + assert {:error, [error], []} = Kernel.ParallelCompiler.require([fixture]) + + assert {^fixture, 3, "this clause " <> _} = error + end) + + assert msg =~ + "Compilation failed due to warnings while using the --warnings-as-errors option\n" + after + Code.compiler_options(warnings_as_errors: warnings_as_errors) + purge([WarningsSample]) + end + end + end +end diff --git a/lib/elixir/test/elixir/kernel/parser_test.exs b/lib/elixir/test/elixir/kernel/parser_test.exs new file mode 100644 index 00000000000..c7e8979768a --- /dev/null +++ b/lib/elixir/test/elixir/kernel/parser_test.exs @@ -0,0 +1,982 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.ParserTest do + use ExUnit.Case, async: true + + describe "nullary ops" do + test "in expressions" do + assert parse!("..") == {:.., [line: 1], []} + end + + test "raises on ambiguous uses" do + assert_raise SyntaxError, ~r/syntax error before: do/, fn -> + parse!("if .. do end") + end + end + end + + describe "unary ops" do + test "in keywords" do + assert parse!("f(!: :ok)") == {:f, [line: 1], [[!: :ok]]} + assert parse!("f @: :ok") == {:f, [line: 1], [[@: :ok]]} + end + + test "ambiguous ops in keywords" do + assert parse!("f(+: :ok)") == {:f, [line: 1], [[+: :ok]]} + assert parse!("f +: :ok") == {:f, [line: 1], [[+: :ok]]} + end + end + + describe "ternary ops" do + test "root" do + assert parse!("1..2//3") == {:"..//", [line: 1], [1, 2, 3]} + assert parse!("(1..2)//3") == {:"..//", [line: 1], [1, 2, 3]} + end + + test "with do-blocks" do + assert parse!("foo do end..bar do end//baz do end") == { + :"..//", + [line: 1], + [ + {:foo, [line: 1], [[do: {:__block__, [], []}]]}, + {:bar, [line: 1], [[do: {:__block__, [], []}]]}, + {:baz, [line: 1], [[do: {:__block__, [], []}]]} + ] + } + end + + test "with no parens" do + assert parse!("1..foo do end//bar bat, baz") == { + :"..//", + [line: 1], + [ + 1, + {:foo, [line: 1], [[do: {:__block__, [], []}]]}, + {:bar, [line: 1], [{:bat, [line: 1], nil}, {:baz, [line: 1], nil}]} + ] + } + end + + test "errors" do + msg = + ~r/the range step operator \(\/\/\) must immediately follow the range definition operator \(\.\.\)/ + + assert_syntax_error(msg, "foo..bar baz//bat") + assert_syntax_error(msg, "foo++bar//bat") + assert_syntax_error(msg, "foo..(bar//bat)") + end + end + + describe "identifier unicode normalization" do + test "nfc normalization is performed" do + # before elixir 1.14, non-nfc would error + # non-nfc: "ç" (code points 0x0063 0x0327) + # nfc-normalized: "ç" (code points 0x00E7) + assert Code.eval_string("ç = 1; ç") == {1, [ç: 1]} + end + + test "elixir's additional normalization is performed" do + # Common micro => Greek mu. See code formatter test too. + assert Code.eval_string("µs = 1; μs") == {1, [{:μs, 1}]} + + # commented out: math symbols capability in elixir + # normalizations, to ensure that we *can* handle codepoints + # that are Common-script and non-ASCII + # assert Code.eval_string("_ℕ𝕩 = 1") == {1, [{:"_ℕ𝕩", 1}]} + end + end + + describe "strings/sigils" do + test "delimiter information for sigils is included" do + string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: false) + + assert parse!("~r/foo/") == + {:sigil_r, [delimiter: "/", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]} + + assert string_to_quoted.("~r[foo]") == + {:sigil_r, [delimiter: "[", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]} + + assert string_to_quoted.("~r\"foo\"") == + {:sigil_r, [delimiter: "\"", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]} + + meta = [delimiter: "\"\"\"", line: 1] + args = {:sigil_S, meta, [{:<<>>, [indentation: 0, line: 1], ["sigil heredoc\n"]}, []]} + assert string_to_quoted.("~S\"\"\"\nsigil heredoc\n\"\"\"") == args + + meta = [delimiter: "'''", line: 1] + args = {:sigil_S, meta, [{:<<>>, [indentation: 0, line: 1], ["sigil heredoc\n"]}, []]} + assert string_to_quoted.("~S'''\nsigil heredoc\n'''") == args + end + + test "sigil newlines" do + assert {:sigil_s, _, [{:<<>>, _, ["here\ndoc"]}, []]} = + Code.string_to_quoted!(~s|~s"here\ndoc"|) + + assert {:sigil_s, _, [{:<<>>, _, ["here\r\ndoc"]}, []]} = + Code.string_to_quoted!(~s|~s"here\r\ndoc"|) + end + + test "string newlines" do + assert Code.string_to_quoted!(~s|"here\ndoc"|) == "here\ndoc" + assert Code.string_to_quoted!(~s|"here\r\ndoc"|) == "here\r\ndoc" + assert Code.string_to_quoted!(~s|"here\\\ndoc"|) == "heredoc" + assert Code.string_to_quoted!(~s|"here\\\r\ndoc"|) == "heredoc" + end + + test "heredoc newlines" do + assert Code.string_to_quoted!(~s|"""\nhere\ndoc\n"""|) == "here\ndoc\n" + assert Code.string_to_quoted!(~s|"""\r\nhere\r\ndoc\r\n"""|) == "here\r\ndoc\r\n" + assert Code.string_to_quoted!(~s| """\n here\n doc\n """|) == "here\ndoc\n" + assert Code.string_to_quoted!(~s| """\r\n here\r\n doc\r\n """|) == "here\r\ndoc\r\n" + assert Code.string_to_quoted!(~s|"""\nhere\\\ndoc\\\n"""|) == "heredoc" + assert Code.string_to_quoted!(~s|"""\r\nhere\\\r\ndoc\\\r\n"""|) == "heredoc" + end + + test "heredoc indentation" do + meta = [delimiter: "'''", line: 1] + args = {:sigil_S, meta, [{:<<>>, [indentation: 2, line: 1], [" sigil heredoc\n"]}, []]} + assert Code.string_to_quoted!("~S'''\n sigil heredoc\n '''") == args + end + end + + describe "string_to_quoted/2" do + test "converts strings to quoted expressions" do + assert Code.string_to_quoted("1 + 2") == {:ok, {:+, [line: 1], [1, 2]}} + + assert Code.string_to_quoted("a.1") == + {:error, {[line: 1, column: 3], "syntax error before: ", "\"1\""}} + end + end + + describe "string_to_quoted/2 and atom handling" do + test "ensures :existing_atoms_only" do + assert Code.string_to_quoted(":there_is_no_such_atom", existing_atoms_only: true) == + {:error, + {[line: 1, column: 1], "unsafe atom does not exist: ", "there_is_no_such_atom"}} + end + + test "encodes atoms" do + ref = make_ref() + + encoder = fn atom, meta -> + assert atom == "there_is_no_such_atom" + assert meta[:line] == 1 + assert meta[:column] == 1 + {:ok, {:my, "atom", ref}} + end + + assert {:ok, {:my, "atom", ^ref}} = + Code.string_to_quoted(":there_is_no_such_atom", static_atoms_encoder: encoder) + end + + test "encodes vars" do + ref = make_ref() + + encoder = fn atom, meta -> + assert atom == "there_is_no_such_var" + assert meta[:line] == 1 + assert meta[:column] == 1 + {:ok, {:my, "atom", ref}} + end + + assert {:ok, {{:my, "atom", ^ref}, [line: 1], nil}} = + Code.string_to_quoted("there_is_no_such_var", static_atoms_encoder: encoder) + end + + test "encodes quoted keyword keys" do + ref = make_ref() + + encoder = fn atom, meta -> + assert atom == "there is no such key" + assert meta[:line] == 1 + assert meta[:column] == 2 + {:ok, {:my, "atom", ref}} + end + + assert {:ok, [{{:my, "atom", ^ref}, true}]} = + Code.string_to_quoted(~S(["there is no such key": true]), + static_atoms_encoder: encoder + ) + end + + test "addresses ambiguities" do + encoder = fn string, _meta -> {:ok, {:atom, string}} end + + # We check a=1 for precedence issues with a!=1, make sure it works + assert Code.string_to_quoted!("a = 1", static_atoms_encoder: encoder) + assert Code.string_to_quoted!("a=1", static_atoms_encoder: encoder) + end + + test "does not encode keywords" do + encoder = fn atom, _meta -> raise "shouldn't be invoked for #{atom}" end + + assert {:ok, {:fn, [line: 1], [{:->, [line: 1], [[1], 2]}]}} = + Code.string_to_quoted("fn 1 -> 2 end", static_atoms_encoder: encoder) + + assert {:ok, {:or, [line: 1], [true, false]}} = + Code.string_to_quoted("true or false", static_atoms_encoder: encoder) + + encoder = fn atom, _meta -> {:ok, {:encoded, atom}} end + + assert {:ok, [encoded: "true", encoded: "do", encoded: "and"]} = + Code.string_to_quoted("[:true, :do, :and]", static_atoms_encoder: encoder) + + assert {:ok, [{{:encoded, "do"}, 1}, {{:encoded, "true"}, 2}, {{:encoded, "end"}, 3}]} = + Code.string_to_quoted("[do: 1, true: 2, end: 3]", static_atoms_encoder: encoder) + end + + test "returns errors on long atoms even when using static_atoms_encoder" do + atom = String.duplicate("a", 256) + + encoder = fn atom, _meta -> {:ok, atom} end + + assert Code.string_to_quoted(atom, static_atoms_encoder: encoder) == + {:error, + {[line: 1, column: 1], "atom length must be less than system limit: ", atom}} + end + + test "may return errors" do + encoder = fn _atom, _meta -> + {:error, "Invalid atom name"} + end + + assert {:error, {[line: 1, column: 1], "Invalid atom name: ", "there_is_no_such_atom"}} = + Code.string_to_quoted(":there_is_no_such_atom", static_atoms_encoder: encoder) + end + + test "may return tuples" do + encoder = fn string, _metadata -> + try do + {:ok, String.to_existing_atom(string)} + rescue + ArgumentError -> + {:ok, {:user_atom, string}} + end + end + + assert {:ok, {:try, _, [[do: {:test, _, [{{:user_atom, "atom_does_not_exist"}, _, []}]}]]}} = + Code.string_to_quoted("try do: test(atom_does_not_exist())", + static_atoms_encoder: encoder + ) + end + end + + describe "string_to_quoted/2 with :columns" do + test "includes column information" do + string_to_quoted = &Code.string_to_quoted(&1, columns: true) + assert string_to_quoted.("1 + 2") == {:ok, {:+, [line: 1, column: 3], [1, 2]}} + + foo = {:foo, [line: 1, column: 1], nil} + bar = {:bar, [line: 1, column: 7], nil} + assert string_to_quoted.("foo + bar") == {:ok, {:+, [line: 1, column: 5], [foo, bar]}} + + nfc_abba = [225, 98, 98, 224] + nfd_abba = [97, 769, 98, 98, 97, 768] + context = [line: 1, column: 8] + expr = "'ábbà' = 1" + + assert string_to_quoted.(String.normalize(expr, :nfc)) == + {:ok, {:=, context, [nfc_abba, 1]}} + + assert string_to_quoted.(String.normalize(expr, :nfd)) == + {:ok, {:=, context, [nfd_abba, 1]}} + end + end + + describe "string_to_quoted/2 with :token_metadata" do + test "adds end_of_expression information to blocks" do + file = """ + one();two() + three() + + four() + + + five() + """ + + args = [ + {:one, + [ + end_of_expression: [newlines: 0, line: 1, column: 6], + closing: [line: 1, column: 5], + line: 1, + column: 1 + ], []}, + {:two, + [ + end_of_expression: [newlines: 1, line: 1, column: 12], + closing: [line: 1, column: 11], + line: 1, + column: 7 + ], []}, + {:three, + [ + end_of_expression: [newlines: 2, line: 2, column: 8], + closing: [line: 2, column: 7], + line: 2, + column: 1 + ], []}, + {:four, + [ + end_of_expression: [newlines: 3, line: 4, column: 7], + closing: [line: 4, column: 6], + line: 4, + column: 1 + ], []}, + {:five, [closing: [line: 7, column: 6], line: 7, column: 1], []} + ] + + assert Code.string_to_quoted!(file, token_metadata: true, columns: true) == + {:__block__, [], args} + end + + test "adds pairing information" do + string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: true) + + assert string_to_quoted.("foo") == {:foo, [line: 1], nil} + assert string_to_quoted.("foo()") == {:foo, [closing: [line: 1], line: 1], []} + + assert string_to_quoted.("foo(\n)") == + {:foo, [newlines: 1, closing: [line: 2], line: 1], []} + + assert string_to_quoted.("%{\n}") == {:%{}, [newlines: 1, closing: [line: 2], line: 1], []} + + assert string_to_quoted.("foo(\n) do\nend") == + {:foo, [do: [line: 2], end: [line: 3], newlines: 1, closing: [line: 2], line: 1], + [[do: {:__block__, [], []}]]} + end + + test "with :literal_encoder" do + opts = [literal_encoder: &{:ok, {:__block__, &2, [&1]}}, token_metadata: true] + string_to_quoted = &Code.string_to_quoted!(&1, opts) + + assert string_to_quoted.(~s("one")) == {:__block__, [delimiter: "\"", line: 1], ["one"]} + assert string_to_quoted.("'one'") == {:__block__, [delimiter: "'", line: 1], ['one']} + assert string_to_quoted.("?é") == {:__block__, [token: "?é", line: 1], [233]} + assert string_to_quoted.("0b10") == {:__block__, [token: "0b10", line: 1], [2]} + assert string_to_quoted.("12") == {:__block__, [token: "12", line: 1], [12]} + assert string_to_quoted.("0o123") == {:__block__, [token: "0o123", line: 1], [83]} + assert string_to_quoted.("0xEF") == {:__block__, [token: "0xEF", line: 1], [239]} + assert string_to_quoted.("12.3") == {:__block__, [token: "12.3", line: 1], [12.3]} + assert string_to_quoted.("nil") == {:__block__, [line: 1], [nil]} + assert string_to_quoted.(":one") == {:__block__, [line: 1], [:one]} + + assert string_to_quoted.("[one: :two]") == { + :__block__, + [{:closing, [line: 1]}, {:line, 1}], + [ + [ + {{:__block__, [format: :keyword, line: 1], [:one]}, + {:__block__, [line: 1], [:two]}} + ] + ] + } + + assert string_to_quoted.("[1]") == + {:__block__, [closing: [line: 1], line: 1], + [[{:__block__, [token: "1", line: 1], [1]}]]} + + assert string_to_quoted.(~s("""\nhello\n""")) == + {:__block__, [delimiter: ~s["""], indentation: 0, line: 1], ["hello\n"]} + + assert string_to_quoted.("'''\nhello\n'''") == + {:__block__, [delimiter: ~s['''], indentation: 0, line: 1], ['hello\n']} + + assert string_to_quoted.(~s[fn (1) -> "hello" end]) == + {:fn, [closing: [line: 1], line: 1], + [ + {:->, [line: 1], + [ + [{:__block__, [token: "1", line: 1, closing: [line: 1], line: 1], [1]}], + {:__block__, [delimiter: "\"", line: 1], ["hello"]} + ]} + ]} + end + + test "adds identifier_location for qualified identifiers" do + string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: true, columns: true) + + assert string_to_quoted.("foo.\nbar") == + {{:., [line: 1, column: 4], + [ + {:foo, [line: 1, column: 1], nil}, + :bar + ]}, [no_parens: true, line: 2, column: 1], []} + + assert string_to_quoted.("foo\n.\nbar") == + {{:., [line: 2, column: 1], + [ + {:foo, [line: 1, column: 1], nil}, + :bar + ]}, [no_parens: true, line: 3, column: 1], []} + + assert string_to_quoted.(~s[Foo.\nbar(1)]) == + {{:., [line: 1, column: 4], + [ + {:__aliases__, [last: [line: 1, column: 1], line: 1, column: 1], [:Foo]}, + :bar + ]}, [closing: [line: 2, column: 6], line: 2, column: 1], [1]} + end + + test "adds metadata for the last alias segment" do + string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: true) + + assert string_to_quoted.("Foo") == {:__aliases__, [last: [line: 1], line: 1], [:Foo]} + + assert string_to_quoted.("Foo.\nBar\n.\nBaz") == + {:__aliases__, [last: [line: 4], line: 1], [:Foo, :Bar, :Baz]} + + assert string_to_quoted.("foo.\nBar\n.\nBaz") == + {:__aliases__, [last: [line: 4], line: 1], [{:foo, [line: 1], nil}, :Bar, :Baz]} + end + end + + describe "token missing errors" do + test "missing paren" do + assert_token_missing( + ~r/nofile:1:9: missing terminator: \) \(for \"\(\" starting at line 1\)/, + 'case 1 (' + ) + end + + test "dot terminator" do + assert_token_missing( + ~r/nofile:1:9: missing terminator: \" \(for function name starting at line 1\)/, + 'foo."bar' + ) + end + + test "sigil terminator" do + assert_token_missing( + ~r/nofile:3:1: missing terminator: " \(for sigil ~r" starting at line 1\)/, + '~r"foo\n\n' + ) + + assert_token_missing( + ~r/nofile:3:1: missing terminator: } \(for sigil ~r{ starting at line 1\)/, + '~r{foo\n\n' + ) + end + + test "string terminator" do + assert_token_missing( + ~r/nofile:1:5: missing terminator: \" \(for string starting at line 1\)/, + '"bar' + ) + end + + test "heredoc with incomplete interpolation" do + assert_token_missing( + ~r/nofile:2:1: missing interpolation terminator: \"}\" \(for heredoc starting at line 1\)/, + '"""\n\#{\n' + ) + end + + test "heredoc terminator" do + assert_token_missing( + ~r/nofile:2:4: missing terminator: \"\"\" \(for heredoc starting at line 1\)/, + '"""\nbar' + ) + + assert_token_missing( + ~r/nofile:2:7: missing terminator: \"\"\" \(for heredoc starting at line 1\)/, + '"""\nbar"""' + ) + end + + test "missing end" do + assert_token_missing( + ~r/nofile:1:9: missing terminator: end \(for \"do\" starting at line 1\)/, + 'foo do 1' + ) + + assert_token_missing( + ~r"HINT: it looks like the \"do\" on line 2 does not have a matching \"end\"", + ''' + defmodule MyApp do + def one do + # end + + def two do + end + end + ''' + ) + end + end + + describe "syntax errors" do + test "invalid heredoc start" do + assert_syntax_error( + ~r/nofile:1:1: heredoc allows only zero or more whitespace characters followed by a new line after \"\"\"/, + '"""bar\n"""' + ) + end + + test "invalid fn" do + assert_syntax_error( + ~r/nofile:1:1: expected anonymous functions to be defined with -> inside: 'fn'/, + 'fn 1 end' + ) + + assert_syntax_error( + ~r/nofile:2: unexpected operator ->. If you want to define multiple clauses,/, + 'fn 1\n2 -> 3 end' + ) + end + + test "invalid token" do + assert_syntax_error( + ~r/nofile:1:1: unexpected token: "#{"\u3164"}" \(column 1, code point U\+3164\)/, + 'ㅤ = 1' + ) + + assert_syntax_error( + ~r/nofile:1:7: unexpected token: "#{"\u200B"}" \(column 7, code point U\+200B\)/, + '[foo: \u200B]\noops' + ) + + assert_syntax_error( + ~r/nofile:1:1: unexpected token: carriage return \(column 1, code point U\+000D\)/, + '\r' + ) + end + + test "invalid bidi in source" do + assert_syntax_error( + ~r"nofile:1:1: invalid bidirectional formatting character in comment: \\u202A", + '# This is a \u202A' + ) + + assert_syntax_error( + ~r"nofile:1:5: invalid bidirectional formatting character in comment: \\u202A", + 'foo. # This is a \u202A' + ) + + assert_syntax_error( + ~r"nofile:1:12: invalid bidirectional formatting character in string: \\u202A. If you want to use such character, use it in its escaped \\u202A form instead", + '"this is a \u202A"' + ) + + assert_syntax_error( + ~r"nofile:1:13: invalid bidirectional formatting character in string: \\u202A. If you want to use such character, use it in its escaped \\u202A form instead", + '"this is a \\\u202A"' + ) + end + + test "reserved tokens" do + assert_syntax_error(~r/nofile:1:1: reserved token: __aliases__/, '__aliases__') + assert_syntax_error(~r/nofile:1:1: reserved token: __block__/, '__block__') + end + + test "invalid alias terminator" do + assert_syntax_error(~r/nofile:1:5: unexpected \( after alias Foo/, 'Foo()') + end + + test "invalid quoted token" do + assert_syntax_error( + ~r/nofile:1:9: syntax error before: \"world\"/, + '"hello" "world"' + ) + + assert_syntax_error( + ~r/nofile:1:3: syntax error before: 'Foobar'/, + '1 Foobar' + ) + + assert_syntax_error( + ~r/nofile:1:5: syntax error before: foo/, + 'Foo.:foo' + ) + + assert_syntax_error( + ~r/nofile:1:5: syntax error before: \"foo\"/, + 'Foo.:"foo\#{:bar}"' + ) + + assert_syntax_error( + ~r/nofile:1:5: syntax error before: \"/, + 'Foo.:"\#{:bar}"' + ) + end + + test "invalid identifier" do + message = fn name -> + ~r/nofile:1:1: invalid character "@" \(code point U\+0040\) in identifier: #{name}/ + end + + assert_syntax_error(message.("foo@"), 'foo@') + assert_syntax_error(message.("foo@"), 'foo@ ') + assert_syntax_error(message.("foo@bar"), 'foo@bar') + + message = fn name -> + ~r/nofile:1:1: invalid character "@" \(code point U\+0040\) in alias: #{name}/ + end + + assert_syntax_error(message.("Foo@"), 'Foo@') + assert_syntax_error(message.("Foo@bar"), 'Foo@bar') + + message = + ~r/nofile:1:1: invalid character "\!" \(code point U\+0021\) in alias \(only ASCII characters, without punctuation, are allowed\): Foo\!/ + + assert_syntax_error(message, 'Foo!') + + message = + ~r/nofile:1:1: invalid character "\?" \(code point U\+003F\) in alias \(only ASCII characters, without punctuation, are allowed\): Foo\?/ + + assert_syntax_error(message, 'Foo?') + + message = + ~r/nofile:1:1: invalid character \"ó\" \(code point U\+00F3\) in alias \(only ASCII characters, without punctuation, are allowed\): Foó/ + + assert_syntax_error(message, 'Foó') + + # token suggestion heuristic: + # "for foO𝚳, NFKC isn't enough because 𝚳 nfkc's to Greek Μ, would be mixed script. + # however the 'confusability skeleton' for that token produces an all-Latin foOM + # and would tokenize -- so suggest that, in case that's what they want" + message = + String.trim(""" + Codepoint failed identifier tokenization, but a simpler form was found. + + Got: + + "foO𝚳" (code points 0x00066 0x0006F 0x0004F 0x1D6B3) + + Hint: You could write the above in a similar way that is accepted by Elixir: + + "foOM" (code points 0x00066 0x0006F 0x0004F 0x0004D) + + See https://hexdocs.pm/elixir/unicode-syntax.html for more information. + | + 1 | foO𝚳 + | ^ + """) + + assert_syntax_error(~r/#{message}/, 'foO𝚳') + + # token suggestion heuristic: + # "for fooی𝚳, both NKFC and confusability would result in mixed scripts, + # because the Farsi letter is confusable with a different Arabic letter. + # Well, can't fix it all at once -- let's check for a suggestion just on + # the one codepoint that triggered this, the 𝚳 -- that would at least + # nudge them forwards." + message = + String.trim(""" + Elixir expects unquoted Unicode atoms, variables, and calls to use allowed codepoints and to be in NFC form. + + Got: + + "𝚳" (code points 0x1D6B3) + + Hint: You could write the above in a compatible format that is accepted by Elixir: + + "Μ" (code points 0x0039C) + + See https://hexdocs.pm/elixir/unicode-syntax.html for more information. + | + 2 | fooی𝚳 + | ^ + """) + + assert_syntax_error(~r/#{message}/, 'fooی𝚳') + end + + test "keyword missing space" do + msg = ~r/nofile:1:1: keyword argument must be followed by space after: foo:/ + + assert_syntax_error(msg, "foo:bar") + assert_syntax_error(msg, "foo:+") + assert_syntax_error(msg, "foo:+1") + end + + test "expression after keyword lists" do + assert_syntax_error( + ~r"unexpected expression after keyword list", + 'call foo: 1, :bar' + ) + + assert_syntax_error( + ~r"unexpected expression after keyword list", + 'call(foo: 1, :bar)' + ) + + assert_syntax_error( + ~r"unexpected expression after keyword list", + '[foo: 1, :bar]' + ) + + assert_syntax_error( + ~r"unexpected expression after keyword list", + '%{foo: 1, :bar => :bar}' + ) + end + + test "syntax errors include formatted snippet" do + message = "nofile:1:5: syntax error before: '*'\n |\n 1 | 1 + * 3\n | ^" + assert_syntax_error(message, "1 + * 3") + end + + test "invalid map start" do + assert_syntax_error( + ~r/nofile:1:7: expected %{ to define a map, got: %\[/, + "{:ok, %[], %{}}" + ) + end + + test "unexpected end" do + assert_syntax_error("nofile:1:3: unexpected reserved word: end", '1 end') + + assert_syntax_error( + ~r" HINT: it looks like the \"end\" on line 2 does not have a matching \"do\" defined before it", + ''' + defmodule MyApp do + def one end + def two do end + end + ''' + ) + + assert_syntax_error( + ~r" HINT: it looks like the \"end\" on line 3 does not have a matching \"do\" defined before it", + ''' + defmodule MyApp do + def one + end + + def two do + end + end + ''' + ) + + assert_syntax_error( + ~r" HINT: it looks like the \"end\" on line 6 does not have a matching \"do\" defined before it", + ''' + defmodule MyApp do + def one do + end + + def two + end + end + ''' + ) + + assert_syntax_error( + ~r"HINT: it looks like the \"do\" on line 3 does not have a matching \"end\"", + ''' + defmodule MyApp do + ( + def one do + # end + + def two do + end + ) + end + ''' + ) + end + + test "invalid keywords" do + assert_syntax_error( + ~r/nofile:1:2: syntax error before: '.'/, + '+.foo' + ) + + assert_syntax_error( + ~r/nofile:1:1: syntax error before: after. \"after\" is a reserved word/, + 'after = 1' + ) + end + + test "before sigil" do + msg = fn x -> "nofile:1:9: syntax error before: sigil ~s starting with content '#{x}'" end + + assert_syntax_error(msg.("bar baz"), '~s(foo) ~s(bar baz)') + assert_syntax_error(msg.(""), '~s(foo) ~s()') + assert_syntax_error(msg.("bar "), '~s(foo) ~s(bar \#{:baz})') + assert_syntax_error(msg.(""), '~s(foo) ~s(\#{:bar} baz)') + end + + test "invalid do" do + assert_syntax_error( + ~r/nofile:1:10: unexpected reserved word: do./, + 'if true, do\n' + ) + + assert_syntax_error(~r/nofile:1:9: unexpected keyword: do:./, 'if true do:\n') + end + + test "invalid parens call" do + msg = + "nofile:1:5: unexpected parentheses. If you are making a function call, do not " <> + "insert spaces between the function name and the opening parentheses. " <> + "Syntax error before: '\\('" + + assert_syntax_error(~r/#{msg}/, 'foo (hello, world)') + end + + test "invalid nested no parens call" do + msg = ~r"nofile:1: unexpected comma. Parentheses are required to solve ambiguity" + + assert_syntax_error(msg, '[foo 1, 2]') + assert_syntax_error(msg, '[foo bar 1, 2]') + assert_syntax_error(msg, '[do: foo 1, 2]') + assert_syntax_error(msg, 'foo(do: bar 1, 2)') + assert_syntax_error(msg, '{foo 1, 2}') + assert_syntax_error(msg, '{foo bar 1, 2}') + assert_syntax_error(msg, 'foo 1, foo 2, 3') + assert_syntax_error(msg, 'foo 1, @bar 3, 4') + assert_syntax_error(msg, 'foo 1, 2 + bar 3, 4') + assert_syntax_error(msg, 'foo(1, foo 2, 3)') + + interpret = fn x -> Macro.to_string(Code.string_to_quoted!(x)) end + assert interpret.("f 1 + g h 2, 3") == "f(1 + g(h(2, 3)))" + + assert interpret.("assert [] = TestRepo.all from p in Post, where: p.title in ^[]") == + "assert [] = TestRepo.all(from(p in Post, where: p.title in ^[]))" + end + + test "invalid atom dot alias" do + msg = + "nofile:1:6: atom cannot be followed by an alias. If the '.' was meant to be " <> + "part of the atom's name, the atom name must be quoted. Syntax error before: '.'" + + assert_syntax_error(~r/#{msg}/, ':foo.Bar') + assert_syntax_error(~r/#{msg}/, ':"+".Bar') + end + + test "invalid map/struct" do + assert_syntax_error(~r/nofile:1:5: syntax error before: '}'/, '%{:a}') + assert_syntax_error(~r/nofile:1:11: syntax error before: '}'/, '%{{:a, :b}}') + assert_syntax_error(~r/nofile:1:8: syntax error before: '{'/, '%{a, b}{a: :b}') + end + + test "invalid interpolation" do + assert_syntax_error( + ~r/nofile:1:17: unexpected token: \). The \"do\" at line 1 is missing terminator \"end\"/, + '"foo\#{case 1 do )}bar"' + ) + end + + test "invalid end of expression" do + # All valid examples + Code.eval_quoted(''' + 1; + 2; + 3 + + (;) + (;1) + (1;) + (1; 2) + + fn -> 1; 2 end + fn -> ; end + + if true do + ; + end + + try do + ; + catch + _, _ -> ; + after + ; + end + ''') + + # All invalid examples + assert_syntax_error(~r/nofile:1:3: syntax error before: ';'/, '1+;\n2') + + assert_syntax_error(~r/nofile:1:8: syntax error before: ';'/, 'max(1, ;2)') + end + + test "invalid new line" do + assert_syntax_error( + "nofile:3:6: unexpectedly reached end of line. The current expression is invalid or incomplete", + 'if true do\n foo = [],\n baz\nend' + ) + end + + test "invalid \"fn do expr end\"" do + assert_syntax_error( + "nofile:1:4: unexpected reserved word: do. Anonymous functions are written as:\n\n fn pattern -> expression end", + 'fn do :ok end' + ) + end + + test "characters literal are printed correctly in syntax errors" do + assert_syntax_error("nofile:1:5: syntax error before: ?a", ':ok ?a') + assert_syntax_error("nofile:1:5: syntax error before: ?\\s", ':ok ?\\s') + assert_syntax_error("nofile:1:5: syntax error before: ?す", ':ok ?す') + end + + test "numbers are printed correctly in syntax errors" do + assert_syntax_error(~r/nofile:1:5: syntax error before: \"12\"/, ':ok 12') + assert_syntax_error(~r/nofile:1:5: syntax error before: \"0b1\"/, ':ok 0b1') + assert_syntax_error(~r/nofile:1:5: syntax error before: \"12.3\"/, ':ok 12.3') + + assert_syntax_error( + ~r"nofile:1:1: invalid character \"_\" after number 123_456", + '123_456_foo' + ) + end + + test "on hex errors" do + msg = + "invalid hex escape character, expected \\\\xHH where H is a hexadecimal digit. Syntax error after: \\\\x" + + assert_syntax_error(~r/nofile:1:2: #{msg}/, ~S["\x"]) + assert_syntax_error(~r/nofile:1:1: #{msg}/, ~S[:"\x"]) + assert_syntax_error(~r/nofile:1:2: #{msg}/, ~S["\x": 123]) + assert_syntax_error(~r/nofile:1:1: #{msg}/, ~s["""\n\\x\n"""]) + end + + test "on unicode errors" do + msg = "invalid Unicode escape character" + + assert_syntax_error(~r/nofile:1:2: #{msg}/, ~S["\u"]) + assert_syntax_error(~r/nofile:1:1: #{msg}/, ~S[:"\u"]) + assert_syntax_error(~r/nofile:1:2: #{msg}/, ~S["\u": 123]) + assert_syntax_error(~r/nofile:1:1: #{msg}/, ~s["""\n\\u\n"""]) + + assert_syntax_error( + ~r/nofile:1:2: invalid or reserved Unicode code point \\u\{FFFFFF\}. Syntax error after: \\u/, + ~S["\u{FFFFFF}"] + ) + end + + test "on interpolation in calls" do + msg = + ~r"interpolation is not allowed when calling function/macro. Found interpolation in a call starting with: \"" + + assert_syntax_error(msg, ".\"\#{}\"") + assert_syntax_error(msg, ".\"a\#{:b}\"c") + end + + test "on long atoms" do + atom = + "@GR{+z]`_XrNla!d0ptDp(amr.oS&,UbT}v$L|rHHXGV{;W!>avHbD[T-G5xrzR6m?rQPot-37B@" + + assert_syntax_error( + ~r"atom length must be less than system limit: ", + ~s[:"#{atom}"] + ) + end + end + + defp parse!(string), do: Code.string_to_quoted!(string) + + defp assert_token_missing(given_message, string) do + assert_raise TokenMissingError, given_message, fn -> parse!(string) end + end + + defp assert_syntax_error(given_message, string) do + assert_raise SyntaxError, given_message, fn -> parse!(string) end + end +end diff --git a/lib/elixir/test/elixir/kernel/quote_test.exs b/lib/elixir/test/elixir/kernel/quote_test.exs index 2d8de03c0ae..c81aec22ed6 100644 --- a/lib/elixir/test/elixir/kernel/quote_test.exs +++ b/lib/elixir/test/elixir/kernel/quote_test.exs @@ -1,109 +1,193 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.QuoteTest do use ExUnit.Case, async: true - test :list do + @some_fun &List.flatten/1 + + test "fun" do + assert is_function(@some_fun) + end + + test "list" do assert quote(do: [1, 2, 3]) == [1, 2, 3] end - test :tuple do + test "tuple" do assert quote(do: {:a, 1}) == {:a, 1} end - test :keep_line do - ## DO NOT MOVE THIS LINE - assert quote(location: :keep, do: bar(1, 2, 3)) == {:bar, [keep: 16], [1, 2, 3]} + test "keep line" do + line = __ENV__.line + 2 + + assert quote(location: :keep, do: bar(1, 2, 3)) == + {:bar, [keep: {Path.relative_to_cwd(__ENV__.file), line}], [1, 2, 3]} end - test :fixed_line do + test "fixed line" do assert quote(line: 3, do: bar(1, 2, 3)) == {:bar, [line: 3], [1, 2, 3]} + assert quote(line: false, do: bar(1, 2, 3)) == {:bar, [], [1, 2, 3]} + assert quote(line: true, do: bar(1, 2, 3)) == {:bar, [line: __ENV__.line], [1, 2, 3]} end - test :quote_line_var do - ## DO NOT MOVE THIS LINE + test "quote line var" do line = __ENV__.line - assert quote(line: line, do: bar(1, 2, 3)) == {:bar, [line: 25], [1, 2, 3]} + assert quote(line: line, do: bar(1, 2, 3)) == {:bar, [line: line], [1, 2, 3]} + + assert_raise ArgumentError, fn -> + line = "oops" + quote(line: line, do: bar(1, 2, 3)) + end + + assert_raise ArgumentError, fn -> + line = true + quote(line: line, do: bar(1, 2, 3)) + end end - test :unquote_call do + test "quote context var" do + context = :dynamic + assert quote(context: context, do: bar) == {:bar, [], :dynamic} + + assert_raise ArgumentError, fn -> + context = "oops" + quote(context: context, do: bar) + end + + assert_raise ArgumentError, fn -> + context = nil + quote(context: context, do: bar) + end + end + + test "operator precedence" do + assert {:+, _, [{:+, _, [1, _]}, 1]} = quote(do: 1 + Foo.l() + 1) + assert {:+, _, [1, {_, _, [{:+, _, [1]}]}]} = quote(do: 1 + Foo.l(+1)) + end + + test "generated" do + assert quote(generated: true, do: bar(1)) == {:bar, [generated: true], [1]} + end + + test "unquote call" do assert quote(do: foo(bar)[unquote(:baz)]) == quote(do: foo(bar)[:baz]) assert quote(do: unquote(:bar)()) == quote(do: bar()) - assert quote(do: unquote(:bar)(1) do 2 + 3 end) == quote(do: bar(1) do 2 + 3 end) + + assert (quote do + unquote(:bar)(1) do + 2 + 3 + end + end) == + (quote do + bar 1 do + 2 + 3 + end + end) + assert quote(do: foo.unquote(:bar)) == quote(do: foo.bar) + assert quote(do: foo.unquote(:bar)()) == quote(do: foo.bar()) assert quote(do: foo.unquote(:bar)(1)) == quote(do: foo.bar(1)) - assert quote(do: foo.unquote(:bar)(1) do 2 + 3 end) == quote(do: foo.bar(1) do 2 + 3 end) + + assert (quote do + foo.unquote(:bar)(1) do + 2 + 3 + end + end) == + (quote do + foo.bar 1 do + 2 + 3 + end + end) + assert quote(do: foo.unquote({:bar, [], nil})) == quote(do: foo.bar) - assert quote(do: foo.unquote({:bar, [], [1,2]})) == quote(do: foo.bar(1,2)) + assert quote(do: foo.unquote({:bar, [], nil})()) == quote(do: foo.bar()) + assert quote(do: foo.unquote({:bar, [], [1, 2]})) == quote(do: foo.bar(1, 2)) - assert Code.eval_quoted(quote(do: Foo.unquote(Bar))) == {Elixir.Foo.Bar, []} - assert Code.eval_quoted(quote(do: Foo.unquote(quote do: Bar))) == {Elixir.Foo.Bar, []} + assert Code.eval_quoted(quote(do: Foo.unquote(Bar))) == {Elixir.Foo.Bar, []} + assert Code.eval_quoted(quote(do: Foo.unquote(quote(do: Bar)))) == {Elixir.Foo.Bar, []} assert_raise ArgumentError, fn -> quote(do: foo.unquote(1)) end end - test :nested_quote do + test "unquote call with dynamic line" do + assert quote(line: String.to_integer("123"), do: Foo.unquote(:bar)()) == + quote(line: 123, do: Foo.bar()) + end + + test "nested quote" do assert {:quote, _, [[do: {:unquote, _, _}]]} = quote(do: quote(do: unquote(x))) end defmacrop nested_quote_in_macro do x = 1 + quote do x = unquote(x) + quote do unquote(x) end end end - test :nested_quote_in_macro do - assert nested_quote_in_macro == 1 + test "nested quote in macro" do + assert nested_quote_in_macro() == 1 end - Enum.each [foo: 1, bar: 2, baz: 3], fn {k, v} -> - def unquote(k)(arg) do - unquote(v) + arg + defmodule Dyn do + for {k, v} <- [foo: 1, bar: 2, baz: 3] do + # Local call unquote + def unquote(k)(), do: unquote(v) + + # Remote call unquote + def unquote(k)(arg), do: __MODULE__.unquote(k)() + arg end end - test :dynamic_definition_with_unquote do - assert foo(1) == 2 - assert bar(2) == 4 - assert baz(3) == 6 + test "dynamic definition with unquote" do + assert Dyn.foo() == 1 + assert Dyn.bar() == 2 + assert Dyn.baz() == 3 + + assert Dyn.foo(1) == 2 + assert Dyn.bar(2) == 4 + assert Dyn.baz(3) == 6 end - test :splice_on_root do + test "splice on root" do contents = [1, 2, 3] - assert quote(do: (unquote_splicing(contents))) == quote do: (1; 2; 3) + + assert quote(do: (unquote_splicing(contents))) == + (quote do + 1 + 2 + 3 + end) end - test :splice_with_tail do + test "splice with tail" do contents = [1, 2, 3] - assert quote(do: [unquote_splicing(contents)|[1, 2, 3]]) == - [1, 2, 3, 1, 2, 3] - assert quote(do: [unquote_splicing(contents)|val]) == - quote(do: [1, 2, 3 | val]) + assert quote(do: [unquote_splicing(contents) | [1, 2, 3]]) == [1, 2, 3, 1, 2, 3] - assert quote(do: [unquote_splicing(contents)|unquote([4])]) == - quote(do: [1, 2, 3, 4]) + assert quote(do: [unquote_splicing(contents) | val]) == quote(do: [1, 2, 3 | val]) + + assert quote(do: [unquote_splicing(contents) | unquote([4])]) == quote(do: [1, 2, 3, 4]) end - test :splice_on_stab do - {fun, []} = - Code.eval_quoted(quote(do: fn(unquote_splicing([1, 2, 3])) -> :ok end), []) + test "splice on stab" do + {fun, []} = Code.eval_quoted(quote(do: fn unquote_splicing([1, 2, 3]) -> :ok end), []) assert fun.(1, 2, 3) == :ok - {fun, []} = - Code.eval_quoted(quote(do: fn(1, unquote_splicing([2, 3])) -> :ok end), []) + {fun, []} = Code.eval_quoted(quote(do: fn 1, unquote_splicing([2, 3]) -> :ok end), []) assert fun.(1, 2, 3) == :ok end - test :splice_on_definition do + test "splice on definition" do defmodule Hello do - def world([unquote_splicing(["foo", "bar"])|rest]) do + def world([unquote_splicing(["foo", "bar"]) | rest]) do rest end end @@ -111,79 +195,147 @@ defmodule Kernel.QuoteTest do assert Hello.world(["foo", "bar", "baz"]) == ["baz"] end - test :splice_on_map do - assert %{unquote_splicing([foo: :bar])} == %{foo: :bar} - assert %{unquote_splicing([foo: :bar]), baz: :bat} == %{foo: :bar, baz: :bat} - assert %{unquote_splicing([foo: :bar]), :baz => :bat} == %{foo: :bar, baz: :bat} - assert %{:baz => :bat, unquote_splicing([foo: :bar])} == %{foo: :bar, baz: :bat} + test "splice on map" do + assert %{unquote_splicing(foo: :bar)} == %{foo: :bar} + assert %{unquote_splicing(foo: :bar), baz: :bat} == %{foo: :bar, baz: :bat} + assert %{unquote_splicing(foo: :bar), :baz => :bat} == %{foo: :bar, baz: :bat} + assert %{:baz => :bat, unquote_splicing(foo: :bar)} == %{foo: :bar, baz: :bat} map = %{foo: :default} - assert %{map | unquote_splicing([foo: :bar])} == %{foo: :bar} + assert %{map | unquote_splicing(foo: :bar)} == %{foo: :bar} + + assert Code.eval_string("quote do: %{unquote_splicing foo: :bar}") == + {{:%{}, [], [foo: :bar]}, []} + + assert Code.eval_string("quote do: %{:baz => :bat, unquote_splicing foo: :bar}") == + {{:%{}, [], [{:baz, :bat}, {:foo, :bar}]}, []} + + assert Code.eval_string("quote do: %{foo bar | baz}") == + {{:%{}, [], [{:foo, [], [{:|, [], [{:bar, [], Elixir}, {:baz, [], Elixir}]}]}]}, []} end - test :when do - assert [{:->,_,[[{:when,_,[1,2,3,4]}],5]}] = quote(do: (1, 2, 3 when 4 -> 5)) - assert [{:->,_,[[{:when,_,[1,2,3,4]}],5]}] = quote(do: ((1, 2, 3) when 4 -> 5)) + test "when" do + assert [{:->, _, [[{:when, _, [1, 2, 3, 4]}], 5]}] = quote(do: (1, 2, 3 when 4 -> 5)) - assert [{:->,_,[[{:when,_,[1,2,3,{:when,_,[4,5]}]}],6]}] = - quote(do: ((1, 2, 3) when 4 when 5 -> 6)) + assert [{:->, _, [[{:when, _, [1, 2, 3, {:when, _, [4, 5]}]}], 6]}] = + quote(do: (1, 2, 3 when 4 when 5 -> 6)) end - test :stab do - assert [{:->, _, [[], nil]}] = (quote do -> end) - assert [{:->, _, [[], nil]}] = (quote do: (->)) + test "stab" do + assert [{:->, _, [[], 1]}] = + (quote do + () -> 1 + end) - assert [{:->, _, [[1], nil]}] = (quote do 1 -> end) - assert [{:->, _, [[1], nil]}] = (quote do: (1 ->)) + assert [{:->, _, [[], 1]}] = quote(do: (() -> 1)) + end - assert [{:->, _, [[], 1]}] = (quote do -> 1 end) - assert [{:->, _, [[], 1]}] = (quote do: (-> 1)) + test "empty block" do + # Since ; is allowed by itself, it must also be allowed inside () + # The exception to this rule is an empty (). While empty expressions + # are allowed, an empty () is ambiguous. We also can't use quote here, + # since the formatter will rewrite (;) to something else. + assert {:ok, {:__block__, [line: 1], []}} = Code.string_to_quoted("(;)") end - test :bind_quoted do - assert quote(bind_quoted: [foo: 1 + 2], do: foo) == {:__block__, [], [ - {:=, [], [{:foo, [], Kernel.QuoteTest}, 3]}, + test "bind quoted" do + args = [ + {:=, [], [{:foo, [line: __ENV__.line + 4], Kernel.QuoteTest}, 3]}, {:foo, [], Kernel.QuoteTest} - ]} - end + ] - test :literals do - assert (quote do: []) == [] - assert (quote do: nil) == nil - assert (quote do [] end) == [] - assert (quote do nil end) == nil + quoted = quote(bind_quoted: [foo: 1 + 2], do: foo) + assert quoted == {:__block__, [], args} end - defmacrop dynamic_opts do - [line: 3] - end + test "literals" do + assert quote(do: []) == [] + assert quote(do: nil) == nil - test :with_dynamic_opts do - assert quote(dynamic_opts, do: bar(1, 2, 3)) == {:bar, [line: 3], [1, 2, 3]} + assert (quote do + [] + end) == [] + + assert (quote do + nil + end) == nil end - test :unary_with_integer_precedence do - assert quote(do: +1.foo) == quote(do: (+1).foo) - assert quote(do: @1.foo) == quote(do: (@1).foo) - assert quote(do: &1.foo) == quote(do: (&1).foo) + defmacrop dynamic_opts do + [line: 3] end - test :operators_slash_arity do - assert {:/, _, [{:+, _, _}, 2]} = quote do: +/2 - assert {:/, _, [{:&&, _, _}, 3]} = quote do: &&/3 + test "with dynamic opts" do + assert quote(dynamic_opts(), do: bar(1, 2, 3)) == {:bar, [line: 3], [1, 2, 3]} + end + + test "unary with integer precedence" do + assert quote(do: +1.foo) == quote(do: +1.foo) + assert quote(do: (@1).foo) == quote(do: (@1).foo) + assert quote(do: &1.foo) == quote(do: &1.foo) + end + + test "pipe precedence" do + assert {:|>, _, [{:|>, _, [{:foo, _, _}, {:bar, _, _}]}, {:baz, _, _}]} = + quote(do: foo |> bar |> baz) + + assert {:|>, _, [{:|>, _, [{:foo, _, _}, {:bar, _, _}]}, {:baz, _, _}]} = + (quote do + foo do + end + |> bar + |> baz + end) + + assert {:|>, _, [{:|>, _, [{:foo, _, _}, {:bar, _, _}]}, {:baz, _, _}]} = + (quote do + foo + |> bar do + end + |> baz + end) + + assert {:|>, _, [{:|>, _, [{:foo, _, _}, {:bar, _, _}]}, {:baz, _, _}]} = + (quote do + foo + |> bar + |> baz do + end + end) + + assert {:|>, _, [{:|>, _, [{:foo, _, _}, {:bar, _, _}]}, {:baz, _, _}]} = + (quote do + foo do + end + |> bar + |> baz do + end + end) + + assert {:|>, _, [{:|>, _, [{:foo, _, _}, {:bar, _, _}]}, {:baz, _, _}]} = + (quote do + foo do + end + |> bar do + end + |> baz do + end + end) end end -## DO NOT MOVE THIS LINE +# DO NOT MOVE THIS LINE defmodule Kernel.QuoteTest.Errors do - defmacro defadd do + def line, do: __ENV__.line + 4 + + defmacro defraise do quote location: :keep do - def add(a, b), do: a + b + def will_raise(_a, _b), do: raise("oops") end end defmacro will_raise do - quote location: :keep, do: raise "omg" + quote(location: :keep, do: raise("oops")) end end @@ -192,44 +344,52 @@ defmodule Kernel.QuoteTest.ErrorsTest do import Kernel.QuoteTest.Errors # Defines the add function - defadd - - test :inside_function_error do - assert_raise ArithmeticError, fn -> - add(:a, :b) + defraise() + + @line line() + test "inside function error" do + try do + will_raise(:a, :b) + rescue + RuntimeError -> + mod = Kernel.QuoteTest.ErrorsTest + file = __ENV__.file |> Path.relative_to_cwd() |> String.to_charlist() + assert [{^mod, :will_raise, 2, [file: ^file, line: @line] ++ _} | _] = __STACKTRACE__ + else + _ -> flunk("expected failure") end - - mod = Kernel.QuoteTest.ErrorsTest - file = __ENV__.file |> Path.relative_to_cwd |> String.to_char_list - assert [{^mod, :add, 2, [file: ^file, line: 181]}|_] = System.stacktrace end - test :outside_function_error do - assert_raise RuntimeError, fn -> - will_raise + @line __ENV__.line + 3 + test "outside function error" do + try do + will_raise() + rescue + RuntimeError -> + mod = Kernel.QuoteTest.ErrorsTest + file = __ENV__.file |> Path.relative_to_cwd() |> String.to_charlist() + assert [{^mod, _, _, [file: ^file, line: @line] ++ _} | _] = __STACKTRACE__ + else + _ -> flunk("expected failure") end - - mod = Kernel.QuoteTest.ErrorsTest - file = __ENV__.file |> Path.relative_to_cwd |> String.to_char_list - assert [{^mod, _, _, [file: ^file, line: 209]}|_] = System.stacktrace end end defmodule Kernel.QuoteTest.VarHygiene do defmacro no_interference do - quote do: a = 1 + quote(do: a = 1) end defmacro write_interference do - quote do: var!(a) = 1 + quote(do: var!(a) = 1) end defmacro read_interference do - quote do: 10 = var!(a) + quote(do: 10 = var!(a)) end defmacro cross_module_interference do - quote do: var!(a, Kernel.QuoteTest.VarHygieneTest) = 1 + quote(do: var!(a, Kernel.QuoteTest.VarHygieneTest) = 1) end end @@ -238,11 +398,11 @@ defmodule Kernel.QuoteTest.VarHygieneTest do import Kernel.QuoteTest.VarHygiene defmacrop cross_module_no_interference do - quote do: a = 10 + quote(do: a = 10) end defmacrop read_cross_module do - quote do: var!(a, __MODULE__) + quote(do: var!(a, __MODULE__)) end defmacrop nested(var, do: block) do @@ -255,44 +415,88 @@ defmodule Kernel.QuoteTest.VarHygieneTest do defmacrop hat do quote do - var = 1 + var = 1 ^var = 1 var end end - test :no_interference do + test "no interference" do a = 10 - no_interference + no_interference() assert a == 10 end - test :cross_module_interference do - cross_module_no_interference - cross_module_interference - assert read_cross_module == 1 + test "cross module interference" do + cross_module_no_interference() + cross_module_interference() + assert read_cross_module() == 1 end - test :write_interference do - write_interference + test "write interference" do + write_interference() assert a == 1 end - test :read_interference do + test "read interference" do a = 10 - read_interference + read_interference() end - test :nested do + test "hat" do + assert hat() == 1 + end + + test "nested macro" do assert (nested 1 do - nested 2 do - :ok + nested 2 do + _ = :ok + end + end) == 1 + end + + test "nested quoted" do + defmodule NestedQuote do + defmacro __using__(_) do + quote unquote: false do + arg = quote(do: arg) + + def test(arg) do + unquote(arg) + end + end end - end) == 1 + end + + defmodule UseNestedQuote do + use NestedQuote + end + + assert UseNestedQuote.test("foo") == "foo" end - test :hat do - assert hat == 1 + test "nested bind quoted" do + defmodule NestedBindQuoted do + defmacrop macro(arg) do + quote bind_quoted: [arg: arg] do + quote bind_quoted: [arg: arg], do: String.duplicate(arg, 2) + end + end + + defmacro __using__(_) do + quote do + def test do + unquote(macro("foo")) + end + end + end + end + + defmodule UseNestedBindQuoted do + use NestedBindQuoted + end + + assert UseNestedBindQuoted.test() == "foofoo" end end @@ -300,11 +504,11 @@ defmodule Kernel.QuoteTest.AliasHygiene do alias Dict, as: SuperDict defmacro dict do - quote do: Dict.Bar + quote(do: Dict.Bar) end defmacro super_dict do - quote do: SuperDict.Bar + quote(do: SuperDict.Bar) end end @@ -313,41 +517,49 @@ defmodule Kernel.QuoteTest.AliasHygieneTest do alias Dict, as: SuperDict - test :annotate_aliases do - assert {:__aliases__, [alias: false], [:Foo, :Bar]} = - quote(do: Foo.Bar) - assert {:__aliases__, [alias: false], [:Dict, :Bar]} = - quote(do: Dict.Bar) - assert {:__aliases__, [alias: Dict.Bar], [:SuperDict, :Bar]} = - quote(do: SuperDict.Bar) + test "annotate aliases" do + assert {:__aliases__, [alias: false], [:Foo, :Bar]} = quote(do: Foo.Bar) + assert {:__aliases__, [alias: false], [:Dict, :Bar]} = quote(do: Dict.Bar) + assert {:__aliases__, [alias: Dict.Bar], [:SuperDict, :Bar]} = quote(do: SuperDict.Bar) + + # Edge-case + assert {:__aliases__, _, [Elixir]} = quote(do: Elixir) end - test :expand_aliases do - assert Code.eval_quoted(quote do: SuperDict.Bar) == {Elixir.Dict.Bar, []} - assert Code.eval_quoted(quote do: alias!(SuperDict.Bar)) == {Elixir.SuperDict.Bar, []} + test "expand aliases" do + assert Code.eval_quoted(quote(do: SuperDict.Bar)) == {Elixir.Dict.Bar, []} + assert Code.eval_quoted(quote(do: alias!(SuperDict.Bar))) == {Elixir.SuperDict.Bar, []} end - test :expand_aliases_without_macro do + test "expand aliases without macro" do alias HashDict, as: SuperDict assert SuperDict.Bar == Elixir.HashDict.Bar end - test :expand_aliases_with_macro_does_not_expand_source_alias do + test "expand aliases with macro does not expand source alias" do alias HashDict, as: Dict, warn: false require Kernel.QuoteTest.AliasHygiene - assert Kernel.QuoteTest.AliasHygiene.dict == Elixir.Dict.Bar + assert Kernel.QuoteTest.AliasHygiene.dict() == Elixir.Dict.Bar end - test :expand_aliases_with_macro_has_higher_preference do + test "expand aliases with macro has higher preference" do alias HashDict, as: SuperDict, warn: false require Kernel.QuoteTest.AliasHygiene - assert Kernel.QuoteTest.AliasHygiene.super_dict == Elixir.Dict.Bar + assert Kernel.QuoteTest.AliasHygiene.super_dict() == Elixir.Dict.Bar end end defmodule Kernel.QuoteTest.ImportsHygieneTest do use ExUnit.Case, async: true + # We are redefining |> and using it inside the quote + # and only inside the quote. This code should still compile. + defmacro x |> f do + quote do + unquote(x) |> unquote(f) + end + end + defmacrop get_list_length do quote do length('hello') @@ -366,11 +578,11 @@ defmodule Kernel.QuoteTest.ImportsHygieneTest do end end - test :expand_imports do + test "expand imports" do import Kernel, except: [length: 1] - assert get_list_length == 5 - assert get_list_length_with_partial == 5 - assert get_list_length_with_function == 5 + assert get_list_length() == 5 + assert get_list_length_with_partial() == 5 + assert get_list_length_with_function() == 5 end defmacrop get_string_length do @@ -381,19 +593,19 @@ defmodule Kernel.QuoteTest.ImportsHygieneTest do end end - test :lazy_expand_imports do + test "lazy expand imports" do import Kernel, except: [length: 1] import String, only: [length: 1] - assert get_string_length == 5 + assert get_string_length() == 5 end - test :lazy_expand_imports_no_conflicts do + test "lazy expand imports no conflicts" do import Kernel, except: [length: 1] import String, only: [length: 1] - assert get_list_length == 5 - assert get_list_length_with_partial == 5 - assert get_list_length_with_function == 5 + assert get_list_length() == 5 + assert get_list_length_with_partial() == 5 + assert get_list_length_with_function() == 5 end defmacrop with_length do @@ -404,7 +616,21 @@ defmodule Kernel.QuoteTest.ImportsHygieneTest do end end - test :explicitly_overridden_imports do - assert with_length == 5 + test "explicitly overridden imports" do + assert with_length() == 5 + end + + defmodule BinaryUtils do + defmacro int32 do + quote do + integer - size(32) + end + end + end + + test "checks the context also for variables to zero-arity functions" do + import BinaryUtils + {:int32, meta, __MODULE__} = quote(do: int32) + assert meta[:imports] == [{BinaryUtils, 0}] end end diff --git a/lib/elixir/test/elixir/kernel/raise_test.exs b/lib/elixir/test/elixir/kernel/raise_test.exs index 2b6567e949a..b4265c639cd 100644 --- a/lib/elixir/test/elixir/kernel/raise_test.exs +++ b/lib/elixir/test/elixir/kernel/raise_test.exs @@ -1,4 +1,4 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.RaiseTest do use ExUnit.Case, async: true @@ -9,8 +9,23 @@ defmodule Kernel.RaiseTest do defp opts, do: [message: "message"] defp struct, do: %RuntimeError{message: "message"} + @compile {:no_warn_undefined, DoNotExist} @trace [{:foo, :bar, 0, []}] + test "raise preserves the stacktrace" do + stacktrace = + try do + raise "a" + rescue + _ -> hd(__STACKTRACE__) + end + + file = __ENV__.file |> Path.relative_to_cwd() |> String.to_charlist() + + assert {__MODULE__, :"test raise preserves the stacktrace", _, [file: ^file, line: 18] ++ _} = + stacktrace + end + test "raise message" do assert_raise RuntimeError, "message", fn -> raise "message" @@ -56,317 +71,497 @@ defmodule Kernel.RaiseTest do end end + if System.otp_release() >= "24" do + test "raise with error_info" do + {exception, stacktrace} = + try do + raise "a" + rescue + e -> {e, __STACKTRACE__} + end + + assert [{__MODULE__, _, _, meta} | _] = stacktrace + assert meta[:error_info] == %{module: Exception} + + assert Exception.format_error(exception, stacktrace) == + %{general: "a", reason: "#Elixir.RuntimeError"} + end + end + test "reraise message" do try do reraise "message", @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end try do var = binary() reraise var, @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end end test "reraise with no arguments" do try do reraise RuntimeError, @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end try do var = atom() reraise var, @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end end test "reraise with arguments" do try do reraise RuntimeError, [message: "message"], @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end try do atom = atom() opts = opts() reraise atom, opts, @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end end test "reraise existing exception" do try do reraise %RuntimeError{message: "message"}, @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end try do var = struct() reraise var, @trace - flunk "should not reach" + flunk("should not reach") rescue RuntimeError -> - assert @trace == :erlang.get_stacktrace() + assert @trace == __STACKTRACE__ end end - test :rescue_with_underscore_no_exception do - result = try do - RescueUndefinedModule.go + test "reraise with invalid stacktrace" do + try do + reraise %RuntimeError{message: "message"}, {:oops, @trace} rescue - _ -> true + ArgumentError -> + {name, arity} = __ENV__.function + assert [{__MODULE__, ^name, ^arity, _} | _] = __STACKTRACE__ end - - assert result end - test :rescue_with_higher_precedence_than_catch do - result = try do - RescueUndefinedModule.go - catch - _, _ -> false - rescue - _ -> true + describe "rescue" do + test "runtime error" do + result = + try do + raise "an exception" + rescue + RuntimeError -> true + catch + :error, _ -> false + end + + assert result + + result = + try do + raise "an exception" + rescue + AnotherError -> true + catch + :error, _ -> false + end + + refute result + end + + test "named runtime error" do + result = + try do + raise "an exception" + rescue + x in [RuntimeError] -> Exception.message(x) + catch + :error, _ -> false + end + + assert result == "an exception" end - assert result - end + test "named runtime or argument error" do + result = + try do + raise "an exception" + rescue + x in [ArgumentError, RuntimeError] -> Exception.message(x) + catch + :error, _ -> false + end - test :rescue_runtime_error do - result = try do - raise "an exception" - rescue - RuntimeError -> true - catch - :error, _ -> false + assert result == "an exception" end - assert result + test "with higher precedence than catch" do + result = + try do + raise "an exception" + rescue + _ -> true + catch + _, _ -> false + end - result = try do - raise "an exception" - rescue - AnotherError -> true - catch - :error, _ -> false + assert result end - refute result - end + test "argument error from Erlang" do + result = + try do + :erlang.error(:badarg) + rescue + ArgumentError -> true + end - test :rescue_named_runtime_error do - result = try do - raise "an exception" - rescue - x in [RuntimeError] -> Exception.message(x) - catch - :error, _ -> false + assert result end - assert result == "an exception" - end + test "argument error from Elixir" do + result = + try do + raise ArgumentError, "" + rescue + ArgumentError -> true + end - test :rescue_argument_error_from_elixir do - result = try do - raise ArgumentError, "" - rescue - ArgumentError -> true + assert result end - assert result - end + test "catch-all variable" do + result = + try do + raise "an exception" + rescue + x -> Exception.message(x) + end - test :rescue_named_with_underscore do - result = try do - raise "an exception" - rescue - x in _ -> Exception.message(x) + assert result == "an exception" end - assert result == "an exception" - end + test "catch-all underscore" do + result = + try do + raise "an exception" + rescue + _ -> true + end - test :wrap_custom_erlang_error do - result = try do - :erlang.error(:sample) - rescue - x in [RuntimeError, ErlangError] -> Exception.message(x) + assert result end - assert result == "erlang error: :sample" - end + test "catch-all unused variable" do + result = + try do + raise "an exception" + rescue + _any -> true + end - test :undefined_function_error do - result = try do - DoNotExist.for_sure() - rescue - x in [UndefinedFunctionError] -> Exception.message(x) + assert result end - assert result == "undefined function: DoNotExist.for_sure/0" + test "catch-all with \"x in _\" syntax" do + result = + try do + raise "an exception" + rescue + exception in _ -> + Exception.message(exception) + end + + assert result == "an exception" + end end - test :function_clause_error do - result = try do - zero(1) - rescue - x in [FunctionClauseError] -> Exception.message(x) + describe "normalize" do + test "wrap custom Erlang error" do + result = + try do + :erlang.error(:sample) + rescue + x in [ErlangError] -> Exception.message(x) + end + + assert result == "Erlang error: :sample" end - assert result == "no function clause matching in Kernel.RaiseTest.zero/1" - end + test "undefined function error" do + result = + try do + DoNotExist.for_sure() + rescue + x in [UndefinedFunctionError] -> Exception.message(x) + end - test :badarg_error do - result = try do - :erlang.error(:badarg) - rescue - x in [ArgumentError] -> Exception.message(x) + assert result == + "function DoNotExist.for_sure/0 is undefined (module DoNotExist is not available)" end - assert result == "argument error" - end + test "function clause error" do + result = + try do + zero(1) + rescue + x in [FunctionClauseError] -> Exception.message(x) + end - test :tuple_badarg_error do - result = try do - :erlang.error({:badarg, [1, 2, 3]}) - rescue - x in [ArgumentError] -> Exception.message(x) + assert result == "no function clause matching in Kernel.RaiseTest.zero/1" end - assert result == "argument error: [1, 2, 3]" - end + test "badarg error" do + result = + try do + :erlang.error(:badarg) + rescue + x in [ArgumentError] -> Exception.message(x) + end - test :badarith_error do - result = try do - :erlang.error(:badarith) - rescue - x in [ArithmeticError] -> Exception.message(x) + assert result == "argument error" end - assert result == "bad argument in arithmetic expression" - end + test "tuple badarg error" do + result = + try do + :erlang.error({:badarg, [1, 2, 3]}) + rescue + x in [ArgumentError] -> Exception.message(x) + end - test :badarity_error do - fun = fn(x) -> x end - string = "#{inspect(fun)} with arity 1 called with 2 arguments (1, 2)" + assert result == "argument error: [1, 2, 3]" + end - result = try do - fun.(1, 2) - rescue - x in [BadArityError] -> Exception.message(x) + test "badarith error" do + result = + try do + :erlang.error(:badarith) + rescue + x in [ArithmeticError] -> Exception.message(x) + end + + assert result == "bad argument in arithmetic expression" end - assert result == string - end + test "badarity error" do + fun = fn x -> x end + string = "#{inspect(fun)} with arity 1 called with 2 arguments (1, 2)" - test :badfun_error do - x = :example - result = try do - x.(2) - rescue - x in [BadFunctionError] -> Exception.message(x) + result = + try do + fun.(1, 2) + rescue + x in [BadArityError] -> Exception.message(x) + end + + assert result == string end - assert result == "expected a function, got: :example" - end + test "badfun error" do + # Avoid "invalid function call" warning + x = fn -> :example end - test :badmatch_error do - x = :example - result = try do - ^x = zero(0) - rescue - x in [MatchError] -> Exception.message(x) + result = + try do + x.().(2) + rescue + x in [BadFunctionError] -> Exception.message(x) + end + + assert result == "expected a function, got: :example" end - assert result == "no match of right hand side value: 0" - end + test "badfun error when the function is gone" do + defmodule BadFunction.Missing do + def fun, do: fn -> :ok end + end - test :case_clause_error do - x = :example - result = try do - case zero(0) do - ^x -> nil + fun = BadFunction.Missing.fun() + + :code.delete(BadFunction.Missing) + + defmodule BadFunction.Missing do + def fun, do: fn -> :another end end - rescue - x in [CaseClauseError] -> Exception.message(x) + + :code.purge(BadFunction.Missing) + + result = + try do + fun.() + rescue + x in [BadFunctionError] -> Exception.message(x) + end + + assert result =~ + ~r/function #Function<[0-9]\.[0-9]*\/0[^>]*> is invalid, likely because it points to an old version of the code/ end - assert result == "no case clause matching: 0" - end + test "badmatch error" do + x = :example - test :cond_clause_error do - result = try do - cond do - !zero(0) -> :ok - end - rescue - x in [CondClauseError] -> Exception.message(x) + result = + try do + ^x = zero(0) + rescue + x in [MatchError] -> Exception.message(x) + end + + assert result == "no match of right hand side value: 0" end - assert result == "no cond clause evaluated to a true value" - end + defp empty_map(), do: %{} - test :try_clause_error do - f = fn() -> :example end - result = try do - try do - f.() - else - :other -> - :ok - end - rescue - x in [TryClauseError] -> Exception.message(x) + test "bad key error" do + result = + try do + %{empty_map() | foo: :bar} + rescue + x in [KeyError] -> Exception.message(x) + end + + assert result == "key :foo not found" + + result = + try do + empty_map().foo + rescue + x in [KeyError] -> Exception.message(x) + end + + assert result == "key :foo not found in: %{}" end - assert result == "no try clause matching: :example" - end + test "bad map error" do + result = + try do + %{zero(0) | foo: :bar} + rescue + x in [BadMapError] -> Exception.message(x) + end - test :undefined_function_error_as_erlang_error do - result = try do - DoNotExist.for_sure() - rescue - x in [ErlangError] -> Exception.message(x) + assert result == "expected a map, got: 0" + end + + test "bad boolean error" do + result = + try do + 1 and true + rescue + x in [BadBooleanError] -> Exception.message(x) + end + + assert result == "expected a boolean on left-side of \"and\", got: 1" + end + + test "case clause error" do + x = :example + + result = + try do + case zero(0) do + ^x -> nil + end + rescue + x in [CaseClauseError] -> Exception.message(x) + end + + assert result == "no case clause matching: 0" + end + + test "cond clause error" do + result = + try do + cond do + !zero(0) -> :ok + end + rescue + x in [CondClauseError] -> Exception.message(x) + end + + assert result == "no cond clause evaluated to a truthy value" + end + + test "try clause error" do + f = fn -> :example end + + result = + try do + try do + f.() + rescue + _exception -> + :ok + else + :other -> + :ok + end + rescue + x in [TryClauseError] -> Exception.message(x) + end + + assert result == "no try clause matching: :example" end - assert result == "undefined function: DoNotExist.for_sure/0" + test "undefined function error as Erlang error" do + result = + try do + DoNotExist.for_sure() + rescue + x in [ErlangError] -> Exception.message(x) + end + + assert result == + "function DoNotExist.for_sure/0 is undefined (module DoNotExist is not available)" + end end defmacrop exceptions do [ErlangError] end - test :with_macros do - result = try do - DoNotExist.for_sure() - rescue - x in exceptions -> Exception.message(x) - end + test "with macros" do + result = + try do + DoNotExist.for_sure() + rescue + x in exceptions() -> Exception.message(x) + end - assert result == "undefined function: DoNotExist.for_sure/0" + assert result == + "function DoNotExist.for_sure/0 is undefined (module DoNotExist is not available)" end defp zero(0), do: 0 diff --git a/lib/elixir/test/elixir/kernel/sigils_test.exs b/lib/elixir/test/elixir/kernel/sigils_test.exs index 28e9f232771..dd7817f67e9 100644 --- a/lib/elixir/test/elixir/kernel/sigils_test.exs +++ b/lib/elixir/test/elixir/kernel/sigils_test.exs @@ -1,21 +1,21 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.SigilsTest do use ExUnit.Case, async: true - test :sigil_s do + test "sigil s" do assert ~s(foo) == "foo" assert ~s(f#{:o}o) == "foo" assert ~s(f\no) == "f\no" end - test :sigil_s_with_heredoc do + test "sigil s with heredoc" do assert " foo\n\n" == ~s""" - f#{:o}o\n - """ + f#{:o}o\n + """ end - test :sigil_S do + test "sigil S" do assert ~S(foo) == "foo" assert ~S[foo] == "foo" assert ~S{foo} == "foo" @@ -25,22 +25,43 @@ defmodule Kernel.SigilsTest do assert ~S/foo/ == "foo" assert ~S|foo| == "foo" assert ~S(f#{o}o) == "f\#{o}o" + assert ~S(f\#{o}o) == "f\\\#{o}o" assert ~S(f\no) == "f\\no" + assert ~S(foo\)) == "foo)" + assert ~S[foo\]] == "foo]" end - test :sigil_S_with_heredoc do + test "sigil S newline" do + assert ~S(foo\ +bar) in ["foo\\\nbar", "foo\\\r\nbar"] + end + + test "sigil S with heredoc" do assert " f\#{o}o\\n\n" == ~S""" - f#{o}o\n - """ + f#{o}o\n + """ + end + + test "sigil S with escaping" do + assert "\"" == ~S"\"" + + assert "\"\"\"\n" == ~S""" + \""" + """ end - test :sigil_c do + test "sigil s/S expand to binary when possible" do + assert Macro.expand(quote(do: ~s(foo)), __ENV__) == "foo" + assert Macro.expand(quote(do: ~S(foo)), __ENV__) == "foo" + end + + test "sigil c" do assert ~c(foo) == 'foo' assert ~c(f#{:o}o) == 'foo' assert ~c(f\no) == 'f\no' end - test :sigil_C do + test "sigil C" do assert ~C(foo) == 'foo' assert ~C[foo] == 'foo' assert ~C{foo} == 'foo' @@ -51,11 +72,19 @@ defmodule Kernel.SigilsTest do assert ~C(f\no) == 'f\\no' end - test :sigil_w do + test "sigil w" do assert ~w() == [] + assert ~w([ , ]) == ["[", ",", "]"] assert ~w(foo bar baz) == ["foo", "bar", "baz"] assert ~w(foo #{:bar} baz) == ["foo", "bar", "baz"] + assert ~w(#{""}) == [] + assert ~w(foo #{""}) == ["foo"] + assert ~w(#{" foo bar "}) == ["foo", "bar"] + + assert ~w(foo\ #{:bar}) == ["foo", "bar"] + assert ~w(foo\ bar) == ["foo", "bar"] + assert ~w( foo bar @@ -66,10 +95,10 @@ defmodule Kernel.SigilsTest do assert ~w(foo bar baz)a == [:foo, :bar, :baz] assert ~w(foo bar baz)c == ['foo', 'bar', 'baz'] - bad_modifier = quote do: ~w(foo bar baz)x + bad_modifier = quote(do: ~w(foo bar baz)x) assert %ArgumentError{} = catch_error(Code.eval_quoted(bad_modifier)) - assert ~w(Foo Bar)a == [:"Foo", :"Bar"] + assert ~w(Foo Bar)a == [:Foo, :Bar] assert ~w(Foo.#{Bar}.Baz)a == [:"Foo.Elixir.Bar.Baz"] assert ~w(Foo.Bar)s == ["Foo.Bar"] assert ~w(Foo.#{Bar})c == ['Foo.Elixir.Bar'] @@ -78,9 +107,13 @@ defmodule Kernel.SigilsTest do assert Macro.expand(quote(do: ~w(a b c)a), __ENV__) == [:a, :b, :c] end - test :sigil_W do + test "sigil W" do + assert ~W() == [] + assert ~W([ , ]) == ["[", ",", "]"] assert ~W(foo #{bar} baz) == ["foo", "\#{bar}", "baz"] + assert ~W(foo\ bar) == ["foo\\", "bar"] + assert ~W( foo bar @@ -91,14 +124,14 @@ defmodule Kernel.SigilsTest do assert ~W(foo bar baz)a == [:foo, :bar, :baz] assert ~W(foo bar baz)c == ['foo', 'bar', 'baz'] - bad_modifier = quote do: ~W(foo bar baz)x + bad_modifier = quote(do: ~W(foo bar baz)x) assert %ArgumentError{} = catch_error(Code.eval_quoted(bad_modifier)) - assert ~W(Foo #{Bar})a == [:"Foo", :"\#{Bar}"] + assert ~W(Foo #{Bar})a == [:Foo, :"\#{Bar}"] assert ~W(Foo.Bar.Baz)a == [:"Foo.Bar.Baz"] end - test :sigils_matching do + test "sigils matching" do assert ~s(f\(oo) == "f(oo" assert ~s(fo\)o) == "fo)o" assert ~s(f\(o\)o) == "f(o)o" diff --git a/lib/elixir/test/elixir/kernel/special_forms_test.exs b/lib/elixir/test/elixir/kernel/special_forms_test.exs new file mode 100644 index 00000000000..e048310c28e --- /dev/null +++ b/lib/elixir/test/elixir/kernel/special_forms_test.exs @@ -0,0 +1,86 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.SpecialFormsTest do + use ExUnit.Case, async: true + + doctest Kernel.SpecialForms + + describe "cond" do + test "does not leak variables for one clause" do + x = 0 + + cond do + true -> + x = 1 + x + end + + assert x == 0 + end + + test "does not leak variables for one clause with non-boolean as catch-all" do + x = 0 + + cond do + :otherwise -> + x = 1 + x + end + + assert x == 0 + end + + test "does not leak variables for multiple clauses" do + x = 0 + + cond do + List.flatten([]) == [] -> + x = 1 + x + + true -> + x = 1 + x + end + + assert x == 0 + end + + test "does not leak variables from conditions" do + x = :not_nil + + result = + cond do + x = List.first([]) -> + x + + true -> + x + end + + assert result == :not_nil + end + + test "does not warn on non-boolean as catch-all" do + cond do + List.flatten([]) == [] -> :good + :otherwise -> :also_good + end + end + + def false_fun(), do: false + + test "cond_clause error keeps line number in stacktrace" do + try do + cond do + false_fun() -> :ok + end + rescue + _ -> + assert [{Kernel.SpecialFormsTest, _, _, meta} | _] = __STACKTRACE__ + assert meta[:file] + assert meta[:line] + end + end + end +end diff --git a/lib/elixir/test/elixir/kernel/string_tokenizer_test.exs b/lib/elixir/test/elixir/kernel/string_tokenizer_test.exs new file mode 100644 index 00000000000..9a9b2923e2b --- /dev/null +++ b/lib/elixir/test/elixir/kernel/string_tokenizer_test.exs @@ -0,0 +1,72 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.StringTokenizerTest do + use ExUnit.Case, async: true + + defp var({var, _, nil}), do: var + defp aliases({:__aliases__, _, [alias]}), do: alias + + test "tokenizes vars" do + assert Code.string_to_quoted!("_12") |> var() == :_12 + assert Code.string_to_quoted!("ola") |> var() == :ola + assert Code.string_to_quoted!("ólá") |> var() == :ólá + assert Code.string_to_quoted!("óLÁ") |> var() == :óLÁ + assert Code.string_to_quoted!("ólá?") |> var() == :ólá? + assert Code.string_to_quoted!("ólá!") |> var() == :ólá! + assert Code.string_to_quoted!("こんにちは世界") |> var() == :こんにちは世界 + assert {:error, _} = Code.string_to_quoted("v@r") + assert {:error, _} = Code.string_to_quoted("1var") + end + + test "tokenizes atoms" do + assert Code.string_to_quoted!(":_12") == :_12 + assert Code.string_to_quoted!(":ola") == :ola + assert Code.string_to_quoted!(":ólá") == :ólá + assert Code.string_to_quoted!(":ólá?") == :ólá? + assert Code.string_to_quoted!(":ólá!") == :ólá! + assert Code.string_to_quoted!(":ól@") == :ól@ + assert Code.string_to_quoted!(":ól@!") == :ól@! + assert Code.string_to_quoted!(":ó@@!") == :ó@@! + assert Code.string_to_quoted!(":Ola") == :Ola + assert Code.string_to_quoted!(":Ólá") == :Ólá + assert Code.string_to_quoted!(":ÓLÁ") == :ÓLÁ + assert Code.string_to_quoted!(":ÓLÁ?") == :ÓLÁ? + assert Code.string_to_quoted!(":ÓLÁ!") == :ÓLÁ! + assert Code.string_to_quoted!(":ÓL@!") == :ÓL@! + assert Code.string_to_quoted!(":Ó@@!") == :Ó@@! + assert Code.string_to_quoted!(":こんにちは世界") == :こんにちは世界 + assert {:error, _} = Code.string_to_quoted(":123") + assert {:error, _} = Code.string_to_quoted(":@123") + end + + test "tokenizes keywords" do + assert Code.string_to_quoted!("[_12: 0]") == [_12: 0] + assert Code.string_to_quoted!("[ola: 0]") == [ola: 0] + assert Code.string_to_quoted!("[ólá: 0]") == [ólá: 0] + assert Code.string_to_quoted!("[ólá?: 0]") == [ólá?: 0] + assert Code.string_to_quoted!("[ólá!: 0]") == [ólá!: 0] + assert Code.string_to_quoted!("[ól@: 0]") == [ól@: 0] + assert Code.string_to_quoted!("[ól@!: 0]") == [ól@!: 0] + assert Code.string_to_quoted!("[ó@@!: 0]") == [ó@@!: 0] + assert Code.string_to_quoted!("[Ola: 0]") == [Ola: 0] + assert Code.string_to_quoted!("[Ólá: 0]") == [Ólá: 0] + assert Code.string_to_quoted!("[ÓLÁ: 0]") == [ÓLÁ: 0] + assert Code.string_to_quoted!("[ÓLÁ?: 0]") == [ÓLÁ?: 0] + assert Code.string_to_quoted!("[ÓLÁ!: 0]") == [ÓLÁ!: 0] + assert Code.string_to_quoted!("[ÓL@!: 0]") == [ÓL@!: 0] + assert Code.string_to_quoted!("[Ó@@!: 0]") == [Ó@@!: 0] + assert Code.string_to_quoted!("[こんにちは世界: 0]") == [こんにちは世界: 0] + assert {:error, _} = Code.string_to_quoted("[123: 0]") + assert {:error, _} = Code.string_to_quoted("[@123: 0]") + end + + test "tokenizes aliases" do + assert Code.string_to_quoted!("Ola") |> aliases() == String.to_atom("Ola") + assert Code.string_to_quoted!("M_123") |> aliases() == String.to_atom("M_123") + assert {:error, _} = Code.string_to_quoted("Óla") + assert {:error, _} = Code.string_to_quoted("Olá") + assert {:error, _} = Code.string_to_quoted("Ol@") + assert {:error, _} = Code.string_to_quoted("Ola?") + assert {:error, _} = Code.string_to_quoted("Ola!") + end +end diff --git a/lib/elixir/test/elixir/kernel/tracers_test.exs b/lib/elixir/test/elixir/kernel/tracers_test.exs new file mode 100644 index 00000000000..e38757a8322 --- /dev/null +++ b/lib/elixir/test/elixir/kernel/tracers_test.exs @@ -0,0 +1,182 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.TracersTest do + use ExUnit.Case + + import Code, only: [compile_string: 1] + + def trace(event, %Macro.Env{} = env) do + send(self(), {event, env}) + :ok + end + + setup_all do + Code.put_compiler_option(:tracers, [__MODULE__]) + Code.put_compiler_option(:parser_options, columns: true) + + on_exit(fn -> + Code.put_compiler_option(:tracers, []) + Code.put_compiler_option(:parser_options, []) + end) + end + + test "traces start and stop" do + compile_string(""" + Foo + """) + + assert_receive {:start, %{lexical_tracker: pid}} when is_pid(pid) + assert_receive {:stop, %{lexical_tracker: pid}} when is_pid(pid) + end + + test "traces alias references" do + compile_string(""" + Foo + """) + + assert_receive {{:alias_reference, meta, Foo}, _} + assert meta[:line] == 1 + assert meta[:column] == 1 + end + + test "traces aliases" do + compile_string(""" + alias Hello.World + World + + alias Foo, as: Bar, warn: true + Bar + """) + + assert_receive {{:alias, meta, Hello.World, World, []}, _} + assert meta[:line] == 1 + assert meta[:column] == 1 + assert_receive {{:alias_expansion, meta, World, Hello.World}, _} + assert meta[:line] == 2 + assert meta[:column] == 1 + + assert_receive {{:alias, meta, Foo, Bar, [as: Bar, warn: true]}, _} + assert meta[:line] == 4 + assert meta[:column] == 1 + assert_receive {{:alias_expansion, meta, Bar, Foo}, _} + assert meta[:line] == 5 + assert meta[:column] == 1 + end + + test "traces imports" do + compile_string(""" + import Integer, only: [is_odd: 1, parse: 1] + true = is_odd(1) + {1, ""} = parse("1") + """) + + assert_receive {{:import, meta, Integer, only: [is_odd: 1, parse: 1]}, _} + assert meta[:line] == 1 + assert meta[:column] == 1 + + assert_receive {{:imported_macro, meta, Integer, :is_odd, 1}, _} + assert meta[:line] == 2 + assert meta[:column] == 8 + + assert_receive {{:imported_function, meta, Integer, :parse, 1}, _} + assert meta[:line] == 3 + assert meta[:column] == 11 + end + + test "traces structs" do + compile_string(""" + %URI{path: "/"} + """) + + assert_receive {{:struct_expansion, meta, URI, [:path]}, _} + assert meta[:line] == 1 + assert meta[:column] == 1 + end + + test "traces remote" do + compile_string(""" + require Integer + true = Integer.is_odd(1) + {1, ""} = Integer.parse("1") + """) + + assert_receive {{:remote_macro, meta, Integer, :is_odd, 1}, _} + assert meta[:line] == 2 + assert meta[:column] == 16 + + assert_receive {{:remote_function, meta, Integer, :parse, 1}, _} + assert meta[:line] == 3 + assert meta[:column] == 19 + end + + test "traces remote via captures" do + compile_string(""" + require Integer + &Integer.is_odd/1 + &Integer.parse/1 + """) + + assert_receive {{:remote_macro, meta, Integer, :is_odd, 1}, _} + assert meta[:line] == 2 + assert meta[:column] == 1 + + assert_receive {{:remote_function, meta, Integer, :parse, 1}, _} + assert meta[:line] == 3 + assert meta[:column] == 1 + end + + test "traces locals" do + compile_string(""" + defmodule Sample do + defmacro foo(arg), do: arg + def bar(arg), do: arg + def baz(arg), do: foo(arg) + bar(arg) + end + """) + + assert_receive {{:local_macro, meta, :foo, 1}, _} + assert meta[:line] == 4 + assert meta[:column] == 21 + + assert_receive {{:local_function, meta, :bar, 1}, _} + assert meta[:line] == 4 + assert meta[:column] == 32 + after + :code.purge(Sample) + :code.delete(Sample) + end + + test "traces locals with capture" do + compile_string(""" + defmodule Sample do + defmacro foo(arg), do: arg + def bar(arg), do: arg + def baz(_), do: {&foo/1, &bar/1} + end + """) + + assert_receive {{:local_macro, meta, :foo, 1}, _} + assert meta[:line] == 4 + assert meta[:column] == 20 + + assert_receive {{:local_function, meta, :bar, 1}, _} + assert meta[:line] == 4 + assert meta[:column] == 28 + after + :code.purge(Sample) + :code.delete(Sample) + end + + test "traces modules" do + compile_string(""" + defmodule Sample do + :ok + end + """) + + assert_receive {{:on_module, <<_::binary>>, :none}, %{module: Sample, function: nil}} + after + :code.purge(Sample) + :code.delete(Sample) + end +end diff --git a/lib/elixir/test/elixir/kernel/typespec_test.exs b/lib/elixir/test/elixir/kernel/typespec_test.exs deleted file mode 100644 index 80908987812..00000000000 --- a/lib/elixir/test/elixir/kernel/typespec_test.exs +++ /dev/null @@ -1,549 +0,0 @@ -Code.require_file "../test_helper.exs", __DIR__ - -defmodule Kernel.TypespecTest do - use ExUnit.Case, async: true - - # This macro allows us to focus on the result of the - # definition and not on the hassles of handling test - # module - defmacrop test_module([{:do, block}]) do - quote do - {:module, _, binary, _} = defmodule TestTypespec do - unquote(block) - end - :code.delete(TestTypespec) - :code.purge(TestTypespec) - binary - end - end - - defp types(module) do - Kernel.Typespec.beam_types(module) - |> Enum.sort - end - - @skip_specs [__info__: 1] - - defp specs(module) do - Kernel.Typespec.beam_specs(module) - |> Enum.reject(fn {sign, _} -> sign in @skip_specs end) - |> Enum.sort() - end - - defp callbacks(module) do - Kernel.Typespec.beam_callbacks(module) - |> Enum.sort - end - - test "invalid type specification" do - assert_raise CompileError, ~r"invalid type specification: mytype = 1", fn -> - test_module do - @type mytype = 1 - end - end - end - - test "invalid function specification" do - assert_raise CompileError, ~r"invalid function type specification: myfun = 1", fn -> - test_module do - @spec myfun = 1 - end - end - end - - test "@type with a single type" do - module = test_module do - @type mytype :: term - end - - assert [type: {:mytype, {:type, _, :term, []}, []}] = - types(module) - end - - test "@type with an atom" do - module = test_module do - @type mytype :: :atom - end - - assert [type: {:mytype, {:atom, _, :atom}, []}] = - types(module) - end - - test "@type with an atom alias" do - module = test_module do - @type mytype :: Atom - end - - assert [type: {:mytype, {:atom, _, Atom}, []}] = - types(module) - end - - test "@type with an integer" do - module = test_module do - @type mytype :: 10 - end - assert [type: {:mytype, {:integer, _, 10}, []}] = - types(module) - end - - test "@type with a negative integer" do - module = test_module do - @type mytype :: -10 - end - - assert [type: {:mytype, {:op, _, :-, {:integer, _, 10}}, []}] = - types(module) - end - - test "@type with a remote type" do - module = test_module do - @type mytype :: Remote.Some.type - @type mytype_arg :: Remote.type(integer) - end - - assert [type: {:mytype, {:remote_type, _, [{:atom, _, Remote.Some}, {:atom, _, :type}, []]}, []}, - type: {:mytype_arg, {:remote_type, _, [{:atom, _, Remote}, {:atom, _, :type}, [{:type, _, :integer, []}]]}, []}] = - types(module) - end - - test "@type with a binary" do - module = test_module do - @type mytype :: binary - end - - assert [type: {:mytype, {:type, _, :binary, []}, []}] = - types(module) - end - - test "@type with an empty binary" do - module = test_module do - @type mytype :: <<>> - end - - assert [type: {:mytype, {:type, _, :binary, [{:integer, _, 0}, {:integer, _, 0}]}, []}] = - types(module) - end - - test "@type with a binary with a base size" do - module = test_module do - @type mytype :: <<_ :: 3>> - end - - assert [type: {:mytype, {:type, _, :binary, [{:integer, _, 3}, {:integer, _, 0}]}, []}] = - types(module) - end - - test "@type with a binary with a unit size" do - module = test_module do - @type mytype :: <<_ :: _ * 8>> - end - - assert [type: {:mytype, {:type, _, :binary, [{:integer, _, 0}, {:integer, _, 8}]}, []}] = - types(module) - end - - test "@type with a range" do - module = test_module do - @type mytype :: range(1, 10) - end - - assert [type: {:mytype, {:type, _, :range, [{:integer, _, 1}, {:integer, _, 10}]}, []}] = - types(module) - end - - test "@type with a range op" do - module = test_module do - @type mytype :: 1..10 - end - - assert [type: {:mytype, {:type, _, :range, [{:integer, _, 1}, {:integer, _, 10}]}, []}] = - types(module) - end - - test "@type with a map" do - module = test_module do - @type mytype :: %{hello: :world} - end - - assert [type: {:mytype, - {:type, _, :map, [ - {:type, _, :map_field_assoc, {:atom, _, :hello}, {:atom, _, :world}} - ]}, - []}] = types(module) - end - - test "@type with a struct" do - module = test_module do - @type mytype :: %User{hello: :world} - end - - assert [type: {:mytype, - {:type, _, :map, [ - {:type, _, :map_field_assoc, {:atom, _, :__struct__}, {:atom, _, User}}, - {:type, _, :map_field_assoc, {:atom, _, :hello}, {:atom, _, :world}} - ]}, - []}] = types(module) - end - - test "@type with a tuple" do - module = test_module do - @type mytype :: tuple - @type mytype1 :: {} - @type mytype2 :: {1, 2} - end - - assert [type: {:mytype, {:type, _, :tuple, :any}, []}, - type: {:mytype1, {:type, _, :tuple, []}, []}, - type: {:mytype2, {:type, _, :tuple, [{:integer, _, 1}, {:integer, _, 2}]}, []}] = - types(module) - end - - test "@type with list shortcuts" do - module = test_module do - @type mytype :: [] - @type mytype1 :: [integer] - @type mytype2 :: [integer, ...] - end - - assert [type: {:mytype, {:type, _, :nil, []}, []}, - type: {:mytype1, {:type, _, :list, [{:type, _, :integer, []}]}, []}, - type: {:mytype2, {:type, _, :nonempty_list, [{:type, _, :integer, []}]}, []}] = - types(module) - end - - test "@type with a fun" do - module = test_module do - @type mytype :: (... -> any) - end - - assert [type: {:mytype, {:type, _, :fun, []}, []}] = - types(module) - end - - test "@type with a fun with multiple arguments and return type" do - module = test_module do - @type mytype :: (integer, integer -> integer) - end - - assert [type: {:mytype, {:type, _, :fun, [{:type, _, :product, - [{:type, _, :integer, []}, {:type, _, :integer, []}]}, - {:type, _, :integer, []}]}, []}] = - types(module) - end - - test "@type with a fun with no arguments and return type" do - module = test_module do - @type mytype :: (() -> integer) - end - - assert [type: {:mytype, {:type, _, :fun, [{:type, _, :product, []}, - {:type, _, :integer, []}]}, []}] = - types(module) - end - - test "@type with a fun with any arity and return type" do - module = test_module do - @type mytype :: (... -> integer) - end - - assert [type: {:mytype, {:type, _, :fun, [{:type, _, :any}, - {:type, _, :integer, []}]}, []}] = - types(module) - end - - test "@type with a union" do - module = test_module do - @type mytype :: integer | char_list | atom - end - - assert [type: {:mytype, {:type, _, :union, [{:type, _, :integer, []}, - {:remote_type, _, [{:atom, _, :elixir}, {:atom, _, :char_list}, []]}, - {:type, _, :atom, []}]}, []}] = - types(module) - end - - test "@type with keywords" do - module = test_module do - @type mytype :: [first: integer, step: integer, last: integer] - end - - assert [type: {:mytype, {:type, _, :list, [ - {:type, _, :union, [ - {:type, _, :tuple, [{:atom, _, :first}, {:type, _, :integer, []}]}, - {:type, _, :tuple, [{:atom, _, :step}, {:type, _, :integer, []}]}, - {:type, _, :tuple, [{:atom, _, :last}, {:type, _, :integer, []}]} - ]} - ]}, []}] = types(module) - end - - test "@type with parameters" do - module = test_module do - @type mytype(x) :: x - @type mytype1(x) :: list(x) - @type mytype2(x, y) :: {x, y} - end - - assert [type: {:mytype, {:var, _, :x}, [{:var, _, :x}]}, - type: {:mytype1, {:type, _, :list, [{:var, _, :x}]}, [{:var, _, :x}]}, - type: {:mytype2, {:type, _, :tuple, [{:var, _, :x}, {:var, _, :y}]}, [{:var, _, :x}, {:var, _, :y}]}] = - types(module) - end - - test "@type with annotations" do - module = test_module do - @type mytype :: (named :: integer) - @type mytype1 :: (a :: integer -> integer) - end - - assert [type: {:mytype, {:ann_type, _, [{:var, _, :named}, {:type, _, :integer, []}]}, []}, - type: {:mytype1, {:type, _, :fun, [{:type, _, :product, [{:ann_type, _, [{:var, _, :a}, {:type, _, :integer, []}]}]}, {:type, _, :integer, []}]}, []}] = - types(module) - end - - test "@opaque(type)" do - module = test_module do - @opaque mytype(x) :: x - end - - assert [opaque: {:mytype, {:var, _, :x}, [{:var, _, :x}]}] = - types(module) - end - - test "@type + opaque" do - module = test_module do - @type mytype :: tuple - @opaque mytype1 :: {} - end - - assert [opaque: {:mytype1, _, []}, - type: {:mytype, _, []},] = - types(module) - end - - test "@type from structs" do - module = test_module do - defstruct name: nil, age: 0 :: non_neg_integer - end - - assert [type: {:t, {:type, _, :map, [ - {:type, _, :map_field_assoc, {:atom, _, :name}, {:type, _, :term, []}}, - {:type, _, :map_field_assoc, {:atom, _, :age}, {:type, _, :non_neg_integer, []}}, - {:type, _, :map_field_assoc, {:atom, _, :__struct__}, {:atom, _, TestTypespec}} - ]}, []}] = types(module) - end - - test "@type from dynamic structs" do - module = test_module do - fields = [name: nil, age: 0] - defstruct fields - end - - assert [type: {:t, {:type, _, :map, [ - {:type, _, :map_field_assoc, {:atom, _, :name}, {:type, _, :term, []}}, - {:type, _, :map_field_assoc, {:atom, _, :age}, {:type, _, :term, []}}, - {:type, _, :map_field_assoc, {:atom, _, :__struct__}, {:atom, _, TestTypespec}} - ]}, []}] = types(module) - end - - test "@type unquote fragment" do - module = test_module do - quoted = quote unquote: false do - name = :mytype - type = :atom - @type unquote(name)() :: unquote(type) - end - Module.eval_quoted(__MODULE__, quoted) |> elem(0) - end - - assert [type: {:mytype, {:atom, _, :atom}, []}] = - types(module) - end - - test "defines_type?" do - test_module do - @type mytype :: tuple - @type mytype(a) :: [a] - assert Kernel.Typespec.defines_type?(__MODULE__, :mytype, 0) - assert Kernel.Typespec.defines_type?(__MODULE__, :mytype, 1) - refute Kernel.Typespec.defines_type?(__MODULE__, :mytype, 2) - end - end - - test "@spec(spec)" do - module = test_module do - def myfun1(x), do: x - def myfun2(), do: :ok - def myfun3(x, y), do: {x, y} - def myfun4(x), do: x - @spec myfun1(integer) :: integer - @spec myfun2() :: integer - @spec myfun3(integer, integer) :: {integer, integer} - @spec myfun4(x :: integer) :: integer - end - - assert [{{:myfun1, 1}, [{:type, _, :fun, [{:type, _, :product, [{:type, _, :integer, []}]}, {:type, _, :integer, []}]}]}, - {{:myfun2, 0}, [{:type, _, :fun, [{:type, _, :product, []}, {:type, _, :integer, []}]}]}, - {{:myfun3, 2}, [{:type, _, :fun, [{:type, _, :product, [{:type, _, :integer, []}, {:type, _, :integer, []}]}, {:type, _, :tuple, [{:type, _, :integer, []}, {:type, _, :integer, []}]}]}]}, - {{:myfun4, 1}, [{:type, _, :fun, [{:type, _, :product, [{:ann_type, _, [{:var, _, :x}, {:type, _, :integer, []}]}]}, {:type, _, :integer, []}]}]}] = - specs(module) - end - - test "@spec(spec) with guards" do - module = test_module do - def myfun1(x), do: x - @spec myfun1(x) :: boolean when [x: integer] - - def myfun2(x), do: x - @spec myfun2(x) :: x when [x: var] - - def myfun3(_x, y), do: y - @spec myfun3(x, y) :: y when [y: x, x: var] - end - - assert [{{:myfun1, 1}, [{:type, _, :bounded_fun, [{:type, _, :fun, [{:type, _, :product, [{:var, _, :x}]}, {:type, _, :boolean, []}]}, [{:type, _, :constraint, [{:atom, _, :is_subtype}, [{:var, _, :x}, {:type, _, :integer, []}]]}]]}]}, - {{:myfun2, 1}, [{:type, _, :fun, [{:type, _, :product, [{:var, _, :x}]}, {:var, _, :x}]}]}, - {{:myfun3, 2}, [{:type, _, :bounded_fun, [{:type, _, :fun, [{:type, _, :product, [{:var, _, :x}, {:var, _, :y}]}, {:var, _, :y}]}, [{:type, _, :constraint, [{:atom, _, :is_subtype}, [{:var, _, :y}, {:var, _, :x}]]}]]}]}] = - specs(module) - end - - test "@callback(callback)" do - module = test_module do - @callback myfun(integer) :: integer - @callback myfun() :: integer - @callback myfun(integer, integer) :: {integer, integer} - end - - assert [{{:myfun, 0}, [{:type, _, :fun, [{:type, _, :product, []}, {:type, _, :integer, []}]}]}, - {{:myfun, 1}, [{:type, _, :fun, [{:type, _, :product, [{:type, _, :integer, []}]}, {:type, _, :integer, []}]}]}, - {{:myfun, 2}, [{:type, _, :fun, [{:type, _, :product, [{:type, _, :integer, []}, {:type, _, :integer, []}]}, {:type, _, :tuple, [{:type, _, :integer, []}, {:type, _, :integer, []}]}]}]}] = - callbacks(module) - end - - test "@spec + @callback" do - module = test_module do - def myfun(x), do: x - @spec myfun(integer) :: integer - @spec myfun(char_list) :: char_list - @callback cb(integer) :: integer - end - - assert [{{:cb, 1}, [{:type, _, :fun, [{:type, _, :product, [{:type, _, :integer, []}]}, {:type, _, :integer, []}]}]}] = - callbacks(module) - - assert [{{:myfun, 1}, [ - {:type, _, :fun, [{:type, _, :product, [ - {:remote_type, _, [{:atom, _, :elixir}, {:atom, _, :char_list}, []]}]}, - {:remote_type, _, [{:atom, _, :elixir}, {:atom, _, :char_list}, []]}]}, - {:type, _, :fun, [{:type, _, :product, [{:type, _, :integer, []}]}, {:type, _, :integer, []}]}]}] = - specs(module) - end - - test "block handling" do - module = test_module do - @spec foo((() -> [ integer ])) :: integer - def foo(_), do: 1 - end - assert [{{:foo, 1}, - [{:type, _, :fun, [{:type, _, :product, [ - {:type, _, :fun, [{:type, _, :product, []}, {:type, _, :list, [{:type, _, :integer, []}]}]}]}, - {:type, _, :integer, []}]}]}] = - specs(module) - end - - # Conversion to AST - - test "type_to_ast" do - quoted = [ - (quote do: @type with_ann() :: (t :: atom())), - (quote do: @type empty_tuple_type() :: {}), - (quote do: @type imm_type_1() :: 1), - (quote do: @type imm_type_2() :: :atom), - (quote do: @type simple_type() :: integer()), - (quote do: @type param_type(p) :: [p]), - (quote do: @type union_type() :: integer() | binary() | boolean()), - (quote do: @type binary_type1() :: <<_ :: _ * 8>>), - (quote do: @type binary_type2() :: <<_ :: 3 * 8>>), - (quote do: @type binary_type3() :: <<_ :: 3>>), - (quote do: @type tuple_type() :: {integer()}), - (quote do: @type ftype() :: (() -> any()) | (() -> integer()) | ((integer() -> integer()))), - (quote do: @type cl() :: char_list()), - (quote do: @type ab() :: as_boolean(term())), - (quote do: @type vaf() :: (... -> any())), - (quote do: @type rng() :: 1 .. 10), - (quote do: @type opts() :: [first: integer(), step: integer(), last: integer()]), - (quote do: @type ops() :: {+1,-1}), - (quote do: @type my_map() :: %{hello: :world}), - (quote do: @type my_struct() :: %User{hello: :world}), - ] |> Enum.sort - - module = test_module do - Module.eval_quoted __MODULE__, quote do: (unquote_splicing(quoted)) - end - - types = types(module) - - Enum.each(Enum.zip(types, quoted), fn {{:type, type}, definition} -> - ast = Kernel.Typespec.type_to_ast(type) - assert Macro.to_string(quote do: @type unquote(ast)) == Macro.to_string(definition) - end) - end - - test "type_to_ast for paren_type" do - type = {:my_type, {:paren_type, 0, [{:type, 0, :integer, []}]}, []} - assert Kernel.Typespec.type_to_ast(type) == - {:::, [], [{:my_type, [], []}, {:integer, [line: 0], []}]} - end - - test "spec_to_ast" do - quoted = [ - (quote do: @spec a() :: integer()), - (quote do: @spec a(atom()) :: integer() | [{}]), - (quote do: @spec a(b) :: integer() when [b: integer()]), - (quote do: @spec a(b) :: b when [b: var]), - (quote do: @spec a(c :: atom()) :: atom()), - ] |> Enum.sort - - module = test_module do - def a, do: 1 - def a(a), do: a - Module.eval_quoted __MODULE__, quote do: (unquote_splicing(quoted)) - end - - specs = Enum.flat_map(specs(module), fn {{_, _}, specs} -> - Enum.map(specs, fn spec -> - quote do: @spec unquote(Kernel.Typespec.spec_to_ast(:a, spec)) - end) - end) |> Enum.sort - - Enum.each(Enum.zip(specs, quoted), fn {spec, definition} -> - assert Macro.to_string(spec) == Macro.to_string(definition) - end) - end - - test "typedoc retrieval" do - {:module, _, binary, _} = defmodule T do - @typedoc "A" - @type a :: any - @typep b :: any - @typedoc "C" - @opaque c(x, y) :: {x, y} - @type d :: any - @spec uses_b() :: b - def uses_b(), do: nil - end - - :code.delete(T) - :code.purge(T) - - assert [ - {{:c, 2}, "C"}, - {{:a, 0}, "A"} - ] = Kernel.Typespec.beam_typedocs(binary) - end - - test "retrieval invalid data" do - assert Kernel.Typespec.beam_typedocs(Unknown) == nil - assert Kernel.Typespec.beam_types(Unknown) == nil - assert Kernel.Typespec.beam_specs(Unknown) == nil - end -end diff --git a/lib/elixir/test/elixir/kernel/warning_test.exs b/lib/elixir/test/elixir/kernel/warning_test.exs index fbfa7dad5f6..7e7d8e0bf93 100644 --- a/lib/elixir/test/elixir/kernel/warning_test.exs +++ b/lib/elixir/test/elixir/kernel/warning_test.exs @@ -1,4 +1,4 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Kernel.WarningTest do use ExUnit.Case @@ -8,486 +8,1965 @@ defmodule Kernel.WarningTest do capture_io(:stderr, fun) end - test :unused_variable do + test "outdented heredoc" do + output = + capture_err(fn -> + Code.eval_string(""" + ''' + outdented + ''' + """) + end) + + assert output =~ "outdented heredoc line" + assert output =~ "nofile:2" + end + + test "does not warn on incomplete tokenization" do + assert {:error, _} = Code.string_to_quoted(~s[:"foobar" do]) + end + + describe "unicode identifier security" do + test "prevents Restricted codepoints in identifiers" do + exception = assert_raise SyntaxError, fn -> Code.string_to_quoted!("_shibㅤ = 1") end + + assert Exception.message(exception) =~ + "unexpected token: \"ㅤ\" (column 6, code point U+3164)" + end + + test "warns on confusables" do + assert capture_err(fn -> Code.string_to_quoted("а=1; a=1") end) =~ + "confusable identifier: 'a' looks like 'а' on line 1" + + assert capture_err(fn -> Code.string_to_quoted("[{:а, 1}, {:a, 1}]") end) =~ + "confusable identifier: 'a' looks like 'а' on line 1" + + assert capture_err(fn -> Code.string_to_quoted("[а: 1, a: 1]") end) =~ + "confusable identifier: 'a' looks like 'а' on line 1" + + assert capture_err(fn -> Code.string_to_quoted("quote do: [а(1), a(1)]") end) =~ + "confusable identifier: 'a' looks like 'а' on line 1" + + assert capture_err(fn -> Code.string_to_quoted("力=1; カ=1") end) =~ + "confusable identifier: 'カ' looks like '力' on line 1" + + # by convention, doesn't warn on ascii-only confusables + assert capture_err(fn -> Code.string_to_quoted("x0 = xO = 1") end) == "" + assert capture_err(fn -> Code.string_to_quoted("l1 = ll = 1") end) == "" + + # works with a custom atom encoder + assert capture_err(fn -> + Code.string_to_quoted("[{:а, 1}, {:a, 1}]", + static_atoms_encoder: fn token, _ -> {:ok, {:wrapped, token}} end + ) + end) =~ + "confusable identifier: 'a' looks like 'а' on line 1" + end + + test "prevents unsafe script mixing in identifiers" do + exception = + assert_raise SyntaxError, fn -> + Code.string_to_quoted!("if аdmin_, do: :ok, else: :err") + end + + assert Exception.message(exception) =~ + "nofile:1:9: invalid mixed-script identifier found: аdmin" + + assert Exception.message(exception) =~ """ + \\u0430 а {Cyrillic} + \\u0064 d {Latin} + \\u006D m {Latin} + \\u0069 i {Latin} + \\u006E n {Latin} + \\u005F _ + """ + + # includes suggestion about what to change + assert Exception.message(exception) =~ """ + Hint: You could write the above in a similar way that is accepted by Elixir: + + "admin_" (code points 0x00061 0x00064 0x0006D 0x00069 0x0006E 0x0005F) + """ + + # a is in cyrillic + assert_raise SyntaxError, ~r/mixed/, fn -> Code.string_to_quoted!("[аdmin: 1]") end + assert_raise SyntaxError, ~r/mixed/, fn -> Code.string_to_quoted!("[{:аdmin, 1}]") end + assert_raise SyntaxError, ~r/mixed/, fn -> Code.string_to_quoted!("quote do: аdmin(1)") end + assert_raise SyntaxError, ~r/mixed/, fn -> Code.string_to_quoted!("рос_api = 1") end + + # T is in cyrillic + assert_raise SyntaxError, ~r/mixed/, fn -> Code.string_to_quoted!("[Тシャツ: 1]") end + end + + test "allows legitimate script mixing" do + # writing systems that legitimately mix multiple scripts, and Common chars like _ + assert capture_err(fn -> Code.eval_string("幻ㄒㄧㄤ = 1") end) == "" + assert capture_err(fn -> Code.eval_string("幻ㄒㄧㄤ1 = 1") end) == "" + assert capture_err(fn -> Code.eval_string("__सवव_1? = 1") end) == "" + + # uts39 5.2 allowed 'highly restrictive' script mixing, like 't-shirt' in Jpan: + assert capture_err(fn -> Code.string_to_quoted!(":Tシャツ") end) == "" + + # elixir's normalizations combine scriptsets of the 'from' and 'to' characters, + # ex: {Common} MICRO => {Greek} MU == {Common, Greek}; Common intersects w/all + assert capture_err(fn -> Code.string_to_quoted!("μs") end) == "" + end + end + + test "operators formed by many of the same character followed by that character" do + output = + capture_err(fn -> + Code.eval_string("quote do: ....()") + end) + + assert output =~ "found \"...\" followed by \".\", please use parens around \"...\" instead" + end + + test "identifier that ends in ! followed by the = operator without a space in between" do + output = capture_err(fn -> Code.eval_string("foo!= 1") end) + assert output =~ "found identifier \"foo!\", ending with \"!\"" + + output = capture_err(fn -> Code.eval_string(":foo!= :foo!") end) + assert output =~ "found atom \":foo!\", ending with \"!\"" + end + + describe "unnecessary quotes" do + test "does not warn for unnecessary quotes in uppercase atoms/keywords" do + assert capture_err(fn -> Code.eval_string(~s/:"Foo"/) end) == "" + assert capture_err(fn -> Code.eval_string(~s/["Foo": :bar]/) end) == "" + assert capture_err(fn -> Code.eval_string(~s/:"Foo"/) end) == "" + assert capture_err(fn -> Code.eval_string(~s/:"foo@bar"/) end) == "" + assert capture_err(fn -> Code.eval_string(~s/:"héllò"/) end) == "" + assert capture_err(fn -> Code.eval_string(~s/:"3L1X1R"/) end) == "" + end + + test "warns for unnecessary quotes" do + assert capture_err(fn -> Code.eval_string(~s/:"foo"/) end) =~ + "found quoted atom \"foo\" but the quotes are not required" + + assert capture_err(fn -> Code.eval_string(~s/["foo": :bar]/) end) =~ + "found quoted keyword \"foo\" but the quotes are not required" + + assert capture_err(fn -> Code.eval_string(~s/[Kernel."length"([])]/) end) =~ + "found quoted call \"length\" but the quotes are not required" + end + end + + test "warns on :: as atom" do + assert capture_err(fn -> Code.eval_string(~s/:::/) end) =~ + "atom ::: must be written between quotes, as in :\"::\", to avoid ambiguity" + end + + test "unused variable" do + output = + capture_err(fn -> + # Note we use compile_string because eval_string does not emit unused vars warning + Code.compile_string(""" + defmodule Sample do + module = 1 + def hello(arg), do: nil + end + file = 2 + file = 3 + file + """) + end) + + assert output =~ "variable \"arg\" is unused" + assert output =~ "variable \"module\" is unused" + assert output =~ "variable \"file\" is unused" + after + purge(Sample) + end + + test "unused variable that could be pinned" do + output = + capture_err(fn -> + # Note we use compile_string because eval_string does not emit unused vars warning + Code.compile_string(""" + defmodule Sample do + def test do + compare_local = "hello" + match?(compare_local, "hello") + + compare_nested = "hello" + case "hello" do + compare_nested -> true + _other -> false + end + end + end + """) + end) + + assert output =~ + "variable \"compare_local\" is unused (there is a variable with the same name in the context, use the pin operator (^) to match on it or prefix this variable with underscore if it is not meant to be used)" + + assert output =~ + "variable \"compare_nested\" is unused (there is a variable with the same name in the context, use the pin operator (^) to match on it or prefix this variable with underscore if it is not meant to be used)" + after + purge(Sample) + end + + test "unused compiler variable" do + output = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + def hello(__MODULE___), do: :ok + def world(_R), do: :ok + end + """) + end) + + assert output =~ "unknown compiler variable \"__MODULE___\"" + refute output =~ "unknown compiler variable \"_R\"" + after + purge(Sample) + end + + test "nested unused variable" do + message = "variable \"x\" is unused" + + assert capture_err(fn -> + assert_raise CompileError, ~r/undefined function x/, fn -> + Code.eval_string(""" + case false do + true -> x = 1 + _ -> 1 + end + x + """) + end + end) =~ message + + assert capture_err(fn -> + assert_raise CompileError, ~r/undefined function x/, fn -> + Code.eval_string(""" + false and (x = 1) + x + """) + end + end) =~ message + + assert capture_err(fn -> + assert_raise CompileError, ~r/undefined function x/, fn -> + Code.eval_string(""" + true or (x = 1) + x + """) + end + end) =~ message + + assert capture_err(fn -> + assert_raise CompileError, ~r/undefined function x/, fn -> + Code.eval_string(""" + if false do + x = 1 + end + x + """) + end + end) =~ message + + assert capture_err(fn -> + assert_raise CompileError, ~r/undefined function x/, fn -> + Code.eval_string(""" + cond do + false -> x = 1 + true -> 1 + end + x + """) + end + end) =~ message + + assert capture_err(fn -> + assert_raise CompileError, ~r/undefined function x/, fn -> + Code.eval_string(""" + receive do + :foo -> x = 1 + after + 0 -> 1 + end + x + """) + end + end) =~ message + + assert capture_err(fn -> + assert_raise CompileError, ~r/undefined function x/, fn -> + Code.eval_string(""" + false && (x = 1) + x + """) + end + end) =~ message + assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - def hello(arg), do: nil - end - """ - end) =~ "warning: variable arg is unused" + assert_raise CompileError, ~r/undefined function x/, fn -> + Code.eval_string(""" + true || (x = 1) + x + """) + end + end) =~ message + + assert capture_err(fn -> + assert_raise CompileError, ~r/undefined function x/, fn -> + Code.eval_string(""" + with true <- true do + x = false + end + x + """) + end + end) =~ message + + assert capture_err(fn -> + assert_raise CompileError, ~r/undefined function x/, fn -> + Code.eval_string(""" + fn -> + x = true + end + x + """) + end + end) =~ message + end + + test "unused variable in redefined function in different file" do + output = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + defmacro __using__(_) do + quote location: :keep do + def function(arg) + end + end + end + """) + + code = """ + defmodule RedefineSample do + use Sample + def function(var123), do: nil + end + """ + + Code.eval_string(code, [], file: "redefine_sample.ex") + end) + + assert output =~ "redefine_sample.ex:3" + assert output =~ "variable \"var123\" is unused" after - purge Sample + purge(Sample) + purge(RedefineSample) end - test :unused_function do + test "useless literal" do + message = "code block contains unused literal \"oops\"" + assert capture_err(fn -> - Code.eval_string """ - defmodule Sample1 do - defp hello, do: nil - end - """ - end) =~ "warning: function hello/0 is unused" + Code.eval_string(""" + "oops" + :ok + """) + end) =~ message assert capture_err(fn -> - Code.eval_string """ - defmodule Sample2 do - defp hello(0), do: hello(1) - defp hello(1), do: :ok - end - """ - end) =~ "function hello/1 is unused" + Code.eval_string(""" + fn -> + "oops" + :ok + end + """) + end) =~ message assert capture_err(fn -> - Code.eval_string ~S""" - defmodule Sample3 do - def a, do: nil - def b, do: d(10) - defp c(x, y \\ 1), do: [x, y] - defp d(x), do: x - end - """ - end) =~ "warning: function c/2 is unused" + Code.eval_string(""" + try do + "oops" + :ok + after + :ok + end + """) + end) =~ message + end + + test "useless attr" do + message = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @foo 1 + @bar 1 + @foo + + def bar do + @bar + :ok + end + end + """) + end) + + assert message =~ "module attribute @foo in code block has no effect as it is never returned " + assert message =~ "module attribute @bar in code block has no effect as it is never returned " after - purge [Sample1, Sample2, Sample3] + purge(Sample) + end + + test "useless var" do + message = "variable foo in code block has no effect as it is never returned " + + assert capture_err(fn -> + Code.eval_string(""" + foo = 1 + foo + :ok + """) + end) =~ message + + assert capture_err(fn -> + Code.eval_string(""" + fn -> + foo = 1 + foo + :ok + end + """) + end) =~ message + + assert capture_err(fn -> + Code.eval_string(""" + try do + foo = 1 + foo + :ok + after + :ok + end + """) + end) =~ message + + assert capture_err(fn -> + Code.eval_string(""" + node() + :ok + """) + end) == "" + end + + test "underscored variable on match" do + assert capture_err(fn -> + Code.eval_string(""" + {_arg, _arg} = {1, 1} + """) + end) =~ "the underscored variable \"_arg\" appears more than once in a match" + end + + test "underscored variable on use" do + assert capture_err(fn -> + Code.eval_string(""" + fn _var -> _var + 1 end + """) + end) =~ "the underscored variable \"_var\" is used after being set" + + assert capture_err(fn -> + Code.eval_string(""" + fn var!(_var, Foo) -> var!(_var, Foo) + 1 end + """) + end) =~ "" end - test :unused_cyclic_functions do + test "unused function" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample1 do + defp hello, do: nil + end + """) + end) =~ "function hello/0 is unused\n nofile:2" + + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample2 do + defp hello(0), do: hello(1) + defp hello(1), do: :ok + end + """) + end) =~ "function hello/1 is unused\n nofile:2" + assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - defp a, do: b - defp b, do: a - end - """ - end) =~ "warning: function a/0 is unused" + Code.eval_string(~S""" + defmodule Sample3 do + def a, do: nil + def b, do: d(10) + defp c(x, y \\ 1), do: [x, y] + defp d(x), do: x + end + """) + end) =~ "function c/2 is unused\n nofile:4" + + assert capture_err(fn -> + Code.eval_string(~S""" + defmodule Sample4 do + def a, do: nil + defp b(x \\ 1, y \\ 1) + defp b(x, y), do: [x, y] + end + """) + end) =~ "function b/2 is unused\n nofile:3" + after + purge([Sample1, Sample2, Sample3, Sample4]) + end + + test "unused cyclic functions" do + message = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + defp a, do: b() + defp b, do: a() + end + """) + end) + + assert message =~ "function a/0 is unused\n nofile:2" + assert message =~ "function b/0 is unused\n nofile:3" after - purge Sample + purge(Sample) end - test :unused_macro do + test "unused macro" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - defmacrop hello, do: nil - end - """ - end) =~ "warning: macro hello/0 is unused" + Code.eval_string(""" + defmodule Sample do + defmacrop hello, do: nil + end + """) + end) =~ "macro hello/0 is unused" after - purge Sample + purge(Sample) end - test :shadowing do + test "shadowing" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - def test(x) do - case x do - {:file, fid} -> fid - {:path, _} -> fn(fid) -> fid end - end - end - end - """ - end) == "" + Code.eval_string(""" + defmodule Sample do + def test(x) do + case x do + {:file, fid} -> fid + {:path, _} -> fn(fid) -> fid end + end + end + end + """) + end) == "" after - purge Sample + purge(Sample) end - test :unused_default_args do + test "unused default args" do + assert capture_err(fn -> + Code.eval_string(~S""" + defmodule Sample1 do + def a, do: b(1, 2, 3) + defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3] + end + """) + end) =~ "default values for the optional arguments in b/3 are never used\n nofile:3" + + assert capture_err(fn -> + Code.eval_string(~S""" + defmodule Sample2 do + def a, do: b(1, 2) + defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3] + end + """) + end) =~ + "the default values for the first 2 optional arguments in b/3 are never used\n nofile:3" + + assert capture_err(fn -> + Code.eval_string(~S""" + defmodule Sample3 do + def a, do: b(1) + defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3] + end + """) + end) =~ + "the default value for the first optional argument in b/3 is never used\n nofile:3" + assert capture_err(fn -> - Code.eval_string ~S""" - defmodule Sample1 do - def a, do: b(1, 2, 3) - defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3] - end - """ - end) =~ "warning: default arguments in b/3 are never used" + Code.eval_string(~S""" + defmodule Sample4 do + def a, do: b(1) + defp b(arg1 \\ 1, arg2, arg3 \\ 3), do: [arg1, arg2, arg3] + end + """) + end) == "" + + assert capture_err(fn -> + Code.eval_string(~S""" + defmodule Sample5 do + def a, do: b(1, 2, 3) + defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3) + + defp b(arg1, arg2, arg3), do: [arg1, arg2, arg3] + end + """) + end) =~ "default values for the optional arguments in b/3 are never used\n nofile:3" assert capture_err(fn -> - Code.eval_string ~S""" - defmodule Sample2 do - def a, do: b(1, 2) - defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3] - end - """ - end) =~ "warning: the first 2 default arguments in b/3 are never used" + Code.eval_string(~S""" + defmodule Sample6 do + def a, do: b(1, 2) + defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3) + defp b(arg1, arg2, arg3), do: [arg1, arg2, arg3] + end + """) + end) =~ + "the default values for the first 2 optional arguments in b/3 are never used\n nofile:3" + after + purge([Sample1, Sample2, Sample3, Sample4, Sample5, Sample6]) + end + + test "unused import" do assert capture_err(fn -> - Code.eval_string ~S""" - defmodule Sample3 do - def a, do: b(1) - defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3] - end - """ - end) =~ "warning: the first default argument in b/3 is never used" + Code.compile_string(""" + defmodule Sample do + import :lists + def a, do: nil + end + """) + end) =~ "unused import :lists\n" assert capture_err(fn -> - Code.eval_string ~S""" - defmodule Sample4 do - def a, do: b(1) - defp b(arg1 \\ 1, arg2, arg3 \\ 3), do: [arg1, arg2, arg3] - end - """ - end) == "" + Code.compile_string(""" + import :lists + """) + end) =~ "unused import :lists\n" + after + purge(Sample) + end + + test "unused import of one of the functions in :only" do + output = + capture_err(fn -> + Code.compile_string(""" + defmodule Sample do + import String, only: [upcase: 1, downcase: 1, trim: 1] + def a, do: upcase("hello") + end + """) + end) + + assert output =~ "unused import String.downcase/1" + assert output =~ "unused import String.trim/1" after - purge [Sample1, Sample2, Sample3, Sample4] + purge(Sample) end - test :unused_import do + test "unused import of any of the functions in :only" do assert capture_err(fn -> - Code.compile_string """ - defmodule Sample do - import :lists, only: [flatten: 1] - def a, do: nil - end - """ - end) =~ "warning: unused import :lists" + Code.compile_string(""" + defmodule Sample do + import String, only: [upcase: 1, downcase: 1] + def a, do: nil + end + """) + end) =~ "unused import String\n" + after + purge(Sample) + end + test "unused alias" do assert capture_err(fn -> - Code.compile_string """ - import :lists, only: [flatten: 1] - """ - end) =~ "warning: unused import :lists" + Code.compile_string(""" + defmodule Sample do + alias :lists, as: List + def a, do: nil + end + """) + end) =~ "unused alias List" after - purge [Sample] + purge(Sample) end - test :unused_alias do + test "unused alias when also import" do assert capture_err(fn -> - Code.compile_string """ - defmodule Sample do - alias :lists, as: List - def a, do: nil - end - """ - end) =~ "warning: unused alias List" + Code.compile_string(""" + defmodule Sample do + alias :lists, as: List + import MapSet + new() + end + """) + end) =~ "unused alias List" after - purge [Sample] + purge(Sample) end - test :unused_inside_dynamic_module do + test "unused inside dynamic module" do import List, only: [flatten: 1], warn: false assert capture_err(fn -> - defmodule Sample do - import String, only: [downcase: 1] + defmodule Sample do + import String, only: [downcase: 1] - def world do - flatten([1,2,3]) - end - end - end) =~ "warning: unused import String" + def world do + flatten([1, 2, 3]) + end + end + end) =~ "unused import String" after - purge [Sample] + purge(Sample) + end + + test "duplicate map keys" do + output = + capture_err(fn -> + defmodule DuplicateMapKeys do + assert %{a: :b, a: :c} == %{a: :c} + assert %{m: :n, m: :o, m: :p} == %{m: :p} + assert %{1 => 2, 1 => 3} == %{1 => 3} + end + end) + + assert output =~ "key :a will be overridden in map" + assert output =~ "key :m will be overridden in map" + assert output =~ "key 1 will be overridden in map" + + assert map_size(%{System.unique_integer() => 1, System.unique_integer() => 2}) == 2 end - test :unused_guard do + test "unused guard" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample1 do - def is_atom_case do - v = "bc" - case v do - _ when is_atom(v) -> :ok - _ -> :fail + Code.eval_string(""" + defmodule Sample do + def atom_case do + v = "bc" + case v do + _ when is_atom(v) -> :ok + _ -> :fail + end + end + end + """) + end) =~ "this check/guard will always yield the same result" + after + purge(Sample) + end + + test "length(list) == 0 in guard" do + error_message = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + def list_case do + v = [] + case v do + _ when length(v) == 0 -> :ok + _ -> :fail + end end end - end - """ - end) =~ "nofile:5: warning: the guard for this clause evaluates to 'false'" - - assert capture_err(fn -> - Code.eval_string """ - defmodule Sample2 do - def is_binary_cond do - v = "bc" - cond do - is_binary(v) -> :bin - true -> :ok + """) + end) + + assert error_message =~ "do not use \"length(v) == 0\" to check if a list is empty" + + assert error_message =~ + "Prefer to pattern match on an empty list or use \"v == []\" as a guard" + after + purge(Sample) + end + + test "length(list) > 0 in guard" do + error_message = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + def list_case do + v = [] + case v do + _ when length(v) > 0 -> :ok + _ -> :fail + end end end - end - """ - end) =~ "nofile:6: warning: this clause cannot match because a previous clause at line 5 always matches" + """) + end) + + assert error_message =~ "do not use \"length(v) > 0\" to check if a list is not empty" + + assert error_message =~ + "Prefer to pattern match on a non-empty list, such as [_ | _], or use \"v != []\" as a guard" after - purge [Sample1, Sample2] + purge(Sample) end - test :empty_clause do + test "late function heads" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample1 do + defmacro __using__(_) do + quote do + def add(a, b), do: a + b + end + end + end + + defmodule Sample2 do + use Sample1 + @doc "hello" + def add(a, b) + end + """) + end) == "" + assert capture_err(fn -> - Code.eval_string """ - defmodule Sample1 do - def hello - end - """ - end) =~ "warning: empty clause provided for nonexistent function or macro hello/0" + Code.eval_string(""" + defmodule Sample3 do + def add(a, b), do: a + b + @doc "hello" + def add(a, b) + end + """) + end) =~ "function head for def add/2 must come at the top of its direct implementation" after - purge [Sample1] + purge([Sample1, Sample2, Sample3]) end - test :used_import_via_alias do + test "used import via alias" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample1 do - import List, only: [flatten: 1] + Code.eval_string(""" + defmodule Sample1 do + import List, only: [flatten: 1] - defmacro generate do - List.duplicate(quote(do: flatten([1,2,3])), 100) - end - end + defmacro generate do + List.duplicate(quote(do: flatten([1, 2, 3])), 100) + end + end + + defmodule Sample2 do + import Sample1 + generate() + end + """) + end) == "" + after + purge([Sample1, Sample2]) + end + + test "clause not match" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + def hello, do: nil + def hello, do: nil + end + """) + end) =~ + ~r"this clause( for hello/0)? cannot match because a previous clause at line 2 always matches" + after + purge(Sample) + end - defmodule Sample2 do - import Sample1 - generate - end - """ - end) == "" + test "generated clause not match" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + defmacro __using__(_) do + quote do + def hello, do: nil + def hello, do: nil + end + end + end + defmodule UseSample do + use Sample + end + """) + end) =~ + ~r"this clause( for hello/0)? cannot match because a previous clause at line 10 always matches" after - purge [Sample1, Sample2] + purge(Sample) + purge(UseSample) + end + + test "deprecated not left in right" do + assert capture_err(fn -> + Code.eval_string("not 1 in [1, 2, 3]") + end) =~ "deprecated" end - test :clause_not_match do + test "clause with defaults should be first" do + message = "def hello/1 has multiple clauses and also declares default values" + assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - def hello, do: nil - def hello, do: nil - end - """ - end) =~ "warning: this clause cannot match because a previous clause at line 2 always matches" + Code.eval_string(~S""" + defmodule Sample1 do + def hello(arg), do: arg + def hello(arg \\ 0), do: arg + end + """) + end) =~ message + + assert capture_err(fn -> + Code.eval_string(~S""" + defmodule Sample2 do + def hello(_arg) + def hello(arg \\ 0), do: arg + end + """) + end) =~ message after - purge Sample + purge([Sample1, Sample2]) end - test :clause_with_defaults_should_be_first do + test "clauses with default should use header" do + message = "def hello/1 has multiple clauses and also declares default values" + assert capture_err(fn -> - Code.eval_string ~S""" - defmodule Sample do - def hello(arg), do: nil - def hello(arg \\ 0), do: nil - end - """ - end) =~ "warning: clause with defaults should be the first clause in def hello/1" + Code.eval_string(~S""" + defmodule Sample do + def hello(arg \\ 0), do: arg + def hello(arg), do: arg + end + """) + end) =~ message after - purge Sample + purge(Sample) end - test :unused_with_local_with_overridable do + test "unused with local with overridable" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - def hello, do: world - defp world, do: :ok - defoverridable [hello: 0] - def hello, do: :ok - end - """ - end) =~ "warning: function world/0 is unused" + Code.eval_string(""" + defmodule Sample do + def hello, do: world() + defp world, do: :ok + defoverridable [hello: 0] + def hello, do: :ok + end + """) + end) =~ "function world/0 is unused" after - purge Sample + purge(Sample) end - test :used_with_local_with_reattached_overridable do + test "undefined module attribute" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - def hello, do: world - defp world, do: :ok - defoverridable [hello: 0, world: 0] - end - """ - end) == "" + Code.eval_string(""" + defmodule Sample do + @foo + end + """) + end) =~ + "undefined module attribute @foo, please remove access to @foo or explicitly set it before access" after - purge Sample + purge(Sample) end - test :undefined_module_attribute do + test "parens with module attribute" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - @foo - end - """ - end) =~ "warning: undefined module attribute @foo, please remove access to @foo or explicitly set it to nil before access" + Code.eval_string(""" + defmodule Sample do + @foo 13 + @foo() + end + """) + end) =~ + "the @foo() notation (with parenthesis) is deprecated, please use @foo (without parenthesis) instead" after - purge Sample + purge(Sample) end - test :undefined_module_attribute_in_function do + test "undefined module attribute in function" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - def hello do - @foo - end - end - """ - end) =~ "warning: undefined module attribute @foo, please remove access to @foo or explicitly set it to nil before access" + Code.eval_string(""" + defmodule Sample do + def hello do + @foo + end + end + """) + end) =~ + "undefined module attribute @foo, please remove access to @foo or explicitly set it before access" + after + purge(Sample) + end + + test "undefined module attribute with file" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @foo + end + """) + end) =~ + "undefined module attribute @foo, please remove access to @foo or explicitly set it before access" after - purge Sample + purge(Sample) end - test :undefined_module_attribute_with_file do + test "parse transform" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - @foo - end - """ - end) =~ "warning: undefined module attribute @foo, please remove access to @foo or explicitly set it to nil before access" + Code.eval_string(""" + defmodule Sample do + @compile {:parse_transform, :ms_transform} + end + """) + end) =~ "@compile {:parse_transform, :ms_transform} is deprecated" + after + purge(Sample) + end + + test "@compile inline no warning for unreachable function" do + refute capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @compile {:inline, foo: 1} + + defp foo(_), do: :ok + end + """) + end) =~ "inlined function foo/1 undefined" after - purge Sample + purge(Sample) end - test :in_guard_empty_list do + test "in guard empty list" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - def a(x) when x in [], do: x - end - """ - end) =~ "warning: the guard for this clause evaluates to 'false'" + Code.eval_string(""" + defmodule Sample do + def a(x) when x in [], do: x + end + """) + end) =~ "this check/guard will always yield the same result" after - purge Sample + purge(Sample) end - test :no_effect_operator do + test "no effect operator" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - def a(x) do - x != :foo - :ok + Code.eval_string(""" + defmodule Sample do + def a(x) do + x != :foo + :ok + end + end + """) + end) =~ "use of operator != has no effect" + after + purge(Sample) + end + + # TODO: Simplify when we require Erlang/OTP 24 + if System.otp_release() >= "24" do + @argument_error_message "the call to Atom.to_string/1" + @arithmetic_error_message "the call to +/2" + else + @argument_error_message "this expression" + @arithmetic_error_message "this expression" + end + + test "eval failure warning" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample1 do + def foo, do: Atom.to_string "abc" + end + """) + end) =~ "#{@argument_error_message} will fail with ArgumentError\n nofile:2" + + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample2 do + def foo, do: 1 + nil + end + """) + end) =~ "#{@arithmetic_error_message} will fail with ArithmeticError\n nofile:2" + after + purge([Sample1, Sample2]) + end + + test "undefined function for behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample1 do + @callback foo :: term + end + + defmodule Sample2 do + @behaviour Sample1 + end + """) + end) =~ + "function foo/0 required by behaviour Sample1 is not implemented (in module Sample2)" + after + purge([Sample1, Sample2]) + end + + test "undefined macro for behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample1 do + @macrocallback foo :: Macro.t + end + + defmodule Sample2 do + @behaviour Sample1 + end + """) + end) =~ + "macro foo/0 required by behaviour Sample1 is not implemented (in module Sample2)" + after + purge([Sample1, Sample2]) + end + + test "wrong kind for behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample1 do + @callback foo :: term + end + + defmodule Sample2 do + @behaviour Sample1 + defmacro foo, do: :ok + end + """) + end) =~ + "function foo/0 required by behaviour Sample1 was implemented as \"defmacro\" but should have been \"def\"" + after + purge([Sample1, Sample2]) + end + + test "conflicting behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample1 do + @callback foo :: term + end + + defmodule Sample2 do + @callback foo :: term + end + + defmodule Sample3 do + @behaviour Sample1 + @behaviour Sample2 + end + """) + end) =~ + "conflicting behaviours found. function foo/0 is required by Sample1 and Sample2 (in module Sample3)" + after + purge([Sample1, Sample2, Sample3]) + end + + test "duplicate behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample1 do + @callback foo :: term + end + + defmodule Sample2 do + @behaviour Sample1 + @behaviour Sample1 + end + """) + end) =~ + "the behavior Sample1 has been declared twice (conflict in function foo/0 in module Sample2)" + after + purge([Sample1, Sample2]) + end + + test "undefined behaviour" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @behaviour UndefinedBehaviour + end + """) + end) =~ "@behaviour UndefinedBehaviour does not exist (in module Sample)" + after + purge(Sample) + end + + test "empty behaviours" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule EmptyBehaviour do + end + defmodule Sample do + @behaviour EmptyBehaviour + end + """) + end) =~ "module EmptyBehaviour is not a behaviour (in module Sample)" + after + purge(Sample) + purge(EmptyBehaviour) + end + + test "undefined behavior" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @behavior Hello + end + """) + end) =~ "@behavior attribute is not supported, please use @behaviour instead" + after + purge(Sample) + end + + test "undefined function for protocol" do + assert capture_err(fn -> + Code.eval_string(""" + defprotocol Sample1 do + def foo(subject) + end + + defimpl Sample1, for: Atom do + end + """) + end) =~ + "function foo/1 required by protocol Sample1 is not implemented (in module Sample1.Atom)" + after + purge([Sample1, Sample1.Atom]) + end + + test "overridden def name" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + def foo(x, 1), do: x + 1 + def foo(), do: nil + def foo(x, 2), do: x * 2 + end + """) + end) =~ + "clauses with the same name should be grouped together, \"def foo/2\" was previously defined (nofile:2)" + after + purge(Sample) + end + + test "overridden def name and arity" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + def foo(x, 1), do: x + 1 + def bar(), do: nil + def foo(x, 2), do: x * 2 + end + """) + end) =~ + "clauses with the same name and arity (number of arguments) should be grouped together, \"def foo/2\" was previously defined (nofile:2)" + after + purge(Sample) + end + + test "warning with overridden file" do + output = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @file "sample" + def foo(x), do: :ok end - end - """ - end) =~ "warning: use of operator != has no effect" + """) + end) + + assert output =~ "variable \"x\" is unused" + assert output =~ "sample:3" after - purge Sample + purge(Sample) + end + + test "warning on unnecessary code point escape" do + assert capture_err(fn -> + Code.eval_string("?\\n + ?\\\\") + end) == "" + + assert capture_err(fn -> + Code.eval_string("?\\w") + end) =~ "unknown escape sequence ?\\w, use ?w instead" end - test :undefined_function_for_behaviour do + test "warning on code point escape" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample1 do - use Behaviour - defcallback foo - end + Code.eval_string("? ") + end) =~ "found ? followed by code point 0x20 (space), please use ?\\s instead" + + assert capture_err(fn -> + Code.eval_string("?\\ ") + end) =~ "found ?\\ followed by code point 0x20 (space), please use ?\\s instead" + end - defmodule Sample2 do - @behaviour Sample1 - end - """ - end) =~ "warning: undefined behaviour function foo/0 (for behaviour Sample1)" + test "duplicated docs in the same clause" do + output = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @doc "Something" + @doc "Another" + def foo, do: :ok + + Module.eval_quoted(__MODULE__, quote(do: @doc false)) + @doc "Doc" + def bar, do: :ok + end + """) + end) + + assert output =~ "redefining @doc attribute previously set at line 2" + assert output =~ "nofile:3: Sample (module)" + refute output =~ "nofile:7" after - purge [Sample1, Sample2, Sample3] + purge(Sample) end - test :undefined_macro_for_behaviour do + test "duplicate docs across clauses" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample1 do + defmacro __using__(_) do + quote do + @doc "hello" + def add(a, 1), do: a + 1 + end + end + end + + defmodule Sample2 do + use Sample1 + @doc "world" + def add(a, 2), do: a + 2 + end + """) + end) == "" + assert capture_err(fn -> - Code.eval_string """ - defmodule Sample1 do - use Behaviour - defmacrocallback foo - end + Code.eval_string(""" + defmodule Sample3 do + @doc "hello" + def add(a, 1), do: a + 1 + @doc "world" + def add(a, b) + end + """) + end) =~ "" - defmodule Sample2 do - @behaviour Sample1 - end - """ - end) =~ "warning: undefined behaviour macro foo/0 (for behaviour Sample1)" + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample4 do + @doc "hello" + def add(a, 1), do: a + 1 + @doc "world" + def add(a, 2), do: a + 2 + end + """) + end) =~ "redefining @doc attribute previously set at line " + after + purge([Sample1, Sample2, Sample3, Sample4]) + end + + test "reserved doc metadata keys" do + output = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @typedoc opaque: false + @type t :: binary + + @doc defaults: 3, since: "1.2.3" + def foo(a), do: a + end + """) + end) + + assert output =~ "ignoring reserved documentation metadata key: :opaque" + assert output =~ "ignoring reserved documentation metadata key: :defaults" + refute output =~ ":since" after - purge [Sample1, Sample2, Sample3] + purge(Sample) end - test :undefined_behavior do + describe "typespecs" do + test "unused types" do + output = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @type pub :: any + @opaque op :: any + @typep priv :: any + @typep priv_args(var1, var2) :: {var1, var2} + @typep priv2 :: any + @typep priv3 :: priv2 | atom + + @spec my_fun(priv3) :: pub + def my_fun(var), do: var + end + """) + end) + + assert output =~ "nofile:4" + assert output =~ "type priv/0 is unused" + assert output =~ "nofile:5" + assert output =~ "type priv_args/2 is unused" + refute output =~ "type pub/0 is unused" + refute output =~ "type op/0 is unused" + refute output =~ "type priv2/0 is unused" + refute output =~ "type priv3/0 is unused" + after + purge(Sample) + end + + test "underspecified opaque types" do + output = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @opaque op1 :: term + @opaque op2 :: any + @opaque op3 :: atom + end + """) + end) + + assert output =~ "nofile:2" + assert output =~ "@opaque type op1/0 is underspecified and therefore meaningless" + assert output =~ "nofile:3" + assert output =~ "@opaque type op2/0 is underspecified and therefore meaningless" + refute output =~ "nofile:4" + refute output =~ "op3" + after + purge(Sample) + end + + test "underscored types variables" do + output = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @type in_typespec_vars(_var1, _var1) :: atom + @type in_typespec(_var2) :: {atom, _var2} + + @spec in_spec(_var3) :: {atom, _var3} when _var3: var + def in_spec(a), do: {:ok, a} + end + """) + end) + + assert output =~ "nofile:2" + assert output =~ ~r/the underscored type variable "_var1" is used more than once/ + assert output =~ "nofile:3" + assert output =~ ~r/the underscored type variable "_var2" is used more than once/ + assert output =~ "nofile:5" + assert output =~ ~r/the underscored type variable "_var3" is used more than once/ + after + purge(Sample) + end + + test "typedoc on typep" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @typedoc "Something" + @typep priv :: any + @spec foo() :: priv + def foo(), do: nil + end + """) + end) =~ "type priv/0 is private, @typedoc's are always discarded for private types" + after + purge(Sample) + end + + test "discouraged types" do + message = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @type foo :: string() + @type bar :: nonempty_string() + end + """) + end) + + string_discouraged = + "string() type use is discouraged. " <> + "For character lists, use charlist() type, for strings, String.t()\n" + + nonempty_string_discouraged = + "nonempty_string() type use is discouraged. " <> + "For non-empty character lists, use nonempty_charlist() type, for strings, String.t()\n" + + assert message =~ string_discouraged + assert message =~ nonempty_string_discouraged + after + purge(Sample) + end + + test "unreachable specs" do + message = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + defp my_fun(x), do: x + @spec my_fun(integer) :: integer + end + """) + end) + + assert message != "" + after + purge(Sample) + end + + test "nested type annotations" do + message = "invalid type annotation. Type annotations cannot be nested" + + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample1 do + @type my_type :: ann_type :: nested_ann_type :: atom + end + """) + end) =~ message + + purge(Sample1) + + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample2 do + @type my_type :: ann_type :: nested_ann_type :: atom | port + end + """) + end) =~ message + + purge(Sample2) + + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample3 do + @spec foo :: {pid, ann_type :: nested_ann_type :: atom} + def foo, do: nil + end + """) + end) =~ message + after + purge([Sample1, Sample2, Sample3]) + end + + test "invalid type annotations" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample1 do + @type my_type :: (pid() :: atom) + end + """) + end) =~ "invalid type annotation. The left side of :: must be a variable, got: pid()" + + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample2 do + @type my_type :: pid | ann_type :: atom + end + """) + end) =~ + "invalid type annotation. The left side of :: must be a variable, got: pid | ann_type. " <> + "Note \"left | right :: ann\" is the same as \"(left | right) :: ann\"" + after + purge([Sample1, Sample2]) + end + end + + test "attribute with no use" do + content = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @at "Something" + end + """) + end) + + assert content =~ "module attribute @at was set but never used" + assert content =~ "nofile:2" + after + purge(Sample) + end + + test "registered attribute with no use" do + content = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + Module.register_attribute(__MODULE__, :at, []) + @at "Something" + end + """) + end) + + assert content =~ "module attribute @at was set but never used" + assert content =~ "nofile:3" + after + purge(Sample) + end + + test "typedoc with no type" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @typedoc "Something" + end + """) + end) =~ "module attribute @typedoc was set but no type follows it" + after + purge(Sample) + end + + test "doc with no function" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + @doc "Something" + end + """) + end) =~ "module attribute @doc was set but no definition follows it" + after + purge(Sample) + end + + test "pipe without explicit parentheses" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - @behavior Hello - end - """ - end) =~ "warning: @behavior attribute is not supported, please use @behaviour instead" + Code.eval_string(""" + [5, 6, 7, 3] + |> Enum.map_join "", &(Integer.to_string(&1)) + |> String.to_integer + """) + end) =~ "parentheses are required when piping into a function call" + end + + test "variable is being expanded to function call" do + output = + capture_err(fn -> + Code.eval_string(""" + self + defmodule Sample do + def my_node(), do: node + end + """) + end) + + assert output =~ "variable \"self\" does not exist and is being expanded to \"self()\"" + assert output =~ "variable \"node\" does not exist and is being expanded to \"node()\"" after - purge [Sample] + purge(Sample) + end + + defmodule User do + defstruct [:name] end - test :undefined_macro_for_protocol do + test ":__struct__ is ignored when using structs" do assert capture_err(fn -> - Code.eval_string """ - defprotocol Sample1 do - def foo(subject) - end + code = """ + assert %Kernel.WarningTest.User{__struct__: Ignored, name: "joe"} == + %Kernel.WarningTest.User{name: "joe"} + """ + + Code.eval_string(code, [], __ENV__) + end) =~ "key :__struct__ is ignored when using structs" + + assert capture_err(fn -> + code = """ + user = %Kernel.WarningTest.User{name: "meg"} + assert %Kernel.WarningTest.User{user | __struct__: Ignored, name: "joe"} == + %Kernel.WarningTest.User{__struct__: Kernel.WarningTest.User, name: "joe"} + """ + + Code.eval_string(code, [], __ENV__) + end) =~ "key :__struct__ is ignored when using structs" + end + + test "catch comes before rescue in try block" do + output = + capture_err(fn -> + Code.eval_string(""" + try do + :trying + catch + _ -> :caught + rescue + _ -> :error + end + """) + end) + + assert output =~ ~s("catch" should always come after "rescue" in try) + end + + test "catch comes before rescue in def" do + output = + capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + def foo do + :trying + catch + _, _ -> :caught + rescue + _ -> :error + end + end + """) + end) - defimpl Sample1, for: Atom do - end - """ - end) =~ "warning: undefined protocol function foo/1 (for protocol Sample1)" + assert output =~ ~s("catch" should always come after "rescue" in def) after - purge [Sample1, Sample1.Atom] + purge(Sample) end - test :overidden_def do + test "unused variable in defguard" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - def foo(x, 1), do: x + 1 - def bar(), do: nil - def foo(x, 2), do: x * 2 - end - """ - end) =~ "nofile:4: warning: clauses for the same def should be grouped together, def foo/2 was previously defined (nofile:2)" + Code.eval_string(""" + defmodule Sample do + defguard foo(bar, baz) when bar + end + """) + end) =~ "variable \"baz\" is unused" after - purge [Sample] + purge(Sample) end - test :warning_with_overridden_file do + test "unused import in defguard" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - @file "sample" - def foo(x), do: :ok - end - """ - end) =~ "sample:3: warning: variable x is unused" + Code.compile_string(""" + defmodule Sample do + import Record + defguard is_record(baz) when baz + end + """) + end) =~ "unused import Record\n" after - purge [Sample] + purge(Sample) end - test :typedoc_on_typep do + test "unused private guard" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - @typedoc "Something" - @typep priv :: any - @spec foo() :: priv - def foo(), do: nil - end - """ - end) =~ "nofile:3: warning: type priv/0 is private, @typedoc's are always discarded for private types" + Code.compile_string(""" + defmodule Sample do + defguardp foo(bar, baz) when bar + baz + end + """) + end) =~ "macro foo/2 is unused\n" after - purge [Sample] + purge(Sample) end - test :typedoc_with_no_type do + test "defguard overriding defmacro" do assert capture_err(fn -> - Code.eval_string """ - defmodule Sample do - @typedoc "Something" - end - """ - end) =~ "nofile:1: warning: @typedoc provided but no type follows it" + Code.eval_string(""" + defmodule Sample do + defmacro foo(bar), do: bar == :bar + defguard foo(baz) when baz == :baz + end + """) + end) =~ + ~r"this clause( for foo/1)? cannot match because a previous clause at line 2 always matches" after - purge [Sample] + purge(Sample) + end + + test "defmacro overriding defguard" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + defguard foo(baz) when baz == :baz + defmacro foo(bar), do: bar == :bar + end + """) + end) =~ + ~r"this clause( for foo/1)? cannot match because a previous clause at line 2 always matches" + after + purge(Sample) + end + + test "struct comparisons" do + expressions = [ + ~s(~N"2018-01-28 12:00:00"), + ~s(~T"12:00:00"), + ~s(~D"2018-01-28"), + "%File.Stat{}" + ] + + for op <- [:<, :>, :<=, :>=], + expression <- expressions do + assert capture_err(fn -> + Code.eval_string("x #{op} #{expression}", x: 1) + end) =~ "invalid comparison with struct literal #{expression}" + + assert capture_err(fn -> + Code.eval_string("#{expression} #{op} x", x: 1) + end) =~ "invalid comparison with struct literal #{expression}" + end + end + + test "deprecated GenServer super on callbacks" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + use GenServer + + def handle_call(a, b, c) do + super(a, b, c) + end + end + """) + end) =~ "calling super for GenServer callback handle_call/3 is deprecated" + after + purge(Sample) + end + + test "super is allowed on GenServer.child_spec/1" do + refute capture_err(fn -> + Code.eval_string(""" + defmodule Sample do + use GenServer + + def child_spec(opts) do + super(opts) + end + end + """) + end) =~ "calling super for GenServer callback child_spec/1 is deprecated" + after + purge(Sample) + end + + test "nested comparison operators" do + message = + capture_err(fn -> + Code.compile_string(""" + 1 < 3 < 5 + """) + end) + + assert message =~ "Elixir does not support nested comparisons" + assert message =~ "1 < 3 < 5" + + message = + capture_err(fn -> + Code.compile_string(""" + x = 5 + y = 7 + 1 < x < y < 10 + """) + end) + + assert message =~ "Elixir does not support nested comparisons" + assert message =~ "1 < x < y < 10" + end + + test "def warns if only clause is else" do + message = + capture_err(fn -> + Code.compile_string(""" + defmodule Sample do + def foo do + :bar + else + _other -> :ok + end + end + """) + end) + + assert message =~ "\"else\" shouldn't be used as the only clause in \"def\"" + after + purge(Sample) + end + + test "try warns if only clause is else" do + message = + capture_err(fn -> + Code.compile_string(""" + try do + :ok + else + other -> other + end + """) + end) + + assert message =~ "\"else\" shouldn't be used as the only clause in \"try\"" + end + + test "sigil w/W warns on trailing comma at macro expansion time" do + for sigil <- ~w(w W), + modifier <- ~w(a s c) do + output = + capture_err(fn -> + {:ok, ast} = + "~#{sigil}(foo, bar baz)#{modifier}" + |> Code.string_to_quoted() + + Macro.expand(ast, __ENV__) + end) + + assert output =~ "the sigils ~w/~W do not allow trailing commas" + end + end + + test "warnings on trailing comma on call" do + assert capture_err(fn -> Code.eval_string("Keyword.merge([], foo: 1,)") end) =~ + "trailing commas are not allowed inside function/macro call arguments" + end + + test "defstruct warns with duplicate keys" do + assert capture_err(fn -> + Code.eval_string(""" + defmodule TestMod do + defstruct [:foo, :bar, foo: 1] + end + """) + end) =~ "duplicate key :foo found in struct" + after + purge(TestMod) + end + + test "deprecate nullary remote zero-arity capture with parens" do + assert capture_err(fn -> + Code.eval_string(""" + import System, only: [pid: 0] + &pid/0 + """) + end) == "" + + assert capture_err(fn -> + Code.eval_string(""" + &System.pid()/0 + """) + end) =~ + "extra parentheses on a remote function capture &System.pid()/0 have been deprecated. Please remove the parentheses: &System.pid/0" end defp purge(list) when is_list(list) do - Enum.each list, &purge/1 + Enum.each(list, &purge/1) end defp purge(module) when is_atom(module) do - :code.delete module - :code.purge module + :code.delete(module) + :code.purge(module) end end diff --git a/lib/elixir/test/elixir/kernel/with_test.exs b/lib/elixir/test/elixir/kernel/with_test.exs new file mode 100644 index 00000000000..d2e202af45f --- /dev/null +++ b/lib/elixir/test/elixir/kernel/with_test.exs @@ -0,0 +1,148 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Kernel.WithTest do + use ExUnit.Case, async: true + + test "basic with" do + assert with({:ok, res} <- ok(41), do: res) == 41 + assert with(res <- four(), do: res + 10) == 14 + end + + test "matching with" do + assert with(_..42 <- 1..42, do: :ok) == :ok + assert with({:ok, res} <- error(), do: res) == :error + assert with({:ok, _} = res <- ok(42), do: elem(res, 1)) == 42 + end + + test "with guards" do + assert with(x when x < 2 <- four(), do: :ok) == 4 + assert with(x when x > 2 <- four(), do: :ok) == :ok + assert with(x when x < 2 when x == 4 <- four(), do: :ok) == :ok + end + + test "pin matching with" do + key = :ok + assert with({^key, res} <- ok(42), do: res) == 42 + end + + test "two levels with" do + result = + with {:ok, n1} <- ok(11), + n2 <- 22, + do: n1 + n2 + + assert result == 33 + + result = + with n1 <- 11, + {:ok, n2} <- error(), + do: n1 + n2 + + assert result == :error + end + + test "binding inside with" do + result = + with {:ok, n1} <- ok(11), + n2 = n1 + 10, + {:ok, n3} <- ok(22), + do: n2 + n3 + + assert result == 43 + + result = + with {:ok, n1} <- ok(11), + n2 = n1 + 10, + {:ok, n3} <- error(), + do: n2 + n3 + + assert result == :error + end + + test "does not leak variables to else" do + state = 1 + + result = + with 1 <- state, + state = 2, + :ok <- error(), + do: state, + else: (_ -> state) + + assert result == 1 + assert state == 1 + end + + test "with shadowing" do + assert with( + a <- + ( + b = 1 + _ = b + 1 + ), + b <- 2, + do: a + b + ) == 3 + end + + test "with extra guards" do + var = + with %_{} = a <- struct(URI), + %_{} <- a do + :ok + end + + assert var == :ok + end + + test "errors in with" do + assert_raise RuntimeError, fn -> + with({:ok, res} <- oops(), do: res) + end + + assert_raise RuntimeError, fn -> + with({:ok, res} <- ok(42), res = res + oops(), do: res) + end + end + + test "else conditions" do + assert (with {:ok, res} <- four() do + res + else + {:error, error} -> error + res -> res + 1 + end) == 5 + + assert (with {:ok, res} <- four() do + res + else + res when res == 4 -> res + 1 + res -> res + end) == 5 + + assert with({:ok, res} <- four(), do: res, else: (_ -> :error)) == :error + end + + test "else conditions with match error" do + assert_raise WithClauseError, "no with clause matching: :error", fn -> + with({:ok, res} <- error(), do: res, else: ({:error, error} -> error)) + end + end + + defp four() do + 4 + end + + defp error() do + :error + end + + defp ok(num) do + {:ok, num} + end + + defp oops() do + raise("oops") + end +end diff --git a/lib/elixir/test/elixir/kernel_test.exs b/lib/elixir/test/elixir/kernel_test.exs index eee1a160d58..f170c94751e 100644 --- a/lib/elixir/test/elixir/kernel_test.exs +++ b/lib/elixir/test/elixir/kernel_test.exs @@ -1,15 +1,109 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule KernelTest do use ExUnit.Case, async: true + doctest Kernel + + def id(arg), do: arg + def id(arg1, arg2), do: {arg1, arg2} + def empty_list(), do: [] + def empty_map, do: %{} + + defp purge(module) do + :code.delete(module) + :code.purge(module) + end + + defp assert_eval_raise(error, msg, string) do + assert_raise error, msg, fn -> + Code.eval_string(string) + end + end + + test "op ambiguity" do + max = 1 + assert max == 1 + assert max(1, 2) == 2 + end + + describe "=/2" do + test "can be reassigned" do + var = 1 + id(var) + var = 2 + assert var == 2 + end + + test "can be reassigned inside a list" do + _ = [var = 1, 2, 3] + id(var) + _ = [var = 2, 3, 4] + assert var == 2 + end + + test "can be reassigned inside a keyword list" do + _ = [a: var = 1, b: 2] + id(var) + _ = [b: var = 2, c: 3] + assert var == 2 + end + + test "can be reassigned inside a call" do + id(var = 1) + id(var) + id(var = 2) + assert var == 2 + end + + test "can be reassigned inside a multi-argument call" do + id(:arg, var = 1) + id(:arg, var) + id(:arg, var = 2) + assert var == 2 + + id(:arg, a: 1, b: var = 2) + id(:arg, var) + id(:arg, b: 2, c: var = 3) + assert var == 3 + end + + test "++/2 works in matches" do + [1, 2] ++ var = [1, 2] + assert var == [] + + [1, 2] ++ var = [1, 2, 3] + assert var == [3] + + 'ab' ++ var = 'abc' + assert var == 'c' + + [:a, :b] ++ var = [:a, :b, :c] + assert var == [:c] + end + end + test "=~/2" do - assert ("abcd" =~ ~r/c(d)/) == true - assert ("abcd" =~ ~r/e/) == false + assert "abcd" =~ ~r/c(d)/ == true + assert "abcd" =~ ~r/e/ == false + assert "abcd" =~ ~R/c(d)/ == true + assert "abcd" =~ ~R/e/ == false string = "^ab+cd*$" - assert (string =~ "ab+") == true - assert (string =~ "bb") == false + assert string =~ "ab+" == true + assert string =~ "bb" == false + + assert "abcd" =~ ~r// == true + assert "abcd" =~ ~R// == true + assert "abcd" =~ "" == true + + assert "" =~ ~r// == true + assert "" =~ ~R// == true + assert "" =~ "" == true + + assert "" =~ "abcd" == false + assert "" =~ ~r/abcd/ == false + assert "" =~ ~R/abcd/ == false assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> 1234 =~ "hello" @@ -18,6 +112,34 @@ defmodule KernelTest do assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> 1234 =~ ~r"hello" end + + assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> + 1234 =~ ~R"hello" + end + + assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> + ~r"hello" =~ "hello" + end + + assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> + ~r"hello" =~ ~r"hello" + end + + assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> + :abcd =~ ~r// + end + + assert_raise FunctionClauseError, "no function clause matching in Kernel.=~/2", fn -> + :abcd =~ "" + end + + assert_raise FunctionClauseError, "no function clause matching in Regex.match?/2", fn -> + "abcd" =~ nil + end + + assert_raise FunctionClauseError, "no function clause matching in Regex.match?/2", fn -> + "abcd" =~ :abcd + end end test "^" do @@ -29,88 +151,25 @@ defmodule KernelTest do end end + # Note we use `==` in assertions so `assert` does not rewrite `match?/2`. test "match?/2" do - assert match?(_, List.first(1)) == true - assert binding([:x]) == [] - a = List.first([0]) assert match?(b when b > a, 1) == true - assert binding([:b]) == [] + assert binding() == [a: 0] assert match?(b when b > a, -1) == false - assert binding([:b]) == [] - end - - test "nil?/1" do - assert nil?(nil) == true - assert nil?(0) == false - assert nil?(false) == false - end - - test "in/2" do - assert 2 in [1, 2, 3] - assert 2 in 1..3 - refute 4 in [1, 2, 3] - refute 4 in 1..3 + assert binding() == [a: 0] - list = [1, 2, 3] - assert 2 in list - refute 4 in list + # Does not warn on underscored variables + assert match?(_unused, a) == true end - @at_list [4,5] - @at_range 6..8 - def fun_in(x) when x in [0], do: :list - def fun_in(x) when x in 1..3, do: :range - def fun_in(x) when x in @at_list, do: :at_list - def fun_in(x) when x in @at_range, do: :at_range + def exported?, do: not_exported?() + defp not_exported?, do: true - test "in/2 in function guard" do - assert fun_in(0) == :list - assert fun_in(2) == :range - assert fun_in(5) == :at_list - assert fun_in(8) == :at_range - end - - defmacrop case_in(x, y) do - quote do - case 0 do - _ when unquote(x) in unquote(y) -> true - _ -> false - end - end - end - - test "in/2 in case guard" do - assert case_in(1, [1,2,3]) == true - assert case_in(1, 1..3) == true - assert case_in(2, 1..3) == true - assert case_in(3, 1..3) == true - assert case_in(-3, -1..-3) == true - end - - test "paren as nil" do - assert nil?(()) == true - assert ((); ();) == nil - assert [ 1, (), 3 ] == [1, nil, 3 ] - assert [do: ()] == [do: nil] - assert {1, (), 3} == {1, nil, 3} - assert (Kernel.&& nil, ()) == nil - assert (Kernel.&& nil, ()) == nil - assert (() && ()) == nil - assert (if(() && ()) do - :ok - else - :error - end) == :error - end - - test "__info__(:macros)" do - assert {:in, 2} in Kernel.__info__(:macros) - end - - test "__info__(:functions)" do - assert not ({:__info__, 1} in Kernel.__info__(:functions)) + test "function_exported?/3" do + assert function_exported?(__MODULE__, :exported?, 0) + refute function_exported?(__MODULE__, :not_exported?, 0) end test "macro_exported?/3" do @@ -119,302 +178,1069 @@ defmodule KernelTest do assert macro_exported?(Kernel, :def, 2) == true assert macro_exported?(Kernel, :def, 3) == false assert macro_exported?(Kernel, :no_such_macro, 2) == false + assert macro_exported?(:erlang, :abs, 1) == false end test "apply/3 and apply/2" do - assert apply(Enum, :reverse, [[1|[2, 3]]]) == [3, 2, 1] + assert apply(Enum, :reverse, [[1 | [2, 3]]]) == [3, 2, 1] assert apply(fn x -> x * 2 end, [2]) == 4 end - test "binding/0, binding/1 and binding/2" do + test "binding/0 and binding/1" do x = 1 - assert binding == [x: 1] - assert binding([:x, :y]) == [x: 1] - assert binding([:x, :y], nil) == [x: 1] + assert binding() == [x: 1] x = 2 - assert binding == [x: 2] + assert binding() == [x: 2] y = 3 - assert binding == [x: 2, y: 3] + assert binding() == [x: 2, y: 3] - var!(x, :foo) = 2 - assert binding(:foo) == [x: 2] - assert binding([:x, :y], :foo) == [x: 2] - end + var!(x, :foo) = 4 + assert binding() == [x: 2, y: 3] + assert binding(:foo) == [x: 4] - defmodule User do - defstruct name: "jose" + # No warnings + _x = 1 + assert binding() == [_x: 1, x: 2, y: 3] end - defmodule UserTuple do - def __struct__({ UserTuple, :ok }) do - %User{} - end + defmodule User do + assert is_map(defstruct name: "john") end test "struct/1 and struct/2" do - assert struct(User) == %User{name: "jose"} + assert struct(User) == %User{name: "john"} - user = struct(User, name: "eric") - assert user == %User{name: "eric"} + user = struct(User, name: "meg") + assert user == %User{name: "meg"} + assert struct(user, %{name: "meg"}) == user assert struct(user, unknown: "key") == user - assert struct(user, %{name: "jose"}) == %User{name: "jose"} + assert struct(user, %{name: "john"}) == %User{name: "john"} assert struct(user, name: "other", __struct__: Post) == %User{name: "other"} + end + + test "struct!/1 and struct!/2" do + assert struct!(User) == %User{name: "john"} + + user = struct!(User, name: "meg") + assert user == %User{name: "meg"} - user_tuple = {UserTuple, :ok} - assert struct(user_tuple, name: "eric") == %User{name: "eric"} + assert_raise KeyError, fn -> + struct!(user, unknown: "key") + end + + assert struct!(user, %{name: "john"}) == %User{name: "john"} + assert struct!(user, name: "other", __struct__: Post) == %User{name: "other"} end - defdelegate my_flatten(list), to: List, as: :flatten - defdelegate [map(callback, list)], to: :lists, append_first: true + test "if/2 with invalid keys" do + error_message = + "invalid or duplicate keys for if, only \"do\" and an optional \"else\" are permitted" - dynamic = :dynamic_flatten - defdelegate unquote(dynamic)(list), to: List, as: :flatten + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("if true, foo: 7") + end - test "defdelefate/2" do - assert my_flatten([[1]]) == [1] + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("if true, do: 6, boo: 7") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("if true, do: 7, do: 6") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("if true, do: 8, else: 7, else: 6") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("if true, else: 6") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("if true, []") + end end - test "defdelegate/2 with :append_first" do - assert map([1], fn(x) -> x + 1 end) == [2] + test "unless/2 with invalid keys" do + error_message = + "invalid or duplicate keys for unless, only \"do\" " <> + "and an optional \"else\" are permitted" + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("unless true, foo: 7") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("unless true, do: 6, boo: 7") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("unless true, do: 7, do: 6") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("unless true, do: 8, else: 7, else: 6") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("unless true, else: 6") + end + + assert_raise ArgumentError, error_message, fn -> + Code.eval_string("unless true, []") + end end - test "defdelegate/2 with unquote" do - assert dynamic_flatten([[1]]) == [1] + test "and/2" do + assert (true and false) == false + assert (true and true) == true + assert (true and 0) == 0 + assert (false and false) == false + assert (false and true) == false + assert (false and 0) == false + assert (false and raise("oops")) == false + assert ((x = true) and not x) == false + assert_raise BadBooleanError, fn -> 0 and 1 end end - test "get_in/2" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} - assert get_in(users, ["josé", :age]) == 27 - assert get_in(users, ["dave", :age]) == nil - assert get_in(nil, ["josé", :age]) == nil + test "or/2" do + assert (true or false) == true + assert (true or true) == true + assert (true or 0) == true + assert (true or raise("foo")) == true + assert (false or false) == false + assert (false or true) == true + assert (false or 0) == 0 + assert ((x = false) or not x) == true + assert_raise BadBooleanError, fn -> 0 or 1 end + end - assert_raise FunctionClauseError, fn -> - get_in(users, []) - end + defp struct?(arg) when is_struct(arg), do: true + defp struct?(_arg), do: false + + defp struct_or_map?(arg) when is_struct(arg) or is_map(arg), do: true + defp struct_or_map?(_arg), do: false + + test "is_struct/1" do + assert is_struct(%{}) == false + assert is_struct([]) == false + assert is_struct(%Macro.Env{}) == true + assert is_struct(%{__struct__: "foo"}) == false + assert struct?(%Macro.Env{}) == true + assert struct?(%{__struct__: "foo"}) == false + assert struct?([]) == false + assert struct?(%{}) == false + end + + test "is_struct/1 and other match works" do + assert struct_or_map?(%Macro.Env{}) == true + assert struct_or_map?(%{}) == true + assert struct_or_map?(10) == false end - test "put_in/3" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} + defp struct?(arg, name) when is_struct(arg, name), do: true + defp struct?(_arg, _name), do: false - assert put_in(nil, ["josé", :age], 28) == - %{"josé" => %{age: 28}} + defp struct_or_map?(arg, name) when is_struct(arg, name) or is_map(arg), do: true + defp struct_or_map?(_arg, _name), do: false - assert put_in(users, ["josé", :age], 28) == - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + defp not_atom(), do: "not atom" - assert put_in(users, ["dave", :age], 19) == - %{"josé" => %{age: 27}, "eric" => %{age: 23}, "dave" => %{age: 19}} + test "is_struct/2" do + assert is_struct(%{}, Macro.Env) == false + assert is_struct([], Macro.Env) == false + assert is_struct(%Macro.Env{}, Macro.Env) == true + assert is_struct(%Macro.Env{}, URI) == false + assert struct?(%Macro.Env{}, Macro.Env) == true + assert struct?(%Macro.Env{}, URI) == false + assert struct?(%{__struct__: "foo"}, "foo") == false + assert struct?(%{__struct__: "foo"}, Macro.Env) == false + assert struct?([], Macro.Env) == false + assert struct?(%{}, Macro.Env) == false - assert_raise FunctionClauseError, fn -> - put_in(users, [], %{}) + assert_raise ArgumentError, "argument error", fn -> + is_struct(%{}, not_atom()) end end - test "put_in/2" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} + test "is_struct/2 and other match works" do + assert struct_or_map?(%{}, "foo") == false + assert struct_or_map?(%{}, Macro.Env) == true + assert struct_or_map?(%Macro.Env{}, Macro.Env) == true + end - assert put_in(nil["josé"][:age], 28) == - %{"josé" => %{age: 28}} + defp exception?(arg) when is_exception(arg), do: true + defp exception?(_arg), do: false + + defp exception_or_map?(arg) when is_exception(arg) or is_map(arg), do: true + defp exception_or_map?(_arg), do: false + + test "is_exception/1" do + assert is_exception(%{}) == false + assert is_exception([]) == false + assert is_exception(%RuntimeError{}) == true + assert is_exception(%{__exception__: "foo"}) == false + assert exception?(%RuntimeError{}) == true + assert exception?(%{__exception__: "foo"}) == false + assert exception?([]) == false + assert exception?(%{}) == false + end - assert put_in(users["josé"][:age], 28) == - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + test "is_exception/1 and other match works" do + assert exception_or_map?(%RuntimeError{}) == true + assert exception_or_map?(%{}) == true + assert exception_or_map?(10) == false + end - assert put_in(users["dave"][:age], 19) == - %{"josé" => %{age: 27}, "eric" => %{age: 23}, "dave" => %{age: 19}} + defp exception?(arg, name) when is_exception(arg, name), do: true + defp exception?(_arg, _name), do: false + + defp exception_or_map?(arg, name) when is_exception(arg, name) or is_map(arg), do: true + defp exception_or_map?(_arg, _name), do: false + + test "is_exception/2" do + assert is_exception(%{}, RuntimeError) == false + assert is_exception([], RuntimeError) == false + assert is_exception(%RuntimeError{}, RuntimeError) == true + assert is_exception(%RuntimeError{}, Macro.Env) == false + assert exception?(%RuntimeError{}, RuntimeError) == true + assert exception?(%RuntimeError{}, Macro.Env) == false + assert exception?(%{__exception__: "foo"}, "foo") == false + assert exception?(%{__exception__: "foo"}, RuntimeError) == false + assert exception?([], RuntimeError) == false + assert exception?(%{}, RuntimeError) == false + + assert_raise ArgumentError, "argument error", fn -> + is_exception(%{}, not_atom()) + end + end + test "is_exception/2 and other match works" do + assert exception_or_map?(%{}, "foo") == false + assert exception_or_map?(%{}, RuntimeError) == true + assert exception_or_map?(%RuntimeError{}, RuntimeError) == true + end - assert put_in(users["josé"].age, 28) == - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + test "then/2" do + assert 1 |> then(fn x -> x * 2 end) == 2 - assert_raise ArgumentError, fn -> - put_in(users["dave"].age, 19) + assert_raise BadArityError, fn -> + 1 |> then(fn x, y -> x * y end) end + end - assert_raise KeyError, fn -> - put_in(users["eric"].unknown, "value") + test "if/2 boolean optimization does not leak variables during expansion" do + if false do + :ok + else + assert Macro.Env.vars(__ENV__) == [] + end + end + + describe ".." do + test "returns 0..-1//1" do + assert (..) == 0..-1//1 end end - test "update_in/3" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} + describe "in/2" do + test "too large list in guards" do + defmodule TooLargeList do + @list Enum.map(1..1024, & &1) + defguard is_value(value) when value in @list + end + end + + test "with literals on right side" do + assert 2 in [1, 2, 3] + assert 2 in 1..3 + refute 4 in [1, 2, 3] + refute 4 in 1..3 + refute 2 in [] + refute false in [] + refute true in [] + end + + test "with expressions on right side" do + list = [1, 2, 3] + empty_list = [] + assert 2 in list + refute 4 in list - assert update_in(nil, ["josé", :age], fn nil -> 28 end) == - %{"josé" => %{age: 28}} + refute 4 in empty_list + refute false in empty_list + refute true in empty_list - assert update_in(users, ["josé", :age], &(&1 + 1)) == - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + assert 2 in [1 | [2, 3]] + assert 3 in [1 | list] - assert update_in(users, ["dave", :age], fn nil -> 19 end) == - %{"josé" => %{age: 27}, "eric" => %{age: 23}, "dave" => %{age: 19}} + some_call = & &1 + refute :x in [1, 2 | some_call.([3, 4])] + assert :x in [1, 2 | some_call.([3, :x])] - assert_raise FunctionClauseError, fn -> - update_in(users, [], fn _ -> %{} end) + assert_raise ArgumentError, fn -> + :x in [1, 2 | some_call.({3, 4})] + end + end + + @at_list1 [4, 5] + @at_range 6..8 + @at_list2 [13, 14] + def fun_in(x) when x in [0], do: :list + def fun_in(x) when x in 1..3, do: :range + def fun_in(x) when x in @at_list1, do: :at_list + def fun_in(x) when x in @at_range, do: :at_range + def fun_in(x) when x in [9 | [10, 11]], do: :list_cons + def fun_in(x) when x in [12 | @at_list2], do: :list_cons_at + def fun_in(x) when x in 21..15//1, do: raise("oops positive") + def fun_in(x) when x in 15..21//-1, do: raise("oops negative") + def fun_in(x) when x in 15..21//2, do: :range_step_2 + def fun_in(x) when x in 15..21//1, do: :range_step_1 + def fun_in(_), do: :none + + test "in function guard" do + assert fun_in(0) == :list + assert fun_in(1) == :range + assert fun_in(2) == :range + assert fun_in(3) == :range + assert fun_in(5) == :at_list + assert fun_in(6) == :at_range + assert fun_in(7) == :at_range + assert fun_in(8) == :at_range + assert fun_in(9) == :list_cons + assert fun_in(10) == :list_cons + assert fun_in(11) == :list_cons + assert fun_in(12) == :list_cons_at + assert fun_in(13) == :list_cons_at + assert fun_in(14) == :list_cons_at + assert fun_in(15) == :range_step_2 + assert fun_in(16) == :range_step_1 + assert fun_in(17) == :range_step_2 + assert fun_in(22) == :none + + assert fun_in(0.0) == :none + assert fun_in(1.0) == :none + assert fun_in(2.0) == :none + assert fun_in(3.0) == :none + assert fun_in(6.0) == :none + assert fun_in(7.0) == :none + assert fun_in(8.0) == :none + assert fun_in(9.0) == :none + assert fun_in(10.0) == :none + assert fun_in(11.0) == :none + assert fun_in(12.0) == :none + assert fun_in(13.0) == :none + assert fun_in(14.0) == :none + assert fun_in(15.0) == :none + assert fun_in(16.0) == :none + assert fun_in(17.0) == :none + end + + def dynamic_in(x, y, z) when x in y..z, do: true + def dynamic_in(_x, _y, _z), do: false + + test "in dynamic range function guard" do + assert dynamic_in(1, 1, 3) + assert dynamic_in(2, 1, 3) + assert dynamic_in(3, 1, 3) + + assert dynamic_in(1, 3, 1) + assert dynamic_in(2, 3, 1) + assert dynamic_in(3, 3, 1) + + refute dynamic_in(0, 1, 3) + refute dynamic_in(4, 1, 3) + refute dynamic_in(0, 3, 1) + refute dynamic_in(4, 3, 1) + + refute dynamic_in(2, 1.0, 3) + refute dynamic_in(2, 1, 3.0) + refute dynamic_in(2.0, 1, 3) end - end - test "update_in/2" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} + def dynamic_step_in(x, y, z, w) when x in y..z//w, do: true + def dynamic_step_in(_x, _y, _z, _w), do: false - assert update_in(nil["josé"][:age], fn nil -> 28 end) == - %{"josé" => %{age: 28}} + test "in dynamic range with step function guard" do + assert dynamic_step_in(1, 1, 3, 1) + assert dynamic_step_in(2, 1, 3, 1) + assert dynamic_step_in(3, 1, 3, 1) - assert update_in(users["josé"][:age], &(&1 + 1)) == - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + refute dynamic_step_in(1, 1, 3, -1) + refute dynamic_step_in(2, 1, 3, -1) + refute dynamic_step_in(3, 1, 3, -1) - assert update_in(users["dave"][:age], fn nil -> 19 end) == - %{"josé" => %{age: 27}, "eric" => %{age: 23}, "dave" => %{age: 19}} + assert dynamic_step_in(1, 3, 1, -1) + assert dynamic_step_in(2, 3, 1, -1) + assert dynamic_step_in(3, 3, 1, -1) - assert update_in(users["josé"].age, &(&1 + 1)) == - %{"josé" => %{age: 28}, "eric" => %{age: 23}} + refute dynamic_step_in(1, 3, 1, 1) + refute dynamic_step_in(2, 3, 1, 1) + refute dynamic_step_in(3, 3, 1, 1) - assert_raise ArgumentError, fn -> - update_in(users["dave"].age, &(&1 + 1)) + assert dynamic_step_in(1, 1, 3, 2) + refute dynamic_step_in(2, 1, 3, 2) + assert dynamic_step_in(3, 1, 3, 2) + assert dynamic_step_in(3, 1, 4, 2) + refute dynamic_step_in(4, 1, 4, 2) end - assert_raise KeyError, fn -> - put_in(users["eric"].unknown, &(&1 + 1)) + defmacrop case_in(x, y) do + quote do + case 0 do + _ when unquote(x) in unquote(y) -> true + _ -> false + end + end + end + + test "in case guard" do + assert case_in(1, [1, 2, 3]) == true + assert case_in(1, 1..3) == true + assert case_in(2, 1..3) == true + assert case_in(3, 1..3) == true + assert case_in(-3, -1..-3) == true end - end - test "get_and_update_in/3" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} + def map_dot(map) when map.field, do: true + def map_dot(_other), do: false - assert get_and_update_in(nil, ["josé", :age], fn nil -> {:ok, 28} end) == - {:ok, %{"josé" => %{age: 28}}} + test "map dot guard" do + refute map_dot(:foo) + refute map_dot(%{}) + refute map_dot(%{field: false}) + assert map_dot(%{field: true}) - assert get_and_update_in(users, ["josé", :age], &{&1, &1 + 1}) == - {27, %{"josé" => %{age: 28}, "eric" => %{age: 23}}} + message = + "cannot invoke remote function in guards. " <> + "If you want to do a map lookup instead, please remove parens from map.field()" - assert_raise FunctionClauseError, fn -> - update_in(users, [], fn _ -> %{} end) + assert_raise CompileError, Regex.compile!(message), fn -> + defmodule MapDot do + def map_dot(map) when map.field(), do: true + end + end + + message = ~r"cannot invoke remote function Module.fun/0 inside guards" + + assert_raise CompileError, message, fn -> + defmodule MapDot do + def map_dot(map) when Module.fun(), do: true + end + end end - end - test "get_and_update_in/2" do - users = %{"josé" => %{age: 27}, "eric" => %{age: 23}} + test "performs all side-effects" do + assert 1 in [1, send(self(), 2)] + assert_received 2 - assert get_and_update_in(nil["josé"][:age], fn nil -> {:ok, 28} end) == - {:ok, %{"josé" => %{age: 28}}} + assert 1 in [1 | send(self(), [2])] + assert_received [2] - assert get_and_update_in(users["josé"].age, &{&1, &1 + 1}) == - {27, %{"josé" => %{age: 28}, "eric" => %{age: 23}}} + assert 2 in [1 | send(self(), [2])] + assert_received [2] + end - assert_raise ArgumentError, fn -> - get_and_update_in(users["dave"].age, &{&1, &1 + 1}) + test "has proper evaluation order" do + a = 1 + assert 1 in [a = 2, a] + # silence unused var warning + _ = a end - assert_raise KeyError, fn -> - get_and_update_in(users["eric"].unknown, &{&1, &1 + 1}) + test "in module body" do + defmodule InSample do + @foo [:a, :b] + true = :a in @foo + end + after + purge(InSample) end - end - test "paths" do - map = empty_map() + test "inside and/2" do + response = %{code: 200} - assert put_in(map[:foo], "bar") == %{foo: "bar"} - assert put_in(empty_map()[:foo], "bar") == %{foo: "bar"} - assert put_in(KernelTest.empty_map()[:foo], "bar") == %{foo: "bar"} - assert put_in(__MODULE__.empty_map()[:foo], "bar") == %{foo: "bar"} + if is_map(response) and response.code in 200..299 do + :pass + end - assert_raise ArgumentError, ~r"access at least one field,", fn -> - Code.eval_quoted(quote(do: put_in(map, "bar")), []) + # This module definition copies internal variable + # defined during in/2 expansion. + Module.create(InVarCopy, nil, __ENV__) + purge(InVarCopy) end - assert_raise ArgumentError, ~r"must start with a variable, local or remote call", fn -> - Code.eval_quoted(quote(do: put_in(map.foo(1, 2)[:bar], "baz")), []) + test "with a non-literal non-escaped compile-time range in guards" do + message = "non-literal range in guard should be escaped with Macro.escape/2" + + assert_eval_raise(ArgumentError, message, """ + defmodule InErrors do + range = 1..3 + def foo(x) when x in unquote(range), do: :ok + end + """) end - end - def empty_map, do: %{} + test "with a non-compile-time range in guards" do + message = ~r/invalid right argument for operator "in", .* got: :hello/ - defmodule PipelineOp do - use ExUnit.Case, async: true + assert_eval_raise(ArgumentError, message, """ + defmodule InErrors do + def foo(x) when x in :hello, do: :ok + end + """) + end - test "simple" do - assert [1, [2], 3] |> List.flatten == [1, 2, 3] + test "with a non-compile-time list cons in guards" do + message = ~r/invalid right argument for operator "in", .* got: \[1 | list\(\)\]/ + + assert_eval_raise(ArgumentError, message, """ + defmodule InErrors do + def list, do: [1] + def foo(x) when x in [1 | list()], do: :ok + end + """) end - test "nested pipelines" do - assert [1, [2], 3] |> List.flatten |> Enum.map(&(&1 * 2)) == [2, 4, 6] + test "with a compile-time non-list in tail in guards" do + message = ~r/invalid right argument for operator "in", .* got: \[1 | 1..3\]/ + + assert_eval_raise(ArgumentError, message, """ + defmodule InErrors do + def foo(x) when x in [1 | 1..3], do: :ok + end + """) end - test "local call" do - assert [1, [2], 3] |> List.flatten |> local == [2, 4, 6] + test "with a non-integer range" do + message = "ranges (first..last) expect both sides to be integers, got: 0..5.0" + + assert_raise ArgumentError, message, fn -> + last = 5.0 + 1 in 0..last + end end - test "pipeline with capture" do - assert Enum.map([1, 2, 3], &(&1 |> twice |> twice)) == [4, 8, 12] + test "hoists variables and keeps order" do + # Ranges + result = expand_to_string(quote(do: rand() in 1..2)) + assert result =~ "var = rand()" + + assert result =~ """ + :erlang.andalso( + :erlang.is_integer(var), + :erlang.andalso(:erlang.>=(var, 1), :erlang.\"=<\"(var, 2)) + )\ + """ + + # Empty list + assert expand_to_string(quote(do: :x in [])) =~ "_ = :x\nfalse" + assert expand_to_string(quote(do: :x in []), :guard) == "false" + + # Lists + result = expand_to_string(quote(do: rand() in [1, 2])) + assert result =~ "var = rand()" + assert result =~ ":erlang.orelse(:erlang.\"=:=\"(var, 1), :erlang.\"=:=\"(var, 2))" + + result = expand_to_string(quote(do: rand() in [1 | [2]])) + assert result =~ ":lists.member(rand(), [1 | [2]]" + + result = expand_to_string(quote(do: rand() in [1 | some_call()])) + assert result =~ ":lists.member(rand(), [1 | some_call()]" end - test "non-call" do - assert 1 |> (&(&1*2)).() == 2 - assert [1] |> (&hd(&1)).() == 1 + defp quote_case_in(left, right) do + quote do + case 0 do + _ when unquote(left) in unquote(right) -> true + end + end + end - import CompileAssertion - assert_compile_fail ArgumentError, "cannot pipe 1 into 2", "1 |> 2" + test "is optimized" do + assert expand_to_string(quote(do: foo in [])) == + "_ = foo\nfalse" + + assert expand_to_string(quote(do: foo in [1, 2, 3])) == """ + :erlang.orelse( + :erlang.orelse(:erlang.\"=:=\"(foo, 1), :erlang.\"=:=\"(foo, 2)), + :erlang.\"=:=\"(foo, 3) + )\ + """ + + assert expand_to_string(quote(do: foo in 0..1)) == """ + :erlang.andalso( + :erlang.is_integer(foo), + :erlang.andalso(:erlang.>=(foo, 0), :erlang.\"=<\"(foo, 1)) + )\ + """ + + assert expand_to_string(quote(do: foo in -1..0)) == """ + :erlang.andalso( + :erlang.is_integer(foo), + :erlang.andalso(:erlang.>=(foo, -1), :erlang.\"=<\"(foo, 0)) + )\ + """ + + assert expand_to_string(quote(do: foo in 1..1)) == + ":erlang.\"=:=\"(foo, 1)" end - defp twice(a), do: a * 2 + defp expand_to_string(ast, environment_or_context \\ __ENV__) - defp local(list) do - Enum.map(list, &(&1 * 2)) + defp expand_to_string(ast, context) when is_atom(context) do + expand_to_string(ast, %{__ENV__ | context: context}) + end + + defp expand_to_string(ast, environment) do + ast + |> Macro.prewalk(&Macro.expand(&1, environment)) + |> Macro.to_string() + end + end + + describe "__info__" do + test ":macros" do + assert {:in, 2} in Kernel.__info__(:macros) + end + + test ":functions" do + refute {:__info__, 1} in Kernel.__info__(:functions) + end + + test ":struct" do + assert Kernel.__info__(:struct) == nil + assert hd(URI.__info__(:struct)) == %{field: :scheme, required: false} + end + + test "others" do + assert Kernel.__info__(:module) == Kernel + assert is_list(Kernel.__info__(:compile)) + assert is_list(Kernel.__info__(:attributes)) end end - defmodule IfScope do - use ExUnit.Case, async: true + describe "@" do + test "setting attribute with do-block" do + exception = + catch_error( + defmodule UpcaseAttrSample do + @foo quote do + :ok + end + end + ) + + assert exception.message =~ "expected 0 or 1 argument for @foo, got 2" + assert exception.message =~ "You probably want to wrap the argument value in parentheses" + end + + test "setting attribute with uppercase" do + message = "module attributes set via @ cannot start with an uppercase letter" - test "variables on nested if" do - if true do - a = 1 - if true do - b = 2 + assert_raise ArgumentError, message, fn -> + defmodule UpcaseAttrSample do + @Upper end end + end - assert a == 1 - assert b == 2 + test "matching attribute" do + assert_raise ArgumentError, ~r"invalid write attribute syntax", fn -> + defmodule MatchAttributeInModule do + @foo = 42 + end + end end + end - test "variables on sibling if" do - if true do - a = 1 + describe "defdelegate" do + defdelegate my_flatten(list), to: List, as: :flatten + + dynamic = :dynamic_flatten + defdelegate unquote(dynamic)(list), to: List, as: :flatten + + test "dispatches to delegated functions" do + assert my_flatten([[1]]) == [1] + end - if true do - b = 2 + test "with unquote" do + assert dynamic_flatten([[1]]) == [1] + end + + test "raises with non-variable arguments" do + assert_raise ArgumentError, + "guards are not allowed in defdelegate/2, got: when is_list(term) or is_binary(term)", + fn -> + string = """ + defmodule IntDelegateWithGuards do + defdelegate foo(term) when is_list(term) or is_binary(term), to: List + end + """ + + Code.eval_string(string, [], __ENV__) + end + + msg = "defdelegate/2 only accepts function parameters, got: 1" + + assert_raise ArgumentError, msg, fn -> + string = """ + defmodule IntDelegate do + defdelegate foo(1), to: List end + """ + + Code.eval_string(string, [], __ENV__) + end - if true do - c = 3 + assert_raise ArgumentError, msg, fn -> + string = """ + defmodule IntOptionDelegate do + defdelegate foo(1 \\\\ 1), to: List end + """ + + Code.eval_string(string, [], __ENV__) end + end - assert a == 1 - assert b == 2 - assert c == 3 + test "raises when :to targeting the delegating module is given without the :as option" do + assert_raise ArgumentError, + ~r/defdelegate function is calling itself, which will lead to an infinite loop. You should either change the value of the :to option or specify the :as option/, + fn -> + defmodule ImplAttributes do + defdelegate foo(), to: __MODULE__ + end + end + end + + defdelegate my_reverse(list \\ []), to: :lists, as: :reverse + defdelegate my_get(map \\ %{}, key, default \\ ""), to: Map, as: :get + + test "accepts variable with optional arguments" do + assert my_reverse() == [] + assert my_reverse([1, 2, 3]) == [3, 2, 1] + + assert my_get("foo") == "" + assert my_get(%{}, "foo") == "" + assert my_get(%{"foo" => "bar"}, "foo") == "bar" + assert my_get(%{}, "foo", "not_found") == "not_found" + end + end + + describe "defmodule" do + test "expects atoms as module names" do + msg = ~r"invalid module name: 3" + + assert_raise CompileError, msg, fn -> + defmodule 1 + 2, do: :ok + end + end + + test "does not accept special atoms as module names" do + special_atoms = [nil, true, false] + + Enum.each(special_atoms, fn special_atom -> + msg = ~r"invalid module name: #{inspect(special_atom)}" + + assert_raise CompileError, msg, fn -> + defmodule special_atom, do: :ok + end + end) + end + + test "does not accept slashes in module names" do + assert_raise CompileError, ~r(invalid module name: :"foo/bar"), fn -> + defmodule :"foo/bar", do: :ok + end + + assert_raise CompileError, ~r(invalid module name: :"foo\\\\bar"), fn -> + defmodule :"foo\\bar", do: :ok + end + end + end + + describe "access" do + defmodule StructAccess do + defstruct [:foo, :bar] + end + + test "get_in/2" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + assert get_in(users, ["john", :age]) == 27 + assert get_in(users, ["dave", :age]) == nil + assert get_in(nil, ["john", :age]) == nil + + map = %{"fruits" => ["banana", "apple", "orange"]} + assert get_in(map, ["fruits", by_index(0)]) == "banana" + assert get_in(map, ["fruits", by_index(3)]) == nil + assert get_in(map, ["unknown", by_index(3)]) == nil + + assert_raise FunctionClauseError, fn -> + get_in(users, []) + end + end + + test "put_in/3" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert put_in(users, ["john", :age], 28) == %{"john" => %{age: 28}, "meg" => %{age: 23}} + + assert_raise FunctionClauseError, fn -> + put_in(users, [], %{}) + end + + assert_raise ArgumentError, "could not put/update key \"john\" on a nil value", fn -> + put_in(nil, ["john", :age], 28) + end + end + + test "put_in/2" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert put_in(users["john"][:age], 28) == %{"john" => %{age: 28}, "meg" => %{age: 23}} + + assert put_in(users["john"].age, 28) == %{"john" => %{age: 28}, "meg" => %{age: 23}} + + struct = %StructAccess{foo: %StructAccess{}} + + assert put_in(struct.foo.bar, :baz) == + %StructAccess{bar: nil, foo: %StructAccess{bar: :baz, foo: nil}} + + assert_raise BadMapError, fn -> + put_in(users["dave"].age, 19) + end + + assert_raise KeyError, fn -> + put_in(users["meg"].unknown, "value") + end + end + + test "update_in/3" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert update_in(users, ["john", :age], &(&1 + 1)) == + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + assert_raise FunctionClauseError, fn -> + update_in(users, [], fn _ -> %{} end) + end + + assert_raise ArgumentError, "could not put/update key \"john\" on a nil value", fn -> + update_in(nil, ["john", :age], fn _ -> %{} end) + end + + assert_raise UndefinedFunctionError, fn -> + pop_in(struct(Sample, []), [:name]) + end + end + + test "update_in/2" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert update_in(users["john"][:age], &(&1 + 1)) == + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + assert update_in(users["john"].age, &(&1 + 1)) == + %{"john" => %{age: 28}, "meg" => %{age: 23}} + + struct = %StructAccess{foo: %StructAccess{bar: 41}} + + assert update_in(struct.foo.bar, &(&1 + 1)) == + %StructAccess{bar: nil, foo: %StructAccess{bar: 42, foo: nil}} + + assert_raise BadMapError, fn -> + update_in(users["dave"].age, &(&1 + 1)) + end + + assert_raise KeyError, fn -> + put_in(users["meg"].unknown, &(&1 + 1)) + end + end + + test "get_and_update_in/3" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert get_and_update_in(users, ["john", :age], &{&1, &1 + 1}) == + {27, %{"john" => %{age: 28}, "meg" => %{age: 23}}} + + map = %{"fruits" => ["banana", "apple", "orange"]} + + assert get_and_update_in(map, ["fruits", by_index(0)], &{&1, String.reverse(&1)}) == + {"banana", %{"fruits" => ["ananab", "apple", "orange"]}} + + assert get_and_update_in(map, ["fruits", by_index(3)], &{&1, &1}) == + {nil, %{"fruits" => ["banana", "apple", "orange"]}} + + assert get_and_update_in(map, ["unknown", by_index(3)], &{&1, []}) == + {:oops, %{"fruits" => ["banana", "apple", "orange"], "unknown" => []}} + + assert_raise FunctionClauseError, fn -> + update_in(users, [], fn _ -> %{} end) + end + end + + test "get_and_update_in/2" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert get_and_update_in(users["john"].age, &{&1, &1 + 1}) == + {27, %{"john" => %{age: 28}, "meg" => %{age: 23}}} + + struct = %StructAccess{foo: %StructAccess{bar: 41}} + + assert get_and_update_in(struct.foo.bar, &{&1, &1 + 1}) == + {41, %StructAccess{bar: nil, foo: %StructAccess{bar: 42, foo: nil}}} + + assert_raise ArgumentError, "could not put/update key \"john\" on a nil value", fn -> + get_and_update_in(nil["john"][:age], fn nil -> {:ok, 28} end) + end + + assert_raise BadMapError, fn -> + get_and_update_in(users["dave"].age, &{&1, &1 + 1}) + end + + assert_raise KeyError, fn -> + get_and_update_in(users["meg"].unknown, &{&1, &1 + 1}) + end + end + + test "pop_in/2" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert pop_in(users, ["john", :age]) == {27, %{"john" => %{}, "meg" => %{age: 23}}} + + assert pop_in(users, ["bob", :age]) == {nil, %{"john" => %{age: 27}, "meg" => %{age: 23}}} + + assert pop_in([], [:foo, :bar]) == {nil, []} + + assert_raise FunctionClauseError, fn -> + pop_in(users, []) + end + + assert_raise FunctionClauseError, "no function clause matching in Kernel.pop_in/2", fn -> + pop_in(users, :not_a_list) + end end - test "variables counter on nested ifs" do - r = (fn() -> 3 end).() # supresses warning at (if r < 0...) - r = r - 1 - r = r - 1 - r = r - 1 + test "pop_in/2 with paths" do + map = %{"fruits" => ["banana", "apple", "orange"]} + + assert pop_in(map, ["fruits", by_index(0)]) == + {"banana", %{"fruits" => ["apple", "orange"]}} + + assert pop_in(map, ["fruits", by_index(3)]) == {nil, map} - if true do - r = r - 1 - if r < 0, do: r = 0 + map = %{"fruits" => [%{name: "banana"}, %{name: "apple"}]} + + assert pop_in(map, ["fruits", by_index(0), :name]) == + {"banana", %{"fruits" => [%{}, %{name: "apple"}]}} + + assert pop_in(map, ["fruits", by_index(3), :name]) == {nil, map} + end + + test "pop_in/1" do + users = %{"john" => %{age: 27}, "meg" => %{age: 23}} + + assert pop_in(users["john"][:age]) == {27, %{"john" => %{}, "meg" => %{age: 23}}} + assert pop_in(users["john"][:name]) == {nil, %{"john" => %{age: 27}, "meg" => %{age: 23}}} + assert pop_in(users["bob"][:age]) == {nil, %{"john" => %{age: 27}, "meg" => %{age: 23}}} + + users = %{john: [age: 27], meg: [age: 23]} + + assert pop_in(users.john[:age]) == {27, %{john: [], meg: [age: 23]}} + assert pop_in(users.john[:name]) == {nil, %{john: [age: 27], meg: [age: 23]}} + + assert pop_in([][:foo][:bar]) == {nil, []} + assert_raise KeyError, fn -> pop_in(users.bob[:age]) end + end + + test "pop_in/1,2 with nils" do + users = %{"john" => nil, "meg" => %{age: 23}} + assert pop_in(users["john"][:age]) == {nil, %{"meg" => %{age: 23}}} + assert pop_in(users, ["john", :age]) == {nil, %{"meg" => %{age: 23}}} + + users = %{john: nil, meg: %{age: 23}} + assert pop_in(users.john[:age]) == {nil, %{john: nil, meg: %{age: 23}}} + assert pop_in(users, [:john, :age]) == {nil, %{meg: %{age: 23}}} + + x = nil + assert_raise ArgumentError, fn -> pop_in(x["john"][:age]) end + assert_raise ArgumentError, fn -> pop_in(nil["john"][:age]) end + assert_raise ArgumentError, fn -> pop_in(nil, ["john", :age]) end + end + + test "with dynamic paths" do + map = empty_map() + + assert put_in(map[:foo], "bar") == %{foo: "bar"} + assert put_in(empty_map()[:foo], "bar") == %{foo: "bar"} + assert put_in(KernelTest.empty_map()[:foo], "bar") == %{foo: "bar"} + assert put_in(__MODULE__.empty_map()[:foo], "bar") == %{foo: "bar"} + + assert_raise ArgumentError, ~r"access at least one element,", fn -> + Code.eval_quoted(quote(do: put_in(map, "bar")), []) + end + + assert_raise ArgumentError, ~r"must start with a variable, local or remote call", fn -> + Code.eval_quoted(quote(do: put_in(map.foo(1, 2)[:bar], "baz")), []) end + end + + def by_index(index) do + fn + :get, nil, _next -> + raise "won't be invoked" + + :get, data, next -> + next.(Enum.at(data, index)) - assert r == 0 + :get_and_update, nil, next -> + next.(:oops) + + :get_and_update, data, next -> + current = Enum.at(data, index) + + case next.(current) do + {get, update} -> {get, List.replace_at(data, index, update)} + :pop -> {current, List.delete_at(data, index)} + end + end end end - defmodule Destructure do - use ExUnit.Case, async: true + describe "pipeline" do + test "simple" do + assert [1, [2], 3] |> List.flatten() == [1, 2, 3] + end + + test "nested" do + assert [1, [2], 3] |> List.flatten() |> Enum.map(&(&1 * 2)) == [2, 4, 6] + end + + test "local call" do + assert [1, [2], 3] |> List.flatten() |> local == [2, 4, 6] + end + + test "with capture" do + assert Enum.map([1, 2, 3], &(&1 |> twice |> twice)) == [4, 8, 12] + end + test "with anonymous functions" do + assert 1 |> (&(&1 * 2)).() == 2 + assert [1] |> (&hd(&1)).() == 1 + end + + defp twice(a), do: a * 2 + + defp local(list) do + Enum.map(list, &(&1 * 2)) + end + end + + describe "destructure" do test "less args" do destructure [x, y, z], [1, 2, 3, 4, 5] assert x == 1 @@ -451,7 +1277,7 @@ defmodule KernelTest do end test "nil values" do - destructure [a, b, c], a_nil + destructure [a, b, c], a_nil() assert a == nil assert b == nil assert c == nil @@ -459,12 +1285,175 @@ defmodule KernelTest do test "invalid match" do a = List.first([3]) - assert_raise CaseClauseError, fn -> - destructure [^a, _b, _c], a_list + + assert_raise MatchError, fn -> + destructure [^a, _b, _c], a_list() end end defp a_list, do: [1, 2, 3] defp a_nil, do: nil end + + describe "use/2" do + import ExUnit.CaptureIO + + defmodule SampleA do + defmacro __using__(opts) do + prefix = Keyword.get(opts, :prefix, "") + IO.puts(prefix <> "A") + end + end + + defmodule SampleB do + defmacro __using__(_) do + IO.puts("B") + end + end + + test "invalid argument is literal" do + message = "invalid arguments for use, expected a compile time atom or alias, got: 42" + + assert_raise ArgumentError, message, fn -> + Code.eval_string("use 42") + end + end + + test "invalid argument is variable" do + message = "invalid arguments for use, expected a compile time atom or alias, got: variable" + + assert_raise ArgumentError, message, fn -> + Code.eval_string("use variable") + end + end + + test "multi-call" do + assert capture_io(fn -> + Code.eval_string("use KernelTest.{SampleA, SampleB,}", [], __ENV__) + end) == "A\nB\n" + end + + test "multi-call with options" do + assert capture_io(fn -> + Code.eval_string(~S|use KernelTest.{SampleA}, prefix: "-"|, [], __ENV__) + end) == "-A\n" + end + + test "multi-call with unquote" do + assert capture_io(fn -> + string = """ + defmodule TestMod do + def main() do + use KernelTest.{SampleB, unquote(:SampleA)} + end + end + """ + + Code.eval_string(string, [], __ENV__) + end) == "B\nA\n" + after + purge(KernelTest.TestMod) + end + end + + test "is_map_key/2" do + assert is_map_key(Map.new([]), :a) == false + assert is_map_key(Map.new(b: 1), :a) == false + assert is_map_key(Map.new(a: 1), :a) == true + + assert_raise BadMapError, fn -> + is_map_key(empty_list(), :a) + end + + case Map.new(a: 1) do + map when is_map_key(map, :a) -> true + _ -> flunk("invalid guard") + end + end + + test "tap/1" do + import ExUnit.CaptureIO + + assert capture_io(fn -> + tap("foo", &IO.puts/1) + end) == "foo\n" + + assert 1 = tap(1, fn x -> x + 1 end) + end + + test "tl/1" do + assert tl([:one]) == [] + assert tl([1, 2, 3]) == [2, 3] + assert_raise ArgumentError, fn -> tl(empty_list()) end + + assert tl([:a | :b]) == :b + assert tl([:a, :b | :c]) == [:b | :c] + end + + test "hd/1" do + assert hd([1, 2, 3, 4]) == 1 + assert_raise ArgumentError, fn -> hd(empty_list()) end + assert hd([1 | 2]) == 1 + end + + test "floor/1" do + assert floor(1) === 1 + assert floor(1.0) === 1 + assert floor(0) === 0 + assert floor(0.0) === 0 + assert floor(-0.0) === 0 + assert floor(1.123) === 1 + assert floor(-10.123) === -11 + assert floor(-10) === -10 + assert floor(-10.0) === -10 + + assert match?(x when floor(x) == 0, 0.2) + end + + test "ceil/1" do + assert ceil(1) === 1 + assert ceil(1.0) === 1 + assert ceil(0) === 0 + assert ceil(0.0) === 0 + assert ceil(-0.0) === 0 + assert ceil(1.123) === 2 + assert ceil(-10.123) === -10 + assert ceil(-10) === -10 + assert ceil(-10.0) === -10 + + assert match?(x when ceil(x) == 1, 0.2) + end + + test "sigil_U/2" do + assert ~U[2015-01-13 13:00:07.123Z] == %DateTime{ + calendar: Calendar.ISO, + day: 13, + hour: 13, + microsecond: {123_000, 3}, + minute: 0, + month: 1, + second: 7, + std_offset: 0, + time_zone: "Etc/UTC", + utc_offset: 0, + year: 2015, + zone_abbr: "UTC" + } + + assert_raise ArgumentError, ~r"reason: :invalid_format", fn -> + Code.eval_string(~s{~U[2015-01-13 13:00]}) + end + + assert_raise ArgumentError, ~r"reason: :invalid_format", fn -> + Code.eval_string(~s{~U[20150113 130007Z]}) + end + + assert_raise ArgumentError, ~r"reason: :missing_offset", fn -> + Code.eval_string(~s{~U[2015-01-13 13:00:07]}) + end + + assert_raise ArgumentError, ~r"reason: :non_utc_offset", fn -> + Code.eval_string(~s{~U[2015-01-13 13:00:07+00:30]}) + end + end end diff --git a/lib/elixir/test/elixir/keyword_test.exs b/lib/elixir/test/elixir/keyword_test.exs index 4387b6026e3..3e8f30986d7 100644 --- a/lib/elixir/test/elixir/keyword_test.exs +++ b/lib/elixir/test/elixir/keyword_test.exs @@ -1,8 +1,10 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule KeywordTest do use ExUnit.Case, async: true + doctest Keyword + test "has a literal syntax" do assert [B: 1] == [{:B, 1}] assert [foo?: :bar] == [{:foo?, :bar}] @@ -11,238 +13,206 @@ defmodule KeywordTest do end test "is a :: operator on ambiguity" do - assert [{:::, _, [{:a, _, _}, {:b, _, _}]}] = quote(do: [a::b]) + assert [{:"::", _, [{:a, _, _}, {:b, _, _}]}] = quote(do: [a :: b]) end test "supports optional comma" do - [a: 1, - b: 2, - c: 3, ] - end - - test "keyword?/1" do - assert Keyword.keyword?([]) - assert Keyword.keyword?([a: 1]) - assert Keyword.keyword?([{Foo, 1}]) - refute Keyword.keyword?([{}]) - refute Keyword.keyword?(<<>>) + assert Code.eval_string("[a: 1, b: 2, c: 3,]") == {[a: 1, b: 2, c: 3], []} end - test "new/0" do - assert Keyword.new == [] + test "implements (almost) all functions in Map" do + assert Map.__info__(:functions) -- Keyword.__info__(:functions) == [from_struct: 1] end - test "new/1" do - assert Keyword.new([{:second_key, 2}, {:first_key, 1}]) == - [first_key: 1, second_key: 2] - end + test "get_and_update/3 raises on bad return value from the argument function" do + message = "the given function must return a two-element tuple or :pop, got: 1" - test "new/2" do - assert Keyword.new([:a, :b], fn x -> {x, x} end) == - [b: :b, a: :a] - end - - test "get/2 and get/3" do - assert Keyword.get(create_keywords, :first_key) == 1 - assert Keyword.get(create_keywords, :second_key) == 2 - assert Keyword.get(create_keywords, :other_key) == nil - assert Keyword.get(create_empty_keywords, :first_key, "default") == "default" - end - - test "fetch!/2" do - assert Keyword.fetch!(create_keywords, :first_key) == 1 - - error = assert_raise KeyError, fn -> - Keyword.fetch!(create_keywords, :unknown) + assert_raise RuntimeError, message, fn -> + Keyword.get_and_update([a: 1], :a, fn value -> value end) end - assert error.key == :unknown - end - - test "keys/1" do - assert Keyword.keys(create_keywords) == [:first_key, :second_key] - assert Keyword.keys(create_empty_keywords) == [] + message = "the given function must return a two-element tuple or :pop, got: nil" - assert_raise FunctionClauseError, fn -> - Keyword.keys([:foo]) + assert_raise RuntimeError, message, fn -> + Keyword.get_and_update([], :a, fn value -> value end) end end - test "values/1" do - assert Keyword.values(create_keywords) == [1, 2] - assert Keyword.values(create_empty_keywords) == [] + test "get_and_update!/3 raises on bad return value from the argument function" do + message = "the given function must return a two-element tuple or :pop, got: 1" - assert_raise FunctionClauseError, fn -> - Keyword.values([:foo]) + assert_raise RuntimeError, message, fn -> + Keyword.get_and_update!([a: 1], :a, fn value -> value end) end end - test "delete/2" do - assert Keyword.delete(create_keywords, :second_key) == [first_key: 1] - assert Keyword.delete(create_keywords, :other_key) == [first_key: 1, second_key: 2] - assert Keyword.delete(create_empty_keywords, :other_key) == [] + test "update!" do + assert Keyword.update!([a: 1, b: 2, a: 3], :a, &(&1 * 2)) == [a: 2, b: 2] + assert Keyword.update!([a: 1, b: 2, c: 3], :b, &(&1 * 2)) == [a: 1, b: 4, c: 3] + end - assert_raise FunctionClauseError, fn -> - Keyword.delete([:foo], :foo) - end + test "replace" do + assert Keyword.replace([a: 1, b: 2, a: 3], :a, :new) == [a: :new, b: 2] + assert Keyword.replace([a: 1, b: 2, a: 3], :a, 1) == [a: 1, b: 2] + assert Keyword.replace([a: 1, b: 2, a: 3, b: 4], :a, 1) == [a: 1, b: 2, b: 4] + assert Keyword.replace([a: 1, b: 2, c: 3, b: 4], :b, :new) == [a: 1, b: :new, c: 3] + assert Keyword.replace([], :b, :new) == [] + assert Keyword.replace([a: 1, b: 2, a: 3], :c, :new) == [a: 1, b: 2, a: 3] end - test "delete/3" do - keywords = [a: 1, b: 2, c: 3, a: 2] - assert Keyword.delete(keywords, :a, 2) == [a: 1, b: 2, c: 3] - assert Keyword.delete(keywords, :a, 1) == [b: 2, c: 3, a: 2] + test "replace!" do + assert Keyword.replace!([a: 1, b: 2, a: 3], :a, :new) == [a: :new, b: 2] + assert Keyword.replace!([a: 1, b: 2, a: 3], :a, 1) == [a: 1, b: 2] + assert Keyword.replace!([a: 1, b: 2, a: 3, b: 4], :a, 1) == [a: 1, b: 2, b: 4] + assert Keyword.replace!([a: 1, b: 2, c: 3, b: 4], :b, :new) == [a: 1, b: :new, c: 3] - assert_raise FunctionClauseError, fn -> - Keyword.delete([:foo], :foo, 0) + assert_raise KeyError, "key :b not found in: []", fn -> + Keyword.replace!([], :b, :new) end - end - - test "put/3" do - assert Keyword.put(create_empty_keywords, :first_key, 1) == [first_key: 1] - assert Keyword.put(create_keywords, :first_key, 3) == [first_key: 3, second_key: 2] - end - test "put_new/3" do - assert Keyword.put_new(create_empty_keywords, :first_key, 1) == [first_key: 1] - assert Keyword.put_new(create_keywords, :first_key, 3) == [first_key: 1, second_key: 2] + assert_raise KeyError, "key :c not found in: [a: 1, b: 2, a: 3]", fn -> + Keyword.replace!([a: 1, b: 2, a: 3], :c, :new) + end end test "merge/2" do - assert Keyword.merge(create_empty_keywords, create_keywords) == [first_key: 1, second_key: 2] - assert Keyword.merge(create_keywords, create_empty_keywords) == [first_key: 1, second_key: 2] - assert Keyword.merge(create_keywords, create_keywords) == [first_key: 1, second_key: 2] - assert Keyword.merge(create_empty_keywords, create_empty_keywords) == [] + assert Keyword.merge([a: 1, b: 2], c: 11, d: 12) == [a: 1, b: 2, c: 11, d: 12] + assert Keyword.merge([], c: 11, d: 12) == [c: 11, d: 12] + assert Keyword.merge([a: 1, b: 2], []) == [a: 1, b: 2] + + message = "expected a keyword list as the first argument, got: [1, 2]" - assert_raise FunctionClauseError, fn -> - Keyword.delete([:foo], [:foo]) + assert_raise ArgumentError, message, fn -> + Keyword.merge([1, 2], c: 11, d: 12) end - end - test "merge/3" do - result = Keyword.merge [a: 1, b: 2], [a: 3, d: 4], fn _k, v1, v2 -> - v1 + v2 + message = "expected a keyword list as the first argument, got: [1 | 2]" + + assert_raise ArgumentError, message, fn -> + Keyword.merge([1 | 2], c: 11, d: 12) end - assert result == [a: 4, b: 2, d: 4] - end - test "has_key?/2" do - assert Keyword.has_key?([a: 1], :a) == true - assert Keyword.has_key?([a: 1], :b) == false - end + message = "expected a keyword list as the second argument, got: [11, 12, 0]" - test "update!/3" do - kw = [a: 1, b: 2, a: 3] - assert Keyword.update!(kw, :a, &(&1 * 2)) == [a: 2, b: 2] - assert_raise KeyError, fn -> - Keyword.update!([a: 1], :b, &(&1 * 2)) + assert_raise ArgumentError, message, fn -> + Keyword.merge([a: 1, b: 2], [11, 12, 0]) end - end - test "update/4" do - kw = [a: 1, b: 2, a: 3] - assert Keyword.update(kw, :a, 13, &(&1 * 2)) == [a: 2, b: 2] - assert Keyword.update([a: 1], :b, 11, &(&1 * 2)) == [a: 1, b: 11] - end + message = "expected a keyword list as the second argument, got: [11 | 12]" - defp create_empty_keywords, do: [] - defp create_keywords, do: [first_key: 1, second_key: 2] -end + assert_raise ArgumentError, message, fn -> + Keyword.merge([a: 1, b: 2], [11 | 12]) + end -defmodule Keyword.DuplicatedTest do - use ExUnit.Case, async: true + # duplicate keys in keywords1 are kept if key is not present in keywords2 + result = [a: 1, b: 2, a: 3, c: 11, d: 12] + assert Keyword.merge([a: 1, b: 2, a: 3], c: 11, d: 12) == result - test "get/2" do - assert Keyword.get(create_keywords, :first_key) == 1 - assert Keyword.get(create_keywords, :second_key) == 2 - assert Keyword.get(create_keywords, :other_key) == nil - assert Keyword.get(create_empty_keywords, :first_key, "default") == "default" - end + result = [b: 2, a: 11] + assert Keyword.merge([a: 1, b: 2, a: 3], a: 11) == result - test "get_values/2" do - assert Keyword.get_values(create_keywords, :first_key) == [1, 2] - assert Keyword.get_values(create_keywords, :second_key) == [2] - assert Keyword.get_values(create_keywords, :other_key) == [] + # duplicate keys in keywords2 are always kept + result = [a: 1, b: 2, c: 11, c: 12, d: 13] + assert Keyword.merge([a: 1, b: 2], c: 11, c: 12, d: 13) == result - assert_raise FunctionClauseError, fn -> - Keyword.get_values([:foo], :foo) - end + # any key in keywords1 is removed if key is present in keyword2 + result = [a: 1, b: 2, c: 11, c: 12, d: 13] + assert Keyword.merge([a: 1, b: 2, c: 3, c: 4], c: 11, c: 12, d: 13) == result end - test "keys/1" do - assert Keyword.keys(create_keywords) == [:first_key, :first_key, :second_key] - assert Keyword.keys(create_empty_keywords) == [] - end + test "merge/3" do + fun = fn _key, value1, value2 -> value1 + value2 end - test "equal?/2" do - assert Keyword.equal? [a: 1, b: 2], [b: 2, a: 1] - refute Keyword.equal? [a: 1, b: 2], [b: 2, c: 3] - end + assert Keyword.merge([a: 1, b: 2], [c: 11, d: 12], fun) == [a: 1, b: 2, c: 11, d: 12] + assert Keyword.merge([], [c: 11, d: 12], fun) == [c: 11, d: 12] + assert Keyword.merge([a: 1, b: 2], [], fun) == [a: 1, b: 2] - test "values/1" do - assert Keyword.values(create_keywords) == [1, 2, 2] - assert Keyword.values(create_empty_keywords) == [] - end + message = "expected a keyword list as the first argument, got: [1, 2]" - test "delete/2" do - assert Keyword.delete(create_keywords, :first_key) == [second_key: 2] - assert Keyword.delete(create_keywords, :other_key) == create_keywords - assert Keyword.delete(create_empty_keywords, :other_key) == [] - end + assert_raise ArgumentError, message, fn -> + Keyword.merge([1, 2], [c: 11, d: 12], fun) + end - test "delete_first/2" do - assert Keyword.delete_first(create_keywords, :first_key) == [first_key: 2, second_key: 2] - assert Keyword.delete_first(create_keywords, :other_key) == [first_key: 1, first_key: 2, second_key: 2] - assert Keyword.delete_first(create_empty_keywords, :other_key) == [] - end + message = "expected a keyword list as the first argument, got: [1 | 2]" - test "put/3" do - assert Keyword.put(create_empty_keywords, :first_key, 1) == [first_key: 1] - assert Keyword.put(create_keywords, :first_key, 1) == [first_key: 1, second_key: 2] - end + assert_raise ArgumentError, message, fn -> + Keyword.merge([1 | 2], [c: 11, d: 12], fun) + end - test "merge/2" do - assert Keyword.merge(create_empty_keywords, create_keywords) == create_keywords - assert Keyword.merge(create_keywords, create_empty_keywords) == create_keywords - assert Keyword.merge(create_keywords, create_keywords) == create_keywords - assert Keyword.merge(create_empty_keywords, create_empty_keywords) == [] - assert Keyword.merge(create_keywords, [first_key: 0]) == [first_key: 0, second_key: 2] - assert Keyword.merge(create_keywords, [first_key: 0, first_key: 3]) == [first_key: 0, first_key: 3, second_key: 2] - end + message = "expected a keyword list as the second argument, got: [{:x, 1}, :y, :z]" - test "merge/3" do - result = Keyword.merge [a: 1, b: 2], [a: 3, d: 4], fn _k, v1, v2 -> - v1 + v2 + assert_raise ArgumentError, message, fn -> + Keyword.merge([a: 1, b: 2], [{:x, 1}, :y, :z], fun) end - assert Keyword.equal?(result, [a: 4, b: 2, d: 4]) - end - test "has_key?/2" do - assert Keyword.has_key?([a: 1], :a) == true - assert Keyword.has_key?([a: 1], :b) == false - end + message = "expected a keyword list as the second argument, got: [:x | :y]" - test "take/2" do - assert Keyword.take([], []) == [] - assert Keyword.take([a: 0, b: 1, a: 2], []) == [] - assert Keyword.take([a: 0, b: 1, a: 2], [:a]) == [a: 0, a: 2] - assert Keyword.take([a: 0, b: 1, a: 2], [:b]) == [b: 1] + assert_raise ArgumentError, message, fn -> + Keyword.merge([a: 1, b: 2], [:x | :y], fun) + end + + message = "expected a keyword list as the second argument, got: [{:x, 1} | :y]" - assert_raise FunctionClauseError, fn -> - Keyword.take([:foo], [:foo]) + assert_raise ArgumentError, message, fn -> + Keyword.merge([a: 1, b: 2], [{:x, 1} | :y], fun) end - end - test "drop/2" do - assert Keyword.drop([], []) == [] - assert Keyword.drop([a: 0, b: 1, a: 2], []) == [a: 0, b: 1, a: 2] - assert Keyword.drop([a: 0, b: 1, a: 2], [:a]) == [b: 1] - assert Keyword.drop([a: 0, b: 1, a: 2], [:b]) == [a: 0, a: 2] + # duplicate keys in keywords1 are left untouched if key is not present in keywords2 + result = [a: 1, b: 2, a: 3, c: 11, d: 12] + assert Keyword.merge([a: 1, b: 2, a: 3], [c: 11, d: 12], fun) == result + + result = [b: 2, a: 12] + assert Keyword.merge([a: 1, b: 2, a: 3], [a: 11], fun) == result + + # duplicate keys in keywords2 are always kept + result = [a: 1, b: 2, c: 11, c: 12, d: 13] + assert Keyword.merge([a: 1, b: 2], [c: 11, c: 12, d: 13], fun) == result + + # every key in keywords1 is replaced with fun result if key is present in keyword2 + result = [a: 1, b: 2, c: 14, c: 54, d: 13] + assert Keyword.merge([a: 1, b: 2, c: 3, c: 4], [c: 11, c: 50, d: 13], fun) == result + end + + test "merge/2 and merge/3 behave exactly the same way" do + fun = fn _key, _value1, value2 -> value2 end + + args = [ + {[a: 1, b: 2], [c: 11, d: 12]}, + {[], [c: 11, d: 12]}, + {[a: 1, b: 2], []}, + {[a: 1, b: 2, a: 3], [c: 11, d: 12]}, + {[a: 1, b: 2, a: 3], [a: 11]}, + {[a: 1, b: 2], [c: 11, c: 12, d: 13]}, + {[a: 1, b: 2, c: 3, c: 4], [c: 11, c: 12, d: 13]} + ] + + args_error = [ + {[1, 2], [c: 11, d: 12]}, + {[1 | 2], [c: 11, d: 12]}, + {[a: 1, b: 2], [11, 12, 0]}, + {[a: 1, b: 2], [11 | 12]}, + {[a: 1, b: 2], [{:x, 1}, :y, :z]}, + {[a: 1, b: 2], [:x | :y]}, + {[a: 1, b: 2], [{:x, 1} | :y]} + ] + + for {arg1, arg2} <- args do + assert Keyword.merge(arg1, arg2) == Keyword.merge(arg1, arg2, fun) + end - assert_raise FunctionClauseError, fn -> - Keyword.drop([:foo], [:foo]) + for {arg1, arg2} <- args_error do + error = assert_raise ArgumentError, fn -> Keyword.merge(arg1, arg2) end + assert_raise ArgumentError, error.message, fn -> Keyword.merge(arg1, arg2, fun) end end end - defp create_empty_keywords, do: [] - defp create_keywords, do: [first_key: 1, first_key: 2, second_key: 2] + test "validate/2 raises on invalid arguments" do + assert_raise ArgumentError, + "expected a keyword list as first argument, got invalid entry: :three", + fn -> Keyword.validate([:three], one: 1, two: 2) end + + assert_raise ArgumentError, + "expected the second argument to be a list of atoms or tuples, got: 3", + fn -> Keyword.validate([three: 3], [:three, 3, :two]) end + end end diff --git a/lib/elixir/test/elixir/list/chars_test.exs b/lib/elixir/test/elixir/list/chars_test.exs index 94389bddb95..50679ccefef 100644 --- a/lib/elixir/test/elixir/list/chars_test.exs +++ b/lib/elixir/test/elixir/list/chars_test.exs @@ -1,37 +1,43 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule List.Chars.AtomTest do use ExUnit.Case, async: true - test :basic do - assert to_char_list(:foo) == 'foo' + test "basic" do + assert to_charlist(:foo) == 'foo' + end + + test "true false nil" do + assert to_charlist(false) == 'false' + assert to_charlist(true) == 'true' + assert to_charlist(nil) == '' end end defmodule List.Chars.BitStringTest do use ExUnit.Case, async: true - test :basic do - assert to_char_list("foo") == 'foo' + test "basic" do + assert to_charlist("foo") == 'foo' end end defmodule List.Chars.NumberTest do use ExUnit.Case, async: true - test :integer do - assert to_char_list(1) == '1' + test "integer" do + assert to_charlist(1) == '1' end - test :float do - assert to_char_list(1.0) == '1.0' + test "float" do + assert to_charlist(1.0) == '1.0' end end defmodule List.Chars.ListTest do use ExUnit.Case, async: true - test :basic do - assert to_char_list([ 1, "b", 3 ]) == [1, "b", 3] + test "basic" do + assert to_charlist([1, "b", 3]) == [1, "b", 3] end end diff --git a/lib/elixir/test/elixir/list_test.exs b/lib/elixir/test/elixir/list_test.exs index 7da04556abc..f022bc9f627 100644 --- a/lib/elixir/test/elixir/list_test.exs +++ b/lib/elixir/test/elixir/list_test.exs @@ -1,121 +1,204 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule ListTest do use ExUnit.Case, async: true - test :cons_cell_precedence do - assert [1|:lists.flatten([2, 3])] == [1, 2, 3] + doctest List + + test "cons cell precedence" do + assert [1 | List.flatten([2, 3])] == [1, 2, 3] end - test :optional_comma do - assert [1] == [ 1, ] - assert [1, 2, 3] == [1, 2, 3, ] + test "optional comma" do + assert Code.eval_string("[1,]") == {[1], []} + assert Code.eval_string("[1, 2, 3,]") == {[1, 2, 3], []} end - test :partial_application do + test "partial application" do assert (&[&1, 2]).(1) == [1, 2] assert (&[&1, &2]).(1, 2) == [1, 2] assert (&[&2, &1]).(2, 1) == [1, 2] - assert (&[&1|&2]).(1, 2) == [1|2] - assert (&[&1, &2|&3]).(1, 2, 3) == [1, 2|3] + assert (&[&1 | &2]).(1, 2) == [1 | 2] + assert (&[&1, &2 | &3]).(1, 2, 3) == [1, 2 | 3] + end + + test "delete/2" do + assert List.delete([:a, :b, :c], :a) == [:b, :c] + assert List.delete([:a, :b, :c], :d) == [:a, :b, :c] + assert List.delete([:a, :b, :b, :c], :b) == [:a, :b, :c] + assert List.delete([], :b) == [] end - test :wrap do + test "wrap/1" do assert List.wrap([1, 2, 3]) == [1, 2, 3] assert List.wrap(1) == [1] assert List.wrap(nil) == [] end - test :flatten do + test "flatten/1" do assert List.flatten([1, 2, 3]) == [1, 2, 3] assert List.flatten([1, [2], 3]) == [1, 2, 3] assert List.flatten([[1, [2], 3]]) == [1, 2, 3] assert List.flatten([]) == [] assert List.flatten([[]]) == [] + assert List.flatten([[], [[], []]]) == [] end - test :flatten_with_tail do + test "flatten/2" do assert List.flatten([1, 2, 3], [4, 5]) == [1, 2, 3, 4, 5] assert List.flatten([1, [2], 3], [4, 5]) == [1, 2, 3, 4, 5] assert List.flatten([[1, [2], 3]], [4, 5]) == [1, 2, 3, 4, 5] + assert List.flatten([1, [], 2], [3, [], 4]) == [1, 2, 3, [], 4] end - test :foldl do + test "foldl/3" do assert List.foldl([1, 2, 3], 0, fn x, y -> x + y end) == 6 assert List.foldl([1, 2, 3], 10, fn x, y -> x + y end) == 16 assert List.foldl([1, 2, 3, 4], 0, fn x, y -> x - y end) == 2 end - test :foldr do + test "foldr/3" do assert List.foldr([1, 2, 3], 0, fn x, y -> x + y end) == 6 assert List.foldr([1, 2, 3], 10, fn x, y -> x + y end) == 16 assert List.foldr([1, 2, 3, 4], 0, fn x, y -> x - y end) == -2 end - test :reverse do - assert Enum.reverse([1, 2, 3]) == [3, 2, 1] - end - - test :duplicate do + test "duplicate/2" do + assert List.duplicate(1, 0) == [] assert List.duplicate(1, 3) == [1, 1, 1] assert List.duplicate([1], 1) == [[1]] end - test :last do + test "first/1" do + assert List.first([]) == nil + assert List.first([], 1) == 1 + assert List.first([1]) == 1 + assert List.first([1, 2, 3]) == 1 + end + + test "last/1" do assert List.last([]) == nil + assert List.last([], 1) == 1 assert List.last([1]) == 1 assert List.last([1, 2, 3]) == 3 end - test :zip do + test "zip/1" do assert List.zip([[1, 4], [2, 5], [3, 6]]) == [{1, 2, 3}, {4, 5, 6}] assert List.zip([[1, 4], [2, 5, 0], [3, 6]]) == [{1, 2, 3}, {4, 5, 6}] assert List.zip([[1], [2, 5], [3, 6]]) == [{1, 2, 3}] assert List.zip([[1, 4], [2, 5], []]) == [] end - test :unzip do - assert List.unzip([{1, 2, 3}, {4, 5, 6}]) == [[1, 4], [2, 5], [3, 6]] - assert List.unzip([{1, 2, 3}, {4, 5}]) == [[1, 4], [2, 5]] - assert List.unzip([[1, 2, 3], [4, 5]]) == [[1, 4], [2, 5]] - assert List.unzip([]) == [] - end - - test :keyfind do + test "keyfind/4" do assert List.keyfind([a: 1, b: 2], :a, 0) == {:a, 1} assert List.keyfind([a: 1, b: 2], 2, 1) == {:b, 2} assert List.keyfind([a: 1, b: 2], :c, 0) == nil end - test :keyreplace do + test "keyreplace/4" do assert List.keyreplace([a: 1, b: 2], :a, 0, {:a, 3}) == [a: 3, b: 2] assert List.keyreplace([a: 1], :b, 0, {:b, 2}) == [a: 1] end - test :keysort do + test "keysort/2" do assert List.keysort([a: 4, b: 3, c: 5], 1) == [b: 3, a: 4, c: 5] assert List.keysort([a: 4, c: 1, b: 2], 0) == [a: 4, b: 2, c: 1] end - test :keystore do + test "keysort/3 with stable sorting" do + collection = [ + {2, 4}, + {1, 5}, + {2, 2}, + {3, 1}, + {4, 3} + ] + + # Stable sorting + assert List.keysort(collection, 0) == [ + {1, 5}, + {2, 4}, + {2, 2}, + {3, 1}, + {4, 3} + ] + + assert List.keysort(collection, 0) == + List.keysort(collection, 0, :asc) + + assert List.keysort(collection, 0, & - assert [] = List.delete_at([], i) + test "delete_at/2" do + for index <- [-1, 0, 1] do + assert List.delete_at([], index) == [] end + assert List.delete_at([1, 2, 3], 0) == [2, 3] assert List.delete_at([1, 2, 3], 2) == [1, 2] assert List.delete_at([1, 2, 3], 3) == [1, 2, 3] @@ -155,13 +239,168 @@ defmodule ListTest do assert List.delete_at([1, 2, 3], -4) == [1, 2, 3] end - test :to_string do + test "pop_at/3" do + for index <- [-1, 0, 1] do + assert List.pop_at([], index) == {nil, []} + end + + assert List.pop_at([1], 1, 2) == {2, [1]} + assert List.pop_at([1, 2, 3], 0) == {1, [2, 3]} + assert List.pop_at([1, 2, 3], 2) == {3, [1, 2]} + assert List.pop_at([1, 2, 3], 3) == {nil, [1, 2, 3]} + assert List.pop_at([1, 2, 3], -1) == {3, [1, 2]} + assert List.pop_at([1, 2, 3], -3) == {1, [2, 3]} + assert List.pop_at([1, 2, 3], -4) == {nil, [1, 2, 3]} + end + + describe "starts_with?/2" do + test "list and prefix are equal" do + assert List.starts_with?([], []) + assert List.starts_with?([1], [1]) + assert List.starts_with?([1, 2, 3], [1, 2, 3]) + end + + test "proper lists" do + refute List.starts_with?([1], [1, 2]) + assert List.starts_with?([1, 2, 3], [1, 2]) + refute List.starts_with?([1, 2, 3], [1, 2, 3, 4]) + end + + test "list is empty" do + refute List.starts_with?([], [1]) + refute List.starts_with?([], [1, 2]) + end + + test "prefix is empty" do + assert List.starts_with?([1], []) + assert List.starts_with?([1, 2], []) + assert List.starts_with?([1, 2, 3], []) + end + + test "only accepts proper lists" do + message = "no function clause matching in List.starts_with?/2" + + assert_raise FunctionClauseError, message, fn -> + List.starts_with?([1 | 2], [1 | 2]) + end + + message = "no function clause matching in List.starts_with?/2" + + assert_raise FunctionClauseError, message, fn -> + List.starts_with?([1, 2], 1) + end + end + end + + test "to_string/1" do assert List.to_string([?æ, ?ß]) == "æß" assert List.to_string([?a, ?b, ?c]) == "abc" + assert List.to_string([]) == "" + assert List.to_string([[], []]) == "" - assert_raise UnicodeConversionError, - "invalid code point 57343", fn -> + assert_raise UnicodeConversionError, "invalid code point 57343", fn -> List.to_string([0xDFFF]) end + + assert_raise UnicodeConversionError, "invalid encoding starting at <<216, 0>>", fn -> + List.to_string(["a", "b", <<0xD800::size(16)>>]) + end + + assert_raise ArgumentError, ~r"cannot convert the given list to a string", fn -> + List.to_string([:a, :b]) + end + end + + test "to_charlist/1" do + assert List.to_charlist([0x00E6, 0x00DF]) == 'æß' + assert List.to_charlist([0x0061, "bc"]) == 'abc' + assert List.to_charlist([0x0064, "ee", ['p']]) == 'deep' + + assert_raise UnicodeConversionError, "invalid code point 57343", fn -> + List.to_charlist([0xDFFF]) + end + + assert_raise UnicodeConversionError, "invalid encoding starting at <<216, 0>>", fn -> + List.to_charlist(["a", "b", <<0xD800::size(16)>>]) + end + + assert_raise ArgumentError, ~r"cannot convert the given list to a charlist", fn -> + List.to_charlist([:a, :b]) + end + end + + describe "myers_difference/2" do + test "follows paper implementation" do + assert List.myers_difference([], []) == [] + assert List.myers_difference([], [1, 2, 3]) == [ins: [1, 2, 3]] + assert List.myers_difference([1, 2, 3], []) == [del: [1, 2, 3]] + assert List.myers_difference([1, 2, 3], [1, 2, 3]) == [eq: [1, 2, 3]] + assert List.myers_difference([1, 2, 3], [1, 4, 2, 3]) == [eq: [1], ins: [4], eq: [2, 3]] + assert List.myers_difference([1, 4, 2, 3], [1, 2, 3]) == [eq: [1], del: [4], eq: [2, 3]] + assert List.myers_difference([1], [[1]]) == [del: [1], ins: [[1]]] + assert List.myers_difference([[1]], [1]) == [del: [[1]], ins: [1]] + end + + test "rearranges inserts and equals for smaller diffs" do + assert List.myers_difference([3, 2, 0, 2], [2, 2, 0, 2]) == + [del: [3], ins: [2], eq: [2, 0, 2]] + + assert List.myers_difference([3, 2, 1, 0, 2], [2, 1, 2, 1, 0, 2]) == + [del: [3], ins: [2, 1], eq: [2, 1, 0, 2]] + + assert List.myers_difference([3, 2, 2, 1, 0, 2], [2, 2, 1, 2, 1, 0, 2]) == + [del: [3], eq: [2, 2, 1], ins: [2, 1], eq: [0, 2]] + + assert List.myers_difference([3, 2, 0, 2], [2, 2, 1, 0, 2]) == + [del: [3], eq: [2], ins: [2, 1], eq: [0, 2]] + end + end + + test "improper?/1" do + assert List.improper?([1 | 2]) + assert List.improper?([1, 2, 3 | 4]) + refute List.improper?([]) + refute List.improper?([1]) + refute List.improper?([[1]]) + refute List.improper?([1, 2]) + refute List.improper?([1, 2, 3]) + + assert_raise FunctionClauseError, fn -> + List.improper?(%{}) + end + end + + describe "ascii_printable?/2" do + test "proper lists without limit" do + assert List.ascii_printable?([]) + assert List.ascii_printable?('abc') + refute(List.ascii_printable?('abc' ++ [0])) + refute List.ascii_printable?('mañana') + + printable_chars = '\a\b\t\n\v\f\r\e' ++ Enum.to_list(32..126) + non_printable_chars = '🌢áéíóúźç©¢🂭' + + assert List.ascii_printable?(printable_chars) + + for char <- printable_chars do + assert List.ascii_printable?([char]) + end + + refute List.ascii_printable?(non_printable_chars) + + for char <- non_printable_chars do + refute List.ascii_printable?([char]) + end + end + + test "proper lists with limit" do + assert List.ascii_printable?([], 100) + assert List.ascii_printable?('abc' ++ [0], 2) + end + + test "improper lists" do + refute List.ascii_printable?('abc' ++ ?d) + assert List.ascii_printable?('abc' ++ ?d, 3) + end end end diff --git a/lib/elixir/test/elixir/macro_test.exs b/lib/elixir/test/elixir/macro_test.exs index 9a1de1c6130..7b75b6d3403 100644 --- a/lib/elixir/test/elixir/macro_test.exs +++ b/lib/elixir/test/elixir/macro_test.exs @@ -1,4 +1,4 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule Macro.ExternalTest do defmacro external do @@ -11,520 +11,1122 @@ defmodule Macro.ExternalTest do end defmacro oror(left, right) do - quote do: unquote(left) || unquote(right) + quote(do: unquote(left) || unquote(right)) end end defmodule MacroTest do use ExUnit.Case, async: true + doctest Macro # Changing the lines above will make compilation # fail since we are asserting on the caller lines import Macro.ExternalTest - ## Escape + describe "escape/2" do + test "returns tuples with size equal to two" do + assert Macro.escape({:a, :b}) == {:a, :b} + end - test :escape_handle_tuples_with_size_different_than_two do - assert {:{}, [], [:a]} == Macro.escape({:a}) - assert {:{}, [], [:a, :b, :c]} == Macro.escape({:a, :b, :c}) - assert {:{}, [], [:a, {:{}, [], [1,2,3]}, :c]} == Macro.escape({:a, {1, 2, 3}, :c}) - end + test "returns lists" do + assert Macro.escape([1, 2, 3]) == [1, 2, 3] + end - test :escape_simply_returns_tuples_with_size_equal_to_two do - assert {:a, :b} == Macro.escape({:a, :b}) - end + test "escapes tuples with size different than two" do + assert Macro.escape({:a}) == {:{}, [], [:a]} + assert Macro.escape({:a, :b, :c}) == {:{}, [], [:a, :b, :c]} + assert Macro.escape({:a, {1, 2, 3}, :c}) == {:{}, [], [:a, {:{}, [], [1, 2, 3]}, :c]} + end - test :escape_simply_returns_any_other_structure do - assert [1, 2, 3] == Macro.escape([1, 2, 3]) - end + test "escapes maps" do + assert Macro.escape(%{a: 1}) == {:%{}, [], [a: 1]} + end - test :escape_handles_maps do - assert {:%{}, [], [a: 1]} = Macro.escape(%{a: 1}) - end + test "escapes bitstring" do + assert {:<<>>, [], args} = Macro.escape(<<300::12>>) + assert [{:"::", [], [1, {:size, [], [4]}]}, {:"::", [], [",", {:binary, [], []}]}] = args + end - test :escape_works_recursively do - assert [1,{:{}, [], [:a,:b,:c]}, 3] == Macro.escape([1, {:a, :b, :c}, 3]) - end + test "escapes recursively" do + assert Macro.escape([1, {:a, :b, :c}, 3]) == [1, {:{}, [], [:a, :b, :c]}, 3] + end - test :escape_improper do - assert [{:|, [], [1,2]}] == Macro.escape([1|2]) - assert [1,{:|, [], [2,3]}] == Macro.escape([1,2|3]) - end + test "escapes improper lists" do + assert Macro.escape([1 | 2]) == [{:|, [], [1, 2]}] + assert Macro.escape([1, 2 | 3]) == [1, {:|, [], [2, 3]}] + end - test :escape_with_unquote do - contents = quote unquote: false, do: unquote(1) - assert Macro.escape(contents, unquote: true) == 1 + test "prunes metadata" do + meta = [nothing: :important, counter: 1] + assert Macro.escape({:foo, meta, []}) == {:{}, [], [:foo, meta, []]} + assert Macro.escape({:foo, meta, []}, prune_metadata: true) == {:{}, [], [:foo, [], []]} + end - contents = quote unquote: false, do: unquote(x) - assert Macro.escape(contents, unquote: true) == {:x, [], MacroTest} - end + test "with unquote" do + contents = quote(unquote: false, do: unquote(1)) + assert Macro.escape(contents, unquote: true) == 1 - defp eval_escaped(contents) do - {eval, []} = Code.eval_quoted(Macro.escape(contents, unquote: true)) - eval - end + contents = quote(unquote: false, do: unquote(x)) + assert Macro.escape(contents, unquote: true) == {:x, [], MacroTest} + end - test :escape_with_remote_unquote do - contents = quote unquote: false, do: Kernel.unquote(:is_atom)(:ok) - assert eval_escaped(contents) == quote(do: Kernel.is_atom(:ok)) - end + defp eval_escaped(contents) do + {eval, []} = Code.eval_quoted(Macro.escape(contents, unquote: true)) + eval + end - test :escape_with_nested_unquote do - contents = quote do - quote do: unquote(x) + test "with remote unquote" do + contents = quote(unquote: false, do: Kernel.unquote(:is_atom)(:ok)) + assert eval_escaped(contents) == quote(do: Kernel.is_atom(:ok)) end - assert eval_escaped(contents) == quote do: (quote do: unquote(x)) - end - test :escape_with_alias_or_no_args_remote_unquote do - contents = quote unquote: false, do: Kernel.unquote(:self) - assert eval_escaped(contents) == quote(do: Kernel.self()) + test "with nested unquote" do + contents = + quote do + quote(do: unquote(x)) + end - contents = quote unquote: false, do: x.unquote(Foo) - assert eval_escaped(contents) == quote(do: x.unquote(Foo)) - end + assert eval_escaped(contents) == quote(do: quote(do: unquote(x))) + end - test :escape_with_splicing do - contents = quote unquote: false, do: [1, 2, 3, 4, 5] - assert Macro.escape(contents, unquote: true) == [1, 2, 3, 4, 5] + test "with alias or no arguments remote unquote" do + contents = quote(unquote: false, do: Kernel.unquote(:self)()) + assert eval_escaped(contents) == quote(do: Kernel.self()) - contents = quote unquote: false, do: [1, 2, unquote_splicing([3, 4, 5])] - assert eval_escaped(contents) == [1, 2, 3, 4, 5] + contents = quote(unquote: false, do: x.unquote(Foo)) + assert eval_escaped(contents) == quote(do: x.unquote(Foo)) + end - contents = quote unquote: false, do: [unquote_splicing([1, 2, 3]), 4, 5] - assert eval_escaped(contents) == [1, 2, 3, 4, 5] + test "with splicing" do + contents = quote(unquote: false, do: [1, 2, 3, 4, 5]) + assert Macro.escape(contents, unquote: true) == [1, 2, 3, 4, 5] - contents = quote unquote: false, do: [unquote_splicing([1, 2, 3]), unquote_splicing([4, 5])] - assert eval_escaped(contents) == [1, 2, 3, 4, 5] + contents = quote(unquote: false, do: [1, 2, unquote_splicing([3, 4, 5])]) + assert eval_escaped(contents) == [1, 2, 3, 4, 5] - contents = quote unquote: false, do: [1, unquote_splicing([2]), 3, unquote_splicing([4]), 5] - assert eval_escaped(contents) == [1, 2, 3, 4, 5] + contents = quote(unquote: false, do: [unquote_splicing([1, 2, 3]), 4, 5]) + assert eval_escaped(contents) == [1, 2, 3, 4, 5] - contents = quote unquote: false, do: [1, unquote_splicing([2]), 3, unquote_splicing([4])|[5]] - assert eval_escaped(contents) == [1, 2, 3, 4, 5] - end + contents = + quote(unquote: false, do: [unquote_splicing([1, 2, 3]), unquote_splicing([4, 5])]) - ## Expansion + assert eval_escaped(contents) == [1, 2, 3, 4, 5] - test :expand_once do - assert {:||, _, _} = Macro.expand_once(quote(do: oror(1, false)), __ENV__) - end + contents = + quote(unquote: false, do: [1, unquote_splicing([2]), 3, unquote_splicing([4]), 5]) - test :expand_once_with_raw_atom do - assert Macro.expand_once(quote(do: :foo), __ENV__) == :foo - end + assert eval_escaped(contents) == [1, 2, 3, 4, 5] - test :expand_once_with_current_module do - assert Macro.expand_once(quote(do: __MODULE__), __ENV__) == __MODULE__ - end + contents = + quote(unquote: false, do: [1, unquote_splicing([2]), 3, unquote_splicing([4]) | [5]]) - test :expand_once_with_main do - assert Macro.expand_once(quote(do: Elixir), __ENV__) == Elixir - end + assert eval_escaped(contents) == [1, 2, 3, 4, 5] + end - test :expand_once_with_simple_alias do - assert Macro.expand_once(quote(do: Foo), __ENV__) == Foo + test "does not add context to quote" do + assert Macro.escape({:quote, [], [[do: :foo]]}) == {:{}, [], [:quote, [], [[do: :foo]]]} + end end - test :expand_once_with_current_module_plus_alias do - assert Macro.expand_once(quote(do: __MODULE__.Foo), __ENV__) == __MODULE__.Foo - end + describe "expand_once/2" do + test "with external macro" do + assert {:||, _, [1, false]} = Macro.expand_once(quote(do: oror(1, false)), __ENV__) + end - test :expand_once_with_main_plus_alias do - assert Macro.expand_once(quote(do: Elixir.Foo), __ENV__) == Foo - end + test "with raw atom" do + assert Macro.expand_once(quote(do: :foo), __ENV__) == :foo + end - test :expand_once_with_custom_alias do - alias Foo, as: Bar - assert Macro.expand_once(quote(do: Bar.Baz), __ENV__) == Foo.Baz - end + test "with current module" do + assert Macro.expand_once(quote(do: __MODULE__), __ENV__) == __MODULE__ + end - test :expand_once_with_main_plus_custom_alias do - alias Foo, as: Bar, warn: false - assert Macro.expand_once(quote(do: Elixir.Bar.Baz), __ENV__) == Elixir.Bar.Baz - end + test "with main" do + assert Macro.expand_once(quote(do: Elixir), __ENV__) == Elixir + end - test :expand_once_with_op do - assert Macro.expand_once(quote(do: Foo.bar.Baz), __ENV__) == (quote do - Foo.bar.Baz - end) - end + test "with simple alias" do + assert Macro.expand_once(quote(do: Foo), __ENV__) == Foo + end - test :expand_once_with_erlang do - assert Macro.expand_once(quote(do: :foo), __ENV__) == :foo - end + test "with current module plus alias" do + assert Macro.expand_once(quote(do: __MODULE__.Foo), __ENV__) == __MODULE__.Foo + end - test :expand_once_env do - env = %{__ENV__ | line: 0} - assert Macro.expand_once(quote(do: __ENV__), env) == {:%{}, [], Map.to_list(env)} - assert Macro.expand_once(quote(do: __ENV__.file), env) == env.file - assert Macro.expand_once(quote(do: __ENV__.unknown), env) == quote(do: __ENV__.unknown) - end + test "with main plus alias" do + assert Macro.expand_once(quote(do: Elixir.Foo), __ENV__) == Foo + end - defmacro local_macro do - :local_macro - end + test "with custom alias" do + alias Foo, as: Bar + assert Macro.expand_once(quote(do: Bar.Baz), __ENV__) == Foo.Baz + end - test :expand_once_local_macro do - assert Macro.expand_once(quote(do: local_macro), __ENV__) == :local_macro - end + test "with main plus custom alias" do + alias Foo, as: Bar, warn: false + assert Macro.expand_once(quote(do: Elixir.Bar.Baz), __ENV__) == Elixir.Bar.Baz + end - test :expand_once_checks_vars do - local_macro = 1 - assert local_macro == 1 - quote = {:local_macro, [], nil} - assert Macro.expand_once(quote, __ENV__) == quote - end + test "with call in alias" do + assert Macro.expand_once(quote(do: Foo.bar().Baz), __ENV__) == quote(do: Foo.bar().Baz) + end - defp expand_once_and_clean(quoted, env) do - cleaner = &Keyword.drop(&1, [:counter]) - quoted - |> Macro.expand_once(env) - |> Macro.prewalk(&Macro.update_meta(&1, cleaner)) - end + test "env" do + env = %{__ENV__ | line: 0} + assert Macro.expand_once(quote(do: __ENV__), env) == {:%{}, [], Map.to_list(env)} + assert Macro.expand_once(quote(do: __ENV__.file), env) == env.file + assert Macro.expand_once(quote(do: __ENV__.unknown), env) == quote(do: __ENV__.unknown) + end - test :expand_once_with_imported_macro do - temp_var = {:x, [], Kernel} - assert expand_once_and_clean(quote(do: 1 || false), __ENV__) == (quote context: Kernel do - case 1 do - unquote(temp_var) when unquote(temp_var) in [false, nil] -> false - unquote(temp_var) -> unquote(temp_var) - end - end) - end + defmacro local_macro(), do: raise("ignored") - test :expand_once_with_require_macro do - temp_var = {:x, [], Kernel} - assert expand_once_and_clean(quote(do: Kernel.||(1, false)), __ENV__) == (quote context: Kernel do - case 1 do - unquote(temp_var) when unquote(temp_var) in [false, nil] -> false - unquote(temp_var) -> unquote(temp_var) - end - end) - end + test "vars" do + expr = {:local_macro, [], nil} + assert Macro.expand_once(expr, __ENV__) == expr + end - test :expand_once_with_not_expandable_expression do - expr = quote(do: other(1, 2, 3)) - assert Macro.expand_once(expr, __ENV__) == expr - end + defp expand_once_and_clean(quoted, env) do + cleaner = &Keyword.drop(&1, [:counter]) + + quoted + |> Macro.expand_once(env) + |> Macro.prewalk(&Macro.update_meta(&1, cleaner)) + end + + test "with imported macro" do + temp_var = {:x, [], Kernel} - @foo 1 - @bar Macro.expand_once(quote(do: @foo), __ENV__) + quoted = + quote context: Kernel do + case 1 do + unquote(temp_var) when :"Elixir.Kernel".in(unquote(temp_var), [false, nil]) -> false + unquote(temp_var) -> unquote(temp_var) + end + end - test :expand_once_with_module_at do - assert @bar == 1 + assert expand_once_and_clean(quote(do: 1 || false), __ENV__) == quoted + end + + test "with require macro" do + temp_var = {:x, [], Kernel} + + quoted = + quote context: Kernel do + case 1 do + unquote(temp_var) when :"Elixir.Kernel".in(unquote(temp_var), [false, nil]) -> false + unquote(temp_var) -> unquote(temp_var) + end + end + + assert expand_once_and_clean(quote(do: Kernel.||(1, false)), __ENV__) == quoted + end + + test "with not expandable expression" do + expr = quote(do: other(1, 2, 3)) + assert Macro.expand_once(expr, __ENV__) == expr + end + + test "does not expand module attributes" do + message = + "could not call Module.get_attribute/2 because the module #{inspect(__MODULE__)} " <> + "is already compiled. Use the Module.__info__/1 callback or Code.fetch_docs/1 instead" + + assert_raise ArgumentError, message, fn -> + Macro.expand_once(quote(do: @foo), __ENV__) + end + end end defp expand_and_clean(quoted, env) do cleaner = &Keyword.drop(&1, [:counter]) + quoted |> Macro.expand(env) |> Macro.prewalk(&Macro.update_meta(&1, cleaner)) end - test :expand do + test "expand/2" do temp_var = {:x, [], Kernel} - assert expand_and_clean(quote(do: oror(1, false)), __ENV__) == (quote context: Kernel do - case 1 do - unquote(temp_var) when unquote(temp_var) in [false, nil] -> false - unquote(temp_var) -> unquote(temp_var) + + quoted = + quote context: Kernel do + case 1 do + unquote(temp_var) when :"Elixir.Kernel".in(unquote(temp_var), [false, nil]) -> false + unquote(temp_var) -> unquote(temp_var) + end end - end) + + assert expand_and_clean(quote(do: oror(1, false)), __ENV__) == quoted end - test :var do + test "var/2" do assert Macro.var(:foo, nil) == {:foo, [], nil} assert Macro.var(:foo, Other) == {:foo, [], Other} end - ## to_string - - test :var_to_string do - assert Macro.to_string(quote do: foo) == "foo" + describe "to_string/1" do + test "converts quoted to string" do + assert Macro.to_string(quote do: hello(world)) == "hello(world)" + end end - test :local_call_to_string do - assert Macro.to_string(quote do: foo(1, 2, 3)) == "foo(1, 2, 3)" - assert Macro.to_string(quote do: foo([1, 2, 3])) == "foo([1, 2, 3])" - end + describe "to_string/2" do + defp macro_to_string(var, fun \\ fn _ast, string -> string end) do + module = Macro + module.to_string(var, fun) + end - test :remote_call_to_string do - assert Macro.to_string(quote do: foo.bar(1, 2, 3)) == "foo.bar(1, 2, 3)" - assert Macro.to_string(quote do: foo.bar([1, 2, 3])) == "foo.bar([1, 2, 3])" - end + test "variable" do + assert macro_to_string(quote(do: foo)) == "foo" + end - test :low_atom_remote_call_to_string do - assert Macro.to_string(quote do: :foo.bar(1, 2, 3)) == ":foo.bar(1, 2, 3)" - end + test "local call" do + assert macro_to_string(quote(do: foo(1, 2, 3))) == "foo(1, 2, 3)" + assert macro_to_string(quote(do: foo([1, 2, 3]))) == "foo([1, 2, 3])" + end - test :big_atom_remote_call_to_string do - assert Macro.to_string(quote do: Foo.Bar.bar(1, 2, 3)) == "Foo.Bar.bar(1, 2, 3)" - end + test "remote call" do + assert macro_to_string(quote(do: foo.bar(1, 2, 3))) == "foo.bar(1, 2, 3)" + assert macro_to_string(quote(do: foo.bar([1, 2, 3]))) == "foo.bar([1, 2, 3])" - test :remote_and_fun_call_to_string do - assert Macro.to_string(quote do: foo.bar.(1, 2, 3)) == "foo.bar().(1, 2, 3)" - assert Macro.to_string(quote do: foo.bar.([1, 2, 3])) == "foo.bar().([1, 2, 3])" - end + quoted = + quote do + (foo do + :ok + end).bar([1, 2, 3]) + end - test :atom_call_to_string do - assert Macro.to_string(quote do: :foo.(1, 2, 3)) == ":foo.(1, 2, 3)" - end + assert macro_to_string(quoted) == "(foo do\n :ok\nend).bar([1, 2, 3])" + end - test :aliases_call_to_string do - assert Macro.to_string(quote do: Foo.Bar.baz(1, 2, 3)) == "Foo.Bar.baz(1, 2, 3)" - assert Macro.to_string(quote do: Foo.Bar.baz([1, 2, 3])) == "Foo.Bar.baz([1, 2, 3])" - end + test "nullary remote call" do + assert macro_to_string(quote do: foo.bar) == "foo.bar" + assert macro_to_string(quote do: foo.bar()) == "foo.bar()" + end - test :arrow_to_string do - assert Macro.to_string(quote do: foo(1, (2 -> 3))) == "foo(1, (2 -> 3))" - end + test "atom remote call" do + assert macro_to_string(quote(do: :foo.bar(1, 2, 3))) == ":foo.bar(1, 2, 3)" + end - test :blocks_to_string do - assert Macro.to_string(quote do: (1; 2; (:foo; :bar); 3)) <> "\n" == """ - ( - 1 - 2 - ( - :foo - :bar - ) - 3 - ) - """ - end + test "remote and fun call" do + assert macro_to_string(quote(do: foo.bar().(1, 2, 3))) == "foo.bar().(1, 2, 3)" + assert macro_to_string(quote(do: foo.bar().([1, 2, 3]))) == "foo.bar().([1, 2, 3])" + end - test :if_else_to_string do - assert Macro.to_string(quote do: (if foo, do: bar, else: baz)) <> "\n" == """ - if(foo) do - bar - else - baz + test "unusual remote atom fun call" do + assert macro_to_string(quote(do: Foo."42"())) == ~s/Foo."42"()/ + assert macro_to_string(quote(do: Foo."Bar"())) == ~s/Foo."Bar"()/ + assert macro_to_string(quote(do: Foo."bar baz"().""())) == ~s/Foo."bar baz"().""()/ + assert macro_to_string(quote(do: Foo."%{}"())) == ~s/Foo."%{}"()/ + assert macro_to_string(quote(do: Foo."..."())) == ~s/Foo."..."()/ end - """ - end - test :case_to_string do - assert Macro.to_string(quote do: (case foo do true -> 0; false -> (1; 2) end)) <> "\n" == """ - case(foo) do - true -> - 0 - false -> + test "atom fun call" do + assert macro_to_string(quote(do: :foo.(1, 2, 3))) == ":foo.(1, 2, 3)" + end + + test "aliases call" do + assert macro_to_string(quote(do: Elixir)) == "Elixir" + assert macro_to_string(quote(do: Foo)) == "Foo" + assert macro_to_string(quote(do: Foo.Bar.baz(1, 2, 3))) == "Foo.Bar.baz(1, 2, 3)" + assert macro_to_string(quote(do: Foo.Bar.baz([1, 2, 3]))) == "Foo.Bar.baz([1, 2, 3])" + assert macro_to_string(quote(do: Foo.bar(<<>>, []))) == "Foo.bar(<<>>, [])" + end + + test "keyword call" do + assert macro_to_string(quote(do: Foo.bar(foo: :bar))) == "Foo.bar(foo: :bar)" + assert macro_to_string(quote(do: Foo.bar("Elixir.Foo": :bar))) == "Foo.bar([{Foo, :bar}])" + end + + test "sigil call" do + assert macro_to_string(quote(do: ~r"123")) == ~S/~r"123"/ + assert macro_to_string(quote(do: ~r"\n123")) == ~S/~r"\n123"/ + assert macro_to_string(quote(do: ~r"12\"3")) == ~S/~r"12\"3"/ + assert macro_to_string(quote(do: ~r/12\/3/u)) == ~S"~r/12\/3/u" + assert macro_to_string(quote(do: ~r{\n123})) == ~S/~r{\n123}/ + assert macro_to_string(quote(do: ~r((1\)(2\)3))) == ~S/~r((1\)(2\)3)/ + assert macro_to_string(quote(do: ~r{\n1{1\}23})) == ~S/~r{\n1{1\}23}/ + assert macro_to_string(quote(do: ~r|12\|3|)) == ~S"~r|12\|3|" + + assert macro_to_string(quote(do: ~r[1#{two}3])) == ~S/~r[1#{two}3]/ + assert macro_to_string(quote(do: ~r[1[#{two}\]3])) == ~S/~r[1[#{two}\]3]/ + assert macro_to_string(quote(do: ~r'1#{two}3'u)) == ~S/~r'1#{two}3'u/ + + assert macro_to_string(quote(do: ~R"123")) == ~S/~R"123"/ + assert macro_to_string(quote(do: ~R"123"u)) == ~S/~R"123"u/ + assert macro_to_string(quote(do: ~R"\n123")) == ~S/~R"\n123"/ + + assert macro_to_string(quote(do: ~S["'(123)'"])) == ~S/~S["'(123)'"]/ + assert macro_to_string(quote(do: ~s"#{"foo"}")) == ~S/~s"#{"foo"}"/ + + assert macro_to_string( + quote do + ~s""" + "\""foo"\"" + """ + end + ) == ~s[~s"""\n"\\""foo"\\""\n"""] + + assert macro_to_string( + quote do + ~s''' + '\''foo'\'' + ''' + end + ) == ~s[~s'''\n'\\''foo'\\''\n'''] + + assert macro_to_string( + quote do + ~s""" + "\"foo\"" + """ + end + ) == ~s[~s"""\n"\\"foo\\""\n"""] + + assert macro_to_string( + quote do + ~s''' + '\"foo\"' + ''' + end + ) == ~s[~s'''\n'\\"foo\\"'\n'''] + + assert macro_to_string( + quote do + ~S""" + "123" + """ + end + ) == ~s[~S"""\n"123"\n"""] + end + + test "tuple call" do + assert macro_to_string(quote(do: alias(Foo.{Bar, Baz, Bong}))) == + "alias(Foo.{Bar, Baz, Bong})" + + assert macro_to_string(quote(do: foo(Foo.{}))) == "foo(Foo.{})" + end + + test "arrow" do + assert macro_to_string(quote(do: foo(1, (2 -> 3)))) == "foo(1, (2 -> 3))" + end + + test "block" do + quoted = + quote do + 1 + 2 + + ( + :foo + :bar + ) + + 3 + end + + expected = """ + ( 1 2 + ( + :foo + :bar + ) + 3 + ) + """ + + assert macro_to_string(quoted) <> "\n" == expected + end + + test "not in" do + assert macro_to_string(quote(do: false not in [])) == "false not in []" end - """ - end - test :fn_to_string do - assert Macro.to_string(quote do: (fn -> 1 + 2 end)) == "fn -> 1 + 2 end" - assert Macro.to_string(quote do: (fn(x) -> x + 1 end)) == "fn x -> x + 1 end" + test "if else" do + expected = """ + if(foo) do + bar + else + baz + end + """ - assert Macro.to_string(quote do: (fn(x) -> y = x + 1; y end)) <> "\n" == """ - fn x -> - y = x + 1 - y + assert macro_to_string(quote(do: if(foo, do: bar, else: baz))) <> "\n" == expected end - """ - assert Macro.to_string(quote do: (fn(x) -> y = x + 1; y; (z) -> z end)) <> "\n" == """ - fn - x -> + test "case" do + quoted = + quote do + case foo do + true -> + 0 + + false -> + 1 + 2 + end + end + + expected = """ + case(foo) do + true -> + 0 + false -> + 1 + 2 + end + """ + + assert macro_to_string(quoted) <> "\n" == expected + end + + test "try" do + quoted = + quote do + try do + foo + catch + _, _ -> + 2 + rescue + ArgumentError -> + 1 + after + 4 + else + _ -> + 3 + end + end + + expected = """ + try do + foo + rescue + ArgumentError -> + 1 + catch + _, _ -> + 2 + else + _ -> + 3 + after + 4 + end + """ + + assert macro_to_string(quoted) <> "\n" == expected + end + + test "fn" do + assert macro_to_string(quote(do: fn -> 1 + 2 end)) == "fn -> 1 + 2 end" + assert macro_to_string(quote(do: fn x -> x + 1 end)) == "fn x -> x + 1 end" + + quoted = + quote do + fn x -> + y = x + 1 + y + end + end + + expected = """ + fn x -> y = x + 1 y - z -> - z + end + """ + + assert macro_to_string(quoted) <> "\n" == expected + + quoted = + quote do + fn + x -> + y = x + 1 + y + + z -> + z + end + end + + expected = """ + fn + x -> + y = x + 1 + y + z -> + z + end + """ + + assert macro_to_string(quoted) <> "\n" == expected + + assert macro_to_string(quote(do: (fn x -> x end).(1))) == "(fn x -> x end).(1)" + + quoted = + quote do + (fn + %{} -> :map + _ -> :other + end).(1) + end + + expected = """ + (fn + %{} -> + :map + _ -> + :other + end).(1) + """ + + assert macro_to_string(quoted) <> "\n" == expected end - """ - end - test :when do - assert Macro.to_string(quote do: (() -> x)) == "(() -> x)" - assert Macro.to_string(quote do: (x when y -> z)) == "(x when y -> z)" - assert Macro.to_string(quote do: (x, y when z -> w)) == "((x, y) when z -> w)" - assert Macro.to_string(quote do: ((x, y) when z -> w)) == "((x, y) when z -> w)" - end + test "range" do + assert macro_to_string(quote(do: unquote(-1..+2))) == "-1..2" + assert macro_to_string(quote(do: Foo.integer()..3)) == "Foo.integer()..3" + assert macro_to_string(quote(do: unquote(-1..+2//-3))) == "-1..2//-3" - test :nested_to_string do - assert Macro.to_string(quote do: (defmodule Foo do def foo do 1 + 1 end end)) <> "\n" == """ - defmodule(Foo) do - def(foo) do - 1 + 1 + assert macro_to_string(quote(do: Foo.integer()..3//Bar.bat())) == + "Foo.integer()..3//Bar.bat()" + end + + test "when" do + assert macro_to_string(quote(do: (() -> x))) == "(() -> x)" + assert macro_to_string(quote(do: (x when y -> z))) == "(x when y -> z)" + assert macro_to_string(quote(do: (x, y when z -> w))) == "((x, y) when z -> w)" + assert macro_to_string(quote(do: (x, y when z -> w))) == "((x, y) when z -> w)" + end + + test "nested" do + quoted = + quote do + defmodule Foo do + def foo do + 1 + 1 + end + end + end + + expected = """ + defmodule(Foo) do + def(foo) do + 1 + 1 + end end + """ + + assert macro_to_string(quoted) <> "\n" == expected end - """ - end - test :op_precedence_to_string do - assert Macro.to_string(quote do: (1 + 2) * (3 - 4)) == "(1 + 2) * (3 - 4)" - assert Macro.to_string(quote do: ((1 + 2) * 3) - 4) == "(1 + 2) * 3 - 4" - assert Macro.to_string(quote do: (1 + 2 + 3) == "(1 + 2 + 3)") - assert Macro.to_string(quote do: (1 + 2 - 3) == "(1 + 2 - 3)") - end + test "operator precedence" do + assert macro_to_string(quote(do: (1 + 2) * (3 - 4))) == "(1 + 2) * (3 - 4)" + assert macro_to_string(quote(do: (1 + 2) * 3 - 4)) == "(1 + 2) * 3 - 4" + assert macro_to_string(quote(do: 1 + 2 + 3)) == "1 + 2 + 3" + assert macro_to_string(quote(do: 1 + 2 - 3)) == "1 + 2 - 3" + end - test :containers_to_string do - assert Macro.to_string(quote do: {}) == "{}" - assert Macro.to_string(quote do: []) == "[]" - assert Macro.to_string(quote do: {1, 2, 3}) == "{1, 2, 3}" - assert Macro.to_string(quote do: [ 1, 2, 3 ]) == "[1, 2, 3]" - assert Macro.to_string(quote do: %{}) == "%{}" - assert Macro.to_string(quote do: %{:foo => :bar}) == "%{foo: :bar}" - assert Macro.to_string(quote do: %{{1,2} => [1,2,3]}) == "%{{1, 2} => [1, 2, 3]}" - assert Macro.to_string(quote do: %{map | "a" => "b"}) == "%{map | \"a\" => \"b\"}" - assert Macro.to_string(quote do: [ 1, 2, 3 ]) == "[1, 2, 3]" - assert Macro.to_string(quote do: << 1, 2, 3 >>) == "<<1, 2, 3>>" - assert Macro.to_string(quote do: << <<1>> >>) == "<< <<1>> >>" - end + test "capture operator" do + assert macro_to_string(quote(do: &foo/0)) == "&foo/0" + assert macro_to_string(quote(do: &Foo.foo/0)) == "&Foo.foo/0" + assert macro_to_string(quote(do: &(&1 + &2))) == "&(&1 + &2)" + assert macro_to_string(quote(do: & &1)) == "&(&1)" + assert macro_to_string(quote(do: & &1.(:x))) == "&(&1.(:x))" + assert macro_to_string(quote(do: (& &1).(:x))) == "(&(&1)).(:x)" + end - test :struct_to_string do - assert Macro.to_string(quote do: %Test{}) == "%Test{}" - assert Macro.to_string(quote do: %Test{foo: 1, bar: 1}) == "%Test{foo: 1, bar: 1}" - assert Macro.to_string(quote do: %Test{struct | foo: 2}) == "%Test{struct | foo: 2}" - assert Macro.to_string(quote do: %Test{} + 1) == "%Test{} + 1" - end + test "containers" do + assert macro_to_string(quote(do: {})) == "{}" + assert macro_to_string(quote(do: [])) == "[]" + assert macro_to_string(quote(do: {1, 2, 3})) == "{1, 2, 3}" + assert macro_to_string(quote(do: [1, 2, 3])) == "[1, 2, 3]" + assert macro_to_string(quote(do: ["Elixir.Foo": :bar])) == "[{Foo, :bar}]" + assert macro_to_string(quote(do: %{})) == "%{}" + assert macro_to_string(quote(do: %{:foo => :bar})) == "%{foo: :bar}" + assert macro_to_string(quote(do: %{:"Elixir.Foo" => :bar})) == "%{Foo => :bar}" + assert macro_to_string(quote(do: %{{1, 2} => [1, 2, 3]})) == "%{{1, 2} => [1, 2, 3]}" + assert macro_to_string(quote(do: %{map | "a" => "b"})) == "%{map | \"a\" => \"b\"}" + assert macro_to_string(quote(do: [1, 2, 3])) == "[1, 2, 3]" + end - test :binary_ops_to_string do - assert Macro.to_string(quote do: 1 + 2) == "1 + 2" - assert Macro.to_string(quote do: [ 1, 2 | 3 ]) == "[1, 2 | 3]" - assert Macro.to_string(quote do: [h|t] = [1, 2, 3]) == "[h | t] = [1, 2, 3]" - assert Macro.to_string(quote do: (x ++ y) ++ z) == "(x ++ y) ++ z" - end + test "struct" do + assert macro_to_string(quote(do: %Test{})) == "%Test{}" + assert macro_to_string(quote(do: %Test{foo: 1, bar: 1})) == "%Test{foo: 1, bar: 1}" + assert macro_to_string(quote(do: %Test{struct | foo: 2})) == "%Test{struct | foo: 2}" + assert macro_to_string(quote(do: %Test{} + 1)) == "%Test{} + 1" + assert macro_to_string(quote(do: %Test{foo(1)} + 2)) == "%Test{foo(1)} + 2" + end - test :unary_ops_to_string do - assert Macro.to_string(quote do: not 1) == "not 1" - assert Macro.to_string(quote do: not foo) == "not foo" - assert Macro.to_string(quote do: -1) == "-1" - assert Macro.to_string(quote do: !(foo > bar)) == "!(foo > bar)" - assert Macro.to_string(quote do: @foo(bar)) == "@foo(bar)" - assert Macro.to_string(quote do: identity(&1)) == "identity(&1)" - assert Macro.to_string(quote do: identity(&foo)) == "identity(&foo)" - end + test "binary operators" do + assert macro_to_string(quote(do: 1 + 2)) == "1 + 2" + assert macro_to_string(quote(do: [1, 2 | 3])) == "[1, 2 | 3]" + assert macro_to_string(quote(do: [h | t] = [1, 2, 3])) == "[h | t] = [1, 2, 3]" + assert macro_to_string(quote(do: (x ++ y) ++ z)) == "(x ++ y) ++ z" + assert macro_to_string(quote(do: (x +++ y) +++ z)) == "(x +++ y) +++ z" + end - test :access_to_string do - assert Macro.to_string(quote do: a[b]) == "a[b]" - assert Macro.to_string(quote do: a[1 + 2]) == "a[1 + 2]" - end + test "unary operators" do + assert macro_to_string(quote(do: not 1)) == "not(1)" + assert macro_to_string(quote(do: not foo)) == "not(foo)" + assert macro_to_string(quote(do: -1)) == "-1" + assert macro_to_string(quote(do: +(+1))) == "+(+1)" + assert macro_to_string(quote(do: !(foo > bar))) == "!(foo > bar)" + assert macro_to_string(quote(do: @foo(bar))) == "@foo(bar)" + assert macro_to_string(quote(do: identity(&1))) == "identity(&1)" + end - test :kw_list do - assert Macro.to_string(quote do: [a: a, b: b]) == "[a: a, b: b]" - assert Macro.to_string(quote do: [a: 1, b: 1 + 2]) == "[a: 1, b: 1 + 2]" - assert Macro.to_string(quote do: ["a.b": 1, c: 1 + 2]) == "[\"a.b\": 1, c: 1 + 2]" - end + test "access" do + assert macro_to_string(quote(do: a[b])) == "a[b]" + assert macro_to_string(quote(do: a[1 + 2])) == "a[1 + 2]" + assert macro_to_string(quote(do: (a || [a: 1])[:a])) == "(a || [a: 1])[:a]" + assert macro_to_string(quote(do: Map.put(%{}, :a, 1)[:a])) == "Map.put(%{}, :a, 1)[:a]" + end - test :string_list do - assert Macro.to_string(quote do: []) == "[]" - assert Macro.to_string(quote do: 'abc') == "'abc'" - end + test "keyword list" do + assert macro_to_string(quote(do: [a: a, b: b])) == "[a: a, b: b]" + assert macro_to_string(quote(do: [a: 1, b: 1 + 2])) == "[a: 1, b: 1 + 2]" + assert macro_to_string(quote(do: ["a.b": 1, c: 1 + 2])) == "[\"a.b\": 1, c: 1 + 2]" + end - test :last_arg_kw_list do - assert Macro.to_string(quote do: foo([])) == "foo([])" - assert Macro.to_string(quote do: foo(x: y)) == "foo(x: y)" - assert Macro.to_string(quote do: foo(x: 1 + 2)) == "foo(x: 1 + 2)" - assert Macro.to_string(quote do: foo(x: y, p: q)) == "foo(x: y, p: q)" - assert Macro.to_string(quote do: foo(a, x: y, p: q)) == "foo(a, x: y, p: q)" + test "interpolation" do + assert macro_to_string(quote(do: "foo#{bar}baz")) == ~S["foo#{bar}baz"] + end + + test "bit syntax" do + ast = quote(do: <<1::8*4>>) + assert macro_to_string(ast) == "<<1::8*4>>" + + ast = quote(do: @type(foo :: <<_::8, _::_*4>>)) + assert macro_to_string(ast) == "@type(foo :: <<_::8, _::_*4>>)" + + ast = quote(do: <<69 - 4::bits-size(8 - 4)-unit(1), 65>>) + assert macro_to_string(ast) == "<<69 - 4::bits-size(8 - 4)-unit(1), 65>>" + + ast = quote(do: <<(<<65>>), 65>>) + assert macro_to_string(ast) == "<<(<<65>>), 65>>" + + ast = quote(do: <<65, (<<65>>)>>) + assert macro_to_string(ast) == "<<65, (<<65>>)>>" + + ast = quote(do: for(<<(a::4 <- <<1, 2>>)>>, do: a)) + assert macro_to_string(ast) == "for(<<(a :: 4 <- <<1, 2>>)>>) do\n a\nend" + end + + test "charlist" do + assert macro_to_string(quote(do: [])) == "[]" + assert macro_to_string(quote(do: 'abc')) == "'abc'" + end + + test "string" do + assert macro_to_string(quote(do: "")) == ~S/""/ + assert macro_to_string(quote(do: "abc")) == ~S/"abc"/ + assert macro_to_string(quote(do: "#{"abc"}")) == ~S/"#{"abc"}"/ + end + + test "last arg keyword list" do + assert macro_to_string(quote(do: foo([]))) == "foo([])" + assert macro_to_string(quote(do: foo(x: y))) == "foo(x: y)" + assert macro_to_string(quote(do: foo(x: 1 + 2))) == "foo(x: 1 + 2)" + assert macro_to_string(quote(do: foo(x: y, p: q))) == "foo(x: y, p: q)" + assert macro_to_string(quote(do: foo(a, x: y, p: q))) == "foo(a, x: y, p: q)" + + assert macro_to_string(quote(do: {[]})) == "{[]}" + assert macro_to_string(quote(do: {[a: b]})) == "{[a: b]}" + assert macro_to_string(quote(do: {x, a: b})) == "{x, [a: b]}" + assert macro_to_string(quote(do: foo(else: a))) == "foo(else: a)" + assert macro_to_string(quote(do: foo(catch: a))) == "foo(catch: a)" + end - assert Macro.to_string(quote do: {[]}) == "{[]}" - assert Macro.to_string(quote do: {[a: b]}) == "{[a: b]}" - assert Macro.to_string(quote do: {x, a: b}) == "{x, [a: b]}" + test "with fun" do + assert macro_to_string(quote(do: foo(1, 2, 3)), fn _, string -> ":#{string}:" end) == + ":foo(:1:, :2:, :3:):" + + assert macro_to_string(quote(do: Bar.foo(1, 2, 3)), fn _, string -> ":#{string}:" end) == + "::Bar:.foo(:1:, :2:, :3:):" + end end - test :to_string_with_fun do - assert Macro.to_string(quote(do: foo(1, 2, 3)), fn _, string -> ":#{string}:" end) == - ":foo(:1:, :2:, :3:):" + test "validate/1" do + ref = make_ref() - assert Macro.to_string(quote(do: Bar.foo(1, 2, 3)), fn _, string -> ":#{string}:" end) == - "::Bar:.foo(:1:, :2:, :3:):" + assert Macro.validate(1) == :ok + assert Macro.validate(1.0) == :ok + assert Macro.validate(:foo) == :ok + assert Macro.validate("bar") == :ok + assert Macro.validate(<<0::8>>) == :ok + assert Macro.validate(self()) == :ok + assert Macro.validate({1, 2}) == :ok + assert Macro.validate({:foo, [], :baz}) == :ok + assert Macro.validate({:foo, [], []}) == :ok + assert Macro.validate([1, 2, 3]) == :ok + + assert Macro.validate(<<0::4>>) == {:error, <<0::4>>} + assert Macro.validate(ref) == {:error, ref} + assert Macro.validate({1, ref}) == {:error, ref} + assert Macro.validate({ref, 2}) == {:error, ref} + assert Macro.validate([1, ref, 3]) == {:error, ref} + assert Macro.validate({:foo, [], 0}) == {:error, {:foo, [], 0}} + assert Macro.validate({:foo, 0, []}) == {:error, {:foo, 0, []}} end - ## decompose_call - - test :decompose_call do - assert Macro.decompose_call(quote do: foo) == {:foo, []} - assert Macro.decompose_call(quote do: foo()) == {:foo, []} - assert Macro.decompose_call(quote do: foo(1, 2, 3)) == {:foo, [1, 2, 3]} - assert Macro.decompose_call(quote do: M.N.foo(1, 2, 3)) == - {{:__aliases__, [alias: false], [:M, :N]}, :foo, [1, 2, 3]} - assert Macro.decompose_call(quote do: :foo.foo(1, 2, 3)) == - {:foo, :foo, [1, 2, 3]} - assert Macro.decompose_call(quote do: 1.(1, 2, 3)) == :error - assert Macro.decompose_call(quote do: "some string") == :error + test "decompose_call/1" do + assert Macro.decompose_call(quote(do: foo)) == {:foo, []} + assert Macro.decompose_call(quote(do: foo())) == {:foo, []} + assert Macro.decompose_call(quote(do: foo(1, 2, 3))) == {:foo, [1, 2, 3]} + + assert Macro.decompose_call(quote(do: M.N.foo(1, 2, 3))) == + {{:__aliases__, [alias: false], [:M, :N]}, :foo, [1, 2, 3]} + + assert Macro.decompose_call(quote(do: :foo.foo(1, 2, 3))) == {:foo, :foo, [1, 2, 3]} + assert Macro.decompose_call(quote(do: 1.(1, 2, 3))) == :error + assert Macro.decompose_call(quote(do: "some string")) == :error + assert Macro.decompose_call(quote(do: {:foo, :bar, :baz})) == :error + assert Macro.decompose_call(quote(do: {:foo, :bar, :baz, 42})) == :error end - ## env + describe "env" do + doctest Macro.Env + + test "prune_compile_info" do + assert %Macro.Env{lexical_tracker: nil, tracers: []} = + Macro.Env.prune_compile_info(%{__ENV__ | lexical_tracker: self(), tracers: [Foo]}) + end - test :env_stacktrace do - env = %{__ENV__ | file: "foo", line: 12} - assert Macro.Env.stacktrace(env) == - [{__MODULE__, :"test env_stacktrace", 1, [file: "foo", line: 12]}] + test "stacktrace" do + env = %{__ENV__ | file: "foo", line: 12} - env = %{env | function: nil} - assert Macro.Env.stacktrace(env) == - [{__MODULE__, :__MODULE__, 0, [file: "foo", line: 12]}] + assert Macro.Env.stacktrace(env) == + [{__MODULE__, :"test env stacktrace", 1, [file: 'foo', line: 12]}] - env = %{env | module: nil} - assert Macro.Env.stacktrace(env) == - [{:elixir_compiler, :__FILE__, 1, [file: "foo", line: 12]}] - end + env = %{env | function: nil} + assert Macro.Env.stacktrace(env) == [{__MODULE__, :__MODULE__, 0, [file: 'foo', line: 12]}] - test :context_modules do - defmodule Foo.Bar do - assert __MODULE__ in __ENV__.context_modules + env = %{env | module: nil} + + assert Macro.Env.stacktrace(env) == + [{:elixir_compiler, :__FILE__, 1, [file: 'foo', line: 12]}] + end + + test "context modules" do + defmodule Foo.Bar do + assert __MODULE__ in __ENV__.context_modules + end + + assert Foo.Bar in __ENV__.context_modules + + Code.compile_string(""" + defmodule Foo.Bar.Compiled do + true = __MODULE__ in __ENV__.context_modules + end + """) + end + + test "to_match/1" do + quote = quote(do: x in []) + + assert {:__block__, [], [{:=, [], [{:_, [], Kernel}, {:x, [], MacroTest}]}, false]} = + Macro.expand_once(quote, __ENV__) + + assert Macro.expand_once(quote, Macro.Env.to_match(__ENV__)) == false + end + + test "prepend_tracer" do + assert %Macro.Env{tracers: [MyCustomTracer | _]} = + Macro.Env.prepend_tracer(__ENV__, MyCustomTracer) end end ## pipe/unpipe - test :pipe do + test "pipe/3" do assert Macro.pipe(1, quote(do: foo), 0) == quote(do: foo(1)) assert Macro.pipe(1, quote(do: foo(2)), 0) == quote(do: foo(1, 2)) assert Macro.pipe(1, quote(do: foo), -1) == quote(do: foo(1)) assert Macro.pipe(2, quote(do: foo(1)), -1) == quote(do: foo(1, 2)) - assert_raise ArgumentError, "cannot pipe 1 into 2", fn -> + assert_raise ArgumentError, ~r"cannot pipe 1 into 2", fn -> Macro.pipe(1, 2, 0) end + + assert_raise ArgumentError, ~r"cannot pipe 1 into \{2, 3\}", fn -> + Macro.pipe(1, {2, 3}, 0) + end + + assert_raise ArgumentError, ~r"cannot pipe 1 into 1 \+ 1, the :\+ operator can", fn -> + Macro.pipe(1, quote(do: 1 + 1), 0) == quote(do: foo(1)) + end + + assert_raise ArgumentError, ~r"cannot pipe 1 into <<1>>", fn -> + Macro.pipe(1, quote(do: <<1>>), 0) + end + + assert_raise ArgumentError, ~r"cannot pipe 1 into the special form unquote/1", fn -> + Macro.pipe(1, quote(do: unquote()), 0) + end + + assert_raise ArgumentError, ~r"piping into a unary operator is not supported", fn -> + Macro.pipe(1, quote(do: +1), 0) + end + + assert_raise ArgumentError, ~r"cannot pipe Macro into Env", fn -> + Macro.pipe(Macro, quote(do: Env), 0) + end + + assert_raise ArgumentError, ~r"cannot pipe 1 into 2 && 3", fn -> + Macro.pipe(1, quote(do: 2 && 3), 0) + end + + message = ~r"cannot pipe :foo into an anonymous function without calling" + + assert_raise ArgumentError, message, fn -> + Macro.pipe(:foo, quote(do: fn x -> x end), 0) + end end - test :unpipe do + test "unpipe/1" do assert Macro.unpipe(quote(do: foo)) == quote(do: [{foo, 0}]) assert Macro.unpipe(quote(do: foo |> bar)) == quote(do: [{foo, 0}, {bar, 0}]) assert Macro.unpipe(quote(do: foo |> bar |> baz)) == quote(do: [{foo, 0}, {bar, 0}, {baz, 0}]) end - ## pre/postwalk - - test :prewalk do - assert prewalk({:foo, [], nil}) == - [{:foo, [], nil}] - - assert prewalk({:foo, [], [1, 2, 3]}) == - [{:foo, [], [1, 2, 3]}, 1, 2, 3] + ## traverse/pre/postwalk + + test "traverse/4" do + assert traverse({:foo, [], nil}) == [{:foo, [], nil}, {:foo, [], nil}] + + assert traverse({:foo, [], [1, 2, 3]}) == + [{:foo, [], [1, 2, 3]}, 1, 1, 2, 2, 3, 3, {:foo, [], [1, 2, 3]}] + + assert traverse({{:., [], [:foo, :bar]}, [], [1, 2, 3]}) == + [ + {{:., [], [:foo, :bar]}, [], [1, 2, 3]}, + {:., [], [:foo, :bar]}, + :foo, + :foo, + :bar, + :bar, + {:., [], [:foo, :bar]}, + 1, + 1, + 2, + 2, + 3, + 3, + {{:., [], [:foo, :bar]}, [], [1, 2, 3]} + ] + + assert traverse({[1, 2, 3], [4, 5, 6]}) == + [ + {[1, 2, 3], [4, 5, 6]}, + [1, 2, 3], + 1, + 1, + 2, + 2, + 3, + 3, + [1, 2, 3], + [4, 5, 6], + 4, + 4, + 5, + 5, + 6, + 6, + [4, 5, 6], + {[1, 2, 3], [4, 5, 6]} + ] + end + + defp traverse(ast) do + Macro.traverse(ast, [], &{&1, [&1 | &2]}, &{&1, [&1 | &2]}) |> elem(1) |> Enum.reverse() + end + + test "prewalk/3" do + assert prewalk({:foo, [], nil}) == [{:foo, [], nil}] + + assert prewalk({:foo, [], [1, 2, 3]}) == [{:foo, [], [1, 2, 3]}, 1, 2, 3] assert prewalk({{:., [], [:foo, :bar]}, [], [1, 2, 3]}) == - [{{:., [], [:foo, :bar]}, [], [1, 2, 3]}, {:., [], [:foo, :bar]}, :foo, :bar, 1, 2, 3] + [ + {{:., [], [:foo, :bar]}, [], [1, 2, 3]}, + {:., [], [:foo, :bar]}, + :foo, + :bar, + 1, + 2, + 3 + ] assert prewalk({[1, 2, 3], [4, 5, 6]}) == - [{[1, 2, 3], [4, 5, 6]}, [1, 2, 3], 1, 2, 3, [4, 5, 6], 4, 5, 6] + [{[1, 2, 3], [4, 5, 6]}, [1, 2, 3], 1, 2, 3, [4, 5, 6], 4, 5, 6] end defp prewalk(ast) do - Macro.prewalk(ast, [], &{&1, [&1|&2]}) |> elem(1) |> Enum.reverse + Macro.prewalk(ast, [], &{&1, [&1 | &2]}) |> elem(1) |> Enum.reverse() end - test :postwalk do - assert postwalk({:foo, [], nil}) == - [{:foo, [], nil}] + test "postwalk/3" do + assert postwalk({:foo, [], nil}) == [{:foo, [], nil}] - assert postwalk({:foo, [], [1, 2, 3]}) == - [1, 2, 3, {:foo, [], [1, 2, 3]}] + assert postwalk({:foo, [], [1, 2, 3]}) == [1, 2, 3, {:foo, [], [1, 2, 3]}] assert postwalk({{:., [], [:foo, :bar]}, [], [1, 2, 3]}) == - [:foo, :bar, {:., [], [:foo, :bar]}, 1, 2, 3, {{:., [], [:foo, :bar]}, [], [1, 2, 3]}] + [ + :foo, + :bar, + {:., [], [:foo, :bar]}, + 1, + 2, + 3, + {{:., [], [:foo, :bar]}, [], [1, 2, 3]} + ] assert postwalk({[1, 2, 3], [4, 5, 6]}) == - [1, 2, 3, [1, 2, 3], 4, 5, 6, [4, 5, 6], {[1, 2, 3], [4, 5, 6]}] + [1, 2, 3, [1, 2, 3], 4, 5, 6, [4, 5, 6], {[1, 2, 3], [4, 5, 6]}] + end + + test "generate_arguments/2" do + assert Macro.generate_arguments(0, __MODULE__) == [] + assert Macro.generate_arguments(1, __MODULE__) == [{:arg1, [], __MODULE__}] + assert Macro.generate_arguments(4, __MODULE__) |> length == 4 end defp postwalk(ast) do - Macro.postwalk(ast, [], &{&1, [&1|&2]}) |> elem(1) |> Enum.reverse + Macro.postwalk(ast, [], &{&1, [&1 | &2]}) |> elem(1) |> Enum.reverse() + end + + test "struct!/2 expands structs multiple levels deep" do + defmodule StructBang do + defstruct [:a, :b] + + assert Macro.struct!(StructBang, __ENV__) == %{__struct__: StructBang, a: nil, b: nil} + + def within_function do + assert Macro.struct!(StructBang, __ENV__) == %{__struct__: StructBang, a: nil, b: nil} + end + + defmodule Nested do + assert Macro.struct!(StructBang, __ENV__) == %{__struct__: StructBang, a: nil, b: nil} + end + end + + assert Macro.struct!(StructBang, __ENV__) == %{__struct__: StructBang, a: nil, b: nil} + end + + test "prewalker/1" do + ast = quote do: :mod.foo(bar({1, 2}), [3, 4, five]) + map = Enum.map(Macro.prewalker(ast), & &1) + + assert map == [ + {{:., [], [:mod, :foo]}, [], [{:bar, [], [{1, 2}]}, [3, 4, {:five, [], MacroTest}]]}, + {:., [], [:mod, :foo]}, + :mod, + :foo, + {:bar, [], [{1, 2}]}, + {1, 2}, + 1, + 2, + [3, 4, {:five, [], MacroTest}], + 3, + 4, + {:five, [], MacroTest} + ] + + assert map == ast |> Macro.prewalk([], &{&1, [&1 | &2]}) |> elem(1) |> Enum.reverse() + assert Enum.zip(Macro.prewalker(ast), []) == Enum.zip(map, []) + + for i <- 0..(length(map) + 1) do + assert Enum.take(Macro.prewalker(ast), i) == Enum.take(map, i) + end + end + + test "postwalker/1" do + ast = quote do: :mod.foo(bar({1, 2}), [3, 4, five]) + map = Enum.map(Macro.postwalker(ast), & &1) + + assert map == [ + :mod, + :foo, + {:., [], [:mod, :foo]}, + 1, + 2, + {1, 2}, + {:bar, [], [{1, 2}]}, + 3, + 4, + {:five, [], MacroTest}, + [3, 4, {:five, [], MacroTest}], + {{:., [], [:mod, :foo]}, [], [{:bar, [], [{1, 2}]}, [3, 4, {:five, [], MacroTest}]]} + ] + + assert map == ast |> Macro.postwalk([], &{&1, [&1 | &2]}) |> elem(1) |> Enum.reverse() + assert Enum.zip(Macro.postwalker(ast), []) == Enum.zip(map, []) + + for i <- 0..(length(map) + 1) do + assert Enum.take(Macro.postwalker(ast), i) == Enum.take(map, i) + end + end + + test "operator?/2" do + assert Macro.operator?(:+, 2) + assert Macro.operator?(:+, 1) + refute Macro.operator?(:+, 0) + end + + test "quoted_literal?/1" do + assert Macro.quoted_literal?(quote(do: "foo")) + assert Macro.quoted_literal?(quote(do: {"foo", 1})) + assert Macro.quoted_literal?(quote(do: %{foo: "bar"})) + assert Macro.quoted_literal?(quote(do: %URI{path: "/"})) + refute Macro.quoted_literal?(quote(do: {"foo", var})) + end + + test "underscore/1" do + assert Macro.underscore("foo") == "foo" + assert Macro.underscore("foo_bar") == "foo_bar" + assert Macro.underscore("Foo") == "foo" + assert Macro.underscore("FooBar") == "foo_bar" + assert Macro.underscore("FOOBar") == "foo_bar" + assert Macro.underscore("FooBAR") == "foo_bar" + assert Macro.underscore("FOO_BAR") == "foo_bar" + assert Macro.underscore("FoBaZa") == "fo_ba_za" + assert Macro.underscore("Foo10") == "foo10" + assert Macro.underscore("FOO10") == "foo10" + assert Macro.underscore("10Foo") == "10_foo" + assert Macro.underscore("FooBar10") == "foo_bar10" + assert Macro.underscore("FooBAR10") == "foo_bar10" + assert Macro.underscore("Foo10Bar") == "foo10_bar" + assert Macro.underscore("Foo.Bar") == "foo/bar" + assert Macro.underscore(Foo.Bar) == "foo/bar" + assert Macro.underscore("API.V1.User") == "api/v1/user" + assert Macro.underscore("") == "" + end + + test "camelize/1" do + assert Macro.camelize("Foo") == "Foo" + assert Macro.camelize("FooBar") == "FooBar" + assert Macro.camelize("foo") == "Foo" + assert Macro.camelize("foo_bar") == "FooBar" + assert Macro.camelize("foo_") == "Foo" + assert Macro.camelize("_foo") == "Foo" + assert Macro.camelize("foo10") == "Foo10" + assert Macro.camelize("_10foo") == "10foo" + assert Macro.camelize("foo_10") == "Foo10" + assert Macro.camelize("foo__10") == "Foo10" + assert Macro.camelize("foo__bar") == "FooBar" + assert Macro.camelize("foo/bar") == "Foo.Bar" + assert Macro.camelize("Foo.Bar") == "Foo.Bar" + assert Macro.camelize("foo1_0") == "Foo10" + assert Macro.camelize("foo_123_4_567") == "Foo1234567" + assert Macro.camelize("FOO_BAR") == "FOO_BAR" + assert Macro.camelize("FOO.BAR") == "FOO.BAR" + assert Macro.camelize("") == "" end end diff --git a/lib/elixir/test/elixir/map_set_test.exs b/lib/elixir/test/elixir/map_set_test.exs new file mode 100644 index 00000000000..e8193ec609d --- /dev/null +++ b/lib/elixir/test/elixir/map_set_test.exs @@ -0,0 +1,171 @@ +Code.require_file("test_helper.exs", __DIR__) + +defmodule MapSetTest do + use ExUnit.Case, async: true + + doctest MapSet + + test "new/1" do + result = MapSet.new(1..5) + assert MapSet.equal?(result, Enum.into(1..5, MapSet.new())) + end + + test "new/2" do + result = MapSet.new(1..5, &(&1 + 2)) + assert MapSet.equal?(result, Enum.into(3..7, MapSet.new())) + end + + test "put/2" do + result = MapSet.put(MapSet.new(), 1) + assert MapSet.equal?(result, MapSet.new([1])) + + result = MapSet.put(MapSet.new([1, 3, 4]), 2) + assert MapSet.equal?(result, MapSet.new(1..4)) + + result = MapSet.put(MapSet.new(5..100), 10) + assert MapSet.equal?(result, MapSet.new(5..100)) + end + + test "union/2" do + result = MapSet.union(MapSet.new([1, 3, 4]), MapSet.new()) + assert MapSet.equal?(result, MapSet.new([1, 3, 4])) + + result = MapSet.union(MapSet.new(5..15), MapSet.new(10..25)) + assert MapSet.equal?(result, MapSet.new(5..25)) + + result = MapSet.union(MapSet.new(1..120), MapSet.new(1..100)) + assert MapSet.equal?(result, MapSet.new(1..120)) + end + + test "intersection/2" do + result = MapSet.intersection(MapSet.new(), MapSet.new(1..21)) + assert MapSet.equal?(result, MapSet.new()) + + result = MapSet.intersection(MapSet.new(1..21), MapSet.new(4..24)) + assert MapSet.equal?(result, MapSet.new(4..21)) + + result = MapSet.intersection(MapSet.new(2..100), MapSet.new(1..120)) + assert MapSet.equal?(result, MapSet.new(2..100)) + end + + test "difference/2" do + result = MapSet.difference(MapSet.new(2..20), MapSet.new()) + assert MapSet.equal?(result, MapSet.new(2..20)) + + result = MapSet.difference(MapSet.new(2..20), MapSet.new(1..21)) + assert MapSet.equal?(result, MapSet.new()) + + result = MapSet.difference(MapSet.new(1..101), MapSet.new(2..100)) + assert MapSet.equal?(result, MapSet.new([1, 101])) + end + + test "disjoint?/2" do + assert MapSet.disjoint?(MapSet.new(), MapSet.new()) + assert MapSet.disjoint?(MapSet.new(1..6), MapSet.new(8..20)) + refute MapSet.disjoint?(MapSet.new(1..6), MapSet.new(5..15)) + refute MapSet.disjoint?(MapSet.new(1..120), MapSet.new(1..6)) + end + + test "subset?/2" do + assert MapSet.subset?(MapSet.new(), MapSet.new()) + assert MapSet.subset?(MapSet.new(1..6), MapSet.new(1..10)) + assert MapSet.subset?(MapSet.new(1..6), MapSet.new(1..120)) + refute MapSet.subset?(MapSet.new(1..120), MapSet.new(1..6)) + end + + test "equal?/2" do + assert MapSet.equal?(MapSet.new(), MapSet.new()) + refute MapSet.equal?(MapSet.new(1..20), MapSet.new(2..21)) + assert MapSet.equal?(MapSet.new(1..120), MapSet.new(1..120)) + end + + test "delete/2" do + result = MapSet.delete(MapSet.new(), 1) + assert MapSet.equal?(result, MapSet.new()) + + result = MapSet.delete(MapSet.new(1..4), 5) + assert MapSet.equal?(result, MapSet.new(1..4)) + + result = MapSet.delete(MapSet.new(1..4), 1) + assert MapSet.equal?(result, MapSet.new(2..4)) + + result = MapSet.delete(MapSet.new(1..4), 2) + assert MapSet.equal?(result, MapSet.new([1, 3, 4])) + end + + test "size/1" do + assert MapSet.size(MapSet.new()) == 0 + assert MapSet.size(MapSet.new(5..15)) == 11 + assert MapSet.size(MapSet.new(2..100)) == 99 + end + + test "to_list/1" do + assert MapSet.to_list(MapSet.new()) == [] + + list = MapSet.to_list(MapSet.new(1..20)) + assert Enum.sort(list) == Enum.to_list(1..20) + + list = MapSet.to_list(MapSet.new(5..120)) + assert Enum.sort(list) == Enum.to_list(5..120) + end + + test "filter/2" do + result = MapSet.filter(MapSet.new([1, nil, 2, false]), & &1) + assert MapSet.equal?(result, MapSet.new(1..2)) + + result = MapSet.filter(MapSet.new(1..10), &(&1 < 2 or &1 > 9)) + assert MapSet.equal?(result, MapSet.new([1, 10])) + + result = MapSet.filter(MapSet.new(~w(A a B b)), fn x -> String.downcase(x) == x end) + assert MapSet.equal?(result, MapSet.new(~w(a b))) + end + + test "reject/2" do + result = MapSet.reject(MapSet.new(1..10), &(&1 < 8)) + assert MapSet.equal?(result, MapSet.new(8..10)) + + result = MapSet.reject(MapSet.new(["a", :b, 1, 1.0]), &is_integer/1) + assert MapSet.equal?(result, MapSet.new(["a", :b, 1.0])) + + result = MapSet.reject(MapSet.new(1..3), fn x -> rem(x, 2) == 0 end) + assert MapSet.equal?(result, MapSet.new([1, 3])) + end + + test "MapSet v1 compatibility" do + result = 1..5 |> map_set_v1() |> MapSet.new() + assert MapSet.equal?(result, MapSet.new(1..5)) + + result = MapSet.put(map_set_v1(1..5), 6) + assert MapSet.equal?(result, MapSet.new(1..6)) + + result = MapSet.union(map_set_v1(1..5), MapSet.new(6..10)) + assert MapSet.equal?(result, MapSet.new(1..10)) + + result = MapSet.intersection(map_set_v1(1..10), MapSet.new(6..15)) + assert MapSet.equal?(result, MapSet.new(6..10)) + + result = MapSet.difference(map_set_v1(1..10), MapSet.new(6..50)) + assert MapSet.equal?(result, MapSet.new(1..5)) + + result = MapSet.delete(map_set_v1(1..10), 1) + assert MapSet.equal?(result, MapSet.new(2..10)) + + assert MapSet.size(map_set_v1(1..5)) == 5 + assert MapSet.to_list(map_set_v1(1..5)) == Enum.to_list(1..5) + + assert MapSet.disjoint?(map_set_v1(1..5), MapSet.new(10..15)) + refute MapSet.disjoint?(map_set_v1(1..5), MapSet.new(5..10)) + + assert MapSet.subset?(map_set_v1(3..7), MapSet.new(1..10)) + refute MapSet.subset?(map_set_v1(7..12), MapSet.new(1..10)) + end + + test "inspect" do + assert inspect(MapSet.new([?a])) == "MapSet.new([97])" + end + + defp map_set_v1(enumerable) do + map = Map.from_keys(Enum.to_list(enumerable), true) + %{__struct__: MapSet, map: map} + end +end diff --git a/lib/elixir/test/elixir/map_test.exs b/lib/elixir/test/elixir/map_test.exs index a005bf0c5ba..149c3b4ffa1 100644 --- a/lib/elixir/test/elixir/map_test.exs +++ b/lib/elixir/test/elixir/map_test.exs @@ -1,52 +1,20 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule MapTest do use ExUnit.Case, async: true - defp empty_map do - %{} - end + doctest Map - defp two_items_map do - %{a: 1, b: 2} - end + @sample %{a: 1, b: 2} - @map %{a: 1, b: 2} + defp sample, do: @sample test "maps in attributes" do - assert @map == %{a: 1, b: 2} + assert @sample == %{a: 1, b: 2} end test "maps when quoted" do - assert (quote do - %{foo: 1} - end) == {:%{}, [], [{:foo, 1}]} - - assert (quote do - % - {foo: 1} - end) == {:%{}, [], [{:foo, 1}]} - end - - test "structs when quoted" do - assert (quote do - %User{foo: 1} - end) == {:%, [], [ - {:__aliases__, [alias: false], [:User]}, - {:%{}, [], [{:foo, 1}]} - ]} - - assert (quote do - % - User{foo: 1} - end) == {:%, [], [ - {:__aliases__, [alias: false], [:User]}, - {:%{}, [], [{:foo, 1}]} - ]} - - assert (quote do - %unquote(User){foo: 1} - end) == {:%, [], [User, {:%{}, [], [{:foo, 1}]}]} + assert quote(do: %{foo: 1}) == {:%{}, [], [{:foo, 1}]} end test "maps keywords and atoms" do @@ -60,76 +28,418 @@ defmodule MapTest do assert a == 1 end + test "maps with generated variables in key" do + assert %{"#{1}" => 1} == %{"1" => 1} + assert %{for(x <- 1..3, do: x) => 1} == %{[1, 2, 3] => 1} + assert %{with(x = 1, do: x) => 1} == %{1 => 1} + assert %{with({:ok, x} <- {:ok, 1}, do: x) => 1} == %{1 => 1} + + assert %{ + try do + raise "error" + rescue + _ -> 1 + end => 1 + } == %{1 => 1} + + assert %{ + try do + throw(1) + catch + x -> x + end => 1 + } == %{1 => 1} + + assert %{ + try do + a = 1 + a + rescue + _ -> 2 + end => 1 + } == %{1 => 1} + + assert %{ + try do + 1 + rescue + _exception -> :exception + else + a -> a + end => 1 + } == %{1 => 1} + end + + test "matching with map as a key" do + assert %{%{1 => 2} => x} = %{%{1 => 2} => 3} + assert x == 3 + end + test "is_map/1" do - assert is_map empty_map - refute is_map(Enum.to_list(empty_map)) + assert is_map(Map.new()) + refute is_map(Enum.to_list(%{})) end test "map_size/1" do - assert map_size(empty_map) == 0 - assert map_size(two_items_map) == 2 + assert map_size(%{}) == 0 + assert map_size(@sample) == 2 end - test "maps with optional comma" do - assert %{a: :b,} == %{a: :b} - assert %{1 => 2,} == %{1 => 2} - assert %{1 => 2, a: :b,} == %{1 => 2, a: :b} + test "new/1" do + assert Map.new(%{a: 1, b: 2}) == %{a: 1, b: 2} + assert Map.new(MapSet.new(a: 1, b: 2, a: 3)) == %{b: 2, a: 3} + end + + test "new/2" do + transformer = fn {key, value} -> {key, value * 2} end + assert Map.new(%{a: 1, b: 2}, transformer) == %{a: 2, b: 4} + assert Map.new(MapSet.new(a: 1, b: 2, a: 3), transformer) == %{b: 4, a: 6} + end + + test "take/2" do + assert Map.take(%{a: 1, b: 2, c: 3}, [:b, :c]) == %{b: 2, c: 3} + assert Map.take(%{a: 1, b: 2, c: 3}, []) == %{} + assert_raise BadMapError, fn -> Map.take(:foo, []) end end - test "maps with duplicate keys" do - assert %{a: :b, a: :c} == %{a: :c} - assert %{1 => 2, 1 => 3} == %{1 => 3} - assert %{:a => :b, a: :c} == %{a: :c} + test "drop/2" do + assert Map.drop(%{a: 1, b: 2, c: 3}, [:b, :c]) == %{a: 1} + assert_raise BadMapError, fn -> Map.drop(:foo, []) end + end + + test "split/2" do + assert Map.split(%{a: 1, b: 2, c: 3}, [:b, :c]) == {%{b: 2, c: 3}, %{a: 1}} + assert_raise BadMapError, fn -> Map.split(:foo, []) end + end + + test "get_and_update/3" do + message = "the given function must return a two-element tuple or :pop, got: 1" + + assert_raise RuntimeError, message, fn -> + Map.get_and_update(%{a: 1}, :a, fn value -> value end) + end + end + + test "get_and_update!/3" do + message = "the given function must return a two-element tuple or :pop, got: 1" + + assert_raise RuntimeError, message, fn -> + Map.get_and_update!(%{a: 1}, :a, fn value -> value end) + end + end + + test "maps with optional comma" do + assert Code.eval_string("%{a: :b,}") == {%{a: :b}, []} + assert Code.eval_string("%{1 => 2,}") == {%{1 => 2}, []} + assert Code.eval_string("%{1 => 2, a: :b,}") == {%{1 => 2, a: :b}, []} end test "update maps" do - assert %{two_items_map | a: 3} == %{a: 3, b: 2} + assert %{sample() | a: 3} == %{a: 3, b: 2} - assert_raise ArgumentError, fn -> - %{two_items_map | c: 3} + assert_raise KeyError, fn -> + %{sample() | c: 3} end end - test "map access" do - assert two_items_map.a == 1 + test "map dot access" do + assert sample().a == 1 assert_raise KeyError, fn -> - two_items_map.c + sample().c + end + end + + test "put/3 optimized by the compiler" do + map = %{a: 1, b: 2} + + assert Map.put(map, :a, 2) == %{a: 2, b: 2} + assert Map.put(map, :c, 3) == %{a: 1, b: 2, c: 3} + + assert Map.put(%{map | a: 2}, :a, 3) == %{a: 3, b: 2} + assert Map.put(%{map | a: 2}, :b, 3) == %{a: 2, b: 3} + + assert Map.put(map, :a, 2) |> Map.put(:a, 3) == %{a: 3, b: 2} + assert Map.put(map, :a, 2) |> Map.put(:c, 3) == %{a: 2, b: 2, c: 3} + assert Map.put(map, :c, 3) |> Map.put(:a, 2) == %{a: 2, b: 2, c: 3} + assert Map.put(map, :c, 3) |> Map.put(:c, 4) == %{a: 1, b: 2, c: 4} + end + + test "merge/2 with map literals optimized by the compiler" do + map = %{a: 1, b: 2} + + assert Map.merge(map, %{a: 2}) == %{a: 2, b: 2} + assert Map.merge(map, %{c: 3}) == %{a: 1, b: 2, c: 3} + assert Map.merge(%{a: 2}, map) == %{a: 1, b: 2} + assert Map.merge(%{c: 3}, map) == %{a: 1, b: 2, c: 3} + + assert Map.merge(%{map | a: 2}, %{a: 3}) == %{a: 3, b: 2} + assert Map.merge(%{map | a: 2}, %{b: 3}) == %{a: 2, b: 3} + assert Map.merge(%{a: 2}, %{map | a: 3}) == %{a: 3, b: 2} + assert Map.merge(%{a: 2}, %{map | b: 3}) == %{a: 1, b: 3} + + assert Map.merge(map, %{a: 2}) |> Map.merge(%{a: 3, c: 3}) == %{a: 3, b: 2, c: 3} + assert Map.merge(map, %{c: 3}) |> Map.merge(%{c: 4}) == %{a: 1, b: 2, c: 4} + assert Map.merge(map, %{a: 3, c: 3}) |> Map.merge(%{a: 2}) == %{a: 2, b: 2, c: 3} + end + + test "merge/3" do + # When first map is bigger + assert Map.merge(%{a: 1, b: 2, c: 3}, %{c: 4, d: 5}, fn :c, 3, 4 -> :x end) == + %{a: 1, b: 2, c: :x, d: 5} + + # When second map is bigger + assert Map.merge(%{b: 2, c: 3}, %{a: 1, c: 4, d: 5}, fn :c, 3, 4 -> :x end) == + %{a: 1, b: 2, c: :x, d: 5} + end + + test "replace/3" do + map = %{c: 3, b: 2, a: 1} + assert Map.replace(map, :b, 10) == %{c: 3, b: 10, a: 1} + assert Map.replace(map, :a, 1) == map + assert Map.replace(map, :x, 1) == map + assert Map.replace(%{}, :x, 1) == %{} + end + + test "replace!/3" do + map = %{c: 3, b: 2, a: 1} + assert Map.replace!(map, :b, 10) == %{c: 3, b: 10, a: 1} + assert Map.replace!(map, :a, 1) == map + + assert_raise KeyError, "key :x not found in: %{a: 1, b: 2, c: 3}", fn -> + Map.replace!(map, :x, 10) + end + + assert_raise KeyError, "key :x not found in: %{}", fn -> + Map.replace!(%{}, :x, 10) end end + test "implements (almost) all functions in Keyword" do + assert Keyword.__info__(:functions) -- Map.__info__(:functions) == [ + delete: 3, + delete_first: 2, + get_values: 2, + keyword?: 1, + pop_first: 2, + pop_first: 3, + pop_values: 2, + validate: 2, + validate!: 2 + ] + end + + test "variable keys" do + x = :key + %{^x => :value} = %{x => :value} + assert %{x => :value} == %{key: :value} + assert (fn %{^x => :value} -> true end).(%{key: :value}) + + map = %{x => :value} + assert %{map | x => :new_value} == %{x => :new_value} + end + defmodule ExternalUser do def __struct__ do - %{__struct__: ThisDoesNotLeak, name: "josé", age: 27} + %{__struct__: __MODULE__, name: "john", age: 27} + end + + def __struct__(kv) do + Enum.reduce(kv, __struct__(), fn {k, v}, acc -> :maps.update(k, v, acc) end) end end + defp empty_map(), do: %{} + test "structs" do - assert %ExternalUser{} == - %{__struct__: ExternalUser, name: "josé", age: 27} + assert %ExternalUser{} == %{__struct__: ExternalUser, name: "john", age: 27} - assert %ExternalUser{name: "valim"} == - %{__struct__: ExternalUser, name: "valim", age: 27} + assert %ExternalUser{name: "meg"} == %{__struct__: ExternalUser, name: "meg", age: 27} user = %ExternalUser{} - assert %ExternalUser{user | name: "valim"} == - %{__struct__: ExternalUser, name: "valim", age: 27} + assert %ExternalUser{user | name: "meg"} == %{__struct__: ExternalUser, name: "meg", age: 27} %ExternalUser{name: name} = %ExternalUser{} - assert name == "josé" + assert name == "john" - map = %{} assert_raise BadStructError, "expected a struct named MapTest.ExternalUser, got: %{}", fn -> - %ExternalUser{map | name: "valim"} + %ExternalUser{empty_map() | name: "meg"} + end + end + + describe "structs with variable name" do + test "extracts the struct module" do + %module{name: "john"} = %ExternalUser{name: "john", age: 27} + assert module == ExternalUser + end + + test "returns the struct on match" do + assert Code.eval_string("%struct{} = %ExternalUser{}", [], __ENV__) == + {%ExternalUser{}, [struct: ExternalUser]} + end + + test "supports the pin operator" do + module = ExternalUser + user = %ExternalUser{name: "john", age: 27} + %^module{name: "john"} = user + end + + test "is supported in case" do + user = %ExternalUser{name: "john", age: 27} + + case user do + %module{} = %{age: 27} -> assert module == ExternalUser + end + end + + defp foo(), do: "foo" + defp destruct1(%module{}), do: module + defp destruct2(%_{}), do: :ok + + test "does not match" do + invalid_struct = %{__struct__: foo()} + + assert_raise CaseClauseError, fn -> + case invalid_struct do + %module{} -> module + end + end + + assert_raise CaseClauseError, fn -> + case invalid_struct do + %_{} -> :ok + end + end + + assert_raise CaseClauseError, fn -> + foo = foo() + + case invalid_struct do + %^foo{} -> :ok + end + end + + assert_raise FunctionClauseError, fn -> + destruct1(invalid_struct) + end + + assert_raise FunctionClauseError, fn -> + destruct2(invalid_struct) + end + + assert_raise MatchError, fn -> + %module{} = invalid_struct + _ = module + end + + assert_raise MatchError, fn -> + %_{} = invalid_struct + end + + assert_raise MatchError, fn -> + foo = foo() + %^foo{} = invalid_struct + end + end + end + + test "structs when using dynamic modules" do + defmodule Module.concat(MapTest, DynamicUser) do + defstruct [:name, :age] + + def sample do + %__MODULE__{} + end + end + end + + test "structs when quoted" do + quoted = + quote do + %User{foo: 1} + end + + assert {:%, [], [aliases, {:%{}, [], [{:foo, 1}]}]} = quoted + assert aliases == {:__aliases__, [alias: false], [:User]} + + quoted = + quote do + %unquote(User){foo: 1} + end + + assert quoted == {:%, [], [User, {:%{}, [], [{:foo, 1}]}]} + end + + test "defstruct can only be used once in a module" do + message = + "defstruct has already been called for TestMod, " <> + "defstruct can only be called once per module" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule TestMod do + defstruct [:foo] + defstruct [:foo] + end + """) end end + test "defstruct allows keys to be enforced" do + message = "the following keys must also be given when building struct TestMod: [:foo]" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule TestMod do + @enforce_keys :foo + defstruct [:foo] + def foo do + %TestMod{} + end + end + """) + end + end + + test "defstruct raises on invalid enforce_keys" do + message = "keys given to @enforce_keys must be atoms, got: \"foo\"" + + assert_raise ArgumentError, message, fn -> + Code.eval_string(""" + defmodule TestMod do + @enforce_keys "foo" + defstruct [:foo] + end + """) + end + end + + test "struct always expands context module" do + Code.compiler_options(ignore_module_conflict: true) + + defmodule LocalPoint do + defstruct x: 0 + def new, do: %LocalPoint{} + end + + assert LocalPoint.new() == %{__struct__: LocalPoint, x: 0} + + defmodule LocalPoint do + defstruct x: 0, y: 0 + def new, do: %LocalPoint{} + end + + assert LocalPoint.new() == %{__struct__: LocalPoint, x: 0, y: 0} + after + Code.compiler_options(ignore_module_conflict: false) + end + defmodule LocalUser do defmodule NestedUser do defstruct [] end - defstruct name: "josé", nested: struct(NestedUser) + defstruct name: "john", nested: struct(NestedUser), context: %{} def new do %LocalUser{} @@ -142,16 +452,18 @@ defmodule MapTest do end end - test "local user" do - assert LocalUser.new == %LocalUser{name: "josé", nested: %LocalUser.NestedUser{}} - assert LocalUser.Context.new == %LocalUser{name: "josé", nested: %LocalUser.NestedUser{}} + test "local and nested structs" do + assert LocalUser.new() == %LocalUser{name: "john", nested: %LocalUser.NestedUser{}} + assert LocalUser.Context.new() == %LocalUser{name: "john", nested: %LocalUser.NestedUser{}} end - defmodule NilUser do - defstruct name: nil, contents: %{} + defmodule :elixir_struct_from_erlang_module do + defstruct [:hello] + def world(%:elixir_struct_from_erlang_module{} = struct), do: struct end - test "nil user" do - assert %NilUser{} == %{__struct__: NilUser, name: nil, contents: %{}} + test "struct from erlang module" do + struct = %:elixir_struct_from_erlang_module{} + assert :elixir_struct_from_erlang_module.world(struct) == struct end end diff --git a/lib/elixir/test/elixir/module/locals_tracker_test.exs b/lib/elixir/test/elixir/module/locals_tracker_test.exs index d24f7eb497d..d16ac729639 100644 --- a/lib/elixir/test/elixir/module/locals_tracker_test.exs +++ b/lib/elixir/test/elixir/module/locals_tracker_test.exs @@ -1,4 +1,4 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Module.LocalsTrackerTest do use ExUnit.Case, async: true @@ -6,145 +6,119 @@ defmodule Module.LocalsTrackerTest do alias Module.LocalsTracker, as: D setup do - {:ok, [pid: D.start_link]} + set = :ets.new(__MODULE__, [:set, :public]) + bag = :ets.new(__MODULE__, [:duplicate_bag, :public]) + [ref: {set, bag}] end ## Locals - test "can add definitions", config do - D.add_definition(config[:pid], :def, {:foo, 1}) - D.add_definition(config[:pid], :defp, {:bar, 1}) + test "functions are reachable when connected through another one", config do + D.add_local(config[:ref], {:public, 1}, {:private, 1}, [line: 1], false) + assert {:private, 1} in D.reachable_from(config[:ref], {:public, 1}) end - test "can add locals", config do - D.add_definition(config[:pid], :def, {:foo, 1}) - D.add_local(config[:pid], {:foo, 1}, {:bar, 1}) - end - - test "public definitions are always reachable", config do - D.add_definition(config[:pid], :def, {:public, 1}) - assert {:public, 1} in D.reachable(config[:pid]) - - D.add_definition(config[:pid], :defmacro, {:public, 2}) - assert {:public, 2} in D.reachable(config[:pid]) - end - - test "private definitions are never reachable", config do - D.add_definition(config[:pid], :defp, {:private, 1}) - refute {:private, 1} in D.reachable(config[:pid]) - - D.add_definition(config[:pid], :defmacrop, {:private, 2}) - refute {:private, 2} in D.reachable(config[:pid]) - end - - test "private definitions are reachable when connected to local", config do - D.add_definition(config[:pid], :defp, {:private, 1}) - refute {:private, 1} in D.reachable(config[:pid]) - - D.add_local(config[:pid], {:private, 1}) - assert {:private, 1} in D.reachable(config[:pid]) - end + test "can yank and reattach nodes", config do + D.add_local(config[:ref], {:foo, 1}, {:bar, 1}, [line: 1], false) - test "private definitions are reachable when connected through a public one", config do - D.add_definition(config[:pid], :defp, {:private, 1}) - refute {:private, 1} in D.reachable(config[:pid]) + outfoo = D.yank(config[:ref], {:foo, 1}) + outbar = D.yank(config[:ref], {:bar, 1}) - D.add_definition(config[:pid], :def, {:public, 1}) - D.add_local(config[:pid], {:public, 1}, {:private, 1}) - assert {:private, 1} in D.reachable(config[:pid]) + D.reattach(config[:ref], {:bar, 1}, :defp, {:bar, 1}, outbar, line: 2) + D.reattach(config[:ref], {:foo, 1}, :def, {:foo, 1}, outfoo, line: 3) + assert {:bar, 1} in D.reachable_from(config[:ref], {:foo, 1}) end - @unused [ - {{:private, 1}, :defp, 0} + @used [ + {{:public, 1}, :def, [], 0} ] test "unused private definitions are marked as so", config do - D.add_definition(config[:pid], :def, {:public, 1}) + D.add_local(config[:ref], {:public, 1}, {:private, 1}, [line: 1], false) - unused = D.collect_unused_locals(config[:pid], @unused) - assert unused == [{:unused_def, {:private, 1}, :defp}] + unused = D.collect_unused_locals(config[:ref], @used, [{{:private, 0}, :defp, [], 0}]) + assert unused == {[private: 0], [{[], {:unused_def, {:private, 0}, :defp}}]} - D.add_local(config[:pid], {:public, 1}, {:private, 1}) - unused = D.collect_unused_locals(config[:pid], @unused) - refute unused == [{:unused_def, {:private, 1}, :defp}] + unused = D.collect_unused_locals(config[:ref], @used, [{{:private, 1}, :defp, [], 0}]) + assert unused == {[], []} end @unused [ - {{:private, 3}, :defp, 3} + {{:private, 3}, :defp, [], 3} ] test "private definitions with unused default arguments", config do - D.add_definition(config[:pid], :def, {:public, 1}) - - unused = D.collect_unused_locals(config[:pid], @unused) - assert unused == [{:unused_def, {:private, 3}, :defp}] + unused = D.collect_unused_locals(config[:ref], @used, @unused) + assert unused == {[private: 3], [{[], {:unused_def, {:private, 3}, :defp}}]} - D.add_local(config[:pid], {:public, 1}, {:private, 3}) - unused = D.collect_unused_locals(config[:pid], @unused) - assert unused == [{:unused_args, {:private, 3}}] + D.add_local(config[:ref], {:public, 1}, {:private, 3}, [line: 1], false) + unused = D.collect_unused_locals(config[:ref], @used, @unused) + assert unused == {[], [{[], {:unused_args, {:private, 3}}}]} end test "private definitions with some unused default arguments", config do - D.add_definition(config[:pid], :def, {:public, 1}) - D.add_local(config[:pid], {:public, 1}, {:private, 1}) - unused = D.collect_unused_locals(config[:pid], @unused) - assert unused == [{:unused_args, {:private, 3}, 1}] + D.add_local(config[:ref], {:public, 1}, {:private, 1}, [line: 1], false) + unused = D.collect_unused_locals(config[:ref], @used, @unused) + assert unused == {[private: 3], [{[], {:unused_args, {:private, 3}, 1}}]} end test "private definitions with all used default arguments", config do - D.add_definition(config[:pid], :def, {:public, 1}) - D.add_local(config[:pid], {:public, 1}, {:private, 0}) - unused = D.collect_unused_locals(config[:pid], @unused) - assert unused == [] + D.add_local(config[:ref], {:public, 1}, {:private, 0}, [line: 1], false) + unused = D.collect_unused_locals(config[:ref], @used, @unused) + assert unused == {[private: 3], []} end - ## Defaults + ### Undefined functions - test "can add defaults", config do - D.add_definition(config[:pid], :def, {:foo, 4}) - D.add_defaults(config[:pid], :def, {:foo, 4}, 2) - end + test "undefined functions are marked as so", config do + D.add_local(config[:ref], {:public, 1}, {:private, 1}, [line: 1], false) - test "defaults are reachable if public", config do - D.add_definition(config[:pid], :def, {:foo, 4}) - D.add_defaults(config[:pid], :def, {:foo, 4}, 2) - assert {:foo, 2} in D.reachable(config[:pid]) - assert {:foo, 3} in D.reachable(config[:pid]) + undefined = D.collect_undefined_locals(config[:ref], @used) + assert undefined == [{[line: 1], {:private, 1}, :undefined_function}] end - test "defaults are not reachable if private", config do - D.add_definition(config[:pid], :defp, {:foo, 4}) - D.add_defaults(config[:pid], :defp, {:foo, 4}, 2) - refute {:foo, 2} in D.reachable(config[:pid]) - refute {:foo, 3} in D.reachable(config[:pid]) - end + ### Incorrect dispatches - test "defaults are connected", config do - D.add_definition(config[:pid], :defp, {:foo, 4}) - D.add_defaults(config[:pid], :defp, {:foo, 4}, 2) - D.add_local(config[:pid], {:foo, 2}) - assert {:foo, 2} in D.reachable(config[:pid]) - assert {:foo, 3} in D.reachable(config[:pid]) - assert {:foo, 4} in D.reachable(config[:pid]) + test "incorrect dispatches are marked as so", config do + {set, _bag} = config[:ref] + :ets.insert(set, {{:def, {:macro, 1}}, :defmacro, [], "nofile", false, {0, true, 0}}) + definitions = [{{:public, 1}, :def, [], 0}, {{:macro, 1}, :defmacro, [], 0}] + + D.add_local(config[:ref], {:public, 1}, {:macro, 1}, [line: 5], false) + + undefined = D.collect_undefined_locals(config[:ref], definitions) + assert undefined == [{[line: 5], {:macro, 1}, :incorrect_dispatch}] end - ## Imports + ## Defaults - test "find imports from dispatch", config do - D.add_import(config[:pid], nil, Module, {:concat, 1}) - assert Module in D.imports_with_dispatch(config[:pid], {:concat, 1}) - refute Module in D.imports_with_dispatch(config[:pid], {:unknown, 1}) + test "defaults are connected to last clause only", config do + D.add_defaults(config[:ref], :defp, {:foo, 4}, 2, line: 1) + D.add_local(config[:ref], {:public, 1}, {:foo, 2}, [line: 2], false) + assert {:foo, 2} in D.reachable_from(config[:ref], {:public, 1}) + refute {:foo, 3} in D.reachable_from(config[:ref], {:public, 1}) + assert {:foo, 4} in D.reachable_from(config[:ref], {:public, 1}) end + ## Imports + test "find import conflicts", config do - refute {[Module], :conflict, 1} in D.collect_imports_conflicts(config[:pid], [conflict: 1]) + entries = [{{:conflict, 1}, :def, [], []}] + refute {[], {Module, {:conflict, 1}}} in D.collect_imports_conflicts(config[:ref], entries) - # Calls outside local functions are not triggered - D.add_import(config[:pid], nil, Module, {:conflict, 1}) - refute {[Module], :conflict, 1} in D.collect_imports_conflicts(config[:pid], [conflict: 1]) + D.add_local(config[:ref], {:public, 1}, {:foo, 2}, [line: 1], false) + D.add_import(config[:ref], {:foo, 2}, Module, {:conflict, 1}) + D.add_import(config[:ref], {:foo, 2}, Module, {:conflict, 1}) + assert {[], {Module, {:conflict, 1}}} in D.collect_imports_conflicts(config[:ref], entries) + end + + defmodule NoPrivate do + defmacrop foo(), do: bar() + defp bar(), do: :baz + def baz(), do: foo() + end - D.add_local(config[:pid], {:foo, 2}) - D.add_import(config[:pid], {:foo, 2}, Module, {:conflict, 1}) - assert {[Module], :conflict, 1} in D.collect_imports_conflicts(config[:pid], [conflict: 1]) + test "does not include unreachable locals" do + assert NoPrivate.module_info(:functions) |> Keyword.take([:foo, :bar, :"MACRO-foo"]) == [] end end diff --git a/lib/elixir/test/elixir/module/types/expr_test.exs b/lib/elixir/test/elixir/module/types/expr_test.exs new file mode 100644 index 00000000000..ec1b1872812 --- /dev/null +++ b/lib/elixir/test/elixir/module/types/expr_test.exs @@ -0,0 +1,303 @@ +Code.require_file("type_helper.exs", __DIR__) + +defmodule Module.Types.ExprTest do + use ExUnit.Case, async: true + + import TypeHelper + + defmodule :"Elixir.Module.Types.ExprTest.Struct" do + defstruct foo: :atom, bar: 123, baz: %{} + end + + test "literal" do + assert quoted_expr(true) == {:ok, {:atom, true}} + assert quoted_expr(false) == {:ok, {:atom, false}} + assert quoted_expr(:foo) == {:ok, {:atom, :foo}} + assert quoted_expr(0) == {:ok, :integer} + assert quoted_expr(0.0) == {:ok, :float} + assert quoted_expr("foo") == {:ok, :binary} + end + + describe "list" do + test "proper" do + assert quoted_expr([]) == {:ok, {:list, :dynamic}} + assert quoted_expr([123]) == {:ok, {:list, :integer}} + assert quoted_expr([123, 456]) == {:ok, {:list, :integer}} + assert quoted_expr([123 | []]) == {:ok, {:list, :integer}} + assert quoted_expr([123, "foo"]) == {:ok, {:list, {:union, [:integer, :binary]}}} + assert quoted_expr([123 | ["foo"]]) == {:ok, {:list, {:union, [:integer, :binary]}}} + end + + test "improper" do + assert quoted_expr([123 | 456]) == {:ok, {:list, :integer}} + assert quoted_expr([123, 456 | 789]) == {:ok, {:list, :integer}} + assert quoted_expr([123 | "foo"]) == {:ok, {:list, {:union, [:integer, :binary]}}} + end + + test "keyword" do + assert quoted_expr(a: 1, b: 2) == + {:ok, + {:list, + {:union, + [ + {:tuple, 2, [{:atom, :a}, :integer]}, + {:tuple, 2, [{:atom, :b}, :integer]} + ]}}} + end + end + + test "tuple" do + assert quoted_expr({}) == {:ok, {:tuple, 0, []}} + assert quoted_expr({:a}) == {:ok, {:tuple, 1, [{:atom, :a}]}} + assert quoted_expr({:a, 123}) == {:ok, {:tuple, 2, [{:atom, :a}, :integer]}} + end + + # Use module attribute to avoid formatter adding parentheses + @mix_module Mix + + test "module call" do + assert quoted_expr(@mix_module.shell) == {:ok, :dynamic} + assert quoted_expr(@mix_module.shell.info) == {:ok, {:var, 0}} + end + + describe "binary" do + test "literal" do + assert quoted_expr(<<"foo"::binary>>) == {:ok, :binary} + assert quoted_expr(<<123::integer>>) == {:ok, :binary} + assert quoted_expr(<<123::utf8>>) == {:ok, :binary} + assert quoted_expr(<<"foo"::utf8>>) == {:ok, :binary} + end + + test "variable" do + assert quoted_expr([foo], <>) == {:ok, :binary} + assert quoted_expr([foo], <>) == {:ok, :binary} + assert quoted_expr([foo], <>) == {:ok, :binary} + assert quoted_expr([foo], <>) == {:ok, :binary} + assert quoted_expr([foo], <>) == {:ok, :binary} + end + + test "infer" do + assert quoted_expr( + ( + foo = 0.0 + <> + ) + ) == {:ok, :binary} + + assert quoted_expr( + ( + foo = 0 + <> + ) + ) == {:ok, :binary} + + assert quoted_expr([foo], {<>, foo}) == + {:ok, {:tuple, 2, [:binary, :integer]}} + + assert quoted_expr([foo], {<>, foo}) == {:ok, {:tuple, 2, [:binary, :binary]}} + + assert quoted_expr([foo], {<>, foo}) == + {:ok, {:tuple, 2, [:binary, {:union, [:integer, :binary]}]}} + + assert {:error, {:unable_unify, {:integer, :binary, _}}} = + quoted_expr( + ( + foo = 0 + <> + ) + ) + + assert {:error, {:unable_unify, {:binary, :integer, _}}} = + quoted_expr([foo], <>) + end + end + + test "variables" do + assert quoted_expr([foo], foo) == {:ok, {:var, 0}} + assert quoted_expr([foo], {foo}) == {:ok, {:tuple, 1, [{:var, 0}]}} + assert quoted_expr([foo, bar], {foo, bar}) == {:ok, {:tuple, 2, [{:var, 0}, {:var, 1}]}} + end + + test "pattern match" do + assert {:error, _} = quoted_expr(:foo = 1) + assert {:error, _} = quoted_expr(1 = :foo) + + assert quoted_expr(:foo = :foo) == {:ok, {:atom, :foo}} + assert quoted_expr(1 = 1) == {:ok, :integer} + end + + test "block" do + assert quoted_expr( + ( + a = 1 + a + ) + ) == {:ok, :integer} + + assert quoted_expr( + ( + a = :foo + a + ) + ) == {:ok, {:atom, :foo}} + + assert {:error, _} = + quoted_expr( + ( + a = 1 + :foo = a + ) + ) + end + + describe "case" do + test "infer pattern" do + assert quoted_expr( + [a], + case a do + :foo = b -> :foo = b + end + ) == {:ok, :dynamic} + + assert {:error, _} = + quoted_expr( + [a], + case a do + :foo = b -> :bar = b + end + ) + end + + test "do not leak pattern/guard inference between clauses" do + assert quoted_expr( + [a], + case a do + :foo = b -> b + :bar = b -> b + end + ) == {:ok, :dynamic} + + assert quoted_expr( + [a], + case a do + b when is_atom(b) -> b + b when is_integer(b) -> b + end + ) == {:ok, :dynamic} + + assert quoted_expr( + [a], + case a do + :foo = b -> :foo = b + :bar = b -> :bar = b + end + ) == {:ok, :dynamic} + end + + test "do not leak body inference between clauses" do + assert quoted_expr( + [a], + case a do + :foo -> + b = :foo + b + + :bar -> + b = :bar + b + end + ) == {:ok, :dynamic} + + assert quoted_expr( + [a, b], + case a do + :foo -> :foo = b + :bar -> :bar = b + end + ) == {:ok, :dynamic} + + assert quoted_expr( + [a, b], + case a do + :foo when is_binary(b) -> b <> "" + :foo when is_list(b) -> b + end + ) == {:ok, :dynamic} + end + end + + test "fn" do + assert quoted_expr(fn :foo = b -> :foo = b end) == {:ok, :dynamic} + + assert {:error, _} = quoted_expr(fn :foo = b -> :bar = b end) + end + + test "receive" do + assert quoted_expr( + receive do + after + 0 -> :ok + end + ) == {:ok, :dynamic} + end + + test "with" do + assert quoted_expr( + [a, b], + with( + :foo <- a, + :bar <- b, + c = :baz, + do: c + ) + ) == {:ok, :dynamic} + + assert quoted_expr( + [a], + ( + with(a = :baz, do: a) + a + ) + ) == {:ok, {:var, 0}} + end + + describe "for comprehension" do + test "with generators and filters" do + assert quoted_expr( + [list], + for( + foo <- list, + is_integer(foo), + do: foo == 123 + ) + ) == {:ok, :dynamic} + end + + test "with unused return" do + assert quoted_expr( + [list, bar], + ( + for( + foo <- list, + is_integer(bar), + do: foo == 123 + ) + + bar + ) + ) == {:ok, {:var, 0}} + end + + test "with reduce" do + assert quoted_expr( + [], + for(i <- [1, 2, 3], do: (acc -> i + acc), reduce: 0) + ) == {:ok, :dynamic} + + assert quoted_expr( + [], + for(i <- [1, 2, 3], do: (_ -> i), reduce: nil) + ) == {:ok, :dynamic} + end + end +end diff --git a/lib/elixir/test/elixir/module/types/integration_test.exs b/lib/elixir/test/elixir/module/types/integration_test.exs new file mode 100644 index 00000000000..175163b599e --- /dev/null +++ b/lib/elixir/test/elixir/module/types/integration_test.exs @@ -0,0 +1,754 @@ +Code.require_file("type_helper.exs", __DIR__) + +defmodule Module.Types.IntegrationTest do + use ExUnit.Case + + import ExUnit.CaptureIO + + setup_all do + previous = Application.get_env(:elixir, :ansi_enabled, false) + Application.put_env(:elixir, :ansi_enabled, false) + on_exit(fn -> Application.put_env(:elixir, :ansi_enabled, previous) end) + end + + describe "ExCk chunk" do + test "writes exports" do + files = %{ + "a.ex" => """ + defmodule A do + defp a, do: :ok + defmacrop b, do: a() + def c, do: b() + defmacro d, do: b() + @deprecated "oops" + def e, do: :ok + end + """, + "b.ex" => """ + defmodule B do + @callback f() :: :ok + end + """, + "c.ex" => """ + defmodule C do + @macrocallback g() :: :ok + end + """ + } + + modules = compile(files) + + assert read_chunk(modules[A]).exports == [ + {{:c, 0}, %{deprecated_reason: nil, kind: :def}}, + {{:d, 0}, %{deprecated_reason: nil, kind: :defmacro}}, + {{:e, 0}, %{deprecated_reason: "oops", kind: :def}} + ] + + assert read_chunk(modules[B]).exports == [ + {{:behaviour_info, 1}, %{deprecated_reason: nil, kind: :def}} + ] + + assert read_chunk(modules[C]).exports == [ + {{:behaviour_info, 1}, %{deprecated_reason: nil, kind: :def}} + ] + end + end + + describe "undefined" do + test "handles Erlang modules" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: :not_a_module.no_module() + def b, do: :lists.no_func() + end + """ + } + + warning = """ + warning: :not_a_module.no_module/0 is undefined (module :not_a_module is not available or is yet to be defined) + a.ex:2: A.a/0 + + warning: :lists.no_func/0 is undefined or private + a.ex:3: A.b/0 + + """ + + assert_warnings(files, warning) + end + + test "handles built in functions" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: Kernel.module_info() + def b, do: Kernel.module_info(:functions) + def c, do: Kernel.__info__(:functions) + def d, do: GenServer.behaviour_info(:callbacks) + def e, do: Kernel.behaviour_info(:callbacks) + end + """ + } + + warning = """ + warning: Kernel.behaviour_info/1 is undefined or private + a.ex:6: A.e/0 + + """ + + assert_warnings(files, warning) + end + + test "handles module body conditionals" do + files = %{ + "a.ex" => """ + defmodule A do + if function_exported?(List, :flatten, 1) do + List.flatten([1, 2, 3]) + else + List.old_flatten([1, 2, 3]) + end + + if function_exported?(List, :flatten, 1) do + def flatten(arg), do: List.flatten(arg) + else + def flatten(arg), do: List.old_flatten(arg) + end + + if function_exported?(List, :flatten, 1) do + def flatten2(arg), do: List.old_flatten(arg) + else + def flatten2(arg), do: List.flatten(arg) + end + end + """ + } + + warning = """ + warning: List.old_flatten/1 is undefined or private. Did you mean: + + * flatten/1 + * flatten/2 + + a.ex:15: A.flatten2/1 + + """ + + assert_warnings(files, warning) + end + + test "reports missing functions" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: A.no_func() + def b, do: A.a() + + @file "external_source.ex" + def c, do: &A.no_func/1 + end + """ + } + + warning = """ + warning: A.no_func/0 is undefined or private + a.ex:2: A.a/0 + + warning: A.no_func/1 is undefined or private + external_source.ex:6: A.c/0 + + """ + + assert_warnings(files, warning) + end + + test "reports missing functions respecting arity" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: :ok + def b, do: A.a(1) + + @file "external_source.ex" + def c, do: A.b(1) + end + """ + } + + warning = """ + warning: A.a/1 is undefined or private. Did you mean: + + * a/0 + + a.ex:3: A.b/0 + + warning: A.b/1 is undefined or private. Did you mean: + + * b/0 + + external_source.ex:6: A.c/0 + + """ + + assert_warnings(files, warning) + end + + test "reports missing modules" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: D.no_module() + + @file "external_source.ex" + def c, do: E.no_module() + end + """ + } + + warning = """ + warning: D.no_module/0 is undefined (module D is not available or is yet to be defined) + a.ex:2: A.a/0 + + warning: E.no_module/0 is undefined (module E is not available or is yet to be defined) + external_source.ex:5: A.c/0 + + """ + + assert_warnings(files, warning) + end + + test "reports missing captures" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: &A.no_func/0 + + @file "external_source.ex" + def c, do: &A.no_func/1 + end + """ + } + + warning = """ + warning: A.no_func/0 is undefined or private + a.ex:2: A.a/0 + + warning: A.no_func/1 is undefined or private + external_source.ex:5: A.c/0 + + """ + + assert_warnings(files, warning) + end + + test "doesn't report missing funcs at compile time" do + files = %{ + "a.ex" => """ + Enum.map([], fn _ -> BadReferencer.no_func4() end) + + if function_exported?(List, :flatten, 1) do + List.flatten([1, 2, 3]) + else + List.old_flatten([1, 2, 3]) + end + """ + } + + assert_no_warnings(files) + end + + test "handles multiple modules in one file" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: B.no_func() + def b, do: B.a() + end + """, + "b.ex" => """ + defmodule B do + def a, do: A.no_func() + def b, do: A.b() + end + """ + } + + warnings = [ + """ + warning: B.no_func/0 is undefined or private + a.ex:2: A.a/0 + """, + """ + warning: A.no_func/0 is undefined or private + b.ex:2: B.a/0 + """ + ] + + assert_warnings(files, warnings) + end + + test "groups multiple warnings in one file" do + files = %{ + "a.ex" => """ + defmodule A do + def a, do: A.no_func() + + @file "external_source.ex" + def b, do: A2.no_func() + + def c, do: A.no_func() + def d, do: A2.no_func() + end + """ + } + + warning = """ + warning: A2.no_func/0 is undefined (module A2 is not available or is yet to be defined) + Invalid call found at 2 locations: + a.ex:8: A.d/0 + external_source.ex:5: A.b/0 + + warning: A.no_func/0 is undefined or private + Invalid call found at 2 locations: + a.ex:2: A.a/0 + a.ex:7: A.c/0 + + """ + + assert_warnings(files, warning) + end + + test "protocols are checked, ignoring missing built-in impls" do + files = %{ + "a.ex" => """ + defprotocol AProtocol do + def func(arg) + end + + defmodule AImplementation do + defimpl AProtocol do + def func(_), do: B.no_func() + end + end + """ + } + + warning = """ + warning: B.no_func/0 is undefined (module B is not available or is yet to be defined) + a.ex:7: AProtocol.AImplementation.func/1 + + """ + + assert_warnings(files, warning) + end + + test "handles Erlang ops" do + files = %{ + "a.ex" => """ + defmodule A do + def a(a, b), do: a and b + def b(a, b), do: a or b + end + """ + } + + assert_no_warnings(files) + end + + test "hints exclude deprecated functions" do + files = %{ + "a.ex" => """ + defmodule A do + def to_charlist(a), do: a + + @deprecated "Use String.to_charlist/1 instead" + def to_char_list(a), do: a + + def c(a), do: A.to_list(a) + end + """ + } + + warning = """ + warning: A.to_list/1 is undefined or private. Did you mean: + + * to_charlist/1 + + a.ex:7: A.c/1 + + """ + + assert_warnings(files, warning) + end + + test "do not warn for module defined in local context" do + files = %{ + "a.ex" => """ + defmodule A do + def a() do + defmodule B do + def b(), do: :ok + end + + B.b() + end + end + """ + } + + assert_no_warnings(files) + end + + test "warn for unrequired module" do + files = %{ + "ab.ex" => """ + defmodule A do + def a(), do: B.b() + end + + defmodule B do + defmacro b(), do: :ok + end + """ + } + + warning = """ + warning: you must require B before invoking the macro B.b/0 + ab.ex:2: A.a/0 + + """ + + assert_warnings(files, warning) + end + + test "excludes local no_warn_undefined" do + files = %{ + "a.ex" => """ + defmodule A do + @compile {:no_warn_undefined, [MissingModule, {MissingModule2, :func, 2}]} + @compile {:no_warn_undefined, {B, :func, 2}} + + def a, do: MissingModule.func(1) + def b, do: MissingModule2.func(1, 2) + def c, do: MissingModule2.func(1) + def d, do: MissingModule3.func(1, 2) + def e, do: B.func(1) + def f, do: B.func(1, 2) + def g, do: B.func(1, 2, 3) + end + """, + "b.ex" => """ + defmodule B do + def func(_), do: :ok + end + """ + } + + warning = """ + warning: MissingModule2.func/1 is undefined (module MissingModule2 is not available or is yet to be defined) + a.ex:7: A.c/0 + + warning: MissingModule3.func/2 is undefined (module MissingModule3 is not available or is yet to be defined) + a.ex:8: A.d/0 + + warning: B.func/3 is undefined or private. Did you mean: + + * func/1 + + a.ex:11: A.g/0 + + """ + + assert_warnings(files, warning) + end + + test "excludes global no_warn_undefined" do + no_warn_undefined = Code.get_compiler_option(:no_warn_undefined) + + try do + Code.compiler_options( + no_warn_undefined: [MissingModule, {MissingModule2, :func, 2}, {B, :func, 2}] + ) + + files = %{ + "a.ex" => """ + defmodule A do + @compile {:no_warn_undefined, [MissingModule, {MissingModule2, :func, 2}]} + @compile {:no_warn_undefined, {B, :func, 2}} + + def a, do: MissingModule.func(1) + def b, do: MissingModule2.func(1, 2) + def c, do: MissingModule2.func(1) + def d, do: MissingModule3.func(1, 2) + def e, do: B.func(1) + def f, do: B.func(1, 2) + def g, do: B.func(1, 2, 3) + end + """, + "b.ex" => """ + defmodule B do + def func(_), do: :ok + end + """ + } + + warning = """ + warning: MissingModule2.func/1 is undefined (module MissingModule2 is not available or is yet to be defined) + a.ex:7: A.c/0 + + warning: MissingModule3.func/2 is undefined (module MissingModule3 is not available or is yet to be defined) + a.ex:8: A.d/0 + + warning: B.func/3 is undefined or private. Did you mean: + + * func/1 + + a.ex:11: A.g/0 + + """ + + assert_warnings(files, warning) + after + Code.compiler_options(no_warn_undefined: no_warn_undefined) + end + end + + test "global no_warn_undefined :all" do + no_warn_undefined = Code.get_compiler_option(:no_warn_undefined) + + try do + Code.compiler_options(no_warn_undefined: :all) + + files = %{ + "a.ex" => """ + defmodule A do + def a, do: MissingModule.func(1) + end + """ + } + + assert_no_warnings(files) + after + Code.compiler_options(no_warn_undefined: no_warn_undefined) + end + end + + test "global no_warn_undefined :all and local exclude" do + no_warn_undefined = Code.get_compiler_option(:no_warn_undefined) + + try do + Code.compiler_options(no_warn_undefined: :all) + + files = %{ + "a.ex" => """ + defmodule A do + @compile {:no_warn_undefined, MissingModule} + + def a, do: MissingModule.func(1) + def b, do: MissingModule2.func(1, 2) + end + """ + } + + assert_no_warnings(files) + after + Code.compiler_options(no_warn_undefined: no_warn_undefined) + end + end + end + + describe "deprecated" do + test "reports functions" do + files = %{ + "a.ex" => """ + defmodule A do + @deprecated "oops" + def a, do: A.a() + end + """ + } + + warning = """ + warning: A.a/0 is deprecated. oops + a.ex:3: A.a/0 + + """ + + assert_warnings(files, warning) + end + + test "reports imported functions" do + files = %{ + "a.ex" => """ + defmodule A do + @deprecated "oops" + def a, do: :ok + end + """, + "b.ex" => """ + defmodule B do + import A + def b, do: a() + end + """ + } + + warning = """ + warning: A.a/0 is deprecated. oops + b.ex:3: B.b/0 + + """ + + assert_warnings(files, warning) + end + + test "reports structs" do + files = %{ + "a.ex" => """ + defmodule A do + @deprecated "oops" + defstruct [:x, :y] + def match(%A{}), do: :ok + def build(:ok), do: %A{} + end + """, + "b.ex" => """ + defmodule B do + def match(%A{}), do: :ok + def build(:ok), do: %A{} + end + """ + } + + warnings = [ + """ + warning: A.__struct__/0 is deprecated. oops + Invalid call found at 2 locations: + a.ex:4: A.match/1 + a.ex:5: A.build/1 + """, + """ + warning: A.__struct__/0 is deprecated. oops + Invalid call found at 2 locations: + b.ex:2: B.match/1 + b.ex:3: B.build/1 + + """ + ] + + assert_warnings(files, warnings) + end + + test "reports module body" do + files = %{ + "a.ex" => """ + defmodule A do + @deprecated "oops" + def a, do: :ok + end + """, + "b.ex" => """ + defmodule B do + require A + A.a() + end + """ + } + + warning = """ + warning: A.a/0 is deprecated. oops + b.ex:3: B + + """ + + assert_warnings(files, warning) + end + + test "reports macro" do + files = %{ + "a.ex" => """ + defmodule A do + @deprecated "oops" + defmacro a, do: :ok + end + """, + "b.ex" => """ + defmodule B do + require A + def b, do: A.a() + end + """ + } + + warning = """ + warning: A.a/0 is deprecated. oops + b.ex:3: B.b/0 + + """ + + assert_warnings(files, warning) + end + end + + defp assert_warnings(files, expected) when is_binary(expected) do + assert capture_compile_warnings(files) == expected + end + + defp assert_warnings(files, expecteds) when is_list(expecteds) do + output = capture_compile_warnings(files) + + Enum.each(expecteds, fn expected -> + assert output =~ expected + end) + end + + defp assert_no_warnings(files) do + assert capture_compile_warnings(files) == "" + end + + defp capture_compile_warnings(files) do + in_tmp(fn -> + paths = generate_files(files) + capture_io(:stderr, fn -> compile_files(paths) end) + end) + end + + defp compile(files) do + in_tmp(fn -> + paths = generate_files(files) + compile_files(paths) + end) + end + + defp compile_files(paths) do + {:ok, modules, _warnings} = Kernel.ParallelCompiler.compile_to_path(paths, ".") + + Map.new(modules, fn module -> + {^module, binary, _filename} = :code.get_object_code(module) + :code.purge(module) + :code.delete(module) + {module, binary} + end) + end + + defp generate_files(files) do + for {file, contents} <- files do + File.write!(file, contents) + file + end + end + + defp read_chunk(binary) do + assert {:ok, {_module, [{'ExCk', chunk}]}} = :beam_lib.chunks(binary, ['ExCk']) + assert {:elixir_checker_v1, map} = :erlang.binary_to_term(chunk) + map + end + + defp in_tmp(fun) do + path = PathHelpers.tmp_path("checker") + + File.rm_rf!(path) + File.mkdir_p!(path) + File.cd!(path, fun) + end +end diff --git a/lib/elixir/test/elixir/module/types/map_test.exs b/lib/elixir/test/elixir/module/types/map_test.exs new file mode 100644 index 00000000000..da6818f2e54 --- /dev/null +++ b/lib/elixir/test/elixir/module/types/map_test.exs @@ -0,0 +1,377 @@ +Code.require_file("type_helper.exs", __DIR__) + +defmodule Module.Types.MapTest do + # This file holds cases for maps and structs. + use ExUnit.Case, async: true + + import TypeHelper + + defmodule :"Elixir.Module.Types.MapTest.Struct" do + defstruct foo: :atom, bar: 123, baz: %{} + end + + test "map" do + assert quoted_expr(%{}) == {:ok, {:map, []}} + assert quoted_expr(%{a: :b}) == {:ok, {:map, [{:required, {:atom, :a}, {:atom, :b}}]}} + assert quoted_expr([a], %{123 => a}) == {:ok, {:map, [{:required, :integer, {:var, 0}}]}} + + assert quoted_expr(%{123 => :foo, 456 => :bar}) == + {:ok, {:map, [{:required, :integer, {:union, [{:atom, :foo}, {:atom, :bar}]}}]}} + end + + test "struct" do + assert quoted_expr(%:"Elixir.Module.Types.MapTest.Struct"{}) == + {:ok, + {:map, + [ + {:required, {:atom, :bar}, :integer}, + {:required, {:atom, :baz}, {:map, []}}, + {:required, {:atom, :foo}, {:atom, :atom}}, + {:required, {:atom, :__struct__}, {:atom, Module.Types.MapTest.Struct}} + ]}} + + assert quoted_expr(%:"Elixir.Module.Types.MapTest.Struct"{foo: 123, bar: :atom}) == + {:ok, + {:map, + [ + {:required, {:atom, :baz}, {:map, []}}, + {:required, {:atom, :foo}, :integer}, + {:required, {:atom, :bar}, {:atom, :atom}}, + {:required, {:atom, :__struct__}, {:atom, Module.Types.MapTest.Struct}} + ]}} + end + + test "map field" do + assert quoted_expr(%{foo: :bar}.foo) == {:ok, {:atom, :bar}} + + assert quoted_expr( + ( + map = %{foo: :bar} + map.foo + ) + ) == {:ok, {:atom, :bar}} + + assert quoted_expr( + [map], + ( + map.foo + map.bar + map + ) + ) == + {:ok, + {:map, + [ + {:required, {:atom, :bar}, {:var, 0}}, + {:required, {:atom, :foo}, {:var, 1}}, + {:optional, :dynamic, :dynamic} + ]}} + + assert quoted_expr( + [map], + ( + :foo = map.foo + :bar = map.bar + map + ) + ) == + {:ok, + {:map, + [ + {:required, {:atom, :bar}, {:atom, :bar}}, + {:required, {:atom, :foo}, {:atom, :foo}}, + {:optional, :dynamic, :dynamic} + ]}} + + assert {:error, + {:unable_unify, + {{:map, [{:required, {:atom, :bar}, {:var, 1}}, {:optional, :dynamic, :dynamic}]}, + {:map, [{:required, {:atom, :foo}, {:atom, :foo}}]}, + _}}} = + quoted_expr( + ( + map = %{foo: :foo} + map.bar + ) + ) + end + + defmodule :"Elixir.Module.Types.MapTest.Struct2" do + defstruct [:field] + end + + test "map and struct fields" do + assert quoted_expr( + [map], + ( + %Module.Types.MapTest.Struct2{} = map + map.field + map + ) + ) == + {:ok, + {:map, + [ + {:required, {:atom, :field}, {:var, 0}}, + {:required, {:atom, :__struct__}, {:atom, Module.Types.MapTest.Struct2}} + ]}} + + assert quoted_expr( + [map], + ( + _ = map.field + %Module.Types.MapTest.Struct2{} = map + ) + ) == + {:ok, + {:map, + [ + {:required, {:atom, :field}, {:var, 0}}, + {:required, {:atom, :__struct__}, {:atom, Module.Types.MapTest.Struct2}} + ]}} + + assert {:error, {:unable_unify, {_, _, _}}} = + quoted_expr( + [map], + ( + %Module.Types.MapTest.Struct2{} = map + map.no_field + ) + ) + + assert {:error, {:unable_unify, {_, _, _}}} = + quoted_expr( + [map], + ( + _ = map.no_field + %Module.Types.MapTest.Struct2{} = map + ) + ) + end + + test "map pattern" do + assert quoted_expr(%{a: :b} = %{a: :b}) == + {:ok, {:map, [{:required, {:atom, :a}, {:atom, :b}}]}} + + assert quoted_expr( + ( + a = :a + %{^a => :b} = %{:a => :b} + ) + ) == {:ok, {:map, [{:required, {:atom, :a}, {:atom, :b}}]}} + + assert quoted_expr( + ( + a = :a + %{{^a, :b} => :c} = %{{:a, :b} => :c} + ) + ) == {:ok, {:map, [{:required, {:tuple, 2, [{:atom, :a}, {:atom, :b}]}, {:atom, :c}}]}} + + assert {:error, + {:unable_unify, + {{:map, [{:required, {:atom, :c}, {:atom, :d}}]}, + {:map, [{:required, {:atom, :a}, {:atom, :b}}, {:optional, :dynamic, :dynamic}]}, + _}}} = quoted_expr(%{a: :b} = %{c: :d}) + + assert {:error, + {:unable_unify, + {{:map, [{:required, {:atom, :b}, {:atom, :error}}]}, + {:map, [{:required, {:var, 0}, {:atom, :ok}}, {:optional, :dynamic, :dynamic}]}, + _}}} = + quoted_expr( + ( + a = :a + %{^a => :ok} = %{:b => :error} + ) + ) + end + + test "map update" do + assert quoted_expr( + ( + map = %{foo: :a} + %{map | foo: :b} + ) + ) == + {:ok, {:map, [{:required, {:atom, :foo}, {:atom, :b}}]}} + + assert quoted_expr([map], %{map | foo: :b}) == + {:ok, + {:map, [{:required, {:atom, :foo}, {:atom, :b}}, {:optional, :dynamic, :dynamic}]}} + + assert {:error, + {:unable_unify, + {{:map, [{:required, {:atom, :foo}, {:atom, :a}}]}, + {:map, [{:required, {:atom, :bar}, :dynamic}, {:optional, :dynamic, :dynamic}]}, + _}}} = + quoted_expr( + ( + map = %{foo: :a} + %{map | bar: :b} + ) + ) + end + + test "struct update" do + assert quoted_expr( + ( + map = %Module.Types.MapTest.Struct2{field: :a} + %Module.Types.MapTest.Struct2{map | field: :b} + ) + ) == + {:ok, + {:map, + [ + {:required, {:atom, :field}, {:atom, :b}}, + {:required, {:atom, :__struct__}, {:atom, Module.Types.MapTest.Struct2}} + ]}} + + # TODO: improve error message to translate to MULTIPLE missing fields + assert {:error, + {:unable_unify, + {{:map, + [ + {:required, {:atom, :foo}, {:var, 1}}, + {:required, {:atom, :field}, {:atom, :b}}, + {:optional, :dynamic, :dynamic} + ]}, + {:map, + [ + {:required, {:atom, :__struct__}, {:atom, Module.Types.MapTest.Struct2}}, + {:required, {:atom, :field}, :dynamic} + ]}, + _}}} = + quoted_expr( + [map], + ( + _ = map.foo + %Module.Types.MapTest.Struct2{map | field: :b} + ) + ) + + assert {:error, + {:unable_unify, + {{:map, [{:required, {:atom, :field}, {:atom, :b}}]}, + {:map, + [ + {:required, {:atom, :__struct__}, {:atom, Module.Types.MapTest.Struct2}}, + {:required, {:atom, :field}, :dynamic} + ]}, + _}}} = + quoted_expr( + ( + map = %{field: :a} + %Module.Types.MapTest.Struct2{map | field: :b} + ) + ) + + assert quoted_expr([map], %Module.Types.MapTest.Struct2{map | field: :b}) == + {:ok, + {:map, + [ + {:required, {:atom, :field}, {:atom, :b}}, + {:required, {:atom, :__struct__}, {:atom, Module.Types.MapTest.Struct2}} + ]}} + + assert {:error, + {:unable_unify, + {{:map, + [ + {:required, {:atom, :field}, {:atom, nil}}, + {:required, {:atom, :__struct__}, {:atom, Module.Types.MapTest.Struct2}} + ]}, + {:map, + [{:required, {:atom, :not_field}, :dynamic}, {:optional, :dynamic, :dynamic}]}, + _}}} = + quoted_expr( + ( + map = %Module.Types.MapTest.Struct2{} + %{map | not_field: :b} + ) + ) + end + + describe "in guards" do + test "not is_struct/2" do + assert quoted_expr([var], [not is_struct(var, URI)], var.name) == {:ok, {:var, 0}} + end + + test "map guards" do + assert quoted_expr([var], [is_map(var)], var.foo) == {:ok, {:var, 0}} + assert quoted_expr([var], [is_map_key(var, :bar)], var.foo) == {:ok, {:var, 0}} + assert quoted_expr([var], [:erlang.map_get(:bar, var)], var.foo) == {:ok, {:var, 0}} + assert quoted_expr([var], [map_size(var) == 1], var.foo) == {:ok, {:var, 0}} + end + end + + test "map creation with bound var keys" do + assert quoted_expr( + [atom, bool, true = var], + [is_atom(atom) and is_boolean(bool)], + %{atom => :atom, bool => :bool, var => true} + ) == + {:ok, + {:map, + [ + {:required, {:atom, true}, {:atom, true}}, + {:required, {:union, [atom: true, atom: false]}, + {:union, [{:atom, :bool}, {:atom, true}]}}, + {:required, :atom, {:union, [{:atom, :atom}, {:atom, :bool}, {:atom, true}]}} + ]}} + + assert quoted_expr( + [atom, bool, true = var], + [is_atom(atom) and is_boolean(bool)], + %{var => true, bool => :bool, atom => :atom} + ) == + {:ok, + {:map, + [ + {:required, {:atom, true}, {:atom, true}}, + {:required, {:union, [atom: true, atom: false]}, + {:union, [{:atom, :bool}, {:atom, true}]}}, + {:required, :atom, {:union, [{:atom, :atom}, {:atom, :bool}, {:atom, true}]}} + ]}} + + assert quoted_expr( + [atom, bool, true = var], + [is_atom(atom) and is_boolean(bool)], + %{var => true, atom => :atom, bool => :bool} + ) == + {:ok, + {:map, + [ + {:required, {:atom, true}, {:atom, true}}, + {:required, {:union, [atom: true, atom: false]}, + {:union, [{:atom, :bool}, {:atom, true}]}}, + {:required, :atom, {:union, [{:atom, :atom}, {:atom, :bool}, {:atom, true}]}} + ]}} + end + + test "map creation with unbound var keys" do + assert quoted_expr( + [var, struct], + ( + map = %{var => :foo} + %^var{} = struct + map + ) + ) == {:ok, {:map, [{:required, :atom, {:atom, :foo}}]}} + + # If we have multiple keys, the unbound key must become required(dynamic) => dynamic + assert quoted_expr( + [var, struct], + ( + map = %{var => :foo, :foo => :bar} + %^var{} = struct + map + ) + ) == + {:ok, + {:map, + [ + {:required, {:atom, :foo}, {:atom, :bar}}, + {:required, :dynamic, :dynamic} + ]}} + end +end diff --git a/lib/elixir/test/elixir/module/types/pattern_test.exs b/lib/elixir/test/elixir/module/types/pattern_test.exs new file mode 100644 index 00000000000..c8bb56d2d9d --- /dev/null +++ b/lib/elixir/test/elixir/module/types/pattern_test.exs @@ -0,0 +1,537 @@ +Code.require_file("type_helper.exs", __DIR__) + +defmodule Module.Types.PatternTest do + use ExUnit.Case, async: true + + alias Module.Types + alias Module.Types.{Unify, Pattern} + + defmacrop quoted_pattern(patterns) do + quote do + {patterns, true} = unquote(Macro.escape(expand_head(patterns, true))) + + Pattern.of_pattern(patterns, new_stack(), new_context()) + |> lift_result() + end + end + + defmacrop quoted_head(patterns, guards \\ []) do + quote do + {patterns, guards} = unquote(Macro.escape(expand_head(patterns, guards))) + + Pattern.of_head(patterns, guards, new_stack(), new_context()) + |> lift_result() + end + end + + defp expand_head(patterns, guards) do + fun = + quote do + fn unquote(patterns) when unquote(guards) -> :ok end + end + + fun = + Macro.prewalk(fun, fn + {var, meta, nil} -> {var, meta, __MODULE__} + other -> other + end) + + {ast, _, _} = :elixir_expand.expand(fun, :elixir_env.env_to_ex(__ENV__), __ENV__) + {:fn, _, [{:->, _, [[{:when, _, [patterns, guards]}], _]}]} = ast + {patterns, guards} + end + + defp new_context() do + Types.context("types_test.ex", TypesTest, {:test, 0}, [], Module.ParallelChecker.test_cache()) + end + + defp new_stack() do + %{ + Types.stack() + | last_expr: {:foo, [], nil} + } + end + + defp lift_result({:ok, types, context}) when is_list(types) do + {types, _context} = Unify.lift_types(types, context) + {:ok, types} + end + + defp lift_result({:ok, type, context}) do + {[type], _context} = Unify.lift_types([type], context) + {:ok, type} + end + + defp lift_result({:error, {type, reason, _context}}) do + {:error, {type, reason}} + end + + defmodule :"Elixir.Module.Types.PatternTest.Struct" do + defstruct foo: :atom, bar: 123, baz: %{} + end + + describe "patterns" do + test "literal" do + assert quoted_pattern(true) == {:ok, {:atom, true}} + assert quoted_pattern(false) == {:ok, {:atom, false}} + assert quoted_pattern(:foo) == {:ok, {:atom, :foo}} + assert quoted_pattern(0) == {:ok, :integer} + assert quoted_pattern(0.0) == {:ok, :float} + assert quoted_pattern("foo") == {:ok, :binary} + end + + test "list" do + assert quoted_pattern([]) == {:ok, {:list, :dynamic}} + assert quoted_pattern([_]) == {:ok, {:list, :dynamic}} + assert quoted_pattern([123]) == {:ok, {:list, :integer}} + assert quoted_pattern([123, 456]) == {:ok, {:list, :integer}} + assert quoted_pattern([123, _]) == {:ok, {:list, :dynamic}} + assert quoted_pattern([_, 456]) == {:ok, {:list, :dynamic}} + assert quoted_pattern([123 | []]) == {:ok, {:list, :integer}} + assert quoted_pattern([123, "foo"]) == {:ok, {:list, {:union, [:integer, :binary]}}} + assert quoted_pattern([123 | ["foo"]]) == {:ok, {:list, {:union, [:integer, :binary]}}} + + # TODO: improper list? + assert quoted_pattern([123 | 456]) == {:ok, {:list, :integer}} + assert quoted_pattern([123, 456 | 789]) == {:ok, {:list, :integer}} + assert quoted_pattern([123 | "foo"]) == {:ok, {:list, {:union, [:integer, :binary]}}} + assert quoted_pattern([123 | _]) == {:ok, {:list, :dynamic}} + assert quoted_pattern([_ | [456]]) == {:ok, {:list, :dynamic}} + assert quoted_pattern([_ | _]) == {:ok, {:list, :dynamic}} + + assert quoted_pattern([] ++ []) == {:ok, {:list, :dynamic}} + assert quoted_pattern([_] ++ _) == {:ok, {:list, :dynamic}} + assert quoted_pattern([123] ++ [456]) == {:ok, {:list, :integer}} + assert quoted_pattern([123] ++ _) == {:ok, {:list, :dynamic}} + assert quoted_pattern([123] ++ ["foo"]) == {:ok, {:list, {:union, [:integer, :binary]}}} + end + + test "tuple" do + assert quoted_pattern({}) == {:ok, {:tuple, 0, []}} + assert quoted_pattern({:a}) == {:ok, {:tuple, 1, [{:atom, :a}]}} + assert quoted_pattern({:a, 123}) == {:ok, {:tuple, 2, [{:atom, :a}, :integer]}} + end + + test "map" do + assert quoted_pattern(%{}) == {:ok, {:map, [{:optional, :dynamic, :dynamic}]}} + + assert quoted_pattern(%{a: :b}) == + {:ok, + {:map, [{:required, {:atom, :a}, {:atom, :b}}, {:optional, :dynamic, :dynamic}]}} + + assert quoted_pattern(%{123 => a}) == + {:ok, + {:map, + [ + {:required, :integer, {:var, 0}}, + {:optional, :dynamic, :dynamic} + ]}} + + assert quoted_pattern(%{123 => :foo, 456 => :bar}) == + {:ok, + {:map, + [ + {:required, :integer, :dynamic}, + {:optional, :dynamic, :dynamic} + ]}} + + assert {:error, {:unable_unify, {:integer, {:atom, :foo}, _}}} = + quoted_pattern(%{a: a = 123, b: a = :foo}) + end + + test "struct" do + assert quoted_pattern(%:"Elixir.Module.Types.PatternTest.Struct"{}) == + {:ok, + {:map, + [ + {:required, {:atom, :__struct__}, {:atom, Module.Types.PatternTest.Struct}}, + {:required, {:atom, :bar}, :dynamic}, + {:required, {:atom, :baz}, :dynamic}, + {:required, {:atom, :foo}, :dynamic} + ]}} + + assert quoted_pattern(%:"Elixir.Module.Types.PatternTest.Struct"{foo: 123, bar: :atom}) == + {:ok, + {:map, + [ + {:required, {:atom, :foo}, :integer}, + {:required, {:atom, :bar}, {:atom, :atom}}, + {:required, {:atom, :__struct__}, {:atom, Module.Types.PatternTest.Struct}}, + {:required, {:atom, :baz}, :dynamic} + ]}} + end + + test "struct var" do + assert quoted_pattern(%var{}) == + {:ok, + {:map, + [{:required, {:atom, :__struct__}, :atom}, {:optional, :dynamic, :dynamic}]}} + + assert quoted_pattern(%var{foo: 123}) == + {:ok, + {:map, + [ + {:required, {:atom, :__struct__}, :atom}, + {:required, {:atom, :foo}, :integer}, + {:optional, :dynamic, :dynamic} + ]}} + + assert quoted_pattern(%var{foo: var}) == + {:ok, + {:map, + [ + {:required, {:atom, :__struct__}, :atom}, + {:required, {:atom, :foo}, :atom}, + {:optional, :dynamic, :dynamic} + ]}} + end + + test "binary" do + assert quoted_pattern(<<"foo"::binary>>) == {:ok, :binary} + assert quoted_pattern(<<123::integer>>) == {:ok, :binary} + assert quoted_pattern(<>) == {:ok, :binary} + assert quoted_pattern(<>) == {:ok, :binary} + assert quoted_pattern(<>) == {:ok, :binary} + assert quoted_pattern(<>) == {:ok, :binary} + assert quoted_pattern(<>) == {:ok, :binary} + assert quoted_pattern(<<123::utf8>>) == {:ok, :binary} + assert quoted_pattern(<<"foo"::utf8>>) == {:ok, :binary} + + assert quoted_pattern({<>, foo}) == {:ok, {:tuple, 2, [:binary, :integer]}} + assert quoted_pattern({<>, foo}) == {:ok, {:tuple, 2, [:binary, :binary]}} + assert quoted_pattern({<>, foo}) == {:ok, {:tuple, 2, [:binary, :integer]}} + + assert {:error, {:unable_unify, {:binary, :integer, _}}} = + quoted_pattern(<>) + end + + test "variables" do + assert quoted_pattern(foo) == {:ok, {:var, 0}} + assert quoted_pattern({foo}) == {:ok, {:tuple, 1, [{:var, 0}]}} + assert quoted_pattern({foo, bar}) == {:ok, {:tuple, 2, [{:var, 0}, {:var, 1}]}} + + assert quoted_pattern(_) == {:ok, :dynamic} + assert quoted_pattern({_ = 123, _}) == {:ok, {:tuple, 2, [:integer, :dynamic]}} + end + + test "assignment" do + assert quoted_pattern(x = y) == {:ok, {:var, 0}} + assert quoted_pattern(x = 123) == {:ok, :integer} + assert quoted_pattern({foo}) == {:ok, {:tuple, 1, [{:var, 0}]}} + assert quoted_pattern({x = y}) == {:ok, {:tuple, 1, [{:var, 0}]}} + + assert quoted_pattern(x = y = 123) == {:ok, :integer} + assert quoted_pattern(x = 123 = y) == {:ok, :integer} + assert quoted_pattern(123 = x = y) == {:ok, :integer} + + assert {:error, {:unable_unify, {{:tuple, 1, [var: 0]}, {:var, 0}, _}}} = + quoted_pattern({x} = x) + end + end + + describe "heads" do + test "variable" do + assert quoted_head([a]) == {:ok, [{:var, 0}]} + assert quoted_head([a, b]) == {:ok, [{:var, 0}, {:var, 1}]} + assert quoted_head([a, a]) == {:ok, [{:var, 0}, {:var, 0}]} + + assert {:ok, [{:var, 0}, {:var, 0}], _} = + Pattern.of_head( + [{:a, [version: 0], :foo}, {:a, [version: 0], :foo}], + [], + new_stack(), + new_context() + ) + + assert {:ok, [{:var, 0}, {:var, 1}], _} = + Pattern.of_head( + [{:a, [version: 0], :foo}, {:a, [version: 1], :foo}], + [], + new_stack(), + new_context() + ) + end + + test "assignment" do + assert quoted_head([x = y, x = y]) == {:ok, [{:var, 0}, {:var, 0}]} + assert quoted_head([x = y, y = x]) == {:ok, [{:var, 0}, {:var, 0}]} + + assert quoted_head([x = :foo, x = y, y = z]) == + {:ok, [{:atom, :foo}, {:atom, :foo}, {:atom, :foo}]} + + assert quoted_head([x = y, y = :foo, y = z]) == + {:ok, [{:atom, :foo}, {:atom, :foo}, {:atom, :foo}]} + + assert quoted_head([x = y, y = z, z = :foo]) == + {:ok, [{:atom, :foo}, {:atom, :foo}, {:atom, :foo}]} + + assert {:error, {:unable_unify, {{:tuple, 1, [var: 1]}, {:var, 0}, _}}} = + quoted_head([{x} = y, {y} = x]) + end + + test "guards" do + assert quoted_head([x], [is_binary(x)]) == {:ok, [:binary]} + + assert quoted_head([x, y], [is_binary(x) and is_atom(y)]) == + {:ok, [:binary, :atom]} + + assert quoted_head([x], [is_binary(x) or is_atom(x)]) == + {:ok, [{:union, [:binary, :atom]}]} + + assert quoted_head([x, x], [is_integer(x)]) == {:ok, [:integer, :integer]} + + assert quoted_head([x = 123], [is_integer(x)]) == {:ok, [:integer]} + + assert quoted_head([x], [is_boolean(x) or is_atom(x)]) == + {:ok, [:atom]} + + assert quoted_head([x], [is_atom(x) or is_boolean(x)]) == + {:ok, [:atom]} + + assert quoted_head([x], [is_tuple(x) or is_atom(x)]) == + {:ok, [{:union, [:tuple, :atom]}]} + + assert quoted_head([x], [is_boolean(x) and is_atom(x)]) == + {:ok, [{:union, [atom: true, atom: false]}]} + + assert quoted_head([x], [is_atom(x) > :foo]) == {:ok, [var: 0]} + + assert quoted_head([x, x = y, y = z], [is_atom(x)]) == + {:ok, [:atom, :atom, :atom]} + + assert quoted_head([x = y, y, y = z], [is_atom(y)]) == + {:ok, [:atom, :atom, :atom]} + + assert quoted_head([x = y, y = z, z], [is_atom(z)]) == + {:ok, [:atom, :atom, :atom]} + + assert quoted_head([x, y], [is_atom(x) or is_integer(y)]) == + {:ok, [{:var, 0}, {:var, 1}]} + + assert quoted_head([x], [is_atom(x) or is_atom(x)]) == + {:ok, [:atom]} + + assert quoted_head([x, y], [(is_atom(x) and is_atom(y)) or (is_atom(x) and is_integer(y))]) == + {:ok, [:atom, union: [:atom, :integer]]} + + assert quoted_head([x, y], [is_atom(x) or is_integer(x)]) == + {:ok, [union: [:atom, :integer], var: 0]} + + assert quoted_head([x, y], [is_atom(y) or is_integer(y)]) == + {:ok, [{:var, 0}, {:union, [:atom, :integer]}]} + + assert quoted_head([x = y], [is_atom(y) or is_integer(y)]) == + {:ok, [{:union, [:atom, :integer]}]} + + assert quoted_head([x = y], [is_atom(x) or is_integer(x)]) == + {:ok, [{:union, [:atom, :integer]}]} + + assert quoted_head([x = y], [is_atom(x) or is_integer(x)]) == + {:ok, [{:union, [:atom, :integer]}]} + + assert quoted_head([x], [true == false or is_integer(x)]) == + {:ok, [var: 0]} + + assert {:error, {:unable_unify, {:binary, :integer, _}}} = + quoted_head([x], [is_binary(x) and is_integer(x)]) + + assert {:error, {:unable_unify, {:tuple, :atom, _}}} = + quoted_head([x], [is_tuple(x) and is_atom(x)]) + + assert {:error, {:unable_unify, {{:atom, true}, :tuple, _}}} = + quoted_head([x], [is_tuple(is_atom(x))]) + end + + test "guard downcast" do + assert {:error, _} = quoted_head([x], [is_atom(x) and is_boolean(x)]) + end + + test "guard and" do + assert quoted_head([], [(true and 1) > 0]) == {:ok, []} + + assert quoted_head( + [struct], + [is_map_key(struct, :map) and map_size(:erlang.map_get(:map, struct))] + ) == {:ok, [{:map, [{:optional, :dynamic, :dynamic}]}]} + end + + test "intersection functions" do + assert quoted_head([x], [+x]) == {:ok, [{:union, [:integer, :float]}]} + assert quoted_head([x], [x + 1]) == {:ok, [{:union, [:float, :integer]}]} + assert quoted_head([x], [x + 1.0]) == {:ok, [{:union, [:integer, :float]}]} + end + + test "nested calls with intersections in guards" do + assert quoted_head([x], [:erlang.rem(x, 2)]) == {:ok, [:integer]} + assert quoted_head([x], [:erlang.rem(x + x, 2)]) == {:ok, [:integer]} + + assert quoted_head([x], [:erlang.bnot(+x)]) == {:ok, [:integer]} + assert quoted_head([x], [:erlang.bnot(x + 1)]) == {:ok, [:integer]} + + assert quoted_head([x], [is_integer(1 + x - 1)]) == {:ok, [:integer]} + + assert quoted_head([x], [is_integer(1 + x - 1) and is_integer(1 + x - 1)]) == + {:ok, [:integer]} + + assert quoted_head([x], [1 - x >= 0]) == {:ok, [{:union, [:float, :integer]}]} + assert quoted_head([x], [1 - x >= 0 and 1 - x < 0]) == {:ok, [{:union, [:float, :integer]}]} + + assert {:error, + {:unable_apply, + {_, [{:var, 0}, :float], _, + [ + {[:integer, :integer], :integer}, + {[:float, {:union, [:integer, :float]}], :float}, + {[{:union, [:integer, :float]}, :float], :float} + ], _}}} = quoted_head([x], [:erlang.bnot(x + 1.0)]) + end + + test "erlang-only guards" do + assert quoted_head([x], [:erlang.size(x)]) == + {:ok, [{:union, [:binary, :tuple]}]} + end + + test "failing guard functions" do + assert quoted_head([x], [length([])]) == {:ok, [{:var, 0}]} + + assert {:error, + {:unable_apply, + {{:erlang, :length, 1}, [{:atom, :foo}], _, [{[{:list, :dynamic}], :integer}], _}}} = + quoted_head([x], [length(:foo)]) + + assert {:error, + {:unable_apply, + {_, [{:union, [{:atom, true}, {:atom, false}]}], _, + [{[{:list, :dynamic}], :integer}], _}}} = quoted_head([x], [length(is_tuple(x))]) + + assert {:error, + {:unable_apply, + {_, [:integer, {:union, [{:atom, true}, {:atom, false}]}], _, + [{[:integer, :tuple], :dynamic}], _}}} = quoted_head([x], [elem(is_tuple(x), 0)]) + + assert {:error, + {:unable_apply, + {_, [{:union, [{:atom, true}, {:atom, false}]}, :integer], _, + [ + {[:integer, :integer], :integer}, + {[:float, {:union, [:integer, :float]}], :float}, + {[{:union, [:integer, :float]}, :float], :float} + ], _}}} = quoted_head([x], [elem({}, is_tuple(x))]) + + assert quoted_head([x], [elem({}, 1)]) == {:ok, [var: 0]} + + assert quoted_head([x], [elem(x, 1) == :foo]) == {:ok, [:tuple]} + + assert quoted_head([x], [is_tuple(x) and elem(x, 1)]) == {:ok, [:tuple]} + + assert quoted_head([x], [length(x) == 0 or elem(x, 1)]) == {:ok, [{:list, :dynamic}]} + + assert quoted_head([x], [ + (is_list(x) and length(x) == 0) or (is_tuple(x) and elem(x, 1)) + ]) == + {:ok, [{:union, [{:list, :dynamic}, :tuple]}]} + + assert quoted_head([x], [ + (length(x) == 0 and is_list(x)) or (elem(x, 1) and is_tuple(x)) + ]) == {:ok, [{:list, :dynamic}]} + + assert quoted_head([x], [elem(x, 1) or is_atom(x)]) == {:ok, [:tuple]} + + assert quoted_head([x], [is_atom(x) or elem(x, 1)]) == {:ok, [{:union, [:atom, :tuple]}]} + + assert quoted_head([x, y], [elem(x, 1) and is_atom(y)]) == {:ok, [:tuple, :atom]} + + assert quoted_head([x, y], [elem(x, 1) or is_atom(y)]) == {:ok, [:tuple, {:var, 0}]} + + assert {:error, {:unable_unify, {:tuple, :atom, _}}} = + quoted_head([x], [elem(x, 1) and is_atom(x)]) + end + + test "map" do + assert quoted_head([%{true: false} = foo, %{} = foo]) == + {:ok, + [ + {:map, + [{:required, {:atom, true}, {:atom, false}}, {:optional, :dynamic, :dynamic}]}, + {:map, + [{:required, {:atom, true}, {:atom, false}}, {:optional, :dynamic, :dynamic}]} + ]} + + assert quoted_head([%{true: bool}], [is_boolean(bool)]) == + {:ok, + [ + {:map, + [ + {:required, {:atom, true}, {:union, [atom: true, atom: false]}}, + {:optional, :dynamic, :dynamic} + ]} + ]} + + assert quoted_head([%{true: true} = foo, %{false: false} = foo]) == + {:ok, + [ + {:map, + [ + {:required, {:atom, false}, {:atom, false}}, + {:required, {:atom, true}, {:atom, true}}, + {:optional, :dynamic, :dynamic} + ]}, + {:map, + [ + {:required, {:atom, false}, {:atom, false}}, + {:required, {:atom, true}, {:atom, true}}, + {:optional, :dynamic, :dynamic} + ]} + ]} + + assert {:error, + {:unable_unify, + {{:map, [{:required, {:atom, true}, {:atom, true}}]}, + {:map, [{:required, {:atom, true}, {:atom, false}}]}, + _}}} = quoted_head([%{true: false} = foo, %{true: true} = foo]) + end + + test "binary in guards" do + assert quoted_head([a, b], [byte_size(a <> b) > 0]) == + {:ok, [:binary, :binary]} + + assert quoted_head([map], [byte_size(map.a <> map.b) > 0]) == + {:ok, [map: [{:optional, :dynamic, :dynamic}]]} + end + + test "struct var guard" do + assert quoted_head([%var{}], [is_atom(var)]) == + {:ok, + [ + {:map, + [{:required, {:atom, :__struct__}, :atom}, {:optional, :dynamic, :dynamic}]} + ]} + + assert {:error, {:unable_unify, {:atom, :integer, _}}} = + quoted_head([%var{}], [is_integer(var)]) + end + + test "tuple_size/1" do + assert quoted_head([x], [tuple_size(x) == 0]) == {:ok, [{:tuple, 0, []}]} + assert quoted_head([x], [0 == tuple_size(x)]) == {:ok, [{:tuple, 0, []}]} + assert quoted_head([x], [tuple_size(x) == 2]) == {:ok, [{:tuple, 2, [:dynamic, :dynamic]}]} + assert quoted_head([x], [is_tuple(x) and tuple_size(x) == 0]) == {:ok, [{:tuple, 0, []}]} + assert quoted_head([x], [tuple_size(x) == 0 and is_tuple(x)]) == {:ok, [{:tuple, 0, []}]} + + assert quoted_head([x = {y}], [is_integer(y) and tuple_size(x) == 1]) == + {:ok, [{:tuple, 1, [:integer]}]} + + assert {:error, {:unable_unify, {{:tuple, 0, []}, :integer, _}}} = + quoted_head([x], [tuple_size(x) == 0 and is_integer(x)]) + + assert {:error, {:unable_unify, {{:tuple, 0, []}, :integer, _}}} = + quoted_head([x], [is_integer(x) and tuple_size(x) == 0]) + + assert {:error, {:unable_unify, {{:tuple, 0, []}, :integer, _}}} = + quoted_head([x], [is_tuple(x) and tuple_size(x) == 0 and is_integer(x)]) + + assert {:error, {:unable_unify, {{:tuple, 1, [:dynamic]}, {:tuple, 0, []}, _}}} = + quoted_head([x = {}], [tuple_size(x) == 1]) + end + end +end diff --git a/lib/elixir/test/elixir/module/types/type_helper.exs b/lib/elixir/test/elixir/module/types/type_helper.exs new file mode 100644 index 00000000000..351461cdf18 --- /dev/null +++ b/lib/elixir/test/elixir/module/types/type_helper.exs @@ -0,0 +1,48 @@ +Code.require_file("../../test_helper.exs", __DIR__) + +defmodule TypeHelper do + alias Module.Types + alias Module.Types.{Pattern, Expr, Unify} + + defmacro quoted_expr(patterns \\ [], guards \\ [], body) do + expr = expand_expr(patterns, guards, body, __CALLER__) + + quote do + TypeHelper.__expr__(unquote(Macro.escape(expr))) + end + end + + def __expr__({patterns, guards, body}) do + with {:ok, _types, context} <- + Pattern.of_head(patterns, guards, new_stack(), new_context()), + {:ok, type, context} <- Expr.of_expr(body, :dynamic, new_stack(), context) do + {[type], _context} = Unify.lift_types([type], context) + {:ok, type} + else + {:error, {type, reason, _context}} -> + {:error, {type, reason}} + end + end + + def expand_expr(patterns, guards, expr, env) do + fun = + quote do + fn unquote(patterns) when unquote(guards) -> unquote(expr) end + end + + {ast, _, _} = :elixir_expand.expand(fun, :elixir_env.env_to_ex(env), env) + {:fn, _, [{:->, _, [[{:when, _, [patterns, guards]}], body]}]} = ast + {patterns, guards, body} + end + + def new_context() do + Types.context("types_test.ex", TypesTest, {:test, 0}, [], Module.ParallelChecker.test_cache()) + end + + def new_stack() do + %{ + Types.stack() + | last_expr: {:foo, [], nil} + } + end +end diff --git a/lib/elixir/test/elixir/module/types/types_test.exs b/lib/elixir/test/elixir/module/types/types_test.exs new file mode 100644 index 00000000000..e82b3eead6a --- /dev/null +++ b/lib/elixir/test/elixir/module/types/types_test.exs @@ -0,0 +1,748 @@ +Code.require_file("type_helper.exs", __DIR__) + +defmodule Module.Types.TypesTest do + use ExUnit.Case, async: true + alias Module.Types + alias Module.Types.{Pattern, Expr} + + defmacro warning(patterns \\ [], guards \\ [], body) do + min_line = min_line(patterns ++ guards ++ [body]) + patterns = reset_line(patterns, min_line) + guards = reset_line(guards, min_line) + body = reset_line(body, min_line) + expr = TypeHelper.expand_expr(patterns, guards, body, __CALLER__) + + quote do + Module.Types.TypesTest.__expr__(unquote(Macro.escape(expr))) + end + end + + defmacro generated(ast) do + Macro.prewalk(ast, fn node -> Macro.update_meta(node, &([generated: true] ++ &1)) end) + end + + def __expr__({patterns, guards, body}) do + with {:ok, _types, context} <- + Pattern.of_head(patterns, guards, TypeHelper.new_stack(), TypeHelper.new_context()), + {:ok, _type, context} <- Expr.of_expr(body, :dynamic, TypeHelper.new_stack(), context) do + case context.warnings do + [warning] -> to_message(:warning, warning) + _ -> :none + end + else + {:error, {type, reason, context}} -> + to_message(:error, {type, reason, context}) + end + end + + defp reset_line(ast, min_line) do + Macro.prewalk(ast, fn ast -> + Macro.update_meta(ast, fn meta -> + Keyword.update!(meta, :line, &(&1 - min_line + 1)) + end) + end) + end + + defp min_line(ast) do + {_ast, min} = + Macro.prewalk(ast, :infinity, fn + {_fun, meta, _args} = ast, min -> {ast, min(min, Keyword.get(meta, :line, 1))} + other, min -> {other, min} + end) + + min + end + + defp to_message(:warning, {module, warning, _location}) do + warning + |> module.format_warning() + |> IO.iodata_to_binary() + end + + defp to_message(:error, {type, reason, context}) do + {Module.Types, error, _location} = Module.Types.error_to_warning(type, reason, context) + + error + |> Module.Types.format_warning() + |> IO.iodata_to_binary() + |> String.trim_trailing("\nConflict found at") + end + + test "expr_to_string/1" do + assert Types.expr_to_string({1, 2}) == "{1, 2}" + assert Types.expr_to_string(quote(do: Foo.bar(arg))) == "Foo.bar(arg)" + assert Types.expr_to_string(quote(do: :erlang.band(a, b))) == "Bitwise.band(a, b)" + assert Types.expr_to_string(quote(do: :erlang.orelse(a, b))) == "a or b" + assert Types.expr_to_string(quote(do: :erlang."=:="(a, b))) == "a === b" + assert Types.expr_to_string(quote(do: :erlang.list_to_atom(a))) == "List.to_atom(a)" + assert Types.expr_to_string(quote(do: :maps.remove(a, b))) == "Map.delete(b, a)" + assert Types.expr_to_string(quote(do: :erlang.element(1, a))) == "elem(a, 0)" + assert Types.expr_to_string(quote(do: :erlang.element(:erlang.+(a, 1), b))) == "elem(b, a)" + end + + test "undefined function warnings" do + assert warning([], URI.unknown("foo")) == + "URI.unknown/1 is undefined or private" + + assert warning([], if(true, do: URI.unknown("foo"))) == + "URI.unknown/1 is undefined or private" + + assert warning([], try(do: :ok, after: URI.unknown("foo"))) == + "URI.unknown/1 is undefined or private" + end + + describe "function head warnings" do + test "warns on literals" do + string = warning([var = 123, var = "abc"], var) + + assert string == """ + incompatible types: + + integer() !~ binary() + + in expression: + + # types_test.ex:1 + var = "abc" + + where "var" was given the type integer() in: + + # types_test.ex:1 + var = 123 + + where "var" was given the type binary() in: + + # types_test.ex:1 + var = "abc" + """ + end + + test "warns on binary patterns" do + string = warning([<>], var) + + assert string == """ + incompatible types: + + integer() !~ binary() + + in expression: + + # types_test.ex:1 + <<..., var::binary>> + + where "var" was given the type integer() in: + + # types_test.ex:1 + <> + + where "var" was given the type binary() in: + + # types_test.ex:1 + <<..., var::binary>> + """ + end + + test "warns on recursive patterns" do + string = warning([{var} = var], var) + + assert string == """ + incompatible types: + + {var1} !~ var1 + + in expression: + + # types_test.ex:1 + {var} = var + + where "var" was given the type {var1} in: + + # types_test.ex:1 + {var} = var + """ + end + + test "warns on guards" do + string = warning([var], [is_integer(var) and is_binary(var)], var) + + assert string == """ + incompatible types: + + integer() !~ binary() + + in expression: + + # types_test.ex:1 + is_binary(var) + + where "var" was given the type integer() in: + + # types_test.ex:1 + is_integer(var) + + where "var" was given the type binary() in: + + # types_test.ex:1 + is_binary(var) + """ + end + + test "warns on guards with multiple variables" do + string = warning([x = y], [is_integer(x) and is_binary(y)], {x, y}) + + assert string == """ + incompatible types: + + integer() !~ binary() + + in expression: + + # types_test.ex:1 + is_binary(y) + + where "x" was given the same type as "y" in: + + # types_test.ex:1 + x = y + + where "y" was given the type integer() in: + + # types_test.ex:1 + is_integer(x) + + where "y" was given the type binary() in: + + # types_test.ex:1 + is_binary(y) + """ + end + + test "warns on guards from cases unless generated" do + string = + warning( + [var], + [is_integer(var)], + case var do + _ when is_binary(var) -> :ok + end + ) + + assert is_binary(string) + + string = + generated( + warning( + [var], + [is_integer(var)], + case var do + _ when is_binary(var) -> :ok + end + ) + ) + + assert string == :none + end + + test "only show relevant traces in warning" do + string = warning([x = y, z], [is_integer(x) and is_binary(y) and is_boolean(z)], {x, y, z}) + + assert string == """ + incompatible types: + + integer() !~ binary() + + in expression: + + # types_test.ex:1 + is_binary(y) + + where "x" was given the same type as "y" in: + + # types_test.ex:1 + x = y + + where "y" was given the type integer() in: + + # types_test.ex:1 + is_integer(x) + + where "y" was given the type binary() in: + + # types_test.ex:1 + is_binary(y) + """ + end + + test "check body" do + string = warning([x], [is_integer(x)], :foo = x) + + assert string == """ + incompatible types: + + integer() !~ :foo + + in expression: + + # types_test.ex:1 + :foo = x + + where "x" was given the type integer() in: + + # types_test.ex:1 + is_integer(x) + + where "x" was given the type :foo in: + + # types_test.ex:1 + :foo = x + """ + end + + test "check binary" do + string = warning([foo], [is_binary(foo)], <>) + + assert string == """ + incompatible types: + + binary() !~ integer() + + in expression: + + # types_test.ex:1 + <> + + where "foo" was given the type binary() in: + + # types_test.ex:1 + is_binary(foo) + + where "foo" was given the type integer() in: + + # types_test.ex:1 + <> + + HINT: all expressions given to binaries are assumed to be of type \ + integer() unless said otherwise. For example, <> assumes "expr" \ + is an integer. Pass a modifier, such as <> or <>, \ + to change the default behaviour. + """ + + string = warning([foo], [is_binary(foo)], <>) + + assert string == """ + incompatible types: + + binary() !~ integer() + + in expression: + + # types_test.ex:1 + <> + + where "foo" was given the type binary() in: + + # types_test.ex:1 + is_binary(foo) + + where "foo" was given the type integer() in: + + # types_test.ex:1 + <> + """ + end + + test "is_tuple warning" do + string = warning([foo], [is_tuple(foo)], {_} = foo) + + assert string == """ + incompatible types: + + tuple() !~ {dynamic()} + + in expression: + + # types_test.ex:1 + {_} = foo + + where "foo" was given the type tuple() in: + + # types_test.ex:1 + is_tuple(foo) + + where "foo" was given the type {dynamic()} in: + + # types_test.ex:1 + {_} = foo + + HINT: use pattern matching or "is_tuple(foo) and tuple_size(foo) == 1" to guard a sized tuple. + """ + end + + test "function call" do + string = warning([foo], [rem(foo, 2.0) == 0], foo) + + assert string == """ + expected Kernel.rem/2 to have signature: + + var1, float() -> dynamic() + + but it has signature: + + integer(), integer() -> integer() + + in expression: + + # types_test.ex:1 + rem(foo, 2.0) + """ + end + + test "operator call" do + string = warning([foo], [foo - :bar == 0], foo) + + assert string == """ + expected Kernel.-/2 to have signature: + + var1, :bar -> dynamic() + + but it has signature: + + integer(), integer() -> integer() + float(), integer() | float() -> float() + integer() | float(), float() -> float() + + in expression: + + # types_test.ex:1 + foo - :bar + """ + end + end + + describe "map warnings" do + test "handling of non-singleton types in maps" do + string = + warning( + [], + ( + event = %{"type" => "order"} + %{"amount" => amount} = event + %{"user" => user} = event + %{"id" => user_id} = user + {:order, user_id, amount} + ) + ) + + assert string == """ + incompatible types: + + binary() !~ map() + + in expression: + + # types_test.ex:5 + %{"id" => user_id} = user + + where "amount" was given the type binary() in: + + # types_test.ex:3 + %{"amount" => amount} = event + + where "amount" was given the same type as "user" in: + + # types_test.ex:4 + %{"user" => user} = event + + where "user" was given the type binary() in: + + # types_test.ex:4 + %{"user" => user} = event + + where "user" was given the type map() in: + + # types_test.ex:5 + %{"id" => user_id} = user + """ + end + + test "show map() when comparing against non-map" do + string = + warning( + [foo], + ( + foo.bar + :atom = foo + ) + ) + + assert string == """ + incompatible types: + + map() !~ :atom + + in expression: + + # types_test.ex:4 + :atom = foo + + where "foo" was given the type map() (due to calling var.field) in: + + # types_test.ex:3 + foo.bar + + where "foo" was given the type :atom in: + + # types_test.ex:4 + :atom = foo + + HINT: "var.field" (without parentheses) implies "var" is a map() while \ + "var.fun()" (with parentheses) implies "var" is an atom() + """ + end + + test "use module as map (without parentheses)" do + string = + warning( + [foo], + ( + %module{} = foo + module.__struct__ + ) + ) + + assert string == """ + incompatible types: + + map() !~ atom() + + in expression: + + # types_test.ex:4 + module.__struct__ + + where "module" was given the type atom() in: + + # types_test.ex:3 + %module{} + + where "module" was given the type map() (due to calling var.field) in: + + # types_test.ex:4 + module.__struct__ + + HINT: "var.field" (without parentheses) implies "var" is a map() while \ + "var.fun()" (with parentheses) implies "var" is an atom() + """ + end + + test "use map as module (with parentheses)" do + string = warning([foo], [is_map(foo)], foo.__struct__()) + + assert string == """ + incompatible types: + + map() !~ atom() + + in expression: + + # types_test.ex:1 + foo.__struct__() + + where "foo" was given the type map() in: + + # types_test.ex:1 + is_map(foo) + + where "foo" was given the type atom() (due to calling var.fun()) in: + + # types_test.ex:1 + foo.__struct__() + + HINT: "var.field" (without parentheses) implies "var" is a map() while \ + "var.fun()" (with parentheses) implies "var" is an atom() + """ + end + + test "non-existent map field warning" do + string = + warning( + ( + map = %{foo: 1} + map.bar + ) + ) + + assert string == """ + undefined field "bar" in expression: + + # types_test.ex:3 + map.bar + + expected one of the following fields: foo + + where "map" was given the type map() in: + + # types_test.ex:2 + map = %{foo: 1} + """ + end + + test "non-existent struct field warning" do + string = + warning( + [foo], + ( + %URI{} = foo + foo.bar + ) + ) + + assert string == """ + undefined field "bar" in expression: + + # types_test.ex:4 + foo.bar + + expected one of the following fields: __struct__, authority, fragment, host, path, port, query, scheme, userinfo + + where "foo" was given the type %URI{} in: + + # types_test.ex:3 + %URI{} = foo + """ + end + + test "expands type variables" do + string = + warning( + [%{foo: key} = event, other_key], + [is_integer(key) and is_atom(other_key)], + %{foo: ^other_key} = event + ) + + assert string == """ + incompatible types: + + %{foo: integer()} !~ %{foo: atom()} + + in expression: + + # types_test.ex:3 + %{foo: ^other_key} = event + + where "event" was given the type %{foo: integer(), optional(dynamic()) => dynamic()} in: + + # types_test.ex:1 + %{foo: key} = event + + where "event" was given the type %{foo: atom(), optional(dynamic()) => dynamic()} in: + + # types_test.ex:3 + %{foo: ^other_key} = event + """ + end + + test "expands map when maps are nested" do + string = + warning( + [map1, map2], + ( + [_var1, _var2] = [map1, map2] + %{} = map1 + %{} = map2.subkey + ) + ) + + assert string == """ + incompatible types: + + %{subkey: var1, optional(dynamic()) => dynamic()} !~ %{optional(dynamic()) => dynamic()} | %{optional(dynamic()) => dynamic()} + + in expression: + + # types_test.ex:5 + map2.subkey + + where "map2" was given the type %{optional(dynamic()) => dynamic()} | %{optional(dynamic()) => dynamic()} in: + + # types_test.ex:3 + [_var1, _var2] = [map1, map2] + + where "map2" was given the type %{subkey: var1, optional(dynamic()) => dynamic()} (due to calling var.field) in: + + # types_test.ex:5 + map2.subkey + + HINT: "var.field" (without parentheses) implies "var" is a map() while "var.fun()" (with parentheses) implies "var" is an atom() + """ + end + end + + describe "regressions" do + test "recursive map fields" do + assert warning( + [queried], + with( + true <- is_nil(queried.foo.bar), + _ = queried.foo + ) do + %{foo: %{other_id: _other_id} = foo} = queried + %{other_id: id} = foo + %{id: id} + end + ) == :none + end + + test "no-recursion on guards with map fields" do + assert warning( + [assigns], + ( + variable_enum = assigns.variable_enum + + case true do + _ when variable_enum != nil -> assigns.variable_enum + end + ) + ) == :none + end + + test "map patterns with pinned keys and field access" do + assert warning( + [x, y], + ( + key_var = y + %{^key_var => _value} = x + key_var2 = y + %{^key_var2 => _value2} = x + y.z + ) + ) == :none + end + + test "map patterns with pinned keys" do + assert warning( + [x, y], + ( + key_var = y + %{^key_var => _value} = x + key_var2 = y + %{^key_var2 => _value2} = x + key_var3 = y + %{^key_var3 => _value3} = x + ) + ) == :none + end + + test "map updates with var key" do + assert warning( + [state0, key0], + ( + state1 = %{state0 | key0 => true} + key1 = key0 + state2 = %{state1 | key1 => true} + state2 + ) + ) == :none + end + end +end diff --git a/lib/elixir/test/elixir/module/types/unify_test.exs b/lib/elixir/test/elixir/module/types/unify_test.exs new file mode 100644 index 00000000000..672fe3892b0 --- /dev/null +++ b/lib/elixir/test/elixir/module/types/unify_test.exs @@ -0,0 +1,770 @@ +Code.require_file("type_helper.exs", __DIR__) + +defmodule Module.Types.UnifyTest do + use ExUnit.Case, async: true + import Module.Types.Unify + alias Module.Types + + defp unify_lift(left, right, context \\ new_context()) do + unify(left, right, new_stack(), context) + |> lift_result() + end + + defp new_context() do + Types.context("types_test.ex", TypesTest, {:test, 0}, [], Module.ParallelChecker.test_cache()) + end + + defp new_stack() do + %{ + Types.stack() + | context: :pattern, + last_expr: {:foo, [], nil} + } + end + + defp unify(left, right, context) do + unify(left, right, new_stack(), context) + end + + defp lift_result({:ok, type, context}) do + {:ok, lift_type(type, context)} + end + + defp lift_result({:error, {type, reason, _context}}) do + {:error, {type, reason}} + end + + defp lift_type(type, context) do + {[type], _context} = lift_types([type], context) + type + end + + defp format_type_string(type, simplify?) do + IO.iodata_to_binary(format_type(type, simplify?)) + end + + describe "unify/3" do + test "literal" do + assert unify_lift({:atom, :foo}, {:atom, :foo}) == {:ok, {:atom, :foo}} + + assert {:error, {:unable_unify, {{:atom, :foo}, {:atom, :bar}, _}}} = + unify_lift({:atom, :foo}, {:atom, :bar}) + end + + test "type" do + assert unify_lift(:integer, :integer) == {:ok, :integer} + assert unify_lift(:binary, :binary) == {:ok, :binary} + assert unify_lift(:atom, :atom) == {:ok, :atom} + + assert {:error, {:unable_unify, {:integer, :atom, _}}} = unify_lift(:integer, :atom) + end + + test "atom subtype" do + assert unify_lift({:atom, true}, :atom) == {:ok, {:atom, true}} + assert {:error, _} = unify_lift(:atom, {:atom, true}) + end + + test "tuple" do + assert unify_lift({:tuple, 0, []}, {:tuple, 0, []}) == {:ok, {:tuple, 0, []}} + + assert unify_lift({:tuple, 1, [:integer]}, {:tuple, 1, [:integer]}) == + {:ok, {:tuple, 1, [:integer]}} + + assert unify_lift({:tuple, 1, [{:atom, :foo}]}, {:tuple, 1, [:atom]}) == + {:ok, {:tuple, 1, [{:atom, :foo}]}} + + assert {:error, {:unable_unify, {{:tuple, 1, [:integer]}, {:tuple, 0, []}, _}}} = + unify_lift({:tuple, 1, [:integer]}, {:tuple, 0, []}) + + assert {:error, {:unable_unify, {:integer, :atom, _}}} = + unify_lift({:tuple, 1, [:integer]}, {:tuple, 1, [:atom]}) + end + + test "list" do + assert unify_lift({:list, :integer}, {:list, :integer}) == {:ok, {:list, :integer}} + + assert {:error, {:unable_unify, {:atom, :integer, _}}} = + unify_lift({:list, :atom}, {:list, :integer}) + end + + test "map" do + assert unify_lift({:map, []}, {:map, []}) == {:ok, {:map, []}} + + assert unify_lift( + {:map, [{:required, :integer, :atom}]}, + {:map, [{:optional, :dynamic, :dynamic}]} + ) == + {:ok, {:map, [{:required, :integer, :atom}]}} + + assert unify_lift( + {:map, [{:optional, :dynamic, :dynamic}]}, + {:map, [{:required, :integer, :atom}]} + ) == + {:ok, {:map, [{:required, :integer, :atom}]}} + + assert unify_lift( + {:map, [{:optional, :dynamic, :dynamic}]}, + {:map, [{:required, :integer, :atom}, {:optional, :dynamic, :dynamic}]} + ) == + {:ok, {:map, [{:required, :integer, :atom}, {:optional, :dynamic, :dynamic}]}} + + assert unify_lift( + {:map, [{:required, :integer, :atom}, {:optional, :dynamic, :dynamic}]}, + {:map, [{:optional, :dynamic, :dynamic}]} + ) == + {:ok, {:map, [{:required, :integer, :atom}, {:optional, :dynamic, :dynamic}]}} + + assert unify_lift( + {:map, [{:required, :integer, :atom}]}, + {:map, [{:required, :integer, :atom}]} + ) == + {:ok, {:map, [{:required, :integer, :atom}]}} + + assert {:error, + {:unable_unify, + {{:map, [{:required, :integer, :atom}]}, {:map, [{:required, :atom, :integer}]}, _}}} = + unify_lift( + {:map, [{:required, :integer, :atom}]}, + {:map, [{:required, :atom, :integer}]} + ) + + assert {:error, {:unable_unify, {{:map, [{:required, :integer, :atom}]}, {:map, []}, _}}} = + unify_lift({:map, [{:required, :integer, :atom}]}, {:map, []}) + + assert {:error, {:unable_unify, {{:map, []}, {:map, [{:required, :integer, :atom}]}, _}}} = + unify_lift({:map, []}, {:map, [{:required, :integer, :atom}]}) + + assert {:error, + {:unable_unify, + {{:map, [{:required, {:atom, :foo}, :integer}]}, + {:map, [{:required, {:atom, :foo}, :atom}]}, + _}}} = + unify_lift( + {:map, [{:required, {:atom, :foo}, :integer}]}, + {:map, [{:required, {:atom, :foo}, :atom}]} + ) + end + + test "map required/optional key" do + assert unify_lift( + {:map, [{:required, {:atom, :foo}, {:atom, :bar}}]}, + {:map, [{:required, {:atom, :foo}, :atom}]} + ) == + {:ok, {:map, [{:required, {:atom, :foo}, {:atom, :bar}}]}} + + assert unify_lift( + {:map, [{:optional, {:atom, :foo}, {:atom, :bar}}]}, + {:map, [{:required, {:atom, :foo}, :atom}]} + ) == + {:ok, {:map, [{:required, {:atom, :foo}, {:atom, :bar}}]}} + + assert unify_lift( + {:map, [{:required, {:atom, :foo}, {:atom, :bar}}]}, + {:map, [{:optional, {:atom, :foo}, :atom}]} + ) == + {:ok, {:map, [{:required, {:atom, :foo}, {:atom, :bar}}]}} + + assert unify_lift( + {:map, [{:optional, {:atom, :foo}, {:atom, :bar}}]}, + {:map, [{:optional, {:atom, :foo}, :atom}]} + ) == + {:ok, {:map, [{:optional, {:atom, :foo}, {:atom, :bar}}]}} + end + + test "map with subtyped keys" do + assert unify_lift( + {:map, [{:required, {:atom, :foo}, :integer}]}, + {:map, [{:required, :atom, :integer}]} + ) == {:ok, {:map, [{:required, {:atom, :foo}, :integer}]}} + + assert unify_lift( + {:map, [{:optional, {:atom, :foo}, :integer}]}, + {:map, [{:required, :atom, :integer}]} + ) == {:ok, {:map, [{:required, {:atom, :foo}, :integer}]}} + + assert unify_lift( + {:map, [{:required, {:atom, :foo}, :integer}]}, + {:map, [{:optional, :atom, :integer}]} + ) == {:ok, {:map, [{:required, {:atom, :foo}, :integer}]}} + + assert unify_lift( + {:map, [{:optional, {:atom, :foo}, :integer}]}, + {:map, [{:optional, :atom, :integer}]} + ) == {:ok, {:map, [{:optional, {:atom, :foo}, :integer}]}} + + assert {:error, + {:unable_unify, + {{:map, [{:required, :atom, :integer}]}, + {:map, [{:required, {:atom, :foo}, :integer}]}, + _}}} = + unify_lift( + {:map, [{:required, :atom, :integer}]}, + {:map, [{:required, {:atom, :foo}, :integer}]} + ) + + assert {:error, + {:unable_unify, + {{:map, [{:optional, :atom, :integer}]}, + {:map, [{:required, {:atom, :foo}, :integer}]}, + _}}} = + unify_lift( + {:map, [{:optional, :atom, :integer}]}, + {:map, [{:required, {:atom, :foo}, :integer}]} + ) + + assert {:error, + {:unable_unify, + {{:map, [{:required, :atom, :integer}]}, + {:map, [{:optional, {:atom, :foo}, :integer}]}, + _}}} = + unify_lift( + {:map, [{:required, :atom, :integer}]}, + {:map, [{:optional, {:atom, :foo}, :integer}]} + ) + + assert unify_lift( + {:map, [{:optional, :atom, :integer}]}, + {:map, [{:optional, {:atom, :foo}, :integer}]} + ) == {:ok, {:map, []}} + + assert unify_lift( + {:map, [{:required, {:atom, :foo}, :integer}]}, + {:map, [{:required, {:atom, :foo}, :integer}]} + ) == {:ok, {:map, [{:required, {:atom, :foo}, :integer}]}} + + assert unify_lift( + {:map, [{:required, {:atom, :foo}, :integer}]}, + {:map, [{:optional, {:atom, :foo}, :integer}]} + ) == {:ok, {:map, [{:required, {:atom, :foo}, :integer}]}} + + assert unify_lift( + {:map, [{:optional, {:atom, :foo}, :integer}]}, + {:map, [{:required, {:atom, :foo}, :integer}]} + ) == {:ok, {:map, [{:required, {:atom, :foo}, :integer}]}} + + assert unify_lift( + {:map, [{:optional, {:atom, :foo}, :integer}]}, + {:map, [{:optional, {:atom, :foo}, :integer}]} + ) == {:ok, {:map, [{:optional, {:atom, :foo}, :integer}]}} + end + + test "map with subtyped and multiple matching keys" do + assert {:error, _} = + unify_lift( + {:map, + [ + {:required, {:atom, :foo}, :integer}, + {:required, :atom, {:union, [:integer, :boolean]}} + ]}, + {:map, [{:required, :atom, :integer}]} + ) + + assert unify_lift( + {:map, + [ + {:required, {:atom, :foo}, :integer}, + {:required, :atom, {:union, [:integer, :boolean]}} + ]}, + {:map, [{:required, :atom, {:union, [:integer, :boolean]}}]} + ) == + {:ok, + {:map, + [ + {:required, {:atom, :foo}, :integer}, + {:required, :atom, {:union, [:integer, :boolean]}} + ]}} + + assert {:error, _} = + unify_lift( + {:map, [{:required, :atom, :integer}]}, + {:map, + [ + {:required, {:atom, :foo}, :integer}, + {:required, :atom, {:union, [:integer, :boolean]}} + ]} + ) + + assert {:error, _} = + unify_lift( + {:map, [{:required, :atom, {:union, [:integer, :boolean]}}]}, + {:map, + [ + {:required, {:atom, :foo}, :integer}, + {:required, :atom, {:union, [:integer, :boolean]}} + ]} + ) + + assert unify_lift( + {:map, [{:required, :atom, :integer}]}, + {:map, + [ + {:optional, {:atom, :foo}, :integer}, + {:optional, :atom, {:union, [:integer, :boolean]}} + ]} + ) == {:ok, {:map, [{:required, :atom, :integer}]}} + + assert unify_lift( + {:map, [{:required, :atom, {:union, [:integer, :boolean]}}]}, + {:map, + [ + {:optional, {:atom, :foo}, :integer}, + {:optional, :atom, {:union, [:integer, :boolean]}} + ]} + ) == {:ok, {:map, [{:required, :atom, {:union, [:integer, :boolean]}}]}} + + assert unify_lift( + {:map, + [ + {:optional, {:atom, :foo}, :integer}, + {:optional, :atom, {:union, [:integer, :boolean]}} + ]}, + {:map, [{:required, :atom, :integer}]} + ) == {:ok, {:map, [{:required, {:atom, :foo}, :integer}]}} + + # TODO: FIX ME + # assert unify_lift( + # {:map, + # [ + # {:optional, {:atom, :foo}, :integer}, + # {:optional, :atom, {:union, [:integer, :boolean]}} + # ]}, + # {:map, [{:required, :atom, {:union, [:integer, :boolean]}}]} + # ) == + # {:ok, + # {:map, + # [ + # {:required, {:atom, :foo}, :integer}, + # {:required, :atom, {:union, [:integer, :boolean]}} + # ]}} + + assert {:error, _} = + unify_lift( + {:map, + [ + {:optional, {:atom, :foo}, :integer}, + {:optional, :atom, {:union, [:integer, :boolean]}} + ]}, + {:map, [{:optional, :atom, :integer}]} + ) + + assert unify_lift( + {:map, + [ + {:optional, {:atom, :foo}, :integer}, + {:optional, :atom, {:union, [:integer, :boolean]}} + ]}, + {:map, [{:optional, :atom, {:union, [:integer, :boolean]}}]} + ) == + {:ok, + {:map, + [ + {:optional, {:atom, :foo}, :integer}, + {:optional, :atom, {:union, [:integer, :boolean]}} + ]}} + + assert unify_lift( + {:map, [{:optional, :atom, :integer}]}, + {:map, + [ + {:optional, {:atom, :foo}, :integer}, + {:optional, :atom, {:union, [:integer, :boolean]}} + ]} + ) == {:ok, {:map, [{:optional, :atom, :integer}]}} + + assert unify_lift( + {:map, [{:optional, :atom, {:union, [:integer, :boolean]}}]}, + {:map, + [ + {:optional, {:atom, :foo}, :integer}, + {:optional, :atom, {:union, [:integer, :boolean]}} + ]} + ) == {:ok, {:map, [{:optional, :atom, {:union, [:integer, :boolean]}}]}} + end + + test "union" do + assert unify_lift({:union, []}, {:union, []}) == {:ok, {:union, []}} + assert unify_lift({:union, [:integer]}, {:union, [:integer]}) == {:ok, {:union, [:integer]}} + + assert unify_lift({:union, [:integer, :atom]}, {:union, [:integer, :atom]}) == + {:ok, {:union, [:integer, :atom]}} + + assert unify_lift({:union, [:integer, :atom]}, {:union, [:atom, :integer]}) == + {:ok, {:union, [:integer, :atom]}} + + assert unify_lift({:union, [{:atom, :bar}]}, {:union, [:atom]}) == + {:ok, {:atom, :bar}} + + assert {:error, {:unable_unify, {:integer, {:union, [:atom]}, _}}} = + unify_lift({:union, [:integer]}, {:union, [:atom]}) + end + + test "dynamic" do + assert unify_lift({:atom, :foo}, :dynamic) == {:ok, {:atom, :foo}} + assert unify_lift(:dynamic, {:atom, :foo}) == {:ok, {:atom, :foo}} + assert unify_lift(:integer, :dynamic) == {:ok, :integer} + assert unify_lift(:dynamic, :integer) == {:ok, :integer} + end + + test "vars" do + {{:var, 0}, var_context} = new_var({:foo, [version: 0], nil}, new_context()) + {{:var, 1}, var_context} = new_var({:bar, [version: 1], nil}, var_context) + + assert {:ok, {:var, 0}, context} = unify({:var, 0}, :integer, var_context) + assert lift_type({:var, 0}, context) == :integer + + assert {:ok, {:var, 0}, context} = unify(:integer, {:var, 0}, var_context) + assert lift_type({:var, 0}, context) == :integer + + assert {:ok, {:var, _}, context} = unify({:var, 0}, {:var, 1}, var_context) + assert {:var, _} = lift_type({:var, 0}, context) + assert {:var, _} = lift_type({:var, 1}, context) + + assert {:ok, {:var, 0}, context} = unify({:var, 0}, :integer, var_context) + assert {:ok, {:var, 1}, context} = unify({:var, 1}, :integer, context) + assert {:ok, {:var, _}, _context} = unify({:var, 0}, {:var, 1}, context) + + assert {:ok, {:var, 0}, context} = unify({:var, 0}, :integer, var_context) + assert {:ok, {:var, 1}, context} = unify({:var, 1}, :integer, context) + assert {:ok, {:var, _}, _context} = unify({:var, 1}, {:var, 0}, context) + + assert {:ok, {:var, 0}, context} = unify({:var, 0}, :integer, var_context) + assert {:ok, {:var, 1}, context} = unify({:var, 1}, :binary, context) + + assert {:error, {:unable_unify, {:integer, :binary, _}}} = + unify_lift({:var, 0}, {:var, 1}, context) + + assert {:ok, {:var, 0}, context} = unify({:var, 0}, :integer, var_context) + assert {:ok, {:var, 1}, context} = unify({:var, 1}, :binary, context) + + assert {:error, {:unable_unify, {:binary, :integer, _}}} = + unify_lift({:var, 1}, {:var, 0}, context) + end + + test "vars inside tuples" do + {{:var, 0}, var_context} = new_var({:foo, [version: 0], nil}, new_context()) + {{:var, 1}, var_context} = new_var({:bar, [version: 1], nil}, var_context) + + assert {:ok, {:tuple, 1, [{:var, 0}]}, context} = + unify({:tuple, 1, [{:var, 0}]}, {:tuple, 1, [:integer]}, var_context) + + assert lift_type({:var, 0}, context) == :integer + + assert {:ok, {:var, 0}, context} = unify({:var, 0}, :integer, var_context) + assert {:ok, {:var, 1}, context} = unify({:var, 1}, :integer, context) + + assert {:ok, {:tuple, 1, [{:var, _}]}, _context} = + unify({:tuple, 1, [{:var, 0}]}, {:tuple, 1, [{:var, 1}]}, context) + + assert {:ok, {:var, 1}, context} = unify({:var, 1}, {:tuple, 1, [{:var, 0}]}, var_context) + assert {:ok, {:var, 0}, context} = unify({:var, 0}, :integer, context) + assert lift_type({:var, 1}, context) == {:tuple, 1, [:integer]} + + assert {:ok, {:var, 0}, context} = unify({:var, 0}, :integer, var_context) + assert {:ok, {:var, 1}, context} = unify({:var, 1}, :binary, context) + + assert {:error, {:unable_unify, {:integer, :binary, _}}} = + unify_lift({:tuple, 1, [{:var, 0}]}, {:tuple, 1, [{:var, 1}]}, context) + end + + # TODO: Vars inside right unions + + test "vars inside left unions" do + {{:var, 0}, var_context} = new_var({:foo, [version: 0], nil}, new_context()) + + assert {:ok, {:var, 0}, context} = + unify({:union, [{:var, 0}, :integer]}, :integer, var_context) + + assert lift_type({:var, 0}, context) == :integer + + assert {:ok, {:var, 0}, context} = + unify({:union, [{:var, 0}, :integer]}, {:union, [:integer, :atom]}, var_context) + + assert lift_type({:var, 0}, context) == {:union, [:integer, :atom]} + + assert {:error, {:unable_unify, {:integer, {:union, [:binary, :atom]}, _}}} = + unify_lift( + {:union, [{:var, 0}, :integer]}, + {:union, [:binary, :atom]}, + var_context + ) + end + + test "recursive type" do + assert {{:var, 0}, var_context} = new_var({:foo, [version: 0], nil}, new_context()) + assert {{:var, 1}, var_context} = new_var({:bar, [version: 1], nil}, var_context) + assert {{:var, 2}, var_context} = new_var({:baz, [version: 2], nil}, var_context) + + assert {:ok, {:var, _}, context} = unify({:var, 0}, {:var, 1}, var_context) + assert {:ok, {:var, _}, context} = unify({:var, 1}, {:var, 0}, context) + assert context.types[0] == {:var, 1} + assert context.types[1] == {:var, 0} + + assert {:ok, {:var, _}, context} = unify({:var, 0}, :tuple, var_context) + assert {:ok, {:var, _}, context} = unify({:var, 1}, {:var, 0}, context) + assert {:ok, {:var, _}, context} = unify({:var, 0}, {:var, 1}, context) + assert context.types[0] == :tuple + assert context.types[1] == {:var, 0} + + assert {:ok, {:var, _}, context} = unify({:var, 0}, {:var, 1}, var_context) + assert {:ok, {:var, _}, context} = unify({:var, 1}, {:var, 2}, context) + assert {:ok, {:var, _}, _context} = unify({:var, 2}, {:var, 0}, context) + assert context.types[0] == {:var, 1} + assert context.types[1] == {:var, 2} + assert context.types[2] == :unbound + + assert {:ok, {:var, _}, context} = unify({:var, 0}, {:var, 1}, var_context) + + assert {:error, {:unable_unify, {{:var, 1}, {:tuple, 1, [{:var, 0}]}, _}}} = + unify_lift({:var, 1}, {:tuple, 1, [{:var, 0}]}, context) + + assert {:ok, {:var, _}, context} = unify({:var, 0}, {:var, 1}, var_context) + assert {:ok, {:var, _}, context} = unify({:var, 1}, {:var, 2}, context) + + assert {:error, {:unable_unify, {{:var, 2}, {:tuple, 1, [{:var, 0}]}, _}}} = + unify_lift({:var, 2}, {:tuple, 1, [{:var, 0}]}, context) + end + + test "error with internal variable" do + context = new_context() + {var_integer, context} = add_var(context) + {var_atom, context} = add_var(context) + + {:ok, _, context} = unify(var_integer, :integer, context) + {:ok, _, context} = unify(var_atom, :atom, context) + + assert {:error, _} = unify(var_integer, var_atom, context) + end + end + + describe "has_unbound_var?/2" do + setup do + context = new_context() + {unbound_var, context} = add_var(context) + {bound_var, context} = add_var(context) + {:ok, _, context} = unify(bound_var, :integer, context) + %{context: context, unbound_var: unbound_var, bound_var: bound_var} + end + + test "returns true when there are unbound vars", + %{context: context, unbound_var: unbound_var} do + assert has_unbound_var?(unbound_var, context) + assert has_unbound_var?({:union, [unbound_var]}, context) + assert has_unbound_var?({:tuple, 1, [unbound_var]}, context) + assert has_unbound_var?({:list, unbound_var}, context) + assert has_unbound_var?({:map, [{:required, unbound_var, :atom}]}, context) + assert has_unbound_var?({:map, [{:required, :atom, unbound_var}]}, context) + end + + test "returns false when there are no unbound vars", + %{context: context, bound_var: bound_var} do + refute has_unbound_var?(bound_var, context) + refute has_unbound_var?({:union, [bound_var]}, context) + refute has_unbound_var?({:tuple, 1, [bound_var]}, context) + refute has_unbound_var?(:integer, context) + refute has_unbound_var?({:list, bound_var}, context) + refute has_unbound_var?({:map, [{:required, :atom, :atom}]}, context) + refute has_unbound_var?({:map, [{:required, bound_var, :atom}]}, context) + refute has_unbound_var?({:map, [{:required, :atom, bound_var}]}, context) + end + end + + describe "subtype?/3" do + test "with simple types" do + assert subtype?({:atom, :foo}, :atom, new_context()) + assert subtype?({:atom, true}, :atom, new_context()) + + refute subtype?(:integer, :binary, new_context()) + refute subtype?(:atom, {:atom, :foo}, new_context()) + refute subtype?(:atom, {:atom, true}, new_context()) + end + + test "with composite types" do + assert subtype?({:list, {:atom, :foo}}, {:list, :atom}, new_context()) + assert subtype?({:tuple, 1, [{:atom, :foo}]}, {:tuple, 1, [:atom]}, new_context()) + + refute subtype?({:list, :atom}, {:list, {:atom, :foo}}, new_context()) + refute subtype?({:tuple, 1, [:atom]}, {:tuple, 1, [{:atom, :foo}]}, new_context()) + refute subtype?({:tuple, 1, [:atom]}, {:tuple, 2, [:atom, :atom]}, new_context()) + refute subtype?({:tuple, 2, [:atom, :atom]}, {:tuple, 1, [:atom]}, new_context()) + + refute subtype?( + {:tuple, 2, [{:atom, :a}, :integer]}, + {:tuple, 2, [{:atom, :b}, :integer]}, + new_context() + ) + end + + test "with maps" do + assert subtype?({:map, [{:optional, :atom, :integer}]}, {:map, []}, new_context()) + + assert subtype?( + {:map, [{:required, :atom, :integer}]}, + {:map, [{:required, :atom, :integer}]}, + new_context() + ) + + assert subtype?( + {:map, [{:required, {:atom, :foo}, :integer}]}, + {:map, [{:required, :atom, :integer}]}, + new_context() + ) + + assert subtype?( + {:map, [{:required, :integer, {:atom, :foo}}]}, + {:map, [{:required, :integer, :atom}]}, + new_context() + ) + + refute subtype?({:map, [{:required, :atom, :integer}]}, {:map, []}, new_context()) + + refute subtype?( + {:map, [{:required, :atom, :integer}]}, + {:map, [{:required, {:atom, :foo}, :integer}]}, + new_context() + ) + + refute subtype?( + {:map, [{:required, :integer, :atom}]}, + {:map, [{:required, :integer, {:atom, :foo}}]}, + new_context() + ) + end + + test "with unions" do + assert subtype?({:union, [{:atom, :foo}]}, {:union, [:atom]}, new_context()) + assert subtype?({:union, [{:atom, :foo}, {:atom, :bar}]}, {:union, [:atom]}, new_context()) + assert subtype?({:union, [{:atom, :foo}]}, {:union, [:integer, :atom]}, new_context()) + + assert subtype?({:atom, :foo}, {:union, [:atom]}, new_context()) + assert subtype?({:atom, :foo}, {:union, [:integer, :atom]}, new_context()) + + assert subtype?({:union, [{:atom, :foo}]}, :atom, new_context()) + assert subtype?({:union, [{:atom, :foo}, {:atom, :bar}]}, :atom, new_context()) + + refute subtype?({:union, [:atom]}, {:union, [{:atom, :foo}]}, new_context()) + refute subtype?({:union, [:atom]}, {:union, [{:atom, :foo}, :integer]}, new_context()) + refute subtype?(:atom, {:union, [{:atom, :foo}, :integer]}, new_context()) + refute subtype?({:union, [:atom]}, {:atom, :foo}, new_context()) + end + end + + test "to_union/2" do + assert to_union([:atom], new_context()) == :atom + assert to_union([:integer, :integer], new_context()) == :integer + assert to_union([{:atom, :foo}, {:atom, :bar}, :atom], new_context()) == :atom + + assert to_union([:binary, :atom], new_context()) == {:union, [:binary, :atom]} + assert to_union([:atom, :binary, :atom], new_context()) == {:union, [:atom, :binary]} + + assert to_union([{:atom, :foo}, :binary, :atom], new_context()) == + {:union, [:binary, :atom]} + + {{:var, 0}, var_context} = new_var({:foo, [version: 0], nil}, new_context()) + assert to_union([{:var, 0}], var_context) == {:var, 0} + + assert to_union([{:tuple, 1, [:integer]}, {:tuple, 1, [:integer]}], new_context()) == + {:tuple, 1, [:integer]} + end + + test "flatten_union/1" do + context = new_context() + assert flatten_union(:binary, context) == [:binary] + assert flatten_union({:atom, :foo}, context) == [{:atom, :foo}] + assert flatten_union({:union, [:binary, {:atom, :foo}]}, context) == [:binary, {:atom, :foo}] + + assert flatten_union({:union, [{:union, [:integer, :binary]}, {:atom, :foo}]}, context) == [ + :integer, + :binary, + {:atom, :foo} + ] + + assert flatten_union({:tuple, 2, [:binary, {:atom, :foo}]}, context) == + [{:tuple, 2, [:binary, {:atom, :foo}]}] + + assert flatten_union({:tuple, 1, [{:union, [:binary, :integer]}]}, context) == + [{:tuple, 1, [:binary]}, {:tuple, 1, [:integer]}] + + assert flatten_union( + {:tuple, 2, [{:union, [:binary, :integer]}, {:union, [:binary, :integer]}]}, + context + ) == + [ + {:tuple, 2, [:binary, :binary]}, + {:tuple, 2, [:binary, :integer]}, + {:tuple, 2, [:integer, :binary]}, + {:tuple, 2, [:integer, :integer]} + ] + + {{:var, 0}, var_context} = new_var({:foo, [version: 0], nil}, new_context()) + assert flatten_union({:var, 0}, var_context) == [{:var, 0}] + + {:ok, {:var, 0}, var_context} = unify({:var, 0}, :integer, var_context) + assert flatten_union({:var, 0}, var_context) == [:integer] + + {{:var, 0}, var_context} = new_var({:foo, [version: 0], nil}, new_context()) + {:ok, {:var, 0}, var_context} = unify({:var, 0}, {:union, [:integer, :float]}, var_context) + assert flatten_union({:var, 0}, var_context) == [:integer, :float] + + {{:var, 0}, var_context} = new_var({:foo, [version: 0], nil}, new_context()) + {{:var, 1}, var_context} = new_var({:bar, [version: 1], nil}, var_context) + {:ok, {:var, 0}, var_context} = unify({:var, 0}, {:var, 1}, var_context) + assert flatten_union({:var, 0}, var_context) == [{:var, 1}] + assert flatten_union({:var, 1}, var_context) == [{:var, 1}] + end + + test "format_type/1" do + assert format_type_string(:binary, false) == "binary()" + assert format_type_string({:atom, true}, false) == "true" + assert format_type_string({:atom, :atom}, false) == ":atom" + assert format_type_string({:list, :binary}, false) == "[binary()]" + assert format_type_string({:tuple, 0, []}, false) == "{}" + assert format_type_string({:tuple, 1, [:integer]}, false) == "{integer()}" + + assert format_type_string({:map, []}, true) == "map()" + assert format_type_string({:map, [{:required, {:atom, :foo}, :atom}]}, true) == "map()" + + assert format_type_string({:map, []}, false) == + "%{}" + + assert format_type_string({:map, [{:required, {:atom, :foo}, :atom}]}, false) == + "%{foo: atom()}" + + assert format_type_string({:map, [{:required, :integer, :atom}]}, false) == + "%{integer() => atom()}" + + assert format_type_string({:map, [{:optional, :integer, :atom}]}, false) == + "%{optional(integer()) => atom()}" + + assert format_type_string({:map, [{:optional, {:atom, :foo}, :atom}]}, false) == + "%{optional(:foo) => atom()}" + + assert format_type_string({:map, [{:required, {:atom, :__struct__}, {:atom, Struct}}]}, false) == + "%Struct{}" + + assert format_type_string( + {:map, + [{:required, {:atom, :__struct__}, {:atom, Struct}}, {:required, :integer, :atom}]}, + false + ) == + "%Struct{integer() => atom()}" + + assert format_type_string({:fun, [{[], :dynamic}]}, false) == "(-> dynamic())" + + assert format_type_string({:fun, [{[:integer], :dynamic}]}, false) == + "(integer() -> dynamic())" + + assert format_type_string({:fun, [{[:integer, :float], :dynamic}]}, false) == + "(integer(), float() -> dynamic())" + + assert format_type_string({:fun, [{[:integer], :dynamic}, {[:integer], :dynamic}]}, false) == + "(integer() -> dynamic(); integer() -> dynamic())" + end + + test "walk/3" do + assert walk(:dynamic, :acc, fn :dynamic, :acc -> {:integer, :bar} end) == {:integer, :bar} + + assert walk({:list, {:tuple, [:integer, :binary]}}, 1, fn type, counter -> + {type, counter + 1} + end) == {{:list, {:tuple, [:integer, :binary]}}, 3} + end +end diff --git a/lib/elixir/test/elixir/module_test.exs b/lib/elixir/test/elixir/module_test.exs index a750e93becd..47786a4cf71 100644 --- a/lib/elixir/test/elixir/module_test.exs +++ b/lib/elixir/test/elixir/module_test.exs @@ -1,4 +1,4 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule ModuleTest.ToBeUsed do def value, do: 1 @@ -9,19 +9,22 @@ defmodule ModuleTest.ToBeUsed do Module.put_attribute(target, :before_compile, __MODULE__) Module.put_attribute(target, :after_compile, __MODULE__) Module.put_attribute(target, :before_compile, {__MODULE__, :callback}) - quote do: (def line, do: __ENV__.line) + quote(do: def(line, do: __ENV__.line)) end defmacro __before_compile__(env) do - quote do: (def before_compile, do: unquote(env.vars)) + quote(do: def(before_compile, do: unquote(Macro.Env.vars(env)))) end - defmacro __after_compile__(%Macro.Env{module: ModuleTest.ToUse, vars: []}, bin) when is_binary(bin) do + defmacro __after_compile__(%Macro.Env{module: ModuleTest.ToUse} = env, bin) + when is_binary(bin) do + [] = Macro.Env.vars(env) # IO.puts "HELLO" end defmacro callback(env) do value = Module.get_attribute(env.module, :has_callback) + quote do def callback_value(true), do: unquote(value) end @@ -29,9 +32,11 @@ defmodule ModuleTest.ToBeUsed do end defmodule ModuleTest.ToUse do - 32 = __ENV__.line # Moving the next line around can make tests fail + # Moving the next line around can make tests fail + 36 = __ENV__.line var = 1 - var # Not available in callbacks + # Not available in callbacks + _ = var def callback_value(false), do: false use ModuleTest.ToBeUsed end @@ -39,109 +44,188 @@ end defmodule ModuleTest do use ExUnit.Case, async: true - Module.register_attribute __MODULE__, :register_example, accumulate: true, persist: true + doctest Module + + Module.register_attribute(__MODULE__, :register_unset_example, persist: true) + Module.register_attribute(__MODULE__, :register_empty_example, accumulate: true, persist: true) + Module.register_attribute(__MODULE__, :register_example, accumulate: true, persist: true) @register_example :it_works @register_example :still_works - contents = quote do: (def eval_quoted_info, do: {__MODULE__, __ENV__.file, __ENV__.line}) - Module.eval_quoted __MODULE__, contents, [], file: "sample.ex", line: 13 + contents = + quote do + def eval_quoted_info, do: {__MODULE__, __ENV__.file, __ENV__.line} + end + + Module.eval_quoted(__MODULE__, contents, [], file: "sample.ex", line: 13) + + defp purge(module) do + :code.purge(module) + :code.delete(module) + end defmacrop in_module(block) do quote do - defmodule Temp, unquote(block) - :code.purge(Temp) - :code.delete(Temp) + defmodule(Temp, unquote(block)) + purge(Temp) + end + end + + test "module attributes returns value" do + in_module do + assert @return([:foo, :bar]) == :ok + _ = @return + end + end + + test "raises on write access attempts from __after_compile__/2" do + contents = + quote do + @after_compile __MODULE__ + + def __after_compile__(%Macro.Env{module: module}, bin) when is_binary(bin) do + Module.put_attribute(module, :foo, 42) + end + end + + assert_raise ArgumentError, + "could not call Module.__put_attribute__/4 because the module ModuleTest.Raise is in read-only mode (@after_compile)", + fn -> + Module.create(ModuleTest.Raise, contents, __ENV__) + end + end + + test "supports read access to module from __after_compile__/2" do + defmodule ModuleTest.NoRaise do + @after_compile __MODULE__ + @foo 42 + + def __after_compile__(%Macro.Env{module: module}, bin) when is_binary(bin) do + send(self(), Module.get_attribute(module, :foo)) + end end + + assert_received 42 + end + + test "in memory modules are tagged as so" do + assert :code.which(__MODULE__) == '' end ## Eval - test :eval_quoted do + test "executes eval_quoted definitions" do assert eval_quoted_info() == {ModuleTest, "sample.ex", 13} end - test :line_from_macro do - assert ModuleTest.ToUse.line == 36 + test "resets last definition information on eval" do + # This should not emit any warning + defmodule LastDefinition do + def foo(0), do: 0 + + Module.eval_quoted( + __ENV__, + quote do + def bar, do: :ok + end + ) + + def foo(1), do: 1 + end + end + + test "retrieves line from use callsite" do + assert ModuleTest.ToUse.line() == 41 end ## Callbacks - test :compile_callback_hook do + test "executes custom before_compile callback" do assert ModuleTest.ToUse.callback_value(true) == true assert ModuleTest.ToUse.callback_value(false) == false end - test :before_compile_callback_hook do - assert ModuleTest.ToUse.before_compile == [] + test "executes default before_compile callback" do + assert ModuleTest.ToUse.before_compile() == [] end - test :on_definition do + def __on_definition__(env, kind, name, args, guards, expr) do + Process.put(env.module, {args, guards, expr}) + assert env.module == ModuleTest.OnDefinition + assert kind == :def + assert name == :hello + assert Module.defines?(env.module, {:hello, 2}) + end + + test "executes on definition callback" do defmodule OnDefinition do @on_definition ModuleTest + def hello(foo, bar) + + assert {[{:foo, _, _}, {:bar, _, _}], [], nil} = Process.get(ModuleTest.OnDefinition) + def hello(foo, bar) do foo + bar end - end - assert Process.get(ModuleTest.OnDefinition) == :called + assert {[{:foo, _, _}, {:bar, _, _}], [], [do: {:+, _, [{:foo, _, nil}, {:bar, _, nil}]}]} = + Process.get(ModuleTest.OnDefinition) + end end - def __on_definition__(env, kind, name, args, guards, expr) do - Process.put(env.module, :called) - assert env.module == ModuleTest.OnDefinition - assert kind == :def - assert name == :hello - assert [{:foo, _, _}, {:bar, _ , _}] = args - assert [] = guards - assert {{:., _, [:erlang, :+]}, _, [{:foo, _, nil}, {:bar, _, nil}]} = expr + defmacro __before_compile__(_) do + quote do + def constant, do: 1 + defoverridable constant: 0 + end end - test :overridable_inside_before_compile do + test "may set overridable inside before_compile callback" do defmodule OverridableWithBeforeCompile do @before_compile ModuleTest end - assert OverridableWithBeforeCompile.constant == 1 + + assert OverridableWithBeforeCompile.constant() == 1 end - test :alias_with_raw_atom do - defmodule :"Elixir.ModuleTest.RawModule" do - def hello, do: :world + describe "__info__(:attributes)" do + test "reserved attributes" do + assert List.keyfind(ExUnit.Server.__info__(:attributes), :behaviour, 0) == + {:behaviour, [GenServer]} end - assert RawModule.hello == :world - end + test "registered attributes" do + assert Enum.filter(__MODULE__.__info__(:attributes), &match?({:register_example, _}, &1)) == + [{:register_example, [:it_works]}, {:register_example, [:still_works]}] + end - defmacro __before_compile__(_) do - quote do - def constant, do: 1 - defoverridable constant: 0 + test "registered attributes with no values are not present" do + refute List.keyfind(__MODULE__.__info__(:attributes), :register_unset_example, 0) + refute List.keyfind(__MODULE__.__info__(:attributes), :register_empty_example, 0) end end - ## Attributes - - test :reserved_attributes do - assert List.keyfind(ExUnit.Server.__info__(:attributes), :behaviour, 0) == {:behaviour, [:gen_server]} - end + @some_attribute [1] + @other_attribute [3, 2, 1] - test :registered_attributes do - assert [{:register_example, [:it_works]}, {:register_example, [:still_works]}] == - Enum.filter __MODULE__.__info__(:attributes), &match?({:register_example, _}, &1) + test "inside function attributes" do + assert @some_attribute == [1] + assert @other_attribute == [3, 2, 1] end - @some_attribute [1] - @other_attribute [3, 2, 1] + test "@compile autoload attribute" do + defmodule NoAutoload do + @compile {:autoload, false} + end - test :inside_function_attributes do - assert [1] = @some_attribute - assert [3, 2, 1] = @other_attribute + refute :code.is_loaded(NoAutoload) end ## Naming - test :concat do - assert Module.concat(Foo, Bar) == Foo.Bar + test "concat" do + assert Module.concat(Foo, Bar) == Foo.Bar assert Module.concat(Foo, :Bar) == Foo.Bar assert Module.concat(Foo, "Bar") == Foo.Bar assert Module.concat(Foo, Bar.Baz) == Foo.Bar.Baz @@ -149,96 +233,374 @@ defmodule ModuleTest do assert Module.concat(Bar, nil) == Elixir.Bar end - test :safe_concat do + test "safe concat" do assert Module.safe_concat(Foo, :Bar) == Foo.Bar + assert_raise ArgumentError, fn -> - Module.safe_concat SafeConcat, Doesnt.Exist + Module.safe_concat(SafeConcat, Doesnt.Exist) end end - test :split do + test "split" do module = Very.Long.Module.Name.And.Even.Longer assert Module.split(module) == ["Very", "Long", "Module", "Name", "And", "Even", "Longer"] assert Module.split("Elixir.Very.Long") == ["Very", "Long"] + + assert_raise ArgumentError, "expected an Elixir module, got: :just_an_atom", fn -> + Module.split(:just_an_atom) + end + + assert_raise ArgumentError, "expected an Elixir module, got: \"Foo\"", fn -> + Module.split("Foo") + end + assert Module.concat(Module.split(module)) == module end - test :__MODULE__ do + test "__MODULE__" do assert Code.eval_string("__MODULE__.Foo") |> elem(0) == Foo end + test "__ENV__.file" do + assert Path.basename(__ENV__.file) == "module_test.exs" + end + + @file "sample.ex" + test "@file sets __ENV__.file" do + assert __ENV__.file == "sample.ex" + end + + test "@file raises when invalid" do + assert_raise ArgumentError, ~r"@file is a built-in module attribute", fn -> + defmodule BadFile do + @file :oops + def my_fun, do: :ok + end + end + end + ## Creation - test :defmodule do - assert match?({:module, Defmodule, binary, 3} when is_binary(binary), defmodule Defmodule do - 1 + 2 - end) + test "defmodule" do + result = + defmodule Defmodule do + 1 + 2 + end + + assert {:module, Defmodule, binary, 3} = result + assert is_binary(binary) end - test :defmodule_with_atom do - assert match?({:module, :root_defmodule, _, _}, defmodule :root_defmodule do - :ok - end) + test "defmodule with atom" do + result = + defmodule :root_defmodule do + :ok + end + + assert {:module, :root_defmodule, _, _} = result end - test :create do + test "does not leak alias from atom" do + defmodule :"Elixir.ModuleTest.RawModule" do + def hello, do: :world + end + + refute __ENV__.aliases[Elixir.ModuleTest] + refute __ENV__.aliases[Elixir.RawModule] + assert ModuleTest.RawModule.hello() == :world + end + + test "does not leak alias from non-atom alias" do + defmodule __MODULE__.NonAtomAlias do + def hello, do: :world + end + + refute __ENV__.aliases[Elixir.ModuleTest] + refute __ENV__.aliases[Elixir.NonAtomAlias] + assert Elixir.ModuleTest.NonAtomAlias.hello() == :world + end + + @compile {:no_warn_undefined, ModuleCreateSample} + + test "create" do contents = quote do def world, do: true end - {:module, ModuleCreateSample, _, _} = - Module.create(ModuleCreateSample, contents, __ENV__) - assert ModuleCreateSample.world + + {:module, ModuleCreateSample, _, _} = Module.create(ModuleCreateSample, contents, __ENV__) + assert ModuleCreateSample.world() end - test :create_with_elixir_as_a_name do + test "create with a reserved module name" do contents = quote do def world, do: true end + assert_raise CompileError, fn -> - {:module, Elixir, _, _} = - Module.create(Elixir, contents, __ENV__) + {:module, Elixir, _, _} = Module.create(Elixir, contents, __ENV__) end end - test :no_function_in_module_body do + @compile {:no_warn_undefined, ModuleTracersSample} + + test "create with propagated tracers" do + contents = + quote do + def world, do: true + end + + env = %{__ENV__ | tracers: [:invalid]} + {:module, ModuleTracersSample, _, _} = Module.create(ModuleTracersSample, contents, env) + assert ModuleTracersSample.world() + end + + @compile {:no_warn_undefined, ModuleHygiene} + + test "create with aliases/var hygiene" do + contents = + quote do + alias List, as: L + + def test do + L.flatten([1, [2], 3]) + end + end + + Module.create(ModuleHygiene, contents, __ENV__) + assert ModuleHygiene.test() == [1, 2, 3] + end + + test "ensure function clauses are sorted (to avoid non-determinism in module vsn)" do + {_, _, binary, _} = + defmodule Ordered do + def foo(:foo), do: :bar + def baz(:baz), do: :bat + end + + {:ok, {ModuleTest.Ordered, [abstract_code: {:raw_abstract_v1, abstract_code}]}} = + :beam_lib.chunks(binary, [:abstract_code]) + + # We need to traverse functions instead of using :exports as exports are sorted + funs = for {:function, _, name, arity, _} <- abstract_code, do: {name, arity} + assert funs == [__info__: 1, baz: 1, foo: 1] + end + + @compile {:no_warn_undefined, ModuleCreateGenerated} + + test "create with generated true does not emit warnings" do + contents = + quote generated: true do + def world, do: true + def world, do: false + end + + {:module, ModuleCreateGenerated, _, _} = + Module.create(ModuleCreateGenerated, contents, __ENV__) + + assert ModuleCreateGenerated.world() + end + + test "uses the debug_info chunk" do + {:module, ModuleCreateDebugInfo, binary, _} = + Module.create(ModuleCreateDebugInfo, :ok, __ENV__) + + {:ok, {_, [debug_info: {:debug_info_v1, backend, data}]}} = + :beam_lib.chunks(binary, [:debug_info]) + + {:ok, map} = backend.debug_info(:elixir_v1, ModuleCreateDebugInfo, data, []) + assert map.module == ModuleCreateDebugInfo + end + + test "uses the debug_info chunk even if debug_info is set to false" do + {:module, ModuleCreateNoDebugInfo, binary, _} = + Module.create(ModuleCreateNoDebugInfo, quote(do: @compile({:debug_info, false})), __ENV__) + + {:ok, {_, [debug_info: {:debug_info_v1, backend, data}]}} = + :beam_lib.chunks(binary, [:debug_info]) + + assert backend.debug_info(:elixir_v1, ModuleCreateNoDebugInfo, data, []) == {:error, :missing} + end + + test "no function in module body" do in_module do assert __ENV__.function == nil end end + test "does not use ETS tables named after the module" do + in_module do + assert :ets.info(__MODULE__) == :undefined + end + end + ## Definitions - test :defines? do + test "defines?" do in_module do - refute Module.defines? __MODULE__, {:foo, 0} + refute Module.defines?(__MODULE__, {:foo, 0}) def foo(), do: bar() - assert Module.defines? __MODULE__, {:foo, 0} - assert Module.defines? __MODULE__, {:foo, 0}, :def + assert Module.defines?(__MODULE__, {:foo, 0}) + assert Module.defines?(__MODULE__, {:foo, 0}, :def) - refute Module.defines? __MODULE__, {:bar, 0}, :defp + refute Module.defines?(__MODULE__, {:bar, 0}, :defp) defp bar(), do: :ok - assert Module.defines? __MODULE__, {:bar, 0}, :defp + assert Module.defines?(__MODULE__, {:bar, 0}, :defp) - refute Module.defines? __MODULE__, {:baz, 0}, :defmacro + refute Module.defines?(__MODULE__, {:baz, 0}, :defmacro) defmacro baz(), do: :ok - assert Module.defines? __MODULE__, {:baz, 0}, :defmacro + assert Module.defines?(__MODULE__, {:baz, 0}, :defmacro) + end + end + + test "definitions in" do + in_module do + defp bar(), do: :ok + def foo(1, 2, 3), do: bar() + + defmacrop macro_bar(), do: 4 + defmacro macro_foo(1, 2, 3), do: macro_bar() + + assert Module.definitions_in(__MODULE__) |> Enum.sort() == + [{:bar, 0}, {:foo, 3}, {:macro_bar, 0}, {:macro_foo, 3}] + + assert Module.definitions_in(__MODULE__, :def) == [foo: 3] + assert Module.definitions_in(__MODULE__, :defp) == [bar: 0] + assert Module.definitions_in(__MODULE__, :defmacro) == [macro_foo: 3] + assert Module.definitions_in(__MODULE__, :defmacrop) == [macro_bar: 0] + + defoverridable foo: 3 + + assert Module.definitions_in(__MODULE__) |> Enum.sort() == + [{:bar, 0}, {:macro_bar, 0}, {:macro_foo, 3}] + + assert Module.definitions_in(__MODULE__, :def) == [] end end - test :definitions_in do + test "get_definition/2 and delete_definition/2" do in_module do - def foo(1, 2, 3), do: 4 + def foo(a, b), do: a + b + + assert {:v1, :def, _, + [ + {_, [{:a, _, nil}, {:b, _, nil}], [], + {{:., _, [:erlang, :+]}, _, [{:a, _, nil}, {:b, _, nil}]}} + ]} = Module.get_definition(__MODULE__, {:foo, 2}) + + assert {:v1, :def, _, nil} = + Module.get_definition(__MODULE__, {:foo, 2}, nillify_clauses: true) + + assert Module.delete_definition(__MODULE__, {:foo, 2}) + assert Module.get_definition(__MODULE__, {:foo, 2}) == nil + refute Module.delete_definition(__MODULE__, {:foo, 2}) + end + end - assert Module.definitions_in(__MODULE__) == [foo: 3] - assert Module.definitions_in(__MODULE__, :def) == [foo: 3] - assert Module.definitions_in(__MODULE__, :defp) == [] + test "make_overridable/2 with invalid arguments" do + contents = + quote do + Module.make_overridable(__MODULE__, [{:foo, 256}]) + end + + message = + "each element in tuple list has to be a {function_name :: atom, arity :: 0..255} " <> + "tuple, got: {:foo, 256}" + + assert_raise ArgumentError, message, fn -> + Module.create(MakeOverridable, contents, __ENV__) + end + after + purge(MakeOverridable) + end + + test "raise when called with already compiled module" do + message = + "could not call Module.get_attribute/2 because the module Enum is already compiled. " <> + "Use the Module.__info__/1 callback or Code.fetch_docs/1 instead" + + assert_raise ArgumentError, message, fn -> + Module.get_attribute(Enum, :moduledoc) end end - test :function do - assert Module.function(:erlang, :atom_to_list, 1).(:hello) == 'hello' - assert is_function Module.function(This, :also_works, 0) + describe "get_attribute/3" do + test "returns a list when the attribute is marked as `accummulate: true`" do + in_module do + Module.register_attribute(__MODULE__, :value, accumulate: true) + Module.put_attribute(__MODULE__, :value, 1) + assert Module.get_attribute(__MODULE__, :value) == [1] + Module.put_attribute(__MODULE__, :value, 2) + assert Module.get_attribute(__MODULE__, :value) == [2, 1] + end + end + + test "returns the value of the attribute if it exists" do + in_module do + Module.put_attribute(__MODULE__, :attribute, 1) + assert Module.get_attribute(__MODULE__, :attribute) == 1 + assert Module.get_attribute(__MODULE__, :attribute, :default) == 1 + end + end + + test "returns the passed default if the attribute does not exist" do + in_module do + assert Module.get_attribute(__MODULE__, :attribute, :default) == :default + end + end + end + + describe "has_attribute?/2 and attributes_in/2" do + test "returns true when attribute has been defined" do + in_module do + @foo 1 + Module.register_attribute(__MODULE__, :bar, []) + Module.register_attribute(__MODULE__, :baz, accumulate: true) + Module.put_attribute(__MODULE__, :qux, 2) + + # silence warning + _ = @foo + + assert Module.has_attribute?(__MODULE__, :foo) + assert :foo in Module.attributes_in(__MODULE__) + assert Module.has_attribute?(__MODULE__, :bar) + assert :bar in Module.attributes_in(__MODULE__) + assert Module.has_attribute?(__MODULE__, :baz) + assert :baz in Module.attributes_in(__MODULE__) + assert Module.has_attribute?(__MODULE__, :qux) + assert :qux in Module.attributes_in(__MODULE__) + end + end + + test "returns false when attribute has not been defined" do + in_module do + refute Module.has_attribute?(__MODULE__, :foo) + end + end + + test "returns false when attribute has been deleted" do + in_module do + @foo 1 + Module.delete_attribute(__MODULE__, :foo) + + refute Module.has_attribute?(__MODULE__, :foo) + end + end + end + + test "@on_load" do + Process.register(self(), :on_load_test_process) + + defmodule OnLoadTest do + @on_load :on_load + + defp on_load do + send(:on_load_test_process, :on_loaded) + :ok + end + end + + assert_received :on_loaded end end diff --git a/lib/elixir/test/elixir/node_test.exs b/lib/elixir/test/elixir/node_test.exs index ff612e4a8cb..7f7da9bf7c2 100644 --- a/lib/elixir/test/elixir/node_test.exs +++ b/lib/elixir/test/elixir/node_test.exs @@ -1,10 +1,13 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule NodeTest do use ExUnit.Case + doctest Node + + @tag :epmd test "start/3 and stop/0" do - assert Node.stop == {:error, :not_found} + assert Node.stop() == {:error, :not_found} assert {:ok, _} = Node.start(:hello, :shortnames, 15000) assert Node.stop() == :ok end diff --git a/lib/elixir/test/elixir/option_parser_test.exs b/lib/elixir/test/elixir/option_parser_test.exs index e4f648f0d97..7f38e426443 100644 --- a/lib/elixir/test/elixir/option_parser_test.exs +++ b/lib/elixir/test/elixir/option_parser_test.exs @@ -1,293 +1,487 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule OptionParserTest do use ExUnit.Case, async: true - test "parses boolean option" do - assert OptionParser.parse(["--docs"]) == {[docs: true], [], []} - end + doctest OptionParser - test "parses alias boolean option as the alias key" do - assert OptionParser.parse(["-d"], aliases: [d: :docs]) - == {[docs: true], [], []} + test "parses --key value option" do + assert OptionParser.parse(["--source", "form_docs/", "other"], switches: [source: :string]) == + {[source: "form_docs/"], ["other"], []} end - test "parses more than one boolean option" do - assert OptionParser.parse(["--docs", "--compile"]) - == {[docs: true, compile: true], [], []} + test "parses --key=value option" do + assert OptionParser.parse(["--source=form_docs/", "other"], switches: [source: :string]) == + {[source: "form_docs/"], ["other"], []} end - test "parses more than one boolean options as the alias" do - assert OptionParser.parse(["-d", "--compile"], aliases: [d: :docs]) - == {[docs: true, compile: true], [], []} + test "parses overrides options by default" do + assert OptionParser.parse( + ["--require", "foo", "--require", "bar", "baz"], + switches: [require: :string] + ) == {[require: "bar"], ["baz"], []} end - test "parses --key value option" do - assert OptionParser.parse(["--source", "form_docs/"]) - == {[source: "form_docs/"], [], []} - end + test "parses multi-word option" do + config = [switches: [hello_world: :boolean]] + assert OptionParser.next(["--hello-world"], config) == {:ok, :hello_world, true, []} + assert OptionParser.next(["--no-hello-world"], config) == {:ok, :hello_world, false, []} - test "parses --key=value option" do - assert OptionParser.parse(["--source=form_docs/", "other"]) - == {[source: "form_docs/"], ["other"], []} - end + assert OptionParser.next(["--no-hello-world"], strict: []) == + {:undefined, "--no-hello-world", nil, []} - test "parses alias --key value option as the alias" do - assert OptionParser.parse(["-s", "from_docs/"], aliases: [s: :source]) - == {[source: "from_docs/"], [], []} - end + assert OptionParser.next(["--no-hello_world"], strict: []) == + {:undefined, "--no-hello_world", nil, []} - test "parses alias --key=value option as the alias" do - assert OptionParser.parse(["-s=from_docs/", "other"], aliases: [s: :source]) - == {[source: "from_docs/"], ["other"], []} - end + config = [strict: [hello_world: :boolean]] + assert OptionParser.next(["--hello-world"], config) == {:ok, :hello_world, true, []} + assert OptionParser.next(["--no-hello-world"], config) == {:ok, :hello_world, false, []} + assert OptionParser.next(["--hello_world"], config) == {:undefined, "--hello_world", nil, []} - test "does not interpret undefined options with value as boolean" do - assert OptionParser.parse(["--no-bool"]) - == {[no_bool: true], [], []} - assert OptionParser.parse(["--no-bool"], strict: []) - == {[], [], [{"--no-bool", nil}]} - assert OptionParser.parse(["--no-bool=...", "other"]) - == {[], ["other"], [{"--no-bool", "..."}]} + assert OptionParser.next(["--no-hello_world"], config) == + {:undefined, "--no-hello_world", nil, []} end - test "does not parse -- as an alias" do - assert OptionParser.parse(["--s=from_docs/"], aliases: [s: :source]) - == {[s: "from_docs/"], [], []} - end + test "parses more than one key-value pair options using switches" do + opts = [switches: [source: :string, docs: :string]] - test "does not parse - as a switch" do - assert OptionParser.parse(["-source=from_docs/"], aliases: [s: :source]) - == {[], [], [{"-source", "from_docs/"}]} - end + assert OptionParser.parse(["--source", "from_docs/", "--docs", "show"], opts) == + {[source: "from_docs/", docs: "show"], [], []} - test "parses configured booleans" do - assert OptionParser.parse(["--docs=false"], switches: [docs: :boolean]) - == {[docs: false], [], []} - assert OptionParser.parse(["--docs=true"], switches: [docs: :boolean]) - == {[docs: true], [], []} - assert OptionParser.parse(["--docs=other"], switches: [docs: :boolean]) - == {[], [], [{"--docs", "other"}]} - assert OptionParser.parse(["--docs="], switches: [docs: :boolean]) - == {[], [], [{"--docs", ""}]} - - assert OptionParser.parse(["--docs", "foo"], switches: [docs: :boolean]) - == {[docs: true], ["foo"], []} - assert OptionParser.parse(["--no-docs", "foo"], switches: [docs: :boolean]) - == {[docs: false], ["foo"], []} - assert OptionParser.parse(["--no-docs=foo", "bar"], switches: [docs: :boolean]) - == {[], ["bar"], [{"--no-docs", "foo"}]} - assert OptionParser.parse(["--no-docs=", "bar"], switches: [docs: :boolean]) - == {[], ["bar"], [{"--no-docs", ""}]} - end + assert OptionParser.parse(["--source", "from_docs/", "--doc", "show"], opts) == + {[source: "from_docs/", doc: "show"], [], []} - test "does not set unparsed booleans" do - assert OptionParser.parse(["foo"], switches: [docs: :boolean]) - == {[], ["foo"], []} + assert OptionParser.parse(["--source", "from_docs/", "--doc=show"], opts) == + {[source: "from_docs/", doc: "show"], [], []} + + assert OptionParser.parse(["--no-bool"], strict: []) == {[], [], [{"--no-bool", nil}]} end - test "keeps options on configured keep" do - args = ["--require", "foo", "--require", "bar", "baz"] - assert OptionParser.parse(args, switches: [require: :keep]) - == {[require: "foo", require: "bar"], ["baz"], []} + test "parses more than one key-value pair options using strict" do + opts = [strict: [source: :string, docs: :string]] + + assert OptionParser.parse(["--source", "from_docs/", "--docs", "show"], opts) == + {[source: "from_docs/", docs: "show"], [], []} + + assert OptionParser.parse(["--source", "from_docs/", "--doc", "show"], opts) == + {[source: "from_docs/"], ["show"], [{"--doc", nil}]} - assert OptionParser.parse(["--require"], switches: [require: :keep]) - == {[], [], [{"--require", nil}]} + assert OptionParser.parse(["--source", "from_docs/", "--doc=show"], opts) == + {[source: "from_docs/"], [], [{"--doc", nil}]} + + assert OptionParser.parse(["--no-bool"], strict: []) == {[], [], [{"--no-bool", nil}]} end - test "parses configured strings" do - assert OptionParser.parse(["--value", "1", "foo"], switches: [value: :string]) - == {[value: "1"], ["foo"], []} - assert OptionParser.parse(["--value=1", "foo"], switches: [value: :string]) - == {[value: "1"], ["foo"], []} - assert OptionParser.parse(["--value"], switches: [value: :string]) - == {[], [], [{"--value", nil}]} - assert OptionParser.parse(["--no-value"], switches: [value: :string]) - == {[], [], [{"--no-value", nil}]} + test "collects multiple invalid options" do + argv = ["--bad", "opt", "foo", "-o", "bad", "bar"] + + assert OptionParser.parse(argv, switches: [bad: :integer]) == + {[], ["foo", "bar"], [{"--bad", "opt"}]} end - test "parses configured integers" do - assert OptionParser.parse(["--value", "1", "foo"], switches: [value: :integer]) - == {[value: 1], ["foo"], []} - assert OptionParser.parse(["--value=1", "foo"], switches: [value: :integer]) - == {[value: 1], ["foo"], []} - assert OptionParser.parse(["--value", "WAT", "foo"], switches: [value: :integer]) - == {[], ["foo"], [{"--value", "WAT"}]} + test "parse/2 raises when using both options: switches and strict" do + assert_raise ArgumentError, ":switches and :strict cannot be given together", fn -> + OptionParser.parse(["--elixir"], switches: [ex: :string], strict: [elixir: :string]) + end end - test "parses configured integers with keep" do - args = ["--value", "1", "--value", "2", "foo"] - assert OptionParser.parse(args, switches: [value: [:integer, :keep]]) - == {[value: 1, value: 2], ["foo"], []} + test "parse/2 raises an exception on invalid switch types/modifiers" do + assert_raise ArgumentError, "invalid switch types/modifiers: :bad", fn -> + OptionParser.parse(["--elixir"], switches: [ex: :bad]) + end - args = ["--value=1", "foo", "--value=2", "bar"] - assert OptionParser.parse(args, switches: [value: [:integer, :keep]]) - == {[value: 1, value: 2], ["foo", "bar"], []} - end + assert_raise ArgumentError, "invalid switch types/modifiers: :bad, :bad_modifier", fn -> + OptionParser.parse(["--elixir"], switches: [ex: [:bad, :bad_modifier]]) + end + end + + test "parse!/2 raises an exception for an unknown option using strict" do + msg = "1 error found!\n--doc-bar : Unknown option. Did you mean --docs-bar?" + + assert_raise OptionParser.ParseError, msg, fn -> + argv = ["--source", "from_docs/", "--doc-bar", "show"] + OptionParser.parse!(argv, strict: [source: :string, docs_bar: :string]) + end - test "parses configured floats" do - assert OptionParser.parse(["--value", "1.0", "foo"], switches: [value: :float]) - == {[value: 1.0], ["foo"], []} - assert OptionParser.parse(["--value=1.0", "foo"], switches: [value: :float]) - == {[value: 1.0], ["foo"], []} - assert OptionParser.parse(["--value", "WAT", "foo"], switches: [value: :float]) - == {[], ["foo"], [{"--value", "WAT"}]} + assert_raise OptionParser.ParseError, "1 error found!\n--foo : Unknown option", fn -> + argv = ["--source", "from_docs/", "--foo", "show"] + OptionParser.parse!(argv, strict: [source: :string, docs: :string]) + end end - test "parses no switches as flags" do - assert OptionParser.parse(["--no-docs", "foo"]) - == {[no_docs: true], ["foo"], []} + test "parse!/2 raises an exception when an option is of the wrong type" do + assert_raise OptionParser.ParseError, fn -> + argv = ["--bad", "opt", "foo", "-o", "bad", "bar"] + OptionParser.parse!(argv, switches: [bad: :integer]) + end end - test "overrides options by default" do - assert OptionParser.parse(["--require", "foo", "--require", "bar", "baz"]) - == {[require: "bar"], ["baz"], []} + test "parse_head!/2 raises an exception when an option is of the wrong type" do + message = "1 error found!\n--number : Expected type integer, got \"lib\"" + + assert_raise OptionParser.ParseError, message, fn -> + argv = ["--number", "lib", "test/enum_test.exs"] + OptionParser.parse_head!(argv, strict: [number: :integer]) + end end - test "parses mixed options" do - args = ["--source", "from_docs/", "--compile", "-x"] - assert OptionParser.parse(args, aliases: [x: :x]) - == {[source: "from_docs/", compile: true, x: true], [], []} + describe "arguments" do + test "parses until --" do + assert OptionParser.parse( + ["--source", "foo", "--", "1", "2", "3"], + switches: [source: :string] + ) == {[source: "foo"], ["1", "2", "3"], []} + + assert OptionParser.parse_head( + ["--source", "foo", "--", "1", "2", "3"], + switches: [source: :string] + ) == {[source: "foo"], ["1", "2", "3"], []} + + assert OptionParser.parse( + ["--source", "foo", "bar", "--", "-x"], + switches: [source: :string] + ) == {[source: "foo"], ["bar", "-x"], []} + + assert OptionParser.parse_head( + ["--source", "foo", "bar", "--", "-x"], + switches: [source: :string] + ) == {[source: "foo"], ["bar", "--", "-x"], []} + end + + test "parses - as argument" do + argv = ["--foo", "-", "-b", "-"] + opts = [strict: [foo: :boolean, boo: :string], aliases: [b: :boo]] + assert OptionParser.parse(argv, opts) == {[foo: true, boo: "-"], ["-"], []} + end + + test "parses until first non-option arguments" do + argv = ["--source", "from_docs/", "test/enum_test.exs", "--verbose"] + + assert OptionParser.parse_head(argv, switches: [source: :string]) == + {[source: "from_docs/"], ["test/enum_test.exs", "--verbose"], []} + end end - test "stops on first non option arguments" do - args = ["--source", "from_docs/", "test/enum_test.exs", "--verbose"] - assert OptionParser.parse_head(args) - == {[source: "from_docs/"], ["test/enum_test.exs", "--verbose"], []} + describe "aliases" do + test "supports boolean aliases" do + assert OptionParser.parse(["-d"], aliases: [d: :docs], switches: [docs: :boolean]) == + {[docs: true], [], []} + end + + test "supports non-boolean aliases" do + assert OptionParser.parse( + ["-s", "from_docs/"], + aliases: [s: :source], + switches: [source: :string] + ) == {[source: "from_docs/"], [], []} + end + + test "supports --key=value aliases" do + assert OptionParser.parse( + ["-s=from_docs/", "other"], + aliases: [s: :source], + switches: [source: :string] + ) == {[source: "from_docs/"], ["other"], []} + end + + test "parses -ab as -a -b" do + opts = [aliases: [a: :first, b: :second], switches: [second: :integer]] + assert OptionParser.parse(["-ab=1"], opts) == {[first: true, second: 1], [], []} + assert OptionParser.parse(["-ab", "1"], opts) == {[first: true, second: 1], [], []} + + opts = [aliases: [a: :first, b: :second], switches: [first: :boolean, second: :boolean]] + assert OptionParser.parse(["-ab"], opts) == {[first: true, second: true], [], []} + assert OptionParser.parse(["-ab3"], opts) == {[first: true], [], [{"-b", "3"}]} + assert OptionParser.parse(["-ab=bar"], opts) == {[first: true], [], [{"-b", "bar"}]} + assert OptionParser.parse(["-ab3=bar"], opts) == {[first: true], [], [{"-b", "3=bar"}]} + assert OptionParser.parse(["-3ab"], opts) == {[], ["-3ab"], []} + end end - test "stops on --" do - options = OptionParser.parse(["--source", "from_docs/", "--", "1", "2", "3"]) - assert options == {[source: "from_docs/"], ["1", "2", "3"], []} + describe "types" do + test "parses configured booleans" do + assert OptionParser.parse(["--docs=false"], switches: [docs: :boolean]) == + {[docs: false], [], []} + + assert OptionParser.parse(["--docs=true"], switches: [docs: :boolean]) == + {[docs: true], [], []} + + assert OptionParser.parse(["--docs=other"], switches: [docs: :boolean]) == + {[], [], [{"--docs", "other"}]} + + assert OptionParser.parse(["--docs="], switches: [docs: :boolean]) == + {[], [], [{"--docs", ""}]} + + assert OptionParser.parse(["--docs", "foo"], switches: [docs: :boolean]) == + {[docs: true], ["foo"], []} + + assert OptionParser.parse(["--no-docs", "foo"], switches: [docs: :boolean]) == + {[docs: false], ["foo"], []} + + assert OptionParser.parse(["--no-docs=foo", "bar"], switches: [docs: :boolean]) == + {[], ["bar"], [{"--no-docs", "foo"}]} + + assert OptionParser.parse(["--no-docs=", "bar"], switches: [docs: :boolean]) == + {[], ["bar"], [{"--no-docs", ""}]} + end + + test "does not set unparsed booleans" do + assert OptionParser.parse(["foo"], switches: [docs: :boolean]) == {[], ["foo"], []} + end + + test "keeps options on configured keep" do + argv = ["--require", "foo", "--require", "bar", "baz"] + + assert OptionParser.parse(argv, switches: [require: :keep]) == + {[require: "foo", require: "bar"], ["baz"], []} + + assert OptionParser.parse(["--require"], switches: [require: :keep]) == + {[], [], [{"--require", nil}]} + end + + test "parses configured strings" do + assert OptionParser.parse(["--value", "1", "foo"], switches: [value: :string]) == + {[value: "1"], ["foo"], []} + + assert OptionParser.parse(["--value=1", "foo"], switches: [value: :string]) == + {[value: "1"], ["foo"], []} - options = OptionParser.parse_head(["--source", "from_docs/", "--", "1", "2", "3"]) - assert options == {[source: "from_docs/"], ["1", "2", "3"], []} + assert OptionParser.parse(["--value"], switches: [value: :string]) == + {[], [], [{"--value", nil}]} - options = OptionParser.parse(["--no-dash", "foo", "bar", "--", "-x"]) - assert options == {[no_dash: true], ["foo", "bar", "-x"], []} + assert OptionParser.parse(["--no-value"], switches: [value: :string]) == + {[no_value: true], [], []} + end - options = OptionParser.parse_head(["--no-dash", "foo", "bar", "--", "-x"]) - assert options == {[no_dash: true], ["foo", "bar", "--", "-x"], []} + test "parses configured counters" do + assert OptionParser.parse(["--verbose"], switches: [verbose: :count]) == + {[verbose: 1], [], []} + + assert OptionParser.parse(["--verbose", "--verbose"], switches: [verbose: :count]) == + {[verbose: 2], [], []} + + argv = ["--verbose", "-v", "-v", "--", "bar"] + opts = [aliases: [v: :verbose], strict: [verbose: :count]] + assert OptionParser.parse(argv, opts) == {[verbose: 3], ["bar"], []} + end + + test "parses configured integers" do + assert OptionParser.parse(["--value", "1", "foo"], switches: [value: :integer]) == + {[value: 1], ["foo"], []} + + assert OptionParser.parse(["--value=1", "foo"], switches: [value: :integer]) == + {[value: 1], ["foo"], []} + + assert OptionParser.parse(["--value", "WAT", "foo"], switches: [value: :integer]) == + {[], ["foo"], [{"--value", "WAT"}]} + end + + test "parses configured integers with keep" do + argv = ["--value", "1", "--value", "2", "foo"] + + assert OptionParser.parse(argv, switches: [value: [:integer, :keep]]) == + {[value: 1, value: 2], ["foo"], []} + + argv = ["--value=1", "foo", "--value=2", "bar"] + + assert OptionParser.parse(argv, switches: [value: [:integer, :keep]]) == + {[value: 1, value: 2], ["foo", "bar"], []} + end + + test "parses configured floats" do + assert OptionParser.parse(["--value", "1.0", "foo"], switches: [value: :float]) == + {[value: 1.0], ["foo"], []} + + assert OptionParser.parse(["--value=1.0", "foo"], switches: [value: :float]) == + {[value: 1.0], ["foo"], []} + + assert OptionParser.parse(["--value", "WAT", "foo"], switches: [value: :float]) == + {[], ["foo"], [{"--value", "WAT"}]} + end + + test "correctly handles negative integers" do + opts = [switches: [option: :integer], aliases: [o: :option]] + assert OptionParser.parse(["arg1", "-o43"], opts) == {[option: 43], ["arg1"], []} + assert OptionParser.parse(["arg1", "-o", "-43"], opts) == {[option: -43], ["arg1"], []} + assert OptionParser.parse(["arg1", "--option=-43"], opts) == {[option: -43], ["arg1"], []} + + assert OptionParser.parse(["arg1", "--option", "-43"], opts) == + {[option: -43], ["arg1"], []} + end + + test "correctly handles negative floating-point numbers" do + opts = [switches: [option: :float], aliases: [o: :option]] + assert OptionParser.parse(["arg1", "-o43.2"], opts) == {[option: 43.2], ["arg1"], []} + assert OptionParser.parse(["arg1", "-o", "-43.2"], opts) == {[option: -43.2], ["arg1"], []} + + assert OptionParser.parse(["arg1", "--option=-43.2"], switches: [option: :float]) == + {[option: -43.2], ["arg1"], []} + + assert OptionParser.parse(["arg1", "--option", "-43.2"], opts) == + {[option: -43.2], ["arg1"], []} + end end - test "goes beyond the first non option arguments" do - args = ["--source", "from_docs/", "test/enum_test.exs", "--verbose"] - assert OptionParser.parse(args) - == {[source: "from_docs/", verbose: true], ["test/enum_test.exs"], []} + describe "next" do + test "with strict good options" do + config = [strict: [str: :string, int: :integer, bool: :boolean]] + assert OptionParser.next(["--str", "hello", "..."], config) == {:ok, :str, "hello", ["..."]} + assert OptionParser.next(["--int=13", "..."], config) == {:ok, :int, 13, ["..."]} + assert OptionParser.next(["--bool=false", "..."], config) == {:ok, :bool, false, ["..."]} + assert OptionParser.next(["--no-bool", "..."], config) == {:ok, :bool, false, ["..."]} + assert OptionParser.next(["--bool", "..."], config) == {:ok, :bool, true, ["..."]} + assert OptionParser.next(["..."], config) == {:error, ["..."]} + end + + test "with strict unknown options" do + config = [strict: [bool: :boolean]] + + assert OptionParser.next(["--str", "13", "..."], config) == + {:undefined, "--str", nil, ["13", "..."]} + + assert OptionParser.next(["--int=hello", "..."], config) == + {:undefined, "--int", "hello", ["..."]} + + assert OptionParser.next(["-no-bool=other", "..."], config) == + {:undefined, "-no-bool", "other", ["..."]} + end + + test "with strict bad type" do + config = [strict: [str: :string, int: :integer, bool: :boolean]] + assert OptionParser.next(["--str", "13", "..."], config) == {:ok, :str, "13", ["..."]} + + assert OptionParser.next(["--int=hello", "..."], config) == + {:invalid, "--int", "hello", ["..."]} + + assert OptionParser.next(["--int", "hello", "..."], config) == + {:invalid, "--int", "hello", ["..."]} + + assert OptionParser.next(["--bool=other", "..."], config) == + {:invalid, "--bool", "other", ["..."]} + end + + test "with strict missing value" do + config = [strict: [str: :string, int: :integer, bool: :boolean]] + assert OptionParser.next(["--str"], config) == {:invalid, "--str", nil, []} + assert OptionParser.next(["--int"], config) == {:invalid, "--int", nil, []} + assert OptionParser.next(["--bool=", "..."], config) == {:invalid, "--bool", "", ["..."]} + + assert OptionParser.next(["--no-bool=", "..."], config) == + {:invalid, "--no-bool", "", ["..."]} + end end - test "parses more than one key/value options" do - assert OptionParser.parse(["--source", "from_docs/", "--docs", "show"]) - == {[source: "from_docs/", docs: "show"], [], []} + test "split" do + assert OptionParser.split(~S[]) == [] + assert OptionParser.split(~S[foo]) == ["foo"] + assert OptionParser.split(~S[foo bar]) == ["foo", "bar"] + assert OptionParser.split(~S[ foo bar ]) == ["foo", "bar"] + assert OptionParser.split(~S[foo\ bar]) == ["foo bar"] + assert OptionParser.split(~S[foo" bar"]) == ["foo bar"] + assert OptionParser.split(~S[foo\" bar\"]) == ["foo\"", "bar\""] + assert OptionParser.split(~S[foo "\ bar\""]) == ["foo", "\\ bar\""] + assert OptionParser.split(~S[foo '\"bar"\'\ ']) == ["foo", "\\\"bar\"'\\ "] end - test "collects multiple invalid options" do - args = ["--bad", "opt", "foo", "-o", "bad", "bar"] - assert OptionParser.parse(args, switches: [bad: :integer]) - == {[], ["foo", "bar"], [{"--bad", "opt"}, {"-o", "bad"}]} + describe "to_argv" do + test "converts options back to switches" do + assert OptionParser.to_argv(foo_bar: "baz") == ["--foo-bar", "baz"] + + assert OptionParser.to_argv(bool: true, bool: false, discarded: nil) == + ["--bool", "--no-bool"] + end + + test "handles :count switch type" do + original = ["--counter", "--counter"] + {opts, [], []} = OptionParser.parse(original, switches: [counter: :count]) + assert original == OptionParser.to_argv(opts, switches: [counter: :count]) + end end +end - test "parses more than one key/value options using strict" do - assert OptionParser.parse(["--source", "from_docs/", "--docs", "show"], - strict: [source: :string, docs: :string]) - == {[source: "from_docs/", docs: "show"], [], []} +defmodule OptionsParserDeprecationsTest do + use ExUnit.Case, async: true - assert OptionParser.parse(["--source", "from_docs/", "--doc", "show"], - strict: [source: :string, docs: :string]) - == {[source: "from_docs/"], ["show"], [{"--doc", nil}]} + @warning ~r[not passing the :switches or :strict option to OptionParser is deprecated] - assert OptionParser.parse(["--source", "from_docs/", "--doc=show"], - strict: [source: :string, docs: :string]) - == {[source: "from_docs/"], [], [{"--doc", nil}]} + def assert_deprecated(fun) do + assert ExUnit.CaptureIO.capture_io(:stderr, fun) =~ @warning end - test "parses - as argument" do - assert OptionParser.parse(["-a", "-", "-", "-b", "-"], aliases: [b: :boo]) - == {[boo: "-"], ["-"], [{"-a", "-"}]} + test "parses boolean option" do + assert_deprecated(fn -> + assert OptionParser.parse(["--docs"]) == {[docs: true], [], []} + end) + end - assert OptionParser.parse(["--foo", "-", "-b", "-"], strict: [foo: :boolean, boo: :string], aliases: [b: :boo]) - == {[foo: true, boo: "-"], ["-"], []} + test "parses more than one boolean option" do + assert_deprecated(fn -> + assert OptionParser.parse(["--docs", "--compile"]) == {[docs: true, compile: true], [], []} + end) end - test "multi-word option" do - config = [switches: [hello_world: :boolean]] - assert OptionParser.next(["--hello-world"], config) - == {:ok, :hello_world, true, []} - assert OptionParser.next(["--no-hello-world"], config) - == {:ok, :hello_world, false, []} - - assert OptionParser.next(["--hello-world"], []) - == {:ok, :hello_world, true, []} - assert OptionParser.next(["--no-hello-world"], []) - == {:ok, :no_hello_world, true, []} - assert OptionParser.next(["--hello_world"], []) - == {:invalid, "--hello_world", nil, []} - assert OptionParser.next(["--no-hello_world"], []) - == {:invalid, "--no-hello_world", nil, []} - - assert OptionParser.next(["--no-hello-world"], strict: []) - == {:undefined, "--no-hello-world", nil, []} - assert OptionParser.next(["--no-hello_world"], strict: []) - == {:undefined, "--no-hello_world", nil, []} + test "parses more than one boolean options as the alias" do + assert_deprecated(fn -> + assert OptionParser.parse(["-d", "--compile"], aliases: [d: :docs]) == + {[docs: true, compile: true], [], []} + end) + end - config = [strict: [hello_world: :boolean]] - assert OptionParser.next(["--hello-world"], config) - == {:ok, :hello_world, true, []} - assert OptionParser.next(["--no-hello-world"], config) - == {:ok, :hello_world, false, []} - assert OptionParser.next(["--hello_world"], config) - == {:undefined, "--hello_world", nil, []} - assert OptionParser.next(["--no-hello_world"], config) - == {:undefined, "--no-hello_world", nil, []} + test "parses --key value option" do + assert_deprecated(fn -> + assert OptionParser.parse(["--source", "form_docs/"]) == {[source: "form_docs/"], [], []} + end) + end + + test "does not interpret undefined options with value as boolean" do + assert_deprecated(fn -> + assert OptionParser.parse(["--no-bool"]) == {[no_bool: true], [], []} + end) + + assert_deprecated(fn -> + assert OptionParser.parse(["--no-bool=...", "other"]) == {[no_bool: "..."], ["other"], []} + end) end - test "next strict: good options" do - config = [strict: [str: :string, int: :integer, bool: :boolean]] - assert OptionParser.next(["--str", "hello", "..."], config) - == {:ok, :str, "hello", ["..."]} - assert OptionParser.next(["--int=13", "..."], config) - == {:ok, :int, 13, ["..."]} - assert OptionParser.next(["--bool=false", "..."], config) - == {:ok, :bool, false, ["..."]} - assert OptionParser.next(["--no-bool", "..."], config) - == {:ok, :bool, false, ["..."]} - assert OptionParser.next(["--bool", "..."], config) - == {:ok, :bool, true, ["..."]} - assert OptionParser.next(["..."], config) - == {:error, ["..."]} + test "parses -ab as -a -b" do + assert_deprecated(fn -> + assert OptionParser.parse(["-ab"], aliases: [a: :first, b: :second]) == + {[first: true, second: true], [], []} + end) end - test "next strict: unknown options" do - config = [strict: [bool: :boolean]] - assert OptionParser.next(["--str", "13", "..."], config) - == {:undefined, "--str", nil, ["13", "..."]} - assert OptionParser.next(["--int=hello", "..."], config) - == {:undefined, "--int", "hello", ["..."]} - assert OptionParser.next(["-no-bool=other", "..."], config) - == {:undefined, "-no-bool", "other", ["..."]} + test "parses mixed options" do + argv = ["--source", "from_docs/", "--compile", "-x"] + + assert_deprecated(fn -> + assert OptionParser.parse(argv, aliases: [x: :x]) == + {[source: "from_docs/", compile: true, x: true], [], []} + end) end - test "next strict: bad type" do - config = [strict: [str: :string, int: :integer, bool: :boolean]] - assert OptionParser.next(["--str", "13", "..."], config) - == {:ok, :str, "13", ["..."]} - assert OptionParser.next(["--int=hello", "..."], config) - == {:invalid, "--int", "hello", ["..."]} - assert OptionParser.next(["--int", "hello", "..."], config) - == {:invalid, "--int", "hello", ["..."]} - assert OptionParser.next(["--bool=other", "..."], config) - == {:invalid, "--bool", "other", ["..."]} + test "parses more than one key-value pair options" do + assert_deprecated(fn -> + assert OptionParser.parse(["--source", "from_docs/", "--docs", "show"]) == + {[source: "from_docs/", docs: "show"], [], []} + end) end - test "next strict: missing value" do - config = [strict: [str: :string, int: :integer, bool: :boolean]] - assert OptionParser.next(["--str"], config) - == {:invalid, "--str", nil, []} - assert OptionParser.next(["--int"], config) - == {:invalid, "--int", nil, []} - assert OptionParser.next(["--bool=", "..."], config) - == {:invalid, "--bool", "", ["..."]} - assert OptionParser.next(["--no-bool=", "..."], config) - == {:undefined, "--no-bool", "", ["..."]} + test "multi-word option" do + assert_deprecated(fn -> + assert OptionParser.next(["--hello-world"], []) == {:ok, :hello_world, true, []} + end) + + assert_deprecated(fn -> + assert OptionParser.next(["--no-hello-world"], []) == {:ok, :no_hello_world, true, []} + end) + + assert_deprecated(fn -> + assert OptionParser.next(["--hello_world"], []) == {:undefined, "--hello_world", nil, []} + end) + + assert_deprecated(fn -> + assert OptionParser.next(["--no-hello_world"], []) == + {:undefined, "--no-hello_world", nil, []} + end) end end diff --git a/lib/elixir/test/elixir/partition_supervisor_test.exs b/lib/elixir/test/elixir/partition_supervisor_test.exs new file mode 100644 index 00000000000..f9ec0c0c156 --- /dev/null +++ b/lib/elixir/test/elixir/partition_supervisor_test.exs @@ -0,0 +1,202 @@ +Code.require_file("test_helper.exs", __DIR__) + +defmodule PartitionSupervisorTest do + use ExUnit.Case, async: true + + describe "child_spec" do + test "uses the atom name as id" do + assert Supervisor.child_spec({PartitionSupervisor, name: Foo}, []) == %{ + id: Foo, + start: {PartitionSupervisor, :start_link, [[name: Foo]]}, + type: :supervisor + } + end + + test "uses the via value as id" do + via = {:via, Foo, {:bar, :baz}} + + assert Supervisor.child_spec({PartitionSupervisor, name: via}, []) == %{ + id: {:bar, :baz}, + start: {PartitionSupervisor, :start_link, [[name: via]]}, + type: :supervisor + } + end + end + + describe "start_link/1" do + test "on success with atom name", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: DynamicSupervisor, + name: config.test + ) + + assert PartitionSupervisor.partitions(config.test) == System.schedulers_online() + + refs = + for _ <- 1..100 do + ref = make_ref() + + DynamicSupervisor.start_child( + {:via, PartitionSupervisor, {config.test, ref}}, + {Agent, fn -> ref end} + ) + + ref + end + + agents = + for {_, pid, _, _} <- PartitionSupervisor.which_children(config.test), + {_, pid, _, _} <- DynamicSupervisor.which_children(pid), + do: Agent.get(pid, & &1) + + assert Enum.sort(refs) == Enum.sort(agents) + end + + test "on success with via name", config do + {:ok, _} = Registry.start_link(keys: :unique, name: PartitionRegistry) + + name = {:via, Registry, {PartitionRegistry, config.test}} + + {:ok, _} = PartitionSupervisor.start_link(child_spec: {Agent, fn -> :hello end}, name: name) + + assert PartitionSupervisor.partitions(name) == System.schedulers_online() + + assert Agent.get({:via, PartitionSupervisor, {name, 0}}, & &1) == :hello + end + + test "with_arguments", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: {Agent, fn -> raise "unused" end}, + with_arguments: fn [_fun], partition -> [fn -> partition end] end, + partitions: 3, + name: config.test + ) + + assert PartitionSupervisor.partitions(config.test) == 3 + + assert Agent.get({:via, PartitionSupervisor, {config.test, 0}}, & &1) == 0 + assert Agent.get({:via, PartitionSupervisor, {config.test, 1}}, & &1) == 1 + assert Agent.get({:via, PartitionSupervisor, {config.test, 2}}, & &1) == 2 + + assert Agent.get({:via, PartitionSupervisor, {config.test, 3}}, & &1) == 0 + assert Agent.get({:via, PartitionSupervisor, {config.test, -1}}, & &1) == 1 + end + + test "raises without name" do + assert_raise ArgumentError, + "the :name option must be given to PartitionSupervisor", + fn -> PartitionSupervisor.start_link(child_spec: DynamicSupervisor) end + end + + test "raises without child_spec" do + assert_raise ArgumentError, + "the :child_spec option must be given to PartitionSupervisor", + fn -> PartitionSupervisor.start_link(name: Foo) end + end + + test "raises on bad partitions" do + assert_raise ArgumentError, + "the :partitions option must be a positive integer, got: 0", + fn -> + PartitionSupervisor.start_link( + name: Foo, + child_spec: DynamicSupervisor, + partitions: 0 + ) + end + end + + test "raises on bad with_arguments" do + assert_raise ArgumentError, + ~r"the :with_arguments option must be a function that receives two arguments", + fn -> + PartitionSupervisor.start_link( + name: Foo, + child_spec: DynamicSupervisor, + with_arguments: 123 + ) + end + end + end + + describe "stop/1" do + test "is synchronous", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: {Agent, fn -> %{} end}, + name: config.test + ) + + assert PartitionSupervisor.stop(config.test) == :ok + assert Process.whereis(config.test) == nil + end + end + + describe "partitions/1" do + test "raises noproc for unknown atom partition supervisor" do + assert {:noproc, _} = catch_exit(PartitionSupervisor.partitions(:unknown)) + end + + test "raises noproc for unknown via partition supervisor", config do + {:ok, _} = Registry.start_link(keys: :unique, name: config.test) + via = {:via, Registry, {config.test, :unknown}} + assert {:noproc, _} = catch_exit(PartitionSupervisor.partitions(via)) + end + end + + describe "which_children/1" do + test "returns all partitions", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: {Agent, fn -> %{} end}, + name: config.test + ) + + assert PartitionSupervisor.partitions(config.test) == System.schedulers_online() + + children = + config.test + |> PartitionSupervisor.which_children() + |> Enum.sort() + + for {child, partition} <- Enum.zip(children, 0..(System.schedulers_online() - 1)) do + via = {:via, PartitionSupervisor, {config.test, partition}} + assert child == {partition, GenServer.whereis(via), :worker, [Agent]} + end + end + end + + describe "count_children/1" do + test "with workers", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: {Agent, fn -> %{} end}, + name: config.test + ) + + partitions = System.schedulers_online() + + assert PartitionSupervisor.count_children(config.test) == + %{active: partitions, specs: partitions, supervisors: 0, workers: partitions} + end + + test "with supervisors", config do + {:ok, _} = + PartitionSupervisor.start_link( + child_spec: DynamicSupervisor, + name: config.test + ) + + partitions = System.schedulers_online() + + assert PartitionSupervisor.count_children(config.test) == + %{active: partitions, specs: partitions, supervisors: partitions, workers: 0} + end + + test "raises noproc for unknown partition supervisor" do + assert {:noproc, _} = catch_exit(PartitionSupervisor.count_children(:unknown)) + end + end +end diff --git a/lib/elixir/test/elixir/path_test.exs b/lib/elixir/test/elixir/path_test.exs index 4fa577c9fa8..6b43b9b5efd 100644 --- a/lib/elixir/test/elixir/path_test.exs +++ b/lib/elixir/test/elixir/path_test.exs @@ -1,91 +1,164 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule PathTest do use ExUnit.Case, async: true - import PathHelpers + doctest Path - if :file.native_name_encoding == :utf8 do - test :wildcard_with_utf8 do - File.mkdir_p(tmp_path("héllò")) - assert Path.wildcard(tmp_path("héllò")) == [tmp_path("héllò")] + if :file.native_name_encoding() == :utf8 do + @tag :tmp_dir + test "wildcard with UTF-8", config do + File.mkdir_p(Path.join(config.tmp_dir, "héllò")) + + assert Path.wildcard(Path.join(config.tmp_dir, "héllò")) == + [Path.join(config.tmp_dir, "héllò")] after - File.rm_rf tmp_path("héllò") + File.rm_rf(Path.join(config.tmp_dir, "héllò")) end end - test :wildcard do - hello = tmp_path("wildcard/.hello") - world = tmp_path("wildcard/.hello/world") + @tag :tmp_dir + test "wildcard/2", config do + hello = Path.join(config.tmp_dir, "wildcard/.hello") + world = Path.join(config.tmp_dir, "wildcard/.hello/world") File.mkdir_p(world) - assert Path.wildcard(tmp_path("wildcard/*/*")) == [] - assert Path.wildcard(tmp_path("wildcard/**/*")) == [] - assert Path.wildcard(tmp_path("wildcard/?hello/world")) == [] + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/*/*")) == [] + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/**/*")) == [] + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/?hello/world")) == [] + + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/*/*"), match_dot: true) == + [world] + + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/**/*"), match_dot: true) == + [hello, world] - assert Path.wildcard(tmp_path("wildcard/*/*"), match_dot: true) == [world] - assert Path.wildcard(tmp_path("wildcard/**/*"), match_dot: true) == [hello, world] - assert Path.wildcard(tmp_path("wildcard/?hello/world"), match_dot: true) == [world] + assert Path.wildcard(Path.join(config.tmp_dir, "wildcard/?hello/world"), match_dot: true) == + [world] after - File.rm_rf tmp_path("wildcard") + File.rm_rf(Path.join(config.tmp_dir, "wildcard")) + end + + test "wildcard/2 raises on null byte" do + assert_raise ArgumentError, ~r/null byte/, fn -> Path.wildcard("foo\0bar") end end - if is_win? do - test :relative do - assert Path.relative("C:/usr/local/bin") == "usr/local/bin" + describe "Windows" do + @describetag :windows + + test "absname/1" do + assert Path.absname("//host/path") == "//host/path" + assert Path.absname("\\\\host\\path") == "//host/path" + assert Path.absname("\\/host\\path") == "//host/path" + assert Path.absname("/\\host\\path") == "//host/path" + + assert Path.absname("c:/") == "c:/" + assert Path.absname("c:/host/path") == "c:/host/path" + + cwd = File.cwd!() + assert Path.absname(cwd |> String.split("/") |> hd()) == cwd + + <> = cwd + random = Enum.random(Enum.to_list(?c..?z) -- [letter]) + assert Path.absname(<>) == <> + end + + test "relative/1" do + assert Path.relative("C:/usr/local/bin") == "usr/local/bin" assert Path.relative("C:\\usr\\local\\bin") == "usr\\local\\bin" - assert Path.relative("C:usr\\local\\bin") == "usr\\local\\bin" + assert Path.relative("C:usr\\local\\bin") == "usr\\local\\bin" - assert Path.relative("/usr/local/bin") == "usr/local/bin" - assert Path.relative("usr/local/bin") == "usr/local/bin" + assert Path.relative("/usr/local/bin") == "usr/local/bin" + assert Path.relative("usr/local/bin") == "usr/local/bin" assert Path.relative("../usr/local/bin") == "../usr/local/bin" end - test :type do - assert Path.type("C:/usr/local/bin") == :absolute + test "relative_to/2" do + assert Path.relative_to("D:/usr/local/foo", "D:/usr/") == "local/foo" + assert Path.relative_to("D:/usr/local/foo", "d:/usr/") == "local/foo" + assert Path.relative_to("d:/usr/local/foo", "D:/usr/") == "local/foo" + assert Path.relative_to("D:/usr/local/foo", "d:/") == "usr/local/foo" + assert Path.relative_to("D:/usr/local/foo", "D:/") == "usr/local/foo" + assert Path.relative_to("D:/usr/local/foo", "d:") == "D:/usr/local/foo" + assert Path.relative_to("D:/usr/local/foo", "D:") == "D:/usr/local/foo" + end + + test "type/1" do + assert Path.type("C:/usr/local/bin") == :absolute assert Path.type('C:\\usr\\local\\bin') == :absolute - assert Path.type("C:usr\\local\\bin") == :volumerelative + assert Path.type("C:usr\\local\\bin") == :volumerelative - assert Path.type("/usr/local/bin") == :volumerelative - assert Path.type('usr/local/bin') == :relative + assert Path.type("/usr/local/bin") == :volumerelative + assert Path.type('usr/local/bin') == :relative assert Path.type("../usr/local/bin") == :relative + + assert Path.type("//host/path") == :absolute + assert Path.type("\\\\host\\path") == :absolute + assert Path.type("/\\host\\path") == :absolute + assert Path.type("\\/host\\path") == :absolute end - else - test :relative do - assert Path.relative("/usr/local/bin") == "usr/local/bin" - assert Path.relative("usr/local/bin") == "usr/local/bin" + + test "split/1" do + assert Path.split("C:\\foo\\bar") == ["c:/", "foo", "bar"] + assert Path.split("C:/foo/bar") == ["c:/", "foo", "bar"] + end + + test "safe_relative/1" do + assert Path.safe_relative("local/foo") == {:ok, "local/foo"} + assert Path.safe_relative("D:/usr/local/foo") == :error + assert Path.safe_relative("d:/usr/local/foo") == :error + assert Path.safe_relative("foo/../..") == :error + end + + test "safe_relative_to/2" do + assert Path.safe_relative_to("local/foo/bar", "local") == {:ok, "local/foo/bar"} + assert Path.safe_relative_to("foo/..", "local") == {:ok, ""} + assert Path.safe_relative_to("..", "local/foo") == :error + assert Path.safe_relative_to("d:/usr/local/foo", "D:/") == :error + assert Path.safe_relative_to("D:/usr/local/foo", "d:/") == :error + end + end + + describe "Unix" do + @describetag :unix + + test "relative/1" do + assert Path.relative("/usr/local/bin") == "usr/local/bin" + assert Path.relative("usr/local/bin") == "usr/local/bin" assert Path.relative("../usr/local/bin") == "../usr/local/bin" + assert Path.relative("/") == "." + assert Path.relative('/') == "." assert Path.relative(['/usr', ?/, "local/bin"]) == "usr/local/bin" end - test :type do - assert Path.type("/usr/local/bin") == :absolute - assert Path.type("usr/local/bin") == :relative + test "type/1" do + assert Path.type("/usr/local/bin") == :absolute + assert Path.type("usr/local/bin") == :relative assert Path.type("../usr/local/bin") == :relative - assert Path.type('/usr/local/bin') == :absolute - assert Path.type('usr/local/bin') == :relative + assert Path.type('/usr/local/bin') == :absolute + assert Path.type('usr/local/bin') == :relative assert Path.type('../usr/local/bin') == :relative - assert Path.type(['/usr/', 'local/bin']) == :absolute - assert Path.type(['usr/', 'local/bin']) == :relative + assert Path.type(['/usr/', 'local/bin']) == :absolute + assert Path.type(['usr/', 'local/bin']) == :relative assert Path.type(['../usr', '/local/bin']) == :relative end end - test :relative_to_cwd do - assert Path.relative_to_cwd(__ENV__.file) == - Path.relative_to(__ENV__.file, System.cwd!) + test "relative_to_cwd/1" do + assert Path.relative_to_cwd(__ENV__.file) == Path.relative_to(__ENV__.file, File.cwd!()) - assert Path.relative_to_cwd(to_char_list(__ENV__.file)) == - Path.relative_to(to_char_list(__ENV__.file), to_char_list(System.cwd!)) + assert Path.relative_to_cwd(to_charlist(__ENV__.file)) == + Path.relative_to(to_charlist(__ENV__.file), to_charlist(File.cwd!())) end - test :absname do - assert (Path.absname("/") |> strip_drive_letter_if_windows) == "/" - assert (Path.absname("/foo") |> strip_drive_letter_if_windows) == "/foo" - assert (Path.absname("/foo/bar") |> strip_drive_letter_if_windows) == "/foo/bar" - assert (Path.absname("/foo/bar/") |> strip_drive_letter_if_windows) == "/foo/bar" - assert (Path.absname("/foo/bar/../bar") |> strip_drive_letter_if_windows) == "/foo/bar/../bar" + test "absname/1,2" do + assert Path.absname("/") |> strip_drive_letter_if_windows == "/" + assert Path.absname("/foo") |> strip_drive_letter_if_windows == "/foo" + assert Path.absname("/./foo") |> strip_drive_letter_if_windows == "/foo" + assert Path.absname("/foo/bar") |> strip_drive_letter_if_windows == "/foo/bar" + assert Path.absname("/foo/bar/") |> strip_drive_letter_if_windows == "/foo/bar" + assert Path.absname("/foo/bar/../bar") |> strip_drive_letter_if_windows == "/foo/bar/../bar" assert Path.absname("bar", "/foo") == "/foo/bar" assert Path.absname("bar/", "/foo") == "/foo/bar" @@ -95,47 +168,59 @@ defmodule PathTest do assert Path.absname(["bar/", ?., ?., ["/bar"]], "/foo") == "/foo/bar/../bar" end - test :expand_path_with_user_home do - home = System.user_home! + test "expand/1,2 with user home" do + home = System.user_home!() |> Path.absname() assert home == Path.expand("~") assert home == Path.expand('~') - assert is_binary Path.expand("~/foo") - assert is_binary Path.expand('~/foo') + assert is_binary(Path.expand("~/foo")) + assert is_binary(Path.expand('~/foo')) assert Path.expand("~/file") == Path.join(home, "file") assert Path.expand("~/file", "whatever") == Path.join(home, "file") - assert Path.expand("file", Path.expand("~")) == Path.expand("~/file") + assert Path.expand("file", Path.expand("~")) == Path.join(home, "file") assert Path.expand("file", "~") == Path.join(home, "file") + assert Path.expand("~file") == Path.join(File.cwd!(), "~file") end - test :expand_path do - assert (Path.expand("/") |> strip_drive_letter_if_windows) == "/" - assert (Path.expand("/foo") |> strip_drive_letter_if_windows) == "/foo" - assert (Path.expand("/foo/bar") |> strip_drive_letter_if_windows) == "/foo/bar" - assert (Path.expand("/foo/bar/") |> strip_drive_letter_if_windows) == "/foo/bar" - assert (Path.expand("/foo/bar/.") |> strip_drive_letter_if_windows)== "/foo/bar" - assert (Path.expand("/foo/bar/../bar") |> strip_drive_letter_if_windows) == "/foo/bar" + test "expand/1,2" do + assert Path.expand("/") |> strip_drive_letter_if_windows == "/" + assert Path.expand("/foo/../..") |> strip_drive_letter_if_windows == "/" + assert Path.expand("/foo") |> strip_drive_letter_if_windows == "/foo" + assert Path.expand("/./foo") |> strip_drive_letter_if_windows == "/foo" + assert Path.expand("/../foo") |> strip_drive_letter_if_windows == "/foo" + assert Path.expand("/foo/bar") |> strip_drive_letter_if_windows == "/foo/bar" + assert Path.expand("/foo/bar/") |> strip_drive_letter_if_windows == "/foo/bar" + assert Path.expand("/foo/bar/.") |> strip_drive_letter_if_windows == "/foo/bar" + assert Path.expand("/foo/bar/../bar") |> strip_drive_letter_if_windows == "/foo/bar" - assert (Path.expand("bar", "/foo") |> strip_drive_letter_if_windows)== "/foo/bar" - assert (Path.expand("bar/", "/foo") |> strip_drive_letter_if_windows)== "/foo/bar" - assert (Path.expand("bar/.", "/foo") |> strip_drive_letter_if_windows)== "/foo/bar" - assert (Path.expand("bar/../bar", "/foo") |> strip_drive_letter_if_windows)== "/foo/bar" - assert (Path.expand("../bar/../bar", "/foo/../foo/../foo") |> strip_drive_letter_if_windows) == "/bar" + assert Path.expand("bar", "/foo") |> strip_drive_letter_if_windows == "/foo/bar" + assert Path.expand("bar/", "/foo") |> strip_drive_letter_if_windows == "/foo/bar" + assert Path.expand("bar/.", "/foo") |> strip_drive_letter_if_windows == "/foo/bar" + assert Path.expand("bar/../bar", "/foo") |> strip_drive_letter_if_windows == "/foo/bar" - assert (Path.expand(['..', ?/, "bar/../bar"], '/foo/../foo/../foo') |> - strip_drive_letter_if_windows) == "/bar" + drive_letter = + Path.expand("../bar/../bar", "/foo/../foo/../foo") |> strip_drive_letter_if_windows - assert Path.expand("bar/../bar", "foo") == Path.expand("foo/bar") + assert drive_letter == "/bar" + + drive_letter = + Path.expand(['..', ?/, "bar/../bar"], '/foo/../foo/../foo') |> strip_drive_letter_if_windows + + assert "/bar" == drive_letter + + assert Path.expand("/..") |> strip_drive_letter_if_windows == "/" - assert (Path.expand("/..") |> strip_drive_letter_if_windows) == "/" + assert Path.expand("bar/../bar", "foo") == Path.expand("foo/bar") end - test :relative_to do + test "relative_to/2" do assert Path.relative_to("/usr/local/foo", "/usr/local") == "foo" assert Path.relative_to("/usr/local/foo", "/") == "usr/local/foo" assert Path.relative_to("/usr/local/foo", "/etc") == "/usr/local/foo" - assert Path.relative_to("/usr/local/foo", "/usr/local/foo") == "/usr/local/foo" + assert Path.relative_to("/usr/local/foo", "/usr/local/foo") == "." + assert Path.relative_to("/usr/local/foo/", "/usr/local/foo") == "." + assert Path.relative_to("/usr/local/foo", "/usr/local/foo/") == "." assert Path.relative_to("usr/local/foo", "usr/local") == "foo" assert Path.relative_to("usr/local/foo", "etc") == "usr/local/foo" @@ -145,14 +230,30 @@ defmodule PathTest do assert Path.relative_to(["usr", ?/, 'local/foo'], 'usr/local') == "foo" end - test :rootname do + test "safe_relative/1" do + assert Path.safe_relative("foo/bar") == {:ok, "foo/bar"} + assert Path.safe_relative("foo/..") == {:ok, ""} + assert Path.safe_relative("./foo") == {:ok, "foo"} + + assert Path.safe_relative("/usr/local/foo") == :error + assert Path.safe_relative("foo/../..") == :error + end + + test "safe_relative_to/2" do + assert Path.safe_relative_to("/usr/local/foo", "/usr/local") == :error + assert Path.safe_relative_to("../../..", "foo/bar") == :error + assert Path.safe_relative_to("../../..", "foo/bar") == :error + assert Path.safe_relative_to("/usr/local/foo", "/") == :error + end + + test "rootname/2" do assert Path.rootname("~/foo/bar.ex", ".ex") == "~/foo/bar" assert Path.rootname("~/foo/bar.exs", ".ex") == "~/foo/bar.exs" assert Path.rootname("~/foo/bar.old.ex", ".ex") == "~/foo/bar.old" assert Path.rootname([?~, '/foo/bar', ".old.ex"], '.ex') == "~/foo/bar.old" end - test :extname do + test "extname/1" do assert Path.extname("foo.erl") == ".erl" assert Path.extname("~/foo/bar") == "" @@ -160,7 +261,7 @@ defmodule PathTest do assert Path.extname('~/foo/bar') == "" end - test :dirname do + test "dirname/1" do assert Path.dirname("/foo/bar.ex") == "/foo" assert Path.dirname("foo/bar.ex") == "foo" @@ -170,7 +271,7 @@ defmodule PathTest do assert Path.dirname([?~, "/foo", '/bar.ex']) == "~/foo" end - test :basename do + test "basename/1,2" do assert Path.basename("foo") == "foo" assert Path.basename("/foo/bar") == "bar" assert Path.basename("/") == "" @@ -182,35 +283,50 @@ defmodule PathTest do assert Path.basename([?~, "/for/bar", '.old.ex'], ".ex") == "bar.old" end - test :join do + test "join/1" do assert Path.join([""]) == "" assert Path.join(["foo"]) == "foo" assert Path.join(["/", "foo", "bar"]) == "/foo/bar" + assert Path.join(["/", "foo", "bar", "/"]) == "/foo/bar" assert Path.join(["~", "foo", "bar"]) == "~/foo/bar" assert Path.join(['/foo/', "/bar/"]) == "/foo/bar" + assert Path.join(["/", ""]) == "/" + assert Path.join(["/", "", "bar"]) == "/bar" + assert Path.join(['foo', [?b, "a", ?r]]) == "foo/bar" + assert Path.join([[?f, 'o', "o"]]) == "foo" end - test :join_two do + test "join/2" do assert Path.join("/foo", "bar") == "/foo/bar" assert Path.join("~", "foo") == "~/foo" - assert Path.join("", "bar") == "/bar" + assert Path.join("", "bar") == "bar" + assert Path.join("bar", "") == "bar" + assert Path.join("", "/bar") == "bar" + assert Path.join("/bar", "") == "/bar" + + assert Path.join("foo", "/bar") == "foo/bar" assert Path.join("/foo", "/bar") == "/foo/bar" - assert Path.join("/foo", "./bar") == "/foo/bar" + assert Path.join("/foo", "/bar") == "/foo/bar" + assert Path.join("/foo", "./bar") == "/foo/./bar" + + assert Path.join("/foo", "/") == "/foo" + assert Path.join("/foo", "/bar/zar/") == "/foo/bar/zar" - assert Path.join([?/, "foo"], "./bar") == "/foo/bar" + assert Path.join([?/, "foo"], "./bar") == "/foo/./bar" + assert Path.join(["/foo", "bar"], ["fiz", "buz"]) == "/foobar/fizbuz" end - test :split_with_binary do + test "split/1" do assert Path.split("") == [] assert Path.split("foo") == ["foo"] assert Path.split("/foo/bar") == ["/", "foo", "bar"] assert Path.split([?/, "foo/bar"]) == ["/", "foo", "bar"] end - if is_win? do - defp strip_drive_letter_if_windows([_d,?:|rest]), do: rest - defp strip_drive_letter_if_windows(<<_d,?:,rest::binary>>), do: rest + if PathHelpers.windows?() do + defp strip_drive_letter_if_windows([_d, ?: | rest]), do: rest + defp strip_drive_letter_if_windows(<<_d, ?:, rest::binary>>), do: rest else defp strip_drive_letter_if_windows(path), do: path end diff --git a/lib/elixir/test/elixir/port_test.exs b/lib/elixir/test/elixir/port_test.exs new file mode 100644 index 00000000000..2579682b1fe --- /dev/null +++ b/lib/elixir/test/elixir/port_test.exs @@ -0,0 +1,34 @@ +Code.require_file("test_helper.exs", __DIR__) + +defmodule PortTest do + use ExUnit.Case, async: true + + test "info/1,2 with registered name" do + {:ok, port} = :gen_udp.open(0) + + assert Port.info(port, :links) == {:links, [self()]} + assert Port.info(port, :registered_name) == {:registered_name, []} + + Process.register(port, __MODULE__) + + assert Port.info(port, :registered_name) == {:registered_name, __MODULE__} + + :ok = :gen_udp.close(port) + + assert Port.info(port, :registered_name) == nil + assert Port.info(port) == nil + end + + # In contrast with other inlined functions, + # it is important to test that monitor/1 is inlined, + # this way we gain the monitor receive optimisation. + test "monitor/1 is inlined" do + assert expand(quote(do: Port.monitor(port())), __ENV__) == + quote(do: :erlang.monitor(:port, port())) + end + + defp expand(expr, env) do + {expr, _, _} = :elixir_expand.expand(expr, :elixir_env.env_to_ex(env), env) + expr + end +end diff --git a/lib/elixir/test/elixir/process_test.exs b/lib/elixir/test/elixir/process_test.exs index be5b984f2f2..ef8ec8200bc 100644 --- a/lib/elixir/test/elixir/process_test.exs +++ b/lib/elixir/test/elixir/process_test.exs @@ -1,41 +1,124 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule ProcessTest do use ExUnit.Case, async: true + doctest Process + + test "dictionary" do + assert Process.put(:foo, :bar) == nil + assert Process.put(:foo, :baz) == :bar + + assert Enum.member?(Process.get_keys(), :foo) + refute Enum.member?(Process.get_keys(), :bar) + refute Enum.member?(Process.get_keys(), :baz) + assert Process.get_keys(:bar) == [] + assert Process.get_keys(:baz) == [:foo] + + assert Process.get(:foo) == :baz + assert Process.delete(:foo) == :baz + assert Process.get(:foo) == nil + end + test "group_leader/2 and group_leader/0" do - another = spawn_link(fn -> :timer.sleep(1000) end) - assert Process.group_leader(self, another) - assert Process.group_leader == another + another = spawn_link(fn -> Process.sleep(1000) end) + assert Process.group_leader(self(), another) + assert Process.group_leader() == another end - test "monitoring functions are inlined by the compiler" do + # In contrast with other inlined functions, + # it is important to test that monitor/1 is inlined, + # this way we gain the monitor receive optimisation. + test "monitor/1 is inlined" do assert expand(quote(do: Process.monitor(pid())), __ENV__) == - quote(do: :erlang.monitor(:process, pid())) + quote(do: :erlang.monitor(:process, pid())) + end + + test "sleep/1" do + assert Process.sleep(0) == :ok end test "info/2" do - pid = spawn fn -> end + pid = spawn(fn -> Process.sleep(1000) end) + assert Process.info(pid, :priority) == {:priority, :normal} + assert Process.info(pid, [:priority]) == [priority: :normal] + Process.exit(pid, :kill) assert Process.info(pid, :backtrace) == nil + assert Process.info(pid, [:backtrace, :status]) == nil end test "info/2 with registered name" do - pid = spawn fn -> end + pid = spawn(fn -> nil end) Process.exit(pid, :kill) - assert Process.info(pid, :registered_name) == - nil + assert Process.info(pid, :registered_name) == nil + assert Process.info(pid, [:registered_name]) == nil + + assert Process.info(self(), :registered_name) == {:registered_name, []} + assert Process.info(self(), [:registered_name]) == [registered_name: []] - assert Process.info(self, :registered_name) == - {:registered_name, []} + Process.register(self(), __MODULE__) + assert Process.info(self(), :registered_name) == {:registered_name, __MODULE__} + assert Process.info(self(), [:registered_name]) == [registered_name: __MODULE__] + end + + test "send_after/3 sends messages once expired" do + Process.send_after(self(), :hello, 10) + assert_receive :hello + end + + test "send_after/4 with absolute time sends message once expired" do + time = System.monotonic_time(:millisecond) + 10 + Process.send_after(self(), :hello, time, abs: true) + assert_receive :hello + end + + test "send_after/3 returns a timer reference that can be read or cancelled" do + timer = Process.send_after(self(), :hello, 100_000) + refute_received :hello + assert is_integer(Process.read_timer(timer)) + assert is_integer(Process.cancel_timer(timer)) + + timer = Process.send_after(self(), :hello, 0) + assert_receive :hello + assert Process.read_timer(timer) == false + assert Process.cancel_timer(timer) == false + + timer = Process.send_after(self(), :hello, 100_000) + assert Process.cancel_timer(timer, async: true) + assert_receive {:cancel_timer, ^timer, result} + assert is_integer(result) + end + + test "exit(pid, :normal) does not cause the target process to exit" do + Process.flag(:trap_exit, true) + + pid = + spawn_link(fn -> + receive do + :done -> nil + end + end) + + true = Process.exit(pid, :normal) + refute_receive {:EXIT, ^pid, :normal} + assert Process.alive?(pid) + + # now exit the process for real so it doesn't hang around + true = Process.exit(pid, :abnormal) + assert_receive {:EXIT, ^pid, :abnormal} + refute Process.alive?(pid) + end - Process.register(self, __MODULE__) - assert Process.info(self, :registered_name) == - {:registered_name, __MODULE__} + test "exit(self(), :normal) causes the calling process to exit" do + Process.flag(:trap_exit, true) + pid = spawn_link(fn -> Process.exit(self(), :normal) end) + assert_receive {:EXIT, ^pid, :normal} + refute Process.alive?(pid) end defp expand(expr, env) do - {expr, _env} = :elixir_exp.expand(expr, env) + {expr, _, _} = :elixir_expand.expand(expr, :elixir_env.env_to_ex(env), env) expr end end diff --git a/lib/elixir/test/elixir/protocol/consolidation_test.exs b/lib/elixir/test/elixir/protocol/consolidation_test.exs new file mode 100644 index 00000000000..6b3fad4d176 --- /dev/null +++ b/lib/elixir/test/elixir/protocol/consolidation_test.exs @@ -0,0 +1,177 @@ +Code.require_file("../test_helper.exs", __DIR__) + +path = Path.expand("../../ebin", __DIR__) +File.mkdir_p!(path) + +files = Path.wildcard(PathHelpers.fixture_path("consolidation/*")) +Kernel.ParallelCompiler.compile_to_path(files, path) + +defmodule Protocol.ConsolidationTest do + use ExUnit.Case, async: true + alias Protocol.ConsolidationTest.{Sample, WithAny} + + defimpl WithAny, for: Map do + def ok(map) do + {:ok, map} + end + end + + defimpl WithAny, for: Any do + def ok(any) do + {:ok, any} + end + end + + defmodule NoImplStruct do + defstruct a: 0, b: 0 + end + + defmodule ImplStruct do + @derive [WithAny] + defstruct a: 0, b: 0 + + defimpl Sample do + @compile {:no_warn_undefined, Unknown} + + def ok(struct) do + Unknown.undefined(struct) + end + end + end + + Code.append_path(path) + + # Any is ignored because there is no fallback + :code.purge(Sample) + :code.delete(Sample) + {:ok, binary} = Protocol.consolidate(Sample, [Any, ImplStruct]) + :code.load_binary(Sample, 'protocol_test.exs', binary) + + @sample_binary binary + + # Any should be moved to the end + :code.purge(WithAny) + :code.delete(WithAny) + {:ok, binary} = Protocol.consolidate(WithAny, [Any, ImplStruct, Map]) + :code.load_binary(WithAny, 'protocol_test.exs', binary) + + test "consolidated?/1" do + assert Protocol.consolidated?(WithAny) + refute Protocol.consolidated?(Enumerable) + end + + test "consolidation prevents new implementations" do + output = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + defimpl WithAny, for: Integer do + def ok(_any), do: :ok + end + end) + + assert output =~ ~r"the .+WithAny protocol has already been consolidated" + after + :code.purge(WithAny.Integer) + :code.delete(WithAny.Integer) + end + + test "consolidated implementations without any" do + assert is_nil(Sample.impl_for(:foo)) + assert is_nil(Sample.impl_for(fn x -> x end)) + assert is_nil(Sample.impl_for(1)) + assert is_nil(Sample.impl_for(1.1)) + assert is_nil(Sample.impl_for([])) + assert is_nil(Sample.impl_for([1, 2, 3])) + assert is_nil(Sample.impl_for({})) + assert is_nil(Sample.impl_for({1, 2, 3})) + assert is_nil(Sample.impl_for("foo")) + assert is_nil(Sample.impl_for(<<1>>)) + assert is_nil(Sample.impl_for(self())) + assert is_nil(Sample.impl_for(%{})) + assert is_nil(Sample.impl_for(hd(:erlang.ports()))) + assert is_nil(Sample.impl_for(make_ref())) + + assert Sample.impl_for(%ImplStruct{}) == Sample.Protocol.ConsolidationTest.ImplStruct + assert Sample.impl_for(%NoImplStruct{}) == nil + end + + test "consolidated implementations with any and tuple fallback" do + assert WithAny.impl_for(%NoImplStruct{}) == WithAny.Any + # Derived + assert WithAny.impl_for(%ImplStruct{}) == WithAny.Any + assert WithAny.impl_for(%{__struct__: "foo"}) == WithAny.Map + assert WithAny.impl_for(%{}) == WithAny.Map + assert WithAny.impl_for(self()) == WithAny.Any + end + + test "consolidation keeps docs" do + {:ok, {Sample, [{'Docs', docs_bin}]}} = :beam_lib.chunks(@sample_binary, ['Docs']) + {:docs_v1, _, _, _, _, _, docs} = :erlang.binary_to_term(docs_bin) + ok_doc = List.keyfind(docs, {:function, :ok, 1}, 0) + + assert {{:function, :ok, 1}, _, ["ok(term)"], %{"en" => "Ok"}, _} = ok_doc + end + + test "consolidation keeps chunks" do + deprecated = [{{:ok, 1}, "Reason"}] + assert deprecated == Sample.__info__(:deprecated) + + {:ok, {Sample, [{'ExCk', check_bin}]}} = :beam_lib.chunks(@sample_binary, ['ExCk']) + assert {:elixir_checker_v1, contents} = :erlang.binary_to_term(check_bin) + export_info = %{deprecated_reason: "Reason", kind: :def} + assert {{:ok, 1}, export_info} in contents.exports + end + + test "consolidation keeps source" do + assert Sample.__info__(:compile)[:source] + end + + test "consolidated keeps callbacks" do + {:ok, callbacks} = Code.Typespec.fetch_callbacks(@sample_binary) + assert callbacks != [] + end + + test "consolidation errors on missing BEAM files" do + defprotocol NoBeam do + def example(arg) + end + + assert Protocol.consolidate(String, []) == {:error, :not_a_protocol} + assert Protocol.consolidate(NoBeam, []) == {:error, :no_beam_info} + end + + test "consolidation updates attributes" do + assert Sample.__protocol__(:consolidated?) + assert Sample.__protocol__(:impls) == {:consolidated, [ImplStruct]} + assert WithAny.__protocol__(:consolidated?) + assert WithAny.__protocol__(:impls) == {:consolidated, [Any, Map, ImplStruct]} + end + + test "consolidation extracts protocols" do + protos = Protocol.extract_protocols([:code.lib_dir(:elixir, :ebin)]) + assert Enumerable in protos + assert Inspect in protos + end + + test "consolidation extracts implementations with charlist path" do + protos = Protocol.extract_impls(Enumerable, [:code.lib_dir(:elixir, :ebin)]) + assert List in protos + assert Function in protos + end + + test "consolidation extracts implementations with binary path" do + protos = Protocol.extract_impls(Enumerable, [Application.app_dir(:elixir, "ebin")]) + assert List in protos + assert Function in protos + end + + test "protocol not implemented" do + message = + "protocol Protocol.ConsolidationTest.Sample not implemented for :foo of type Atom. " <> + "This protocol is implemented for the following type(s): Protocol.ConsolidationTest.ImplStruct" + + assert_raise Protocol.UndefinedError, message, fn -> + sample = Sample + sample.ok(:foo) + end + end +end diff --git a/lib/elixir/test/elixir/protocol_test.exs b/lib/elixir/test/elixir/protocol_test.exs index 4392b189946..fe8b8f57737 100644 --- a/lib/elixir/test/elixir/protocol_test.exs +++ b/lib/elixir/test/elixir/protocol_test.exs @@ -1,26 +1,35 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule ProtocolTest do use ExUnit.Case, async: true - defprotocol Sample do - @type t :: any - @doc "Ok" - @spec ok(t) :: boolean - def ok(thing) - end + doctest Protocol - defprotocol WithAny do - @fallback_to_any true - @doc "Ok" - def ok(thing) - end + {_, _, sample_binary, _} = + defprotocol Sample do + @type t :: any + @doc "Ok" + @deprecated "Reason" + @spec ok(t) :: boolean + def ok(term) + end + + @sample_binary sample_binary + + {_, _, with_any_binary, _} = + defprotocol WithAny do + @fallback_to_any true + @doc "Ok" + def ok(term) + end + + @with_any_binary with_any_binary defprotocol Derivable do def ok(a) end - defimpl Derivable, for: Map do + defimpl Derivable, for: Any do defmacro __deriving__(module, struct, options) do quote do defimpl Derivable, for: unquote(module) do @@ -57,6 +66,8 @@ defmodule ProtocolTest do defstruct a: 0, b: 0 defimpl Sample do + @compile {:no_warn_undefined, Unknown} + def ok(struct) do Unknown.undefined(struct) end @@ -64,56 +75,67 @@ defmodule ProtocolTest do end test "protocol implementations without any" do - assert nil? Sample.impl_for(:foo) - assert nil? Sample.impl_for(fn(x) -> x end) - assert nil? Sample.impl_for(1) - assert nil? Sample.impl_for(1.1) - assert nil? Sample.impl_for([]) - assert nil? Sample.impl_for([1, 2, 3]) - assert nil? Sample.impl_for({}) - assert nil? Sample.impl_for({1, 2, 3}) - assert nil? Sample.impl_for("foo") - assert nil? Sample.impl_for(<<1>>) - assert nil? Sample.impl_for(%{}) - assert nil? Sample.impl_for(self) - assert nil? Sample.impl_for(hd(:erlang.ports)) - assert nil? Sample.impl_for(make_ref) - - assert Sample.impl_for(%ImplStruct{}) == - Sample.ProtocolTest.ImplStruct - assert Sample.impl_for(%NoImplStruct{}) == - nil - end - - test "protocol implementation with any and structs fallback" do - assert WithAny.impl_for(%NoImplStruct{}) == WithAny.Any - assert WithAny.impl_for(%ImplStruct{}) == WithAny.Map # Derived + assert is_nil(Sample.impl_for(:foo)) + assert is_nil(Sample.impl_for(fn x -> x end)) + assert is_nil(Sample.impl_for(1)) + assert is_nil(Sample.impl_for(1.1)) + assert is_nil(Sample.impl_for([])) + assert is_nil(Sample.impl_for([1, 2, 3])) + assert is_nil(Sample.impl_for({})) + assert is_nil(Sample.impl_for({1, 2, 3})) + assert is_nil(Sample.impl_for("foo")) + assert is_nil(Sample.impl_for(<<1>>)) + assert is_nil(Sample.impl_for(%{})) + assert is_nil(Sample.impl_for(self())) + assert is_nil(Sample.impl_for(hd(:erlang.ports()))) + assert is_nil(Sample.impl_for(make_ref())) + + assert Sample.impl_for(%ImplStruct{}) == Sample.ProtocolTest.ImplStruct + assert Sample.impl_for(%NoImplStruct{}) == nil + end + + test "protocol implementation with Any and struct fallbacks" do + assert WithAny.impl_for(%NoImplStruct{}) == WithAny.Any + # Derived + assert WithAny.impl_for(%ImplStruct{}) == WithAny.Any assert WithAny.impl_for(%{__struct__: "foo"}) == WithAny.Map - assert WithAny.impl_for(%{}) == WithAny.Map - assert WithAny.impl_for(self) == WithAny.Any + assert WithAny.impl_for(%{}) == WithAny.Map + assert WithAny.impl_for(self()) == WithAny.Any end test "protocol not implemented" do - assert_raise Protocol.UndefinedError, "protocol ProtocolTest.Sample not implemented for :foo", fn -> - Sample.ok(:foo) + message = "protocol ProtocolTest.Sample not implemented for :foo of type Atom" + + assert_raise Protocol.UndefinedError, message, fn -> + sample = Sample + sample.ok(:foo) end end - test "protocol documentation" do + test "protocol documentation and deprecated" do import PathHelpers - write_beam(defprotocol SampleDocsProto do - @type t :: any - @doc "Ok" - @spec ok(t) :: boolean - def ok(thing) - end) + write_beam( + defprotocol SampleDocsProto do + @type t :: any + @doc "Ok" + @deprecated "Reason" + @spec ok(t) :: boolean + def ok(term) + end + ) - docs = Code.get_docs(SampleDocsProto, :docs) - assert {{:ok, 1}, _, :def, [{:thing, _, nil}], "Ok"} = - List.keyfind(docs, {:ok, 1}, 0) + {:docs_v1, _, _, _, _, _, docs} = Code.fetch_docs(SampleDocsProto) + + assert {{:function, :ok, 1}, _, ["ok(term)"], %{"en" => "Ok"}, _} = + List.keyfind(docs, {:function, :ok, 1}, 0) + + deprecated = SampleDocsProto.__info__(:deprecated) + assert [{{:ok, 1}, "Reason"}] = deprecated end + @compile {:no_warn_undefined, WithAll} + test "protocol keeps underlying UndefinedFunctionError" do assert_raise UndefinedFunctionError, fn -> WithAll.ok(%ImplStruct{}) @@ -121,34 +143,49 @@ defmodule ProtocolTest do end test "protocol defines callbacks" do - assert get_callbacks(Sample, :ok, 1) == - [{:type, 9, :fun, [{:type, 9, :product, [{:type, 9, :t, []}]}, {:type, 9, :boolean, []}]}] + assert [{:type, 13, :fun, args}] = get_callbacks(@sample_binary, :ok, 1) + assert args == [{:type, 13, :product, [{:user_type, 13, :t, []}]}, {:type, 13, :boolean, []}] - assert get_callbacks(WithAny, :ok, 1) == - [{:type, 16, :fun, [{:type, 16, :product, [{:type, 16, :t, []}]}, {:type, 16, :term, []}]}] + assert [{:type, 23, :fun, args}] = get_callbacks(@with_any_binary, :ok, 1) + assert args == [{:type, 23, :product, [{:user_type, 23, :t, []}]}, {:type, 23, :term, []}] end - test "protocol defines attributes" do - assert Sample.__info__(:attributes)[:protocol] == [fallback_to_any: false, consolidated: false] - assert WithAny.__info__(:attributes)[:protocol] == [fallback_to_any: true, consolidated: false] + test "protocol defines functions and attributes" do + assert Sample.__protocol__(:module) == Sample + assert Sample.__protocol__(:functions) == [ok: 1] + refute Sample.__protocol__(:consolidated?) + assert Sample.__protocol__(:impls) == :not_consolidated + assert Sample.__info__(:attributes)[:__protocol__] == [fallback_to_any: false] + + assert WithAny.__protocol__(:module) == WithAny + assert WithAny.__protocol__(:functions) == [ok: 1] + refute WithAny.__protocol__(:consolidated?) + assert WithAny.__protocol__(:impls) == :not_consolidated + assert WithAny.__info__(:attributes)[:__protocol__] == [fallback_to_any: true] end test "defimpl" do - defprotocol Attribute do - def test(thing) - end + module = Module.concat(Sample, ImplStruct) + assert module.__impl__(:for) == ImplStruct + assert module.__impl__(:target) == module + assert module.__impl__(:protocol) == Sample + assert module.__info__(:attributes)[:__impl__] == [protocol: Sample, for: ImplStruct] + end - defimpl Attribute, for: ImplStruct do - def test(_) do - {@protocol, @for} - end - end + test "defimpl with implicit derive" do + module = Module.concat(WithAny, ImplStruct) + assert module.__impl__(:for) == ImplStruct + assert module.__impl__(:target) == WithAny.Any + assert module.__impl__(:protocol) == WithAny + assert module.__info__(:attributes)[:__impl__] == [protocol: WithAny, for: ImplStruct] + end - assert Attribute.test(%ImplStruct{}) == {Attribute, ImplStruct} - assert Attribute.ProtocolTest.ImplStruct.__impl__(:protocol) == Attribute - assert Attribute.ProtocolTest.ImplStruct.__impl__(:for) == ImplStruct - assert Attribute.ProtocolTest.ImplStruct.__info__(:attributes)[:impl] == - [protocol: Attribute, for: ImplStruct] + test "defimpl with explicit derive" do + module = Module.concat(Derivable, ImplStruct) + assert module.__impl__(:for) == ImplStruct + assert module.__impl__(:target) == module + assert module.__impl__(:protocol) == Derivable + assert module.__info__(:attributes)[:__impl__] == [protocol: Derivable, for: ImplStruct] end test "defimpl with multiple for" do @@ -164,25 +201,35 @@ defmodule ProtocolTest do assert Multi.test(:a) == :a end - defp get_callbacks(module, name, arity) do - callbacks = for {:callback, info} <- module.__info__(:attributes), do: hd(info) + test "defimpl without :for option when outside a module" do + msg = "defimpl/3 expects a :for option when declared outside a module" + + assert_raise ArgumentError, msg, fn -> + ast = + quote do + defimpl Sample do + def ok(_term), do: true + end + end + + Code.eval_quoted(ast, [], %{__ENV__ | module: nil}) + end + end + + defp get_callbacks(beam, name, arity) do + {:ok, callbacks} = Code.Typespec.fetch_callbacks(beam) List.keyfind(callbacks, {name, arity}, 0) |> elem(1) end - test "derives protocol" do + test "derives protocol implicitly" do struct = %ImplStruct{a: 1, b: 1} assert WithAny.ok(struct) == {:ok, struct} - end - test "derived protocol keeps local file/line info" do - assert ProtocolTest.WithAny.ProtocolTest.ImplStruct.__info__(:compile)[:source] == - String.to_char_list(__ENV__.file) + struct = %NoImplStruct{a: 1, b: 1} + assert WithAny.ok(struct) == {:ok, struct} end - test "custom derive implementation" do - struct = %ImplStruct{a: 1, b: 1} - assert Derivable.ok(struct) == {:ok, struct, %ImplStruct{}, []} - + test "derives protocol explicitly" do struct = %ImplStruct{a: 1, b: 1} assert Derivable.ok(struct) == {:ok, struct, %ImplStruct{}, []} @@ -192,19 +239,18 @@ defmodule ProtocolTest do end end - test "custom derive implementation with options" do + test "derives protocol explicitly with options" do defmodule AnotherStruct do @derive [{Derivable, :ok}] @derive [WithAny] defstruct a: 0, b: 0 end - struct = struct AnotherStruct, a: 1, b: 1 - assert Derivable.ok(struct) == - {:ok, struct, struct(AnotherStruct), :ok} + struct = struct(AnotherStruct, a: 1, b: 1) + assert Derivable.ok(struct) == {:ok, struct, struct(AnotherStruct), :ok} end - test "custom derive implementation via API" do + test "derive protocol explicitly via API" do defmodule InlineStruct do defstruct a: 0, b: 0 end @@ -212,156 +258,131 @@ defmodule ProtocolTest do require Protocol assert Protocol.derive(Derivable, InlineStruct, :oops) == :ok - struct = struct InlineStruct, a: 1, b: 1 - assert Derivable.ok(struct) == - {:ok, struct, struct(InlineStruct), :oops} + struct = struct(InlineStruct, a: 1, b: 1) + assert Derivable.ok(struct) == {:ok, struct, struct(InlineStruct), :oops} end - test "cannot derive without a map implementation" do - assert_raise ArgumentError, - ~r"#{inspect Sample.Map} is not available, cannot derive #{inspect Sample}", fn -> - defmodule NotCompiled do - @derive [Sample] - defstruct hello: :world - end - end + test "derived implementation keeps local file/line info" do + assert ProtocolTest.WithAny.ProtocolTest.ImplStruct.__info__(:compile)[:source] == + String.to_charlist(__ENV__.file) end -end -path = Path.expand("../ebin", __DIR__) -File.mkdir_p!(path) + describe "warnings" do + import ExUnit.CaptureIO -compile = fn {:module, module, binary, _} -> - File.write!("#{path}/#{module}.beam", binary) -end + test "with no definitions" do + assert capture_io(:stderr, fn -> + defprotocol SampleWithNoDefinitions do + end + end) =~ "protocols must define at least one function, but none was defined" + end -defmodule Protocol.ConsolidationTest do - use ExUnit.Case, async: true + test "when @callbacks and friends are defined inside a protocol" do + message = + capture_io(:stderr, fn -> + defprotocol SampleWithCallbacks do + @spec with_specs(any(), keyword()) :: tuple() + def with_specs(term, options \\ []) - compile.( - defprotocol Sample do - @type t :: any - @doc "Ok" - @spec ok(t) :: boolean - def ok(thing) - end - ) + @spec with_specs_and_when(any(), opts) :: tuple() when opts: keyword + def with_specs_and_when(term, options \\ []) - compile.( - defprotocol WithAny do - @fallback_to_any true - @doc "Ok" - def ok(thing) - end - ) + def without_specs(term, options \\ []) - defimpl WithAny, for: Map do - def ok(map) do - {:ok, map} - end - end + @callback foo :: {:ok, term} + @callback foo(term) :: {:ok, term} + @callback foo(term, keyword) :: {:ok, term, keyword} - defimpl WithAny, for: Any do - def ok(any) do - {:ok, any} - end - end + @callback foo_when :: {:ok, x} when x: term + @callback foo_when(x) :: {:ok, x} when x: term + @callback foo_when(x, opts) :: {:ok, x, opts} when x: term, opts: keyword - defmodule NoImplStruct do - defstruct a: 0, b: 0 - end + @macrocallback bar(term) :: {:ok, term} + @macrocallback bar(term, keyword) :: {:ok, term, keyword} - defmodule ImplStruct do - @derive [WithAny] - defstruct a: 0, b: 0 + @optional_callbacks [foo: 1, foo: 2] + @optional_callbacks [without_specs: 2] + end + end) - defimpl Sample do - def ok(struct) do - Unknown.undefined(struct) - end - end - end + assert message =~ + "cannot define @callback foo/0 inside protocol, use def/1 to outline your protocol definition" - Code.append_path(path) + assert message =~ + "cannot define @callback foo/1 inside protocol, use def/1 to outline your protocol definition" - # Any is ignored because there is no fallback - :code.purge(Sample) - :code.delete(Sample) - {:ok, binary} = Protocol.consolidate(Sample, [Any, ImplStruct]) - :code.load_binary(Sample, 'protocol_test.exs', binary) + assert message =~ + "cannot define @callback foo/2 inside protocol, use def/1 to outline your protocol definition" - # Any should be moved to the end - :code.purge(WithAny) - :code.delete(WithAny) - {:ok, binary} = Protocol.consolidate(WithAny, [Any, ImplStruct, Map]) - :code.load_binary(WithAny, 'protocol_test.exs', binary) + assert message =~ + "cannot define @callback foo_when/0 inside protocol, use def/1 to outline your protocol definition" - test "consolidated?/1" do - assert Protocol.consolidated?(WithAny) - refute Protocol.consolidated?(Enumerable) - end + assert message =~ + "cannot define @callback foo_when/1 inside protocol, use def/1 to outline your protocol definition" - test "consolidated implementations without any" do - assert nil? Sample.impl_for(:foo) - assert nil? Sample.impl_for(fn(x) -> x end) - assert nil? Sample.impl_for(1) - assert nil? Sample.impl_for(1.1) - assert nil? Sample.impl_for([]) - assert nil? Sample.impl_for([1, 2, 3]) - assert nil? Sample.impl_for({}) - assert nil? Sample.impl_for({1, 2, 3}) - assert nil? Sample.impl_for("foo") - assert nil? Sample.impl_for(<<1>>) - assert nil? Sample.impl_for(self) - assert nil? Sample.impl_for(%{}) - assert nil? Sample.impl_for(hd(:erlang.ports)) - assert nil? Sample.impl_for(make_ref) - - assert Sample.impl_for(%ImplStruct{}) == - Sample.Protocol.ConsolidationTest.ImplStruct - assert Sample.impl_for(%NoImplStruct{}) == - nil - end + assert message =~ + "cannot define @callback foo_when/2 inside protocol, use def/1 to outline your protocol definition" - test "consolidated implementations with any and tuple fallback" do - assert WithAny.impl_for(%NoImplStruct{}) == WithAny.Any - assert WithAny.impl_for(%ImplStruct{}) == WithAny.Map # Derived - assert WithAny.impl_for(%{__struct__: "foo"}) == WithAny.Map - assert WithAny.impl_for(%{}) == WithAny.Map - assert WithAny.impl_for(self) == WithAny.Any - end + assert message =~ + "cannot define @macrocallback bar/1 inside protocol, use def/1 to outline your protocol definition" - test "consolidation keeps docs" do - docs = Code.get_docs(Sample, :docs) - assert {{:ok, 1}, _, :def, [{:thing, _, nil}], "Ok"} = - List.keyfind(docs, {:ok, 1}, 0) - end + assert message =~ + "cannot define @macrocallback bar/2 inside protocol, use def/1 to outline your protocol definition" - test "consolidated keeps callbacks" do - callbacks = for {:callback, info} <- Sample.__info__(:attributes), do: hd(info) - assert callbacks != [] - end + assert message =~ + "cannot define @optional_callbacks inside protocol, all of the protocol definitions are required" + end - test "consolidation errors on missing beams" do - defprotocol NoBeam, do: nil - assert Protocol.consolidate(String, []) == {:error, :not_a_protocol} - assert Protocol.consolidate(NoBeam, []) == {:error, :no_beam_info} - end + test "when deriving after struct" do + assert capture_io(:stderr, fn -> + defmodule DeriveTooLate do + defstruct [] + @derive [{Derivable, :ok}] + end + end) =~ + "module attribute @derive was set after defstruct, all @derive calls must come before defstruct" + end - test "consolidation updates attributes" do - assert Sample.__info__(:attributes)[:protocol] == [fallback_to_any: false, consolidated: true] - assert WithAny.__info__(:attributes)[:protocol] == [fallback_to_any: true, consolidated: true] + test "when deriving with no struct" do + assert capture_io(:stderr, fn -> + defmodule DeriveNeverUsed do + @derive [{Derivable, :ok}] + end + end) =~ + "module attribute @derive was set but never used (it must come before defstruct)" + end end - test "consolidation extracts protocols" do - protos = Protocol.extract_protocols([:code.lib_dir(:elixir, :ebin)]) - assert Enumerable in protos - assert Inspect in protos + describe "errors" do + test "cannot derive without any implementation" do + assert_raise ArgumentError, + ~r"could not load module #{inspect(Sample.Any)} due to reason :nofile, cannot derive #{inspect(Sample)}", + fn -> + defmodule NotCompiled do + @derive [Sample] + defstruct hello: :world + end + end + end end +end + +defmodule Protocol.DebugInfoTest do + use ExUnit.Case + + test "protocols always keep debug_info" do + Code.compiler_options(debug_info: false) + + {:module, _, binary, _} = + defprotocol DebugInfoProto do + def example(info) + end + + assert {:ok, {DebugInfoProto, [debug_info: debug_info]}} = + :beam_lib.chunks(binary, [:debug_info]) - test "consolidation extracts implementations" do - protos = Protocol.extract_impls(Enumerable, [:code.lib_dir(:elixir, :ebin)]) - assert List in protos - assert Function in protos + assert {:debug_info_v1, :elixir_erl, {:elixir_v1, _, _}} = debug_info + after + Code.compiler_options(debug_info: true) end end diff --git a/lib/elixir/test/elixir/range_test.exs b/lib/elixir/test/elixir/range_test.exs index cde88cc2fb5..dddbddc3695 100644 --- a/lib/elixir/test/elixir/range_test.exs +++ b/lib/elixir/test/elixir/range_test.exs @@ -1,41 +1,151 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule RangeTest do use ExUnit.Case, async: true - test :precedence do - assert Enum.to_list(1..3+2) == [1, 2, 3, 4, 5] - assert 1..3 |> Enum.to_list == [1, 2, 3] + doctest Range + + defp reverse(first..last) do + last..first + end + + defp assert_disjoint(r1, r2) do + disjoint_assertions(r1, r2, true) + end + + defp assert_overlap(r1, r2) do + disjoint_assertions(r1, r2, false) + end + + defp disjoint_assertions(r1, r2, expected) do + # The caller should choose pairs of representative ranges, and we take care + # here of commuting them. + Enum.each([[r1, r2], [r2, r1]], fn [a, b] -> + assert Range.disjoint?(a, b) == expected + assert Range.disjoint?(reverse(a), b) == expected + assert Range.disjoint?(a, reverse(b)) == expected + assert Range.disjoint?(reverse(a), reverse(b)) == expected + end) end - test :op do + test "new" do + assert Range.new(1, 3) == 1..3//1 + assert Range.new(3, 1) == 3..1//-1 + assert Range.new(1, 3, 2) == 1..3//2 + assert Range.new(3, 1, -2) == 3..1//-2 + end + + test "op" do assert (1..3).first == 1 - assert (1..3).last == 3 + assert (1..3).last == 3 + assert (1..3).step == 1 + assert (3..1).step == -1 + assert (1..3//2).step == 2 end - test :range? do - assert Range.range?(1..3) - refute Range.range?(0) + test "inspect" do + assert inspect(1..3) == "1..3" + assert inspect(1..3//2) == "1..3//2" + + assert inspect(3..1) == "3..1//-1" + assert inspect(3..1//1) == "3..1//1" end - test :enum do - refute Enum.empty?(1..1) + test "shift" do + assert Range.shift(0..10//2, 2) == 4..14//2 + assert Range.shift(10..0//-2, 2) == 6..-4//-2 + assert Range.shift(10..0//-2, -2) == 14..4//-2 + end + + test "limits are integer only" do + first = 1.0 + last = 3.0 + message = "ranges (first..last) expect both sides to be integers, got: 1.0..3.0" + assert_raise ArgumentError, message, fn -> first..last end - assert Enum.member?(1..3, 2) - refute Enum.member?(1..3, 0) - refute Enum.member?(1..3, 4) - refute Enum.member?(3..1, 0) - refute Enum.member?(3..1, 4) + first = [] + last = [] + message = "ranges (first..last) expect both sides to be integers, got: []..[]" + assert_raise ArgumentError, message, fn -> first..last end + end - assert Enum.count(1..3) == 3 - assert Enum.count(3..1) == 3 + test "step is a non-zero integer" do + step = 1.0 + message = ~r"the step to be a non-zero integer" + assert_raise ArgumentError, message, fn -> 1..3//step end - assert Enum.map(1..3, &(&1 * 2)) == [2, 4, 6] - assert Enum.map(3..1, &(&1 * 2)) == [6, 4, 2] + step = 0 + message = ~r"the step to be a non-zero integer" + assert_raise ArgumentError, message, fn -> 1..3//step end end - test :inspect do - assert inspect(1..3) == "1..3" - assert inspect(3..1) == "3..1" + describe "disjoint?" do + test "returns true for disjoint ranges" do + assert_disjoint(1..5, 6..9) + assert_disjoint(-3..1, 2..3) + assert_disjoint(-7..-5, -3..-1) + + assert Range.disjoint?(1..1, 2..2) == true + assert Range.disjoint?(2..2, 1..1) == true + end + + test "returns false for ranges with common endpoints" do + assert_overlap(1..5, 5..9) + assert_overlap(-1..0, 0..1) + assert_overlap(-7..-5, -5..-1) + end + + test "returns false for ranges that overlap" do + assert_overlap(1..5, 3..7) + assert_overlap(-3..1, -1..3) + assert_overlap(-7..-5, -5..-1) + + assert Range.disjoint?(1..1, 1..1) == false + end + end + + describe "old ranges" do + test "inspect" do + asc = %{__struct__: Range, first: 1, last: 3} + desc = %{__struct__: Range, first: 3, last: 1} + + assert inspect(asc) == "1..3" + assert inspect(desc) == "3..1//-1" + end + + test "enum" do + asc = %{__struct__: Range, first: 1, last: 3} + desc = %{__struct__: Range, first: 3, last: 1} + + assert Enum.to_list(asc) == [1, 2, 3] + assert Enum.member?(asc, 2) + assert Enum.count(asc) == 3 + assert Enum.drop(asc, 1) == [2, 3] + assert Enum.slice([1, 2, 3, 4, 5, 6], asc) == [2, 3, 4] + # testing private Enum.aggregate + assert Enum.max(asc) == 3 + assert Enum.sum(asc) == 6 + assert Enum.min_max(asc) == {1, 3} + assert Enum.reduce(asc, 0, fn a, b -> a + b end) == 6 + + assert Enum.to_list(desc) == [3, 2, 1] + assert Enum.member?(desc, 2) + assert Enum.count(desc) == 3 + assert Enum.drop(desc, 1) == [2, 1] + assert Enum.slice([1, 2, 3, 4, 5, 6], desc) == [] + # testing private Enum.aggregate + assert Enum.max(desc) == 3 + assert Enum.sum(desc) == 6 + assert Enum.min_max(desc) == {1, 3} + assert Enum.reduce(desc, 0, fn a, b -> a + b end) == 6 + end + + test "string" do + asc = %{__struct__: Range, first: 1, last: 3} + desc = %{__struct__: Range, first: 3, last: 1} + + assert String.slice("elixir", asc) == "lix" + assert String.slice("elixir", desc) == "" + end end end diff --git a/lib/elixir/test/elixir/record_test.exs b/lib/elixir/test/elixir/record_test.exs index 4916b01d56b..ac0164a6b7e 100644 --- a/lib/elixir/test/elixir/record_test.exs +++ b/lib/elixir/test/elixir/record_test.exs @@ -1,16 +1,27 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule RecordTest do use ExUnit.Case, async: true require Record + doctest Record test "extract/2 extracts information from an Erlang file" do - assert Record.extract(:file_info, from_lib: "kernel/include/file.hrl") == - [size: :undefined, type: :undefined, access: :undefined, atime: :undefined, - mtime: :undefined, ctime: :undefined, mode: :undefined, links: :undefined, - major_device: :undefined, minor_device: :undefined, inode: :undefined, - uid: :undefined, gid: :undefined] + assert Record.extract(:file_info, from_lib: "kernel/include/file.hrl") == [ + size: :undefined, + type: :undefined, + access: :undefined, + atime: :undefined, + mtime: :undefined, + ctime: :undefined, + mode: :undefined, + links: :undefined, + major_device: :undefined, + minor_device: :undefined, + inode: :undefined, + uid: :undefined, + gid: :undefined + ] end test "extract/2 handles nested records too" do @@ -24,58 +35,251 @@ defmodule RecordTest do defstruct Record.extract(:file_info, from_lib: "kernel/include/file.hrl") end - assert %{__struct__: StructExtract, size: :undefined} = - StructExtract.__struct__ + assert %{__struct__: StructExtract, size: :undefined} = StructExtract.__struct__() + end + + test "extract_all/1 extracts all records information from an Erlang file" do + all_extract = Record.extract_all(from_lib: "kernel/include/file.hrl") + # has been stable over the very long time + assert length(all_extract) == 2 + assert all_extract[:file_info] + assert all_extract[:file_descriptor] end # We need indirection to avoid warnings defp record?(data, kind) do - Record.record?(data, kind) + Record.is_record(data, kind) end - test "record?/2" do - assert record?({User, "jose", 27}, User) - refute record?({User, "jose", 27}, Author) + test "is_record/2" do + assert record?({User, "meg", 27}, User) + refute record?({User, "meg", 27}, Author) refute record?(13, Author) + refute record?({"user", "meg", 27}, "user") + refute record?({}, User) + refute record?([], User) end # We need indirection to avoid warnings defp record?(data) do - Record.record?(data) + Record.is_record(data) end - test "record?/1" do - assert record?({User, "jose", 27}) - refute record?({"jose", 27}) + test "is_record/1" do + assert record?({User, "john", 27}) + refute record?({"john", 27}) refute record?(13) + refute record?({}) end - Record.defrecord :timestamp, [:date, :time] - Record.defrecord :user, __MODULE__, name: "José", age: 25 - Record.defrecordp :file_info, Record.extract(:file_info, from_lib: "kernel/include/file.hrl") + def record_in_guard?(term) when Record.is_record(term), do: true + def record_in_guard?(_), do: false + + def record_in_guard?(term, kind) when Record.is_record(term, kind), do: true + def record_in_guard?(_, _), do: false + + test "is_record/1,2 (in guard)" do + assert record_in_guard?({User, "john", 27}) + refute record_in_guard?({"user", "john", 27}) + + assert record_in_guard?({User, "john", 27}, User) + refute record_in_guard?({"user", "john", 27}, "user") + end + + Record.defrecord(:timestamp, [:date, :time]) + Record.defrecord(:user, __MODULE__, name: "john", age: 25) + + Record.defrecordp(:file_info, Record.extract(:file_info, from_lib: "kernel/include/file.hrl")) + + Record.defrecordp( + :certificate, + :OTPCertificate, + Record.extract(:OTPCertificate, from_lib: "public_key/include/public_key.hrl") + ) - test "records generates macros that generates tuples" do + test "records are tagged" do + assert elem(file_info(), 0) == :file_info + end + + test "records macros" do record = user() - assert user(record, :name) == "José" - assert user(record, :age) == 25 + assert user(record, :name) == "john" + assert user(record, :age) == 25 - record = user(record, name: "Eric") - assert user(record, :name) == "Eric" + record = user(record, name: "meg") + assert user(record, :name) == "meg" - assert elem(record, user(:name)) == "Eric" + assert elem(record, user(:name)) == "meg" assert elem(record, 0) == RecordTest user(name: name) = record - assert name == "Eric" + assert name == "meg" + + assert user(:name) == 1 end - test "records with no tag" do - assert elem(file_info(), 0) == :file_info + test "records with default values" do + record = user(_: :_, name: "meg") + assert user(record, :name) == "meg" + assert user(record, :age) == :_ + assert match?(user(_: _), user()) + end + + test "records preserve side-effects order" do + user = + user( + age: send(self(), :age), + name: send(self(), :name) + ) + + assert Process.info(self(), :messages) == {:messages, [:age, :name]} + + _ = + user(user, + age: send(self(), :update_age), + name: send(self(), :update_name) + ) + + assert Process.info(self(), :messages) == + {:messages, [:age, :name, :update_age, :update_name]} + end + + test "nested records preserve side-effects order" do + user = + user( + age: + user( + age: send(self(), :inner_age), + name: send(self(), :inner_name) + ), + name: send(self(), :name) + ) + + assert user == {RecordTest, :name, {RecordTest, :inner_name, :inner_age}} + assert for(_ <- 1..3, do: assert_receive(_)) == [:inner_age, :inner_name, :name] + + user = + user( + name: send(self(), :name), + age: + user( + age: send(self(), :inner_age), + name: send(self(), :inner_name) + ) + ) + + assert user == {RecordTest, :name, {RecordTest, :inner_name, :inner_age}} + assert for(_ <- 1..3, do: assert_receive(_)) == [:name, :inner_age, :inner_name] + end + + Record.defrecord( + :defaults, + struct: ~D[2016-01-01], + map: %{}, + tuple_zero: {}, + tuple_one: {1}, + tuple_two: {1, 2}, + tuple_three: {1, 2, 3}, + list: [1, 2, 3], + call: MapSet.new(), + string: "abc", + binary: <<1, 2, 3>>, + charlist: 'abc' + ) + + test "records with literal defaults and on-the-fly record" do + assert defaults(defaults()) == [ + struct: ~D[2016-01-01], + map: %{}, + tuple_zero: {}, + tuple_one: {1}, + tuple_two: {1, 2}, + tuple_three: {1, 2, 3}, + list: [1, 2, 3], + call: MapSet.new(), + string: "abc", + binary: <<1, 2, 3>>, + charlist: 'abc' + ] + + assert defaults(defaults(), :struct) == ~D[2016-01-01] + assert defaults(defaults(), :map) == %{} + assert defaults(defaults(), :tuple_zero) == {} + assert defaults(defaults(), :tuple_one) == {1} + assert defaults(defaults(), :tuple_two) == {1, 2} + assert defaults(defaults(), :tuple_three) == {1, 2, 3} + assert defaults(defaults(), :list) == [1, 2, 3] + assert defaults(defaults(), :call) == MapSet.new() + assert defaults(defaults(), :string) == "abc" + assert defaults(defaults(), :binary) == <<1, 2, 3>> + assert defaults(defaults(), :charlist) == 'abc' + end + + test "records with literal defaults and record in a variable" do + defaults = defaults() + + assert defaults(defaults) == [ + struct: ~D[2016-01-01], + map: %{}, + tuple_zero: {}, + tuple_one: {1}, + tuple_two: {1, 2}, + tuple_three: {1, 2, 3}, + list: [1, 2, 3], + call: MapSet.new(), + string: "abc", + binary: <<1, 2, 3>>, + charlist: 'abc' + ] + + assert defaults(defaults, :struct) == ~D[2016-01-01] + assert defaults(defaults, :map) == %{} + assert defaults(defaults, :tuple_zero) == {} + assert defaults(defaults, :tuple_one) == {1} + assert defaults(defaults, :tuple_two) == {1, 2} + assert defaults(defaults, :tuple_three) == {1, 2, 3} + assert defaults(defaults, :list) == [1, 2, 3] + assert defaults(defaults, :call) == MapSet.new() + assert defaults(defaults, :string) == "abc" + assert defaults(defaults, :binary) == <<1, 2, 3>> + assert defaults(defaults, :charlist) == 'abc' end test "records with dynamic arguments" do record = file_info() assert file_info(record, :size) == :undefined + + record = user() + assert user(record) == [name: "john", age: 25] + assert user(user()) == [name: "john", age: 25] + + msg = + "expected argument to be a literal atom, literal keyword or a :file_info record, " <> + "got runtime: {RecordTest, \"john\", 25}" + + assert_raise ArgumentError, msg, fn -> + file_info(record) + end + + pretender = {RecordTest, "john"} + + msg = + "expected argument to be a RecordTest record with 2 fields, " <> + "got: {RecordTest, \"john\"}" + + assert_raise ArgumentError, msg, fn -> + user(pretender) + end + + pretender = {RecordTest, "john", 25, []} + + msg = + "expected argument to be a RecordTest record with 2 fields, " <> + "got: {RecordTest, \"john\", 25, []}" + + assert_raise ArgumentError, msg, fn -> + user(pretender) + end end test "records visibility" do @@ -83,6 +287,12 @@ defmodule RecordTest do refute macro_exported?(__MODULE__, :file_info, 1) end + test "records reflection" do + assert %{fields: [:name, :age], kind: :defrecord, name: :user, tag: RecordTest} in @__records__ + + assert %{fields: [:date, :time], kind: :defrecord, name: :timestamp, tag: :timestamp} in @__records__ + end + test "records with no defaults" do record = timestamp() assert timestamp(record, :date) == nil @@ -92,4 +302,80 @@ defmodule RecordTest do assert timestamp(record, :date) == :foo assert timestamp(record, :time) == :bar end + + test "records defined multiple times" do + msg = "cannot define record :r because a definition r/0 already exists" + + assert_raise ArgumentError, msg, fn -> + defmodule M do + import Record + defrecord :r, [:a] + defrecord :r, [:a] + end + end + end + + test "macro and record with the same name defined" do + msg = "cannot define record :a because a definition a/1 already exists" + + assert_raise ArgumentError, msg, fn -> + defmodule M do + defmacro a(_) do + end + + require Record + Record.defrecord(:a, [:a]) + end + end + + msg = "cannot define record :a because a definition a/2 already exists" + + assert_raise ArgumentError, msg, fn -> + defmodule M do + defmacro a(_, _) do + end + + require Record + Record.defrecord(:a, [:a]) + end + end + end + + describe "warnings" do + import ExUnit.CaptureIO + + test "warns on bad record update input" do + assert capture_io(:stderr, fn -> + defmodule RecordSample do + require Record + Record.defrecord(:user, __MODULE__, name: "john", age: 25) + + def fun do + user(user(), _: :_, name: "meg") + end + end + end) =~ + "updating a record with a default (:_) is equivalent to creating a new record" + after + purge(RecordSample) + end + + test "defrecord warns with duplicate keys" do + assert capture_io(:stderr, fn -> + Code.eval_string(""" + defmodule RecordSample do + import Record + defrecord :r, [:foo, :bar, foo: 1] + end + """) + end) =~ "duplicate key :foo found in record" + after + purge(RecordSample) + end + + defp purge(module) when is_atom(module) do + :code.delete(module) + :code.purge(module) + end + end end diff --git a/lib/elixir/test/elixir/regex_test.exs b/lib/elixir/test/elixir/regex_test.exs index 702c7594f66..008b9f273e1 100644 --- a/lib/elixir/test/elixir/regex_test.exs +++ b/lib/elixir/test/elixir/regex_test.exs @@ -1,47 +1,57 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule RegexTest do use ExUnit.Case, async: true - test :multiline do + @re_21_3_little %Regex{ + re_pattern: + {:re_pattern, 1, 0, 0, + <<69, 82, 67, 80, 94, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 255, 255, 255, 255, 255, 255, 255, + 255, 99, 0, 0, 0, 0, 0, 1, 0, 0, 0, 64, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 102, 111, 111, 0, 131, 0, 20, 29, 99, 133, + 0, 7, 0, 1, 29, 100, 119, 0, 5, 29, 101, 120, 0, 12, 120, 0, 20, 0>>}, + re_version: {"8.42 2018-03-20", :little}, + source: "c(?d|e)" + } + + @re_21_3_big %Regex{ + re_pattern: + {:re_pattern, 1, 0, 0, + <<80, 67, 82, 69, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 17, 255, 255, 255, 255, 255, 255, 255, + 255, 0, 99, 0, 0, 0, 0, 0, 1, 0, 0, 0, 56, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 102, 111, 111, 0, 131, 0, 20, 29, 99, 133, 0, 7, 0, 1, 29, 100, 119, + 0, 5, 29, 101, 120, 0, 12, 120, 0, 20, 0>>}, + re_version: {"8.42 2018-03-20", :big}, + source: "c(?d|e)" + } + + @re_19_3_little %Regex{ + re_pattern: + {:re_pattern, 1, 0, 0, + <<69, 82, 67, 80, 94, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 255, 255, 255, 255, 255, 255, 255, + 255, 99, 0, 0, 0, 0, 0, 1, 0, 0, 0, 64, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 102, 111, 111, 0, 125, 0, 20, 29, 99, 127, + 0, 7, 0, 1, 29, 100, 113, 0, 5, 29, 101, 114, 0, 12, 114, 0, 20, 0>>}, + re_version: {"8.33 2013-05-29", :little}, + source: "c(?d|e)" + } + + doctest Regex + + test "multiline" do refute Regex.match?(~r/^b$/, "a\nb\nc") assert Regex.match?(~r/^b$/m, "a\nb\nc") end - test :precedence do + test "precedence" do assert {"aa", :unknown} |> elem(0) =~ ~r/(a)\1/ end - test :backreference do + test "backreference" do assert "aa" =~ ~r/(a)\1/ end - test :compile! do - assert Regex.regex?(Regex.compile!("foo")) - - assert_raise Regex.CompileError, ~r/position 0$/, fn -> - Regex.compile!("*foo") - end - end - - test :compile do - {:ok, regex} = Regex.compile("foo") - assert Regex.regex?(regex) - assert {:error, _} = Regex.compile("*foo") - assert {:error, _} = Regex.compile("foo", "y") - end - - test :compile_with_erl_opts do - {:ok, regex} = Regex.compile("foo\\sbar", [:dotall, {:newline, :anycrlf}]) - assert "foo\nbar" =~ regex - end - - test :regex? do - assert Regex.regex?(~r/foo/) - refute Regex.regex?(0) - end - - test :source do + test "source" do src = "foo" assert Regex.source(Regex.compile!(src)) == src assert Regex.source(~r/#{src}/) == src @@ -55,53 +65,92 @@ defmodule RegexTest do assert Regex.source(~r/#{src}/) == src end - test :literal_source do + test "literal source" do assert Regex.source(Regex.compile!("foo")) == "foo" assert Regex.source(~r"foo") == "foo" - assert Regex.re_pattern(Regex.compile!("foo")) - == Regex.re_pattern(~r"foo") + assert Regex.re_pattern(Regex.compile!("foo")) == Regex.re_pattern(~r"foo") assert Regex.source(Regex.compile!("\a\b\d\e\f\n\r\s\t\v")) == "\a\b\d\e\f\n\r\s\t\v" assert Regex.source(~r<\a\b\d\e\f\n\r\s\t\v>) == "\a\\b\\d\\e\f\n\r\\s\t\v" - assert Regex.re_pattern(Regex.compile!("\a\b\d\e\f\n\r\s\t\v")) - == Regex.re_pattern(~r"\a\010\177\033\f\n\r \t\v") + + assert Regex.re_pattern(Regex.compile!("\a\b\d\e\f\n\r\s\t\v")) == + Regex.re_pattern(~r"\a\010\177\033\f\n\r \t\v") assert Regex.source(Regex.compile!("\a\\b\\d\e\f\n\r\\s\t\v")) == "\a\\b\\d\e\f\n\r\\s\t\v" assert Regex.source(~r<\a\\b\\d\\e\f\n\r\\s\t\v>) == "\a\\\\b\\\\d\\\\e\f\n\r\\\\s\t\v" - assert Regex.re_pattern(Regex.compile!("\a\\b\\d\e\f\n\r\\s\t\v")) - == Regex.re_pattern(~r"\a\b\d\e\f\n\r\s\t\v") - end - test :opts do - assert Regex.opts(Regex.compile!("foo", "i")) == "i" + assert Regex.re_pattern(Regex.compile!("\a\\b\\d\e\f\n\r\\s\t\v")) == + Regex.re_pattern(~r"\a\b\d\e\f\n\r\s\t\v") end - test :unicode do - assert "josé" =~ ~r"\p{Latin}$"u + test "Unicode" do + assert "olá" =~ ~r"\p{Latin}$"u refute "£" =~ ~r/\p{Lu}/u + # Non breaking space matches [[:space:]] with Unicode + assert <<0xA0::utf8>> =~ ~r/[[:space:]]/u + assert <<0xA0::utf8>> =~ ~r/\s/u assert <>> =~ ~r/<.>/ - refute <>> =~ ~r/<.>/u end - test :names do + test "ungreedy" do + assert Regex.run(~r/[\d ]+/, "1 2 3 4 5"), ["1 2 3 4 5"] + assert Regex.run(~r/[\d ]?+/, "1 2 3 4 5"), ["1"] + assert Regex.run(~r/[\d ]+/U, "1 2 3 4 5"), ["1"] + end + + test "compile/1" do + {:ok, %Regex{}} = Regex.compile("foo") + assert {:error, _} = Regex.compile("*foo") + assert {:error, _} = Regex.compile("foo", "y") + assert {:error, _} = Regex.compile("foo", "uy") + end + + test "compile/1 with Erlang options" do + {:ok, regex} = Regex.compile("foo\\sbar", [:dotall, {:newline, :anycrlf}]) + assert "foo\nbar" =~ regex + end + + test "compile!/1" do + assert %Regex{} = Regex.compile!("foo") + + assert_raise Regex.CompileError, ~r/position 0$/, fn -> + Regex.compile!("*foo") + end + end + + test "recompile/1" do + new_regex = ~r/foo/ + {:ok, %Regex{}} = Regex.recompile(new_regex) + assert %Regex{} = Regex.recompile!(new_regex) + + old_regex = Map.delete(~r/foo/, :re_version) + {:ok, %Regex{}} = Regex.recompile(old_regex) + assert %Regex{} = Regex.recompile!(old_regex) + end + + test "opts/1" do + assert Regex.opts(Regex.compile!("foo", "i")) == "i" + end + + test "names/1" do assert Regex.names(~r/(?foo)/) == ["FOO"] end - test :match? do + test "match?/2" do assert Regex.match?(~r/foo/, "foo") refute Regex.match?(~r/foo/, "FOO") assert Regex.match?(~r/foo/i, "FOO") assert Regex.match?(~r/\d{1,3}/i, "123") - assert Regex.match?(~r/foo/, "afooa") + assert Regex.match?(~r/foo/, "afooa") refute Regex.match?(~r/^foo/, "afooa") - assert Regex.match?(~r/^foo/, "fooa") + assert Regex.match?(~r/^foo/, "fooa") refute Regex.match?(~r/foo$/, "afooa") - assert Regex.match?(~r/foo$/, "afoo") + assert Regex.match?(~r/foo$/, "afoo") end - test :named_captures do + test "named_captures/2" do assert Regex.named_captures(~r/(?c)(?d)/, "abcd") == %{"bar" => "d", "foo" => "c"} assert Regex.named_captures(~r/c(?d)/, "abcd") == %{"foo" => "d"} assert Regex.named_captures(~r/c(?d)/, "no_match") == nil @@ -109,61 +158,154 @@ defmodule RegexTest do assert Regex.named_captures(~r/c(.)/, "cat") == %{} end - test :sigil_R do + test "sigil R" do assert Regex.match?(~R/f#{1,3}o/, "f#o") end - test :run do + test "run/2" do assert Regex.run(~r"c(d)", "abcd") == ["cd", "d"] assert Regex.run(~r"e", "abcd") == nil end - test :run_with_all_names do + test "run/3 with :all_names as the value of the :capture option" do assert Regex.run(~r/c(?d)/, "abcd", capture: :all_names) == ["d"] assert Regex.run(~r/c(?d)/, "no_match", capture: :all_names) == nil assert Regex.run(~r/c(?d|e)/, "abcd abce", capture: :all_names) == ["d"] end - test :run_with_indexes do + test "run/3 with :index as the value of the :return option" do assert Regex.run(~r"c(d)", "abcd", return: :index) == [{2, 2}, {3, 1}] assert Regex.run(~r"e", "abcd", return: :index) == nil end - test :scan do + test "run/3 with :offset" do + assert Regex.run(~r"^foo", "foobar", offset: 0) == ["foo"] + assert Regex.run(~r"^foo", "foobar", offset: 2) == nil + assert Regex.run(~r"^foo", "foobar", offset: 2, return: :index) == nil + assert Regex.run(~r"bar", "foobar", offset: 2, return: :index) == [{3, 3}] + end + + test "run/3 with regexes compiled in different systems" do + assert Regex.run(@re_21_3_little, "abcd abce", capture: :all_names) == ["d"] + assert Regex.run(@re_21_3_big, "abcd abce", capture: :all_names) == ["d"] + assert Regex.run(@re_19_3_little, "abcd abce", capture: :all_names) == ["d"] + end + + test "run/3 with regexes with options compiled in different systems" do + assert Regex.run(%{~r/foo/i | re_version: "bad version"}, "FOO") == ["FOO"] + end + + test "scan/2" do assert Regex.scan(~r"c(d|e)", "abcd abce") == [["cd", "d"], ["ce", "e"]] assert Regex.scan(~r"c(?:d|e)", "abcd abce") == [["cd"], ["ce"]] assert Regex.scan(~r"e", "abcd") == [] end - test :scan_with_all_names do + test "scan/2 with :all_names as the value of the :capture option" do assert Regex.scan(~r/cd/, "abcd", capture: :all_names) == [] assert Regex.scan(~r/c(?d)/, "abcd", capture: :all_names) == [["d"]] assert Regex.scan(~r/c(?d)/, "no_match", capture: :all_names) == [] assert Regex.scan(~r/c(?d|e)/, "abcd abce", capture: :all_names) == [["d"], ["e"]] end - test :split do + test "scan/2 with :offset" do + assert Regex.scan(~r"^foo", "foobar", offset: 0) == [["foo"]] + assert Regex.scan(~r"^foo", "foobar", offset: 1) == [] + end + + test "scan/2 with regexes compiled in different systems" do + assert Regex.scan(@re_21_3_little, "abcd abce", capture: :all_names) == [["d"], ["e"]] + assert Regex.scan(@re_21_3_big, "abcd abce", capture: :all_names) == [["d"], ["e"]] + assert Regex.scan(@re_19_3_little, "abcd abce", capture: :all_names) == [["d"], ["e"]] + end + + test "scan/2 with regexes with options compiled in different systems" do + assert Regex.scan(%{~r/foo/i | re_version: "bad version"}, "FOO") == [["FOO"]] + end + + test "split/2,3" do assert Regex.split(~r",", "") == [""] + assert Regex.split(~r",", "", trim: true) == [] + assert Regex.split(~r",", "", trim: true, parts: 2) == [] + + assert Regex.split(~r"=", "key=") == ["key", ""] + assert Regex.split(~r"=", "=value") == ["", "value"] + assert Regex.split(~r" ", "foo bar baz") == ["foo", "bar", "baz"] - assert Regex.split(~r" ", "foo bar baz", parts: 0) == ["foo", "bar", "baz"] assert Regex.split(~r" ", "foo bar baz", parts: :infinity) == ["foo", "bar", "baz"] assert Regex.split(~r" ", "foo bar baz", parts: 10) == ["foo", "bar", "baz"] assert Regex.split(~r" ", "foo bar baz", parts: 2) == ["foo", "bar baz"] - assert Regex.split(~r"\s", "foobar") == ["foobar"] + assert Regex.split(~r" ", " foo bar baz ") == ["", "foo", "bar", "baz", ""] assert Regex.split(~r" ", " foo bar baz ", trim: true) == ["foo", "bar", "baz"] - assert Regex.split(~r"=", "key=") == ["key", ""] - assert Regex.split(~r"=", "=value") == ["", "value"] + assert Regex.split(~r" ", " foo bar baz ", parts: 2) == ["", "foo bar baz "] + assert Regex.split(~r" ", " foo bar baz ", trim: true, parts: 2) == ["foo", "bar baz "] + end + + test "split/3 with the :on option" do + assert Regex.split(~r/()abc()/, "xabcxabcx", on: :none) == ["xabcxabcx"] + + parts = ["x", "abc", "x", "abc", "x"] + assert Regex.split(~r/()abc()/, "xabcxabcx", on: :all_but_first) == parts + + assert Regex.split(~r/(?)abc(?)/, "xabcxabcx", on: [:first, :last]) == parts + + parts = ["xabc", "xabc", "x"] + assert Regex.split(~r/(?)abc(?)/, "xabcxabcx", on: [:last, :first]) == parts + + assert Regex.split(~r/a(?b)c/, "abc", on: [:second]) == ["a", "c"] + + parts = ["a", "c adc a", "c"] + assert Regex.split(~r/a(?b)c|a(?d)c/, "abc adc abc", on: [:second]) == parts + + assert Regex.split(~r/a(?b)c|a(?d)c/, "abc adc abc", on: [:second, :fourth]) == + ["a", "c a", "c a", "c"] + end + + test "split/3 with the :include_captures option" do + assert Regex.split(~r/([ln])/, "Erlang", include_captures: true) == ["Er", "l", "a", "n", "g"] + assert Regex.split(~r/([kw])/, "Elixir", include_captures: true) == ["Elixir"] + + assert Regex.split(~r/([Ee]lixir)/, "Elixir", include_captures: true, trim: true) == + ["Elixir"] + + assert Regex.split(~r/([Ee]lixir)/, "Elixir", include_captures: true, trim: false) == + ["", "Elixir", ""] + + assert Regex.split(~r//, "abc", include_captures: true) == + ["", "", "a", "", "b", "", "c", "", ""] + + assert Regex.split(~r/a/, "abc", include_captures: true) == ["", "a", "bc"] + assert Regex.split(~r/c/, "abc", include_captures: true) == ["ab", "c", ""] + + assert Regex.split(~r/[Ei]/, "Elixir", include_captures: true, parts: 2) == + ["", "E", "lixir"] + + assert Regex.split(~r/[Ei]/, "Elixir", include_captures: true, parts: 3) == + ["", "E", "l", "i", "xir"] + + assert Regex.split(~r/[Ei]/, "Elixir", include_captures: true, parts: 2, trim: true) == + ["E", "lixir"] + + assert Regex.split(~r/[Ei]/, "Elixir", include_captures: true, parts: 3, trim: true) == + ["E", "l", "i", "xir"] end - test :replace do - assert Regex.replace(~r(d), "abc", "d") == "abc" - assert Regex.replace(~r(b), "abc", "d") == "adc" - assert Regex.replace(~r(b), "abc", "[\\0]") == "a[b]c" + test "replace/3,4" do + assert Regex.replace(~r/d/, "abc", "d") == "abc" + assert Regex.replace(~r/b/, "abc", "d") == "adc" + assert Regex.replace(~r/b/, "abc", "[\\0]") == "a[b]c" assert Regex.replace(~r[(b)], "abc", "[\\1]") == "a[b]c" + assert Regex.replace(~r[(b)], "abc", "[\\2]") == "a[]c" + assert Regex.replace(~r[(b)], "abc", "[\\3]") == "a[]c" + assert Regex.replace(~r/b/, "abc", "[\\g{0}]") == "a[b]c" + assert Regex.replace(~r[(b)], "abc", "[\\g{1}]") == "a[b]c" + + assert Regex.replace(~r/b/, "abcbe", "d") == "adcde" + assert Regex.replace(~r/b/, "abcbe", "d", global: false) == "adcbe" - assert Regex.replace(~r(b), "abcbe", "d") == "adcde" - assert Regex.replace(~r(b), "abcbe", "d", global: false) == "adcbe" + assert Regex.replace(~r/ /, "first third", "\\second\\") == "first\\second\\third" + assert Regex.replace(~r/ /, "first third", "\\\\second\\\\") == "first\\second\\third" assert Regex.replace(~r[a(b)c], "abcabc", fn -> "ac" end) == "acac" assert Regex.replace(~r[a(b)c], "abcabc", fn "abc" -> "ac" end) == "acac" @@ -172,7 +314,7 @@ defmodule RegexTest do assert Regex.replace(~r[a(b)c], "abcabc", fn "abc", "b" -> "ac" end, global: false) == "acabc" end - test :escape do + test "escape" do assert matches_escaped?(".") refute matches_escaped?(".", "x") @@ -190,10 +332,18 @@ defmodule RegexTest do assert matches_escaped?("\\A \\z") assert matches_escaped?(" x ") - assert matches_escaped?("  x    x ") # unicode spaces here + # Unicode spaces here + assert matches_escaped?("  x    x ") assert matches_escaped?("# lol") - assert matches_escaped?("\\A.^$*+?()[{\\| \t\n\xff\\z #hello\x{202F}\x{205F}") + assert matches_escaped?("\\A.^$*+?()[{\\| \t\n\x20\\z #hello\u202F\u205F") + assert Regex.match?(Regex.compile!("[" <> Regex.escape("!-#") <> "]"), "-") + + assert Regex.escape("{}") == "\\{\\}" + assert Regex.escape("[]") == "\\[\\]" + + assert Regex.escape("{foo}") == "\\{foo\\}" + assert Regex.escape("[foo]") == "\\[foo\\]" end defp matches_escaped?(string) do @@ -201,6 +351,6 @@ defmodule RegexTest do end defp matches_escaped?(string, match) do - Regex.match? ~r/#{Regex.escape(string)}/simxu, match + Regex.match?(~r/#{Regex.escape(string)}/simx, match) end end diff --git a/lib/elixir/test/elixir/registry_test.exs b/lib/elixir/test/elixir/registry_test.exs new file mode 100644 index 00000000000..68d099467dd --- /dev/null +++ b/lib/elixir/test/elixir/registry_test.exs @@ -0,0 +1,1043 @@ +Code.require_file("test_helper.exs", __DIR__) + +defmodule RegistryTest do + use ExUnit.Case, async: true + doctest Registry, except: [:moduledoc] + + setup config do + keys = config[:keys] || :unique + partitions = config[:partitions] || 1 + listeners = List.wrap(config[:listener]) + opts = [keys: keys, name: config.test, partitions: partitions, listeners: listeners] + {:ok, _} = start_supervised({Registry, opts}) + {:ok, %{registry: config.test, partitions: partitions}} + end + + for {describe, partitions} <- ["with 1 partition": 1, "with 8 partitions": 8] do + describe "unique #{describe}" do + @describetag keys: :unique, partitions: partitions + + test "starts configured number of partitions", %{registry: registry, partitions: partitions} do + assert length(Supervisor.which_children(registry)) == partitions + end + + test "counts 0 keys in an empty registry", %{registry: registry} do + assert 0 == Registry.count(registry) + end + + test "counts the number of keys in a registry", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + + assert 2 == Registry.count(registry) + end + + test "has unique registrations", %{registry: registry} do + {:ok, pid} = Registry.register(registry, "hello", :value) + assert is_pid(pid) + assert Registry.keys(registry, self()) == ["hello"] + assert Registry.values(registry, "hello", self()) == [:value] + + assert {:error, {:already_registered, pid}} = Registry.register(registry, "hello", :value) + assert pid == self() + assert Registry.keys(registry, self()) == ["hello"] + assert Registry.values(registry, "hello", self()) == [:value] + + {:ok, pid} = Registry.register(registry, "world", :value) + assert is_pid(pid) + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello", "world"] + end + + test "has unique registrations across processes", %{registry: registry} do + {_, task} = register_task(registry, "hello", :value) + Process.link(Process.whereis(registry)) + assert Registry.keys(registry, task) == ["hello"] + assert Registry.values(registry, "hello", task) == [:value] + + assert {:error, {:already_registered, ^task}} = + Registry.register(registry, "hello", :recent) + + assert Registry.keys(registry, self()) == [] + assert Registry.values(registry, "hello", self()) == [] + + {:links, links} = Process.info(self(), :links) + assert Process.whereis(registry) in links + end + + test "has unique registrations even if partition is delayed", %{registry: registry} do + {owner, task} = register_task(registry, "hello", :value) + + assert Registry.register(registry, "hello", :other) == + {:error, {:already_registered, task}} + + :sys.suspend(owner) + kill_and_assert_down(task) + Registry.register(registry, "hello", :other) + assert Registry.lookup(registry, "hello") == [{self(), :other}] + end + + test "supports match patterns", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + assert Registry.match(registry, "hello", {1, :_, :_}) == [{self(), value}] + assert Registry.match(registry, "hello", {1.0, :_, :_}) == [] + assert Registry.match(registry, "hello", {:_, :atom, :_}) == [{self(), value}] + assert Registry.match(registry, "hello", {:"$1", :_, :"$1"}) == [{self(), value}] + assert Registry.match(registry, "hello", :_) == [{self(), value}] + assert Registry.match(registry, :_, :_) == [] + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + assert Registry.match(registry, "world", %{b: "b"}) == [{self(), value2}] + end + + test "supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert Registry.match(registry, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 1}]) == + [{self(), value}] + + assert Registry.match(registry, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 2}]) == [] + + assert Registry.match(registry, "hello", {:_, :"$1", :_}, [{:is_atom, :"$1"}]) == + [{self(), value}] + end + + test "count_match supports match patterns", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + assert 1 == Registry.count_match(registry, "hello", {1, :_, :_}) + assert 0 == Registry.count_match(registry, "hello", {1.0, :_, :_}) + assert 1 == Registry.count_match(registry, "hello", {:_, :atom, :_}) + assert 1 == Registry.count_match(registry, "hello", {:"$1", :_, :"$1"}) + assert 1 == Registry.count_match(registry, "hello", :_) + assert 0 == Registry.count_match(registry, :_, :_) + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + assert 1 == Registry.count_match(registry, "world", %{b: "b"}) + end + + test "count_match supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert 1 == Registry.count_match(registry, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 1}]) + assert 0 == Registry.count_match(registry, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 2}]) + assert 1 == Registry.count_match(registry, "hello", {:_, :"$1", :_}, [{:is_atom, :"$1"}]) + end + + test "unregister_match supports patterns", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + + Registry.unregister_match(registry, "hello", {2, :_, :_}) + assert Registry.lookup(registry, "hello") == [{self(), value}] + Registry.unregister_match(registry, "hello", {1.0, :_, :_}) + assert Registry.lookup(registry, "hello") == [{self(), value}] + Registry.unregister_match(registry, "hello", {:_, :atom, :_}) + assert Registry.lookup(registry, "hello") == [] + end + + test "unregister_match supports guards", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + + Registry.unregister_match(registry, "hello", {:"$1", :_, :_}, [{:<, :"$1", 2}]) + assert Registry.lookup(registry, "hello") == [] + end + + test "unregister_match supports tricky keys", %{registry: registry} do + {:ok, _} = Registry.register(registry, :_, :foo) + {:ok, _} = Registry.register(registry, "hello", "b") + + Registry.unregister_match(registry, :_, :foo) + assert Registry.lookup(registry, :_) == [] + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello"] + end + + test "compares using ===", %{registry: registry} do + {:ok, _} = Registry.register(registry, 1.0, :value) + {:ok, _} = Registry.register(registry, 1, :value) + assert Registry.keys(registry, self()) |> Enum.sort() == [1, 1.0] + end + + test "updates current process value", %{registry: registry} do + assert Registry.update_value(registry, "hello", &raise/1) == :error + register_task(registry, "hello", :value) + assert Registry.update_value(registry, "hello", &raise/1) == :error + + Registry.register(registry, "world", 1) + assert Registry.lookup(registry, "world") == [{self(), 1}] + assert Registry.update_value(registry, "world", &(&1 + 1)) == {2, 1} + assert Registry.lookup(registry, "world") == [{self(), 2}] + end + + test "dispatches to a single key", %{registry: registry} do + fun = fn _ -> raise "will never be invoked" end + assert Registry.dispatch(registry, "hello", fun) == :ok + + {:ok, _} = Registry.register(registry, "hello", :value) + + fun = fn [{pid, value}] -> send(pid, {:dispatch, value}) end + assert Registry.dispatch(registry, "hello", fun) + + assert_received {:dispatch, :value} + end + + test "unregisters process by key", %{registry: registry} do + :ok = Registry.unregister(registry, "hello") + + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello", "world"] + + :ok = Registry.unregister(registry, "hello") + assert Registry.keys(registry, self()) == ["world"] + + :ok = Registry.unregister(registry, "world") + assert Registry.keys(registry, self()) == [] + end + + test "unregisters with no entries", %{registry: registry} do + assert Registry.unregister(registry, "hello") == :ok + end + + test "unregisters with tricky keys", %{registry: registry} do + {:ok, _} = Registry.register(registry, :_, :foo) + {:ok, _} = Registry.register(registry, "hello", "b") + + Registry.unregister(registry, :_) + assert Registry.lookup(registry, :_) == [] + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello"] + end + + @tag listener: :"unique_listener_#{partitions}" + test "allows listeners", %{registry: registry, listener: listener} do + Process.register(self(), listener) + {_, task} = register_task(registry, "hello", :world) + assert_received {:register, ^registry, "hello", ^task, :world} + + self = self() + {:ok, _} = Registry.register(registry, "world", :value) + assert_received {:register, ^registry, "world", ^self, :value} + + :ok = Registry.unregister(registry, "world") + assert_received {:unregister, ^registry, "world", ^self} + end + + test "links and unlinks on register/unregister", %{registry: registry} do + {:ok, pid} = Registry.register(registry, "hello", :value) + {:links, links} = Process.info(self(), :links) + assert pid in links + + {:ok, pid} = Registry.register(registry, "world", :value) + {:links, links} = Process.info(self(), :links) + assert pid in links + + :ok = Registry.unregister(registry, "hello") + {:links, links} = Process.info(self(), :links) + assert pid in links + + :ok = Registry.unregister(registry, "world") + {:links, links} = Process.info(self(), :links) + refute pid in links + end + + test "raises on unknown registry name" do + assert_raise ArgumentError, ~r/unknown registry/, fn -> + Registry.register(:unknown, "hello", :value) + end + end + + test "via callbacks", %{registry: registry} do + name = {:via, Registry, {registry, "hello"}} + + # register_name + {:ok, pid} = Agent.start_link(fn -> 0 end, name: name) + + # send + assert Agent.update(name, &(&1 + 1)) == :ok + + # whereis_name + assert Agent.get(name, & &1) == 1 + + # unregister_name + assert {:error, _} = Agent.start(fn -> raise "oops" end) + + # errors + assert {:error, {:already_started, ^pid}} = Agent.start(fn -> 0 end, name: name) + end + + test "uses value provided in via", %{registry: registry} do + name = {:via, Registry, {registry, "hello", :value}} + {:ok, pid} = Agent.start_link(fn -> 0 end, name: name) + assert Registry.lookup(registry, "hello") == [{pid, :value}] + end + + test "empty list for empty registry", %{registry: registry} do + assert Registry.select(registry, [{{:_, :_, :_}, [], [:"$_"]}]) == [] + end + + test "select all", %{registry: registry} do + name = {:via, Registry, {registry, "hello"}} + {:ok, pid} = Agent.start_link(fn -> 0 end, name: name) + {:ok, _} = Registry.register(registry, "world", :value) + + assert Registry.select(registry, [{{:"$1", :"$2", :"$3"}, [], [{{:"$1", :"$2", :"$3"}}]}]) + |> Enum.sort() == + [{"hello", pid, nil}, {"world", self(), :value}] + end + + test "select supports full match specs", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{"hello", :"$2", :"$3"}, [], [{{"hello", :"$2", :"$3"}}]} + ]) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{:"$1", self(), :"$3"}, [], [{{:"$1", self(), :"$3"}}]} + ]) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{:"$1", :"$2", value}, [], [{{:"$1", :"$2", {value}}}]} + ]) + + assert [] == + Registry.select(registry, [ + {{"world", :"$2", :"$3"}, [], [{{"world", :"$2", :"$3"}}]} + ]) + + assert [] == Registry.select(registry, [{{:"$1", :"$2", {1.0, :_, :_}}, [], [:"$_"]}]) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{:"$1", :"$2", {:"$3", :atom, :"$4"}}, [], + [{{:"$1", :"$2", {{:"$3", :atom, :"$4"}}}}]} + ]) + + assert [{"hello", self(), {1, :atom, 1}}] == + Registry.select(registry, [ + {{:"$1", :"$2", {:"$3", :"$4", :"$3"}}, [], + [{{:"$1", :"$2", {{:"$3", :"$4", :"$3"}}}}]} + ]) + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + + assert [:match] == + Registry.select(registry, [{{"world", self(), %{b: "b"}}, [], [:match]}]) + + assert ["hello", "world"] == + Registry.select(registry, [{{:"$1", :_, :_}, [], [:"$1"]}]) |> Enum.sort() + end + + test "select supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert [{"hello", self(), {1, :atom, 2}}] == + Registry.select(registry, [ + {{:"$1", :"$2", {:"$3", :"$4", :"$5"}}, [{:>, :"$5", 1}], + [{{:"$1", :"$2", {{:"$3", :"$4", :"$5"}}}}]} + ]) + + assert [] == + Registry.select(registry, [ + {{:_, :_, {:_, :_, :"$1"}}, [{:>, :"$1", 2}], [:"$_"]} + ]) + + assert ["hello"] == + Registry.select(registry, [ + {{:"$1", :_, {:_, :"$2", :_}}, [{:is_atom, :"$2"}], [:"$1"]} + ]) + end + + test "select allows multiple specs", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + + assert ["hello", "world"] == + Registry.select(registry, [ + {{"hello", :_, :_}, [], [{:element, 1, :"$_"}]}, + {{"world", :_, :_}, [], [{:element, 1, :"$_"}]} + ]) + |> Enum.sort() + end + + test "select raises on incorrect shape of match spec", %{registry: registry} do + assert_raise ArgumentError, fn -> + Registry.select(registry, [{:_, [], []}]) + end + end + + test "count_select supports match specs", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + assert 1 == Registry.count_select(registry, [{{:_, :_, value}, [], [true]}]) + assert 1 == Registry.count_select(registry, [{{"hello", :_, :_}, [], [true]}]) + assert 1 == Registry.count_select(registry, [{{:_, :_, {1, :atom, :_}}, [], [true]}]) + assert 1 == Registry.count_select(registry, [{{:_, :_, {:"$1", :_, :"$1"}}, [], [true]}]) + assert 0 == Registry.count_select(registry, [{{"hello", :_, nil}, [], [true]}]) + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + assert 1 == Registry.count_select(registry, [{{"world", :_, :_}, [], [true]}]) + end + + test "count_select supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert 1 == + Registry.count_select(registry, [ + {{:_, :_, {:_, :"$1", :_}}, [{:is_atom, :"$1"}], [true]} + ]) + + assert 1 == + Registry.count_select(registry, [ + {{:_, :_, {:_, :_, :"$1"}}, [{:>, :"$1", 1}], [true]} + ]) + + assert 0 == + Registry.count_select(registry, [ + {{:_, :_, {:_, :_, :"$1"}}, [{:>, :"$1", 2}], [true]} + ]) + end + + test "count_select allows multiple specs", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + + assert 2 == + Registry.count_select(registry, [ + {{"hello", :_, :_}, [], [true]}, + {{"world", :_, :_}, [], [true]} + ]) + end + + test "count_select raises on incorrect shape of match spec", %{registry: registry} do + assert_raise ArgumentError, fn -> + Registry.count_select(registry, [{:_, [], []}]) + end + end + + test "doesn't grow ets on already_registered", + %{registry: registry, partitions: partitions} do + assert sum_pid_entries(registry, partitions) == 0 + + {:ok, pid} = Registry.register(registry, "hello", :value) + assert is_pid(pid) + assert sum_pid_entries(registry, partitions) == 1 + + {:ok, pid} = Registry.register(registry, "world", :value) + assert is_pid(pid) + assert sum_pid_entries(registry, partitions) == 2 + + assert {:error, {:already_registered, _pid}} = + Registry.register(registry, "hello", :value) + + assert sum_pid_entries(registry, partitions) == 2 + end + + test "doesn't grow ets on already_registered across processes", + %{registry: registry, partitions: partitions} do + assert sum_pid_entries(registry, partitions) == 0 + + {_, task} = register_task(registry, "hello", :value) + Process.link(Process.whereis(registry)) + + assert sum_pid_entries(registry, partitions) == 1 + + {:ok, pid} = Registry.register(registry, "world", :value) + assert is_pid(pid) + assert sum_pid_entries(registry, partitions) == 2 + + assert {:error, {:already_registered, ^task}} = + Registry.register(registry, "hello", :recent) + + assert sum_pid_entries(registry, partitions) == 2 + end + end + end + + for {describe, partitions} <- ["with 1 partition": 1, "with 8 partitions": 8] do + describe "duplicate #{describe}" do + @describetag keys: :duplicate, partitions: partitions + + test "starts configured number of partitions", %{registry: registry, partitions: partitions} do + assert length(Supervisor.which_children(registry)) == partitions + end + + test "counts 0 keys in an empty registry", %{registry: registry} do + assert 0 == Registry.count(registry) + end + + test "counts the number of keys in a registry", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "hello", :value) + + assert 2 == Registry.count(registry) + end + + test "has duplicate registrations", %{registry: registry} do + {:ok, pid} = Registry.register(registry, "hello", :value) + assert is_pid(pid) + assert Registry.keys(registry, self()) == ["hello"] + assert Registry.values(registry, "hello", self()) == [:value] + + assert {:ok, pid} = Registry.register(registry, "hello", :value) + assert is_pid(pid) + assert Registry.keys(registry, self()) == ["hello", "hello"] + assert Registry.values(registry, "hello", self()) == [:value, :value] + + {:ok, pid} = Registry.register(registry, "world", :value) + assert is_pid(pid) + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello", "hello", "world"] + end + + test "has duplicate registrations across processes", %{registry: registry} do + {_, task} = register_task(registry, "hello", :world) + assert Registry.keys(registry, self()) == [] + assert Registry.keys(registry, task) == ["hello"] + assert Registry.values(registry, "hello", self()) == [] + assert Registry.values(registry, "hello", task) == [:world] + + assert {:ok, _pid} = Registry.register(registry, "hello", :value) + assert Registry.keys(registry, self()) == ["hello"] + assert Registry.values(registry, "hello", self()) == [:value] + end + + test "compares using matches", %{registry: registry} do + {:ok, _} = Registry.register(registry, 1.0, :value) + {:ok, _} = Registry.register(registry, 1, :value) + assert Registry.keys(registry, self()) |> Enum.sort() == [1, 1.0] + end + + test "dispatches to multiple keys in serial", %{registry: registry} do + Process.flag(:trap_exit, true) + parent = self() + + fun = fn _ -> raise "will never be invoked" end + assert Registry.dispatch(registry, "hello", fun, parallel: false) == :ok + + {:ok, _} = Registry.register(registry, "hello", :value1) + {:ok, _} = Registry.register(registry, "hello", :value2) + {:ok, _} = Registry.register(registry, "world", :value3) + + fun = fn entries -> + assert parent == self() + for {pid, value} <- entries, do: send(pid, {:dispatch, value}) + end + + assert Registry.dispatch(registry, "hello", fun, parallel: false) + + assert_received {:dispatch, :value1} + assert_received {:dispatch, :value2} + refute_received {:dispatch, :value3} + + fun = fn entries -> + assert parent == self() + for {pid, value} <- entries, do: send(pid, {:dispatch, value}) + end + + assert Registry.dispatch(registry, "world", fun, parallel: false) + + refute_received {:dispatch, :value1} + refute_received {:dispatch, :value2} + assert_received {:dispatch, :value3} + + refute_received {:EXIT, _, _} + end + + test "dispatches to multiple keys in parallel", context do + %{registry: registry, partitions: partitions} = context + Process.flag(:trap_exit, true) + parent = self() + + fun = fn _ -> raise "will never be invoked" end + assert Registry.dispatch(registry, "hello", fun, parallel: true) == :ok + + {:ok, _} = Registry.register(registry, "hello", :value1) + {:ok, _} = Registry.register(registry, "hello", :value2) + {:ok, _} = Registry.register(registry, "world", :value3) + + fun = fn entries -> + if partitions == 8 do + assert parent != self() + else + assert parent == self() + end + + for {pid, value} <- entries, do: send(pid, {:dispatch, value}) + end + + assert Registry.dispatch(registry, "hello", fun, parallel: true) + + assert_received {:dispatch, :value1} + assert_received {:dispatch, :value2} + refute_received {:dispatch, :value3} + + fun = fn entries -> + if partitions == 8 do + assert parent != self() + else + assert parent == self() + end + + for {pid, value} <- entries, do: send(pid, {:dispatch, value}) + end + + assert Registry.dispatch(registry, "world", fun, parallel: true) + + refute_received {:dispatch, :value1} + refute_received {:dispatch, :value2} + assert_received {:dispatch, :value3} + + refute_received {:EXIT, _, _} + end + + test "unregisters by key", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello", "hello", "world"] + + :ok = Registry.unregister(registry, "hello") + assert Registry.keys(registry, self()) == ["world"] + + :ok = Registry.unregister(registry, "world") + assert Registry.keys(registry, self()) == [] + end + + test "unregisters with no entries", %{registry: registry} do + assert Registry.unregister(registry, "hello") == :ok + end + + test "unregisters with tricky keys", %{registry: registry} do + {:ok, _} = Registry.register(registry, :_, :foo) + {:ok, _} = Registry.register(registry, :_, :bar) + {:ok, _} = Registry.register(registry, "hello", "a") + {:ok, _} = Registry.register(registry, "hello", "b") + + Registry.unregister(registry, :_) + assert Registry.keys(registry, self()) |> Enum.sort() == ["hello", "hello"] + end + + test "supports match patterns", %{registry: registry} do + value1 = {1, :atom, 1} + value2 = {2, :atom, 2} + + {:ok, _} = Registry.register(registry, "hello", value1) + {:ok, _} = Registry.register(registry, "hello", value2) + + assert Registry.match(registry, "hello", {1, :_, :_}) == [{self(), value1}] + assert Registry.match(registry, "hello", {1.0, :_, :_}) == [] + + assert Registry.match(registry, "hello", {:_, :atom, :_}) |> Enum.sort() == + [{self(), value1}, {self(), value2}] + + assert Registry.match(registry, "hello", {:"$1", :_, :"$1"}) |> Enum.sort() == + [{self(), value1}, {self(), value2}] + + assert Registry.match(registry, "hello", {2, :_, :_}) == [{self(), value2}] + assert Registry.match(registry, "hello", {2.0, :_, :_}) == [] + end + + test "supports guards", %{registry: registry} do + value1 = {1, :atom, 1} + value2 = {2, :atom, 2} + + {:ok, _} = Registry.register(registry, "hello", value1) + {:ok, _} = Registry.register(registry, "hello", value2) + + assert Registry.match(registry, "hello", {:"$1", :_, :_}, [{:<, :"$1", 2}]) == + [{self(), value1}] + + assert Registry.match(registry, "hello", {:"$1", :_, :_}, [{:>, :"$1", 3}]) == [] + + assert Registry.match(registry, "hello", {:"$1", :_, :_}, [{:<, :"$1", 3}]) |> Enum.sort() == + [{self(), value1}, {self(), value2}] + + assert Registry.match(registry, "hello", {:_, :"$1", :_}, [{:is_atom, :"$1"}]) + |> Enum.sort() == [{self(), value1}, {self(), value2}] + end + + test "count_match supports match patterns", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + assert 1 == Registry.count_match(registry, "hello", {1, :_, :_}) + assert 0 == Registry.count_match(registry, "hello", {1.0, :_, :_}) + assert 1 == Registry.count_match(registry, "hello", {:_, :atom, :_}) + assert 1 == Registry.count_match(registry, "hello", {:"$1", :_, :"$1"}) + assert 1 == Registry.count_match(registry, "hello", :_) + assert 0 == Registry.count_match(registry, :_, :_) + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + assert 1 == Registry.count_match(registry, "world", %{b: "b"}) + end + + test "count_match supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert 1 == Registry.count_match(registry, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 1}]) + assert 0 == Registry.count_match(registry, "hello", {:_, :_, :"$1"}, [{:>, :"$1", 2}]) + assert 1 == Registry.count_match(registry, "hello", {:_, :"$1", :_}, [{:is_atom, :"$1"}]) + end + + test "unregister_match supports patterns", %{registry: registry} do + value1 = {1, :atom, 1} + value2 = {2, :atom, 2} + + {:ok, _} = Registry.register(registry, "hello", value1) + {:ok, _} = Registry.register(registry, "hello", value2) + + Registry.unregister_match(registry, "hello", {2, :_, :_}) + assert Registry.lookup(registry, "hello") == [{self(), value1}] + + {:ok, _} = Registry.register(registry, "hello", value2) + Registry.unregister_match(registry, "hello", {2.0, :_, :_}) + assert Registry.lookup(registry, "hello") == [{self(), value1}, {self(), value2}] + Registry.unregister_match(registry, "hello", {:_, :atom, :_}) + assert Registry.lookup(registry, "hello") == [] + end + + test "unregister_match supports guards", %{registry: registry} do + value1 = {1, :atom, 1} + value2 = {2, :atom, 2} + + {:ok, _} = Registry.register(registry, "hello", value1) + {:ok, _} = Registry.register(registry, "hello", value2) + + Registry.unregister_match(registry, "hello", {:"$1", :_, :_}, [{:<, :"$1", 2}]) + assert Registry.lookup(registry, "hello") == [{self(), value2}] + end + + test "unregister_match supports tricky keys", %{registry: registry} do + {:ok, _} = Registry.register(registry, :_, :foo) + {:ok, _} = Registry.register(registry, :_, :bar) + {:ok, _} = Registry.register(registry, "hello", "a") + {:ok, _} = Registry.register(registry, "hello", "b") + + Registry.unregister_match(registry, :_, :foo) + assert Registry.lookup(registry, :_) == [{self(), :bar}] + + assert Registry.keys(registry, self()) |> Enum.sort() == [:_, "hello", "hello"] + end + + @tag listener: :"duplicate_listener_#{partitions}" + test "allows listeners", %{registry: registry, listener: listener} do + Process.register(self(), listener) + {_, task} = register_task(registry, "hello", :world) + assert_received {:register, ^registry, "hello", ^task, :world} + + self = self() + {:ok, _} = Registry.register(registry, "hello", :value) + assert_received {:register, ^registry, "hello", ^self, :value} + + :ok = Registry.unregister(registry, "hello") + assert_received {:unregister, ^registry, "hello", ^self} + end + + test "links and unlinks on register/unregister", %{registry: registry} do + {:ok, pid} = Registry.register(registry, "hello", :value) + {:links, links} = Process.info(self(), :links) + assert pid in links + + {:ok, pid} = Registry.register(registry, "world", :value) + {:links, links} = Process.info(self(), :links) + assert pid in links + + :ok = Registry.unregister(registry, "hello") + {:links, links} = Process.info(self(), :links) + assert pid in links + + :ok = Registry.unregister(registry, "world") + {:links, links} = Process.info(self(), :links) + refute pid in links + end + + test "raises on unknown registry name" do + assert_raise ArgumentError, ~r/unknown registry/, fn -> + Registry.register(:unknown, "hello", :value) + end + end + + test "raises if attempt to be used on via", %{registry: registry} do + assert_raise ArgumentError, ":via is not supported for duplicate registries", fn -> + name = {:via, Registry, {registry, "hello"}} + Agent.start_link(fn -> 0 end, name: name) + end + end + + test "empty list for empty registry", %{registry: registry} do + assert Registry.select(registry, [{{:_, :_, :_}, [], [:"$_"]}]) == [] + end + + test "select all", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "hello", :value) + + assert Registry.select(registry, [{{:"$1", :"$2", :"$3"}, [], [{{:"$1", :"$2", :"$3"}}]}]) + |> Enum.sort() == + [{"hello", self(), :value}, {"hello", self(), :value}] + end + + test "select supports full match specs", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{"hello", :"$2", :"$3"}, [], [{{"hello", :"$2", :"$3"}}]} + ]) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{:"$1", self(), :"$3"}, [], [{{:"$1", self(), :"$3"}}]} + ]) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{:"$1", :"$2", value}, [], [{{:"$1", :"$2", {value}}}]} + ]) + + assert [] == + Registry.select(registry, [ + {{"world", :"$2", :"$3"}, [], [{{"world", :"$2", :"$3"}}]} + ]) + + assert [] == Registry.select(registry, [{{:"$1", :"$2", {1.0, :_, :_}}, [], [:"$_"]}]) + + assert [{"hello", self(), value}] == + Registry.select(registry, [ + {{:"$1", :"$2", {:"$3", :atom, :"$4"}}, [], + [{{:"$1", :"$2", {{:"$3", :atom, :"$4"}}}}]} + ]) + + assert [{"hello", self(), {1, :atom, 1}}] == + Registry.select(registry, [ + {{:"$1", :"$2", {:"$3", :"$4", :"$3"}}, [], + [{{:"$1", :"$2", {{:"$3", :"$4", :"$3"}}}}]} + ]) + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + + assert [:match] == + Registry.select(registry, [{{"world", self(), %{b: "b"}}, [], [:match]}]) + + assert ["hello", "world"] == + Registry.select(registry, [{{:"$1", :_, :_}, [], [:"$1"]}]) |> Enum.sort() + end + + test "select supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert [{"hello", self(), {1, :atom, 2}}] == + Registry.select(registry, [ + {{:"$1", :"$2", {:"$3", :"$4", :"$5"}}, [{:>, :"$5", 1}], + [{{:"$1", :"$2", {{:"$3", :"$4", :"$5"}}}}]} + ]) + + assert [] == + Registry.select(registry, [ + {{:_, :_, {:_, :_, :"$1"}}, [{:>, :"$1", 2}], [:"$_"]} + ]) + + assert ["hello"] == + Registry.select(registry, [ + {{:"$1", :_, {:_, :"$2", :_}}, [{:is_atom, :"$2"}], [:"$1"]} + ]) + end + + test "select allows multiple specs", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + + assert ["hello", "world"] == + Registry.select(registry, [ + {{"hello", :_, :_}, [], [{:element, 1, :"$_"}]}, + {{"world", :_, :_}, [], [{:element, 1, :"$_"}]} + ]) + |> Enum.sort() + end + + test "count_select supports match specs", %{registry: registry} do + value = {1, :atom, 1} + {:ok, _} = Registry.register(registry, "hello", value) + assert 1 == Registry.count_select(registry, [{{:_, :_, value}, [], [true]}]) + assert 1 == Registry.count_select(registry, [{{"hello", :_, :_}, [], [true]}]) + assert 1 == Registry.count_select(registry, [{{:_, :_, {1, :atom, :_}}, [], [true]}]) + assert 1 == Registry.count_select(registry, [{{:_, :_, {:"$1", :_, :"$1"}}, [], [true]}]) + assert 0 == Registry.count_select(registry, [{{"hello", :_, nil}, [], [true]}]) + + value2 = %{a: "a", b: "b"} + {:ok, _} = Registry.register(registry, "world", value2) + assert 1 == Registry.count_select(registry, [{{"world", :_, :_}, [], [true]}]) + end + + test "count_select supports guard conditions", %{registry: registry} do + value = {1, :atom, 2} + {:ok, _} = Registry.register(registry, "hello", value) + + assert 1 == + Registry.count_select(registry, [ + {{:_, :_, {:_, :"$1", :_}}, [{:is_atom, :"$1"}], [true]} + ]) + + assert 1 == + Registry.count_select(registry, [ + {{:_, :_, {:_, :_, :"$1"}}, [{:>, :"$1", 1}], [true]} + ]) + + assert 0 == + Registry.count_select(registry, [ + {{:_, :_, {:_, :_, :"$1"}}, [{:>, :"$1", 2}], [true]} + ]) + end + + test "count_select allows multiple specs", %{registry: registry} do + {:ok, _} = Registry.register(registry, "hello", :value) + {:ok, _} = Registry.register(registry, "world", :value) + + assert 2 == + Registry.count_select(registry, [ + {{"hello", :_, :_}, [], [true]}, + {{"world", :_, :_}, [], [true]} + ]) + end + end + end + + # Note: those tests relies on internals + for keys <- [:unique, :duplicate] do + describe "clean up #{keys} registry on process crash" do + @describetag keys: keys + + @tag partitions: 8 + test "with 8 partitions", %{registry: registry} do + {_, task1} = register_task(registry, "hello", :value) + {_, task2} = register_task(registry, "world", :value) + + kill_and_assert_down(task1) + kill_and_assert_down(task2) + + # pid might be in different partition to key so need to sync with all + # partitions before checking ETS tables are empty. + for i <- 0..7 do + [{_, _, {partition, _}}] = :ets.lookup(registry, i) + GenServer.call(partition, :sync) + end + + for i <- 0..7 do + [{_, key, {_, pid}}] = :ets.lookup(registry, i) + assert :ets.tab2list(key) == [] + assert :ets.tab2list(pid) == [] + end + end + + @tag partitions: 1 + test "with 1 partition", %{registry: registry} do + {_, task1} = register_task(registry, "hello", :value) + {_, task2} = register_task(registry, "world", :value) + + kill_and_assert_down(task1) + kill_and_assert_down(task2) + + [{-1, {_, _, key, {partition, pid}, _}}] = :ets.lookup(registry, -1) + GenServer.call(partition, :sync) + assert :ets.tab2list(key) == [] + assert :ets.tab2list(pid) == [] + end + end + end + + test "child_spec/1 uses :name as :id" do + assert %{id: :custom_name} = Registry.child_spec(name: :custom_name) + assert %{id: Registry} = Registry.child_spec([]) + end + + test "raises if :name is missing" do + assert_raise ArgumentError, ~r/expected :name option to be present/, fn -> + Registry.start_link(keys: :unique) + end + end + + test "raises if :name is not an atom" do + assert_raise ArgumentError, ~r/expected :name to be an atom, got/, fn -> + Registry.start_link(keys: :unique, name: []) + end + end + + test "raises if :compressed is not a boolean" do + assert_raise ArgumentError, ~r/expected :compressed to be a boolean, got/, fn -> + Registry.start_link(keys: :unique, name: :name, compressed: :fail) + end + end + + test "unregistration on crash with {registry, key, value} via tuple", %{registry: registry} do + name = {:via, Registry, {registry, :name, :value}} + spec = %{id: :foo, start: {Agent, :start_link, [fn -> raise "some error" end, [name: name]]}} + assert {:error, {error, _childspec}} = start_supervised(spec) + assert {%RuntimeError{message: "some error"}, _stacktrace} = error + end + + test "send works", %{registry: registry} do + name = {registry, "self"} + Registry.register_name(name, self()) + GenServer.cast({:via, Registry, name}, :message) + assert_received {:"$gen_cast", :message} + end + + test "send works with value", %{registry: registry} do + name = {registry, "self", "value"} + Registry.register_name(name, self()) + GenServer.cast({:via, Registry, name}, :message) + assert_received {:"$gen_cast", :message} + end + + defp register_task(registry, key, value) do + parent = self() + + {:ok, task} = + Task.start(fn -> + send(parent, Registry.register(registry, key, value)) + Process.sleep(:infinity) + end) + + assert_receive {:ok, owner} + {owner, task} + end + + defp kill_and_assert_down(pid) do + ref = Process.monitor(pid) + Process.exit(pid, :kill) + assert_receive {:DOWN, ^ref, _, _, _} + end + + defp sum_pid_entries(registry, partitions) do + Enum.map(0..(partitions - 1), &Module.concat(registry, "PIDPartition#{&1}")) + |> sum_ets_entries() + end + + defp sum_ets_entries(table_names) do + table_names + |> Enum.map(&ets_entries/1) + |> Enum.sum() + end + + defp ets_entries(table_name) do + :ets.all() + |> Enum.find_value(fn id -> :ets.info(id, :name) == table_name and :ets.info(id, :size) end) + end +end diff --git a/lib/elixir/test/elixir/set_test.exs b/lib/elixir/test/elixir/set_test.exs deleted file mode 100644 index 52545b8a6fd..00000000000 --- a/lib/elixir/test/elixir/set_test.exs +++ /dev/null @@ -1,189 +0,0 @@ -Code.require_file "test_helper.exs", __DIR__ - -# A TestSet implementation used only for testing. -defmodule TestSet do - defstruct list: [] - def new(list \\ []) when is_list(list) do - %TestSet{list: list} - end - - def reduce(%TestSet{list: list}, acc, fun) do - Enumerable.reduce(list, acc, fun) - end - - def member?(%TestSet{list: list}, v) do - v in list - end - - def size(%TestSet{list: list}) do - length(list) - end -end - -defmodule SetTest.Common do - defmacro __using__(_) do - quote location: :keep do - defp new_set(list \\ []) do - Enum.into list, set_impl.new - end - - defp new_set(list, fun) do - Enum.into list, set_impl.new, fun - end - - defp int_set() do - Enum.into [1, 2, 3], set_impl.new - end - - test "delete/2" do - result = Set.delete(new_set([1, 2, 3]), 2) - assert Set.equal?(result, new_set([1, 3])) - end - - test "delete/2 with match" do - refute Set.member?(Set.delete(int_set, 1), 1) - assert Set.member?(Set.delete(int_set, 1.0), 1) - end - - test "difference/2" do - result = Set.difference(new_set([1, 2, 3]), new_set([3])) - assert Set.equal?(result, new_set([1, 2])) - end - - test "difference/2 with match" do - refute Set.member?(Set.difference(int_set, new_set([1])), 1) - assert Set.member?(Set.difference(int_set, new_set([1.0])), 1) - end - - test "difference/2 with other set" do - result = Set.difference(new_set([1, 2, 3]), TestSet.new([3])) - assert Set.equal?(result, new_set([1, 2])) - end - - test "disjoint?/2" do - assert Set.disjoint?(new_set([1, 2, 3]), new_set([4, 5 ,6])) - refute Set.disjoint?(new_set([1, 2, 3]), new_set([3, 4 ,5])) - end - - test "disjoint/2 with other set" do - assert Set.disjoint?(new_set([1, 2, 3]), TestSet.new([4, 5 ,6])) - refute Set.disjoint?(new_set([1, 2, 3]), TestSet.new([3, 4 ,5])) - end - - test "equal?/2" do - assert Set.equal?(new_set([1, 2, 3]), new_set([3, 2, 1])) - refute Set.equal?(new_set([1, 2, 3]), new_set([3.0, 2.0, 1.0])) - end - - test "equal?/2 with other set" do - assert Set.equal?(new_set([1, 2, 3]), TestSet.new([3, 2, 1])) - refute Set.equal?(new_set([1, 2, 3]), TestSet.new([3.0, 2.0, 1.0])) - end - - test "intersection/2" do - result = Set.intersection(new_set([1, 2, 3]), new_set([2, 3, 4])) - assert Set.equal?(result, new_set([2, 3])) - end - - test "intersection/2 with match" do - assert Set.member?(Set.intersection(int_set, new_set([1])), 1) - refute Set.member?(Set.intersection(int_set, new_set([1.0])), 1) - end - - test "intersection/2 with other set" do - result = Set.intersection(new_set([1, 2, 3]), TestSet.new([2, 3, 4])) - assert Set.equal?(result, new_set([2, 3])) - end - - test "member?/2" do - assert Set.member?(new_set([1, 2, 3]), 2) - refute Set.member?(new_set([1, 2, 3]), 4) - refute Set.member?(new_set([1, 2, 3]), 1.0) - end - - test "put/2" do - result = Set.put(new_set([1, 2]), 3) - assert Set.equal?(result, new_set([1, 2, 3])) - end - - test "put/2 with match" do - assert Set.size(Set.put(int_set, 1)) == 3 - assert Set.size(Set.put(int_set, 1.0)) == 4 - end - - test "size/1" do - assert Set.size(new_set([1, 2, 3])) == 3 - end - - test "subset?/2" do - assert Set.subset?(new_set([1, 2]), new_set([1, 2, 3])) - refute Set.subset?(new_set([1, 2, 3]), new_set([1, 2])) - end - - test "subset/2 with match?" do - assert Set.subset?(new_set([1]), int_set) - refute Set.subset?(new_set([1.0]), int_set) - end - - test "subset?/2 with other set" do - assert Set.subset?(new_set([1, 2]), TestSet.new([1, 2, 3])) - refute Set.subset?(new_set([1, 2, 3]), TestSet.new([1, 2])) - end - - test "to_list/1" do - assert Set.to_list(new_set([1, 2, 3])) |> Enum.sort == [1, 2, 3] - end - - test "union/2" do - result = Set.union(new_set([1, 2, 3]), new_set([2, 3, 4])) - assert Set.equal?(result, new_set([1, 2, 3, 4])) - end - - test "union/2 with match" do - assert Set.size(Set.union(int_set, new_set([1]))) == 3 - assert Set.size(Set.union(int_set, new_set([1.0]))) == 4 - end - - test "union/2 with other set" do - result = Set.union(new_set([1, 2, 3]), TestSet.new([2, 3, 4])) - assert Set.equal?(result, new_set([1, 2, 3, 4])) - end - - test "is enumerable" do - assert Enum.member?(int_set, 1) - refute Enum.member?(int_set, 1.0) - assert Enum.sort(int_set) == [1,2,3] - end - - test "is collectable" do - assert Set.equal?(new_set([1, 1, 2, 3, 3, 3]), new_set([1, 2, 3])) - assert Set.equal?(new_set([1, 1, 2, 3, 3, 3], &(&1 * 2)), new_set([2, 4, 6])) - assert Collectable.empty(new_set([1, 2, 3])) == new_set - end - - test "is zippable" do - set = new_set(1..8) - list = Dict.to_list(set) - assert Enum.zip(list, list) == Enum.zip(set, set) - - set = new_set(1..100) - list = Dict.to_list(set) - assert Enum.zip(list, list) == Enum.zip(set, set) - end - - test "unsupported set" do - assert_raise ArgumentError, "unsupported set: :bad_set", fn -> - Set.to_list :bad_set - end - end - end - end -end - -defmodule Set.HashSetTest do - use ExUnit.Case, async: true - use SetTest.Common - - doctest Set - def set_impl, do: HashSet -end diff --git a/lib/elixir/test/elixir/stream_test.exs b/lib/elixir/test/elixir/stream_test.exs index bc1e3ad64b5..8e9e4f23cc7 100644 --- a/lib/elixir/test/elixir/stream_test.exs +++ b/lib/elixir/test/elixir/stream_test.exs @@ -1,13 +1,47 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule StreamTest do use ExUnit.Case, async: true + doctest Stream + + defmodule Pdict do + defstruct [] + + defimpl Collectable do + def into(struct) do + fun = fn + _, {:cont, x} -> Process.put(:stream_cont, [x | Process.get(:stream_cont)]) + _, :done -> Process.put(:stream_done, true) + _, :halt -> Process.put(:stream_halt, true) + end + + {struct, fun} + end + end + end + + defmodule HaltAcc do + defstruct [:acc] + + defimpl Enumerable do + def count(_lazy), do: {:error, __MODULE__} + + def member?(_lazy, _value), do: {:error, __MODULE__} + + def slice(_lazy), do: {:error, __MODULE__} + + def reduce(lazy, _acc, _fun) do + {:halted, Enum.to_list(lazy.acc)} + end + end + end + test "streams as enumerables" do - stream = Stream.map([1,2,3], &(&1 * 2)) + stream = Stream.map([1, 2, 3], &(&1 * 2)) # Reduce - assert Enum.map(stream, &(&1 + 1)) == [3,5,7] + assert Enum.map(stream, &(&1 + 1)) == [3, 5, 7] # Member assert Enum.member?(stream, 4) refute Enum.member?(stream, 1) @@ -16,203 +50,359 @@ defmodule StreamTest do end test "streams are composable" do - stream = Stream.map([1,2,3], &(&1 * 2)) - assert is_lazy(stream) + stream = Stream.map([1, 2, 3], &(&1 * 2)) + assert lazy?(stream) stream = Stream.map(stream, &(&1 + 1)) - assert is_lazy(stream) - - assert Enum.to_list(stream) == [3,5,7] - end - - test "chunk/2, chunk/3 and chunk/4" do - assert Stream.chunk([1, 2, 3, 4, 5], 2) |> Enum.to_list == - [[1, 2], [3, 4]] - assert Stream.chunk([1, 2, 3, 4, 5], 2, 2, [6]) |> Enum.to_list == - [[1, 2], [3, 4], [5, 6]] - assert Stream.chunk([1, 2, 3, 4, 5, 6], 3, 2) |> Enum.to_list == - [[1, 2, 3], [3, 4, 5]] - assert Stream.chunk([1, 2, 3, 4, 5, 6], 2, 3) |> Enum.to_list == - [[1, 2], [4, 5]] - assert Stream.chunk([1, 2, 3, 4, 5, 6], 3, 2, []) |> Enum.to_list == - [[1, 2, 3], [3, 4, 5], [5, 6]] - assert Stream.chunk([1, 2, 3, 4, 5, 6], 3, 3, []) |> Enum.to_list == - [[1, 2, 3], [4, 5, 6]] - assert Stream.chunk([1, 2, 3, 4, 5], 4, 4, 6..10) |> Enum.to_list == - [[1, 2, 3, 4], [5, 6, 7, 8]] - end - - test "chunk/4 is zippable" do - stream = Stream.chunk([1, 2, 3, 4, 5, 6], 3, 2, []) - list = Enum.to_list(stream) + assert lazy?(stream) + + assert Enum.to_list(stream) == [3, 5, 7] + end + + test "chunk_every/2, chunk_every/3 and chunk_every/4" do + assert Stream.chunk_every([1, 2, 3, 4, 5], 2) |> Enum.to_list() == [[1, 2], [3, 4], [5]] + + assert Stream.chunk_every([1, 2, 3, 4, 5], 2, 2, [6]) |> Enum.to_list() == + [[1, 2], [3, 4], [5, 6]] + + assert Stream.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, :discard) |> Enum.to_list() == + [[1, 2, 3], [3, 4, 5]] + + assert Stream.chunk_every([1, 2, 3, 4, 5, 6], 2, 3, :discard) |> Enum.to_list() == + [[1, 2], [4, 5]] + + assert Stream.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, []) |> Enum.to_list() == + [[1, 2, 3], [3, 4, 5], [5, 6]] + + assert Stream.chunk_every([1, 2, 3, 4, 5, 6], 3, 3, []) |> Enum.to_list() == + [[1, 2, 3], [4, 5, 6]] + + assert Stream.chunk_every([1, 2, 3, 4, 5], 4, 4, 6..10) |> Enum.to_list() == + [[1, 2, 3, 4], [5, 6, 7, 8]] + end + + test "chunk_every/4 is zippable" do + stream = Stream.chunk_every([1, 2, 3, 4, 5, 6], 3, 2, []) + list = Enum.to_list(stream) assert Enum.zip(list, list) == Enum.zip(stream, stream) end + test "chunk_every/4 is haltable" do + assert 1..10 |> Stream.take(6) |> Stream.chunk_every(4, 4, [7, 8]) |> Enum.to_list() == + [[1, 2, 3, 4], [5, 6, 7, 8]] + + assert 1..10 + |> Stream.take(6) + |> Stream.chunk_every(4, 4, [7, 8]) + |> Stream.take(3) + |> Enum.to_list() == [[1, 2, 3, 4], [5, 6, 7, 8]] + + assert 1..10 + |> Stream.take(6) + |> Stream.chunk_every(4, 4, [7, 8]) + |> Stream.take(2) + |> Enum.to_list() == [[1, 2, 3, 4], [5, 6, 7, 8]] + + assert 1..10 + |> Stream.take(6) + |> Stream.chunk_every(4, 4, [7, 8]) + |> Stream.take(1) + |> Enum.to_list() == [[1, 2, 3, 4]] + + assert 1..6 |> Stream.take(6) |> Stream.chunk_every(4, 4, [7, 8]) |> Enum.to_list() == + [[1, 2, 3, 4], [5, 6, 7, 8]] + end + test "chunk_by/2" do stream = Stream.chunk_by([1, 2, 2, 3, 4, 4, 6, 7, 7], &(rem(&1, 2) == 1)) - assert is_lazy(stream) - assert Enum.to_list(stream) == - [[1], [2, 2], [3], [4, 4, 6], [7, 7]] - assert stream |> Stream.take(3) |> Enum.to_list == - [[1], [2, 2], [3]] + assert lazy?(stream) + assert Enum.to_list(stream) == [[1], [2, 2], [3], [4, 4, 6], [7, 7]] + assert stream |> Stream.take(3) |> Enum.to_list() == [[1], [2, 2], [3]] + assert 1..10 |> Stream.chunk_every(2) |> Enum.take(2) == [[1, 2], [3, 4]] end test "chunk_by/2 is zippable" do stream = Stream.chunk_by([1, 2, 2, 3], &(rem(&1, 2) == 1)) - list = Enum.to_list(stream) + list = Enum.to_list(stream) assert Enum.zip(list, list) == Enum.zip(stream, stream) end + test "chunk_while/4" do + chunk_fun = fn i, acc -> + cond do + i > 10 -> {:halt, acc} + rem(i, 2) == 0 -> {:cont, Enum.reverse([i | acc]), []} + true -> {:cont, [i | acc]} + end + end + + after_fun = fn + [] -> {:cont, []} + acc -> {:cont, Enum.reverse(acc), []} + end + + assert Stream.chunk_while([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [], chunk_fun, after_fun) + |> Enum.to_list() == [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + assert Stream.chunk_while(0..9, [], chunk_fun, after_fun) |> Enum.to_list() == + [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9]] + + assert Stream.chunk_while(0..10, [], chunk_fun, after_fun) |> Enum.to_list() == + [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + assert Stream.chunk_while(0..11, [], chunk_fun, after_fun) |> Enum.to_list() == + [[0], [1, 2], [3, 4], [5, 6], [7, 8], [9, 10]] + + assert Stream.chunk_while([5, 7, 9, 11], [], chunk_fun, after_fun) |> Enum.to_list() == + [[5, 7, 9]] + end + + test "chunk_while/4 with inner halt" do + chunk_fun = fn + i, [] -> + {:cont, [i]} + + i, chunk -> + if rem(i, 2) == 0 do + {:cont, Enum.reverse(chunk), [i]} + else + {:cont, [i | chunk]} + end + end + + after_fun = fn + [] -> {:cont, []} + chunk -> {:cont, Enum.reverse(chunk), []} + end + + assert Stream.chunk_while([1, 2, 3, 4, 5], [], chunk_fun, after_fun) |> Enum.at(0) == [1] + end + test "concat/1" do stream = Stream.concat([1..3, [], [4, 5, 6], [], 7..9]) assert is_function(stream) - assert Enum.to_list(stream) == [1,2,3,4,5,6,7,8,9] - assert Enum.take(stream, 5) == [1,2,3,4,5] + assert Enum.to_list(stream) == [1, 2, 3, 4, 5, 6, 7, 8, 9] + assert Enum.take(stream, 5) == [1, 2, 3, 4, 5] stream = Stream.concat([1..3, [4, 5, 6], Stream.cycle(7..100)]) assert is_function(stream) - assert Enum.take(stream, 13) == [1,2,3,4,5,6,7,8,9,10,11,12,13] + assert Enum.take(stream, 13) == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] end test "concat/2" do stream = Stream.concat(1..3, 4..6) assert is_function(stream) - assert Stream.cycle(stream) |> Enum.take(16) == [1,2,3,4,5,6,1,2,3,4,5,6,1,2,3,4] + + assert Stream.cycle(stream) |> Enum.take(16) == + [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4] stream = Stream.concat(1..3, []) assert is_function(stream) - assert Stream.cycle(stream) |> Enum.take(5) == [1,2,3,1,2] + assert Stream.cycle(stream) |> Enum.take(5) == [1, 2, 3, 1, 2] stream = Stream.concat(1..6, Stream.cycle(7..9)) assert is_function(stream) - assert Stream.drop(stream, 3) |> Enum.take(13) == [4,5,6,7,8,9,7,8,9,7,8,9,7] + assert Stream.drop(stream, 3) |> Enum.take(13) == [4, 5, 6, 7, 8, 9, 7, 8, 9, 7, 8, 9, 7] stream = Stream.concat(Stream.cycle(1..3), Stream.cycle(4..6)) assert is_function(stream) - assert Enum.take(stream, 13) == [1,2,3,1,2,3,1,2,3,1,2,3,1] + assert Enum.take(stream, 13) == [1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1] + end + + test "concat/2 is zippable" do + stream = 1..2 |> Stream.take(2) |> Stream.concat(3..4) + assert Enum.zip(1..4, [1, 2, 3, 4]) == Enum.zip(1..4, stream) end test "concat/2 does not intercept wrapped lazy enumeration" do # concat returns a lazy enumeration that does not halt assert Stream.concat([[0], Stream.map([1, 2, 3], & &1), [4]]) |> Stream.take_while(fn x -> x <= 4 end) - |> Enum.to_list == [0, 1, 2, 3, 4] + |> Enum.to_list() == [0, 1, 2, 3, 4] # concat returns a lazy enumeration that does halts assert Stream.concat([[0], Stream.take_while(1..6, &(&1 <= 3)), [4]]) |> Stream.take_while(fn x -> x <= 4 end) - |> Enum.to_list == [0, 1, 2, 3, 4] + |> Enum.to_list() == [0, 1, 2, 3, 4] end test "cycle/1" do - stream = Stream.cycle([1,2,3]) + stream = Stream.cycle([1, 2, 3]) assert is_function(stream) - assert Stream.cycle([1,2,3]) |> Stream.take(5) |> Enum.to_list == [1,2,3,1,2] - assert Enum.take(stream, 5) == [1,2,3,1,2] + assert_raise ArgumentError, "cannot cycle over an empty enumerable", fn -> + Stream.cycle([]) + end + + assert_raise ArgumentError, "cannot cycle over an empty enumerable", fn -> + Stream.cycle(%{}) |> Enum.to_list() + end + + assert Stream.cycle([1, 2, 3]) |> Stream.take(5) |> Enum.to_list() == [1, 2, 3, 1, 2] + assert Enum.take(stream, 5) == [1, 2, 3, 1, 2] end test "cycle/1 is zippable" do - stream = Stream.cycle([1,2,3]) - assert Enum.zip(1..6, [1,2,3,1,2,3]) == Enum.zip(1..6, stream) + stream = Stream.cycle([1, 2, 3]) + assert Enum.zip(1..6, [1, 2, 3, 1, 2, 3]) == Enum.zip(1..6, stream) end test "cycle/1 with inner stream" do - assert [1,2,3] |> Stream.take(2) |> Stream.cycle |> Enum.take(4) == - [1,2,1,2] + assert [1, 2, 3] |> Stream.take(2) |> Stream.cycle() |> Enum.take(4) == [1, 2, 1, 2] + end + + test "cycle/1 with cycle/1 with cycle/1" do + assert [1] |> Stream.cycle() |> Stream.cycle() |> Stream.cycle() |> Enum.take(5) == + [1, 1, 1, 1, 1] + end + + test "dedup/1 is lazy" do + assert lazy?(Stream.dedup([1, 2, 3])) + end + + test "dedup/1" do + assert Stream.dedup([1, 1, 2, 1, 1, 2, 1]) |> Enum.to_list() == [1, 2, 1, 2, 1] + assert Stream.dedup([2, 1, 1, 2, 1]) |> Enum.to_list() == [2, 1, 2, 1] + assert Stream.dedup([1, 2, 3, 4]) |> Enum.to_list() == [1, 2, 3, 4] + assert Stream.dedup([1, 1.0, 2.0, 2]) |> Enum.to_list() == [1, 1.0, 2.0, 2] + assert Stream.dedup([]) |> Enum.to_list() == [] + + assert Stream.dedup([nil, nil, true, {:value, true}]) |> Enum.to_list() == + [nil, true, {:value, true}] + + assert Stream.dedup([nil]) |> Enum.to_list() == [nil] + end + + test "dedup_by/2" do + assert Stream.dedup_by([{1, :x}, {2, :y}, {2, :z}, {1, :x}], fn {x, _} -> x end) + |> Enum.to_list() == [{1, :x}, {2, :y}, {1, :x}] end test "drop/2" do stream = Stream.drop(1..10, 5) - assert is_lazy(stream) - assert Enum.to_list(stream) == [6,7,8,9,10] + assert lazy?(stream) + assert Enum.to_list(stream) == [6, 7, 8, 9, 10] - assert Enum.to_list(Stream.drop(1..5, 0)) == [1,2,3,4,5] + assert Enum.to_list(Stream.drop(1..5, 0)) == [1, 2, 3, 4, 5] assert Enum.to_list(Stream.drop(1..3, 5)) == [] nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.drop(nats, 2) |> Enum.take(5) == [3,4,5,6,7] + assert Stream.drop(nats, 2) |> Enum.take(5) == [3, 4, 5, 6, 7] end test "drop/2 with negative count" do stream = Stream.drop(1..10, -5) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,2,3,4,5] + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 2, 3, 4, 5] stream = Stream.drop(1..10, -5) - list = Enum.to_list(stream) + list = Enum.to_list(stream) assert Enum.zip(list, list) == Enum.zip(stream, stream) end test "drop/2 with negative count stream entries" do - par = self - pid = spawn_link fn -> - Enum.each Stream.drop(&inbox_stream/2, -3), - fn x -> send par, {:stream, x} end - end + par = self() + + pid = + spawn_link(fn -> + Enum.each(Stream.drop(&inbox_stream/2, -3), fn x -> send(par, {:stream, x}) end) + end) - send pid, {:stream, 1} - send pid, {:stream, 2} - send pid, {:stream, 3} + send(pid, {:stream, 1}) + send(pid, {:stream, 2}) + send(pid, {:stream, 3}) refute_receive {:stream, 1} - send pid, {:stream, 4} + send(pid, {:stream, 4}) assert_receive {:stream, 1} - send pid, {:stream, 5} + send(pid, {:stream, 5}) assert_receive {:stream, 2} refute_receive {:stream, 3} end + test "drop_every/2" do + assert 1..10 + |> Stream.drop_every(2) + |> Enum.to_list() == [2, 4, 6, 8, 10] + + assert 1..10 + |> Stream.drop_every(3) + |> Enum.to_list() == [2, 3, 5, 6, 8, 9] + + assert 1..10 + |> Stream.drop(2) + |> Stream.drop_every(2) + |> Stream.drop(1) + |> Enum.to_list() == [6, 8, 10] + + assert 1..10 + |> Stream.drop_every(0) + |> Enum.to_list() == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + assert [] + |> Stream.drop_every(10) + |> Enum.to_list() == [] + end + + test "drop_every/2 without non-negative integer" do + assert_raise FunctionClauseError, fn -> + Stream.drop_every(1..10, -1) + end + + assert_raise FunctionClauseError, fn -> + Stream.drop_every(1..10, 3.33) + end + end + test "drop_while/2" do stream = Stream.drop_while(1..10, &(&1 <= 5)) - assert is_lazy(stream) - assert Enum.to_list(stream) == [6,7,8,9,10] + assert lazy?(stream) + assert Enum.to_list(stream) == [6, 7, 8, 9, 10] - assert Enum.to_list(Stream.drop_while(1..5, &(&1 <= 0))) == [1,2,3,4,5] + assert Enum.to_list(Stream.drop_while(1..5, &(&1 <= 0))) == [1, 2, 3, 4, 5] assert Enum.to_list(Stream.drop_while(1..3, &(&1 <= 5))) == [] nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.drop_while(nats, &(&1 <= 5)) |> Enum.take(5) == [6,7,8,9,10] + assert Stream.drop_while(nats, &(&1 <= 5)) |> Enum.take(5) == [6, 7, 8, 9, 10] + end + + test "duplicate/2" do + stream = Stream.duplicate(7, 7) + + assert is_function(stream) + assert stream |> Stream.take(5) |> Enum.to_list() == [7, 7, 7, 7, 7] + assert Enum.to_list(stream) == [7, 7, 7, 7, 7, 7, 7] end test "each/2" do Process.put(:stream_each, []) - stream = Stream.each([1,2,3], fn x -> - Process.put(:stream_each, [x|Process.get(:stream_each)]) - end) + stream = + Stream.each([1, 2, 3], fn x -> + Process.put(:stream_each, [x | Process.get(:stream_each)]) + end) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,2,3] - assert Process.get(:stream_each) == [3,2,1] + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 2, 3] + assert Process.get(:stream_each) == [3, 2, 1] end test "filter/2" do - stream = Stream.filter([1,2,3], fn(x) -> rem(x, 2) == 0 end) - assert is_lazy(stream) + stream = Stream.filter([1, 2, 3], fn x -> rem(x, 2) == 0 end) + assert lazy?(stream) assert Enum.to_list(stream) == [2] nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.filter(nats, &(rem(&1, 2) == 0)) |> Enum.take(5) == [2,4,6,8,10] - end - - test "filter_map/3" do - stream = Stream.filter_map([1,2,3], fn(x) -> rem(x, 2) == 0 end, &(&1 * 2)) - assert is_lazy(stream) - assert Enum.to_list(stream) == [4] - - nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.filter_map(nats, &(rem(&1, 2) == 0), &(&1 * 2)) - |> Enum.take(5) == [4,8,12,16,20] + assert Stream.filter(nats, &(rem(&1, 2) == 0)) |> Enum.take(5) == [2, 4, 6, 8, 10] end test "flat_map/2" do stream = Stream.flat_map([1, 2, 3], &[&1, &1 * 2]) - assert is_lazy(stream) + assert lazy?(stream) assert Enum.to_list(stream) == [1, 2, 2, 4, 3, 6] nats = Stream.iterate(1, &(&1 + 1)) @@ -222,59 +412,67 @@ defmodule StreamTest do test "flat_map/2 does not intercept wrapped lazy enumeration" do # flat_map returns a lazy enumeration that does not halt assert [1, 2, 3, -1, -2] - |> Stream.flat_map(fn x -> Stream.map([x, x+1], & &1) end) + |> Stream.flat_map(fn x -> Stream.map([x, x + 1], & &1) end) |> Stream.take_while(fn x -> x >= 0 end) - |> Enum.to_list == [1, 2, 2, 3, 3, 4] + |> Enum.to_list() == [1, 2, 2, 3, 3, 4] # flat_map returns a lazy enumeration that does halts assert [1, 2, 3, -1, -2] - |> Stream.flat_map(fn x -> Stream.take_while([x, x+1, x+2], &(&1 <= x + 1)) end) + |> Stream.flat_map(fn x -> Stream.take_while([x, x + 1, x + 2], &(&1 <= x + 1)) end) |> Stream.take_while(fn x -> x >= 0 end) - |> Enum.to_list == [1, 2, 2, 3, 3, 4] + |> Enum.to_list() == [1, 2, 2, 3, 3, 4] # flat_map returns a lazy enumeration that does halts wrapped in an enumerable assert [1, 2, 3, -1, -2] - |> Stream.flat_map(fn x -> Stream.concat([x], Stream.take_while([x+1, x+2], &(&1 <= x + 1))) end) + |> Stream.flat_map(fn x -> + Stream.concat([x], Stream.take_while([x + 1, x + 2], &(&1 <= x + 1))) + end) |> Stream.take_while(fn x -> x >= 0 end) - |> Enum.to_list == [1, 2, 2, 3, 3, 4] + |> Enum.to_list() == [1, 2, 2, 3, 3, 4] end test "flat_map/2 is zippable" do - stream = [1, 2, 3, -1, -2] - |> Stream.flat_map(fn x -> Stream.map([x, x+1], & &1) end) - |> Stream.take_while(fn x -> x >= 0 end) - list = Enum.to_list(stream) + stream = + [1, 2, 3, -1, -2] + |> Stream.flat_map(fn x -> Stream.map([x, x + 1], & &1) end) + |> Stream.take_while(fn x -> x >= 0 end) + + list = Enum.to_list(stream) assert Enum.zip(list, list) == Enum.zip(stream, stream) end test "flat_map/2 does not leave inner stream suspended" do - stream = Stream.flat_map [1,2,3], - fn i -> - Stream.resource(fn -> i end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_flat_map, true) end) - end + stream = + Stream.flat_map([1, 2, 3], fn i -> + Stream.resource(fn -> i end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_flat_map, true) + end) + end) Process.put(:stream_flat_map, false) - assert stream |> Enum.take(3) == [1,2,3] + assert stream |> Enum.take(3) == [1, 2, 3] assert Process.get(:stream_flat_map) end test "flat_map/2 does not leave outer stream suspended" do - stream = Stream.resource(fn -> 1 end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_flat_map, true) end) + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_flat_map, true) + end) + stream = Stream.flat_map(stream, fn i -> [i, i + 1, i + 2] end) Process.put(:stream_flat_map, false) - assert stream |> Enum.take(3) == [1,2,3] + assert stream |> Enum.take(3) == [1, 2, 3] assert Process.get(:stream_flat_map) end test "flat_map/2 closes on error" do - stream = Stream.resource(fn -> 1 end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_flat_map, true) end) + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_flat_map, true) + end) + stream = Stream.flat_map(stream, fn _ -> throw(:error) end) Process.put(:stream_flat_map, false) @@ -282,20 +480,58 @@ defmodule StreamTest do assert Process.get(:stream_flat_map) end + test "flat_map/2 with inner flat_map/2" do + stream = + Stream.flat_map(1..5, fn x -> + Stream.flat_map([x], fn x -> + x..(x * x) + end) + |> Stream.map(&(&1 * 1)) + end) + + assert Enum.take(stream, 5) == [1, 2, 3, 4, 3] + end + + test "flat_map/2 properly halts both inner and outer stream when inner stream is halted" do + # Fixes a bug that, when the inner stream was done, + # sending it a halt would cause it to return the + # inner stream was halted, forcing flat_map to get + # the next value from the outer stream, evaluate it, + # get another inner stream, just to halt it. + # 2 should never be used + assert [1, 2] + |> Stream.flat_map(fn 1 -> Stream.repeatedly(fn -> 1 end) end) + |> Stream.flat_map(fn 1 -> Stream.repeatedly(fn -> 1 end) end) + |> Enum.take(1) == [1] + end + + test "interval/1" do + stream = Stream.interval(10) + {time_us, value} = :timer.tc(fn -> Enum.take(stream, 5) end) + + assert value == [0, 1, 2, 3, 4] + assert time_us >= 50000 + end + + test "interval/1 with infinity" do + stream = Stream.interval(:infinity) + spawn(Stream, :run, [stream]) + end + test "into/2 and run/1" do Process.put(:stream_cont, []) Process.put(:stream_done, false) Process.put(:stream_halt, false) - stream = Stream.into([1, 2, 3], collectable_pdict) + stream = Stream.into([1, 2, 3], %Pdict{}) - assert is_lazy(stream) + assert lazy?(stream) assert Stream.run(stream) == :ok - assert Process.get(:stream_cont) == [3,2,1] + assert Process.get(:stream_cont) == [3, 2, 1] assert Process.get(:stream_done) refute Process.get(:stream_halt) - stream = Stream.into(fn _, _ -> raise "error" end, collectable_pdict) + stream = Stream.into(fn _, _ -> raise "error" end, %Pdict{}) catch_error(Stream.run(stream)) assert Process.get(:stream_halt) end @@ -305,9 +541,9 @@ defmodule StreamTest do Process.put(:stream_done, false) Process.put(:stream_halt, false) - stream = Stream.into([1, 2, 3], collectable_pdict, fn x -> x*2 end) + stream = Stream.into([1, 2, 3], %Pdict{}, fn x -> x * 2 end) - assert is_lazy(stream) + assert lazy?(stream) assert Enum.to_list(stream) == [1, 2, 3] assert Process.get(:stream_cont) == [6, 4, 2] assert Process.get(:stream_done) @@ -319,9 +555,9 @@ defmodule StreamTest do Process.put(:stream_done, false) Process.put(:stream_halt, false) - stream = Stream.into([1, 2, 3], collectable_pdict) + stream = Stream.into([1, 2, 3], %Pdict{}) - assert is_lazy(stream) + assert lazy?(stream) assert Enum.take(stream, 1) == [1] assert Process.get(:stream_cont) == [1] assert Process.get(:stream_done) @@ -330,29 +566,60 @@ defmodule StreamTest do test "transform/3" do stream = Stream.transform([1, 2, 3], 0, &{[&1, &2], &1 + &2}) - assert is_lazy(stream) + assert lazy?(stream) assert Enum.to_list(stream) == [1, 0, 2, 1, 3, 3] nats = Stream.iterate(1, &(&1 + 1)) assert Stream.transform(nats, 0, &{[&1, &2], &1 + &2}) |> Enum.take(6) == [1, 0, 2, 1, 3, 3] end + test "transform/3 with early halt" do + stream = + fn -> throw(:error) end + |> Stream.repeatedly() + |> Stream.transform(nil, &{[&1, &2], &1}) + + assert {:halted, nil} = Enumerable.reduce(stream, {:halt, nil}, fn _, _ -> throw(:error) end) + end + + test "transform/3 with early suspend" do + stream = + Stream.repeatedly(fn -> throw(:error) end) + |> Stream.transform(nil, &{[&1, &2], &1}) + + assert {:suspended, nil, _} = + Enumerable.reduce(stream, {:suspend, nil}, fn _, _ -> throw(:error) end) + end + test "transform/3 with halt" do - stream = Stream.resource(fn -> 1 end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_transform, true) end) - stream = Stream.transform(stream, 0, fn i, acc -> if acc < 3, do: {[i], acc + 1}, else: {:halt, acc} end) + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_transform, true) + end) + + stream = + Stream.transform(stream, 0, fn i, acc -> + if acc < 3, do: {[i], acc + 1}, else: {:halt, acc} + end) Process.put(:stream_transform, false) - assert Enum.to_list(stream) == [1,2,3] + assert Enum.to_list(stream) == [1, 2, 3] assert Process.get(:stream_transform) end + test "transform/3 (via flat_map) handles multiple returns from suspension" do + assert [false] + |> Stream.take(1) + |> Stream.concat([true]) + |> Stream.flat_map(&[&1]) + |> Enum.to_list() == [false, true] + end + test "iterate/2" do - stream = Stream.iterate(0, &(&1+2)) - assert Enum.take(stream, 5) == [0,2,4,6,8] - stream = Stream.iterate(5, &(&1+2)) - assert Enum.take(stream, 5) == [5,7,9,11,13] + stream = Stream.iterate(0, &(&1 + 2)) + assert Enum.take(stream, 5) == [0, 2, 4, 6, 8] + stream = Stream.iterate(5, &(&1 + 2)) + assert Enum.take(stream, 5) == [5, 7, 9, 11, 13] # Only calculate values if needed stream = Stream.iterate("HELLO", &raise/1) @@ -360,49 +627,173 @@ defmodule StreamTest do end test "map/2" do - stream = Stream.map([1,2,3], &(&1 * 2)) - assert is_lazy(stream) - assert Enum.to_list(stream) == [2,4,6] + stream = Stream.map([1, 2, 3], &(&1 * 2)) + assert lazy?(stream) + assert Enum.to_list(stream) == [2, 4, 6] nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.map(nats, &(&1 * 2)) |> Enum.take(5) == [2,4,6,8,10] + assert Stream.map(nats, &(&1 * 2)) |> Enum.take(5) == [2, 4, 6, 8, 10] assert Stream.map(nats, &(&1 - 2)) |> Stream.map(&(&1 * 2)) |> Enum.take(3) == [-2, 0, 2] end + test "map_every/3" do + assert 1..10 + |> Stream.map_every(2, &(&1 * 2)) + |> Enum.to_list() == [2, 2, 6, 4, 10, 6, 14, 8, 18, 10] + + assert 1..10 + |> Stream.map_every(3, &(&1 * 2)) + |> Enum.to_list() == [2, 2, 3, 8, 5, 6, 14, 8, 9, 20] + + assert 1..10 + |> Stream.drop(2) + |> Stream.map_every(2, &(&1 * 2)) + |> Stream.drop(1) + |> Enum.to_list() == [4, 10, 6, 14, 8, 18, 10] + + assert 1..5 + |> Stream.map_every(0, &(&1 * 2)) + |> Enum.to_list() == [1, 2, 3, 4, 5] + + assert [] + |> Stream.map_every(10, &(&1 * 2)) + |> Enum.to_list() == [] + + assert_raise FunctionClauseError, fn -> + Stream.map_every(1..10, -1, &(&1 * 2)) + end + + assert_raise FunctionClauseError, fn -> + Stream.map_every(1..10, 3.33, &(&1 * 2)) + end + end + test "reject/2" do - stream = Stream.reject([1,2,3], fn(x) -> rem(x, 2) == 0 end) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,3] + stream = Stream.reject([1, 2, 3], fn x -> rem(x, 2) == 0 end) + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 3] nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.reject(nats, &(rem(&1, 2) == 0)) |> Enum.take(5) == [1,3,5,7,9] + assert Stream.reject(nats, &(rem(&1, 2) == 0)) |> Enum.take(5) == [1, 3, 5, 7, 9] end test "repeatedly/1" do stream = Stream.repeatedly(fn -> 1 end) - assert Enum.take(stream, 5) == [1,1,1,1,1] - stream = Stream.repeatedly(&:random.uniform/0) - [r1,r2] = Enum.take(stream, 2) + assert Enum.take(stream, 5) == [1, 1, 1, 1, 1] + stream = Stream.repeatedly(&:rand.uniform/0) + [r1, r2] = Enum.take(stream, 2) assert r1 != r2 end - test "resource/3 closes on errors" do - stream = Stream.resource(fn -> 1 end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_resource, true) end) + test "resource/3 closes on outer errors" do + stream = + Stream.resource( + fn -> 1 end, + fn + 2 -> throw(:error) + acc -> {[acc], acc + 1} + end, + fn 2 -> Process.put(:stream_resource, true) end + ) Process.put(:stream_resource, false) - stream = Stream.map(stream, fn x -> if x > 2, do: throw(:error), else: x end) assert catch_throw(Enum.to_list(stream)) == :error assert Process.get(:stream_resource) end + test "resource/3 closes with correct accumulator on outer errors with inner single-element list" do + stream = + Stream.resource( + fn -> :start end, + fn _ -> {[:error], :end} end, + fn acc -> Process.put(:stream_resource, acc) end + ) + |> Stream.map(fn :error -> throw(:error) end) + + Process.put(:stream_resource, nil) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_resource) == :end + end + + test "resource/3 closes with correct accumulator on outer errors with inner list" do + stream = + Stream.resource( + fn -> :start end, + fn _ -> {[:ok, :error], :end} end, + fn acc -> Process.put(:stream_resource, acc) end + ) + |> Stream.map(fn acc -> if acc == :error, do: throw(:error), else: acc end) + + Process.put(:stream_resource, nil) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_resource) == :end + end + + test "resource/3 closes with correct accumulator on outer errors with inner enum" do + stream = + Stream.resource( + fn -> 1 end, + fn acc -> {acc..(acc + 2), acc + 1} end, + fn acc -> Process.put(:stream_resource, acc) end + ) + |> Stream.map(fn x -> if x > 2, do: throw(:error), else: x end) + + Process.put(:stream_resource, nil) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_resource) == 2 + end + test "resource/3 is zippable" do - stream = Stream.resource(fn -> 1 end, - fn 10 -> nil - acc -> {acc, acc + 1} - end, - fn _ -> Process.put(:stream_resource, true) end) + transform_fun = fn + 10 -> {:halt, 10} + acc -> {[acc], acc + 1} + end + + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) + + list = Enum.to_list(stream) + Process.put(:stream_resource, false) + assert Enum.zip(list, list) == Enum.zip(stream, stream) + assert Process.get(:stream_resource) + end + + test "resource/3 returning inner empty list" do + transform_fun = fn acc -> if rem(acc, 2) == 0, do: {[], acc + 1}, else: {[acc], acc + 1} end + stream = Stream.resource(fn -> 1 end, transform_fun, fn _ -> :ok end) + + assert Enum.take(stream, 5) == [1, 3, 5, 7, 9] + end + + test "resource/3 halts with inner list" do + transform_fun = fn acc -> {[acc, acc + 1, acc + 2], acc + 1} end + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) + + Process.put(:stream_resource, false) + assert Enum.take(stream, 5) == [1, 2, 3, 2, 3] + assert Process.get(:stream_resource) + end + + test "resource/3 closes on errors with inner list" do + transform_fun = fn acc -> {[acc, acc + 1, acc + 2], acc + 1} end + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) + + Process.put(:stream_resource, false) + stream = Stream.map(stream, fn x -> if x > 2, do: throw(:error), else: x end) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_resource) + end + + test "resource/3 is zippable with inner list" do + transform_fun = fn + 10 -> {:halt, 10} + acc -> {[acc, acc + 1, acc + 2], acc + 1} + end + + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) list = Enum.to_list(stream) Process.put(:stream_resource, false) @@ -410,159 +801,591 @@ defmodule StreamTest do assert Process.get(:stream_resource) end + test "resource/3 halts with inner enum" do + transform_fun = fn acc -> {acc..(acc + 2), acc + 1} end + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) + + Process.put(:stream_resource, false) + assert Enum.take(stream, 5) == [1, 2, 3, 2, 3] + assert Process.get(:stream_resource) + end + + test "resource/3 closes on errors with inner enum" do + transform_fun = fn acc -> {acc..(acc + 2), acc + 1} end + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) + + Process.put(:stream_resource, false) + stream = Stream.map(stream, fn x -> if x > 2, do: throw(:error), else: x end) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_resource) + end + + test "resource/3 is zippable with inner enum" do + transform_fun = fn + 10 -> {:halt, 10} + acc -> {acc..(acc + 2), acc + 1} + end + + after_fun = fn _ -> Process.put(:stream_resource, true) end + stream = Stream.resource(fn -> 1 end, transform_fun, after_fun) + + list = Enum.to_list(stream) + Process.put(:stream_resource, false) + assert Enum.zip(list, list) == Enum.zip(stream, stream) + assert Process.get(:stream_resource) + end + + test "transform/4" do + transform_fun = fn x, acc -> {[x, x + acc], x} end + after_fun = fn 10 -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..10, fn -> 0 end, transform_fun, after_fun) + Process.put(:stream_transform, false) + + assert Enum.to_list(stream) == + [1, 1, 2, 3, 3, 5, 4, 7, 5, 9, 6, 11, 7, 13, 8, 15, 9, 17, 10, 19] + + assert Process.get(:stream_transform) + end + + test "transform/4 with early halt" do + after_fun = fn nil -> Process.put(:stream_transform, true) end + + stream = + fn -> throw(:error) end + |> Stream.repeatedly() + |> Stream.transform(fn -> nil end, &{[&1, &2], &1}, after_fun) + + Process.put(:stream_transform, false) + assert {:halted, nil} = Enumerable.reduce(stream, {:halt, nil}, fn _, _ -> throw(:error) end) + assert Process.get(:stream_transform) + end + + test "transform/4 with early suspend" do + after_fun = fn nil -> Process.put(:stream_transform, true) end + + stream = + fn -> throw(:error) end + |> Stream.repeatedly() + |> Stream.transform(fn -> nil end, &{[&1, &2], &1}, after_fun) + + refute Process.get(:stream_transform) + + assert {:suspended, nil, _} = + Enumerable.reduce(stream, {:suspend, nil}, fn _, _ -> throw(:error) end) + end + + test "transform/4 closes on outer errors" do + transform_fun = fn + 3, _ -> throw(:error) + x, acc -> {[x + acc], x} + end + + after_fun = fn 2 -> Process.put(:stream_transform, true) end + + stream = Stream.transform(1..10, fn -> 0 end, transform_fun, after_fun) + + Process.put(:stream_transform, false) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_transform) + end + + test "transform/4 closes on nested errors" do + transform_fun = fn + 3, _ -> throw(:error) + x, acc -> {[x + acc], x} + end + + after_fun = fn _ -> Process.put(:stream_transform_inner, true) end + outer_after_fun = fn 0 -> Process.put(:stream_transform_outer, true) end + + stream = + 1..10 + |> Stream.transform(fn -> 0 end, transform_fun, after_fun) + |> Stream.transform(fn -> 0 end, fn x, acc -> {[x], acc} end, outer_after_fun) + + Process.put(:stream_transform_inner, false) + Process.put(:stream_transform_outer, false) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_transform_inner) + assert Process.get(:stream_transform_outer) + end + + test "transform/4 is zippable" do + transform_fun = fn + 10, acc -> {:halt, acc} + x, acc -> {[x + acc], x} + end + + after_fun = fn 9 -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..20, fn -> 0 end, transform_fun, after_fun) + + list = Enum.to_list(stream) + Process.put(:stream_transform, false) + assert Enum.zip(list, list) == Enum.zip(stream, stream) + assert Process.get(:stream_transform) + end + + test "transform/4 halts with inner list" do + transform_fun = fn x, acc -> {[x, x + 1, x + 2], acc} end + after_fun = fn :acc -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..10, fn -> :acc end, transform_fun, after_fun) + + Process.put(:stream_transform, false) + assert Enum.take(stream, 5) == [1, 2, 3, 2, 3] + assert Process.get(:stream_transform) + end + + test "transform/4 closes on errors with inner list" do + transform_fun = fn x, acc -> {[x, x + 1, x + 2], acc} end + after_fun = fn :acc -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..10, fn -> :acc end, transform_fun, after_fun) + + Process.put(:stream_transform, false) + stream = Stream.map(stream, fn x -> if x > 2, do: throw(:error), else: x end) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_transform) + end + + test "transform/4 is zippable with inner list" do + transform_fun = fn + 10, acc -> {:halt, acc} + x, acc -> {[x, x + 1, x + 2], acc} + end + + after_fun = fn :inner -> Process.put(:stream_transform, true) end + + stream = Stream.transform(1..20, fn -> :inner end, transform_fun, after_fun) + + list = Enum.to_list(stream) + Process.put(:stream_transform, false) + assert Enum.zip(list, list) == Enum.zip(stream, stream) + assert Process.get(:stream_transform) + end + + test "transform/4 halts with inner enum" do + transform_fun = fn x, acc -> {x..(x + 2), acc} end + after_fun = fn :acc -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..10, fn -> :acc end, transform_fun, after_fun) + + Process.put(:stream_transform, false) + assert Enum.take(stream, 5) == [1, 2, 3, 2, 3] + assert Process.get(:stream_transform) + end + + test "transform/4 closes on errors with inner enum" do + transform_fun = fn x, acc -> {x..(x + 2), acc} end + after_fun = fn :acc -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..10, fn -> :acc end, transform_fun, after_fun) + + Process.put(:stream_transform, false) + stream = Stream.map(stream, fn x -> if x > 2, do: throw(:error), else: x end) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_transform) + end + + test "transform/4 is zippable with inner enum" do + transform_fun = fn + 10, acc -> {:halt, acc} + x, acc -> {x..(x + 2), acc} + end + + after_fun = fn :inner -> Process.put(:stream_transform, true) end + stream = Stream.transform(1..20, fn -> :inner end, transform_fun, after_fun) + + list = Enum.to_list(stream) + Process.put(:stream_transform, false) + assert Enum.zip(list, list) == Enum.zip(stream, stream) + assert Process.get(:stream_transform) + end + + test "transform/5 emits last elements on done" do + stream = + Stream.transform( + 1..5//2, + fn -> 0 end, + fn i, _acc -> {i..(i + 1), i + 1} end, + fn 6 -> {7..10, 10} end, + fn i when is_integer(i) -> Process.put(__MODULE__, i) end + ) + + assert Enum.to_list(stream) == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + assert Process.get(__MODULE__) == 10 + + assert Enum.take(stream, 3) == [1, 2, 3] + assert Process.get(__MODULE__) == 4 + + assert Enum.take(stream, 4) == [1, 2, 3, 4] + assert Process.get(__MODULE__) == 4 + + assert Enum.take(stream, 7) == [1, 2, 3, 4, 5, 6, 7] + assert Process.get(__MODULE__) == 10 + end + + test "transform/5 emits last elements on inner halt done" do + stream = + Stream.transform( + Stream.take(1..15//2, 3), + fn -> 0 end, + fn i, _acc -> {i..(i + 1), i + 1} end, + fn 6 -> {7..10, 10} end, + fn i when is_integer(i) -> Process.put(__MODULE__, i) end + ) + + assert Enum.to_list(stream) == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + assert Process.get(__MODULE__) == 10 + + assert Enum.take(stream, 3) == [1, 2, 3] + assert Process.get(__MODULE__) == 4 + + assert Enum.take(stream, 4) == [1, 2, 3, 4] + assert Process.get(__MODULE__) == 4 + + assert Enum.take(stream, 7) == [1, 2, 3, 4, 5, 6, 7] + assert Process.get(__MODULE__) == 10 + end + test "scan/2" do stream = Stream.scan(1..5, &(&1 + &2)) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,3,6,10,15] - assert Stream.scan([], &(&1 + &2)) |> Enum.to_list == [] + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 3, 6, 10, 15] + assert Stream.scan([], &(&1 + &2)) |> Enum.to_list() == [] end test "scan/3" do stream = Stream.scan(1..5, 0, &(&1 + &2)) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,3,6,10,15] - assert Stream.scan([], 0, &(&1 + &2)) |> Enum.to_list == [] + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 3, 6, 10, 15] + assert Stream.scan([], 0, &(&1 + &2)) |> Enum.to_list() == [] end test "take/2" do stream = Stream.take(1..1000, 5) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,2,3,4,5] + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 2, 3, 4, 5] assert Enum.to_list(Stream.take(1..1000, 0)) == [] - assert Enum.to_list(Stream.take(1..3, 5)) == [1,2,3] + assert Enum.to_list(Stream.take([], 5)) == [] + assert Enum.to_list(Stream.take(1..3, 5)) == [1, 2, 3] nats = Stream.iterate(1, &(&1 + 1)) - assert Enum.to_list(Stream.take(nats, 5)) == [1,2,3,4,5] + assert Enum.to_list(Stream.take(nats, 5)) == [1, 2, 3, 4, 5] stream = Stream.drop(1..100, 5) - assert Stream.take(stream, 5) |> Enum.to_list == [6,7,8,9,10] + assert Stream.take(stream, 5) |> Enum.to_list() == [6, 7, 8, 9, 10] stream = 1..5 |> Stream.take(10) |> Stream.drop(15) assert {[], []} = Enum.split(stream, 5) stream = 1..20 |> Stream.take(10 + 5) |> Stream.drop(4) - assert Enum.to_list(stream) == [5,6,7,8,9,10,11,12,13,14,15] + assert Enum.to_list(stream) == [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + end + + test "take/2 does not consume next element on halt" do + assert [false, true] + |> Stream.each(&(&1 && raise("oops"))) + |> Stream.take(1) + |> Stream.take_while(& &1) + |> Enum.to_list() == [] + end + + test "take/2 does not consume next element on suspend" do + assert [false, true] + |> Stream.each(&(&1 && raise("oops"))) + |> Stream.take(1) + |> Stream.flat_map(&[&1]) + |> Enum.to_list() == [false] end test "take/2 with negative count" do Process.put(:stream_each, []) stream = Stream.take(1..100, -5) - assert is_lazy(stream) + assert lazy?(stream) - stream = Stream.each(stream, &Process.put(:stream_each, [&1|Process.get(:stream_each)])) - assert Enum.to_list(stream) == [96,97,98,99,100] - assert Process.get(:stream_each) == [100,99,98,97,96] + stream = Stream.each(stream, &Process.put(:stream_each, [&1 | Process.get(:stream_each)])) + assert Enum.to_list(stream) == [96, 97, 98, 99, 100] + assert Process.get(:stream_each) == [100, 99, 98, 97, 96] end test "take/2 is zippable" do stream = Stream.take(1..1000, 5) - list = Enum.to_list(stream) + list = Enum.to_list(stream) assert Enum.zip(list, list) == Enum.zip(stream, stream) end test "take_every/2" do assert 1..10 |> Stream.take_every(2) - |> Enum.to_list == [1, 3, 5, 7, 9] + |> Enum.to_list() == [1, 3, 5, 7, 9] + + assert 1..10 + |> Stream.take_every(3) + |> Enum.to_list() == [1, 4, 7, 10] assert 1..10 |> Stream.drop(2) |> Stream.take_every(2) |> Stream.drop(1) - |> Enum.to_list == [5, 7, 9] + |> Enum.to_list() == [5, 7, 9] + + assert 1..10 + |> Stream.take_every(0) + |> Enum.to_list() == [] + + assert [] + |> Stream.take_every(10) + |> Enum.to_list() == [] + end + + test "take_every/2 without non-negative integer" do + assert_raise FunctionClauseError, fn -> + Stream.take_every(1..10, -1) + end + + assert_raise FunctionClauseError, fn -> + Stream.take_every(1..10, 3.33) + end end test "take_while/2" do stream = Stream.take_while(1..1000, &(&1 <= 5)) - assert is_lazy(stream) - assert Enum.to_list(stream) == [1,2,3,4,5] + assert lazy?(stream) + assert Enum.to_list(stream) == [1, 2, 3, 4, 5] assert Enum.to_list(Stream.take_while(1..1000, &(&1 <= 0))) == [] - assert Enum.to_list(Stream.take_while(1..3, &(&1 <= 5))) == [1,2,3] + assert Enum.to_list(Stream.take_while(1..3, &(&1 <= 5))) == [1, 2, 3] nats = Stream.iterate(1, &(&1 + 1)) - assert Enum.to_list(Stream.take_while(nats, &(&1 <= 5))) == [1,2,3,4,5] + assert Enum.to_list(Stream.take_while(nats, &(&1 <= 5))) == [1, 2, 3, 4, 5] stream = Stream.drop(1..100, 5) - assert Stream.take_while(stream, &(&1 < 11)) |> Enum.to_list == [6,7,8,9,10] + assert Stream.take_while(stream, &(&1 < 11)) |> Enum.to_list() == [6, 7, 8, 9, 10] + end + + test "timer/1" do + stream = Stream.timer(10) + + {time_us, value} = :timer.tc(fn -> Enum.to_list(stream) end) + + assert value == [0] + # We check for >= 5000 (us) instead of >= 10000 (us) + # because the resolution on Windows system is not high + # enough and we would get a difference of 9000 from + # time to time. So a value halfway is good enough. + assert time_us >= 5000 + end + + test "timer/1 with infinity" do + stream = Stream.timer(:infinity) + spawn(Stream, :run, [stream]) end test "unfold/2" do - stream = Stream.unfold(10, fn x -> if x > 0, do: {x, x-1}, else: nil end) + stream = Stream.unfold(10, fn x -> if x > 0, do: {x, x - 1} end) assert Enum.take(stream, 5) == [10, 9, 8, 7, 6] - stream = Stream.unfold(5, fn x -> if x > 0, do: {x, x-1}, else: nil end) + stream = Stream.unfold(5, fn x -> if x > 0, do: {x, x - 1} end) assert Enum.to_list(stream) == [5, 4, 3, 2, 1] end test "unfold/2 only calculates values if needed" do - stream = Stream.unfold(1, fn x -> if x > 0, do: {x, x-1}, else: throw(:boom) end) + stream = Stream.unfold(1, fn x -> if x > 0, do: {x, x - 1}, else: throw(:boom) end) assert Enum.take(stream, 1) == [1] - stream = Stream.unfold(5, fn x -> if x > 0, do: {x, x-1}, else: nil end) + stream = Stream.unfold(5, fn x -> if x > 0, do: {x, x - 1} end) assert Enum.to_list(Stream.take(stream, 2)) == [5, 4] end test "unfold/2 is zippable" do - stream = Stream.unfold(10, fn x -> if x > 0, do: {x, x-1}, else: nil end) - list = Enum.to_list(stream) + stream = Stream.unfold(10, fn x -> if x > 0, do: {x, x - 1} end) + list = Enum.to_list(stream) assert Enum.zip(list, list) == Enum.zip(stream, stream) end - test "uniq/1" do - assert Stream.uniq([1, 2, 3, 2, 1]) |> Enum.to_list == - [1, 2, 3] + test "uniq/1 & uniq/2" do + assert Stream.uniq([1, 2, 3, 2, 1]) |> Enum.to_list() == [1, 2, 3] + end + + test "uniq_by/2" do + assert Stream.uniq_by([{1, :x}, {2, :y}, {1, :z}], fn {x, _} -> x end) |> Enum.to_list() == + [{1, :x}, {2, :y}] - assert Stream.uniq([{1, :x}, {2, :y}, {1, :z}], fn {x, _} -> x end) |> Enum.to_list == - [{1,:x}, {2,:y}] + assert Stream.uniq_by([a: {:tea, 2}, b: {:tea, 2}, c: {:coffee, 1}], fn {_, y} -> y end) + |> Enum.to_list() == [a: {:tea, 2}, c: {:coffee, 1}] end test "zip/2" do concat = Stream.concat(1..3, 4..6) - cycle = Stream.cycle([:a, :b, :c]) - assert Stream.zip(concat, cycle) |> Enum.to_list == - [{1,:a},{2,:b},{3,:c},{4,:a},{5,:b},{6,:c}] + cycle = Stream.cycle([:a, :b, :c]) + + assert Stream.zip(concat, cycle) |> Enum.to_list() == + [{1, :a}, {2, :b}, {3, :c}, {4, :a}, {5, :b}, {6, :c}] + end + + test "zip_with/3" do + concat = Stream.concat(1..3, 4..6) + cycle = Stream.cycle([:a, :b, :c]) + zip_fun = &List.to_tuple([&1, &2]) + + assert Stream.zip_with(concat, cycle, zip_fun) |> Enum.to_list() == + [{1, :a}, {2, :b}, {3, :c}, {4, :a}, {5, :b}, {6, :c}] + + stream = Stream.concat(1..3, 4..6) + other_stream = fn _, _ -> {:cont, [1, 2]} end + result = Stream.zip_with(stream, other_stream, fn a, b -> a + b end) |> Enum.to_list() + assert result == [2, 4] + end + + test "zip_with/2" do + concat = Stream.concat(1..3, 4..6) + cycle = Stream.cycle([:a, :b, :c]) + zip_fun = &List.to_tuple/1 + + assert Stream.zip_with([concat, cycle], zip_fun) |> Enum.to_list() == + [{1, :a}, {2, :b}, {3, :c}, {4, :a}, {5, :b}, {6, :c}] + + assert Stream.chunk_every([0, 1, 2, 3], 2) |> Stream.zip_with(zip_fun) |> Enum.to_list() == + [{0, 2}, {1, 3}] + + stream = %HaltAcc{acc: 1..3} + assert Stream.zip_with([1..3, stream], zip_fun) |> Enum.to_list() == [{1, 1}, {2, 2}, {3, 3}] + + range_cycle = Stream.cycle(1..2) + + assert Stream.zip_with([1..3, range_cycle], zip_fun) |> Enum.to_list() == [ + {1, 1}, + {2, 2}, + {3, 1} + ] + end + + test "zip_with/2 does not leave streams suspended" do + zip_with_fun = &List.to_tuple/1 + + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_zip_with, true) + end) + + Process.put(:stream_zip_with, false) + + assert Stream.zip_with([[:a, :b, :c], stream], zip_with_fun) |> Enum.to_list() == [ + a: 1, + b: 2, + c: 3 + ] + + assert Process.get(:stream_zip_with) + + Process.put(:stream_zip_with, false) + + assert Stream.zip_with([stream, [:a, :b, :c]], zip_with_fun) |> Enum.to_list() == [ + {1, :a}, + {2, :b}, + {3, :c} + ] + + assert Process.get(:stream_zip_with) + end + + test "zip_with/2 does not leave streams suspended on halt" do + zip_with_fun = &List.to_tuple/1 + + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_zip_with, :done) + end) + + assert Stream.zip_with([[:a, :b, :c, :d, :e], stream], zip_with_fun) |> Enum.take(3) == [ + a: 1, + b: 2, + c: 3 + ] + + assert Process.get(:stream_zip_with) == :done + end + + test "zip_with/2 closes on inner error" do + zip_with_fun = &List.to_tuple/1 + stream = Stream.into([1, 2, 3], %Pdict{}) + + stream = + Stream.zip_with([stream, Stream.map([:a, :b, :c], fn _ -> throw(:error) end)], zip_with_fun) + + Process.put(:stream_done, false) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_done) + end + + test "zip_with/2 closes on outer error" do + zip_with_fun = &List.to_tuple/1 + + stream = + Stream.zip_with([Stream.into([1, 2, 3], %Pdict{}), [:a, :b, :c]], zip_with_fun) + |> Stream.map(fn _ -> throw(:error) end) + + Process.put(:stream_done, false) + assert catch_throw(Enum.to_list(stream)) == :error + assert Process.get(:stream_done) + end + + test "zip/1" do + concat = Stream.concat(1..3, 4..6) + cycle = Stream.cycle([:a, :b, :c]) + + assert Stream.zip([concat, cycle]) |> Enum.to_list() == + [{1, :a}, {2, :b}, {3, :c}, {4, :a}, {5, :b}, {6, :c}] + + assert Stream.chunk_every([0, 1, 2, 3], 2) |> Stream.zip() |> Enum.to_list() == + [{0, 2}, {1, 3}] + + stream = %HaltAcc{acc: 1..3} + assert Stream.zip([1..3, stream]) |> Enum.to_list() == [{1, 1}, {2, 2}, {3, 3}] + + range_cycle = Stream.cycle(1..2) + assert Stream.zip([1..3, range_cycle]) |> Enum.to_list() == [{1, 1}, {2, 2}, {3, 1}] end - test "zip/2 does not leave streams suspended" do - stream = Stream.resource(fn -> 1 end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_zip, true) end) + test "zip/1 does not leave streams suspended" do + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_zip, true) + end) Process.put(:stream_zip, false) - assert Stream.zip([:a, :b, :c], stream) |> Enum.to_list == [a: 1, b: 2, c: 3] + assert Stream.zip([[:a, :b, :c], stream]) |> Enum.to_list() == [a: 1, b: 2, c: 3] assert Process.get(:stream_zip) Process.put(:stream_zip, false) - assert Stream.zip(stream, [:a, :b, :c]) |> Enum.to_list == [{1, :a}, {2, :b}, {3, :c}] + assert Stream.zip([stream, [:a, :b, :c]]) |> Enum.to_list() == [{1, :a}, {2, :b}, {3, :c}] assert Process.get(:stream_zip) end - test "zip/2 does not leave streams suspended on halt" do - stream = Stream.resource(fn -> 1 end, - fn acc -> {acc, acc + 1} end, - fn _ -> Process.put(:stream_zip, :done) end) + test "zip/1 does not leave streams suspended on halt" do + stream = + Stream.resource(fn -> 1 end, fn acc -> {[acc], acc + 1} end, fn _ -> + Process.put(:stream_zip, :done) + end) - assert Stream.zip([:a, :b, :c, :d, :e], stream) |> Enum.take(3) == - [a: 1, b: 2, c: 3] + assert Stream.zip([[:a, :b, :c, :d, :e], stream]) |> Enum.take(3) == [a: 1, b: 2, c: 3] assert Process.get(:stream_zip) == :done end - test "zip/2 closes on inner error" do - stream = Stream.into([1, 2, 3], collectable_pdict) - stream = Stream.zip(stream, Stream.map([:a, :b, :c], fn _ -> throw(:error) end)) + test "zip/1 closes on inner error" do + stream = Stream.into([1, 2, 3], %Pdict{}) + stream = Stream.zip([stream, Stream.map([:a, :b, :c], fn _ -> throw(:error) end)]) Process.put(:stream_done, false) assert catch_throw(Enum.to_list(stream)) == :error assert Process.get(:stream_done) end - test "zip/2 closes on outer error" do - stream = Stream.into([1, 2, 3], collectable_pdict) - |> Stream.zip([:a, :b, :c]) - |> Stream.map(fn _ -> throw(:error) end) + test "zip/1 closes on outer error" do + stream = + Stream.zip([Stream.into([1, 2, 3], %Pdict{}), [:a, :b, :c]]) + |> Stream.map(fn _ -> throw(:error) end) Process.put(:stream_done, false) assert catch_throw(Enum.to_list(stream)) == :error @@ -570,24 +1393,41 @@ defmodule StreamTest do end test "with_index/2" do - stream = Stream.with_index([1,2,3]) - assert is_lazy(stream) - assert Enum.to_list(stream) == [{1,0},{2,1},{3,2}] + stream = Stream.with_index([1, 2, 3]) + assert lazy?(stream) + assert Enum.to_list(stream) == [{1, 0}, {2, 1}, {3, 2}] + + stream = Stream.with_index([1, 2, 3], 10) + assert Enum.to_list(stream) == [{1, 10}, {2, 11}, {3, 12}] nats = Stream.iterate(1, &(&1 + 1)) - assert Stream.with_index(nats) |> Enum.take(3) == [{1,0},{2,1},{3,2}] + assert Stream.with_index(nats) |> Enum.take(3) == [{1, 0}, {2, 1}, {3, 2}] end - defp is_lazy(stream) do - match?(%Stream{}, stream) or is_function(stream, 2) + test "intersperse/2 is lazy" do + assert lazy?(Stream.intersperse([], 0)) end - defp collectable_pdict do - fn - _, {:cont, x} -> Process.put(:stream_cont, [x|Process.get(:stream_cont)]) - _, :done -> Process.put(:stream_done, true) - _, :halt -> Process.put(:stream_halt, true) - end + test "intersperse/2 on an empty list" do + assert Enum.to_list(Stream.intersperse([], 0)) == [] + end + + test "intersperse/2 on a single element list" do + assert Enum.to_list(Stream.intersperse([1], 0)) == [1] + end + + test "intersperse/2 on a multiple elements list" do + assert Enum.to_list(Stream.intersperse(1..3, 0)) == [1, 0, 2, 0, 3] + end + + test "intersperse/2 is zippable" do + stream = Stream.intersperse(1..10, 0) + list = Enum.to_list(stream) + assert Enum.zip(list, list) == Enum.zip(stream, stream) + end + + defp lazy?(stream) do + match?(%Stream{}, stream) or is_function(stream, 2) end defp inbox_stream({:suspend, acc}, f) do @@ -600,8 +1440,8 @@ defmodule StreamTest do defp inbox_stream({:cont, acc}, f) do receive do - {:stream, item} -> - inbox_stream(f.(item, acc), f) + {:stream, element} -> + inbox_stream(f.(element, acc), f) end end end diff --git a/lib/elixir/test/elixir/string/chars_test.exs b/lib/elixir/test/elixir/string/chars_test.exs index 173439287bb..02a397a485c 100644 --- a/lib/elixir/test/elixir/string/chars_test.exs +++ b/lib/elixir/test/elixir/string/chars_test.exs @@ -1,28 +1,30 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule String.Chars.AtomTest do use ExUnit.Case, async: true - test :basic do + doctest String.Chars + + test "basic" do assert to_string(:foo) == "foo" end - test :empty do + test "empty" do assert to_string(:"") == "" end - test :true_false_nil do + test "true false nil" do assert to_string(false) == "false" assert to_string(true) == "true" assert to_string(nil) == "" end - test :with_uppercase do + test "with uppercase" do assert to_string(:fOO) == "fOO" assert to_string(:FOO) == "FOO" end - test :alias_atom do + test "alias atom" do assert to_string(Foo.Bar) == "Elixir.Foo.Bar" end end @@ -30,7 +32,7 @@ end defmodule String.Chars.BitStringTest do use ExUnit.Case, async: true - test :binary do + test "binary" do assert to_string("foo") == "foo" assert to_string(<>) == "abc" assert to_string("我今天要学习.") == "我今天要学习." @@ -40,11 +42,11 @@ end defmodule String.Chars.NumberTest do use ExUnit.Case, async: true - test :integer do + test "integer" do assert to_string(100) == "100" end - test :float do + test "float" do assert to_string(1.0) == "1.0" assert to_string(1.0e10) == "1.0e10" end @@ -53,66 +55,110 @@ end defmodule String.Chars.ListTest do use ExUnit.Case, async: true - test :basic do - assert to_string([ 1, "b", 3 ]) == <<1, 98, 3>> + test "basic" do + assert to_string([1, "b", 3]) == <<1, 98, 3>> end - test :printable do + test "printable" do assert to_string('abc') == "abc" end - test :char_list do - assert to_string([0, 1, 2, 3, 255]) == - <<0, 1, 2, 3, 195, 191>> + test "charlist" do + assert to_string([0, 1, 2, 3, 255]) == <<0, 1, 2, 3, 195, 191>> assert to_string([0, [1, "hello"], 2, [["bye"]]]) == - <<0, 1, 104, 101, 108, 108, 111, 2, 98, 121, 101>> + <<0, 1, 104, 101, 108, 108, 111, 2, 98, 121, 101>> end - test :empty do + test "empty" do assert to_string([]) == "" end end +defmodule String.Chars.Version.RequirementTest do + use ExUnit.Case, async: true + + test "version requirement" do + {:ok, requirement} = Version.parse_requirement("== 2.0.1") + assert String.Chars.to_string(requirement) == "== 2.0.1" + end +end + +defmodule String.Chars.URITest do + use ExUnit.Case, async: true + + test "uri" do + uri = URI.parse("http://google.com") + assert String.Chars.to_string(uri) == "http://google.com" + + uri_no_host = URI.parse("/foo/bar") + assert String.Chars.to_string(uri_no_host) == "/foo/bar" + end +end + defmodule String.Chars.ErrorsTest do use ExUnit.Case, async: true - test :bitstring do - assert_raise Protocol.UndefinedError, - "protocol String.Chars not implemented for <<0, 1::size(4)>>, " <> - "cannot convert a bitstring to a string", fn -> - to_string(<<1 :: [size(12), integer, signed]>>) + defmodule Foo do + defstruct foo: "bar" + end + + test "bitstring" do + message = + "protocol String.Chars not implemented for <<0, 1::size(4)>> of type BitString, cannot convert a bitstring to a string" + + assert_raise Protocol.UndefinedError, message, fn -> + to_string(<<1::size(12)-integer-signed>>) end end - test :tuple do - assert_raise Protocol.UndefinedError, "protocol String.Chars not implemented for {1, 2, 3}", fn -> + test "tuple" do + message = "protocol String.Chars not implemented for {1, 2, 3} of type Tuple" + + assert_raise Protocol.UndefinedError, message, fn -> to_string({1, 2, 3}) end end - test :pid do - assert_raise Protocol.UndefinedError, ~r"^protocol String\.Chars not implemented for #PID<.+?>$", fn -> + test "PID" do + message = ~r"^protocol String\.Chars not implemented for #PID<.+?> of type PID$" + + assert_raise Protocol.UndefinedError, message, fn -> to_string(self()) end end - test :ref do - assert_raise Protocol.UndefinedError, ~r"^protocol String\.Chars not implemented for #Reference<.+?>$", fn -> + test "ref" do + message = ~r"^protocol String\.Chars not implemented for #Reference<.+?> of type Reference$" + + assert_raise Protocol.UndefinedError, message, fn -> to_string(make_ref()) == "" end end - test :function do - assert_raise Protocol.UndefinedError, ~r"^protocol String\.Chars not implemented for #Function<.+?>$", fn -> - to_string(fn -> end) + test "function" do + message = ~r"^protocol String\.Chars not implemented for #Function<.+?> of type Function$" + + assert_raise Protocol.UndefinedError, message, fn -> + to_string(fn -> nil end) end end - test :port do - [port|_] = Port.list - assert_raise Protocol.UndefinedError, ~r"^protocol String\.Chars not implemented for #Port<.+?>$", fn -> + test "port" do + [port | _] = Port.list() + message = ~r"^protocol String\.Chars not implemented for #Port<.+?> of type Port$" + + assert_raise Protocol.UndefinedError, message, fn -> to_string(port) end end + + test "user-defined struct" do + message = + "protocol String\.Chars not implemented for %String.Chars.ErrorsTest.Foo{foo: \"bar\"} of type String.Chars.ErrorsTest.Foo (a struct)" + + assert_raise Protocol.UndefinedError, message, fn -> + to_string(%Foo{}) + end + end end diff --git a/lib/elixir/test/elixir/string_io_test.exs b/lib/elixir/test/elixir/string_io_test.exs index da589631d29..cb89e46fc20 100644 --- a/lib/elixir/test/elixir/string_io_test.exs +++ b/lib/elixir/test/elixir/string_io_test.exs @@ -1,233 +1,395 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule StringIOTest do use ExUnit.Case, async: true - test "start and stop" do - {:ok, pid} = StringIO.open("") - assert StringIO.close(pid) == {:ok, {"", ""}} - end + doctest StringIO - test "start_link and stop" do + test "open and close" do {:ok, pid} = StringIO.open("") assert StringIO.close(pid) == {:ok, {"", ""}} end - test "peek" do + test "contents" do {:ok, pid} = StringIO.open("abc") IO.write(pid, "edf") assert StringIO.contents(pid) == {"abc", "edf"} end - ## IO module - - def start(string, opts \\ []) do - StringIO.open(string, opts) |> elem(1) + test "flush" do + {:ok, pid} = StringIO.open("") + IO.write(pid, "edf") + assert StringIO.flush(pid) == "edf" + assert StringIO.contents(pid) == {"", ""} end - def contents(pid) do - StringIO.contents(pid) - end + ## IO module test "IO.read :line with \\n" do - pid = start("abc\n") + {:ok, pid} = StringIO.open("abc\n") assert IO.read(pid, :line) == "abc\n" assert IO.read(pid, :line) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end test "IO.read :line with \\rn" do - pid = start("abc\r\n") + {:ok, pid} = StringIO.open("abc\r\n") assert IO.read(pid, :line) == "abc\n" assert IO.read(pid, :line) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end test "IO.read :line without line break" do - pid = start("abc") + {:ok, pid} = StringIO.open("abc") assert IO.read(pid, :line) == "abc" assert IO.read(pid, :line) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end - test "IO.read :line with invalid utf8" do - pid = start(<< 130, 227, 129, 132, 227, 129, 134 >>) + test "IO.read :line with UTF-8" do + {:ok, pid} = StringIO.open("⼊\n") + assert IO.read(pid, :line) == "⼊\n" + assert IO.read(pid, :line) == :eof + assert StringIO.contents(pid) == {"", ""} + end + + test "IO.read :line with invalid UTF-8" do + {:ok, pid} = StringIO.open(<<130, 227, 129, 132, 227, 129, 134>>) assert IO.read(pid, :line) == {:error, :collect_line} - assert contents(pid) == {<< 130, 227, 129, 132, 227, 129, 134 >>, ""} + assert StringIO.contents(pid) == {<<130, 227, 129, 132, 227, 129, 134>>, ""} end test "IO.read count" do - pid = start("abc") + {:ok, pid} = StringIO.open("abc") assert IO.read(pid, 2) == "ab" assert IO.read(pid, 8) == "c" assert IO.read(pid, 1) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end - test "IO.read count with utf8" do - pid = start("あいう") + test "IO.read count with UTF-8" do + {:ok, pid} = StringIO.open("あいう") assert IO.read(pid, 2) == "あい" assert IO.read(pid, 8) == "う" assert IO.read(pid, 1) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end - test "IO.read count with invalid utf8" do - pid = start(<< 130, 227, 129, 132, 227, 129, 134 >>) + test "IO.read count with invalid UTF-8" do + {:ok, pid} = StringIO.open(<<130, 227, 129, 132, 227, 129, 134>>) assert IO.read(pid, 2) == {:error, :invalid_unicode} - assert contents(pid) == {<< 130, 227, 129, 132, 227, 129, 134 >>, ""} + assert StringIO.contents(pid) == {<<130, 227, 129, 132, 227, 129, 134>>, ""} end test "IO.binread :line with \\n" do - pid = start("abc\n") + {:ok, pid} = StringIO.open("abc\n") assert IO.binread(pid, :line) == "abc\n" assert IO.binread(pid, :line) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end test "IO.binread :line with \\r\\n" do - pid = start("abc\r\n") + {:ok, pid} = StringIO.open("abc\r\n") assert IO.binread(pid, :line) == "abc\n" assert IO.binread(pid, :line) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end test "IO.binread :line without line break" do - pid = start("abc") + {:ok, pid} = StringIO.open("abc") assert IO.binread(pid, :line) == "abc" assert IO.binread(pid, :line) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} + end + + test "IO.binread :line with raw bytes" do + {:ok, pid} = StringIO.open(<<181, 255, 194, ?\n>>) + assert IO.binread(pid, :line) == <<181, 255, 194, ?\n>> + assert IO.binread(pid, :line) == :eof + assert StringIO.contents(pid) == {"", ""} end test "IO.binread count" do - pid = start("abc") + {:ok, pid} = StringIO.open("abc") assert IO.binread(pid, 2) == "ab" assert IO.binread(pid, 8) == "c" assert IO.binread(pid, 1) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end - test "IO.binread count with utf8" do - pid = start("あいう") - assert IO.binread(pid, 2) == << 227, 129 >> - assert IO.binread(pid, 8) == << 130, 227, 129, 132, 227, 129, 134 >> + test "IO.binread count with UTF-8" do + {:ok, pid} = StringIO.open("あいう") + assert IO.binread(pid, 2) == <<227, 129>> + assert IO.binread(pid, 8) == <<130, 227, 129, 132, 227, 129, 134>> assert IO.binread(pid, 1) == :eof - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end test "IO.write" do - pid = start("") + {:ok, pid} = StringIO.open("") assert IO.write(pid, "foo") == :ok - assert contents(pid) == {"", "foo"} + assert StringIO.contents(pid) == {"", "foo"} end - test "IO.write with utf8" do - pid = start("") + test "IO.write with UTF-8" do + {:ok, pid} = StringIO.open("") assert IO.write(pid, "あいう") == :ok - assert contents(pid) == {"", "あいう"} + assert StringIO.contents(pid) == {"", "あいう"} + end + + test "IO.write with non-printable arguments" do + {:ok, pid} = StringIO.open("") + + assert_raise ArgumentError, fn -> + IO.write(pid, [<<1::1>>]) + end + + assert_raise ErlangError, ~r/no_translation/, fn -> + IO.write(pid, <<222>>) + end end test "IO.binwrite" do - pid = start("") + {:ok, pid} = StringIO.open("") assert IO.binwrite(pid, "foo") == :ok - assert contents(pid) == {"", "foo"} + assert StringIO.contents(pid) == {"", "foo"} end - test "IO.binwrite with utf8" do - pid = start("") + test "IO.binwrite with UTF-8" do + {:ok, pid} = StringIO.open("") assert IO.binwrite(pid, "あいう") == :ok - assert contents(pid) == {"", "あいう"} + + binary = + <<195, 163, 194, 129, 194, 130, 195, 163>> <> + <<194, 129, 194, 132, 195, 163, 194, 129, 194, 134>> + + assert StringIO.contents(pid) == {"", binary} + end + + test "IO.binwrite with bytes" do + {:ok, pid} = StringIO.open("") + assert IO.binwrite(pid, <<127, 128>>) == :ok + assert StringIO.contents(pid) == {"", <<127, 194, 128>>} + end + + test "IO.binwrite with bytes and latin1 encoding" do + {:ok, pid} = StringIO.open("", encoding: :latin1) + assert IO.binwrite(pid, <<127, 128>>) == :ok + assert StringIO.contents(pid) == {"", <<127, 128>>} end test "IO.puts" do - pid = start("") + {:ok, pid} = StringIO.open("") assert IO.puts(pid, "abc") == :ok - assert contents(pid) == {"", "abc\n"} + assert StringIO.contents(pid) == {"", "abc\n"} + end + + test "IO.puts with non-printable arguments" do + {:ok, pid} = StringIO.open("") + + assert_raise ArgumentError, fn -> + IO.puts(pid, [<<1::1>>]) + end end test "IO.inspect" do - pid = start("") + {:ok, pid} = StringIO.open("") assert IO.inspect(pid, {}, []) == {} - assert contents(pid) == {"", "{}\n"} + assert StringIO.contents(pid) == {"", "{}\n"} end test "IO.getn" do - pid = start("abc") + {:ok, pid} = StringIO.open("abc") assert IO.getn(pid, ">", 2) == "ab" - assert contents(pid) == {"c", ""} + assert StringIO.contents(pid) == {"c", ""} end - test "IO.getn with utf8" do - pid = start("あいう") + test "IO.getn with UTF-8" do + {:ok, pid} = StringIO.open("あいう") assert IO.getn(pid, ">", 2) == "あい" - assert contents(pid) == {"う", ""} + assert StringIO.contents(pid) == {"う", ""} end - test "IO.getn with invalid utf8" do - pid = start(<< 130, 227, 129, 132, 227, 129, 134 >>) + test "IO.getn with invalid UTF-8" do + {:ok, pid} = StringIO.open(<<130, 227, 129, 132, 227, 129, 134>>) assert IO.getn(pid, ">", 2) == {:error, :invalid_unicode} - assert contents(pid) == {<< 130, 227, 129, 132, 227, 129, 134 >>, ""} + assert StringIO.contents(pid) == {<<130, 227, 129, 132, 227, 129, 134>>, ""} end test "IO.getn with capture_prompt" do - pid = start("abc", capture_prompt: true) + {:ok, pid} = StringIO.open("abc", capture_prompt: true) assert IO.getn(pid, ">", 2) == "ab" - assert contents(pid) == {"c", ">"} + assert StringIO.contents(pid) == {"c", ">"} end test "IO.gets with \\n" do - pid = start("abc\nd") + {:ok, pid} = StringIO.open("abc\nd") assert IO.gets(pid, ">") == "abc\n" - assert contents(pid) == {"d", ""} + assert StringIO.contents(pid) == {"d", ""} end test "IO.gets with \\r\\n" do - pid = start("abc\r\nd") + {:ok, pid} = StringIO.open("abc\r\nd") assert IO.gets(pid, ">") == "abc\n" - assert contents(pid) == {"d", ""} + assert StringIO.contents(pid) == {"d", ""} end test "IO.gets without line breaks" do - pid = start("abc") + {:ok, pid} = StringIO.open("abc") assert IO.gets(pid, ">") == "abc" - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end - test "IO.gets with invalid utf8" do - pid = start(<< 130, 227, 129, 132, 227, 129, 134 >>) + test "IO.gets with invalid UTF-8" do + {:ok, pid} = StringIO.open(<<130, 227, 129, 132, 227, 129, 134>>) assert IO.gets(pid, ">") == {:error, :collect_line} - assert contents(pid) == {<< 130, 227, 129, 132, 227, 129, 134 >>, ""} + assert StringIO.contents(pid) == {<<130, 227, 129, 132, 227, 129, 134>>, ""} end test "IO.gets with capture_prompt" do - pid = start("abc\n", capture_prompt: true) + {:ok, pid} = StringIO.open("abc\n", capture_prompt: true) assert IO.gets(pid, ">") == "abc\n" - assert contents(pid) == {"", ">"} + assert StringIO.contents(pid) == {"", ">"} end test ":io.get_password" do - pid = start("abc\n") + {:ok, pid} = StringIO.open("abc\n") assert :io.get_password(pid) == "abc\n" - assert contents(pid) == {"", ""} + assert StringIO.contents(pid) == {"", ""} end test "IO.stream" do - pid = start("abc") - assert IO.stream(pid, 2) |> Enum.to_list == ["ab", "c"] - assert contents(pid) == {"", ""} + {:ok, pid} = StringIO.open("abc") + assert IO.stream(pid, 2) |> Enum.to_list() == ["ab", "c"] + assert StringIO.contents(pid) == {"", ""} end - test "IO.stream with invalid utf8" do - pid = start(<< 130, 227, 129, 132, 227, 129, 134 >>) - assert_raise IO.StreamError, fn-> - IO.stream(pid, 2) |> Enum.to_list + test "IO.stream with invalid UTF-8" do + {:ok, pid} = StringIO.open(<<130, 227, 129, 132, 227, 129, 134>>) + + assert_raise IO.StreamError, fn -> + IO.stream(pid, 2) |> Enum.to_list() end - assert contents(pid) == {<< 130, 227, 129, 132, 227, 129, 134 >>, ""} + + assert StringIO.contents(pid) == {<<130, 227, 129, 132, 227, 129, 134>>, ""} end test "IO.binstream" do - pid = start("abc") - assert IO.stream(pid, 2) |> Enum.to_list == ["ab", "c"] - assert contents(pid) == {"", ""} + {:ok, pid} = StringIO.open("abc") + assert IO.stream(pid, 2) |> Enum.to_list() == ["ab", "c"] + assert StringIO.contents(pid) == {"", ""} + end + + defp get_until(pid, encoding, prompt, module, function) do + :io.request(pid, {:get_until, encoding, prompt, module, function, []}) + end + + defmodule GetUntilCallbacks do + def until_eof(continuation, :eof) do + {:done, continuation, :eof} + end + + def until_eof(continuation, content) do + {:more, continuation ++ content} + end + + def until_eof_then_try_more('magic-stop-prefix' ++ continuation, :eof) do + {:done, continuation, :eof} + end + + def until_eof_then_try_more(continuation, :eof) do + {:more, 'magic-stop-prefix' ++ continuation} + end + + def until_eof_then_try_more(continuation, content) do + {:more, continuation ++ content} + end + + def up_to_3_bytes(continuation, :eof) do + {:done, continuation, :eof} + end + + def up_to_3_bytes(continuation, content) do + case continuation ++ content do + [a, b, c | tail] -> {:done, [a, b, c], tail} + str -> {:more, str} + end + end + + def up_to_3_bytes_discard_rest(continuation, :eof) do + {:done, continuation, :eof} + end + + def up_to_3_bytes_discard_rest(continuation, content) do + case continuation ++ content do + [a, b, c | _tail] -> {:done, [a, b, c], :eof} + str -> {:more, str} + end + end + end + + test "get_until with up_to_3_bytes" do + {:ok, pid} = StringIO.open("abcdefg") + result = get_until(pid, :unicode, "", GetUntilCallbacks, :up_to_3_bytes) + assert result == "abc" + assert IO.read(pid, :all) == "defg" + end + + test "get_until with up_to_3_bytes_discard_rest" do + {:ok, pid} = StringIO.open("abcdefg") + result = get_until(pid, :unicode, "", GetUntilCallbacks, :up_to_3_bytes_discard_rest) + assert result == "abc" + assert IO.read(pid, :all) == "" + end + + test "get_until with until_eof" do + {:ok, pid} = StringIO.open("abc\nd") + result = get_until(pid, :unicode, "", GetUntilCallbacks, :until_eof) + assert result == "abc\nd" + end + + test "get_until with until_eof and \\r\\n" do + {:ok, pid} = StringIO.open("abc\r\nd") + result = get_until(pid, :unicode, "", GetUntilCallbacks, :until_eof) + assert result == "abc\r\nd" + end + + test "get_until with until_eof capturing prompt" do + {:ok, pid} = StringIO.open("abc\nd", capture_prompt: true) + result = get_until(pid, :unicode, ">", GetUntilCallbacks, :until_eof) + assert result == "abc\nd" + assert StringIO.contents(pid) == {"", ">>>"} + end + + test "get_until with until_eof_then_try_more" do + {:ok, pid} = StringIO.open("abc\nd") + result = get_until(pid, :unicode, "", GetUntilCallbacks, :until_eof_then_try_more) + assert result == "abc\nd" + end + + test "get_until with invalid UTF-8" do + {:ok, pid} = StringIO.open(<<130, 227, 129, 132, 227, 129, 134>>) + result = get_until(pid, :unicode, "", GetUntilCallbacks, :until_eof) + assert result == :error + end + + test "get_until with raw bytes (latin1)" do + {:ok, pid} = StringIO.open(<<181, 255, 194, ?\n>>) + result = get_until(pid, :latin1, "", GetUntilCallbacks, :until_eof) + assert result == <<181, 255, 194, ?\n>> + end + + test ":io.erl_scan_form/2" do + {:ok, pid} = StringIO.open("1.") + result = :io.scan_erl_form(pid, 'p>') + assert result == {:ok, [{:integer, 1, 1}, {:dot, 1}], 1} + assert StringIO.contents(pid) == {"", ""} + end + + test ":io.erl_scan_form/2 with capture_prompt" do + {:ok, pid} = StringIO.open("1.", capture_prompt: true) + result = :io.scan_erl_form(pid, 'p>') + assert result == {:ok, [{:integer, 1, 1}, {:dot, 1}], 1} + assert StringIO.contents(pid) == {"", "p>p>"} end end diff --git a/lib/elixir/test/elixir/string_test.exs b/lib/elixir/test/elixir/string_test.exs index f6f38e7624d..ff437c9c54c 100644 --- a/lib/elixir/test/elixir/string_test.exs +++ b/lib/elixir/test/elixir/string_test.exs @@ -1,22 +1,18 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule StringTest do use ExUnit.Case, async: true - test :integer_codepoints do - assert ?é == 233 - assert ?\xE9 == 233 - assert ?\351 == 233 - end + doctest String - test :next_codepoint do + test "next_codepoint/1" do assert String.next_codepoint("ésoj") == {"é", "soj"} assert String.next_codepoint(<<255>>) == {<<255>>, ""} assert String.next_codepoint("") == nil end - # test cases described in http://mortoray.com/2013/11/27/the-string-type-is-broken/ - test :unicode do + # test cases described in https://mortoray.com/2013/11/27/the-string-type-is-broken/ + test "Unicode" do assert String.reverse("noël") == "lëon" assert String.slice("noël", 0..2) == "noë" assert String.length("noël") == 4 @@ -26,41 +22,75 @@ defmodule StringTest do assert String.reverse("") == "" assert String.upcase("baffle") == "BAFFLE" + + assert String.equivalent?("noël", "noël") end - test :split do - assert String.split("") == [""] + test "split/1,2,3" do + assert String.split("") == [] assert String.split("foo bar") == ["foo", "bar"] assert String.split(" foo bar") == ["foo", "bar"] assert String.split("foo bar ") == ["foo", "bar"] assert String.split(" foo bar ") == ["foo", "bar"] assert String.split("foo\t\n\v\f\r\sbar\n") == ["foo", "bar"] - assert String.split("foo" <> <<31>> <> "bar") == ["foo", "bar"] assert String.split("foo" <> <<194, 133>> <> "bar") == ["foo", "bar"] + # information separators are not considered whitespace + assert String.split("foo\u001Fbar") == ["foo\u001Fbar"] + # no-break space is excluded + assert String.split("foo\00A0bar") == ["foo\00A0bar"] + assert String.split("foo\u202Fbar") == ["foo\u202Fbar"] - assert String.split("", ",") == [""] assert String.split("a,b,c", ",") == ["a", "b", "c"] assert String.split("a,b", ".") == ["a,b"] assert String.split("1,2 3,4", [" ", ","]) == ["1", "2", "3", "4"] + + assert String.split("", ",") == [""] assert String.split(" a b c ", " ") == ["", "a", "b", "c", ""] + assert String.split(" a b c ", " ", parts: :infinity) == ["", "a", "b", "c", ""] + assert String.split(" a b c ", " ", parts: 1) == [" a b c "] + assert String.split(" a b c ", " ", parts: 2) == ["", "a b c "] + assert String.split("", ",", trim: true) == [] assert String.split(" a b c ", " ", trim: true) == ["a", "b", "c"] - assert String.split(" a b c ", " ", trim: true, parts: 0) == ["a", "b", "c"] assert String.split(" a b c ", " ", trim: true, parts: :infinity) == ["a", "b", "c"] assert String.split(" a b c ", " ", trim: true, parts: 1) == [" a b c "] + assert String.split(" a b c ", " ", trim: true, parts: 2) == ["a", "b c "] - assert String.split("abé", "") == ["a", "b", "é", ""] - assert String.split("abé", "", parts: 0) == ["a", "b", "é", ""] + assert String.split("abé", "") == ["", "a", "b", "é", ""] + assert String.split("abé", "", parts: :infinity) == ["", "a", "b", "é", ""] assert String.split("abé", "", parts: 1) == ["abé"] - assert String.split("abé", "", parts: 2) == ["a", "bé"] - assert String.split("abé", "", parts: 10) == ["a", "b", "é", ""] + assert String.split("abé", "", parts: 2) == ["", "abé"] + assert String.split("abé", "", parts: 3) == ["", "a", "bé"] + assert String.split("abé", "", parts: 4) == ["", "a", "b", "é"] + assert String.split("abé", "", parts: 5) == ["", "a", "b", "é", ""] + assert String.split("abé", "", parts: 10) == ["", "a", "b", "é", ""] assert String.split("abé", "", trim: true) == ["a", "b", "é"] - assert String.split("abé", "", trim: true, parts: 0) == ["a", "b", "é"] + assert String.split("abé", "", trim: true, parts: :infinity) == ["a", "b", "é"] assert String.split("abé", "", trim: true, parts: 2) == ["a", "bé"] + assert String.split("abé", "", trim: true, parts: 3) == ["a", "b", "é"] + assert String.split("abé", "", trim: true, parts: 4) == ["a", "b", "é"] + + assert String.split("noël", "") == ["", "n", "o", "ë", "l", ""] + assert String.split("x-", "-", parts: 2, trim: true) == ["x"] + assert String.split("x-x-", "-", parts: 3, trim: true) == ["x", "x"] + + assert String.split("hello", []) == ["hello"] + assert String.split("hello", [], trim: true) == ["hello"] + assert String.split("", []) == [""] + assert String.split("", [], trim: true) == [] + + assert_raise ArgumentError, fn -> + String.split("a,b,c", [""]) + end + + assert_raise ArgumentError, fn -> + String.split("a,b,c", [""]) + end end - test :split_with_regex do + test "split/2,3 with regex" do assert String.split("", ~r{,}) == [""] + assert String.split("", ~r{,}, trim: true) == [] assert String.split("a,b", ~r{,}) == ["a", "b"] assert String.split("a,b,c", ~r{,}) == ["a", "b", "c"] assert String.split("a,b,c", ~r{,}, parts: 2) == ["a", "b,c"] @@ -69,7 +99,38 @@ defmodule StringTest do assert String.split("a,b", ~r{\.}) == ["a,b"] end - test :split_at do + test "split/2,3 with compiled pattern" do + pattern = :binary.compile_pattern("-") + + assert String.split("x-", pattern) == ["x", ""] + assert String.split("x-", pattern, parts: 2, trim: true) == ["x"] + assert String.split("x-x-", pattern, parts: 3, trim: true) == ["x", "x"] + end + + test "splitter/2,3" do + assert String.splitter("a,b,c", ",") |> Enum.to_list() == ["a", "b", "c"] + assert String.splitter("a,b", ".") |> Enum.to_list() == ["a,b"] + assert String.splitter("1,2 3,4", [" ", ","]) |> Enum.to_list() == ["1", "2", "3", "4"] + assert String.splitter("", ",") |> Enum.to_list() == [""] + + assert String.splitter("", ",", trim: true) |> Enum.to_list() == [] + assert String.splitter(" a b c ", " ", trim: true) |> Enum.to_list() == ["a", "b", "c"] + assert String.splitter(" a b c ", " ", trim: true) |> Enum.take(1) == ["a"] + assert String.splitter(" a b c ", " ", trim: true) |> Enum.take(2) == ["a", "b"] + + assert String.splitter("hello", []) |> Enum.to_list() == ["hello"] + assert String.splitter("hello", [], trim: true) |> Enum.to_list() == ["hello"] + assert String.splitter("", []) |> Enum.to_list() == [""] + assert String.splitter("", [], trim: true) |> Enum.to_list() == [] + + assert String.splitter("1,2 3,4 5", "") |> Enum.take(4) == ["", "1", ",", "2"] + + assert_raise ArgumentError, fn -> + String.splitter("a", [""]) + end + end + + test "split_at/2" do assert String.split_at("", 0) == {"", ""} assert String.split_at("", -1) == {"", ""} assert String.split_at("", 1) == {"", ""} @@ -84,37 +145,91 @@ defmodule StringTest do assert String.split_at("abc", -3) == {"", "abc"} assert String.split_at("abc", -4) == {"", "abc"} assert String.split_at("abc", -1000) == {"", "abc"} + + assert_raise FunctionClauseError, fn -> + String.split_at("abc", 0.1) + end + + assert_raise FunctionClauseError, fn -> + String.split_at("abc", -0.1) + end end - test :upcase do - assert String.upcase("123 abcd 456 efg hij ( %$#) kl mnop @ qrst = -_ uvwxyz") == "123 ABCD 456 EFG HIJ ( %$#) KL MNOP @ QRST = -_ UVWXYZ" + test "split_at/2 with invalid guard" do + assert String.split_at(<>, 2) == {<>, <<10, ?a>>} + assert String.split_at(<<107, 205, 135, 184>>, 1) == {<<107, 205, 135>>, <<184>>} + end + + test "upcase/1" do + assert String.upcase("123 abcd 456 efg hij ( %$#) kl mnop @ qrst = -_ uvwxyz") == + "123 ABCD 456 EFG HIJ ( %$#) KL MNOP @ QRST = -_ UVWXYZ" + assert String.upcase("") == "" assert String.upcase("abcD") == "ABCD" end - test :upcase_utf8 do + test "upcase/1 with UTF-8" do assert String.upcase("& % # àáâ ãäå 1 2 ç æ") == "& % # ÀÁ ÃÄÅ 1 2 Ç Æ" assert String.upcase("àáâãäåæçèéêëìíîïðñòóôõöøùúûüýþ") == "ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ" end - test :upcase_utf8_multibyte do + test "upcase/1 with UTF-8 multibyte" do assert String.upcase("straße") == "STRASSE" assert String.upcase("áüÈß") == "ÁÜÈSS" end - test :downcase do - assert String.downcase("123 ABcD 456 EfG HIJ ( %$#) KL MNOP @ QRST = -_ UVWXYZ") == "123 abcd 456 efg hij ( %$#) kl mnop @ qrst = -_ uvwxyz" + test "upcase/1 with ascii" do + assert String.upcase("olá", :ascii) == "OLá" + end + + test "upcase/1 with turkic" do + assert String.upcase("ıi", :turkic) == "Iİ" + assert String.upcase("Iİ", :turkic) == "Iİ" + end + + test "downcase/1" do + assert String.downcase("123 ABcD 456 EfG HIJ ( %$#) KL MNOP @ QRST = -_ UVWXYZ") == + "123 abcd 456 efg hij ( %$#) kl mnop @ qrst = -_ uvwxyz" + assert String.downcase("abcD") == "abcd" assert String.downcase("") == "" end - test :downcase_utf8 do + test "downcase/1 with UTF-8" do assert String.downcase("& % # ÀÁ ÃÄÅ 1 2 Ç Æ") == "& % # àáâ ãäå 1 2 ç æ" assert String.downcase("ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ") == "àáâãäåæçèéêëìíîïðñòóôõöøùúûüýþ" assert String.downcase("áüÈß") == "áüèß" end - test :capitalize do + test "downcase/1 with greek final sigma" do + assert String.downcase("Σ") == "σ" + assert String.downcase("ΣΣ") == "σσ" + assert String.downcase("Σ ΣΣ") == "σ σσ" + assert String.downcase("ΜΕΣ'ΑΠΟ") == "μεσ'απο" + assert String.downcase("ΑΣ'ΤΟΥΣ") == "ασ'τουσ" + + assert String.downcase("Σ", :greek) == "σ" + assert String.downcase("Σ ΣΣ", :greek) == "σ σς" + assert String.downcase("Σ ΣΑΣ Σ", :greek) == "σ σας σ" + assert String.downcase("ΜΕΣ'ΑΠΟ", :greek) == "μεσ'απο" + assert String.downcase("ΑΣ'ΤΟΥΣ", :greek) == "ασ'τους" + end + + test "downcase/1 with ascii" do + assert String.downcase("OLÁ", :ascii) == "olÁ" + end + + test "downcase/1 with turkic" do + assert String.downcase("Iİ", :turkic) == "ıi" + assert String.downcase("İ", :turkic) == "i" + + assert String.downcase("ıi", :turkic) == "ıi" + assert String.downcase("i", :turkic) == "i" + + assert String.downcase("İ") == "i̇" + end + + test "capitalize/1" do assert String.capitalize("") == "" assert String.capitalize("abc") == "Abc" assert String.capitalize("ABC") == "Abc" @@ -124,7 +239,7 @@ defmodule StringTest do assert String.capitalize(" aBc1") == " abc1" end - test :capitalize_utf8 do + test "capitalize/1 with UTF-8" do assert String.capitalize("àáâ") == "Àáâ" assert String.capitalize("ÀÁÂ") == "Àáâ" assert String.capitalize("âáà") == "Âáà" @@ -134,126 +249,345 @@ defmodule StringTest do assert String.capitalize("fin") == "Fin" end - test :rstrip do - assert String.rstrip("") == "" - assert String.rstrip(" abc ") == " abc" - assert String.rstrip(" abc a") == " abc a" - assert String.rstrip("a abc a\n\n") == "a abc a" - assert String.rstrip("a abc a\t\n\v\f\r\s") == "a abc a" - assert String.rstrip("a abc a " <> <<31>>) == "a abc a" - assert String.rstrip("a abc a" <> <<194, 133>>) == "a abc a" - assert String.rstrip(" abc aa", ?a) == " abc " - assert String.rstrip(" abc __", ?_) == " abc " - assert String.rstrip(" cat 猫猫", ?猫) == " cat " - end - - test :lstrip do - assert String.lstrip("") == "" - assert String.lstrip(" abc ") == "abc " - assert String.lstrip("a abc a") == "a abc a" - assert String.lstrip("\n\na abc a") == "a abc a" - assert String.lstrip("\t\n\v\f\r\sa abc a") == "a abc a" - assert String.lstrip(<<31>> <> " a abc a") == "a abc a" - assert String.lstrip(<<194, 133>> <> "a abc a") == "a abc a" - assert String.lstrip("__ abc _", ?_) == " abc _" - assert String.lstrip("猫猫 cat ", ?猫) == " cat " - end - - test :strip do - assert String.strip("") == "" - assert String.strip(" abc ") == "abc" - assert String.strip("a abc a\n\n") == "a abc a" - assert String.strip("a abc a\t\n\v\f\r\s") == "a abc a" - assert String.strip("___ abc ___", ?_) == " abc " - assert String.strip("猫猫猫 cat 猫猫猫", ?猫) == " cat " - end - - test :rjust do - assert String.rjust("", 5) == " " - assert String.rjust("abc", 5) == " abc" - assert String.rjust(" abc ", 9) == " abc " - assert String.rjust("猫", 5) == " 猫" - assert String.rjust("abc", 5, ?-) == "--abc" - assert String.rjust("abc", 5, ?猫) == "猫猫abc" - end - - test :ljust do - assert String.ljust("", 5) == " " - assert String.ljust("abc", 5) == "abc " - assert String.ljust(" abc ", 9) == " abc " - assert String.ljust("猫", 5) == "猫 " - assert String.ljust("abc", 5, ?-) == "abc--" - assert String.ljust("abc", 5, ?猫) == "abc猫猫" - end - - test :reverse do + test "capitalize/1 with ascii" do + assert String.capitalize("àáâ", :ascii) == "àáâ" + assert String.capitalize("aáA", :ascii) == "Aáa" + end + + test "capitalize/1 with turkic" do + assert String.capitalize("iii", :turkic) == "İii" + assert String.capitalize("ııı", :turkic) == "Iıı" + assert String.capitalize("İii", :turkic) == "İii" + assert String.capitalize("Iıı", :turkic) == "Iıı" + end + + test "replace_leading/3" do + assert String.replace_leading("aa abc ", "a", "b") == "bb abc " + assert String.replace_leading("__ abc ", "_", "b") == "bb abc " + assert String.replace_leading("aaaaaaaa ", "a", "b") == "bbbbbbbb " + assert String.replace_leading("aaaaaaaa ", "aaa", "b") == "bbaa " + assert String.replace_leading("aaaaaaaaa", "a", "b") == "bbbbbbbbb" + assert String.replace_leading("]]]]]]", "]", "[]") == "[][][][][][]" + assert String.replace_leading("]]]]]]]]", "]", "") == "" + assert String.replace_leading("]]]]]] ]", "]", "") == " ]" + assert String.replace_leading("猫猫 cat ", "猫", "й") == "йй cat " + assert String.replace_leading("test", "t", "T") == "Test" + assert String.replace_leading("t", "t", "T") == "T" + assert String.replace_leading("aaa", "b", "c") == "aaa" + + message = ~r/cannot use an empty string/ + + assert_raise ArgumentError, message, fn -> + String.replace_leading("foo", "", "bar") + end + + assert_raise ArgumentError, message, fn -> + String.replace_leading("", "", "bar") + end + end + + test "replace_trailing/3" do + assert String.replace_trailing(" abc aa", "a", "b") == " abc bb" + assert String.replace_trailing(" abc __", "_", "b") == " abc bb" + assert String.replace_trailing(" aaaaaaaa", "a", "b") == " bbbbbbbb" + assert String.replace_trailing(" aaaaaaaa", "aaa", "b") == " aabb" + assert String.replace_trailing("aaaaaaaaa", "a", "b") == "bbbbbbbbb" + assert String.replace_trailing("]]]]]]", "]", "[]") == "[][][][][][]" + assert String.replace_trailing("]]]]]]]]", "]", "") == "" + assert String.replace_trailing("] ]]]]]]", "]", "") == "] " + assert String.replace_trailing(" cat 猫猫", "猫", "й") == " cat йй" + assert String.replace_trailing("test", "t", "T") == "tesT" + assert String.replace_trailing("t", "t", "T") == "T" + assert String.replace_trailing("aaa", "b", "c") == "aaa" + + message = ~r/cannot use an empty string/ + + assert_raise ArgumentError, message, fn -> + String.replace_trailing("foo", "", "bar") + end + + assert_raise ArgumentError, message, fn -> + String.replace_trailing("", "", "bar") + end + end + + test "trim/1,2" do + assert String.trim("") == "" + assert String.trim(" abc ") == "abc" + assert String.trim("a abc a\n\n") == "a abc a" + assert String.trim("a abc a\t\n\v\f\r\s") == "a abc a" + + assert String.trim("___ abc ___", "_") == " abc " + assert String.trim("猫猫猫cat猫猫猫", "猫猫") == "猫cat猫" + # no-break space + assert String.trim("\u00A0a abc a\u00A0") == "a abc a" + # whitespace defined as a range + assert String.trim("\u2008a abc a\u2005") == "a abc a" + end + + test "trim_leading/1,2" do + assert String.trim_leading("") == "" + assert String.trim_leading(" abc ") == "abc " + assert String.trim_leading("a abc a") == "a abc a" + assert String.trim_leading("\n\na abc a") == "a abc a" + assert String.trim_leading("\t\n\v\f\r\sa abc a") == "a abc a" + assert String.trim_leading(<<194, 133, "a abc a">>) == "a abc a" + # information separators are not whitespace + assert String.trim_leading("\u001F a abc a") == "\u001F a abc a" + # no-break space + assert String.trim_leading("\u00A0 a abc a") == "a abc a" + + assert String.trim_leading("aa aaa", "aaa") == "aa aaa" + assert String.trim_leading("aaa aaa", "aa") == "a aaa" + assert String.trim_leading("aa abc ", "a") == " abc " + assert String.trim_leading("__ abc ", "_") == " abc " + assert String.trim_leading("aaaaaaaaa ", "a") == " " + assert String.trim_leading("aaaaaaaaaa", "a") == "" + assert String.trim_leading("]]]]]] ]", "]") == " ]" + assert String.trim_leading("猫猫 cat ", "猫") == " cat " + assert String.trim_leading("test", "t") == "est" + assert String.trim_leading("t", "t") == "" + assert String.trim_leading("", "t") == "" + end + + test "trim_trailing/1,2" do + assert String.trim_trailing("") == "" + assert String.trim_trailing("1\n") == "1" + assert String.trim_trailing("\r\n") == "" + assert String.trim_trailing(" abc ") == " abc" + assert String.trim_trailing(" abc a") == " abc a" + assert String.trim_trailing("a abc a\n\n") == "a abc a" + assert String.trim_trailing("a abc a\t\n\v\f\r\s") == "a abc a" + assert String.trim_trailing(<<"a abc a", 194, 133>>) == "a abc a" + # information separators are not whitespace + assert String.trim_trailing("a abc a \u001F") == "a abc a \u001F" + # no-break space + assert String.trim_trailing("a abc a \u00A0") == "a abc a" + + assert String.trim_trailing("aaa aa", "aaa") == "aaa aa" + assert String.trim_trailing("aaa aaa", "aa") == "aaa a" + assert String.trim_trailing(" abc aa", "a") == " abc " + assert String.trim_trailing(" abc __", "_") == " abc " + assert String.trim_trailing(" aaaaaaaaa", "a") == " " + assert String.trim_trailing("aaaaaaaaaa", "a") == "" + assert String.trim_trailing("] ]]]]]]", "]") == "] " + assert String.trim_trailing(" cat 猫猫", "猫") == " cat " + assert String.trim_trailing("test", "t") == "tes" + assert String.trim_trailing("t", "t") == "" + assert String.trim_trailing("", "t") == "" + end + + test "pad_leading/2,3" do + assert String.pad_leading("", 5) == " " + assert String.pad_leading("abc", 5) == " abc" + assert String.pad_leading(" abc ", 9) == " abc " + assert String.pad_leading("猫", 5) == " 猫" + assert String.pad_leading("-", 0) == "-" + assert String.pad_leading("-", 1) == "-" + + assert String.pad_leading("---", 5, "abc") == "ab---" + assert String.pad_leading("---", 9, "abc") == "abcabc---" + + assert String.pad_leading("---", 5, ["abc"]) == "abcabc---" + assert String.pad_leading("--", 6, ["a", "bc"]) == "abcabc--" + + assert_raise FunctionClauseError, fn -> + String.pad_leading("-", -1) + end + + assert_raise FunctionClauseError, fn -> + String.pad_leading("-", 1, []) + end + + message = "expected a string padding element, got: 10" + + assert_raise ArgumentError, message, fn -> + String.pad_leading("-", 3, ["-", 10]) + end + end + + test "pad_trailing/2,3" do + assert String.pad_trailing("", 5) == " " + assert String.pad_trailing("abc", 5) == "abc " + assert String.pad_trailing(" abc ", 9) == " abc " + assert String.pad_trailing("猫", 5) == "猫 " + assert String.pad_trailing("-", 0) == "-" + assert String.pad_trailing("-", 1) == "-" + + assert String.pad_trailing("---", 5, "abc") == "---ab" + assert String.pad_trailing("---", 9, "abc") == "---abcabc" + + assert String.pad_trailing("---", 5, ["abc"]) == "---abcabc" + assert String.pad_trailing("--", 6, ["a", "bc"]) == "--abcabc" + + assert_raise FunctionClauseError, fn -> + String.pad_trailing("-", -1) + end + + assert_raise FunctionClauseError, fn -> + String.pad_trailing("-", 1, []) + end + + message = "expected a string padding element, got: 10" + + assert_raise ArgumentError, message, fn -> + String.pad_trailing("-", 3, ["-", 10]) + end + end + + test "reverse/1" do assert String.reverse("") == "" assert String.reverse("abc") == "cba" assert String.reverse("Hello World") == "dlroW olleH" assert String.reverse("Hello ∂og") == "go∂ olleH" assert String.reverse("Ā̀stute") == "etutsĀ̀" assert String.reverse(String.reverse("Hello World")) == "Hello World" + assert String.reverse(String.reverse("Hello \r\n World")) == "Hello \r\n World" end - test :replace do - assert String.replace("a,b,c", ",", "-") == "a-b-c" - assert String.replace("a,b,c", [",", "b"], "-") == "a---c" + describe "replace/3" do + test "with empty string and string replacement" do + assert String.replace("elixir", "", "") == "elixir" + assert String.replace("ELIXIR", "", ".") == ".E.L.I.X.I.R." + assert String.replace("ELIXIR", "", ".", global: true) == ".E.L.I.X.I.R." + assert String.replace("ELIXIR", "", ".", global: false) == ".ELIXIR" - assert String.replace("a,b,c", ",", "-", global: false) == "a-b,c" - assert String.replace("a,b,c", [",", "b"], "-", global: false) == "a-b,c" - assert String.replace("ãéã", "é", "e", global: false) == "ãeã" + assert_raise ArgumentError, fn -> + String.replace("elixir", [""], "") + end + end + + test "with empty pattern list" do + assert String.replace("elixir", [], "anything") == "elixir" + end + + test "with match pattern and string replacement" do + assert String.replace("a,b,c", ",", "-") == "a-b-c" + assert String.replace("a,b,c", [",", "b"], "-") == "a---c" - assert String.replace("a,b,c", ",", "[]", insert_replaced: 2) == "a[],b[],c" - assert String.replace("a,b,c", ",", "[]", insert_replaced: [1, 1]) == "a[,,]b[,,]c" - assert String.replace("a,b,c", "b", "[]", insert_replaced: 1, global: false) == "a,[b],c" + assert String.replace("a,b,c", ",", "-", global: false) == "a-b,c" + assert String.replace("a,b,c", [",", "b"], "-", global: false) == "a-b,c" + assert String.replace("ãéã", "é", "e", global: false) == "ãeã" + end + + test "with regex and string replacement" do + assert String.replace("a,b,c", ~r/,(.)/, ",\\1\\1") == "a,bb,cc" + assert String.replace("a,b,c", ~r/,(.)/, ",\\1\\1", global: false) == "a,bb,c" + end - assert String.replace("a,b,c", ~r/,(.)/, ",\\1\\1") == "a,bb,cc" - assert String.replace("a,b,c", ~r/,(.)/, ",\\1\\1", global: false) == "a,bb,c" + test "with empty string and function replacement" do + assert String.replace("elixir", "", fn "" -> "" end) == "elixir" + assert String.replace("ELIXIR", "", fn "" -> "." end) == ".E.L.I.X.I.R." + assert String.replace("ELIXIR", "", fn "" -> "." end, global: true) == ".E.L.I.X.I.R." + assert String.replace("ELIXIR", "", fn "" -> "." end, global: false) == ".ELIXIR" + + assert String.replace("elixir", "", fn "" -> [""] end) == "elixir" + assert String.replace("ELIXIR", "", fn "" -> ["."] end) == ".E.L.I.X.I.R." + assert String.replace("ELIXIR", "", fn "" -> ["."] end, global: true) == ".E.L.I.X.I.R." + assert String.replace("ELIXIR", "", fn "" -> ["."] end, global: false) == ".ELIXIR" + end + + test "with match pattern and function replacement" do + assert String.replace("a,b,c", ",", fn "," -> "-" end) == "a-b-c" + assert String.replace("a,b,c", [",", "b"], fn x -> "[#{x}]" end) == "a[,][b][,]c" + assert String.replace("a,b,c", [",", "b"], fn x -> [?[, x, ?]] end) == "a[,][b][,]c" + + assert String.replace("a,b,c", ",", fn "," -> "-" end, global: false) == "a-b,c" + assert String.replace("a,b,c", [",", "b"], fn x -> "[#{x}]" end, global: false) == "a[,]b,c" + assert String.replace("ãéã", "é", fn "é" -> "e" end, global: false) == "ãeã" + end + + test "with regex and function replacement" do + assert String.replace("a,b,c", ~r/,(.)/, fn x -> "#{x}#{x}" end) == "a,b,b,c,c" + assert String.replace("a,b,c", ~r/,(.)/, fn x -> [x, x] end) == "a,b,b,c,c" + assert String.replace("a,b,c", ~r/,(.)/, fn x -> "#{x}#{x}" end, global: false) == "a,b,b,c" + assert String.replace("a,b,c", ~r/,(.)/, fn x -> [x, x] end, global: false) == "a,b,b,c" + end end - test :duplicate do + describe "replace/4" do + test "with incorrect params" do + assert_raise FunctionClauseError, "no function clause matching in String.replace/4", fn -> + String.replace("a,b,c", "a,b,c", ",", "") + end + end + end + + test "duplicate/2" do assert String.duplicate("abc", 0) == "" assert String.duplicate("abc", 1) == "abc" assert String.duplicate("abc", 2) == "abcabc" assert String.duplicate("&ã$", 2) == "&ã$&ã$" + + assert_raise ArgumentError, fn -> + String.duplicate("abc", -1) + end end - test :codepoints do + test "codepoints/1" do assert String.codepoints("elixir") == ["e", "l", "i", "x", "i", "r"] - assert String.codepoints("elixír") == ["e", "l", "i", "x", "í", "r"] # slovak - assert String.codepoints("ոգելից ըմպելիք") == ["ո", "գ", "ե", "լ", "ի", "ց", " ", "ը", "մ", "պ", "ե", "լ", "ի", "ք"] # armenian - assert String.codepoints("эліксір") == ["э", "л", "і", "к", "с", "і", "р"] # belarussian - assert String.codepoints("ελιξήριο") == ["ε", "λ", "ι", "ξ", "ή", "ρ", "ι", "ο"] # greek - assert String.codepoints("סם חיים") == ["ס", "ם", " ", "ח", "י", "י", "ם"] # hebraic - assert String.codepoints("अमृत") == ["अ", "म", "ृ", "त"] # hindi - assert String.codepoints("স্পর্শমণি") == ["স", "্", "প", "র", "্", "শ", "ম", "ণ", "ি"] # bengali - assert String.codepoints("સર્વશ્રેષ્ઠ ઇલાજ") == ["સ", "ર", "્", "વ", "શ", "્", "ર", "ે", "ષ", "્", "ઠ", " ", "ઇ", "લ", "ા", "જ"] # gujarati - assert String.codepoints("世界中の一番") == ["世", "界", "中", "の", "一", "番"] # japanese + # slovak + assert String.codepoints("elixír") == ["e", "l", "i", "x", "í", "r"] + # armenian + assert String.codepoints("ոգելից ըմպելիք") == + ["ո", "գ", "ե", "լ", "ի", "ց", " ", "ը", "մ", "պ", "ե", "լ", "ի", "ք"] + + # belarussian + assert String.codepoints("эліксір") == ["э", "л", "і", "к", "с", "і", "р"] + # greek + assert String.codepoints("ελιξήριο") == ["ε", "λ", "ι", "ξ", "ή", "ρ", "ι", "ο"] + # hebraic + assert String.codepoints("סם חיים") == ["ס", "ם", " ", "ח", "י", "י", "ם"] + # hindi + assert String.codepoints("अमृत") == ["अ", "म", "ृ", "त"] + # bengali + assert String.codepoints("স্পর্শমণি") == ["স", "্", "প", "র", "্", "শ", "ম", "ণ", "ি"] + # gujarati + assert String.codepoints("સર્વશ્રેષ્ઠ ઇલાજ") == + ["સ", "ર", "્", "વ", "શ", "્", "ર", "ે", "ષ", "્", "ઠ", " ", "ઇ", "લ", "ા", "જ"] + + # japanese + assert String.codepoints("世界中の一番") == ["世", "界", "中", "の", "一", "番"] assert String.codepoints("がガちゃ") == ["が", "ガ", "ち", "ゃ"] assert String.codepoints("") == [] + assert String.codepoints("ϖͲϥЫݎߟΈټϘለДШव׆ש؇؊صلټܗݎޥޘ߉ऌ૫ሏᶆ℆ℙℱ ⅚Ⅷ↠∈⌘①ffi") == - ["ϖ", "Ͳ", "ϥ", "Ы", "ݎ", "ߟ", "Έ", "ټ", "Ϙ", "ለ", "Д", "Ш", "व", "׆", "ש", "؇", "؊", "ص", "ل", "ټ", "ܗ", "ݎ", "ޥ", "ޘ", "߉", "ऌ", "૫", "ሏ", "ᶆ", "℆", "ℙ", "ℱ", " ", "⅚", "Ⅷ", "↠", "∈", "⌘", "①", "ffi"] + ["ϖ", "Ͳ", "ϥ", "Ы", "ݎ", "ߟ", "Έ"] ++ + ["ټ", "Ϙ", "ለ", "Д", "Ш", "व"] ++ + ["׆", "ש", "؇", "؊", "ص", "ل", "ټ"] ++ + ["ܗ", "ݎ", "ޥ", "ޘ", "߉", "ऌ", "૫"] ++ + ["ሏ", "ᶆ", "℆", "ℙ", "ℱ", " ", "⅚"] ++ ["Ⅷ", "↠", "∈", "⌘", "①", "ffi"] end - test :graphemes do + test "equivalent?/2" do + assert String.equivalent?("", "") + assert String.equivalent?("elixir", "elixir") + assert String.equivalent?("뢴", "뢴") + assert String.equivalent?("ṩ", "ṩ") + refute String.equivalent?("ELIXIR", "elixir") + refute String.equivalent?("døge", "dóge") + end + + test "graphemes/1" do # Extended assert String.graphemes("Ā̀stute") == ["Ā̀", "s", "t", "u", "t", "e"] # CLRF - assert String.graphemes("\n\r\f") == ["\n\r", "\f"] + assert String.graphemes("\r\n\f") == ["\r\n", "\f"] # Regional indicator - assert String.graphemes("\x{1F1E6}\x{1F1E7}\x{1F1E8}") == ["\x{1F1E6}\x{1F1E7}\x{1F1E8}"] + assert String.graphemes("\u{1F1E6}\u{1F1E7}") == ["\u{1F1E6}\u{1F1E7}"] + assert String.graphemes("\u{1F1E6}\u{1F1E7}\u{1F1E8}") == ["\u{1F1E6}\u{1F1E7}", "\u{1F1E8}"] # Hangul - assert String.graphemes("\x{1100}\x{115D}\x{B4A4}") == ["ᄀᅝ뒤"] + assert String.graphemes("\u1100\u115D\uB4A4") == ["ᄀᅝ뒤"] # Special Marking with Extended - assert String.graphemes("a\x{0300}\x{0903}") == ["a\x{0300}\x{0903}"] + assert String.graphemes("a\u0300\u0903") == ["a\u0300\u0903"] end - test :next_grapheme do + test "next_grapheme/1" do assert String.next_grapheme("Ā̀stute") == {"Ā̀", "stute"} assert String.next_grapheme("") == nil end - test :first do + test "first/1" do assert String.first("elixir") == "e" assert String.first("íelixr") == "í" assert String.first("եոգլից ըմպելիք") == "ե" @@ -265,7 +599,7 @@ defmodule StringTest do assert String.first("") == nil end - test :last do + test "last/1" do assert String.last("elixir") == "r" assert String.last("elixrí") == "í" assert String.last("եոգլից ըմպելիքե") == "ե" @@ -277,7 +611,7 @@ defmodule StringTest do assert String.last("") == nil end - test :length do + test "length/1" do assert String.length("elixir") == 6 assert String.length("elixrí") == 6 assert String.length("եոգլից") == 6 @@ -286,10 +620,11 @@ defmodule StringTest do assert String.length("סם ייםח") == 7 assert String.length("がガちゃ") == 4 assert String.length("Ā̀stute") == 6 + assert String.length("👨‍👩‍👧‍👦") == 1 assert String.length("") == 0 end - test :at do + test "at/2" do assert String.at("л", 0) == "л" assert String.at("elixir", 1) == "l" assert String.at("がガちゃ", 2) == "ち" @@ -299,9 +634,17 @@ defmodule StringTest do assert String.at("л", -3) == nil assert String.at("Ā̀stute", 1) == "s" assert String.at("elixir", 6) == nil + + assert_raise FunctionClauseError, fn -> + String.at("elixir", 0.1) + end + + assert_raise FunctionClauseError, fn -> + String.at("elixir", -0.1) + end end - test :slice do + test "slice/3" do assert String.slice("elixir", 1, 3) == "lix" assert String.slice("あいうえお", 2, 2) == "うえ" assert String.slice("ειξήριολ", 2, 3) == "ξήρ" @@ -311,9 +654,9 @@ defmodule StringTest do assert String.slice("elixir", -3, 2) == "xi" assert String.slice("あいうえお", -4, 3) == "いうえ" assert String.slice("ειξήριολ", -5, 3) == "ήρι" - assert String.slice("elixir", -10, 1) == "" - assert String.slice("あいうえお", -10, 2) == "" - assert String.slice("ειξήριολ", -10, 3) == "" + assert String.slice("elixir", -10, 1) == "e" + assert String.slice("あいうえお", -10, 2) == "あい" + assert String.slice("ειξήριολ", -10, 3) == "ειξ" assert String.slice("elixir", 8, 2) == "" assert String.slice("あいうえお", 6, 2) == "" assert String.slice("ειξήριολ", 8, 1) == "" @@ -321,13 +664,17 @@ defmodule StringTest do assert String.slice("elixir", 0, 0) == "" assert String.slice("elixir", 5, 0) == "" assert String.slice("elixir", -5, 0) == "" + assert String.slice("elixir", -10, 10) == "elixir" assert String.slice("", 0, 1) == "" assert String.slice("", 1, 1) == "" + end + test "slice/2" do assert String.slice("elixir", 0..-2) == "elixi" assert String.slice("elixir", 1..3) == "lix" assert String.slice("elixir", -5..-3) == "lix" assert String.slice("elixir", -5..3) == "lix" + assert String.slice("elixir", -10..10) == "elixir" assert String.slice("あいうえお", 2..3) == "うえ" assert String.slice("ειξήριολ", 2..4) == "ξήρ" assert String.slice("elixir", 3..6) == "xir" @@ -344,139 +691,283 @@ defmodule StringTest do assert String.slice("", 1..1) == "" assert String.slice("あいうえお", -2..-4) == "" assert String.slice("あいうえお", -10..-15) == "" + assert String.slice("hello あいうえお Unicode", 8..-1) == "うえお Unicode" + assert String.slice("abc", -1..14) == "c" + assert String.slice("a·̀ͯ‿.⁀:", 0..-2) == "a·̀ͯ‿.⁀" + + assert_raise FunctionClauseError, fn -> + String.slice(nil, 0..1) + end end - test :valid? do + test "slice/2 with steps" do + assert String.slice("elixir", 0..-2//2) == "eii" + assert String.slice("elixir", 1..3//2) == "lx" + assert String.slice("elixir", -5..-3//2) == "lx" + assert String.slice("elixir", -5..3//2) == "lx" + assert String.slice("あいうえお", 2..3//2) == "う" + assert String.slice("ειξήριολ", 2..4//2) == "ξρ" + assert String.slice("elixir", 3..6//2) == "xr" + assert String.slice("あいうえお", 3..7//2) == "え" + assert String.slice("ειξήριολ", 5..8//2) == "ιλ" + assert String.slice("elixir", -3..-2//2) == "x" + assert String.slice("あいうえお", -4..-2//2) == "いえ" + assert String.slice("ειξήριολ", -5..-3//2) == "ήι" + assert String.slice("elixir", 8..9//2) == "" + assert String.slice("", 0..0//2) == "" + assert String.slice("", 1..1//2) == "" + assert String.slice("あいうえお", -2..-4//2) == "" + assert String.slice("あいうえお", -10..-15//2) == "" + assert String.slice("hello あいうえお Unicode", 8..-1//2) == "うおUioe" + assert String.slice("abc", -1..14//2) == "c" + assert String.slice("a·̀ͯ‿.⁀:", 0..-2//2) == "a‿⁀" + end + + test "valid?/1" do assert String.valid?("afds") assert String.valid?("øsdfh") assert String.valid?("dskfjあska") + assert String.valid?(<<0xEF, 0xB7, 0x90>>) - refute String.valid?(<<0xffff :: 16>>) - refute String.valid?("asd" <> <<0xffff :: 16>>) - end - - test :valid_character? do - assert String.valid_character?("a") - assert String.valid_character?("ø") - assert String.valid_character?("あ") - - refute String.valid_character?("\x{ffff}") - refute String.valid_character?("ab") + refute String.valid?(<<0xFFFF::16>>) + refute String.valid?("asd" <> <<0xFFFF::16>>) end - test :chunk_valid do + test "chunk/2 with :valid trait" do assert String.chunk("", :valid) == [] - assert String.chunk("ødskfjあ\011ska", :valid) - == ["ødskfjあ\011ska"] - assert String.chunk("abc\x{0ffff}def", :valid) - == ["abc", <<0x0ffff::utf8>>, "def"] - assert String.chunk("\x{0fffe}\x{3ffff}привет\x{0ffff}мир", :valid) - == [<<0x0fffe::utf8, 0x3ffff::utf8>>, "привет", <<0x0ffff::utf8>>, "мир"] - assert String.chunk("日本\x{0ffff}\x{fdef}ござございます\x{fdd0}", :valid) - == ["日本", <<0x0ffff::utf8, 0xfdef::utf8>>, "ござございます", <<0xfdd0::utf8>>] + assert String.chunk("ødskfjあ\x11ska", :valid) == ["ødskfjあ\x11ska"] end - test :chunk_printable do + test "chunk/2 with :printable trait" do assert String.chunk("", :printable) == [] - assert String.chunk("ødskfjあska", :printable) - == ["ødskfjあska"] - assert String.chunk("abc\x{0ffff}def", :printable) - == ["abc", <<0x0ffff::utf8>>, "def"] - assert String.chunk("\006ab\005cdef\003\000", :printable) - == [<<06>>, "ab", <<05>>, "cdef", <<03, 0>>] - end - - test :starts_with? do - ## Normal cases ## - assert String.starts_with? "hello", "he" - assert String.starts_with? "hello", "hello" - assert String.starts_with? "hello", ["hellö", "hell"] - assert String.starts_with? "エリクシア", "エリ" - refute String.starts_with? "hello", "lo" - refute String.starts_with? "hello", "hellö" - refute String.starts_with? "hello", ["hellö", "goodbye"] - refute String.starts_with? "エリクシア", "仙丹" - - ## Edge cases ## - assert String.starts_with? "", "" - assert String.starts_with? "", ["", "a"] - assert String.starts_with? "b", ["", "a"] - - assert String.starts_with? "abc", "" - assert String.starts_with? "abc", [""] - - refute String.starts_with? "", "abc" - refute String.starts_with? "", [" "] - - ## Sanity checks ## - assert String.starts_with? "", ["", ""] - assert String.starts_with? "abc", ["", ""] - end - - test :ends_with? do - ## Normal cases ## - assert String.ends_with? "hello", "lo" - assert String.ends_with? "hello", "hello" - assert String.ends_with? "hello", ["hell", "lo", "xx"] - assert String.ends_with? "hello", ["hellö", "lo"] - assert String.ends_with? "エリクシア", "シア" - refute String.ends_with? "hello", "he" - refute String.ends_with? "hello", "hellö" - refute String.ends_with? "hello", ["hel", "goodbye"] - refute String.ends_with? "エリクシア", "仙丹" - - ## Edge cases ## - assert String.ends_with? "", "" - assert String.ends_with? "", ["", "a"] - refute String.ends_with? "", ["a", "b"] - - assert String.ends_with? "abc", "" - assert String.ends_with? "abc", ["", "x"] - - refute String.ends_with? "", "abc" - refute String.ends_with? "", [" "] - - ## Sanity checks ## - assert String.ends_with? "", ["", ""] - assert String.ends_with? "abc", ["", ""] - end - - test :contains? do - ## Normal cases ## - assert String.contains? "elixir of life", "of" - assert String.contains? "エリクシア", "シ" - assert String.contains? "elixir of life", ["mercury", "life"] - refute String.contains? "elixir of life", "death" - refute String.contains? "エリクシア", "仙" - refute String.contains? "elixir of life", ["death", "mercury", "eternal life"] - - ## Edge cases ## - assert String.contains? "", "" - assert String.contains? "abc", "" - assert String.contains? "abc", ["", "x"] - - refute String.contains? "", " " - refute String.contains? "", "a" - - ## Sanity checks ## - assert String.contains? "", ["", ""] - assert String.contains? "abc", ["", ""] - end - - test :to_char_list do - assert String.to_char_list("æß") == [?æ, ?ß] - assert String.to_char_list("abc") == [?a, ?b, ?c] - - assert_raise UnicodeConversionError, - "invalid encoding starting at <<223, 255>>", fn -> - String.to_char_list(<< 0xDF, 0xFF >>) + assert String.chunk("ødskfjあska", :printable) == ["ødskfjあska"] + assert String.chunk("abc\u{0FFFF}def", :printable) == ["abc", <<0x0FFFF::utf8>>, "def"] + + assert String.chunk("\x06ab\x05cdef\x03\0", :printable) == + [<<6>>, "ab", <<5>>, "cdef", <<3, 0>>] + end + + test "starts_with?/2" do + assert String.starts_with?("hello", "he") + assert String.starts_with?("hello", "hello") + refute String.starts_with?("hello", []) + assert String.starts_with?("hello", "") + assert String.starts_with?("hello", [""]) + assert String.starts_with?("hello", ["hellö", "hell"]) + assert String.starts_with?("エリクシア", "エリ") + refute String.starts_with?("hello", "lo") + refute String.starts_with?("hello", "hellö") + refute String.starts_with?("hello", ["hellö", "goodbye"]) + refute String.starts_with?("エリクシア", "仙丹") + end + + test "ends_with?/2" do + assert String.ends_with?("hello", "lo") + assert String.ends_with?("hello", "hello") + refute String.ends_with?("hello", []) + assert String.ends_with?("hello", ["hell", "lo", "xx"]) + assert String.ends_with?("hello", ["hellö", "lo"]) + assert String.ends_with?("エリクシア", "シア") + refute String.ends_with?("hello", "he") + refute String.ends_with?("hello", "hellö") + refute String.ends_with?("hello", ["hel", "goodbye"]) + refute String.ends_with?("エリクシア", "仙丹") + end + + test "contains?/2" do + assert String.contains?("elixir of life", "of") + assert String.contains?("エリクシア", "シ") + refute String.contains?("elixir of life", []) + assert String.contains?("elixir of life", "") + assert String.contains?("elixir of life", [""]) + assert String.contains?("elixir of life", ["mercury", "life"]) + refute String.contains?("elixir of life", "death") + refute String.contains?("エリクシア", "仙") + refute String.contains?("elixir of life", ["death", "mercury", "eternal life"]) + end + + test "to_charlist/1" do + assert String.to_charlist("æß") == [?æ, ?ß] + assert String.to_charlist("abc") == [?a, ?b, ?c] + + assert_raise UnicodeConversionError, "invalid encoding starting at <<223, 255>>", fn -> + String.to_charlist(<<0xDF, 0xFF>>) end - assert_raise UnicodeConversionError, - "incomplete encoding starting at <<195>>", fn -> - String.to_char_list(<< 106, 111, 115, 195 >>) + assert_raise UnicodeConversionError, "incomplete encoding starting at <<195>>", fn -> + String.to_charlist(<<106, 111, 115, 195>>) end end + + test "to_float/1" do + assert String.to_float("3.0") == 3.0 + + three = fn -> "3" end + assert_raise ArgumentError, fn -> String.to_float(three.()) end + end + + test "jaro_distance/2" do + assert String.jaro_distance("same", "same") == 1.0 + assert String.jaro_distance("any", "") == 0.0 + assert String.jaro_distance("", "any") == 0.0 + assert String.jaro_distance("martha", "marhta") == 0.9444444444444445 + assert String.jaro_distance("martha", "marhha") == 0.888888888888889 + assert String.jaro_distance("marhha", "martha") == 0.888888888888889 + assert String.jaro_distance("dwayne", "duane") == 0.8222222222222223 + assert String.jaro_distance("dixon", "dicksonx") == 0.7666666666666666 + assert String.jaro_distance("xdicksonx", "dixon") == 0.7851851851851852 + assert String.jaro_distance("shackleford", "shackelford") == 0.9696969696969697 + assert String.jaro_distance("dunningham", "cunnigham") == 0.8962962962962964 + assert String.jaro_distance("nichleson", "nichulson") == 0.9259259259259259 + assert String.jaro_distance("jones", "johnson") == 0.7904761904761904 + assert String.jaro_distance("massey", "massie") == 0.888888888888889 + assert String.jaro_distance("abroms", "abrams") == 0.888888888888889 + assert String.jaro_distance("hardin", "martinez") == 0.7222222222222222 + assert String.jaro_distance("itman", "smith") == 0.4666666666666666 + assert String.jaro_distance("jeraldine", "geraldine") == 0.9259259259259259 + assert String.jaro_distance("michelle", "michael") == 0.8690476190476191 + assert String.jaro_distance("julies", "julius") == 0.888888888888889 + assert String.jaro_distance("tanya", "tonya") == 0.8666666666666667 + assert String.jaro_distance("sean", "susan") == 0.7833333333333333 + assert String.jaro_distance("jon", "john") == 0.9166666666666666 + assert String.jaro_distance("jon", "jan") == 0.7777777777777777 + assert String.jaro_distance("семена", "стремя") == 0.6666666666666666 + end + + test "myers_difference/2" do + assert String.myers_difference("", "abc") == [ins: "abc"] + assert String.myers_difference("abc", "") == [del: "abc"] + assert String.myers_difference("", "") == [] + assert String.myers_difference("abc", "abc") == [eq: "abc"] + assert String.myers_difference("abc", "aйbc") == [eq: "a", ins: "й", eq: "bc"] + assert String.myers_difference("aйbc", "abc") == [eq: "a", del: "й", eq: "bc"] + end + + test "normalize/2" do + assert String.normalize("ŝ", :nfd) == "ŝ" + assert String.normalize("ḇravô", :nfd) == "ḇravô" + assert String.normalize("ṩierra", :nfd) == "ṩierra" + assert String.normalize("뢴", :nfd) == "뢴" + assert String.normalize("êchǭ", :nfc) == "êchǭ" + assert String.normalize("거̄", :nfc) == "거̄" + assert String.normalize("뢴", :nfc) == "뢴" + + ## Error cases + assert String.normalize(<<15, 216>>, :nfc) == <<15, 216>> + assert String.normalize(<<15, 216>>, :nfd) == <<15, 216>> + assert String.normalize(<<216, 15>>, :nfc) == <<216, 15>> + assert String.normalize(<<216, 15>>, :nfd) == <<216, 15>> + + assert String.normalize(<<15, 216>>, :nfkc) == <<15, 216>> + assert String.normalize(<<15, 216>>, :nfkd) == <<15, 216>> + assert String.normalize(<<216, 15>>, :nfkc) == <<216, 15>> + assert String.normalize(<<216, 15>>, :nfkd) == <<216, 15>> + + ## Cases from NormalizationTest.txt + + # 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F + # 05B1 05B8 05B9 0591 05C3 05B0 05AC 059F + # HEBREW POINT QAMATS, HEBREW POINT HOLAM, HEBREW POINT HATAF SEGOL, + # HEBREW ACCENT ETNAHTA, HEBREW PUNCTUATION SOF PASUQ, HEBREW POINT SHEVA, + # HEBREW ACCENT ILUY, HEBREW ACCENT QARNEY PARA + assert String.normalize("ֱָֹ֑׃ְ֬֟", :nfc) == "ֱָֹ֑׃ְ֬֟" + + # 095D (exclusion list) + # 0922 093C + # DEVANAGARI LETTER RHA + assert String.normalize("ढ़", :nfc) == "ढ़" + + # 0061 0315 0300 05AE 0340 0062 + # 00E0 05AE 0300 0315 0062 + # LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, + # HEBREW ACCENT ZINOR, COMBINING GRAVE TONE MARK, LATIN SMALL LETTER B + assert String.normalize("à֮̀̕b", :nfc) == "à֮̀̕b" + + # 0344 + # 0308 0301 + # COMBINING GREEK DIALYTIKA TONOS + assert String.normalize("\u0344", :nfc) == "\u0308\u0301" + + # 115B9 0334 115AF + # 115B9 0334 115AF + # SIDDHAM VOWEL SIGN AI, COMBINING TILDE OVERLAY, SIDDHAM VOWEL SIGN AA + assert String.normalize("𑖹̴𑖯", :nfc) == "𑖹̴𑖯" + + # HEBREW ACCENT ETNAHTA, HEBREW PUNCTUATION SOF PASUQ, HEBREW POINT SHEVA, + # HEBREW ACCENT ILUY, HEBREW ACCENT QARNEY PARA + assert String.normalize("ֱָֹ֑׃ְ֬֟", :nfc) == "ֱָֹ֑׃ְ֬֟" + + # 095D (exclusion list) + # HEBREW ACCENT ETNAHTA, HEBREW PUNCTUATION SOF PASUQ, HEBREW POINT SHEVA, + # HEBREW ACCENT ILUY, HEBREW ACCENT QARNEY PARA + assert String.normalize("ֱָֹ֑׃ְ֬֟", :nfc) == "ֱָֹ֑׃ְ֬֟" + + # 095D (exclusion list) + # 0922 093C + # DEVANAGARI LETTER RHA + assert String.normalize("ढ़", :nfc) == "ढ़" + + # 0061 0315 0300 05AE 0340 0062 + # 00E0 05AE 0300 0315 0062 + # LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, + # HEBREW ACCENT ZINOR, COMBINING GRAVE TONE MARK, LATIN SMALL LETTER B + assert String.normalize("à֮̀̕b", :nfc) == "à֮̀̕b" + + # 0344 + # 0308 0301 + # COMBINING GREEK DIALYTIKA TONOS + assert String.normalize("\u0344", :nfc) == "\u0308\u0301" + + # 115B9 0334 115AF + # 115B9 0334 115AF + # SIDDHAM VOWEL SIGN AI, COMBINING TILDE OVERLAY, SIDDHAM VOWEL SIGN AA + assert String.normalize("𑖹̴𑖯", :nfc) == "𑖹̴𑖯" + + # (ff; ff; ff; ff; ff; ) LATIN SMALL LIGATURE FF + # FB00;FB00;FB00;0066 0066;0066 0066; + assert String.normalize("ff", :nfkd) == "\u0066\u0066" + + # (fl; fl; fl; fl; fl; ) LATIN SMALL LIGATURE FL + # FB02;FB02;FB02;0066 006C;0066 006C; + assert String.normalize("fl", :nfkd) == "\u0066\u006C" + + # (ſt; ſt; ſt; st; st; ) LATIN SMALL LIGATURE LONG S T + # FB05;FB05;FB05;0073 0074;0073 0074; + assert String.normalize("ſt", :nfkd) == "\u0073\u0074" + + # (st; st; st; st; st; ) LATIN SMALL LIGATURE ST + # FB06;FB06;FB06;0073 0074;0073 0074; + assert String.normalize("\u0073\u0074", :nfkc) == "\u0073\u0074" + + # (ﬓ; ﬓ; ﬓ; մն; մն; ) ARMENIAN SMALL LIGATURE MEN NOW + # FB13;FB13;FB13;0574 0576;0574 0576; + assert String.normalize("\u0574\u0576", :nfkc) == "\u0574\u0576" + end + + # Carriage return can be a grapheme cluster if followed by + # newline so we test some corner cases here. + test "carriage return" do + assert String.at("\r\t\v", 0) == "\r" + assert String.at("\r\t\v", 1) == "\t" + assert String.at("\r\t\v", 2) == "\v" + assert String.at("\xFF\r\t\v", 1) == "\r" + assert String.at("\r\xFF\t\v", 2) == "\t" + assert String.at("\r\t\xFF\v", 3) == "\v" + + assert String.last("\r\t\v") == "\v" + assert String.last("\r\xFF\t\xFF\v") == "\v" + + assert String.next_grapheme("\r\t\v") == {"\r", "\t\v"} + assert String.next_grapheme("\t\v") == {"\t", "\v"} + assert String.next_grapheme("\v") == {"\v", ""} + + assert String.length("\r\t\v") == 3 + assert String.length("\r\xFF\t\v") == 4 + assert String.length("\r\t\xFF\v") == 4 + + assert String.bag_distance("\r\t\xFF\v", "\xFF\r\n\xFF") == 0.25 + assert String.split("\r\t\v", "") == ["", "\r", "\t", "\v", ""] + end end diff --git a/lib/elixir/test/elixir/supervisor/spec_test.exs b/lib/elixir/test/elixir/supervisor/spec_test.exs deleted file mode 100644 index a15234bcef3..00000000000 --- a/lib/elixir/test/elixir/supervisor/spec_test.exs +++ /dev/null @@ -1,85 +0,0 @@ -Code.require_file "../test_helper.exs", __DIR__ - -defmodule Supervisor.SpecTest do - use ExUnit.Case, async: true - - import Supervisor.Spec - - test "worker/3" do - assert worker(Foo, [1, 2, 3]) == { - Foo, - {Foo, :start_link, [1, 2, 3]}, - :permanent, - 5000, - :worker, - [Foo] - } - - opts = [id: :sample, function: :start, modules: :dynamic, - restart: :temporary, shutdown: :brutal_kill] - - assert worker(Foo, [1, 2, 3], opts) == { - :sample, - {Foo, :start, [1, 2, 3]}, - :temporary, - :brutal_kill, - :worker, - :dynamic - } - end - - test "worker/3 with GenEvent" do - assert worker(GenEvent, [[name: :hello]]) == { - GenEvent, - {GenEvent, :start_link, [[name: :hello]]}, - :permanent, - 5000, - :worker, - :dynamic - } - end - - test "supervisor/3" do - assert supervisor(Foo, [1, 2, 3]) == { - Foo, - {Foo, :start_link, [1, 2, 3]}, - :permanent, - :infinity, - :supervisor, - [Foo] - } - - opts = [id: :sample, function: :start, modules: :dynamic, - restart: :temporary, shutdown: :brutal_kill] - - assert supervisor(Foo, [1, 2, 3], opts) == { - :sample, - {Foo, :start, [1, 2, 3]}, - :temporary, - :brutal_kill, - :supervisor, - :dynamic - } - end - - test "supervise/2" do - assert supervise([], strategy: :one_for_one) == { - :ok, {{:one_for_one, 5, 5}, []} - } - - children = [worker(GenEvent, [])] - options = [strategy: :one_for_all, max_restarts: 1, max_seconds: 1] - - assert supervise(children, options) == { - :ok, {{:one_for_all, 1, 1}, children} - } - end - - test "supervise/2 with duplicated ids" do - children = [worker(GenEvent, []), worker(GenEvent, [])] - - assert_raise ArgumentError, fn -> - supervise(children, strategy: :one_for_one) - end - end -end diff --git a/lib/elixir/test/elixir/supervisor_test.exs b/lib/elixir/test/elixir/supervisor_test.exs index eed8887e107..6f18bcc6f9f 100644 --- a/lib/elixir/test/elixir/supervisor_test.exs +++ b/lib/elixir/test/elixir/supervisor_test.exs @@ -1,4 +1,4 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule SupervisorTest do use ExUnit.Case, async: true @@ -6,43 +6,154 @@ defmodule SupervisorTest do defmodule Stack do use GenServer - def start_link(state, opts) do + def start_link({state, opts}) do GenServer.start_link(__MODULE__, state, opts) end - def handle_call(:pop, _from, [h|t]) do + def init(args) do + {:ok, args} + end + + def handle_call(:pop, _from, [h | t]) do {:reply, h, t} end def handle_call(:stop, _from, stack) do - # There is a race condition in between genserver terminations. + # There is a race condition between genserver terminations. # So we will explicitly unregister it here. try do - self |> Process.info(:registered_name) |> elem(1) |> Process.unregister + self() |> Process.info(:registered_name) |> elem(1) |> Process.unregister() rescue _ -> :ok end + {:stop, :normal, :ok, stack} end - def handle_cast({:push, h}, _from, t) do - {:noreply, [h|t]} + def handle_cast({:push, h}, t) do + {:noreply, [h | t]} end end defmodule Stack.Sup do use Supervisor - def init({arg, opts}) do - children = [worker(Stack, [arg, opts])] - supervise(children, strategy: :one_for_one) + def init(pair) do + Supervisor.init([{Stack, pair}], strategy: :one_for_one) + end + end + + test "generates child_spec/1" do + assert Stack.Sup.child_spec([:hello]) == %{ + id: Stack.Sup, + start: {Stack.Sup, :start_link, [[:hello]]}, + type: :supervisor + } + + defmodule CustomSup do + use Supervisor, + id: :id, + restart: :temporary, + start: {:foo, :bar, []} + + def init(arg) do + arg + end end + + assert CustomSup.child_spec([:hello]) == %{ + id: :id, + restart: :temporary, + start: {:foo, :bar, []}, + type: :supervisor + } end - import Supervisor.Spec + test "child_spec/2" do + assert Supervisor.child_spec(Task, []) == + %{id: Task, restart: :temporary, start: {Task, :start_link, [[]]}} + + assert Supervisor.child_spec({Task, :foo}, []) == + %{id: Task, restart: :temporary, start: {Task, :start_link, [:foo]}} + + assert Supervisor.child_spec(%{id: Task}, []) == %{id: Task} + + assert Supervisor.child_spec( + Task, + id: :foo, + start: {:foo, :bar, []}, + restart: :permanent, + shutdown: :infinity + ) == %{id: :foo, start: {:foo, :bar, []}, restart: :permanent, shutdown: :infinity} + + message = ~r"The module SupervisorTest was given as a child.*\nbut it does not implement"m + + assert_raise ArgumentError, message, fn -> + Supervisor.child_spec(SupervisorTest, []) + end + + message = ~r"The module Unknown was given as a child.*but it does not exist"m + + assert_raise ArgumentError, message, fn -> + Supervisor.child_spec(Unknown, []) + end + + message = ~r"supervisors expect each child to be one of" + + assert_raise ArgumentError, message, fn -> + Supervisor.child_spec("other", []) + end + end + + test "init/2" do + flags = %{intensity: 3, period: 5, strategy: :one_for_one} + children = [%{id: Task, restart: :temporary, start: {Task, :start_link, [[]]}}] + assert Supervisor.init([Task], strategy: :one_for_one) == {:ok, {flags, children}} + + flags = %{intensity: 1, period: 2, strategy: :one_for_all} + children = [%{id: Task, restart: :temporary, start: {Task, :start_link, [:foo]}}] + + assert Supervisor.init( + [{Task, :foo}], + strategy: :one_for_all, + max_restarts: 1, + max_seconds: 2 + ) == {:ok, {flags, children}} + + assert_raise ArgumentError, "expected :strategy option to be given", fn -> + Supervisor.init([], []) + end + end + + test "init/2 with old and new child specs" do + flags = %{intensity: 3, period: 5, strategy: :one_for_one} + + children = [ + %{id: Task, restart: :temporary, start: {Task, :start_link, [[]]}}, + old_spec = {Task, {Task, :start_link, []}, :permanent, 5000, :worker, [Task]} + ] + + assert Supervisor.init([Task, old_spec], strategy: :one_for_one) == + {:ok, {flags, children}} + end + + test "start_link/2 with via" do + Supervisor.start_link([], strategy: :one_for_one, name: {:via, :global, :via_sup}) + assert Supervisor.which_children({:via, :global, :via_sup}) == [] + end + + test "start_link/3 with global" do + Supervisor.start_link([], strategy: :one_for_one, name: {:global, :global_sup}) + assert Supervisor.which_children({:global, :global_sup}) == [] + end + + test "start_link/3 with local" do + Supervisor.start_link([], strategy: :one_for_one, name: :my_sup) + assert Supervisor.which_children(:my_sup) == [] + end test "start_link/2" do - children = [worker(Stack, [[:hello], [name: :dyn_stack]])] + children = [{Stack, {[:hello], [name: :dyn_stack]}}] {:ok, pid} = Supervisor.start_link(children, strategy: :one_for_one) wait_until_registered(:dyn_stack) @@ -51,32 +162,88 @@ defmodule SupervisorTest do wait_until_registered(:dyn_stack) assert GenServer.call(:dyn_stack, :pop) == :hello + Supervisor.stop(pid) - Process.exit(pid, :normal) + assert_raise ArgumentError, ~r"expected :name option to be one of the following:", fn -> + name = "my_gen_server_name" + Supervisor.start_link(children, name: name, strategy: :one_for_one) + end + + assert_raise ArgumentError, ~r"expected :name option to be one of the following:", fn -> + name = {:invalid_tuple, "my_gen_server_name"} + Supervisor.start_link(children, name: name, strategy: :one_for_one) + end + + assert_raise ArgumentError, ~r"expected :name option to be one of the following:", fn -> + name = {:via, "Via", "my_gen_server_name"} + Supervisor.start_link(children, name: name, strategy: :one_for_one) + end end - test "start_link/3" do - {:ok, pid} = Supervisor.start_link(Stack.Sup, {[:hello], [name: :stat_stack]}, name: :stack_sup) - wait_until_registered(:stack_sup) + test "start_link/2 with old and new specs" do + children = [ + {Stack, {[:hello], []}}, + {:old_stack, {SupervisorTest.Stack, :start_link, [{[:hello], []}]}, :permanent, 5000, + :worker, [SupervisorTest.Stack]} + ] + + {:ok, _} = Supervisor.start_link(children, strategy: :one_for_one) + end + test "start_link/3" do + {:ok, pid} = Supervisor.start_link(Stack.Sup, {[:hello], [name: :stat_stack]}) + wait_until_registered(:stat_stack) assert GenServer.call(:stat_stack, :pop) == :hello - Process.exit(pid, :normal) + Supervisor.stop(pid) end - test "*_child functions" do + describe "start_child/2" do + test "supports old child spec" do + {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) + child = {Task, {Task, :start_link, [fn -> :ok end]}, :temporary, 5000, :worker, [Task]} + assert {:ok, pid} = Supervisor.start_child(pid, child) + assert is_pid(pid) + end + + test "supports new child spec as tuple" do + {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) + child = %{id: Task, restart: :temporary, start: {Task, :start_link, [fn -> :ok end]}} + assert {:ok, pid} = Supervisor.start_child(pid, child) + assert is_pid(pid) + end + + test "supports new child spec" do + {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) + child = {Task, fn -> :timer.sleep(:infinity) end} + assert {:ok, pid} = Supervisor.start_child(pid, child) + assert is_pid(pid) + end + + test "with invalid child spec" do + {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) + + assert Supervisor.start_child(pid, %{}) == {:error, :missing_id} + assert Supervisor.start_child(pid, {1, 2, 3, 4, 5, 6}) == {:error, {:invalid_mfa, 2}} + + assert Supervisor.start_child(pid, %{id: 1, start: {Task, :foo, :bar}}) == + {:error, {:invalid_mfa, {Task, :foo, :bar}}} + end + end + + test "child life-cycle" do {:ok, pid} = Supervisor.start_link([], strategy: :one_for_one) assert Supervisor.which_children(pid) == [] - assert Supervisor.count_children(pid) == - %{specs: 0, active: 0, supervisors: 0, workers: 0} + assert Supervisor.count_children(pid) == %{specs: 0, active: 0, supervisors: 0, workers: 0} - {:ok, stack} = Supervisor.start_child(pid, worker(Stack, [[:hello], []])) + child_spec = Supervisor.child_spec({Stack, {[:hello], []}}, []) + {:ok, stack} = Supervisor.start_child(pid, child_spec) assert GenServer.call(stack, :pop) == :hello assert Supervisor.which_children(pid) == - [{SupervisorTest.Stack, stack, :worker, [SupervisorTest.Stack]}] - assert Supervisor.count_children(pid) == - %{specs: 1, active: 1, supervisors: 0, workers: 1} + [{SupervisorTest.Stack, stack, :worker, [SupervisorTest.Stack]}] + + assert Supervisor.count_children(pid) == %{specs: 1, active: 1, supervisors: 0, workers: 1} assert Supervisor.delete_child(pid, Stack) == {:error, :running} assert Supervisor.terminate_child(pid, Stack) == :ok @@ -86,8 +253,7 @@ defmodule SupervisorTest do assert Supervisor.terminate_child(pid, Stack) == :ok assert Supervisor.delete_child(pid, Stack) == :ok - - Process.exit(pid, :normal) + Supervisor.stop(pid) end defp wait_until_registered(name) do diff --git a/lib/elixir/test/elixir/system_test.exs b/lib/elixir/test/elixir/system_test.exs index 13949d346ec..2df500ed0d7 100644 --- a/lib/elixir/test/elixir/system_test.exs +++ b/lib/elixir/test/elixir/system_test.exs @@ -1,78 +1,318 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule SystemTest do use ExUnit.Case import PathHelpers - test "build_info" do - assert is_map System.build_info - assert not nil?(System.build_info[:version]) - assert not nil?(System.build_info[:tag]) - assert not nil?(System.build_info[:date]) - end + test "build_info/0" do + build_info = System.build_info() + assert is_map(build_info) + assert is_binary(build_info[:build]) + assert is_binary(build_info[:date]) + assert is_binary(build_info[:revision]) + assert is_binary(build_info[:version]) + assert is_binary(build_info[:otp_release]) + + if build_info[:revision] != "" do + assert String.length(build_info[:revision]) >= 7 + end - test "cwd" do - assert is_binary System.cwd - assert is_binary System.cwd! + version_file = Path.join([__DIR__, "../../../..", "VERSION"]) |> Path.expand() + {:ok, version} = File.read(version_file) + assert build_info[:version] == String.trim(version) + assert build_info[:build] =~ "compiled with Erlang/OTP" end - if :file.native_name_encoding == :utf8 do - test "cwd_with_utf8" do - File.mkdir_p(tmp_path("héllò")) + test "user_home/0" do + assert is_binary(System.user_home()) + assert is_binary(System.user_home!()) + end - File.cd!(tmp_path("héllò"), fn -> - assert Path.basename(System.cwd!) == "héllò" - end) - after - File.rm_rf tmp_path("héllò") - end + test "tmp_dir/0" do + assert is_binary(System.tmp_dir()) + assert is_binary(System.tmp_dir!()) end - test "user_home" do - assert is_binary System.user_home - assert is_binary System.user_home! + test "endianness/0" do + assert System.endianness() in [:little, :big] + assert System.endianness() == System.compiled_endianness() end - test "tmp_dir" do - assert is_binary System.tmp_dir - assert is_binary System.tmp_dir! + test "pid/0" do + assert is_binary(System.pid()) end - test "argv" do - list = elixir('-e "IO.inspect System.argv" -- -o opt arg1 arg2 --long-opt 10') - {args, _} = Code.eval_string list, [] + test "argv/0" do + list = elixir('-e "IO.inspect System.argv()" -- -o opt arg1 arg2 --long-opt 10') + {args, _} = Code.eval_string(list, []) assert args == ["-o", "opt", "arg1", "arg2", "--long-opt", "10"] end @test_var "SYSTEM_ELIXIR_ENV_TEST_VAR" - test "env" do + test "*_env/*" do assert System.get_env(@test_var) == nil + assert System.get_env(@test_var, "SAMPLE") == "SAMPLE" + assert System.fetch_env(@test_var) == :error + + message = "could not fetch environment variable #{inspect(@test_var)} because it is not set" + assert_raise ArgumentError, message, fn -> System.fetch_env!(@test_var) end + System.put_env(@test_var, "SAMPLE") + assert System.get_env(@test_var) == "SAMPLE" assert System.get_env()[@test_var] == "SAMPLE" + assert System.fetch_env(@test_var) == {:ok, "SAMPLE"} + assert System.fetch_env!(@test_var) == "SAMPLE" System.delete_env(@test_var) assert System.get_env(@test_var) == nil System.put_env(%{@test_var => "OTHER_SAMPLE"}) assert System.get_env(@test_var) == "OTHER_SAMPLE" + + assert_raise ArgumentError, ~r[cannot execute System.put_env/2 for key with \"=\"], fn -> + System.put_env("FOO=BAR", "BAZ") + end + end + + test "cmd/2 raises for null bytes" do + assert_raise ArgumentError, ~r"cannot execute System.cmd/3 for program with null byte", fn -> + System.cmd("null\0byte", []) + end + end + + test "cmd/3 raises with non-binary arguments" do + assert_raise ArgumentError, ~r"all arguments for System.cmd/3 must be binaries", fn -> + System.cmd("ls", ['/usr']) + end + end + + describe "Windows" do + @describetag :windows + + test "cmd/2" do + assert {"hello\r\n", 0} = System.cmd("cmd", ~w[/c echo hello]) + end + + test "cmd/3 (with options)" do + opts = [ + into: [], + cd: File.cwd!(), + env: %{"foo" => "bar", "baz" => nil}, + arg0: "echo", + stderr_to_stdout: true, + parallelism: true + ] + + assert {["hello\r\n"], 0} = System.cmd("cmd", ~w[/c echo hello], opts) + end + + @echo "echo-elixir-test" + @tag :tmp_dir + test "cmd/3 with absolute and relative paths", config do + echo = Path.join(config.tmp_dir, @echo) + File.mkdir_p!(Path.dirname(echo)) + File.cp!(System.find_executable("cmd"), echo) + + File.cd!(Path.dirname(echo), fn -> + # There is a bug in OTP where find_executable is finding + # entries on the current directory. If this is the case, + # we should avoid the assertion below. + unless System.find_executable(@echo) do + assert :enoent = catch_error(System.cmd(@echo, ~w[/c echo hello])) + end + + assert {"hello\r\n", 0} = + System.cmd(Path.join(File.cwd!(), @echo), ~w[/c echo hello], [{:arg0, "echo"}]) + end) + end + + test "shell/1" do + assert {"hello\r\n", 0} = System.shell("echo hello") + end + + test "shell/2 (with options)" do + opts = [ + into: [], + cd: File.cwd!(), + env: %{"foo" => "bar", "baz" => nil}, + stderr_to_stdout: true, + parallelism: true + ] + + assert {["bar\r\n"], 0} = System.shell("echo %foo%", opts) + end + end + + describe "Unix" do + @describetag :unix + + test "cmd/2" do + assert {"hello\n", 0} = System.cmd("echo", ["hello"]) + end + + test "cmd/3 (with options)" do + opts = [ + into: [], + cd: File.cwd!(), + env: %{"foo" => "bar", "baz" => nil}, + arg0: "echo", + stderr_to_stdout: true, + parallelism: true + ] + + assert {["hello\n"], 0} = System.cmd("echo", ["hello"], opts) + end + + @echo "echo-elixir-test" + @tag :tmp_dir + test "cmd/3 with absolute and relative paths", config do + echo = Path.join(config.tmp_dir, @echo) + File.mkdir_p!(Path.dirname(echo)) + File.cp!(System.find_executable("echo"), echo) + + File.cd!(Path.dirname(echo), fn -> + # There is a bug in OTP where find_executable is finding + # entries on the current directory. If this is the case, + # we should avoid the assertion below. + unless System.find_executable(@echo) do + assert :enoent = catch_error(System.cmd(@echo, ["hello"])) + end + + assert {"hello\n", 0} = + System.cmd(Path.join(File.cwd!(), @echo), ["hello"], [{:arg0, "echo"}]) + end) + end + + test "shell/1" do + assert {"hello\n", 0} = System.shell("echo hello") + end + + test "shell/1 with interpolation" do + assert {"1\n2\n", 0} = System.shell("x=1; echo $x; echo '2'") + end + + @tag timeout: 1_000 + test "shell/1 returns when command awaits input" do + assert {"", 0} = System.shell("cat") + end + + test "shell/1 with comment" do + assert {"1\n", 0} = System.shell("echo '1' # comment") + end + + test "shell/2 (with options)" do + opts = [ + into: [], + cd: File.cwd!(), + env: %{"foo" => "bar", "baz" => nil}, + stderr_to_stdout: true + ] + + assert {["bar\n"], 0} = System.shell("echo $foo", opts) + end end - test "cmd" do - assert is_binary(System.cmd "echo hello") - assert is_list(System.cmd 'echo hello') + @tag :unix + test "vm signals" do + assert System.trap_signal(:sigquit, :example, fn -> :ok end) == {:ok, :example} + assert System.trap_signal(:sigquit, :example, fn -> :ok end) == {:error, :already_registered} + assert {:ok, ref} = System.trap_signal(:sigquit, fn -> :ok end) + + assert System.untrap_signal(:sigquit, :example) == :ok + assert System.trap_signal(:sigquit, :example, fn -> :ok end) == {:ok, :example} + assert System.trap_signal(:sigquit, ref, fn -> :ok end) == {:error, :already_registered} + + assert System.untrap_signal(:sigusr1, :example) == {:error, :not_found} + assert System.untrap_signal(:sigquit, :example) == :ok + assert System.untrap_signal(:sigquit, :example) == {:error, :not_found} + assert System.untrap_signal(:sigquit, ref) == :ok + assert System.untrap_signal(:sigquit, ref) == {:error, :not_found} end - test "find_executable with binary" do + @tag :unix + test "os signals" do + parent = self() + + assert System.trap_signal(:sighup, :example, fn -> + send(parent, :sighup_called) + :ok + end) == {:ok, :example} + + {"", 0} = System.cmd("kill", ["-s", "hup", System.pid()]) + + assert_receive :sighup_called + after + System.untrap_signal(:sighup, :example) + end + + test "find_executable/1" do assert System.find_executable("erl") - assert is_binary System.find_executable("erl") + assert is_binary(System.find_executable("erl")) assert !System.find_executable("does-not-really-exist-from-elixir") + + message = ~r"cannot execute System.find_executable/1 for program with null byte" + + assert_raise ArgumentError, message, fn -> + System.find_executable("null\0byte") + end + end + + test "monotonic_time/0" do + assert is_integer(System.monotonic_time()) + end + + test "monotonic_time/1" do + assert is_integer(System.monotonic_time(:nanosecond)) + assert abs(System.monotonic_time(:microsecond)) < abs(System.monotonic_time(:nanosecond)) + end + + test "system_time/0" do + assert is_integer(System.system_time()) + end + + test "system_time/1" do + assert is_integer(System.system_time(:nanosecond)) + assert abs(System.system_time(:microsecond)) < abs(System.system_time(:nanosecond)) + end + + test "time_offset/0 and time_offset/1" do + assert is_integer(System.time_offset()) + assert is_integer(System.time_offset(:second)) + end + + test "os_time/0" do + assert is_integer(System.os_time()) + end + + test "os_time/1" do + assert is_integer(System.os_time(:nanosecond)) + assert abs(System.os_time(:microsecond)) < abs(System.os_time(:nanosecond)) + end + + test "unique_integer/0 and unique_integer/1" do + assert is_integer(System.unique_integer()) + assert System.unique_integer([:positive]) > 0 + + assert System.unique_integer([:positive, :monotonic]) < + System.unique_integer([:positive, :monotonic]) + end + + test "convert_time_unit/3" do + time = System.monotonic_time(:nanosecond) + assert abs(System.convert_time_unit(time, :nanosecond, :microsecond)) < abs(time) + end + + test "schedulers/0" do + assert System.schedulers() >= 1 + end + + test "schedulers_online/0" do + assert System.schedulers_online() >= 1 end - test "find_executable with list" do - assert System.find_executable('erl') - assert is_list System.find_executable('erl') - assert !System.find_executable('does-not-really-exist-from-elixir') + test "otp_release/0" do + assert is_binary(System.otp_release()) end end diff --git a/lib/elixir/test/elixir/task/supervisor_test.exs b/lib/elixir/test/elixir/task/supervisor_test.exs index 154773b9dbf..aa4e004e5f9 100644 --- a/lib/elixir/test/elixir/task/supervisor_test.exs +++ b/lib/elixir/test/elixir/task/supervisor_test.exs @@ -1,65 +1,220 @@ -Code.require_file "../test_helper.exs", __DIR__ +Code.require_file("../test_helper.exs", __DIR__) defmodule Task.SupervisorTest do - use ExUnit.Case, async: true + use ExUnit.Case + + @moduletag :capture_log setup do {:ok, pid} = Task.Supervisor.start_link() {:ok, supervisor: pid} end - setup do - :error_logger.tty(false) - on_exit fn -> :error_logger.tty(true) end - :ok - end - def wait_and_send(caller, atom) do - send caller, :ready + send(caller, :ready) receive do: (true -> true) - send caller, atom + send(caller, atom) end - test "async/1", config do - parent = self() - fun = fn -> wait_and_send(parent, :done) end - task = Task.Supervisor.async(config[:supervisor], fun) + def sleep(number) do + Process.sleep(number) + number + end + + test "can be supervised directly", config do + modules = [{Task.Supervisor, name: config.test}] + assert {:ok, _} = Supervisor.start_link(modules, strategy: :one_for_one) + assert Process.whereis(config.test) + end + + test "start with spawn_opt" do + {:ok, pid} = Task.Supervisor.start_link(spawn_opt: [priority: :high]) + assert Process.info(pid, :priority) == {:priority, :high} + end + + test "multiple supervisors can be supervised and identified with simple child spec" do + {:ok, _} = Registry.start_link(keys: :unique, name: TaskSup.Registry) + + children = [ + {Task.Supervisor, strategy: :one_for_one, name: :simple_name}, + {Task.Supervisor, strategy: :one_for_one, name: {:global, :global_name}}, + {Task.Supervisor, + strategy: :one_for_one, name: {:via, Registry, {TaskSup.Registry, "via_name"}}} + ] + + assert {:ok, supsup} = Supervisor.start_link(children, strategy: :one_for_one) + + assert {:ok, no_name_dynsup} = + Supervisor.start_child(supsup, {Task.Supervisor, strategy: :one_for_one}) + + assert Task.Supervisor.children(:simple_name) == [] + assert Task.Supervisor.children({:global, :global_name}) == [] + assert Task.Supervisor.children({:via, Registry, {TaskSup.Registry, "via_name"}}) == [] + assert Task.Supervisor.children(no_name_dynsup) == [] + + assert Supervisor.start_child(supsup, {Task.Supervisor, strategy: :one_for_one}) == + {:error, {:already_started, no_name_dynsup}} + end + + test "counts and returns children", config do + assert Task.Supervisor.children(config[:supervisor]) == [] + + assert Supervisor.count_children(config[:supervisor]) == + %{active: 0, specs: 0, supervisors: 0, workers: 0} + + assert DynamicSupervisor.count_children(config[:supervisor]) == + %{active: 0, specs: 0, supervisors: 0, workers: 0} + end + + describe "async/1" do + test "spawns tasks under the supervisor", config do + parent = self() + fun = fn -> wait_and_send(parent, :done) end + task = Task.Supervisor.async(config[:supervisor], fun) + assert Task.Supervisor.children(config[:supervisor]) == [task.pid] + + # Assert the struct + assert task.__struct__ == Task + assert is_pid(task.pid) + assert is_reference(task.ref) + + # Assert the link + {:links, links} = Process.info(self(), :links) + assert task.pid in links + + receive do: (:ready -> :ok) + + # Assert the initial call + {:name, fun_name} = Function.info(fun, :name) + assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(task.pid) + + # Run the task + send(task.pid, true) + + # Assert response and monitoring messages + ref = task.ref + assert_receive {^ref, :done} + assert_receive {:DOWN, ^ref, _, _, :normal} + end + + test "with custom shutdown", config do + Process.flag(:trap_exit, true) + parent = self() + + fun = fn -> wait_and_send(parent, :done) end + %{pid: pid} = Task.Supervisor.async(config[:supervisor], fun, shutdown: :brutal_kill) + Process.exit(config[:supervisor], :shutdown) + assert_receive {:DOWN, _, _, ^pid, :killed} + end + + test "raises when :max_children is reached" do + {:ok, sup} = Task.Supervisor.start_link(max_children: 1) + Task.Supervisor.async(sup, fn -> Process.sleep(:infinity) end) + + assert_raise RuntimeError, ~r/reached the maximum number of tasks/, fn -> + Task.Supervisor.async(sup, fn -> :ok end) + end + end + + test "with $callers", config do + sup = config[:supervisor] + grandparent = self() + + Task.Supervisor.async(sup, fn -> + parent = self() + assert Process.get(:"$callers") == [grandparent] + assert Process.get(:"$ancestors") == [sup, grandparent] + + Task.Supervisor.async(sup, fn -> + assert Process.get(:"$callers") == [parent, grandparent] + assert Process.get(:"$ancestors") == [sup, grandparent] + end) + |> Task.await() + end) + |> Task.await() + end + end + + test "async/3", config do + args = [self(), :done] + task = Task.Supervisor.async(config[:supervisor], __MODULE__, :wait_and_send, args) assert Task.Supervisor.children(config[:supervisor]) == [task.pid] - # Assert the struct + receive do: (:ready -> :ok) + assert {__MODULE__, :wait_and_send, 2} === :proc_lib.translate_initial_call(task.pid) + + send(task.pid, true) assert task.__struct__ == Task - assert is_pid task.pid - assert is_reference task.ref + assert task.mfa == {__MODULE__, :wait_and_send, 2} + assert Task.await(task) == :done + end - # Assert the link - {:links, links} = Process.info(self, :links) - assert task.pid in links + describe "async_nolink/1" do + test "spawns a task under the supervisor without linking to the caller", config do + parent = self() + fun = fn -> wait_and_send(parent, :done) end + task = Task.Supervisor.async_nolink(config[:supervisor], fun) + assert Task.Supervisor.children(config[:supervisor]) == [task.pid] - receive do: (:ready -> :ok) + # Assert the struct + assert task.__struct__ == Task + assert is_pid(task.pid) + assert is_reference(task.ref) + assert task.mfa == {:erlang, :apply, 2} + + # Refute the link + {:links, links} = Process.info(self(), :links) + refute task.pid in links + + receive do: (:ready -> :ok) - # Assert the initial call - {:name, fun_name} = :erlang.fun_info(fun, :name) - assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(task.pid) + # Assert the initial call + {:name, fun_name} = Function.info(fun, :name) + assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(task.pid) - # Run the task - send task.pid, true + # Run the task + send(task.pid, true) - # Assert response and monitoring messages - ref = task.ref - assert_receive {^ref, :done} - assert_receive {:DOWN, ^ref, _, _, :normal} + # Assert response and monitoring messages + ref = task.ref + assert_receive {^ref, :done} + assert_receive {:DOWN, ^ref, _, _, :normal} + end + + test "with custom shutdown", config do + Process.flag(:trap_exit, true) + parent = self() + + fun = fn -> wait_and_send(parent, :done) end + %{pid: pid} = Task.Supervisor.async_nolink(config[:supervisor], fun, shutdown: :brutal_kill) + + Process.exit(config[:supervisor], :shutdown) + assert_receive {:DOWN, _, _, ^pid, :killed} + end + + test "raises when :max_children is reached" do + {:ok, sup} = Task.Supervisor.start_link(max_children: 1) + + Task.Supervisor.async_nolink(sup, fn -> Process.sleep(:infinity) end) + + assert_raise RuntimeError, ~r/reached the maximum number of tasks/, fn -> + Task.Supervisor.async_nolink(sup, fn -> :ok end) + end + end end - test "async/3", config do - task = Task.Supervisor.async(config[:supervisor], __MODULE__, :wait_and_send, [self(), :done]) + test "async_nolink/3", config do + args = [self(), :done] + task = Task.Supervisor.async_nolink(config[:supervisor], __MODULE__, :wait_and_send, args) assert Task.Supervisor.children(config[:supervisor]) == [task.pid] receive do: (:ready -> :ok) assert {__MODULE__, :wait_and_send, 2} === :proc_lib.translate_initial_call(task.pid) - send task.pid, true + send(task.pid, true) assert task.__struct__ == Task + assert task.mfa == {__MODULE__, :wait_and_send, 2} assert Task.await(task) == :done end @@ -69,57 +224,321 @@ defmodule Task.SupervisorTest do {:ok, pid} = Task.Supervisor.start_child(config[:supervisor], fun) assert Task.Supervisor.children(config[:supervisor]) == [pid] - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) refute pid in links receive do: (:ready -> :ok) - {:name, fun_name} = :erlang.fun_info(fun, :name) + {:name, fun_name} = Function.info(fun, :name) assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(pid) - send pid, true + send(pid, true) assert_receive :done end test "start_child/3", config do - {:ok, pid} = Task.Supervisor.start_child(config[:supervisor], __MODULE__, :wait_and_send, [self(), :done]) + args = [self(), :done] + + {:ok, pid} = + Task.Supervisor.start_child(config[:supervisor], __MODULE__, :wait_and_send, args) + assert Task.Supervisor.children(config[:supervisor]) == [pid] - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) refute pid in links receive do: (:ready -> :ok) assert {__MODULE__, :wait_and_send, 2} === :proc_lib.translate_initial_call(pid) - send pid, true + send(pid, true) + assert_receive :done + + assert_raise FunctionClauseError, fn -> + Task.Supervisor.start_child(config[:supervisor], __MODULE__, :wait_and_send, :illegal_arg) + end + + assert_raise FunctionClauseError, fn -> + args = [self(), :done] + Task.Supervisor.start_child(config[:supervisor], __MODULE__, "wait_and_send", args) + end + end + + test "start_child/1 with custom shutdown", config do + Process.flag(:trap_exit, true) + parent = self() + + fun = fn -> wait_and_send(parent, :done) end + {:ok, pid} = Task.Supervisor.start_child(config[:supervisor], fun, shutdown: :brutal_kill) + + Process.monitor(pid) + Process.exit(config[:supervisor], :shutdown) + assert_receive {:DOWN, _, _, ^pid, :killed} + end + + test "start_child/1 with custom restart", config do + parent = self() + + fun = fn -> wait_and_send(parent, :done) end + {:ok, pid} = Task.Supervisor.start_child(config[:supervisor], fun, restart: :permanent) + + assert_receive :ready + Process.monitor(pid) + Process.exit(pid, :shutdown) + assert_receive {:DOWN, _, _, ^pid, :shutdown} + assert_receive :ready + end + + test "start_child/1 with $callers", config do + sup = config[:supervisor] + grandparent = self() + + Task.Supervisor.start_child(sup, fn -> + parent = self() + assert Process.get(:"$callers") == [grandparent] + assert Process.get(:"$ancestors") == [sup, grandparent] + + Task.Supervisor.start_child(sup, fn -> + assert Process.get(:"$callers") == [parent, grandparent] + assert Process.get(:"$ancestors") == [sup, grandparent] + send(grandparent, :done) + end) + end) + assert_receive :done end test "terminate_child/2", config do - {:ok, pid} = Task.Supervisor.start_child(config[:supervisor], __MODULE__, :wait_and_send, [self(), :done]) + args = [self(), :done] + + {:ok, pid} = + Task.Supervisor.start_child(config[:supervisor], __MODULE__, :wait_and_send, args) + assert Task.Supervisor.children(config[:supervisor]) == [pid] assert Task.Supervisor.terminate_child(config[:supervisor], pid) == :ok assert Task.Supervisor.children(config[:supervisor]) == [] - assert Task.Supervisor.terminate_child(config[:supervisor], pid) == :ok + assert Task.Supervisor.terminate_child(config[:supervisor], pid) == {:error, :not_found} end - test "await/1 exits on task throw", config do - Process.flag(:trap_exit, true) - task = Task.Supervisor.async(config[:supervisor], fn -> throw :unknown end) - assert {{{:nocatch, :unknown}, _}, {Task, :await, [^task, 5000]}} = - catch_exit(Task.await(task)) + describe "await/1" do + if System.otp_release() >= "24" do + test "demonitors and unalias on timeout", config do + task = + Task.Supervisor.async(config[:supervisor], fn -> + assert_receive :go + :done + end) + + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + new_ref = Process.monitor(task.pid) + old_ref = task.ref + + send(task.pid, :go) + assert_receive {:DOWN, ^new_ref, _, _, _} + refute_received {^old_ref, :done} + refute_received {:DOWN, ^old_ref, _, _, _} + end + end + + test "exits on task throw", config do + Process.flag(:trap_exit, true) + task = Task.Supervisor.async(config[:supervisor], fn -> throw(:unknown) end) + + assert {{{:nocatch, :unknown}, _}, {Task, :await, [^task, 5000]}} = + catch_exit(Task.await(task)) + end + + test "exits on task error", config do + Process.flag(:trap_exit, true) + task = Task.Supervisor.async(config[:supervisor], fn -> raise "oops" end) + assert {{%RuntimeError{}, _}, {Task, :await, [^task, 5000]}} = catch_exit(Task.await(task)) + end + + test "exits on task exit", config do + Process.flag(:trap_exit, true) + task = Task.Supervisor.async(config[:supervisor], fn -> exit(:unknown) end) + assert {:unknown, {Task, :await, [^task, 5000]}} = catch_exit(Task.await(task)) + end end - test "await/1 exits on task error", config do - Process.flag(:trap_exit, true) - task = Task.Supervisor.async(config[:supervisor], fn -> raise "oops" end) - assert {{%RuntimeError{}, _}, {Task, :await, [^task, 5000]}} = - catch_exit(Task.await(task)) + describe "async_stream" do + @opts [] + test "streams an enumerable with fun", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream(1..4, &sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "streams an enumerable with mfa", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream(1..4, __MODULE__, :sleep, [], @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "streams an enumerable without leaking tasks", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream(1..4, &sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + + refute_received _ + end + + test "streams an enumerable with slowest first", %{supervisor: supervisor} do + Process.flag(:trap_exit, true) + + assert supervisor + |> Task.Supervisor.async_stream(4..1, &sleep/1, @opts) + |> Enum.to_list() == [ok: 4, ok: 3, ok: 2, ok: 1] + end + + test "streams an enumerable with exits", %{supervisor: supervisor} do + Process.flag(:trap_exit, true) + + assert supervisor + |> Task.Supervisor.async_stream(1..4, &yield_and_exit(Integer.to_string(&1)), @opts) + |> Enum.to_list() == [exit: "1", exit: "2", exit: "3", exit: "4"] + end + + test "shuts down unused tasks", %{supervisor: supervisor} do + collection = [0, :infinity, :infinity, :infinity] + + assert supervisor + |> Task.Supervisor.async_stream(collection, &sleep/1, @opts) + |> Enum.take(1) == [ok: 0] + + assert Process.info(self(), :links) == {:links, [supervisor]} + end + + test "shuts down unused tasks without leaking messages", %{supervisor: supervisor} do + collection = [0, :infinity, :infinity, :infinity] + + assert supervisor + |> Task.Supervisor.async_stream(collection, &sleep/1, @opts) + |> Enum.take(1) == [ok: 0] + + refute_received _ + end + + test "raises an error if :max_children is reached with clean stream shutdown", + %{supervisor: unused_supervisor} do + {:ok, supervisor} = Task.Supervisor.start_link(max_children: 1) + collection = [:infinity, :infinity, :infinity] + + assert_raise RuntimeError, ~r/reached the maximum number of tasks/, fn -> + supervisor + |> Task.Supervisor.async_stream(collection, &sleep/1, max_concurrency: 2) + |> Enum.to_list() + end + + {:links, links} = Process.info(self(), :links) + assert MapSet.new(links) == MapSet.new([unused_supervisor, supervisor]) + refute_received _ + end + + test "with $callers", config do + sup = config[:supervisor] + grandparent = self() + + Task.Supervisor.async_stream(sup, [1], fn 1 -> + parent = self() + assert Process.get(:"$callers") == [grandparent] + assert Process.get(:"$ancestors") == [sup, grandparent] + + Task.Supervisor.async_stream(sup, [1], fn 1 -> + assert Process.get(:"$callers") == [parent, grandparent] + assert Process.get(:"$ancestors") == [sup, grandparent] + send(grandparent, :done) + end) + |> Stream.run() + end) + |> Stream.run() + + assert_receive :done + end + + test "consuming from another process", config do + parent = self() + stream = Task.Supervisor.async_stream(config[:supervisor], [1, 2, 3], &send(parent, &1)) + Task.start(Stream, :run, [stream]) + assert_receive 1 + assert_receive 2 + assert_receive 3 + end end - test "await/1 exits on task exit", config do - Process.flag(:trap_exit, true) - task = Task.Supervisor.async(config[:supervisor], fn -> exit :unknown end) - assert {:unknown, {Task, :await, [^task, 5000]}} = - catch_exit(Task.await(task)) + describe "async_stream_nolink" do + @opts [max_concurrency: 4] + + test "streams an enumerable with fun", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream_nolink(1..4, &sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "streams an enumerable with mfa", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream_nolink(1..4, __MODULE__, :sleep, [], @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "streams an enumerable without leaking tasks", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream_nolink(1..4, &sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + + refute_received _ + end + + test "streams an enumerable with slowest first", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream_nolink(4..1, &sleep/1, @opts) + |> Enum.to_list() == [ok: 4, ok: 3, ok: 2, ok: 1] + end + + test "streams an enumerable with exits", %{supervisor: supervisor} do + assert supervisor + |> Task.Supervisor.async_stream_nolink(1..4, &yield_and_exit/1, @opts) + |> Enum.to_list() == [exit: 1, exit: 2, exit: 3, exit: 4] + end + + test "shuts down unused tasks", %{supervisor: supervisor} do + collection = [0, :infinity, :infinity, :infinity] + + assert supervisor + |> Task.Supervisor.async_stream_nolink(collection, &sleep/1, @opts) + |> Enum.take(1) == [ok: 0] + + assert Process.info(self(), :links) == {:links, [supervisor]} + end + + test "shuts down unused tasks without leaking messages", %{supervisor: supervisor} do + collection = [0, :infinity, :infinity, :infinity] + + assert supervisor + |> Task.Supervisor.async_stream_nolink(collection, &sleep/1, @opts) + |> Enum.take(1) == [ok: 0] + + refute_received _ + end + + test "raises an error if :max_children is reached with clean stream shutdown", + %{supervisor: unused_supervisor} do + {:ok, supervisor} = Task.Supervisor.start_link(max_children: 1) + collection = [:infinity, :infinity, :infinity] + + assert_raise RuntimeError, ~r/reached the maximum number of tasks/, fn -> + supervisor + |> Task.Supervisor.async_stream_nolink(collection, &sleep/1, max_concurrency: 2) + |> Enum.to_list() + end + + {:links, links} = Process.info(self(), :links) + assert MapSet.new(links) == MapSet.new([unused_supervisor, supervisor]) + refute_received _ + end + end + + def yield_and_exit(value) do + # We call yield first so we give the parent a chance to monitor + :erlang.yield() + :erlang.exit(value) end end diff --git a/lib/elixir/test/elixir/task_test.exs b/lib/elixir/test/elixir/task_test.exs index a75d80228b7..71598071837 100644 --- a/lib/elixir/test/elixir/task_test.exs +++ b/lib/elixir/test/elixir/task_test.exs @@ -1,18 +1,66 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule TaskTest do - use ExUnit.Case, async: true - - setup do - :error_logger.tty(false) - on_exit fn -> :error_logger.tty(true) end - :ok - end + use ExUnit.Case + doctest Task + @moduletag :capture_log def wait_and_send(caller, atom) do - send caller, :ready + send(caller, :ready) receive do: (true -> true) - send caller, atom + send(caller, atom) + end + + defp create_task_in_other_process do + caller = self() + spawn(fn -> send(caller, Task.async(fn -> nil end)) end) + receive do: (task -> task) + end + + defp create_dummy_task(reason) do + {pid, ref} = spawn_monitor(Kernel, :exit, [reason]) + + receive do + {:DOWN, ^ref, _, _, _} -> + %Task{ref: ref, pid: pid, owner: self(), mfa: {__MODULE__, :create_dummy_task, 1}} + end + end + + def sleep(number) do + Process.sleep(number) + number + end + + def wait_until_down(task) do + ref = Process.monitor(task.pid) + assert_receive {:DOWN, ^ref, _, _, _} + end + + test "can be supervised directly" do + assert {:ok, _} = Supervisor.start_link([{Task, fn -> :ok end}], strategy: :one_for_one) + end + + test "generates child_spec/1" do + defmodule MyTask do + use Task + end + + assert MyTask.child_spec([:hello]) == %{ + id: MyTask, + restart: :temporary, + start: {MyTask, :start_link, [[:hello]]} + } + + defmodule CustomTask do + use Task, id: :id, restart: :permanent, shutdown: :infinity, start: {:foo, :bar, []} + end + + assert CustomTask.child_spec([:hello]) == %{ + id: :id, + restart: :permanent, + shutdown: :infinity, + start: {:foo, :bar, []} + } end test "async/1" do @@ -22,21 +70,22 @@ defmodule TaskTest do # Assert the struct assert task.__struct__ == Task - assert is_pid task.pid - assert is_reference task.ref + assert is_pid(task.pid) + assert is_reference(task.ref) + assert task.mfa == {:erlang, :apply, 2} # Assert the link - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) assert task.pid in links receive do: (:ready -> :ok) # Assert the initial call - {:name, fun_name} = :erlang.fun_info(fun, :name) + {:name, fun_name} = Function.info(fun, :name) assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(task.pid) # Run the task - send task.pid, true + send(task.pid, true) # Assert response and monitoring messages ref = task.ref @@ -47,98 +96,931 @@ defmodule TaskTest do test "async/3" do task = Task.async(__MODULE__, :wait_and_send, [self(), :done]) assert task.__struct__ == Task + assert task.mfa == {__MODULE__, :wait_and_send, 2} - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) assert task.pid in links receive do: (:ready -> :ok) - assert {__MODULE__, :wait_and_send, 2} === :proc_lib.translate_initial_call(task.pid) send(task.pid, true) - assert Task.await(task) === :done assert_receive :done end + test "async with $callers" do + grandparent = self() + + Task.async(fn -> + parent = self() + assert Process.get(:"$callers") == [grandparent] + + Task.async(fn -> + assert Process.get(:"$callers") == [parent, grandparent] + end) + |> Task.await() + end) + |> Task.await() + end + + test "start/1" do + parent = self() + fun = fn -> wait_and_send(parent, :done) end + {:ok, pid} = Task.start(fun) + + {:links, links} = Process.info(self(), :links) + refute pid in links + + receive do: (:ready -> :ok) + + {:name, fun_name} = Function.info(fun, :name) + assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(pid) + + send(pid, true) + assert_receive :done + end + + test "start/3" do + {:ok, pid} = Task.start(__MODULE__, :wait_and_send, [self(), :done]) + + {:links, links} = Process.info(self(), :links) + refute pid in links + + receive do: (:ready -> :ok) + + assert {__MODULE__, :wait_and_send, 2} === :proc_lib.translate_initial_call(pid) + + send(pid, true) + assert_receive :done + end + + test "completed/1" do + task = Task.completed(:done) + assert task.__struct__ == Task + + refute task.pid + + assert Task.await(task) == :done + end + test "start_link/1" do parent = self() fun = fn -> wait_and_send(parent, :done) end {:ok, pid} = Task.start_link(fun) - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) assert pid in links receive do: (:ready -> :ok) - {:name, fun_name} = :erlang.fun_info(fun, :name) + {:name, fun_name} = Function.info(fun, :name) assert {__MODULE__, fun_name, 0} === :proc_lib.translate_initial_call(pid) - send pid, true + send(pid, true) assert_receive :done end test "start_link/3" do {:ok, pid} = Task.start_link(__MODULE__, :wait_and_send, [self(), :done]) - {:links, links} = Process.info(self, :links) + {:links, links} = Process.info(self(), :links) assert pid in links receive do: (:ready -> :ok) assert {__MODULE__, :wait_and_send, 2} === :proc_lib.translate_initial_call(pid) - send pid, true + send(pid, true) + assert_receive :done + end + + test "start_link with $callers" do + grandparent = self() + + Task.start_link(fn -> + parent = self() + assert Process.get(:"$callers") == [grandparent] + + Task.start_link(fn -> + assert Process.get(:"$callers") == [parent, grandparent] + send(grandparent, :done) + end) + end) + assert_receive :done end - test "await/1 exits on timeout" do - task = %Task{ref: make_ref()} - assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + if System.otp_release() >= "24" do + describe "ignore/1" do + test "discards on time replies" do + task = Task.async(fn -> :ok end) + wait_until_down(task) + assert Task.ignore(task) == {:ok, :ok} + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + end + + test "discards late replies" do + task = Task.async(fn -> assert_receive(:go) && :ok end) + assert Task.ignore(task) == nil + send(task.pid, :go) + wait_until_down(task) + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + end + + test "discards on-time failures" do + Process.flag(:trap_exit, true) + task = Task.async(fn -> exit(:oops) end) + wait_until_down(task) + assert Task.ignore(task) == {:exit, :oops} + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + end + + test "discards late failures" do + task = Task.async(fn -> assert_receive(:go) && exit(:oops) end) + assert Task.ignore(task) == nil + send(task.pid, :go) + wait_until_down(task) + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + end + + test "exits on :noconnection" do + ref = make_ref() + task = %Task{ref: ref, pid: self(), owner: self(), mfa: {__MODULE__, :test, 1}} + send(self(), {:DOWN, ref, self(), self(), :noconnection}) + assert catch_exit(Task.ignore(task)) |> elem(0) == {:nodedown, :nonode@nohost} + end + end end - test "await/1 exits on normal exit" do - task = Task.async(fn -> exit :normal end) - assert catch_exit(Task.await(task)) == {:normal, {Task, :await, [task, 5000]}} + describe "await/2" do + if System.otp_release() >= "24" do + test "demonitors and unalias on timeout" do + task = + Task.async(fn -> + assert_receive :go + :done + end) + + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + send(task.pid, :go) + ref = task.ref + + wait_until_down(task) + refute_received {^ref, :done} + refute_received {:DOWN, ^ref, _, _, _} + end + end + + test "exits on timeout" do + task = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + assert catch_exit(Task.await(task, 0)) == {:timeout, {Task, :await, [task, 0]}} + end + + test "exits on normal exit" do + task = Task.async(fn -> exit(:normal) end) + assert catch_exit(Task.await(task)) == {:normal, {Task, :await, [task, 5000]}} + end + + test "exits on task throw" do + Process.flag(:trap_exit, true) + task = Task.async(fn -> throw(:unknown) end) + + assert {{{:nocatch, :unknown}, _}, {Task, :await, [^task, 5000]}} = + catch_exit(Task.await(task)) + end + + test "exits on task error" do + Process.flag(:trap_exit, true) + task = Task.async(fn -> raise "oops" end) + assert {{%RuntimeError{}, _}, {Task, :await, [^task, 5000]}} = catch_exit(Task.await(task)) + end + + @compile {:no_warn_undefined, :module_does_not_exist} + + test "exits on task undef module error" do + Process.flag(:trap_exit, true) + task = Task.async(&:module_does_not_exist.undef/0) + + assert {exit_status, mfa} = catch_exit(Task.await(task)) + assert {:undef, [{:module_does_not_exist, :undef, _, _} | _]} = exit_status + assert {Task, :await, [^task, 5000]} = mfa + end + + @compile {:no_warn_undefined, {TaskTest, :undef, 0}} + + test "exits on task undef function error" do + Process.flag(:trap_exit, true) + task = Task.async(&TaskTest.undef/0) + + assert {{:undef, [{TaskTest, :undef, _, _} | _]}, {Task, :await, [^task, 5000]}} = + catch_exit(Task.await(task)) + end + + test "exits on task exit" do + Process.flag(:trap_exit, true) + task = Task.async(fn -> exit(:unknown) end) + assert {:unknown, {Task, :await, [^task, 5000]}} = catch_exit(Task.await(task)) + end + + test "exits on :noconnection" do + ref = make_ref() + task = %Task{ref: ref, pid: self(), owner: self(), mfa: {__MODULE__, :test, 1}} + send(self(), {:DOWN, ref, :process, self(), :noconnection}) + assert catch_exit(Task.await(task)) |> elem(0) == {:nodedown, :nonode@nohost} + end + + test "exits on :noconnection from named monitor" do + ref = make_ref() + task = %Task{ref: ref, owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + send(self(), {:DOWN, ref, :process, {:name, :node}, :noconnection}) + assert catch_exit(Task.await(task)) |> elem(0) == {:nodedown, :node} + end + + test "raises when invoked from a non-owner process" do + task = create_task_in_other_process() + + message = + "task #{inspect(task)} must be queried from the owner " <> + "but was queried from #{inspect(self())}" + + assert_raise ArgumentError, message, fn -> Task.await(task, 1) end + end + end + + describe "await_many/2" do + test "returns list of replies" do + tasks = for val <- [1, 3, 9], do: Task.async(fn -> val end) + assert Task.await_many(tasks) == [1, 3, 9] + end + + test "returns replies in input order ignoring response order" do + refs = [ref_1 = make_ref(), ref_2 = make_ref(), ref_3 = make_ref()] + + tasks = + Enum.map(refs, fn ref -> + %Task{ref: ref, owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + end) + + send(self(), {ref_2, 3}) + send(self(), {ref_3, 9}) + send(self(), {ref_1, 1}) + assert Task.await_many(tasks) == [1, 3, 9] + end + + test "returns an empty list immediately" do + assert Task.await_many([]) == [] + end + + test "ignores messages from other processes" do + other_ref = make_ref() + tasks = for val <- [:a, :b], do: Task.async(fn -> val end) + send(self(), other_ref) + send(self(), {other_ref, :z}) + send(self(), {:DOWN, other_ref, :process, 1, :goodbye}) + assert Task.await_many(tasks) == [:a, :b] + assert_received ^other_ref + assert_received {^other_ref, :z} + assert_received {:DOWN, ^other_ref, :process, 1, :goodbye} + end + + test "ignores additional messages after reply" do + refs = [ref_1 = make_ref(), ref_2 = make_ref()] + + tasks = + Enum.map(refs, fn ref -> + %Task{ref: ref, owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + end) + + send(self(), {ref_2, :b}) + send(self(), {ref_2, :other}) + send(self(), {ref_1, :a}) + assert Task.await_many(tasks) == [:a, :b] + assert_received {^ref_2, :other} + end + + test "exits on timeout" do + tasks = [Task.async(fn -> Process.sleep(:infinity) end)] + assert catch_exit(Task.await_many(tasks, 0)) == {:timeout, {Task, :await_many, [tasks, 0]}} + end + + test "exits with same reason when task exits" do + tasks = [Task.async(fn -> exit(:normal) end)] + assert catch_exit(Task.await_many(tasks)) == {:normal, {Task, :await_many, [tasks, 5000]}} + end + + test "exits immediately when any task exits" do + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + Task.async(fn -> exit(:normal) end) + ] + + assert catch_exit(Task.await_many(tasks)) == {:normal, {Task, :await_many, [tasks, 5000]}} + end + + test "exits immediately when any task crashes" do + Process.flag(:trap_exit, true) + + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + Task.async(fn -> exit(:unknown) end) + ] + + assert catch_exit(Task.await_many(tasks)) == {:unknown, {Task, :await_many, [tasks, 5000]}} + + # Make sure all monitors are cleared up afterwards too + Enum.each(tasks, &Process.exit(&1.pid, :kill)) + refute_received {:DOWN, _, _, _, _} + end + + test "exits immediately when any task throws" do + Process.flag(:trap_exit, true) + + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + Task.async(fn -> throw(:unknown) end) + ] + + assert {{{:nocatch, :unknown}, _}, {Task, :await_many, [^tasks, 5000]}} = + catch_exit(Task.await_many(tasks)) + end + + test "exits immediately on any task error" do + Process.flag(:trap_exit, true) + + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + Task.async(fn -> raise "oops" end) + ] + + assert {{%RuntimeError{}, _}, {Task, :await_many, [^tasks, 5000]}} = + catch_exit(Task.await_many(tasks)) + end + + test "exits immediately on :noconnection" do + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + %Task{ref: ref = make_ref(), owner: self(), pid: self(), mfa: {__MODULE__, :test, 1}} + ] + + send(self(), {:DOWN, ref, :process, self(), :noconnection}) + assert catch_exit(Task.await_many(tasks)) |> elem(0) == {:nodedown, :nonode@nohost} + end + + test "exits immediately on :noconnection from named monitor" do + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + %Task{ref: ref = make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + ] + + send(self(), {:DOWN, ref, :process, {:name, :node}, :noconnection}) + assert catch_exit(Task.await_many(tasks)) |> elem(0) == {:nodedown, :node} + end + + test "raises when invoked from a non-owner process" do + tasks = [ + Task.async(fn -> Process.sleep(:infinity) end), + bad_task = create_task_in_other_process() + ] + + message = + "task #{inspect(bad_task)} must be queried from the owner " <> + "but was queried from #{inspect(self())}" + + assert_raise ArgumentError, message, fn -> Task.await_many(tasks, 1) end + end end - test "await/1 exits on task throw" do - Process.flag(:trap_exit, true) - task = Task.async(fn -> throw :unknown end) - assert {{{:nocatch, :unknown}, _}, {Task, :await, [^task, 5000]}} = - catch_exit(Task.await(task)) + describe "yield/2" do + test "returns {:ok, result} when reply and :DOWN in message queue" do + task = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, self(), :abnormal}) + assert Task.yield(task, 0) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns nil on timeout" do + task = %Task{ref: make_ref(), pid: nil, owner: self(), mfa: {__MODULE__, :test, 1}} + assert Task.yield(task, 0) == nil + end + + test "return exit on normal exit" do + task = Task.async(fn -> exit(:normal) end) + assert Task.yield(task) == {:exit, :normal} + end + + test "exits on :noconnection" do + ref = make_ref() + task = %Task{ref: ref, pid: self(), owner: self(), mfa: {__MODULE__, :test, 1}} + send(self(), {:DOWN, ref, self(), self(), :noconnection}) + assert catch_exit(Task.yield(task)) |> elem(0) == {:nodedown, :nonode@nohost} + end + + test "raises when invoked from a non-owner process" do + task = create_task_in_other_process() + + message = + "task #{inspect(task)} must be queried from the owner " <> + "but was queried from #{inspect(self())}" + + assert_raise ArgumentError, message, fn -> Task.yield(task, 1) end + end end - test "await/1 exits on task error" do - Process.flag(:trap_exit, true) - task = Task.async(fn -> raise "oops" end) - assert {{%RuntimeError{}, _}, {Task, :await, [^task, 5000]}} = - catch_exit(Task.await(task)) + describe "yield_many/2" do + test "returns {:ok, result} when reply and :DOWN in message queue" do + task = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, self(), :abnormal}) + assert Task.yield_many([task], 0) == [{task, {:ok, :result}}] + refute_received {:DOWN, _, _, _, _} + end + + test "returns nil on timeout" do + task = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + assert Task.yield_many([task], 0) == [{task, nil}] + end + + test "return exit on normal exit" do + task = Task.async(fn -> exit(:normal) end) + assert Task.yield_many([task]) == [{task, {:exit, :normal}}] + end + + test "exits on :noconnection" do + ref = make_ref() + task = %Task{ref: ref, pid: self(), owner: self(), mfa: {__MODULE__, :test, 1}} + send(self(), {:DOWN, ref, :process, self(), :noconnection}) + assert catch_exit(Task.yield_many([task])) |> elem(0) == {:nodedown, :nonode@nohost} + end + + test "raises when invoked from a non-owner process" do + task = create_task_in_other_process() + + message = + "task #{inspect(task)} must be queried from the owner " <> + "but was queried from #{inspect(self())}" + + assert_raise ArgumentError, message, fn -> Task.yield_many([task], 1) end + end + + test "returns results from multiple tasks" do + task1 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + task2 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + task3 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + + send(self(), {task1.ref, :result}) + send(self(), {:DOWN, task3.ref, :process, self(), :normal}) + + assert Task.yield_many([task1, task2, task3], 0) == + [{task1, {:ok, :result}}, {task2, nil}, {task3, {:exit, :normal}}] + end + + test "returns results on infinity timeout" do + task1 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + task2 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + task3 = %Task{ref: make_ref(), owner: self(), pid: nil, mfa: {__MODULE__, :test, 1}} + + send(self(), {task1.ref, :result}) + send(self(), {task2.ref, :result}) + send(self(), {:DOWN, task3.ref, :process, self(), :normal}) + + assert Task.yield_many([task1, task2, task3], :infinity) == + [{task1, {:ok, :result}}, {task2, {:ok, :result}}, {task3, {:exit, :normal}}] + end end - test "await/1 exits on task exit" do - Process.flag(:trap_exit, true) - task = Task.async(fn -> exit :unknown end) - assert {:unknown, {Task, :await, [^task, 5000]}} = - catch_exit(Task.await(task)) + describe "shutdown/2" do + test "returns {:ok, result} when reply and abnormal :DOWN in message queue" do + task = create_dummy_task(:abnormal) + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, task.pid, :abnormal}) + assert Task.shutdown(task) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns {:ok, result} when reply and normal :DOWN in message queue" do + task = create_dummy_task(:normal) + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, task.pid, :normal}) + assert Task.shutdown(task) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns {:ok, result} when reply and shut down :DOWN in message queue" do + task = create_dummy_task(:shutdown) + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, task.pid, :shutdown}) + assert Task.shutdown(task) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns nil on shutting down task" do + task = Task.async(:timer, :sleep, [:infinity]) + assert Task.shutdown(task) == nil + end + + test "returns exit on abnormal :DOWN in message queue" do + task = create_dummy_task(:abnormal) + send(self(), {:DOWN, task.ref, :process, task.pid, :abnormal}) + assert Task.shutdown(task) == {:exit, :abnormal} + end + + test "returns exit on normal :DOWN in message queue" do + task = create_dummy_task(:normal) + send(self(), {:DOWN, task.ref, :process, task.pid, :normal}) + assert Task.shutdown(task) == {:exit, :normal} + end + + test "returns nil on shutdown :DOWN in message queue" do + task = create_dummy_task(:shutdown) + send(self(), {:DOWN, task.ref, :process, task.pid, :shutdown}) + assert Task.shutdown(task) == nil + end + + test "returns exit on killed :DOWN in message queue" do + task = create_dummy_task(:killed) + send(self(), {:DOWN, task.ref, :process, task.pid, :killed}) + assert Task.shutdown(task) == {:exit, :killed} + end + + test "exits on noconnection :DOWN in message queue" do + task = create_dummy_task(:noconnection) + send(self(), {:DOWN, task.ref, :process, task.pid, :noconnection}) + + assert catch_exit(Task.shutdown(task)) == + {{:nodedown, node()}, {Task, :shutdown, [task, 5000]}} + end + + test "raises if task PID is nil" do + task = %Task{ref: make_ref(), owner: nil, pid: nil, mfa: {__MODULE__, :test, 1}} + message = "task #{inspect(task)} does not have an associated task process" + assert_raise ArgumentError, message, fn -> Task.shutdown(task) end + end + + test "raises when invoked from a non-owner process" do + task = create_task_in_other_process() + + message = + "task #{inspect(task)} must be queried from the owner " <> + "but was queried from #{inspect(self())}" + + assert_raise ArgumentError, message, fn -> Task.shutdown(task) end + end + + test "returns nil on killing task" do + caller = self() + + task = + Task.async(fn -> + Process.flag(:trap_exit, true) + wait_and_send(caller, :ready) + Process.sleep(:infinity) + end) + + receive do: (:ready -> :ok) + + assert Task.shutdown(task, :brutal_kill) == nil + refute_received {:DOWN, _, _, _, _} + end + + test "returns {:exit, :noproc} if task handled" do + task = create_dummy_task(:noproc) + assert Task.shutdown(task) == {:exit, :noproc} + end end - test "await/1 exits on :noconnection" do - ref = make_ref() - task = %Task{ref: ref, pid: self()} - send self(), {:DOWN, ref, self(), self(), :noconnection} - assert catch_exit(Task.await(task)) |> elem(0) == {:nodedown, :nonode@nohost} + describe "shutdown/2 with :brutal_kill" do + test "returns {:ok, result} when reply and abnormal :DOWN in message queue" do + task = create_dummy_task(:abnormal) + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, task.pid, :abnormal}) + assert Task.shutdown(task, :brutal_kill) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns {:ok, result} when reply and normal :DOWN in message queue" do + task = create_dummy_task(:normal) + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, task.pid, :normal}) + assert Task.shutdown(task, :brutal_kill) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns {:ok, result} when reply and shut down :DOWN in message queue" do + task = create_dummy_task(:shutdown) + send(self(), {task.ref, :result}) + send(self(), {:DOWN, task.ref, :process, task.pid, :shutdown}) + assert Task.shutdown(task, :brutal_kill) == {:ok, :result} + refute_received {:DOWN, _, _, _, _} + end + + test "returns nil on killed :DOWN in message queue" do + task = create_dummy_task(:killed) + send(self(), {:DOWN, task.ref, :process, task.pid, :killed}) + assert Task.shutdown(task, :brutal_kill) == nil + end + + test "returns exit on abnormal :DOWN in message queue" do + task = create_dummy_task(:abnormal) + send(self(), {:DOWN, task.ref, :process, task.pid, :abnormal}) + assert Task.shutdown(task, :brutal_kill) == {:exit, :abnormal} + end + + test "returns exit on normal :DOWN in message queue" do + task = create_dummy_task(:normal) + send(self(), {:DOWN, task.ref, :process, task.pid, :normal}) + assert Task.shutdown(task, :brutal_kill) == {:exit, :normal} + end + + test "returns exit on shutdown :DOWN in message queue" do + task = create_dummy_task(:shutdown) + send(self(), {:DOWN, task.ref, :process, task.pid, :shutdown}) + assert Task.shutdown(task, :brutal_kill) == {:exit, :shutdown} + end + + test "exits on noconnection :DOWN in message queue" do + task = create_dummy_task(:noconnection) + send(self(), {:DOWN, task.ref, :process, task.pid, :noconnection}) + + assert catch_exit(Task.shutdown(task, :brutal_kill)) == + {{:nodedown, node()}, {Task, :shutdown, [task, :brutal_kill]}} + end + + test "returns exit on killing task after shutdown timeout" do + caller = self() + + task = + Task.async(fn -> + Process.flag(:trap_exit, true) + wait_and_send(caller, :ready) + Process.sleep(:infinity) + end) + + receive do: (:ready -> :ok) + assert Task.shutdown(task, 1) == {:exit, :killed} + end + + test "returns {:exit, :noproc} if task handled" do + task = create_dummy_task(:noproc) + assert Task.shutdown(task, :brutal_kill) == {:exit, :noproc} + end end - test "find/2" do - task = %Task{ref: make_ref} - assert Task.find([task], {make_ref, :ok}) == nil - assert Task.find([task], {task.ref, :ok}) == {:ok, task} + describe "async_stream/2" do + test "timeout" do + assert catch_exit([:infinity] |> Task.async_stream(&sleep/1, timeout: 0) |> Enum.to_list()) == + {:timeout, {Task.Supervised, :stream, [0]}} + + refute_received _ + end - assert Task.find([task], {:DOWN, make_ref, :process, self, :kill}) == nil - msg = {:DOWN, task.ref, :process, self, :kill} - assert catch_exit(Task.find([task], msg)) == - {:kill, {Task, :find, [[task], msg]}} + test "streams an enumerable with ordered: false" do + opts = [max_concurrency: 1, ordered: false] + + assert 4..1 + |> Task.async_stream(&sleep(&1 * 100), opts) + |> Enum.to_list() == [ok: 400, ok: 300, ok: 200, ok: 100] + + opts = [max_concurrency: 4, ordered: false] + + assert 4..1 + |> Task.async_stream(&sleep(&1 * 100), opts) + |> Enum.to_list() == [ok: 100, ok: 200, ok: 300, ok: 400] + end + + test "streams an enumerable with ordered: false, on_timeout: :kill_task" do + opts = [max_concurrency: 4, ordered: false, on_timeout: :kill_task, timeout: 50] + + assert [100, 1, 100, 1] + |> Task.async_stream(&sleep/1, opts) + |> Enum.to_list() == [ok: 1, ok: 1, exit: :timeout, exit: :timeout] + + refute_received _ + end + + test "streams an enumerable with infinite timeout" do + [ok: :ok] = Task.async_stream([1], fn _ -> :ok end, timeout: :infinity) |> Enum.to_list() + end + + test "does not allow streaming with max_concurrency = 0" do + assert_raise ArgumentError, ":max_concurrency must be an integer greater than zero", fn -> + Task.async_stream([1], fn _ -> :ok end, max_concurrency: 0) |> Enum.to_list() + end + end + + test "streams with fake down messages on the inbox" do + parent = self() + + assert Task.async_stream([:ok], fn :ok -> + {:links, links} = Process.info(self(), :links) + + for link <- links do + send(link, {:DOWN, make_ref(), :process, parent, :oops}) + end + + :ok + end) + |> Enum.to_list() == [ok: :ok] + end + + test "with $callers" do + grandparent = self() + + Task.async_stream([1], fn 1 -> + parent = self() + assert Process.get(:"$callers") == [grandparent] + + Task.async_stream([1], fn 1 -> + assert Process.get(:"$callers") == [parent, grandparent] + send(grandparent, :done) + end) + |> Stream.run() + end) + |> Stream.run() + + assert_receive :done + end + + test "consuming from another process" do + parent = self() + stream = Task.async_stream([1, 2, 3], &send(parent, &1)) + Task.start(Stream, :run, [stream]) + assert_receive 1 + assert_receive 2 + assert_receive 3 + end end + for {desc, concurrency} <- [==: 4, <: 2, >: 8] do + describe "async_stream with max_concurrency #{desc} tasks" do + @opts [max_concurrency: concurrency] + + test "streams an enumerable with fun" do + assert 1..4 + |> Task.async_stream(&sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "streams an enumerable with mfa" do + assert 1..4 + |> Task.async_stream(__MODULE__, :sleep, [], @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "streams an enumerable without leaking tasks" do + assert 1..4 + |> Task.async_stream(&sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + + refute_received _ + end + + test "streams an enumerable with slowest first" do + Process.flag(:trap_exit, true) + + assert 4..1 + |> Task.async_stream(&sleep/1, @opts) + |> Enum.to_list() == [ok: 4, ok: 3, ok: 2, ok: 1] + end + + test "streams an enumerable with exits" do + Process.flag(:trap_exit, true) + + assert 1..4 + |> Task.async_stream(&exit/1, @opts) + |> Enum.to_list() == [exit: 1, exit: 2, exit: 3, exit: 4] + + refute_received {:EXIT, _, _} + end + + test "shuts down unused tasks" do + assert [0, :infinity, :infinity, :infinity] + |> Task.async_stream(&sleep/1, @opts) + |> Enum.take(1) == [ok: 0] + + assert Process.info(self(), :links) == {:links, []} + end + + test "shuts down unused tasks without leaking messages" do + assert [0, :infinity, :infinity, :infinity] + |> Task.async_stream(&sleep/1, @opts) + |> Enum.take(1) == [ok: 0] + + refute_received _ + end + + test "is zippable on success" do + task = 1..4 |> Task.async_stream(&sleep/1, @opts) |> Stream.map(&elem(&1, 1)) + assert Enum.zip(task, task) == [{1, 1}, {2, 2}, {3, 3}, {4, 4}] + end + + test "is zippable on failure" do + Process.flag(:trap_exit, true) + task = 1..4 |> Task.async_stream(&exit/1, @opts) |> Stream.map(&elem(&1, 1)) + assert Enum.zip(task, task) == [{1, 1}, {2, 2}, {3, 3}, {4, 4}] + end + + test "is zippable with slowest first" do + task = 4..1 |> Task.async_stream(&sleep/1, @opts) |> Stream.map(&elem(&1, 1)) + assert Enum.zip(task, task) == [{4, 4}, {3, 3}, {2, 2}, {1, 1}] + end + + test "with inner halt on success" do + assert 1..8 + |> Stream.take(4) + |> Task.async_stream(&sleep/1, @opts) + |> Enum.to_list() == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "with inner halt on failure" do + Process.flag(:trap_exit, true) + + assert 1..8 + |> Stream.take(4) + |> Task.async_stream(&exit/1, @opts) + |> Enum.to_list() == [exit: 1, exit: 2, exit: 3, exit: 4] + end + + test "with inner halt and slowest first" do + assert 8..1 + |> Stream.take(4) + |> Task.async_stream(&sleep/1, @opts) + |> Enum.to_list() == [ok: 8, ok: 7, ok: 6, ok: 5] + end + + test "with outer halt on success" do + assert 1..8 + |> Task.async_stream(&sleep/1, @opts) + |> Enum.take(4) == [ok: 1, ok: 2, ok: 3, ok: 4] + end + + test "with outer halt on failure" do + Process.flag(:trap_exit, true) + + assert 1..8 + |> Task.async_stream(&exit/1, @opts) + |> Enum.take(4) == [exit: 1, exit: 2, exit: 3, exit: 4] + end + + test "with outer halt and slowest first" do + assert 8..1 + |> Task.async_stream(&sleep/1, @opts) + |> Enum.take(4) == [ok: 8, ok: 7, ok: 6, ok: 5] + end + + test "terminates inner effect" do + stream = + 1..4 + |> Task.async_stream(&sleep/1, @opts) + |> Stream.transform(fn -> :ok end, fn x, acc -> {[x], acc} end, fn _ -> + Process.put(:stream_transform, true) + end) + + Process.put(:stream_transform, false) + assert Enum.to_list(stream) == [ok: 1, ok: 2, ok: 3, ok: 4] + assert Process.get(:stream_transform) + end + + test "terminates outer effect" do + stream = + 1..4 + |> Stream.transform(fn -> :ok end, fn x, acc -> {[x], acc} end, fn _ -> + Process.put(:stream_transform, true) + end) + |> Task.async_stream(&sleep/1, @opts) + + Process.put(:stream_transform, false) + assert Enum.to_list(stream) == [ok: 1, ok: 2, ok: 3, ok: 4] + assert Process.get(:stream_transform) + end + + test "with :on_timeout set to :kill_task" do + opts = Keyword.merge(@opts, on_timeout: :kill_task, timeout: 50) + + assert [100, 1, 100, 1] + |> Task.async_stream(&sleep/1, opts) + |> Enum.to_list() == [exit: :timeout, ok: 1, exit: :timeout, ok: 1] + + refute_received _ + end + + test "with timeout and :zip_input_on_exit set to true" do + opts = Keyword.merge(@opts, zip_input_on_exit: true, on_timeout: :kill_task, timeout: 50) + + assert [1, 100] + |> Task.async_stream(&sleep/1, opts) + |> Enum.to_list() == [ok: 1, exit: {100, :timeout}] + end + + test "with outer halt on failure and :zip_input_on_exit" do + Process.flag(:trap_exit, true) + opts = Keyword.merge(@opts, zip_input_on_exit: true) + + assert 1..8 + |> Task.async_stream(&exit/1, opts) + |> Enum.take(4) == [exit: {1, 1}, exit: {2, 2}, exit: {3, 3}, exit: {4, 4}] + end + end + end end diff --git a/lib/elixir/test/elixir/test_helper.exs b/lib/elixir/test/elixir/test_helper.exs index 6edf43dd100..0425019ee05 100644 --- a/lib/elixir/test/elixir/test_helper.exs +++ b/lib/elixir/test/elixir/test_helper.exs @@ -1,12 +1,10 @@ -ExUnit.start [trace: "--trace" in System.argv] - # Beam files compiled on demand path = Path.expand("../../tmp/beams", __DIR__) File.rm_rf!(path) File.mkdir_p!(path) Code.prepend_path(path) -Code.compiler_options debug_info: true +Code.compiler_options(debug_info: true) defmodule PathHelpers do def fixture_path() do @@ -18,15 +16,15 @@ defmodule PathHelpers do end def fixture_path(extra) do - Path.join(fixture_path, extra) + Path.join(fixture_path(), extra) end def tmp_path(extra) do - Path.join(tmp_path, extra) + Path.join(tmp_path(), extra) end def elixir(args) do - runcmd(elixir_executable, args) + run_cmd(elixir_executable(), args) end def elixir_executable do @@ -34,13 +32,21 @@ defmodule PathHelpers do end def elixirc(args) do - runcmd(elixirc_executable, args) + run_cmd(elixirc_executable(), args) end def elixirc_executable do executable_path("elixirc") end + def iex(args) do + run_cmd(iex_executable(), args) + end + + def iex_executable do + executable_path("iex") + end + def write_beam({:module, name, bin, _} = res) do File.mkdir_p!(unquote(path)) beam_path = Path.join(unquote(path), Atom.to_string(name) <> ".beam") @@ -48,58 +54,49 @@ defmodule PathHelpers do res end - defp runcmd(executable,args) do - :os.cmd :binary.bin_to_list("#{executable} #{IO.chardata_to_string(args)}#{redirect_std_err_on_win}") + defp run_cmd(executable, args) do + '#{executable} #{IO.chardata_to_string(args)}#{redirect_std_err_on_win()}' + |> :os.cmd() + |> :binary.list_to_bin() end defp executable_path(name) do - Path.expand("../../../../bin/#{name}#{executable_extension}", __DIR__) + Path.expand("../../../../bin/#{name}#{executable_extension()}", __DIR__) end - if match? {:win32, _}, :os.type do - def is_win?, do: true + if match?({:win32, _}, :os.type()) do + def windows?, do: true def executable_extension, do: ".bat" def redirect_std_err_on_win, do: " 2>&1" else - def is_win?, do: false + def windows?, do: false def executable_extension, do: "" def redirect_std_err_on_win, do: "" end end -defmodule CompileAssertion do - import ExUnit.Assertions - - def assert_compile_fail(exception, string) do - case format_rescue(string) do - {^exception, _} -> :ok - error -> - raise ExUnit.AssertionError, - left: inspect(elem(error, 0)), - right: inspect(exception), - message: "Expected match" +defmodule CodeFormatterHelpers do + defmacro assert_same(good, opts \\ []) do + quote bind_quoted: [good: good, opts: opts] do + assert IO.iodata_to_binary(Code.format_string!(good, opts)) == String.trim(good) end end - def assert_compile_fail(exception, message, string) do - case format_rescue(string) do - {^exception, ^message} -> :ok - error -> - raise ExUnit.AssertionError, - left: "#{inspect elem(error, 0)}[message: #{inspect elem(error, 1)}]", - right: "#{inspect exception}[message: #{inspect message}]", - message: "Expected match" + defmacro assert_format(bad, good, opts \\ []) do + quote bind_quoted: [bad: bad, good: good, opts: opts] do + result = String.trim(good) + assert IO.iodata_to_binary(Code.format_string!(bad, opts)) == result + assert IO.iodata_to_binary(Code.format_string!(good, opts)) == result end end +end - defp format_rescue(expr) do - result = try do - :elixir.eval(to_char_list(expr), []) - nil - rescue - error -> {error.__struct__, Exception.message(error)} - end +assert_timeout = String.to_integer(System.get_env("ELIXIR_ASSERT_TIMEOUT") || "500") +epmd_exclude = if match?({:win32, _}, :os.type()), do: [epmd: true], else: [] +os_exclude = if PathHelpers.windows?(), do: [unix: true], else: [windows: true] - result || flunk(message: "Expected expression to fail") - end -end +ExUnit.start( + trace: "--trace" in System.argv(), + assert_receive_timeout: assert_timeout, + exclude: epmd_exclude ++ os_exclude +) diff --git a/lib/elixir/test/elixir/tuple_test.exs b/lib/elixir/test/elixir/tuple_test.exs index 54af5e603b6..2de0724213a 100644 --- a/lib/elixir/test/elixir/tuple_test.exs +++ b/lib/elixir/test/elixir/tuple_test.exs @@ -1,35 +1,39 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule TupleTest do use ExUnit.Case, async: true - test :elem do + doctest Tuple + + # Tuple-related functions in the Kernel module. + + test "Kernel.elem/2" do assert elem({:a, :b, :c}, 1) == :b end - test :put_elem do + test "Kernel.put_elem/3" do assert put_elem({:a, :b, :c}, 1, :d) == {:a, :d, :c} end - test :keywords do + test "keyword syntax is supported in tuple literals" do assert {1, 2, three: :four} == {1, 2, [three: :four]} end - test :optional_comma do - assert {1} == {1,} - assert {1, 2, 3} == {1, 2, 3,} + test "optional comma is supported in tuple literals" do + assert Code.eval_string("{1,}") == {{1}, []} + assert Code.eval_string("{1, 2, 3,}") == {{1, 2, 3}, []} end - test :partial_application do + test "partial application" do assert (&{&1, 2}).(1) == {1, 2} assert (&{&1, &2}).(1, 2) == {1, 2} assert (&{&2, &1}).(2, 1) == {1, 2} end # Tuple module - # We check two variants due to inlining. + # We check two variants of each function due to inlining. - test :duplicate do + test "duplicate/2" do assert Tuple.duplicate(:foo, 0) == {} assert Tuple.duplicate(:foo, 3) == {:foo, :foo, :foo} @@ -38,17 +42,24 @@ defmodule TupleTest do assert mod.duplicate(:foo, 3) == {:foo, :foo, :foo} end - test :insert_at do + test "insert_at/3" do assert Tuple.insert_at({:bar, :baz}, 0, :foo) == {:foo, :bar, :baz} mod = Tuple assert mod.insert_at({:bar, :baz}, 0, :foo) == {:foo, :bar, :baz} end - test :delete_at do + test "append/2" do + assert Tuple.append({:foo, :bar}, :baz) == {:foo, :bar, :baz} + + mod = Tuple + assert mod.append({:foo, :bar}, :baz) == {:foo, :bar, :baz} + end + + test "delete_at/2" do assert Tuple.delete_at({:foo, :bar, :baz}, 0) == {:bar, :baz} mod = Tuple assert mod.delete_at({:foo, :bar, :baz}, 0) == {:bar, :baz} end -end \ No newline at end of file +end diff --git a/lib/elixir/test/elixir/typespec_test.exs b/lib/elixir/test/elixir/typespec_test.exs new file mode 100644 index 00000000000..2cf0e5eb033 --- /dev/null +++ b/lib/elixir/test/elixir/typespec_test.exs @@ -0,0 +1,1606 @@ +Code.require_file("test_helper.exs", __DIR__) + +# Holds tests for both Kernel.Typespec and Code.Typespec +defmodule TypespecTest do + use ExUnit.Case, async: true + alias TypespecTest.TypespecSample + + defstruct [:hello] + + defmacrop test_module(do: block) do + quote do + {:module, _, bytecode, _} = + defmodule TypespecSample do + unquote(block) + end + + :code.delete(TypespecSample) + :code.purge(TypespecSample) + bytecode + end + end + + defp types(bytecode) do + bytecode + |> Code.Typespec.fetch_types() + |> elem(1) + |> Enum.sort() + end + + @skip_specs [__info__: 1] + + defp specs(bytecode) do + bytecode + |> Code.Typespec.fetch_specs() + |> elem(1) + |> Enum.reject(fn {sign, _} -> sign in @skip_specs end) + |> Enum.sort() + end + + defp callbacks(bytecode) do + bytecode + |> Code.Typespec.fetch_callbacks() + |> elem(1) + |> Enum.sort() + end + + describe "Kernel.Typespec errors" do + test "invalid type specification" do + assert_raise CompileError, ~r"invalid type specification: my_type = 1", fn -> + test_module do + @type my_type = 1 + end + end + end + + test "unexpected expression in typespec" do + assert_raise CompileError, ~r"unexpected expression in typespec: \"foobar\"", fn -> + test_module do + @type my_type :: "foobar" + end + end + end + + test "invalid function specification" do + assert_raise CompileError, ~r"invalid type specification: \"not a spec\"", fn -> + test_module do + @spec "not a spec" + end + end + + assert_raise CompileError, ~r"invalid type specification: 1 :: 2", fn -> + test_module do + @spec 1 :: 2 + end + end + end + + test "undefined type" do + assert_raise CompileError, ~r"type foo/0 undefined", fn -> + test_module do + @type omg :: foo + end + end + + assert_raise CompileError, ~r"type foo/2 undefined", fn -> + test_module do + @type omg :: foo(atom, integer) + end + end + + assert_raise CompileError, ~r"type bar/0 undefined", fn -> + test_module do + @spec foo(bar, integer) :: {atom, integer} + def foo(var1, var2), do: {var1, var2} + end + end + + assert_raise CompileError, ~r"type foo/0 undefined", fn -> + test_module do + @type omg :: __MODULE__.foo() + end + end + end + + test "redefined type" do + assert_raise CompileError, + ~r"type foo/0 is already defined in test/elixir/typespec_test.exs:110", + fn -> + test_module do + @type foo :: atom + @type foo :: integer + end + end + + assert_raise CompileError, + ~r"type foo/2 is already defined in test/elixir/typespec_test.exs:120", + fn -> + test_module do + @type foo :: atom + @type foo(var1, var2) :: {var1, var2} + @type foo(x, y) :: {x, y} + end + end + + assert_raise CompileError, + ~r"type foo/0 is already defined in test/elixir/typespec_test.exs:129", + fn -> + test_module do + @type foo :: atom + @typep foo :: integer + end + end + end + + test "type variable unused (singleton type variable)" do + assert_raise CompileError, ~r"type variable x is used only once", fn -> + test_module do + @type foo(x) :: integer + end + end + end + + test "type variable starting with underscore" do + test_module do + assert @type(foo(_hello) :: integer) == :ok + end + end + + test "type variable named _" do + assert_raise CompileError, ~r"type variable '_' is invalid", fn -> + test_module do + @type foo(_) :: integer + end + end + + assert_raise CompileError, ~r"type variable '_' is invalid", fn -> + test_module do + @type foo(_, _) :: integer + end + end + end + + test "spec for undefined function" do + assert_raise CompileError, ~r"spec for undefined function omg/0", fn -> + test_module do + @spec omg :: atom + end + end + end + + test "spec variable used only once (singleton type variable)" do + assert_raise CompileError, ~r"type variable x is used only once", fn -> + test_module do + @spec foo(x, integer) :: integer when x: var + def foo(x, y), do: x + y + end + end + end + + test "invalid optional callback" do + assert_raise CompileError, ~r"invalid optional callback :foo", fn -> + test_module do + @optional_callbacks :foo + end + end + end + + test "unknown optional callback" do + assert_raise CompileError, ~r"unknown callback foo/1 given as optional callback", fn -> + test_module do + @optional_callbacks foo: 1 + end + end + end + + test "repeated optional callback" do + message = ~r"foo/1 has been specified as optional callback more than once" + + assert_raise CompileError, message, fn -> + test_module do + @callback foo(:ok) :: :ok + @optional_callbacks foo: 1, foo: 1 + end + end + end + + test "behaviour_info/1 explicitly defined alongside @callback/@macrocallback" do + message = ~r"cannot define @callback attribute for foo/1 when behaviour_info/1" + + assert_raise CompileError, message, fn -> + test_module do + @callback foo(:ok) :: :ok + def behaviour_info(_), do: [] + end + end + + message = ~r"cannot define @macrocallback attribute for foo/1 when behaviour_info/1" + + assert_raise CompileError, message, fn -> + test_module do + @macrocallback foo(:ok) :: :ok + def behaviour_info(_), do: [] + end + end + end + + test "default is not supported" do + assert_raise ArgumentError, fn -> + test_module do + @callback hello(num \\ 0 :: integer) :: integer + end + end + + assert_raise ArgumentError, fn -> + test_module do + @callback hello(num :: integer \\ 0) :: integer + end + end + + assert_raise ArgumentError, fn -> + test_module do + @macrocallback hello(num \\ 0 :: integer) :: Macro.t() + end + end + + assert_raise ArgumentError, fn -> + test_module do + @macrocallback hello(num :: integer \\ 0) :: Macro.t() + end + end + + assert_raise ArgumentError, fn -> + test_module do + @spec hello(num \\ 0 :: integer) :: integer + end + end + + assert_raise ArgumentError, fn -> + test_module do + @spec hello(num :: integer \\ 0) :: integer + end + end + end + + test "@spec shows readable error message when return type is missing" do + message = ~r"type specification missing return type: my_fun\(integer\)" + + assert_raise CompileError, message, fn -> + test_module do + @spec my_fun(integer) + end + end + end + end + + describe "Kernel.Typespec definitions" do + test "typespec declarations return :ok" do + test_module do + def foo(), do: nil + + assert @type(foo :: any()) == :ok + assert @typep(foop :: any()) == :ok + assert @spec(foo() :: nil) == :ok + assert @opaque(my_type :: atom) == :ok + assert @callback(foo(foop) :: integer) == :ok + assert @macrocallback(foo(integer) :: integer) == :ok + end + end + + test "@type with a single type" do + bytecode = + test_module do + @type my_type :: term + end + + assert [type: {:my_type, {:type, _, :term, []}, []}] = types(bytecode) + end + + test "@type with an atom/alias" do + bytecode = + test_module do + @type foo :: :foo + @type bar :: Bar + end + + assert [ + type: {:bar, {:atom, _, Bar}, []}, + type: {:foo, {:atom, _, :foo}, []} + ] = types(bytecode) + end + + test "@type with an integer" do + bytecode = + test_module do + @type pos :: 10 + @type neg :: -10 + end + + assert [ + type: {:neg, {:op, _, :-, {:integer, _, 10}}, []}, + type: {:pos, {:integer, _, 10}, []} + ] = types(bytecode) + end + + test "@type with a tuple" do + bytecode = + test_module do + @type tup :: tuple() + @type one :: {123} + end + + assert [ + type: {:one, {:type, _, :tuple, [{:integer, _, 123}]}, []}, + type: {:tup, {:type, _, :tuple, :any}, []} + ] = types(bytecode) + end + + test "@type with a remote type" do + bytecode = + test_module do + @type my_type :: Remote.Some.type() + @type my_type_arg :: Remote.type(integer) + end + + assert [type: my_type, type: my_type_arg] = types(bytecode) + + assert {:my_type, type, []} = my_type + assert {:remote_type, _, [{:atom, _, Remote.Some}, {:atom, _, :type}, []]} = type + + assert {:my_type_arg, type, []} = my_type_arg + assert {:remote_type, _, args} = type + assert [{:atom, _, Remote}, {:atom, _, :type}, [{:type, _, :integer, []}]] = args + end + + test "@type with a binary" do + bytecode = + test_module do + @type bin :: binary + @type empty :: <<>> + @type size :: <<_::3>> + @type unit :: <<_::_*8>> + @type size_and_unit :: <<_::3, _::_*8>> + end + + assert [ + type: {:bin, {:type, _, :binary, []}, []}, + type: {:empty, {:type, _, :binary, [{:integer, _, 0}, {:integer, _, 0}]}, []}, + type: {:size, {:type, _, :binary, [{:integer, _, 3}, {:integer, _, 0}]}, []}, + type: + {:size_and_unit, {:type, _, :binary, [{:integer, _, 3}, {:integer, _, 8}]}, []}, + type: {:unit, {:type, _, :binary, [{:integer, _, 0}, {:integer, _, 8}]}, []} + ] = types(bytecode) + end + + test "@type with invalid binary spec" do + assert_raise CompileError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::3*8>> + end + end + + assert_raise CompileError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::atom>> + end + end + + assert_raise CompileError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::integer>> + end + end + + assert_raise CompileError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::(-4)>> + end + end + + assert_raise CompileError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::3, _::_*atom>> + end + end + + assert_raise CompileError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::3, _::_*(-8)>> + end + end + + assert_raise CompileError, ~r"invalid binary specification", fn -> + test_module do + @type my_type :: <<_::3, _::_*257>> + end + end + end + + test "@type with a range op" do + bytecode = + test_module do + @type range1 :: 1..10 + @type range2 :: -1..1 + end + + assert [ + {:type, {:range1, {:type, _, :range, range1_args}, []}}, + {:type, {:range2, {:type, _, :range, range2_args}, []}} + ] = types(bytecode) + + assert [{:integer, _, 1}, {:integer, _, 10}] = range1_args + assert [{:op, _, :-, {:integer, _, 1}}, {:integer, _, 1}] = range2_args + end + + test "@type with invalid range" do + assert_raise CompileError, ~r"invalid range specification", fn -> + test_module do + @type my_type :: atom..10 + end + end + end + + test "@type with a keyword map" do + bytecode = + test_module do + @type my_type :: %{hello: :world} + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :map, [arg]} = type + assert {:type, _, :map_field_exact, [{:atom, _, :hello}, {:atom, _, :world}]} = arg + end + + test "@type with a map" do + bytecode = + test_module do + @type my_type :: %{required(:a) => :b, optional(:c) => :d} + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :map, [arg1, arg2]} = type + assert {:type, _, :map_field_exact, [{:atom, _, :a}, {:atom, _, :b}]} = arg1 + assert {:type, _, :map_field_assoc, [{:atom, _, :c}, {:atom, _, :d}]} = arg2 + end + + test "@type with a struct" do + bytecode = + test_module do + defstruct hello: nil, other: nil + @type my_type :: %TypespecSample{hello: :world} + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :map, [struct, arg1, arg2]} = type + assert {:type, _, :map_field_exact, struct_args} = struct + assert [{:atom, _, :__struct__}, {:atom, _, TypespecSample}] = struct_args + assert {:type, _, :map_field_exact, [{:atom, _, :hello}, {:atom, _, :world}]} = arg1 + assert {:type, _, :map_field_exact, [{:atom, _, :other}, {:type, _, :term, []}]} = arg2 + end + + test "@type with an exception struct" do + bytecode = + test_module do + defexception [:message] + @type my_type :: %TypespecSample{} + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :map, [struct, arg1, arg2]} = type + assert {:type, _, :map_field_exact, struct_args} = struct + assert [{:atom, _, :__struct__}, {:atom, _, TypespecSample}] = struct_args + assert {:type, _, :map_field_exact, [{:atom, _, :__exception__}, {:atom, _, true}]} = arg1 + assert {:type, _, :map_field_exact, [{:atom, _, :message}, {:type, _, :term, []}]} = arg2 + end + + @fields Enum.map(10..42, &{:"f#{&1}", :ok}) + + test "@type with a large struct" do + bytecode = + test_module do + defstruct unquote(@fields) + @type my_type :: %TypespecSample{unquote_splicing(@fields)} + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :map, [struct, arg1, arg2 | _]} = type + assert {:type, _, :map_field_exact, struct_args} = struct + assert [{:atom, _, :__struct__}, {:atom, _, TypespecSample}] = struct_args + assert {:type, _, :map_field_exact, [{:atom, _, :f10}, {:atom, _, :ok}]} = arg1 + assert {:type, _, :map_field_exact, [{:atom, _, :f11}, {:atom, _, :ok}]} = arg2 + end + + test "@type with struct does not @enforce_keys" do + bytecode = + test_module do + @enforce_keys [:other] + defstruct hello: nil, other: nil + @type my_type :: %TypespecSample{hello: :world} + end + + assert [type: {:my_type, _type, []}] = types(bytecode) + end + + test "@type with undefined struct" do + assert_raise CompileError, ~r"ThisModuleDoesNotExist.__struct__/0 is undefined", fn -> + test_module do + @type my_type :: %ThisModuleDoesNotExist{} + end + end + + assert_raise CompileError, ~r"cannot access struct TypespecTest.TypespecSample", fn -> + test_module do + @type my_type :: %TypespecSample{} + end + end + end + + test "@type with a struct with undefined field" do + assert_raise CompileError, + ~r"undefined field :no_field on struct TypespecTest.TypespecSample", + fn -> + test_module do + defstruct [:hello, :eric] + @type my_type :: %TypespecSample{no_field: :world} + end + end + + assert_raise CompileError, + ~r"undefined field :no_field on struct TypespecTest.TypespecSample", + fn -> + test_module do + defstruct [:hello, :eric] + @type my_type :: %__MODULE__{no_field: :world} + end + end + end + + test "@type when overriding Elixir built-in" do + assert_raise CompileError, ~r"type struct/0 is a built-in type", fn -> + test_module do + @type struct :: :oops + end + end + end + + test "@type when overriding Erlang built-in" do + assert_raise CompileError, ~r"type list/0 is a built-in type", fn -> + test_module do + @type list :: :oops + end + end + end + + test "@type with public record" do + bytecode = + test_module do + require Record + Record.defrecord(:timestamp, date: 1, time: 2) + @type my_type :: record(:timestamp, time: :foo) + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :tuple, [timestamp, term, foo]} = type + assert {:atom, 0, :timestamp} = timestamp + assert {:ann_type, 0, [{:var, 0, :date}, {:type, 0, :term, []}]} = term + assert {:ann_type, 0, [{:var, 0, :time}, {:atom, 0, :foo}]} = foo + end + + test "@type with private record" do + bytecode = + test_module do + require Record + Record.defrecordp(:timestamp, date: 1, time: 2) + @type my_type :: record(:timestamp, time: :foo) + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :tuple, args} = type + + assert [ + {:atom, 0, :timestamp}, + {:ann_type, 0, [{:var, 0, :date}, {:type, 0, :term, []}]}, + {:ann_type, 0, [{:var, 0, :time}, {:atom, 0, :foo}]} + ] = args + end + + test "@type with named record" do + bytecode = + test_module do + require Record + Record.defrecord(:timestamp, :my_timestamp, date: 1, time: 2) + @type my_type :: record(:timestamp, time: :foo) + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :tuple, [my_timestamp, term, _foo]} = type + assert {:atom, 0, :my_timestamp} = my_timestamp + assert {:ann_type, 0, [{:var, 0, :date}, {:type, 0, :term, []}]} = term + assert {:ann_type, 0, [{:var, 0, :time}, {:atom, 0, :foo}]} + end + + test "@type with undefined record" do + assert_raise CompileError, ~r"unknown record :this_record_does_not_exist", fn -> + test_module do + @type my_type :: record(:this_record_does_not_exist, []) + end + end + end + + test "@type with a record with undefined field" do + assert_raise CompileError, ~r"undefined field no_field on record :timestamp", fn -> + test_module do + require Record + Record.defrecord(:timestamp, date: 1, time: 2) + @type my_type :: record(:timestamp, no_field: :foo) + end + end + end + + test "@type with a record which declares the name as the type `atom` rather than an atom literal" do + assert_raise CompileError, ~r"expected the record name to be an atom literal", fn -> + test_module do + @type my_type :: record(atom, field: :foo) + end + end + end + + test "@type can be named record" do + bytecode = + test_module do + @type record :: binary + @spec foo?(record) :: boolean + def foo?(_), do: true + end + + assert [type: {:record, {:type, _, :binary, []}, []}] = types(bytecode) + end + + test "@type with an invalid map notation" do + assert_raise CompileError, ~r"invalid map specification", fn -> + test_module do + @type content :: %{atom | String.t() => term} + end + end + end + + test "@type with list shortcuts" do + bytecode = + test_module do + @type my_type :: [] + @type my_type1 :: [integer] + @type my_type2 :: [integer, ...] + end + + assert [ + type: {:my_type, {:type, _, nil, []}, []}, + type: {:my_type1, {:type, _, :list, [{:type, _, :integer, []}]}, []}, + type: {:my_type2, {:type, _, :nonempty_list, [{:type, _, :integer, []}]}, []} + ] = types(bytecode) + end + + test "@type with a fun" do + bytecode = + test_module do + @type my_type :: (... -> any) + end + + assert [type: {:my_type, {:type, _, :fun, []}, []}] = types(bytecode) + end + + test "@type with a fun with multiple arguments and return type" do + bytecode = + test_module do + @type my_type :: (integer, integer -> integer) + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :fun, [args, return_type]} = type + assert {:type, _, :product, [{:type, _, :integer, []}, {:type, _, :integer, []}]} = args + assert {:type, _, :integer, []} = return_type + end + + test "@type with a fun with no arguments and return type" do + bytecode = + test_module do + @type my_type :: (() -> integer) + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :fun, [{:type, _, :product, []}, {:type, _, :integer, []}]} = type + end + + test "@type with a fun with any arity and return type" do + bytecode = + test_module do + @type my_type :: (... -> integer) + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :fun, [{:type, _, :any}, {:type, _, :integer, []}]} = type + end + + test "@type with a union" do + bytecode = + test_module do + @type my_type :: integer | charlist | atom + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :union, [integer, charlist, atom]} = type + assert {:type, _, :integer, []} = integer + assert {:remote_type, _, [{:atom, _, :elixir}, {:atom, _, :charlist}, []]} = charlist + assert {:type, _, :atom, []} = atom + end + + test "@type with keywords" do + bytecode = + test_module do + @type my_type :: [first: integer, step: integer, last: integer] + end + + assert [type: {:my_type, type, []}] = types(bytecode) + assert {:type, _, :list, [{:type, _, :union, union_types}]} = type + + assert [ + {:type, _, :tuple, [{:atom, _, :first}, {:type, _, :integer, []}]}, + {:type, _, :tuple, [{:atom, _, :step}, {:type, _, :integer, []}]}, + {:type, _, :tuple, [{:atom, _, :last}, {:type, _, :integer, []}]} + ] = union_types + end + + test "@type with parameters" do + bytecode = + test_module do + @type my_type(x) :: x + @type my_type1(x) :: list(x) + @type my_type2(x, y) :: {x, y} + end + + assert [ + type: {:my_type, {:var, _, :x}, [{:var, _, :x}]}, + type: {:my_type1, {:type, _, :list, [{:var, _, :x}]}, [{:var, _, :x}]}, + type: {:my_type2, my_type2, [{:var, _, :x}, {:var, _, :y}]} + ] = types(bytecode) + + assert {:type, _, :tuple, [{:var, _, :x}, {:var, _, :y}]} = my_type2 + end + + test "@type with annotations" do + bytecode = + test_module do + @type my_type :: named :: integer + @type my_type1 :: (a :: integer -> integer) + end + + assert [type: {:my_type, my_type, []}, type: {:my_type1, my_type1, []}] = types(bytecode) + + assert {:ann_type, _, [{:var, _, :named}, {:type, _, :integer, []}]} = my_type + + assert {:type, _, :fun, [fun_args, fun_return]} = my_type1 + assert {:type, _, :product, [{:ann_type, _, [a, {:type, _, :integer, []}]}]} = fun_args + assert {:var, _, :a} = a + assert {:type, _, :integer, []} = fun_return + end + + test "@type unquote fragment" do + quoted = + quote unquote: false do + name = :my_type + type = :foo + @type unquote(name)() :: unquote(type) + end + + bytecode = + test_module do + Module.eval_quoted(__MODULE__, quoted) + end + + assert [type: {:my_type, {:atom, _, :foo}, []}] = types(bytecode) + end + + test "@type with module attributes" do + bytecode = + test_module do + @keyword Keyword + @type kw :: @keyword.t + @type kw(value) :: @keyword.t(value) + end + + assert [type: {:kw, kw, _}, type: {:kw, kw_with_value, [{:var, _, :value}]}] = + types(bytecode) + + assert {:remote_type, _, [{:atom, _, Keyword}, {:atom, _, :t}, []]} = kw + assert {:remote_type, _, kw_with_value_args} = kw_with_value + assert [{:atom, _, Keyword}, {:atom, _, :t}, [{:var, _, :value}]] = kw_with_value_args + end + + test "@type with a reserved signature" do + assert_raise CompileError, + ~r"type required\/1 is a reserved type and it cannot be defined", + fn -> + test_module do + @type required(arg) :: any() + end + end + + assert_raise CompileError, + ~r"type optional\/1 is a reserved type and it cannot be defined", + fn -> + test_module do + @type optional(arg) :: any() + end + end + + assert_raise CompileError, + ~r"type required\/1 is a reserved type and it cannot be defined", + fn -> + test_module do + @typep required(arg) :: any() + end + end + + assert_raise CompileError, + ~r"type optional\/1 is a reserved type and it cannot be defined", + fn -> + test_module do + @typep optional(arg) :: any() + end + end + + assert_raise CompileError, + ~r"type required\/1 is a reserved type and it cannot be defined", + fn -> + test_module do + @opaque required(arg) :: any() + end + end + + assert_raise CompileError, + ~r"type optional\/1 is a reserved type and it cannot be defined", + fn -> + test_module do + @opaque optional(arg) :: any() + end + end + end + + test "invalid remote @type with module attribute that does not evaluate to a module" do + assert_raise CompileError, ~r/\(@foo is "bar"\)/, fn -> + test_module do + @foo "bar" + @type t :: @foo.t + end + end + end + + test "defines_type?" do + test_module do + @type my_type :: tuple + @type my_type(a) :: [a] + assert Kernel.Typespec.defines_type?(__MODULE__, {:my_type, 0}) + assert Kernel.Typespec.defines_type?(__MODULE__, {:my_type, 1}) + refute Kernel.Typespec.defines_type?(__MODULE__, {:my_type, 2}) + end + end + + test "spec_to_callback/2" do + bytecode = + test_module do + @spec foo() :: term() + def foo(), do: :ok + Kernel.Typespec.spec_to_callback(__MODULE__, {:foo, 0}) + end + + assert specs(bytecode) == callbacks(bytecode) + end + + test "@opaque" do + bytecode = + test_module do + @opaque my_type(x) :: x + end + + assert [opaque: {:my_type, {:var, _, :x}, [{:var, _, :x}]}] = types(bytecode) + end + + test "@spec" do + bytecode = + test_module do + def my_fun1(x), do: x + def my_fun2(), do: :ok + def my_fun3(x, y), do: {x, y} + def my_fun4(x), do: x + @spec my_fun1(integer) :: integer + @spec my_fun2() :: integer + @spec my_fun3(integer, integer) :: {integer, integer} + @spec my_fun4(x :: integer) :: integer + end + + assert [my_fun1, my_fun2, my_fun3, my_fun4] = specs(bytecode) + + assert {{:my_fun1, 1}, [{:type, _, :fun, args}]} = my_fun1 + assert [{:type, _, :product, [{:type, _, :integer, []}]}, {:type, _, :integer, []}] = args + + assert {{:my_fun2, 0}, [{:type, _, :fun, args}]} = my_fun2 + assert [{:type, _, :product, []}, {:type, _, :integer, []}] = args + + assert {{:my_fun3, 2}, [{:type, _, :fun, [arg1, arg2]}]} = my_fun3 + assert {:type, _, :product, [{:type, _, :integer, []}, {:type, _, :integer, []}]} = arg1 + assert {:type, _, :tuple, [{:type, _, :integer, []}, {:type, _, :integer, []}]} = arg2 + + assert {{:my_fun4, 1}, [{:type, _, :fun, args}]} = my_fun4 + assert [x, {:type, _, :integer, []}] = args + assert {:type, _, :product, [{:ann_type, _, [{:var, _, :x}, {:type, _, :integer, []}]}]} = x + end + + test "@spec with vars matching built-ins" do + bytecode = + test_module do + def my_fun1(x), do: x + def my_fun2(x), do: x + @spec my_fun1(tuple) :: tuple + @spec my_fun2(tuple) :: tuple when tuple: {integer, integer} + end + + assert [my_fun1, my_fun2] = specs(bytecode) + + assert {{:my_fun1, 1}, [{:type, _, :fun, args}]} = my_fun1 + assert [{:type, _, :product, [{:type, _, :tuple, :any}]}, {:type, _, :tuple, :any}] = args + + assert {{:my_fun2, 1}, [{:type, _, :bounded_fun, args}]} = my_fun2 + + assert [type, _] = args + + assert {:type, _, :fun, [{:type, _, :product, [{:var, _, :tuple}]}, {:var, _, :tuple}]} = + type + end + + test "@spec with guards" do + bytecode = + test_module do + def my_fun1(x), do: x + @spec my_fun1(x) :: boolean when x: integer + + def my_fun2(x), do: x + @spec my_fun2(x) :: x when x: var + + def my_fun3(_x, y), do: y + @spec my_fun3(x, y) :: y when y: x, x: var + end + + assert [my_fun1, my_fun2, my_fun3] = specs(bytecode) + + assert {{:my_fun1, 1}, [{:type, _, :bounded_fun, args}]} = my_fun1 + assert [{:type, _, :fun, [product, {:type, _, :boolean, []}]}, constraints] = args + assert {:type, _, :product, [{:var, _, :x}]} = product + assert [{:type, _, :constraint, subtype}] = constraints + assert [{:atom, _, :is_subtype}, [{:var, _, :x}, {:type, _, :integer, []}]] = subtype + + assert {{:my_fun2, 1}, [{:type, _, :fun, args}]} = my_fun2 + assert [{:type, _, :product, [{:var, _, :x}]}, {:var, _, :x}] = args + + assert {{:my_fun3, 2}, [{:type, _, :bounded_fun, args}]} = my_fun3 + assert [{:type, _, :fun, fun_type}, [{:type, _, :constraint, constraint_type}]] = args + assert [{:type, _, :product, [{:var, _, :x}, {:var, _, :y}]}, {:var, _, :y}] = fun_type + assert [{:atom, _, :is_subtype}, [{:var, _, :y}, {:var, _, :x}]] = constraint_type + end + + test "@type, @opaque, and @typep as module attributes" do + defmodule TypeModuleAttributes do + @type type1 :: boolean + @opaque opaque1 :: boolean + @typep typep1 :: boolean + + def type1, do: @type + def opaque1, do: @opaque + def typep1, do: @typep + + @type type2 :: atom + @type type3 :: pid + @opaque opaque2 :: atom + @opaque opaque3 :: pid + @typep typep2 :: atom + + def type2, do: @type + def opaque2, do: @opaque + def typep2, do: @typep + + # Avoid unused warnings + @spec foo(typep1) :: typep2 + def foo(_x), do: :ok + end + + assert [ + {:type, {:"::", _, [{:type1, _, _}, {:boolean, _, _}]}, {TypeModuleAttributes, _}} + ] = TypeModuleAttributes.type1() + + assert [ + {:type, {:"::", _, [{:type3, _, _}, {:pid, _, _}]}, {TypeModuleAttributes, _}}, + {:type, {:"::", _, [{:type2, _, _}, {:atom, _, _}]}, {TypeModuleAttributes, _}}, + {:type, {:"::", _, [{:type1, _, _}, {:boolean, _, _}]}, {TypeModuleAttributes, _}} + ] = TypeModuleAttributes.type2() + + assert [ + {:opaque, {:"::", _, [{:opaque1, _, _}, {:boolean, _, _}]}, + {TypeModuleAttributes, _}} + ] = TypeModuleAttributes.opaque1() + + assert [ + {:opaque, {:"::", _, [{:opaque3, _, _}, {:pid, _, _}]}, {TypeModuleAttributes, _}}, + {:opaque, {:"::", _, [{:opaque2, _, _}, {:atom, _, _}]}, + {TypeModuleAttributes, _}}, + {:opaque, {:"::", _, [{:opaque1, _, _}, {:boolean, _, _}]}, + {TypeModuleAttributes, _}} + ] = TypeModuleAttributes.opaque2() + + assert [ + {:typep, {:"::", _, [{:typep1, _, _}, {:boolean, _, _}]}, + {TypeModuleAttributes, _}} + ] = TypeModuleAttributes.typep1() + + assert [ + {:typep, {:"::", _, [{:typep2, _, _}, {:atom, _, _}]}, {TypeModuleAttributes, _}}, + {:typep, {:"::", _, [{:typep1, _, _}, {:boolean, _, _}]}, + {TypeModuleAttributes, _}} + ] = TypeModuleAttributes.typep2() + after + :code.delete(TypeModuleAttributes) + :code.purge(TypeModuleAttributes) + end + + test "@spec, @callback, and @macrocallback as module attributes" do + defmodule SpecModuleAttributes do + @callback callback1 :: integer + @macrocallback macrocallback1 :: integer + + @spec spec1 :: boolean + def spec1, do: @spec + + @callback callback2 :: var when var: boolean + @macrocallback macrocallback2 :: var when var: boolean + + @spec spec2 :: atom + def spec2, do: @spec + + @spec spec3 :: pid + def spec3, do: :ok + def spec4, do: @spec + + def callback, do: @callback + def macrocallback, do: @macrocallback + end + + assert [ + {:spec, {:"::", _, [{:spec1, _, _}, {:boolean, _, _}]}, {SpecModuleAttributes, _}} + ] = SpecModuleAttributes.spec1() + + assert [ + {:spec, {:"::", _, [{:spec2, _, _}, {:atom, _, _}]}, {SpecModuleAttributes, _}}, + {:spec, {:"::", _, [{:spec1, _, _}, {:boolean, _, _}]}, {SpecModuleAttributes, _}} + ] = SpecModuleAttributes.spec2() + + assert [ + {:spec, {:"::", _, [{:spec3, _, _}, {:pid, _, _}]}, {SpecModuleAttributes, _}}, + {:spec, {:"::", _, [{:spec2, _, _}, {:atom, _, _}]}, {SpecModuleAttributes, _}}, + {:spec, {:"::", _, [{:spec1, _, _}, {:boolean, _, _}]}, {SpecModuleAttributes, _}} + ] = SpecModuleAttributes.spec4() + + assert [ + {:callback, + {:when, _, + [{:"::", _, [{:callback2, _, _}, {:var, _, _}]}, [var: {:boolean, _, _}]]}, + {SpecModuleAttributes, _}}, + {:callback, {:"::", _, [{:callback1, _, _}, {:integer, _, _}]}, + {SpecModuleAttributes, _}} + ] = SpecModuleAttributes.callback() + + assert [ + {:macrocallback, + {:when, _, + [{:"::", _, [{:macrocallback2, _, _}, {:var, _, _}]}, [var: {:boolean, _, _}]]}, + {SpecModuleAttributes, _}}, + {:macrocallback, {:"::", _, [{:macrocallback1, _, _}, {:integer, _, _}]}, + {SpecModuleAttributes, _}} + ] = SpecModuleAttributes.macrocallback() + after + :code.delete(SpecModuleAttributes) + :code.purge(SpecModuleAttributes) + end + + test "@callback" do + bytecode = + test_module do + @callback my_fun(integer) :: integer + @callback my_fun(list) :: list + @callback my_fun() :: integer + @callback my_fun(integer, integer) :: {integer, integer} + end + + assert [my_fun_0, my_fun_1, my_fun_2] = callbacks(bytecode) + + assert {{:my_fun, 0}, [{:type, _, :fun, args}]} = my_fun_0 + assert [{:type, _, :product, []}, {:type, _, :integer, []}] = args + + assert {{:my_fun, 1}, [clause1, clause2]} = my_fun_1 + assert {:type, _, :fun, args1} = clause1 + assert [{:type, _, :product, [{:type, _, :integer, []}]}, {:type, _, :integer, []}] = args1 + assert {:type, _, :fun, args2} = clause2 + assert [{:type, _, :product, [{:type, _, :list, []}]}, {:type, _, :list, []}] = args2 + + assert {{:my_fun, 2}, [{:type, _, :fun, [args_type, return_type]}]} = my_fun_2 + + assert {:type, _, :product, [{:type, _, :integer, []}, {:type, _, :integer, []}]} = + args_type + + assert {:type, _, :tuple, [{:type, _, :integer, []}, {:type, _, :integer, []}]} = + return_type + end + + test "block handling" do + bytecode = + test_module do + @spec foo((() -> [integer])) :: integer + def foo(_), do: 1 + end + + assert [{{:foo, 1}, [{:type, _, :fun, [args, return]}]}] = specs(bytecode) + assert {:type, _, :product, [{:type, _, :fun, fun_args}]} = args + assert [{:type, _, :product, []}, {:type, _, :list, [{:type, _, :integer, []}]}] = fun_args + assert {:type, _, :integer, []} = return + end + end + + describe "Code.Typespec" do + test "type_to_quoted" do + quoted = + Enum.sort([ + quote(do: @type(tuple(arg) :: {:tuple, arg})), + quote(do: @type(with_ann() :: t :: atom())), + quote(do: @type(a_tuple() :: tuple())), + quote(do: @type(empty_tuple() :: {})), + quote(do: @type(one_tuple() :: {:foo})), + quote(do: @type(two_tuple() :: {:foo, :bar})), + quote(do: @type(custom_tuple() :: tuple(:foo))), + quote(do: @type(imm_type_1() :: 1)), + quote(do: @type(imm_type_2() :: :foo)), + quote(do: @type(simple_type() :: integer())), + quote(do: @type(param_type(p) :: [p])), + quote(do: @type(union_type() :: integer() | binary() | boolean())), + quote(do: @type(binary_type1() :: <<_::_*8>>)), + quote(do: @type(binary_type2() :: <<_::3>>)), + quote(do: @type(binary_type3() :: <<_::3, _::_*8>>)), + quote(do: @type(tuple_type() :: {integer()})), + quote( + do: @type(ftype() :: (() -> any()) | (() -> integer()) | (integer() -> integer())) + ), + quote(do: @type(cl() :: charlist())), + quote(do: @type(st() :: struct())), + quote(do: @type(ab() :: as_boolean(term()))), + quote(do: @type(kw() :: keyword())), + quote(do: @type(kwt() :: keyword(term()))), + quote(do: @type(vaf() :: (... -> any()))), + quote(do: @type(rng() :: 1..10)), + quote(do: @type(opts() :: [first: integer(), step: integer(), last: integer()])), + quote(do: @type(ops() :: {+1, -1})), + quote(do: @type(map(arg) :: {:map, arg})), + quote(do: @type(a_map() :: map())), + quote(do: @type(empty_map() :: %{})), + quote(do: @type(my_map() :: %{hello: :world})), + quote(do: @type(my_req_map() :: %{required(0) => :foo})), + quote(do: @type(my_opt_map() :: %{optional(0) => :foo})), + quote(do: @type(my_struct() :: %TypespecTest{hello: :world})), + quote(do: @type(custom_map() :: map(:foo))), + quote(do: @type(list1() :: list())), + quote(do: @type(list2() :: [0])), + quote(do: @type(list3() :: [...])), + quote(do: @type(list4() :: [0, ...])), + quote(do: @type(nil_list() :: [])) + ]) + + bytecode = + test_module do + Module.eval_quoted(__MODULE__, quoted) + end + + types = types(bytecode) + + Enum.each(Enum.zip(types, quoted), fn {{:type, type}, definition} -> + ast = Code.Typespec.type_to_quoted(type) + assert Macro.to_string(quote(do: @type(unquote(ast)))) == Macro.to_string(definition) + end) + end + + test "type_to_quoted for paren_type" do + type = {:my_type, {:paren_type, 0, [{:type, 0, :integer, []}]}, []} + + assert Code.Typespec.type_to_quoted(type) == + {:"::", [], [{:my_type, [], []}, {:integer, [line: 0], []}]} + end + + test "spec_to_quoted" do + quoted = + Enum.sort([ + quote(do: @spec(foo() :: integer())), + quote(do: @spec(foo() :: union())), + quote(do: @spec(foo() :: union(integer()))), + quote(do: @spec(foo() :: truly_union())), + quote(do: @spec(foo(union()) :: union())), + quote(do: @spec(foo(union(integer())) :: union(integer()))), + quote(do: @spec(foo(truly_union()) :: truly_union())), + quote(do: @spec(foo(atom()) :: integer() | [{}])), + quote(do: @spec(foo(arg) :: integer() when [arg: integer()])), + quote(do: @spec(foo(arg) :: arg when [arg: var])), + quote(do: @spec(foo(arg :: atom()) :: atom())) + ]) + + bytecode = + test_module do + @type union :: any() + @type union(t) :: t + @type truly_union :: list | map | union + + def foo(), do: 1 + def foo(arg), do: arg + Module.eval_quoted(__MODULE__, quote(do: (unquote_splicing(quoted)))) + end + + specs = + Enum.flat_map(specs(bytecode), fn {{_, _}, specs} -> + Enum.map(specs, fn spec -> + quote(do: @spec(unquote(Code.Typespec.spec_to_quoted(:foo, spec)))) + end) + end) + + specs_with_quoted = specs |> Enum.sort() |> Enum.zip(quoted) + + Enum.each(specs_with_quoted, fn {spec, definition} -> + assert Macro.to_string(spec) == Macro.to_string(definition) + end) + end + + test "spec_to_quoted with maps with __struct__ key" do + defmodule A do + defstruct [:key] + end + + defmodule B do + defstruct [:key] + end + + bytecode = + test_module do + @spec single_struct(%A{}) :: :ok + def single_struct(arg), do: {:ok, arg} + + @spec single_struct_key(%{__struct__: A}) :: :ok + def single_struct_key(arg), do: {:ok, arg} + + @spec single_struct_key_type(%{__struct__: atom()}) :: :ok + def single_struct_key_type(arg), do: {:ok, arg} + + @spec union_struct(%A{} | %B{}) :: :ok + def union_struct(arg), do: {:ok, arg} + + @spec union_struct_key(%{__struct__: A | B}) :: :ok + def union_struct_key(arg), do: {:ok, arg} + + @spec union_struct_key_type(%{__struct__: atom() | A | binary()}) :: :ok + def union_struct_key_type(arg), do: {:ok, arg} + end + + [ + {{:single_struct, 1}, [ast_single_struct]}, + {{:single_struct_key, 1}, [ast_single_struct_key]}, + {{:single_struct_key_type, 1}, [ast_single_struct_key_type]}, + {{:union_struct, 1}, [ast_union_struct]}, + {{:union_struct_key, 1}, [ast_union_struct_key]}, + {{:union_struct_key_type, 1}, [ast_union_struct_key_type]} + ] = specs(bytecode) + + assert Code.Typespec.spec_to_quoted(:single_struct, ast_single_struct) + |> Macro.to_string() == + "single_struct(%TypespecTest.A{key: term()}) :: :ok" + + assert Code.Typespec.spec_to_quoted(:single_struct_key, ast_single_struct_key) + |> Macro.to_string() == + "single_struct_key(%TypespecTest.A{}) :: :ok" + + assert Code.Typespec.spec_to_quoted(:single_struct_key_type, ast_single_struct_key_type) + |> Macro.to_string() == + "single_struct_key_type(%{__struct__: atom()}) :: :ok" + + assert Code.Typespec.spec_to_quoted(:union_struct, ast_union_struct) |> Macro.to_string() == + "union_struct(%TypespecTest.A{key: term()} | %TypespecTest.B{key: term()}) :: :ok" + + assert Code.Typespec.spec_to_quoted(:union_struct_key, ast_union_struct_key) + |> Macro.to_string() == + "union_struct_key(%{__struct__: TypespecTest.A | TypespecTest.B}) :: :ok" + + assert Code.Typespec.spec_to_quoted(:union_struct_key_type, ast_union_struct_key_type) + |> Macro.to_string() == + "union_struct_key_type(%{__struct__: atom() | TypespecTest.A | binary()}) :: :ok" + end + + test "non-variables are given as arguments" do + msg = ~r/The type one_bad_variable\/1 has an invalid argument\(s\): String.t\(\)/ + + assert_raise CompileError, msg, fn -> + test_module do + @type one_bad_variable(String.t()) :: String.t() + end + end + + msg = ~r/The type two_bad_variables\/2 has an invalid argument\(s\): :ok, Enumerable.t\(\)/ + + assert_raise CompileError, msg, fn -> + test_module do + @type two_bad_variables(:ok, Enumerable.t()) :: {:ok, []} + end + end + + msg = ~r/The type one_bad_one_good\/2 has an invalid argument\(s\): \"\"/ + + assert_raise CompileError, msg, fn -> + test_module do + @type one_bad_one_good(input1, "") :: {:ok, input1} + end + end + end + + test "retrieval invalid data" do + assert Code.Typespec.fetch_types(Unknown) == :error + assert Code.Typespec.fetch_specs(Unknown) == :error + end + + # This is a test that implements all types specified in lib/elixir/pages/typespecs.md + test "documented types and their AST" do + defmodule SomeStruct do + defstruct [:key] + end + + quoted = + Enum.sort([ + ## Basic types + quote(do: @type(basic_any() :: any())), + quote(do: @type(basic_none() :: none())), + quote(do: @type(basic_atom() :: atom())), + quote(do: @type(basic_map() :: map())), + quote(do: @type(basic_pid() :: pid())), + quote(do: @type(basic_port() :: port())), + quote(do: @type(basic_reference() :: reference())), + quote(do: @type(basic_struct() :: struct())), + quote(do: @type(basic_tuple() :: tuple())), + + # Numbers + quote(do: @type(basic_float() :: float())), + quote(do: @type(basic_integer() :: integer())), + quote(do: @type(basic_neg_integer() :: neg_integer())), + quote(do: @type(basic_non_neg_integer() :: non_neg_integer())), + quote(do: @type(basic_pos_integer() :: pos_integer())), + + # Lists + quote(do: @type(basic_list_type() :: list(integer()))), + quote(do: @type(basic_nonempty_list_type() :: nonempty_list(integer()))), + quote do + @type basic_maybe_improper_list_type() :: maybe_improper_list(integer(), atom()) + end, + quote do + @type basic_nonempty_improper_list_type() :: nonempty_improper_list(integer(), atom()) + end, + quote do + @type basic_nonempty_maybe_improper_list_type() :: + nonempty_maybe_improper_list(integer(), atom()) + end, + + ## Literals + quote(do: @type(literal_atom() :: :atom)), + quote(do: @type(literal_integer() :: 1)), + quote(do: @type(literal_integers() :: 1..10)), + quote(do: @type(literal_empty_bitstring() :: <<>>)), + quote(do: @type(literal_size_0() :: <<_::0>>)), + quote(do: @type(literal_unit_1() :: <<_::_*1>>)), + quote(do: @type(literal_size_1_unit_8() :: <<_::100, _::_*256>>)), + quote(do: @type(literal_function_arity_any() :: (... -> integer()))), + quote(do: @type(literal_function_arity_0() :: (() -> integer()))), + quote(do: @type(literal_function_arity_2() :: (integer(), atom() -> integer()))), + quote(do: @type(literal_list_type() :: [integer()])), + quote(do: @type(literal_empty_list() :: [])), + quote(do: @type(literal_list_nonempty() :: [...])), + quote(do: @type(literal_nonempty_list_type() :: [atom(), ...])), + quote(do: @type(literal_keyword_list_fixed_key() :: [key: integer()])), + quote(do: @type(literal_keyword_list_fixed_key2() :: [{:key, integer()}])), + quote(do: @type(literal_keyword_list_type_key() :: [{binary(), integer()}])), + quote(do: @type(literal_empty_map() :: %{})), + quote(do: @type(literal_map_with_key() :: %{key: integer()})), + quote( + do: @type(literal_map_with_required_key() :: %{required(bitstring()) => integer()}) + ), + quote( + do: @type(literal_map_with_optional_key() :: %{optional(bitstring()) => integer()}) + ), + quote(do: @type(literal_struct_all_fields_any_type() :: %SomeStruct{})), + quote(do: @type(literal_struct_all_fields_key_type() :: %SomeStruct{key: integer()})), + quote(do: @type(literal_empty_tuple() :: {})), + quote(do: @type(literal_2_element_tuple() :: {1, 2})), + + ## Built-in types + quote(do: @type(built_in_term() :: term())), + quote(do: @type(built_in_arity() :: arity())), + quote(do: @type(built_in_as_boolean() :: as_boolean(:t))), + quote(do: @type(built_in_binary() :: binary())), + quote(do: @type(built_in_bitstring() :: bitstring())), + quote(do: @type(built_in_boolean() :: boolean())), + quote(do: @type(built_in_byte() :: byte())), + quote(do: @type(built_in_char() :: char())), + quote(do: @type(built_in_charlist() :: charlist())), + quote(do: @type(built_in_nonempty_charlist() :: nonempty_charlist())), + quote(do: @type(built_in_fun() :: fun())), + quote(do: @type(built_in_function() :: function())), + quote(do: @type(built_in_identifier() :: identifier())), + quote(do: @type(built_in_iodata() :: iodata())), + quote(do: @type(built_in_iolist() :: iolist())), + quote(do: @type(built_in_keyword() :: keyword())), + quote(do: @type(built_in_keyword_value_type() :: keyword(:t))), + quote(do: @type(built_in_list() :: list())), + quote(do: @type(built_in_nonempty_list() :: nonempty_list())), + quote(do: @type(built_in_maybe_improper_list() :: maybe_improper_list())), + quote( + do: @type(built_in_nonempty_maybe_improper_list() :: nonempty_maybe_improper_list()) + ), + quote(do: @type(built_in_mfa() :: mfa())), + quote(do: @type(built_in_module() :: module())), + quote(do: @type(built_in_no_return() :: no_return())), + quote(do: @type(built_in_node() :: node())), + quote(do: @type(built_in_number() :: number())), + quote(do: @type(built_in_struct() :: struct())), + quote(do: @type(built_in_timeout() :: timeout())), + + ## Remote types + quote(do: @type(remote_enum_t0() :: Enum.t())), + quote(do: @type(remote_keyword_t1() :: Keyword.t(integer()))) + ]) + + bytecode = + test_module do + Module.eval_quoted(__MODULE__, quoted) + end + + types = types(bytecode) + + Enum.each(Enum.zip(types, quoted), fn {{:type, type}, definition} -> + ast = Code.Typespec.type_to_quoted(type) + ast_string = Macro.to_string(quote(do: @type(unquote(ast)))) + + case type do + # These cases do not translate directly to their own string version. + {:basic_list_type, _, _} -> + assert ast_string == "@type basic_list_type() :: [integer()]" + + {:basic_nonempty_list_type, _, _} -> + assert ast_string == "@type basic_nonempty_list_type() :: [integer(), ...]" + + {:literal_empty_bitstring, _, _} -> + assert ast_string == "@type literal_empty_bitstring() :: <<_::0>>" + + {:literal_keyword_list_fixed_key, _, _} -> + assert ast_string == "@type literal_keyword_list_fixed_key() :: [{:key, integer()}]" + + {:literal_keyword_list_fixed_key2, _, _} -> + assert ast_string == "@type literal_keyword_list_fixed_key2() :: [{:key, integer()}]" + + {:literal_struct_all_fields_any_type, _, _} -> + assert ast_string == + "@type literal_struct_all_fields_any_type() :: %TypespecTest.SomeStruct{key: term()}" + + {:literal_struct_all_fields_key_type, _, _} -> + assert ast_string == + "@type literal_struct_all_fields_key_type() :: %TypespecTest.SomeStruct{key: integer()}" + + {:built_in_fun, _, _} -> + assert ast_string == "@type built_in_fun() :: (... -> any())" + + {:built_in_nonempty_list, _, _} -> + assert ast_string == "@type built_in_nonempty_list() :: [...]" + + _ -> + assert ast_string == Macro.to_string(definition) + end + end) + end + end + + describe "behaviour_info" do + defmodule SampleCallbacks do + @callback first(integer) :: integer + @callback foo(atom(), binary) :: binary + @callback bar(External.hello(), my_var :: binary) :: binary + @callback guarded(my_var) :: my_var when my_var: binary + @callback orr(atom | integer) :: atom + @callback literal(123, {atom}, :foo, [integer], true) :: atom + @macrocallback last(integer) :: Macro.t() + @macrocallback last() :: atom + @optional_callbacks bar: 2, last: 0 + @optional_callbacks first: 1 + end + + test "defines callbacks" do + expected_callbacks = [ + "MACRO-last": 1, + "MACRO-last": 2, + bar: 2, + first: 1, + foo: 2, + guarded: 1, + literal: 5, + orr: 1 + ] + + assert Enum.sort(SampleCallbacks.behaviour_info(:callbacks)) == expected_callbacks + end + + test "defines optional callbacks" do + assert Enum.sort(SampleCallbacks.behaviour_info(:optional_callbacks)) == + ["MACRO-last": 1, bar: 2, first: 1] + end + end + + @tag tmp_dir: true + test "erlang module", c do + erlc(c, :typespec_test_mod, """ + -module(typespec_test_mod). + -export([f/1]). + -export_type([t/1]). + + -type t(X) :: list(X). + + -spec f(X) -> X. + f(X) -> X. + """) + + [type: type] = types(:typespec_test_mod) + line = 5 + + assert Code.Typespec.type_to_quoted(type) == + {:"::", [], [{:t, [], [{:x, [line: line], nil}]}, [{:x, [line: line], nil}]]} + + [{{:f, 1}, [spec]}] = specs(:typespec_test_mod) + line = 7 + + assert Code.Typespec.spec_to_quoted(:f, spec) == + {:when, [line: line], + [ + {:"::", [line: line], + [{:f, [line: line], [{:x, [line: line], nil}]}, {:x, [line: line], nil}]}, + [x: {:var, [line: line], nil}] + ]} + end + + defp erlc(context, module, code) do + dir = context.tmp_dir + + src_path = Path.join([dir, "#{module}.erl"]) + src_path |> Path.dirname() |> File.mkdir_p!() + File.write!(src_path, code) + + ebin_dir = Path.join(dir, "ebin") + File.mkdir_p!(ebin_dir) + + {:ok, module} = + :compile.file(String.to_charlist(src_path), [ + :debug_info, + outdir: String.to_charlist(ebin_dir) + ]) + + true = Code.prepend_path(ebin_dir) + {:module, ^module} = :code.load_file(module) + + ExUnit.Callbacks.on_exit(fn -> + :code.purge(module) + :code.delete(module) + File.rm_rf!(dir) + end) + + :ok + end +end diff --git a/lib/elixir/test/elixir/uri_test.exs b/lib/elixir/test/elixir/uri_test.exs index 76126d74766..4a553c48350 100644 --- a/lib/elixir/test/elixir/uri_test.exs +++ b/lib/elixir/test/elixir/uri_test.exs @@ -1,202 +1,695 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule URITest do use ExUnit.Case, async: true - test :encode do + doctest URI + + test "encode/1,2" do assert URI.encode("4_test.is-s~") == "4_test.is-s~" + assert URI.encode("\r\n&<%>\" ゆ", &URI.char_unreserved?/1) == - "%0D%0A%26%3C%25%3E%22%20%E3%82%86" + "%0D%0A%26%3C%25%3E%22%20%E3%82%86" end - test :encode_www_form do + test "encode_www_form/1" do assert URI.encode_www_form("4test ~1.x") == "4test+~1.x" assert URI.encode_www_form("poll:146%") == "poll%3A146%25" assert URI.encode_www_form("/\n+/ゆ") == "%2F%0A%2B%2F%E3%82%86" end - test :encode_query do + test "encode_query/1,2" do assert URI.encode_query([{:foo, :bar}, {:baz, :quux}]) == "foo=bar&baz=quux" assert URI.encode_query([{"foo", "bar"}, {"baz", "quux"}]) == "foo=bar&baz=quux" + assert URI.encode_query([{"foo z", :bar}]) == "foo+z=bar" + assert URI.encode_query([{"foo z", :bar}], :rfc3986) == "foo%20z=bar" + assert URI.encode_query([{"foo z", :bar}], :www_form) == "foo+z=bar" + + assert URI.encode_query([{"foo[]", "+=/?&# Ñ"}]) == + "foo%5B%5D=%2B%3D%2F%3F%26%23+%C3%91" + + assert URI.encode_query([{"foo[]", "+=/?&# Ñ"}], :rfc3986) == + "foo%5B%5D=%2B%3D%2F%3F%26%23%20%C3%91" + + assert URI.encode_query([{"foo[]", "+=/?&# Ñ"}], :www_form) == + "foo%5B%5D=%2B%3D%2F%3F%26%23+%C3%91" assert_raise ArgumentError, fn -> URI.encode_query([{"foo", 'bar'}]) end + + assert_raise ArgumentError, fn -> + URI.encode_query([{'foo', "bar"}]) + end end - test :decode_query do - assert URI.decode_query("", []) == [] + test "decode_query/1,2,3" do assert URI.decode_query("", %{}) == %{} + assert URI.decode_query("safe=off", %{"cookie" => "foo"}) == + %{"safe" => "off", "cookie" => "foo"} + assert URI.decode_query("q=search%20query&cookie=ab%26cd&block+buster=") == - %{"block buster" => "", "cookie" => "ab&cd", "q" => "search query"} + %{"block buster" => "", "cookie" => "ab&cd", "q" => "search query"} - assert URI.decode_query("something=weird%3Dhappening") == - %{"something" => "weird=happening"} + assert URI.decode_query("q=search%20query&cookie=ab%26cd&block+buster=", %{}, :rfc3986) == + %{"block+buster" => "", "cookie" => "ab&cd", "q" => "search query"} - assert URI.decode_query("garbage") == - %{"garbage" => nil} - assert URI.decode_query("=value") == - %{"" => "value"} - assert URI.decode_query("something=weird=happening") == - %{"something" => "weird=happening"} + assert URI.decode_query("something=weird%3Dhappening") == %{"something" => "weird=happening"} + + assert URI.decode_query("=") == %{"" => ""} + assert URI.decode_query("key") == %{"key" => ""} + assert URI.decode_query("key=") == %{"key" => ""} + assert URI.decode_query("=value") == %{"" => "value"} + assert URI.decode_query("something=weird=happening") == %{"something" => "weird=happening"} end - test :decoder do - decoder = URI.query_decoder("q=search%20query&cookie=ab%26cd&block%20buster=") + test "query_decoder/1,2" do + decoder = URI.query_decoder("q=search%20query&cookie=ab%26cd&block+buster=") expected = [{"q", "search query"}, {"cookie", "ab&cd"}, {"block buster", ""}] - assert Enum.map(decoder, &(&1)) == expected + assert Enum.map(decoder, & &1) == expected + + decoder = URI.query_decoder("q=search%20query&cookie=ab%26cd&block+buster=", :rfc3986) + expected = [{"q", "search query"}, {"cookie", "ab&cd"}, {"block+buster", ""}] + assert Enum.map(decoder, & &1) == expected end - test :decode do + test "decode/1" do assert URI.decode("%0D%0A%26%3C%25%3E%22%20%E3%82%86") == "\r\n&<%>\" ゆ" assert URI.decode("%2f%41%4a%55") == "/AJU" assert URI.decode("4_t+st.is-s~") == "4_t+st.is-s~" - - assert_raise ArgumentError, ~R/malformed URI/, fn -> - URI.decode("% invalid") - end - assert_raise ArgumentError, ~R/malformed URI/, fn -> - URI.decode("invalid%") - end + assert URI.decode("% invalid") == "% invalid" + assert URI.decode("invalid %") == "invalid %" + assert URI.decode("%%") == "%%" end - test :decode_www_form do + test "decode_www_form/1" do assert URI.decode_www_form("%3Eval+ue%2B") == ">val ue+" assert URI.decode_www_form("%E3%82%86+") == "ゆ " + assert URI.decode_www_form("% invalid") == "% invalid" + assert URI.decode_www_form("invalid %") == "invalid %" + assert URI.decode_www_form("%%") == "%%" end - test :parse_uri do - assert URI.parse(uri = %URI{scheme: "http", host: "foo.com"}) == uri - end + describe "new/1" do + test "empty" do + assert URI.new("") == {:ok, %URI{}} + end - test :parse_http do - assert %URI{scheme: "http", host: "foo.com", path: "/path/to/something", - query: "foo=bar&bar=foo", fragment: "fragment", port: 80, - authority: "foo.com", userinfo: nil} == - URI.parse("http://foo.com/path/to/something?foo=bar&bar=foo#fragment") + test "errors on bad URIs" do + assert URI.new("/>") == {:error, ">"} + assert URI.new(":https") == {:error, ":"} + assert URI.new("ht\0tps://foo.com") == {:error, "\0"} + end end - test :parse_https do - assert %URI{scheme: "https", host: "foo.com", authority: "foo.com", - query: nil, fragment: nil, port: 443, path: nil, userinfo: nil} == - URI.parse("https://foo.com") - end + describe "new!/1" do + test "returns the given URI if a %URI{} struct is given" do + assert URI.new!(uri = %URI{scheme: "http", host: "foo.com"}) == uri + end - test :parse_file do - assert %URI{scheme: "file", host: nil, path: "/foo/bar/baz", userinfo: nil, - query: nil, fragment: nil, port: nil, authority: nil} == - URI.parse("file:///foo/bar/baz") - end + test "works with HTTP scheme" do + expected_uri = %URI{ + scheme: "http", + host: "foo.com", + path: "/path/to/something", + query: "foo=bar&bar=foo", + fragment: "fragment", + port: 80, + userinfo: nil + } + + assert URI.new!("http://foo.com/path/to/something?foo=bar&bar=foo#fragment") == + expected_uri + end - test :parse_ftp do - assert %URI{scheme: "ftp", host: "private.ftp-servers.example.com", - userinfo: "user001:secretpassword", authority: "user001:secretpassword@private.ftp-servers.example.com", - path: "/mydirectory/myfile.txt", query: nil, fragment: nil, - port: 21} == - URI.parse("ftp://user001:secretpassword@private.ftp-servers.example.com/mydirectory/myfile.txt") - end + test "works with HTTPS scheme" do + expected_uri = %URI{ + scheme: "https", + host: "foo.com", + query: nil, + fragment: nil, + port: 443, + path: nil, + userinfo: nil + } + + assert URI.new!("https://foo.com") == expected_uri + end - test :parse_sftp do - assert %URI{scheme: "sftp", host: "private.ftp-servers.example.com", - userinfo: "user001:secretpassword", authority: "user001:secretpassword@private.ftp-servers.example.com", - path: "/mydirectory/myfile.txt", query: nil, fragment: nil, port: 22} == - URI.parse("sftp://user001:secretpassword@private.ftp-servers.example.com/mydirectory/myfile.txt") - end + test "works with file scheme" do + expected_uri = %URI{ + scheme: "file", + host: "", + path: "/foo/bar/baz", + userinfo: nil, + query: nil, + fragment: nil, + port: nil + } + + assert URI.new!("file:///foo/bar/baz") == expected_uri + end - test :parse_tftp do - assert %URI{scheme: "tftp", host: "private.ftp-servers.example.com", - userinfo: "user001:secretpassword", authority: "user001:secretpassword@private.ftp-servers.example.com", - path: "/mydirectory/myfile.txt", query: nil, fragment: nil, port: 69} == - URI.parse("tftp://user001:secretpassword@private.ftp-servers.example.com/mydirectory/myfile.txt") - end + test "works with FTP scheme" do + expected_uri = %URI{ + scheme: "ftp", + host: "private.ftp-server.example.com", + userinfo: "user001:password", + path: "/my_directory/my_file.txt", + query: nil, + fragment: nil, + port: 21 + } + + ftp = "ftp://user001:password@private.ftp-server.example.com/my_directory/my_file.txt" + assert URI.new!(ftp) == expected_uri + end + test "works with SFTP scheme" do + expected_uri = %URI{ + scheme: "sftp", + host: "private.ftp-server.example.com", + userinfo: "user001:password", + path: "/my_directory/my_file.txt", + query: nil, + fragment: nil, + port: 22 + } + + sftp = "sftp://user001:password@private.ftp-server.example.com/my_directory/my_file.txt" + assert URI.new!(sftp) == expected_uri + end - test :parse_ldap do - assert %URI{scheme: "ldap", host: nil, authority: nil, userinfo: nil, - path: "/dc=example,dc=com", query: "?sub?(givenName=John)", - fragment: nil, port: 389} == - URI.parse("ldap:///dc=example,dc=com??sub?(givenName=John)") - assert %URI{scheme: "ldap", host: "ldap.example.com", authority: "ldap.example.com", - userinfo: nil, path: "/cn=John%20Doe,dc=example,dc=com", fragment: nil, - port: 389, query: nil} == - URI.parse("ldap://ldap.example.com/cn=John%20Doe,dc=example,dc=com") - end + test "works with TFTP scheme" do + expected_uri = %URI{ + scheme: "tftp", + host: "private.ftp-server.example.com", + userinfo: "user001:password", + path: "/my_directory/my_file.txt", + query: nil, + fragment: nil, + port: 69 + } + + tftp = "tftp://user001:password@private.ftp-server.example.com/my_directory/my_file.txt" + assert URI.new!(tftp) == expected_uri + end - test :parse_splits_authority do - assert %URI{scheme: "http", host: "foo.com", path: nil, - query: nil, fragment: nil, port: 4444, - authority: "foo:bar@foo.com:4444", - userinfo: "foo:bar"} == - URI.parse("http://foo:bar@foo.com:4444") - assert %URI{scheme: "https", host: "foo.com", path: nil, - query: nil, fragment: nil, port: 443, - authority: "foo:bar@foo.com", userinfo: "foo:bar"} == - URI.parse("https://foo:bar@foo.com") - assert %URI{scheme: "http", host: "foo.com", path: nil, - query: nil, fragment: nil, port: 4444, - authority: "foo.com:4444", userinfo: nil} == - URI.parse("http://foo.com:4444") + test "works with LDAP scheme" do + expected_uri = %URI{ + scheme: "ldap", + host: "", + userinfo: nil, + path: "/dc=example,dc=com", + query: "?sub?(givenName=John)", + fragment: nil, + port: 389 + } + + assert URI.new!("ldap:///dc=example,dc=com??sub?(givenName=John)") == expected_uri + + expected_uri = %URI{ + scheme: "ldap", + host: "ldap.example.com", + userinfo: nil, + path: "/cn=John%20Doe,dc=foo,dc=com", + fragment: nil, + port: 389, + query: nil + } + + assert URI.new!("ldap://ldap.example.com/cn=John%20Doe,dc=foo,dc=com") == expected_uri + end + + test "can parse IPv6 addresses" do + addresses = [ + # undefined + "::", + # loopback + "::1", + # unicast + "1080::8:800:200C:417A", + # multicast + "FF01::101", + # link-local + "fe80::", + # abbreviated + "2607:f3f0:2:0:216:3cff:fef0:174a", + # mixed hex case + "2607:f3F0:2:0:216:3cFf:Fef0:174A", + # complete + "2051:0db8:2d5a:3521:8313:ffad:1242:8e2e", + # embedded IPv4 + "::00:192.168.10.184" + ] + + Enum.each(addresses, fn addr -> + simple_uri = URI.new!("http://[#{addr}]/") + assert simple_uri.host == addr + + userinfo_uri = URI.new!("http://user:pass@[#{addr}]/") + assert userinfo_uri.host == addr + assert userinfo_uri.userinfo == "user:pass" + + port_uri = URI.new!("http://[#{addr}]:2222/") + assert port_uri.host == addr + assert port_uri.port == 2222 + + userinfo_port_uri = URI.new!("http://user:pass@[#{addr}]:2222/") + assert userinfo_port_uri.host == addr + assert userinfo_port_uri.userinfo == "user:pass" + assert userinfo_port_uri.port == 2222 + end) + end + + test "downcases the scheme" do + assert URI.new!("hTtP://google.com").scheme == "http" + end + + test "preserves empty fragments" do + assert URI.new!("http://example.com#").fragment == "" + assert URI.new!("http://example.com/#").fragment == "" + assert URI.new!("http://example.com/test#").fragment == "" + end + + test "preserves an empty query" do + assert URI.new!("http://foo.com/?").query == "" + end end - test :default_port do + test "default_port/1,2" do assert URI.default_port("http") == 80 - assert URI.default_port("unknown") == nil + try do + URI.default_port("http", 8000) + assert URI.default_port("http") == 8000 + after + URI.default_port("http", 80) + end + + assert URI.default_port("unknown") == nil URI.default_port("unknown", 13) assert URI.default_port("unknown") == 13 end - test :parse_bad_uris do - assert URI.parse("https:??@?F?@#>F//23/") - assert URI.parse("") - assert URI.parse(":https") - assert URI.parse("https") + test "to_string/1 and Kernel.to_string/1" do + assert to_string(URI.new!("http://google.com")) == "http://google.com" + assert to_string(URI.new!("http://google.com:443")) == "http://google.com:443" + assert to_string(URI.new!("https://google.com:443")) == "https://google.com" + assert to_string(URI.new!("file:/path")) == "file:/path" + assert to_string(URI.new!("file:///path")) == "file:///path" + assert to_string(URI.new!("file://///path")) == "file://///path" + assert to_string(URI.new!("http://lol:wut@google.com")) == "http://lol:wut@google.com" + assert to_string(URI.new!("http://google.com/elixir")) == "http://google.com/elixir" + assert to_string(URI.new!("http://google.com?q=lol")) == "http://google.com?q=lol" + assert to_string(URI.new!("http://google.com?q=lol#omg")) == "http://google.com?q=lol#omg" + assert to_string(URI.new!("//google.com/elixir")) == "//google.com/elixir" + assert to_string(URI.new!("//google.com:8080/elixir")) == "//google.com:8080/elixir" + assert to_string(URI.new!("//user:password@google.com/")) == "//user:password@google.com/" + assert to_string(URI.new!("http://[2001:db8::]:8080")) == "http://[2001:db8::]:8080" + assert to_string(URI.new!("http://[2001:db8::]")) == "http://[2001:db8::]" + + assert URI.to_string(URI.new!("http://google.com")) == "http://google.com" + assert URI.to_string(URI.new!("gid:hello/123")) == "gid:hello/123" + + assert URI.to_string(URI.new!("//user:password@google.com/")) == + "//user:password@google.com/" + + assert_raise ArgumentError, + ~r":path in URI must be empty or an absolute path if URL has a :host", + fn -> %URI{host: "foo.com", path: "hello/123"} |> URI.to_string() end end - test :ipv6_addresses do - addrs = [ - "::", # undefined - "::1", # loopback - "1080::8:800:200C:417A", # unicast - "FF01::101", # multicast - "2607:f3f0:2:0:216:3cff:fef0:174a", # abbreviated - "2607:f3F0:2:0:216:3cFf:Fef0:174A", # mixed hex case - "2051:0db8:2d5a:3521:8313:ffad:1242:8e2e", # complete - "::00:192.168.10.184" # embedded IPv4 - ] - - Enum.each addrs, fn(addr) -> - simple_uri = URI.parse("http://[#{addr}]/") - assert simple_uri.host == addr - - userinfo_uri = URI.parse("http://user:pass@[#{addr}]/") - assert userinfo_uri.host == addr - assert userinfo_uri.userinfo == "user:pass" - - port_uri = URI.parse("http://[#{addr}]:2222/") - assert port_uri.host == addr - assert port_uri.port == 2222 - - userinfo_port_uri = URI.parse("http://user:pass@[#{addr}]:2222/") - assert userinfo_port_uri.host == addr - assert userinfo_port_uri.userinfo == "user:pass" - assert userinfo_port_uri.port == 2222 + test "merge/2" do + assert_raise ArgumentError, "you must merge onto an absolute URI", fn -> + URI.merge("/relative", "") end + + assert URI.merge("http://google.com/foo", "http://example.com/baz") + |> to_string == "http://example.com/baz" + + assert URI.merge("http://google.com/foo", "http://example.com/.././bar/../../baz") + |> to_string == "http://example.com/baz" + + assert URI.merge("http://google.com/foo", "//example.com/baz") + |> to_string == "http://example.com/baz" + + assert URI.merge("http://google.com/foo", "//example.com/.././bar/../../../baz") + |> to_string == "http://example.com/baz" + + assert URI.merge("http://example.com", URI.new!("/foo")) + |> to_string == "http://example.com/foo" + + assert URI.merge("http://example.com", URI.new!("/.././bar/../../../baz")) + |> to_string == "http://example.com/baz" + + base = URI.new!("http://example.com/foo/bar") + assert URI.merge(base, "") |> to_string == "http://example.com/foo/bar" + assert URI.merge(base, "#fragment") |> to_string == "http://example.com/foo/bar#fragment" + assert URI.merge(base, "?query") |> to_string == "http://example.com/foo/bar?query" + assert URI.merge(base, %URI{}) |> to_string == "http://example.com/foo/bar" + + assert URI.merge(base, %URI{fragment: "fragment"}) + |> to_string == "http://example.com/foo/bar#fragment" + + base = URI.new!("http://example.com") + assert URI.merge(base, "/foo") |> to_string == "http://example.com/foo" + assert URI.merge(base, "foo") |> to_string == "http://example.com/foo" + + base = URI.new!("http://example.com/foo/bar") + assert URI.merge(base, "/baz") |> to_string == "http://example.com/baz" + assert URI.merge(base, "baz") |> to_string == "http://example.com/foo/baz" + assert URI.merge(base, "../baz") |> to_string == "http://example.com/baz" + assert URI.merge(base, ".././baz") |> to_string == "http://example.com/baz" + assert URI.merge(base, "./baz") |> to_string == "http://example.com/foo/baz" + assert URI.merge(base, "bar/./baz") |> to_string == "http://example.com/foo/bar/baz" + + base = URI.new!("http://example.com/foo/bar/") + assert URI.merge(base, "/baz") |> to_string == "http://example.com/baz" + assert URI.merge(base, "baz") |> to_string == "http://example.com/foo/bar/baz" + assert URI.merge(base, "../baz") |> to_string == "http://example.com/foo/baz" + assert URI.merge(base, ".././baz") |> to_string == "http://example.com/foo/baz" + assert URI.merge(base, "./baz") |> to_string == "http://example.com/foo/bar/baz" + assert URI.merge(base, "bar/./baz") |> to_string == "http://example.com/foo/bar/bar/baz" + + base = URI.new!("http://example.com/foo/bar/baz") + assert URI.merge(base, "../../foobar") |> to_string == "http://example.com/foobar" + assert URI.merge(base, "../../../foobar") |> to_string == "http://example.com/foobar" + assert URI.merge(base, "../../../../../../foobar") |> to_string == "http://example.com/foobar" + + base = URI.new!("http://example.com/foo/../bar") + assert URI.merge(base, "baz") |> to_string == "http://example.com/baz" + + base = URI.new!("http://example.com/foo/./bar") + assert URI.merge(base, "baz") |> to_string == "http://example.com/foo/baz" + + base = URI.new!("http://example.com/foo?query1") + assert URI.merge(base, "?query2") |> to_string == "http://example.com/foo?query2" + assert URI.merge(base, "") |> to_string == "http://example.com/foo?query1" + + base = URI.new!("http://example.com/foo#fragment1") + assert URI.merge(base, "#fragment2") |> to_string == "http://example.com/foo#fragment2" + assert URI.merge(base, "") |> to_string == "http://example.com/foo" + + page_url = "https://example.com/guide/" + image_url = "https://images.example.com/t/1600x/https://images.example.com/foo.jpg" + + assert URI.merge(URI.new!(page_url), URI.new!(image_url)) |> to_string == + "https://images.example.com/t/1600x/https://images.example.com/foo.jpg" end - test :downcase_scheme do - assert URI.parse("hTtP://google.com").scheme == "http" + ## Deprecate API + + describe "authority" do + test "to_string" do + assert URI.to_string(%URI{authority: "foo@example.com:80"}) == + "//foo@example.com:80" + + assert URI.to_string(%URI{userinfo: "bar", host: "example.org", port: 81}) == + "//bar@example.org:81" + + assert URI.to_string(%URI{ + authority: "foo@example.com:80", + userinfo: "bar", + host: "example.org", + port: 81 + }) == + "//bar@example.org:81" + end end - test :to_string do - assert to_string(URI.parse("http://google.com")) == "http://google.com" - assert to_string(URI.parse("http://google.com:443")) == "http://google.com:443" - assert to_string(URI.parse("https://google.com:443")) == "https://google.com" - assert to_string(URI.parse("http://lol:wut@google.com")) == "http://lol:wut@google.com" - assert to_string(URI.parse("http://google.com/elixir")) == "http://google.com/elixir" - assert to_string(URI.parse("http://google.com?q=lol")) == "http://google.com?q=lol" - assert to_string(URI.parse("http://google.com?q=lol#omg")) == "http://google.com?q=lol#omg" + describe "parse/1" do + test "returns the given URI if a %URI{} struct is given" do + assert URI.parse(uri = %URI{scheme: "http", host: "foo.com"}) == uri + end + + test "works with HTTP scheme" do + expected_uri = %URI{ + scheme: "http", + host: "foo.com", + path: "/path/to/something", + query: "foo=bar&bar=foo", + fragment: "fragment", + port: 80, + authority: "foo.com", + userinfo: nil + } + + assert URI.parse("http://foo.com/path/to/something?foo=bar&bar=foo#fragment") == + expected_uri + end + + test "works with HTTPS scheme" do + expected_uri = %URI{ + scheme: "https", + host: "foo.com", + authority: "foo.com", + query: nil, + fragment: nil, + port: 443, + path: nil, + userinfo: nil + } + + assert URI.parse("https://foo.com") == expected_uri + end + + test "works with \"file\" scheme" do + expected_uri = %URI{ + scheme: "file", + host: "", + path: "/foo/bar/baz", + userinfo: nil, + query: nil, + fragment: nil, + port: nil, + authority: "" + } + + assert URI.parse("file:///foo/bar/baz") == expected_uri + end + + test "works with FTP scheme" do + expected_uri = %URI{ + scheme: "ftp", + host: "private.ftp-server.example.com", + userinfo: "user001:password", + authority: "user001:password@private.ftp-server.example.com", + path: "/my_directory/my_file.txt", + query: nil, + fragment: nil, + port: 21 + } + + ftp = "ftp://user001:password@private.ftp-server.example.com/my_directory/my_file.txt" + assert URI.parse(ftp) == expected_uri + end + + test "works with SFTP scheme" do + expected_uri = %URI{ + scheme: "sftp", + host: "private.ftp-server.example.com", + userinfo: "user001:password", + authority: "user001:password@private.ftp-server.example.com", + path: "/my_directory/my_file.txt", + query: nil, + fragment: nil, + port: 22 + } + + sftp = "sftp://user001:password@private.ftp-server.example.com/my_directory/my_file.txt" + assert URI.parse(sftp) == expected_uri + end + + test "works with TFTP scheme" do + expected_uri = %URI{ + scheme: "tftp", + host: "private.ftp-server.example.com", + userinfo: "user001:password", + authority: "user001:password@private.ftp-server.example.com", + path: "/my_directory/my_file.txt", + query: nil, + fragment: nil, + port: 69 + } + + tftp = "tftp://user001:password@private.ftp-server.example.com/my_directory/my_file.txt" + assert URI.parse(tftp) == expected_uri + end + + test "works with LDAP scheme" do + expected_uri = %URI{ + scheme: "ldap", + host: "", + authority: "", + userinfo: nil, + path: "/dc=example,dc=com", + query: "?sub?(givenName=John)", + fragment: nil, + port: 389 + } + + assert URI.parse("ldap:///dc=example,dc=com??sub?(givenName=John)") == expected_uri + + expected_uri = %URI{ + scheme: "ldap", + host: "ldap.example.com", + authority: "ldap.example.com", + userinfo: nil, + path: "/cn=John%20Doe,dc=foo,dc=com", + fragment: nil, + port: 389, + query: nil + } + + assert URI.parse("ldap://ldap.example.com/cn=John%20Doe,dc=foo,dc=com") == expected_uri + end + + test "works with WebSocket scheme" do + expected_uri = %URI{ + authority: "ws.example.com", + fragment: "content", + host: "ws.example.com", + path: "/path/to", + port: 80, + query: "here", + scheme: "ws", + userinfo: nil + } + + assert URI.parse("ws://ws.example.com/path/to?here#content") == expected_uri + end + + test "works with WebSocket Secure scheme" do + expected_uri = %URI{ + authority: "ws.example.com", + fragment: "content", + host: "ws.example.com", + path: "/path/to", + port: 443, + query: "here", + scheme: "wss", + userinfo: nil + } + + assert URI.parse("wss://ws.example.com/path/to?here#content") == expected_uri + end + + test "splits authority" do + expected_uri = %URI{ + scheme: "http", + host: "foo.com", + path: nil, + query: nil, + fragment: nil, + port: 4444, + authority: "foo:bar@foo.com:4444", + userinfo: "foo:bar" + } + + assert URI.parse("http://foo:bar@foo.com:4444") == expected_uri + + expected_uri = %URI{ + scheme: "https", + host: "foo.com", + path: nil, + query: nil, + fragment: nil, + port: 443, + authority: "foo:bar@foo.com", + userinfo: "foo:bar" + } + + assert URI.parse("https://foo:bar@foo.com") == expected_uri + + expected_uri = %URI{ + scheme: "http", + host: "foo.com", + path: nil, + query: nil, + fragment: nil, + port: 4444, + authority: "foo.com:4444", + userinfo: nil + } + + assert URI.parse("http://foo.com:4444") == expected_uri + end + + test "can parse bad URIs" do + assert URI.parse("") + assert URI.parse("https:??@?F?@#>F//23/") + + assert URI.parse(":https").path == ":https" + assert URI.parse("https").path == "https" + assert URI.parse("ht\0tps://foo.com").path == "ht\0tps://foo.com" + end + + test "can parse IPv6 addresses" do + addresses = [ + # undefined + "::", + # loopback + "::1", + # unicast + "1080::8:800:200C:417A", + # multicast + "FF01::101", + # link-local + "fe80::", + # abbreviated + "2607:f3f0:2:0:216:3cff:fef0:174a", + # mixed hex case + "2607:f3F0:2:0:216:3cFf:Fef0:174A", + # complete + "2051:0db8:2d5a:3521:8313:ffad:1242:8e2e", + # embedded IPv4 + "::00:192.168.10.184" + ] + + Enum.each(addresses, fn addr -> + simple_uri = URI.parse("http://[#{addr}]/") + assert simple_uri.authority == "[#{addr}]" + assert simple_uri.host == addr + + userinfo_uri = URI.parse("http://user:pass@[#{addr}]/") + assert userinfo_uri.authority == "user:pass@[#{addr}]" + assert userinfo_uri.host == addr + assert userinfo_uri.userinfo == "user:pass" + + port_uri = URI.parse("http://[#{addr}]:2222/") + assert port_uri.authority == "[#{addr}]:2222" + assert port_uri.host == addr + assert port_uri.port == 2222 + + userinfo_port_uri = URI.parse("http://user:pass@[#{addr}]:2222/") + assert userinfo_port_uri.authority == "user:pass@[#{addr}]:2222" + assert userinfo_port_uri.host == addr + assert userinfo_port_uri.userinfo == "user:pass" + assert userinfo_port_uri.port == 2222 + end) + end + + test "downcases the scheme" do + assert URI.parse("hTtP://google.com").scheme == "http" + end + + test "preserves empty fragments" do + assert URI.parse("http://example.com#").fragment == "" + assert URI.parse("http://example.com/#").fragment == "" + assert URI.parse("http://example.com/test#").fragment == "" + end + + test "preserves an empty query" do + assert URI.parse("http://foo.com/?").query == "" + end + + test "merges empty path" do + base = URI.parse("http://example.com") + assert URI.merge(base, "/foo") |> to_string == "http://example.com/foo" + assert URI.merge(base, "foo") |> to_string == "http://example.com/foo" + end end end diff --git a/lib/elixir/test/elixir/version_test.exs b/lib/elixir/test/elixir/version_test.exs index 04c96b5d08f..c960c3f05f2 100644 --- a/lib/elixir/test/elixir/version_test.exs +++ b/lib/elixir/test/elixir/version_test.exs @@ -1,199 +1,338 @@ -Code.require_file "test_helper.exs", __DIR__ +Code.require_file("test_helper.exs", __DIR__) defmodule VersionTest do - use ExUnit.Case, async: true - alias Version.Parser, as: P - alias Version, as: V - - test "compare" do - assert :gt == V.compare("1.0.1", "1.0.0") - assert :gt == V.compare("1.1.0", "1.0.1") - assert :gt == V.compare("2.1.1", "1.2.2") - assert :gt == V.compare("1.0.0", "1.0.0-dev") - assert :gt == V.compare("1.2.3-dev", "0.1.2") - assert :gt == V.compare("1.0.0-a.b", "1.0.0-a") - assert :gt == V.compare("1.0.0-b", "1.0.0-a.b") - assert :gt == V.compare("1.0.0-a", "1.0.0-0") - assert :gt == V.compare("1.0.0-a.b", "1.0.0-a.a") - - assert :lt == V.compare("1.0.0", "1.0.1") - assert :lt == V.compare("1.0.1", "1.1.0") - assert :lt == V.compare("1.2.2", "2.1.1") - assert :lt == V.compare("1.0.0-dev", "1.0.0") - assert :lt == V.compare("0.1.2", "1.2.3-dev") - assert :lt == V.compare("1.0.0-a", "1.0.0-a.b") - assert :lt == V.compare("1.0.0-a.b", "1.0.0-b") - assert :lt == V.compare("1.0.0-0", "1.0.0-a") - assert :lt == V.compare("1.0.0-a.a", "1.0.0-a.b") - - assert :eq == V.compare("1.0.0", "1.0.0") - assert :eq == V.compare("1.0.0-dev", "1.0.0-dev") - assert :eq == V.compare("1.0.0-a", "1.0.0-a") - end - - test "invalid compare" do - assert_raise V.InvalidVersionError, fn -> - V.compare("1.0", "1.0.0") + use ExUnit.Case, async: true + + doctest Version + + alias Version.Parser + + test "compare/2 with valid versions" do + assert Version.compare("1.0.1", "1.0.0") == :gt + assert Version.compare("1.1.0", "1.0.1") == :gt + assert Version.compare("2.1.1", "1.2.2") == :gt + assert Version.compare("1.0.0", "1.0.0-dev") == :gt + assert Version.compare("1.2.3-dev", "0.1.2") == :gt + assert Version.compare("1.0.0-a.b", "1.0.0-a") == :gt + assert Version.compare("1.0.0-b", "1.0.0-a.b") == :gt + assert Version.compare("1.0.0-a", "1.0.0-0") == :gt + assert Version.compare("1.0.0-a.b", "1.0.0-a.a") == :gt + + assert Version.compare("1.0.0", "1.0.1") == :lt + assert Version.compare("1.0.1", "1.1.0") == :lt + assert Version.compare("1.2.2", "2.1.1") == :lt + assert Version.compare("1.0.0-dev", "1.0.0") == :lt + assert Version.compare("0.1.2", "1.2.3-dev") == :lt + assert Version.compare("1.0.0-a", "1.0.0-a.b") == :lt + assert Version.compare("1.0.0-a.b", "1.0.0-b") == :lt + assert Version.compare("1.0.0-0", "1.0.0-a") == :lt + assert Version.compare("1.0.0-a.a", "1.0.0-a.b") == :lt + + assert Version.compare("1.0.0", "1.0.0") == :eq + assert Version.compare("1.0.0-dev", "1.0.0-dev") == :eq + assert Version.compare("1.0.0-a", "1.0.0-a") == :eq + assert Version.compare("1.5.0-rc.0", "1.5.0-rc0") == :lt + end + + test "compare/2 with invalid versions" do + assert_raise Version.InvalidVersionError, fn -> + Version.compare("1.0", "1.0.0") end - assert_raise V.InvalidVersionError, fn -> - V.compare("1.0.0-dev", "1.0") + assert_raise Version.InvalidVersionError, fn -> + Version.compare("1.0.0-dev", "1.0") end - assert_raise V.InvalidVersionError, fn -> - V.compare("foo", "1.0.0-a") + assert_raise Version.InvalidVersionError, fn -> + Version.compare("foo", "1.0.0-a") end end test "lexes specifications properly" do - assert P.lexer("== != > >= < <= ~>", []) == [:'==', :'!=', :'>', :'>=', :'<', :'<=', :'~>'] - assert P.lexer("2.3.0", []) == [:'==', "2.3.0"] - assert P.lexer("!2.3.0", []) == [:'!=', "2.3.0"] - assert P.lexer(">>=", []) == [:'>', :'>='] - assert P.lexer(">2.4.0", []) == [:'>', "2.4.0"] - assert P.lexer(" > 2.4.0", []) == [:'>', "2.4.0"] + assert Parser.lexer("== > >= < <= ~>") |> Enum.reverse() == [:==, :>, :>=, :<, :<=, :~>] + assert Parser.lexer("2.3.0") |> Enum.reverse() == [:==, "2.3.0"] + assert Parser.lexer(">>=") |> Enum.reverse() == [:>, :>=] + assert Parser.lexer(">2.4.0") |> Enum.reverse() == [:>, "2.4.0"] + assert Parser.lexer("> 2.4.0") |> Enum.reverse() == [:>, "2.4.0"] + assert Parser.lexer(" > 2.4.0") |> Enum.reverse() == [:>, "2.4.0"] + assert Parser.lexer(" or 2.1.0") |> Enum.reverse() == [:or, :==, "2.1.0"] + assert Parser.lexer(" and 2.1.0") |> Enum.reverse() == [:and, :==, "2.1.0"] + + assert Parser.lexer(">= 2.0.0 and < 2.1.0") |> Enum.reverse() == + [:>=, "2.0.0", :and, :<, "2.1.0"] + + assert Parser.lexer(">= 2.0.0 or < 2.1.0") |> Enum.reverse() == + [:>=, "2.0.0", :or, :<, "2.1.0"] end - test "parse" do - assert {:ok, %V{major: 1, minor: 2, patch: 3}} = V.parse("1.2.3") - assert {:ok, %V{major: 1, minor: 4, patch: 5}} = V.parse("1.4.5+ignore") - assert {:ok, %V{major: 1, minor: 4, patch: 5, pre: ["6-g3318bd5"]}} = V.parse("1.4.5-6-g3318bd5") - assert {:ok, %V{major: 1, minor: 4, patch: 5, pre: [6, 7, "eight"]}} = V.parse("1.4.5-6.7.eight") - assert {:ok, %V{major: 1, minor: 4, patch: 5, pre: ["6-g3318bd5"]}} = V.parse("1.4.5-6-g3318bd5+ignore") - - assert :error = V.parse("foobar") - assert :error = V.parse("2.3") - assert :error = V.parse("2") - assert :error = V.parse("2.3.0-01") + test "parse/1" do + assert {:ok, %Version{major: 1, minor: 2, patch: 3}} = Version.parse("1.2.3") + + assert {:ok, %Version{major: 1, minor: 4, patch: 5, build: "ignore"}} = + Version.parse("1.4.5+ignore") + + assert {:ok, %Version{major: 0, minor: 0, patch: 1, build: "sha.0702245"}} = + Version.parse("0.0.1+sha.0702245") + + assert {:ok, %Version{major: 1, minor: 4, patch: 5, pre: ["6-g3318bd5"]}} = + Version.parse("1.4.5-6-g3318bd5") + + assert {:ok, %Version{major: 1, minor: 4, patch: 5, pre: [6, 7, "eight"]}} = + Version.parse("1.4.5-6.7.eight") + + assert {:ok, %Version{major: 1, minor: 4, patch: 5, pre: ["6-g3318bd5"]}} = + Version.parse("1.4.5-6-g3318bd5+ignore") + + assert Version.parse("foobar") == :error + assert Version.parse("2") == :error + assert Version.parse("2.") == :error + assert Version.parse("2.3") == :error + assert Version.parse("2.3.") == :error + assert Version.parse("2.3.0-") == :error + assert Version.parse("2.3.0+") == :error + assert Version.parse("2.3.0.") == :error + assert Version.parse("2.3.0.4") == :error + assert Version.parse("2.3.-rc.1") == :error + assert Version.parse("2.3.+rc.1") == :error + assert Version.parse("2.3.0-01") == :error + assert Version.parse("2.3.00-1") == :error + assert Version.parse("2.3.00") == :error + assert Version.parse("2.03.0") == :error + assert Version.parse("02.3.0") == :error + assert Version.parse("0. 0.0") == :error + assert Version.parse("0.1.0-&&pre") == :error end - test "to_string" do - assert V.parse("1.0.0") |> elem(1) |> to_string == "1.0.0" - assert V.parse("1.0.0-dev") |> elem(1) |> to_string == "1.0.0-dev" - assert V.parse("1.0.0+lol") |> elem(1) |> to_string == "1.0.0+lol" - assert V.parse("1.0.0-dev+lol") |> elem(1) |> to_string == "1.0.0-dev+lol" + test "to_string/1" do + assert Version.parse!("1.0.0") |> Version.to_string() == "1.0.0" + assert Version.parse!("1.0.0-dev") |> Version.to_string() == "1.0.0-dev" + assert Version.parse!("1.0.0+lol") |> Version.to_string() == "1.0.0+lol" + assert Version.parse!("1.0.0-dev+lol") |> Version.to_string() == "1.0.0-dev+lol" + assert Version.parse!("1.0.0-dev+lol.4") |> Version.to_string() == "1.0.0-dev+lol.4" + assert Version.parse!("1.0.0-0") |> Version.to_string() == "1.0.0-0" + assert Version.parse!("1.0.0-rc.0") |> Version.to_string() == "1.0.0-rc.0" + assert %Version{major: 1, minor: 0, patch: 0} |> Version.to_string() == "1.0.0" end - test "invalid match" do - assert_raise V.InvalidVersionError, fn -> - V.match?("foo", "2.3.0") + test "match?/2 with invalid versions" do + assert_raise Version.InvalidVersionError, fn -> + Version.match?("foo", "2.3.0") end - assert_raise V.InvalidVersionError, fn -> - V.match?("2.3", "2.3.0") + assert_raise Version.InvalidVersionError, fn -> + Version.match?("2.3", "2.3.0") end - assert_raise V.InvalidRequirementError, fn -> - V.match?("2.3.0", "foo") + assert_raise Version.InvalidRequirementError, fn -> + Version.match?("2.3.0", "foo") end - assert_raise V.InvalidRequirementError, fn -> - V.match?("2.3.0", "2.3") + assert_raise Version.InvalidRequirementError, fn -> + Version.match?("2.3.0", "2.3") end end test "==" do - assert V.match?("2.3.0", "2.3.0") - refute V.match?("2.4.0", "2.3.0") + assert Version.match?("2.3.0", "2.3.0") + refute Version.match?("2.4.0", "2.3.0") - assert V.match?("2.3.0", "== 2.3.0") - refute V.match?("2.4.0", "== 2.3.0") + assert Version.match?("2.3.0", "== 2.3.0") + refute Version.match?("2.4.0", "== 2.3.0") - assert V.match?("1.0.0", "1.0.0") - assert V.match?("1.0.0", "1.0.0") + assert Version.match?("1.0.0", "1.0.0") + assert Version.match?("1.0.0", "1.0.0") - assert V.match?("1.2.3-alpha", "1.2.3-alpha") + assert Version.match?("1.2.3-alpha", "1.2.3-alpha") - assert V.match?("0.9.3", "== 0.9.3+dev") + assert Version.match?("0.9.3", "== 0.9.3+dev") + + {:ok, vsn} = Version.parse("2.3.0") + assert Version.match?(vsn, "2.3.0") end test "!=" do - assert V.match?("2.4.0", "!2.3.0") - refute V.match?("2.3.0", "!2.3.0") + ExUnit.CaptureIO.capture_io(:stderr, fn -> + assert Version.match?("2.4.0", "!2.3.0") + refute Version.match?("2.3.0", "!2.3.0") - assert V.match?("2.4.0", "!= 2.3.0") - refute V.match?("2.3.0", "!= 2.3.0") + assert Version.match?("2.4.0", "!= 2.3.0") + refute Version.match?("2.3.0", "!= 2.3.0") + end) end test ">" do - assert V.match?("2.4.0", "> 2.3.0") - refute V.match?("2.2.0", "> 2.3.0") - refute V.match?("2.3.0", "> 2.3.0") - - assert V.match?("1.2.3", "> 1.2.3-alpha") - assert V.match?("1.2.3-alpha.1", "> 1.2.3-alpha") - assert V.match?("1.2.3-alpha.beta.sigma", "> 1.2.3-alpha.beta") - refute V.match?("1.2.3-alpha.10", "< 1.2.3-alpha.1") - refute V.match?("0.10.2-dev", "> 0.10.2") + assert Version.match?("2.4.0", "> 2.3.0") + refute Version.match?("2.2.0", "> 2.3.0") + refute Version.match?("2.3.0", "> 2.3.0") + + assert Version.match?("1.2.3", "> 1.2.3-alpha") + assert Version.match?("1.2.3-alpha.1", "> 1.2.3-alpha") + assert Version.match?("1.2.3-alpha.beta.sigma", "> 1.2.3-alpha.beta") + refute Version.match?("1.2.3-alpha.10", "< 1.2.3-alpha.1") + refute Version.match?("0.10.2-dev", "> 0.10.2") + + refute Version.match?("1.5.0-rc.0", "> 1.5.0-rc0") + assert Version.match?("1.5.0-rc0", "> 1.5.0-rc.0") end test ">=" do - assert V.match?("2.4.0", ">= 2.3.0") - refute V.match?("2.2.0", ">= 2.3.0") - assert V.match?("2.3.0", ">= 2.3.0") + assert Version.match?("2.4.0", ">= 2.3.0") + refute Version.match?("2.2.0", ">= 2.3.0") + assert Version.match?("2.3.0", ">= 2.3.0") - assert V.match?("2.0.0", ">= 1.0.0") - assert V.match?("1.0.0", ">= 1.0.0") + assert Version.match?("2.0.0", ">= 1.0.0") + assert Version.match?("1.0.0", ">= 1.0.0") + + refute Version.match?("1.5.0-rc.0", ">= 1.5.0-rc0") + assert Version.match?("1.5.0-rc0", ">= 1.5.0-rc.0") end test "<" do - assert V.match?("2.2.0", "< 2.3.0") - refute V.match?("2.4.0", "< 2.3.0") - refute V.match?("2.3.0", "< 2.3.0") + assert Version.match?("2.2.0", "< 2.3.0") + refute Version.match?("2.4.0", "< 2.3.0") + refute Version.match?("2.3.0", "< 2.3.0") - assert V.match?("0.10.2-dev", "< 0.10.2") + assert Version.match?("0.10.2-dev", "< 0.10.2") - refute V.match?("1.0.0", "< 1.0.0-dev") - refute V.match?("1.2.3-dev", "< 0.1.2") + refute Version.match?("1.0.0", "< 1.0.0-dev") + refute Version.match?("1.2.3-dev", "< 0.1.2") end test "<=" do - assert V.match?("2.2.0", "<= 2.3.0") - refute V.match?("2.4.0", "<= 2.3.0") - assert V.match?("2.3.0", "<= 2.3.0") + assert Version.match?("2.2.0", "<= 2.3.0") + refute Version.match?("2.4.0", "<= 2.3.0") + assert Version.match?("2.3.0", "<= 2.3.0") end - test "~>" do - assert V.match?("3.0.0", "~> 3.0") - assert V.match?("3.2.0", "~> 3.0") - refute V.match?("4.0.0", "~> 3.0") - refute V.match?("4.4.0", "~> 3.0") - - assert V.match?("3.0.2", "~> 3.0.0") - assert V.match?("3.0.0", "~> 3.0.0") - refute V.match?("3.1.0", "~> 3.0.0") - refute V.match?("3.4.0", "~> 3.0.0") + describe "~>" do + test "regular cases" do + assert Version.match?("3.0.0", "~> 3.0") + assert Version.match?("3.2.0", "~> 3.0") + refute Version.match?("4.0.0", "~> 3.0") + refute Version.match?("4.4.0", "~> 3.0") + + assert Version.match?("3.0.2", "~> 3.0.0") + assert Version.match?("3.0.0", "~> 3.0.0") + refute Version.match?("3.1.0", "~> 3.0.0") + refute Version.match?("3.4.0", "~> 3.0.0") + + assert Version.match?("3.6.0", "~> 3.5") + assert Version.match?("3.5.0", "~> 3.5") + refute Version.match?("4.0.0", "~> 3.5") + refute Version.match?("5.0.0", "~> 3.5") + + assert Version.match?("3.5.2", "~> 3.5.0") + assert Version.match?("3.5.4", "~> 3.5.0") + refute Version.match?("3.6.0", "~> 3.5.0") + refute Version.match?("3.6.3", "~> 3.5.0") + + assert Version.match?("0.9.3", "~> 0.9.3-dev") + refute Version.match?("0.10.0", "~> 0.9.3-dev") + + refute Version.match?("0.3.0-dev", "~> 0.2.0") + + assert Version.match?("1.11.0-dev", "~> 1.11-dev") + assert Version.match?("1.11.0", "~> 1.11-dev") + assert Version.match?("1.12.0", "~> 1.11-dev") + refute Version.match?("1.10.0", "~> 1.11-dev") + refute Version.match?("2.0.0", "~> 1.11-dev") + + refute Version.match?("1.5.0-rc.0", "~> 1.5.0-rc0") + assert Version.match?("1.5.0-rc0", "~> 1.5.0-rc.0") + + assert_raise Version.InvalidRequirementError, fn -> + Version.match?("3.0.0", "~> 3") + end + end - assert V.match?("3.6.0", "~> 3.5") - assert V.match?("3.5.0", "~> 3.5") - refute V.match?("4.0.0", "~> 3.5") - refute V.match?("5.0.0", "~> 3.5") + test "~> will never include pre-release versions of its upper bound" do + refute Version.match?("2.2.0-dev", "~> 2.1.0") + refute Version.match?("2.2.0-dev", "~> 2.1.0", allow_pre: false) + refute Version.match?("2.2.0-dev", "~> 2.1.0-dev") + refute Version.match?("2.2.0-dev", "~> 2.1.0-dev", allow_pre: false) + end + end - assert V.match?("3.5.2", "~> 3.5.0") - assert V.match?("3.5.4", "~> 3.5.0") - refute V.match?("3.6.0", "~> 3.5.0") - refute V.match?("3.6.3", "~> 3.5.0") + test "allow_pre" do + assert Version.match?("1.1.0", "~> 1.0", allow_pre: true) + assert Version.match?("1.1.0", "~> 1.0", allow_pre: false) + assert Version.match?("1.1.0-beta", "~> 1.0", allow_pre: true) + refute Version.match?("1.1.0-beta", "~> 1.0", allow_pre: false) + assert Version.match?("1.0.1-beta", "~> 1.0.0-beta", allow_pre: false) + + assert Version.match?("1.1.0", ">= 1.0.0", allow_pre: true) + assert Version.match?("1.1.0", ">= 1.0.0", allow_pre: false) + assert Version.match?("1.1.0-beta", ">= 1.0.0", allow_pre: true) + refute Version.match?("1.1.0-beta", ">= 1.0.0", allow_pre: false) + assert Version.match?("1.1.0-beta", ">= 1.0.0-beta", allow_pre: false) + end - assert V.match?("0.9.3", "~> 0.9.3-dev") - refute V.match?("0.10.0", "~> 0.9.3-dev") + test "and" do + assert Version.match?("0.9.3", "> 0.9.0 and < 0.10.0") + refute Version.match?("0.10.2", "> 0.9.0 and < 0.10.0") + end - refute V.match?("0.3.0-dev", "~> 0.2.0") + test "or" do + assert Version.match?("0.9.1", "0.9.1 or 0.9.3 or 0.9.5") + assert Version.match?("0.9.3", "0.9.1 or 0.9.3 or 0.9.5") + assert Version.match?("0.9.5", "0.9.1 or 0.9.3 or 0.9.5") + refute Version.match?("0.9.6", "0.9.1 or 0.9.3 or 0.9.5") + end - assert_raise V.InvalidRequirementError, fn -> - V.match?("3.0.0", "~> 3") - end + test "and/or" do + req = "< 0.2.0 and >= 0.1.0 or >= 0.7.0" + assert Version.match?("0.1.0", req) + assert Version.match?("0.1.5", req) + refute Version.match?("0.3.0", req) + refute Version.match?("0.6.0", req) + assert Version.match?("0.7.0", req) + assert Version.match?("0.7.5", req) + + req = ">= 0.7.0 or < 0.2.0 and >= 0.1.0" + assert Version.match?("0.1.0", req) + assert Version.match?("0.1.5", req) + refute Version.match?("0.3.0", req) + refute Version.match?("0.6.0", req) + assert Version.match?("0.7.0", req) + assert Version.match?("0.7.5", req) + + req = "< 0.2.0 and >= 0.1.0 or < 0.8.0 and >= 0.7.0" + assert Version.match?("0.1.0", req) + assert Version.match?("0.1.5", req) + refute Version.match?("0.3.0", req) + refute Version.match?("0.6.0", req) + assert Version.match?("0.7.0", req) + assert Version.match?("0.7.5", req) + + req = "== 0.2.0 or >= 0.3.0 and < 0.4.0 or == 0.7.0" + assert Version.match?("0.2.0", req) + refute Version.match?("0.2.5", req) + assert Version.match?("0.3.0", req) + assert Version.match?("0.3.5", req) + refute Version.match?("0.4.0", req) + assert Version.match?("0.7.0", req) end - test "and" do - assert V.match?("0.9.3", "> 0.9.0 and < 0.10.0") - refute V.match?("0.10.2", "> 0.9.0 and < 0.10.0") + test "compile_requirement/1" do + {:ok, req} = Version.parse_requirement("1.2.3") + assert req == Version.compile_requirement(req) + + assert_raise(FunctionClauseError, fn -> + Version.compile_requirement("~> 1.2.3") + end) end - test "or" do - assert V.match?("0.9.1", "0.9.1 or 0.9.3 or 0.9.5") - assert V.match?("0.9.3", "0.9.1 or 0.9.3 or 0.9.5") - assert V.match?("0.9.5", "0.9.1 or 0.9.3 or 0.9.5") + test "compile requirement" do + {:ok, req} = Version.parse_requirement("1.2.3") + req = Version.compile_requirement(req) + + assert Version.match?("1.2.3", req) + refute Version.match?("1.2.4", req) - refute V.match?("0.9.6", "0.9.1 or 0.9.3 or 0.9.5") + assert Version.parse_requirement("1 . 2 . 3") == :error + assert Version.parse_requirement("== >= 1.2.3") == :error + assert Version.parse_requirement("1.2.3 and or 4.5.6") == :error + assert Version.parse_requirement(">= 1") == :error + assert Version.parse_requirement("1.2.3 >=") == :error end end diff --git a/lib/elixir/test/erlang/atom_test.erl b/lib/elixir/test/erlang/atom_test.erl index 3b84b90fb44..89c8cc098ff 100644 --- a/lib/elixir/test/erlang/atom_test.erl +++ b/lib/elixir/test/erlang/atom_test.erl @@ -1,42 +1,43 @@ -module(atom_test). -export([kv/1]). --include("elixir.hrl"). -include_lib("eunit/include/eunit.hrl"). eval(Content) -> - {Value, Binding, _, _} = elixir:eval(Content, []), + {Value, Binding, _} = + elixir:eval_forms(elixir:'string_to_quoted!'(Content, 1, 1, <<"nofile">>, []), [], []), {Value, Binding}. -kv([{Key,nil}]) -> Key. +kv([{Key, nil}]) -> Key. atom_with_punctuation_test() -> - {foo@bar,[]} = eval(":foo@bar"), - {'a?',[]} = eval(":a?"), - {'a!',[]} = eval(":a!"), - {'||',[]} = eval(":||"), - {'...',[]} = eval(":..."). + {foo@bar, []} = eval(":foo@bar"), + {'a?', []} = eval(":a?"), + {'a!', []} = eval(":a!"), + {'||', []} = eval(":||"), + {'...', []} = eval(":..."). atom_quoted_call_test() -> - {3,[]} = eval("Kernel.'+'(1, 2)"). + {3, []} = eval("Kernel.'+'(1, 2)"). kv_with_quotes_test() -> - {'foo bar',[]} = eval(":atom_test.kv(\"foo bar\": nil)"). + {'foo bar', []} = eval(":atom_test.kv(\"foo bar\": nil)"). kv_with_interpolation_test() -> - {'foo',[]} = eval(":atom_test.kv(\"#{\"foo\"}\": nil)"), - {'foo',[]} = eval(":atom_test.kv(\"#{\"fo\"}o\": nil)"), - {'foo',_} = eval("a = \"f\"; :atom_test.kv(\"#{a}#{\"o\"}o\": nil)"). + {'foo', []} = eval(":atom_test.kv(\"#{\"foo\"}\": nil)"), + {'foo', []} = eval(":atom_test.kv(\"#{\"fo\"}o\": nil)"), + {'foo', _} = eval("a = \"f\"; :atom_test.kv(\"#{a}#{\"o\"}o\": nil)"). quoted_atom_test() -> - {foo,[]} = eval(":\"foo\""), - {foo,[]} = eval(":'foo'"). + {'+', []} = eval(":\"+\""), + {'+', []} = eval(":'+'"), + {'foo bar', []} = eval(":\"foo bar\""). atom_with_interpolation_test() -> - {foo,[]} = eval(":\"f#{\"o\"}o\""), - {foo,_} = eval("a=\"foo\"; :\"#{a}\""), - {foo,_} = eval("a=\"oo\"; :\"f#{a}\""), - {foo,_} = eval("a=\"fo\"; :\"#{a}o\""), - {fof,_} = eval("a=\"f\"; :\"#{a}o#{a}\""). + {foo, []} = eval(":\"f#{\"o\"}o\""), + {foo, _} = eval("a=\"foo\"; :\"#{a}\""), + {foo, _} = eval("a=\"oo\"; :\"f#{a}\""), + {foo, _} = eval("a=\"fo\"; :\"#{a}o\""), + {fof, _} = eval("a=\"f\"; :\"#{a}o#{a}\""). quoted_atom_chars_are_escaped_test() -> - {'"',[]} = eval(":\"\\\"\""). + {'"', []} = eval(":\"\\\"\""). diff --git a/lib/elixir/test/erlang/control_test.erl b/lib/elixir/test/erlang/control_test.erl index de7fbac7fac..338a7de4d82 100644 --- a/lib/elixir/test/erlang/control_test.erl +++ b/lib/elixir/test/erlang/control_test.erl @@ -1,283 +1,125 @@ -module(control_test). --include("elixir.hrl"). -include_lib("eunit/include/eunit.hrl"). -eval(Content) -> - {Value, Binding, _, _} = elixir:eval(Content, []), - {Value, Binding}. - to_erl(String) -> - Forms = elixir:'string_to_quoted!'(String, 1, <<"nofile">>, []), - {Expr, _, _} = elixir:quoted_to_erl(Forms, elixir:env_for_eval([])), + Forms = elixir:'string_to_quoted!'(String, 1, 1, <<"nofile">>, []), + {Expr, _, _, _} = elixir:quoted_to_erl(Forms, elixir:env_for_eval([])), Expr. -% Booleans - -booleans_test() -> - {nil, _} = eval("nil"), - {true, _} = eval("true"), - {false, _} = eval("false"). - -% If - -if_else_kv_args_test() -> - {1, _} = eval("if(true, do: 1)"), - {nil, _} = eval("if(false, do: 1)"), - {2, _} = eval("if(false, do: 1, else: 2)"). - -if_else_kv_blocks_test() -> - {2, _} = eval("if(false) do\n1\nelse\n2\nend"), - {2, _} = eval("if(false) do\n1\n3\nelse\n2\nend"), - {2, _} = eval("if(false) do 1 else 2 end"), - {2, _} = eval("if(false) do 1;else 2; end"), - {3, _} = eval("if(false) do 1;else 2; 3; end"). - -vars_if_test() -> - F = fun() -> - {1, [{foo,1}]} = eval("if foo = 1 do; true; else false; end; foo"), - eval("defmodule Bar do\ndef foo, do: 1\ndef bar(x) do\nif x do; foo = 2; else foo = foo; end; foo; end\nend"), - {1, _} = eval("Bar.bar(false)"), - {2, _} = eval("Bar.bar(true)") - end, - test_helper:run_and_remove(F, ['Elixir.Bar']). - -multi_assigned_if_test() -> - {3, _} = eval("x = 1\nif true do\nx = 2\nx = 3\nelse true\nend\nx"), - {3, _} = eval("x = 1\nif true do\n^x = 1\nx = 2\nx = 3\nelse true\nend\nx"), - {1, _} = eval("if true do\nx = 1\nelse true\nend\nx"), - {nil, _} = eval("if false do\nx = 1\nelse true\nend\nx"). - -multi_line_if_test() -> - {1, _} = eval("if true\ndo\n1\nelse\n2\nend"). - -% Try - -try_test() -> - {2, _} = eval("try do\n:foo.bar\ncatch\n:error, :undef -> 2\nend"). - -try_else_test() -> - {true, _} = eval("try do\n1\nelse 2 -> false\n1 -> true\nrescue\nErlangError -> nil\nend"), - {true, _} = eval("try do\n1\nelse {x,y} -> false\nx -> true\nrescue\nErlangError -> nil\nend"), - {true, _} = eval("try do\n{1,2}\nelse {3,4} -> false\n_ -> true\nrescue\nErlangError -> nil\nend"). - -% Receive - -receive_test() -> - {10, _} = eval("send self(), :foo\nreceive do\n:foo -> 10\nend"), - {20, _} = eval("send self(), :bar\nreceive do\n:foo -> 10\n_ -> 20\nend"), - {30, _} = eval("receive do\nafter 1 -> 30\nend"). - -vars_receive_test() -> - {10, _} = eval("send self(), :foo\nreceive do\n:foo ->\na = 10\n:bar -> nil\nend\na"), - {nil, _} = eval("send self(), :bar\nreceive do\n:foo ->\nb = 10\n_ -> 20\nend\nb"), - {30, _} = eval("receive do\n:foo -> nil\nafter\n1 -> c = 30\nend\nc"), - {30, _} = eval("x = 1\nreceive do\n:foo -> nil\nafter\nx -> c = 30\nend\nc"). - -% Case - -case_test() -> - {true, []} = eval("case 1 do\n2 -> false\n1 -> true\nend"), - {true, []} = eval("case 1 do\n{x,y} -> false\nx -> true\nend"), - {true, []} = eval("case {1,2} do;{3,4} -> false\n_ -> true\nend"). - -case_with_do_ambiguity_test() -> - {true,_} = eval("case Atom.to_char_list(true) do\n_ -> true\nend"). - -case_with_match_do_ambiguity_test() -> - {true,_} = eval("case x = Atom.to_char_list(true) do\n_ -> true\nend"). - -case_with_unary_do_ambiguity_test() -> - {false,_} = eval("! case Atom.to_char_list(true) do\n_ -> true\nend"). - -multi_assigned_case_test() -> - {3, _} = eval("x = 1\ncase true do\n true ->\nx = 2\nx = 3\n_ -> true\nend\nx"), - {3, _} = eval("x = 1\ncase 1 do\n ^x -> x = 2\nx = 3\n_ -> true\nend\nx"), - {1, _} = eval("case true do\ntrue -> x = 1\n_ -> true\nend\nx"), - {nil, _} = eval("case true do\nfalse -> x = 1\n_ -> true\nend\nx"). - -vars_case_test() -> - F = fun() -> - eval("defmodule Bar do\ndef foo, do: 1\ndef bar(x) do\ncase x do\ntrue -> foo = 2\nfalse -> foo = foo\nend\nfoo\nend\nend"), - {1, _} = eval("Bar.bar(false)"), - {2, _} = eval("Bar.bar(true)") - end, - test_helper:run_and_remove(F, ['Elixir.Bar']). - -% Comparison - -equal_test() -> - {true,_} = eval(":a == :a"), - {true,_} = eval("1 == 1"), - {true,_} = eval("{1,2} == {1,2}"), - {false,_} = eval("1 == 2"), - {false,_} = eval("{1,2} == {1,3}"). - -not_equal_test() -> - {false,_} = eval(":a != :a"), - {false,_} = eval("1 != 1"), - {false,_} = eval("{1,2} != {1,2}"), - {true,_} = eval("1 != 2"), - {true,_} = eval("{1,2} != {1,3}"). - -not_exclamation_mark_test() -> - {false,_} = eval("! :a"), - {false,_} = eval("!true"), - {false,_} = eval("!1"), - {false,_} = eval("![]"), - {true,_} = eval("!nil"), - {true,_} = eval("!false"). - -notnot_exclamation_mark_test() -> - {true,_} = eval("!! :a"), - {true,_} = eval("!!true"), - {true,_} = eval("!!1"), - {true,_} = eval("!![]"), - {false,_} = eval("!!nil"), - {false,_} = eval("!!false"). - -less_greater_test() -> - {true,_} = eval("1 < 2"), - {true,_} = eval("1 < :a"), - {false,_} = eval("1 < 1.0"), - {false,_} = eval("1 < 1"), - {true,_} = eval("1 <= 1.0"), - {true,_} = eval("1 <= 1"), - {true,_} = eval("1 <= :a"), - {false,_} = eval("1 > 2"), - {false,_} = eval("1 > :a"), - {false,_} = eval("1 > 1.0"), - {false,_} = eval("1 > 1"), - {true,_} = eval("1 >= 1.0"), - {true,_} = eval("1 >= 1"), - {false,_} = eval("1 >= :a"). - -integer_and_float_test() -> - {true,_} = eval("1 == 1"), - {false,_} = eval("1 != 1"), - {true,_} = eval("1 == 1.0"), - {false,_} = eval("1 != 1.0"), - {true,_} = eval("1 === 1"), - {false,_} = eval("1 !== 1"), - {false,_} = eval("1 === 1.0"), - {true,_} = eval("1 !== 1.0"). - -and_test() -> - F = fun() -> - eval("defmodule Bar do\ndef foo, do: true\ndef bar, do: false\n def baz(x), do: x == 1\nend"), - {true, _} = eval("true and true"), - {false, _} = eval("true and false"), - {false, _} = eval("false and true"), - {false, _} = eval("false and false"), - {true, _} = eval("Bar.foo and Bar.foo"), - {false, _} = eval("Bar.foo and Bar.bar"), - {true, _} = eval("Bar.foo and Bar.baz 1"), - {false, _} = eval("Bar.foo and Bar.baz 2"), - {true, _} = eval("false and false or true"), - {3, _} = eval("Bar.foo and 1 + 2"), - {false, _} = eval("Bar.bar and :erlang.error(:bad)"), - ?assertError({badarg, 1}, eval("1 and 2")) - end, - test_helper:run_and_remove(F, ['Elixir.Bar']). - -or_test() -> - F = fun() -> - eval("defmodule Bar do\ndef foo, do: true\ndef bar, do: false\n def baz(x), do: x == 1\nend"), - {true, _} = eval("true or true"), - {true, _} = eval("true or false"), - {true, _} = eval("false or true"), - {false, _} = eval("false or false"), - {true, _} = eval("Bar.foo or Bar.foo"), - {true, _} = eval("Bar.foo or Bar.bar"), - {false, _} = eval("Bar.bar or Bar.bar"), - {true, _} = eval("Bar.bar or Bar.baz 1"), - {false, _} = eval("Bar.bar or Bar.baz 2"), - {3, _} = eval("Bar.bar or 1 + 2"), - {true, _} = eval("Bar.foo or :erlang.error(:bad)"), - ?assertError({badarg, 1}, eval("1 or 2")) - end, - test_helper:run_and_remove(F, ['Elixir.Bar']). - -not_test() -> - {false, _} = eval("not true"), - {true, _} = eval("not false"), - ?assertError(badarg, eval("not 1")). - -andand_test() -> - F = fun() -> - eval("defmodule Bar do\ndef foo, do: true\ndef bar, do: false\n def baz(x), do: x == 1\nend"), - {true, _} = eval("Kernel.&&(true, true)"), - {true, _} = eval("true && true"), - {false, _} = eval("true && false"), - {false, _} = eval("false && true"), - {false, _} = eval("false && false"), - {nil, _} = eval("true && nil"), - {nil, _} = eval("nil && true"), - {false, _} = eval("false && nil"), - {true, _} = eval("Bar.foo && Bar.foo"), - {false, _} = eval("Bar.foo && Bar.bar"), - {true, _} = eval("Bar.foo && Bar.baz 1"), - {false, _} = eval("Bar.foo && Bar.baz 2"), - {true, _} = eval("1 == 1 && 2 < 3"), - {3, _} = eval("Bar.foo && 1 + 2"), - {false, _} = eval("Bar.bar && :erlang.error(:bad)"), - {2, _} = eval("1 && 2"), - {nil, _} = eval("nil && 2") - end, - test_helper:run_and_remove(F, ['Elixir.Bar']). - -andand_with_literal_test() -> - {[nil, nil, nil], _} = eval("[nil && 2, nil && 3, nil && 4]"). - -oror_test() -> - F = fun() -> - eval("defmodule Bar do\ndef foo, do: true\ndef bar, do: false\n def baz(x), do: x == 1\nend"), - {true, _} = eval("Kernel.||(false, true)"), - {true, _} = eval("true || true"), - {true, _} = eval("true || false"), - {true, _} = eval("false || true"), - {false, _} = eval("false || false"), - {false, _} = eval("nil || false"), - {nil, _} = eval("false || nil"), - {true, _} = eval("false || nil || true"), - {true, _} = eval("Bar.foo || Bar.foo"), - {true, _} = eval("Bar.foo || Bar.bar"), - {false, _} = eval("Bar.bar || Bar.bar"), - {true, _} = eval("Bar.bar || Bar.baz 1"), - {false, _} = eval("Bar.bar || Bar.baz 2"), - {false, _} = eval("1 == 2 || 2 > 3"), - {3, _} = eval("Bar.bar || 1 + 2"), - {true, _} = eval("Bar.foo || :erlang.error(:bad)"), - {1, _} = eval("1 || 2"), - {2, _} = eval("nil || 2"), - {true, _} = eval("false && false || true") - end, - test_helper:run_and_remove(F, ['Elixir.Bar']). +cond_line_test() -> + {'case', 1, _, + [{clause, 2, _, _, _}, + {clause, 3, _, _, _}] + } = to_erl("cond do\n 1 -> :ok\n 2 -> :ok\nend"). % Optimized optimized_if_test() -> {'case', _, _, - [{clause,_,[{atom,_,false}],[],[{atom,_,else}]}, - {clause,_,[{atom,_,true}],[],[{atom,_,do}]}] - } = to_erl("if is_list([]), do: :do, else: :else"). + [{clause, _, [{atom, _, false}], [], [{atom, _, else}]}, + {clause, _, [{atom, _, true}], [], [{atom, _, do}]}] + } = to_erl("if is_list([]), do: :do, else: :else"). optimized_andand_test() -> {'case', _, _, - [{clause,_, - [{var,_,Var}], - [[{op,_,'orelse',_,_}]], - [{var,_,Var}]}, - {clause,_,[{var,_,'_'}],[],[{atom,0,done}]}] - } = to_erl("is_list([]) && :done"). + [{clause, _, + [{var, _, Var}], + [[{op, _, 'orelse', _, _}]], + [{var, _, Var}]}, + {clause, _, [{var, _, '_'}], [], [{atom, 1, done}]}] + } = to_erl("is_list([]) && :done"). optimized_oror_test() -> {'case', _, _, - [{clause,1, - [{var,1,_}], - [[{op,1,'orelse',_,_}]], - [{atom,0,done}]}, - {clause,1,[{var,1,Var}],[],[{var,1,Var}]}] - } = to_erl("is_list([]) || :done"). + [{clause, 1, + [{var, 1, _}], + [[{op, 1, 'orelse', _, _}]], + [{atom, 1, done}]}, + {clause, 1, [{var, 1, Var}], [], [{var, 1, Var}]}] + } = to_erl("is_list([]) || :done"). + +optimized_and_test() -> + {'case',_, _, + [{clause, _, [{atom, _, false}], [], [{atom, _, false}]}, + {clause, _, [{atom, _, true}], [], [{atom, _, done}]}] + } = to_erl("is_list([]) and :done"). + +optimized_or_test() -> + {'case', _, _, + [{clause, _, [{atom, _, false}], [], [{atom, _, done}]}, + {clause, _, [{atom, _, true}], [], [{atom, _, true}]}] + } = to_erl("is_list([]) or :done"). no_after_in_try_test() -> - {'try', _, [_], [_], _, []} = to_erl("try do :foo.bar() else _ -> :ok end"). \ No newline at end of file + {'try', _, [_], [], [_], []} = to_erl("try do :foo.bar() catch _ -> :ok end"). + +optimized_inspect_interpolation_test() -> + {bin, _, + [{bin_element, _, + {call, _, {remote, _,{atom, _, 'Elixir.Kernel'}, {atom, _, inspect}}, [_]}, + default, [binary]}]} = to_erl("\"#{inspect(1)}\""). + +optimized_map_put_test() -> + {map, _, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 1}}, + {map_field_assoc, _, {atom, _, b}, {integer, _, 2}}] + } = to_erl("Map.put(%{a: 1}, :b, 2)"). + +optimized_map_put_variable_test() -> + {block, _, + [_, + {map, _, {var, _, _}, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 1}}] + }] + } = to_erl("x = %{}; Map.put(x, :a, 1)"). + +optimized_nested_map_put_variable_test() -> + {block, _, + [_, + {map, _, {var, _, _}, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 1}}, + {map_field_assoc, _, {atom, _, b}, {integer, _, 2}}] + }] + } = to_erl("x = %{}; Map.put(Map.put(x, :a, 1), :b, 2)"). + +optimized_map_merge_test() -> + {map, _, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 1}}, + {map_field_assoc, _, {atom, _, b}, {integer, _, 2}}, + {map_field_assoc, _, {atom, _, c}, {integer, _, 3}}] + } = to_erl("Map.merge(%{a: 1, b: 2}, %{c: 3})"). + +optimized_map_merge_variable_test() -> + {block, _, + [_, + {map, _, {var, _, _}, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 1}}] + }] + } = to_erl("x = %{}; Map.merge(x, %{a: 1})"). + +optimized_map_update_and_merge_test() -> + {block, _, + [_, + {map, _, {var, _, _}, + [{map_field_exact, _, {atom, _, a}, {integer, _, 2}}, + {map_field_assoc, _, {atom, _, b}, {integer, _, 3}}] + }] + } = to_erl("x = %{a: 1}; Map.merge(%{x | a: 2}, %{b: 3})"), + {block, _, + [_, + {call, _, {remote, _, {atom, _, maps}, {atom, _, merge}}, + [{map, _, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 2}}]}, + {map, _, {var, _, _}, + [{map_field_exact, _, {atom, _, b}, {integer, _, 3}}]}] + }] + } = to_erl("x = %{a: 1}; Map.merge(%{a: 2}, %{x | b: 3})"). + +optimized_nested_map_merge_variable_test() -> + {block, _, + [_, + {map, _, {var, _, _}, + [{map_field_assoc, _, {atom, _, a}, {integer, _, 1}}, + {map_field_assoc, _, {atom, _, b}, {integer, _, 2}}] + }] + } = to_erl("x = %{}; Map.merge(Map.merge(x, %{a: 1}), %{b: 2})"). diff --git a/lib/elixir/test/erlang/function_test.erl b/lib/elixir/test/erlang/function_test.erl index 8f875a5817b..e0a8c89c508 100644 --- a/lib/elixir/test/erlang/function_test.erl +++ b/lib/elixir/test/erlang/function_test.erl @@ -2,21 +2,22 @@ -include_lib("eunit/include/eunit.hrl"). eval(Content) -> - {Value, Binding, _, _} = elixir:eval(Content, []), - {Value, Binding}. + {Value, Binding, _} = + elixir:eval_forms(elixir:'string_to_quoted!'(Content, 1, 1, <<"nofile">>, []), [], []), + {Value, lists:sort(Binding)}. function_arg_do_end_test() -> {3, _} = eval("if true do\n1 + 2\nend"), {nil, _} = eval("if true do end"). function_stab_end_test() -> - {_, [{a, Fun1}]} = eval("a = fn -> end"), - nil = Fun1(), - {_, [{a, Fun2}]} = eval("a = fn() -> end"), - nil = Fun2(), {_, [{a, Fun3}]} = eval("a = fn -> 1 + 2 end"), 3 = Fun3(). +function_stab_newlines_test() -> + {_, [{a, Fun3}]} = eval("a = fn\n->\n1 + 2\nend"), + 3 = Fun3(). + function_stab_many_test() -> {_, [{a, Fun}]} = eval("a = fn\n{:foo, x} -> x\n{:bar, x} -> x\nend"), 1 = Fun({foo, 1}), @@ -29,47 +30,47 @@ function_stab_inline_test() -> function_with_args_test() -> {Fun, _} = eval("fn(a, b) -> a + b end"), - 3 = Fun(1,2). + 3 = Fun(1, 2). function_with_kv_args_test() -> {Fun, _} = eval("fn(a, [other: b, another: c]) -> a + b + c end"), - 6 = Fun(1,[{other,2}, {another,3}]). + 6 = Fun(1, [{other, 2}, {another, 3}]). function_as_closure_test() -> - {_, [{a, Res1}|_]} = eval("b = 1; a = fn -> b + 2 end"), + {_, [{a, Res1} | _]} = eval("b = 1; a = fn -> b + 2 end"), 3 = Res1(). function_apply_test() -> - {3,_} = eval("a = fn -> 3 end; apply a, []"). + {3, _} = eval("a = fn -> 3 end; apply a, []"). function_apply_with_args_test() -> - {3,_} = eval("a = fn b -> b + 2 end; apply a, [1]"). + {3, _} = eval("a = fn b -> b + 2 end; apply a, [1]"). function_apply_and_clojure_test() -> - {3,_} = eval("b = 1; a = fn -> b + 2 end; apply a, []"). + {3, _} = eval("b = 1; a = fn -> b + 2 end; apply a, []"). function_parens_test() -> - {0,_} = eval("(fn() -> 0 end).()"), - {1,_} = eval("(fn(1) -> 1 end).(1)"), - {3,_} = eval("(fn(1, 2) -> 3 end).(1, 2)"), + {0, _} = eval("(fn() -> 0 end).()"), + {1, _} = eval("(fn(1) -> 1 end).(1)"), + {3, _} = eval("(fn(1, 2) -> 3 end).(1, 2)"), - {0,_} = eval("(fn () -> 0 end).()"), - {1,_} = eval("(fn (1) -> 1 end).(1)"), - {3,_} = eval("(fn (1, 2) -> 3 end).(1, 2)"). + {0, _} = eval("(fn() -> 0 end).()"), + {1, _} = eval("(fn(1) -> 1 end).(1)"), + {3, _} = eval("(fn(1, 2) -> 3 end).(1, 2)"). %% Function calls function_call_test() -> - {3, _} = eval("x = fn a, b -> a + b end\nx.(1,2)"). + {3, _} = eval("x = fn a, b -> a + b end\nx.(1, 2)"). function_call_without_arg_test() -> {3, _} = eval("x = fn -> 2 + 1 end\nx.()"). function_call_do_end_test() -> - {[1,[{do,2},{else,3}]], _} = eval("x = fn a, b -> [a,b] end\nx.(1) do\n2\nelse 3\nend"). + {[1, [{do, 2}, {else, 3}]], _} = eval("x = fn a, b -> [a, b] end\nx.(1) do\n2\nelse 3\nend"). function_call_with_assignment_test() -> - {3, [{a,_},{c, 3}]} = eval("a = fn x -> x + 2 end; c = a.(1)"). + {3, [{a, _}, {c, 3}]} = eval("a = fn x -> x + 2 end; c = a.(1)"). function_calls_with_multiple_expressions_test() -> {26, _} = eval("a = fn a, b -> a + b end; a.((3 + 4 - 1), (2 * 10))"). @@ -78,29 +79,29 @@ function_calls_with_multiple_args_with_line_breaks_test() -> {5, _} = eval("a = fn a, b -> a + b end; a.(\n3,\n2\n)"). function_calls_with_parenthesis_test() -> - {3, [{a,_},{b,1}]} = eval("a = (fn x -> x + 2 end).(b = 1)"). + {3, [{a, _}, {b, 1}]} = eval("a = (fn x -> x + 2 end).(b = 1)"). function_call_with_a_single_space_test() -> - {3, _} = eval("a = fn a, b -> a + b end; a. (1,2)"), - {3, _} = eval("a = fn a, b -> a + b end; a .(1,2)"). + {3, _} = eval("a = fn a, b -> a + b end; a. (1, 2)"), + {3, _} = eval("a = fn a, b -> a + b end; a .(1, 2)"). function_call_with_spaces_test() -> - {3, _} = eval("a = fn a, b -> a + b end; a . (1,2)"). + {3, _} = eval("a = fn a, b -> a + b end; a . (1, 2)"). function_call_without_assigning_with_spaces_test() -> - {3, _} = eval("(fn a, b -> a + b end) . (1,2)"). + {3, _} = eval("(fn a, b -> a + b end) . (1, 2)"). function_call_with_assignment_and_spaces_test() -> - {3, [{a,_},{c,3}]} = eval("a = fn x -> x + 2 end; c = a . (1)"). + {3, [{a, _}, {c, 3}]} = eval("a = fn x -> x + 2 end; c = a . (1)"). function_call_with_multiple_spaces_test() -> - {3, _} = eval("a = fn a, b -> a + b end; a . (1,2)"). + {3, _} = eval("a = fn a, b -> a + b end; a . (1, 2)"). function_call_with_multiline_test() -> - {3, _} = eval("a = fn a, b -> a + b end; a . \n (1,2)"). + {3, _} = eval("a = fn a, b -> a + b end; a . \n (1, 2)"). function_call_with_tabs_test() -> - {3, _} = eval("a = fn a, b -> a + b end; a .\n\t(1,2)"). + {3, _} = eval("a = fn a, b -> a + b end; a .\n\t(1, 2)"). function_call_with_args_and_nested_when_test() -> {Fun, _} = eval("fn a, b when a == 1 when b == 2 -> a + b end"), diff --git a/lib/elixir/test/erlang/match_test.erl b/lib/elixir/test/erlang/match_test.erl deleted file mode 100644 index 317b066dadf..00000000000 --- a/lib/elixir/test/erlang/match_test.erl +++ /dev/null @@ -1,118 +0,0 @@ --module(match_test). --include_lib("eunit/include/eunit.hrl"). - -eval(Content) -> eval(Content, []). - -eval(Content, Initial) -> - {Value, Binding, _, _} = elixir:eval(Content, Initial), - {Value, Binding}. - -no_assignment_test() -> - {nil, []} = eval(""). - -% Var/assignment test -arithmetic_test() -> - ?assertError({badmatch, _}, eval("-1 = 1")). - -assignment_test() -> - {1, [{a, 1}]} = eval("a = 1"). - -not_single_assignment_test() -> - {2, [{a, 2}]} = eval("a = 1\na = 2\na"), - {1, [{a, 1}]} = eval("{a,a} = {1,1}\na"), - {2, [{a, 2}]} = eval("a = 1\n{^a,a} = {1,2}\na"), - ?assertError({badmatch, _}, eval("{a,a} = {1,2}")), - ?assertError({badmatch, _}, eval("{1 = a,a} = {1,2}")), - ?assertError({badmatch, _}, eval("{a = 1,a} = {1,2}")), - ?assertError({badmatch, _}, eval("a = 0;{a,a} = {1,2}")), - ?assertError({badmatch, _}, eval("a = 0;{1 = a,a} = {1,2}")), - ?assertError({badmatch, _}, eval("a = 1\n^a = 2")). - -duplicated_assignment_on_module_with_tuple_test() -> - F = fun() -> - eval("defmodule Foo do\ndef v({a, _left}, {a, _right}), do: a\nend"), - {1,_} = eval("Foo.v({1, :foo}, {1, :bar})"), - ?assertError(function_clause, eval("Foo.v({1, :foo}, {2, :bar})")) - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -duplicated_assignment_on_module_with_list_test() -> - F = fun() -> - eval("defmodule Foo do\ndef v([ a, _left ], [ a, _right ]), do: a\nend"), - {1,_} = eval("Foo.v([ 1, :foo ], [ 1, :bar ])"), - ?assertError(function_clause, eval("Foo.v([ 1, :foo ], [ 2, :bar ])")) - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -multiline_assignment_test() -> - {1, [{a, 1}]} = eval("a =\n1"), - {1, [{a, 1}, {b, 1}]} = eval("a = 1\nb = 1"). - -multiple_assignment_test() -> - {1, [{a, 1}, {b, 1}]} = eval("a = b = 1"). - -multiple_assignment_with_parens_test() -> - {1, [{a, 1}, {b, 1}]} = eval("a = (b = 1)"). - -multiple_assignment_with_left_parens_test() -> - {1, [{a, 1}, {b, 1}]} = eval("(a) = (b = 1)"). - -multiple_assignment_with_expression_test() -> - {-4, [{a, -4}, {b, -4}]} = eval("a = (b = -(2 * 2))"). - -multiple_assignment_with_binding_expression_test() -> - {3, [{a, 3}, {b, 1}]} = eval("a = (2 + b)", [{b, 1}]). - -underscore_assignment_test() -> - {1, []} = eval("_ = 1"). - -assignment_precedence_test() -> - {_, [{x,{'__block__', _, [1,2,3]}}]} = eval("x = quote do\n1\n2\n3\nend"). - -% Tuples match -simple_tuple_test() -> - {{}, _} = eval("a = {}"), - {{1,2,3}, _} = eval("a = {1, 2, 3}"), - {{1,2,3}, _} = eval("a = {1, 1 + 1, 3}"), - {{1,{2},3}, _} = eval("a = {1, {2}, 3}"). - -tuple_match_test() -> - {_, _} = eval("{1,2,3} = {1, 2, 3}"), - ?assertError({badmatch, _}, eval("{1, 3, 2} = {1, 2, 3}")). - -% Lists match -simple_list_test() -> - {[], _} = eval("a = []"), - {[1,2,3], _} = eval("a = [1, 2, 3]"), - {[1,2,3], _} = eval("a = [1, 1 + 1, 3]"), - {[1,[2],3], _} = eval("a = [1, [2], 3]"), - {[1,{2},3], _} = eval("a = [1, {2}, 3]"). - -list_match_test() -> - {_, _} = eval("[1, 2, 3] = [1, 2, 3]"), - ?assertError({badmatch, _}, eval("[1, 3, 2] = [1, 2, 3]")). - -list_vars_test() -> - {[3,1], [{x,3}]} = eval("x = 1\n[x = x + 2, x]"). - -head_and_tail_test() -> - {_,[{h,1},{t,[2,3]}]} = eval("[h|t] = [1,2,3]"), - {_,[{h,2},{t,[3]}]} = eval("[1,h|t] = [1,2,3]"), - {_,[{t,[3]}]} = eval("[1,2|t] = [1,2,3]"), - {_,[{h,1}]} = eval("[h|[2,3]] = [1,2,3]"), - {_,[{t,[2,3]}]} = eval("[+1|t] = [1,2,3]"), - ?assertError({badmatch, _}, eval("[2,h|t] = [1,2,3]")). - -% Keyword match - -orrdict_match_test() -> - {[{a,1},{b,2}], _} = eval("a = [a: 1, b: 2]"). - -% Function match - -function_clause_test() -> - F = fun() -> - eval("defmodule Foo do\ndef a([{_k,_}=e|_]), do: e\nend"), - {{foo,bar},_} = eval("Foo.a([{:foo,:bar}])") - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). \ No newline at end of file diff --git a/lib/elixir/test/erlang/module_test.erl b/lib/elixir/test/erlang/module_test.erl deleted file mode 100644 index 1b9b835f2e7..00000000000 --- a/lib/elixir/test/erlang/module_test.erl +++ /dev/null @@ -1,112 +0,0 @@ --module(module_test). --include_lib("eunit/include/eunit.hrl"). - -eval(Content) -> - {Value, Binding, _, _} = elixir:eval(Content, []), - {Value, Binding}. - -definition_test() -> - F = fun() -> - eval("defmodule Foo.Bar.Baz, do: nil") - end, - test_helper:run_and_remove(F, ['Elixir.Foo.Bar.Baz']). - -module_vars_test() -> - F = fun() -> - eval("a = 1; b = 2; c = 3; defmodule Foo do\n1 = a; 2 = b; 3 = c\nend") - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -function_test() -> - F = fun() -> - eval("defmodule Foo.Bar.Baz do\ndef sum(a, b) do\na + b\nend\nend"), - 3 = 'Elixir.Foo.Bar.Baz':sum(1, 2) - end, - test_helper:run_and_remove(F, ['Elixir.Foo.Bar.Baz']). - -quote_unquote_splicing_test() -> - {{'{}', [], [1,2,3,4,5]}, _} = eval("x = [2,3,4]\nquote do: {1, unquote_splicing(x), 5}"). - -def_shortcut_test() -> - F = fun() -> - {1,[]} = eval("defmodule Foo do\ndef version, do: 1\nend\nFoo.version") - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -macro_test() -> - F = fun() -> - {'Elixir.Foo',[]} = eval("defmodule Foo do\ndef version, do: __MODULE__\nend\nFoo.version"), - {nil,[]} = eval("__MODULE__") - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -macro_line_test() -> - F = fun() -> - ?assertMatch({2, []}, eval("defmodule Foo do\ndef line, do: __ENV__.line\nend\nFoo.line")), - ?assertMatch({1, []}, eval("__ENV__.line")) - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -def_default_test() -> - F = fun() -> - eval("defmodule Foo do\ndef version(x \\\\ 1), do: x\nend"), - ?assertEqual({1, []}, eval("Foo.version")), - ?assertEqual({2, []}, eval("Foo.version(2)")) - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -def_left_default_test() -> - F = fun() -> - eval("defmodule Foo do\ndef version(x \\\\ 1, y), do: x + y\nend"), - ?assertEqual({4, []}, eval("Foo.version(3)")), - ?assertEqual({5, []}, eval("Foo.version(2, 3)")) - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -def_with_guard_test() -> - F = fun() -> - eval("defmodule Foo do\ndef v(x) when x < 10, do: true\ndef v(x) when x >= 10, do: false\nend"), - {true,_} = eval("Foo.v(0)"), - {false,_} = eval("Foo.v(20)") - end, - test_helper:run_and_remove(F, ['Elixir.Foo']). - -do_end_test() -> - F = fun() -> - eval("defmodule Foo do\ndef a, do: 1\ndefmodule Bar do\ndef b, do: 2\nend\ndef c, do: 3\nend"), - {1,_} = eval("Foo.a"), - {2,_} = eval("Foo.Bar.b"), - {3,_} = eval("Foo.c") - end, - test_helper:run_and_remove(F, ['Elixir.Foo', 'Elixir.Foo.Bar']). - -nesting_test() -> - F = fun() -> - eval("defmodule Foo do\ndefmodule Elixir.Bar do\ndef b, do: 2\nend\nend"), - {2,_} = eval("Bar.b") - end, - test_helper:run_and_remove(F, ['Elixir.Foo', 'Elixir.Bar']). - -dot_alias_test() -> - {'Elixir.Foo.Bar.Baz', _} = eval("Foo.Bar.Baz"). - -dot_dyn_alias_test() -> - {'Elixir.Foo.Bar.Baz', _} = eval("a = Foo.Bar; a.Baz"). - -single_ref_test() -> - {'Elixir.Foo', _} = eval("Foo"), - {'Elixir.Foo', _} = eval("Elixir.Foo"). - -nested_ref_test() -> - {'Elixir.Foo.Bar.Baz', _} = eval("Foo.Bar.Baz"). - -module_with_elixir_as_a_name_test() -> - ?assertError(#{'__struct__' := 'Elixir.CompileError'}, eval("defmodule Elixir do\nend")). - -dynamic_defmodule_test() -> - F = fun() -> - eval("defmodule Foo do\ndef a(name) do\ndefmodule name, do: (def x, do: 1)\nend\nend"), - {_,_} = eval("Foo.a(Bar)"), - {1,_} = eval("Bar.x") - end, - test_helper:run_and_remove(F, ['Elixir.Foo', 'Elixir.Bar']). \ No newline at end of file diff --git a/lib/elixir/test/erlang/operators_test.erl b/lib/elixir/test/erlang/operators_test.erl deleted file mode 100644 index 6ef07a89f14..00000000000 --- a/lib/elixir/test/erlang/operators_test.erl +++ /dev/null @@ -1,92 +0,0 @@ --module(operators_test). --include_lib("eunit/include/eunit.hrl"). - -eval(Content) -> - {Value, Binding, _, _} = elixir:eval(Content, []), - {Value, Binding}. - -separator_test() -> - {334,[]} = eval("3_34"), - {600,[]} = eval("2_00+45_5-5_5"). - -integer_sum_test() -> - {3,[]} = eval("1+2"), - {6,[]} = eval("1+2+3"), - {6,[]} = eval("1+2 +3"), - {6,[]} = eval("1 + 2 + 3"). - -integer_sum_minus_test() -> - {-4,[]} = eval("1-2-3"), - {0,[]} = eval("1+2-3"), - {0,[]} = eval("1 + 2 - 3"). - -integer_mult_test() -> - {6,[]} = eval("1*2*3"), - {6,[]} = eval("1 * 2 * 3"). - -integer_div_test() -> - {0.5,[]} = eval("1 / 2"), - {2.0,[]} = eval("4 / 2"). - -integer_div_rem_test() -> - {2,[]} = eval("div 5, 2"), - {1,[]} = eval("rem 5, 2"). - -integer_mult_div_test() -> - {1.0,[]} = eval("2*1/2"), - {6.0,[]} = eval("3 * 4 / 2"). - -integer_without_parens_test() -> - {17,[]} = eval("3 * 5 + 2"), - {17,[]} = eval("2 + 3 * 5"), - {6.0,[]} = eval("4 / 4 + 5"). - -integer_with_parens_test() -> - {21,[]} = eval("3 * (5 + 2)"), - {21,[]} = eval("3 * (((5 + (2))))"), - {25,[]} = eval("(2 + 3) * 5"), - {0.25,[]} = eval("4 / (11 + 5)"). - -integer_with_unary_test() -> - {2,[]} = eval("- 1 * - 2"). - -integer_eol_test() -> - {3,[]} = eval("1 +\n2"), - {2,[]} = eval("1 *\n2"), - {8,[]} = eval("1 + 2\n3 + 5"), - {8,[]} = eval("1 + 2\n\n\n3 + 5"), - {8,[]} = eval("1 + 2;\n\n3 + 5"), - {8,[]} = eval("1 + (\n2\n) + 3 + 2"), - {8,[]} = eval("1 + (\n\n 2\n\n) + 3 + 2"), - {3,[]} = eval(";1 + 2"), - ?assertError(#{'__struct__' := 'Elixir.SyntaxError'}, eval("1 + 2;\n;\n3 + 5")). - -float_with_parens_and_unary_test() -> - {-21.0,[]} = eval("-3.0 * (5 + 2)"), - {25.0,[]} = eval("(2 + 3.0) * 5"), - {0.25,[]} = eval("4 / (11.0 + 5)"). - -operators_precedence_test() -> - {2, _} = eval("max -1, 2"), - {5, []} = eval("abs -10 + 5"), - {15, []} = eval("abs(-10) + 5"). - -operators_variables_precedence_test() -> - {30, _} = eval("a = 10\nb= 20\na+b"), - {30, _} = eval("a = 10\nb= 20\na + b"). - -operators_variables_precedence_on_namespaces_test() -> - F = fun() -> - eval("defmodule Foo do; def l, do: 1; end; defmodule Bar do; def l(_x), do: 1; end"), - {3,[]} = eval("1 + Foo.l + 1"), - {3,[]} = eval("1 + Foo.l+1"), - {2,[]} = eval("1 + Bar.l +1") - end, - test_helper:run_and_remove(F, ['Elixir.Foo', 'Elixir.Bar']). - -add_add_op_test() -> - {[1,2,3,4],[]} = eval("[1,2] ++ [3,4]"). - -minus_minus_op_test() -> - {[1,2],[]} = eval("[1,2,3] -- [3]"), - {[1,2,3],[]} = eval("[1,2,3] -- [3] -- [3]"). \ No newline at end of file diff --git a/lib/elixir/test/erlang/string_test.erl b/lib/elixir/test/erlang/string_test.erl index aa787110fbb..cc54ec11d14 100644 --- a/lib/elixir/test/erlang/string_test.erl +++ b/lib/elixir/test/erlang/string_test.erl @@ -1,69 +1,82 @@ -module(string_test). --include("elixir.hrl"). +-include("../../src/elixir.hrl"). -include_lib("eunit/include/eunit.hrl"). eval(Content) -> - {Value, Binding, _, _} = elixir:eval(Content, []), + {Value, Binding, _} = + elixir:eval_forms(elixir:'string_to_quoted!'(Content, 1, 1, <<"nofile">>, []), [], []), {Value, Binding}. extract_interpolations(String) -> - element(2, elixir_interpolation:extract(1, - #elixir_tokenizer{file = <<"nofile">>}, true, String ++ [$"], $")). + case elixir_interpolation:extract(1, 1, #elixir_tokenizer{}, true, String ++ [$"], $") of + {error, Error} -> + Error; + {_, _, Parts, _, _} -> + Parts + end. % Interpolations extract_interpolations_without_interpolation_test() -> - [<<"foo">>] = extract_interpolations("foo"). + ["foo"] = extract_interpolations("foo"). extract_interpolations_with_escaped_interpolation_test() -> - [<<"f#{o}o">>] = extract_interpolations("f\\#{o}o"). + ["f\\#{o}o"] = extract_interpolations("f\\#{o}o"), + {1, 8, ["f\\#{o}o"], [], _} = + elixir_interpolation:extract(1, 2, #elixir_tokenizer{}, true, "f\\#{o}o\"", $"). extract_interpolations_with_interpolation_test() -> - [<<"f">>, - {1,[{atom,1,o}]}, - <<"o">>] = extract_interpolations("f#{:o}o"). + ["f", + {{1, 2, nil}, {1, 6, nil}, [{atom, {1, 4, _}, o}]}, + "o"] = extract_interpolations("f#{:o}o"). extract_interpolations_with_two_interpolations_test() -> - [<<"f">>, - {1,[{atom,1,o}]},{1,[{atom,1,o}]}, - <<"o">>] = extract_interpolations("f#{:o}#{:o}o"). + ["f", + {{1, 2, nil}, {1, 6, nil}, [{atom, {1, 4, _}, o}]}, + {{1, 7, nil}, {1, 11, nil}, [{atom, {1, 9, _}, o}]}, + "o"] = extract_interpolations("f#{:o}#{:o}o"). extract_interpolations_with_only_two_interpolations_test() -> - [{1,[{atom,1,o}]}, - {1,[{atom,1,o}]}] = extract_interpolations("#{:o}#{:o}"). + [{{1, 1, nil}, {1, 5, nil}, [{atom, {1, 3, _}, o}]}, + {{1, 6, nil}, {1, 10, nil}, [{atom, {1, 8, _}, o}]}] = extract_interpolations("#{:o}#{:o}"). extract_interpolations_with_tuple_inside_interpolation_test() -> - [<<"f">>, - {1,[{'{',1},{number,1,1},{'}',1}]}, - <<"o">>] = extract_interpolations("f#{{1}}o"). + ["f", + {{1, 2, nil}, {1, 7, nil}, [{'{', {1, 4, nil}}, {int, {1, 5, 1}, "1"}, {'}', {1, 6, nil}}]}, + "o"] = extract_interpolations("f#{{1}}o"). extract_interpolations_with_many_expressions_inside_interpolation_test() -> - [<<"f">>, - {1,[{number,1,1},{eol,1,newline},{number,2,2}]}, - <<"o">>] = extract_interpolations("f#{1\n2}o"). + ["f", + {{1, 2, nil}, {2, 2, nil}, [{int, {1, 4, 1}, "1"}, {eol, {1, 5, 1}}, {int, {2, 1, 2}, "2"}]}, + "o"] = extract_interpolations("f#{1\n2}o"). extract_interpolations_with_right_curly_inside_string_inside_interpolation_test() -> - [<<"f">>, - {1,[{bin_string,1,[<<"f}o">>]}]}, - <<"o">>] = extract_interpolations("f#{\"f}o\"}o"). + ["f", + {{1, 2, nil}, {1, 9, nil}, [{bin_string, {1, 4, nil}, [<<"f}o">>]}]}, + "o"] = extract_interpolations("f#{\"f}o\"}o"). extract_interpolations_with_left_curly_inside_string_inside_interpolation_test() -> - [<<"f">>, - {1,[{bin_string,1,[<<"f{o">>]}]}, - <<"o">>] = extract_interpolations("f#{\"f{o\"}o"). + ["f", + {{1, 2, nil}, {1, 9, nil}, [{bin_string, {1, 4, nil}, [<<"f{o">>]}]}, + "o"] = extract_interpolations("f#{\"f{o\"}o"). extract_interpolations_with_escaped_quote_inside_string_inside_interpolation_test() -> - [<<"f">>, - {1,[{bin_string,1,[<<"f\"o">>]}]}, - <<"o">>] = extract_interpolations("f#{\"f\\\"o\"}o"). + ["f", + {{1, 2, nil}, {1, 10, nil}, [{bin_string, {1, 4, nil}, [<<"f\"o">>]}]}, + "o"] = extract_interpolations("f#{\"f\\\"o\"}o"). extract_interpolations_with_less_than_operation_inside_interpolation_test() -> - [<<"f">>, - {1,[{number,1,1},{rel_op,1,'<'},{number,1,2}]}, - <<"o">>] = extract_interpolations("f#{1<2}o"). + ["f", + {{1, 2, nil}, {1, 7, nil}, [{int, {1, 4, 1}, "1"}, {rel_op, {1, 5, nil}, '<'}, {int, {1, 6, 2}, "2"}]}, + "o"] = extract_interpolations("f#{1<2}o"). + +extract_interpolations_with_an_escaped_character_test() -> + ["f", + {{1, 2, nil}, {1, 16, nil}, [{char, {1, 4, "?\\a"}, 7}, {rel_op, {1, 8, nil}, '>'}, {char, {1, 10, "?\\a"}, 7}]} + ] = extract_interpolations("f#{?\\a > ?\\a }"). extract_interpolations_with_invalid_expression_inside_interpolation_test() -> - {1,"invalid token: ",":1}o\""} = extract_interpolations("f#{:1}o"). + {1, 4, "unexpected token: ", _} = extract_interpolations("f#{:1}o"). %% Bin strings @@ -166,11 +179,11 @@ list_string_with_the_end_of_line_slash_test() -> {"fo", _} = eval("'f\\\r\no'"). char_test() -> - {99,[]} = eval("?1 + ?2"), - {10,[]} = eval("?\\n"), - {40,[]} = eval("?\\("). + {99, []} = eval("?1 + ?2"), + {10, []} = eval("?\\n"), + {40, []} = eval("?("). %% Binaries -bitstr_with_integer_test() -> +bitstr_with_int_test() -> {<<"fdo">>, _} = eval("<< \"f\", 50+50, \"o\" >>"). diff --git a/lib/elixir/test/erlang/test_helper.erl b/lib/elixir/test/erlang/test_helper.erl index e0890855250..d32dfdf2826 100644 --- a/lib/elixir/test/erlang/test_helper.erl +++ b/lib/elixir/test/erlang/test_helper.erl @@ -1,19 +1,15 @@ -module(test_helper). --include("elixir.hrl"). -export([test/0, run_and_remove/2, throw_elixir/1, throw_erlang/1]). -define(TESTS, [ atom_test, control_test, function_test, - match_test, - module_test, - operators_test, string_test, tokenizer_test ]). test() -> - application:start(elixir), + application:ensure_all_started(elixir), case eunit:test(?TESTS) of error -> erlang:halt(1); _Res -> erlang:halt(0) @@ -30,8 +26,8 @@ run_and_remove(Fun, Modules) -> % Throws an error with the Erlang Abstract Form from the Elixir string throw_elixir(String) -> - Forms = elixir:'string_to_quoted!'(String, 1, <<"nofile">>, []), - {Expr, _, _} = elixir:quoted_to_erl(Forms, elixir:env_for_eval([])), + Forms = elixir:'string_to_quoted!'(String, 1, 1, <<"nofile">>, []), + {Expr, _, _, _} = elixir:quoted_to_erl(Forms, elixir:env_for_eval([])), erlang:error(io:format("~p~n", [Expr])). % Throws an error with the Erlang Abstract Form from the Erlang string diff --git a/lib/elixir/test/erlang/tokenizer_test.erl b/lib/elixir/test/erlang/tokenizer_test.erl index 4cd713860e8..006cd879ac2 100644 --- a/lib/elixir/test/erlang/tokenizer_test.erl +++ b/lib/elixir/test/erlang/tokenizer_test.erl @@ -1,146 +1,278 @@ -module(tokenizer_test). --include("elixir.hrl"). -include_lib("eunit/include/eunit.hrl"). tokenize(String) -> - {ok, _Line, Result} = elixir_tokenizer:tokenize(String, 1, []), + tokenize(String, []). + +tokenize(String, Opts) -> + {ok, _Line, _Column, _Warnings, Result} = elixir_tokenizer:tokenize(String, 1, Opts), Result. tokenize_error(String) -> - {error, Error, _, _} = elixir_tokenizer:tokenize(String, 1, []), + {error, Error, _, _, _} = elixir_tokenizer:tokenize(String, 1, []), Error. type_test() -> - [{number,1,1},{type_op,1,'::'},{number,1,3}] = tokenize("1 :: 3"), - [{identifier,1,foo}, - {'.',1}, - {paren_identifier,1,'::'}, - {'(',1}, - {number,1,3}, - {')',1}] = tokenize("foo.::(3)"). + [{int, {1, 1, 1}, "1"}, + {type_op, {1, 3, nil}, '::'}, + {int, {1, 6, 3}, "3"}] = tokenize("1 :: 3"), + [{'true', {1, 1, nil}}, + {type_op, {1, 5, nil}, '::'}, + {int, {1, 7, 3}, "3"}] = tokenize("true::3"), + [{identifier, {1, 1, _}, name}, + {'.', {1, 5, nil}}, + {paren_identifier, {1, 6, _}, '::'}, + {'(', {1, 8, nil}}, + {int, {1, 9, 3}, "3"}, + {')', {1, 10, nil}}] = tokenize("name.::(3)"). arithmetic_test() -> - [{number,1,1},{dual_op,1,'+'},{number,1,2},{dual_op,1,'+'},{number,1,3}] = tokenize("1 + 2 + 3"). + [{int, {1, 1, 1}, "1"}, + {dual_op, {1, 3, nil}, '+'}, + {int, {1, 5, 2}, "2"}, + {dual_op, {1, 7, nil}, '+'}, + {int, {1, 9, 3}, "3"}] = tokenize("1 + 2 + 3"). op_kw_test() -> - [{atom,1,foo},{dual_op,1,'+'},{atom,1,bar}] = tokenize(":foo+:bar"). + [{atom, {1, 1, _}, foo}, + {dual_op, {1, 5, nil}, '+'}, + {atom, {1, 6, _}, bar}] = tokenize(":foo+:bar"). scientific_test() -> - [{number, 1, 0.1}] = tokenize("1.0e-1"). + [{flt, {1, 1, 0.1}, "1.0e-1"}] = tokenize("1.0e-1"), + [{flt, {1, 1, 0.1}, "1.0E-1"}] = tokenize("1.0E-1"), + [{flt, {1, 1, 1.2345678e-7}, "1_234.567_8e-10"}] = tokenize("1_234.567_8e-10"), + {1, 1, "invalid float number ", "1.0e309"} = tokenize_error("1.0e309"). hex_bin_octal_test() -> - [{number,1,255}] = tokenize("0xFF"), - [{number,1,255}] = tokenize("0Xff"), - [{number,1,63}] = tokenize("077"), - [{number,1,63}] = tokenize("077"), - [{number,1,3}] = tokenize("0b11"), - [{number,1,3}] = tokenize("0B11"). + [{int, {1, 1, 255}, "0xFF"}] = tokenize("0xFF"), + [{int, {1, 1, 255}, "0xF_F"}] = tokenize("0xF_F"), + [{int, {1, 1, 63}, "0o77"}] = tokenize("0o77"), + [{int, {1, 1, 63}, "0o7_7"}] = tokenize("0o7_7"), + [{int, {1, 1, 3}, "0b11"}] = tokenize("0b11"), + [{int, {1, 1, 3}, "0b1_1"}] = tokenize("0b1_1"). unquoted_atom_test() -> - [{atom, 1, '+'}] = tokenize(":+"), - [{atom, 1, '-'}] = tokenize(":-"), - [{atom, 1, '*'}] = tokenize(":*"), - [{atom, 1, '/'}] = tokenize(":/"), - [{atom, 1, '='}] = tokenize(":="), - [{atom, 1, '&&'}] = tokenize(":&&"). + [{atom, {1, 1, _}, '+'}] = tokenize(":+"), + [{atom, {1, 1, _}, '-'}] = tokenize(":-"), + [{atom, {1, 1, _}, '*'}] = tokenize(":*"), + [{atom, {1, 1, _}, '/'}] = tokenize(":/"), + [{atom, {1, 1, _}, '='}] = tokenize(":="), + [{atom, {1, 1, _}, '&&'}] = tokenize(":&&"). quoted_atom_test() -> - [{atom_unsafe, 1, [<<"foo bar">>]}] = tokenize(":\"foo bar\""). + [{atom_quoted, {1, 1, nil}, 'foo bar'}] = tokenize(":\"foo bar\""). oversized_atom_test() -> - OversizedAtom = [$:|string:copies("a", 256)], - {1, "atom length must be less than system limit", ":"} = tokenize_error(OversizedAtom). + OversizedAtom = string:copies("a", 256), + {1, 1, "atom length must be less than system limit: ", OversizedAtom} = + tokenize_error([$: | OversizedAtom]). op_atom_test() -> - [{atom,1,f0_1}] = tokenize(":f0_1"). + [{atom, {1, 1, _}, f0_1}] = tokenize(":f0_1"). kw_test() -> - [{kw_identifier, 1, do}] = tokenize("do: "), - [{kw_identifier_unsafe, 1, [<<"foo bar">>]}] = tokenize("\"foo bar\": "). + [{kw_identifier, {1, 1, _}, do}] = tokenize("do: "), + [{kw_identifier, {1, 1, _}, a@}] = tokenize("a@: "), + [{kw_identifier, {1, 1, _}, 'A@'}] = tokenize("A@: "), + [{kw_identifier, {1, 1, _}, a@b}] = tokenize("a@b: "), + [{kw_identifier, {1, 1, _}, 'A@!'}] = tokenize("A@!: "), + [{kw_identifier, {1, 1, _}, 'a@!'}] = tokenize("a@!: "), + [{kw_identifier, {1, 1, _}, foo}, {bin_string, {1, 6, nil}, [<<"bar">>]}] = tokenize("foo: \"bar\""), + [{kw_identifier, {1, 1, _}, '+'}, {bin_string, {1, 6, nil}, [<<"bar">>]}] = tokenize("\"+\": \"bar\""). -integer_test() -> - [{number, 1, 123}] = tokenize("123"), - [{number, 1, 123},{eol, 1, ';'}] = tokenize("123;"), - [{eol, 1, newline}, {number, 3, 123}] = tokenize("\n\n123"), - [{number, 1, 123}, {number, 1, 234}] = tokenize(" 123 234 "). +int_test() -> + [{int, {1, 1, 123}, "123"}] = tokenize("123"), + [{int, {1, 1, 123}, "123"}, {';', {1, 4, 0}}] = tokenize("123;"), + [{eol, {1, 1, 2}}, {int, {3, 1, 123}, "123"}] = tokenize("\n\n123"), + [{int, {1, 3, 123}, "123"}, {int, {1, 8, 234}, "234"}] = tokenize(" 123 234 "), + [{int, {1, 1, 7}, "007"}] = tokenize("007"), + [{int, {1, 1, 100000}, "0100000"}] = tokenize("0100000"). float_test() -> - [{number, 1, 12.3}] = tokenize("12.3"), - [{number, 1, 12.3},{eol, 1, ';'}] = tokenize("12.3;"), - [{eol, 1, newline}, {number, 3, 12.3}] = tokenize("\n\n12.3"), - [{number, 1, 12.3}, {number, 1, 23.4}] = tokenize(" 12.3 23.4 "). - -comments_test() -> - [{number, 1, 1},{eol, 1, newline},{number,2,2}] = tokenize("1 # Comment\n2"). + [{flt, {1, 1, 12.3}, "12.3"}] = tokenize("12.3"), + [{flt, {1, 1, 12.3}, "12.3"}, {';', {1, 5, 0}}] = tokenize("12.3;"), + [{eol, {1, 1, 2}}, {flt, {3, 1, 12.3}, "12.3"}] = tokenize("\n\n12.3"), + [{flt, {1, 3, 12.3}, "12.3"}, {flt, {1, 9, 23.4}, "23.4"}] = tokenize(" 12.3 23.4 "), + [{flt, {1, 1, 12.3}, "00_12.3_00"}] = tokenize("00_12.3_00"), + OversizedFloat = string:copies("9", 310) ++ ".0", + {1, 1, "invalid float number ", OversizedFloat} = tokenize_error(OversizedFloat). identifier_test() -> - [{identifier,1,abc}] = tokenize("abc "), - [{identifier,1,'abc?'}] = tokenize("abc?"), - [{identifier,1,'abc!'}] = tokenize("abc!"), - [{identifier,1,'a0c!'}] = tokenize("a0c!"), - [{paren_identifier,1,'a0c'},{'(',1},{')',1}] = tokenize("a0c()"), - [{paren_identifier,1,'a0c!'},{'(',1},{')',1}] = tokenize("a0c!()"). + [{identifier, {1, 1, _}, abc}] = tokenize("abc "), + [{identifier, {1, 1, _}, 'abc?'}] = tokenize("abc?"), + [{identifier, {1, 1, _}, 'abc!'}] = tokenize("abc!"), + [{identifier, {1, 1, _}, 'a0c!'}] = tokenize("a0c!"), + [{paren_identifier, {1, 1, _}, 'a0c'}, {'(', {1, 4, nil}}, {')', {1, 5, nil}}] = tokenize("a0c()"), + [{paren_identifier, {1, 1, _}, 'a0c!'}, {'(', {1, 5, nil}}, {')', {1, 6, nil}}] = tokenize("a0c!()"). module_macro_test() -> - [{identifier,1,'__MODULE__'}] = tokenize("__MODULE__"). + [{identifier, {1, 1, _}, '__MODULE__'}] = tokenize("__MODULE__"). triple_dot_test() -> - [{identifier,1,'...'}] = tokenize("..."), - [{'.',1},{identifier,1,'..'}] = tokenize(". .."). + [{identifier, {1, 1, _}, '...'}] = tokenize("..."), + [{'.', {1, 1, nil}}, {identifier, {1, 3, _}, '..'}] = tokenize(". .."). dot_test() -> - [{identifier,1,foo}, - {'.',1}, - {identifier,1,bar}, - {'.',1}, - {identifier,1,baz}] = tokenize("foo.bar.baz"). + [{identifier, {1, 1, _}, foo}, + {'.', {1, 4, nil}}, + {identifier, {1, 5, _}, bar}, + {'.', {1, 8, nil}}, + {identifier, {1, 9, _}, baz}] = tokenize("foo.bar.baz"). dot_keyword_test() -> - [{identifier,1,foo}, - {'.',1}, - {identifier,1,do}] = tokenize("foo.do"). + [{identifier, {1, 1, _}, foo}, + {'.', {1, 4, nil}}, + {identifier, {1, 5, _}, do}] = tokenize("foo.do"). newline_test() -> - [{identifier,1,foo}, - {'.',2}, - {identifier,2,bar}] = tokenize("foo\n.bar"), - [{number,1,1}, - {two_op,2,'++'}, - {number,2,2}] = tokenize("1\n++2"). + [{identifier, {1, 1, _}, foo}, + {'.', {2, 1, nil}}, + {identifier, {2, 2, _}, bar}] = tokenize("foo\n.bar"), + [{int, {1, 1, 1}, "1"}, + {concat_op, {2, 1, 1}, '++'}, + {int, {2, 3, 2}, "2"}] = tokenize("1\n++2"). + +dot_newline_operator_test() -> + [{identifier, {1, 1, _}, foo}, + {'.', {1, 4, nil}}, + {identifier, {2, 1, _}, '+'}, + {int, {2, 2, 1}, "1"}] = tokenize("foo.\n+1"), + [{identifier, {1, 1, _}, foo}, + {'.', {1, 4, nil}}, + {identifier, {2, 1, _}, '+'}, + {int, {2, 2, 1}, "1"}] = tokenize("foo.#bar\n+1"). + +dot_call_operator_test() -> + [{identifier, {1, 1, _}, f}, + {dot_call_op, {1, 2, nil}, '.'}, + {'(', {1, 3, nil}}, + {')', {1, 4, nil}}] = tokenize("f.()"). aliases_test() -> - [{'aliases',1,['Foo']}] = tokenize("Foo"), - [{'aliases',1,['Foo']}, - {'.',1}, - {'aliases',1,['Bar']}, - {'.',1}, - {'aliases',1,['Baz']}] = tokenize("Foo.Bar.Baz"). + [{'alias', {1, 1, _}, 'Foo'}] = tokenize("Foo"), + [{'alias', {1, 1, _}, 'Foo'}, + {'.', {1, 4, nil}}, + {'alias', {1, 5, _}, 'Bar'}, + {'.', {1, 8, nil}}, + {'alias', {1, 9, _}, 'Baz'}] = tokenize("Foo.Bar.Baz"). string_test() -> - [{bin_string,1,[<<"foo">>]}] = tokenize("\"foo\""), - [{list_string,1,[<<"foo">>]}] = tokenize("'foo'"). + [{bin_string, {1, 1, nil}, [<<"foo">>]}] = tokenize("\"foo\""), + [{bin_string, {1, 1, nil}, [<<"f\"">>]}] = tokenize("\"f\\\"\""), + [{list_string, {1, 1, nil}, [<<"foo">>]}] = tokenize("'foo'"). + +heredoc_test() -> + [{bin_heredoc, {1, 1, nil}, 0, [<<"heredoc\n">>]}] = tokenize("\"\"\"\nheredoc\n\"\"\""), + [{bin_heredoc, {1, 1, nil}, 1, [<<"heredoc\n">>]}, {';', {3, 5, 0}}] = tokenize("\"\"\"\n heredoc\n \"\"\";"). empty_string_test() -> - [{bin_string,1,[<<>>]}] = tokenize("\"\""), - [{list_string,1,[<<>>]}] = tokenize("''"). + [{bin_string, {1, 1, nil}, [<<>>]}] = tokenize("\"\""), + [{list_string, {1, 1, nil}, [<<>>]}] = tokenize("''"). + +concat_test() -> + [{identifier, {1, 1, _}, x}, + {concat_op, {1, 3, nil}, '++'}, + {identifier, {1, 6, _}, y}] = tokenize("x ++ y"), + [{identifier, {1, 1, _}, x}, + {concat_op, {1, 3, nil}, '+++'}, + {identifier, {1, 7, _}, y}] = tokenize("x +++ y"). -addadd_test() -> - [{identifier,1,x},{two_op,1,'++'},{identifier,1,y}] = tokenize("x ++ y"). +space_test() -> + [{op_identifier, {1, 1, _}, foo}, + {dual_op, {1, 5, nil}, '-'}, + {int, {1, 6, 2}, "2"}] = tokenize("foo -2"), + [{op_identifier, {1, 1, _}, foo}, + {dual_op, {1, 6, nil}, '-'}, + {int, {1, 7, 2}, "2"}] = tokenize("foo -2"). chars_test() -> - [{number,1,97}] = tokenize("?a"), - [{number,1,99}] = tokenize("?c"), - [{number,1,7}] = tokenize("?\\a"), - [{number,1,10}] = tokenize("?\\n"), - [{number,1,92}] = tokenize("?\\\\"), - [{number,1,10}] = tokenize("?\\xa"), - [{number,1,26}] = tokenize("?\\X1a"), - [{number,1,6}] = tokenize("?\\6"), - [{number,1,49}] = tokenize("?\\61"), - [{number,1,255}] = tokenize("?\\377"), - [{number,1,10}] = tokenize("?\\x{a}"), - [{number,1,171}] = tokenize("?\\x{ab}"), - [{number,1,2748}] = tokenize("?\\x{abc}"), - [{number,1,43981}] = tokenize("?\\x{abcd}"), - [{number,1,703710}] = tokenize("?\\x{abcde}"), - [{number,1,1092557}] = tokenize("?\\x{10abcd}"). + [{char, {1, 1, "?a"}, 97}] = tokenize("?a"), + [{char, {1, 1, "?c"}, 99}] = tokenize("?c"), + [{char, {1, 1, "?\\0"}, 0}] = tokenize("?\\0"), + [{char, {1, 1, "?\\a"}, 7}] = tokenize("?\\a"), + [{char, {1, 1, "?\\n"}, 10}] = tokenize("?\\n"), + [{char, {1, 1, "?\\\\"}, 92}] = tokenize("?\\\\"). + +interpolation_test() -> + [{bin_string, {1, 1, nil}, [<<"f">>, {{1, 3, nil},{1, 7, nil}, [{identifier, {1, 5, _}, oo}]}]}, + {concat_op, {1, 10, nil}, '<>'}, + {bin_string, {1, 13, nil}, [<<>>]}] = tokenize("\"f#{oo}\" <> \"\""). + +capture_test() -> + % Parens precedence + [{capture_op, {1, 1, nil}, '&'}, + {unary_op, {1, 2, nil}, 'not'}, + {int, {1, 6, 1}, "1"}, + {',', {1, 7, 0}}, + {int, {1, 9, 2}, "2"}] = tokenize("¬ 1, 2"), + + % Operators + [{capture_op, {1, 1, nil}, '&'}, + {identifier, {1, 2, _}, '||'}, + {mult_op, {1, 4, nil}, '/'}, + {int, {1, 5, 2}, "2"}] = tokenize("&||/2"), + [{capture_op, {1, 1, nil}, '&'}, + {identifier, {1, 2, _}, 'or'}, + {mult_op, {1, 4, nil}, '/'}, + {int, {1, 5, 2}, "2"}] = tokenize("&or/2"), + [{capture_op,{1,1,nil},'&'}, + {identifier,{1,3,_},'+'}, + {mult_op,{1,4,nil},'/'}, + {int,{1,5,1},"1"}] = tokenize("& +/1"), + [{capture_op,{1,1,nil},'&'}, + {identifier,{1,3,_},'&'}, + {mult_op,{1,4,nil},'/'}, + {int,{1,5,1},"1"}] = tokenize("& &/1"), + [{capture_op,{1,1,nil},'&'}, + {identifier,{1,3,_},'..//'}, + {mult_op,{1,7,nil},'/'}, + {int,{1,8,3},"3"}] = tokenize("& ..///3"), + [{capture_op, {1,1,nil}, '&'}, + {identifier, {1,3,_}, '/'}, + {mult_op, {1,5,nil}, '/'}, + {int, {1,6,2}, "2"}] = tokenize("& / /2"), + [{capture_op, {1,1,nil}, '&'}, + {identifier, {1,2,_}, '/'}, + {mult_op, {1,4,nil}, '/'}, + {int, {1,5,2}, "2"}] = tokenize("&/ /2"), + + % Only operators + [{identifier,{1,1,_},'&'}, + {mult_op,{1,2,nil},'/'}, + {int,{1,3,1},"1"}] = tokenize("&/1"), + [{identifier,{1,1,_},'+'}, + {mult_op,{1,2,nil},'/'}, + {int,{1,3,1},"1"}] = tokenize("+/1"), + [{identifier, {1,1,_}, '/'}, + {mult_op, {1,3,nil}, '/'}, + {int, {1,4,2}, "2"}] = tokenize("/ /2"), + [{identifier, {1,1,_}, '..//'}, + {mult_op, {1,5,nil}, '/'}, + {int, {1,6,3}, "3"}] = tokenize("..///3"). + +vc_merge_conflict_test() -> + {1, 1, "found an unexpected version control marker, please resolve the conflicts: ", "<<<<<<< HEAD"} = + tokenize_error("<<<<<<< HEAD\n[1, 2, 3]"). + +sigil_terminator_test() -> + [{sigil, {1, 1, nil}, 114, [<<"foo">>], "", nil, <<"/">>}] = tokenize("~r/foo/"), + [{sigil, {1, 1, nil}, 114, [<<"foo">>], "", nil, <<"[">>}] = tokenize("~r[foo]"), + [{sigil, {1, 1, nil}, 114, [<<"foo">>], "", nil, <<"\"">>}] = tokenize("~r\"foo\""), + [{sigil, {1, 1, nil}, 114, [<<"foo">>], "", nil, <<"/">>}, + {comp_op, {1, 9, nil}, '=='}, + {identifier, {1, 12, _}, bar}] = tokenize("~r/foo/ == bar"), + [{sigil, {1, 1, nil}, 114, [<<"foo">>], "iu", nil, <<"/">>}, + {comp_op, {1, 11, nil}, '=='}, + {identifier, {1, 14, _}, bar}] = tokenize("~r/foo/iu == bar"), + [{sigil, {1, 1, nil}, 77, [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8"). + +sigil_heredoc_test() -> + [{sigil, {1, 1, nil}, 83, [<<"sigil heredoc\n">>], "", 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""), + [{sigil, {1, 1, nil}, 83, [<<"sigil heredoc\n">>], "", 0, <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''"), + [{sigil, {1, 1, nil}, 83, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~S\"\"\"\n sigil heredoc\n \"\"\""), + [{sigil, {1, 1, nil}, 115, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~s\"\"\"\n sigil heredoc\n \"\"\""). + +invalid_sigil_delimiter_test() -> + {1, 1, "invalid sigil delimiter: ", Message} = tokenize_error("~s\\"), + true = lists:prefix("\"\\\" (column 3, code point U+005C)", lists:flatten(Message)). diff --git a/lib/elixir/unicode/GraphemeBreakProperty.txt b/lib/elixir/unicode/GraphemeBreakProperty.txt deleted file mode 100644 index f13970a2567..00000000000 --- a/lib/elixir/unicode/GraphemeBreakProperty.txt +++ /dev/null @@ -1,1252 +0,0 @@ -000D ; CR # Cc -000A ; LF # Cc -0000..0009 ; Control # Cc [10] .. -000B..000C ; Control # Cc [2] .. -000E..001F ; Control # Cc [18] .. -007F..009F ; Control # Cc [33] .. -00AD ; Control # Cf SOFT HYPHEN -0600..0605 ; Control # Cf [6] ARABIC NUMBER SIGN..ARABIC NUMBER MARK ABOVE -061C ; Control # Cf ARABIC LETTER MARK -06DD ; Control # Cf ARABIC END OF AYAH -070F ; Control # Cf SYRIAC ABBREVIATION MARK -180E ; Control # Cf MONGOLIAN VOWEL SEPARATOR -200B ; Control # Cf ZERO WIDTH SPACE -200E..200F ; Control # Cf [2] LEFT-TO-RIGHT MARK..RIGHT-TO-LEFT MARK -2028 ; Control # Zl LINE SEPARATOR -2029 ; Control # Zp PARAGRAPH SEPARATOR -202A..202E ; Control # Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE -2060..2064 ; Control # Cf [5] WORD JOINER..INVISIBLE PLUS -2065 ; Control # Cn -2066..206F ; Control # Cf [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIGIT SHAPES -D800..DFFF ; Control # Cs [2048] .. -FEFF ; Control # Cf ZERO WIDTH NO-BREAK SPACE -FFF0..FFF8 ; Control # Cn [9] .. -FFF9..FFFB ; Control # Cf [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR -110BD ; Control # Cf KAITHI NUMBER SIGN -1BCA0..1BCA3 ; Control # Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP -1D173..1D17A ; Control # Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE -E0000 ; Control # Cn -E0001 ; Control # Cf LANGUAGE TAG -E0002..E001F ; Control # Cn [30] .. -E0020..E007F ; Control # Cf [96] TAG SPACE..CANCEL TAG -E0080..E00FF ; Control # Cn [128] .. -E01F0..E0FFF ; Control # Cn [3600] .. -0300..036F ; Extend # Mn [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X -0483..0487 ; Extend # Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE -0488..0489 ; Extend # Me [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN -0591..05BD ; Extend # Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG -05BF ; Extend # Mn HEBREW POINT RAFE -05C1..05C2 ; Extend # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT -05C4..05C5 ; Extend # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT -05C7 ; Extend # Mn HEBREW POINT QAMATS QATAN -0610..061A ; Extend # Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA -064B..065F ; Extend # Mn [21] ARABIC FATHATAN..ARABIC WAVY HAMZA BELOW -0670 ; Extend # Mn ARABIC LETTER SUPERSCRIPT ALEF -06D6..06DC ; Extend # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN -06DF..06E4 ; Extend # Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA -06E7..06E8 ; Extend # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON -06EA..06ED ; Extend # Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM -0711 ; Extend # Mn SYRIAC LETTER SUPERSCRIPT ALAPH -0730..074A ; Extend # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH -07A6..07B0 ; Extend # Mn [11] THAANA ABAFILI..THAANA SUKUN -07EB..07F3 ; Extend # Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE -0816..0819 ; Extend # Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH -081B..0823 ; Extend # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A -0825..0827 ; Extend # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U -0829..082D ; Extend # Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA -0859..085B ; Extend # Mn [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK -08E4..0902 ; Extend # Mn [31] ARABIC CURLY FATHA..DEVANAGARI SIGN ANUSVARA -093A ; Extend # Mn DEVANAGARI VOWEL SIGN OE -093C ; Extend # Mn DEVANAGARI SIGN NUKTA -0941..0948 ; Extend # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI -094D ; Extend # Mn DEVANAGARI SIGN VIRAMA -0951..0957 ; Extend # Mn [7] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI VOWEL SIGN UUE -0962..0963 ; Extend # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL -0981 ; Extend # Mn BENGALI SIGN CANDRABINDU -09BC ; Extend # Mn BENGALI SIGN NUKTA -09BE ; Extend # Mc BENGALI VOWEL SIGN AA -09C1..09C4 ; Extend # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR -09CD ; Extend # Mn BENGALI SIGN VIRAMA -09D7 ; Extend # Mc BENGALI AU LENGTH MARK -09E2..09E3 ; Extend # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL -0A01..0A02 ; Extend # Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI -0A3C ; Extend # Mn GURMUKHI SIGN NUKTA -0A41..0A42 ; Extend # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU -0A47..0A48 ; Extend # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI -0A4B..0A4D ; Extend # Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA -0A51 ; Extend # Mn GURMUKHI SIGN UDAAT -0A70..0A71 ; Extend # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK -0A75 ; Extend # Mn GURMUKHI SIGN YAKASH -0A81..0A82 ; Extend # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA -0ABC ; Extend # Mn GUJARATI SIGN NUKTA -0AC1..0AC5 ; Extend # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E -0AC7..0AC8 ; Extend # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI -0ACD ; Extend # Mn GUJARATI SIGN VIRAMA -0AE2..0AE3 ; Extend # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL -0B01 ; Extend # Mn ORIYA SIGN CANDRABINDU -0B3C ; Extend # Mn ORIYA SIGN NUKTA -0B3E ; Extend # Mc ORIYA VOWEL SIGN AA -0B3F ; Extend # Mn ORIYA VOWEL SIGN I -0B41..0B44 ; Extend # Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR -0B4D ; Extend # Mn ORIYA SIGN VIRAMA -0B56 ; Extend # Mn ORIYA AI LENGTH MARK -0B57 ; Extend # Mc ORIYA AU LENGTH MARK -0B62..0B63 ; Extend # Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL -0B82 ; Extend # Mn TAMIL SIGN ANUSVARA -0BBE ; Extend # Mc TAMIL VOWEL SIGN AA -0BC0 ; Extend # Mn TAMIL VOWEL SIGN II -0BCD ; Extend # Mn TAMIL SIGN VIRAMA -0BD7 ; Extend # Mc TAMIL AU LENGTH MARK -0C00 ; Extend # Mn TELUGU SIGN COMBINING CANDRABINDU ABOVE -0C3E..0C40 ; Extend # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II -0C46..0C48 ; Extend # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI -0C4A..0C4D ; Extend # Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA -0C55..0C56 ; Extend # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK -0C62..0C63 ; Extend # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL -0C81 ; Extend # Mn KANNADA SIGN CANDRABINDU -0CBC ; Extend # Mn KANNADA SIGN NUKTA -0CBF ; Extend # Mn KANNADA VOWEL SIGN I -0CC2 ; Extend # Mc KANNADA VOWEL SIGN UU -0CC6 ; Extend # Mn KANNADA VOWEL SIGN E -0CCC..0CCD ; Extend # Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA -0CD5..0CD6 ; Extend # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK -0CE2..0CE3 ; Extend # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL -0D01 ; Extend # Mn MALAYALAM SIGN CANDRABINDU -0D3E ; Extend # Mc MALAYALAM VOWEL SIGN AA -0D41..0D44 ; Extend # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR -0D4D ; Extend # Mn MALAYALAM SIGN VIRAMA -0D57 ; Extend # Mc MALAYALAM AU LENGTH MARK -0D62..0D63 ; Extend # Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL -0DCA ; Extend # Mn SINHALA SIGN AL-LAKUNA -0DCF ; Extend # Mc SINHALA VOWEL SIGN AELA-PILLA -0DD2..0DD4 ; Extend # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA -0DD6 ; Extend # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA -0DDF ; Extend # Mc SINHALA VOWEL SIGN GAYANUKITTA -0E31 ; Extend # Mn THAI CHARACTER MAI HAN-AKAT -0E34..0E3A ; Extend # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU -0E47..0E4E ; Extend # Mn [8] THAI CHARACTER MAITAIKHU..THAI CHARACTER YAMAKKAN -0EB1 ; Extend # Mn LAO VOWEL SIGN MAI KAN -0EB4..0EB9 ; Extend # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU -0EBB..0EBC ; Extend # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO -0EC8..0ECD ; Extend # Mn [6] LAO TONE MAI EK..LAO NIGGAHITA -0F18..0F19 ; Extend # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS -0F35 ; Extend # Mn TIBETAN MARK NGAS BZUNG NYI ZLA -0F37 ; Extend # Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS -0F39 ; Extend # Mn TIBETAN MARK TSA -PHRU -0F71..0F7E ; Extend # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO -0F80..0F84 ; Extend # Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIBETAN MARK HALANTA -0F86..0F87 ; Extend # Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS -0F8D..0F97 ; Extend # Mn [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA -0F99..0FBC ; Extend # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA -0FC6 ; Extend # Mn TIBETAN SYMBOL PADMA GDAN -102D..1030 ; Extend # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU -1032..1037 ; Extend # Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIGN DOT BELOW -1039..103A ; Extend # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT -103D..103E ; Extend # Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA -1058..1059 ; Extend # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL -105E..1060 ; Extend # Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA -1071..1074 ; Extend # Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE -1082 ; Extend # Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA -1085..1086 ; Extend # Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y -108D ; Extend # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE -109D ; Extend # Mn MYANMAR VOWEL SIGN AITON AI -135D..135F ; Extend # Mn [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK -1712..1714 ; Extend # Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN VIRAMA -1732..1734 ; Extend # Mn [3] HANUNOO VOWEL SIGN I..HANUNOO SIGN PAMUDPOD -1752..1753 ; Extend # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U -1772..1773 ; Extend # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U -17B4..17B5 ; Extend # Mn [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA -17B7..17BD ; Extend # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA -17C6 ; Extend # Mn KHMER SIGN NIKAHIT -17C9..17D3 ; Extend # Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT -17DD ; Extend # Mn KHMER SIGN ATTHACAN -180B..180D ; Extend # Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE -18A9 ; Extend # Mn MONGOLIAN LETTER ALI GALI DAGALGA -1920..1922 ; Extend # Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U -1927..1928 ; Extend # Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O -1932 ; Extend # Mn LIMBU SMALL LETTER ANUSVARA -1939..193B ; Extend # Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I -1A17..1A18 ; Extend # Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U -1A1B ; Extend # Mn BUGINESE VOWEL SIGN AE -1A56 ; Extend # Mn TAI THAM CONSONANT SIGN MEDIAL LA -1A58..1A5E ; Extend # Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA -1A60 ; Extend # Mn TAI THAM SIGN SAKOT -1A62 ; Extend # Mn TAI THAM VOWEL SIGN MAI SAT -1A65..1A6C ; Extend # Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW -1A73..1A7C ; Extend # Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN KHUEN-LUE KARAN -1A7F ; Extend # Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT -1AB0..1ABD ; Extend # Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW -1ABE ; Extend # Me COMBINING PARENTHESES OVERLAY -1B00..1B03 ; Extend # Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG -1B34 ; Extend # Mn BALINESE SIGN REREKAN -1B36..1B3A ; Extend # Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA -1B3C ; Extend # Mn BALINESE VOWEL SIGN LA LENGA -1B42 ; Extend # Mn BALINESE VOWEL SIGN PEPET -1B6B..1B73 ; Extend # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG -1B80..1B81 ; Extend # Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR -1BA2..1BA5 ; Extend # Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU -1BA8..1BA9 ; Extend # Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG -1BAB..1BAD ; Extend # Mn [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA -1BE6 ; Extend # Mn BATAK SIGN TOMPI -1BE8..1BE9 ; Extend # Mn [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE -1BED ; Extend # Mn BATAK VOWEL SIGN KARO O -1BEF..1BF1 ; Extend # Mn [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H -1C2C..1C33 ; Extend # Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T -1C36..1C37 ; Extend # Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA -1CD0..1CD2 ; Extend # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA -1CD4..1CE0 ; Extend # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA -1CE2..1CE8 ; Extend # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL -1CED ; Extend # Mn VEDIC SIGN TIRYAK -1CF4 ; Extend # Mn VEDIC TONE CANDRA ABOVE -1CF8..1CF9 ; Extend # Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE -1DC0..1DF5 ; Extend # Mn [54] COMBINING DOTTED GRAVE ACCENT..COMBINING UP TACK ABOVE -1DFC..1DFF ; Extend # Mn [4] COMBINING DOUBLE INVERTED BREVE BELOW..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW -200C..200D ; Extend # Cf [2] ZERO WIDTH NON-JOINER..ZERO WIDTH JOINER -20D0..20DC ; Extend # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE -20DD..20E0 ; Extend # Me [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH -20E1 ; Extend # Mn COMBINING LEFT RIGHT ARROW ABOVE -20E2..20E4 ; Extend # Me [3] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING UPWARD POINTING TRIANGLE -20E5..20F0 ; Extend # Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE -2CEF..2CF1 ; Extend # Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS -2D7F ; Extend # Mn TIFINAGH CONSONANT JOINER -2DE0..2DFF ; Extend # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS -302A..302D ; Extend # Mn [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOGRAPHIC ENTERING TONE MARK -302E..302F ; Extend # Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK -3099..309A ; Extend # Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK -A66F ; Extend # Mn COMBINING CYRILLIC VZMET -A670..A672 ; Extend # Me [3] COMBINING CYRILLIC TEN MILLIONS SIGN..COMBINING CYRILLIC THOUSAND MILLIONS SIGN -A674..A67D ; Extend # Mn [10] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC PAYEROK -A69F ; Extend # Mn COMBINING CYRILLIC LETTER IOTIFIED E -A6F0..A6F1 ; Extend # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS -A802 ; Extend # Mn SYLOTI NAGRI SIGN DVISVARA -A806 ; Extend # Mn SYLOTI NAGRI SIGN HASANTA -A80B ; Extend # Mn SYLOTI NAGRI SIGN ANUSVARA -A825..A826 ; Extend # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E -A8C4 ; Extend # Mn SAURASHTRA SIGN VIRAMA -A8E0..A8F1 ; Extend # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA -A926..A92D ; Extend # Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE CALYA PLOPHU -A947..A951 ; Extend # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R -A980..A982 ; Extend # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR -A9B3 ; Extend # Mn JAVANESE SIGN CECAK TELU -A9B6..A9B9 ; Extend # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT -A9BC ; Extend # Mn JAVANESE VOWEL SIGN PEPET -A9E5 ; Extend # Mn MYANMAR SIGN SHAN SAW -AA29..AA2E ; Extend # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE -AA31..AA32 ; Extend # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE -AA35..AA36 ; Extend # Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA -AA43 ; Extend # Mn CHAM CONSONANT SIGN FINAL NG -AA4C ; Extend # Mn CHAM CONSONANT SIGN FINAL M -AA7C ; Extend # Mn MYANMAR SIGN TAI LAING TONE-2 -AAB0 ; Extend # Mn TAI VIET MAI KANG -AAB2..AAB4 ; Extend # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U -AAB7..AAB8 ; Extend # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA -AABE..AABF ; Extend # Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK -AAC1 ; Extend # Mn TAI VIET TONE MAI THO -AAEC..AAED ; Extend # Mn [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI MAYEK VOWEL SIGN AAI -AAF6 ; Extend # Mn MEETEI MAYEK VIRAMA -ABE5 ; Extend # Mn MEETEI MAYEK VOWEL SIGN ANAP -ABE8 ; Extend # Mn MEETEI MAYEK VOWEL SIGN UNAP -ABED ; Extend # Mn MEETEI MAYEK APUN IYEK -FB1E ; Extend # Mn HEBREW POINT JUDEO-SPANISH VARIKA -FE00..FE0F ; Extend # Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 -FE20..FE2D ; Extend # Mn [14] COMBINING LIGATURE LEFT HALF..COMBINING CONJOINING MACRON BELOW -FF9E..FF9F ; Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK -101FD ; Extend # Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE -102E0 ; Extend # Mn COPTIC EPACT THOUSANDS MARK -10376..1037A ; Extend # Mn [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII -10A01..10A03 ; Extend # Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R -10A05..10A06 ; Extend # Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O -10A0C..10A0F ; Extend # Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA -10A38..10A3A ; Extend # Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW -10A3F ; Extend # Mn KHAROSHTHI VIRAMA -10AE5..10AE6 ; Extend # Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW -11001 ; Extend # Mn BRAHMI SIGN ANUSVARA -11038..11046 ; Extend # Mn [15] BRAHMI VOWEL SIGN AA..BRAHMI VIRAMA -1107F..11081 ; Extend # Mn [3] BRAHMI NUMBER JOINER..KAITHI SIGN ANUSVARA -110B3..110B6 ; Extend # Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI -110B9..110BA ; Extend # Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA -11100..11102 ; Extend # Mn [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA -11127..1112B ; Extend # Mn [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU -1112D..11134 ; Extend # Mn [8] CHAKMA VOWEL SIGN AI..CHAKMA MAAYYAA -11173 ; Extend # Mn MAHAJANI SIGN NUKTA -11180..11181 ; Extend # Mn [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA -111B6..111BE ; Extend # Mn [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O -1122F..11231 ; Extend # Mn [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI -11234 ; Extend # Mn KHOJKI SIGN ANUSVARA -11236..11237 ; Extend # Mn [2] KHOJKI SIGN NUKTA..KHOJKI SIGN SHADDA -112DF ; Extend # Mn KHUDAWADI SIGN ANUSVARA -112E3..112EA ; Extend # Mn [8] KHUDAWADI VOWEL SIGN U..KHUDAWADI SIGN VIRAMA -11301 ; Extend # Mn GRANTHA SIGN CANDRABINDU -1133C ; Extend # Mn GRANTHA SIGN NUKTA -1133E ; Extend # Mc GRANTHA VOWEL SIGN AA -11340 ; Extend # Mn GRANTHA VOWEL SIGN II -11357 ; Extend # Mc GRANTHA AU LENGTH MARK -11366..1136C ; Extend # Mn [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX -11370..11374 ; Extend # Mn [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA -114B0 ; Extend # Mc TIRHUTA VOWEL SIGN AA -114B3..114B8 ; Extend # Mn [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL -114BA ; Extend # Mn TIRHUTA VOWEL SIGN SHORT E -114BD ; Extend # Mc TIRHUTA VOWEL SIGN SHORT O -114BF..114C0 ; Extend # Mn [2] TIRHUTA SIGN CANDRABINDU..TIRHUTA SIGN ANUSVARA -114C2..114C3 ; Extend # Mn [2] TIRHUTA SIGN VIRAMA..TIRHUTA SIGN NUKTA -115AF ; Extend # Mc SIDDHAM VOWEL SIGN AA -115B2..115B5 ; Extend # Mn [4] SIDDHAM VOWEL SIGN U..SIDDHAM VOWEL SIGN VOCALIC RR -115BC..115BD ; Extend # Mn [2] SIDDHAM SIGN CANDRABINDU..SIDDHAM SIGN ANUSVARA -115BF..115C0 ; Extend # Mn [2] SIDDHAM SIGN VIRAMA..SIDDHAM SIGN NUKTA -11633..1163A ; Extend # Mn [8] MODI VOWEL SIGN U..MODI VOWEL SIGN AI -1163D ; Extend # Mn MODI SIGN ANUSVARA -1163F..11640 ; Extend # Mn [2] MODI SIGN VIRAMA..MODI SIGN ARDHACANDRA -116AB ; Extend # Mn TAKRI SIGN ANUSVARA -116AD ; Extend # Mn TAKRI VOWEL SIGN AA -116B0..116B5 ; Extend # Mn [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU -116B7 ; Extend # Mn TAKRI SIGN NUKTA -16AF0..16AF4 ; Extend # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE -16B30..16B36 ; Extend # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM -16F8F..16F92 ; Extend # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW -1BC9D..1BC9E ; Extend # Mn [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK -1D165 ; Extend # Mc MUSICAL SYMBOL COMBINING STEM -1D167..1D169 ; Extend # Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 -1D16E..1D172 ; Extend # Mc [5] MUSICAL SYMBOL COMBINING FLAG-1..MUSICAL SYMBOL COMBINING FLAG-5 -1D17B..1D182 ; Extend # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE -1D185..1D18B ; Extend # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE -1D1AA..1D1AD ; Extend # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO -1D242..1D244 ; Extend # Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME -1E8D0..1E8D6 ; Extend # Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS -E0100..E01EF ; Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 -1F1E6..1F1FF ; Regional_Indicator # So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z -0903 ; SpacingMark # Mc DEVANAGARI SIGN VISARGA -093B ; SpacingMark # Mc DEVANAGARI VOWEL SIGN OOE -093E..0940 ; SpacingMark # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II -0949..094C ; SpacingMark # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU -094E..094F ; SpacingMark # Mc [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW -0982..0983 ; SpacingMark # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA -09BF..09C0 ; SpacingMark # Mc [2] BENGALI VOWEL SIGN I..BENGALI VOWEL SIGN II -09C7..09C8 ; SpacingMark # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI -09CB..09CC ; SpacingMark # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU -0A03 ; SpacingMark # Mc GURMUKHI SIGN VISARGA -0A3E..0A40 ; SpacingMark # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II -0A83 ; SpacingMark # Mc GUJARATI SIGN VISARGA -0ABE..0AC0 ; SpacingMark # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II -0AC9 ; SpacingMark # Mc GUJARATI VOWEL SIGN CANDRA O -0ACB..0ACC ; SpacingMark # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU -0B02..0B03 ; SpacingMark # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA -0B40 ; SpacingMark # Mc ORIYA VOWEL SIGN II -0B47..0B48 ; SpacingMark # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI -0B4B..0B4C ; SpacingMark # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU -0BBF ; SpacingMark # Mc TAMIL VOWEL SIGN I -0BC1..0BC2 ; SpacingMark # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU -0BC6..0BC8 ; SpacingMark # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI -0BCA..0BCC ; SpacingMark # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU -0C01..0C03 ; SpacingMark # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA -0C41..0C44 ; SpacingMark # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR -0C82..0C83 ; SpacingMark # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA -0CBE ; SpacingMark # Mc KANNADA VOWEL SIGN AA -0CC0..0CC1 ; SpacingMark # Mc [2] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN U -0CC3..0CC4 ; SpacingMark # Mc [2] KANNADA VOWEL SIGN VOCALIC R..KANNADA VOWEL SIGN VOCALIC RR -0CC7..0CC8 ; SpacingMark # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI -0CCA..0CCB ; SpacingMark # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO -0D02..0D03 ; SpacingMark # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA -0D3F..0D40 ; SpacingMark # Mc [2] MALAYALAM VOWEL SIGN I..MALAYALAM VOWEL SIGN II -0D46..0D48 ; SpacingMark # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI -0D4A..0D4C ; SpacingMark # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU -0D82..0D83 ; SpacingMark # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA -0DD0..0DD1 ; SpacingMark # Mc [2] SINHALA VOWEL SIGN KETTI AEDA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA -0DD8..0DDE ; SpacingMark # Mc [7] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA -0DF2..0DF3 ; SpacingMark # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA -0E33 ; SpacingMark # Lo THAI CHARACTER SARA AM -0EB3 ; SpacingMark # Lo LAO VOWEL SIGN AM -0F3E..0F3F ; SpacingMark # Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES -0F7F ; SpacingMark # Mc TIBETAN SIGN RNAM BCAD -1031 ; SpacingMark # Mc MYANMAR VOWEL SIGN E -103B..103C ; SpacingMark # Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA -1056..1057 ; SpacingMark # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR -1084 ; SpacingMark # Mc MYANMAR VOWEL SIGN SHAN E -17B6 ; SpacingMark # Mc KHMER VOWEL SIGN AA -17BE..17C5 ; SpacingMark # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU -17C7..17C8 ; SpacingMark # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU -1923..1926 ; SpacingMark # Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU -1929..192B ; SpacingMark # Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA -1930..1931 ; SpacingMark # Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA -1933..1938 ; SpacingMark # Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA -19B5..19B7 ; SpacingMark # Mc [3] NEW TAI LUE VOWEL SIGN E..NEW TAI LUE VOWEL SIGN O -19BA ; SpacingMark # Mc NEW TAI LUE VOWEL SIGN AY -1A19..1A1A ; SpacingMark # Mc [2] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN O -1A55 ; SpacingMark # Mc TAI THAM CONSONANT SIGN MEDIAL RA -1A57 ; SpacingMark # Mc TAI THAM CONSONANT SIGN LA TANG LAI -1A6D..1A72 ; SpacingMark # Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI -1B04 ; SpacingMark # Mc BALINESE SIGN BISAH -1B35 ; SpacingMark # Mc BALINESE VOWEL SIGN TEDUNG -1B3B ; SpacingMark # Mc BALINESE VOWEL SIGN RA REPA TEDUNG -1B3D..1B41 ; SpacingMark # Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG -1B43..1B44 ; SpacingMark # Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG -1B82 ; SpacingMark # Mc SUNDANESE SIGN PANGWISAD -1BA1 ; SpacingMark # Mc SUNDANESE CONSONANT SIGN PAMINGKAL -1BA6..1BA7 ; SpacingMark # Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG -1BAA ; SpacingMark # Mc SUNDANESE SIGN PAMAAEH -1BE7 ; SpacingMark # Mc BATAK VOWEL SIGN E -1BEA..1BEC ; SpacingMark # Mc [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O -1BEE ; SpacingMark # Mc BATAK VOWEL SIGN U -1BF2..1BF3 ; SpacingMark # Mc [2] BATAK PANGOLAT..BATAK PANONGONAN -1C24..1C2B ; SpacingMark # Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU -1C34..1C35 ; SpacingMark # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG -1CE1 ; SpacingMark # Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA -1CF2..1CF3 ; SpacingMark # Mc [2] VEDIC SIGN ARDHAVISARGA..VEDIC SIGN ROTATED ARDHAVISARGA -A823..A824 ; SpacingMark # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I -A827 ; SpacingMark # Mc SYLOTI NAGRI VOWEL SIGN OO -A880..A881 ; SpacingMark # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA -A8B4..A8C3 ; SpacingMark # Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU -A952..A953 ; SpacingMark # Mc [2] REJANG CONSONANT SIGN H..REJANG VIRAMA -A983 ; SpacingMark # Mc JAVANESE SIGN WIGNYAN -A9B4..A9B5 ; SpacingMark # Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG -A9BA..A9BB ; SpacingMark # Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE -A9BD..A9C0 ; SpacingMark # Mc [4] JAVANESE CONSONANT SIGN KERET..JAVANESE PANGKON -AA2F..AA30 ; SpacingMark # Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI -AA33..AA34 ; SpacingMark # Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA -AA4D ; SpacingMark # Mc CHAM CONSONANT SIGN FINAL H -AAEB ; SpacingMark # Mc MEETEI MAYEK VOWEL SIGN II -AAEE..AAEF ; SpacingMark # Mc [2] MEETEI MAYEK VOWEL SIGN AU..MEETEI MAYEK VOWEL SIGN AAU -AAF5 ; SpacingMark # Mc MEETEI MAYEK VOWEL SIGN VISARGA -ABE3..ABE4 ; SpacingMark # Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP -ABE6..ABE7 ; SpacingMark # Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP -ABE9..ABEA ; SpacingMark # Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG -ABEC ; SpacingMark # Mc MEETEI MAYEK LUM IYEK -11000 ; SpacingMark # Mc BRAHMI SIGN CANDRABINDU -11002 ; SpacingMark # Mc BRAHMI SIGN VISARGA -11082 ; SpacingMark # Mc KAITHI SIGN VISARGA -110B0..110B2 ; SpacingMark # Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II -110B7..110B8 ; SpacingMark # Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU -1112C ; SpacingMark # Mc CHAKMA VOWEL SIGN E -11182 ; SpacingMark # Mc SHARADA SIGN VISARGA -111B3..111B5 ; SpacingMark # Mc [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II -111BF..111C0 ; SpacingMark # Mc [2] SHARADA VOWEL SIGN AU..SHARADA SIGN VIRAMA -1122C..1122E ; SpacingMark # Mc [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II -11232..11233 ; SpacingMark # Mc [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU -11235 ; SpacingMark # Mc KHOJKI SIGN VIRAMA -112E0..112E2 ; SpacingMark # Mc [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II -11302..11303 ; SpacingMark # Mc [2] GRANTHA SIGN ANUSVARA..GRANTHA SIGN VISARGA -1133F ; SpacingMark # Mc GRANTHA VOWEL SIGN I -11341..11344 ; SpacingMark # Mc [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR -11347..11348 ; SpacingMark # Mc [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI -1134B..1134D ; SpacingMark # Mc [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA -11362..11363 ; SpacingMark # Mc [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL -114B1..114B2 ; SpacingMark # Mc [2] TIRHUTA VOWEL SIGN I..TIRHUTA VOWEL SIGN II -114B9 ; SpacingMark # Mc TIRHUTA VOWEL SIGN E -114BB..114BC ; SpacingMark # Mc [2] TIRHUTA VOWEL SIGN AI..TIRHUTA VOWEL SIGN O -114BE ; SpacingMark # Mc TIRHUTA VOWEL SIGN AU -114C1 ; SpacingMark # Mc TIRHUTA SIGN VISARGA -115B0..115B1 ; SpacingMark # Mc [2] SIDDHAM VOWEL SIGN I..SIDDHAM VOWEL SIGN II -115B8..115BB ; SpacingMark # Mc [4] SIDDHAM VOWEL SIGN E..SIDDHAM VOWEL SIGN AU -115BE ; SpacingMark # Mc SIDDHAM SIGN VISARGA -11630..11632 ; SpacingMark # Mc [3] MODI VOWEL SIGN AA..MODI VOWEL SIGN II -1163B..1163C ; SpacingMark # Mc [2] MODI VOWEL SIGN O..MODI VOWEL SIGN AU -1163E ; SpacingMark # Mc MODI SIGN VISARGA -116AC ; SpacingMark # Mc TAKRI SIGN VISARGA -116AE..116AF ; SpacingMark # Mc [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II -116B6 ; SpacingMark # Mc TAKRI SIGN VIRAMA -16F51..16F7E ; SpacingMark # Mc [46] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN NG -1D166 ; SpacingMark # Mc MUSICAL SYMBOL COMBINING SPRECHGESANG STEM -1D16D ; SpacingMark # Mc MUSICAL SYMBOL COMBINING AUGMENTATION DOT -1100..115F ; L # Lo [96] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG FILLER -A960..A97C ; L # Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH -1160..11A7 ; V # Lo [72] HANGUL JUNGSEONG FILLER..HANGUL JUNGSEONG O-YAE -D7B0..D7C6 ; V # Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E -11A8..11FF ; T # Lo [88] HANGUL JONGSEONG KIYEOK..HANGUL JONGSEONG SSANGNIEUN -D7CB..D7FB ; T # Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH -AC00 ; LV # Lo HANGUL SYLLABLE GA -AC1C ; LV # Lo HANGUL SYLLABLE GAE -AC38 ; LV # Lo HANGUL SYLLABLE GYA -AC54 ; LV # Lo HANGUL SYLLABLE GYAE -AC70 ; LV # Lo HANGUL SYLLABLE GEO -AC8C ; LV # Lo HANGUL SYLLABLE GE -ACA8 ; LV # Lo HANGUL SYLLABLE GYEO -ACC4 ; LV # Lo HANGUL SYLLABLE GYE -ACE0 ; LV # Lo HANGUL SYLLABLE GO -ACFC ; LV # Lo HANGUL SYLLABLE GWA -AD18 ; LV # Lo HANGUL SYLLABLE GWAE -AD34 ; LV # Lo HANGUL SYLLABLE GOE -AD50 ; LV # Lo HANGUL SYLLABLE GYO -AD6C ; LV # Lo HANGUL SYLLABLE GU -AD88 ; LV # Lo HANGUL SYLLABLE GWEO -ADA4 ; LV # Lo HANGUL SYLLABLE GWE -ADC0 ; LV # Lo HANGUL SYLLABLE GWI -ADDC ; LV # Lo HANGUL SYLLABLE GYU -ADF8 ; LV # Lo HANGUL SYLLABLE GEU -AE14 ; LV # Lo HANGUL SYLLABLE GYI -AE30 ; LV # Lo HANGUL SYLLABLE GI -AE4C ; LV # Lo HANGUL SYLLABLE GGA -AE68 ; LV # Lo HANGUL SYLLABLE GGAE -AE84 ; LV # Lo HANGUL SYLLABLE GGYA -AEA0 ; LV # Lo HANGUL SYLLABLE GGYAE -AEBC ; LV # Lo HANGUL SYLLABLE GGEO -AED8 ; LV # Lo HANGUL SYLLABLE GGE -AEF4 ; LV # Lo HANGUL SYLLABLE GGYEO -AF10 ; LV # Lo HANGUL SYLLABLE GGYE -AF2C ; LV # Lo HANGUL SYLLABLE GGO -AF48 ; LV # Lo HANGUL SYLLABLE GGWA -AF64 ; LV # Lo HANGUL SYLLABLE GGWAE -AF80 ; LV # Lo HANGUL SYLLABLE GGOE -AF9C ; LV # Lo HANGUL SYLLABLE GGYO -AFB8 ; LV # Lo HANGUL SYLLABLE GGU -AFD4 ; LV # Lo HANGUL SYLLABLE GGWEO -AFF0 ; LV # Lo HANGUL SYLLABLE GGWE -B00C ; LV # Lo HANGUL SYLLABLE GGWI -B028 ; LV # Lo HANGUL SYLLABLE GGYU -B044 ; LV # Lo HANGUL SYLLABLE GGEU -B060 ; LV # Lo HANGUL SYLLABLE GGYI -B07C ; LV # Lo HANGUL SYLLABLE GGI -B098 ; LV # Lo HANGUL SYLLABLE NA -B0B4 ; LV # Lo HANGUL SYLLABLE NAE -B0D0 ; LV # Lo HANGUL SYLLABLE NYA -B0EC ; LV # Lo HANGUL SYLLABLE NYAE -B108 ; LV # Lo HANGUL SYLLABLE NEO -B124 ; LV # Lo HANGUL SYLLABLE NE -B140 ; LV # Lo HANGUL SYLLABLE NYEO -B15C ; LV # Lo HANGUL SYLLABLE NYE -B178 ; LV # Lo HANGUL SYLLABLE NO -B194 ; LV # Lo HANGUL SYLLABLE NWA -B1B0 ; LV # Lo HANGUL SYLLABLE NWAE -B1CC ; LV # Lo HANGUL SYLLABLE NOE -B1E8 ; LV # Lo HANGUL SYLLABLE NYO -B204 ; LV # Lo HANGUL SYLLABLE NU -B220 ; LV # Lo HANGUL SYLLABLE NWEO -B23C ; LV # Lo HANGUL SYLLABLE NWE -B258 ; LV # Lo HANGUL SYLLABLE NWI -B274 ; LV # Lo HANGUL SYLLABLE NYU -B290 ; LV # Lo HANGUL SYLLABLE NEU -B2AC ; LV # Lo HANGUL SYLLABLE NYI -B2C8 ; LV # Lo HANGUL SYLLABLE NI -B2E4 ; LV # Lo HANGUL SYLLABLE DA -B300 ; LV # Lo HANGUL SYLLABLE DAE -B31C ; LV # Lo HANGUL SYLLABLE DYA -B338 ; LV # Lo HANGUL SYLLABLE DYAE -B354 ; LV # Lo HANGUL SYLLABLE DEO -B370 ; LV # Lo HANGUL SYLLABLE DE -B38C ; LV # Lo HANGUL SYLLABLE DYEO -B3A8 ; LV # Lo HANGUL SYLLABLE DYE -B3C4 ; LV # Lo HANGUL SYLLABLE DO -B3E0 ; LV # Lo HANGUL SYLLABLE DWA -B3FC ; LV # Lo HANGUL SYLLABLE DWAE -B418 ; LV # Lo HANGUL SYLLABLE DOE -B434 ; LV # Lo HANGUL SYLLABLE DYO -B450 ; LV # Lo HANGUL SYLLABLE DU -B46C ; LV # Lo HANGUL SYLLABLE DWEO -B488 ; LV # Lo HANGUL SYLLABLE DWE -B4A4 ; LV # Lo HANGUL SYLLABLE DWI -B4C0 ; LV # Lo HANGUL SYLLABLE DYU -B4DC ; LV # Lo HANGUL SYLLABLE DEU -B4F8 ; LV # Lo HANGUL SYLLABLE DYI -B514 ; LV # Lo HANGUL SYLLABLE DI -B530 ; LV # Lo HANGUL SYLLABLE DDA -B54C ; LV # Lo HANGUL SYLLABLE DDAE -B568 ; LV # Lo HANGUL SYLLABLE DDYA -B584 ; LV # Lo HANGUL SYLLABLE DDYAE -B5A0 ; LV # Lo HANGUL SYLLABLE DDEO -B5BC ; LV # Lo HANGUL SYLLABLE DDE -B5D8 ; LV # Lo HANGUL SYLLABLE DDYEO -B5F4 ; LV # Lo HANGUL SYLLABLE DDYE -B610 ; LV # Lo HANGUL SYLLABLE DDO -B62C ; LV # Lo HANGUL SYLLABLE DDWA -B648 ; LV # Lo HANGUL SYLLABLE DDWAE -B664 ; LV # Lo HANGUL SYLLABLE DDOE -B680 ; LV # Lo HANGUL SYLLABLE DDYO -B69C ; LV # Lo HANGUL SYLLABLE DDU -B6B8 ; LV # Lo HANGUL SYLLABLE DDWEO -B6D4 ; LV # Lo HANGUL SYLLABLE DDWE -B6F0 ; LV # Lo HANGUL SYLLABLE DDWI -B70C ; LV # Lo HANGUL SYLLABLE DDYU -B728 ; LV # Lo HANGUL SYLLABLE DDEU -B744 ; LV # Lo HANGUL SYLLABLE DDYI -B760 ; LV # Lo HANGUL SYLLABLE DDI -B77C ; LV # Lo HANGUL SYLLABLE RA -B798 ; LV # Lo HANGUL SYLLABLE RAE -B7B4 ; LV # Lo HANGUL SYLLABLE RYA -B7D0 ; LV # Lo HANGUL SYLLABLE RYAE -B7EC ; LV # Lo HANGUL SYLLABLE REO -B808 ; LV # Lo HANGUL SYLLABLE RE -B824 ; LV # Lo HANGUL SYLLABLE RYEO -B840 ; LV # Lo HANGUL SYLLABLE RYE -B85C ; LV # Lo HANGUL SYLLABLE RO -B878 ; LV # Lo HANGUL SYLLABLE RWA -B894 ; LV # Lo HANGUL SYLLABLE RWAE -B8B0 ; LV # Lo HANGUL SYLLABLE ROE -B8CC ; LV # Lo HANGUL SYLLABLE RYO -B8E8 ; LV # Lo HANGUL SYLLABLE RU -B904 ; LV # Lo HANGUL SYLLABLE RWEO -B920 ; LV # Lo HANGUL SYLLABLE RWE -B93C ; LV # Lo HANGUL SYLLABLE RWI -B958 ; LV # Lo HANGUL SYLLABLE RYU -B974 ; LV # Lo HANGUL SYLLABLE REU -B990 ; LV # Lo HANGUL SYLLABLE RYI -B9AC ; LV # Lo HANGUL SYLLABLE RI -B9C8 ; LV # Lo HANGUL SYLLABLE MA -B9E4 ; LV # Lo HANGUL SYLLABLE MAE -BA00 ; LV # Lo HANGUL SYLLABLE MYA -BA1C ; LV # Lo HANGUL SYLLABLE MYAE -BA38 ; LV # Lo HANGUL SYLLABLE MEO -BA54 ; LV # Lo HANGUL SYLLABLE ME -BA70 ; LV # Lo HANGUL SYLLABLE MYEO -BA8C ; LV # Lo HANGUL SYLLABLE MYE -BAA8 ; LV # Lo HANGUL SYLLABLE MO -BAC4 ; LV # Lo HANGUL SYLLABLE MWA -BAE0 ; LV # Lo HANGUL SYLLABLE MWAE -BAFC ; LV # Lo HANGUL SYLLABLE MOE -BB18 ; LV # Lo HANGUL SYLLABLE MYO -BB34 ; LV # Lo HANGUL SYLLABLE MU -BB50 ; LV # Lo HANGUL SYLLABLE MWEO -BB6C ; LV # Lo HANGUL SYLLABLE MWE -BB88 ; LV # Lo HANGUL SYLLABLE MWI -BBA4 ; LV # Lo HANGUL SYLLABLE MYU -BBC0 ; LV # Lo HANGUL SYLLABLE MEU -BBDC ; LV # Lo HANGUL SYLLABLE MYI -BBF8 ; LV # Lo HANGUL SYLLABLE MI -BC14 ; LV # Lo HANGUL SYLLABLE BA -BC30 ; LV # Lo HANGUL SYLLABLE BAE -BC4C ; LV # Lo HANGUL SYLLABLE BYA -BC68 ; LV # Lo HANGUL SYLLABLE BYAE -BC84 ; LV # Lo HANGUL SYLLABLE BEO -BCA0 ; LV # Lo HANGUL SYLLABLE BE -BCBC ; LV # Lo HANGUL SYLLABLE BYEO -BCD8 ; LV # Lo HANGUL SYLLABLE BYE -BCF4 ; LV # Lo HANGUL SYLLABLE BO -BD10 ; LV # Lo HANGUL SYLLABLE BWA -BD2C ; LV # Lo HANGUL SYLLABLE BWAE -BD48 ; LV # Lo HANGUL SYLLABLE BOE -BD64 ; LV # Lo HANGUL SYLLABLE BYO -BD80 ; LV # Lo HANGUL SYLLABLE BU -BD9C ; LV # Lo HANGUL SYLLABLE BWEO -BDB8 ; LV # Lo HANGUL SYLLABLE BWE -BDD4 ; LV # Lo HANGUL SYLLABLE BWI -BDF0 ; LV # Lo HANGUL SYLLABLE BYU -BE0C ; LV # Lo HANGUL SYLLABLE BEU -BE28 ; LV # Lo HANGUL SYLLABLE BYI -BE44 ; LV # Lo HANGUL SYLLABLE BI -BE60 ; LV # Lo HANGUL SYLLABLE BBA -BE7C ; LV # Lo HANGUL SYLLABLE BBAE -BE98 ; LV # Lo HANGUL SYLLABLE BBYA -BEB4 ; LV # Lo HANGUL SYLLABLE BBYAE -BED0 ; LV # Lo HANGUL SYLLABLE BBEO -BEEC ; LV # Lo HANGUL SYLLABLE BBE -BF08 ; LV # Lo HANGUL SYLLABLE BBYEO -BF24 ; LV # Lo HANGUL SYLLABLE BBYE -BF40 ; LV # Lo HANGUL SYLLABLE BBO -BF5C ; LV # Lo HANGUL SYLLABLE BBWA -BF78 ; LV # Lo HANGUL SYLLABLE BBWAE -BF94 ; LV # Lo HANGUL SYLLABLE BBOE -BFB0 ; LV # Lo HANGUL SYLLABLE BBYO -BFCC ; LV # Lo HANGUL SYLLABLE BBU -BFE8 ; LV # Lo HANGUL SYLLABLE BBWEO -C004 ; LV # Lo HANGUL SYLLABLE BBWE -C020 ; LV # Lo HANGUL SYLLABLE BBWI -C03C ; LV # Lo HANGUL SYLLABLE BBYU -C058 ; LV # Lo HANGUL SYLLABLE BBEU -C074 ; LV # Lo HANGUL SYLLABLE BBYI -C090 ; LV # Lo HANGUL SYLLABLE BBI -C0AC ; LV # Lo HANGUL SYLLABLE SA -C0C8 ; LV # Lo HANGUL SYLLABLE SAE -C0E4 ; LV # Lo HANGUL SYLLABLE SYA -C100 ; LV # Lo HANGUL SYLLABLE SYAE -C11C ; LV # Lo HANGUL SYLLABLE SEO -C138 ; LV # Lo HANGUL SYLLABLE SE -C154 ; LV # Lo HANGUL SYLLABLE SYEO -C170 ; LV # Lo HANGUL SYLLABLE SYE -C18C ; LV # Lo HANGUL SYLLABLE SO -C1A8 ; LV # Lo HANGUL SYLLABLE SWA -C1C4 ; LV # Lo HANGUL SYLLABLE SWAE -C1E0 ; LV # Lo HANGUL SYLLABLE SOE -C1FC ; LV # Lo HANGUL SYLLABLE SYO -C218 ; LV # Lo HANGUL SYLLABLE SU -C234 ; LV # Lo HANGUL SYLLABLE SWEO -C250 ; LV # Lo HANGUL SYLLABLE SWE -C26C ; LV # Lo HANGUL SYLLABLE SWI -C288 ; LV # Lo HANGUL SYLLABLE SYU -C2A4 ; LV # Lo HANGUL SYLLABLE SEU -C2C0 ; LV # Lo HANGUL SYLLABLE SYI -C2DC ; LV # Lo HANGUL SYLLABLE SI -C2F8 ; LV # Lo HANGUL SYLLABLE SSA -C314 ; LV # Lo HANGUL SYLLABLE SSAE -C330 ; LV # Lo HANGUL SYLLABLE SSYA -C34C ; LV # Lo HANGUL SYLLABLE SSYAE -C368 ; LV # Lo HANGUL SYLLABLE SSEO -C384 ; LV # Lo HANGUL SYLLABLE SSE -C3A0 ; LV # Lo HANGUL SYLLABLE SSYEO -C3BC ; LV # Lo HANGUL SYLLABLE SSYE -C3D8 ; LV # Lo HANGUL SYLLABLE SSO -C3F4 ; LV # Lo HANGUL SYLLABLE SSWA -C410 ; LV # Lo HANGUL SYLLABLE SSWAE -C42C ; LV # Lo HANGUL SYLLABLE SSOE -C448 ; LV # Lo HANGUL SYLLABLE SSYO -C464 ; LV # Lo HANGUL SYLLABLE SSU -C480 ; LV # Lo HANGUL SYLLABLE SSWEO -C49C ; LV # Lo HANGUL SYLLABLE SSWE -C4B8 ; LV # Lo HANGUL SYLLABLE SSWI -C4D4 ; LV # Lo HANGUL SYLLABLE SSYU -C4F0 ; LV # Lo HANGUL SYLLABLE SSEU -C50C ; LV # Lo HANGUL SYLLABLE SSYI -C528 ; LV # Lo HANGUL SYLLABLE SSI -C544 ; LV # Lo HANGUL SYLLABLE A -C560 ; LV # Lo HANGUL SYLLABLE AE -C57C ; LV # Lo HANGUL SYLLABLE YA -C598 ; LV # Lo HANGUL SYLLABLE YAE -C5B4 ; LV # Lo HANGUL SYLLABLE EO -C5D0 ; LV # Lo HANGUL SYLLABLE E -C5EC ; LV # Lo HANGUL SYLLABLE YEO -C608 ; LV # Lo HANGUL SYLLABLE YE -C624 ; LV # Lo HANGUL SYLLABLE O -C640 ; LV # Lo HANGUL SYLLABLE WA -C65C ; LV # Lo HANGUL SYLLABLE WAE -C678 ; LV # Lo HANGUL SYLLABLE OE -C694 ; LV # Lo HANGUL SYLLABLE YO -C6B0 ; LV # Lo HANGUL SYLLABLE U -C6CC ; LV # Lo HANGUL SYLLABLE WEO -C6E8 ; LV # Lo HANGUL SYLLABLE WE -C704 ; LV # Lo HANGUL SYLLABLE WI -C720 ; LV # Lo HANGUL SYLLABLE YU -C73C ; LV # Lo HANGUL SYLLABLE EU -C758 ; LV # Lo HANGUL SYLLABLE YI -C774 ; LV # Lo HANGUL SYLLABLE I -C790 ; LV # Lo HANGUL SYLLABLE JA -C7AC ; LV # Lo HANGUL SYLLABLE JAE -C7C8 ; LV # Lo HANGUL SYLLABLE JYA -C7E4 ; LV # Lo HANGUL SYLLABLE JYAE -C800 ; LV # Lo HANGUL SYLLABLE JEO -C81C ; LV # Lo HANGUL SYLLABLE JE -C838 ; LV # Lo HANGUL SYLLABLE JYEO -C854 ; LV # Lo HANGUL SYLLABLE JYE -C870 ; LV # Lo HANGUL SYLLABLE JO -C88C ; LV # Lo HANGUL SYLLABLE JWA -C8A8 ; LV # Lo HANGUL SYLLABLE JWAE -C8C4 ; LV # Lo HANGUL SYLLABLE JOE -C8E0 ; LV # Lo HANGUL SYLLABLE JYO -C8FC ; LV # Lo HANGUL SYLLABLE JU -C918 ; LV # Lo HANGUL SYLLABLE JWEO -C934 ; LV # Lo HANGUL SYLLABLE JWE -C950 ; LV # Lo HANGUL SYLLABLE JWI -C96C ; LV # Lo HANGUL SYLLABLE JYU -C988 ; LV # Lo HANGUL SYLLABLE JEU -C9A4 ; LV # Lo HANGUL SYLLABLE JYI -C9C0 ; LV # Lo HANGUL SYLLABLE JI -C9DC ; LV # Lo HANGUL SYLLABLE JJA -C9F8 ; LV # Lo HANGUL SYLLABLE JJAE -CA14 ; LV # Lo HANGUL SYLLABLE JJYA -CA30 ; LV # Lo HANGUL SYLLABLE JJYAE -CA4C ; LV # Lo HANGUL SYLLABLE JJEO -CA68 ; LV # Lo HANGUL SYLLABLE JJE -CA84 ; LV # Lo HANGUL SYLLABLE JJYEO -CAA0 ; LV # Lo HANGUL SYLLABLE JJYE -CABC ; LV # Lo HANGUL SYLLABLE JJO -CAD8 ; LV # Lo HANGUL SYLLABLE JJWA -CAF4 ; LV # Lo HANGUL SYLLABLE JJWAE -CB10 ; LV # Lo HANGUL SYLLABLE JJOE -CB2C ; LV # Lo HANGUL SYLLABLE JJYO -CB48 ; LV # Lo HANGUL SYLLABLE JJU -CB64 ; LV # Lo HANGUL SYLLABLE JJWEO -CB80 ; LV # Lo HANGUL SYLLABLE JJWE -CB9C ; LV # Lo HANGUL SYLLABLE JJWI -CBB8 ; LV # Lo HANGUL SYLLABLE JJYU -CBD4 ; LV # Lo HANGUL SYLLABLE JJEU -CBF0 ; LV # Lo HANGUL SYLLABLE JJYI -CC0C ; LV # Lo HANGUL SYLLABLE JJI -CC28 ; LV # Lo HANGUL SYLLABLE CA -CC44 ; LV # Lo HANGUL SYLLABLE CAE -CC60 ; LV # Lo HANGUL SYLLABLE CYA -CC7C ; LV # Lo HANGUL SYLLABLE CYAE -CC98 ; LV # Lo HANGUL SYLLABLE CEO -CCB4 ; LV # Lo HANGUL SYLLABLE CE -CCD0 ; LV # Lo HANGUL SYLLABLE CYEO -CCEC ; LV # Lo HANGUL SYLLABLE CYE -CD08 ; LV # Lo HANGUL SYLLABLE CO -CD24 ; LV # Lo HANGUL SYLLABLE CWA -CD40 ; LV # Lo HANGUL SYLLABLE CWAE -CD5C ; LV # Lo HANGUL SYLLABLE COE -CD78 ; LV # Lo HANGUL SYLLABLE CYO -CD94 ; LV # Lo HANGUL SYLLABLE CU -CDB0 ; LV # Lo HANGUL SYLLABLE CWEO -CDCC ; LV # Lo HANGUL SYLLABLE CWE -CDE8 ; LV # Lo HANGUL SYLLABLE CWI -CE04 ; LV # Lo HANGUL SYLLABLE CYU -CE20 ; LV # Lo HANGUL SYLLABLE CEU -CE3C ; LV # Lo HANGUL SYLLABLE CYI -CE58 ; LV # Lo HANGUL SYLLABLE CI -CE74 ; LV # Lo HANGUL SYLLABLE KA -CE90 ; LV # Lo HANGUL SYLLABLE KAE -CEAC ; LV # Lo HANGUL SYLLABLE KYA -CEC8 ; LV # Lo HANGUL SYLLABLE KYAE -CEE4 ; LV # Lo HANGUL SYLLABLE KEO -CF00 ; LV # Lo HANGUL SYLLABLE KE -CF1C ; LV # Lo HANGUL SYLLABLE KYEO -CF38 ; LV # Lo HANGUL SYLLABLE KYE -CF54 ; LV # Lo HANGUL SYLLABLE KO -CF70 ; LV # Lo HANGUL SYLLABLE KWA -CF8C ; LV # Lo HANGUL SYLLABLE KWAE -CFA8 ; LV # Lo HANGUL SYLLABLE KOE -CFC4 ; LV # Lo HANGUL SYLLABLE KYO -CFE0 ; LV # Lo HANGUL SYLLABLE KU -CFFC ; LV # Lo HANGUL SYLLABLE KWEO -D018 ; LV # Lo HANGUL SYLLABLE KWE -D034 ; LV # Lo HANGUL SYLLABLE KWI -D050 ; LV # Lo HANGUL SYLLABLE KYU -D06C ; LV # Lo HANGUL SYLLABLE KEU -D088 ; LV # Lo HANGUL SYLLABLE KYI -D0A4 ; LV # Lo HANGUL SYLLABLE KI -D0C0 ; LV # Lo HANGUL SYLLABLE TA -D0DC ; LV # Lo HANGUL SYLLABLE TAE -D0F8 ; LV # Lo HANGUL SYLLABLE TYA -D114 ; LV # Lo HANGUL SYLLABLE TYAE -D130 ; LV # Lo HANGUL SYLLABLE TEO -D14C ; LV # Lo HANGUL SYLLABLE TE -D168 ; LV # Lo HANGUL SYLLABLE TYEO -D184 ; LV # Lo HANGUL SYLLABLE TYE -D1A0 ; LV # Lo HANGUL SYLLABLE TO -D1BC ; LV # Lo HANGUL SYLLABLE TWA -D1D8 ; LV # Lo HANGUL SYLLABLE TWAE -D1F4 ; LV # Lo HANGUL SYLLABLE TOE -D210 ; LV # Lo HANGUL SYLLABLE TYO -D22C ; LV # Lo HANGUL SYLLABLE TU -D248 ; LV # Lo HANGUL SYLLABLE TWEO -D264 ; LV # Lo HANGUL SYLLABLE TWE -D280 ; LV # Lo HANGUL SYLLABLE TWI -D29C ; LV # Lo HANGUL SYLLABLE TYU -D2B8 ; LV # Lo HANGUL SYLLABLE TEU -D2D4 ; LV # Lo HANGUL SYLLABLE TYI -D2F0 ; LV # Lo HANGUL SYLLABLE TI -D30C ; LV # Lo HANGUL SYLLABLE PA -D328 ; LV # Lo HANGUL SYLLABLE PAE -D344 ; LV # Lo HANGUL SYLLABLE PYA -D360 ; LV # Lo HANGUL SYLLABLE PYAE -D37C ; LV # Lo HANGUL SYLLABLE PEO -D398 ; LV # Lo HANGUL SYLLABLE PE -D3B4 ; LV # Lo HANGUL SYLLABLE PYEO -D3D0 ; LV # Lo HANGUL SYLLABLE PYE -D3EC ; LV # Lo HANGUL SYLLABLE PO -D408 ; LV # Lo HANGUL SYLLABLE PWA -D424 ; LV # Lo HANGUL SYLLABLE PWAE -D440 ; LV # Lo HANGUL SYLLABLE POE -D45C ; LV # Lo HANGUL SYLLABLE PYO -D478 ; LV # Lo HANGUL SYLLABLE PU -D494 ; LV # Lo HANGUL SYLLABLE PWEO -D4B0 ; LV # Lo HANGUL SYLLABLE PWE -D4CC ; LV # Lo HANGUL SYLLABLE PWI -D4E8 ; LV # Lo HANGUL SYLLABLE PYU -D504 ; LV # Lo HANGUL SYLLABLE PEU -D520 ; LV # Lo HANGUL SYLLABLE PYI -D53C ; LV # Lo HANGUL SYLLABLE PI -D558 ; LV # Lo HANGUL SYLLABLE HA -D574 ; LV # Lo HANGUL SYLLABLE HAE -D590 ; LV # Lo HANGUL SYLLABLE HYA -D5AC ; LV # Lo HANGUL SYLLABLE HYAE -D5C8 ; LV # Lo HANGUL SYLLABLE HEO -D5E4 ; LV # Lo HANGUL SYLLABLE HE -D600 ; LV # Lo HANGUL SYLLABLE HYEO -D61C ; LV # Lo HANGUL SYLLABLE HYE -D638 ; LV # Lo HANGUL SYLLABLE HO -D654 ; LV # Lo HANGUL SYLLABLE HWA -D670 ; LV # Lo HANGUL SYLLABLE HWAE -D68C ; LV # Lo HANGUL SYLLABLE HOE -D6A8 ; LV # Lo HANGUL SYLLABLE HYO -D6C4 ; LV # Lo HANGUL SYLLABLE HU -D6E0 ; LV # Lo HANGUL SYLLABLE HWEO -D6FC ; LV # Lo HANGUL SYLLABLE HWE -D718 ; LV # Lo HANGUL SYLLABLE HWI -D734 ; LV # Lo HANGUL SYLLABLE HYU -D750 ; LV # Lo HANGUL SYLLABLE HEU -D76C ; LV # Lo HANGUL SYLLABLE HYI -D788 ; LV # Lo HANGUL SYLLABLE HI -AC01..AC1B ; LVT # Lo [27] HANGUL SYLLABLE GAG..HANGUL SYLLABLE GAH -AC1D..AC37 ; LVT # Lo [27] HANGUL SYLLABLE GAEG..HANGUL SYLLABLE GAEH -AC39..AC53 ; LVT # Lo [27] HANGUL SYLLABLE GYAG..HANGUL SYLLABLE GYAH -AC55..AC6F ; LVT # Lo [27] HANGUL SYLLABLE GYAEG..HANGUL SYLLABLE GYAEH -AC71..AC8B ; LVT # Lo [27] HANGUL SYLLABLE GEOG..HANGUL SYLLABLE GEOH -AC8D..ACA7 ; LVT # Lo [27] HANGUL SYLLABLE GEG..HANGUL SYLLABLE GEH -ACA9..ACC3 ; LVT # Lo [27] HANGUL SYLLABLE GYEOG..HANGUL SYLLABLE GYEOH -ACC5..ACDF ; LVT # Lo [27] HANGUL SYLLABLE GYEG..HANGUL SYLLABLE GYEH -ACE1..ACFB ; LVT # Lo [27] HANGUL SYLLABLE GOG..HANGUL SYLLABLE GOH -ACFD..AD17 ; LVT # Lo [27] HANGUL SYLLABLE GWAG..HANGUL SYLLABLE GWAH -AD19..AD33 ; LVT # Lo [27] HANGUL SYLLABLE GWAEG..HANGUL SYLLABLE GWAEH -AD35..AD4F ; LVT # Lo [27] HANGUL SYLLABLE GOEG..HANGUL SYLLABLE GOEH -AD51..AD6B ; LVT # Lo [27] HANGUL SYLLABLE GYOG..HANGUL SYLLABLE GYOH -AD6D..AD87 ; LVT # Lo [27] HANGUL SYLLABLE GUG..HANGUL SYLLABLE GUH -AD89..ADA3 ; LVT # Lo [27] HANGUL SYLLABLE GWEOG..HANGUL SYLLABLE GWEOH -ADA5..ADBF ; LVT # Lo [27] HANGUL SYLLABLE GWEG..HANGUL SYLLABLE GWEH -ADC1..ADDB ; LVT # Lo [27] HANGUL SYLLABLE GWIG..HANGUL SYLLABLE GWIH -ADDD..ADF7 ; LVT # Lo [27] HANGUL SYLLABLE GYUG..HANGUL SYLLABLE GYUH -ADF9..AE13 ; LVT # Lo [27] HANGUL SYLLABLE GEUG..HANGUL SYLLABLE GEUH -AE15..AE2F ; LVT # Lo [27] HANGUL SYLLABLE GYIG..HANGUL SYLLABLE GYIH -AE31..AE4B ; LVT # Lo [27] HANGUL SYLLABLE GIG..HANGUL SYLLABLE GIH -AE4D..AE67 ; LVT # Lo [27] HANGUL SYLLABLE GGAG..HANGUL SYLLABLE GGAH -AE69..AE83 ; LVT # Lo [27] HANGUL SYLLABLE GGAEG..HANGUL SYLLABLE GGAEH -AE85..AE9F ; LVT # Lo [27] HANGUL SYLLABLE GGYAG..HANGUL SYLLABLE GGYAH -AEA1..AEBB ; LVT # Lo [27] HANGUL SYLLABLE GGYAEG..HANGUL SYLLABLE GGYAEH -AEBD..AED7 ; LVT # Lo [27] HANGUL SYLLABLE GGEOG..HANGUL SYLLABLE GGEOH -AED9..AEF3 ; LVT # Lo [27] HANGUL SYLLABLE GGEG..HANGUL SYLLABLE GGEH -AEF5..AF0F ; LVT # Lo [27] HANGUL SYLLABLE GGYEOG..HANGUL SYLLABLE GGYEOH -AF11..AF2B ; LVT # Lo [27] HANGUL SYLLABLE GGYEG..HANGUL SYLLABLE GGYEH -AF2D..AF47 ; LVT # Lo [27] HANGUL SYLLABLE GGOG..HANGUL SYLLABLE GGOH -AF49..AF63 ; LVT # Lo [27] HANGUL SYLLABLE GGWAG..HANGUL SYLLABLE GGWAH -AF65..AF7F ; LVT # Lo [27] HANGUL SYLLABLE GGWAEG..HANGUL SYLLABLE GGWAEH -AF81..AF9B ; LVT # Lo [27] HANGUL SYLLABLE GGOEG..HANGUL SYLLABLE GGOEH -AF9D..AFB7 ; LVT # Lo [27] HANGUL SYLLABLE GGYOG..HANGUL SYLLABLE GGYOH -AFB9..AFD3 ; LVT # Lo [27] HANGUL SYLLABLE GGUG..HANGUL SYLLABLE GGUH -AFD5..AFEF ; LVT # Lo [27] HANGUL SYLLABLE GGWEOG..HANGUL SYLLABLE GGWEOH -AFF1..B00B ; LVT # Lo [27] HANGUL SYLLABLE GGWEG..HANGUL SYLLABLE GGWEH -B00D..B027 ; LVT # Lo [27] HANGUL SYLLABLE GGWIG..HANGUL SYLLABLE GGWIH -B029..B043 ; LVT # Lo [27] HANGUL SYLLABLE GGYUG..HANGUL SYLLABLE GGYUH -B045..B05F ; LVT # Lo [27] HANGUL SYLLABLE GGEUG..HANGUL SYLLABLE GGEUH -B061..B07B ; LVT # Lo [27] HANGUL SYLLABLE GGYIG..HANGUL SYLLABLE GGYIH -B07D..B097 ; LVT # Lo [27] HANGUL SYLLABLE GGIG..HANGUL SYLLABLE GGIH -B099..B0B3 ; LVT # Lo [27] HANGUL SYLLABLE NAG..HANGUL SYLLABLE NAH -B0B5..B0CF ; LVT # Lo [27] HANGUL SYLLABLE NAEG..HANGUL SYLLABLE NAEH -B0D1..B0EB ; LVT # Lo [27] HANGUL SYLLABLE NYAG..HANGUL SYLLABLE NYAH -B0ED..B107 ; LVT # Lo [27] HANGUL SYLLABLE NYAEG..HANGUL SYLLABLE NYAEH -B109..B123 ; LVT # Lo [27] HANGUL SYLLABLE NEOG..HANGUL SYLLABLE NEOH -B125..B13F ; LVT # Lo [27] HANGUL SYLLABLE NEG..HANGUL SYLLABLE NEH -B141..B15B ; LVT # Lo [27] HANGUL SYLLABLE NYEOG..HANGUL SYLLABLE NYEOH -B15D..B177 ; LVT # Lo [27] HANGUL SYLLABLE NYEG..HANGUL SYLLABLE NYEH -B179..B193 ; LVT # Lo [27] HANGUL SYLLABLE NOG..HANGUL SYLLABLE NOH -B195..B1AF ; LVT # Lo [27] HANGUL SYLLABLE NWAG..HANGUL SYLLABLE NWAH -B1B1..B1CB ; LVT # Lo [27] HANGUL SYLLABLE NWAEG..HANGUL SYLLABLE NWAEH -B1CD..B1E7 ; LVT # Lo [27] HANGUL SYLLABLE NOEG..HANGUL SYLLABLE NOEH -B1E9..B203 ; LVT # Lo [27] HANGUL SYLLABLE NYOG..HANGUL SYLLABLE NYOH -B205..B21F ; LVT # Lo [27] HANGUL SYLLABLE NUG..HANGUL SYLLABLE NUH -B221..B23B ; LVT # Lo [27] HANGUL SYLLABLE NWEOG..HANGUL SYLLABLE NWEOH -B23D..B257 ; LVT # Lo [27] HANGUL SYLLABLE NWEG..HANGUL SYLLABLE NWEH -B259..B273 ; LVT # Lo [27] HANGUL SYLLABLE NWIG..HANGUL SYLLABLE NWIH -B275..B28F ; LVT # Lo [27] HANGUL SYLLABLE NYUG..HANGUL SYLLABLE NYUH -B291..B2AB ; LVT # Lo [27] HANGUL SYLLABLE NEUG..HANGUL SYLLABLE NEUH -B2AD..B2C7 ; LVT # Lo [27] HANGUL SYLLABLE NYIG..HANGUL SYLLABLE NYIH -B2C9..B2E3 ; LVT # Lo [27] HANGUL SYLLABLE NIG..HANGUL SYLLABLE NIH -B2E5..B2FF ; LVT # Lo [27] HANGUL SYLLABLE DAG..HANGUL SYLLABLE DAH -B301..B31B ; LVT # Lo [27] HANGUL SYLLABLE DAEG..HANGUL SYLLABLE DAEH -B31D..B337 ; LVT # Lo [27] HANGUL SYLLABLE DYAG..HANGUL SYLLABLE DYAH -B339..B353 ; LVT # Lo [27] HANGUL SYLLABLE DYAEG..HANGUL SYLLABLE DYAEH -B355..B36F ; LVT # Lo [27] HANGUL SYLLABLE DEOG..HANGUL SYLLABLE DEOH -B371..B38B ; LVT # Lo [27] HANGUL SYLLABLE DEG..HANGUL SYLLABLE DEH -B38D..B3A7 ; LVT # Lo [27] HANGUL SYLLABLE DYEOG..HANGUL SYLLABLE DYEOH -B3A9..B3C3 ; LVT # Lo [27] HANGUL SYLLABLE DYEG..HANGUL SYLLABLE DYEH -B3C5..B3DF ; LVT # Lo [27] HANGUL SYLLABLE DOG..HANGUL SYLLABLE DOH -B3E1..B3FB ; LVT # Lo [27] HANGUL SYLLABLE DWAG..HANGUL SYLLABLE DWAH -B3FD..B417 ; LVT # Lo [27] HANGUL SYLLABLE DWAEG..HANGUL SYLLABLE DWAEH -B419..B433 ; LVT # Lo [27] HANGUL SYLLABLE DOEG..HANGUL SYLLABLE DOEH -B435..B44F ; LVT # Lo [27] HANGUL SYLLABLE DYOG..HANGUL SYLLABLE DYOH -B451..B46B ; LVT # Lo [27] HANGUL SYLLABLE DUG..HANGUL SYLLABLE DUH -B46D..B487 ; LVT # Lo [27] HANGUL SYLLABLE DWEOG..HANGUL SYLLABLE DWEOH -B489..B4A3 ; LVT # Lo [27] HANGUL SYLLABLE DWEG..HANGUL SYLLABLE DWEH -B4A5..B4BF ; LVT # Lo [27] HANGUL SYLLABLE DWIG..HANGUL SYLLABLE DWIH -B4C1..B4DB ; LVT # Lo [27] HANGUL SYLLABLE DYUG..HANGUL SYLLABLE DYUH -B4DD..B4F7 ; LVT # Lo [27] HANGUL SYLLABLE DEUG..HANGUL SYLLABLE DEUH -B4F9..B513 ; LVT # Lo [27] HANGUL SYLLABLE DYIG..HANGUL SYLLABLE DYIH -B515..B52F ; LVT # Lo [27] HANGUL SYLLABLE DIG..HANGUL SYLLABLE DIH -B531..B54B ; LVT # Lo [27] HANGUL SYLLABLE DDAG..HANGUL SYLLABLE DDAH -B54D..B567 ; LVT # Lo [27] HANGUL SYLLABLE DDAEG..HANGUL SYLLABLE DDAEH -B569..B583 ; LVT # Lo [27] HANGUL SYLLABLE DDYAG..HANGUL SYLLABLE DDYAH -B585..B59F ; LVT # Lo [27] HANGUL SYLLABLE DDYAEG..HANGUL SYLLABLE DDYAEH -B5A1..B5BB ; LVT # Lo [27] HANGUL SYLLABLE DDEOG..HANGUL SYLLABLE DDEOH -B5BD..B5D7 ; LVT # Lo [27] HANGUL SYLLABLE DDEG..HANGUL SYLLABLE DDEH -B5D9..B5F3 ; LVT # Lo [27] HANGUL SYLLABLE DDYEOG..HANGUL SYLLABLE DDYEOH -B5F5..B60F ; LVT # Lo [27] HANGUL SYLLABLE DDYEG..HANGUL SYLLABLE DDYEH -B611..B62B ; LVT # Lo [27] HANGUL SYLLABLE DDOG..HANGUL SYLLABLE DDOH -B62D..B647 ; LVT # Lo [27] HANGUL SYLLABLE DDWAG..HANGUL SYLLABLE DDWAH -B649..B663 ; LVT # Lo [27] HANGUL SYLLABLE DDWAEG..HANGUL SYLLABLE DDWAEH -B665..B67F ; LVT # Lo [27] HANGUL SYLLABLE DDOEG..HANGUL SYLLABLE DDOEH -B681..B69B ; LVT # Lo [27] HANGUL SYLLABLE DDYOG..HANGUL SYLLABLE DDYOH -B69D..B6B7 ; LVT # Lo [27] HANGUL SYLLABLE DDUG..HANGUL SYLLABLE DDUH -B6B9..B6D3 ; LVT # Lo [27] HANGUL SYLLABLE DDWEOG..HANGUL SYLLABLE DDWEOH -B6D5..B6EF ; LVT # Lo [27] HANGUL SYLLABLE DDWEG..HANGUL SYLLABLE DDWEH -B6F1..B70B ; LVT # Lo [27] HANGUL SYLLABLE DDWIG..HANGUL SYLLABLE DDWIH -B70D..B727 ; LVT # Lo [27] HANGUL SYLLABLE DDYUG..HANGUL SYLLABLE DDYUH -B729..B743 ; LVT # Lo [27] HANGUL SYLLABLE DDEUG..HANGUL SYLLABLE DDEUH -B745..B75F ; LVT # Lo [27] HANGUL SYLLABLE DDYIG..HANGUL SYLLABLE DDYIH -B761..B77B ; LVT # Lo [27] HANGUL SYLLABLE DDIG..HANGUL SYLLABLE DDIH -B77D..B797 ; LVT # Lo [27] HANGUL SYLLABLE RAG..HANGUL SYLLABLE RAH -B799..B7B3 ; LVT # Lo [27] HANGUL SYLLABLE RAEG..HANGUL SYLLABLE RAEH -B7B5..B7CF ; LVT # Lo [27] HANGUL SYLLABLE RYAG..HANGUL SYLLABLE RYAH -B7D1..B7EB ; LVT # Lo [27] HANGUL SYLLABLE RYAEG..HANGUL SYLLABLE RYAEH -B7ED..B807 ; LVT # Lo [27] HANGUL SYLLABLE REOG..HANGUL SYLLABLE REOH -B809..B823 ; LVT # Lo [27] HANGUL SYLLABLE REG..HANGUL SYLLABLE REH -B825..B83F ; LVT # Lo [27] HANGUL SYLLABLE RYEOG..HANGUL SYLLABLE RYEOH -B841..B85B ; LVT # Lo [27] HANGUL SYLLABLE RYEG..HANGUL SYLLABLE RYEH -B85D..B877 ; LVT # Lo [27] HANGUL SYLLABLE ROG..HANGUL SYLLABLE ROH -B879..B893 ; LVT # Lo [27] HANGUL SYLLABLE RWAG..HANGUL SYLLABLE RWAH -B895..B8AF ; LVT # Lo [27] HANGUL SYLLABLE RWAEG..HANGUL SYLLABLE RWAEH -B8B1..B8CB ; LVT # Lo [27] HANGUL SYLLABLE ROEG..HANGUL SYLLABLE ROEH -B8CD..B8E7 ; LVT # Lo [27] HANGUL SYLLABLE RYOG..HANGUL SYLLABLE RYOH -B8E9..B903 ; LVT # Lo [27] HANGUL SYLLABLE RUG..HANGUL SYLLABLE RUH -B905..B91F ; LVT # Lo [27] HANGUL SYLLABLE RWEOG..HANGUL SYLLABLE RWEOH -B921..B93B ; LVT # Lo [27] HANGUL SYLLABLE RWEG..HANGUL SYLLABLE RWEH -B93D..B957 ; LVT # Lo [27] HANGUL SYLLABLE RWIG..HANGUL SYLLABLE RWIH -B959..B973 ; LVT # Lo [27] HANGUL SYLLABLE RYUG..HANGUL SYLLABLE RYUH -B975..B98F ; LVT # Lo [27] HANGUL SYLLABLE REUG..HANGUL SYLLABLE REUH -B991..B9AB ; LVT # Lo [27] HANGUL SYLLABLE RYIG..HANGUL SYLLABLE RYIH -B9AD..B9C7 ; LVT # Lo [27] HANGUL SYLLABLE RIG..HANGUL SYLLABLE RIH -B9C9..B9E3 ; LVT # Lo [27] HANGUL SYLLABLE MAG..HANGUL SYLLABLE MAH -B9E5..B9FF ; LVT # Lo [27] HANGUL SYLLABLE MAEG..HANGUL SYLLABLE MAEH -BA01..BA1B ; LVT # Lo [27] HANGUL SYLLABLE MYAG..HANGUL SYLLABLE MYAH -BA1D..BA37 ; LVT # Lo [27] HANGUL SYLLABLE MYAEG..HANGUL SYLLABLE MYAEH -BA39..BA53 ; LVT # Lo [27] HANGUL SYLLABLE MEOG..HANGUL SYLLABLE MEOH -BA55..BA6F ; LVT # Lo [27] HANGUL SYLLABLE MEG..HANGUL SYLLABLE MEH -BA71..BA8B ; LVT # Lo [27] HANGUL SYLLABLE MYEOG..HANGUL SYLLABLE MYEOH -BA8D..BAA7 ; LVT # Lo [27] HANGUL SYLLABLE MYEG..HANGUL SYLLABLE MYEH -BAA9..BAC3 ; LVT # Lo [27] HANGUL SYLLABLE MOG..HANGUL SYLLABLE MOH -BAC5..BADF ; LVT # Lo [27] HANGUL SYLLABLE MWAG..HANGUL SYLLABLE MWAH -BAE1..BAFB ; LVT # Lo [27] HANGUL SYLLABLE MWAEG..HANGUL SYLLABLE MWAEH -BAFD..BB17 ; LVT # Lo [27] HANGUL SYLLABLE MOEG..HANGUL SYLLABLE MOEH -BB19..BB33 ; LVT # Lo [27] HANGUL SYLLABLE MYOG..HANGUL SYLLABLE MYOH -BB35..BB4F ; LVT # Lo [27] HANGUL SYLLABLE MUG..HANGUL SYLLABLE MUH -BB51..BB6B ; LVT # Lo [27] HANGUL SYLLABLE MWEOG..HANGUL SYLLABLE MWEOH -BB6D..BB87 ; LVT # Lo [27] HANGUL SYLLABLE MWEG..HANGUL SYLLABLE MWEH -BB89..BBA3 ; LVT # Lo [27] HANGUL SYLLABLE MWIG..HANGUL SYLLABLE MWIH -BBA5..BBBF ; LVT # Lo [27] HANGUL SYLLABLE MYUG..HANGUL SYLLABLE MYUH -BBC1..BBDB ; LVT # Lo [27] HANGUL SYLLABLE MEUG..HANGUL SYLLABLE MEUH -BBDD..BBF7 ; LVT # Lo [27] HANGUL SYLLABLE MYIG..HANGUL SYLLABLE MYIH -BBF9..BC13 ; LVT # Lo [27] HANGUL SYLLABLE MIG..HANGUL SYLLABLE MIH -BC15..BC2F ; LVT # Lo [27] HANGUL SYLLABLE BAG..HANGUL SYLLABLE BAH -BC31..BC4B ; LVT # Lo [27] HANGUL SYLLABLE BAEG..HANGUL SYLLABLE BAEH -BC4D..BC67 ; LVT # Lo [27] HANGUL SYLLABLE BYAG..HANGUL SYLLABLE BYAH -BC69..BC83 ; LVT # Lo [27] HANGUL SYLLABLE BYAEG..HANGUL SYLLABLE BYAEH -BC85..BC9F ; LVT # Lo [27] HANGUL SYLLABLE BEOG..HANGUL SYLLABLE BEOH -BCA1..BCBB ; LVT # Lo [27] HANGUL SYLLABLE BEG..HANGUL SYLLABLE BEH -BCBD..BCD7 ; LVT # Lo [27] HANGUL SYLLABLE BYEOG..HANGUL SYLLABLE BYEOH -BCD9..BCF3 ; LVT # Lo [27] HANGUL SYLLABLE BYEG..HANGUL SYLLABLE BYEH -BCF5..BD0F ; LVT # Lo [27] HANGUL SYLLABLE BOG..HANGUL SYLLABLE BOH -BD11..BD2B ; LVT # Lo [27] HANGUL SYLLABLE BWAG..HANGUL SYLLABLE BWAH -BD2D..BD47 ; LVT # Lo [27] HANGUL SYLLABLE BWAEG..HANGUL SYLLABLE BWAEH -BD49..BD63 ; LVT # Lo [27] HANGUL SYLLABLE BOEG..HANGUL SYLLABLE BOEH -BD65..BD7F ; LVT # Lo [27] HANGUL SYLLABLE BYOG..HANGUL SYLLABLE BYOH -BD81..BD9B ; LVT # Lo [27] HANGUL SYLLABLE BUG..HANGUL SYLLABLE BUH -BD9D..BDB7 ; LVT # Lo [27] HANGUL SYLLABLE BWEOG..HANGUL SYLLABLE BWEOH -BDB9..BDD3 ; LVT # Lo [27] HANGUL SYLLABLE BWEG..HANGUL SYLLABLE BWEH -BDD5..BDEF ; LVT # Lo [27] HANGUL SYLLABLE BWIG..HANGUL SYLLABLE BWIH -BDF1..BE0B ; LVT # Lo [27] HANGUL SYLLABLE BYUG..HANGUL SYLLABLE BYUH -BE0D..BE27 ; LVT # Lo [27] HANGUL SYLLABLE BEUG..HANGUL SYLLABLE BEUH -BE29..BE43 ; LVT # Lo [27] HANGUL SYLLABLE BYIG..HANGUL SYLLABLE BYIH -BE45..BE5F ; LVT # Lo [27] HANGUL SYLLABLE BIG..HANGUL SYLLABLE BIH -BE61..BE7B ; LVT # Lo [27] HANGUL SYLLABLE BBAG..HANGUL SYLLABLE BBAH -BE7D..BE97 ; LVT # Lo [27] HANGUL SYLLABLE BBAEG..HANGUL SYLLABLE BBAEH -BE99..BEB3 ; LVT # Lo [27] HANGUL SYLLABLE BBYAG..HANGUL SYLLABLE BBYAH -BEB5..BECF ; LVT # Lo [27] HANGUL SYLLABLE BBYAEG..HANGUL SYLLABLE BBYAEH -BED1..BEEB ; LVT # Lo [27] HANGUL SYLLABLE BBEOG..HANGUL SYLLABLE BBEOH -BEED..BF07 ; LVT # Lo [27] HANGUL SYLLABLE BBEG..HANGUL SYLLABLE BBEH -BF09..BF23 ; LVT # Lo [27] HANGUL SYLLABLE BBYEOG..HANGUL SYLLABLE BBYEOH -BF25..BF3F ; LVT # Lo [27] HANGUL SYLLABLE BBYEG..HANGUL SYLLABLE BBYEH -BF41..BF5B ; LVT # Lo [27] HANGUL SYLLABLE BBOG..HANGUL SYLLABLE BBOH -BF5D..BF77 ; LVT # Lo [27] HANGUL SYLLABLE BBWAG..HANGUL SYLLABLE BBWAH -BF79..BF93 ; LVT # Lo [27] HANGUL SYLLABLE BBWAEG..HANGUL SYLLABLE BBWAEH -BF95..BFAF ; LVT # Lo [27] HANGUL SYLLABLE BBOEG..HANGUL SYLLABLE BBOEH -BFB1..BFCB ; LVT # Lo [27] HANGUL SYLLABLE BBYOG..HANGUL SYLLABLE BBYOH -BFCD..BFE7 ; LVT # Lo [27] HANGUL SYLLABLE BBUG..HANGUL SYLLABLE BBUH -BFE9..C003 ; LVT # Lo [27] HANGUL SYLLABLE BBWEOG..HANGUL SYLLABLE BBWEOH -C005..C01F ; LVT # Lo [27] HANGUL SYLLABLE BBWEG..HANGUL SYLLABLE BBWEH -C021..C03B ; LVT # Lo [27] HANGUL SYLLABLE BBWIG..HANGUL SYLLABLE BBWIH -C03D..C057 ; LVT # Lo [27] HANGUL SYLLABLE BBYUG..HANGUL SYLLABLE BBYUH -C059..C073 ; LVT # Lo [27] HANGUL SYLLABLE BBEUG..HANGUL SYLLABLE BBEUH -C075..C08F ; LVT # Lo [27] HANGUL SYLLABLE BBYIG..HANGUL SYLLABLE BBYIH -C091..C0AB ; LVT # Lo [27] HANGUL SYLLABLE BBIG..HANGUL SYLLABLE BBIH -C0AD..C0C7 ; LVT # Lo [27] HANGUL SYLLABLE SAG..HANGUL SYLLABLE SAH -C0C9..C0E3 ; LVT # Lo [27] HANGUL SYLLABLE SAEG..HANGUL SYLLABLE SAEH -C0E5..C0FF ; LVT # Lo [27] HANGUL SYLLABLE SYAG..HANGUL SYLLABLE SYAH -C101..C11B ; LVT # Lo [27] HANGUL SYLLABLE SYAEG..HANGUL SYLLABLE SYAEH -C11D..C137 ; LVT # Lo [27] HANGUL SYLLABLE SEOG..HANGUL SYLLABLE SEOH -C139..C153 ; LVT # Lo [27] HANGUL SYLLABLE SEG..HANGUL SYLLABLE SEH -C155..C16F ; LVT # Lo [27] HANGUL SYLLABLE SYEOG..HANGUL SYLLABLE SYEOH -C171..C18B ; LVT # Lo [27] HANGUL SYLLABLE SYEG..HANGUL SYLLABLE SYEH -C18D..C1A7 ; LVT # Lo [27] HANGUL SYLLABLE SOG..HANGUL SYLLABLE SOH -C1A9..C1C3 ; LVT # Lo [27] HANGUL SYLLABLE SWAG..HANGUL SYLLABLE SWAH -C1C5..C1DF ; LVT # Lo [27] HANGUL SYLLABLE SWAEG..HANGUL SYLLABLE SWAEH -C1E1..C1FB ; LVT # Lo [27] HANGUL SYLLABLE SOEG..HANGUL SYLLABLE SOEH -C1FD..C217 ; LVT # Lo [27] HANGUL SYLLABLE SYOG..HANGUL SYLLABLE SYOH -C219..C233 ; LVT # Lo [27] HANGUL SYLLABLE SUG..HANGUL SYLLABLE SUH -C235..C24F ; LVT # Lo [27] HANGUL SYLLABLE SWEOG..HANGUL SYLLABLE SWEOH -C251..C26B ; LVT # Lo [27] HANGUL SYLLABLE SWEG..HANGUL SYLLABLE SWEH -C26D..C287 ; LVT # Lo [27] HANGUL SYLLABLE SWIG..HANGUL SYLLABLE SWIH -C289..C2A3 ; LVT # Lo [27] HANGUL SYLLABLE SYUG..HANGUL SYLLABLE SYUH -C2A5..C2BF ; LVT # Lo [27] HANGUL SYLLABLE SEUG..HANGUL SYLLABLE SEUH -C2C1..C2DB ; LVT # Lo [27] HANGUL SYLLABLE SYIG..HANGUL SYLLABLE SYIH -C2DD..C2F7 ; LVT # Lo [27] HANGUL SYLLABLE SIG..HANGUL SYLLABLE SIH -C2F9..C313 ; LVT # Lo [27] HANGUL SYLLABLE SSAG..HANGUL SYLLABLE SSAH -C315..C32F ; LVT # Lo [27] HANGUL SYLLABLE SSAEG..HANGUL SYLLABLE SSAEH -C331..C34B ; LVT # Lo [27] HANGUL SYLLABLE SSYAG..HANGUL SYLLABLE SSYAH -C34D..C367 ; LVT # Lo [27] HANGUL SYLLABLE SSYAEG..HANGUL SYLLABLE SSYAEH -C369..C383 ; LVT # Lo [27] HANGUL SYLLABLE SSEOG..HANGUL SYLLABLE SSEOH -C385..C39F ; LVT # Lo [27] HANGUL SYLLABLE SSEG..HANGUL SYLLABLE SSEH -C3A1..C3BB ; LVT # Lo [27] HANGUL SYLLABLE SSYEOG..HANGUL SYLLABLE SSYEOH -C3BD..C3D7 ; LVT # Lo [27] HANGUL SYLLABLE SSYEG..HANGUL SYLLABLE SSYEH -C3D9..C3F3 ; LVT # Lo [27] HANGUL SYLLABLE SSOG..HANGUL SYLLABLE SSOH -C3F5..C40F ; LVT # Lo [27] HANGUL SYLLABLE SSWAG..HANGUL SYLLABLE SSWAH -C411..C42B ; LVT # Lo [27] HANGUL SYLLABLE SSWAEG..HANGUL SYLLABLE SSWAEH -C42D..C447 ; LVT # Lo [27] HANGUL SYLLABLE SSOEG..HANGUL SYLLABLE SSOEH -C449..C463 ; LVT # Lo [27] HANGUL SYLLABLE SSYOG..HANGUL SYLLABLE SSYOH -C465..C47F ; LVT # Lo [27] HANGUL SYLLABLE SSUG..HANGUL SYLLABLE SSUH -C481..C49B ; LVT # Lo [27] HANGUL SYLLABLE SSWEOG..HANGUL SYLLABLE SSWEOH -C49D..C4B7 ; LVT # Lo [27] HANGUL SYLLABLE SSWEG..HANGUL SYLLABLE SSWEH -C4B9..C4D3 ; LVT # Lo [27] HANGUL SYLLABLE SSWIG..HANGUL SYLLABLE SSWIH -C4D5..C4EF ; LVT # Lo [27] HANGUL SYLLABLE SSYUG..HANGUL SYLLABLE SSYUH -C4F1..C50B ; LVT # Lo [27] HANGUL SYLLABLE SSEUG..HANGUL SYLLABLE SSEUH -C50D..C527 ; LVT # Lo [27] HANGUL SYLLABLE SSYIG..HANGUL SYLLABLE SSYIH -C529..C543 ; LVT # Lo [27] HANGUL SYLLABLE SSIG..HANGUL SYLLABLE SSIH -C545..C55F ; LVT # Lo [27] HANGUL SYLLABLE AG..HANGUL SYLLABLE AH -C561..C57B ; LVT # Lo [27] HANGUL SYLLABLE AEG..HANGUL SYLLABLE AEH -C57D..C597 ; LVT # Lo [27] HANGUL SYLLABLE YAG..HANGUL SYLLABLE YAH -C599..C5B3 ; LVT # Lo [27] HANGUL SYLLABLE YAEG..HANGUL SYLLABLE YAEH -C5B5..C5CF ; LVT # Lo [27] HANGUL SYLLABLE EOG..HANGUL SYLLABLE EOH -C5D1..C5EB ; LVT # Lo [27] HANGUL SYLLABLE EG..HANGUL SYLLABLE EH -C5ED..C607 ; LVT # Lo [27] HANGUL SYLLABLE YEOG..HANGUL SYLLABLE YEOH -C609..C623 ; LVT # Lo [27] HANGUL SYLLABLE YEG..HANGUL SYLLABLE YEH -C625..C63F ; LVT # Lo [27] HANGUL SYLLABLE OG..HANGUL SYLLABLE OH -C641..C65B ; LVT # Lo [27] HANGUL SYLLABLE WAG..HANGUL SYLLABLE WAH -C65D..C677 ; LVT # Lo [27] HANGUL SYLLABLE WAEG..HANGUL SYLLABLE WAEH -C679..C693 ; LVT # Lo [27] HANGUL SYLLABLE OEG..HANGUL SYLLABLE OEH -C695..C6AF ; LVT # Lo [27] HANGUL SYLLABLE YOG..HANGUL SYLLABLE YOH -C6B1..C6CB ; LVT # Lo [27] HANGUL SYLLABLE UG..HANGUL SYLLABLE UH -C6CD..C6E7 ; LVT # Lo [27] HANGUL SYLLABLE WEOG..HANGUL SYLLABLE WEOH -C6E9..C703 ; LVT # Lo [27] HANGUL SYLLABLE WEG..HANGUL SYLLABLE WEH -C705..C71F ; LVT # Lo [27] HANGUL SYLLABLE WIG..HANGUL SYLLABLE WIH -C721..C73B ; LVT # Lo [27] HANGUL SYLLABLE YUG..HANGUL SYLLABLE YUH -C73D..C757 ; LVT # Lo [27] HANGUL SYLLABLE EUG..HANGUL SYLLABLE EUH -C759..C773 ; LVT # Lo [27] HANGUL SYLLABLE YIG..HANGUL SYLLABLE YIH -C775..C78F ; LVT # Lo [27] HANGUL SYLLABLE IG..HANGUL SYLLABLE IH -C791..C7AB ; LVT # Lo [27] HANGUL SYLLABLE JAG..HANGUL SYLLABLE JAH -C7AD..C7C7 ; LVT # Lo [27] HANGUL SYLLABLE JAEG..HANGUL SYLLABLE JAEH -C7C9..C7E3 ; LVT # Lo [27] HANGUL SYLLABLE JYAG..HANGUL SYLLABLE JYAH -C7E5..C7FF ; LVT # Lo [27] HANGUL SYLLABLE JYAEG..HANGUL SYLLABLE JYAEH -C801..C81B ; LVT # Lo [27] HANGUL SYLLABLE JEOG..HANGUL SYLLABLE JEOH -C81D..C837 ; LVT # Lo [27] HANGUL SYLLABLE JEG..HANGUL SYLLABLE JEH -C839..C853 ; LVT # Lo [27] HANGUL SYLLABLE JYEOG..HANGUL SYLLABLE JYEOH -C855..C86F ; LVT # Lo [27] HANGUL SYLLABLE JYEG..HANGUL SYLLABLE JYEH -C871..C88B ; LVT # Lo [27] HANGUL SYLLABLE JOG..HANGUL SYLLABLE JOH -C88D..C8A7 ; LVT # Lo [27] HANGUL SYLLABLE JWAG..HANGUL SYLLABLE JWAH -C8A9..C8C3 ; LVT # Lo [27] HANGUL SYLLABLE JWAEG..HANGUL SYLLABLE JWAEH -C8C5..C8DF ; LVT # Lo [27] HANGUL SYLLABLE JOEG..HANGUL SYLLABLE JOEH -C8E1..C8FB ; LVT # Lo [27] HANGUL SYLLABLE JYOG..HANGUL SYLLABLE JYOH -C8FD..C917 ; LVT # Lo [27] HANGUL SYLLABLE JUG..HANGUL SYLLABLE JUH -C919..C933 ; LVT # Lo [27] HANGUL SYLLABLE JWEOG..HANGUL SYLLABLE JWEOH -C935..C94F ; LVT # Lo [27] HANGUL SYLLABLE JWEG..HANGUL SYLLABLE JWEH -C951..C96B ; LVT # Lo [27] HANGUL SYLLABLE JWIG..HANGUL SYLLABLE JWIH -C96D..C987 ; LVT # Lo [27] HANGUL SYLLABLE JYUG..HANGUL SYLLABLE JYUH -C989..C9A3 ; LVT # Lo [27] HANGUL SYLLABLE JEUG..HANGUL SYLLABLE JEUH -C9A5..C9BF ; LVT # Lo [27] HANGUL SYLLABLE JYIG..HANGUL SYLLABLE JYIH -C9C1..C9DB ; LVT # Lo [27] HANGUL SYLLABLE JIG..HANGUL SYLLABLE JIH -C9DD..C9F7 ; LVT # Lo [27] HANGUL SYLLABLE JJAG..HANGUL SYLLABLE JJAH -C9F9..CA13 ; LVT # Lo [27] HANGUL SYLLABLE JJAEG..HANGUL SYLLABLE JJAEH -CA15..CA2F ; LVT # Lo [27] HANGUL SYLLABLE JJYAG..HANGUL SYLLABLE JJYAH -CA31..CA4B ; LVT # Lo [27] HANGUL SYLLABLE JJYAEG..HANGUL SYLLABLE JJYAEH -CA4D..CA67 ; LVT # Lo [27] HANGUL SYLLABLE JJEOG..HANGUL SYLLABLE JJEOH -CA69..CA83 ; LVT # Lo [27] HANGUL SYLLABLE JJEG..HANGUL SYLLABLE JJEH -CA85..CA9F ; LVT # Lo [27] HANGUL SYLLABLE JJYEOG..HANGUL SYLLABLE JJYEOH -CAA1..CABB ; LVT # Lo [27] HANGUL SYLLABLE JJYEG..HANGUL SYLLABLE JJYEH -CABD..CAD7 ; LVT # Lo [27] HANGUL SYLLABLE JJOG..HANGUL SYLLABLE JJOH -CAD9..CAF3 ; LVT # Lo [27] HANGUL SYLLABLE JJWAG..HANGUL SYLLABLE JJWAH -CAF5..CB0F ; LVT # Lo [27] HANGUL SYLLABLE JJWAEG..HANGUL SYLLABLE JJWAEH -CB11..CB2B ; LVT # Lo [27] HANGUL SYLLABLE JJOEG..HANGUL SYLLABLE JJOEH -CB2D..CB47 ; LVT # Lo [27] HANGUL SYLLABLE JJYOG..HANGUL SYLLABLE JJYOH -CB49..CB63 ; LVT # Lo [27] HANGUL SYLLABLE JJUG..HANGUL SYLLABLE JJUH -CB65..CB7F ; LVT # Lo [27] HANGUL SYLLABLE JJWEOG..HANGUL SYLLABLE JJWEOH -CB81..CB9B ; LVT # Lo [27] HANGUL SYLLABLE JJWEG..HANGUL SYLLABLE JJWEH -CB9D..CBB7 ; LVT # Lo [27] HANGUL SYLLABLE JJWIG..HANGUL SYLLABLE JJWIH -CBB9..CBD3 ; LVT # Lo [27] HANGUL SYLLABLE JJYUG..HANGUL SYLLABLE JJYUH -CBD5..CBEF ; LVT # Lo [27] HANGUL SYLLABLE JJEUG..HANGUL SYLLABLE JJEUH -CBF1..CC0B ; LVT # Lo [27] HANGUL SYLLABLE JJYIG..HANGUL SYLLABLE JJYIH -CC0D..CC27 ; LVT # Lo [27] HANGUL SYLLABLE JJIG..HANGUL SYLLABLE JJIH -CC29..CC43 ; LVT # Lo [27] HANGUL SYLLABLE CAG..HANGUL SYLLABLE CAH -CC45..CC5F ; LVT # Lo [27] HANGUL SYLLABLE CAEG..HANGUL SYLLABLE CAEH -CC61..CC7B ; LVT # Lo [27] HANGUL SYLLABLE CYAG..HANGUL SYLLABLE CYAH -CC7D..CC97 ; LVT # Lo [27] HANGUL SYLLABLE CYAEG..HANGUL SYLLABLE CYAEH -CC99..CCB3 ; LVT # Lo [27] HANGUL SYLLABLE CEOG..HANGUL SYLLABLE CEOH -CCB5..CCCF ; LVT # Lo [27] HANGUL SYLLABLE CEG..HANGUL SYLLABLE CEH -CCD1..CCEB ; LVT # Lo [27] HANGUL SYLLABLE CYEOG..HANGUL SYLLABLE CYEOH -CCED..CD07 ; LVT # Lo [27] HANGUL SYLLABLE CYEG..HANGUL SYLLABLE CYEH -CD09..CD23 ; LVT # Lo [27] HANGUL SYLLABLE COG..HANGUL SYLLABLE COH -CD25..CD3F ; LVT # Lo [27] HANGUL SYLLABLE CWAG..HANGUL SYLLABLE CWAH -CD41..CD5B ; LVT # Lo [27] HANGUL SYLLABLE CWAEG..HANGUL SYLLABLE CWAEH -CD5D..CD77 ; LVT # Lo [27] HANGUL SYLLABLE COEG..HANGUL SYLLABLE COEH -CD79..CD93 ; LVT # Lo [27] HANGUL SYLLABLE CYOG..HANGUL SYLLABLE CYOH -CD95..CDAF ; LVT # Lo [27] HANGUL SYLLABLE CUG..HANGUL SYLLABLE CUH -CDB1..CDCB ; LVT # Lo [27] HANGUL SYLLABLE CWEOG..HANGUL SYLLABLE CWEOH -CDCD..CDE7 ; LVT # Lo [27] HANGUL SYLLABLE CWEG..HANGUL SYLLABLE CWEH -CDE9..CE03 ; LVT # Lo [27] HANGUL SYLLABLE CWIG..HANGUL SYLLABLE CWIH -CE05..CE1F ; LVT # Lo [27] HANGUL SYLLABLE CYUG..HANGUL SYLLABLE CYUH -CE21..CE3B ; LVT # Lo [27] HANGUL SYLLABLE CEUG..HANGUL SYLLABLE CEUH -CE3D..CE57 ; LVT # Lo [27] HANGUL SYLLABLE CYIG..HANGUL SYLLABLE CYIH -CE59..CE73 ; LVT # Lo [27] HANGUL SYLLABLE CIG..HANGUL SYLLABLE CIH -CE75..CE8F ; LVT # Lo [27] HANGUL SYLLABLE KAG..HANGUL SYLLABLE KAH -CE91..CEAB ; LVT # Lo [27] HANGUL SYLLABLE KAEG..HANGUL SYLLABLE KAEH -CEAD..CEC7 ; LVT # Lo [27] HANGUL SYLLABLE KYAG..HANGUL SYLLABLE KYAH -CEC9..CEE3 ; LVT # Lo [27] HANGUL SYLLABLE KYAEG..HANGUL SYLLABLE KYAEH -CEE5..CEFF ; LVT # Lo [27] HANGUL SYLLABLE KEOG..HANGUL SYLLABLE KEOH -CF01..CF1B ; LVT # Lo [27] HANGUL SYLLABLE KEG..HANGUL SYLLABLE KEH -CF1D..CF37 ; LVT # Lo [27] HANGUL SYLLABLE KYEOG..HANGUL SYLLABLE KYEOH -CF39..CF53 ; LVT # Lo [27] HANGUL SYLLABLE KYEG..HANGUL SYLLABLE KYEH -CF55..CF6F ; LVT # Lo [27] HANGUL SYLLABLE KOG..HANGUL SYLLABLE KOH -CF71..CF8B ; LVT # Lo [27] HANGUL SYLLABLE KWAG..HANGUL SYLLABLE KWAH -CF8D..CFA7 ; LVT # Lo [27] HANGUL SYLLABLE KWAEG..HANGUL SYLLABLE KWAEH -CFA9..CFC3 ; LVT # Lo [27] HANGUL SYLLABLE KOEG..HANGUL SYLLABLE KOEH -CFC5..CFDF ; LVT # Lo [27] HANGUL SYLLABLE KYOG..HANGUL SYLLABLE KYOH -CFE1..CFFB ; LVT # Lo [27] HANGUL SYLLABLE KUG..HANGUL SYLLABLE KUH -CFFD..D017 ; LVT # Lo [27] HANGUL SYLLABLE KWEOG..HANGUL SYLLABLE KWEOH -D019..D033 ; LVT # Lo [27] HANGUL SYLLABLE KWEG..HANGUL SYLLABLE KWEH -D035..D04F ; LVT # Lo [27] HANGUL SYLLABLE KWIG..HANGUL SYLLABLE KWIH -D051..D06B ; LVT # Lo [27] HANGUL SYLLABLE KYUG..HANGUL SYLLABLE KYUH -D06D..D087 ; LVT # Lo [27] HANGUL SYLLABLE KEUG..HANGUL SYLLABLE KEUH -D089..D0A3 ; LVT # Lo [27] HANGUL SYLLABLE KYIG..HANGUL SYLLABLE KYIH -D0A5..D0BF ; LVT # Lo [27] HANGUL SYLLABLE KIG..HANGUL SYLLABLE KIH -D0C1..D0DB ; LVT # Lo [27] HANGUL SYLLABLE TAG..HANGUL SYLLABLE TAH -D0DD..D0F7 ; LVT # Lo [27] HANGUL SYLLABLE TAEG..HANGUL SYLLABLE TAEH -D0F9..D113 ; LVT # Lo [27] HANGUL SYLLABLE TYAG..HANGUL SYLLABLE TYAH -D115..D12F ; LVT # Lo [27] HANGUL SYLLABLE TYAEG..HANGUL SYLLABLE TYAEH -D131..D14B ; LVT # Lo [27] HANGUL SYLLABLE TEOG..HANGUL SYLLABLE TEOH -D14D..D167 ; LVT # Lo [27] HANGUL SYLLABLE TEG..HANGUL SYLLABLE TEH -D169..D183 ; LVT # Lo [27] HANGUL SYLLABLE TYEOG..HANGUL SYLLABLE TYEOH -D185..D19F ; LVT # Lo [27] HANGUL SYLLABLE TYEG..HANGUL SYLLABLE TYEH -D1A1..D1BB ; LVT # Lo [27] HANGUL SYLLABLE TOG..HANGUL SYLLABLE TOH -D1BD..D1D7 ; LVT # Lo [27] HANGUL SYLLABLE TWAG..HANGUL SYLLABLE TWAH -D1D9..D1F3 ; LVT # Lo [27] HANGUL SYLLABLE TWAEG..HANGUL SYLLABLE TWAEH -D1F5..D20F ; LVT # Lo [27] HANGUL SYLLABLE TOEG..HANGUL SYLLABLE TOEH -D211..D22B ; LVT # Lo [27] HANGUL SYLLABLE TYOG..HANGUL SYLLABLE TYOH -D22D..D247 ; LVT # Lo [27] HANGUL SYLLABLE TUG..HANGUL SYLLABLE TUH -D249..D263 ; LVT # Lo [27] HANGUL SYLLABLE TWEOG..HANGUL SYLLABLE TWEOH -D265..D27F ; LVT # Lo [27] HANGUL SYLLABLE TWEG..HANGUL SYLLABLE TWEH -D281..D29B ; LVT # Lo [27] HANGUL SYLLABLE TWIG..HANGUL SYLLABLE TWIH -D29D..D2B7 ; LVT # Lo [27] HANGUL SYLLABLE TYUG..HANGUL SYLLABLE TYUH -D2B9..D2D3 ; LVT # Lo [27] HANGUL SYLLABLE TEUG..HANGUL SYLLABLE TEUH -D2D5..D2EF ; LVT # Lo [27] HANGUL SYLLABLE TYIG..HANGUL SYLLABLE TYIH -D2F1..D30B ; LVT # Lo [27] HANGUL SYLLABLE TIG..HANGUL SYLLABLE TIH -D30D..D327 ; LVT # Lo [27] HANGUL SYLLABLE PAG..HANGUL SYLLABLE PAH -D329..D343 ; LVT # Lo [27] HANGUL SYLLABLE PAEG..HANGUL SYLLABLE PAEH -D345..D35F ; LVT # Lo [27] HANGUL SYLLABLE PYAG..HANGUL SYLLABLE PYAH -D361..D37B ; LVT # Lo [27] HANGUL SYLLABLE PYAEG..HANGUL SYLLABLE PYAEH -D37D..D397 ; LVT # Lo [27] HANGUL SYLLABLE PEOG..HANGUL SYLLABLE PEOH -D399..D3B3 ; LVT # Lo [27] HANGUL SYLLABLE PEG..HANGUL SYLLABLE PEH -D3B5..D3CF ; LVT # Lo [27] HANGUL SYLLABLE PYEOG..HANGUL SYLLABLE PYEOH -D3D1..D3EB ; LVT # Lo [27] HANGUL SYLLABLE PYEG..HANGUL SYLLABLE PYEH -D3ED..D407 ; LVT # Lo [27] HANGUL SYLLABLE POG..HANGUL SYLLABLE POH -D409..D423 ; LVT # Lo [27] HANGUL SYLLABLE PWAG..HANGUL SYLLABLE PWAH -D425..D43F ; LVT # Lo [27] HANGUL SYLLABLE PWAEG..HANGUL SYLLABLE PWAEH -D441..D45B ; LVT # Lo [27] HANGUL SYLLABLE POEG..HANGUL SYLLABLE POEH -D45D..D477 ; LVT # Lo [27] HANGUL SYLLABLE PYOG..HANGUL SYLLABLE PYOH -D479..D493 ; LVT # Lo [27] HANGUL SYLLABLE PUG..HANGUL SYLLABLE PUH -D495..D4AF ; LVT # Lo [27] HANGUL SYLLABLE PWEOG..HANGUL SYLLABLE PWEOH -D4B1..D4CB ; LVT # Lo [27] HANGUL SYLLABLE PWEG..HANGUL SYLLABLE PWEH -D4CD..D4E7 ; LVT # Lo [27] HANGUL SYLLABLE PWIG..HANGUL SYLLABLE PWIH -D4E9..D503 ; LVT # Lo [27] HANGUL SYLLABLE PYUG..HANGUL SYLLABLE PYUH -D505..D51F ; LVT # Lo [27] HANGUL SYLLABLE PEUG..HANGUL SYLLABLE PEUH -D521..D53B ; LVT # Lo [27] HANGUL SYLLABLE PYIG..HANGUL SYLLABLE PYIH -D53D..D557 ; LVT # Lo [27] HANGUL SYLLABLE PIG..HANGUL SYLLABLE PIH -D559..D573 ; LVT # Lo [27] HANGUL SYLLABLE HAG..HANGUL SYLLABLE HAH -D575..D58F ; LVT # Lo [27] HANGUL SYLLABLE HAEG..HANGUL SYLLABLE HAEH -D591..D5AB ; LVT # Lo [27] HANGUL SYLLABLE HYAG..HANGUL SYLLABLE HYAH -D5AD..D5C7 ; LVT # Lo [27] HANGUL SYLLABLE HYAEG..HANGUL SYLLABLE HYAEH -D5C9..D5E3 ; LVT # Lo [27] HANGUL SYLLABLE HEOG..HANGUL SYLLABLE HEOH -D5E5..D5FF ; LVT # Lo [27] HANGUL SYLLABLE HEG..HANGUL SYLLABLE HEH -D601..D61B ; LVT # Lo [27] HANGUL SYLLABLE HYEOG..HANGUL SYLLABLE HYEOH -D61D..D637 ; LVT # Lo [27] HANGUL SYLLABLE HYEG..HANGUL SYLLABLE HYEH -D639..D653 ; LVT # Lo [27] HANGUL SYLLABLE HOG..HANGUL SYLLABLE HOH -D655..D66F ; LVT # Lo [27] HANGUL SYLLABLE HWAG..HANGUL SYLLABLE HWAH -D671..D68B ; LVT # Lo [27] HANGUL SYLLABLE HWAEG..HANGUL SYLLABLE HWAEH -D68D..D6A7 ; LVT # Lo [27] HANGUL SYLLABLE HOEG..HANGUL SYLLABLE HOEH -D6A9..D6C3 ; LVT # Lo [27] HANGUL SYLLABLE HYOG..HANGUL SYLLABLE HYOH -D6C5..D6DF ; LVT # Lo [27] HANGUL SYLLABLE HUG..HANGUL SYLLABLE HUH -D6E1..D6FB ; LVT # Lo [27] HANGUL SYLLABLE HWEOG..HANGUL SYLLABLE HWEOH -D6FD..D717 ; LVT # Lo [27] HANGUL SYLLABLE HWEG..HANGUL SYLLABLE HWEH -D719..D733 ; LVT # Lo [27] HANGUL SYLLABLE HWIG..HANGUL SYLLABLE HWIH -D735..D74F ; LVT # Lo [27] HANGUL SYLLABLE HYUG..HANGUL SYLLABLE HYUH -D751..D76B ; LVT # Lo [27] HANGUL SYLLABLE HEUG..HANGUL SYLLABLE HEUH -D76D..D787 ; LVT # Lo [27] HANGUL SYLLABLE HYIG..HANGUL SYLLABLE HYIH -D789..D7A3 ; LVT # Lo [27] HANGUL SYLLABLE HIG..HANGUL SYLLABLE HIH diff --git a/lib/elixir/unicode/IdentifierType.txt b/lib/elixir/unicode/IdentifierType.txt new file mode 100644 index 00000000000..af06f3badc8 --- /dev/null +++ b/lib/elixir/unicode/IdentifierType.txt @@ -0,0 +1,2456 @@ +# IdentifierType.txt +# Date: 2021-08-12, 01:13:33 GMT +# © 2021 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Unicode Security Mechanisms for UTS #39 +# Version: 14.0.0 +# +# For documentation and usage, see http://www.unicode.org/reports/tr39 +# +# Format +# +# Field 0: code point +# Field 1: set of Identifier_Type values (see Table 1 of http://www.unicode.org/reports/tr39) +# +# Any missing code points have the Identifier_Type value Not_Character +# +# For the purpose of regular expressions, the property Identifier_Type is defined as +# mapping each code point to a set of enumerated values. +# The short name of Identifier_Type is the same as the long name. +# The possible values are: +# Not_Character, Deprecated, Default_Ignorable, Not_NFKC, Not_XID, +# Exclusion, Obsolete, Technical, Uncommon_Use, Limited_Use, Inclusion, Recommended +# The short name of each value is the same as its long name. +# The default property value for all Unicode code points U+0000..U+10FFFF +# not mentioned in this data file is Not_Character. +# As usual, sets are unordered, with no duplicate values. + + +# Identifier_Type: Recommended + +0030..0039 ; Recommended # 1.1 [10] DIGIT ZERO..DIGIT NINE +0041..005A ; Recommended # 1.1 [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +005F ; Recommended # 1.1 LOW LINE +0061..007A ; Recommended # 1.1 [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00C0..00D6 ; Recommended # 1.1 [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; Recommended # 1.1 [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..0131 ; Recommended # 1.1 [58] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER DOTLESS I +0134..013E ; Recommended # 1.1 [11] LATIN CAPITAL LETTER J WITH CIRCUMFLEX..LATIN SMALL LETTER L WITH CARON +0141..0148 ; Recommended # 1.1 [8] LATIN CAPITAL LETTER L WITH STROKE..LATIN SMALL LETTER N WITH CARON +014A..017E ; Recommended # 1.1 [53] LATIN CAPITAL LETTER ENG..LATIN SMALL LETTER Z WITH CARON +018F ; Recommended # 1.1 LATIN CAPITAL LETTER SCHWA +01A0..01A1 ; Recommended # 1.1 [2] LATIN CAPITAL LETTER O WITH HORN..LATIN SMALL LETTER O WITH HORN +01AF..01B0 ; Recommended # 1.1 [2] LATIN CAPITAL LETTER U WITH HORN..LATIN SMALL LETTER U WITH HORN +01CD..01DC ; Recommended # 1.1 [16] LATIN CAPITAL LETTER A WITH CARON..LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE +01DE..01E3 ; Recommended # 1.1 [6] LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON..LATIN SMALL LETTER AE WITH MACRON +01E6..01F0 ; Recommended # 1.1 [11] LATIN CAPITAL LETTER G WITH CARON..LATIN SMALL LETTER J WITH CARON +01F4..01F5 ; Recommended # 1.1 [2] LATIN CAPITAL LETTER G WITH ACUTE..LATIN SMALL LETTER G WITH ACUTE +01F8..01F9 ; Recommended # 3.0 [2] LATIN CAPITAL LETTER N WITH GRAVE..LATIN SMALL LETTER N WITH GRAVE +01FA..0217 ; Recommended # 1.1 [30] LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE..LATIN SMALL LETTER U WITH INVERTED BREVE +0218..021B ; Recommended # 3.0 [4] LATIN CAPITAL LETTER S WITH COMMA BELOW..LATIN SMALL LETTER T WITH COMMA BELOW +021E..021F ; Recommended # 3.0 [2] LATIN CAPITAL LETTER H WITH CARON..LATIN SMALL LETTER H WITH CARON +0226..0233 ; Recommended # 3.0 [14] LATIN CAPITAL LETTER A WITH DOT ABOVE..LATIN SMALL LETTER Y WITH MACRON +0259 ; Recommended # 1.1 LATIN SMALL LETTER SCHWA +02BB..02BC ; Recommended # 1.1 [2] MODIFIER LETTER TURNED COMMA..MODIFIER LETTER APOSTROPHE +02EC ; Recommended # 3.0 MODIFIER LETTER VOICING +0300..0304 ; Recommended # 1.1 [5] COMBINING GRAVE ACCENT..COMBINING MACRON +0306..030C ; Recommended # 1.1 [7] COMBINING BREVE..COMBINING CARON +030F..0311 ; Recommended # 1.1 [3] COMBINING DOUBLE GRAVE ACCENT..COMBINING INVERTED BREVE +0313..0314 ; Recommended # 1.1 [2] COMBINING COMMA ABOVE..COMBINING REVERSED COMMA ABOVE +031B ; Recommended # 1.1 COMBINING HORN +0323..0328 ; Recommended # 1.1 [6] COMBINING DOT BELOW..COMBINING OGONEK +032D..032E ; Recommended # 1.1 [2] COMBINING CIRCUMFLEX ACCENT BELOW..COMBINING BREVE BELOW +0330..0331 ; Recommended # 1.1 [2] COMBINING TILDE BELOW..COMBINING MACRON BELOW +0335 ; Recommended # 1.1 COMBINING SHORT STROKE OVERLAY +0338..0339 ; Recommended # 1.1 [2] COMBINING LONG SOLIDUS OVERLAY..COMBINING RIGHT HALF RING BELOW +0342 ; Recommended # 1.1 COMBINING GREEK PERISPOMENI +0345 ; Recommended # 1.1 COMBINING GREEK YPOGEGRAMMENI +037B..037D ; Recommended # 5.0 [3] GREEK SMALL REVERSED LUNATE SIGMA SYMBOL..GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL +0386 ; Recommended # 1.1 GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Recommended # 1.1 [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Recommended # 1.1 GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; Recommended # 1.1 [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03CE ; Recommended # 1.1 [44] GREEK CAPITAL LETTER SIGMA..GREEK SMALL LETTER OMEGA WITH TONOS +03FC..03FF ; Recommended # 4.1 [4] GREEK RHO WITH STROKE SYMBOL..GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL +0400 ; Recommended # 3.0 CYRILLIC CAPITAL LETTER IE WITH GRAVE +0401..040C ; Recommended # 1.1 [12] CYRILLIC CAPITAL LETTER IO..CYRILLIC CAPITAL LETTER KJE +040D ; Recommended # 3.0 CYRILLIC CAPITAL LETTER I WITH GRAVE +040E..044F ; Recommended # 1.1 [66] CYRILLIC CAPITAL LETTER SHORT U..CYRILLIC SMALL LETTER YA +0450 ; Recommended # 3.0 CYRILLIC SMALL LETTER IE WITH GRAVE +0451..045C ; Recommended # 1.1 [12] CYRILLIC SMALL LETTER IO..CYRILLIC SMALL LETTER KJE +045D ; Recommended # 3.0 CYRILLIC SMALL LETTER I WITH GRAVE +045E..045F ; Recommended # 1.1 [2] CYRILLIC SMALL LETTER SHORT U..CYRILLIC SMALL LETTER DZHE +048A..048B ; Recommended # 3.2 [2] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER SHORT I WITH TAIL +048C..048F ; Recommended # 3.0 [4] CYRILLIC CAPITAL LETTER SEMISOFT SIGN..CYRILLIC SMALL LETTER ER WITH TICK +0490..04C4 ; Recommended # 1.1 [53] CYRILLIC CAPITAL LETTER GHE WITH UPTURN..CYRILLIC SMALL LETTER KA WITH HOOK +04C5..04C6 ; Recommended # 3.2 [2] CYRILLIC CAPITAL LETTER EL WITH TAIL..CYRILLIC SMALL LETTER EL WITH TAIL +04C7..04C8 ; Recommended # 1.1 [2] CYRILLIC CAPITAL LETTER EN WITH HOOK..CYRILLIC SMALL LETTER EN WITH HOOK +04C9..04CA ; Recommended # 3.2 [2] CYRILLIC CAPITAL LETTER EN WITH TAIL..CYRILLIC SMALL LETTER EN WITH TAIL +04CB..04CC ; Recommended # 1.1 [2] CYRILLIC CAPITAL LETTER KHAKASSIAN CHE..CYRILLIC SMALL LETTER KHAKASSIAN CHE +04CD..04CE ; Recommended # 3.2 [2] CYRILLIC CAPITAL LETTER EM WITH TAIL..CYRILLIC SMALL LETTER EM WITH TAIL +04CF ; Recommended # 5.0 CYRILLIC SMALL LETTER PALOCHKA +04D0..04EB ; Recommended # 1.1 [28] CYRILLIC CAPITAL LETTER A WITH BREVE..CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS +04EC..04ED ; Recommended # 3.0 [2] CYRILLIC CAPITAL LETTER E WITH DIAERESIS..CYRILLIC SMALL LETTER E WITH DIAERESIS +04EE..04F5 ; Recommended # 1.1 [8] CYRILLIC CAPITAL LETTER U WITH MACRON..CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F6..04F7 ; Recommended # 4.1 [2] CYRILLIC CAPITAL LETTER GHE WITH DESCENDER..CYRILLIC SMALL LETTER GHE WITH DESCENDER +04F8..04F9 ; Recommended # 1.1 [2] CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS..CYRILLIC SMALL LETTER YERU WITH DIAERESIS +04FA..04FF ; Recommended # 5.0 [6] CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK..CYRILLIC SMALL LETTER HA WITH STROKE +0510..0513 ; Recommended # 5.0 [4] CYRILLIC CAPITAL LETTER REVERSED ZE..CYRILLIC SMALL LETTER EL WITH HOOK +0514..0523 ; Recommended # 5.1 [16] CYRILLIC CAPITAL LETTER LHA..CYRILLIC SMALL LETTER EN WITH MIDDLE HOOK +0524..0525 ; Recommended # 5.2 [2] CYRILLIC CAPITAL LETTER PE WITH DESCENDER..CYRILLIC SMALL LETTER PE WITH DESCENDER +0526..0527 ; Recommended # 6.0 [2] CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER..CYRILLIC SMALL LETTER SHHA WITH DESCENDER +0528..0529 ; Recommended # 7.0 [2] CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK..CYRILLIC SMALL LETTER EN WITH LEFT HOOK +052E..052F ; Recommended # 7.0 [2] CYRILLIC CAPITAL LETTER EL WITH DESCENDER..CYRILLIC SMALL LETTER EL WITH DESCENDER +0531..0556 ; Recommended # 1.1 [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; Recommended # 1.1 ARMENIAN MODIFIER LETTER LEFT HALF RING +0561..0586 ; Recommended # 1.1 [38] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LETTER FEH +05B4 ; Recommended # 1.1 HEBREW POINT HIRIQ +05D0..05EA ; Recommended # 1.1 [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05EF ; Recommended # 11.0 HEBREW YOD TRIANGLE +05F0..05F2 ; Recommended # 1.1 [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0620 ; Recommended # 6.0 ARABIC LETTER KASHMIRI YEH +0621..063A ; Recommended # 1.1 [26] ARABIC LETTER HAMZA..ARABIC LETTER GHAIN +063B..063F ; Recommended # 5.1 [5] ARABIC LETTER KEHEH WITH TWO DOTS ABOVE..ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE +0641..0652 ; Recommended # 1.1 [18] ARABIC LETTER FEH..ARABIC SUKUN +0653..0655 ; Recommended # 3.0 [3] ARABIC MADDAH ABOVE..ARABIC HAMZA BELOW +0660..0669 ; Recommended # 1.1 [10] ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE +0670..0672 ; Recommended # 1.1 [3] ARABIC LETTER SUPERSCRIPT ALEF..ARABIC LETTER ALEF WITH WAVY HAMZA ABOVE +0674 ; Recommended # 1.1 ARABIC LETTER HIGH HAMZA +0679..068D ; Recommended # 1.1 [21] ARABIC LETTER TTEH..ARABIC LETTER DDAHAL +068F..06A0 ; Recommended # 1.1 [18] ARABIC LETTER DAL WITH THREE DOTS ABOVE DOWNWARDS..ARABIC LETTER AIN WITH THREE DOTS ABOVE +06A2..06B7 ; Recommended # 1.1 [22] ARABIC LETTER FEH WITH DOT MOVED BELOW..ARABIC LETTER LAM WITH THREE DOTS ABOVE +06B8..06B9 ; Recommended # 3.0 [2] ARABIC LETTER LAM WITH THREE DOTS BELOW..ARABIC LETTER NOON WITH DOT BELOW +06BA..06BE ; Recommended # 1.1 [5] ARABIC LETTER NOON GHUNNA..ARABIC LETTER HEH DOACHASHMEE +06BF ; Recommended # 3.0 ARABIC LETTER TCHEH WITH DOT ABOVE +06C0..06CE ; Recommended # 1.1 [15] ARABIC LETTER HEH WITH YEH ABOVE..ARABIC LETTER YEH WITH SMALL V +06CF ; Recommended # 3.0 ARABIC LETTER WAW WITH DOT ABOVE +06D0..06D3 ; Recommended # 1.1 [4] ARABIC LETTER E..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; Recommended # 1.1 ARABIC LETTER AE +06E5..06E6 ; Recommended # 1.1 [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06EE..06EF ; Recommended # 4.0 [2] ARABIC LETTER DAL WITH INVERTED V..ARABIC LETTER REH WITH INVERTED V +06F0..06F9 ; Recommended # 1.1 [10] EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE +06FA..06FC ; Recommended # 3.0 [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +06FF ; Recommended # 4.0 ARABIC LETTER HEH WITH INVERTED V +0750..076D ; Recommended # 4.1 [30] ARABIC LETTER BEH WITH THREE DOTS HORIZONTALLY BELOW..ARABIC LETTER SEEN WITH TWO DOTS VERTICALLY ABOVE +076E..077F ; Recommended # 5.1 [18] ARABIC LETTER HAH WITH SMALL ARABIC LETTER TAH BELOW..ARABIC LETTER KAF WITH TWO DOTS ABOVE +0780..07B0 ; Recommended # 3.0 [49] THAANA LETTER HAA..THAANA SUKUN +07B1 ; Recommended # 3.2 THAANA LETTER NAA +0870..0887 ; Recommended # 14.0 [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT +0889..088E ; Recommended # 14.0 [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL +08A0 ; Recommended # 6.1 ARABIC LETTER BEH WITH SMALL V BELOW +08A1 ; Recommended # 7.0 ARABIC LETTER BEH WITH HAMZA ABOVE +08A2..08AC ; Recommended # 6.1 [11] ARABIC LETTER JEEM WITH TWO DOTS ABOVE..ARABIC LETTER ROHINGYA YEH +08B2 ; Recommended # 7.0 ARABIC LETTER ZAIN WITH INVERTED V ABOVE +08B5 ; Recommended # 14.0 ARABIC LETTER QAF WITH DOT BELOW AND NO DOTS ABOVE +08B6..08BD ; Recommended # 9.0 [8] ARABIC LETTER BEH WITH SMALL MEEM ABOVE..ARABIC LETTER AFRICAN NOON +08BE..08C7 ; Recommended # 13.0 [10] ARABIC LETTER PEH WITH SMALL V..ARABIC LETTER LAM WITH SMALL ARABIC LETTER TAH ABOVE +08C8..08C9 ; Recommended # 14.0 [2] ARABIC LETTER GRAF..ARABIC SMALL FARSI YEH +0901..0903 ; Recommended # 1.1 [3] DEVANAGARI SIGN CANDRABINDU..DEVANAGARI SIGN VISARGA +0904 ; Recommended # 4.0 DEVANAGARI LETTER SHORT A +0905..0939 ; Recommended # 1.1 [53] DEVANAGARI LETTER A..DEVANAGARI LETTER HA +093A..093B ; Recommended # 6.0 [2] DEVANAGARI VOWEL SIGN OE..DEVANAGARI VOWEL SIGN OOE +093C..094D ; Recommended # 1.1 [18] DEVANAGARI SIGN NUKTA..DEVANAGARI SIGN VIRAMA +094F ; Recommended # 6.0 DEVANAGARI VOWEL SIGN AW +0950 ; Recommended # 1.1 DEVANAGARI OM +0956..0957 ; Recommended # 6.0 [2] DEVANAGARI VOWEL SIGN UE..DEVANAGARI VOWEL SIGN UUE +0960..0963 ; Recommended # 1.1 [4] DEVANAGARI LETTER VOCALIC RR..DEVANAGARI VOWEL SIGN VOCALIC LL +0966..096F ; Recommended # 1.1 [10] DEVANAGARI DIGIT ZERO..DEVANAGARI DIGIT NINE +0971..0972 ; Recommended # 5.1 [2] DEVANAGARI SIGN HIGH SPACING DOT..DEVANAGARI LETTER CANDRA A +0973..0977 ; Recommended # 6.0 [5] DEVANAGARI LETTER OE..DEVANAGARI LETTER UUE +0979..097A ; Recommended # 5.2 [2] DEVANAGARI LETTER ZHA..DEVANAGARI LETTER HEAVY YA +097B..097C ; Recommended # 5.0 [2] DEVANAGARI LETTER GGA..DEVANAGARI LETTER JJA +097D ; Recommended # 4.1 DEVANAGARI LETTER GLOTTAL STOP +097E..097F ; Recommended # 5.0 [2] DEVANAGARI LETTER DDDA..DEVANAGARI LETTER BBA +0981..0983 ; Recommended # 1.1 [3] BENGALI SIGN CANDRABINDU..BENGALI SIGN VISARGA +0985..098C ; Recommended # 1.1 [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; Recommended # 1.1 [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; Recommended # 1.1 [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; Recommended # 1.1 [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; Recommended # 1.1 BENGALI LETTER LA +09B6..09B9 ; Recommended # 1.1 [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BC ; Recommended # 1.1 BENGALI SIGN NUKTA +09BD ; Recommended # 4.0 BENGALI SIGN AVAGRAHA +09BE..09C4 ; Recommended # 1.1 [7] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8 ; Recommended # 1.1 [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CD ; Recommended # 1.1 [3] BENGALI VOWEL SIGN O..BENGALI SIGN VIRAMA +09CE ; Recommended # 4.1 BENGALI LETTER KHANDA TA +09D7 ; Recommended # 1.1 BENGALI AU LENGTH MARK +09E0..09E3 ; Recommended # 1.1 [4] BENGALI LETTER VOCALIC RR..BENGALI VOWEL SIGN VOCALIC LL +09E6..09F1 ; Recommended # 1.1 [12] BENGALI DIGIT ZERO..BENGALI LETTER RA WITH LOWER DIAGONAL +09FE ; Recommended # 11.0 BENGALI SANDHI MARK +0A01 ; Recommended # 4.0 GURMUKHI SIGN ADAK BINDI +0A02 ; Recommended # 1.1 GURMUKHI SIGN BINDI +0A03 ; Recommended # 4.0 GURMUKHI SIGN VISARGA +0A05..0A0A ; Recommended # 1.1 [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; Recommended # 1.1 [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; Recommended # 1.1 [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; Recommended # 1.1 [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32 ; Recommended # 1.1 GURMUKHI LETTER LA +0A35 ; Recommended # 1.1 GURMUKHI LETTER VA +0A38..0A39 ; Recommended # 1.1 [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A3C ; Recommended # 1.1 GURMUKHI SIGN NUKTA +0A3E..0A42 ; Recommended # 1.1 [5] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; Recommended # 1.1 [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4D ; Recommended # 1.1 [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA +0A5C ; Recommended # 1.1 GURMUKHI LETTER RRA +0A66..0A74 ; Recommended # 1.1 [15] GURMUKHI DIGIT ZERO..GURMUKHI EK ONKAR +0A81..0A83 ; Recommended # 1.1 [3] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN VISARGA +0A85..0A8B ; Recommended # 1.1 [7] GUJARATI LETTER A..GUJARATI LETTER VOCALIC R +0A8C ; Recommended # 4.0 GUJARATI LETTER VOCALIC L +0A8D ; Recommended # 1.1 GUJARATI VOWEL CANDRA E +0A8F..0A91 ; Recommended # 1.1 [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; Recommended # 1.1 [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; Recommended # 1.1 [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; Recommended # 1.1 [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; Recommended # 1.1 [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABC..0AC5 ; Recommended # 1.1 [10] GUJARATI SIGN NUKTA..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC9 ; Recommended # 1.1 [3] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACD ; Recommended # 1.1 [3] GUJARATI VOWEL SIGN O..GUJARATI SIGN VIRAMA +0AD0 ; Recommended # 1.1 GUJARATI OM +0AE0 ; Recommended # 1.1 GUJARATI LETTER VOCALIC RR +0AE1..0AE3 ; Recommended # 4.0 [3] GUJARATI LETTER VOCALIC LL..GUJARATI VOWEL SIGN VOCALIC LL +0AE6..0AEF ; Recommended # 1.1 [10] GUJARATI DIGIT ZERO..GUJARATI DIGIT NINE +0AFA..0AFF ; Recommended # 10.0 [6] GUJARATI SIGN SUKUN..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE +0B01..0B03 ; Recommended # 1.1 [3] ORIYA SIGN CANDRABINDU..ORIYA SIGN VISARGA +0B05..0B0C ; Recommended # 1.1 [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; Recommended # 1.1 [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; Recommended # 1.1 [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; Recommended # 1.1 [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; Recommended # 1.1 [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B35 ; Recommended # 4.0 ORIYA LETTER VA +0B36..0B39 ; Recommended # 1.1 [4] ORIYA LETTER SHA..ORIYA LETTER HA +0B3C..0B43 ; Recommended # 1.1 [8] ORIYA SIGN NUKTA..ORIYA VOWEL SIGN VOCALIC R +0B47..0B48 ; Recommended # 1.1 [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4D ; Recommended # 1.1 [3] ORIYA VOWEL SIGN O..ORIYA SIGN VIRAMA +0B55 ; Recommended # 13.0 ORIYA SIGN OVERLINE +0B56..0B57 ; Recommended # 1.1 [2] ORIYA AI LENGTH MARK..ORIYA AU LENGTH MARK +0B5F..0B61 ; Recommended # 1.1 [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B66..0B6F ; Recommended # 1.1 [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE +0B71 ; Recommended # 4.0 ORIYA LETTER WA +0B82..0B83 ; Recommended # 1.1 [2] TAMIL SIGN ANUSVARA..TAMIL SIGN VISARGA +0B85..0B8A ; Recommended # 1.1 [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; Recommended # 1.1 [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; Recommended # 1.1 [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; Recommended # 1.1 [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; Recommended # 1.1 TAMIL LETTER JA +0B9E..0B9F ; Recommended # 1.1 [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; Recommended # 1.1 [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; Recommended # 1.1 [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB5 ; Recommended # 1.1 [8] TAMIL LETTER MA..TAMIL LETTER VA +0BB6 ; Recommended # 4.1 TAMIL LETTER SHA +0BB7..0BB9 ; Recommended # 1.1 [3] TAMIL LETTER SSA..TAMIL LETTER HA +0BBE..0BC2 ; Recommended # 1.1 [5] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN UU +0BC6..0BC8 ; Recommended # 1.1 [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCD ; Recommended # 1.1 [4] TAMIL VOWEL SIGN O..TAMIL SIGN VIRAMA +0BD0 ; Recommended # 5.1 TAMIL OM +0BD7 ; Recommended # 1.1 TAMIL AU LENGTH MARK +0BE6 ; Recommended # 4.1 TAMIL DIGIT ZERO +0BE7..0BEF ; Recommended # 1.1 [9] TAMIL DIGIT ONE..TAMIL DIGIT NINE +0C01..0C03 ; Recommended # 1.1 [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C04 ; Recommended # 11.0 TELUGU SIGN COMBINING ANUSVARA ABOVE +0C05..0C0C ; Recommended # 1.1 [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10 ; Recommended # 1.1 [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; Recommended # 1.1 [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C33 ; Recommended # 1.1 [10] TELUGU LETTER PA..TELUGU LETTER LLA +0C35..0C39 ; Recommended # 1.1 [5] TELUGU LETTER VA..TELUGU LETTER HA +0C3C ; Recommended # 14.0 TELUGU SIGN NUKTA +0C3D ; Recommended # 5.1 TELUGU SIGN AVAGRAHA +0C3E..0C44 ; Recommended # 1.1 [7] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN VOCALIC RR +0C46..0C48 ; Recommended # 1.1 [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4D ; Recommended # 1.1 [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA +0C55..0C56 ; Recommended # 1.1 [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C5D ; Recommended # 14.0 TELUGU LETTER NAKAARA POLLU +0C60..0C61 ; Recommended # 1.1 [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C66..0C6F ; Recommended # 1.1 [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE +0C80 ; Recommended # 9.0 KANNADA SIGN SPACING CANDRABINDU +0C82..0C83 ; Recommended # 1.1 [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0C85..0C8C ; Recommended # 1.1 [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90 ; Recommended # 1.1 [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; Recommended # 1.1 [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3 ; Recommended # 1.1 [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9 ; Recommended # 1.1 [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBC..0CBD ; Recommended # 4.0 [2] KANNADA SIGN NUKTA..KANNADA SIGN AVAGRAHA +0CBE..0CC4 ; Recommended # 1.1 [7] KANNADA VOWEL SIGN AA..KANNADA VOWEL SIGN VOCALIC RR +0CC6..0CC8 ; Recommended # 1.1 [3] KANNADA VOWEL SIGN E..KANNADA VOWEL SIGN AI +0CCA..0CCD ; Recommended # 1.1 [4] KANNADA VOWEL SIGN O..KANNADA SIGN VIRAMA +0CD5..0CD6 ; Recommended # 1.1 [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CDD ; Recommended # 14.0 KANNADA LETTER NAKAARA POLLU +0CE0..0CE1 ; Recommended # 1.1 [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0CE2..0CE3 ; Recommended # 5.0 [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL +0CE6..0CEF ; Recommended # 1.1 [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE +0CF1..0CF2 ; Recommended # 5.0 [2] KANNADA SIGN JIHVAMULIYA..KANNADA SIGN UPADHMANIYA +0D00 ; Recommended # 10.0 MALAYALAM SIGN COMBINING ANUSVARA ABOVE +0D02..0D03 ; Recommended # 1.1 [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D05..0D0C ; Recommended # 1.1 [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L +0D0E..0D10 ; Recommended # 1.1 [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D28 ; Recommended # 1.1 [23] MALAYALAM LETTER O..MALAYALAM LETTER NA +0D29 ; Recommended # 6.0 MALAYALAM LETTER NNNA +0D2A..0D39 ; Recommended # 1.1 [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA +0D3A ; Recommended # 6.0 MALAYALAM LETTER TTTA +0D3D ; Recommended # 5.1 MALAYALAM SIGN AVAGRAHA +0D3E..0D43 ; Recommended # 1.1 [6] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN VOCALIC R +0D46..0D48 ; Recommended # 1.1 [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4D ; Recommended # 1.1 [4] MALAYALAM VOWEL SIGN O..MALAYALAM SIGN VIRAMA +0D4E ; Recommended # 6.0 MALAYALAM LETTER DOT REPH +0D54..0D56 ; Recommended # 9.0 [3] MALAYALAM LETTER CHILLU M..MALAYALAM LETTER CHILLU LLL +0D57 ; Recommended # 1.1 MALAYALAM AU LENGTH MARK +0D60..0D61 ; Recommended # 1.1 [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL +0D66..0D6F ; Recommended # 1.1 [10] MALAYALAM DIGIT ZERO..MALAYALAM DIGIT NINE +0D7A..0D7F ; Recommended # 5.1 [6] MALAYALAM LETTER CHILLU NN..MALAYALAM LETTER CHILLU K +0D82..0D83 ; Recommended # 3.0 [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0D85..0D8E ; Recommended # 3.0 [10] SINHALA LETTER AYANNA..SINHALA LETTER IRUUYANNA +0D91..0D96 ; Recommended # 3.0 [6] SINHALA LETTER EYANNA..SINHALA LETTER AUYANNA +0D9A..0DA5 ; Recommended # 3.0 [12] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER TAALUJA SANYOOGA NAAKSIKYAYA +0DA7..0DB1 ; Recommended # 3.0 [11] SINHALA LETTER ALPAPRAANA TTAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB ; Recommended # 3.0 [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD ; Recommended # 3.0 SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6 ; Recommended # 3.0 [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0DCA ; Recommended # 3.0 SINHALA SIGN AL-LAKUNA +0DCF..0DD4 ; Recommended # 3.0 [6] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6 ; Recommended # 3.0 SINHALA VOWEL SIGN DIGA PAA-PILLA +0DD8..0DDE ; Recommended # 3.0 [7] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA +0DF2 ; Recommended # 3.0 SINHALA VOWEL SIGN DIGA GAETTA-PILLA +0E01..0E32 ; Recommended # 1.1 [50] THAI CHARACTER KO KAI..THAI CHARACTER SARA AA +0E34..0E3A ; Recommended # 1.1 [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E40..0E4E ; Recommended # 1.1 [15] THAI CHARACTER SARA E..THAI CHARACTER YAMAKKAN +0E50..0E59 ; Recommended # 1.1 [10] THAI DIGIT ZERO..THAI DIGIT NINE +0E81..0E82 ; Recommended # 1.1 [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84 ; Recommended # 1.1 LAO LETTER KHO TAM +0E86 ; Recommended # 12.0 LAO LETTER PALI GHA +0E87..0E88 ; Recommended # 1.1 [2] LAO LETTER NGO..LAO LETTER CO +0E89 ; Recommended # 12.0 LAO LETTER PALI CHA +0E8A ; Recommended # 1.1 LAO LETTER SO TAM +0E8C ; Recommended # 12.0 LAO LETTER PALI JHA +0E8D ; Recommended # 1.1 LAO LETTER NYO +0E8E..0E93 ; Recommended # 12.0 [6] LAO LETTER PALI NYA..LAO LETTER PALI NNA +0E94..0E97 ; Recommended # 1.1 [4] LAO LETTER DO..LAO LETTER THO TAM +0E98 ; Recommended # 12.0 LAO LETTER PALI DHA +0E99..0E9F ; Recommended # 1.1 [7] LAO LETTER NO..LAO LETTER FO SUNG +0EA0 ; Recommended # 12.0 LAO LETTER PALI BHA +0EA1..0EA3 ; Recommended # 1.1 [3] LAO LETTER MO..LAO LETTER LO LING +0EA5 ; Recommended # 1.1 LAO LETTER LO LOOT +0EA7 ; Recommended # 1.1 LAO LETTER WO +0EA8..0EA9 ; Recommended # 12.0 [2] LAO LETTER SANSKRIT SHA..LAO LETTER SANSKRIT SSA +0EAA..0EAB ; Recommended # 1.1 [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG +0EAC ; Recommended # 12.0 LAO LETTER PALI LLA +0EAD..0EB2 ; Recommended # 1.1 [6] LAO LETTER O..LAO VOWEL SIGN AA +0EB4..0EB9 ; Recommended # 1.1 [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU +0EBA ; Recommended # 12.0 LAO SIGN PALI VIRAMA +0EBB..0EBD ; Recommended # 1.1 [3] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN NYO +0EC0..0EC4 ; Recommended # 1.1 [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6 ; Recommended # 1.1 LAO KO LA +0EC8..0ECD ; Recommended # 1.1 [6] LAO TONE MAI EK..LAO NIGGAHITA +0ED0..0ED9 ; Recommended # 1.1 [10] LAO DIGIT ZERO..LAO DIGIT NINE +0EDE..0EDF ; Recommended # 6.1 [2] LAO LETTER KHMU GO..LAO LETTER KHMU NYO +0F00 ; Recommended # 2.0 TIBETAN SYLLABLE OM +0F20..0F29 ; Recommended # 2.0 [10] TIBETAN DIGIT ZERO..TIBETAN DIGIT NINE +0F35 ; Recommended # 2.0 TIBETAN MARK NGAS BZUNG NYI ZLA +0F37 ; Recommended # 2.0 TIBETAN MARK NGAS BZUNG SGOR RTAGS +0F3E..0F42 ; Recommended # 2.0 [5] TIBETAN SIGN YAR TSHES..TIBETAN LETTER GA +0F44..0F47 ; Recommended # 2.0 [4] TIBETAN LETTER NGA..TIBETAN LETTER JA +0F49..0F4C ; Recommended # 2.0 [4] TIBETAN LETTER NYA..TIBETAN LETTER DDA +0F4E..0F51 ; Recommended # 2.0 [4] TIBETAN LETTER NNA..TIBETAN LETTER DA +0F53..0F56 ; Recommended # 2.0 [4] TIBETAN LETTER NA..TIBETAN LETTER BA +0F58..0F5B ; Recommended # 2.0 [4] TIBETAN LETTER MA..TIBETAN LETTER DZA +0F5D..0F68 ; Recommended # 2.0 [12] TIBETAN LETTER WA..TIBETAN LETTER A +0F6A ; Recommended # 3.0 TIBETAN LETTER FIXED-FORM RA +0F6B..0F6C ; Recommended # 5.1 [2] TIBETAN LETTER KKA..TIBETAN LETTER RRA +0F71..0F72 ; Recommended # 2.0 [2] TIBETAN VOWEL SIGN AA..TIBETAN VOWEL SIGN I +0F74 ; Recommended # 2.0 TIBETAN VOWEL SIGN U +0F7A..0F80 ; Recommended # 2.0 [7] TIBETAN VOWEL SIGN E..TIBETAN VOWEL SIGN REVERSED I +0F82..0F84 ; Recommended # 2.0 [3] TIBETAN SIGN NYI ZLA NAA DA..TIBETAN MARK HALANTA +0F86..0F8B ; Recommended # 2.0 [6] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN GRU MED RGYINGS +0F8C..0F8F ; Recommended # 6.0 [4] TIBETAN SIGN INVERTED MCHU CAN..TIBETAN SUBJOINED SIGN INVERTED MCHU CAN +0F90..0F92 ; Recommended # 2.0 [3] TIBETAN SUBJOINED LETTER KA..TIBETAN SUBJOINED LETTER GA +0F94..0F95 ; Recommended # 2.0 [2] TIBETAN SUBJOINED LETTER NGA..TIBETAN SUBJOINED LETTER CA +0F96 ; Recommended # 3.0 TIBETAN SUBJOINED LETTER CHA +0F97 ; Recommended # 2.0 TIBETAN SUBJOINED LETTER JA +0F99..0F9C ; Recommended # 2.0 [4] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER DDA +0F9E..0FA1 ; Recommended # 2.0 [4] TIBETAN SUBJOINED LETTER NNA..TIBETAN SUBJOINED LETTER DA +0FA3..0FA6 ; Recommended # 2.0 [4] TIBETAN SUBJOINED LETTER NA..TIBETAN SUBJOINED LETTER BA +0FA8..0FAB ; Recommended # 2.0 [4] TIBETAN SUBJOINED LETTER MA..TIBETAN SUBJOINED LETTER DZA +0FAD ; Recommended # 2.0 TIBETAN SUBJOINED LETTER WA +0FAE..0FB0 ; Recommended # 3.0 [3] TIBETAN SUBJOINED LETTER ZHA..TIBETAN SUBJOINED LETTER -A +0FB1..0FB7 ; Recommended # 2.0 [7] TIBETAN SUBJOINED LETTER YA..TIBETAN SUBJOINED LETTER HA +0FB8 ; Recommended # 3.0 TIBETAN SUBJOINED LETTER A +0FBA..0FBC ; Recommended # 3.0 [3] TIBETAN SUBJOINED LETTER FIXED-FORM WA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +0FC6 ; Recommended # 3.0 TIBETAN SYMBOL PADMA GDAN +1000..1021 ; Recommended # 3.0 [34] MYANMAR LETTER KA..MYANMAR LETTER A +1022 ; Recommended # 5.1 MYANMAR LETTER SHAN A +1023..1027 ; Recommended # 3.0 [5] MYANMAR LETTER I..MYANMAR LETTER E +1028 ; Recommended # 5.1 MYANMAR LETTER MON E +1029..102A ; Recommended # 3.0 [2] MYANMAR LETTER O..MYANMAR LETTER AU +102B ; Recommended # 5.1 MYANMAR VOWEL SIGN TALL AA +102C..1032 ; Recommended # 3.0 [7] MYANMAR VOWEL SIGN AA..MYANMAR VOWEL SIGN AI +1033..1035 ; Recommended # 5.1 [3] MYANMAR VOWEL SIGN MON II..MYANMAR VOWEL SIGN E ABOVE +1036..1039 ; Recommended # 3.0 [4] MYANMAR SIGN ANUSVARA..MYANMAR SIGN VIRAMA +103A..103F ; Recommended # 5.1 [6] MYANMAR SIGN ASAT..MYANMAR LETTER GREAT SA +1040..1049 ; Recommended # 3.0 [10] MYANMAR DIGIT ZERO..MYANMAR DIGIT NINE +1050..1059 ; Recommended # 3.0 [10] MYANMAR LETTER SHA..MYANMAR VOWEL SIGN VOCALIC LL +105A..1099 ; Recommended # 5.1 [64] MYANMAR LETTER MON NGA..MYANMAR SHAN DIGIT NINE +109A..109D ; Recommended # 5.2 [4] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON AI +10C7 ; Recommended # 6.1 GEORGIAN CAPITAL LETTER YN +10CD ; Recommended # 6.1 GEORGIAN CAPITAL LETTER AEN +10D0..10F0 ; Recommended # 1.1 [33] GEORGIAN LETTER AN..GEORGIAN LETTER HAE +10F7..10F8 ; Recommended # 3.2 [2] GEORGIAN LETTER YN..GEORGIAN LETTER ELIFI +10F9..10FA ; Recommended # 4.1 [2] GEORGIAN LETTER TURNED GAN..GEORGIAN LETTER AIN +10FD..10FF ; Recommended # 6.1 [3] GEORGIAN LETTER AEN..GEORGIAN LETTER LABIAL SIGN +1200..1206 ; Recommended # 3.0 [7] ETHIOPIC SYLLABLE HA..ETHIOPIC SYLLABLE HO +1207 ; Recommended # 4.1 ETHIOPIC SYLLABLE HOA +1208..1246 ; Recommended # 3.0 [63] ETHIOPIC SYLLABLE LA..ETHIOPIC SYLLABLE QO +1247 ; Recommended # 4.1 ETHIOPIC SYLLABLE QOA +1248 ; Recommended # 3.0 ETHIOPIC SYLLABLE QWA +124A..124D ; Recommended # 3.0 [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256 ; Recommended # 3.0 [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258 ; Recommended # 3.0 ETHIOPIC SYLLABLE QHWA +125A..125D ; Recommended # 3.0 [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1286 ; Recommended # 3.0 [39] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XO +1287 ; Recommended # 4.1 ETHIOPIC SYLLABLE XOA +1288 ; Recommended # 3.0 ETHIOPIC SYLLABLE XWA +128A..128D ; Recommended # 3.0 [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12AE ; Recommended # 3.0 [31] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KO +12AF ; Recommended # 4.1 ETHIOPIC SYLLABLE KOA +12B0 ; Recommended # 3.0 ETHIOPIC SYLLABLE KWA +12B2..12B5 ; Recommended # 3.0 [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE ; Recommended # 3.0 [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0 ; Recommended # 3.0 ETHIOPIC SYLLABLE KXWA +12C2..12C5 ; Recommended # 3.0 [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12CE ; Recommended # 3.0 [7] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE WO +12CF ; Recommended # 4.1 ETHIOPIC SYLLABLE WOA +12D0..12D6 ; Recommended # 3.0 [7] ETHIOPIC SYLLABLE PHARYNGEAL A..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..12EE ; Recommended # 3.0 [23] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE YO +12EF ; Recommended # 4.1 ETHIOPIC SYLLABLE YOA +12F0..130E ; Recommended # 3.0 [31] ETHIOPIC SYLLABLE DA..ETHIOPIC SYLLABLE GO +130F ; Recommended # 4.1 ETHIOPIC SYLLABLE GOA +1310 ; Recommended # 3.0 ETHIOPIC SYLLABLE GWA +1312..1315 ; Recommended # 3.0 [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..131E ; Recommended # 3.0 [7] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE GGO +131F ; Recommended # 4.1 ETHIOPIC SYLLABLE GGWAA +1320..1346 ; Recommended # 3.0 [39] ETHIOPIC SYLLABLE THA..ETHIOPIC SYLLABLE TZO +1347 ; Recommended # 4.1 ETHIOPIC SYLLABLE TZOA +1348..135A ; Recommended # 3.0 [19] ETHIOPIC SYLLABLE FA..ETHIOPIC SYLLABLE FYA +135D..135E ; Recommended # 6.0 [2] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING VOWEL LENGTH MARK +135F ; Recommended # 4.1 ETHIOPIC COMBINING GEMINATION MARK +1380..138F ; Recommended # 4.1 [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..ETHIOPIC SYLLABLE PWE +1780..17A2 ; Recommended # 3.0 [35] KHMER LETTER KA..KHMER LETTER QA +17A5..17A7 ; Recommended # 3.0 [3] KHMER INDEPENDENT VOWEL QI..KHMER INDEPENDENT VOWEL QU +17A9..17B3 ; Recommended # 3.0 [11] KHMER INDEPENDENT VOWEL QUU..KHMER INDEPENDENT VOWEL QAU +17B6..17CD ; Recommended # 3.0 [24] KHMER VOWEL SIGN AA..KHMER SIGN TOANDAKHIAT +17D0 ; Recommended # 3.0 KHMER SIGN SAMYOK SANNYA +17D2 ; Recommended # 3.0 KHMER SIGN COENG +17D7 ; Recommended # 3.0 KHMER SIGN LEK TOO +17DC ; Recommended # 3.0 KHMER SIGN AVAKRAHASANYA +17E0..17E9 ; Recommended # 3.0 [10] KHMER DIGIT ZERO..KHMER DIGIT NINE +1C90..1CBA ; Recommended # 11.0 [43] GEORGIAN MTAVRULI CAPITAL LETTER AN..GEORGIAN MTAVRULI CAPITAL LETTER AIN +1CBD..1CBF ; Recommended # 11.0 [3] GEORGIAN MTAVRULI CAPITAL LETTER AEN..GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN +1E00..1E99 ; Recommended # 1.1 [154] LATIN CAPITAL LETTER A WITH RING BELOW..LATIN SMALL LETTER Y WITH RING ABOVE +1E9E ; Recommended # 5.1 LATIN CAPITAL LETTER SHARP S +1EA0..1EF9 ; Recommended # 1.1 [90] LATIN CAPITAL LETTER A WITH DOT BELOW..LATIN SMALL LETTER Y WITH TILDE +1F00..1F15 ; Recommended # 1.1 [22] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; Recommended # 1.1 [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; Recommended # 1.1 [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; Recommended # 1.1 [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Recommended # 1.1 [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; Recommended # 1.1 GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Recommended # 1.1 GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Recommended # 1.1 GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F70 ; Recommended # 1.1 [18] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER ALPHA WITH VARIA +1F72 ; Recommended # 1.1 GREEK SMALL LETTER EPSILON WITH VARIA +1F74 ; Recommended # 1.1 GREEK SMALL LETTER ETA WITH VARIA +1F76 ; Recommended # 1.1 GREEK SMALL LETTER IOTA WITH VARIA +1F78 ; Recommended # 1.1 GREEK SMALL LETTER OMICRON WITH VARIA +1F7A ; Recommended # 1.1 GREEK SMALL LETTER UPSILON WITH VARIA +1F7C ; Recommended # 1.1 GREEK SMALL LETTER OMEGA WITH VARIA +1F80..1FB4 ; Recommended # 1.1 [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBA ; Recommended # 1.1 [5] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH VARIA +1FBC ; Recommended # 1.1 GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FC2..1FC4 ; Recommended # 1.1 [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FC8 ; Recommended # 1.1 [3] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER EPSILON WITH VARIA +1FCA ; Recommended # 1.1 GREEK CAPITAL LETTER ETA WITH VARIA +1FCC ; Recommended # 1.1 GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD2 ; Recommended # 1.1 [3] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA +1FD6..1FDA ; Recommended # 1.1 [5] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH VARIA +1FE0..1FE2 ; Recommended # 1.1 [3] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA +1FE4..1FEA ; Recommended # 1.1 [7] GREEK SMALL LETTER RHO WITH PSILI..GREEK CAPITAL LETTER UPSILON WITH VARIA +1FEC ; Recommended # 1.1 GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; Recommended # 1.1 [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FF8 ; Recommended # 1.1 [3] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMICRON WITH VARIA +1FFA ; Recommended # 1.1 GREEK CAPITAL LETTER OMEGA WITH VARIA +1FFC ; Recommended # 1.1 GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2D27 ; Recommended # 6.1 GEORGIAN SMALL LETTER YN +2D2D ; Recommended # 6.1 GEORGIAN SMALL LETTER AEN +2D80..2D96 ; Recommended # 4.1 [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SYLLABLE GGWE +2DA0..2DA6 ; Recommended # 4.1 [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SYLLABLE SSO +2DA8..2DAE ; Recommended # 4.1 [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SYLLABLE CCO +2DB0..2DB6 ; Recommended # 4.1 [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SYLLABLE ZZO +2DB8..2DBE ; Recommended # 4.1 [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC SYLLABLE CCHO +2DC0..2DC6 ; Recommended # 4.1 [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SYLLABLE QYO +2DC8..2DCE ; Recommended # 4.1 [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SYLLABLE KYO +2DD0..2DD6 ; Recommended # 4.1 [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SYLLABLE XYO +2DD8..2DDE ; Recommended # 4.1 [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SYLLABLE GYO +3005..3007 ; Recommended # 1.1 [3] IDEOGRAPHIC ITERATION MARK..IDEOGRAPHIC NUMBER ZERO +3041..3094 ; Recommended # 1.1 [84] HIRAGANA LETTER SMALL A..HIRAGANA LETTER VU +3095..3096 ; Recommended # 3.2 [2] HIRAGANA LETTER SMALL KA..HIRAGANA LETTER SMALL KE +3099..309A ; Recommended # 1.1 [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309D..309E ; Recommended # 1.1 [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +30A1..30FA ; Recommended # 1.1 [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FC..30FE ; Recommended # 1.1 [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +3105..312C ; Recommended # 1.1 [40] BOPOMOFO LETTER B..BOPOMOFO LETTER GN +312D ; Recommended # 5.1 BOPOMOFO LETTER IH +312F ; Recommended # 11.0 BOPOMOFO LETTER NN +31A0..31B7 ; Recommended # 3.0 [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H +31B8..31BA ; Recommended # 6.0 [3] BOPOMOFO LETTER GH..BOPOMOFO LETTER ZY +31BB..31BF ; Recommended # 13.0 [5] BOPOMOFO FINAL LETTER G..BOPOMOFO LETTER AH +3400..4DB5 ; Recommended # 3.0 [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 +4DB6..4DBF ; Recommended # 13.0 [10] CJK UNIFIED IDEOGRAPH-4DB6..CJK UNIFIED IDEOGRAPH-4DBF +4E00..9FA5 ; Recommended # 1.1 [20902] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FA5 +9FA6..9FBB ; Recommended # 4.1 [22] CJK UNIFIED IDEOGRAPH-9FA6..CJK UNIFIED IDEOGRAPH-9FBB +9FBC..9FC3 ; Recommended # 5.1 [8] CJK UNIFIED IDEOGRAPH-9FBC..CJK UNIFIED IDEOGRAPH-9FC3 +9FC4..9FCB ; Recommended # 5.2 [8] CJK UNIFIED IDEOGRAPH-9FC4..CJK UNIFIED IDEOGRAPH-9FCB +9FCC ; Recommended # 6.1 CJK UNIFIED IDEOGRAPH-9FCC +9FCD..9FD5 ; Recommended # 8.0 [9] CJK UNIFIED IDEOGRAPH-9FCD..CJK UNIFIED IDEOGRAPH-9FD5 +9FD6..9FEA ; Recommended # 10.0 [21] CJK UNIFIED IDEOGRAPH-9FD6..CJK UNIFIED IDEOGRAPH-9FEA +9FEB..9FEF ; Recommended # 11.0 [5] CJK UNIFIED IDEOGRAPH-9FEB..CJK UNIFIED IDEOGRAPH-9FEF +9FF0..9FFC ; Recommended # 13.0 [13] CJK UNIFIED IDEOGRAPH-9FF0..CJK UNIFIED IDEOGRAPH-9FFC +9FFD..9FFF ; Recommended # 14.0 [3] CJK UNIFIED IDEOGRAPH-9FFD..CJK UNIFIED IDEOGRAPH-9FFF +A67F ; Recommended # 5.1 CYRILLIC PAYEROK +A717..A71A ; Recommended # 5.0 [4] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOWER RIGHT CORNER ANGLE +A71B..A71F ; Recommended # 5.1 [5] MODIFIER LETTER RAISED UP ARROW..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A788 ; Recommended # 5.1 MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A78D ; Recommended # 6.0 LATIN CAPITAL LETTER TURNED H +A792..A793 ; Recommended # 6.1 [2] LATIN CAPITAL LETTER C WITH BAR..LATIN SMALL LETTER C WITH BAR +A7AA ; Recommended # 6.1 LATIN CAPITAL LETTER H WITH HOOK +A7AE ; Recommended # 9.0 LATIN CAPITAL LETTER SMALL CAPITAL I +A7B8..A7B9 ; Recommended # 11.0 [2] LATIN CAPITAL LETTER U WITH STROKE..LATIN SMALL LETTER U WITH STROKE +A7C0..A7C1 ; Recommended # 14.0 [2] LATIN CAPITAL LETTER OLD POLISH O..LATIN SMALL LETTER OLD POLISH O +A7C2..A7C6 ; Recommended # 12.0 [5] LATIN CAPITAL LETTER ANGLICANA W..LATIN CAPITAL LETTER Z WITH PALATAL HOOK +A7C7..A7CA ; Recommended # 13.0 [4] LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY +A7D0..A7D1 ; Recommended # 14.0 [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G +A7D3 ; Recommended # 14.0 LATIN SMALL LETTER DOUBLE THORN +A7D5..A7D9 ; Recommended # 14.0 [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S +A9E7..A9FE ; Recommended # 7.0 [24] MYANMAR LETTER TAI LAING NYA..MYANMAR LETTER TAI LAING BHA +AA60..AA76 ; Recommended # 5.2 [23] MYANMAR LETTER KHAMTI GA..MYANMAR LOGOGRAM KHAMTI HM +AA7A..AA7B ; Recommended # 5.2 [2] MYANMAR LETTER AITON RA..MYANMAR SIGN PAO KAREN TONE +AA7C..AA7F ; Recommended # 7.0 [4] MYANMAR SIGN TAI LAING TONE-2..MYANMAR LETTER SHWE PALAUNG SHA +AB01..AB06 ; Recommended # 6.0 [6] ETHIOPIC SYLLABLE TTHU..ETHIOPIC SYLLABLE TTHO +AB09..AB0E ; Recommended # 6.0 [6] ETHIOPIC SYLLABLE DDHU..ETHIOPIC SYLLABLE DDHO +AB11..AB16 ; Recommended # 6.0 [6] ETHIOPIC SYLLABLE DZU..ETHIOPIC SYLLABLE DZO +AB20..AB26 ; Recommended # 6.0 [7] ETHIOPIC SYLLABLE CCHHA..ETHIOPIC SYLLABLE CCHHO +AB28..AB2E ; Recommended # 6.0 [7] ETHIOPIC SYLLABLE BBA..ETHIOPIC SYLLABLE BBO +AB66..AB67 ; Recommended # 12.0 [2] LATIN SMALL LETTER DZ DIGRAPH WITH RETROFLEX HOOK..LATIN SMALL LETTER TS DIGRAPH WITH RETROFLEX HOOK +AC00..D7A3 ; Recommended # 2.0 [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +FA0E..FA0F ; Recommended # 1.1 [2] CJK COMPATIBILITY IDEOGRAPH-FA0E..CJK COMPATIBILITY IDEOGRAPH-FA0F +FA11 ; Recommended # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA11 +FA13..FA14 ; Recommended # 1.1 [2] CJK COMPATIBILITY IDEOGRAPH-FA13..CJK COMPATIBILITY IDEOGRAPH-FA14 +FA1F ; Recommended # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA1F +FA21 ; Recommended # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA21 +FA23..FA24 ; Recommended # 1.1 [2] CJK COMPATIBILITY IDEOGRAPH-FA23..CJK COMPATIBILITY IDEOGRAPH-FA24 +FA27..FA29 ; Recommended # 1.1 [3] CJK COMPATIBILITY IDEOGRAPH-FA27..CJK COMPATIBILITY IDEOGRAPH-FA29 +11301 ; Recommended # 7.0 GRANTHA SIGN CANDRABINDU +11303 ; Recommended # 7.0 GRANTHA SIGN VISARGA +1133B ; Recommended # 11.0 COMBINING BINDU BELOW +1133C ; Recommended # 7.0 GRANTHA SIGN NUKTA +16FF0..16FF1 ; Recommended # 13.0 [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY +1B11F..1B122 ; Recommended # 14.0 [4] HIRAGANA LETTER ARCHAIC WU..KATAKANA LETTER ARCHAIC WU +1B150..1B152 ; Recommended # 12.0 [3] HIRAGANA LETTER SMALL WI..HIRAGANA LETTER SMALL WO +1B164..1B167 ; Recommended # 12.0 [4] KATAKANA LETTER SMALL WI..KATAKANA LETTER SMALL N +1DF00..1DF1E ; Recommended # 14.0 [31] LATIN SMALL LETTER FENG DIGRAPH WITH TRILL..LATIN SMALL LETTER S WITH CURL +1E7E0..1E7E6 ; Recommended # 14.0 [7] ETHIOPIC SYLLABLE HHYA..ETHIOPIC SYLLABLE HHYO +1E7E8..1E7EB ; Recommended # 14.0 [4] ETHIOPIC SYLLABLE GURAGE HHWA..ETHIOPIC SYLLABLE HHWE +1E7ED..1E7EE ; Recommended # 14.0 [2] ETHIOPIC SYLLABLE GURAGE MWI..ETHIOPIC SYLLABLE GURAGE MWEE +1E7F0..1E7FE ; Recommended # 14.0 [15] ETHIOPIC SYLLABLE GURAGE QWI..ETHIOPIC SYLLABLE GURAGE PWEE +20000..2A6D6 ; Recommended # 3.1 [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 +2A6D7..2A6DD ; Recommended # 13.0 [7] CJK UNIFIED IDEOGRAPH-2A6D7..CJK UNIFIED IDEOGRAPH-2A6DD +2A6DE..2A6DF ; Recommended # 14.0 [2] CJK UNIFIED IDEOGRAPH-2A6DE..CJK UNIFIED IDEOGRAPH-2A6DF +2A700..2B734 ; Recommended # 5.2 [4149] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B734 +2B735..2B738 ; Recommended # 14.0 [4] CJK UNIFIED IDEOGRAPH-2B735..CJK UNIFIED IDEOGRAPH-2B738 +2B740..2B81D ; Recommended # 6.0 [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D +2B820..2CEA1 ; Recommended # 8.0 [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 +2CEB0..2EBE0 ; Recommended # 10.0 [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 +30000..3134A ; Recommended # 13.0 [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A + +# Total code points: 107938 + +# Identifier_Type: Inclusion + +0027 ; Inclusion # 1.1 APOSTROPHE +002D..002E ; Inclusion # 1.1 [2] HYPHEN-MINUS..FULL STOP +003A ; Inclusion # 1.1 COLON +00B7 ; Inclusion # 1.1 MIDDLE DOT +0375 ; Inclusion # 1.1 GREEK LOWER NUMERAL SIGN +058A ; Inclusion # 3.0 ARMENIAN HYPHEN +05F3..05F4 ; Inclusion # 1.1 [2] HEBREW PUNCTUATION GERESH..HEBREW PUNCTUATION GERSHAYIM +06FD..06FE ; Inclusion # 3.0 [2] ARABIC SIGN SINDHI AMPERSAND..ARABIC SIGN SINDHI POSTPOSITION MEN +0F0B ; Inclusion # 2.0 TIBETAN MARK INTERSYLLABIC TSHEG +200C..200D ; Inclusion # 1.1 [2] ZERO WIDTH NON-JOINER..ZERO WIDTH JOINER +2010 ; Inclusion # 1.1 HYPHEN +2019 ; Inclusion # 1.1 RIGHT SINGLE QUOTATION MARK +2027 ; Inclusion # 1.1 HYPHENATION POINT +30A0 ; Inclusion # 3.2 KATAKANA-HIRAGANA DOUBLE HYPHEN +30FB ; Inclusion # 1.1 KATAKANA MIDDLE DOT + +# Total code points: 19 + +# Identifier_Type: Limited_Use + +0710..072C ; Limited_Use # 3.0 [29] SYRIAC LETTER ALAPH..SYRIAC LETTER TAW +072D..072F ; Limited_Use # 4.0 [3] SYRIAC LETTER PERSIAN BHETH..SYRIAC LETTER PERSIAN DHALATH +0730..073F ; Limited_Use # 3.0 [16] SYRIAC PTHAHA ABOVE..SYRIAC RWAHA +074D..074F ; Limited_Use # 4.0 [3] SYRIAC LETTER SOGDIAN ZHAIN..SYRIAC LETTER SOGDIAN FE +07C0..07E7 ; Limited_Use # 5.0 [40] NKO DIGIT ZERO..NKO LETTER NYA WOLOSO +07EB..07F5 ; Limited_Use # 5.0 [11] NKO COMBINING SHORT HIGH TONE..NKO LOW TONE APOSTROPHE +07FD ; Limited_Use # 11.0 NKO DANTAYALAN +0840..085B ; Limited_Use # 6.0 [28] MANDAIC LETTER HALQA..MANDAIC GEMINATION MARK +0860..086A ; Limited_Use # 10.0 [11] SYRIAC LETTER MALAYALAM NGA..SYRIAC LETTER MALAYALAM SSA +13A0..13F4 ; Limited_Use # 3.0 [85] CHEROKEE LETTER A..CHEROKEE LETTER YV +13F5 ; Limited_Use # 8.0 CHEROKEE LETTER MV +13F8..13FD ; Limited_Use # 8.0 [6] CHEROKEE SMALL LETTER YE..CHEROKEE SMALL LETTER MV +1401..166C ; Limited_Use # 3.0 [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166F..1676 ; Limited_Use # 3.0 [8] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS NNGAA +1677..167F ; Limited_Use # 5.2 [9] CANADIAN SYLLABICS WOODS-CREE THWEE..CANADIAN SYLLABICS BLACKFOOT W +18B0..18F5 ; Limited_Use # 5.2 [70] CANADIAN SYLLABICS OY..CANADIAN SYLLABICS CARRIER DENTAL S +1900..191C ; Limited_Use # 4.0 [29] LIMBU VOWEL-CARRIER LETTER..LIMBU LETTER HA +191D..191E ; Limited_Use # 7.0 [2] LIMBU LETTER GYAN..LIMBU LETTER TRA +1920..192B ; Limited_Use # 4.0 [12] LIMBU VOWEL SIGN A..LIMBU SUBJOINED LETTER WA +1930..193B ; Limited_Use # 4.0 [12] LIMBU SMALL LETTER KA..LIMBU SIGN SA-I +1946..196D ; Limited_Use # 4.0 [40] LIMBU DIGIT ZERO..TAI LE LETTER AI +1970..1974 ; Limited_Use # 4.0 [5] TAI LE LETTER TONE-2..TAI LE LETTER TONE-6 +1980..19A9 ; Limited_Use # 4.1 [42] NEW TAI LUE LETTER HIGH QA..NEW TAI LUE LETTER LOW XVA +19AA..19AB ; Limited_Use # 5.2 [2] NEW TAI LUE LETTER HIGH SUA..NEW TAI LUE LETTER LOW SUA +19B0..19C9 ; Limited_Use # 4.1 [26] NEW TAI LUE VOWEL SIGN VOWEL SHORTENER..NEW TAI LUE TONE MARK-2 +19D0..19D9 ; Limited_Use # 4.1 [10] NEW TAI LUE DIGIT ZERO..NEW TAI LUE DIGIT NINE +19DA ; Limited_Use # 5.2 NEW TAI LUE THAM DIGIT ONE +1A20..1A5E ; Limited_Use # 5.2 [63] TAI THAM LETTER HIGH KA..TAI THAM CONSONANT SIGN SA +1A60..1A7C ; Limited_Use # 5.2 [29] TAI THAM SIGN SAKOT..TAI THAM SIGN KHUEN-LUE KARAN +1A7F..1A89 ; Limited_Use # 5.2 [11] TAI THAM COMBINING CRYPTOGRAMMIC DOT..TAI THAM HORA DIGIT NINE +1A90..1A99 ; Limited_Use # 5.2 [10] TAI THAM THAM DIGIT ZERO..TAI THAM THAM DIGIT NINE +1AA7 ; Limited_Use # 5.2 TAI THAM SIGN MAI YAMOK +1B00..1B4B ; Limited_Use # 5.0 [76] BALINESE SIGN ULU RICEM..BALINESE LETTER ASYURA SASAK +1B4C ; Limited_Use # 14.0 BALINESE LETTER ARCHAIC JNYA +1B50..1B59 ; Limited_Use # 5.0 [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE +1B80..1BAA ; Limited_Use # 5.1 [43] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PAMAAEH +1BAB..1BAD ; Limited_Use # 6.1 [3] SUNDANESE SIGN VIRAMA..SUNDANESE CONSONANT SIGN PASANGAN WA +1BAE..1BB9 ; Limited_Use # 5.1 [12] SUNDANESE LETTER KHA..SUNDANESE DIGIT NINE +1BBA..1BBF ; Limited_Use # 6.1 [6] SUNDANESE AVAGRAHA..SUNDANESE LETTER FINAL M +1BC0..1BF3 ; Limited_Use # 6.0 [52] BATAK LETTER A..BATAK PANONGONAN +1C00..1C37 ; Limited_Use # 5.1 [56] LEPCHA LETTER KA..LEPCHA SIGN NUKTA +1C40..1C49 ; Limited_Use # 5.1 [10] LEPCHA DIGIT ZERO..LEPCHA DIGIT NINE +1C4D..1C7D ; Limited_Use # 5.1 [49] LEPCHA LETTER TTA..OL CHIKI AHAD +2D30..2D65 ; Limited_Use # 4.1 [54] TIFINAGH LETTER YA..TIFINAGH LETTER YAZZ +2D66..2D67 ; Limited_Use # 6.1 [2] TIFINAGH LETTER YE..TIFINAGH LETTER YO +2D7F ; Limited_Use # 6.0 TIFINAGH CONSONANT JOINER +A000..A48C ; Limited_Use # 3.0 [1165] YI SYLLABLE IT..YI SYLLABLE YYR +A4D0..A4FD ; Limited_Use # 5.2 [46] LISU LETTER BA..LISU LETTER TONE MYA JEU +A500..A60C ; Limited_Use # 5.1 [269] VAI SYLLABLE EE..VAI SYLLABLE LENGTHENER +A613..A629 ; Limited_Use # 5.1 [23] VAI SYMBOL FEENG..VAI DIGIT NINE +A6A0..A6F1 ; Limited_Use # 5.2 [82] BAMUM LETTER A..BAMUM COMBINING MARK TUKWENTIS +A800..A827 ; Limited_Use # 4.1 [40] SYLOTI NAGRI LETTER A..SYLOTI NAGRI VOWEL SIGN OO +A82C ; Limited_Use # 13.0 SYLOTI NAGRI SIGN ALTERNATE HASANTA +A880..A8C4 ; Limited_Use # 5.1 [69] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VIRAMA +A8C5 ; Limited_Use # 9.0 SAURASHTRA SIGN CANDRABINDU +A8D0..A8D9 ; Limited_Use # 5.1 [10] SAURASHTRA DIGIT ZERO..SAURASHTRA DIGIT NINE +A900..A92D ; Limited_Use # 5.1 [46] KAYAH LI DIGIT ZERO..KAYAH LI TONE CALYA PLOPHU +A980..A9C0 ; Limited_Use # 5.2 [65] JAVANESE SIGN PANYANGGA..JAVANESE PANGKON +A9D0..A9D9 ; Limited_Use # 5.2 [10] JAVANESE DIGIT ZERO..JAVANESE DIGIT NINE +AA00..AA36 ; Limited_Use # 5.1 [55] CHAM LETTER A..CHAM CONSONANT SIGN WA +AA40..AA4D ; Limited_Use # 5.1 [14] CHAM LETTER FINAL K..CHAM CONSONANT SIGN FINAL H +AA50..AA59 ; Limited_Use # 5.1 [10] CHAM DIGIT ZERO..CHAM DIGIT NINE +AA80..AAC2 ; Limited_Use # 5.2 [67] TAI VIET LETTER LOW KO..TAI VIET TONE MAI SONG +AADB..AADD ; Limited_Use # 5.2 [3] TAI VIET SYMBOL KON..TAI VIET SYMBOL SAM +AAE0..AAEF ; Limited_Use # 6.1 [16] MEETEI MAYEK LETTER E..MEETEI MAYEK VOWEL SIGN AAU +AAF2..AAF6 ; Limited_Use # 6.1 [5] MEETEI MAYEK ANJI..MEETEI MAYEK VIRAMA +AB70..ABBF ; Limited_Use # 8.0 [80] CHEROKEE SMALL LETTER A..CHEROKEE SMALL LETTER YA +ABC0..ABEA ; Limited_Use # 5.2 [43] MEETEI MAYEK LETTER KOK..MEETEI MAYEK VOWEL SIGN NUNG +ABEC..ABED ; Limited_Use # 5.2 [2] MEETEI MAYEK LUM IYEK..MEETEI MAYEK APUN IYEK +ABF0..ABF9 ; Limited_Use # 5.2 [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DIGIT NINE +104B0..104D3 ; Limited_Use # 9.0 [36] OSAGE CAPITAL LETTER A..OSAGE CAPITAL LETTER ZHA +104D8..104FB ; Limited_Use # 9.0 [36] OSAGE SMALL LETTER A..OSAGE SMALL LETTER ZHA +10D00..10D27 ; Limited_Use # 11.0 [40] HANIFI ROHINGYA LETTER A..HANIFI ROHINGYA SIGN TASSI +10D30..10D39 ; Limited_Use # 11.0 [10] HANIFI ROHINGYA DIGIT ZERO..HANIFI ROHINGYA DIGIT NINE +11100..11134 ; Limited_Use # 6.1 [53] CHAKMA SIGN CANDRABINDU..CHAKMA MAAYYAA +11136..1113F ; Limited_Use # 6.1 [10] CHAKMA DIGIT ZERO..CHAKMA DIGIT NINE +11144..11146 ; Limited_Use # 11.0 [3] CHAKMA LETTER LHAA..CHAKMA VOWEL SIGN EI +11147 ; Limited_Use # 13.0 CHAKMA LETTER VAA +11400..1144A ; Limited_Use # 9.0 [75] NEWA LETTER A..NEWA SIDDHI +11450..11459 ; Limited_Use # 9.0 [10] NEWA DIGIT ZERO..NEWA DIGIT NINE +1145E ; Limited_Use # 11.0 NEWA SANDHI MARK +1145F ; Limited_Use # 12.0 NEWA LETTER VEDIC ANUSVARA +11460..11461 ; Limited_Use # 13.0 [2] NEWA SIGN JIHVAMULIYA..NEWA SIGN UPADHMANIYA +11AB0..11ABF ; Limited_Use # 14.0 [16] CANADIAN SYLLABICS NATTILIK HI..CANADIAN SYLLABICS SPA +11D60..11D65 ; Limited_Use # 11.0 [6] GUNJALA GONDI LETTER A..GUNJALA GONDI LETTER UU +11D67..11D68 ; Limited_Use # 11.0 [2] GUNJALA GONDI LETTER EE..GUNJALA GONDI LETTER AI +11D6A..11D8E ; Limited_Use # 11.0 [37] GUNJALA GONDI LETTER OO..GUNJALA GONDI VOWEL SIGN UU +11D90..11D91 ; Limited_Use # 11.0 [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI +11D93..11D98 ; Limited_Use # 11.0 [6] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI OM +11DA0..11DA9 ; Limited_Use # 11.0 [10] GUNJALA GONDI DIGIT ZERO..GUNJALA GONDI DIGIT NINE +11FB0 ; Limited_Use # 13.0 LISU LETTER YHA +16800..16A38 ; Limited_Use # 6.0 [569] BAMUM LETTER PHASE-A NGKUE MFON..BAMUM LETTER PHASE-F VUEQ +16F00..16F44 ; Limited_Use # 6.1 [69] MIAO LETTER PA..MIAO LETTER HHA +16F45..16F4A ; Limited_Use # 12.0 [6] MIAO LETTER BRI..MIAO LETTER RTE +16F4F ; Limited_Use # 12.0 MIAO SIGN CONSONANT MODIFIER BAR +16F50..16F7E ; Limited_Use # 6.1 [47] MIAO LETTER NASALIZATION..MIAO VOWEL SIGN NG +16F7F..16F87 ; Limited_Use # 12.0 [9] MIAO VOWEL SIGN UOG..MIAO VOWEL SIGN UI +16F8F..16F9F ; Limited_Use # 6.1 [17] MIAO TONE RIGHT..MIAO LETTER REFORMED TONE-8 +1E100..1E12C ; Limited_Use # 12.0 [45] NYIAKENG PUACHUE HMONG LETTER MA..NYIAKENG PUACHUE HMONG LETTER W +1E130..1E13D ; Limited_Use # 12.0 [14] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG SYLLABLE LENGTHENER +1E140..1E149 ; Limited_Use # 12.0 [10] NYIAKENG PUACHUE HMONG DIGIT ZERO..NYIAKENG PUACHUE HMONG DIGIT NINE +1E14E ; Limited_Use # 12.0 NYIAKENG PUACHUE HMONG LOGOGRAM NYAJ +1E2C0..1E2F9 ; Limited_Use # 12.0 [58] WANCHO LETTER AA..WANCHO DIGIT NINE +1E900..1E94A ; Limited_Use # 9.0 [75] ADLAM CAPITAL LETTER ALIF..ADLAM NUKTA +1E94B ; Limited_Use # 12.0 ADLAM NASALIZATION MARK +1E950..1E959 ; Limited_Use # 9.0 [10] ADLAM DIGIT ZERO..ADLAM DIGIT NINE + +# Total code points: 5033 + +# Identifier_Type: Limited_Use Technical + +0740..074A ; Limited_Use Technical # 3.0 [11] SYRIAC FEMININE DOT..SYRIAC BARREKH +1B6B..1B73 ; Limited_Use Technical # 5.0 [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG +1DFA ; Limited_Use Technical # 14.0 COMBINING DOT BELOW LEFT + +# Total code points: 21 + +# Identifier_Type: Limited_Use Exclusion + +A9CF ; Limited_Use Exclusion # 5.2 JAVANESE PANGRANGKEP + +# Total code points: 1 + +# Identifier_Type: Limited_Use Obsolete + +07E8..07EA ; Limited_Use Obsolete # 5.0 [3] NKO LETTER JONA JA..NKO LETTER JONA RA +07FA ; Limited_Use Obsolete # 5.0 NKO LAJANYALAN +A610..A612 ; Limited_Use Obsolete # 5.1 [3] VAI SYLLABLE NDOLE FA..VAI SYLLABLE NDOLE SOO +A62A..A62B ; Limited_Use Obsolete # 5.1 [2] VAI SYLLABLE NDOLE MA..VAI SYLLABLE NDOLE DO + +# Total code points: 9 + +# Identifier_Type: Limited_Use Not_XID + +0700..070D ; Limited_Use Not_XID # 3.0 [14] SYRIAC END OF PARAGRAPH..SYRIAC HARKLEAN ASTERISCUS +070F ; Limited_Use Not_XID # 3.0 SYRIAC ABBREVIATION MARK +07F6..07F9 ; Limited_Use Not_XID # 5.0 [4] NKO SYMBOL OO DENNEN..NKO EXCLAMATION MARK +07FE..07FF ; Limited_Use Not_XID # 11.0 [2] NKO DOROME SIGN..NKO TAMAN SIGN +085E ; Limited_Use Not_XID # 6.0 MANDAIC PUNCTUATION +1400 ; Limited_Use Not_XID # 5.2 CANADIAN SYLLABICS HYPHEN +166D..166E ; Limited_Use Not_XID # 3.0 [2] CANADIAN SYLLABICS CHI SIGN..CANADIAN SYLLABICS FULL STOP +1940 ; Limited_Use Not_XID # 4.0 LIMBU SIGN LOO +1944..1945 ; Limited_Use Not_XID # 4.0 [2] LIMBU EXCLAMATION MARK..LIMBU QUESTION MARK +19DE..19DF ; Limited_Use Not_XID # 4.1 [2] NEW TAI LUE SIGN LAE..NEW TAI LUE SIGN LAEV +1AA0..1AA6 ; Limited_Use Not_XID # 5.2 [7] TAI THAM SIGN WIANG..TAI THAM SIGN REVERSED ROTATED RANA +1AA8..1AAD ; Limited_Use Not_XID # 5.2 [6] TAI THAM SIGN KAAN..TAI THAM SIGN CAANG +1B5A..1B6A ; Limited_Use Not_XID # 5.0 [17] BALINESE PANTI..BALINESE MUSICAL SYMBOL DANG GEDE +1B74..1B7C ; Limited_Use Not_XID # 5.0 [9] BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DUG..BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING +1B7D..1B7E ; Limited_Use Not_XID # 14.0 [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG +1BFC..1BFF ; Limited_Use Not_XID # 6.0 [4] BATAK SYMBOL BINDU NA METEK..BATAK SYMBOL BINDU PANGOLAT +1C3B..1C3F ; Limited_Use Not_XID # 5.1 [5] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION TSHOOK +1C7E..1C7F ; Limited_Use Not_XID # 5.1 [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD +1CC0..1CC7 ; Limited_Use Not_XID # 6.1 [8] SUNDANESE PUNCTUATION BINDU SURYA..SUNDANESE PUNCTUATION BINDU BA SATANGA +2D70 ; Limited_Use Not_XID # 6.0 TIFINAGH SEPARATOR MARK +A490..A4A1 ; Limited_Use Not_XID # 3.0 [18] YI RADICAL QOT..YI RADICAL GA +A4A2..A4A3 ; Limited_Use Not_XID # 3.2 [2] YI RADICAL ZUP..YI RADICAL CYT +A4A4..A4B3 ; Limited_Use Not_XID # 3.0 [16] YI RADICAL DDUR..YI RADICAL JO +A4B4 ; Limited_Use Not_XID # 3.2 YI RADICAL NZUP +A4B5..A4C0 ; Limited_Use Not_XID # 3.0 [12] YI RADICAL JJY..YI RADICAL SHAT +A4C1 ; Limited_Use Not_XID # 3.2 YI RADICAL ZUR +A4C2..A4C4 ; Limited_Use Not_XID # 3.0 [3] YI RADICAL SHOP..YI RADICAL ZZIET +A4C5 ; Limited_Use Not_XID # 3.2 YI RADICAL NBIE +A4C6 ; Limited_Use Not_XID # 3.0 YI RADICAL KE +A4FE..A4FF ; Limited_Use Not_XID # 5.2 [2] LISU PUNCTUATION COMMA..LISU PUNCTUATION FULL STOP +A60D..A60F ; Limited_Use Not_XID # 5.1 [3] VAI COMMA..VAI QUESTION MARK +A6F2..A6F7 ; Limited_Use Not_XID # 5.2 [6] BAMUM NJAEMLI..BAMUM QUESTION MARK +A828..A82B ; Limited_Use Not_XID # 4.1 [4] SYLOTI NAGRI POETRY MARK-1..SYLOTI NAGRI POETRY MARK-4 +A8CE..A8CF ; Limited_Use Not_XID # 5.1 [2] SAURASHTRA DANDA..SAURASHTRA DOUBLE DANDA +A92F ; Limited_Use Not_XID # 5.1 KAYAH LI SIGN SHYA +A9C1..A9CD ; Limited_Use Not_XID # 5.2 [13] JAVANESE LEFT RERENGGAN..JAVANESE TURNED PADA PISELEH +A9DE..A9DF ; Limited_Use Not_XID # 5.2 [2] JAVANESE PADA TIRTA TUMETES..JAVANESE PADA ISEN-ISEN +AA5C..AA5F ; Limited_Use Not_XID # 5.1 [4] CHAM PUNCTUATION SPIRAL..CHAM PUNCTUATION TRIPLE DANDA +AADE..AADF ; Limited_Use Not_XID # 5.2 [2] TAI VIET SYMBOL HO HOI..TAI VIET SYMBOL KOI KOI +AAF0..AAF1 ; Limited_Use Not_XID # 6.1 [2] MEETEI MAYEK CHEIKHAN..MEETEI MAYEK AHANG KHUDAM +ABEB ; Limited_Use Not_XID # 5.2 MEETEI MAYEK CHEIKHEI +11140..11143 ; Limited_Use Not_XID # 6.1 [4] CHAKMA SECTION MARK..CHAKMA QUESTION MARK +1144B..1144F ; Limited_Use Not_XID # 9.0 [5] NEWA DANDA..NEWA ABBREVIATION SIGN +1145A ; Limited_Use Not_XID # 13.0 NEWA DOUBLE COMMA +1145B ; Limited_Use Not_XID # 9.0 NEWA PLACEHOLDER MARK +1145D ; Limited_Use Not_XID # 9.0 NEWA INSERTION SIGN +1E14F ; Limited_Use Not_XID # 12.0 NYIAKENG PUACHUE HMONG CIRCLED CA +1E2FF ; Limited_Use Not_XID # 12.0 WANCHO NGUN SIGN +1E95E..1E95F ; Limited_Use Not_XID # 9.0 [2] ADLAM INITIAL EXCLAMATION MARK..ADLAM INITIAL QUESTION MARK + +# Total code points: 204 + +# Identifier_Type: Uncommon_Use + +0181..018C ; Uncommon_Use # 1.1 [12] LATIN CAPITAL LETTER B WITH HOOK..LATIN SMALL LETTER D WITH TOPBAR +018E ; Uncommon_Use # 1.1 LATIN CAPITAL LETTER REVERSED E +0190..019F ; Uncommon_Use # 1.1 [16] LATIN CAPITAL LETTER OPEN E..LATIN CAPITAL LETTER O WITH MIDDLE TILDE +01A2..01A9 ; Uncommon_Use # 1.1 [8] LATIN CAPITAL LETTER OI..LATIN CAPITAL LETTER ESH +01AC..01AE ; Uncommon_Use # 1.1 [3] LATIN CAPITAL LETTER T WITH HOOK..LATIN CAPITAL LETTER T WITH RETROFLEX HOOK +01B1..01B8 ; Uncommon_Use # 1.1 [8] LATIN CAPITAL LETTER UPSILON..LATIN CAPITAL LETTER EZH REVERSED +01BC..01BD ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER TONE FIVE..LATIN SMALL LETTER TONE FIVE +01DD ; Uncommon_Use # 1.1 LATIN SMALL LETTER TURNED E +01E4..01E5 ; Uncommon_Use # 1.1 [2] LATIN CAPITAL LETTER G WITH STROKE..LATIN SMALL LETTER G WITH STROKE +0220 ; Uncommon_Use # 3.2 LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0221 ; Uncommon_Use # 4.0 LATIN SMALL LETTER D WITH CURL +0222..0225 ; Uncommon_Use # 3.0 [4] LATIN CAPITAL LETTER OU..LATIN SMALL LETTER Z WITH HOOK +0237..0241 ; Uncommon_Use # 4.1 [11] LATIN SMALL LETTER DOTLESS J..LATIN CAPITAL LETTER GLOTTAL STOP +0242..024F ; Uncommon_Use # 5.0 [14] LATIN SMALL LETTER GLOTTAL STOP..LATIN SMALL LETTER Y WITH STROKE +0305 ; Uncommon_Use # 1.1 COMBINING OVERLINE +030D ; Uncommon_Use # 1.1 COMBINING VERTICAL LINE ABOVE +0316 ; Uncommon_Use # 1.1 COMBINING GRAVE ACCENT BELOW +0321..0322 ; Uncommon_Use # 1.1 [2] COMBINING PALATALIZED HOOK BELOW..COMBINING RETROFLEX HOOK BELOW +0332 ; Uncommon_Use # 1.1 COMBINING LOW LINE +0334 ; Uncommon_Use # 1.1 COMBINING TILDE OVERLAY +0336 ; Uncommon_Use # 1.1 COMBINING LONG STROKE OVERLAY +0358 ; Uncommon_Use # 4.1 COMBINING DOT ABOVE RIGHT +0591..05A1 ; Uncommon_Use # 2.0 [17] HEBREW ACCENT ETNAHTA..HEBREW ACCENT PAZER +05A3..05AF ; Uncommon_Use # 2.0 [13] HEBREW ACCENT MUNAH..HEBREW MARK MASORA CIRCLE +05B0..05B3 ; Uncommon_Use # 1.1 [4] HEBREW POINT SHEVA..HEBREW POINT HATAF QAMATS +05B5..05B9 ; Uncommon_Use # 1.1 [5] HEBREW POINT TSERE..HEBREW POINT HOLAM +05BA ; Uncommon_Use # 5.0 HEBREW POINT HOLAM HASER FOR VAV +05BB..05BD ; Uncommon_Use # 1.1 [3] HEBREW POINT QUBUTS..HEBREW POINT METEG +05BF ; Uncommon_Use # 1.1 HEBREW POINT RAFE +05C1..05C2 ; Uncommon_Use # 1.1 [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4 ; Uncommon_Use # 2.0 HEBREW MARK UPPER DOT +0610..0615 ; Uncommon_Use # 4.0 [6] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL HIGH TAH +0616..061A ; Uncommon_Use # 5.1 [5] ARABIC SMALL HIGH LIGATURE ALEF WITH LAM WITH YEH..ARABIC SMALL KASRA +0656..0658 ; Uncommon_Use # 4.0 [3] ARABIC SUBSCRIPT ALEF..ARABIC MARK NOON GHUNNA +0659..065E ; Uncommon_Use # 4.1 [6] ARABIC ZWARAKAY..ARABIC FATHA WITH TWO DOTS +065F ; Uncommon_Use # 6.0 ARABIC WAVY HAMZA BELOW +06D6..06DC ; Uncommon_Use # 1.1 [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06DF..06E4 ; Uncommon_Use # 1.1 [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA +06E7..06E8 ; Uncommon_Use # 1.1 [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06EA..06ED ; Uncommon_Use # 1.1 [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM +0898..089F ; Uncommon_Use # 14.0 [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA +08B3..08B4 ; Uncommon_Use # 8.0 [2] ARABIC LETTER AIN WITH THREE DOTS BELOW..ARABIC LETTER KAF WITH DOT BELOW +08CA..08D2 ; Uncommon_Use # 14.0 [9] ARABIC SMALL HIGH FARSI YEH..ARABIC LARGE ROUND DOT INSIDE CIRCLE BELOW +08D3 ; Uncommon_Use # 11.0 ARABIC SMALL LOW WAW +08D4..08E1 ; Uncommon_Use # 9.0 [14] ARABIC SMALL HIGH WORD AR-RUB..ARABIC SMALL HIGH SIGN SAFHA +08E3 ; Uncommon_Use # 8.0 ARABIC TURNED DAMMA BELOW +08E4..08FE ; Uncommon_Use # 6.1 [27] ARABIC CURLY FATHA..ARABIC DAMMA WITH DOT +08FF ; Uncommon_Use # 7.0 ARABIC MARK SIDEWAYS NOON GHUNNA +0900 ; Uncommon_Use # 5.2 DEVANAGARI SIGN INVERTED CANDRABINDU +0955 ; Uncommon_Use # 5.2 DEVANAGARI VOWEL SIGN CANDRA LONG E +0A51 ; Uncommon_Use # 5.1 GURMUKHI SIGN UDAAT +0A75 ; Uncommon_Use # 5.1 GURMUKHI SIGN YAKASH +0AF9 ; Uncommon_Use # 8.0 GUJARATI LETTER ZHA +0B44 ; Uncommon_Use # 5.1 ORIYA VOWEL SIGN VOCALIC RR +0B62..0B63 ; Uncommon_Use # 5.1 [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL +0C5A ; Uncommon_Use # 8.0 TELUGU LETTER RRRA +0C62..0C63 ; Uncommon_Use # 5.1 [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL +0D44 ; Uncommon_Use # 5.1 MALAYALAM VOWEL SIGN VOCALIC RR +0D62..0D63 ; Uncommon_Use # 5.1 [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL +0F39 ; Uncommon_Use # 2.0 TIBETAN MARK TSA -PHRU +1AC1..1ACE ; Uncommon_Use # 14.0 [14] COMBINING LEFT PARENTHESIS ABOVE LEFT..COMBINING LATIN SMALL LETTER INSULAR T +2054 ; Uncommon_Use # 4.0 INVERTED UNDERTIE +2C68..2C6C ; Uncommon_Use # 5.0 [5] LATIN SMALL LETTER H WITH DESCENDER..LATIN SMALL LETTER Z WITH DESCENDER +A66F ; Uncommon_Use # 5.1 COMBINING CYRILLIC VZMET +A67C..A67D ; Uncommon_Use # 5.1 [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILLIC PAYEROK +A78B..A78C ; Uncommon_Use # 5.1 [2] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER SALTILLO +A78F ; Uncommon_Use # 8.0 LATIN LETTER SINOLOGICAL DOT +A7B2..A7B7 ; Uncommon_Use # 8.0 [6] LATIN CAPITAL LETTER J WITH CROSSED-TAIL..LATIN SMALL LETTER OMEGA +AB60..AB63 ; Uncommon_Use # 8.0 [4] LATIN SMALL LETTER SAKHA YAT..LATIN SMALL LETTER UO +10780 ; Uncommon_Use # 14.0 MODIFIER LETTER SMALL CAPITAL AA +1AFF0..1AFF3 ; Uncommon_Use # 14.0 [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5 +1AFF5..1AFFB ; Uncommon_Use # 14.0 [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5 +1AFFD..1AFFE ; Uncommon_Use # 14.0 [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8 + +# Total code points: 308 + +# Identifier_Type: Uncommon_Use Technical + +0253..0254 ; Uncommon_Use Technical # 1.1 [2] LATIN SMALL LETTER B WITH HOOK..LATIN SMALL LETTER OPEN O +0256..0257 ; Uncommon_Use Technical # 1.1 [2] LATIN SMALL LETTER D WITH TAIL..LATIN SMALL LETTER D WITH HOOK +025B ; Uncommon_Use Technical # 1.1 LATIN SMALL LETTER OPEN E +0263 ; Uncommon_Use Technical # 1.1 LATIN SMALL LETTER GAMMA +0268..0269 ; Uncommon_Use Technical # 1.1 [2] LATIN SMALL LETTER I WITH STROKE..LATIN SMALL LETTER IOTA +0272 ; Uncommon_Use Technical # 1.1 LATIN SMALL LETTER N WITH LEFT HOOK +0289 ; Uncommon_Use Technical # 1.1 LATIN SMALL LETTER U BAR +0292 ; Uncommon_Use Technical # 1.1 LATIN SMALL LETTER EZH +05C7 ; Uncommon_Use Technical # 4.1 HEBREW POINT QAMATS QATAN +0D8F..0D90 ; Uncommon_Use Technical # 3.0 [2] SINHALA LETTER ILUYANNA..SINHALA LETTER ILUUYANNA +0DA6 ; Uncommon_Use Technical # 3.0 SINHALA LETTER SANYAKA JAYANNA +0DDF ; Uncommon_Use Technical # 3.0 SINHALA VOWEL SIGN GAYANUKITTA +0DF3 ; Uncommon_Use Technical # 3.0 SINHALA VOWEL SIGN DIGA GAYANUKITTA +FB1E ; Uncommon_Use Technical # 1.1 HEBREW POINT JUDEO-SPANISH VARIKA +FE2E..FE2F ; Uncommon_Use Technical # 8.0 [2] COMBINING CYRILLIC TITLO LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF + +# Total code points: 20 + +# Identifier_Type: Uncommon_Use Technical Not_XID + +1D1DE..1D1E8 ; Uncommon_Use Technical Not_XID # 8.0 [11] MUSICAL SYMBOL KIEVAN C CLEF..MUSICAL SYMBOL KIEVAN FLAT SIGN + +# Total code points: 11 + +# Identifier_Type: Uncommon_Use Exclusion + +18A9 ; Uncommon_Use Exclusion # 3.0 MONGOLIAN LETTER ALI GALI DAGALGA +16A40..16A5E ; Uncommon_Use Exclusion # 7.0 [31] MRO LETTER TA..MRO LETTER TEK +16A60..16A69 ; Uncommon_Use Exclusion # 7.0 [10] MRO DIGIT ZERO..MRO DIGIT NINE + +# Total code points: 42 + +# Identifier_Type: Uncommon_Use Obsolete + +05A2 ; Uncommon_Use Obsolete # 4.1 HEBREW ACCENT ATNAH HAFUKH +05C5 ; Uncommon_Use Obsolete # 4.1 HEBREW MARK LOWER DOT +A69E ; Uncommon_Use Obsolete # 8.0 COMBINING CYRILLIC LETTER EF +A8FD ; Uncommon_Use Obsolete # 8.0 DEVANAGARI JAIN OM + +# Total code points: 4 + +# Identifier_Type: Uncommon_Use Obsolete Not_XID + +A8FC ; Uncommon_Use Obsolete Not_XID # 8.0 DEVANAGARI SIGN SIDDHAM + +# Total code points: 1 + +# Identifier_Type: Uncommon_Use Not_XID + +218A..218B ; Uncommon_Use Not_XID # 8.0 [2] TURNED DIGIT TWO..TURNED DIGIT THREE +2BEC..2BEF ; Uncommon_Use Not_XID # 8.0 [4] LEFTWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS..DOWNWARDS TWO-HEADED ARROW WITH TRIANGLE ARROWHEADS +1F54F ; Uncommon_Use Not_XID # 8.0 BOWL OF HYGIEIA + +# Total code points: 7 + +# Identifier_Type: Technical + +0180 ; Technical # 1.1 LATIN SMALL LETTER B WITH STROKE +01C0..01C3 ; Technical # 1.1 [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +0234..0236 ; Technical # 4.0 [3] LATIN SMALL LETTER L WITH CURL..LATIN SMALL LETTER T WITH CURL +0250..0252 ; Technical # 1.1 [3] LATIN SMALL LETTER TURNED A..LATIN SMALL LETTER TURNED ALPHA +0255 ; Technical # 1.1 LATIN SMALL LETTER C WITH CURL +0258 ; Technical # 1.1 LATIN SMALL LETTER REVERSED E +025A ; Technical # 1.1 LATIN SMALL LETTER SCHWA WITH HOOK +025C..0262 ; Technical # 1.1 [7] LATIN SMALL LETTER REVERSED OPEN E..LATIN LETTER SMALL CAPITAL G +0264..0267 ; Technical # 1.1 [4] LATIN SMALL LETTER RAMS HORN..LATIN SMALL LETTER HENG WITH HOOK +026A..0271 ; Technical # 1.1 [8] LATIN LETTER SMALL CAPITAL I..LATIN SMALL LETTER M WITH HOOK +0273..0276 ; Technical # 1.1 [4] LATIN SMALL LETTER N WITH RETROFLEX HOOK..LATIN LETTER SMALL CAPITAL OE +0278..027B ; Technical # 1.1 [4] LATIN SMALL LETTER PHI..LATIN SMALL LETTER TURNED R WITH HOOK +027D..0288 ; Technical # 1.1 [12] LATIN SMALL LETTER R WITH TAIL..LATIN SMALL LETTER T WITH RETROFLEX HOOK +028A..0291 ; Technical # 1.1 [8] LATIN SMALL LETTER UPSILON..LATIN SMALL LETTER Z WITH CURL +0293..029D ; Technical # 1.1 [11] LATIN SMALL LETTER EZH WITH CURL..LATIN SMALL LETTER J WITH CROSSED-TAIL +029F..02A8 ; Technical # 1.1 [10] LATIN LETTER SMALL CAPITAL L..LATIN SMALL LETTER TC DIGRAPH WITH CURL +02A9..02AD ; Technical # 3.0 [5] LATIN SMALL LETTER FENG DIGRAPH..LATIN LETTER BIDENTAL PERCUSSIVE +02AE..02AF ; Technical # 4.0 [2] LATIN SMALL LETTER TURNED H WITH FISHHOOK..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +02B9..02BA ; Technical # 1.1 [2] MODIFIER LETTER PRIME..MODIFIER LETTER DOUBLE PRIME +02BD..02C1 ; Technical # 1.1 [5] MODIFIER LETTER REVERSED COMMA..MODIFIER LETTER REVERSED GLOTTAL STOP +02C6..02D1 ; Technical # 1.1 [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02EE ; Technical # 3.0 MODIFIER LETTER DOUBLE APOSTROPHE +030E ; Technical # 1.1 COMBINING DOUBLE VERTICAL LINE ABOVE +0312 ; Technical # 1.1 COMBINING TURNED COMMA ABOVE +0315 ; Technical # 1.1 COMBINING COMMA ABOVE RIGHT +0317..031A ; Technical # 1.1 [4] COMBINING ACUTE ACCENT BELOW..COMBINING LEFT ANGLE ABOVE +031C..0320 ; Technical # 1.1 [5] COMBINING LEFT HALF RING BELOW..COMBINING MINUS SIGN BELOW +0329..032C ; Technical # 1.1 [4] COMBINING VERTICAL LINE BELOW..COMBINING CARON BELOW +032F ; Technical # 1.1 COMBINING INVERTED BREVE BELOW +0333 ; Technical # 1.1 COMBINING DOUBLE LOW LINE +0337 ; Technical # 1.1 COMBINING SHORT SOLIDUS OVERLAY +033A..033F ; Technical # 1.1 [6] COMBINING INVERTED BRIDGE BELOW..COMBINING DOUBLE OVERLINE +0346..034E ; Technical # 3.0 [9] COMBINING BRIDGE ABOVE..COMBINING UPWARDS ARROW BELOW +0350..0357 ; Technical # 4.0 [8] COMBINING RIGHT ARROWHEAD ABOVE..COMBINING RIGHT HALF RING ABOVE +0359..035C ; Technical # 4.1 [4] COMBINING ASTERISK BELOW..COMBINING DOUBLE BREVE BELOW +035D..035F ; Technical # 4.0 [3] COMBINING DOUBLE BREVE..COMBINING DOUBLE MACRON BELOW +0360..0361 ; Technical # 1.1 [2] COMBINING DOUBLE TILDE..COMBINING DOUBLE INVERTED BREVE +0362 ; Technical # 3.0 COMBINING DOUBLE RIGHTWARDS ARROW BELOW +03CF ; Technical # 5.1 GREEK CAPITAL KAI SYMBOL +03D7 ; Technical # 3.0 GREEK KAI SYMBOL +0560 ; Technical # 11.0 ARMENIAN SMALL LETTER TURNED AYB +0588 ; Technical # 11.0 ARMENIAN SMALL LETTER YI WITH STROKE +0953..0954 ; Technical # 1.1 [2] DEVANAGARI GRAVE ACCENT..DEVANAGARI ACUTE ACCENT +0D81 ; Technical # 13.0 SINHALA SIGN CANDRABINDU +0F18..0F19 ; Technical # 2.0 [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS +17CE..17CF ; Technical # 3.0 [2] KHMER SIGN KAKABAT..KHMER SIGN AHSDA +1ABF..1AC0 ; Technical # 13.0 [2] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER TURNED W BELOW +1D00..1D2B ; Technical # 4.0 [44] LATIN LETTER SMALL CAPITAL A..CYRILLIC LETTER SMALL CAPITAL EL +1D2F ; Technical # 4.0 MODIFIER LETTER CAPITAL BARRED B +1D3B ; Technical # 4.0 MODIFIER LETTER CAPITAL REVERSED N +1D4E ; Technical # 4.0 MODIFIER LETTER SMALL TURNED I +1D6B ; Technical # 4.0 LATIN SMALL LETTER UE +1D6C..1D77 ; Technical # 4.1 [12] LATIN SMALL LETTER B WITH MIDDLE TILDE..LATIN SMALL LETTER TURNED G +1D79..1D9A ; Technical # 4.1 [34] LATIN SMALL LETTER INSULAR G..LATIN SMALL LETTER EZH WITH RETROFLEX HOOK +1DC4..1DCA ; Technical # 5.0 [7] COMBINING MACRON-ACUTE..COMBINING LATIN SMALL LETTER R BELOW +1DCB..1DCD ; Technical # 5.1 [3] COMBINING BREVE-MACRON..COMBINING DOUBLE CIRCUMFLEX ABOVE +1DCF..1DD0 ; Technical # 5.1 [2] COMBINING ZIGZAG BELOW..COMBINING IS BELOW +1DE7..1DF5 ; Technical # 7.0 [15] COMBINING LATIN SMALL LETTER ALPHA..COMBINING UP TACK ABOVE +1DF6..1DF9 ; Technical # 10.0 [4] COMBINING KAVYKA ABOVE RIGHT..COMBINING WIDE INVERTED BRIDGE BELOW +1DFB ; Technical # 9.0 COMBINING DELETION MARK +1DFC ; Technical # 6.0 COMBINING DOUBLE INVERTED BREVE BELOW +1DFD ; Technical # 5.2 COMBINING ALMOST EQUAL TO BELOW +1DFE..1DFF ; Technical # 5.0 [2] COMBINING LEFT ARROWHEAD ABOVE..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW +1E9C..1E9D ; Technical # 5.1 [2] LATIN SMALL LETTER LONG S WITH DIAGONAL STROKE..LATIN SMALL LETTER LONG S WITH HIGH STROKE +1E9F ; Technical # 5.1 LATIN SMALL LETTER DELTA +1EFA..1EFF ; Technical # 5.1 [6] LATIN CAPITAL LETTER MIDDLE-WELSH LL..LATIN SMALL LETTER Y WITH LOOP +203F..2040 ; Technical # 1.1 [2] UNDERTIE..CHARACTER TIE +20D0..20DC ; Technical # 1.1 [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20E1 ; Technical # 1.1 COMBINING LEFT RIGHT ARROW ABOVE +20E5..20EA ; Technical # 3.2 [6] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING LEFTWARDS ARROW OVERLAY +20EB ; Technical # 4.1 COMBINING LONG DOUBLE SOLIDUS OVERLAY +20EC..20EF ; Technical # 5.0 [4] COMBINING RIGHTWARDS HARPOON WITH BARB DOWNWARDS..COMBINING RIGHT ARROW BELOW +20F0 ; Technical # 5.1 COMBINING ASTERISK ABOVE +2118 ; Technical # 1.1 SCRIPT CAPITAL P +212E ; Technical # 1.1 ESTIMATED SYMBOL +2C60..2C67 ; Technical # 5.0 [8] LATIN CAPITAL LETTER L WITH DOUBLE BAR..LATIN CAPITAL LETTER H WITH DESCENDER +2C77 ; Technical # 5.0 LATIN SMALL LETTER TAILLESS PHI +2C78..2C7B ; Technical # 5.1 [4] LATIN SMALL LETTER E WITH NOTCH..LATIN LETTER SMALL CAPITAL TURNED E +3021..302D ; Technical # 1.1 [13] HANGZHOU NUMERAL ONE..IDEOGRAPHIC ENTERING TONE MARK +3031..3035 ; Technical # 1.1 [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +303B..303C ; Technical # 3.2 [2] VERTICAL IDEOGRAPHIC ITERATION MARK..MASU MARK +A78E ; Technical # 6.0 LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT +A7AF ; Technical # 11.0 LATIN LETTER SMALL CAPITAL Q +A7BA..A7BF ; Technical # 12.0 [6] LATIN CAPITAL LETTER GLOTTAL A..LATIN SMALL LETTER GLOTTAL U +A7FA ; Technical # 6.0 LATIN LETTER SMALL CAPITAL TURNED M +AB68 ; Technical # 13.0 LATIN SMALL LETTER TURNED R WITH MIDDLE TILDE +FE20..FE23 ; Technical # 1.1 [4] COMBINING LIGATURE LEFT HALF..COMBINING DOUBLE TILDE RIGHT HALF +FE24..FE26 ; Technical # 5.1 [3] COMBINING MACRON LEFT HALF..COMBINING CONJOINING MACRON +FE27..FE2D ; Technical # 7.0 [7] COMBINING LIGATURE LEFT HALF BELOW..COMBINING CONJOINING MACRON BELOW +FE73 ; Technical # 3.2 ARABIC TAIL FRAGMENT +1CF00..1CF2D ; Technical # 14.0 [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT +1CF30..1CF46 ; Technical # 14.0 [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG +1D165..1D169 ; Technical # 3.1 [5] MUSICAL SYMBOL COMBINING STEM..MUSICAL SYMBOL COMBINING TREMOLO-3 +1D16D..1D172 ; Technical # 3.1 [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 +1D17B..1D182 ; Technical # 3.1 [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE +1D185..1D18B ; Technical # 3.1 [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE +1D1AA..1D1AD ; Technical # 3.1 [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO + +# Total code points: 500 + +# Identifier_Type: Technical Exclusion + +2CF0..2CF1 ; Technical Exclusion # 5.2 [2] COPTIC COMBINING SPIRITUS ASPER..COPTIC COMBINING SPIRITUS LENIS + +# Total code points: 2 + +# Identifier_Type: Technical Obsolete + +018D ; Technical Obsolete # 1.1 LATIN SMALL LETTER TURNED DELTA +01AA..01AB ; Technical Obsolete # 1.1 [2] LATIN LETTER REVERSED ESH LOOP..LATIN SMALL LETTER T WITH PALATAL HOOK +01BA..01BB ; Technical Obsolete # 1.1 [2] LATIN SMALL LETTER EZH WITH TAIL..LATIN LETTER TWO WITH STROKE +01BE ; Technical Obsolete # 1.1 LATIN LETTER INVERTED GLOTTAL STOP WITH STROKE +0277 ; Technical Obsolete # 1.1 LATIN SMALL LETTER CLOSED OMEGA +027C ; Technical Obsolete # 1.1 LATIN SMALL LETTER R WITH LONG LEG +029E ; Technical Obsolete # 1.1 LATIN SMALL LETTER TURNED K +03F3 ; Technical Obsolete # 1.1 GREEK LETTER YOT +0484..0486 ; Technical Obsolete # 1.1 [3] COMBINING CYRILLIC PALATALIZATION..COMBINING CYRILLIC PSILI PNEUMATA +0487 ; Technical Obsolete # 5.1 COMBINING CYRILLIC POKRYTIE +0D04 ; Technical Obsolete # 13.0 MALAYALAM LETTER VEDIC ANUSVARA +17D1 ; Technical Obsolete # 3.0 KHMER SIGN VIRIAM +17DD ; Technical Obsolete # 4.0 KHMER SIGN ATTHACAN +1DC0..1DC3 ; Technical Obsolete # 4.1 [4] COMBINING DOTTED GRAVE ACCENT..COMBINING SUSPENSION MARK +1DCE ; Technical Obsolete # 5.1 COMBINING OGONEK ABOVE +1DD1..1DE6 ; Technical Obsolete # 5.1 [22] COMBINING UR ABOVE..COMBINING LATIN SMALL LETTER Z +2180..2182 ; Technical Obsolete # 1.1 [3] ROMAN NUMERAL ONE THOUSAND C D..ROMAN NUMERAL TEN THOUSAND +2183 ; Technical Obsolete # 3.0 ROMAN NUMERAL REVERSED ONE HUNDRED +302E..302F ; Technical Obsolete # 1.1 [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK +A722..A72F ; Technical Obsolete # 5.1 [14] LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF..LATIN SMALL LETTER CUATRILLO WITH COMMA +1D242..1D244 ; Technical Obsolete # 4.1 [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME + +# Total code points: 67 + +# Identifier_Type: Technical Obsolete Not_XID + +2E00..2E0D ; Technical Obsolete Not_XID # 4.1 [14] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT RAISED OMISSION BRACKET + +# Total code points: 14 + +# Identifier_Type: Technical Not_XID + +20DD..20E0 ; Technical Not_XID # 1.1 [4] COMBINING ENCLOSING CIRCLE..COMBINING ENCLOSING CIRCLE BACKSLASH +20E2..20E3 ; Technical Not_XID # 3.0 [2] COMBINING ENCLOSING SCREEN..COMBINING ENCLOSING KEYCAP +20E4 ; Technical Not_XID # 3.2 COMBINING ENCLOSING UPWARD POINTING TRIANGLE +24EB..24FE ; Technical Not_XID # 3.2 [20] NEGATIVE CIRCLED NUMBER ELEVEN..DOUBLE CIRCLED NUMBER TEN +24FF ; Technical Not_XID # 4.0 NEGATIVE CIRCLED DIGIT ZERO +2800..28FF ; Technical Not_XID # 3.0 [256] BRAILLE PATTERN BLANK..BRAILLE PATTERN DOTS-12345678 +327F ; Technical Not_XID # 1.1 KOREAN STANDARD SYMBOL +4DC0..4DFF ; Technical Not_XID # 4.0 [64] HEXAGRAM FOR THE CREATIVE HEAVEN..HEXAGRAM FOR BEFORE COMPLETION +A708..A716 ; Technical Not_XID # 4.1 [15] MODIFIER LETTER EXTRA-HIGH DOTTED TONE BAR..MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR +FBB2..FBC1 ; Technical Not_XID # 6.0 [16] ARABIC SYMBOL DOT ABOVE..ARABIC SYMBOL SMALL TAH BELOW +FBC2 ; Technical Not_XID # 14.0 ARABIC SYMBOL WASLA ABOVE +FD3E..FD3F ; Technical Not_XID # 1.1 [2] ORNATE LEFT PARENTHESIS..ORNATE RIGHT PARENTHESIS +FD40..FD4F ; Technical Not_XID # 14.0 [16] ARABIC LIGATURE RAHIMAHU ALLAAH..ARABIC LIGATURE RAHIMAHUM ALLAAH +FDCF ; Technical Not_XID # 14.0 ARABIC LIGATURE SALAAMUHU ALAYNAA +FDFD ; Technical Not_XID # 4.0 ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM +FDFE..FDFF ; Technical Not_XID # 14.0 [2] ARABIC LIGATURE SUBHAANAHU WA TAAALAA..ARABIC LIGATURE AZZA WA JALL +FE45..FE46 ; Technical Not_XID # 3.2 [2] SESAME DOT..WHITE SESAME DOT +1CF50..1CFC3 ; Technical Not_XID # 14.0 [116] ZNAMENNY NEUME KRYUK..ZNAMENNY NEUME PAUK +1D000..1D0F5 ; Technical Not_XID # 3.1 [246] BYZANTINE MUSICAL SYMBOL PSILI..BYZANTINE MUSICAL SYMBOL GORGON NEO KATO +1D100..1D126 ; Technical Not_XID # 3.1 [39] MUSICAL SYMBOL SINGLE BARLINE..MUSICAL SYMBOL DRUM CLEF-2 +1D129 ; Technical Not_XID # 5.1 MUSICAL SYMBOL MULTIPLE MEASURE REST +1D12A..1D15D ; Technical Not_XID # 3.1 [52] MUSICAL SYMBOL DOUBLE SHARP..MUSICAL SYMBOL WHOLE NOTE +1D16A..1D16C ; Technical Not_XID # 3.1 [3] MUSICAL SYMBOL FINGERED TREMOLO-1..MUSICAL SYMBOL FINGERED TREMOLO-3 +1D183..1D184 ; Technical Not_XID # 3.1 [2] MUSICAL SYMBOL ARPEGGIATO UP..MUSICAL SYMBOL ARPEGGIATO DOWN +1D18C..1D1A9 ; Technical Not_XID # 3.1 [30] MUSICAL SYMBOL RINFORZANDO..MUSICAL SYMBOL DEGREE SLASH +1D1AE..1D1BA ; Technical Not_XID # 3.1 [13] MUSICAL SYMBOL PEDAL MARK..MUSICAL SYMBOL SEMIBREVIS BLACK +1D1C1..1D1DD ; Technical Not_XID # 3.1 [29] MUSICAL SYMBOL LONGA PERFECTA REST..MUSICAL SYMBOL PES SUBPUNCTIS +1D1E9..1D1EA ; Technical Not_XID # 14.0 [2] MUSICAL SYMBOL SORI..MUSICAL SYMBOL KORON +1D300..1D356 ; Technical Not_XID # 4.0 [87] MONOGRAM FOR EARTH..TETRAGRAM FOR FOSTERING + +# Total code points: 1025 + +# Identifier_Type: Exclusion + +03E2..03EF ; Exclusion # 1.1 [14] COPTIC CAPITAL LETTER SHEI..COPTIC SMALL LETTER DEI +0800..082D ; Exclusion # 5.2 [46] SAMARITAN LETTER ALAF..SAMARITAN MARK NEQUDAA +1681..169A ; Exclusion # 3.0 [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +16A0..16EA ; Exclusion # 3.0 [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EE..16F0 ; Exclusion # 3.0 [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +16F1..16F8 ; Exclusion # 7.0 [8] RUNIC LETTER K..RUNIC LETTER FRANKS CASKET AESC +1700..170C ; Exclusion # 3.2 [13] TAGALOG LETTER A..TAGALOG LETTER YA +170D ; Exclusion # 14.0 TAGALOG LETTER RA +170E..1714 ; Exclusion # 3.2 [7] TAGALOG LETTER LA..TAGALOG SIGN VIRAMA +1715 ; Exclusion # 14.0 TAGALOG SIGN PAMUDPOD +171F ; Exclusion # 14.0 TAGALOG LETTER ARCHAIC RA +1720..1734 ; Exclusion # 3.2 [21] HANUNOO LETTER A..HANUNOO SIGN PAMUDPOD +1740..1753 ; Exclusion # 3.2 [20] BUHID LETTER A..BUHID VOWEL SIGN U +1760..176C ; Exclusion # 3.2 [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770 ; Exclusion # 3.2 [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1772..1773 ; Exclusion # 3.2 [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +1810..1819 ; Exclusion # 3.0 [10] MONGOLIAN DIGIT ZERO..MONGOLIAN DIGIT NINE +1820..1877 ; Exclusion # 3.0 [88] MONGOLIAN LETTER A..MONGOLIAN LETTER MANCHU ZHA +1878 ; Exclusion # 11.0 MONGOLIAN LETTER CHA WITH TWO DOTS +1880..18A8 ; Exclusion # 3.0 [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA +18AA ; Exclusion # 5.1 MONGOLIAN LETTER MANCHU ALI GALI LHA +1A00..1A1B ; Exclusion # 4.1 [28] BUGINESE LETTER KA..BUGINESE VOWEL SIGN AE +1CFA ; Exclusion # 12.0 VEDIC SIGN DOUBLE ANUSVARA ANTARGOMUKHA +2C00..2C2E ; Exclusion # 4.1 [47] GLAGOLITIC CAPITAL LETTER AZU..GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE +2C2F ; Exclusion # 14.0 GLAGOLITIC CAPITAL LETTER CAUDATE CHRIVI +2C30..2C5E ; Exclusion # 4.1 [47] GLAGOLITIC SMALL LETTER AZU..GLAGOLITIC SMALL LETTER LATINATE MYSLITE +2C5F ; Exclusion # 14.0 GLAGOLITIC SMALL LETTER CAUDATE CHRIVI +2C80..2CE4 ; Exclusion # 4.1 [101] COPTIC CAPITAL LETTER ALFA..COPTIC SYMBOL KAI +2CEB..2CEF ; Exclusion # 5.2 [5] COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI..COPTIC COMBINING NI ABOVE +2CF2..2CF3 ; Exclusion # 6.1 [2] COPTIC CAPITAL LETTER BOHAIRIC KHEI..COPTIC SMALL LETTER BOHAIRIC KHEI +A840..A873 ; Exclusion # 5.0 [52] PHAGS-PA LETTER KA..PHAGS-PA LETTER CANDRABINDU +A930..A953 ; Exclusion # 5.1 [36] REJANG LETTER KA..REJANG VIRAMA +10000..1000B ; Exclusion # 4.0 [12] LINEAR B SYLLABLE B008 A..LINEAR B SYLLABLE B046 JE +1000D..10026 ; Exclusion # 4.0 [26] LINEAR B SYLLABLE B036 JO..LINEAR B SYLLABLE B032 QO +10028..1003A ; Exclusion # 4.0 [19] LINEAR B SYLLABLE B060 RA..LINEAR B SYLLABLE B042 WO +1003C..1003D ; Exclusion # 4.0 [2] LINEAR B SYLLABLE B017 ZA..LINEAR B SYLLABLE B074 ZE +1003F..1004D ; Exclusion # 4.0 [15] LINEAR B SYLLABLE B020 ZO..LINEAR B SYLLABLE B091 TWO +10050..1005D ; Exclusion # 4.0 [14] LINEAR B SYMBOL B018..LINEAR B SYMBOL B089 +10080..100FA ; Exclusion # 4.0 [123] LINEAR B IDEOGRAM B100 MAN..LINEAR B IDEOGRAM VESSEL B305 +10280..1029C ; Exclusion # 5.1 [29] LYCIAN LETTER A..LYCIAN LETTER X +102A0..102D0 ; Exclusion # 5.1 [49] CARIAN LETTER A..CARIAN LETTER UUU3 +10300..1031E ; Exclusion # 3.1 [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU +1031F ; Exclusion # 7.0 OLD ITALIC LETTER ESS +1032D..1032F ; Exclusion # 10.0 [3] OLD ITALIC LETTER YE..OLD ITALIC LETTER SOUTHERN TSE +10330..1034A ; Exclusion # 3.1 [27] GOTHIC LETTER AHSA..GOTHIC LETTER NINE HUNDRED +10350..1037A ; Exclusion # 7.0 [43] OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII +10380..1039D ; Exclusion # 4.0 [30] UGARITIC LETTER ALPA..UGARITIC LETTER SSU +103A0..103C3 ; Exclusion # 4.1 [36] OLD PERSIAN SIGN A..OLD PERSIAN SIGN HA +103C8..103CF ; Exclusion # 4.1 [8] OLD PERSIAN SIGN AURAMAZDAA..OLD PERSIAN SIGN BUUMISH +103D1..103D5 ; Exclusion # 4.1 [5] OLD PERSIAN NUMBER ONE..OLD PERSIAN NUMBER HUNDRED +10400..10425 ; Exclusion # 3.1 [38] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER ENG +10426..10427 ; Exclusion # 4.0 [2] DESERET CAPITAL LETTER OI..DESERET CAPITAL LETTER EW +10428..1044D ; Exclusion # 3.1 [38] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER ENG +1044E..1049D ; Exclusion # 4.0 [80] DESERET SMALL LETTER OI..OSMANYA LETTER OO +104A0..104A9 ; Exclusion # 4.0 [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE +10500..10527 ; Exclusion # 7.0 [40] ELBASAN LETTER A..ELBASAN LETTER KHE +10530..10563 ; Exclusion # 7.0 [52] CAUCASIAN ALBANIAN LETTER ALT..CAUCASIAN ALBANIAN LETTER KIW +10570..1057A ; Exclusion # 14.0 [11] VITHKUQI CAPITAL LETTER A..VITHKUQI CAPITAL LETTER GA +1057C..1058A ; Exclusion # 14.0 [15] VITHKUQI CAPITAL LETTER HA..VITHKUQI CAPITAL LETTER RE +1058C..10592 ; Exclusion # 14.0 [7] VITHKUQI CAPITAL LETTER SE..VITHKUQI CAPITAL LETTER XE +10594..10595 ; Exclusion # 14.0 [2] VITHKUQI CAPITAL LETTER Y..VITHKUQI CAPITAL LETTER ZE +10597..105A1 ; Exclusion # 14.0 [11] VITHKUQI SMALL LETTER A..VITHKUQI SMALL LETTER GA +105A3..105B1 ; Exclusion # 14.0 [15] VITHKUQI SMALL LETTER HA..VITHKUQI SMALL LETTER RE +105B3..105B9 ; Exclusion # 14.0 [7] VITHKUQI SMALL LETTER SE..VITHKUQI SMALL LETTER XE +105BB..105BC ; Exclusion # 14.0 [2] VITHKUQI SMALL LETTER Y..VITHKUQI SMALL LETTER ZE +10600..10736 ; Exclusion # 7.0 [311] LINEAR A SIGN AB001..LINEAR A SIGN A664 +10740..10755 ; Exclusion # 7.0 [22] LINEAR A SIGN A701 A..LINEAR A SIGN A732 JE +10760..10767 ; Exclusion # 7.0 [8] LINEAR A SIGN A800..LINEAR A SIGN A807 +10800..10805 ; Exclusion # 4.0 [6] CYPRIOT SYLLABLE A..CYPRIOT SYLLABLE JA +10808 ; Exclusion # 4.0 CYPRIOT SYLLABLE JO +1080A..10835 ; Exclusion # 4.0 [44] CYPRIOT SYLLABLE KA..CYPRIOT SYLLABLE WO +10837..10838 ; Exclusion # 4.0 [2] CYPRIOT SYLLABLE XA..CYPRIOT SYLLABLE XE +1083C ; Exclusion # 4.0 CYPRIOT SYLLABLE ZA +1083F ; Exclusion # 4.0 CYPRIOT SYLLABLE ZO +10840..10855 ; Exclusion # 5.2 [22] IMPERIAL ARAMAIC LETTER ALEPH..IMPERIAL ARAMAIC LETTER TAW +10860..10876 ; Exclusion # 7.0 [23] PALMYRENE LETTER ALEPH..PALMYRENE LETTER TAW +10880..1089E ; Exclusion # 7.0 [31] NABATAEAN LETTER FINAL ALEPH..NABATAEAN LETTER TAW +108E0..108F2 ; Exclusion # 8.0 [19] HATRAN LETTER ALEPH..HATRAN LETTER QOPH +108F4..108F5 ; Exclusion # 8.0 [2] HATRAN LETTER SHIN..HATRAN LETTER TAW +10900..10915 ; Exclusion # 5.0 [22] PHOENICIAN LETTER ALF..PHOENICIAN LETTER TAU +10920..10939 ; Exclusion # 5.1 [26] LYDIAN LETTER A..LYDIAN LETTER C +10980..109B7 ; Exclusion # 6.1 [56] MEROITIC HIEROGLYPHIC LETTER A..MEROITIC CURSIVE LETTER DA +109BE..109BF ; Exclusion # 6.1 [2] MEROITIC CURSIVE LOGOGRAM RMT..MEROITIC CURSIVE LOGOGRAM IMN +10A00..10A03 ; Exclusion # 4.1 [4] KHAROSHTHI LETTER A..KHAROSHTHI VOWEL SIGN VOCALIC R +10A05..10A06 ; Exclusion # 4.1 [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O +10A0C..10A13 ; Exclusion # 4.1 [8] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI LETTER GHA +10A15..10A17 ; Exclusion # 4.1 [3] KHAROSHTHI LETTER CA..KHAROSHTHI LETTER JA +10A19..10A33 ; Exclusion # 4.1 [27] KHAROSHTHI LETTER NYA..KHAROSHTHI LETTER TTTHA +10A34..10A35 ; Exclusion # 11.0 [2] KHAROSHTHI LETTER TTTA..KHAROSHTHI LETTER VHA +10A38..10A3A ; Exclusion # 4.1 [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW +10A3F ; Exclusion # 4.1 KHAROSHTHI VIRAMA +10A60..10A7C ; Exclusion # 5.2 [29] OLD SOUTH ARABIAN LETTER HE..OLD SOUTH ARABIAN LETTER THETH +10A80..10A9C ; Exclusion # 7.0 [29] OLD NORTH ARABIAN LETTER HEH..OLD NORTH ARABIAN LETTER ZAH +10AC0..10AC7 ; Exclusion # 7.0 [8] MANICHAEAN LETTER ALEPH..MANICHAEAN LETTER WAW +10AC9..10AE6 ; Exclusion # 7.0 [30] MANICHAEAN LETTER ZAYIN..MANICHAEAN ABBREVIATION MARK BELOW +10B00..10B35 ; Exclusion # 5.2 [54] AVESTAN LETTER A..AVESTAN LETTER HE +10B40..10B55 ; Exclusion # 5.2 [22] INSCRIPTIONAL PARTHIAN LETTER ALEPH..INSCRIPTIONAL PARTHIAN LETTER TAW +10B60..10B72 ; Exclusion # 5.2 [19] INSCRIPTIONAL PAHLAVI LETTER ALEPH..INSCRIPTIONAL PAHLAVI LETTER TAW +10B80..10B91 ; Exclusion # 7.0 [18] PSALTER PAHLAVI LETTER ALEPH..PSALTER PAHLAVI LETTER TAW +10C00..10C48 ; Exclusion # 5.2 [73] OLD TURKIC LETTER ORKHON A..OLD TURKIC LETTER ORKHON BASH +10C80..10CB2 ; Exclusion # 8.0 [51] OLD HUNGARIAN CAPITAL LETTER A..OLD HUNGARIAN CAPITAL LETTER US +10CC0..10CF2 ; Exclusion # 8.0 [51] OLD HUNGARIAN SMALL LETTER A..OLD HUNGARIAN SMALL LETTER US +10E80..10EA9 ; Exclusion # 13.0 [42] YEZIDI LETTER ELIF..YEZIDI LETTER ET +10EAB..10EAC ; Exclusion # 13.0 [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK +10EB0..10EB1 ; Exclusion # 13.0 [2] YEZIDI LETTER LAM WITH DOT ABOVE..YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE +10F00..10F1C ; Exclusion # 11.0 [29] OLD SOGDIAN LETTER ALEPH..OLD SOGDIAN LETTER FINAL TAW WITH VERTICAL TAIL +10F27 ; Exclusion # 11.0 OLD SOGDIAN LIGATURE AYIN-DALETH +10F30..10F50 ; Exclusion # 11.0 [33] SOGDIAN LETTER ALEPH..SOGDIAN COMBINING STROKE BELOW +10F70..10F85 ; Exclusion # 14.0 [22] OLD UYGHUR LETTER ALEPH..OLD UYGHUR COMBINING TWO DOTS BELOW +10FB0..10FC4 ; Exclusion # 13.0 [21] CHORASMIAN LETTER ALEPH..CHORASMIAN LETTER TAW +10FE0..10FF6 ; Exclusion # 12.0 [23] ELYMAIC LETTER ALEPH..ELYMAIC LIGATURE ZAYIN-YODH +11000..11046 ; Exclusion # 6.0 [71] BRAHMI SIGN CANDRABINDU..BRAHMI VIRAMA +11066..1106F ; Exclusion # 6.0 [10] BRAHMI DIGIT ZERO..BRAHMI DIGIT NINE +11070..11075 ; Exclusion # 14.0 [6] BRAHMI SIGN OLD TAMIL VIRAMA..BRAHMI LETTER OLD TAMIL LLA +1107F ; Exclusion # 7.0 BRAHMI NUMBER JOINER +11080..110BA ; Exclusion # 5.2 [59] KAITHI SIGN CANDRABINDU..KAITHI SIGN NUKTA +110C2 ; Exclusion # 14.0 KAITHI VOWEL SIGN VOCALIC R +110D0..110E8 ; Exclusion # 6.1 [25] SORA SOMPENG LETTER SAH..SORA SOMPENG LETTER MAE +110F0..110F9 ; Exclusion # 6.1 [10] SORA SOMPENG DIGIT ZERO..SORA SOMPENG DIGIT NINE +11150..11173 ; Exclusion # 7.0 [36] MAHAJANI LETTER A..MAHAJANI SIGN NUKTA +11176 ; Exclusion # 7.0 MAHAJANI LIGATURE SHRI +11180..111C4 ; Exclusion # 6.1 [69] SHARADA SIGN CANDRABINDU..SHARADA OM +111C9..111CC ; Exclusion # 8.0 [4] SHARADA SANDHI MARK..SHARADA EXTRA SHORT VOWEL MARK +111CE..111CF ; Exclusion # 13.0 [2] SHARADA VOWEL SIGN PRISHTHAMATRA E..SHARADA SIGN INVERTED CANDRABINDU +111D0..111D9 ; Exclusion # 6.1 [10] SHARADA DIGIT ZERO..SHARADA DIGIT NINE +111DA ; Exclusion # 7.0 SHARADA EKAM +111DC ; Exclusion # 8.0 SHARADA HEADSTROKE +11200..11211 ; Exclusion # 7.0 [18] KHOJKI LETTER A..KHOJKI LETTER JJA +11213..11237 ; Exclusion # 7.0 [37] KHOJKI LETTER NYA..KHOJKI SIGN SHADDA +1123E ; Exclusion # 9.0 KHOJKI SIGN SUKUN +11280..11286 ; Exclusion # 8.0 [7] MULTANI LETTER A..MULTANI LETTER GA +11288 ; Exclusion # 8.0 MULTANI LETTER GHA +1128A..1128D ; Exclusion # 8.0 [4] MULTANI LETTER CA..MULTANI LETTER JJA +1128F..1129D ; Exclusion # 8.0 [15] MULTANI LETTER NYA..MULTANI LETTER BA +1129F..112A8 ; Exclusion # 8.0 [10] MULTANI LETTER BHA..MULTANI LETTER RHA +112B0..112EA ; Exclusion # 7.0 [59] KHUDAWADI LETTER A..KHUDAWADI SIGN VIRAMA +112F0..112F9 ; Exclusion # 7.0 [10] KHUDAWADI DIGIT ZERO..KHUDAWADI DIGIT NINE +11300 ; Exclusion # 8.0 GRANTHA SIGN COMBINING ANUSVARA ABOVE +11302 ; Exclusion # 7.0 GRANTHA SIGN ANUSVARA +11305..1130C ; Exclusion # 7.0 [8] GRANTHA LETTER A..GRANTHA LETTER VOCALIC L +1130F..11310 ; Exclusion # 7.0 [2] GRANTHA LETTER EE..GRANTHA LETTER AI +11313..11328 ; Exclusion # 7.0 [22] GRANTHA LETTER OO..GRANTHA LETTER NA +1132A..11330 ; Exclusion # 7.0 [7] GRANTHA LETTER PA..GRANTHA LETTER RA +11332..11333 ; Exclusion # 7.0 [2] GRANTHA LETTER LA..GRANTHA LETTER LLA +11335..11339 ; Exclusion # 7.0 [5] GRANTHA LETTER VA..GRANTHA LETTER HA +1133D..11344 ; Exclusion # 7.0 [8] GRANTHA SIGN AVAGRAHA..GRANTHA VOWEL SIGN VOCALIC RR +11347..11348 ; Exclusion # 7.0 [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI +1134B..1134D ; Exclusion # 7.0 [3] GRANTHA VOWEL SIGN OO..GRANTHA SIGN VIRAMA +11350 ; Exclusion # 8.0 GRANTHA OM +11357 ; Exclusion # 7.0 GRANTHA AU LENGTH MARK +1135D..11363 ; Exclusion # 7.0 [7] GRANTHA SIGN PLUTA..GRANTHA VOWEL SIGN VOCALIC LL +11366..1136C ; Exclusion # 7.0 [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX +11370..11374 ; Exclusion # 7.0 [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA +11480..114C5 ; Exclusion # 7.0 [70] TIRHUTA ANJI..TIRHUTA GVANG +114C7 ; Exclusion # 7.0 TIRHUTA OM +114D0..114D9 ; Exclusion # 7.0 [10] TIRHUTA DIGIT ZERO..TIRHUTA DIGIT NINE +11580..115B5 ; Exclusion # 7.0 [54] SIDDHAM LETTER A..SIDDHAM VOWEL SIGN VOCALIC RR +115B8..115C0 ; Exclusion # 7.0 [9] SIDDHAM VOWEL SIGN E..SIDDHAM SIGN NUKTA +115D8..115DD ; Exclusion # 8.0 [6] SIDDHAM LETTER THREE-CIRCLE ALTERNATE I..SIDDHAM VOWEL SIGN ALTERNATE UU +11600..11640 ; Exclusion # 7.0 [65] MODI LETTER A..MODI SIGN ARDHACANDRA +11644 ; Exclusion # 7.0 MODI SIGN HUVA +11650..11659 ; Exclusion # 7.0 [10] MODI DIGIT ZERO..MODI DIGIT NINE +11680..116B7 ; Exclusion # 6.1 [56] TAKRI LETTER A..TAKRI SIGN NUKTA +116B8 ; Exclusion # 12.0 TAKRI LETTER ARCHAIC KHA +116C0..116C9 ; Exclusion # 6.1 [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE +11700..11719 ; Exclusion # 8.0 [26] AHOM LETTER KA..AHOM LETTER JHA +1171A ; Exclusion # 11.0 AHOM LETTER ALTERNATE BA +1171D..1172B ; Exclusion # 8.0 [15] AHOM CONSONANT SIGN MEDIAL LA..AHOM SIGN KILLER +11730..11739 ; Exclusion # 8.0 [10] AHOM DIGIT ZERO..AHOM DIGIT NINE +11740..11746 ; Exclusion # 14.0 [7] AHOM LETTER CA..AHOM LETTER LLA +11800..1183A ; Exclusion # 11.0 [59] DOGRA LETTER A..DOGRA SIGN NUKTA +118A0..118E9 ; Exclusion # 7.0 [74] WARANG CITI CAPITAL LETTER NGAA..WARANG CITI DIGIT NINE +118FF ; Exclusion # 7.0 WARANG CITI OM +11900..11906 ; Exclusion # 13.0 [7] DIVES AKURU LETTER A..DIVES AKURU LETTER E +11909 ; Exclusion # 13.0 DIVES AKURU LETTER O +1190C..11913 ; Exclusion # 13.0 [8] DIVES AKURU LETTER KA..DIVES AKURU LETTER JA +11915..11916 ; Exclusion # 13.0 [2] DIVES AKURU LETTER NYA..DIVES AKURU LETTER TTA +11918..11935 ; Exclusion # 13.0 [30] DIVES AKURU LETTER DDA..DIVES AKURU VOWEL SIGN E +11937..11938 ; Exclusion # 13.0 [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O +1193B..11943 ; Exclusion # 13.0 [9] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN NUKTA +11950..11959 ; Exclusion # 13.0 [10] DIVES AKURU DIGIT ZERO..DIVES AKURU DIGIT NINE +119A0..119A7 ; Exclusion # 12.0 [8] NANDINAGARI LETTER A..NANDINAGARI LETTER VOCALIC RR +119AA..119D7 ; Exclusion # 12.0 [46] NANDINAGARI LETTER E..NANDINAGARI VOWEL SIGN VOCALIC RR +119DA..119E1 ; Exclusion # 12.0 [8] NANDINAGARI VOWEL SIGN E..NANDINAGARI SIGN AVAGRAHA +119E3..119E4 ; Exclusion # 12.0 [2] NANDINAGARI HEADSTROKE..NANDINAGARI VOWEL SIGN PRISHTHAMATRA E +11A00..11A3E ; Exclusion # 10.0 [63] ZANABAZAR SQUARE LETTER A..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA +11A47 ; Exclusion # 10.0 ZANABAZAR SQUARE SUBJOINER +11A50..11A83 ; Exclusion # 10.0 [52] SOYOMBO LETTER A..SOYOMBO LETTER KSSA +11A84..11A85 ; Exclusion # 12.0 [2] SOYOMBO SIGN JIHVAMULIYA..SOYOMBO SIGN UPADHMANIYA +11A86..11A99 ; Exclusion # 10.0 [20] SOYOMBO CLUSTER-INITIAL LETTER RA..SOYOMBO SUBJOINER +11A9D ; Exclusion # 11.0 SOYOMBO MARK PLUTA +11AC0..11AF8 ; Exclusion # 7.0 [57] PAU CIN HAU LETTER PA..PAU CIN HAU GLOTTAL STOP FINAL +11C00..11C08 ; Exclusion # 9.0 [9] BHAIKSUKI LETTER A..BHAIKSUKI LETTER VOCALIC L +11C0A..11C36 ; Exclusion # 9.0 [45] BHAIKSUKI LETTER E..BHAIKSUKI VOWEL SIGN VOCALIC L +11C38..11C40 ; Exclusion # 9.0 [9] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN AVAGRAHA +11C50..11C59 ; Exclusion # 9.0 [10] BHAIKSUKI DIGIT ZERO..BHAIKSUKI DIGIT NINE +11C72..11C8F ; Exclusion # 9.0 [30] MARCHEN LETTER KA..MARCHEN LETTER A +11C92..11CA7 ; Exclusion # 9.0 [22] MARCHEN SUBJOINED LETTER KA..MARCHEN SUBJOINED LETTER ZA +11CA9..11CB6 ; Exclusion # 9.0 [14] MARCHEN SUBJOINED LETTER YA..MARCHEN SIGN CANDRABINDU +11D00..11D06 ; Exclusion # 10.0 [7] MASARAM GONDI LETTER A..MASARAM GONDI LETTER E +11D08..11D09 ; Exclusion # 10.0 [2] MASARAM GONDI LETTER AI..MASARAM GONDI LETTER O +11D0B..11D36 ; Exclusion # 10.0 [44] MASARAM GONDI LETTER AU..MASARAM GONDI VOWEL SIGN VOCALIC R +11D3A ; Exclusion # 10.0 MASARAM GONDI VOWEL SIGN E +11D3C..11D3D ; Exclusion # 10.0 [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O +11D3F..11D47 ; Exclusion # 10.0 [9] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI RA-KARA +11D50..11D59 ; Exclusion # 10.0 [10] MASARAM GONDI DIGIT ZERO..MASARAM GONDI DIGIT NINE +11EE0..11EF6 ; Exclusion # 11.0 [23] MAKASAR LETTER KA..MAKASAR VOWEL SIGN O +12000..1236E ; Exclusion # 5.0 [879] CUNEIFORM SIGN A..CUNEIFORM SIGN ZUM +1236F..12398 ; Exclusion # 7.0 [42] CUNEIFORM SIGN KAP ELAMITE..CUNEIFORM SIGN UM TIMES ME +12399 ; Exclusion # 8.0 CUNEIFORM SIGN U U +12400..12462 ; Exclusion # 5.0 [99] CUNEIFORM NUMERIC SIGN TWO ASH..CUNEIFORM NUMERIC SIGN OLD ASSYRIAN ONE QUARTER +12463..1246E ; Exclusion # 7.0 [12] CUNEIFORM NUMERIC SIGN ONE QUARTER GUR..CUNEIFORM NUMERIC SIGN NINE U VARIANT FORM +12480..12543 ; Exclusion # 8.0 [196] CUNEIFORM SIGN AB TIMES NUN TENU..CUNEIFORM SIGN ZU5 TIMES THREE DISH TENU +12F90..12FF0 ; Exclusion # 14.0 [97] CYPRO-MINOAN SIGN CM001..CYPRO-MINOAN SIGN CM114 +13000..1342E ; Exclusion # 5.2 [1071] EGYPTIAN HIEROGLYPH A001..EGYPTIAN HIEROGLYPH AA032 +14400..14646 ; Exclusion # 8.0 [583] ANATOLIAN HIEROGLYPH A001..ANATOLIAN HIEROGLYPH A530 +16A70..16ABE ; Exclusion # 14.0 [79] TANGSA LETTER OZ..TANGSA LETTER ZA +16AC0..16AC9 ; Exclusion # 14.0 [10] TANGSA DIGIT ZERO..TANGSA DIGIT NINE +16AD0..16AED ; Exclusion # 7.0 [30] BASSA VAH LETTER ENNI..BASSA VAH LETTER I +16AF0..16AF4 ; Exclusion # 7.0 [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE +16B00..16B36 ; Exclusion # 7.0 [55] PAHAWH HMONG VOWEL KEEB..PAHAWH HMONG MARK CIM TAUM +16B40..16B43 ; Exclusion # 7.0 [4] PAHAWH HMONG SIGN VOS SEEV..PAHAWH HMONG SIGN IB YAM +16B50..16B59 ; Exclusion # 7.0 [10] PAHAWH HMONG DIGIT ZERO..PAHAWH HMONG DIGIT NINE +16B63..16B77 ; Exclusion # 7.0 [21] PAHAWH HMONG SIGN VOS LUB..PAHAWH HMONG SIGN CIM NRES TOS +16B7D..16B8F ; Exclusion # 7.0 [19] PAHAWH HMONG CLAN SIGN TSHEEJ..PAHAWH HMONG CLAN SIGN VWJ +16E40..16E7F ; Exclusion # 11.0 [64] MEDEFAIDRIN CAPITAL LETTER M..MEDEFAIDRIN SMALL LETTER Y +16FE0 ; Exclusion # 9.0 TANGUT ITERATION MARK +16FE1 ; Exclusion # 10.0 NUSHU ITERATION MARK +16FE4 ; Exclusion # 13.0 KHITAN SMALL SCRIPT FILLER +17000..187EC ; Exclusion # 9.0 [6125] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187EC +187ED..187F1 ; Exclusion # 11.0 [5] TANGUT IDEOGRAPH-187ED..TANGUT IDEOGRAPH-187F1 +187F2..187F7 ; Exclusion # 12.0 [6] TANGUT IDEOGRAPH-187F2..TANGUT IDEOGRAPH-187F7 +18800..18AF2 ; Exclusion # 9.0 [755] TANGUT COMPONENT-001..TANGUT COMPONENT-755 +18AF3..18CD5 ; Exclusion # 13.0 [483] TANGUT COMPONENT-756..KHITAN SMALL SCRIPT CHARACTER-18CD5 +18D00..18D08 ; Exclusion # 13.0 [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08 +1B170..1B2FB ; Exclusion # 10.0 [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB +1BC00..1BC6A ; Exclusion # 7.0 [107] DUPLOYAN LETTER H..DUPLOYAN LETTER VOCALIC M +1BC70..1BC7C ; Exclusion # 7.0 [13] DUPLOYAN AFFIX LEFT HORIZONTAL SECANT..DUPLOYAN AFFIX ATTACHED TANGENT HOOK +1BC80..1BC88 ; Exclusion # 7.0 [9] DUPLOYAN AFFIX HIGH ACUTE..DUPLOYAN AFFIX HIGH VERTICAL +1BC90..1BC99 ; Exclusion # 7.0 [10] DUPLOYAN AFFIX LOW ACUTE..DUPLOYAN AFFIX LOW ARROW +1BC9D..1BC9E ; Exclusion # 7.0 [2] DUPLOYAN THICK LETTER SELECTOR..DUPLOYAN DOUBLE MARK +1DA00..1DA36 ; Exclusion # 8.0 [55] SIGNWRITING HEAD RIM..SIGNWRITING AIR SUCKING IN +1DA3B..1DA6C ; Exclusion # 8.0 [50] SIGNWRITING MOUTH CLOSED NEUTRAL..SIGNWRITING EXCITEMENT +1DA75 ; Exclusion # 8.0 SIGNWRITING UPPER BODY TILTING FROM HIP JOINTS +1DA84 ; Exclusion # 8.0 SIGNWRITING LOCATION HEAD NECK +1DA9B..1DA9F ; Exclusion # 8.0 [5] SIGNWRITING FILL MODIFIER-2..SIGNWRITING FILL MODIFIER-6 +1DAA1..1DAAF ; Exclusion # 8.0 [15] SIGNWRITING ROTATION MODIFIER-2..SIGNWRITING ROTATION MODIFIER-16 +1E000..1E006 ; Exclusion # 9.0 [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE +1E008..1E018 ; Exclusion # 9.0 [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU +1E01B..1E021 ; Exclusion # 9.0 [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI +1E023..1E024 ; Exclusion # 9.0 [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS +1E026..1E02A ; Exclusion # 9.0 [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA +1E290..1E2AE ; Exclusion # 14.0 [31] TOTO LETTER PA..TOTO SIGN RISING TONE +1E800..1E8C4 ; Exclusion # 7.0 [197] MENDE KIKAKUI SYLLABLE M001 KI..MENDE KIKAKUI SYLLABLE M060 NYON +1E8D0..1E8D6 ; Exclusion # 7.0 [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS + +# Total code points: 15930 + +# Identifier_Type: Exclusion Not_XID + +0830..083E ; Exclusion Not_XID # 5.2 [15] SAMARITAN PUNCTUATION NEQUDAA..SAMARITAN PUNCTUATION ANNAAU +1680 ; Exclusion Not_XID # 3.0 OGHAM SPACE MARK +169B..169C ; Exclusion Not_XID # 3.0 [2] OGHAM FEATHER MARK..OGHAM REVERSED FEATHER MARK +1735..1736 ; Exclusion Not_XID # 3.2 [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION +1800..180A ; Exclusion Not_XID # 3.0 [11] MONGOLIAN BIRGA..MONGOLIAN NIRUGU +1A1E..1A1F ; Exclusion Not_XID # 4.1 [2] BUGINESE PALLAWA..BUGINESE END OF SECTION +2CE5..2CEA ; Exclusion Not_XID # 4.1 [6] COPTIC SYMBOL MI RO..COPTIC SYMBOL SHIMA SIMA +2CF9..2CFF ; Exclusion Not_XID # 4.1 [7] COPTIC OLD NUBIAN FULL STOP..COPTIC MORPHOLOGICAL DIVIDER +A874..A877 ; Exclusion Not_XID # 5.0 [4] PHAGS-PA SINGLE HEAD MARK..PHAGS-PA MARK DOUBLE SHAD +A95F ; Exclusion Not_XID # 5.1 REJANG SECTION MARK +10100..10102 ; Exclusion Not_XID # 4.0 [3] AEGEAN WORD SEPARATOR LINE..AEGEAN CHECK MARK +10107..10133 ; Exclusion Not_XID # 4.0 [45] AEGEAN NUMBER ONE..AEGEAN NUMBER NINETY THOUSAND +10137..1013F ; Exclusion Not_XID # 4.0 [9] AEGEAN WEIGHT BASE UNIT..AEGEAN MEASURE THIRD SUBUNIT +10320..10323 ; Exclusion Not_XID # 3.1 [4] OLD ITALIC NUMERAL ONE..OLD ITALIC NUMERAL FIFTY +1039F ; Exclusion Not_XID # 4.0 UGARITIC WORD DIVIDER +103D0 ; Exclusion Not_XID # 4.1 OLD PERSIAN WORD DIVIDER +1056F ; Exclusion Not_XID # 7.0 CAUCASIAN ALBANIAN CITATION MARK +10857..1085F ; Exclusion Not_XID # 5.2 [9] IMPERIAL ARAMAIC SECTION SIGN..IMPERIAL ARAMAIC NUMBER TEN THOUSAND +10877..1087F ; Exclusion Not_XID # 7.0 [9] PALMYRENE LEFT-POINTING FLEURON..PALMYRENE NUMBER TWENTY +108A7..108AF ; Exclusion Not_XID # 7.0 [9] NABATAEAN NUMBER ONE..NABATAEAN NUMBER ONE HUNDRED +108FB..108FF ; Exclusion Not_XID # 8.0 [5] HATRAN NUMBER ONE..HATRAN NUMBER ONE HUNDRED +10916..10919 ; Exclusion Not_XID # 5.0 [4] PHOENICIAN NUMBER ONE..PHOENICIAN NUMBER ONE HUNDRED +1091A..1091B ; Exclusion Not_XID # 5.2 [2] PHOENICIAN NUMBER TWO..PHOENICIAN NUMBER THREE +1091F ; Exclusion Not_XID # 5.0 PHOENICIAN WORD SEPARATOR +1093F ; Exclusion Not_XID # 5.1 LYDIAN TRIANGULAR MARK +109BC..109BD ; Exclusion Not_XID # 8.0 [2] MEROITIC CURSIVE FRACTION ELEVEN TWELFTHS..MEROITIC CURSIVE FRACTION ONE HALF +109C0..109CF ; Exclusion Not_XID # 8.0 [16] MEROITIC CURSIVE NUMBER ONE..MEROITIC CURSIVE NUMBER SEVENTY +109D2..109FF ; Exclusion Not_XID # 8.0 [46] MEROITIC CURSIVE NUMBER ONE HUNDRED..MEROITIC CURSIVE FRACTION TEN TWELFTHS +10A40..10A47 ; Exclusion Not_XID # 4.1 [8] KHAROSHTHI DIGIT ONE..KHAROSHTHI NUMBER ONE THOUSAND +10A48 ; Exclusion Not_XID # 11.0 KHAROSHTHI FRACTION ONE HALF +10A50..10A58 ; Exclusion Not_XID # 4.1 [9] KHAROSHTHI PUNCTUATION DOT..KHAROSHTHI PUNCTUATION LINES +10A7D..10A7F ; Exclusion Not_XID # 5.2 [3] OLD SOUTH ARABIAN NUMBER ONE..OLD SOUTH ARABIAN NUMERIC INDICATOR +10A9D..10A9F ; Exclusion Not_XID # 7.0 [3] OLD NORTH ARABIAN NUMBER ONE..OLD NORTH ARABIAN NUMBER TWENTY +10AC8 ; Exclusion Not_XID # 7.0 MANICHAEAN SIGN UD +10AEB..10AF6 ; Exclusion Not_XID # 7.0 [12] MANICHAEAN NUMBER ONE..MANICHAEAN PUNCTUATION LINE FILLER +10B39..10B3F ; Exclusion Not_XID # 5.2 [7] AVESTAN ABBREVIATION MARK..LARGE ONE RING OVER TWO RINGS PUNCTUATION +10B58..10B5F ; Exclusion Not_XID # 5.2 [8] INSCRIPTIONAL PARTHIAN NUMBER ONE..INSCRIPTIONAL PARTHIAN NUMBER ONE THOUSAND +10B78..10B7F ; Exclusion Not_XID # 5.2 [8] INSCRIPTIONAL PAHLAVI NUMBER ONE..INSCRIPTIONAL PAHLAVI NUMBER ONE THOUSAND +10B99..10B9C ; Exclusion Not_XID # 7.0 [4] PSALTER PAHLAVI SECTION MARK..PSALTER PAHLAVI FOUR DOTS WITH DOT +10BA9..10BAF ; Exclusion Not_XID # 7.0 [7] PSALTER PAHLAVI NUMBER ONE..PSALTER PAHLAVI NUMBER ONE HUNDRED +10CFA..10CFF ; Exclusion Not_XID # 8.0 [6] OLD HUNGARIAN NUMBER ONE..OLD HUNGARIAN NUMBER ONE THOUSAND +10EAD ; Exclusion Not_XID # 13.0 YEZIDI HYPHENATION MARK +10F1D..10F26 ; Exclusion Not_XID # 11.0 [10] OLD SOGDIAN NUMBER ONE..OLD SOGDIAN FRACTION ONE HALF +10F51..10F59 ; Exclusion Not_XID # 11.0 [9] SOGDIAN NUMBER ONE..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT +10F86..10F89 ; Exclusion Not_XID # 14.0 [4] OLD UYGHUR PUNCTUATION BAR..OLD UYGHUR PUNCTUATION FOUR DOTS +10FC5..10FCB ; Exclusion Not_XID # 13.0 [7] CHORASMIAN NUMBER ONE..CHORASMIAN NUMBER ONE HUNDRED +11047..1104D ; Exclusion Not_XID # 6.0 [7] BRAHMI DANDA..BRAHMI PUNCTUATION LOTUS +11052..11065 ; Exclusion Not_XID # 6.0 [20] BRAHMI NUMBER ONE..BRAHMI NUMBER ONE THOUSAND +110BB..110BC ; Exclusion Not_XID # 5.2 [2] KAITHI ABBREVIATION SIGN..KAITHI ENUMERATION SIGN +110BD ; Exclusion Not_XID # 5.2 KAITHI NUMBER SIGN +110BE..110C1 ; Exclusion Not_XID # 5.2 [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA +110CD ; Exclusion Not_XID # 11.0 KAITHI NUMBER SIGN ABOVE +11174..11175 ; Exclusion Not_XID # 7.0 [2] MAHAJANI ABBREVIATION SIGN..MAHAJANI SECTION MARK +111C5..111C8 ; Exclusion Not_XID # 6.1 [4] SHARADA DANDA..SHARADA SEPARATOR +111CD ; Exclusion Not_XID # 7.0 SHARADA SUTRA MARK +111DB ; Exclusion Not_XID # 8.0 SHARADA SIGN SIDDHAM +111DD..111DF ; Exclusion Not_XID # 8.0 [3] SHARADA CONTINUATION SIGN..SHARADA SECTION MARK-2 +11238..1123D ; Exclusion Not_XID # 7.0 [6] KHOJKI DANDA..KHOJKI ABBREVIATION SIGN +112A9 ; Exclusion Not_XID # 8.0 MULTANI SECTION MARK +114C6 ; Exclusion Not_XID # 7.0 TIRHUTA ABBREVIATION SIGN +115C1..115C9 ; Exclusion Not_XID # 7.0 [9] SIDDHAM SIGN SIDDHAM..SIDDHAM END OF TEXT MARK +115CA..115D7 ; Exclusion Not_XID # 8.0 [14] SIDDHAM SECTION MARK WITH TRIDENT AND U-SHAPED ORNAMENTS..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES +11641..11643 ; Exclusion Not_XID # 7.0 [3] MODI DANDA..MODI ABBREVIATION SIGN +11660..1166C ; Exclusion Not_XID # 9.0 [13] MONGOLIAN BIRGA WITH ORNAMENT..MONGOLIAN TURNED SWIRL BIRGA WITH DOUBLE ORNAMENT +116B9 ; Exclusion Not_XID # 14.0 TAKRI ABBREVIATION SIGN +1173A..1173F ; Exclusion Not_XID # 8.0 [6] AHOM NUMBER TEN..AHOM SYMBOL VI +1183B ; Exclusion Not_XID # 11.0 DOGRA ABBREVIATION SIGN +118EA..118F2 ; Exclusion Not_XID # 7.0 [9] WARANG CITI NUMBER TEN..WARANG CITI NUMBER NINETY +11944..11946 ; Exclusion Not_XID # 13.0 [3] DIVES AKURU DOUBLE DANDA..DIVES AKURU END OF TEXT MARK +119E2 ; Exclusion Not_XID # 12.0 NANDINAGARI SIGN SIDDHAM +11A3F..11A46 ; Exclusion Not_XID # 10.0 [8] ZANABAZAR SQUARE INITIAL HEAD MARK..ZANABAZAR SQUARE CLOSING DOUBLE-LINED HEAD MARK +11A9A..11A9C ; Exclusion Not_XID # 10.0 [3] SOYOMBO MARK TSHEG..SOYOMBO MARK DOUBLE SHAD +11A9E..11AA2 ; Exclusion Not_XID # 10.0 [5] SOYOMBO HEAD MARK WITH MOON AND SUN AND TRIPLE FLAME..SOYOMBO TERMINAL MARK-2 +11C41..11C45 ; Exclusion Not_XID # 9.0 [5] BHAIKSUKI DANDA..BHAIKSUKI GAP FILLER-2 +11C5A..11C6C ; Exclusion Not_XID # 9.0 [19] BHAIKSUKI NUMBER ONE..BHAIKSUKI HUNDREDS UNIT MARK +11C70..11C71 ; Exclusion Not_XID # 9.0 [2] MARCHEN HEAD MARK..MARCHEN MARK SHAD +11EF7..11EF8 ; Exclusion Not_XID # 11.0 [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION +12470..12473 ; Exclusion Not_XID # 5.0 [4] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL TRICOLON +12474 ; Exclusion Not_XID # 7.0 CUNEIFORM PUNCTUATION SIGN DIAGONAL QUADCOLON +12FF1..12FF2 ; Exclusion Not_XID # 14.0 [2] CYPRO-MINOAN SIGN CM301..CYPRO-MINOAN SIGN CM302 +13430..13438 ; Exclusion Not_XID # 12.0 [9] EGYPTIAN HIEROGLYPH VERTICAL JOINER..EGYPTIAN HIEROGLYPH END SEGMENT +16A6E..16A6F ; Exclusion Not_XID # 7.0 [2] MRO DANDA..MRO DOUBLE DANDA +16AF5 ; Exclusion Not_XID # 7.0 BASSA VAH FULL STOP +16B37..16B3F ; Exclusion Not_XID # 7.0 [9] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN XYEEM FAIB +16B44..16B45 ; Exclusion Not_XID # 7.0 [2] PAHAWH HMONG SIGN XAUS..PAHAWH HMONG SIGN CIM TSOV ROG +16B5B..16B61 ; Exclusion Not_XID # 7.0 [7] PAHAWH HMONG NUMBER TENS..PAHAWH HMONG NUMBER TRILLIONS +16E80..16E9A ; Exclusion Not_XID # 11.0 [27] MEDEFAIDRIN DIGIT ZERO..MEDEFAIDRIN EXCLAMATION OH +1BC9C ; Exclusion Not_XID # 7.0 DUPLOYAN SIGN O WITH CROSS +1BC9F ; Exclusion Not_XID # 7.0 DUPLOYAN PUNCTUATION CHINOOK FULL STOP +1D800..1D9FF ; Exclusion Not_XID # 8.0 [512] SIGNWRITING HAND-FIST INDEX..SIGNWRITING HEAD +1DA37..1DA3A ; Exclusion Not_XID # 8.0 [4] SIGNWRITING AIR BLOW SMALL ROTATIONS..SIGNWRITING BREATH EXHALE +1DA6D..1DA74 ; Exclusion Not_XID # 8.0 [8] SIGNWRITING SHOULDER HIP SPINE..SIGNWRITING TORSO-FLOORPLANE TWISTING +1DA76..1DA83 ; Exclusion Not_XID # 8.0 [14] SIGNWRITING LIMB COMBINATION..SIGNWRITING LOCATION DEPTH +1DA85..1DA8B ; Exclusion Not_XID # 8.0 [7] SIGNWRITING LOCATION TORSO..SIGNWRITING PARENTHESIS +1E8C7..1E8CF ; Exclusion Not_XID # 7.0 [9] MENDE KIKAKUI DIGIT ONE..MENDE KIKAKUI DIGIT NINE + +# Total code points: 1105 + +# Identifier_Type: Obsolete + +01B9 ; Obsolete # 1.1 LATIN SMALL LETTER EZH REVERSED +01BF ; Obsolete # 1.1 LATIN LETTER WYNN +01F6..01F7 ; Obsolete # 3.0 [2] LATIN CAPITAL LETTER HWAIR..LATIN CAPITAL LETTER WYNN +021C..021D ; Obsolete # 3.0 [2] LATIN CAPITAL LETTER YOGH..LATIN SMALL LETTER YOGH +0363..036F ; Obsolete # 3.2 [13] COMBINING LATIN SMALL LETTER A..COMBINING LATIN SMALL LETTER X +0370..0373 ; Obsolete # 5.1 [4] GREEK CAPITAL LETTER HETA..GREEK SMALL LETTER ARCHAIC SAMPI +0376..0377 ; Obsolete # 5.1 [2] GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA..GREEK SMALL LETTER PAMPHYLIAN DIGAMMA +037F ; Obsolete # 7.0 GREEK CAPITAL LETTER YOT +03D8..03D9 ; Obsolete # 3.2 [2] GREEK LETTER ARCHAIC KOPPA..GREEK SMALL LETTER ARCHAIC KOPPA +03DA ; Obsolete # 1.1 GREEK LETTER STIGMA +03DB ; Obsolete # 3.0 GREEK SMALL LETTER STIGMA +03DC ; Obsolete # 1.1 GREEK LETTER DIGAMMA +03DD ; Obsolete # 3.0 GREEK SMALL LETTER DIGAMMA +03DE ; Obsolete # 1.1 GREEK LETTER KOPPA +03DF ; Obsolete # 3.0 GREEK SMALL LETTER KOPPA +03E0 ; Obsolete # 1.1 GREEK LETTER SAMPI +03E1 ; Obsolete # 3.0 GREEK SMALL LETTER SAMPI +03F7..03F8 ; Obsolete # 4.0 [2] GREEK CAPITAL LETTER SHO..GREEK SMALL LETTER SHO +03FA..03FB ; Obsolete # 4.0 [2] GREEK CAPITAL LETTER SAN..GREEK SMALL LETTER SAN +0460..0481 ; Obsolete # 1.1 [34] CYRILLIC CAPITAL LETTER OMEGA..CYRILLIC SMALL LETTER KOPPA +0483 ; Obsolete # 1.1 COMBINING CYRILLIC TITLO +0500..050F ; Obsolete # 3.2 [16] CYRILLIC CAPITAL LETTER KOMI DE..CYRILLIC SMALL LETTER KOMI TJE +052A..052D ; Obsolete # 7.0 [4] CYRILLIC CAPITAL LETTER DZZHE..CYRILLIC SMALL LETTER DCHE +0640 ; Obsolete # 1.1 ARABIC TATWEEL +066E..066F ; Obsolete # 3.2 [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +068E ; Obsolete # 1.1 ARABIC LETTER DUL +06A1 ; Obsolete # 1.1 ARABIC LETTER DOTLESS FEH +08AD..08B1 ; Obsolete # 7.0 [5] ARABIC LETTER LOW ALEF..ARABIC LETTER STRAIGHT WAW +094E ; Obsolete # 5.2 DEVANAGARI VOWEL SIGN PRISHTHAMATRA E +0951..0952 ; Obsolete # 1.1 [2] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI STRESS SIGN ANUDATTA +0978 ; Obsolete # 7.0 DEVANAGARI LETTER MARWARI DDA +0980 ; Obsolete # 7.0 BENGALI ANJI +09FC ; Obsolete # 10.0 BENGALI LETTER VEDIC ANUSVARA +0C00 ; Obsolete # 7.0 TELUGU SIGN COMBINING CANDRABINDU ABOVE +0C34 ; Obsolete # 7.0 TELUGU LETTER LLLA +0C58..0C59 ; Obsolete # 5.1 [2] TELUGU LETTER TSA..TELUGU LETTER DZA +0C81 ; Obsolete # 7.0 KANNADA SIGN CANDRABINDU +0CDE ; Obsolete # 1.1 KANNADA LETTER FA +0D01 ; Obsolete # 7.0 MALAYALAM SIGN CANDRABINDU +0D3B..0D3C ; Obsolete # 10.0 [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA +0D5F ; Obsolete # 8.0 MALAYALAM LETTER ARCHAIC II +0DE6..0DEF ; Obsolete # 7.0 [10] SINHALA LITH DIGIT ZERO..SINHALA LITH DIGIT NINE +10A0..10C5 ; Obsolete # 1.1 [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10F1..10F6 ; Obsolete # 1.1 [6] GEORGIAN LETTER HE..GEORGIAN LETTER FI +1100..1159 ; Obsolete # 1.1 [90] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG YEORINHIEUH +115A..115E ; Obsolete # 5.2 [5] HANGUL CHOSEONG KIYEOK-TIKEUT..HANGUL CHOSEONG TIKEUT-RIEUL +1161..11A2 ; Obsolete # 1.1 [66] HANGUL JUNGSEONG A..HANGUL JUNGSEONG SSANGARAEA +11A3..11A7 ; Obsolete # 5.2 [5] HANGUL JUNGSEONG A-EU..HANGUL JUNGSEONG O-YAE +11A8..11F9 ; Obsolete # 1.1 [82] HANGUL JONGSEONG KIYEOK..HANGUL JONGSEONG YEORINHIEUH +11FA..11FF ; Obsolete # 5.2 [6] HANGUL JONGSEONG KIYEOK-NIEUN..HANGUL JONGSEONG SSANGNIEUN +1369..1371 ; Obsolete # 3.0 [9] ETHIOPIC DIGIT ONE..ETHIOPIC DIGIT NINE +17A8 ; Obsolete # 3.0 KHMER INDEPENDENT VOWEL QUK +17D3 ; Obsolete # 3.0 KHMER SIGN BATHAMASAT +1AB0..1ABD ; Obsolete # 7.0 [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW +1C80..1C88 ; Obsolete # 9.0 [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK +1CD0..1CD2 ; Obsolete # 5.2 [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA +1CD4..1CF2 ; Obsolete # 5.2 [31] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC SIGN ARDHAVISARGA +1CF3..1CF6 ; Obsolete # 6.1 [4] VEDIC SIGN ROTATED ARDHAVISARGA..VEDIC SIGN UPADHMANIYA +1CF7 ; Obsolete # 10.0 VEDIC SIGN ATIKRAMA +1CF8..1CF9 ; Obsolete # 7.0 [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE +2132 ; Obsolete # 1.1 TURNED CAPITAL F +214E ; Obsolete # 5.0 TURNED SMALL F +2184 ; Obsolete # 5.0 LATIN SMALL LETTER REVERSED C +2185..2188 ; Obsolete # 5.1 [4] ROMAN NUMERAL SIX LATE FORM..ROMAN NUMERAL ONE HUNDRED THOUSAND +2C6D..2C6F ; Obsolete # 5.1 [3] LATIN CAPITAL LETTER ALPHA..LATIN CAPITAL LETTER TURNED A +2C70 ; Obsolete # 5.2 LATIN CAPITAL LETTER TURNED ALPHA +2C71..2C73 ; Obsolete # 5.1 [3] LATIN SMALL LETTER V WITH RIGHT HOOK..LATIN SMALL LETTER W WITH HOOK +2C74..2C76 ; Obsolete # 5.0 [3] LATIN SMALL LETTER V WITH CURL..LATIN SMALL LETTER HALF H +2C7E..2C7F ; Obsolete # 5.2 [2] LATIN CAPITAL LETTER S WITH SWASH TAIL..LATIN CAPITAL LETTER Z WITH SWASH TAIL +2D00..2D25 ; Obsolete # 4.1 [38] GEORGIAN SMALL LETTER AN..GEORGIAN SMALL LETTER HOE +2DE0..2DFF ; Obsolete # 5.1 [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS +312E ; Obsolete # 10.0 BOPOMOFO LETTER O WITH DOT ABOVE +31F0..31FF ; Obsolete # 3.2 [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +A640..A65F ; Obsolete # 5.1 [32] CYRILLIC CAPITAL LETTER ZEMLYA..CYRILLIC SMALL LETTER YN +A660..A661 ; Obsolete # 6.0 [2] CYRILLIC CAPITAL LETTER REVERSED TSE..CYRILLIC SMALL LETTER REVERSED TSE +A662..A66E ; Obsolete # 5.1 [13] CYRILLIC CAPITAL LETTER SOFT DE..CYRILLIC LETTER MULTIOCULAR O +A674..A67B ; Obsolete # 6.1 [8] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC LETTER OMEGA +A680..A697 ; Obsolete # 5.1 [24] CYRILLIC CAPITAL LETTER DWE..CYRILLIC SMALL LETTER SHWE +A698..A69B ; Obsolete # 7.0 [4] CYRILLIC CAPITAL LETTER DOUBLE O..CYRILLIC SMALL LETTER CROSSED O +A69F ; Obsolete # 6.1 COMBINING CYRILLIC LETTER IOTIFIED E +A730..A76F ; Obsolete # 5.1 [64] LATIN LETTER SMALL CAPITAL F..LATIN SMALL LETTER CON +A771..A787 ; Obsolete # 5.1 [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T +A790..A791 ; Obsolete # 6.0 [2] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER N WITH DESCENDER +A794..A79F ; Obsolete # 7.0 [12] LATIN SMALL LETTER C WITH PALATAL HOOK..LATIN SMALL LETTER VOLAPUK UE +A7A0..A7A9 ; Obsolete # 6.0 [10] LATIN CAPITAL LETTER G WITH OBLIQUE STROKE..LATIN SMALL LETTER S WITH OBLIQUE STROKE +A7AB..A7AD ; Obsolete # 7.0 [3] LATIN CAPITAL LETTER REVERSED OPEN E..LATIN CAPITAL LETTER L WITH BELT +A7B0..A7B1 ; Obsolete # 7.0 [2] LATIN CAPITAL LETTER TURNED K..LATIN CAPITAL LETTER TURNED T +A7F5..A7F6 ; Obsolete # 13.0 [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H +A7F7 ; Obsolete # 7.0 LATIN EPIGRAPHIC LETTER SIDEWAYS I +A7FB..A7FF ; Obsolete # 5.1 [5] LATIN EPIGRAPHIC LETTER REVERSED F..LATIN EPIGRAPHIC LETTER ARCHAIC M +A8E0..A8F7 ; Obsolete # 5.2 [24] COMBINING DEVANAGARI DIGIT ZERO..DEVANAGARI SIGN CANDRABINDU AVAGRAHA +A8FB ; Obsolete # 5.2 DEVANAGARI HEADSTROKE +A8FE..A8FF ; Obsolete # 11.0 [2] DEVANAGARI LETTER AY..DEVANAGARI VOWEL SIGN AY +A960..A97C ; Obsolete # 5.2 [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANGUL CHOSEONG SSANGYEORINHIEUH +A9E0..A9E6 ; Obsolete # 7.0 [7] MYANMAR LETTER SHAN GHA..MYANMAR MODIFIER LETTER SHAN REDUPLICATION +AB30..AB5A ; Obsolete # 7.0 [43] LATIN SMALL LETTER BARRED ALPHA..LATIN SMALL LETTER Y WITH SHORT RIGHT LEG +AB64..AB65 ; Obsolete # 7.0 [2] LATIN SMALL LETTER INVERTED ALPHA..GREEK LETTER SMALL CAPITAL OMEGA +D7B0..D7C6 ; Obsolete # 5.2 [23] HANGUL JUNGSEONG O-YEO..HANGUL JUNGSEONG ARAEA-E +D7CB..D7FB ; Obsolete # 5.2 [49] HANGUL JONGSEONG NIEUN-RIEUL..HANGUL JONGSEONG PHIEUPH-THIEUTH +10140..10174 ; Obsolete # 4.1 [53] GREEK ACROPHONIC ATTIC ONE QUARTER..GREEK ACROPHONIC STRATIAN FIFTY MNAS +101FD ; Obsolete # 5.1 PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE +102E0 ; Obsolete # 7.0 COPTIC EPACT THOUSANDS MARK +16FE3 ; Obsolete # 12.0 OLD CHINESE ITERATION MARK +1B000..1B001 ; Obsolete # 6.0 [2] KATAKANA LETTER ARCHAIC E..HIRAGANA LETTER ARCHAIC YE +1B002..1B11E ; Obsolete # 10.0 [285] HENTAIGANA LETTER A-1..HENTAIGANA LETTER N-MU-MO-2 + +# Total code points: 1341 + +# Identifier_Type: Obsolete Not_XID + +0482 ; Obsolete Not_XID # 1.1 CYRILLIC THOUSANDS SIGN +0488..0489 ; Obsolete Not_XID # 3.0 [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN +05C6 ; Obsolete Not_XID # 4.1 HEBREW PUNCTUATION NUN HAFUKHA +17D8 ; Obsolete Not_XID # 3.0 KHMER SIGN BEYYAL +1CD3 ; Obsolete Not_XID # 5.2 VEDIC SIGN NIHSHVASA +2056 ; Obsolete Not_XID # 4.1 THREE DOT PUNCTUATION +2058..205E ; Obsolete Not_XID # 4.1 [7] FOUR DOT PUNCTUATION..VERTICAL FOUR DOTS +2127 ; Obsolete Not_XID # 1.1 INVERTED OHM SIGN +214F ; Obsolete Not_XID # 5.1 SYMBOL FOR SAMARITAN SOURCE +2E0E..2E16 ; Obsolete Not_XID # 4.1 [9] EDITORIAL CORONIS..DOTTED RIGHT-POINTING ANGLE +2E2A..2E30 ; Obsolete Not_XID # 5.1 [7] TWO DOTS OVER ONE DOT PUNCTUATION..RING POINT +2E31 ; Obsolete Not_XID # 5.2 WORD SEPARATOR MIDDLE DOT +2E32 ; Obsolete Not_XID # 6.1 TURNED COMMA +2E35 ; Obsolete Not_XID # 6.1 TURNED SEMICOLON +2E39 ; Obsolete Not_XID # 6.1 TOP HALF SECTION SIGN +301E ; Obsolete Not_XID # 1.1 DOUBLE PRIME QUOTATION MARK +A670..A673 ; Obsolete Not_XID # 5.1 [4] COMBINING CYRILLIC TEN MILLIONS SIGN..SLAVONIC ASTERISK +A700..A707 ; Obsolete Not_XID # 4.1 [8] MODIFIER LETTER CHINESE TONE YIN PING..MODIFIER LETTER CHINESE TONE YANG RU +A8F8..A8FA ; Obsolete Not_XID # 5.2 [3] DEVANAGARI SIGN PUSHPIKA..DEVANAGARI CARET +101D0..101FC ; Obsolete Not_XID # 5.1 [45] PHAISTOS DISC SIGN PEDESTRIAN..PHAISTOS DISC SIGN WAVY BAND +102E1..102FB ; Obsolete Not_XID # 7.0 [27] COPTIC EPACT DIGIT ONE..COPTIC EPACT NUMBER NINE HUNDRED +1D200..1D241 ; Obsolete Not_XID # 4.1 [66] GREEK VOCAL NOTATION SYMBOL-1..GREEK INSTRUMENTAL NOTATION SYMBOL-54 +1D245 ; Obsolete Not_XID # 4.1 GREEK MUSICAL LEIMMA + +# Total code points: 191 + +# Identifier_Type: Not_XID + +0009..000D ; Not_XID # 1.1 [5] .. +0020..0026 ; Not_XID # 1.1 [7] SPACE..AMPERSAND +0028..002C ; Not_XID # 1.1 [5] LEFT PARENTHESIS..COMMA +002F ; Not_XID # 1.1 SOLIDUS +003B..0040 ; Not_XID # 1.1 [6] SEMICOLON..COMMERCIAL AT +005B..005E ; Not_XID # 1.1 [4] LEFT SQUARE BRACKET..CIRCUMFLEX ACCENT +0060 ; Not_XID # 1.1 GRAVE ACCENT +007B..007E ; Not_XID # 1.1 [4] LEFT CURLY BRACKET..TILDE +0085 ; Not_XID # 1.1 +00A1..00A7 ; Not_XID # 1.1 [7] INVERTED EXCLAMATION MARK..SECTION SIGN +00A9 ; Not_XID # 1.1 COPYRIGHT SIGN +00AB..00AC ; Not_XID # 1.1 [2] LEFT-POINTING DOUBLE ANGLE QUOTATION MARK..NOT SIGN +00AE ; Not_XID # 1.1 REGISTERED SIGN +00B0..00B1 ; Not_XID # 1.1 [2] DEGREE SIGN..PLUS-MINUS SIGN +00B6 ; Not_XID # 1.1 PILCROW SIGN +00BB ; Not_XID # 1.1 RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +00BF ; Not_XID # 1.1 INVERTED QUESTION MARK +00D7 ; Not_XID # 1.1 MULTIPLICATION SIGN +00F7 ; Not_XID # 1.1 DIVISION SIGN +02C2..02C5 ; Not_XID # 1.1 [4] MODIFIER LETTER LEFT ARROWHEAD..MODIFIER LETTER DOWN ARROWHEAD +02D2..02D7 ; Not_XID # 1.1 [6] MODIFIER LETTER CENTRED RIGHT HALF RING..MODIFIER LETTER MINUS SIGN +02DE ; Not_XID # 1.1 MODIFIER LETTER RHOTIC HOOK +02DF ; Not_XID # 3.0 MODIFIER LETTER CROSS ACCENT +02E5..02E9 ; Not_XID # 1.1 [5] MODIFIER LETTER EXTRA-HIGH TONE BAR..MODIFIER LETTER EXTRA-LOW TONE BAR +02EA..02EB ; Not_XID # 3.0 [2] MODIFIER LETTER YIN DEPARTING TONE MARK..MODIFIER LETTER YANG DEPARTING TONE MARK +02ED ; Not_XID # 3.0 MODIFIER LETTER UNASPIRATED +02EF..02FF ; Not_XID # 4.0 [17] MODIFIER LETTER LOW DOWN ARROWHEAD..MODIFIER LETTER LOW LEFT ARROW +03F6 ; Not_XID # 3.2 GREEK REVERSED LUNATE EPSILON SYMBOL +055A..055F ; Not_XID # 1.1 [6] ARMENIAN APOSTROPHE..ARMENIAN ABBREVIATION MARK +0589 ; Not_XID # 1.1 ARMENIAN FULL STOP +058D..058E ; Not_XID # 7.0 [2] RIGHT-FACING ARMENIAN ETERNITY SIGN..LEFT-FACING ARMENIAN ETERNITY SIGN +058F ; Not_XID # 6.1 ARMENIAN DRAM SIGN +05BE ; Not_XID # 1.1 HEBREW PUNCTUATION MAQAF +05C0 ; Not_XID # 1.1 HEBREW PUNCTUATION PASEQ +05C3 ; Not_XID # 1.1 HEBREW PUNCTUATION SOF PASUQ +0600..0603 ; Not_XID # 4.0 [4] ARABIC NUMBER SIGN..ARABIC SIGN SAFHA +0604 ; Not_XID # 6.1 ARABIC SIGN SAMVAT +0605 ; Not_XID # 7.0 ARABIC NUMBER MARK ABOVE +0606..060A ; Not_XID # 5.1 [5] ARABIC-INDIC CUBE ROOT..ARABIC-INDIC PER TEN THOUSAND SIGN +060B ; Not_XID # 4.1 AFGHANI SIGN +060C ; Not_XID # 1.1 ARABIC COMMA +060D..060F ; Not_XID # 4.0 [3] ARABIC DATE SEPARATOR..ARABIC SIGN MISRA +061B ; Not_XID # 1.1 ARABIC SEMICOLON +061D ; Not_XID # 14.0 ARABIC END OF TEXT MARK +061E ; Not_XID # 4.1 ARABIC TRIPLE DOT PUNCTUATION MARK +061F ; Not_XID # 1.1 ARABIC QUESTION MARK +066A..066D ; Not_XID # 1.1 [4] ARABIC PERCENT SIGN..ARABIC FIVE POINTED STAR +06D4 ; Not_XID # 1.1 ARABIC FULL STOP +06DD ; Not_XID # 1.1 ARABIC END OF AYAH +06DE ; Not_XID # 1.1 ARABIC START OF RUB EL HIZB +06E9 ; Not_XID # 1.1 ARABIC PLACE OF SAJDAH +0888 ; Not_XID # 14.0 ARABIC RAISED ROUND DOT +0890..0891 ; Not_XID # 14.0 [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE +08E2 ; Not_XID # 9.0 ARABIC DISPUTED END OF AYAH +0964..0965 ; Not_XID # 1.1 [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA +0970 ; Not_XID # 1.1 DEVANAGARI ABBREVIATION SIGN +09F2..09FA ; Not_XID # 1.1 [9] BENGALI RUPEE MARK..BENGALI ISSHAR +09FB ; Not_XID # 5.2 BENGALI GANDA MARK +09FD ; Not_XID # 10.0 BENGALI ABBREVIATION SIGN +0A76 ; Not_XID # 11.0 GURMUKHI ABBREVIATION SIGN +0AF0 ; Not_XID # 6.1 GUJARATI ABBREVIATION SIGN +0AF1 ; Not_XID # 4.0 GUJARATI RUPEE SIGN +0B70 ; Not_XID # 1.1 ORIYA ISSHAR +0B72..0B77 ; Not_XID # 6.0 [6] ORIYA FRACTION ONE QUARTER..ORIYA FRACTION THREE SIXTEENTHS +0BF0..0BF2 ; Not_XID # 1.1 [3] TAMIL NUMBER TEN..TAMIL NUMBER ONE THOUSAND +0BF3..0BFA ; Not_XID # 4.0 [8] TAMIL DAY SIGN..TAMIL NUMBER SIGN +0C77 ; Not_XID # 12.0 TELUGU SIGN SIDDHAM +0C78..0C7F ; Not_XID # 5.1 [8] TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR..TELUGU SIGN TUUMU +0C84 ; Not_XID # 11.0 KANNADA SIGN SIDDHAM +0D4F ; Not_XID # 9.0 MALAYALAM SIGN PARA +0D58..0D5E ; Not_XID # 9.0 [7] MALAYALAM FRACTION ONE ONE-HUNDRED-AND-SIXTIETH..MALAYALAM FRACTION ONE FIFTH +0D70..0D75 ; Not_XID # 5.1 [6] MALAYALAM NUMBER TEN..MALAYALAM FRACTION THREE QUARTERS +0D76..0D78 ; Not_XID # 9.0 [3] MALAYALAM FRACTION ONE SIXTEENTH..MALAYALAM FRACTION THREE SIXTEENTHS +0D79 ; Not_XID # 5.1 MALAYALAM DATE MARK +0DF4 ; Not_XID # 3.0 SINHALA PUNCTUATION KUNDDALIYA +0E3F ; Not_XID # 1.1 THAI CURRENCY SYMBOL BAHT +0E4F ; Not_XID # 1.1 THAI CHARACTER FONGMAN +0E5A..0E5B ; Not_XID # 1.1 [2] THAI CHARACTER ANGKHANKHU..THAI CHARACTER KHOMUT +0F01..0F0A ; Not_XID # 2.0 [10] TIBETAN MARK GTER YIG MGO TRUNCATED A..TIBETAN MARK BKA- SHOG YIG MGO +0F0D..0F17 ; Not_XID # 2.0 [11] TIBETAN MARK SHAD..TIBETAN ASTROLOGICAL SIGN SGRA GCAN -CHAR RTAGS +0F1A..0F1F ; Not_XID # 2.0 [6] TIBETAN SIGN RDEL DKAR GCIG..TIBETAN SIGN RDEL DKAR RDEL NAG +0F2A..0F34 ; Not_XID # 2.0 [11] TIBETAN DIGIT HALF ONE..TIBETAN MARK BSDUS RTAGS +0F36 ; Not_XID # 2.0 TIBETAN MARK CARET -DZUD RTAGS BZHI MIG CAN +0F38 ; Not_XID # 2.0 TIBETAN MARK CHE MGO +0F3A..0F3D ; Not_XID # 2.0 [4] TIBETAN MARK GUG RTAGS GYON..TIBETAN MARK ANG KHANG GYAS +0F85 ; Not_XID # 2.0 TIBETAN MARK PALUTA +0FBE..0FC5 ; Not_XID # 3.0 [8] TIBETAN KU RU KHA..TIBETAN SYMBOL RDO RJE +0FC7..0FCC ; Not_XID # 3.0 [6] TIBETAN SYMBOL RDO RJE RGYA GRAM..TIBETAN SYMBOL NOR BU BZHI -KHYIL +0FCE ; Not_XID # 5.1 TIBETAN SIGN RDEL NAG RDEL DKAR +0FCF ; Not_XID # 3.0 TIBETAN SIGN RDEL NAG GSUM +0FD0..0FD1 ; Not_XID # 4.1 [2] TIBETAN MARK BSKA- SHOG GI MGO RGYAN..TIBETAN MARK MNYAM YIG GI MGO RGYAN +0FD2..0FD4 ; Not_XID # 5.1 [3] TIBETAN MARK NYIS TSHEG..TIBETAN MARK CLOSING BRDA RNYING YIG MGO SGAB MA +0FD5..0FD8 ; Not_XID # 5.2 [4] RIGHT-FACING SVASTI SIGN..LEFT-FACING SVASTI SIGN WITH DOTS +0FD9..0FDA ; Not_XID # 6.0 [2] TIBETAN MARK LEADING MCHAN RTAGS..TIBETAN MARK TRAILING MCHAN RTAGS +104A..104F ; Not_XID # 3.0 [6] MYANMAR SIGN LITTLE SECTION..MYANMAR SYMBOL GENITIVE +109E..109F ; Not_XID # 5.1 [2] MYANMAR SYMBOL SHAN ONE..MYANMAR SYMBOL SHAN EXCLAMATION +10FB ; Not_XID # 1.1 GEORGIAN PARAGRAPH SEPARATOR +1360 ; Not_XID # 4.1 ETHIOPIC SECTION MARK +1361..1368 ; Not_XID # 3.0 [8] ETHIOPIC WORDSPACE..ETHIOPIC PARAGRAPH SEPARATOR +1372..137C ; Not_XID # 3.0 [11] ETHIOPIC NUMBER TEN..ETHIOPIC NUMBER TEN THOUSAND +1390..1399 ; Not_XID # 4.1 [10] ETHIOPIC TONAL MARK YIZET..ETHIOPIC TONAL MARK KURT +16EB..16ED ; Not_XID # 3.0 [3] RUNIC SINGLE PUNCTUATION..RUNIC CROSS PUNCTUATION +17D4..17D6 ; Not_XID # 3.0 [3] KHMER SIGN KHAN..KHMER SIGN CAMNUC PII KUUH +17D9..17DB ; Not_XID # 3.0 [3] KHMER SIGN PHNAEK MUAN..KHMER CURRENCY SYMBOL RIEL +17F0..17F9 ; Not_XID # 4.0 [10] KHMER SYMBOL LEK ATTAK SON..KHMER SYMBOL LEK ATTAK PRAM-BUON +19E0..19FF ; Not_XID # 4.0 [32] KHMER SYMBOL PATHAMASAT..KHMER SYMBOL DAP-PRAM ROC +1ABE ; Not_XID # 7.0 COMBINING PARENTHESES OVERLAY +2012..2016 ; Not_XID # 1.1 [5] FIGURE DASH..DOUBLE VERTICAL LINE +2018 ; Not_XID # 1.1 LEFT SINGLE QUOTATION MARK +201A..2023 ; Not_XID # 1.1 [10] SINGLE LOW-9 QUOTATION MARK..TRIANGULAR BULLET +2028..2029 ; Not_XID # 1.1 [2] LINE SEPARATOR..PARAGRAPH SEPARATOR +2030..2032 ; Not_XID # 1.1 [3] PER MILLE SIGN..PRIME +2035 ; Not_XID # 1.1 REVERSED PRIME +2038..203B ; Not_XID # 1.1 [4] CARET..REFERENCE MARK +203D ; Not_XID # 1.1 INTERROBANG +2041..2046 ; Not_XID # 1.1 [6] CARET INSERTION POINT..RIGHT SQUARE BRACKET WITH QUILL +204A..204D ; Not_XID # 3.0 [4] TIRONIAN SIGN ET..BLACK RIGHTWARDS BULLET +204E..2052 ; Not_XID # 3.2 [5] LOW ASTERISK..COMMERCIAL MINUS SIGN +2053 ; Not_XID # 4.0 SWUNG DASH +2055 ; Not_XID # 4.1 FLOWER PUNCTUATION MARK +20A0..20A7 ; Not_XID # 1.1 [8] EURO-CURRENCY SIGN..PESETA SIGN +20A9..20AA ; Not_XID # 1.1 [2] WON SIGN..NEW SHEQEL SIGN +20AB ; Not_XID # 2.0 DONG SIGN +20AC ; Not_XID # 2.1 EURO SIGN +20AD..20AF ; Not_XID # 3.0 [3] KIP SIGN..DRACHMA SIGN +20B0..20B1 ; Not_XID # 3.2 [2] GERMAN PENNY SIGN..PESO SIGN +20B2..20B5 ; Not_XID # 4.1 [4] GUARANI SIGN..CEDI SIGN +20B6..20B8 ; Not_XID # 5.2 [3] LIVRE TOURNOIS SIGN..TENGE SIGN +20B9 ; Not_XID # 6.0 INDIAN RUPEE SIGN +20BA ; Not_XID # 6.2 TURKISH LIRA SIGN +20BB..20BD ; Not_XID # 7.0 [3] NORDIC MARK SIGN..RUBLE SIGN +20BE ; Not_XID # 8.0 LARI SIGN +20BF ; Not_XID # 10.0 BITCOIN SIGN +20C0 ; Not_XID # 14.0 SOM SIGN +2104 ; Not_XID # 1.1 CENTRE LINE SYMBOL +2108 ; Not_XID # 1.1 SCRUPLE +2114 ; Not_XID # 1.1 L B BAR SYMBOL +2117 ; Not_XID # 1.1 SOUND RECORDING COPYRIGHT +211E..211F ; Not_XID # 1.1 [2] PRESCRIPTION TAKE..RESPONSE +2123 ; Not_XID # 1.1 VERSICLE +2125 ; Not_XID # 1.1 OUNCE SIGN +2129 ; Not_XID # 1.1 TURNED GREEK SMALL LETTER IOTA +213A ; Not_XID # 3.0 ROTATED CAPITAL Q +2141..2144 ; Not_XID # 3.2 [4] TURNED SANS-SERIF CAPITAL G..TURNED SANS-SERIF CAPITAL Y +214A..214B ; Not_XID # 3.2 [2] PROPERTY LINE..TURNED AMPERSAND +214C ; Not_XID # 4.1 PER SIGN +214D ; Not_XID # 5.0 AKTIESELSKAB +2190..21EA ; Not_XID # 1.1 [91] LEFTWARDS ARROW..UPWARDS WHITE ARROW FROM BAR +21EB..21F3 ; Not_XID # 3.0 [9] UPWARDS WHITE ARROW ON PEDESTAL..UP DOWN WHITE ARROW +21F4..21FF ; Not_XID # 3.2 [12] RIGHT ARROW WITH SMALL CIRCLE..LEFT RIGHT OPEN-HEADED ARROW +2200..222B ; Not_XID # 1.1 [44] FOR ALL..INTEGRAL +222E ; Not_XID # 1.1 CONTOUR INTEGRAL +2231..22F1 ; Not_XID # 1.1 [193] CLOCKWISE INTEGRAL..DOWN RIGHT DIAGONAL ELLIPSIS +22F2..22FF ; Not_XID # 3.2 [14] ELEMENT OF WITH LONG HORIZONTAL STROKE..Z NOTATION BAG MEMBERSHIP +2300 ; Not_XID # 1.1 DIAMETER SIGN +2301 ; Not_XID # 3.0 ELECTRIC ARROW +2302..2328 ; Not_XID # 1.1 [39] HOUSE..KEYBOARD +232B..237A ; Not_XID # 1.1 [80] ERASE TO THE LEFT..APL FUNCTIONAL SYMBOL ALPHA +237B ; Not_XID # 3.0 NOT CHECK MARK +237C ; Not_XID # 3.2 RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +237D..239A ; Not_XID # 3.0 [30] SHOULDERED OPEN BOX..CLEAR SCREEN SYMBOL +239B..23CE ; Not_XID # 3.2 [52] LEFT PARENTHESIS UPPER HOOK..RETURN SYMBOL +23CF..23D0 ; Not_XID # 4.0 [2] EJECT SYMBOL..VERTICAL LINE EXTENSION +23D1..23DB ; Not_XID # 4.1 [11] METRICAL BREVE..FUSE +23DC..23E7 ; Not_XID # 5.0 [12] TOP PARENTHESIS..ELECTRICAL INTERSECTION +23E8 ; Not_XID # 5.2 DECIMAL EXPONENT SYMBOL +23E9..23F3 ; Not_XID # 6.0 [11] BLACK RIGHT-POINTING DOUBLE TRIANGLE..HOURGLASS WITH FLOWING SAND +23F4..23FA ; Not_XID # 7.0 [7] BLACK MEDIUM LEFT-POINTING TRIANGLE..BLACK CIRCLE FOR RECORD +23FB..23FE ; Not_XID # 9.0 [4] POWER SYMBOL..POWER SLEEP SYMBOL +23FF ; Not_XID # 10.0 OBSERVER EYE SYMBOL +2400..2424 ; Not_XID # 1.1 [37] SYMBOL FOR NULL..SYMBOL FOR NEWLINE +2425..2426 ; Not_XID # 3.0 [2] SYMBOL FOR DELETE FORM TWO..SYMBOL FOR SUBSTITUTE FORM TWO +2440..244A ; Not_XID # 1.1 [11] OCR HOOK..OCR DOUBLE BACKSLASH +2500..2595 ; Not_XID # 1.1 [150] BOX DRAWINGS LIGHT HORIZONTAL..RIGHT ONE EIGHTH BLOCK +2596..259F ; Not_XID # 3.2 [10] QUADRANT LOWER LEFT..QUADRANT UPPER RIGHT AND LOWER LEFT AND LOWER RIGHT +25A0..25EF ; Not_XID # 1.1 [80] BLACK SQUARE..LARGE CIRCLE +25F0..25F7 ; Not_XID # 3.0 [8] WHITE SQUARE WITH UPPER LEFT QUADRANT..WHITE CIRCLE WITH UPPER RIGHT QUADRANT +25F8..25FF ; Not_XID # 3.2 [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE +2600..2613 ; Not_XID # 1.1 [20] BLACK SUN WITH RAYS..SALTIRE +2614..2615 ; Not_XID # 4.0 [2] UMBRELLA WITH RAIN DROPS..HOT BEVERAGE +2616..2617 ; Not_XID # 3.2 [2] WHITE SHOGI PIECE..BLACK SHOGI PIECE +2618 ; Not_XID # 4.1 SHAMROCK +2619 ; Not_XID # 3.0 REVERSED ROTATED FLORAL HEART BULLET +261A..266F ; Not_XID # 1.1 [86] BLACK LEFT POINTING INDEX..MUSIC SHARP SIGN +2670..2671 ; Not_XID # 3.0 [2] WEST SYRIAC CROSS..EAST SYRIAC CROSS +2672..267D ; Not_XID # 3.2 [12] UNIVERSAL RECYCLING SYMBOL..PARTIALLY-RECYCLED PAPER SYMBOL +267E..267F ; Not_XID # 4.1 [2] PERMANENT PAPER SIGN..WHEELCHAIR SYMBOL +2680..2689 ; Not_XID # 3.2 [10] DIE FACE-1..BLACK CIRCLE WITH TWO WHITE DOTS +268A..2691 ; Not_XID # 4.0 [8] MONOGRAM FOR YANG..BLACK FLAG +2692..269C ; Not_XID # 4.1 [11] HAMMER AND PICK..FLEUR-DE-LIS +269D ; Not_XID # 5.1 OUTLINED WHITE STAR +269E..269F ; Not_XID # 5.2 [2] THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT +26A0..26A1 ; Not_XID # 4.0 [2] WARNING SIGN..HIGH VOLTAGE SIGN +26A2..26B1 ; Not_XID # 4.1 [16] DOUBLED FEMALE SIGN..FUNERAL URN +26B2 ; Not_XID # 5.0 NEUTER +26B3..26BC ; Not_XID # 5.1 [10] CERES..SESQUIQUADRATE +26BD..26BF ; Not_XID # 5.2 [3] SOCCER BALL..SQUARED KEY +26C0..26C3 ; Not_XID # 5.1 [4] WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING +26C4..26CD ; Not_XID # 5.2 [10] SNOWMAN WITHOUT SNOW..DISABLED CAR +26CE ; Not_XID # 6.0 OPHIUCHUS +26CF..26E1 ; Not_XID # 5.2 [19] PICK..RESTRICTED LEFT ENTRY-2 +26E2 ; Not_XID # 6.0 ASTRONOMICAL SYMBOL FOR URANUS +26E3 ; Not_XID # 5.2 HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE +26E4..26E7 ; Not_XID # 6.0 [4] PENTAGRAM..INVERTED PENTAGRAM +26E8..26FF ; Not_XID # 5.2 [24] BLACK CROSS ON SHIELD..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE +2700 ; Not_XID # 7.0 BLACK SAFETY SCISSORS +2701..2704 ; Not_XID # 1.1 [4] UPPER BLADE SCISSORS..WHITE SCISSORS +2705 ; Not_XID # 6.0 WHITE HEAVY CHECK MARK +2706..2709 ; Not_XID # 1.1 [4] TELEPHONE LOCATION SIGN..ENVELOPE +270A..270B ; Not_XID # 6.0 [2] RAISED FIST..RAISED HAND +270C..2727 ; Not_XID # 1.1 [28] VICTORY HAND..WHITE FOUR POINTED STAR +2728 ; Not_XID # 6.0 SPARKLES +2729..274B ; Not_XID # 1.1 [35] STRESS OUTLINED WHITE STAR..HEAVY EIGHT TEARDROP-SPOKED PROPELLER ASTERISK +274C ; Not_XID # 6.0 CROSS MARK +274D ; Not_XID # 1.1 SHADOWED WHITE CIRCLE +274E ; Not_XID # 6.0 NEGATIVE SQUARED CROSS MARK +274F..2752 ; Not_XID # 1.1 [4] LOWER RIGHT DROP-SHADOWED WHITE SQUARE..UPPER RIGHT SHADOWED WHITE SQUARE +2753..2755 ; Not_XID # 6.0 [3] BLACK QUESTION MARK ORNAMENT..WHITE EXCLAMATION MARK ORNAMENT +2756 ; Not_XID # 1.1 BLACK DIAMOND MINUS WHITE X +2757 ; Not_XID # 5.2 HEAVY EXCLAMATION MARK SYMBOL +2758..275E ; Not_XID # 1.1 [7] LIGHT VERTICAL BAR..HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT +275F..2760 ; Not_XID # 6.0 [2] HEAVY LOW SINGLE COMMA QUOTATION MARK ORNAMENT..HEAVY LOW DOUBLE COMMA QUOTATION MARK ORNAMENT +2761..2767 ; Not_XID # 1.1 [7] CURVED STEM PARAGRAPH SIGN ORNAMENT..ROTATED FLORAL HEART BULLET +2768..2775 ; Not_XID # 3.2 [14] MEDIUM LEFT PARENTHESIS ORNAMENT..MEDIUM RIGHT CURLY BRACKET ORNAMENT +2776..2794 ; Not_XID # 1.1 [31] DINGBAT NEGATIVE CIRCLED DIGIT ONE..HEAVY WIDE-HEADED RIGHTWARDS ARROW +2795..2797 ; Not_XID # 6.0 [3] HEAVY PLUS SIGN..HEAVY DIVISION SIGN +2798..27AF ; Not_XID # 1.1 [24] HEAVY SOUTH EAST ARROW..NOTCHED LOWER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW +27B0 ; Not_XID # 6.0 CURLY LOOP +27B1..27BE ; Not_XID # 1.1 [14] NOTCHED UPPER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW..OPEN-OUTLINED RIGHTWARDS ARROW +27BF ; Not_XID # 6.0 DOUBLE CURLY LOOP +27C0..27C6 ; Not_XID # 4.1 [7] THREE DIMENSIONAL ANGLE..RIGHT S-SHAPED BAG DELIMITER +27C7..27CA ; Not_XID # 5.0 [4] OR WITH DOT INSIDE..VERTICAL BAR WITH HORIZONTAL STROKE +27CB ; Not_XID # 6.1 MATHEMATICAL RISING DIAGONAL +27CC ; Not_XID # 5.1 LONG DIVISION +27CD ; Not_XID # 6.1 MATHEMATICAL FALLING DIAGONAL +27CE..27CF ; Not_XID # 6.0 [2] SQUARED LOGICAL AND..SQUARED LOGICAL OR +27D0..27EB ; Not_XID # 3.2 [28] WHITE DIAMOND WITH CENTRED DOT..MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET +27EC..27EF ; Not_XID # 5.1 [4] MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET..MATHEMATICAL RIGHT FLATTENED PARENTHESIS +27F0..27FF ; Not_XID # 3.2 [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2900..2A0B ; Not_XID # 3.2 [268] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..SUMMATION WITH INTEGRAL +2A0D..2A73 ; Not_XID # 3.2 [103] FINITE PART INTEGRAL..EQUALS SIGN ABOVE TILDE OPERATOR +2A77..2ADB ; Not_XID # 3.2 [101] EQUALS SIGN WITH TWO DOTS ABOVE AND TWO DOTS BELOW..TRANSVERSAL INTERSECTION +2ADD..2AFF ; Not_XID # 3.2 [35] NONFORKING..N-ARY WHITE VERTICAL BAR +2B00..2B0D ; Not_XID # 4.0 [14] NORTH EAST WHITE ARROW..UP DOWN BLACK ARROW +2B0E..2B13 ; Not_XID # 4.1 [6] RIGHTWARDS ARROW WITH TIP DOWNWARDS..SQUARE WITH BOTTOM HALF BLACK +2B14..2B1A ; Not_XID # 5.0 [7] SQUARE WITH UPPER RIGHT DIAGONAL HALF BLACK..DOTTED SQUARE +2B1B..2B1F ; Not_XID # 5.1 [5] BLACK LARGE SQUARE..BLACK PENTAGON +2B20..2B23 ; Not_XID # 5.0 [4] WHITE PENTAGON..HORIZONTAL BLACK HEXAGON +2B24..2B4C ; Not_XID # 5.1 [41] BLACK LARGE CIRCLE..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR +2B4D..2B4F ; Not_XID # 7.0 [3] DOWNWARDS TRIANGLE-HEADED ZIGZAG ARROW..SHORT BACKSLANTED SOUTH ARROW +2B50..2B54 ; Not_XID # 5.1 [5] WHITE MEDIUM STAR..WHITE RIGHT-POINTING PENTAGON +2B55..2B59 ; Not_XID # 5.2 [5] HEAVY LARGE CIRCLE..HEAVY CIRCLED SALTIRE +2B5A..2B73 ; Not_XID # 7.0 [26] SLANTED NORTH ARROW WITH HOOKED HEAD..DOWNWARDS TRIANGLE-HEADED ARROW TO BAR +2B76..2B95 ; Not_XID # 7.0 [32] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..RIGHTWARDS BLACK ARROW +2B97 ; Not_XID # 13.0 SYMBOL FOR TYPE A ELECTRONICS +2B98..2BB9 ; Not_XID # 7.0 [34] THREE-D TOP-LIGHTED LEFTWARDS EQUILATERAL ARROWHEAD..UP ARROWHEAD IN A RECTANGLE BOX +2BBA..2BBC ; Not_XID # 11.0 [3] OVERLAPPING WHITE SQUARES..OVERLAPPING BLACK SQUARES +2BBD..2BC8 ; Not_XID # 7.0 [12] BALLOT BOX WITH LIGHT X..BLACK MEDIUM RIGHT-POINTING TRIANGLE CENTRED +2BC9 ; Not_XID # 12.0 NEPTUNE FORM TWO +2BCA..2BD1 ; Not_XID # 7.0 [8] TOP HALF BLACK CIRCLE..UNCERTAINTY SIGN +2BD2 ; Not_XID # 10.0 GROUP MARK +2BD3..2BEB ; Not_XID # 11.0 [25] PLUTO FORM TWO..STAR WITH RIGHT HALF BLACK +2BF0..2BFE ; Not_XID # 11.0 [15] ERIS FORM ONE..REVERSED RIGHT ANGLE +2BFF ; Not_XID # 12.0 HELLSCHREIBER PAUSE SYMBOL +2E17 ; Not_XID # 4.1 DOUBLE OBLIQUE HYPHEN +2E18..2E1B ; Not_XID # 5.1 [4] INVERTED INTERROBANG..TILDE WITH RING ABOVE +2E1C..2E1D ; Not_XID # 4.1 [2] LEFT LOW PARAPHRASE BRACKET..RIGHT LOW PARAPHRASE BRACKET +2E1E..2E29 ; Not_XID # 5.1 [12] TILDE WITH DOT ABOVE..RIGHT DOUBLE PARENTHESIS +2E33..2E34 ; Not_XID # 6.1 [2] RAISED DOT..RAISED COMMA +2E36..2E38 ; Not_XID # 6.1 [3] DAGGER WITH LEFT GUARD..TURNED DAGGER +2E3A..2E3B ; Not_XID # 6.1 [2] TWO-EM DASH..THREE-EM DASH +2E3C..2E42 ; Not_XID # 7.0 [7] STENOGRAPHIC FULL STOP..DOUBLE LOW-REVERSED-9 QUOTATION MARK +2E43..2E44 ; Not_XID # 9.0 [2] DASH WITH LEFT UPTURN..DOUBLE SUSPENSION MARK +2E45..2E49 ; Not_XID # 10.0 [5] INVERTED LOW KAVYKA..DOUBLE STACKED COMMA +2E4A..2E4E ; Not_XID # 11.0 [5] DOTTED SOLIDUS..PUNCTUS ELEVATUS MARK +2E4F ; Not_XID # 12.0 CORNISH VERSE DIVIDER +2E50..2E52 ; Not_XID # 13.0 [3] CROSS PATTY WITH RIGHT CROSSBAR..TIRONIAN SIGN CAPITAL ET +2E53..2E5D ; Not_XID # 14.0 [11] MEDIEVAL EXCLAMATION MARK..OBLIQUE HYPHEN +2E80..2E99 ; Not_XID # 3.0 [26] CJK RADICAL REPEAT..CJK RADICAL RAP +2E9B..2E9E ; Not_XID # 3.0 [4] CJK RADICAL CHOKE..CJK RADICAL DEATH +2EA0..2EF2 ; Not_XID # 3.0 [83] CJK RADICAL CIVILIAN..CJK RADICAL J-SIMPLIFIED TURTLE +2FF0..2FFB ; Not_XID # 3.0 [12] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID +3001..3004 ; Not_XID # 1.1 [4] IDEOGRAPHIC COMMA..JAPANESE INDUSTRIAL STANDARD SYMBOL +3008..301D ; Not_XID # 1.1 [22] LEFT ANGLE BRACKET..REVERSED DOUBLE PRIME QUOTATION MARK +301F..3020 ; Not_XID # 1.1 [2] LOW DOUBLE PRIME QUOTATION MARK..POSTAL MARK FACE +3030 ; Not_XID # 1.1 WAVY DASH +3037 ; Not_XID # 1.1 IDEOGRAPHIC TELEGRAPH LINE FEED SEPARATOR SYMBOL +303D ; Not_XID # 3.2 PART ALTERNATION MARK +303E ; Not_XID # 3.0 IDEOGRAPHIC VARIATION INDICATOR +303F ; Not_XID # 1.1 IDEOGRAPHIC HALF FILL SPACE +3190..3191 ; Not_XID # 1.1 [2] IDEOGRAPHIC ANNOTATION LINKING MARK..IDEOGRAPHIC ANNOTATION REVERSE MARK +31C0..31CF ; Not_XID # 4.1 [16] CJK STROKE T..CJK STROKE N +31D0..31E3 ; Not_XID # 5.1 [20] CJK STROKE H..CJK STROKE Q +3248..324F ; Not_XID # 5.2 [8] CIRCLED NUMBER TEN ON BLACK SQUARE..CIRCLED NUMBER EIGHTY ON BLACK SQUARE +A67E ; Not_XID # 5.1 CYRILLIC KAVYKA +A720..A721 ; Not_XID # 5.0 [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE +A789..A78A ; Not_XID # 5.1 [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN +A830..A839 ; Not_XID # 5.2 [10] NORTH INDIC FRACTION ONE QUARTER..NORTH INDIC QUANTITY MARK +A92E ; Not_XID # 5.1 KAYAH LI SIGN CWI +AA77..AA79 ; Not_XID # 5.2 [3] MYANMAR SYMBOL AITON EXCLAMATION..MYANMAR SYMBOL AITON TWO +AB5B ; Not_XID # 7.0 MODIFIER BREVE WITH INVERTED BREVE +AB6A..AB6B ; Not_XID # 13.0 [2] MODIFIER LETTER LEFT TACK..MODIFIER LETTER RIGHT TACK +FFF9..FFFB ; Not_XID # 3.0 [3] INTERLINEAR ANNOTATION ANCHOR..INTERLINEAR ANNOTATION TERMINATOR +FFFC ; Not_XID # 2.1 OBJECT REPLACEMENT CHARACTER +FFFD ; Not_XID # 1.1 REPLACEMENT CHARACTER +10175..1018A ; Not_XID # 4.1 [22] GREEK ONE HALF SIGN..GREEK ZERO SIGN +1018B..1018C ; Not_XID # 7.0 [2] GREEK ONE QUARTER SIGN..GREEK SINUSOID SIGN +1018D..1018E ; Not_XID # 9.0 [2] GREEK INDICTION SIGN..NOMISMA SIGN +10190..1019B ; Not_XID # 5.1 [12] ROMAN SEXTANS SIGN..ROMAN CENTURIAL SIGN +1019C ; Not_XID # 13.0 ASCIA SYMBOL +101A0 ; Not_XID # 7.0 GREEK SYMBOL TAU RHO +10E60..10E7E ; Not_XID # 5.2 [31] RUMI DIGIT ONE..RUMI FRACTION TWO THIRDS +111E1..111F4 ; Not_XID # 7.0 [20] SINHALA ARCHAIC DIGIT ONE..SINHALA ARCHAIC NUMBER ONE THOUSAND +11FC0..11FF1 ; Not_XID # 12.0 [50] TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH..TAMIL SIGN VAKAIYARAA +11FFF ; Not_XID # 12.0 TAMIL PUNCTUATION END OF TEXT +16FE2 ; Not_XID # 12.0 OLD CHINESE HOOK MARK +1D2E0..1D2F3 ; Not_XID # 11.0 [20] MAYAN NUMERAL ZERO..MAYAN NUMERAL NINETEEN +1D360..1D371 ; Not_XID # 5.0 [18] COUNTING ROD UNIT DIGIT ONE..COUNTING ROD TENS DIGIT NINE +1D372..1D378 ; Not_XID # 11.0 [7] IDEOGRAPHIC TALLY MARK ONE..TALLY MARK FIVE +1EC71..1ECB4 ; Not_XID # 11.0 [68] INDIC SIYAQ NUMBER ONE..INDIC SIYAQ ALTERNATE LAKH MARK +1ED01..1ED3D ; Not_XID # 12.0 [61] OTTOMAN SIYAQ NUMBER ONE..OTTOMAN SIYAQ FRACTION ONE SIXTH +1EEF0..1EEF1 ; Not_XID # 6.1 [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL +1F000..1F02B ; Not_XID # 5.1 [44] MAHJONG TILE EAST WIND..MAHJONG TILE BACK +1F030..1F093 ; Not_XID # 5.1 [100] DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06 +1F0A0..1F0AE ; Not_XID # 6.0 [15] PLAYING CARD BACK..PLAYING CARD KING OF SPADES +1F0B1..1F0BE ; Not_XID # 6.0 [14] PLAYING CARD ACE OF HEARTS..PLAYING CARD KING OF HEARTS +1F0BF ; Not_XID # 7.0 PLAYING CARD RED JOKER +1F0C1..1F0CF ; Not_XID # 6.0 [15] PLAYING CARD ACE OF DIAMONDS..PLAYING CARD BLACK JOKER +1F0D1..1F0DF ; Not_XID # 6.0 [15] PLAYING CARD ACE OF CLUBS..PLAYING CARD WHITE JOKER +1F0E0..1F0F5 ; Not_XID # 7.0 [22] PLAYING CARD FOOL..PLAYING CARD TRUMP-21 +1F10B..1F10C ; Not_XID # 7.0 [2] DINGBAT CIRCLED SANS-SERIF DIGIT ZERO..DINGBAT NEGATIVE CIRCLED SANS-SERIF DIGIT ZERO +1F10D..1F10F ; Not_XID # 13.0 [3] CIRCLED ZERO WITH SLASH..CIRCLED DOLLAR SIGN WITH OVERLAID BACKSLASH +1F12F ; Not_XID # 11.0 COPYLEFT SYMBOL +1F150..1F156 ; Not_XID # 6.0 [7] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER G +1F157 ; Not_XID # 5.2 NEGATIVE CIRCLED LATIN CAPITAL LETTER H +1F158..1F15E ; Not_XID # 6.0 [7] NEGATIVE CIRCLED LATIN CAPITAL LETTER I..NEGATIVE CIRCLED LATIN CAPITAL LETTER O +1F15F ; Not_XID # 5.2 NEGATIVE CIRCLED LATIN CAPITAL LETTER P +1F160..1F169 ; Not_XID # 6.0 [10] NEGATIVE CIRCLED LATIN CAPITAL LETTER Q..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z +1F16D..1F16F ; Not_XID # 13.0 [3] CIRCLED CC..CIRCLED HUMAN FIGURE +1F170..1F178 ; Not_XID # 6.0 [9] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER I +1F179 ; Not_XID # 5.2 NEGATIVE SQUARED LATIN CAPITAL LETTER J +1F17A ; Not_XID # 6.0 NEGATIVE SQUARED LATIN CAPITAL LETTER K +1F17B..1F17C ; Not_XID # 5.2 [2] NEGATIVE SQUARED LATIN CAPITAL LETTER L..NEGATIVE SQUARED LATIN CAPITAL LETTER M +1F17D..1F17E ; Not_XID # 6.0 [2] NEGATIVE SQUARED LATIN CAPITAL LETTER N..NEGATIVE SQUARED LATIN CAPITAL LETTER O +1F17F ; Not_XID # 5.2 NEGATIVE SQUARED LATIN CAPITAL LETTER P +1F180..1F189 ; Not_XID # 6.0 [10] NEGATIVE SQUARED LATIN CAPITAL LETTER Q..NEGATIVE SQUARED LATIN CAPITAL LETTER Z +1F18A..1F18D ; Not_XID # 5.2 [4] CROSSED NEGATIVE SQUARED LATIN CAPITAL LETTER P..NEGATIVE SQUARED SA +1F18E..1F18F ; Not_XID # 6.0 [2] NEGATIVE SQUARED AB..NEGATIVE SQUARED WC +1F191..1F19A ; Not_XID # 6.0 [10] SQUARED CL..SQUARED VS +1F19B..1F1AC ; Not_XID # 9.0 [18] SQUARED THREE D..SQUARED VOD +1F1AD ; Not_XID # 13.0 MASK WORK SYMBOL +1F1E6..1F1FF ; Not_XID # 6.0 [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z +1F260..1F265 ; Not_XID # 10.0 [6] ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI +1F300..1F320 ; Not_XID # 6.0 [33] CYCLONE..SHOOTING STAR +1F321..1F32C ; Not_XID # 7.0 [12] THERMOMETER..WIND BLOWING FACE +1F32D..1F32F ; Not_XID # 8.0 [3] HOT DOG..BURRITO +1F330..1F335 ; Not_XID # 6.0 [6] CHESTNUT..CACTUS +1F336 ; Not_XID # 7.0 HOT PEPPER +1F337..1F37C ; Not_XID # 6.0 [70] TULIP..BABY BOTTLE +1F37D ; Not_XID # 7.0 FORK AND KNIFE WITH PLATE +1F37E..1F37F ; Not_XID # 8.0 [2] BOTTLE WITH POPPING CORK..POPCORN +1F380..1F393 ; Not_XID # 6.0 [20] RIBBON..GRADUATION CAP +1F394..1F39F ; Not_XID # 7.0 [12] HEART WITH TIP ON THE LEFT..ADMISSION TICKETS +1F3A0..1F3C4 ; Not_XID # 6.0 [37] CAROUSEL HORSE..SURFER +1F3C5 ; Not_XID # 7.0 SPORTS MEDAL +1F3C6..1F3CA ; Not_XID # 6.0 [5] TROPHY..SWIMMER +1F3CB..1F3CE ; Not_XID # 7.0 [4] WEIGHT LIFTER..RACING CAR +1F3CF..1F3D3 ; Not_XID # 8.0 [5] CRICKET BAT AND BALL..TABLE TENNIS PADDLE AND BALL +1F3D4..1F3DF ; Not_XID # 7.0 [12] SNOW CAPPED MOUNTAIN..STADIUM +1F3E0..1F3F0 ; Not_XID # 6.0 [17] HOUSE BUILDING..EUROPEAN CASTLE +1F3F1..1F3F7 ; Not_XID # 7.0 [7] WHITE PENNANT..LABEL +1F3F8..1F3FF ; Not_XID # 8.0 [8] BADMINTON RACQUET AND SHUTTLECOCK..EMOJI MODIFIER FITZPATRICK TYPE-6 +1F400..1F43E ; Not_XID # 6.0 [63] RAT..PAW PRINTS +1F43F ; Not_XID # 7.0 CHIPMUNK +1F440 ; Not_XID # 6.0 EYES +1F441 ; Not_XID # 7.0 EYE +1F442..1F4F7 ; Not_XID # 6.0 [182] EAR..CAMERA +1F4F8 ; Not_XID # 7.0 CAMERA WITH FLASH +1F4F9..1F4FC ; Not_XID # 6.0 [4] VIDEO CAMERA..VIDEOCASSETTE +1F4FD..1F4FE ; Not_XID # 7.0 [2] FILM PROJECTOR..PORTABLE STEREO +1F4FF ; Not_XID # 8.0 PRAYER BEADS +1F500..1F53D ; Not_XID # 6.0 [62] TWISTED RIGHTWARDS ARROWS..DOWN-POINTING SMALL RED TRIANGLE +1F53E..1F53F ; Not_XID # 7.0 [2] LOWER RIGHT SHADOWED WHITE CIRCLE..UPPER RIGHT SHADOWED WHITE CIRCLE +1F540..1F543 ; Not_XID # 6.1 [4] CIRCLED CROSS POMMEE..NOTCHED LEFT SEMICIRCLE WITH THREE DOTS +1F544..1F54A ; Not_XID # 7.0 [7] NOTCHED RIGHT SEMICIRCLE WITH THREE DOTS..DOVE OF PEACE +1F54B..1F54E ; Not_XID # 8.0 [4] KAABA..MENORAH WITH NINE BRANCHES +1F550..1F567 ; Not_XID # 6.0 [24] CLOCK FACE ONE OCLOCK..CLOCK FACE TWELVE-THIRTY +1F568..1F579 ; Not_XID # 7.0 [18] RIGHT SPEAKER..JOYSTICK +1F57A ; Not_XID # 9.0 MAN DANCING +1F57B..1F5A3 ; Not_XID # 7.0 [41] LEFT HAND TELEPHONE RECEIVER..BLACK DOWN POINTING BACKHAND INDEX +1F5A4 ; Not_XID # 9.0 BLACK HEART +1F5A5..1F5FA ; Not_XID # 7.0 [86] DESKTOP COMPUTER..WORLD MAP +1F5FB..1F5FF ; Not_XID # 6.0 [5] MOUNT FUJI..MOYAI +1F600 ; Not_XID # 6.1 GRINNING FACE +1F601..1F610 ; Not_XID # 6.0 [16] GRINNING FACE WITH SMILING EYES..NEUTRAL FACE +1F611 ; Not_XID # 6.1 EXPRESSIONLESS FACE +1F612..1F614 ; Not_XID # 6.0 [3] UNAMUSED FACE..PENSIVE FACE +1F615 ; Not_XID # 6.1 CONFUSED FACE +1F616 ; Not_XID # 6.0 CONFOUNDED FACE +1F617 ; Not_XID # 6.1 KISSING FACE +1F618 ; Not_XID # 6.0 FACE THROWING A KISS +1F619 ; Not_XID # 6.1 KISSING FACE WITH SMILING EYES +1F61A ; Not_XID # 6.0 KISSING FACE WITH CLOSED EYES +1F61B ; Not_XID # 6.1 FACE WITH STUCK-OUT TONGUE +1F61C..1F61E ; Not_XID # 6.0 [3] FACE WITH STUCK-OUT TONGUE AND WINKING EYE..DISAPPOINTED FACE +1F61F ; Not_XID # 6.1 WORRIED FACE +1F620..1F625 ; Not_XID # 6.0 [6] ANGRY FACE..DISAPPOINTED BUT RELIEVED FACE +1F626..1F627 ; Not_XID # 6.1 [2] FROWNING FACE WITH OPEN MOUTH..ANGUISHED FACE +1F628..1F62B ; Not_XID # 6.0 [4] FEARFUL FACE..TIRED FACE +1F62C ; Not_XID # 6.1 GRIMACING FACE +1F62D ; Not_XID # 6.0 LOUDLY CRYING FACE +1F62E..1F62F ; Not_XID # 6.1 [2] FACE WITH OPEN MOUTH..HUSHED FACE +1F630..1F633 ; Not_XID # 6.0 [4] FACE WITH OPEN MOUTH AND COLD SWEAT..FLUSHED FACE +1F634 ; Not_XID # 6.1 SLEEPING FACE +1F635..1F640 ; Not_XID # 6.0 [12] DIZZY FACE..WEARY CAT FACE +1F641..1F642 ; Not_XID # 7.0 [2] SLIGHTLY FROWNING FACE..SLIGHTLY SMILING FACE +1F643..1F644 ; Not_XID # 8.0 [2] UPSIDE-DOWN FACE..FACE WITH ROLLING EYES +1F645..1F64F ; Not_XID # 6.0 [11] FACE WITH NO GOOD GESTURE..PERSON WITH FOLDED HANDS +1F650..1F67F ; Not_XID # 7.0 [48] NORTH WEST POINTING LEAF..REVERSE CHECKER BOARD +1F680..1F6C5 ; Not_XID # 6.0 [70] ROCKET..LEFT LUGGAGE +1F6C6..1F6CF ; Not_XID # 7.0 [10] TRIANGLE WITH ROUNDED CORNERS..BED +1F6D0 ; Not_XID # 8.0 PLACE OF WORSHIP +1F6D1..1F6D2 ; Not_XID # 9.0 [2] OCTAGONAL SIGN..SHOPPING TROLLEY +1F6D3..1F6D4 ; Not_XID # 10.0 [2] STUPA..PAGODA +1F6D5 ; Not_XID # 12.0 HINDU TEMPLE +1F6D6..1F6D7 ; Not_XID # 13.0 [2] HUT..ELEVATOR +1F6DD..1F6DF ; Not_XID # 14.0 [3] PLAYGROUND SLIDE..RING BUOY +1F6E0..1F6EC ; Not_XID # 7.0 [13] HAMMER AND WRENCH..AIRPLANE ARRIVING +1F6F0..1F6F3 ; Not_XID # 7.0 [4] SATELLITE..PASSENGER SHIP +1F6F4..1F6F6 ; Not_XID # 9.0 [3] SCOOTER..CANOE +1F6F7..1F6F8 ; Not_XID # 10.0 [2] SLED..FLYING SAUCER +1F6F9 ; Not_XID # 11.0 SKATEBOARD +1F6FA ; Not_XID # 12.0 AUTO RICKSHAW +1F6FB..1F6FC ; Not_XID # 13.0 [2] PICKUP TRUCK..ROLLER SKATE +1F700..1F773 ; Not_XID # 6.0 [116] ALCHEMICAL SYMBOL FOR QUINTESSENCE..ALCHEMICAL SYMBOL FOR HALF OUNCE +1F780..1F7D4 ; Not_XID # 7.0 [85] BLACK LEFT-POINTING ISOSCELES RIGHT TRIANGLE..HEAVY TWELVE POINTED PINWHEEL STAR +1F7D5..1F7D8 ; Not_XID # 11.0 [4] CIRCLED TRIANGLE..NEGATIVE CIRCLED SQUARE +1F7E0..1F7EB ; Not_XID # 12.0 [12] LARGE ORANGE CIRCLE..LARGE BROWN SQUARE +1F7F0 ; Not_XID # 14.0 HEAVY EQUALS SIGN +1F800..1F80B ; Not_XID # 7.0 [12] LEFTWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD..DOWNWARDS ARROW WITH LARGE TRIANGLE ARROWHEAD +1F810..1F847 ; Not_XID # 7.0 [56] LEFTWARDS ARROW WITH SMALL EQUILATERAL ARROWHEAD..DOWNWARDS HEAVY ARROW +1F850..1F859 ; Not_XID # 7.0 [10] LEFTWARDS SANS-SERIF ARROW..UP DOWN SANS-SERIF ARROW +1F860..1F887 ; Not_XID # 7.0 [40] WIDE-HEADED LEFTWARDS LIGHT BARB ARROW..WIDE-HEADED SOUTH WEST VERY HEAVY BARB ARROW +1F890..1F8AD ; Not_XID # 7.0 [30] LEFTWARDS TRIANGLE ARROWHEAD..WHITE ARROW SHAFT WIDTH TWO THIRDS +1F8B0..1F8B1 ; Not_XID # 13.0 [2] ARROW POINTING UPWARDS THEN NORTH WEST..ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST +1F900..1F90B ; Not_XID # 10.0 [12] CIRCLED CROSS FORMEE WITH FOUR DOTS..DOWNWARD FACING NOTCHED HOOK WITH DOT +1F90C ; Not_XID # 13.0 PINCHED FINGERS +1F90D..1F90F ; Not_XID # 12.0 [3] WHITE HEART..PINCHING HAND +1F910..1F918 ; Not_XID # 8.0 [9] ZIPPER-MOUTH FACE..SIGN OF THE HORNS +1F919..1F91E ; Not_XID # 9.0 [6] CALL ME HAND..HAND WITH INDEX AND MIDDLE FINGERS CROSSED +1F91F ; Not_XID # 10.0 I LOVE YOU HAND SIGN +1F920..1F927 ; Not_XID # 9.0 [8] FACE WITH COWBOY HAT..SNEEZING FACE +1F928..1F92F ; Not_XID # 10.0 [8] FACE WITH ONE EYEBROW RAISED..SHOCKED FACE WITH EXPLODING HEAD +1F930 ; Not_XID # 9.0 PREGNANT WOMAN +1F931..1F932 ; Not_XID # 10.0 [2] BREAST-FEEDING..PALMS UP TOGETHER +1F933..1F93E ; Not_XID # 9.0 [12] SELFIE..HANDBALL +1F93F ; Not_XID # 12.0 DIVING MASK +1F940..1F94B ; Not_XID # 9.0 [12] WILTED FLOWER..MARTIAL ARTS UNIFORM +1F94C ; Not_XID # 10.0 CURLING STONE +1F94D..1F94F ; Not_XID # 11.0 [3] LACROSSE STICK AND BALL..FLYING DISC +1F950..1F95E ; Not_XID # 9.0 [15] CROISSANT..PANCAKES +1F95F..1F96B ; Not_XID # 10.0 [13] DUMPLING..CANNED FOOD +1F96C..1F970 ; Not_XID # 11.0 [5] LEAFY GREEN..SMILING FACE WITH SMILING EYES AND THREE HEARTS +1F971 ; Not_XID # 12.0 YAWNING FACE +1F972 ; Not_XID # 13.0 SMILING FACE WITH TEAR +1F973..1F976 ; Not_XID # 11.0 [4] FACE WITH PARTY HORN AND PARTY HAT..FREEZING FACE +1F977..1F978 ; Not_XID # 13.0 [2] NINJA..DISGUISED FACE +1F979 ; Not_XID # 14.0 FACE HOLDING BACK TEARS +1F97A ; Not_XID # 11.0 FACE WITH PLEADING EYES +1F97B ; Not_XID # 12.0 SARI +1F97C..1F97F ; Not_XID # 11.0 [4] LAB COAT..FLAT SHOE +1F980..1F984 ; Not_XID # 8.0 [5] CRAB..UNICORN FACE +1F985..1F991 ; Not_XID # 9.0 [13] EAGLE..SQUID +1F992..1F997 ; Not_XID # 10.0 [6] GIRAFFE FACE..CRICKET +1F998..1F9A2 ; Not_XID # 11.0 [11] KANGAROO..SWAN +1F9A3..1F9A4 ; Not_XID # 13.0 [2] MAMMOTH..DODO +1F9A5..1F9AA ; Not_XID # 12.0 [6] SLOTH..OYSTER +1F9AB..1F9AD ; Not_XID # 13.0 [3] BEAVER..SEAL +1F9AE..1F9AF ; Not_XID # 12.0 [2] GUIDE DOG..PROBING CANE +1F9B0..1F9B9 ; Not_XID # 11.0 [10] EMOJI COMPONENT RED HAIR..SUPERVILLAIN +1F9BA..1F9BF ; Not_XID # 12.0 [6] SAFETY VEST..MECHANICAL LEG +1F9C0 ; Not_XID # 8.0 CHEESE WEDGE +1F9C1..1F9C2 ; Not_XID # 11.0 [2] CUPCAKE..SALT SHAKER +1F9C3..1F9CA ; Not_XID # 12.0 [8] BEVERAGE BOX..ICE CUBE +1F9CB ; Not_XID # 13.0 BUBBLE TEA +1F9CC ; Not_XID # 14.0 TROLL +1F9CD..1F9CF ; Not_XID # 12.0 [3] STANDING PERSON..DEAF PERSON +1F9D0..1F9E6 ; Not_XID # 10.0 [23] FACE WITH MONOCLE..SOCKS +1F9E7..1F9FF ; Not_XID # 11.0 [25] RED GIFT ENVELOPE..NAZAR AMULET +1FA00..1FA53 ; Not_XID # 12.0 [84] NEUTRAL CHESS KING..BLACK CHESS KNIGHT-BISHOP +1FA60..1FA6D ; Not_XID # 11.0 [14] XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER +1FA70..1FA73 ; Not_XID # 12.0 [4] BALLET SHOES..SHORTS +1FA74 ; Not_XID # 13.0 THONG SANDAL +1FA78..1FA7A ; Not_XID # 12.0 [3] DROP OF BLOOD..STETHOSCOPE +1FA7B..1FA7C ; Not_XID # 14.0 [2] X-RAY..CRUTCH +1FA80..1FA82 ; Not_XID # 12.0 [3] YO-YO..PARACHUTE +1FA83..1FA86 ; Not_XID # 13.0 [4] BOOMERANG..NESTING DOLLS +1FA90..1FA95 ; Not_XID # 12.0 [6] RINGED PLANET..BANJO +1FA96..1FAA8 ; Not_XID # 13.0 [19] MILITARY HELMET..ROCK +1FAA9..1FAAC ; Not_XID # 14.0 [4] MIRROR BALL..HAMSA +1FAB0..1FAB6 ; Not_XID # 13.0 [7] FLY..FEATHER +1FAB7..1FABA ; Not_XID # 14.0 [4] LOTUS..NEST WITH EGGS +1FAC0..1FAC2 ; Not_XID # 13.0 [3] ANATOMICAL HEART..PEOPLE HUGGING +1FAC3..1FAC5 ; Not_XID # 14.0 [3] PREGNANT MAN..PERSON WITH CROWN +1FAD0..1FAD6 ; Not_XID # 13.0 [7] BLUEBERRIES..TEAPOT +1FAD7..1FAD9 ; Not_XID # 14.0 [3] POURING LIQUID..JAR +1FAE0..1FAE7 ; Not_XID # 14.0 [8] MELTING FACE..BUBBLES +1FAF0..1FAF6 ; Not_XID # 14.0 [7] HAND WITH INDEX FINGER AND THUMB CROSSED..HEART HANDS +1FB00..1FB92 ; Not_XID # 13.0 [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK +1FB94..1FBCA ; Not_XID # 13.0 [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON + +# Total code points: 5640 + +# Identifier_Type: Not_NFKC + +00A0 ; Not_NFKC # 1.1 NO-BREAK SPACE +00A8 ; Not_NFKC # 1.1 DIAERESIS +00AA ; Not_NFKC # 1.1 FEMININE ORDINAL INDICATOR +00AF ; Not_NFKC # 1.1 MACRON +00B2..00B5 ; Not_NFKC # 1.1 [4] SUPERSCRIPT TWO..MICRO SIGN +00B8..00BA ; Not_NFKC # 1.1 [3] CEDILLA..MASCULINE ORDINAL INDICATOR +00BC..00BE ; Not_NFKC # 1.1 [3] VULGAR FRACTION ONE QUARTER..VULGAR FRACTION THREE QUARTERS +0132..0133 ; Not_NFKC # 1.1 [2] LATIN CAPITAL LIGATURE IJ..LATIN SMALL LIGATURE IJ +013F..0140 ; Not_NFKC # 1.1 [2] LATIN CAPITAL LETTER L WITH MIDDLE DOT..LATIN SMALL LETTER L WITH MIDDLE DOT +017F ; Not_NFKC # 1.1 LATIN SMALL LETTER LONG S +01C4..01CC ; Not_NFKC # 1.1 [9] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER NJ +01F1..01F3 ; Not_NFKC # 1.1 [3] LATIN CAPITAL LETTER DZ..LATIN SMALL LETTER DZ +02B0..02B8 ; Not_NFKC # 1.1 [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02D8..02DD ; Not_NFKC # 1.1 [6] BREVE..DOUBLE ACUTE ACCENT +02E0..02E4 ; Not_NFKC # 1.1 [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +0340..0341 ; Not_NFKC # 1.1 [2] COMBINING GRAVE TONE MARK..COMBINING ACUTE TONE MARK +0343..0344 ; Not_NFKC # 1.1 [2] COMBINING GREEK KORONIS..COMBINING GREEK DIALYTIKA TONOS +0374 ; Not_NFKC # 1.1 GREEK NUMERAL SIGN +037A ; Not_NFKC # 1.1 GREEK YPOGEGRAMMENI +037E ; Not_NFKC # 1.1 GREEK QUESTION MARK +0384..0385 ; Not_NFKC # 1.1 [2] GREEK TONOS..GREEK DIALYTIKA TONOS +0387 ; Not_NFKC # 1.1 GREEK ANO TELEIA +03D0..03D6 ; Not_NFKC # 1.1 [7] GREEK BETA SYMBOL..GREEK PI SYMBOL +03F0..03F2 ; Not_NFKC # 1.1 [3] GREEK KAPPA SYMBOL..GREEK LUNATE SIGMA SYMBOL +03F4..03F5 ; Not_NFKC # 3.1 [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +03F9 ; Not_NFKC # 4.0 GREEK CAPITAL LUNATE SIGMA SYMBOL +0587 ; Not_NFKC # 1.1 ARMENIAN SMALL LIGATURE ECH YIWN +0675..0678 ; Not_NFKC # 1.1 [4] ARABIC LETTER HIGH HAMZA ALEF..ARABIC LETTER HIGH HAMZA YEH +0958..095F ; Not_NFKC # 1.1 [8] DEVANAGARI LETTER QA..DEVANAGARI LETTER YYA +09DC..09DD ; Not_NFKC # 1.1 [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF ; Not_NFKC # 1.1 BENGALI LETTER YYA +0A33 ; Not_NFKC # 1.1 GURMUKHI LETTER LLA +0A36 ; Not_NFKC # 1.1 GURMUKHI LETTER SHA +0A59..0A5B ; Not_NFKC # 1.1 [3] GURMUKHI LETTER KHHA..GURMUKHI LETTER ZA +0A5E ; Not_NFKC # 1.1 GURMUKHI LETTER FA +0B5C..0B5D ; Not_NFKC # 1.1 [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0E33 ; Not_NFKC # 1.1 THAI CHARACTER SARA AM +0EB3 ; Not_NFKC # 1.1 LAO VOWEL SIGN AM +0EDC..0EDD ; Not_NFKC # 1.1 [2] LAO HO NO..LAO HO MO +0F0C ; Not_NFKC # 2.0 TIBETAN MARK DELIMITER TSHEG BSTAR +0F43 ; Not_NFKC # 2.0 TIBETAN LETTER GHA +0F4D ; Not_NFKC # 2.0 TIBETAN LETTER DDHA +0F52 ; Not_NFKC # 2.0 TIBETAN LETTER DHA +0F57 ; Not_NFKC # 2.0 TIBETAN LETTER BHA +0F5C ; Not_NFKC # 2.0 TIBETAN LETTER DZHA +0F69 ; Not_NFKC # 2.0 TIBETAN LETTER KSSA +0F73 ; Not_NFKC # 2.0 TIBETAN VOWEL SIGN II +0F75..0F76 ; Not_NFKC # 2.0 [2] TIBETAN VOWEL SIGN UU..TIBETAN VOWEL SIGN VOCALIC R +0F78 ; Not_NFKC # 2.0 TIBETAN VOWEL SIGN VOCALIC L +0F81 ; Not_NFKC # 2.0 TIBETAN VOWEL SIGN REVERSED II +0F93 ; Not_NFKC # 2.0 TIBETAN SUBJOINED LETTER GHA +0F9D ; Not_NFKC # 2.0 TIBETAN SUBJOINED LETTER DDHA +0FA2 ; Not_NFKC # 2.0 TIBETAN SUBJOINED LETTER DHA +0FA7 ; Not_NFKC # 2.0 TIBETAN SUBJOINED LETTER BHA +0FAC ; Not_NFKC # 2.0 TIBETAN SUBJOINED LETTER DZHA +0FB9 ; Not_NFKC # 2.0 TIBETAN SUBJOINED LETTER KSSA +10FC ; Not_NFKC # 4.1 MODIFIER LETTER GEORGIAN NAR +1D2C..1D2E ; Not_NFKC # 4.0 [3] MODIFIER LETTER CAPITAL A..MODIFIER LETTER CAPITAL B +1D30..1D3A ; Not_NFKC # 4.0 [11] MODIFIER LETTER CAPITAL D..MODIFIER LETTER CAPITAL N +1D3C..1D4D ; Not_NFKC # 4.0 [18] MODIFIER LETTER CAPITAL O..MODIFIER LETTER SMALL G +1D4F..1D6A ; Not_NFKC # 4.0 [28] MODIFIER LETTER SMALL K..GREEK SUBSCRIPT SMALL LETTER CHI +1D78 ; Not_NFKC # 4.1 MODIFIER LETTER CYRILLIC EN +1D9B..1DBF ; Not_NFKC # 4.1 [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +1E9A ; Not_NFKC # 1.1 LATIN SMALL LETTER A WITH RIGHT HALF RING +1E9B ; Not_NFKC # 2.0 LATIN SMALL LETTER LONG S WITH DOT ABOVE +1F71 ; Not_NFKC # 1.1 GREEK SMALL LETTER ALPHA WITH OXIA +1F73 ; Not_NFKC # 1.1 GREEK SMALL LETTER EPSILON WITH OXIA +1F75 ; Not_NFKC # 1.1 GREEK SMALL LETTER ETA WITH OXIA +1F77 ; Not_NFKC # 1.1 GREEK SMALL LETTER IOTA WITH OXIA +1F79 ; Not_NFKC # 1.1 GREEK SMALL LETTER OMICRON WITH OXIA +1F7B ; Not_NFKC # 1.1 GREEK SMALL LETTER UPSILON WITH OXIA +1F7D ; Not_NFKC # 1.1 GREEK SMALL LETTER OMEGA WITH OXIA +1FBB ; Not_NFKC # 1.1 GREEK CAPITAL LETTER ALPHA WITH OXIA +1FBD..1FC1 ; Not_NFKC # 1.1 [5] GREEK KORONIS..GREEK DIALYTIKA AND PERISPOMENI +1FC9 ; Not_NFKC # 1.1 GREEK CAPITAL LETTER EPSILON WITH OXIA +1FCB ; Not_NFKC # 1.1 GREEK CAPITAL LETTER ETA WITH OXIA +1FCD..1FCF ; Not_NFKC # 1.1 [3] GREEK PSILI AND VARIA..GREEK PSILI AND PERISPOMENI +1FD3 ; Not_NFKC # 1.1 GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FDB ; Not_NFKC # 1.1 GREEK CAPITAL LETTER IOTA WITH OXIA +1FDD..1FDF ; Not_NFKC # 1.1 [3] GREEK DASIA AND VARIA..GREEK DASIA AND PERISPOMENI +1FE3 ; Not_NFKC # 1.1 GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +1FEB ; Not_NFKC # 1.1 GREEK CAPITAL LETTER UPSILON WITH OXIA +1FED..1FEF ; Not_NFKC # 1.1 [3] GREEK DIALYTIKA AND VARIA..GREEK VARIA +1FF9 ; Not_NFKC # 1.1 GREEK CAPITAL LETTER OMICRON WITH OXIA +1FFB ; Not_NFKC # 1.1 GREEK CAPITAL LETTER OMEGA WITH OXIA +1FFD..1FFE ; Not_NFKC # 1.1 [2] GREEK OXIA..GREEK DASIA +2000..200A ; Not_NFKC # 1.1 [11] EN QUAD..HAIR SPACE +2011 ; Not_NFKC # 1.1 NON-BREAKING HYPHEN +2017 ; Not_NFKC # 1.1 DOUBLE LOW LINE +2024..2026 ; Not_NFKC # 1.1 [3] ONE DOT LEADER..HORIZONTAL ELLIPSIS +202F ; Not_NFKC # 3.0 NARROW NO-BREAK SPACE +2033..2034 ; Not_NFKC # 1.1 [2] DOUBLE PRIME..TRIPLE PRIME +2036..2037 ; Not_NFKC # 1.1 [2] REVERSED DOUBLE PRIME..REVERSED TRIPLE PRIME +203C ; Not_NFKC # 1.1 DOUBLE EXCLAMATION MARK +203E ; Not_NFKC # 1.1 OVERLINE +2047 ; Not_NFKC # 3.2 DOUBLE QUESTION MARK +2048..2049 ; Not_NFKC # 3.0 [2] QUESTION EXCLAMATION MARK..EXCLAMATION QUESTION MARK +2057 ; Not_NFKC # 3.2 QUADRUPLE PRIME +205F ; Not_NFKC # 3.2 MEDIUM MATHEMATICAL SPACE +2070 ; Not_NFKC # 1.1 SUPERSCRIPT ZERO +2071 ; Not_NFKC # 3.2 SUPERSCRIPT LATIN SMALL LETTER I +2074..208E ; Not_NFKC # 1.1 [27] SUPERSCRIPT FOUR..SUBSCRIPT RIGHT PARENTHESIS +2090..2094 ; Not_NFKC # 4.1 [5] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER SCHWA +2095..209C ; Not_NFKC # 6.0 [8] LATIN SUBSCRIPT SMALL LETTER H..LATIN SUBSCRIPT SMALL LETTER T +20A8 ; Not_NFKC # 1.1 RUPEE SIGN +2100..2103 ; Not_NFKC # 1.1 [4] ACCOUNT OF..DEGREE CELSIUS +2105..2107 ; Not_NFKC # 1.1 [3] CARE OF..EULER CONSTANT +2109..2113 ; Not_NFKC # 1.1 [11] DEGREE FAHRENHEIT..SCRIPT SMALL L +2115..2116 ; Not_NFKC # 1.1 [2] DOUBLE-STRUCK CAPITAL N..NUMERO SIGN +2119..211D ; Not_NFKC # 1.1 [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2120..2122 ; Not_NFKC # 1.1 [3] SERVICE MARK..TRADE MARK SIGN +2124 ; Not_NFKC # 1.1 DOUBLE-STRUCK CAPITAL Z +2126 ; Not_NFKC # 1.1 OHM SIGN +2128 ; Not_NFKC # 1.1 BLACK-LETTER CAPITAL Z +212A..212D ; Not_NFKC # 1.1 [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212F..2131 ; Not_NFKC # 1.1 [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2138 ; Not_NFKC # 1.1 [6] SCRIPT CAPITAL M..DALET SYMBOL +2139 ; Not_NFKC # 3.0 INFORMATION SOURCE +213B ; Not_NFKC # 4.0 FACSIMILE SIGN +213C ; Not_NFKC # 4.1 DOUBLE-STRUCK SMALL PI +213D..2140 ; Not_NFKC # 3.2 [4] DOUBLE-STRUCK SMALL GAMMA..DOUBLE-STRUCK N-ARY SUMMATION +2145..2149 ; Not_NFKC # 3.2 [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +2150..2152 ; Not_NFKC # 5.2 [3] VULGAR FRACTION ONE SEVENTH..VULGAR FRACTION ONE TENTH +2153..217F ; Not_NFKC # 1.1 [45] VULGAR FRACTION ONE THIRD..SMALL ROMAN NUMERAL ONE THOUSAND +2189 ; Not_NFKC # 5.2 VULGAR FRACTION ZERO THIRDS +222C..222D ; Not_NFKC # 1.1 [2] DOUBLE INTEGRAL..TRIPLE INTEGRAL +222F..2230 ; Not_NFKC # 1.1 [2] SURFACE INTEGRAL..VOLUME INTEGRAL +2460..24EA ; Not_NFKC # 1.1 [139] CIRCLED DIGIT ONE..CIRCLED DIGIT ZERO +2A0C ; Not_NFKC # 3.2 QUADRUPLE INTEGRAL OPERATOR +2A74..2A76 ; Not_NFKC # 3.2 [3] DOUBLE COLON EQUAL..THREE CONSECUTIVE EQUALS SIGNS +2ADC ; Not_NFKC # 3.2 FORKING +2C7C..2C7D ; Not_NFKC # 5.1 [2] LATIN SUBSCRIPT SMALL LETTER J..MODIFIER LETTER CAPITAL V +2D6F ; Not_NFKC # 4.1 TIFINAGH MODIFIER LETTER LABIALIZATION MARK +2E9F ; Not_NFKC # 3.0 CJK RADICAL MOTHER +2EF3 ; Not_NFKC # 3.0 CJK RADICAL C-SIMPLIFIED TURTLE +2F00..2FD5 ; Not_NFKC # 3.0 [214] KANGXI RADICAL ONE..KANGXI RADICAL FLUTE +3000 ; Not_NFKC # 1.1 IDEOGRAPHIC SPACE +3036 ; Not_NFKC # 1.1 CIRCLED POSTAL MARK +3038..303A ; Not_NFKC # 3.0 [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +309B..309C ; Not_NFKC # 1.1 [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309F ; Not_NFKC # 3.2 HIRAGANA DIGRAPH YORI +30FF ; Not_NFKC # 3.2 KATAKANA DIGRAPH KOTO +3131..3163 ; Not_NFKC # 1.1 [51] HANGUL LETTER KIYEOK..HANGUL LETTER I +3165..318E ; Not_NFKC # 1.1 [42] HANGUL LETTER SSANGNIEUN..HANGUL LETTER ARAEAE +3192..319F ; Not_NFKC # 1.1 [14] IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION MAN MARK +3200..321C ; Not_NFKC # 1.1 [29] PARENTHESIZED HANGUL KIYEOK..PARENTHESIZED HANGUL CIEUC U +321D..321E ; Not_NFKC # 4.0 [2] PARENTHESIZED KOREAN CHARACTER OJEON..PARENTHESIZED KOREAN CHARACTER O HU +3220..3243 ; Not_NFKC # 1.1 [36] PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH REACH +3244..3247 ; Not_NFKC # 5.2 [4] CIRCLED IDEOGRAPH QUESTION..CIRCLED IDEOGRAPH KOTO +3250 ; Not_NFKC # 4.0 PARTNERSHIP SIGN +3251..325F ; Not_NFKC # 3.2 [15] CIRCLED NUMBER TWENTY ONE..CIRCLED NUMBER THIRTY FIVE +3260..327B ; Not_NFKC # 1.1 [28] CIRCLED HANGUL KIYEOK..CIRCLED HANGUL HIEUH A +327C..327D ; Not_NFKC # 4.0 [2] CIRCLED KOREAN CHARACTER CHAMKO..CIRCLED KOREAN CHARACTER JUEUI +327E ; Not_NFKC # 4.1 CIRCLED HANGUL IEUNG U +3280..32B0 ; Not_NFKC # 1.1 [49] CIRCLED IDEOGRAPH ONE..CIRCLED IDEOGRAPH NIGHT +32B1..32BF ; Not_NFKC # 3.2 [15] CIRCLED NUMBER THIRTY SIX..CIRCLED NUMBER FIFTY +32C0..32CB ; Not_NFKC # 1.1 [12] IDEOGRAPHIC TELEGRAPH SYMBOL FOR JANUARY..IDEOGRAPHIC TELEGRAPH SYMBOL FOR DECEMBER +32CC..32CF ; Not_NFKC # 4.0 [4] SQUARE HG..LIMITED LIABILITY SIGN +32D0..32FE ; Not_NFKC # 1.1 [47] CIRCLED KATAKANA A..CIRCLED KATAKANA WO +32FF ; Not_NFKC # 12.1 SQUARE ERA NAME REIWA +3300..3376 ; Not_NFKC # 1.1 [119] SQUARE APAATO..SQUARE PC +3377..337A ; Not_NFKC # 4.0 [4] SQUARE DM..SQUARE IU +337B..33DD ; Not_NFKC # 1.1 [99] SQUARE ERA NAME HEISEI..SQUARE WB +33DE..33DF ; Not_NFKC # 4.0 [2] SQUARE V OVER M..SQUARE A OVER M +33E0..33FE ; Not_NFKC # 1.1 [31] IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY ONE..IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THIRTY-ONE +33FF ; Not_NFKC # 4.0 SQUARE GAL +A69C..A69D ; Not_NFKC # 7.0 [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN +A770 ; Not_NFKC # 5.1 MODIFIER LETTER US +A7F2..A7F4 ; Not_NFKC # 14.0 [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q +A7F8..A7F9 ; Not_NFKC # 6.1 [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE +AB5C..AB5F ; Not_NFKC # 7.0 [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK +AB69 ; Not_NFKC # 13.0 MODIFIER LETTER SMALL TURNED W +F900..FA0D ; Not_NFKC # 1.1 [270] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA0D +FA10 ; Not_NFKC # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA10 +FA12 ; Not_NFKC # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA12 +FA15..FA1E ; Not_NFKC # 1.1 [10] CJK COMPATIBILITY IDEOGRAPH-FA15..CJK COMPATIBILITY IDEOGRAPH-FA1E +FA20 ; Not_NFKC # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA20 +FA22 ; Not_NFKC # 1.1 CJK COMPATIBILITY IDEOGRAPH-FA22 +FA25..FA26 ; Not_NFKC # 1.1 [2] CJK COMPATIBILITY IDEOGRAPH-FA25..CJK COMPATIBILITY IDEOGRAPH-FA26 +FA2A..FA2D ; Not_NFKC # 1.1 [4] CJK COMPATIBILITY IDEOGRAPH-FA2A..CJK COMPATIBILITY IDEOGRAPH-FA2D +FA2E..FA2F ; Not_NFKC # 6.1 [2] CJK COMPATIBILITY IDEOGRAPH-FA2E..CJK COMPATIBILITY IDEOGRAPH-FA2F +FA30..FA6A ; Not_NFKC # 3.2 [59] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6A +FA6B..FA6D ; Not_NFKC # 5.2 [3] CJK COMPATIBILITY IDEOGRAPH-FA6B..CJK COMPATIBILITY IDEOGRAPH-FA6D +FA70..FAD9 ; Not_NFKC # 4.1 [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +FB00..FB06 ; Not_NFKC # 1.1 [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Not_NFKC # 1.1 [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D ; Not_NFKC # 3.0 HEBREW LETTER YOD WITH HIRIQ +FB1F..FB36 ; Not_NFKC # 1.1 [24] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C ; Not_NFKC # 1.1 [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E ; Not_NFKC # 1.1 HEBREW LETTER MEM WITH DAGESH +FB40..FB41 ; Not_NFKC # 1.1 [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44 ; Not_NFKC # 1.1 [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FBB1 ; Not_NFKC # 1.1 [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3..FD3D ; Not_NFKC # 1.1 [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD50..FD8F ; Not_NFKC # 1.1 [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7 ; Not_NFKC # 1.1 [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0..FDFB ; Not_NFKC # 1.1 [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU +FDFC ; Not_NFKC # 3.2 RIAL SIGN +FE10..FE19 ; Not_NFKC # 4.1 [10] PRESENTATION FORM FOR VERTICAL COMMA..PRESENTATION FORM FOR VERTICAL HORIZONTAL ELLIPSIS +FE30..FE44 ; Not_NFKC # 1.1 [21] PRESENTATION FORM FOR VERTICAL TWO DOT LEADER..PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET +FE47..FE48 ; Not_NFKC # 4.0 [2] PRESENTATION FORM FOR VERTICAL LEFT SQUARE BRACKET..PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET +FE49..FE52 ; Not_NFKC # 1.1 [10] DASHED OVERLINE..SMALL FULL STOP +FE54..FE66 ; Not_NFKC # 1.1 [19] SMALL SEMICOLON..SMALL EQUALS SIGN +FE68..FE6B ; Not_NFKC # 1.1 [4] SMALL REVERSE SOLIDUS..SMALL COMMERCIAL AT +FE70..FE72 ; Not_NFKC # 1.1 [3] ARABIC FATHATAN ISOLATED FORM..ARABIC DAMMATAN ISOLATED FORM +FE74 ; Not_NFKC # 1.1 ARABIC KASRATAN ISOLATED FORM +FE76..FEFC ; Not_NFKC # 1.1 [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF01..FF5E ; Not_NFKC # 1.1 [94] FULLWIDTH EXCLAMATION MARK..FULLWIDTH TILDE +FF5F..FF60 ; Not_NFKC # 3.2 [2] FULLWIDTH LEFT WHITE PARENTHESIS..FULLWIDTH RIGHT WHITE PARENTHESIS +FF61..FF9F ; Not_NFKC # 1.1 [63] HALFWIDTH IDEOGRAPHIC FULL STOP..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA1..FFBE ; Not_NFKC # 1.1 [30] HALFWIDTH HANGUL LETTER KIYEOK..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7 ; Not_NFKC # 1.1 [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF ; Not_NFKC # 1.1 [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7 ; Not_NFKC # 1.1 [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC ; Not_NFKC # 1.1 [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +FFE0..FFE6 ; Not_NFKC # 1.1 [7] FULLWIDTH CENT SIGN..FULLWIDTH WON SIGN +FFE8..FFEE ; Not_NFKC # 1.1 [7] HALFWIDTH FORMS LIGHT VERTICAL..HALFWIDTH WHITE CIRCLE +10781..10785 ; Not_NFKC # 14.0 [5] MODIFIER LETTER SUPERSCRIPT TRIANGULAR COLON..MODIFIER LETTER SMALL B WITH HOOK +10787..107B0 ; Not_NFKC # 14.0 [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK +107B2..107BA ; Not_NFKC # 14.0 [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL +1D15E..1D164 ; Not_NFKC # 3.1 [7] MUSICAL SYMBOL HALF NOTE..MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE +1D1BB..1D1C0 ; Not_NFKC # 3.1 [6] MUSICAL SYMBOL MINIMA..MUSICAL SYMBOL FUSA BLACK +1D400..1D454 ; Not_NFKC # 3.1 [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Not_NFKC # 3.1 [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Not_NFKC # 3.1 [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Not_NFKC # 3.1 MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Not_NFKC # 3.1 [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Not_NFKC # 3.1 [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Not_NFKC # 3.1 [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Not_NFKC # 3.1 MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C0 ; Not_NFKC # 3.1 [4] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL K +1D4C1 ; Not_NFKC # 4.0 MATHEMATICAL SCRIPT SMALL L +1D4C2..1D4C3 ; Not_NFKC # 3.1 [2] MATHEMATICAL SCRIPT SMALL M..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Not_NFKC # 3.1 [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Not_NFKC # 3.1 [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Not_NFKC # 3.1 [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Not_NFKC # 3.1 [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Not_NFKC # 3.1 [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Not_NFKC # 3.1 [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Not_NFKC # 3.1 [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Not_NFKC # 3.1 MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Not_NFKC # 3.1 [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A3 ; Not_NFKC # 3.1 [338] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL MONOSPACE SMALL Z +1D6A4..1D6A5 ; Not_NFKC # 4.1 [2] MATHEMATICAL ITALIC SMALL DOTLESS I..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D7C9 ; Not_NFKC # 3.1 [290] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL +1D7CA..1D7CB ; Not_NFKC # 5.0 [2] MATHEMATICAL BOLD CAPITAL DIGAMMA..MATHEMATICAL BOLD SMALL DIGAMMA +1D7CE..1D7FF ; Not_NFKC # 3.1 [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE +1EE00..1EE03 ; Not_NFKC # 6.1 [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL +1EE05..1EE1F ; Not_NFKC # 6.1 [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF +1EE21..1EE22 ; Not_NFKC # 6.1 [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM +1EE24 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL INITIAL HEH +1EE27 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL INITIAL HAH +1EE29..1EE32 ; Not_NFKC # 6.1 [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF +1EE34..1EE37 ; Not_NFKC # 6.1 [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH +1EE39 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL INITIAL DAD +1EE3B ; Not_NFKC # 6.1 ARABIC MATHEMATICAL INITIAL GHAIN +1EE42 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED JEEM +1EE47 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED HAH +1EE49 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED YEH +1EE4B ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED LAM +1EE4D..1EE4F ; Not_NFKC # 6.1 [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN +1EE51..1EE52 ; Not_NFKC # 6.1 [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF +1EE54 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED SHEEN +1EE57 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED KHAH +1EE59 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED DAD +1EE5B ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED GHAIN +1EE5D ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED DOTLESS NOON +1EE5F ; Not_NFKC # 6.1 ARABIC MATHEMATICAL TAILED DOTLESS QAF +1EE61..1EE62 ; Not_NFKC # 6.1 [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM +1EE64 ; Not_NFKC # 6.1 ARABIC MATHEMATICAL STRETCHED HEH +1EE67..1EE6A ; Not_NFKC # 6.1 [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF +1EE6C..1EE72 ; Not_NFKC # 6.1 [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF +1EE74..1EE77 ; Not_NFKC # 6.1 [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH +1EE79..1EE7C ; Not_NFKC # 6.1 [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH +1EE7E ; Not_NFKC # 6.1 ARABIC MATHEMATICAL STRETCHED DOTLESS FEH +1EE80..1EE89 ; Not_NFKC # 6.1 [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH +1EE8B..1EE9B ; Not_NFKC # 6.1 [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN +1EEA1..1EEA3 ; Not_NFKC # 6.1 [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL +1EEA5..1EEA9 ; Not_NFKC # 6.1 [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH +1EEAB..1EEBB ; Not_NFKC # 6.1 [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN +1F100..1F10A ; Not_NFKC # 5.2 [11] DIGIT ZERO FULL STOP..DIGIT NINE COMMA +1F110..1F12E ; Not_NFKC # 5.2 [31] PARENTHESIZED LATIN CAPITAL LETTER A..CIRCLED WZ +1F130 ; Not_NFKC # 6.0 SQUARED LATIN CAPITAL LETTER A +1F131 ; Not_NFKC # 5.2 SQUARED LATIN CAPITAL LETTER B +1F132..1F13C ; Not_NFKC # 6.0 [11] SQUARED LATIN CAPITAL LETTER C..SQUARED LATIN CAPITAL LETTER M +1F13D ; Not_NFKC # 5.2 SQUARED LATIN CAPITAL LETTER N +1F13E ; Not_NFKC # 6.0 SQUARED LATIN CAPITAL LETTER O +1F13F ; Not_NFKC # 5.2 SQUARED LATIN CAPITAL LETTER P +1F140..1F141 ; Not_NFKC # 6.0 [2] SQUARED LATIN CAPITAL LETTER Q..SQUARED LATIN CAPITAL LETTER R +1F142 ; Not_NFKC # 5.2 SQUARED LATIN CAPITAL LETTER S +1F143..1F145 ; Not_NFKC # 6.0 [3] SQUARED LATIN CAPITAL LETTER T..SQUARED LATIN CAPITAL LETTER V +1F146 ; Not_NFKC # 5.2 SQUARED LATIN CAPITAL LETTER W +1F147..1F149 ; Not_NFKC # 6.0 [3] SQUARED LATIN CAPITAL LETTER X..SQUARED LATIN CAPITAL LETTER Z +1F14A..1F14E ; Not_NFKC # 5.2 [5] SQUARED HV..SQUARED PPV +1F14F ; Not_NFKC # 6.0 SQUARED WC +1F16A..1F16B ; Not_NFKC # 6.1 [2] RAISED MC SIGN..RAISED MD SIGN +1F16C ; Not_NFKC # 12.0 RAISED MR SIGN +1F190 ; Not_NFKC # 5.2 SQUARE DJ +1F200 ; Not_NFKC # 5.2 SQUARE HIRAGANA HOKA +1F201..1F202 ; Not_NFKC # 6.0 [2] SQUARED KATAKANA KOKO..SQUARED KATAKANA SA +1F210..1F231 ; Not_NFKC # 5.2 [34] SQUARED CJK UNIFIED IDEOGRAPH-624B..SQUARED CJK UNIFIED IDEOGRAPH-6253 +1F232..1F23A ; Not_NFKC # 6.0 [9] SQUARED CJK UNIFIED IDEOGRAPH-7981..SQUARED CJK UNIFIED IDEOGRAPH-55B6 +1F23B ; Not_NFKC # 9.0 SQUARED CJK UNIFIED IDEOGRAPH-914D +1F240..1F248 ; Not_NFKC # 5.2 [9] TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-672C..TORTOISE SHELL BRACKETED CJK UNIFIED IDEOGRAPH-6557 +1F250..1F251 ; Not_NFKC # 6.0 [2] CIRCLED IDEOGRAPH ADVANTAGE..CIRCLED IDEOGRAPH ACCEPT +1FBF0..1FBF9 ; Not_NFKC # 13.0 [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE +2F800..2FA1D ; Not_NFKC # 3.1 [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + +# Total code points: 4859 + +# Identifier_Type: Default_Ignorable + +00AD ; Default_Ignorable # 1.1 SOFT HYPHEN +034F ; Default_Ignorable # 3.2 COMBINING GRAPHEME JOINER +061C ; Default_Ignorable # 6.3 ARABIC LETTER MARK +115F..1160 ; Default_Ignorable # 1.1 [2] HANGUL CHOSEONG FILLER..HANGUL JUNGSEONG FILLER +17B4..17B5 ; Default_Ignorable # 3.0 [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA +180B..180D ; Default_Ignorable # 3.0 [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE +180E ; Default_Ignorable # 3.0 MONGOLIAN VOWEL SEPARATOR +180F ; Default_Ignorable # 14.0 MONGOLIAN FREE VARIATION SELECTOR FOUR +200B ; Default_Ignorable # 1.1 ZERO WIDTH SPACE +200E..200F ; Default_Ignorable # 1.1 [2] LEFT-TO-RIGHT MARK..RIGHT-TO-LEFT MARK +202A..202E ; Default_Ignorable # 1.1 [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE +2060..2063 ; Default_Ignorable # 3.2 [4] WORD JOINER..INVISIBLE SEPARATOR +2064 ; Default_Ignorable # 5.1 INVISIBLE PLUS +2066..2069 ; Default_Ignorable # 6.3 [4] LEFT-TO-RIGHT ISOLATE..POP DIRECTIONAL ISOLATE +3164 ; Default_Ignorable # 1.1 HANGUL FILLER +FE00..FE0F ; Default_Ignorable # 3.2 [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 +FEFF ; Default_Ignorable # 1.1 ZERO WIDTH NO-BREAK SPACE +FFA0 ; Default_Ignorable # 1.1 HALFWIDTH HANGUL FILLER +1BCA0..1BCA3 ; Default_Ignorable # 7.0 [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP +1D173..1D17A ; Default_Ignorable # 3.1 [8] MUSICAL SYMBOL BEGIN BEAM..MUSICAL SYMBOL END PHRASE +E0020..E007F ; Default_Ignorable # 3.1 [96] TAG SPACE..CANCEL TAG +E0100..E01EF ; Default_Ignorable # 4.0 [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 + +# Total code points: 396 + +# Identifier_Type: Deprecated + +0149 ; Deprecated # 1.1 LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +0673 ; Deprecated # 1.1 ARABIC LETTER ALEF WITH WAVY HAMZA BELOW +0F77 ; Deprecated # 2.0 TIBETAN VOWEL SIGN VOCALIC RR +0F79 ; Deprecated # 2.0 TIBETAN VOWEL SIGN VOCALIC LL +17A3..17A4 ; Deprecated # 3.0 [2] KHMER INDEPENDENT VOWEL QAQ..KHMER INDEPENDENT VOWEL QAA +206A..206F ; Deprecated # 1.1 [6] INHIBIT SYMMETRIC SWAPPING..NOMINAL DIGIT SHAPES +2329..232A ; Deprecated # 1.1 [2] LEFT-POINTING ANGLE BRACKET..RIGHT-POINTING ANGLE BRACKET +E0001 ; Deprecated # 3.1 LANGUAGE TAG + +# Total code points: 15 diff --git a/lib/elixir/unicode/PropList.txt b/lib/elixir/unicode/PropList.txt new file mode 100644 index 00000000000..0a5a9346828 --- /dev/null +++ b/lib/elixir/unicode/PropList.txt @@ -0,0 +1,1743 @@ +# PropList-14.0.0.txt +# Date: 2021-08-12, 23:13:05 GMT +# © 2021 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see http://www.unicode.org/reports/tr44/ + +# ================================================ + +0009..000D ; White_Space # Cc [5] .. +0020 ; White_Space # Zs SPACE +0085 ; White_Space # Cc +00A0 ; White_Space # Zs NO-BREAK SPACE +1680 ; White_Space # Zs OGHAM SPACE MARK +2000..200A ; White_Space # Zs [11] EN QUAD..HAIR SPACE +2028 ; White_Space # Zl LINE SEPARATOR +2029 ; White_Space # Zp PARAGRAPH SEPARATOR +202F ; White_Space # Zs NARROW NO-BREAK SPACE +205F ; White_Space # Zs MEDIUM MATHEMATICAL SPACE +3000 ; White_Space # Zs IDEOGRAPHIC SPACE + +# Total code points: 25 + +# ================================================ + +061C ; Bidi_Control # Cf ARABIC LETTER MARK +200E..200F ; Bidi_Control # Cf [2] LEFT-TO-RIGHT MARK..RIGHT-TO-LEFT MARK +202A..202E ; Bidi_Control # Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-LEFT OVERRIDE +2066..2069 ; Bidi_Control # Cf [4] LEFT-TO-RIGHT ISOLATE..POP DIRECTIONAL ISOLATE + +# Total code points: 12 + +# ================================================ + +200C..200D ; Join_Control # Cf [2] ZERO WIDTH NON-JOINER..ZERO WIDTH JOINER + +# Total code points: 2 + +# ================================================ + +002D ; Dash # Pd HYPHEN-MINUS +058A ; Dash # Pd ARMENIAN HYPHEN +05BE ; Dash # Pd HEBREW PUNCTUATION MAQAF +1400 ; Dash # Pd CANADIAN SYLLABICS HYPHEN +1806 ; Dash # Pd MONGOLIAN TODO SOFT HYPHEN +2010..2015 ; Dash # Pd [6] HYPHEN..HORIZONTAL BAR +2053 ; Dash # Po SWUNG DASH +207B ; Dash # Sm SUPERSCRIPT MINUS +208B ; Dash # Sm SUBSCRIPT MINUS +2212 ; Dash # Sm MINUS SIGN +2E17 ; Dash # Pd DOUBLE OBLIQUE HYPHEN +2E1A ; Dash # Pd HYPHEN WITH DIAERESIS +2E3A..2E3B ; Dash # Pd [2] TWO-EM DASH..THREE-EM DASH +2E40 ; Dash # Pd DOUBLE HYPHEN +2E5D ; Dash # Pd OBLIQUE HYPHEN +301C ; Dash # Pd WAVE DASH +3030 ; Dash # Pd WAVY DASH +30A0 ; Dash # Pd KATAKANA-HIRAGANA DOUBLE HYPHEN +FE31..FE32 ; Dash # Pd [2] PRESENTATION FORM FOR VERTICAL EM DASH..PRESENTATION FORM FOR VERTICAL EN DASH +FE58 ; Dash # Pd SMALL EM DASH +FE63 ; Dash # Pd SMALL HYPHEN-MINUS +FF0D ; Dash # Pd FULLWIDTH HYPHEN-MINUS +10EAD ; Dash # Pd YEZIDI HYPHENATION MARK + +# Total code points: 30 + +# ================================================ + +002D ; Hyphen # Pd HYPHEN-MINUS +00AD ; Hyphen # Cf SOFT HYPHEN +058A ; Hyphen # Pd ARMENIAN HYPHEN +1806 ; Hyphen # Pd MONGOLIAN TODO SOFT HYPHEN +2010..2011 ; Hyphen # Pd [2] HYPHEN..NON-BREAKING HYPHEN +2E17 ; Hyphen # Pd DOUBLE OBLIQUE HYPHEN +30FB ; Hyphen # Po KATAKANA MIDDLE DOT +FE63 ; Hyphen # Pd SMALL HYPHEN-MINUS +FF0D ; Hyphen # Pd FULLWIDTH HYPHEN-MINUS +FF65 ; Hyphen # Po HALFWIDTH KATAKANA MIDDLE DOT + +# Total code points: 11 + +# ================================================ + +0022 ; Quotation_Mark # Po QUOTATION MARK +0027 ; Quotation_Mark # Po APOSTROPHE +00AB ; Quotation_Mark # Pi LEFT-POINTING DOUBLE ANGLE QUOTATION MARK +00BB ; Quotation_Mark # Pf RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +2018 ; Quotation_Mark # Pi LEFT SINGLE QUOTATION MARK +2019 ; Quotation_Mark # Pf RIGHT SINGLE QUOTATION MARK +201A ; Quotation_Mark # Ps SINGLE LOW-9 QUOTATION MARK +201B..201C ; Quotation_Mark # Pi [2] SINGLE HIGH-REVERSED-9 QUOTATION MARK..LEFT DOUBLE QUOTATION MARK +201D ; Quotation_Mark # Pf RIGHT DOUBLE QUOTATION MARK +201E ; Quotation_Mark # Ps DOUBLE LOW-9 QUOTATION MARK +201F ; Quotation_Mark # Pi DOUBLE HIGH-REVERSED-9 QUOTATION MARK +2039 ; Quotation_Mark # Pi SINGLE LEFT-POINTING ANGLE QUOTATION MARK +203A ; Quotation_Mark # Pf SINGLE RIGHT-POINTING ANGLE QUOTATION MARK +2E42 ; Quotation_Mark # Ps DOUBLE LOW-REVERSED-9 QUOTATION MARK +300C ; Quotation_Mark # Ps LEFT CORNER BRACKET +300D ; Quotation_Mark # Pe RIGHT CORNER BRACKET +300E ; Quotation_Mark # Ps LEFT WHITE CORNER BRACKET +300F ; Quotation_Mark # Pe RIGHT WHITE CORNER BRACKET +301D ; Quotation_Mark # Ps REVERSED DOUBLE PRIME QUOTATION MARK +301E..301F ; Quotation_Mark # Pe [2] DOUBLE PRIME QUOTATION MARK..LOW DOUBLE PRIME QUOTATION MARK +FE41 ; Quotation_Mark # Ps PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET +FE42 ; Quotation_Mark # Pe PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET +FE43 ; Quotation_Mark # Ps PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET +FE44 ; Quotation_Mark # Pe PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET +FF02 ; Quotation_Mark # Po FULLWIDTH QUOTATION MARK +FF07 ; Quotation_Mark # Po FULLWIDTH APOSTROPHE +FF62 ; Quotation_Mark # Ps HALFWIDTH LEFT CORNER BRACKET +FF63 ; Quotation_Mark # Pe HALFWIDTH RIGHT CORNER BRACKET + +# Total code points: 30 + +# ================================================ + +0021 ; Terminal_Punctuation # Po EXCLAMATION MARK +002C ; Terminal_Punctuation # Po COMMA +002E ; Terminal_Punctuation # Po FULL STOP +003A..003B ; Terminal_Punctuation # Po [2] COLON..SEMICOLON +003F ; Terminal_Punctuation # Po QUESTION MARK +037E ; Terminal_Punctuation # Po GREEK QUESTION MARK +0387 ; Terminal_Punctuation # Po GREEK ANO TELEIA +0589 ; Terminal_Punctuation # Po ARMENIAN FULL STOP +05C3 ; Terminal_Punctuation # Po HEBREW PUNCTUATION SOF PASUQ +060C ; Terminal_Punctuation # Po ARABIC COMMA +061B ; Terminal_Punctuation # Po ARABIC SEMICOLON +061D..061F ; Terminal_Punctuation # Po [3] ARABIC END OF TEXT MARK..ARABIC QUESTION MARK +06D4 ; Terminal_Punctuation # Po ARABIC FULL STOP +0700..070A ; Terminal_Punctuation # Po [11] SYRIAC END OF PARAGRAPH..SYRIAC CONTRACTION +070C ; Terminal_Punctuation # Po SYRIAC HARKLEAN METOBELUS +07F8..07F9 ; Terminal_Punctuation # Po [2] NKO COMMA..NKO EXCLAMATION MARK +0830..083E ; Terminal_Punctuation # Po [15] SAMARITAN PUNCTUATION NEQUDAA..SAMARITAN PUNCTUATION ANNAAU +085E ; Terminal_Punctuation # Po MANDAIC PUNCTUATION +0964..0965 ; Terminal_Punctuation # Po [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA +0E5A..0E5B ; Terminal_Punctuation # Po [2] THAI CHARACTER ANGKHANKHU..THAI CHARACTER KHOMUT +0F08 ; Terminal_Punctuation # Po TIBETAN MARK SBRUL SHAD +0F0D..0F12 ; Terminal_Punctuation # Po [6] TIBETAN MARK SHAD..TIBETAN MARK RGYA GRAM SHAD +104A..104B ; Terminal_Punctuation # Po [2] MYANMAR SIGN LITTLE SECTION..MYANMAR SIGN SECTION +1361..1368 ; Terminal_Punctuation # Po [8] ETHIOPIC WORDSPACE..ETHIOPIC PARAGRAPH SEPARATOR +166E ; Terminal_Punctuation # Po CANADIAN SYLLABICS FULL STOP +16EB..16ED ; Terminal_Punctuation # Po [3] RUNIC SINGLE PUNCTUATION..RUNIC CROSS PUNCTUATION +1735..1736 ; Terminal_Punctuation # Po [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION +17D4..17D6 ; Terminal_Punctuation # Po [3] KHMER SIGN KHAN..KHMER SIGN CAMNUC PII KUUH +17DA ; Terminal_Punctuation # Po KHMER SIGN KOOMUUT +1802..1805 ; Terminal_Punctuation # Po [4] MONGOLIAN COMMA..MONGOLIAN FOUR DOTS +1808..1809 ; Terminal_Punctuation # Po [2] MONGOLIAN MANCHU COMMA..MONGOLIAN MANCHU FULL STOP +1944..1945 ; Terminal_Punctuation # Po [2] LIMBU EXCLAMATION MARK..LIMBU QUESTION MARK +1AA8..1AAB ; Terminal_Punctuation # Po [4] TAI THAM SIGN KAAN..TAI THAM SIGN SATKAANKUU +1B5A..1B5B ; Terminal_Punctuation # Po [2] BALINESE PANTI..BALINESE PAMADA +1B5D..1B5F ; Terminal_Punctuation # Po [3] BALINESE CARIK PAMUNGKAH..BALINESE CARIK PAREREN +1B7D..1B7E ; Terminal_Punctuation # Po [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG +1C3B..1C3F ; Terminal_Punctuation # Po [5] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION TSHOOK +1C7E..1C7F ; Terminal_Punctuation # Po [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD +203C..203D ; Terminal_Punctuation # Po [2] DOUBLE EXCLAMATION MARK..INTERROBANG +2047..2049 ; Terminal_Punctuation # Po [3] DOUBLE QUESTION MARK..EXCLAMATION QUESTION MARK +2E2E ; Terminal_Punctuation # Po REVERSED QUESTION MARK +2E3C ; Terminal_Punctuation # Po STENOGRAPHIC FULL STOP +2E41 ; Terminal_Punctuation # Po REVERSED COMMA +2E4C ; Terminal_Punctuation # Po MEDIEVAL COMMA +2E4E..2E4F ; Terminal_Punctuation # Po [2] PUNCTUS ELEVATUS MARK..CORNISH VERSE DIVIDER +2E53..2E54 ; Terminal_Punctuation # Po [2] MEDIEVAL EXCLAMATION MARK..MEDIEVAL QUESTION MARK +3001..3002 ; Terminal_Punctuation # Po [2] IDEOGRAPHIC COMMA..IDEOGRAPHIC FULL STOP +A4FE..A4FF ; Terminal_Punctuation # Po [2] LISU PUNCTUATION COMMA..LISU PUNCTUATION FULL STOP +A60D..A60F ; Terminal_Punctuation # Po [3] VAI COMMA..VAI QUESTION MARK +A6F3..A6F7 ; Terminal_Punctuation # Po [5] BAMUM FULL STOP..BAMUM QUESTION MARK +A876..A877 ; Terminal_Punctuation # Po [2] PHAGS-PA MARK SHAD..PHAGS-PA MARK DOUBLE SHAD +A8CE..A8CF ; Terminal_Punctuation # Po [2] SAURASHTRA DANDA..SAURASHTRA DOUBLE DANDA +A92F ; Terminal_Punctuation # Po KAYAH LI SIGN SHYA +A9C7..A9C9 ; Terminal_Punctuation # Po [3] JAVANESE PADA PANGKAT..JAVANESE PADA LUNGSI +AA5D..AA5F ; Terminal_Punctuation # Po [3] CHAM PUNCTUATION DANDA..CHAM PUNCTUATION TRIPLE DANDA +AADF ; Terminal_Punctuation # Po TAI VIET SYMBOL KOI KOI +AAF0..AAF1 ; Terminal_Punctuation # Po [2] MEETEI MAYEK CHEIKHAN..MEETEI MAYEK AHANG KHUDAM +ABEB ; Terminal_Punctuation # Po MEETEI MAYEK CHEIKHEI +FE50..FE52 ; Terminal_Punctuation # Po [3] SMALL COMMA..SMALL FULL STOP +FE54..FE57 ; Terminal_Punctuation # Po [4] SMALL SEMICOLON..SMALL EXCLAMATION MARK +FF01 ; Terminal_Punctuation # Po FULLWIDTH EXCLAMATION MARK +FF0C ; Terminal_Punctuation # Po FULLWIDTH COMMA +FF0E ; Terminal_Punctuation # Po FULLWIDTH FULL STOP +FF1A..FF1B ; Terminal_Punctuation # Po [2] FULLWIDTH COLON..FULLWIDTH SEMICOLON +FF1F ; Terminal_Punctuation # Po FULLWIDTH QUESTION MARK +FF61 ; Terminal_Punctuation # Po HALFWIDTH IDEOGRAPHIC FULL STOP +FF64 ; Terminal_Punctuation # Po HALFWIDTH IDEOGRAPHIC COMMA +1039F ; Terminal_Punctuation # Po UGARITIC WORD DIVIDER +103D0 ; Terminal_Punctuation # Po OLD PERSIAN WORD DIVIDER +10857 ; Terminal_Punctuation # Po IMPERIAL ARAMAIC SECTION SIGN +1091F ; Terminal_Punctuation # Po PHOENICIAN WORD SEPARATOR +10A56..10A57 ; Terminal_Punctuation # Po [2] KHAROSHTHI PUNCTUATION DANDA..KHAROSHTHI PUNCTUATION DOUBLE DANDA +10AF0..10AF5 ; Terminal_Punctuation # Po [6] MANICHAEAN PUNCTUATION STAR..MANICHAEAN PUNCTUATION TWO DOTS +10B3A..10B3F ; Terminal_Punctuation # Po [6] TINY TWO DOTS OVER ONE DOT PUNCTUATION..LARGE ONE RING OVER TWO RINGS PUNCTUATION +10B99..10B9C ; Terminal_Punctuation # Po [4] PSALTER PAHLAVI SECTION MARK..PSALTER PAHLAVI FOUR DOTS WITH DOT +10F55..10F59 ; Terminal_Punctuation # Po [5] SOGDIAN PUNCTUATION TWO VERTICAL BARS..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT +10F86..10F89 ; Terminal_Punctuation # Po [4] OLD UYGHUR PUNCTUATION BAR..OLD UYGHUR PUNCTUATION FOUR DOTS +11047..1104D ; Terminal_Punctuation # Po [7] BRAHMI DANDA..BRAHMI PUNCTUATION LOTUS +110BE..110C1 ; Terminal_Punctuation # Po [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA +11141..11143 ; Terminal_Punctuation # Po [3] CHAKMA DANDA..CHAKMA QUESTION MARK +111C5..111C6 ; Terminal_Punctuation # Po [2] SHARADA DANDA..SHARADA DOUBLE DANDA +111CD ; Terminal_Punctuation # Po SHARADA SUTRA MARK +111DE..111DF ; Terminal_Punctuation # Po [2] SHARADA SECTION MARK-1..SHARADA SECTION MARK-2 +11238..1123C ; Terminal_Punctuation # Po [5] KHOJKI DANDA..KHOJKI DOUBLE SECTION MARK +112A9 ; Terminal_Punctuation # Po MULTANI SECTION MARK +1144B..1144D ; Terminal_Punctuation # Po [3] NEWA DANDA..NEWA COMMA +1145A..1145B ; Terminal_Punctuation # Po [2] NEWA DOUBLE COMMA..NEWA PLACEHOLDER MARK +115C2..115C5 ; Terminal_Punctuation # Po [4] SIDDHAM DANDA..SIDDHAM SEPARATOR BAR +115C9..115D7 ; Terminal_Punctuation # Po [15] SIDDHAM END OF TEXT MARK..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES +11641..11642 ; Terminal_Punctuation # Po [2] MODI DANDA..MODI DOUBLE DANDA +1173C..1173E ; Terminal_Punctuation # Po [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI +11944 ; Terminal_Punctuation # Po DIVES AKURU DOUBLE DANDA +11946 ; Terminal_Punctuation # Po DIVES AKURU END OF TEXT MARK +11A42..11A43 ; Terminal_Punctuation # Po [2] ZANABAZAR SQUARE MARK SHAD..ZANABAZAR SQUARE MARK DOUBLE SHAD +11A9B..11A9C ; Terminal_Punctuation # Po [2] SOYOMBO MARK SHAD..SOYOMBO MARK DOUBLE SHAD +11AA1..11AA2 ; Terminal_Punctuation # Po [2] SOYOMBO TERMINAL MARK-1..SOYOMBO TERMINAL MARK-2 +11C41..11C43 ; Terminal_Punctuation # Po [3] BHAIKSUKI DANDA..BHAIKSUKI WORD SEPARATOR +11C71 ; Terminal_Punctuation # Po MARCHEN MARK SHAD +11EF7..11EF8 ; Terminal_Punctuation # Po [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION +12470..12474 ; Terminal_Punctuation # Po [5] CUNEIFORM PUNCTUATION SIGN OLD ASSYRIAN WORD DIVIDER..CUNEIFORM PUNCTUATION SIGN DIAGONAL QUADCOLON +16A6E..16A6F ; Terminal_Punctuation # Po [2] MRO DANDA..MRO DOUBLE DANDA +16AF5 ; Terminal_Punctuation # Po BASSA VAH FULL STOP +16B37..16B39 ; Terminal_Punctuation # Po [3] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN CIM CHEEM +16B44 ; Terminal_Punctuation # Po PAHAWH HMONG SIGN XAUS +16E97..16E98 ; Terminal_Punctuation # Po [2] MEDEFAIDRIN COMMA..MEDEFAIDRIN FULL STOP +1BC9F ; Terminal_Punctuation # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP +1DA87..1DA8A ; Terminal_Punctuation # Po [4] SIGNWRITING COMMA..SIGNWRITING COLON + +# Total code points: 276 + +# ================================================ + +005E ; Other_Math # Sk CIRCUMFLEX ACCENT +03D0..03D2 ; Other_Math # L& [3] GREEK BETA SYMBOL..GREEK UPSILON WITH HOOK SYMBOL +03D5 ; Other_Math # L& GREEK PHI SYMBOL +03F0..03F1 ; Other_Math # L& [2] GREEK KAPPA SYMBOL..GREEK RHO SYMBOL +03F4..03F5 ; Other_Math # L& [2] GREEK CAPITAL THETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +2016 ; Other_Math # Po DOUBLE VERTICAL LINE +2032..2034 ; Other_Math # Po [3] PRIME..TRIPLE PRIME +2040 ; Other_Math # Pc CHARACTER TIE +2061..2064 ; Other_Math # Cf [4] FUNCTION APPLICATION..INVISIBLE PLUS +207D ; Other_Math # Ps SUPERSCRIPT LEFT PARENTHESIS +207E ; Other_Math # Pe SUPERSCRIPT RIGHT PARENTHESIS +208D ; Other_Math # Ps SUBSCRIPT LEFT PARENTHESIS +208E ; Other_Math # Pe SUBSCRIPT RIGHT PARENTHESIS +20D0..20DC ; Other_Math # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20E1 ; Other_Math # Mn COMBINING LEFT RIGHT ARROW ABOVE +20E5..20E6 ; Other_Math # Mn [2] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING DOUBLE VERTICAL STROKE OVERLAY +20EB..20EF ; Other_Math # Mn [5] COMBINING LONG DOUBLE SOLIDUS OVERLAY..COMBINING RIGHT ARROW BELOW +2102 ; Other_Math # L& DOUBLE-STRUCK CAPITAL C +2107 ; Other_Math # L& EULER CONSTANT +210A..2113 ; Other_Math # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Other_Math # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Other_Math # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Other_Math # L& DOUBLE-STRUCK CAPITAL Z +2128 ; Other_Math # L& BLACK-LETTER CAPITAL Z +2129 ; Other_Math # So TURNED GREEK SMALL LETTER IOTA +212C..212D ; Other_Math # L& [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C +212F..2131 ; Other_Math # L& [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2134 ; Other_Math # L& [2] SCRIPT CAPITAL M..SCRIPT SMALL O +2135..2138 ; Other_Math # Lo [4] ALEF SYMBOL..DALET SYMBOL +213C..213F ; Other_Math # L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STRUCK CAPITAL PI +2145..2149 ; Other_Math # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +2195..2199 ; Other_Math # So [5] UP DOWN ARROW..SOUTH WEST ARROW +219C..219F ; Other_Math # So [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW +21A1..21A2 ; Other_Math # So [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL +21A4..21A5 ; Other_Math # So [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR +21A7 ; Other_Math # So DOWNWARDS ARROW FROM BAR +21A9..21AD ; Other_Math # So [5] LEFTWARDS ARROW WITH HOOK..LEFT RIGHT WAVE ARROW +21B0..21B1 ; Other_Math # So [2] UPWARDS ARROW WITH TIP LEFTWARDS..UPWARDS ARROW WITH TIP RIGHTWARDS +21B6..21B7 ; Other_Math # So [2] ANTICLOCKWISE TOP SEMICIRCLE ARROW..CLOCKWISE TOP SEMICIRCLE ARROW +21BC..21CD ; Other_Math # So [18] LEFTWARDS HARPOON WITH BARB UPWARDS..LEFTWARDS DOUBLE ARROW WITH STROKE +21D0..21D1 ; Other_Math # So [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW +21D3 ; Other_Math # So DOWNWARDS DOUBLE ARROW +21D5..21DB ; Other_Math # So [7] UP DOWN DOUBLE ARROW..RIGHTWARDS TRIPLE ARROW +21DD ; Other_Math # So RIGHTWARDS SQUIGGLE ARROW +21E4..21E5 ; Other_Math # So [2] LEFTWARDS ARROW TO BAR..RIGHTWARDS ARROW TO BAR +2308 ; Other_Math # Ps LEFT CEILING +2309 ; Other_Math # Pe RIGHT CEILING +230A ; Other_Math # Ps LEFT FLOOR +230B ; Other_Math # Pe RIGHT FLOOR +23B4..23B5 ; Other_Math # So [2] TOP SQUARE BRACKET..BOTTOM SQUARE BRACKET +23B7 ; Other_Math # So RADICAL SYMBOL BOTTOM +23D0 ; Other_Math # So VERTICAL LINE EXTENSION +23E2 ; Other_Math # So WHITE TRAPEZIUM +25A0..25A1 ; Other_Math # So [2] BLACK SQUARE..WHITE SQUARE +25AE..25B6 ; Other_Math # So [9] BLACK VERTICAL RECTANGLE..BLACK RIGHT-POINTING TRIANGLE +25BC..25C0 ; Other_Math # So [5] BLACK DOWN-POINTING TRIANGLE..BLACK LEFT-POINTING TRIANGLE +25C6..25C7 ; Other_Math # So [2] BLACK DIAMOND..WHITE DIAMOND +25CA..25CB ; Other_Math # So [2] LOZENGE..WHITE CIRCLE +25CF..25D3 ; Other_Math # So [5] BLACK CIRCLE..CIRCLE WITH UPPER HALF BLACK +25E2 ; Other_Math # So BLACK LOWER RIGHT TRIANGLE +25E4 ; Other_Math # So BLACK UPPER LEFT TRIANGLE +25E7..25EC ; Other_Math # So [6] SQUARE WITH LEFT HALF BLACK..WHITE UP-POINTING TRIANGLE WITH DOT +2605..2606 ; Other_Math # So [2] BLACK STAR..WHITE STAR +2640 ; Other_Math # So FEMALE SIGN +2642 ; Other_Math # So MALE SIGN +2660..2663 ; Other_Math # So [4] BLACK SPADE SUIT..BLACK CLUB SUIT +266D..266E ; Other_Math # So [2] MUSIC FLAT SIGN..MUSIC NATURAL SIGN +27C5 ; Other_Math # Ps LEFT S-SHAPED BAG DELIMITER +27C6 ; Other_Math # Pe RIGHT S-SHAPED BAG DELIMITER +27E6 ; Other_Math # Ps MATHEMATICAL LEFT WHITE SQUARE BRACKET +27E7 ; Other_Math # Pe MATHEMATICAL RIGHT WHITE SQUARE BRACKET +27E8 ; Other_Math # Ps MATHEMATICAL LEFT ANGLE BRACKET +27E9 ; Other_Math # Pe MATHEMATICAL RIGHT ANGLE BRACKET +27EA ; Other_Math # Ps MATHEMATICAL LEFT DOUBLE ANGLE BRACKET +27EB ; Other_Math # Pe MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET +27EC ; Other_Math # Ps MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET +27ED ; Other_Math # Pe MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET +27EE ; Other_Math # Ps MATHEMATICAL LEFT FLATTENED PARENTHESIS +27EF ; Other_Math # Pe MATHEMATICAL RIGHT FLATTENED PARENTHESIS +2983 ; Other_Math # Ps LEFT WHITE CURLY BRACKET +2984 ; Other_Math # Pe RIGHT WHITE CURLY BRACKET +2985 ; Other_Math # Ps LEFT WHITE PARENTHESIS +2986 ; Other_Math # Pe RIGHT WHITE PARENTHESIS +2987 ; Other_Math # Ps Z NOTATION LEFT IMAGE BRACKET +2988 ; Other_Math # Pe Z NOTATION RIGHT IMAGE BRACKET +2989 ; Other_Math # Ps Z NOTATION LEFT BINDING BRACKET +298A ; Other_Math # Pe Z NOTATION RIGHT BINDING BRACKET +298B ; Other_Math # Ps LEFT SQUARE BRACKET WITH UNDERBAR +298C ; Other_Math # Pe RIGHT SQUARE BRACKET WITH UNDERBAR +298D ; Other_Math # Ps LEFT SQUARE BRACKET WITH TICK IN TOP CORNER +298E ; Other_Math # Pe RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +298F ; Other_Math # Ps LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +2990 ; Other_Math # Pe RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER +2991 ; Other_Math # Ps LEFT ANGLE BRACKET WITH DOT +2992 ; Other_Math # Pe RIGHT ANGLE BRACKET WITH DOT +2993 ; Other_Math # Ps LEFT ARC LESS-THAN BRACKET +2994 ; Other_Math # Pe RIGHT ARC GREATER-THAN BRACKET +2995 ; Other_Math # Ps DOUBLE LEFT ARC GREATER-THAN BRACKET +2996 ; Other_Math # Pe DOUBLE RIGHT ARC LESS-THAN BRACKET +2997 ; Other_Math # Ps LEFT BLACK TORTOISE SHELL BRACKET +2998 ; Other_Math # Pe RIGHT BLACK TORTOISE SHELL BRACKET +29D8 ; Other_Math # Ps LEFT WIGGLY FENCE +29D9 ; Other_Math # Pe RIGHT WIGGLY FENCE +29DA ; Other_Math # Ps LEFT DOUBLE WIGGLY FENCE +29DB ; Other_Math # Pe RIGHT DOUBLE WIGGLY FENCE +29FC ; Other_Math # Ps LEFT-POINTING CURVED ANGLE BRACKET +29FD ; Other_Math # Pe RIGHT-POINTING CURVED ANGLE BRACKET +FE61 ; Other_Math # Po SMALL ASTERISK +FE63 ; Other_Math # Pd SMALL HYPHEN-MINUS +FE68 ; Other_Math # Po SMALL REVERSE SOLIDUS +FF3C ; Other_Math # Po FULLWIDTH REVERSE SOLIDUS +FF3E ; Other_Math # Sk FULLWIDTH CIRCUMFLEX ACCENT +1D400..1D454 ; Other_Math # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Other_Math # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Other_Math # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Other_Math # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Other_Math # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Other_Math # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Other_Math # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Other_Math # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C3 ; Other_Math # L& [7] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Other_Math # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Other_Math # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Other_Math # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Other_Math # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Other_Math # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Other_Math # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Other_Math # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Other_Math # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Other_Math # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A5 ; Other_Math # L& [340] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL ITALIC SMALL DOTLESS J +1D6A8..1D6C0 ; Other_Math # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C2..1D6DA ; Other_Math # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6FA ; Other_Math # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FC..1D714 ; Other_Math # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D734 ; Other_Math # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D736..1D74E ; Other_Math # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D76E ; Other_Math # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D770..1D788 ; Other_Math # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D7A8 ; Other_Math # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7AA..1D7C2 ; Other_Math # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7CB ; Other_Math # L& [8] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD SMALL DIGAMMA +1D7CE..1D7FF ; Other_Math # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE +1EE00..1EE03 ; Other_Math # Lo [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL +1EE05..1EE1F ; Other_Math # Lo [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF +1EE21..1EE22 ; Other_Math # Lo [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM +1EE24 ; Other_Math # Lo ARABIC MATHEMATICAL INITIAL HEH +1EE27 ; Other_Math # Lo ARABIC MATHEMATICAL INITIAL HAH +1EE29..1EE32 ; Other_Math # Lo [10] ARABIC MATHEMATICAL INITIAL YEH..ARABIC MATHEMATICAL INITIAL QAF +1EE34..1EE37 ; Other_Math # Lo [4] ARABIC MATHEMATICAL INITIAL SHEEN..ARABIC MATHEMATICAL INITIAL KHAH +1EE39 ; Other_Math # Lo ARABIC MATHEMATICAL INITIAL DAD +1EE3B ; Other_Math # Lo ARABIC MATHEMATICAL INITIAL GHAIN +1EE42 ; Other_Math # Lo ARABIC MATHEMATICAL TAILED JEEM +1EE47 ; Other_Math # Lo ARABIC MATHEMATICAL TAILED HAH +1EE49 ; Other_Math # Lo ARABIC MATHEMATICAL TAILED YEH +1EE4B ; Other_Math # Lo ARABIC MATHEMATICAL TAILED LAM +1EE4D..1EE4F ; Other_Math # Lo [3] ARABIC MATHEMATICAL TAILED NOON..ARABIC MATHEMATICAL TAILED AIN +1EE51..1EE52 ; Other_Math # Lo [2] ARABIC MATHEMATICAL TAILED SAD..ARABIC MATHEMATICAL TAILED QAF +1EE54 ; Other_Math # Lo ARABIC MATHEMATICAL TAILED SHEEN +1EE57 ; Other_Math # Lo ARABIC MATHEMATICAL TAILED KHAH +1EE59 ; Other_Math # Lo ARABIC MATHEMATICAL TAILED DAD +1EE5B ; Other_Math # Lo ARABIC MATHEMATICAL TAILED GHAIN +1EE5D ; Other_Math # Lo ARABIC MATHEMATICAL TAILED DOTLESS NOON +1EE5F ; Other_Math # Lo ARABIC MATHEMATICAL TAILED DOTLESS QAF +1EE61..1EE62 ; Other_Math # Lo [2] ARABIC MATHEMATICAL STRETCHED BEH..ARABIC MATHEMATICAL STRETCHED JEEM +1EE64 ; Other_Math # Lo ARABIC MATHEMATICAL STRETCHED HEH +1EE67..1EE6A ; Other_Math # Lo [4] ARABIC MATHEMATICAL STRETCHED HAH..ARABIC MATHEMATICAL STRETCHED KAF +1EE6C..1EE72 ; Other_Math # Lo [7] ARABIC MATHEMATICAL STRETCHED MEEM..ARABIC MATHEMATICAL STRETCHED QAF +1EE74..1EE77 ; Other_Math # Lo [4] ARABIC MATHEMATICAL STRETCHED SHEEN..ARABIC MATHEMATICAL STRETCHED KHAH +1EE79..1EE7C ; Other_Math # Lo [4] ARABIC MATHEMATICAL STRETCHED DAD..ARABIC MATHEMATICAL STRETCHED DOTLESS BEH +1EE7E ; Other_Math # Lo ARABIC MATHEMATICAL STRETCHED DOTLESS FEH +1EE80..1EE89 ; Other_Math # Lo [10] ARABIC MATHEMATICAL LOOPED ALEF..ARABIC MATHEMATICAL LOOPED YEH +1EE8B..1EE9B ; Other_Math # Lo [17] ARABIC MATHEMATICAL LOOPED LAM..ARABIC MATHEMATICAL LOOPED GHAIN +1EEA1..1EEA3 ; Other_Math # Lo [3] ARABIC MATHEMATICAL DOUBLE-STRUCK BEH..ARABIC MATHEMATICAL DOUBLE-STRUCK DAL +1EEA5..1EEA9 ; Other_Math # Lo [5] ARABIC MATHEMATICAL DOUBLE-STRUCK WAW..ARABIC MATHEMATICAL DOUBLE-STRUCK YEH +1EEAB..1EEBB ; Other_Math # Lo [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN + +# Total code points: 1362 + +# ================================================ + +0030..0039 ; Hex_Digit # Nd [10] DIGIT ZERO..DIGIT NINE +0041..0046 ; Hex_Digit # L& [6] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER F +0061..0066 ; Hex_Digit # L& [6] LATIN SMALL LETTER A..LATIN SMALL LETTER F +FF10..FF19 ; Hex_Digit # Nd [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NINE +FF21..FF26 ; Hex_Digit # L& [6] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER F +FF41..FF46 ; Hex_Digit # L& [6] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER F + +# Total code points: 44 + +# ================================================ + +0030..0039 ; ASCII_Hex_Digit # Nd [10] DIGIT ZERO..DIGIT NINE +0041..0046 ; ASCII_Hex_Digit # L& [6] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER F +0061..0066 ; ASCII_Hex_Digit # L& [6] LATIN SMALL LETTER A..LATIN SMALL LETTER F + +# Total code points: 22 + +# ================================================ + +0345 ; Other_Alphabetic # Mn COMBINING GREEK YPOGEGRAMMENI +05B0..05BD ; Other_Alphabetic # Mn [14] HEBREW POINT SHEVA..HEBREW POINT METEG +05BF ; Other_Alphabetic # Mn HEBREW POINT RAFE +05C1..05C2 ; Other_Alphabetic # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4..05C5 ; Other_Alphabetic # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT +05C7 ; Other_Alphabetic # Mn HEBREW POINT QAMATS QATAN +0610..061A ; Other_Alphabetic # Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA +064B..0657 ; Other_Alphabetic # Mn [13] ARABIC FATHATAN..ARABIC INVERTED DAMMA +0659..065F ; Other_Alphabetic # Mn [7] ARABIC ZWARAKAY..ARABIC WAVY HAMZA BELOW +0670 ; Other_Alphabetic # Mn ARABIC LETTER SUPERSCRIPT ALEF +06D6..06DC ; Other_Alphabetic # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06E1..06E4 ; Other_Alphabetic # Mn [4] ARABIC SMALL HIGH DOTLESS HEAD OF KHAH..ARABIC SMALL HIGH MADDA +06E7..06E8 ; Other_Alphabetic # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06ED ; Other_Alphabetic # Mn ARABIC SMALL LOW MEEM +0711 ; Other_Alphabetic # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0730..073F ; Other_Alphabetic # Mn [16] SYRIAC PTHAHA ABOVE..SYRIAC RWAHA +07A6..07B0 ; Other_Alphabetic # Mn [11] THAANA ABAFILI..THAANA SUKUN +0816..0817 ; Other_Alphabetic # Mn [2] SAMARITAN MARK IN..SAMARITAN MARK IN-ALAF +081B..0823 ; Other_Alphabetic # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A +0825..0827 ; Other_Alphabetic # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U +0829..082C ; Other_Alphabetic # Mn [4] SAMARITAN VOWEL SIGN LONG I..SAMARITAN VOWEL SIGN SUKUN +08D4..08DF ; Other_Alphabetic # Mn [12] ARABIC SMALL HIGH WORD AR-RUB..ARABIC SMALL HIGH WORD WAQFA +08E3..08E9 ; Other_Alphabetic # Mn [7] ARABIC TURNED DAMMA BELOW..ARABIC CURLY KASRATAN +08F0..0902 ; Other_Alphabetic # Mn [19] ARABIC OPEN FATHATAN..DEVANAGARI SIGN ANUSVARA +0903 ; Other_Alphabetic # Mc DEVANAGARI SIGN VISARGA +093A ; Other_Alphabetic # Mn DEVANAGARI VOWEL SIGN OE +093B ; Other_Alphabetic # Mc DEVANAGARI VOWEL SIGN OOE +093E..0940 ; Other_Alphabetic # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II +0941..0948 ; Other_Alphabetic # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +0949..094C ; Other_Alphabetic # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU +094E..094F ; Other_Alphabetic # Mc [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW +0955..0957 ; Other_Alphabetic # Mn [3] DEVANAGARI VOWEL SIGN CANDRA LONG E..DEVANAGARI VOWEL SIGN UUE +0962..0963 ; Other_Alphabetic # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0981 ; Other_Alphabetic # Mn BENGALI SIGN CANDRABINDU +0982..0983 ; Other_Alphabetic # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA +09BE..09C0 ; Other_Alphabetic # Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II +09C1..09C4 ; Other_Alphabetic # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8 ; Other_Alphabetic # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CC ; Other_Alphabetic # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU +09D7 ; Other_Alphabetic # Mc BENGALI AU LENGTH MARK +09E2..09E3 ; Other_Alphabetic # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +0A01..0A02 ; Other_Alphabetic # Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI SIGN BINDI +0A03 ; Other_Alphabetic # Mc GURMUKHI SIGN VISARGA +0A3E..0A40 ; Other_Alphabetic # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II +0A41..0A42 ; Other_Alphabetic # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; Other_Alphabetic # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4C ; Other_Alphabetic # Mn [2] GURMUKHI VOWEL SIGN OO..GURMUKHI VOWEL SIGN AU +0A51 ; Other_Alphabetic # Mn GURMUKHI SIGN UDAAT +0A70..0A71 ; Other_Alphabetic # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A75 ; Other_Alphabetic # Mn GURMUKHI SIGN YAKASH +0A81..0A82 ; Other_Alphabetic # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0A83 ; Other_Alphabetic # Mc GUJARATI SIGN VISARGA +0ABE..0AC0 ; Other_Alphabetic # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II +0AC1..0AC5 ; Other_Alphabetic # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8 ; Other_Alphabetic # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0AC9 ; Other_Alphabetic # Mc GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACC ; Other_Alphabetic # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU +0AE2..0AE3 ; Other_Alphabetic # Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJARATI VOWEL SIGN VOCALIC LL +0AFA..0AFC ; Other_Alphabetic # Mn [3] GUJARATI SIGN SUKUN..GUJARATI SIGN MADDAH +0B01 ; Other_Alphabetic # Mn ORIYA SIGN CANDRABINDU +0B02..0B03 ; Other_Alphabetic # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA +0B3E ; Other_Alphabetic # Mc ORIYA VOWEL SIGN AA +0B3F ; Other_Alphabetic # Mn ORIYA VOWEL SIGN I +0B40 ; Other_Alphabetic # Mc ORIYA VOWEL SIGN II +0B41..0B44 ; Other_Alphabetic # Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC RR +0B47..0B48 ; Other_Alphabetic # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4C ; Other_Alphabetic # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU +0B56 ; Other_Alphabetic # Mn ORIYA AI LENGTH MARK +0B57 ; Other_Alphabetic # Mc ORIYA AU LENGTH MARK +0B62..0B63 ; Other_Alphabetic # Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA VOWEL SIGN VOCALIC LL +0B82 ; Other_Alphabetic # Mn TAMIL SIGN ANUSVARA +0BBE..0BBF ; Other_Alphabetic # Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I +0BC0 ; Other_Alphabetic # Mn TAMIL VOWEL SIGN II +0BC1..0BC2 ; Other_Alphabetic # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU +0BC6..0BC8 ; Other_Alphabetic # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCC ; Other_Alphabetic # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU +0BD7 ; Other_Alphabetic # Mc TAMIL AU LENGTH MARK +0C00 ; Other_Alphabetic # Mn TELUGU SIGN COMBINING CANDRABINDU ABOVE +0C01..0C03 ; Other_Alphabetic # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C3E..0C40 ; Other_Alphabetic # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II +0C41..0C44 ; Other_Alphabetic # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR +0C46..0C48 ; Other_Alphabetic # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4C ; Other_Alphabetic # Mn [3] TELUGU VOWEL SIGN O..TELUGU VOWEL SIGN AU +0C55..0C56 ; Other_Alphabetic # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C62..0C63 ; Other_Alphabetic # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL +0C81 ; Other_Alphabetic # Mn KANNADA SIGN CANDRABINDU +0C82..0C83 ; Other_Alphabetic # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0CBE ; Other_Alphabetic # Mc KANNADA VOWEL SIGN AA +0CBF ; Other_Alphabetic # Mn KANNADA VOWEL SIGN I +0CC0..0CC4 ; Other_Alphabetic # Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR +0CC6 ; Other_Alphabetic # Mn KANNADA VOWEL SIGN E +0CC7..0CC8 ; Other_Alphabetic # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI +0CCA..0CCB ; Other_Alphabetic # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CCC ; Other_Alphabetic # Mn KANNADA VOWEL SIGN AU +0CD5..0CD6 ; Other_Alphabetic # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CE2..0CE3 ; Other_Alphabetic # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL +0D00..0D01 ; Other_Alphabetic # Mn [2] MALAYALAM SIGN COMBINING ANUSVARA ABOVE..MALAYALAM SIGN CANDRABINDU +0D02..0D03 ; Other_Alphabetic # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D3E..0D40 ; Other_Alphabetic # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II +0D41..0D44 ; Other_Alphabetic # Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC RR +0D46..0D48 ; Other_Alphabetic # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4C ; Other_Alphabetic # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU +0D57 ; Other_Alphabetic # Mc MALAYALAM AU LENGTH MARK +0D62..0D63 ; Other_Alphabetic # Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MALAYALAM VOWEL SIGN VOCALIC LL +0D81 ; Other_Alphabetic # Mn SINHALA SIGN CANDRABINDU +0D82..0D83 ; Other_Alphabetic # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0DCF..0DD1 ; Other_Alphabetic # Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA +0DD2..0DD4 ; Other_Alphabetic # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6 ; Other_Alphabetic # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA +0DD8..0DDF ; Other_Alphabetic # Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA +0DF2..0DF3 ; Other_Alphabetic # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA +0E31 ; Other_Alphabetic # Mn THAI CHARACTER MAI HAN-AKAT +0E34..0E3A ; Other_Alphabetic # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E4D ; Other_Alphabetic # Mn THAI CHARACTER NIKHAHIT +0EB1 ; Other_Alphabetic # Mn LAO VOWEL SIGN MAI KAN +0EB4..0EB9 ; Other_Alphabetic # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU +0EBB..0EBC ; Other_Alphabetic # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO +0ECD ; Other_Alphabetic # Mn LAO NIGGAHITA +0F71..0F7E ; Other_Alphabetic # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO +0F7F ; Other_Alphabetic # Mc TIBETAN SIGN RNAM BCAD +0F80..0F81 ; Other_Alphabetic # Mn [2] TIBETAN VOWEL SIGN REVERSED I..TIBETAN VOWEL SIGN REVERSED II +0F8D..0F97 ; Other_Alphabetic # Mn [11] TIBETAN SUBJOINED SIGN LCE TSA CAN..TIBETAN SUBJOINED LETTER JA +0F99..0FBC ; Other_Alphabetic # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +102B..102C ; Other_Alphabetic # Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMAR VOWEL SIGN AA +102D..1030 ; Other_Alphabetic # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU +1031 ; Other_Alphabetic # Mc MYANMAR VOWEL SIGN E +1032..1036 ; Other_Alphabetic # Mn [5] MYANMAR VOWEL SIGN AI..MYANMAR SIGN ANUSVARA +1038 ; Other_Alphabetic # Mc MYANMAR SIGN VISARGA +103B..103C ; Other_Alphabetic # Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..MYANMAR CONSONANT SIGN MEDIAL RA +103D..103E ; Other_Alphabetic # Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..MYANMAR CONSONANT SIGN MEDIAL HA +1056..1057 ; Other_Alphabetic # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR +1058..1059 ; Other_Alphabetic # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL +105E..1060 ; Other_Alphabetic # Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL NA..MYANMAR CONSONANT SIGN MON MEDIAL LA +1062..1064 ; Other_Alphabetic # Mc [3] MYANMAR VOWEL SIGN SGAW KAREN EU..MYANMAR TONE MARK SGAW KAREN KE PHO +1067..106D ; Other_Alphabetic # Mc [7] MYANMAR VOWEL SIGN WESTERN PWO KAREN EU..MYANMAR SIGN WESTERN PWO KAREN TONE-5 +1071..1074 ; Other_Alphabetic # Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..MYANMAR VOWEL SIGN KAYAH EE +1082 ; Other_Alphabetic # Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA +1083..1084 ; Other_Alphabetic # Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMAR VOWEL SIGN SHAN E +1085..1086 ; Other_Alphabetic # Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..MYANMAR VOWEL SIGN SHAN FINAL Y +1087..108C ; Other_Alphabetic # Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 +108D ; Other_Alphabetic # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE +108F ; Other_Alphabetic # Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 +109A..109C ; Other_Alphabetic # Mc [3] MYANMAR SIGN KHAMTI TONE-1..MYANMAR VOWEL SIGN AITON A +109D ; Other_Alphabetic # Mn MYANMAR VOWEL SIGN AITON AI +1712..1713 ; Other_Alphabetic # Mn [2] TAGALOG VOWEL SIGN I..TAGALOG VOWEL SIGN U +1732..1733 ; Other_Alphabetic # Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U +1752..1753 ; Other_Alphabetic # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U +1772..1773 ; Other_Alphabetic # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +17B6 ; Other_Alphabetic # Mc KHMER VOWEL SIGN AA +17B7..17BD ; Other_Alphabetic # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA +17BE..17C5 ; Other_Alphabetic # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU +17C6 ; Other_Alphabetic # Mn KHMER SIGN NIKAHIT +17C7..17C8 ; Other_Alphabetic # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU +1885..1886 ; Other_Alphabetic # Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA +18A9 ; Other_Alphabetic # Mn MONGOLIAN LETTER ALI GALI DAGALGA +1920..1922 ; Other_Alphabetic # Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SIGN U +1923..1926 ; Other_Alphabetic # Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL SIGN AU +1927..1928 ; Other_Alphabetic # Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SIGN O +1929..192B ; Other_Alphabetic # Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU SUBJOINED LETTER WA +1930..1931 ; Other_Alphabetic # Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL LETTER NGA +1932 ; Other_Alphabetic # Mn LIMBU SMALL LETTER ANUSVARA +1933..1938 ; Other_Alphabetic # Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL LETTER LA +1A17..1A18 ; Other_Alphabetic # Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U +1A19..1A1A ; Other_Alphabetic # Mc [2] BUGINESE VOWEL SIGN E..BUGINESE VOWEL SIGN O +1A1B ; Other_Alphabetic # Mn BUGINESE VOWEL SIGN AE +1A55 ; Other_Alphabetic # Mc TAI THAM CONSONANT SIGN MEDIAL RA +1A56 ; Other_Alphabetic # Mn TAI THAM CONSONANT SIGN MEDIAL LA +1A57 ; Other_Alphabetic # Mc TAI THAM CONSONANT SIGN LA TANG LAI +1A58..1A5E ; Other_Alphabetic # Mn [7] TAI THAM SIGN MAI KANG LAI..TAI THAM CONSONANT SIGN SA +1A61 ; Other_Alphabetic # Mc TAI THAM VOWEL SIGN A +1A62 ; Other_Alphabetic # Mn TAI THAM VOWEL SIGN MAI SAT +1A63..1A64 ; Other_Alphabetic # Mc [2] TAI THAM VOWEL SIGN AA..TAI THAM VOWEL SIGN TALL AA +1A65..1A6C ; Other_Alphabetic # Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VOWEL SIGN OA BELOW +1A6D..1A72 ; Other_Alphabetic # Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM VOWEL SIGN THAM AI +1A73..1A74 ; Other_Alphabetic # Mn [2] TAI THAM VOWEL SIGN OA ABOVE..TAI THAM SIGN MAI KANG +1ABF..1AC0 ; Other_Alphabetic # Mn [2] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER TURNED W BELOW +1ACC..1ACE ; Other_Alphabetic # Mn [3] COMBINING LATIN SMALL LETTER INSULAR G..COMBINING LATIN SMALL LETTER INSULAR T +1B00..1B03 ; Other_Alphabetic # Mn [4] BALINESE SIGN ULU RICEM..BALINESE SIGN SURANG +1B04 ; Other_Alphabetic # Mc BALINESE SIGN BISAH +1B35 ; Other_Alphabetic # Mc BALINESE VOWEL SIGN TEDUNG +1B36..1B3A ; Other_Alphabetic # Mn [5] BALINESE VOWEL SIGN ULU..BALINESE VOWEL SIGN RA REPA +1B3B ; Other_Alphabetic # Mc BALINESE VOWEL SIGN RA REPA TEDUNG +1B3C ; Other_Alphabetic # Mn BALINESE VOWEL SIGN LA LENGA +1B3D..1B41 ; Other_Alphabetic # Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUNG..BALINESE VOWEL SIGN TALING REPA TEDUNG +1B42 ; Other_Alphabetic # Mn BALINESE VOWEL SIGN PEPET +1B43 ; Other_Alphabetic # Mc BALINESE VOWEL SIGN PEPET TEDUNG +1B80..1B81 ; Other_Alphabetic # Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE SIGN PANGLAYAR +1B82 ; Other_Alphabetic # Mc SUNDANESE SIGN PANGWISAD +1BA1 ; Other_Alphabetic # Mc SUNDANESE CONSONANT SIGN PAMINGKAL +1BA2..1BA5 ; Other_Alphabetic # Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA..SUNDANESE VOWEL SIGN PANYUKU +1BA6..1BA7 ; Other_Alphabetic # Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..SUNDANESE VOWEL SIGN PANOLONG +1BA8..1BA9 ; Other_Alphabetic # Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUNDANESE VOWEL SIGN PANEULEUNG +1BAC..1BAD ; Other_Alphabetic # Mn [2] SUNDANESE CONSONANT SIGN PASANGAN MA..SUNDANESE CONSONANT SIGN PASANGAN WA +1BE7 ; Other_Alphabetic # Mc BATAK VOWEL SIGN E +1BE8..1BE9 ; Other_Alphabetic # Mn [2] BATAK VOWEL SIGN PAKPAK E..BATAK VOWEL SIGN EE +1BEA..1BEC ; Other_Alphabetic # Mc [3] BATAK VOWEL SIGN I..BATAK VOWEL SIGN O +1BED ; Other_Alphabetic # Mn BATAK VOWEL SIGN KARO O +1BEE ; Other_Alphabetic # Mc BATAK VOWEL SIGN U +1BEF..1BF1 ; Other_Alphabetic # Mn [3] BATAK VOWEL SIGN U FOR SIMALUNGUN SA..BATAK CONSONANT SIGN H +1C24..1C2B ; Other_Alphabetic # Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA VOWEL SIGN UU +1C2C..1C33 ; Other_Alphabetic # Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSONANT SIGN T +1C34..1C35 ; Other_Alphabetic # Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEPCHA CONSONANT SIGN KANG +1C36 ; Other_Alphabetic # Mn LEPCHA SIGN RAN +1DE7..1DF4 ; Other_Alphabetic # Mn [14] COMBINING LATIN SMALL LETTER ALPHA..COMBINING LATIN SMALL LETTER U WITH DIAERESIS +24B6..24E9 ; Other_Alphabetic # So [52] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN SMALL LETTER Z +2DE0..2DFF ; Other_Alphabetic # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS +A674..A67B ; Other_Alphabetic # Mn [8] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC LETTER OMEGA +A69E..A69F ; Other_Alphabetic # Mn [2] COMBINING CYRILLIC LETTER EF..COMBINING CYRILLIC LETTER IOTIFIED E +A802 ; Other_Alphabetic # Mn SYLOTI NAGRI SIGN DVISVARA +A80B ; Other_Alphabetic # Mn SYLOTI NAGRI SIGN ANUSVARA +A823..A824 ; Other_Alphabetic # Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI NAGRI VOWEL SIGN I +A825..A826 ; Other_Alphabetic # Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI NAGRI VOWEL SIGN E +A827 ; Other_Alphabetic # Mc SYLOTI NAGRI VOWEL SIGN OO +A880..A881 ; Other_Alphabetic # Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHTRA SIGN VISARGA +A8B4..A8C3 ; Other_Alphabetic # Mc [16] SAURASHTRA CONSONANT SIGN HAARU..SAURASHTRA VOWEL SIGN AU +A8C5 ; Other_Alphabetic # Mn SAURASHTRA SIGN CANDRABINDU +A8FF ; Other_Alphabetic # Mn DEVANAGARI VOWEL SIGN AY +A926..A92A ; Other_Alphabetic # Mn [5] KAYAH LI VOWEL UE..KAYAH LI VOWEL O +A947..A951 ; Other_Alphabetic # Mn [11] REJANG VOWEL SIGN I..REJANG CONSONANT SIGN R +A952 ; Other_Alphabetic # Mc REJANG CONSONANT SIGN H +A980..A982 ; Other_Alphabetic # Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE SIGN LAYAR +A983 ; Other_Alphabetic # Mc JAVANESE SIGN WIGNYAN +A9B4..A9B5 ; Other_Alphabetic # Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANESE VOWEL SIGN TOLONG +A9B6..A9B9 ; Other_Alphabetic # Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE VOWEL SIGN SUKU MENDUT +A9BA..A9BB ; Other_Alphabetic # Mc [2] JAVANESE VOWEL SIGN TALING..JAVANESE VOWEL SIGN DIRGA MURE +A9BC..A9BD ; Other_Alphabetic # Mn [2] JAVANESE VOWEL SIGN PEPET..JAVANESE CONSONANT SIGN KERET +A9BE..A9BF ; Other_Alphabetic # Mc [2] JAVANESE CONSONANT SIGN PENGKAL..JAVANESE CONSONANT SIGN CAKRA +A9E5 ; Other_Alphabetic # Mn MYANMAR SIGN SHAN SAW +AA29..AA2E ; Other_Alphabetic # Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIGN OE +AA2F..AA30 ; Other_Alphabetic # Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI +AA31..AA32 ; Other_Alphabetic # Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIGN UE +AA33..AA34 ; Other_Alphabetic # Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSONANT SIGN RA +AA35..AA36 ; Other_Alphabetic # Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSONANT SIGN WA +AA43 ; Other_Alphabetic # Mn CHAM CONSONANT SIGN FINAL NG +AA4C ; Other_Alphabetic # Mn CHAM CONSONANT SIGN FINAL M +AA4D ; Other_Alphabetic # Mc CHAM CONSONANT SIGN FINAL H +AA7B ; Other_Alphabetic # Mc MYANMAR SIGN PAO KAREN TONE +AA7C ; Other_Alphabetic # Mn MYANMAR SIGN TAI LAING TONE-2 +AA7D ; Other_Alphabetic # Mc MYANMAR SIGN TAI LAING TONE-5 +AAB0 ; Other_Alphabetic # Mn TAI VIET MAI KANG +AAB2..AAB4 ; Other_Alphabetic # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U +AAB7..AAB8 ; Other_Alphabetic # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA +AABE ; Other_Alphabetic # Mn TAI VIET VOWEL AM +AAEB ; Other_Alphabetic # Mc MEETEI MAYEK VOWEL SIGN II +AAEC..AAED ; Other_Alphabetic # Mn [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI MAYEK VOWEL SIGN AAI +AAEE..AAEF ; Other_Alphabetic # Mc [2] MEETEI MAYEK VOWEL SIGN AU..MEETEI MAYEK VOWEL SIGN AAU +AAF5 ; Other_Alphabetic # Mc MEETEI MAYEK VOWEL SIGN VISARGA +ABE3..ABE4 ; Other_Alphabetic # Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEETEI MAYEK VOWEL SIGN INAP +ABE5 ; Other_Alphabetic # Mn MEETEI MAYEK VOWEL SIGN ANAP +ABE6..ABE7 ; Other_Alphabetic # Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEETEI MAYEK VOWEL SIGN SOUNAP +ABE8 ; Other_Alphabetic # Mn MEETEI MAYEK VOWEL SIGN UNAP +ABE9..ABEA ; Other_Alphabetic # Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..MEETEI MAYEK VOWEL SIGN NUNG +FB1E ; Other_Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA +10376..1037A ; Other_Alphabetic # Mn [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII +10A01..10A03 ; Other_Alphabetic # Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROSHTHI VOWEL SIGN VOCALIC R +10A05..10A06 ; Other_Alphabetic # Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROSHTHI VOWEL SIGN O +10A0C..10A0F ; Other_Alphabetic # Mn [4] KHAROSHTHI VOWEL LENGTH MARK..KHAROSHTHI SIGN VISARGA +10D24..10D27 ; Other_Alphabetic # Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI +10EAB..10EAC ; Other_Alphabetic # Mn [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK +11000 ; Other_Alphabetic # Mc BRAHMI SIGN CANDRABINDU +11001 ; Other_Alphabetic # Mn BRAHMI SIGN ANUSVARA +11002 ; Other_Alphabetic # Mc BRAHMI SIGN VISARGA +11038..11045 ; Other_Alphabetic # Mn [14] BRAHMI VOWEL SIGN AA..BRAHMI VOWEL SIGN AU +11073..11074 ; Other_Alphabetic # Mn [2] BRAHMI VOWEL SIGN OLD TAMIL SHORT E..BRAHMI VOWEL SIGN OLD TAMIL SHORT O +11082 ; Other_Alphabetic # Mc KAITHI SIGN VISARGA +110B0..110B2 ; Other_Alphabetic # Mc [3] KAITHI VOWEL SIGN AA..KAITHI VOWEL SIGN II +110B3..110B6 ; Other_Alphabetic # Mn [4] KAITHI VOWEL SIGN U..KAITHI VOWEL SIGN AI +110B7..110B8 ; Other_Alphabetic # Mc [2] KAITHI VOWEL SIGN O..KAITHI VOWEL SIGN AU +110C2 ; Other_Alphabetic # Mn KAITHI VOWEL SIGN VOCALIC R +11100..11102 ; Other_Alphabetic # Mn [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA +11127..1112B ; Other_Alphabetic # Mn [5] CHAKMA VOWEL SIGN A..CHAKMA VOWEL SIGN UU +1112C ; Other_Alphabetic # Mc CHAKMA VOWEL SIGN E +1112D..11132 ; Other_Alphabetic # Mn [6] CHAKMA VOWEL SIGN AI..CHAKMA AU MARK +11145..11146 ; Other_Alphabetic # Mc [2] CHAKMA VOWEL SIGN AA..CHAKMA VOWEL SIGN EI +11180..11181 ; Other_Alphabetic # Mn [2] SHARADA SIGN CANDRABINDU..SHARADA SIGN ANUSVARA +11182 ; Other_Alphabetic # Mc SHARADA SIGN VISARGA +111B3..111B5 ; Other_Alphabetic # Mc [3] SHARADA VOWEL SIGN AA..SHARADA VOWEL SIGN II +111B6..111BE ; Other_Alphabetic # Mn [9] SHARADA VOWEL SIGN U..SHARADA VOWEL SIGN O +111BF ; Other_Alphabetic # Mc SHARADA VOWEL SIGN AU +111CE ; Other_Alphabetic # Mc SHARADA VOWEL SIGN PRISHTHAMATRA E +111CF ; Other_Alphabetic # Mn SHARADA SIGN INVERTED CANDRABINDU +1122C..1122E ; Other_Alphabetic # Mc [3] KHOJKI VOWEL SIGN AA..KHOJKI VOWEL SIGN II +1122F..11231 ; Other_Alphabetic # Mn [3] KHOJKI VOWEL SIGN U..KHOJKI VOWEL SIGN AI +11232..11233 ; Other_Alphabetic # Mc [2] KHOJKI VOWEL SIGN O..KHOJKI VOWEL SIGN AU +11234 ; Other_Alphabetic # Mn KHOJKI SIGN ANUSVARA +11237 ; Other_Alphabetic # Mn KHOJKI SIGN SHADDA +1123E ; Other_Alphabetic # Mn KHOJKI SIGN SUKUN +112DF ; Other_Alphabetic # Mn KHUDAWADI SIGN ANUSVARA +112E0..112E2 ; Other_Alphabetic # Mc [3] KHUDAWADI VOWEL SIGN AA..KHUDAWADI VOWEL SIGN II +112E3..112E8 ; Other_Alphabetic # Mn [6] KHUDAWADI VOWEL SIGN U..KHUDAWADI VOWEL SIGN AU +11300..11301 ; Other_Alphabetic # Mn [2] GRANTHA SIGN COMBINING ANUSVARA ABOVE..GRANTHA SIGN CANDRABINDU +11302..11303 ; Other_Alphabetic # Mc [2] GRANTHA SIGN ANUSVARA..GRANTHA SIGN VISARGA +1133E..1133F ; Other_Alphabetic # Mc [2] GRANTHA VOWEL SIGN AA..GRANTHA VOWEL SIGN I +11340 ; Other_Alphabetic # Mn GRANTHA VOWEL SIGN II +11341..11344 ; Other_Alphabetic # Mc [4] GRANTHA VOWEL SIGN U..GRANTHA VOWEL SIGN VOCALIC RR +11347..11348 ; Other_Alphabetic # Mc [2] GRANTHA VOWEL SIGN EE..GRANTHA VOWEL SIGN AI +1134B..1134C ; Other_Alphabetic # Mc [2] GRANTHA VOWEL SIGN OO..GRANTHA VOWEL SIGN AU +11357 ; Other_Alphabetic # Mc GRANTHA AU LENGTH MARK +11362..11363 ; Other_Alphabetic # Mc [2] GRANTHA VOWEL SIGN VOCALIC L..GRANTHA VOWEL SIGN VOCALIC LL +11435..11437 ; Other_Alphabetic # Mc [3] NEWA VOWEL SIGN AA..NEWA VOWEL SIGN II +11438..1143F ; Other_Alphabetic # Mn [8] NEWA VOWEL SIGN U..NEWA VOWEL SIGN AI +11440..11441 ; Other_Alphabetic # Mc [2] NEWA VOWEL SIGN O..NEWA VOWEL SIGN AU +11443..11444 ; Other_Alphabetic # Mn [2] NEWA SIGN CANDRABINDU..NEWA SIGN ANUSVARA +11445 ; Other_Alphabetic # Mc NEWA SIGN VISARGA +114B0..114B2 ; Other_Alphabetic # Mc [3] TIRHUTA VOWEL SIGN AA..TIRHUTA VOWEL SIGN II +114B3..114B8 ; Other_Alphabetic # Mn [6] TIRHUTA VOWEL SIGN U..TIRHUTA VOWEL SIGN VOCALIC LL +114B9 ; Other_Alphabetic # Mc TIRHUTA VOWEL SIGN E +114BA ; Other_Alphabetic # Mn TIRHUTA VOWEL SIGN SHORT E +114BB..114BE ; Other_Alphabetic # Mc [4] TIRHUTA VOWEL SIGN AI..TIRHUTA VOWEL SIGN AU +114BF..114C0 ; Other_Alphabetic # Mn [2] TIRHUTA SIGN CANDRABINDU..TIRHUTA SIGN ANUSVARA +114C1 ; Other_Alphabetic # Mc TIRHUTA SIGN VISARGA +115AF..115B1 ; Other_Alphabetic # Mc [3] SIDDHAM VOWEL SIGN AA..SIDDHAM VOWEL SIGN II +115B2..115B5 ; Other_Alphabetic # Mn [4] SIDDHAM VOWEL SIGN U..SIDDHAM VOWEL SIGN VOCALIC RR +115B8..115BB ; Other_Alphabetic # Mc [4] SIDDHAM VOWEL SIGN E..SIDDHAM VOWEL SIGN AU +115BC..115BD ; Other_Alphabetic # Mn [2] SIDDHAM SIGN CANDRABINDU..SIDDHAM SIGN ANUSVARA +115BE ; Other_Alphabetic # Mc SIDDHAM SIGN VISARGA +115DC..115DD ; Other_Alphabetic # Mn [2] SIDDHAM VOWEL SIGN ALTERNATE U..SIDDHAM VOWEL SIGN ALTERNATE UU +11630..11632 ; Other_Alphabetic # Mc [3] MODI VOWEL SIGN AA..MODI VOWEL SIGN II +11633..1163A ; Other_Alphabetic # Mn [8] MODI VOWEL SIGN U..MODI VOWEL SIGN AI +1163B..1163C ; Other_Alphabetic # Mc [2] MODI VOWEL SIGN O..MODI VOWEL SIGN AU +1163D ; Other_Alphabetic # Mn MODI SIGN ANUSVARA +1163E ; Other_Alphabetic # Mc MODI SIGN VISARGA +11640 ; Other_Alphabetic # Mn MODI SIGN ARDHACANDRA +116AB ; Other_Alphabetic # Mn TAKRI SIGN ANUSVARA +116AC ; Other_Alphabetic # Mc TAKRI SIGN VISARGA +116AD ; Other_Alphabetic # Mn TAKRI VOWEL SIGN AA +116AE..116AF ; Other_Alphabetic # Mc [2] TAKRI VOWEL SIGN I..TAKRI VOWEL SIGN II +116B0..116B5 ; Other_Alphabetic # Mn [6] TAKRI VOWEL SIGN U..TAKRI VOWEL SIGN AU +1171D..1171F ; Other_Alphabetic # Mn [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA +11720..11721 ; Other_Alphabetic # Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA +11722..11725 ; Other_Alphabetic # Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU +11726 ; Other_Alphabetic # Mc AHOM VOWEL SIGN E +11727..1172A ; Other_Alphabetic # Mn [4] AHOM VOWEL SIGN AW..AHOM VOWEL SIGN AM +1182C..1182E ; Other_Alphabetic # Mc [3] DOGRA VOWEL SIGN AA..DOGRA VOWEL SIGN II +1182F..11837 ; Other_Alphabetic # Mn [9] DOGRA VOWEL SIGN U..DOGRA SIGN ANUSVARA +11838 ; Other_Alphabetic # Mc DOGRA SIGN VISARGA +11930..11935 ; Other_Alphabetic # Mc [6] DIVES AKURU VOWEL SIGN AA..DIVES AKURU VOWEL SIGN E +11937..11938 ; Other_Alphabetic # Mc [2] DIVES AKURU VOWEL SIGN AI..DIVES AKURU VOWEL SIGN O +1193B..1193C ; Other_Alphabetic # Mn [2] DIVES AKURU SIGN ANUSVARA..DIVES AKURU SIGN CANDRABINDU +11940 ; Other_Alphabetic # Mc DIVES AKURU MEDIAL YA +11942 ; Other_Alphabetic # Mc DIVES AKURU MEDIAL RA +119D1..119D3 ; Other_Alphabetic # Mc [3] NANDINAGARI VOWEL SIGN AA..NANDINAGARI VOWEL SIGN II +119D4..119D7 ; Other_Alphabetic # Mn [4] NANDINAGARI VOWEL SIGN U..NANDINAGARI VOWEL SIGN VOCALIC RR +119DA..119DB ; Other_Alphabetic # Mn [2] NANDINAGARI VOWEL SIGN E..NANDINAGARI VOWEL SIGN AI +119DC..119DF ; Other_Alphabetic # Mc [4] NANDINAGARI VOWEL SIGN O..NANDINAGARI SIGN VISARGA +119E4 ; Other_Alphabetic # Mc NANDINAGARI VOWEL SIGN PRISHTHAMATRA E +11A01..11A0A ; Other_Alphabetic # Mn [10] ZANABAZAR SQUARE VOWEL SIGN I..ZANABAZAR SQUARE VOWEL LENGTH MARK +11A35..11A38 ; Other_Alphabetic # Mn [4] ZANABAZAR SQUARE SIGN CANDRABINDU..ZANABAZAR SQUARE SIGN ANUSVARA +11A39 ; Other_Alphabetic # Mc ZANABAZAR SQUARE SIGN VISARGA +11A3B..11A3E ; Other_Alphabetic # Mn [4] ZANABAZAR SQUARE CLUSTER-FINAL LETTER YA..ZANABAZAR SQUARE CLUSTER-FINAL LETTER VA +11A51..11A56 ; Other_Alphabetic # Mn [6] SOYOMBO VOWEL SIGN I..SOYOMBO VOWEL SIGN OE +11A57..11A58 ; Other_Alphabetic # Mc [2] SOYOMBO VOWEL SIGN AI..SOYOMBO VOWEL SIGN AU +11A59..11A5B ; Other_Alphabetic # Mn [3] SOYOMBO VOWEL SIGN VOCALIC R..SOYOMBO VOWEL LENGTH MARK +11A8A..11A96 ; Other_Alphabetic # Mn [13] SOYOMBO FINAL CONSONANT SIGN G..SOYOMBO SIGN ANUSVARA +11A97 ; Other_Alphabetic # Mc SOYOMBO SIGN VISARGA +11C2F ; Other_Alphabetic # Mc BHAIKSUKI VOWEL SIGN AA +11C30..11C36 ; Other_Alphabetic # Mn [7] BHAIKSUKI VOWEL SIGN I..BHAIKSUKI VOWEL SIGN VOCALIC L +11C38..11C3D ; Other_Alphabetic # Mn [6] BHAIKSUKI VOWEL SIGN E..BHAIKSUKI SIGN ANUSVARA +11C3E ; Other_Alphabetic # Mc BHAIKSUKI SIGN VISARGA +11C92..11CA7 ; Other_Alphabetic # Mn [22] MARCHEN SUBJOINED LETTER KA..MARCHEN SUBJOINED LETTER ZA +11CA9 ; Other_Alphabetic # Mc MARCHEN SUBJOINED LETTER YA +11CAA..11CB0 ; Other_Alphabetic # Mn [7] MARCHEN SUBJOINED LETTER RA..MARCHEN VOWEL SIGN AA +11CB1 ; Other_Alphabetic # Mc MARCHEN VOWEL SIGN I +11CB2..11CB3 ; Other_Alphabetic # Mn [2] MARCHEN VOWEL SIGN U..MARCHEN VOWEL SIGN E +11CB4 ; Other_Alphabetic # Mc MARCHEN VOWEL SIGN O +11CB5..11CB6 ; Other_Alphabetic # Mn [2] MARCHEN SIGN ANUSVARA..MARCHEN SIGN CANDRABINDU +11D31..11D36 ; Other_Alphabetic # Mn [6] MASARAM GONDI VOWEL SIGN AA..MASARAM GONDI VOWEL SIGN VOCALIC R +11D3A ; Other_Alphabetic # Mn MASARAM GONDI VOWEL SIGN E +11D3C..11D3D ; Other_Alphabetic # Mn [2] MASARAM GONDI VOWEL SIGN AI..MASARAM GONDI VOWEL SIGN O +11D3F..11D41 ; Other_Alphabetic # Mn [3] MASARAM GONDI VOWEL SIGN AU..MASARAM GONDI SIGN VISARGA +11D43 ; Other_Alphabetic # Mn MASARAM GONDI SIGN CANDRA +11D47 ; Other_Alphabetic # Mn MASARAM GONDI RA-KARA +11D8A..11D8E ; Other_Alphabetic # Mc [5] GUNJALA GONDI VOWEL SIGN AA..GUNJALA GONDI VOWEL SIGN UU +11D90..11D91 ; Other_Alphabetic # Mn [2] GUNJALA GONDI VOWEL SIGN EE..GUNJALA GONDI VOWEL SIGN AI +11D93..11D94 ; Other_Alphabetic # Mc [2] GUNJALA GONDI VOWEL SIGN OO..GUNJALA GONDI VOWEL SIGN AU +11D95 ; Other_Alphabetic # Mn GUNJALA GONDI SIGN ANUSVARA +11D96 ; Other_Alphabetic # Mc GUNJALA GONDI SIGN VISARGA +11EF3..11EF4 ; Other_Alphabetic # Mn [2] MAKASAR VOWEL SIGN I..MAKASAR VOWEL SIGN U +11EF5..11EF6 ; Other_Alphabetic # Mc [2] MAKASAR VOWEL SIGN E..MAKASAR VOWEL SIGN O +16F4F ; Other_Alphabetic # Mn MIAO SIGN CONSONANT MODIFIER BAR +16F51..16F87 ; Other_Alphabetic # Mc [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI +16F8F..16F92 ; Other_Alphabetic # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW +16FF0..16FF1 ; Other_Alphabetic # Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY +1BC9E ; Other_Alphabetic # Mn DUPLOYAN DOUBLE MARK +1E000..1E006 ; Other_Alphabetic # Mn [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE +1E008..1E018 ; Other_Alphabetic # Mn [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU +1E01B..1E021 ; Other_Alphabetic # Mn [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI +1E023..1E024 ; Other_Alphabetic # Mn [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS +1E026..1E02A ; Other_Alphabetic # Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA +1E947 ; Other_Alphabetic # Mn ADLAM HAMZA +1F130..1F149 ; Other_Alphabetic # So [26] SQUARED LATIN CAPITAL LETTER A..SQUARED LATIN CAPITAL LETTER Z +1F150..1F169 ; Other_Alphabetic # So [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z +1F170..1F189 ; Other_Alphabetic # So [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z + +# Total code points: 1404 + +# ================================================ + +3006 ; Ideographic # Lo IDEOGRAPHIC CLOSING MARK +3007 ; Ideographic # Nl IDEOGRAPHIC NUMBER ZERO +3021..3029 ; Ideographic # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +3038..303A ; Ideographic # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +3400..4DBF ; Ideographic # Lo [6592] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DBF +4E00..9FFF ; Ideographic # Lo [20992] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FFF +F900..FA6D ; Ideographic # Lo [366] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA6D +FA70..FAD9 ; Ideographic # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILITY IDEOGRAPH-FAD9 +16FE4 ; Ideographic # Mn KHITAN SMALL SCRIPT FILLER +17000..187F7 ; Ideographic # Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7 +18800..18CD5 ; Ideographic # Lo [1238] TANGUT COMPONENT-001..KHITAN SMALL SCRIPT CHARACTER-18CD5 +18D00..18D08 ; Ideographic # Lo [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08 +1B170..1B2FB ; Ideographic # Lo [396] NUSHU CHARACTER-1B170..NUSHU CHARACTER-1B2FB +20000..2A6DF ; Ideographic # Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF +2A700..2B738 ; Ideographic # Lo [4153] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B738 +2B740..2B81D ; Ideographic # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D +2B820..2CEA1 ; Ideographic # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 +2CEB0..2EBE0 ; Ideographic # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 +2F800..2FA1D ; Ideographic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D +30000..3134A ; Ideographic # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A + +# Total code points: 101661 + +# ================================================ + +005E ; Diacritic # Sk CIRCUMFLEX ACCENT +0060 ; Diacritic # Sk GRAVE ACCENT +00A8 ; Diacritic # Sk DIAERESIS +00AF ; Diacritic # Sk MACRON +00B4 ; Diacritic # Sk ACUTE ACCENT +00B7 ; Diacritic # Po MIDDLE DOT +00B8 ; Diacritic # Sk CEDILLA +02B0..02C1 ; Diacritic # Lm [18] MODIFIER LETTER SMALL H..MODIFIER LETTER REVERSED GLOTTAL STOP +02C2..02C5 ; Diacritic # Sk [4] MODIFIER LETTER LEFT ARROWHEAD..MODIFIER LETTER DOWN ARROWHEAD +02C6..02D1 ; Diacritic # Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT..MODIFIER LETTER HALF TRIANGULAR COLON +02D2..02DF ; Diacritic # Sk [14] MODIFIER LETTER CENTRED RIGHT HALF RING..MODIFIER LETTER CROSS ACCENT +02E0..02E4 ; Diacritic # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02E5..02EB ; Diacritic # Sk [7] MODIFIER LETTER EXTRA-HIGH TONE BAR..MODIFIER LETTER YANG DEPARTING TONE MARK +02EC ; Diacritic # Lm MODIFIER LETTER VOICING +02ED ; Diacritic # Sk MODIFIER LETTER UNASPIRATED +02EE ; Diacritic # Lm MODIFIER LETTER DOUBLE APOSTROPHE +02EF..02FF ; Diacritic # Sk [17] MODIFIER LETTER LOW DOWN ARROWHEAD..MODIFIER LETTER LOW LEFT ARROW +0300..034E ; Diacritic # Mn [79] COMBINING GRAVE ACCENT..COMBINING UPWARDS ARROW BELOW +0350..0357 ; Diacritic # Mn [8] COMBINING RIGHT ARROWHEAD ABOVE..COMBINING RIGHT HALF RING ABOVE +035D..0362 ; Diacritic # Mn [6] COMBINING DOUBLE BREVE..COMBINING DOUBLE RIGHTWARDS ARROW BELOW +0374 ; Diacritic # Lm GREEK NUMERAL SIGN +0375 ; Diacritic # Sk GREEK LOWER NUMERAL SIGN +037A ; Diacritic # Lm GREEK YPOGEGRAMMENI +0384..0385 ; Diacritic # Sk [2] GREEK TONOS..GREEK DIALYTIKA TONOS +0483..0487 ; Diacritic # Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE +0559 ; Diacritic # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0591..05A1 ; Diacritic # Mn [17] HEBREW ACCENT ETNAHTA..HEBREW ACCENT PAZER +05A3..05BD ; Diacritic # Mn [27] HEBREW ACCENT MUNAH..HEBREW POINT METEG +05BF ; Diacritic # Mn HEBREW POINT RAFE +05C1..05C2 ; Diacritic # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4 ; Diacritic # Mn HEBREW MARK UPPER DOT +064B..0652 ; Diacritic # Mn [8] ARABIC FATHATAN..ARABIC SUKUN +0657..0658 ; Diacritic # Mn [2] ARABIC INVERTED DAMMA..ARABIC MARK NOON GHUNNA +06DF..06E0 ; Diacritic # Mn [2] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH UPRIGHT RECTANGULAR ZERO +06E5..06E6 ; Diacritic # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06EA..06EC ; Diacritic # Mn [3] ARABIC EMPTY CENTRE LOW STOP..ARABIC ROUNDED HIGH STOP WITH FILLED CENTRE +0730..074A ; Diacritic # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH +07A6..07B0 ; Diacritic # Mn [11] THAANA ABAFILI..THAANA SUKUN +07EB..07F3 ; Diacritic # Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE +07F4..07F5 ; Diacritic # Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW TONE APOSTROPHE +0818..0819 ; Diacritic # Mn [2] SAMARITAN MARK OCCLUSION..SAMARITAN MARK DAGESH +0898..089F ; Diacritic # Mn [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA +08C9 ; Diacritic # Lm ARABIC SMALL FARSI YEH +08CA..08D2 ; Diacritic # Mn [9] ARABIC SMALL HIGH FARSI YEH..ARABIC LARGE ROUND DOT INSIDE CIRCLE BELOW +08E3..08FE ; Diacritic # Mn [28] ARABIC TURNED DAMMA BELOW..ARABIC DAMMA WITH DOT +093C ; Diacritic # Mn DEVANAGARI SIGN NUKTA +094D ; Diacritic # Mn DEVANAGARI SIGN VIRAMA +0951..0954 ; Diacritic # Mn [4] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI ACUTE ACCENT +0971 ; Diacritic # Lm DEVANAGARI SIGN HIGH SPACING DOT +09BC ; Diacritic # Mn BENGALI SIGN NUKTA +09CD ; Diacritic # Mn BENGALI SIGN VIRAMA +0A3C ; Diacritic # Mn GURMUKHI SIGN NUKTA +0A4D ; Diacritic # Mn GURMUKHI SIGN VIRAMA +0ABC ; Diacritic # Mn GUJARATI SIGN NUKTA +0ACD ; Diacritic # Mn GUJARATI SIGN VIRAMA +0AFD..0AFF ; Diacritic # Mn [3] GUJARATI SIGN THREE-DOT NUKTA ABOVE..GUJARATI SIGN TWO-CIRCLE NUKTA ABOVE +0B3C ; Diacritic # Mn ORIYA SIGN NUKTA +0B4D ; Diacritic # Mn ORIYA SIGN VIRAMA +0B55 ; Diacritic # Mn ORIYA SIGN OVERLINE +0BCD ; Diacritic # Mn TAMIL SIGN VIRAMA +0C3C ; Diacritic # Mn TELUGU SIGN NUKTA +0C4D ; Diacritic # Mn TELUGU SIGN VIRAMA +0CBC ; Diacritic # Mn KANNADA SIGN NUKTA +0CCD ; Diacritic # Mn KANNADA SIGN VIRAMA +0D3B..0D3C ; Diacritic # Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA +0D4D ; Diacritic # Mn MALAYALAM SIGN VIRAMA +0DCA ; Diacritic # Mn SINHALA SIGN AL-LAKUNA +0E47..0E4C ; Diacritic # Mn [6] THAI CHARACTER MAITAIKHU..THAI CHARACTER THANTHAKHAT +0E4E ; Diacritic # Mn THAI CHARACTER YAMAKKAN +0EBA ; Diacritic # Mn LAO SIGN PALI VIRAMA +0EC8..0ECC ; Diacritic # Mn [5] LAO TONE MAI EK..LAO CANCELLATION MARK +0F18..0F19 ; Diacritic # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS +0F35 ; Diacritic # Mn TIBETAN MARK NGAS BZUNG NYI ZLA +0F37 ; Diacritic # Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS +0F39 ; Diacritic # Mn TIBETAN MARK TSA -PHRU +0F3E..0F3F ; Diacritic # Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SIGN MAR TSHES +0F82..0F84 ; Diacritic # Mn [3] TIBETAN SIGN NYI ZLA NAA DA..TIBETAN MARK HALANTA +0F86..0F87 ; Diacritic # Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS +0FC6 ; Diacritic # Mn TIBETAN SYMBOL PADMA GDAN +1037 ; Diacritic # Mn MYANMAR SIGN DOT BELOW +1039..103A ; Diacritic # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT +1063..1064 ; Diacritic # Mc [2] MYANMAR TONE MARK SGAW KAREN HATHI..MYANMAR TONE MARK SGAW KAREN KE PHO +1069..106D ; Diacritic # Mc [5] MYANMAR SIGN WESTERN PWO KAREN TONE-1..MYANMAR SIGN WESTERN PWO KAREN TONE-5 +1087..108C ; Diacritic # Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR SIGN SHAN COUNCIL TONE-3 +108D ; Diacritic # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE +108F ; Diacritic # Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 +109A..109B ; Diacritic # Mc [2] MYANMAR SIGN KHAMTI TONE-1..MYANMAR SIGN KHAMTI TONE-3 +135D..135F ; Diacritic # Mn [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK +1714 ; Diacritic # Mn TAGALOG SIGN VIRAMA +1715 ; Diacritic # Mc TAGALOG SIGN PAMUDPOD +17C9..17D3 ; Diacritic # Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN BATHAMASAT +17DD ; Diacritic # Mn KHMER SIGN ATTHACAN +1939..193B ; Diacritic # Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I +1A75..1A7C ; Diacritic # Mn [8] TAI THAM SIGN TONE-1..TAI THAM SIGN KHUEN-LUE KARAN +1A7F ; Diacritic # Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT +1AB0..1ABD ; Diacritic # Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW +1ABE ; Diacritic # Me COMBINING PARENTHESES OVERLAY +1AC1..1ACB ; Diacritic # Mn [11] COMBINING LEFT PARENTHESIS ABOVE LEFT..COMBINING TRIPLE ACUTE ACCENT +1B34 ; Diacritic # Mn BALINESE SIGN REREKAN +1B44 ; Diacritic # Mc BALINESE ADEG ADEG +1B6B..1B73 ; Diacritic # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG +1BAA ; Diacritic # Mc SUNDANESE SIGN PAMAAEH +1BAB ; Diacritic # Mn SUNDANESE SIGN VIRAMA +1C36..1C37 ; Diacritic # Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA +1C78..1C7D ; Diacritic # Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD +1CD0..1CD2 ; Diacritic # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA +1CD3 ; Diacritic # Po VEDIC SIGN NIHSHVASA +1CD4..1CE0 ; Diacritic # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA +1CE1 ; Diacritic # Mc VEDIC TONE ATHARVAVEDIC INDEPENDENT SVARITA +1CE2..1CE8 ; Diacritic # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL +1CED ; Diacritic # Mn VEDIC SIGN TIRYAK +1CF4 ; Diacritic # Mn VEDIC TONE CANDRA ABOVE +1CF7 ; Diacritic # Mc VEDIC SIGN ATIKRAMA +1CF8..1CF9 ; Diacritic # Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE +1D2C..1D6A ; Diacritic # Lm [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI +1DC4..1DCF ; Diacritic # Mn [12] COMBINING MACRON-ACUTE..COMBINING ZIGZAG BELOW +1DF5..1DFF ; Diacritic # Mn [11] COMBINING UP TACK ABOVE..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW +1FBD ; Diacritic # Sk GREEK KORONIS +1FBF..1FC1 ; Diacritic # Sk [3] GREEK PSILI..GREEK DIALYTIKA AND PERISPOMENI +1FCD..1FCF ; Diacritic # Sk [3] GREEK PSILI AND VARIA..GREEK PSILI AND PERISPOMENI +1FDD..1FDF ; Diacritic # Sk [3] GREEK DASIA AND VARIA..GREEK DASIA AND PERISPOMENI +1FED..1FEF ; Diacritic # Sk [3] GREEK DIALYTIKA AND VARIA..GREEK VARIA +1FFD..1FFE ; Diacritic # Sk [2] GREEK OXIA..GREEK DASIA +2CEF..2CF1 ; Diacritic # Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS +2E2F ; Diacritic # Lm VERTICAL TILDE +302A..302D ; Diacritic # Mn [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOGRAPHIC ENTERING TONE MARK +302E..302F ; Diacritic # Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK +3099..309A ; Diacritic # Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309B..309C ; Diacritic # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +30FC ; Diacritic # Lm KATAKANA-HIRAGANA PROLONGED SOUND MARK +A66F ; Diacritic # Mn COMBINING CYRILLIC VZMET +A67C..A67D ; Diacritic # Mn [2] COMBINING CYRILLIC KAVYKA..COMBINING CYRILLIC PAYEROK +A67F ; Diacritic # Lm CYRILLIC PAYEROK +A69C..A69D ; Diacritic # Lm [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN +A6F0..A6F1 ; Diacritic # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS +A700..A716 ; Diacritic # Sk [23] MODIFIER LETTER CHINESE TONE YIN PING..MODIFIER LETTER EXTRA-LOW LEFT-STEM TONE BAR +A717..A71F ; Diacritic # Lm [9] MODIFIER LETTER DOT VERTICAL BAR..MODIFIER LETTER LOW INVERTED EXCLAMATION MARK +A720..A721 ; Diacritic # Sk [2] MODIFIER LETTER STRESS AND HIGH TONE..MODIFIER LETTER STRESS AND LOW TONE +A788 ; Diacritic # Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT +A789..A78A ; Diacritic # Sk [2] MODIFIER LETTER COLON..MODIFIER LETTER SHORT EQUALS SIGN +A7F8..A7F9 ; Diacritic # Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE +A8C4 ; Diacritic # Mn SAURASHTRA SIGN VIRAMA +A8E0..A8F1 ; Diacritic # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA +A92B..A92D ; Diacritic # Mn [3] KAYAH LI TONE PLOPHU..KAYAH LI TONE CALYA PLOPHU +A92E ; Diacritic # Po KAYAH LI SIGN CWI +A953 ; Diacritic # Mc REJANG VIRAMA +A9B3 ; Diacritic # Mn JAVANESE SIGN CECAK TELU +A9C0 ; Diacritic # Mc JAVANESE PANGKON +A9E5 ; Diacritic # Mn MYANMAR SIGN SHAN SAW +AA7B ; Diacritic # Mc MYANMAR SIGN PAO KAREN TONE +AA7C ; Diacritic # Mn MYANMAR SIGN TAI LAING TONE-2 +AA7D ; Diacritic # Mc MYANMAR SIGN TAI LAING TONE-5 +AABF ; Diacritic # Mn TAI VIET TONE MAI EK +AAC0 ; Diacritic # Lo TAI VIET TONE MAI NUENG +AAC1 ; Diacritic # Mn TAI VIET TONE MAI THO +AAC2 ; Diacritic # Lo TAI VIET TONE MAI SONG +AAF6 ; Diacritic # Mn MEETEI MAYEK VIRAMA +AB5B ; Diacritic # Sk MODIFIER BREVE WITH INVERTED BREVE +AB5C..AB5F ; Diacritic # Lm [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK +AB69 ; Diacritic # Lm MODIFIER LETTER SMALL TURNED W +AB6A..AB6B ; Diacritic # Sk [2] MODIFIER LETTER LEFT TACK..MODIFIER LETTER RIGHT TACK +ABEC ; Diacritic # Mc MEETEI MAYEK LUM IYEK +ABED ; Diacritic # Mn MEETEI MAYEK APUN IYEK +FB1E ; Diacritic # Mn HEBREW POINT JUDEO-SPANISH VARIKA +FE20..FE2F ; Diacritic # Mn [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF +FF3E ; Diacritic # Sk FULLWIDTH CIRCUMFLEX ACCENT +FF40 ; Diacritic # Sk FULLWIDTH GRAVE ACCENT +FF70 ; Diacritic # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF9E..FF9F ; Diacritic # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFE3 ; Diacritic # Sk FULLWIDTH MACRON +102E0 ; Diacritic # Mn COPTIC EPACT THOUSANDS MARK +10780..10785 ; Diacritic # Lm [6] MODIFIER LETTER SMALL CAPITAL AA..MODIFIER LETTER SMALL B WITH HOOK +10787..107B0 ; Diacritic # Lm [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK +107B2..107BA ; Diacritic # Lm [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL +10AE5..10AE6 ; Diacritic # Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW +10D22..10D23 ; Diacritic # Lo [2] HANIFI ROHINGYA MARK SAKIN..HANIFI ROHINGYA MARK NA KHONNA +10D24..10D27 ; Diacritic # Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI +10F46..10F50 ; Diacritic # Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW +10F82..10F85 ; Diacritic # Mn [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW +11046 ; Diacritic # Mn BRAHMI VIRAMA +11070 ; Diacritic # Mn BRAHMI SIGN OLD TAMIL VIRAMA +110B9..110BA ; Diacritic # Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN NUKTA +11133..11134 ; Diacritic # Mn [2] CHAKMA VIRAMA..CHAKMA MAAYYAA +11173 ; Diacritic # Mn MAHAJANI SIGN NUKTA +111C0 ; Diacritic # Mc SHARADA SIGN VIRAMA +111CA..111CC ; Diacritic # Mn [3] SHARADA SIGN NUKTA..SHARADA EXTRA SHORT VOWEL MARK +11235 ; Diacritic # Mc KHOJKI SIGN VIRAMA +11236 ; Diacritic # Mn KHOJKI SIGN NUKTA +112E9..112EA ; Diacritic # Mn [2] KHUDAWADI SIGN NUKTA..KHUDAWADI SIGN VIRAMA +1133C ; Diacritic # Mn GRANTHA SIGN NUKTA +1134D ; Diacritic # Mc GRANTHA SIGN VIRAMA +11366..1136C ; Diacritic # Mn [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX +11370..11374 ; Diacritic # Mn [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA +11442 ; Diacritic # Mn NEWA SIGN VIRAMA +11446 ; Diacritic # Mn NEWA SIGN NUKTA +114C2..114C3 ; Diacritic # Mn [2] TIRHUTA SIGN VIRAMA..TIRHUTA SIGN NUKTA +115BF..115C0 ; Diacritic # Mn [2] SIDDHAM SIGN VIRAMA..SIDDHAM SIGN NUKTA +1163F ; Diacritic # Mn MODI SIGN VIRAMA +116B6 ; Diacritic # Mc TAKRI SIGN VIRAMA +116B7 ; Diacritic # Mn TAKRI SIGN NUKTA +1172B ; Diacritic # Mn AHOM SIGN KILLER +11839..1183A ; Diacritic # Mn [2] DOGRA SIGN VIRAMA..DOGRA SIGN NUKTA +1193D ; Diacritic # Mc DIVES AKURU SIGN HALANTA +1193E ; Diacritic # Mn DIVES AKURU VIRAMA +11943 ; Diacritic # Mn DIVES AKURU SIGN NUKTA +119E0 ; Diacritic # Mn NANDINAGARI SIGN VIRAMA +11A34 ; Diacritic # Mn ZANABAZAR SQUARE SIGN VIRAMA +11A47 ; Diacritic # Mn ZANABAZAR SQUARE SUBJOINER +11A99 ; Diacritic # Mn SOYOMBO SUBJOINER +11C3F ; Diacritic # Mn BHAIKSUKI SIGN VIRAMA +11D42 ; Diacritic # Mn MASARAM GONDI SIGN NUKTA +11D44..11D45 ; Diacritic # Mn [2] MASARAM GONDI SIGN HALANTA..MASARAM GONDI VIRAMA +11D97 ; Diacritic # Mn GUNJALA GONDI VIRAMA +16AF0..16AF4 ; Diacritic # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE +16B30..16B36 ; Diacritic # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM +16F8F..16F92 ; Diacritic # Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW +16F93..16F9F ; Diacritic # Lm [13] MIAO LETTER TONE-2..MIAO LETTER REFORMED TONE-8 +16FF0..16FF1 ; Diacritic # Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY +1AFF0..1AFF3 ; Diacritic # Lm [4] KATAKANA LETTER MINNAN TONE-2..KATAKANA LETTER MINNAN TONE-5 +1AFF5..1AFFB ; Diacritic # Lm [7] KATAKANA LETTER MINNAN TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-5 +1AFFD..1AFFE ; Diacritic # Lm [2] KATAKANA LETTER MINNAN NASALIZED TONE-7..KATAKANA LETTER MINNAN NASALIZED TONE-8 +1CF00..1CF2D ; Diacritic # Mn [46] ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT..ZNAMENNY COMBINING MARK KRYZH ON LEFT +1CF30..1CF46 ; Diacritic # Mn [23] ZNAMENNY COMBINING TONAL RANGE MARK MRACHNO..ZNAMENNY PRIZNAK MODIFIER ROG +1D167..1D169 ; Diacritic # Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3 +1D16D..1D172 ; Diacritic # Mc [6] MUSICAL SYMBOL COMBINING AUGMENTATION DOT..MUSICAL SYMBOL COMBINING FLAG-5 +1D17B..1D182 ; Diacritic # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE +1D185..1D18B ; Diacritic # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE +1D1AA..1D1AD ; Diacritic # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO +1E130..1E136 ; Diacritic # Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D +1E2AE ; Diacritic # Mn TOTO SIGN RISING TONE +1E2EC..1E2EF ; Diacritic # Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI +1E8D0..1E8D6 ; Diacritic # Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS +1E944..1E946 ; Diacritic # Mn [3] ADLAM ALIF LENGTHENER..ADLAM GEMINATION MARK +1E948..1E94A ; Diacritic # Mn [3] ADLAM CONSONANT MODIFIER..ADLAM NUKTA + +# Total code points: 1064 + +# ================================================ + +00B7 ; Extender # Po MIDDLE DOT +02D0..02D1 ; Extender # Lm [2] MODIFIER LETTER TRIANGULAR COLON..MODIFIER LETTER HALF TRIANGULAR COLON +0640 ; Extender # Lm ARABIC TATWEEL +07FA ; Extender # Lm NKO LAJANYALAN +0B55 ; Extender # Mn ORIYA SIGN OVERLINE +0E46 ; Extender # Lm THAI CHARACTER MAIYAMOK +0EC6 ; Extender # Lm LAO KO LA +180A ; Extender # Po MONGOLIAN NIRUGU +1843 ; Extender # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1AA7 ; Extender # Lm TAI THAM SIGN MAI YAMOK +1C36 ; Extender # Mn LEPCHA SIGN RAN +1C7B ; Extender # Lm OL CHIKI RELAA +3005 ; Extender # Lm IDEOGRAPHIC ITERATION MARK +3031..3035 ; Extender # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +309D..309E ; Extender # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +30FC..30FE ; Extender # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +A015 ; Extender # Lm YI SYLLABLE WU +A60C ; Extender # Lm VAI SYLLABLE LENGTHENER +A9CF ; Extender # Lm JAVANESE PANGRANGKEP +A9E6 ; Extender # Lm MYANMAR MODIFIER LETTER SHAN REDUPLICATION +AA70 ; Extender # Lm MYANMAR MODIFIER LETTER KHAMTI REDUPLICATION +AADD ; Extender # Lm TAI VIET SYMBOL SAM +AAF3..AAF4 ; Extender # Lm [2] MEETEI MAYEK SYLLABLE REPETITION MARK..MEETEI MAYEK WORD REPETITION MARK +FF70 ; Extender # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +10781..10782 ; Extender # Lm [2] MODIFIER LETTER SUPERSCRIPT TRIANGULAR COLON..MODIFIER LETTER SUPERSCRIPT HALF TRIANGULAR COLON +1135D ; Extender # Lo GRANTHA SIGN PLUTA +115C6..115C8 ; Extender # Po [3] SIDDHAM REPETITION MARK-1..SIDDHAM REPETITION MARK-3 +11A98 ; Extender # Mn SOYOMBO GEMINATION MARK +16B42..16B43 ; Extender # Lm [2] PAHAWH HMONG SIGN VOS NRUA..PAHAWH HMONG SIGN IB YAM +16FE0..16FE1 ; Extender # Lm [2] TANGUT ITERATION MARK..NUSHU ITERATION MARK +16FE3 ; Extender # Lm OLD CHINESE ITERATION MARK +1E13C..1E13D ; Extender # Lm [2] NYIAKENG PUACHUE HMONG SIGN XW XW..NYIAKENG PUACHUE HMONG SYLLABLE LENGTHENER +1E944..1E946 ; Extender # Mn [3] ADLAM ALIF LENGTHENER..ADLAM GEMINATION MARK + +# Total code points: 50 + +# ================================================ + +00AA ; Other_Lowercase # Lo FEMININE ORDINAL INDICATOR +00BA ; Other_Lowercase # Lo MASCULINE ORDINAL INDICATOR +02B0..02B8 ; Other_Lowercase # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02C0..02C1 ; Other_Lowercase # Lm [2] MODIFIER LETTER GLOTTAL STOP..MODIFIER LETTER REVERSED GLOTTAL STOP +02E0..02E4 ; Other_Lowercase # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +0345 ; Other_Lowercase # Mn COMBINING GREEK YPOGEGRAMMENI +037A ; Other_Lowercase # Lm GREEK YPOGEGRAMMENI +1D2C..1D6A ; Other_Lowercase # Lm [63] MODIFIER LETTER CAPITAL A..GREEK SUBSCRIPT SMALL LETTER CHI +1D78 ; Other_Lowercase # Lm MODIFIER LETTER CYRILLIC EN +1D9B..1DBF ; Other_Lowercase # Lm [37] MODIFIER LETTER SMALL TURNED ALPHA..MODIFIER LETTER SMALL THETA +2071 ; Other_Lowercase # Lm SUPERSCRIPT LATIN SMALL LETTER I +207F ; Other_Lowercase # Lm SUPERSCRIPT LATIN SMALL LETTER N +2090..209C ; Other_Lowercase # Lm [13] LATIN SUBSCRIPT SMALL LETTER A..LATIN SUBSCRIPT SMALL LETTER T +2170..217F ; Other_Lowercase # Nl [16] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL ONE THOUSAND +24D0..24E9 ; Other_Lowercase # So [26] CIRCLED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z +2C7C..2C7D ; Other_Lowercase # Lm [2] LATIN SUBSCRIPT SMALL LETTER J..MODIFIER LETTER CAPITAL V +A69C..A69D ; Other_Lowercase # Lm [2] MODIFIER LETTER CYRILLIC HARD SIGN..MODIFIER LETTER CYRILLIC SOFT SIGN +A770 ; Other_Lowercase # Lm MODIFIER LETTER US +A7F8..A7F9 ; Other_Lowercase # Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE +AB5C..AB5F ; Other_Lowercase # Lm [4] MODIFIER LETTER SMALL HENG..MODIFIER LETTER SMALL U WITH LEFT HOOK +10780 ; Other_Lowercase # Lm MODIFIER LETTER SMALL CAPITAL AA +10783..10785 ; Other_Lowercase # Lm [3] MODIFIER LETTER SMALL AE..MODIFIER LETTER SMALL B WITH HOOK +10787..107B0 ; Other_Lowercase # Lm [42] MODIFIER LETTER SMALL DZ DIGRAPH..MODIFIER LETTER SMALL V WITH RIGHT HOOK +107B2..107BA ; Other_Lowercase # Lm [9] MODIFIER LETTER SMALL CAPITAL Y..MODIFIER LETTER SMALL S WITH CURL + +# Total code points: 244 + +# ================================================ + +2160..216F ; Other_Uppercase # Nl [16] ROMAN NUMERAL ONE..ROMAN NUMERAL ONE THOUSAND +24B6..24CF ; Other_Uppercase # So [26] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN CAPITAL LETTER Z +1F130..1F149 ; Other_Uppercase # So [26] SQUARED LATIN CAPITAL LETTER A..SQUARED LATIN CAPITAL LETTER Z +1F150..1F169 ; Other_Uppercase # So [26] NEGATIVE CIRCLED LATIN CAPITAL LETTER A..NEGATIVE CIRCLED LATIN CAPITAL LETTER Z +1F170..1F189 ; Other_Uppercase # So [26] NEGATIVE SQUARED LATIN CAPITAL LETTER A..NEGATIVE SQUARED LATIN CAPITAL LETTER Z + +# Total code points: 120 + +# ================================================ + +FDD0..FDEF ; Noncharacter_Code_Point # Cn [32] .. +FFFE..FFFF ; Noncharacter_Code_Point # Cn [2] .. +1FFFE..1FFFF ; Noncharacter_Code_Point # Cn [2] .. +2FFFE..2FFFF ; Noncharacter_Code_Point # Cn [2] .. +3FFFE..3FFFF ; Noncharacter_Code_Point # Cn [2] .. +4FFFE..4FFFF ; Noncharacter_Code_Point # Cn [2] .. +5FFFE..5FFFF ; Noncharacter_Code_Point # Cn [2] .. +6FFFE..6FFFF ; Noncharacter_Code_Point # Cn [2] .. +7FFFE..7FFFF ; Noncharacter_Code_Point # Cn [2] .. +8FFFE..8FFFF ; Noncharacter_Code_Point # Cn [2] .. +9FFFE..9FFFF ; Noncharacter_Code_Point # Cn [2] .. +AFFFE..AFFFF ; Noncharacter_Code_Point # Cn [2] .. +BFFFE..BFFFF ; Noncharacter_Code_Point # Cn [2] .. +CFFFE..CFFFF ; Noncharacter_Code_Point # Cn [2] .. +DFFFE..DFFFF ; Noncharacter_Code_Point # Cn [2] .. +EFFFE..EFFFF ; Noncharacter_Code_Point # Cn [2] .. +FFFFE..FFFFF ; Noncharacter_Code_Point # Cn [2] .. +10FFFE..10FFFF; Noncharacter_Code_Point # Cn [2] .. + +# Total code points: 66 + +# ================================================ + +09BE ; Other_Grapheme_Extend # Mc BENGALI VOWEL SIGN AA +09D7 ; Other_Grapheme_Extend # Mc BENGALI AU LENGTH MARK +0B3E ; Other_Grapheme_Extend # Mc ORIYA VOWEL SIGN AA +0B57 ; Other_Grapheme_Extend # Mc ORIYA AU LENGTH MARK +0BBE ; Other_Grapheme_Extend # Mc TAMIL VOWEL SIGN AA +0BD7 ; Other_Grapheme_Extend # Mc TAMIL AU LENGTH MARK +0CC2 ; Other_Grapheme_Extend # Mc KANNADA VOWEL SIGN UU +0CD5..0CD6 ; Other_Grapheme_Extend # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0D3E ; Other_Grapheme_Extend # Mc MALAYALAM VOWEL SIGN AA +0D57 ; Other_Grapheme_Extend # Mc MALAYALAM AU LENGTH MARK +0DCF ; Other_Grapheme_Extend # Mc SINHALA VOWEL SIGN AELA-PILLA +0DDF ; Other_Grapheme_Extend # Mc SINHALA VOWEL SIGN GAYANUKITTA +1B35 ; Other_Grapheme_Extend # Mc BALINESE VOWEL SIGN TEDUNG +200C ; Other_Grapheme_Extend # Cf ZERO WIDTH NON-JOINER +302E..302F ; Other_Grapheme_Extend # Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK +FF9E..FF9F ; Other_Grapheme_Extend # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +1133E ; Other_Grapheme_Extend # Mc GRANTHA VOWEL SIGN AA +11357 ; Other_Grapheme_Extend # Mc GRANTHA AU LENGTH MARK +114B0 ; Other_Grapheme_Extend # Mc TIRHUTA VOWEL SIGN AA +114BD ; Other_Grapheme_Extend # Mc TIRHUTA VOWEL SIGN SHORT O +115AF ; Other_Grapheme_Extend # Mc SIDDHAM VOWEL SIGN AA +11930 ; Other_Grapheme_Extend # Mc DIVES AKURU VOWEL SIGN AA +1D165 ; Other_Grapheme_Extend # Mc MUSICAL SYMBOL COMBINING STEM +1D16E..1D172 ; Other_Grapheme_Extend # Mc [5] MUSICAL SYMBOL COMBINING FLAG-1..MUSICAL SYMBOL COMBINING FLAG-5 +E0020..E007F ; Other_Grapheme_Extend # Cf [96] TAG SPACE..CANCEL TAG + +# Total code points: 127 + +# ================================================ + +2FF0..2FF1 ; IDS_Binary_Operator # So [2] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER ABOVE TO BELOW +2FF4..2FFB ; IDS_Binary_Operator # So [8] IDEOGRAPHIC DESCRIPTION CHARACTER FULL SURROUND..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID + +# Total code points: 10 + +# ================================================ + +2FF2..2FF3 ; IDS_Trinary_Operator # So [2] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO MIDDLE AND RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER ABOVE TO MIDDLE AND BELOW + +# Total code points: 2 + +# ================================================ + +2E80..2E99 ; Radical # So [26] CJK RADICAL REPEAT..CJK RADICAL RAP +2E9B..2EF3 ; Radical # So [89] CJK RADICAL CHOKE..CJK RADICAL C-SIMPLIFIED TURTLE +2F00..2FD5 ; Radical # So [214] KANGXI RADICAL ONE..KANGXI RADICAL FLUTE + +# Total code points: 329 + +# ================================================ + +3400..4DBF ; Unified_Ideograph # Lo [6592] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DBF +4E00..9FFF ; Unified_Ideograph # Lo [20992] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FFF +FA0E..FA0F ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA0E..CJK COMPATIBILITY IDEOGRAPH-FA0F +FA11 ; Unified_Ideograph # Lo CJK COMPATIBILITY IDEOGRAPH-FA11 +FA13..FA14 ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA13..CJK COMPATIBILITY IDEOGRAPH-FA14 +FA1F ; Unified_Ideograph # Lo CJK COMPATIBILITY IDEOGRAPH-FA1F +FA21 ; Unified_Ideograph # Lo CJK COMPATIBILITY IDEOGRAPH-FA21 +FA23..FA24 ; Unified_Ideograph # Lo [2] CJK COMPATIBILITY IDEOGRAPH-FA23..CJK COMPATIBILITY IDEOGRAPH-FA24 +FA27..FA29 ; Unified_Ideograph # Lo [3] CJK COMPATIBILITY IDEOGRAPH-FA27..CJK COMPATIBILITY IDEOGRAPH-FA29 +20000..2A6DF ; Unified_Ideograph # Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF +2A700..2B738 ; Unified_Ideograph # Lo [4153] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B738 +2B740..2B81D ; Unified_Ideograph # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D +2B820..2CEA1 ; Unified_Ideograph # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 +2CEB0..2EBE0 ; Unified_Ideograph # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 +30000..3134A ; Unified_Ideograph # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A + +# Total code points: 92865 + +# ================================================ + +034F ; Other_Default_Ignorable_Code_Point # Mn COMBINING GRAPHEME JOINER +115F..1160 ; Other_Default_Ignorable_Code_Point # Lo [2] HANGUL CHOSEONG FILLER..HANGUL JUNGSEONG FILLER +17B4..17B5 ; Other_Default_Ignorable_Code_Point # Mn [2] KHMER VOWEL INHERENT AQ..KHMER VOWEL INHERENT AA +2065 ; Other_Default_Ignorable_Code_Point # Cn +3164 ; Other_Default_Ignorable_Code_Point # Lo HANGUL FILLER +FFA0 ; Other_Default_Ignorable_Code_Point # Lo HALFWIDTH HANGUL FILLER +FFF0..FFF8 ; Other_Default_Ignorable_Code_Point # Cn [9] .. +E0000 ; Other_Default_Ignorable_Code_Point # Cn +E0002..E001F ; Other_Default_Ignorable_Code_Point # Cn [30] .. +E0080..E00FF ; Other_Default_Ignorable_Code_Point # Cn [128] .. +E01F0..E0FFF ; Other_Default_Ignorable_Code_Point # Cn [3600] .. + +# Total code points: 3776 + +# ================================================ + +0149 ; Deprecated # L& LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +0673 ; Deprecated # Lo ARABIC LETTER ALEF WITH WAVY HAMZA BELOW +0F77 ; Deprecated # Mn TIBETAN VOWEL SIGN VOCALIC RR +0F79 ; Deprecated # Mn TIBETAN VOWEL SIGN VOCALIC LL +17A3..17A4 ; Deprecated # Lo [2] KHMER INDEPENDENT VOWEL QAQ..KHMER INDEPENDENT VOWEL QAA +206A..206F ; Deprecated # Cf [6] INHIBIT SYMMETRIC SWAPPING..NOMINAL DIGIT SHAPES +2329 ; Deprecated # Ps LEFT-POINTING ANGLE BRACKET +232A ; Deprecated # Pe RIGHT-POINTING ANGLE BRACKET +E0001 ; Deprecated # Cf LANGUAGE TAG + +# Total code points: 15 + +# ================================================ + +0069..006A ; Soft_Dotted # L& [2] LATIN SMALL LETTER I..LATIN SMALL LETTER J +012F ; Soft_Dotted # L& LATIN SMALL LETTER I WITH OGONEK +0249 ; Soft_Dotted # L& LATIN SMALL LETTER J WITH STROKE +0268 ; Soft_Dotted # L& LATIN SMALL LETTER I WITH STROKE +029D ; Soft_Dotted # L& LATIN SMALL LETTER J WITH CROSSED-TAIL +02B2 ; Soft_Dotted # Lm MODIFIER LETTER SMALL J +03F3 ; Soft_Dotted # L& GREEK LETTER YOT +0456 ; Soft_Dotted # L& CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I +0458 ; Soft_Dotted # L& CYRILLIC SMALL LETTER JE +1D62 ; Soft_Dotted # Lm LATIN SUBSCRIPT SMALL LETTER I +1D96 ; Soft_Dotted # L& LATIN SMALL LETTER I WITH RETROFLEX HOOK +1DA4 ; Soft_Dotted # Lm MODIFIER LETTER SMALL I WITH STROKE +1DA8 ; Soft_Dotted # Lm MODIFIER LETTER SMALL J WITH CROSSED-TAIL +1E2D ; Soft_Dotted # L& LATIN SMALL LETTER I WITH TILDE BELOW +1ECB ; Soft_Dotted # L& LATIN SMALL LETTER I WITH DOT BELOW +2071 ; Soft_Dotted # Lm SUPERSCRIPT LATIN SMALL LETTER I +2148..2149 ; Soft_Dotted # L& [2] DOUBLE-STRUCK ITALIC SMALL I..DOUBLE-STRUCK ITALIC SMALL J +2C7C ; Soft_Dotted # Lm LATIN SUBSCRIPT SMALL LETTER J +1D422..1D423 ; Soft_Dotted # L& [2] MATHEMATICAL BOLD SMALL I..MATHEMATICAL BOLD SMALL J +1D456..1D457 ; Soft_Dotted # L& [2] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL ITALIC SMALL J +1D48A..1D48B ; Soft_Dotted # L& [2] MATHEMATICAL BOLD ITALIC SMALL I..MATHEMATICAL BOLD ITALIC SMALL J +1D4BE..1D4BF ; Soft_Dotted # L& [2] MATHEMATICAL SCRIPT SMALL I..MATHEMATICAL SCRIPT SMALL J +1D4F2..1D4F3 ; Soft_Dotted # L& [2] MATHEMATICAL BOLD SCRIPT SMALL I..MATHEMATICAL BOLD SCRIPT SMALL J +1D526..1D527 ; Soft_Dotted # L& [2] MATHEMATICAL FRAKTUR SMALL I..MATHEMATICAL FRAKTUR SMALL J +1D55A..1D55B ; Soft_Dotted # L& [2] MATHEMATICAL DOUBLE-STRUCK SMALL I..MATHEMATICAL DOUBLE-STRUCK SMALL J +1D58E..1D58F ; Soft_Dotted # L& [2] MATHEMATICAL BOLD FRAKTUR SMALL I..MATHEMATICAL BOLD FRAKTUR SMALL J +1D5C2..1D5C3 ; Soft_Dotted # L& [2] MATHEMATICAL SANS-SERIF SMALL I..MATHEMATICAL SANS-SERIF SMALL J +1D5F6..1D5F7 ; Soft_Dotted # L& [2] MATHEMATICAL SANS-SERIF BOLD SMALL I..MATHEMATICAL SANS-SERIF BOLD SMALL J +1D62A..1D62B ; Soft_Dotted # L& [2] MATHEMATICAL SANS-SERIF ITALIC SMALL I..MATHEMATICAL SANS-SERIF ITALIC SMALL J +1D65E..1D65F ; Soft_Dotted # L& [2] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL I..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL J +1D692..1D693 ; Soft_Dotted # L& [2] MATHEMATICAL MONOSPACE SMALL I..MATHEMATICAL MONOSPACE SMALL J +1DF1A ; Soft_Dotted # L& LATIN SMALL LETTER I WITH STROKE AND RETROFLEX HOOK + +# Total code points: 47 + +# ================================================ + +0E40..0E44 ; Logical_Order_Exception # Lo [5] THAI CHARACTER SARA E..THAI CHARACTER SARA AI MAIMALAI +0EC0..0EC4 ; Logical_Order_Exception # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +19B5..19B7 ; Logical_Order_Exception # Lo [3] NEW TAI LUE VOWEL SIGN E..NEW TAI LUE VOWEL SIGN O +19BA ; Logical_Order_Exception # Lo NEW TAI LUE VOWEL SIGN AY +AAB5..AAB6 ; Logical_Order_Exception # Lo [2] TAI VIET VOWEL E..TAI VIET VOWEL O +AAB9 ; Logical_Order_Exception # Lo TAI VIET VOWEL UEA +AABB..AABC ; Logical_Order_Exception # Lo [2] TAI VIET VOWEL AUE..TAI VIET VOWEL AY + +# Total code points: 19 + +# ================================================ + +1885..1886 ; Other_ID_Start # Mn [2] MONGOLIAN LETTER ALI GALI BALUDA..MONGOLIAN LETTER ALI GALI THREE BALUDA +2118 ; Other_ID_Start # Sm SCRIPT CAPITAL P +212E ; Other_ID_Start # So ESTIMATED SYMBOL +309B..309C ; Other_ID_Start # Sk [2] KATAKANA-HIRAGANA VOICED SOUND MARK..KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK + +# Total code points: 6 + +# ================================================ + +00B7 ; Other_ID_Continue # Po MIDDLE DOT +0387 ; Other_ID_Continue # Po GREEK ANO TELEIA +1369..1371 ; Other_ID_Continue # No [9] ETHIOPIC DIGIT ONE..ETHIOPIC DIGIT NINE +19DA ; Other_ID_Continue # No NEW TAI LUE THAM DIGIT ONE + +# Total code points: 12 + +# ================================================ + +0021 ; Sentence_Terminal # Po EXCLAMATION MARK +002E ; Sentence_Terminal # Po FULL STOP +003F ; Sentence_Terminal # Po QUESTION MARK +0589 ; Sentence_Terminal # Po ARMENIAN FULL STOP +061D..061F ; Sentence_Terminal # Po [3] ARABIC END OF TEXT MARK..ARABIC QUESTION MARK +06D4 ; Sentence_Terminal # Po ARABIC FULL STOP +0700..0702 ; Sentence_Terminal # Po [3] SYRIAC END OF PARAGRAPH..SYRIAC SUBLINEAR FULL STOP +07F9 ; Sentence_Terminal # Po NKO EXCLAMATION MARK +0837 ; Sentence_Terminal # Po SAMARITAN PUNCTUATION MELODIC QITSA +0839 ; Sentence_Terminal # Po SAMARITAN PUNCTUATION QITSA +083D..083E ; Sentence_Terminal # Po [2] SAMARITAN PUNCTUATION SOF MASHFAAT..SAMARITAN PUNCTUATION ANNAAU +0964..0965 ; Sentence_Terminal # Po [2] DEVANAGARI DANDA..DEVANAGARI DOUBLE DANDA +104A..104B ; Sentence_Terminal # Po [2] MYANMAR SIGN LITTLE SECTION..MYANMAR SIGN SECTION +1362 ; Sentence_Terminal # Po ETHIOPIC FULL STOP +1367..1368 ; Sentence_Terminal # Po [2] ETHIOPIC QUESTION MARK..ETHIOPIC PARAGRAPH SEPARATOR +166E ; Sentence_Terminal # Po CANADIAN SYLLABICS FULL STOP +1735..1736 ; Sentence_Terminal # Po [2] PHILIPPINE SINGLE PUNCTUATION..PHILIPPINE DOUBLE PUNCTUATION +1803 ; Sentence_Terminal # Po MONGOLIAN FULL STOP +1809 ; Sentence_Terminal # Po MONGOLIAN MANCHU FULL STOP +1944..1945 ; Sentence_Terminal # Po [2] LIMBU EXCLAMATION MARK..LIMBU QUESTION MARK +1AA8..1AAB ; Sentence_Terminal # Po [4] TAI THAM SIGN KAAN..TAI THAM SIGN SATKAANKUU +1B5A..1B5B ; Sentence_Terminal # Po [2] BALINESE PANTI..BALINESE PAMADA +1B5E..1B5F ; Sentence_Terminal # Po [2] BALINESE CARIK SIKI..BALINESE CARIK PAREREN +1B7D..1B7E ; Sentence_Terminal # Po [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG +1C3B..1C3C ; Sentence_Terminal # Po [2] LEPCHA PUNCTUATION TA-ROL..LEPCHA PUNCTUATION NYET THYOOM TA-ROL +1C7E..1C7F ; Sentence_Terminal # Po [2] OL CHIKI PUNCTUATION MUCAAD..OL CHIKI PUNCTUATION DOUBLE MUCAAD +203C..203D ; Sentence_Terminal # Po [2] DOUBLE EXCLAMATION MARK..INTERROBANG +2047..2049 ; Sentence_Terminal # Po [3] DOUBLE QUESTION MARK..EXCLAMATION QUESTION MARK +2E2E ; Sentence_Terminal # Po REVERSED QUESTION MARK +2E3C ; Sentence_Terminal # Po STENOGRAPHIC FULL STOP +2E53..2E54 ; Sentence_Terminal # Po [2] MEDIEVAL EXCLAMATION MARK..MEDIEVAL QUESTION MARK +3002 ; Sentence_Terminal # Po IDEOGRAPHIC FULL STOP +A4FF ; Sentence_Terminal # Po LISU PUNCTUATION FULL STOP +A60E..A60F ; Sentence_Terminal # Po [2] VAI FULL STOP..VAI QUESTION MARK +A6F3 ; Sentence_Terminal # Po BAMUM FULL STOP +A6F7 ; Sentence_Terminal # Po BAMUM QUESTION MARK +A876..A877 ; Sentence_Terminal # Po [2] PHAGS-PA MARK SHAD..PHAGS-PA MARK DOUBLE SHAD +A8CE..A8CF ; Sentence_Terminal # Po [2] SAURASHTRA DANDA..SAURASHTRA DOUBLE DANDA +A92F ; Sentence_Terminal # Po KAYAH LI SIGN SHYA +A9C8..A9C9 ; Sentence_Terminal # Po [2] JAVANESE PADA LINGSA..JAVANESE PADA LUNGSI +AA5D..AA5F ; Sentence_Terminal # Po [3] CHAM PUNCTUATION DANDA..CHAM PUNCTUATION TRIPLE DANDA +AAF0..AAF1 ; Sentence_Terminal # Po [2] MEETEI MAYEK CHEIKHAN..MEETEI MAYEK AHANG KHUDAM +ABEB ; Sentence_Terminal # Po MEETEI MAYEK CHEIKHEI +FE52 ; Sentence_Terminal # Po SMALL FULL STOP +FE56..FE57 ; Sentence_Terminal # Po [2] SMALL QUESTION MARK..SMALL EXCLAMATION MARK +FF01 ; Sentence_Terminal # Po FULLWIDTH EXCLAMATION MARK +FF0E ; Sentence_Terminal # Po FULLWIDTH FULL STOP +FF1F ; Sentence_Terminal # Po FULLWIDTH QUESTION MARK +FF61 ; Sentence_Terminal # Po HALFWIDTH IDEOGRAPHIC FULL STOP +10A56..10A57 ; Sentence_Terminal # Po [2] KHAROSHTHI PUNCTUATION DANDA..KHAROSHTHI PUNCTUATION DOUBLE DANDA +10F55..10F59 ; Sentence_Terminal # Po [5] SOGDIAN PUNCTUATION TWO VERTICAL BARS..SOGDIAN PUNCTUATION HALF CIRCLE WITH DOT +10F86..10F89 ; Sentence_Terminal # Po [4] OLD UYGHUR PUNCTUATION BAR..OLD UYGHUR PUNCTUATION FOUR DOTS +11047..11048 ; Sentence_Terminal # Po [2] BRAHMI DANDA..BRAHMI DOUBLE DANDA +110BE..110C1 ; Sentence_Terminal # Po [4] KAITHI SECTION MARK..KAITHI DOUBLE DANDA +11141..11143 ; Sentence_Terminal # Po [3] CHAKMA DANDA..CHAKMA QUESTION MARK +111C5..111C6 ; Sentence_Terminal # Po [2] SHARADA DANDA..SHARADA DOUBLE DANDA +111CD ; Sentence_Terminal # Po SHARADA SUTRA MARK +111DE..111DF ; Sentence_Terminal # Po [2] SHARADA SECTION MARK-1..SHARADA SECTION MARK-2 +11238..11239 ; Sentence_Terminal # Po [2] KHOJKI DANDA..KHOJKI DOUBLE DANDA +1123B..1123C ; Sentence_Terminal # Po [2] KHOJKI SECTION MARK..KHOJKI DOUBLE SECTION MARK +112A9 ; Sentence_Terminal # Po MULTANI SECTION MARK +1144B..1144C ; Sentence_Terminal # Po [2] NEWA DANDA..NEWA DOUBLE DANDA +115C2..115C3 ; Sentence_Terminal # Po [2] SIDDHAM DANDA..SIDDHAM DOUBLE DANDA +115C9..115D7 ; Sentence_Terminal # Po [15] SIDDHAM END OF TEXT MARK..SIDDHAM SECTION MARK WITH CIRCLES AND FOUR ENCLOSURES +11641..11642 ; Sentence_Terminal # Po [2] MODI DANDA..MODI DOUBLE DANDA +1173C..1173E ; Sentence_Terminal # Po [3] AHOM SIGN SMALL SECTION..AHOM SIGN RULAI +11944 ; Sentence_Terminal # Po DIVES AKURU DOUBLE DANDA +11946 ; Sentence_Terminal # Po DIVES AKURU END OF TEXT MARK +11A42..11A43 ; Sentence_Terminal # Po [2] ZANABAZAR SQUARE MARK SHAD..ZANABAZAR SQUARE MARK DOUBLE SHAD +11A9B..11A9C ; Sentence_Terminal # Po [2] SOYOMBO MARK SHAD..SOYOMBO MARK DOUBLE SHAD +11C41..11C42 ; Sentence_Terminal # Po [2] BHAIKSUKI DANDA..BHAIKSUKI DOUBLE DANDA +11EF7..11EF8 ; Sentence_Terminal # Po [2] MAKASAR PASSIMBANG..MAKASAR END OF SECTION +16A6E..16A6F ; Sentence_Terminal # Po [2] MRO DANDA..MRO DOUBLE DANDA +16AF5 ; Sentence_Terminal # Po BASSA VAH FULL STOP +16B37..16B38 ; Sentence_Terminal # Po [2] PAHAWH HMONG SIGN VOS THOM..PAHAWH HMONG SIGN VOS TSHAB CEEB +16B44 ; Sentence_Terminal # Po PAHAWH HMONG SIGN XAUS +16E98 ; Sentence_Terminal # Po MEDEFAIDRIN FULL STOP +1BC9F ; Sentence_Terminal # Po DUPLOYAN PUNCTUATION CHINOOK FULL STOP +1DA88 ; Sentence_Terminal # Po SIGNWRITING FULL STOP + +# Total code points: 152 + +# ================================================ + +180B..180D ; Variation_Selector # Mn [3] MONGOLIAN FREE VARIATION SELECTOR ONE..MONGOLIAN FREE VARIATION SELECTOR THREE +180F ; Variation_Selector # Mn MONGOLIAN FREE VARIATION SELECTOR FOUR +FE00..FE0F ; Variation_Selector # Mn [16] VARIATION SELECTOR-1..VARIATION SELECTOR-16 +E0100..E01EF ; Variation_Selector # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 + +# Total code points: 260 + +# ================================================ + +0009..000D ; Pattern_White_Space # Cc [5] .. +0020 ; Pattern_White_Space # Zs SPACE +0085 ; Pattern_White_Space # Cc +200E..200F ; Pattern_White_Space # Cf [2] LEFT-TO-RIGHT MARK..RIGHT-TO-LEFT MARK +2028 ; Pattern_White_Space # Zl LINE SEPARATOR +2029 ; Pattern_White_Space # Zp PARAGRAPH SEPARATOR + +# Total code points: 11 + +# ================================================ + +0021..0023 ; Pattern_Syntax # Po [3] EXCLAMATION MARK..NUMBER SIGN +0024 ; Pattern_Syntax # Sc DOLLAR SIGN +0025..0027 ; Pattern_Syntax # Po [3] PERCENT SIGN..APOSTROPHE +0028 ; Pattern_Syntax # Ps LEFT PARENTHESIS +0029 ; Pattern_Syntax # Pe RIGHT PARENTHESIS +002A ; Pattern_Syntax # Po ASTERISK +002B ; Pattern_Syntax # Sm PLUS SIGN +002C ; Pattern_Syntax # Po COMMA +002D ; Pattern_Syntax # Pd HYPHEN-MINUS +002E..002F ; Pattern_Syntax # Po [2] FULL STOP..SOLIDUS +003A..003B ; Pattern_Syntax # Po [2] COLON..SEMICOLON +003C..003E ; Pattern_Syntax # Sm [3] LESS-THAN SIGN..GREATER-THAN SIGN +003F..0040 ; Pattern_Syntax # Po [2] QUESTION MARK..COMMERCIAL AT +005B ; Pattern_Syntax # Ps LEFT SQUARE BRACKET +005C ; Pattern_Syntax # Po REVERSE SOLIDUS +005D ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET +005E ; Pattern_Syntax # Sk CIRCUMFLEX ACCENT +0060 ; Pattern_Syntax # Sk GRAVE ACCENT +007B ; Pattern_Syntax # Ps LEFT CURLY BRACKET +007C ; Pattern_Syntax # Sm VERTICAL LINE +007D ; Pattern_Syntax # Pe RIGHT CURLY BRACKET +007E ; Pattern_Syntax # Sm TILDE +00A1 ; Pattern_Syntax # Po INVERTED EXCLAMATION MARK +00A2..00A5 ; Pattern_Syntax # Sc [4] CENT SIGN..YEN SIGN +00A6 ; Pattern_Syntax # So BROKEN BAR +00A7 ; Pattern_Syntax # Po SECTION SIGN +00A9 ; Pattern_Syntax # So COPYRIGHT SIGN +00AB ; Pattern_Syntax # Pi LEFT-POINTING DOUBLE ANGLE QUOTATION MARK +00AC ; Pattern_Syntax # Sm NOT SIGN +00AE ; Pattern_Syntax # So REGISTERED SIGN +00B0 ; Pattern_Syntax # So DEGREE SIGN +00B1 ; Pattern_Syntax # Sm PLUS-MINUS SIGN +00B6 ; Pattern_Syntax # Po PILCROW SIGN +00BB ; Pattern_Syntax # Pf RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK +00BF ; Pattern_Syntax # Po INVERTED QUESTION MARK +00D7 ; Pattern_Syntax # Sm MULTIPLICATION SIGN +00F7 ; Pattern_Syntax # Sm DIVISION SIGN +2010..2015 ; Pattern_Syntax # Pd [6] HYPHEN..HORIZONTAL BAR +2016..2017 ; Pattern_Syntax # Po [2] DOUBLE VERTICAL LINE..DOUBLE LOW LINE +2018 ; Pattern_Syntax # Pi LEFT SINGLE QUOTATION MARK +2019 ; Pattern_Syntax # Pf RIGHT SINGLE QUOTATION MARK +201A ; Pattern_Syntax # Ps SINGLE LOW-9 QUOTATION MARK +201B..201C ; Pattern_Syntax # Pi [2] SINGLE HIGH-REVERSED-9 QUOTATION MARK..LEFT DOUBLE QUOTATION MARK +201D ; Pattern_Syntax # Pf RIGHT DOUBLE QUOTATION MARK +201E ; Pattern_Syntax # Ps DOUBLE LOW-9 QUOTATION MARK +201F ; Pattern_Syntax # Pi DOUBLE HIGH-REVERSED-9 QUOTATION MARK +2020..2027 ; Pattern_Syntax # Po [8] DAGGER..HYPHENATION POINT +2030..2038 ; Pattern_Syntax # Po [9] PER MILLE SIGN..CARET +2039 ; Pattern_Syntax # Pi SINGLE LEFT-POINTING ANGLE QUOTATION MARK +203A ; Pattern_Syntax # Pf SINGLE RIGHT-POINTING ANGLE QUOTATION MARK +203B..203E ; Pattern_Syntax # Po [4] REFERENCE MARK..OVERLINE +2041..2043 ; Pattern_Syntax # Po [3] CARET INSERTION POINT..HYPHEN BULLET +2044 ; Pattern_Syntax # Sm FRACTION SLASH +2045 ; Pattern_Syntax # Ps LEFT SQUARE BRACKET WITH QUILL +2046 ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET WITH QUILL +2047..2051 ; Pattern_Syntax # Po [11] DOUBLE QUESTION MARK..TWO ASTERISKS ALIGNED VERTICALLY +2052 ; Pattern_Syntax # Sm COMMERCIAL MINUS SIGN +2053 ; Pattern_Syntax # Po SWUNG DASH +2055..205E ; Pattern_Syntax # Po [10] FLOWER PUNCTUATION MARK..VERTICAL FOUR DOTS +2190..2194 ; Pattern_Syntax # Sm [5] LEFTWARDS ARROW..LEFT RIGHT ARROW +2195..2199 ; Pattern_Syntax # So [5] UP DOWN ARROW..SOUTH WEST ARROW +219A..219B ; Pattern_Syntax # Sm [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE +219C..219F ; Pattern_Syntax # So [4] LEFTWARDS WAVE ARROW..UPWARDS TWO HEADED ARROW +21A0 ; Pattern_Syntax # Sm RIGHTWARDS TWO HEADED ARROW +21A1..21A2 ; Pattern_Syntax # So [2] DOWNWARDS TWO HEADED ARROW..LEFTWARDS ARROW WITH TAIL +21A3 ; Pattern_Syntax # Sm RIGHTWARDS ARROW WITH TAIL +21A4..21A5 ; Pattern_Syntax # So [2] LEFTWARDS ARROW FROM BAR..UPWARDS ARROW FROM BAR +21A6 ; Pattern_Syntax # Sm RIGHTWARDS ARROW FROM BAR +21A7..21AD ; Pattern_Syntax # So [7] DOWNWARDS ARROW FROM BAR..LEFT RIGHT WAVE ARROW +21AE ; Pattern_Syntax # Sm LEFT RIGHT ARROW WITH STROKE +21AF..21CD ; Pattern_Syntax # So [31] DOWNWARDS ZIGZAG ARROW..LEFTWARDS DOUBLE ARROW WITH STROKE +21CE..21CF ; Pattern_Syntax # Sm [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE +21D0..21D1 ; Pattern_Syntax # So [2] LEFTWARDS DOUBLE ARROW..UPWARDS DOUBLE ARROW +21D2 ; Pattern_Syntax # Sm RIGHTWARDS DOUBLE ARROW +21D3 ; Pattern_Syntax # So DOWNWARDS DOUBLE ARROW +21D4 ; Pattern_Syntax # Sm LEFT RIGHT DOUBLE ARROW +21D5..21F3 ; Pattern_Syntax # So [31] UP DOWN DOUBLE ARROW..UP DOWN WHITE ARROW +21F4..22FF ; Pattern_Syntax # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP +2300..2307 ; Pattern_Syntax # So [8] DIAMETER SIGN..WAVY LINE +2308 ; Pattern_Syntax # Ps LEFT CEILING +2309 ; Pattern_Syntax # Pe RIGHT CEILING +230A ; Pattern_Syntax # Ps LEFT FLOOR +230B ; Pattern_Syntax # Pe RIGHT FLOOR +230C..231F ; Pattern_Syntax # So [20] BOTTOM RIGHT CROP..BOTTOM RIGHT CORNER +2320..2321 ; Pattern_Syntax # Sm [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL +2322..2328 ; Pattern_Syntax # So [7] FROWN..KEYBOARD +2329 ; Pattern_Syntax # Ps LEFT-POINTING ANGLE BRACKET +232A ; Pattern_Syntax # Pe RIGHT-POINTING ANGLE BRACKET +232B..237B ; Pattern_Syntax # So [81] ERASE TO THE LEFT..NOT CHECK MARK +237C ; Pattern_Syntax # Sm RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +237D..239A ; Pattern_Syntax # So [30] SHOULDERED OPEN BOX..CLEAR SCREEN SYMBOL +239B..23B3 ; Pattern_Syntax # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM +23B4..23DB ; Pattern_Syntax # So [40] TOP SQUARE BRACKET..FUSE +23DC..23E1 ; Pattern_Syntax # Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET +23E2..2426 ; Pattern_Syntax # So [69] WHITE TRAPEZIUM..SYMBOL FOR SUBSTITUTE FORM TWO +2427..243F ; Pattern_Syntax # Cn [25] .. +2440..244A ; Pattern_Syntax # So [11] OCR HOOK..OCR DOUBLE BACKSLASH +244B..245F ; Pattern_Syntax # Cn [21] .. +2500..25B6 ; Pattern_Syntax # So [183] BOX DRAWINGS LIGHT HORIZONTAL..BLACK RIGHT-POINTING TRIANGLE +25B7 ; Pattern_Syntax # Sm WHITE RIGHT-POINTING TRIANGLE +25B8..25C0 ; Pattern_Syntax # So [9] BLACK RIGHT-POINTING SMALL TRIANGLE..BLACK LEFT-POINTING TRIANGLE +25C1 ; Pattern_Syntax # Sm WHITE LEFT-POINTING TRIANGLE +25C2..25F7 ; Pattern_Syntax # So [54] BLACK LEFT-POINTING SMALL TRIANGLE..WHITE CIRCLE WITH UPPER RIGHT QUADRANT +25F8..25FF ; Pattern_Syntax # Sm [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE +2600..266E ; Pattern_Syntax # So [111] BLACK SUN WITH RAYS..MUSIC NATURAL SIGN +266F ; Pattern_Syntax # Sm MUSIC SHARP SIGN +2670..2767 ; Pattern_Syntax # So [248] WEST SYRIAC CROSS..ROTATED FLORAL HEART BULLET +2768 ; Pattern_Syntax # Ps MEDIUM LEFT PARENTHESIS ORNAMENT +2769 ; Pattern_Syntax # Pe MEDIUM RIGHT PARENTHESIS ORNAMENT +276A ; Pattern_Syntax # Ps MEDIUM FLATTENED LEFT PARENTHESIS ORNAMENT +276B ; Pattern_Syntax # Pe MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT +276C ; Pattern_Syntax # Ps MEDIUM LEFT-POINTING ANGLE BRACKET ORNAMENT +276D ; Pattern_Syntax # Pe MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT +276E ; Pattern_Syntax # Ps HEAVY LEFT-POINTING ANGLE QUOTATION MARK ORNAMENT +276F ; Pattern_Syntax # Pe HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT +2770 ; Pattern_Syntax # Ps HEAVY LEFT-POINTING ANGLE BRACKET ORNAMENT +2771 ; Pattern_Syntax # Pe HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT +2772 ; Pattern_Syntax # Ps LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT +2773 ; Pattern_Syntax # Pe LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT +2774 ; Pattern_Syntax # Ps MEDIUM LEFT CURLY BRACKET ORNAMENT +2775 ; Pattern_Syntax # Pe MEDIUM RIGHT CURLY BRACKET ORNAMENT +2794..27BF ; Pattern_Syntax # So [44] HEAVY WIDE-HEADED RIGHTWARDS ARROW..DOUBLE CURLY LOOP +27C0..27C4 ; Pattern_Syntax # Sm [5] THREE DIMENSIONAL ANGLE..OPEN SUPERSET +27C5 ; Pattern_Syntax # Ps LEFT S-SHAPED BAG DELIMITER +27C6 ; Pattern_Syntax # Pe RIGHT S-SHAPED BAG DELIMITER +27C7..27E5 ; Pattern_Syntax # Sm [31] OR WITH DOT INSIDE..WHITE SQUARE WITH RIGHTWARDS TICK +27E6 ; Pattern_Syntax # Ps MATHEMATICAL LEFT WHITE SQUARE BRACKET +27E7 ; Pattern_Syntax # Pe MATHEMATICAL RIGHT WHITE SQUARE BRACKET +27E8 ; Pattern_Syntax # Ps MATHEMATICAL LEFT ANGLE BRACKET +27E9 ; Pattern_Syntax # Pe MATHEMATICAL RIGHT ANGLE BRACKET +27EA ; Pattern_Syntax # Ps MATHEMATICAL LEFT DOUBLE ANGLE BRACKET +27EB ; Pattern_Syntax # Pe MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET +27EC ; Pattern_Syntax # Ps MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET +27ED ; Pattern_Syntax # Pe MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET +27EE ; Pattern_Syntax # Ps MATHEMATICAL LEFT FLATTENED PARENTHESIS +27EF ; Pattern_Syntax # Pe MATHEMATICAL RIGHT FLATTENED PARENTHESIS +27F0..27FF ; Pattern_Syntax # Sm [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2800..28FF ; Pattern_Syntax # So [256] BRAILLE PATTERN BLANK..BRAILLE PATTERN DOTS-12345678 +2900..2982 ; Pattern_Syntax # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON +2983 ; Pattern_Syntax # Ps LEFT WHITE CURLY BRACKET +2984 ; Pattern_Syntax # Pe RIGHT WHITE CURLY BRACKET +2985 ; Pattern_Syntax # Ps LEFT WHITE PARENTHESIS +2986 ; Pattern_Syntax # Pe RIGHT WHITE PARENTHESIS +2987 ; Pattern_Syntax # Ps Z NOTATION LEFT IMAGE BRACKET +2988 ; Pattern_Syntax # Pe Z NOTATION RIGHT IMAGE BRACKET +2989 ; Pattern_Syntax # Ps Z NOTATION LEFT BINDING BRACKET +298A ; Pattern_Syntax # Pe Z NOTATION RIGHT BINDING BRACKET +298B ; Pattern_Syntax # Ps LEFT SQUARE BRACKET WITH UNDERBAR +298C ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET WITH UNDERBAR +298D ; Pattern_Syntax # Ps LEFT SQUARE BRACKET WITH TICK IN TOP CORNER +298E ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +298F ; Pattern_Syntax # Ps LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER +2990 ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER +2991 ; Pattern_Syntax # Ps LEFT ANGLE BRACKET WITH DOT +2992 ; Pattern_Syntax # Pe RIGHT ANGLE BRACKET WITH DOT +2993 ; Pattern_Syntax # Ps LEFT ARC LESS-THAN BRACKET +2994 ; Pattern_Syntax # Pe RIGHT ARC GREATER-THAN BRACKET +2995 ; Pattern_Syntax # Ps DOUBLE LEFT ARC GREATER-THAN BRACKET +2996 ; Pattern_Syntax # Pe DOUBLE RIGHT ARC LESS-THAN BRACKET +2997 ; Pattern_Syntax # Ps LEFT BLACK TORTOISE SHELL BRACKET +2998 ; Pattern_Syntax # Pe RIGHT BLACK TORTOISE SHELL BRACKET +2999..29D7 ; Pattern_Syntax # Sm [63] DOTTED FENCE..BLACK HOURGLASS +29D8 ; Pattern_Syntax # Ps LEFT WIGGLY FENCE +29D9 ; Pattern_Syntax # Pe RIGHT WIGGLY FENCE +29DA ; Pattern_Syntax # Ps LEFT DOUBLE WIGGLY FENCE +29DB ; Pattern_Syntax # Pe RIGHT DOUBLE WIGGLY FENCE +29DC..29FB ; Pattern_Syntax # Sm [32] INCOMPLETE INFINITY..TRIPLE PLUS +29FC ; Pattern_Syntax # Ps LEFT-POINTING CURVED ANGLE BRACKET +29FD ; Pattern_Syntax # Pe RIGHT-POINTING CURVED ANGLE BRACKET +29FE..2AFF ; Pattern_Syntax # Sm [258] TINY..N-ARY WHITE VERTICAL BAR +2B00..2B2F ; Pattern_Syntax # So [48] NORTH EAST WHITE ARROW..WHITE VERTICAL ELLIPSE +2B30..2B44 ; Pattern_Syntax # Sm [21] LEFT ARROW WITH SMALL CIRCLE..RIGHTWARDS ARROW THROUGH SUPERSET +2B45..2B46 ; Pattern_Syntax # So [2] LEFTWARDS QUADRUPLE ARROW..RIGHTWARDS QUADRUPLE ARROW +2B47..2B4C ; Pattern_Syntax # Sm [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR +2B4D..2B73 ; Pattern_Syntax # So [39] DOWNWARDS TRIANGLE-HEADED ZIGZAG ARROW..DOWNWARDS TRIANGLE-HEADED ARROW TO BAR +2B74..2B75 ; Pattern_Syntax # Cn [2] .. +2B76..2B95 ; Pattern_Syntax # So [32] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..RIGHTWARDS BLACK ARROW +2B96 ; Pattern_Syntax # Cn +2B97..2BFF ; Pattern_Syntax # So [105] SYMBOL FOR TYPE A ELECTRONICS..HELLSCHREIBER PAUSE SYMBOL +2E00..2E01 ; Pattern_Syntax # Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER +2E02 ; Pattern_Syntax # Pi LEFT SUBSTITUTION BRACKET +2E03 ; Pattern_Syntax # Pf RIGHT SUBSTITUTION BRACKET +2E04 ; Pattern_Syntax # Pi LEFT DOTTED SUBSTITUTION BRACKET +2E05 ; Pattern_Syntax # Pf RIGHT DOTTED SUBSTITUTION BRACKET +2E06..2E08 ; Pattern_Syntax # Po [3] RAISED INTERPOLATION MARKER..DOTTED TRANSPOSITION MARKER +2E09 ; Pattern_Syntax # Pi LEFT TRANSPOSITION BRACKET +2E0A ; Pattern_Syntax # Pf RIGHT TRANSPOSITION BRACKET +2E0B ; Pattern_Syntax # Po RAISED SQUARE +2E0C ; Pattern_Syntax # Pi LEFT RAISED OMISSION BRACKET +2E0D ; Pattern_Syntax # Pf RIGHT RAISED OMISSION BRACKET +2E0E..2E16 ; Pattern_Syntax # Po [9] EDITORIAL CORONIS..DOTTED RIGHT-POINTING ANGLE +2E17 ; Pattern_Syntax # Pd DOUBLE OBLIQUE HYPHEN +2E18..2E19 ; Pattern_Syntax # Po [2] INVERTED INTERROBANG..PALM BRANCH +2E1A ; Pattern_Syntax # Pd HYPHEN WITH DIAERESIS +2E1B ; Pattern_Syntax # Po TILDE WITH RING ABOVE +2E1C ; Pattern_Syntax # Pi LEFT LOW PARAPHRASE BRACKET +2E1D ; Pattern_Syntax # Pf RIGHT LOW PARAPHRASE BRACKET +2E1E..2E1F ; Pattern_Syntax # Po [2] TILDE WITH DOT ABOVE..TILDE WITH DOT BELOW +2E20 ; Pattern_Syntax # Pi LEFT VERTICAL BAR WITH QUILL +2E21 ; Pattern_Syntax # Pf RIGHT VERTICAL BAR WITH QUILL +2E22 ; Pattern_Syntax # Ps TOP LEFT HALF BRACKET +2E23 ; Pattern_Syntax # Pe TOP RIGHT HALF BRACKET +2E24 ; Pattern_Syntax # Ps BOTTOM LEFT HALF BRACKET +2E25 ; Pattern_Syntax # Pe BOTTOM RIGHT HALF BRACKET +2E26 ; Pattern_Syntax # Ps LEFT SIDEWAYS U BRACKET +2E27 ; Pattern_Syntax # Pe RIGHT SIDEWAYS U BRACKET +2E28 ; Pattern_Syntax # Ps LEFT DOUBLE PARENTHESIS +2E29 ; Pattern_Syntax # Pe RIGHT DOUBLE PARENTHESIS +2E2A..2E2E ; Pattern_Syntax # Po [5] TWO DOTS OVER ONE DOT PUNCTUATION..REVERSED QUESTION MARK +2E2F ; Pattern_Syntax # Lm VERTICAL TILDE +2E30..2E39 ; Pattern_Syntax # Po [10] RING POINT..TOP HALF SECTION SIGN +2E3A..2E3B ; Pattern_Syntax # Pd [2] TWO-EM DASH..THREE-EM DASH +2E3C..2E3F ; Pattern_Syntax # Po [4] STENOGRAPHIC FULL STOP..CAPITULUM +2E40 ; Pattern_Syntax # Pd DOUBLE HYPHEN +2E41 ; Pattern_Syntax # Po REVERSED COMMA +2E42 ; Pattern_Syntax # Ps DOUBLE LOW-REVERSED-9 QUOTATION MARK +2E43..2E4F ; Pattern_Syntax # Po [13] DASH WITH LEFT UPTURN..CORNISH VERSE DIVIDER +2E50..2E51 ; Pattern_Syntax # So [2] CROSS PATTY WITH RIGHT CROSSBAR..CROSS PATTY WITH LEFT CROSSBAR +2E52..2E54 ; Pattern_Syntax # Po [3] TIRONIAN SIGN CAPITAL ET..MEDIEVAL QUESTION MARK +2E55 ; Pattern_Syntax # Ps LEFT SQUARE BRACKET WITH STROKE +2E56 ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET WITH STROKE +2E57 ; Pattern_Syntax # Ps LEFT SQUARE BRACKET WITH DOUBLE STROKE +2E58 ; Pattern_Syntax # Pe RIGHT SQUARE BRACKET WITH DOUBLE STROKE +2E59 ; Pattern_Syntax # Ps TOP HALF LEFT PARENTHESIS +2E5A ; Pattern_Syntax # Pe TOP HALF RIGHT PARENTHESIS +2E5B ; Pattern_Syntax # Ps BOTTOM HALF LEFT PARENTHESIS +2E5C ; Pattern_Syntax # Pe BOTTOM HALF RIGHT PARENTHESIS +2E5D ; Pattern_Syntax # Pd OBLIQUE HYPHEN +2E5E..2E7F ; Pattern_Syntax # Cn [34] .. +3001..3003 ; Pattern_Syntax # Po [3] IDEOGRAPHIC COMMA..DITTO MARK +3008 ; Pattern_Syntax # Ps LEFT ANGLE BRACKET +3009 ; Pattern_Syntax # Pe RIGHT ANGLE BRACKET +300A ; Pattern_Syntax # Ps LEFT DOUBLE ANGLE BRACKET +300B ; Pattern_Syntax # Pe RIGHT DOUBLE ANGLE BRACKET +300C ; Pattern_Syntax # Ps LEFT CORNER BRACKET +300D ; Pattern_Syntax # Pe RIGHT CORNER BRACKET +300E ; Pattern_Syntax # Ps LEFT WHITE CORNER BRACKET +300F ; Pattern_Syntax # Pe RIGHT WHITE CORNER BRACKET +3010 ; Pattern_Syntax # Ps LEFT BLACK LENTICULAR BRACKET +3011 ; Pattern_Syntax # Pe RIGHT BLACK LENTICULAR BRACKET +3012..3013 ; Pattern_Syntax # So [2] POSTAL MARK..GETA MARK +3014 ; Pattern_Syntax # Ps LEFT TORTOISE SHELL BRACKET +3015 ; Pattern_Syntax # Pe RIGHT TORTOISE SHELL BRACKET +3016 ; Pattern_Syntax # Ps LEFT WHITE LENTICULAR BRACKET +3017 ; Pattern_Syntax # Pe RIGHT WHITE LENTICULAR BRACKET +3018 ; Pattern_Syntax # Ps LEFT WHITE TORTOISE SHELL BRACKET +3019 ; Pattern_Syntax # Pe RIGHT WHITE TORTOISE SHELL BRACKET +301A ; Pattern_Syntax # Ps LEFT WHITE SQUARE BRACKET +301B ; Pattern_Syntax # Pe RIGHT WHITE SQUARE BRACKET +301C ; Pattern_Syntax # Pd WAVE DASH +301D ; Pattern_Syntax # Ps REVERSED DOUBLE PRIME QUOTATION MARK +301E..301F ; Pattern_Syntax # Pe [2] DOUBLE PRIME QUOTATION MARK..LOW DOUBLE PRIME QUOTATION MARK +3020 ; Pattern_Syntax # So POSTAL MARK FACE +3030 ; Pattern_Syntax # Pd WAVY DASH +FD3E ; Pattern_Syntax # Pe ORNATE LEFT PARENTHESIS +FD3F ; Pattern_Syntax # Ps ORNATE RIGHT PARENTHESIS +FE45..FE46 ; Pattern_Syntax # Po [2] SESAME DOT..WHITE SESAME DOT + +# Total code points: 2760 + +# ================================================ + +0600..0605 ; Prepended_Concatenation_Mark # Cf [6] ARABIC NUMBER SIGN..ARABIC NUMBER MARK ABOVE +06DD ; Prepended_Concatenation_Mark # Cf ARABIC END OF AYAH +070F ; Prepended_Concatenation_Mark # Cf SYRIAC ABBREVIATION MARK +0890..0891 ; Prepended_Concatenation_Mark # Cf [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE +08E2 ; Prepended_Concatenation_Mark # Cf ARABIC DISPUTED END OF AYAH +110BD ; Prepended_Concatenation_Mark # Cf KAITHI NUMBER SIGN +110CD ; Prepended_Concatenation_Mark # Cf KAITHI NUMBER SIGN ABOVE + +# Total code points: 13 + +# ================================================ + +1F1E6..1F1FF ; Regional_Indicator # So [26] REGIONAL INDICATOR SYMBOL LETTER A..REGIONAL INDICATOR SYMBOL LETTER Z + +# Total code points: 26 + +# EOF diff --git a/lib/elixir/unicode/PropertyValueAliases.txt b/lib/elixir/unicode/PropertyValueAliases.txt new file mode 100644 index 00000000000..f0cb26bdab8 --- /dev/null +++ b/lib/elixir/unicode/PropertyValueAliases.txt @@ -0,0 +1,1615 @@ +# PropertyValueAliases-14.0.0.txt +# Date: 2021-05-10, 21:08:53 GMT +# © 2021 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use, see http://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see http://www.unicode.org/reports/tr44/ +# +# This file contains aliases for property values used in the UCD. +# These names can be used for XML formats of UCD data, for regular-expression +# property tests, and other programmatic textual descriptions of Unicode data. +# +# The names may be translated in appropriate environments, and additional +# aliases may be useful. +# +# FORMAT +# +# Each line describes a property value name. +# This consists of three or more fields, separated by semicolons. +# +# First Field: The first field describes the property for which that +# property value name is used. +# +# Second Field: The second field is the short name for the property value. +# It is typically an abbreviation, but in a number of cases it is simply +# a duplicate of the "long name" in the third field. +# +# Third Field: The third field is the long name for the property value, +# typically the formal name used in documentation about the property value. +# +# In the case of Canonical_Combining_Class (ccc), there are 4 fields: +# The second field is numeric, the third is the short name, and the fourth is the long name. +# +# The above are the preferred aliases. Other aliases may be listed in additional fields. +# +# Loose matching should be applied to all property names and property values, with +# the exception of String Property values. With loose matching of property names and +# values, the case distinctions, whitespace, hyphens, and '_' are ignored. +# For Numeric Property values, numeric equivalence is applied: thus "01.00" +# is equivalent to "1". +# +# NOTE: Property value names are NOT unique across properties. For example: +# +# AL means Arabic Letter for the Bidi_Class property, and +# AL means Above_Left for the Canonical_Combining_Class property, and +# AL means Alphabetic for the Line_Break property. +# +# In addition, some property names may be the same as some property value names. +# For example: +# +# sc means the Script property, and +# Sc means the General_Category property value Currency_Symbol (Sc) +# +# The combination of property value and property name is, however, unique. +# +# For more information, see UAX #44, Unicode Character Database, and +# UTS #18, Unicode Regular Expressions. +# ================================================ + + +# ASCII_Hex_Digit (AHex) + +AHex; N ; No ; F ; False +AHex; Y ; Yes ; T ; True + +# Age (age) + +age; 1.1 ; V1_1 +age; 2.0 ; V2_0 +age; 2.1 ; V2_1 +age; 3.0 ; V3_0 +age; 3.1 ; V3_1 +age; 3.2 ; V3_2 +age; 4.0 ; V4_0 +age; 4.1 ; V4_1 +age; 5.0 ; V5_0 +age; 5.1 ; V5_1 +age; 5.2 ; V5_2 +age; 6.0 ; V6_0 +age; 6.1 ; V6_1 +age; 6.2 ; V6_2 +age; 6.3 ; V6_3 +age; 7.0 ; V7_0 +age; 8.0 ; V8_0 +age; 9.0 ; V9_0 +age; 10.0 ; V10_0 +age; 11.0 ; V11_0 +age; 12.0 ; V12_0 +age; 12.1 ; V12_1 +age; 13.0 ; V13_0 +age; 14.0 ; V14_0 +age; NA ; Unassigned + +# Alphabetic (Alpha) + +Alpha; N ; No ; F ; False +Alpha; Y ; Yes ; T ; True + +# Bidi_Class (bc) + +bc ; AL ; Arabic_Letter +bc ; AN ; Arabic_Number +bc ; B ; Paragraph_Separator +bc ; BN ; Boundary_Neutral +bc ; CS ; Common_Separator +bc ; EN ; European_Number +bc ; ES ; European_Separator +bc ; ET ; European_Terminator +bc ; FSI ; First_Strong_Isolate +bc ; L ; Left_To_Right +bc ; LRE ; Left_To_Right_Embedding +bc ; LRI ; Left_To_Right_Isolate +bc ; LRO ; Left_To_Right_Override +bc ; NSM ; Nonspacing_Mark +bc ; ON ; Other_Neutral +bc ; PDF ; Pop_Directional_Format +bc ; PDI ; Pop_Directional_Isolate +bc ; R ; Right_To_Left +bc ; RLE ; Right_To_Left_Embedding +bc ; RLI ; Right_To_Left_Isolate +bc ; RLO ; Right_To_Left_Override +bc ; S ; Segment_Separator +bc ; WS ; White_Space + +# Bidi_Control (Bidi_C) + +Bidi_C; N ; No ; F ; False +Bidi_C; Y ; Yes ; T ; True + +# Bidi_Mirrored (Bidi_M) + +Bidi_M; N ; No ; F ; False +Bidi_M; Y ; Yes ; T ; True + +# Bidi_Mirroring_Glyph (bmg) + +# @missing: 0000..10FFFF; Bidi_Mirroring_Glyph; + +# Bidi_Paired_Bracket (bpb) + +# @missing: 0000..10FFFF; Bidi_Paired_Bracket; + +# Bidi_Paired_Bracket_Type (bpt) + +bpt; c ; Close +bpt; n ; None +bpt; o ; Open +# @missing: 0000..10FFFF; Bidi_Paired_Bracket_Type; n + +# Block (blk) + +blk; Adlam ; Adlam +blk; Aegean_Numbers ; Aegean_Numbers +blk; Ahom ; Ahom +blk; Alchemical ; Alchemical_Symbols +blk; Alphabetic_PF ; Alphabetic_Presentation_Forms +blk; Anatolian_Hieroglyphs ; Anatolian_Hieroglyphs +blk; Ancient_Greek_Music ; Ancient_Greek_Musical_Notation +blk; Ancient_Greek_Numbers ; Ancient_Greek_Numbers +blk; Ancient_Symbols ; Ancient_Symbols +blk; Arabic ; Arabic +blk; Arabic_Ext_A ; Arabic_Extended_A +blk; Arabic_Ext_B ; Arabic_Extended_B +blk; Arabic_Math ; Arabic_Mathematical_Alphabetic_Symbols +blk; Arabic_PF_A ; Arabic_Presentation_Forms_A ; Arabic_Presentation_Forms-A +blk; Arabic_PF_B ; Arabic_Presentation_Forms_B +blk; Arabic_Sup ; Arabic_Supplement +blk; Armenian ; Armenian +blk; Arrows ; Arrows +blk; ASCII ; Basic_Latin +blk; Avestan ; Avestan +blk; Balinese ; Balinese +blk; Bamum ; Bamum +blk; Bamum_Sup ; Bamum_Supplement +blk; Bassa_Vah ; Bassa_Vah +blk; Batak ; Batak +blk; Bengali ; Bengali +blk; Bhaiksuki ; Bhaiksuki +blk; Block_Elements ; Block_Elements +blk; Bopomofo ; Bopomofo +blk; Bopomofo_Ext ; Bopomofo_Extended +blk; Box_Drawing ; Box_Drawing +blk; Brahmi ; Brahmi +blk; Braille ; Braille_Patterns +blk; Buginese ; Buginese +blk; Buhid ; Buhid +blk; Byzantine_Music ; Byzantine_Musical_Symbols +blk; Carian ; Carian +blk; Caucasian_Albanian ; Caucasian_Albanian +blk; Chakma ; Chakma +blk; Cham ; Cham +blk; Cherokee ; Cherokee +blk; Cherokee_Sup ; Cherokee_Supplement +blk; Chess_Symbols ; Chess_Symbols +blk; Chorasmian ; Chorasmian +blk; CJK ; CJK_Unified_Ideographs +blk; CJK_Compat ; CJK_Compatibility +blk; CJK_Compat_Forms ; CJK_Compatibility_Forms +blk; CJK_Compat_Ideographs ; CJK_Compatibility_Ideographs +blk; CJK_Compat_Ideographs_Sup ; CJK_Compatibility_Ideographs_Supplement +blk; CJK_Ext_A ; CJK_Unified_Ideographs_Extension_A +blk; CJK_Ext_B ; CJK_Unified_Ideographs_Extension_B +blk; CJK_Ext_C ; CJK_Unified_Ideographs_Extension_C +blk; CJK_Ext_D ; CJK_Unified_Ideographs_Extension_D +blk; CJK_Ext_E ; CJK_Unified_Ideographs_Extension_E +blk; CJK_Ext_F ; CJK_Unified_Ideographs_Extension_F +blk; CJK_Ext_G ; CJK_Unified_Ideographs_Extension_G +blk; CJK_Radicals_Sup ; CJK_Radicals_Supplement +blk; CJK_Strokes ; CJK_Strokes +blk; CJK_Symbols ; CJK_Symbols_And_Punctuation +blk; Compat_Jamo ; Hangul_Compatibility_Jamo +blk; Control_Pictures ; Control_Pictures +blk; Coptic ; Coptic +blk; Coptic_Epact_Numbers ; Coptic_Epact_Numbers +blk; Counting_Rod ; Counting_Rod_Numerals +blk; Cuneiform ; Cuneiform +blk; Cuneiform_Numbers ; Cuneiform_Numbers_And_Punctuation +blk; Currency_Symbols ; Currency_Symbols +blk; Cypriot_Syllabary ; Cypriot_Syllabary +blk; Cypro_Minoan ; Cypro_Minoan +blk; Cyrillic ; Cyrillic +blk; Cyrillic_Ext_A ; Cyrillic_Extended_A +blk; Cyrillic_Ext_B ; Cyrillic_Extended_B +blk; Cyrillic_Ext_C ; Cyrillic_Extended_C +blk; Cyrillic_Sup ; Cyrillic_Supplement ; Cyrillic_Supplementary +blk; Deseret ; Deseret +blk; Devanagari ; Devanagari +blk; Devanagari_Ext ; Devanagari_Extended +blk; Diacriticals ; Combining_Diacritical_Marks +blk; Diacriticals_Ext ; Combining_Diacritical_Marks_Extended +blk; Diacriticals_For_Symbols ; Combining_Diacritical_Marks_For_Symbols; Combining_Marks_For_Symbols +blk; Diacriticals_Sup ; Combining_Diacritical_Marks_Supplement +blk; Dingbats ; Dingbats +blk; Dives_Akuru ; Dives_Akuru +blk; Dogra ; Dogra +blk; Domino ; Domino_Tiles +blk; Duployan ; Duployan +blk; Early_Dynastic_Cuneiform ; Early_Dynastic_Cuneiform +blk; Egyptian_Hieroglyph_Format_Controls; Egyptian_Hieroglyph_Format_Controls +blk; Egyptian_Hieroglyphs ; Egyptian_Hieroglyphs +blk; Elbasan ; Elbasan +blk; Elymaic ; Elymaic +blk; Emoticons ; Emoticons +blk; Enclosed_Alphanum ; Enclosed_Alphanumerics +blk; Enclosed_Alphanum_Sup ; Enclosed_Alphanumeric_Supplement +blk; Enclosed_CJK ; Enclosed_CJK_Letters_And_Months +blk; Enclosed_Ideographic_Sup ; Enclosed_Ideographic_Supplement +blk; Ethiopic ; Ethiopic +blk; Ethiopic_Ext ; Ethiopic_Extended +blk; Ethiopic_Ext_A ; Ethiopic_Extended_A +blk; Ethiopic_Ext_B ; Ethiopic_Extended_B +blk; Ethiopic_Sup ; Ethiopic_Supplement +blk; Geometric_Shapes ; Geometric_Shapes +blk; Geometric_Shapes_Ext ; Geometric_Shapes_Extended +blk; Georgian ; Georgian +blk; Georgian_Ext ; Georgian_Extended +blk; Georgian_Sup ; Georgian_Supplement +blk; Glagolitic ; Glagolitic +blk; Glagolitic_Sup ; Glagolitic_Supplement +blk; Gothic ; Gothic +blk; Grantha ; Grantha +blk; Greek ; Greek_And_Coptic +blk; Greek_Ext ; Greek_Extended +blk; Gujarati ; Gujarati +blk; Gunjala_Gondi ; Gunjala_Gondi +blk; Gurmukhi ; Gurmukhi +blk; Half_And_Full_Forms ; Halfwidth_And_Fullwidth_Forms +blk; Half_Marks ; Combining_Half_Marks +blk; Hangul ; Hangul_Syllables +blk; Hanifi_Rohingya ; Hanifi_Rohingya +blk; Hanunoo ; Hanunoo +blk; Hatran ; Hatran +blk; Hebrew ; Hebrew +blk; High_PU_Surrogates ; High_Private_Use_Surrogates +blk; High_Surrogates ; High_Surrogates +blk; Hiragana ; Hiragana +blk; IDC ; Ideographic_Description_Characters +blk; Ideographic_Symbols ; Ideographic_Symbols_And_Punctuation +blk; Imperial_Aramaic ; Imperial_Aramaic +blk; Indic_Number_Forms ; Common_Indic_Number_Forms +blk; Indic_Siyaq_Numbers ; Indic_Siyaq_Numbers +blk; Inscriptional_Pahlavi ; Inscriptional_Pahlavi +blk; Inscriptional_Parthian ; Inscriptional_Parthian +blk; IPA_Ext ; IPA_Extensions +blk; Jamo ; Hangul_Jamo +blk; Jamo_Ext_A ; Hangul_Jamo_Extended_A +blk; Jamo_Ext_B ; Hangul_Jamo_Extended_B +blk; Javanese ; Javanese +blk; Kaithi ; Kaithi +blk; Kana_Ext_A ; Kana_Extended_A +blk; Kana_Ext_B ; Kana_Extended_B +blk; Kana_Sup ; Kana_Supplement +blk; Kanbun ; Kanbun +blk; Kangxi ; Kangxi_Radicals +blk; Kannada ; Kannada +blk; Katakana ; Katakana +blk; Katakana_Ext ; Katakana_Phonetic_Extensions +blk; Kayah_Li ; Kayah_Li +blk; Kharoshthi ; Kharoshthi +blk; Khitan_Small_Script ; Khitan_Small_Script +blk; Khmer ; Khmer +blk; Khmer_Symbols ; Khmer_Symbols +blk; Khojki ; Khojki +blk; Khudawadi ; Khudawadi +blk; Lao ; Lao +blk; Latin_1_Sup ; Latin_1_Supplement ; Latin_1 +blk; Latin_Ext_A ; Latin_Extended_A +blk; Latin_Ext_Additional ; Latin_Extended_Additional +blk; Latin_Ext_B ; Latin_Extended_B +blk; Latin_Ext_C ; Latin_Extended_C +blk; Latin_Ext_D ; Latin_Extended_D +blk; Latin_Ext_E ; Latin_Extended_E +blk; Latin_Ext_F ; Latin_Extended_F +blk; Latin_Ext_G ; Latin_Extended_G +blk; Lepcha ; Lepcha +blk; Letterlike_Symbols ; Letterlike_Symbols +blk; Limbu ; Limbu +blk; Linear_A ; Linear_A +blk; Linear_B_Ideograms ; Linear_B_Ideograms +blk; Linear_B_Syllabary ; Linear_B_Syllabary +blk; Lisu ; Lisu +blk; Lisu_Sup ; Lisu_Supplement +blk; Low_Surrogates ; Low_Surrogates +blk; Lycian ; Lycian +blk; Lydian ; Lydian +blk; Mahajani ; Mahajani +blk; Mahjong ; Mahjong_Tiles +blk; Makasar ; Makasar +blk; Malayalam ; Malayalam +blk; Mandaic ; Mandaic +blk; Manichaean ; Manichaean +blk; Marchen ; Marchen +blk; Masaram_Gondi ; Masaram_Gondi +blk; Math_Alphanum ; Mathematical_Alphanumeric_Symbols +blk; Math_Operators ; Mathematical_Operators +blk; Mayan_Numerals ; Mayan_Numerals +blk; Medefaidrin ; Medefaidrin +blk; Meetei_Mayek ; Meetei_Mayek +blk; Meetei_Mayek_Ext ; Meetei_Mayek_Extensions +blk; Mende_Kikakui ; Mende_Kikakui +blk; Meroitic_Cursive ; Meroitic_Cursive +blk; Meroitic_Hieroglyphs ; Meroitic_Hieroglyphs +blk; Miao ; Miao +blk; Misc_Arrows ; Miscellaneous_Symbols_And_Arrows +blk; Misc_Math_Symbols_A ; Miscellaneous_Mathematical_Symbols_A +blk; Misc_Math_Symbols_B ; Miscellaneous_Mathematical_Symbols_B +blk; Misc_Pictographs ; Miscellaneous_Symbols_And_Pictographs +blk; Misc_Symbols ; Miscellaneous_Symbols +blk; Misc_Technical ; Miscellaneous_Technical +blk; Modi ; Modi +blk; Modifier_Letters ; Spacing_Modifier_Letters +blk; Modifier_Tone_Letters ; Modifier_Tone_Letters +blk; Mongolian ; Mongolian +blk; Mongolian_Sup ; Mongolian_Supplement +blk; Mro ; Mro +blk; Multani ; Multani +blk; Music ; Musical_Symbols +blk; Myanmar ; Myanmar +blk; Myanmar_Ext_A ; Myanmar_Extended_A +blk; Myanmar_Ext_B ; Myanmar_Extended_B +blk; Nabataean ; Nabataean +blk; Nandinagari ; Nandinagari +blk; NB ; No_Block +blk; New_Tai_Lue ; New_Tai_Lue +blk; Newa ; Newa +blk; NKo ; NKo +blk; Number_Forms ; Number_Forms +blk; Nushu ; Nushu +blk; Nyiakeng_Puachue_Hmong ; Nyiakeng_Puachue_Hmong +blk; OCR ; Optical_Character_Recognition +blk; Ogham ; Ogham +blk; Ol_Chiki ; Ol_Chiki +blk; Old_Hungarian ; Old_Hungarian +blk; Old_Italic ; Old_Italic +blk; Old_North_Arabian ; Old_North_Arabian +blk; Old_Permic ; Old_Permic +blk; Old_Persian ; Old_Persian +blk; Old_Sogdian ; Old_Sogdian +blk; Old_South_Arabian ; Old_South_Arabian +blk; Old_Turkic ; Old_Turkic +blk; Old_Uyghur ; Old_Uyghur +blk; Oriya ; Oriya +blk; Ornamental_Dingbats ; Ornamental_Dingbats +blk; Osage ; Osage +blk; Osmanya ; Osmanya +blk; Ottoman_Siyaq_Numbers ; Ottoman_Siyaq_Numbers +blk; Pahawh_Hmong ; Pahawh_Hmong +blk; Palmyrene ; Palmyrene +blk; Pau_Cin_Hau ; Pau_Cin_Hau +blk; Phags_Pa ; Phags_Pa +blk; Phaistos ; Phaistos_Disc +blk; Phoenician ; Phoenician +blk; Phonetic_Ext ; Phonetic_Extensions +blk; Phonetic_Ext_Sup ; Phonetic_Extensions_Supplement +blk; Playing_Cards ; Playing_Cards +blk; Psalter_Pahlavi ; Psalter_Pahlavi +blk; PUA ; Private_Use_Area ; Private_Use +blk; Punctuation ; General_Punctuation +blk; Rejang ; Rejang +blk; Rumi ; Rumi_Numeral_Symbols +blk; Runic ; Runic +blk; Samaritan ; Samaritan +blk; Saurashtra ; Saurashtra +blk; Sharada ; Sharada +blk; Shavian ; Shavian +blk; Shorthand_Format_Controls ; Shorthand_Format_Controls +blk; Siddham ; Siddham +blk; Sinhala ; Sinhala +blk; Sinhala_Archaic_Numbers ; Sinhala_Archaic_Numbers +blk; Small_Forms ; Small_Form_Variants +blk; Small_Kana_Ext ; Small_Kana_Extension +blk; Sogdian ; Sogdian +blk; Sora_Sompeng ; Sora_Sompeng +blk; Soyombo ; Soyombo +blk; Specials ; Specials +blk; Sundanese ; Sundanese +blk; Sundanese_Sup ; Sundanese_Supplement +blk; Sup_Arrows_A ; Supplemental_Arrows_A +blk; Sup_Arrows_B ; Supplemental_Arrows_B +blk; Sup_Arrows_C ; Supplemental_Arrows_C +blk; Sup_Math_Operators ; Supplemental_Mathematical_Operators +blk; Sup_PUA_A ; Supplementary_Private_Use_Area_A +blk; Sup_PUA_B ; Supplementary_Private_Use_Area_B +blk; Sup_Punctuation ; Supplemental_Punctuation +blk; Sup_Symbols_And_Pictographs ; Supplemental_Symbols_And_Pictographs +blk; Super_And_Sub ; Superscripts_And_Subscripts +blk; Sutton_SignWriting ; Sutton_SignWriting +blk; Syloti_Nagri ; Syloti_Nagri +blk; Symbols_And_Pictographs_Ext_A ; Symbols_And_Pictographs_Extended_A +blk; Symbols_For_Legacy_Computing ; Symbols_For_Legacy_Computing +blk; Syriac ; Syriac +blk; Syriac_Sup ; Syriac_Supplement +blk; Tagalog ; Tagalog +blk; Tagbanwa ; Tagbanwa +blk; Tags ; Tags +blk; Tai_Le ; Tai_Le +blk; Tai_Tham ; Tai_Tham +blk; Tai_Viet ; Tai_Viet +blk; Tai_Xuan_Jing ; Tai_Xuan_Jing_Symbols +blk; Takri ; Takri +blk; Tamil ; Tamil +blk; Tamil_Sup ; Tamil_Supplement +blk; Tangsa ; Tangsa +blk; Tangut ; Tangut +blk; Tangut_Components ; Tangut_Components +blk; Tangut_Sup ; Tangut_Supplement +blk; Telugu ; Telugu +blk; Thaana ; Thaana +blk; Thai ; Thai +blk; Tibetan ; Tibetan +blk; Tifinagh ; Tifinagh +blk; Tirhuta ; Tirhuta +blk; Toto ; Toto +blk; Transport_And_Map ; Transport_And_Map_Symbols +blk; UCAS ; Unified_Canadian_Aboriginal_Syllabics; Canadian_Syllabics +blk; UCAS_Ext ; Unified_Canadian_Aboriginal_Syllabics_Extended +blk; UCAS_Ext_A ; Unified_Canadian_Aboriginal_Syllabics_Extended_A +blk; Ugaritic ; Ugaritic +blk; Vai ; Vai +blk; Vedic_Ext ; Vedic_Extensions +blk; Vertical_Forms ; Vertical_Forms +blk; Vithkuqi ; Vithkuqi +blk; VS ; Variation_Selectors +blk; VS_Sup ; Variation_Selectors_Supplement +blk; Wancho ; Wancho +blk; Warang_Citi ; Warang_Citi +blk; Yezidi ; Yezidi +blk; Yi_Radicals ; Yi_Radicals +blk; Yi_Syllables ; Yi_Syllables +blk; Yijing ; Yijing_Hexagram_Symbols +blk; Zanabazar_Square ; Zanabazar_Square +blk; Znamenny_Music ; Znamenny_Musical_Notation + +# Canonical_Combining_Class (ccc) + +ccc; 0; NR ; Not_Reordered +ccc; 1; OV ; Overlay +ccc; 6; HANR ; Han_Reading +ccc; 7; NK ; Nukta +ccc; 8; KV ; Kana_Voicing +ccc; 9; VR ; Virama +ccc; 10; CCC10 ; CCC10 +ccc; 11; CCC11 ; CCC11 +ccc; 12; CCC12 ; CCC12 +ccc; 13; CCC13 ; CCC13 +ccc; 14; CCC14 ; CCC14 +ccc; 15; CCC15 ; CCC15 +ccc; 16; CCC16 ; CCC16 +ccc; 17; CCC17 ; CCC17 +ccc; 18; CCC18 ; CCC18 +ccc; 19; CCC19 ; CCC19 +ccc; 20; CCC20 ; CCC20 +ccc; 21; CCC21 ; CCC21 +ccc; 22; CCC22 ; CCC22 +ccc; 23; CCC23 ; CCC23 +ccc; 24; CCC24 ; CCC24 +ccc; 25; CCC25 ; CCC25 +ccc; 26; CCC26 ; CCC26 +ccc; 27; CCC27 ; CCC27 +ccc; 28; CCC28 ; CCC28 +ccc; 29; CCC29 ; CCC29 +ccc; 30; CCC30 ; CCC30 +ccc; 31; CCC31 ; CCC31 +ccc; 32; CCC32 ; CCC32 +ccc; 33; CCC33 ; CCC33 +ccc; 34; CCC34 ; CCC34 +ccc; 35; CCC35 ; CCC35 +ccc; 36; CCC36 ; CCC36 +ccc; 84; CCC84 ; CCC84 +ccc; 91; CCC91 ; CCC91 +ccc; 103; CCC103 ; CCC103 +ccc; 107; CCC107 ; CCC107 +ccc; 118; CCC118 ; CCC118 +ccc; 122; CCC122 ; CCC122 +ccc; 129; CCC129 ; CCC129 +ccc; 130; CCC130 ; CCC130 +ccc; 132; CCC132 ; CCC132 +ccc; 133; CCC133 ; CCC133 # RESERVED +ccc; 200; ATBL ; Attached_Below_Left +ccc; 202; ATB ; Attached_Below +ccc; 214; ATA ; Attached_Above +ccc; 216; ATAR ; Attached_Above_Right +ccc; 218; BL ; Below_Left +ccc; 220; B ; Below +ccc; 222; BR ; Below_Right +ccc; 224; L ; Left +ccc; 226; R ; Right +ccc; 228; AL ; Above_Left +ccc; 230; A ; Above +ccc; 232; AR ; Above_Right +ccc; 233; DB ; Double_Below +ccc; 234; DA ; Double_Above +ccc; 240; IS ; Iota_Subscript + +# Case_Folding (cf) + +# @missing: 0000..10FFFF; Case_Folding; + +# Case_Ignorable (CI) + +CI ; N ; No ; F ; False +CI ; Y ; Yes ; T ; True + +# Cased (Cased) + +Cased; N ; No ; F ; False +Cased; Y ; Yes ; T ; True + +# Changes_When_Casefolded (CWCF) + +CWCF; N ; No ; F ; False +CWCF; Y ; Yes ; T ; True + +# Changes_When_Casemapped (CWCM) + +CWCM; N ; No ; F ; False +CWCM; Y ; Yes ; T ; True + +# Changes_When_Lowercased (CWL) + +CWL; N ; No ; F ; False +CWL; Y ; Yes ; T ; True + +# Changes_When_NFKC_Casefolded (CWKCF) + +CWKCF; N ; No ; F ; False +CWKCF; Y ; Yes ; T ; True + +# Changes_When_Titlecased (CWT) + +CWT; N ; No ; F ; False +CWT; Y ; Yes ; T ; True + +# Changes_When_Uppercased (CWU) + +CWU; N ; No ; F ; False +CWU; Y ; Yes ; T ; True + +# Composition_Exclusion (CE) + +CE ; N ; No ; F ; False +CE ; Y ; Yes ; T ; True + +# Dash (Dash) + +Dash; N ; No ; F ; False +Dash; Y ; Yes ; T ; True + +# Decomposition_Mapping (dm) + +# @missing: 0000..10FFFF; Decomposition_Mapping; + +# Decomposition_Type (dt) + +dt ; Can ; Canonical ; can +dt ; Com ; Compat ; com +dt ; Enc ; Circle ; enc +dt ; Fin ; Final ; fin +dt ; Font ; Font ; font +dt ; Fra ; Fraction ; fra +dt ; Init ; Initial ; init +dt ; Iso ; Isolated ; iso +dt ; Med ; Medial ; med +dt ; Nar ; Narrow ; nar +dt ; Nb ; Nobreak ; nb +dt ; None ; None ; none +dt ; Sml ; Small ; sml +dt ; Sqr ; Square ; sqr +dt ; Sub ; Sub ; sub +dt ; Sup ; Super ; sup +dt ; Vert ; Vertical ; vert +dt ; Wide ; Wide ; wide + +# Default_Ignorable_Code_Point (DI) + +DI ; N ; No ; F ; False +DI ; Y ; Yes ; T ; True + +# Deprecated (Dep) + +Dep; N ; No ; F ; False +Dep; Y ; Yes ; T ; True + +# Diacritic (Dia) + +Dia; N ; No ; F ; False +Dia; Y ; Yes ; T ; True + +# East_Asian_Width (ea) + +ea ; A ; Ambiguous +ea ; F ; Fullwidth +ea ; H ; Halfwidth +ea ; N ; Neutral +ea ; Na ; Narrow +ea ; W ; Wide + +# Emoji (Emoji) + +Emoji; N ; No ; F ; False +Emoji; Y ; Yes ; T ; True + +# Emoji_Component (EComp) + +EComp; N ; No ; F ; False +EComp; Y ; Yes ; T ; True + +# Emoji_Modifier (EMod) + +EMod; N ; No ; F ; False +EMod; Y ; Yes ; T ; True + +# Emoji_Modifier_Base (EBase) + +EBase; N ; No ; F ; False +EBase; Y ; Yes ; T ; True + +# Emoji_Presentation (EPres) + +EPres; N ; No ; F ; False +EPres; Y ; Yes ; T ; True + +# Equivalent_Unified_Ideograph (EqUIdeo) + +# @missing: 0000..10FFFF; Equivalent_Unified_Ideograph; + +# Expands_On_NFC (XO_NFC) + +XO_NFC; N ; No ; F ; False +XO_NFC; Y ; Yes ; T ; True + +# Expands_On_NFD (XO_NFD) + +XO_NFD; N ; No ; F ; False +XO_NFD; Y ; Yes ; T ; True + +# Expands_On_NFKC (XO_NFKC) + +XO_NFKC; N ; No ; F ; False +XO_NFKC; Y ; Yes ; T ; True + +# Expands_On_NFKD (XO_NFKD) + +XO_NFKD; N ; No ; F ; False +XO_NFKD; Y ; Yes ; T ; True + +# Extended_Pictographic (ExtPict) + +ExtPict; N ; No ; F ; False +ExtPict; Y ; Yes ; T ; True + +# Extender (Ext) + +Ext; N ; No ; F ; False +Ext; Y ; Yes ; T ; True + +# FC_NFKC_Closure (FC_NFKC) + +# @missing: 0000..10FFFF; FC_NFKC_Closure; + +# Full_Composition_Exclusion (Comp_Ex) + +Comp_Ex; N ; No ; F ; False +Comp_Ex; Y ; Yes ; T ; True + +# General_Category (gc) + +gc ; C ; Other # Cc | Cf | Cn | Co | Cs +gc ; Cc ; Control ; cntrl +gc ; Cf ; Format +gc ; Cn ; Unassigned +gc ; Co ; Private_Use +gc ; Cs ; Surrogate +gc ; L ; Letter # Ll | Lm | Lo | Lt | Lu +gc ; LC ; Cased_Letter # Ll | Lt | Lu +gc ; Ll ; Lowercase_Letter +gc ; Lm ; Modifier_Letter +gc ; Lo ; Other_Letter +gc ; Lt ; Titlecase_Letter +gc ; Lu ; Uppercase_Letter +gc ; M ; Mark ; Combining_Mark # Mc | Me | Mn +gc ; Mc ; Spacing_Mark +gc ; Me ; Enclosing_Mark +gc ; Mn ; Nonspacing_Mark +gc ; N ; Number # Nd | Nl | No +gc ; Nd ; Decimal_Number ; digit +gc ; Nl ; Letter_Number +gc ; No ; Other_Number +gc ; P ; Punctuation ; punct # Pc | Pd | Pe | Pf | Pi | Po | Ps +gc ; Pc ; Connector_Punctuation +gc ; Pd ; Dash_Punctuation +gc ; Pe ; Close_Punctuation +gc ; Pf ; Final_Punctuation +gc ; Pi ; Initial_Punctuation +gc ; Po ; Other_Punctuation +gc ; Ps ; Open_Punctuation +gc ; S ; Symbol # Sc | Sk | Sm | So +gc ; Sc ; Currency_Symbol +gc ; Sk ; Modifier_Symbol +gc ; Sm ; Math_Symbol +gc ; So ; Other_Symbol +gc ; Z ; Separator # Zl | Zp | Zs +gc ; Zl ; Line_Separator +gc ; Zp ; Paragraph_Separator +gc ; Zs ; Space_Separator +# @missing: 0000..10FFFF; General_Category; Unassigned + +# Grapheme_Base (Gr_Base) + +Gr_Base; N ; No ; F ; False +Gr_Base; Y ; Yes ; T ; True + +# Grapheme_Cluster_Break (GCB) + +GCB; CN ; Control +GCB; CR ; CR +GCB; EB ; E_Base +GCB; EBG ; E_Base_GAZ +GCB; EM ; E_Modifier +GCB; EX ; Extend +GCB; GAZ ; Glue_After_Zwj +GCB; L ; L +GCB; LF ; LF +GCB; LV ; LV +GCB; LVT ; LVT +GCB; PP ; Prepend +GCB; RI ; Regional_Indicator +GCB; SM ; SpacingMark +GCB; T ; T +GCB; V ; V +GCB; XX ; Other +GCB; ZWJ ; ZWJ + +# Grapheme_Extend (Gr_Ext) + +Gr_Ext; N ; No ; F ; False +Gr_Ext; Y ; Yes ; T ; True + +# Grapheme_Link (Gr_Link) + +Gr_Link; N ; No ; F ; False +Gr_Link; Y ; Yes ; T ; True + +# Hangul_Syllable_Type (hst) + +hst; L ; Leading_Jamo +hst; LV ; LV_Syllable +hst; LVT ; LVT_Syllable +hst; NA ; Not_Applicable +hst; T ; Trailing_Jamo +hst; V ; Vowel_Jamo + +# Hex_Digit (Hex) + +Hex; N ; No ; F ; False +Hex; Y ; Yes ; T ; True + +# Hyphen (Hyphen) + +Hyphen; N ; No ; F ; False +Hyphen; Y ; Yes ; T ; True + +# IDS_Binary_Operator (IDSB) + +IDSB; N ; No ; F ; False +IDSB; Y ; Yes ; T ; True + +# IDS_Trinary_Operator (IDST) + +IDST; N ; No ; F ; False +IDST; Y ; Yes ; T ; True + +# ID_Continue (IDC) + +IDC; N ; No ; F ; False +IDC; Y ; Yes ; T ; True + +# ID_Start (IDS) + +IDS; N ; No ; F ; False +IDS; Y ; Yes ; T ; True + +# ISO_Comment (isc) + +# @missing: 0000..10FFFF; ISO_Comment; + +# Ideographic (Ideo) + +Ideo; N ; No ; F ; False +Ideo; Y ; Yes ; T ; True + +# Indic_Positional_Category (InPC) + +InPC; Bottom ; Bottom +InPC; Bottom_And_Left ; Bottom_And_Left +InPC; Bottom_And_Right ; Bottom_And_Right +InPC; Left ; Left +InPC; Left_And_Right ; Left_And_Right +InPC; NA ; NA +InPC; Overstruck ; Overstruck +InPC; Right ; Right +InPC; Top ; Top +InPC; Top_And_Bottom ; Top_And_Bottom +InPC; Top_And_Bottom_And_Left ; Top_And_Bottom_And_Left +InPC; Top_And_Bottom_And_Right ; Top_And_Bottom_And_Right +InPC; Top_And_Left ; Top_And_Left +InPC; Top_And_Left_And_Right ; Top_And_Left_And_Right +InPC; Top_And_Right ; Top_And_Right +InPC; Visual_Order_Left ; Visual_Order_Left + +# Indic_Syllabic_Category (InSC) + +InSC; Avagraha ; Avagraha +InSC; Bindu ; Bindu +InSC; Brahmi_Joining_Number ; Brahmi_Joining_Number +InSC; Cantillation_Mark ; Cantillation_Mark +InSC; Consonant ; Consonant +InSC; Consonant_Dead ; Consonant_Dead +InSC; Consonant_Final ; Consonant_Final +InSC; Consonant_Head_Letter ; Consonant_Head_Letter +InSC; Consonant_Initial_Postfixed ; Consonant_Initial_Postfixed +InSC; Consonant_Killer ; Consonant_Killer +InSC; Consonant_Medial ; Consonant_Medial +InSC; Consonant_Placeholder ; Consonant_Placeholder +InSC; Consonant_Preceding_Repha ; Consonant_Preceding_Repha +InSC; Consonant_Prefixed ; Consonant_Prefixed +InSC; Consonant_Subjoined ; Consonant_Subjoined +InSC; Consonant_Succeeding_Repha ; Consonant_Succeeding_Repha +InSC; Consonant_With_Stacker ; Consonant_With_Stacker +InSC; Gemination_Mark ; Gemination_Mark +InSC; Invisible_Stacker ; Invisible_Stacker +InSC; Joiner ; Joiner +InSC; Modifying_Letter ; Modifying_Letter +InSC; Non_Joiner ; Non_Joiner +InSC; Nukta ; Nukta +InSC; Number ; Number +InSC; Number_Joiner ; Number_Joiner +InSC; Other ; Other +InSC; Pure_Killer ; Pure_Killer +InSC; Register_Shifter ; Register_Shifter +InSC; Syllable_Modifier ; Syllable_Modifier +InSC; Tone_Letter ; Tone_Letter +InSC; Tone_Mark ; Tone_Mark +InSC; Virama ; Virama +InSC; Visarga ; Visarga +InSC; Vowel ; Vowel +InSC; Vowel_Dependent ; Vowel_Dependent +InSC; Vowel_Independent ; Vowel_Independent + +# Jamo_Short_Name (JSN) + +JSN; A ; A +JSN; AE ; AE +JSN; B ; B +JSN; BB ; BB +JSN; BS ; BS +JSN; C ; C +JSN; D ; D +JSN; DD ; DD +JSN; E ; E +JSN; EO ; EO +JSN; EU ; EU +JSN; G ; G +JSN; GG ; GG +JSN; GS ; GS +JSN; H ; H +JSN; I ; I +JSN; J ; J +JSN; JJ ; JJ +JSN; K ; K +JSN; L ; L +JSN; LB ; LB +JSN; LG ; LG +JSN; LH ; LH +JSN; LM ; LM +JSN; LP ; LP +JSN; LS ; LS +JSN; LT ; LT +JSN; M ; M +JSN; N ; N +JSN; NG ; NG +JSN; NH ; NH +JSN; NJ ; NJ +JSN; O ; O +JSN; OE ; OE +JSN; P ; P +JSN; R ; R +JSN; S ; S +JSN; SS ; SS +JSN; T ; T +JSN; U ; U +JSN; WA ; WA +JSN; WAE ; WAE +JSN; WE ; WE +JSN; WEO ; WEO +JSN; WI ; WI +JSN; YA ; YA +JSN; YAE ; YAE +JSN; YE ; YE +JSN; YEO ; YEO +JSN; YI ; YI +JSN; YO ; YO +JSN; YU ; YU +# @missing: 0000..10FFFF; Jamo_Short_Name; + +# Join_Control (Join_C) + +Join_C; N ; No ; F ; False +Join_C; Y ; Yes ; T ; True + +# Joining_Group (jg) + +jg ; African_Feh ; African_Feh +jg ; African_Noon ; African_Noon +jg ; African_Qaf ; African_Qaf +jg ; Ain ; Ain +jg ; Alaph ; Alaph +jg ; Alef ; Alef +jg ; Beh ; Beh +jg ; Beth ; Beth +jg ; Burushaski_Yeh_Barree ; Burushaski_Yeh_Barree +jg ; Dal ; Dal +jg ; Dalath_Rish ; Dalath_Rish +jg ; E ; E +jg ; Farsi_Yeh ; Farsi_Yeh +jg ; Fe ; Fe +jg ; Feh ; Feh +jg ; Final_Semkath ; Final_Semkath +jg ; Gaf ; Gaf +jg ; Gamal ; Gamal +jg ; Hah ; Hah +jg ; Hanifi_Rohingya_Kinna_Ya ; Hanifi_Rohingya_Kinna_Ya +jg ; Hanifi_Rohingya_Pa ; Hanifi_Rohingya_Pa +jg ; He ; He +jg ; Heh ; Heh +jg ; Heh_Goal ; Heh_Goal +jg ; Heth ; Heth +jg ; Kaf ; Kaf +jg ; Kaph ; Kaph +jg ; Khaph ; Khaph +jg ; Knotted_Heh ; Knotted_Heh +jg ; Lam ; Lam +jg ; Lamadh ; Lamadh +jg ; Malayalam_Bha ; Malayalam_Bha +jg ; Malayalam_Ja ; Malayalam_Ja +jg ; Malayalam_Lla ; Malayalam_Lla +jg ; Malayalam_Llla ; Malayalam_Llla +jg ; Malayalam_Nga ; Malayalam_Nga +jg ; Malayalam_Nna ; Malayalam_Nna +jg ; Malayalam_Nnna ; Malayalam_Nnna +jg ; Malayalam_Nya ; Malayalam_Nya +jg ; Malayalam_Ra ; Malayalam_Ra +jg ; Malayalam_Ssa ; Malayalam_Ssa +jg ; Malayalam_Tta ; Malayalam_Tta +jg ; Manichaean_Aleph ; Manichaean_Aleph +jg ; Manichaean_Ayin ; Manichaean_Ayin +jg ; Manichaean_Beth ; Manichaean_Beth +jg ; Manichaean_Daleth ; Manichaean_Daleth +jg ; Manichaean_Dhamedh ; Manichaean_Dhamedh +jg ; Manichaean_Five ; Manichaean_Five +jg ; Manichaean_Gimel ; Manichaean_Gimel +jg ; Manichaean_Heth ; Manichaean_Heth +jg ; Manichaean_Hundred ; Manichaean_Hundred +jg ; Manichaean_Kaph ; Manichaean_Kaph +jg ; Manichaean_Lamedh ; Manichaean_Lamedh +jg ; Manichaean_Mem ; Manichaean_Mem +jg ; Manichaean_Nun ; Manichaean_Nun +jg ; Manichaean_One ; Manichaean_One +jg ; Manichaean_Pe ; Manichaean_Pe +jg ; Manichaean_Qoph ; Manichaean_Qoph +jg ; Manichaean_Resh ; Manichaean_Resh +jg ; Manichaean_Sadhe ; Manichaean_Sadhe +jg ; Manichaean_Samekh ; Manichaean_Samekh +jg ; Manichaean_Taw ; Manichaean_Taw +jg ; Manichaean_Ten ; Manichaean_Ten +jg ; Manichaean_Teth ; Manichaean_Teth +jg ; Manichaean_Thamedh ; Manichaean_Thamedh +jg ; Manichaean_Twenty ; Manichaean_Twenty +jg ; Manichaean_Waw ; Manichaean_Waw +jg ; Manichaean_Yodh ; Manichaean_Yodh +jg ; Manichaean_Zayin ; Manichaean_Zayin +jg ; Meem ; Meem +jg ; Mim ; Mim +jg ; No_Joining_Group ; No_Joining_Group +jg ; Noon ; Noon +jg ; Nun ; Nun +jg ; Nya ; Nya +jg ; Pe ; Pe +jg ; Qaf ; Qaf +jg ; Qaph ; Qaph +jg ; Reh ; Reh +jg ; Reversed_Pe ; Reversed_Pe +jg ; Rohingya_Yeh ; Rohingya_Yeh +jg ; Sad ; Sad +jg ; Sadhe ; Sadhe +jg ; Seen ; Seen +jg ; Semkath ; Semkath +jg ; Shin ; Shin +jg ; Straight_Waw ; Straight_Waw +jg ; Swash_Kaf ; Swash_Kaf +jg ; Syriac_Waw ; Syriac_Waw +jg ; Tah ; Tah +jg ; Taw ; Taw +jg ; Teh_Marbuta ; Teh_Marbuta +jg ; Teh_Marbuta_Goal ; Hamza_On_Heh_Goal +jg ; Teth ; Teth +jg ; Thin_Yeh ; Thin_Yeh +jg ; Vertical_Tail ; Vertical_Tail +jg ; Waw ; Waw +jg ; Yeh ; Yeh +jg ; Yeh_Barree ; Yeh_Barree +jg ; Yeh_With_Tail ; Yeh_With_Tail +jg ; Yudh ; Yudh +jg ; Yudh_He ; Yudh_He +jg ; Zain ; Zain +jg ; Zhain ; Zhain + +# Joining_Type (jt) + +jt ; C ; Join_Causing +jt ; D ; Dual_Joining +jt ; L ; Left_Joining +jt ; R ; Right_Joining +jt ; T ; Transparent +jt ; U ; Non_Joining + +# Line_Break (lb) + +lb ; AI ; Ambiguous +lb ; AL ; Alphabetic +lb ; B2 ; Break_Both +lb ; BA ; Break_After +lb ; BB ; Break_Before +lb ; BK ; Mandatory_Break +lb ; CB ; Contingent_Break +lb ; CJ ; Conditional_Japanese_Starter +lb ; CL ; Close_Punctuation +lb ; CM ; Combining_Mark +lb ; CP ; Close_Parenthesis +lb ; CR ; Carriage_Return +lb ; EB ; E_Base +lb ; EM ; E_Modifier +lb ; EX ; Exclamation +lb ; GL ; Glue +lb ; H2 ; H2 +lb ; H3 ; H3 +lb ; HL ; Hebrew_Letter +lb ; HY ; Hyphen +lb ; ID ; Ideographic +lb ; IN ; Inseparable ; Inseperable +lb ; IS ; Infix_Numeric +lb ; JL ; JL +lb ; JT ; JT +lb ; JV ; JV +lb ; LF ; Line_Feed +lb ; NL ; Next_Line +lb ; NS ; Nonstarter +lb ; NU ; Numeric +lb ; OP ; Open_Punctuation +lb ; PO ; Postfix_Numeric +lb ; PR ; Prefix_Numeric +lb ; QU ; Quotation +lb ; RI ; Regional_Indicator +lb ; SA ; Complex_Context +lb ; SG ; Surrogate +lb ; SP ; Space +lb ; SY ; Break_Symbols +lb ; WJ ; Word_Joiner +lb ; XX ; Unknown +lb ; ZW ; ZWSpace +lb ; ZWJ ; ZWJ + +# Logical_Order_Exception (LOE) + +LOE; N ; No ; F ; False +LOE; Y ; Yes ; T ; True + +# Lowercase (Lower) + +Lower; N ; No ; F ; False +Lower; Y ; Yes ; T ; True + +# Lowercase_Mapping (lc) + +# @missing: 0000..10FFFF; Lowercase_Mapping; + +# Math (Math) + +Math; N ; No ; F ; False +Math; Y ; Yes ; T ; True + +# NFC_Quick_Check (NFC_QC) + +NFC_QC; M ; Maybe +NFC_QC; N ; No +NFC_QC; Y ; Yes + +# NFD_Quick_Check (NFD_QC) + +NFD_QC; N ; No +NFD_QC; Y ; Yes + +# NFKC_Casefold (NFKC_CF) + +# @missing: 0000..10FFFF; NFKC_Casefold; + +# NFKC_Quick_Check (NFKC_QC) + +NFKC_QC; M ; Maybe +NFKC_QC; N ; No +NFKC_QC; Y ; Yes + +# NFKD_Quick_Check (NFKD_QC) + +NFKD_QC; N ; No +NFKD_QC; Y ; Yes + +# Name (na) + +# @missing: 0000..10FFFF; Name; + +# Name_Alias (Name_Alias) + +# @missing: 0000..10FFFF; Name_Alias; + +# Noncharacter_Code_Point (NChar) + +NChar; N ; No ; F ; False +NChar; Y ; Yes ; T ; True + +# Numeric_Type (nt) + +nt ; De ; Decimal +nt ; Di ; Digit +nt ; None ; None +nt ; Nu ; Numeric + +# Numeric_Value (nv) + +# @missing: 0000..10FFFF; Numeric_Value; NaN + +# Other_Alphabetic (OAlpha) + +OAlpha; N ; No ; F ; False +OAlpha; Y ; Yes ; T ; True + +# Other_Default_Ignorable_Code_Point (ODI) + +ODI; N ; No ; F ; False +ODI; Y ; Yes ; T ; True + +# Other_Grapheme_Extend (OGr_Ext) + +OGr_Ext; N ; No ; F ; False +OGr_Ext; Y ; Yes ; T ; True + +# Other_ID_Continue (OIDC) + +OIDC; N ; No ; F ; False +OIDC; Y ; Yes ; T ; True + +# Other_ID_Start (OIDS) + +OIDS; N ; No ; F ; False +OIDS; Y ; Yes ; T ; True + +# Other_Lowercase (OLower) + +OLower; N ; No ; F ; False +OLower; Y ; Yes ; T ; True + +# Other_Math (OMath) + +OMath; N ; No ; F ; False +OMath; Y ; Yes ; T ; True + +# Other_Uppercase (OUpper) + +OUpper; N ; No ; F ; False +OUpper; Y ; Yes ; T ; True + +# Pattern_Syntax (Pat_Syn) + +Pat_Syn; N ; No ; F ; False +Pat_Syn; Y ; Yes ; T ; True + +# Pattern_White_Space (Pat_WS) + +Pat_WS; N ; No ; F ; False +Pat_WS; Y ; Yes ; T ; True + +# Prepended_Concatenation_Mark (PCM) + +PCM; N ; No ; F ; False +PCM; Y ; Yes ; T ; True + +# Quotation_Mark (QMark) + +QMark; N ; No ; F ; False +QMark; Y ; Yes ; T ; True + +# Radical (Radical) + +Radical; N ; No ; F ; False +Radical; Y ; Yes ; T ; True + +# Regional_Indicator (RI) + +RI ; N ; No ; F ; False +RI ; Y ; Yes ; T ; True + +# Script (sc) + +sc ; Adlm ; Adlam +sc ; Aghb ; Caucasian_Albanian +sc ; Ahom ; Ahom +sc ; Arab ; Arabic +sc ; Armi ; Imperial_Aramaic +sc ; Armn ; Armenian +sc ; Avst ; Avestan +sc ; Bali ; Balinese +sc ; Bamu ; Bamum +sc ; Bass ; Bassa_Vah +sc ; Batk ; Batak +sc ; Beng ; Bengali +sc ; Bhks ; Bhaiksuki +sc ; Bopo ; Bopomofo +sc ; Brah ; Brahmi +sc ; Brai ; Braille +sc ; Bugi ; Buginese +sc ; Buhd ; Buhid +sc ; Cakm ; Chakma +sc ; Cans ; Canadian_Aboriginal +sc ; Cari ; Carian +sc ; Cham ; Cham +sc ; Cher ; Cherokee +sc ; Chrs ; Chorasmian +sc ; Copt ; Coptic ; Qaac +sc ; Cpmn ; Cypro_Minoan +sc ; Cprt ; Cypriot +sc ; Cyrl ; Cyrillic +sc ; Deva ; Devanagari +sc ; Diak ; Dives_Akuru +sc ; Dogr ; Dogra +sc ; Dsrt ; Deseret +sc ; Dupl ; Duployan +sc ; Egyp ; Egyptian_Hieroglyphs +sc ; Elba ; Elbasan +sc ; Elym ; Elymaic +sc ; Ethi ; Ethiopic +sc ; Geor ; Georgian +sc ; Glag ; Glagolitic +sc ; Gong ; Gunjala_Gondi +sc ; Gonm ; Masaram_Gondi +sc ; Goth ; Gothic +sc ; Gran ; Grantha +sc ; Grek ; Greek +sc ; Gujr ; Gujarati +sc ; Guru ; Gurmukhi +sc ; Hang ; Hangul +sc ; Hani ; Han +sc ; Hano ; Hanunoo +sc ; Hatr ; Hatran +sc ; Hebr ; Hebrew +sc ; Hira ; Hiragana +sc ; Hluw ; Anatolian_Hieroglyphs +sc ; Hmng ; Pahawh_Hmong +sc ; Hmnp ; Nyiakeng_Puachue_Hmong +sc ; Hrkt ; Katakana_Or_Hiragana +sc ; Hung ; Old_Hungarian +sc ; Ital ; Old_Italic +sc ; Java ; Javanese +sc ; Kali ; Kayah_Li +sc ; Kana ; Katakana +sc ; Khar ; Kharoshthi +sc ; Khmr ; Khmer +sc ; Khoj ; Khojki +sc ; Kits ; Khitan_Small_Script +sc ; Knda ; Kannada +sc ; Kthi ; Kaithi +sc ; Lana ; Tai_Tham +sc ; Laoo ; Lao +sc ; Latn ; Latin +sc ; Lepc ; Lepcha +sc ; Limb ; Limbu +sc ; Lina ; Linear_A +sc ; Linb ; Linear_B +sc ; Lisu ; Lisu +sc ; Lyci ; Lycian +sc ; Lydi ; Lydian +sc ; Mahj ; Mahajani +sc ; Maka ; Makasar +sc ; Mand ; Mandaic +sc ; Mani ; Manichaean +sc ; Marc ; Marchen +sc ; Medf ; Medefaidrin +sc ; Mend ; Mende_Kikakui +sc ; Merc ; Meroitic_Cursive +sc ; Mero ; Meroitic_Hieroglyphs +sc ; Mlym ; Malayalam +sc ; Modi ; Modi +sc ; Mong ; Mongolian +sc ; Mroo ; Mro +sc ; Mtei ; Meetei_Mayek +sc ; Mult ; Multani +sc ; Mymr ; Myanmar +sc ; Nand ; Nandinagari +sc ; Narb ; Old_North_Arabian +sc ; Nbat ; Nabataean +sc ; Newa ; Newa +sc ; Nkoo ; Nko +sc ; Nshu ; Nushu +sc ; Ogam ; Ogham +sc ; Olck ; Ol_Chiki +sc ; Orkh ; Old_Turkic +sc ; Orya ; Oriya +sc ; Osge ; Osage +sc ; Osma ; Osmanya +sc ; Ougr ; Old_Uyghur +sc ; Palm ; Palmyrene +sc ; Pauc ; Pau_Cin_Hau +sc ; Perm ; Old_Permic +sc ; Phag ; Phags_Pa +sc ; Phli ; Inscriptional_Pahlavi +sc ; Phlp ; Psalter_Pahlavi +sc ; Phnx ; Phoenician +sc ; Plrd ; Miao +sc ; Prti ; Inscriptional_Parthian +sc ; Rjng ; Rejang +sc ; Rohg ; Hanifi_Rohingya +sc ; Runr ; Runic +sc ; Samr ; Samaritan +sc ; Sarb ; Old_South_Arabian +sc ; Saur ; Saurashtra +sc ; Sgnw ; SignWriting +sc ; Shaw ; Shavian +sc ; Shrd ; Sharada +sc ; Sidd ; Siddham +sc ; Sind ; Khudawadi +sc ; Sinh ; Sinhala +sc ; Sogd ; Sogdian +sc ; Sogo ; Old_Sogdian +sc ; Sora ; Sora_Sompeng +sc ; Soyo ; Soyombo +sc ; Sund ; Sundanese +sc ; Sylo ; Syloti_Nagri +sc ; Syrc ; Syriac +sc ; Tagb ; Tagbanwa +sc ; Takr ; Takri +sc ; Tale ; Tai_Le +sc ; Talu ; New_Tai_Lue +sc ; Taml ; Tamil +sc ; Tang ; Tangut +sc ; Tavt ; Tai_Viet +sc ; Telu ; Telugu +sc ; Tfng ; Tifinagh +sc ; Tglg ; Tagalog +sc ; Thaa ; Thaana +sc ; Thai ; Thai +sc ; Tibt ; Tibetan +sc ; Tirh ; Tirhuta +sc ; Tnsa ; Tangsa +sc ; Toto ; Toto +sc ; Ugar ; Ugaritic +sc ; Vaii ; Vai +sc ; Vith ; Vithkuqi +sc ; Wara ; Warang_Citi +sc ; Wcho ; Wancho +sc ; Xpeo ; Old_Persian +sc ; Xsux ; Cuneiform +sc ; Yezi ; Yezidi +sc ; Yiii ; Yi +sc ; Zanb ; Zanabazar_Square +sc ; Zinh ; Inherited ; Qaai +sc ; Zyyy ; Common +sc ; Zzzz ; Unknown + +# Script_Extensions (scx) + +# @missing: 0000..10FFFF; Script_Extensions;