diff --git a/lib/spitfire.ex b/lib/spitfire.ex index 7d95cc1..af6b54f 100644 --- a/lib/spitfire.ex +++ b/lib/spitfire.ex @@ -248,13 +248,16 @@ defmodule Spitfire do :alias -> &parse_alias/1 :"<<" -> &parse_bitstring/1 :kw_identifier when is_list or is_map -> &parse_kw_identifier/1 + :kw_identifier_safe when is_list or is_map -> &parse_kw_identifier/1 :kw_identifier_unsafe when is_list or is_map -> &parse_kw_identifier/1 :kw_identifier when not is_list and not is_map -> &parse_bracketless_kw_list/1 + :kw_identifier_safe when not is_list and not is_map -> &parse_bracketless_kw_list/1 :kw_identifier_unsafe when not is_list and not is_map -> &parse_bracketless_kw_list/1 :int -> &parse_int/1 :flt -> &parse_float/1 :atom -> &parse_atom/1 :atom_quoted -> &parse_atom/1 + :atom_safe -> &parse_atom/1 :atom_unsafe -> &parse_atom/1 true -> &parse_boolean/1 false -> &parse_boolean/1 @@ -535,6 +538,9 @@ defmodule Spitfire do end end + defp map_kw_identifier_to_atom_token(:kw_identifier_safe), do: :atom_safe + defp map_kw_identifier_to_atom_token(:kw_identifier_unsafe), do: :atom_unsafe + defp parse_kw_identifier(%{current_token: {:kw_identifier, meta, token}} = parser) do trace "parse_kw_identifier", trace_meta(parser) do token = encode_literal(parser, token, meta) @@ -546,9 +552,10 @@ defmodule Spitfire do end end - defp parse_kw_identifier(%{current_token: {:kw_identifier_unsafe, meta, tokens}} = parser) do - trace "parse_kw_identifier (unsafe)", trace_meta(parser) do - {atom, parser} = parse_atom(%{parser | current_token: {:atom_unsafe, meta, tokens}}) + defp parse_kw_identifier(%{current_token: {type, meta, tokens}} = parser) + when type in [:kw_identifier_safe, :kw_identifier_unsafe] do + trace "parse_kw_identifier (#{type})", trace_meta(parser) do + {atom, parser} = parse_atom(%{parser | current_token: {map_kw_identifier_to_atom_token(type), meta, tokens}}) parser = parser |> next_token() |> eat_eol() {expr, parser} = parse_expression(parser, @kw_identifier, false, false, false) @@ -584,9 +591,10 @@ defmodule Spitfire do end end - defp parse_bracketless_kw_list(%{current_token: {:kw_identifier_unsafe, meta, tokens}} = parser) do - trace "parse_bracketless_kw_list (unsafe)", trace_meta(parser) do - {atom, parser} = parse_atom(%{parser | current_token: {:atom_unsafe, meta, tokens}}) + defp parse_bracketless_kw_list(%{current_token: {type, meta, tokens}} = parser) + when type in [:kw_identifier_safe, :kw_identifier_unsafe] do + trace "parse_bracketless_kw_list (#{type})", trace_meta(parser) do + {atom, parser} = parse_atom(%{parser | current_token: {map_kw_identifier_to_atom_token(type), meta, tokens}}) parser = parser |> next_token() |> eat_eol() atom = @@ -1274,11 +1282,18 @@ defmodule Spitfire do end end - defp parse_atom(%{current_token: {:atom_unsafe, _, tokens}} = parser) do - trace "parse_atom (unsafe)", trace_meta(parser) do + defp parse_atom(%{current_token: {type, _, tokens}} = parser) when type in [:atom_safe, :atom_unsafe] do + trace "parse_atom (#{type})", trace_meta(parser) do meta = current_meta(parser) {args, parser} = parse_interpolation(parser, tokens) - {{{:., meta, [:erlang, :binary_to_atom]}, [{:delimiter, ~S'"'} | meta], [{:<<>>, meta, args}, :utf8]}, parser} + + binary_to_atom_op = + case type do + :atom_safe -> :binary_to_existing_atom + :atom_unsafe -> :binary_to_atom + end + + {{{:., meta, [:erlang, binary_to_atom_op]}, [{:delimiter, ~S'"'} | meta], [{:<<>>, meta, args}, :utf8]}, parser} end end diff --git a/test/spitfire_test.exs b/test/spitfire_test.exs index afee842..9406b58 100644 --- a/test/spitfire_test.exs +++ b/test/spitfire_test.exs @@ -2911,6 +2911,78 @@ defmodule SpitfireTest do end end + describe "safe atoms" do + test "parses quoted atoms" do + code = """ + :"abc#{Enum.random(1..10_000)}" + """ + + # tokenizer returns error + assert Spitfire.parse(code, existing_atoms_only: true) == {:ok, {:__block__, [], []}} + + code = """ + :"abc\#{foo}" + """ + + # tokenizer returns atom_safe token + assert Spitfire.parse(code, existing_atoms_only: true) == s2q(code, existing_atoms_only: true) + end + + test "parses quoted kw identifiers" do + # TODO this code crashes the parser + # ** (FunctionClauseError) no function clause matching in Spitfire.peek_token/1 + + # The following arguments were given to Spitfire.peek_token/1: + + # # 1 + # %{tokens: [nil | :eot], literal_encoder: nil, errors: [{[line: 1, column: 1], "missing closing bracket for list"}, {[], "unknown token: eot"}], current_token: {:"]", nil}, fuel: 150, peek_token: nil, nesting: 0} + + # Attempted function clauses (showing 7 out of 7): + + # defp peek_token(%{peek_token: {:stab_op, _, token}}) + # defp peek_token(%{peek_token: {type, _, _, _}}) when type === :list_heredoc or type === :bin_heredoc + # defp peek_token(%{peek_token: {token, _, _}}) + # defp peek_token(%{peek_token: {token, _}}) + # defp peek_token(%{peek_token: {token, _, _, _, _, _, _}}) + # defp peek_token(%{peek_token: :eof}) + # defp peek_token(%{tokens: :eot}) + + # code: assert Spitfire.parse(code, existing_atoms_only: true) == {:ok, {:__block__, [], []}} + # stacktrace: + # (spitfire 0.1.5) lib/spitfire.ex:2386: Spitfire.peek_token/1 + # (spitfire 0.1.5) lib/spitfire.ex:318: anonymous fn/7 in Spitfire.parse_expression/6 + # (spitfire 0.1.5) lib/spitfire/while.ex:49: Spitfire.While.do_while/2 + # (spitfire 0.1.5) lib/spitfire.ex:196: anonymous fn/1 in Spitfire.parse_program/1 + # (spitfire 0.1.5) lib/spitfire/while.ex:5: Spitfire.While2.recurse/3 + # (spitfire 0.1.5) lib/spitfire.ex:195: Spitfire.parse_program/1 + # (spitfire 0.1.5) lib/spitfire.ex:140: Spitfire.parse/2 + # test/spitfire_test.exs:2936: (test) + # code = "[\"abc#{Enum.random(1..10000)}\": 1]" + + # # tokenizer return error + # assert Spitfire.parse(code, existing_atoms_only: true) == {:ok, {:__block__, [], []}} + + # TODO this code errors with a wrong message + # { + # :error, + # {:%{}, [{:closing, []}, {:line, 1}, {:column, 1}], [{:__block__, [error: true], []}]}, + # [{[], "unknown token: eot"}, {[], "missing closing brace for map"}] + # } + + # code = "%{\"abc#{Enum.random(1..10000)}\": 1}" + + # # tokenizer return error + # assert Spitfire.parse(code, existing_atoms_only: true) == {:ok, {:__block__, [], []}} + + code = """ + ["abc\#{foo}": 1] + """ + + # tokenizer returns kw_identifier_safe token + assert Spitfire.parse(code, existing_atoms_only: true) == s2q(code, existing_atoms_only: true) + end + end + defp s2q(code, opts \\ []) do Code.string_to_quoted(code, Keyword.merge([columns: true, token_metadata: true, emit_warnings: false], opts)) end