Consistently raise UnicodeConversionError in tokenizer (#14589)

Author: lukaszsamson

lib/elixir/src/elixir_tokenizer.erl

@@ -1032,18 +1032,42 @@ unsafe_to_atom(Binary, Line, Column, #elixir_tokenizer{existing_atoms_only=true}
   try
     {ok, binary_to_existing_atom(Binary, utf8)}
   catch
-    error:badarg -> {error, {?LOC(Line, Column), "unsafe atom does not exist: ", elixir_utils:characters_to_list(Binary)}}
+    error:badarg -> 
+      % Check if it's a UTF-8 issue by trying to convert to list
+      elixir_utils:characters_to_list(Binary),
+      % If we get here, it's not a UTF-8 issue
+      {error, {?LOC(Line, Column), "unsafe atom does not exist: ", elixir_utils:characters_to_list(Binary)}}
+  end;
+unsafe_to_atom(Binary, Line, Column, #elixir_tokenizer{}) when is_binary(Binary) ->
+  try
+    {ok, binary_to_atom(Binary, utf8)}
+  catch
+    error:badarg ->
+      % Try to convert using elixir_utils to get proper UnicodeConversionError
+      elixir_utils:characters_to_list(Binary),
+      % If we get here, it's not a UTF-8 issue, so it's some other badarg
+      {error, {?LOC(Line, Column), "invalid atom: ", elixir_utils:characters_to_list(Binary)}}
   end;
-unsafe_to_atom(Binary, _Line, _Column, #elixir_tokenizer{}) when is_binary(Binary) ->
-  {ok, binary_to_atom(Binary, utf8)};
 unsafe_to_atom(List, Line, Column, #elixir_tokenizer{existing_atoms_only=true}) when is_list(List) ->
   try
     {ok, list_to_existing_atom(List)}
   catch
-    error:badarg -> {error, {?LOC(Line, Column), "unsafe atom does not exist: ", List}}
+    error:badarg ->
+      % Try to convert using elixir_utils to get proper UnicodeConversionError
+      elixir_utils:characters_to_binary(List),
+      % If we get here, it's not a UTF-8 issue
+      {error, {?LOC(Line, Column), "unsafe atom does not exist: ", List}}
   end;
-unsafe_to_atom(List, _Line, _Column, #elixir_tokenizer{}) when is_list(List) ->
-  {ok, list_to_atom(List)}.
+unsafe_to_atom(List, Line, Column, #elixir_tokenizer{}) when is_list(List) ->
+  try
+    {ok, list_to_atom(List)}
+  catch
+    error:badarg ->
+      % Try to convert using elixir_utils to get proper UnicodeConversionError
+      elixir_utils:characters_to_binary(List),
+      % If we get here, it's not a UTF-8 issue, so it's some other badarg
+      {error, {?LOC(Line, Column), "invalid atom: ", List}}
+  end.
 
 collect_modifiers([H | T], Buffer) when ?is_downcase(H) or ?is_upcase(H) or ?is_digit(H) ->
   collect_modifiers(T, [H | Buffer]);

lib/elixir/test/elixir/code_test.exs

@@ -514,6 +514,27 @@ defmodule CodeTest do
              }
   end
 
+  test "string_to_quoted raises UnicodeConversionError for invalid UTF-8 in quoted atoms and function calls" do
+    invalid_utf8_cases = [
+      # Quoted atom
+      ~S{:"\xFF"},
+      ~S{:'\xFF'},
+      # Quoted function call
+      ~S{foo."\xFF"()},
+      ~S{foo.'\xFF'()}
+    ]
+
+    for code <- invalid_utf8_cases do
+      assert_raise UnicodeConversionError, fn ->
+        Code.string_to_quoted!(code)
+      end
+
+      assert_raise UnicodeConversionError, fn ->
+        Code.string_to_quoted!(code, existing_atoms_only: true)
+      end
+    end
+  end
+
   @tag :requires_source
   test "compile source" do
     assert __MODULE__.__info__(:compile)[:source] == String.to_charlist(__ENV__.file)