fix crash when LSP cursor position is exactly in the middle of UTF16 high surrogate pair

Author: lukaszsamson

apps/debug_adapter/lib/debug_adapter/utils.ex

@@ -28,22 +28,39 @@ defmodule ElixirLS.DebugAdapter.Utils do
   def dap_character_to_elixir(_utf8_line, dap_character) when dap_character <= 0, do: 0
 
   def dap_character_to_elixir(utf8_line, dap_character) do
-    utf16_line =
-      utf8_line
-      |> characters_to_binary!(:utf8, :utf16)
-
-    byte_size = byte_size(utf16_line)
-
-    utf8_character =
-      utf16_line
-      |> (&binary_part(
-            &1,
-            0,
-            min(dap_character * 2, byte_size)
-          )).()
-      |> characters_to_binary!(:utf16, :utf8)
-      |> String.length()
-
-    utf8_character
+    utf16_line = characters_to_binary!(utf8_line, :utf8, :utf16)
+    max_bytes = byte_size(utf16_line)
+
+    # LSP character -> code units -> bytes
+    offset0 = dap_character * 2
+    offset = clamp_offset_to_surrogate_boundary(utf16_line, offset0, max_bytes)
+
+    partial = binary_part(utf16_line, 0, offset)
+    partial_utf8 = characters_to_binary!(partial, :utf16, :utf8)
+    String.length(partial_utf8)
+  end
+
+  # “Clamp” helper. 
+  # - If offset is out of bounds, keep it within [0, max_bytes].
+  # - Then check if we landed *immediately* after a high surrogate (0xD800..0xDBFF);
+  #   if so, subtract 2 to avoid slicing in the middle.
+  defp clamp_offset_to_surrogate_boundary(_bin, offset, max_bytes) when offset >= max_bytes,
+    do: max_bytes
+
+  defp clamp_offset_to_surrogate_boundary(_bin, offset, _max_bytes) when offset <= 0,
+    do: 0
+
+  defp clamp_offset_to_surrogate_boundary(bin, offset, _max_bytes) do
+    # We know 0 < offset < max_bytes at this point
+    # Look at the 2 bytes immediately before `offset`
+    <<_::binary-size(offset - 2), maybe_high::binary-size(2), _::binary>> = bin
+    code_unit = :binary.decode_unsigned(maybe_high, :big)
+
+    # If that 16-bit code_unit is a high surrogate, we've sliced in half
+    if code_unit in 0xD800..0xDBFF do
+      offset - 2
+    else
+      offset
+    end
   end
 end

apps/debug_adapter/test/utils_test.exs

@@ -62,5 +62,11 @@ defmodule ElixirLS.DebugAdapter.UtilsTest do
     test "dap_character_to_elixir utf8" do
       assert 1 == Utils.dap_character_to_elixir("🏳️‍🌈abcde", 6)
     end
+
+    test "dap_character_to_elixir index inside high surrogate pair" do
+      assert 6 == Utils.dap_character_to_elixir("Hello 🙌 World", 6)
+      assert 6 == Utils.dap_character_to_elixir("Hello 🙌 World", 7)
+      assert 7 == Utils.dap_character_to_elixir("Hello 🙌 World", 8)
+    end
   end
 end

apps/language_server/lib/language_server/source_file.ex

@@ -44,30 +44,6 @@ defmodule ElixirLS.LanguageServer.SourceFile do
     do_lines_with_endings(rest, line <> <<char::utf8>>)
   end
 
-  def text_before(text, position_line, position_character) do
-    text
-    |> lines
-    |> Enum.with_index()
-    |> Enum.reduce_while([], fn
-      {line, count}, acc when count < position_line ->
-        {:cont, [line, ?\n | acc]}
-
-      {line, count}, acc when count == position_line ->
-        slice =
-          characters_to_binary!(line, :utf8, :utf16)
-          |> (&binary_part(
-                &1,
-                0,
-                min(position_character * 2, byte_size(&1))
-              )).()
-          |> characters_to_binary!(:utf16, :utf8)
-
-        {:halt, [slice, ?\n | acc]}
-    end)
-    |> Enum.reverse()
-    |> IO.iodata_to_binary()
-  end
-
   def apply_content_changes(%__MODULE__{} = source_file, []) do
     source_file
   end
@@ -339,23 +315,40 @@ defmodule ElixirLS.LanguageServer.SourceFile do
   def lsp_character_to_elixir(_utf8_line, lsp_character) when lsp_character <= 0, do: 1
 
   def lsp_character_to_elixir(utf8_line, lsp_character) do
-    utf16_line =
-      utf8_line
-      |> characters_to_binary!(:utf8, :utf16)
-
-    byte_size = byte_size(utf16_line)
-
-    utf8_character =
-      utf16_line
-      |> (&binary_part(
-            &1,
-            0,
-            min(lsp_character * 2, byte_size)
-          )).()
-      |> characters_to_binary!(:utf16, :utf8)
-      |> String.length()
-
-    utf8_character + 1
+    utf16_line = characters_to_binary!(utf8_line, :utf8, :utf16)
+    max_bytes = byte_size(utf16_line)
+
+    # LSP character -> code units -> bytes
+    offset0 = lsp_character * 2
+    offset = clamp_offset_to_surrogate_boundary(utf16_line, offset0, max_bytes)
+
+    partial = binary_part(utf16_line, 0, offset)
+    partial_utf8 = characters_to_binary!(partial, :utf16, :utf8)
+    String.length(partial_utf8) + 1
+  end
+
+  # “Clamp” helper. 
+  # - If offset is out of bounds, keep it within [0, max_bytes].
+  # - Then check if we landed *immediately* after a high surrogate (0xD800..0xDBFF);
+  #   if so, subtract 2 to avoid slicing in the middle.
+  defp clamp_offset_to_surrogate_boundary(_bin, offset, max_bytes) when offset >= max_bytes,
+    do: max_bytes
+
+  defp clamp_offset_to_surrogate_boundary(_bin, offset, _max_bytes) when offset <= 0,
+    do: 0
+
+  defp clamp_offset_to_surrogate_boundary(bin, offset, _max_bytes) do
+    # We know 0 < offset < max_bytes at this point
+    # Look at the 2 bytes immediately before `offset`
+    <<_::binary-size(offset - 2), maybe_high::binary-size(2), _::binary>> = bin
+    code_unit = :binary.decode_unsigned(maybe_high, :big)
+
+    # If that 16-bit code_unit is a high surrogate, we've sliced in half
+    if code_unit in 0xD800..0xDBFF do
+      offset - 2
+    else
+      offset
+    end
   end
 
   def lsp_position_to_elixir(_urf8_text_or_lines, {lsp_line, _lsp_character}) when lsp_line < 0,

apps/language_server/test/source_file_test.exs

@@ -653,6 +653,12 @@ defmodule ElixirLS.LanguageServer.SourceFileTest do
       assert {1, 2} == SourceFile.lsp_position_to_elixir("🏳️‍🌈abcde", {0, 6})
     end
 
+    test "lsp_position_to_elixir single line index inside high surrogate pair" do
+      assert {1, 7} == SourceFile.lsp_position_to_elixir("Hello 🙌 World", {0, 6})
+      assert {1, 7} == SourceFile.lsp_position_to_elixir("Hello 🙌 World", {0, 7})
+      assert {1, 8} == SourceFile.lsp_position_to_elixir("Hello 🙌 World", {0, 8})
+    end
+
     test "lsp_position_to_elixir multi line" do
       assert {2, 2} == SourceFile.lsp_position_to_elixir("abcde\n1234", {1, 1})
     end