feat: folding ranges

apps/expert/lib/expert.ex

@@ -234,6 +234,9 @@ defmodule Expert do
       %GenLSP.Requests.WorkspaceSymbol{} ->
         {:ok, Handlers.WorkspaceSymbol}
 
+      %GenLSP.Requests.TextDocumentFoldingRange{} ->
+        {:ok, Handlers.FoldingRange}
+
       %request_module{} ->
         {:error, {:unhandled, request_module}}
     end

apps/expert/lib/expert/provider/handlers/folding_range.ex

@@ -0,0 +1,147 @@
+defmodule Expert.Provider.Handlers.FoldingRange do
+  @moduledoc """
+  ## Methodology
+
+  ### High level
+
+  We make multiple passes (currently 4) through the source text and create
+  folding ranges from each pass.
+  Then we merge the ranges from each pass to provide the final ranges.
+  Each pass gets a priority to help break ties (the priority is an integer,
+  higher integers win).
+
+  ### Indentation pass (priority: 1)
+
+  We use the indentation level -- determined by the column of the first
+  non-whitespace character on each line -- to provide baseline ranges.
+  All ranges from this pass are `kind?: :region` ranges.
+
+  ### Comment block pass (priority: 2)
+
+  We let "comment blocks", consecutive lines starting with `#`, from regions.
+  All ranges from this pass are `kind?: :comment` ranges.
+
+  ### Token-pairs pass (priority: 3)
+
+  We use pairs of tokens, e.g. `do` and `end`, to provide another pass of
+  ranges.
+  All ranges from this pass are `kind?: :region` ranges.
+
+  ### Special tokens pass (priority: 3)
+
+  We find strings (regular/charlist strings/heredocs) and sigils in a pass as
+  they're delimited by a few special tokens.
+  Ranges from this pass are either
+  - `kind?: :comment` if the token is paired with `@doc` or `@moduledoc`, or
+  - `kind?: :region` otherwise.
+
+  ## Notes
+
+  Each pass may return ranges in any order.
+  But all ranges are valid, i.e. end_line > start_line.
+  """
+  alias Expert.Provider.Handlers.FoldingRange
+  alias Expert.Provider.Handlers.FoldingRange.Token
+  alias Forge.Document
+  alias GenLSP.Requests
+  alias GenLSP.Structures
+
+  import Forge.Document.Line
+
+  def handle(%Requests.TextDocumentFoldingRange{params: %Structures.FoldingRangeParams{} = params}, _config) do
+    document = Document.Container.context_document(params, nil)
+
+    input = document_to_input(document)
+
+    passes_with_priority = [
+      {1, FoldingRange.Indentation},
+      {2, FoldingRange.CommentBlock},
+      {3, FoldingRange.TokenPair},
+      {3, FoldingRange.SpecialToken}
+    ]
+
+    ranges =
+      passes_with_priority
+      |> Enum.map(fn {priority, pass} ->
+        ranges = ranges_from_pass(pass, input)
+        {priority, ranges}
+      end)
+      |> merge_ranges_with_priorities()
+
+    {:ok, ranges}
+  end
+
+  def document_to_input(document) do
+    %{
+      tokens: tokens(document),
+      lines: lines(document)
+    }
+  end
+
+  defp tokens(document) do
+    text = Document.to_string(document)
+    Token.format_string(text)
+  end
+
+  defp lines(document) do
+    for idx <- 1..(Forge.Document.Lines.size(document.lines) - 1) do
+      {:ok, line} = Forge.Document.Lines.fetch_line(document.lines, idx)
+      {line, indentation(line)}
+    end
+  end
+
+  defp indentation(line) do
+    text = line(line, :text)
+    ascii? = line(line, :ascii?)
+    full_length = line_length(text, ascii?)
+    trimmed = String.trim_leading(text)
+    trimmed_length = line_length(trimmed, ascii?)
+
+    if {full_length, trimmed_length} == {0, 0} do
+      nil
+    else
+      full_length - trimmed_length
+    end
+  end
+
+  defp line_length(text, true) do
+    text
+    |> byte_size()
+    |> div(2)
+  end
+
+  defp line_length(text, false) do
+    text
+    |> characters_to_binary!(:utf8, :utf16)
+    |> byte_size()
+    |> div(2)
+  end
+
+  defp characters_to_binary!(binary, from, to) do
+    case :unicode.characters_to_binary(binary, from, to) do
+      result when is_binary(result) -> result
+    end
+  end
+
+  defp ranges_from_pass(pass, input) do
+    with {:ok, ranges} <- pass.provide_ranges(input) do
+      ranges
+    else
+      _ -> []
+    end
+  end
+
+  defp merge_ranges_with_priorities(range_lists_with_priorities) do
+    range_lists_with_priorities
+    |> Enum.flat_map(fn {priority, ranges} -> Enum.zip(Stream.cycle([priority]), ranges) end)
+    |> Enum.group_by(fn {_priority, range} -> range.start_line end)
+    |> Enum.map(fn {_start, ranges_with_priority} ->
+      {_priority, range} =
+        ranges_with_priority
+        |> Enum.max_by(fn {priority, range} -> {priority, range.end_line} end)
+
+      range
+    end)
+    |> Enum.sort_by(& &1.start_line)
+  end
+end

apps/expert/lib/expert/provider/handlers/folding_range/comment_block.ex

@@ -0,0 +1,56 @@
+defmodule Expert.Provider.Handlers.FoldingRange.CommentBlock do
+  @moduledoc """
+  Code folding based on comment blocks
+
+  Note that this implementation can create comment ranges inside heredocs.
+  It's a little sloppy, but it shouldn't be very impactful.
+  We'd have to merge the token and line representations of the source text to
+  mitigate this issue, so we've left it as is for now.
+  """
+  alias Expert.Provider.Handlers.FoldingRange.Helpers
+
+  import Forge.Document.Line
+
+  def provide_ranges(%{lines: lines}) do
+    ranges =
+      lines
+      |> Enum.map(&extract_cell/1)
+      |> group_comments()
+      |> Enum.filter(fn group -> length(group) > 1 end)
+      |> Enum.map(&convert_comment_group_to_range/1)
+
+    {:ok, ranges}
+  end
+
+  def extract_cell({line(line_number: line), indentation}), do: {line, indentation}
+
+  def group_comments(lines) do
+    lines
+    |> Enum.reduce([[]], fn
+      {_, cell, "#"}, [[{_, "#"} | _] = head | tail] ->
+        [[{cell, "#"} | head] | tail]
+
+      {_, cell, "#"}, [[] | tail] ->
+        [[{cell, "#"}] | tail]
+
+      _, [[{_, "#"} | _] | _] = acc ->
+        [[] | acc]
+
+      _, acc ->
+        acc
+    end)
+  end
+
+  defp convert_comment_group_to_range(group) do
+    {{{end_line, _}, _}, {{start_line, _}, _}} =
+      Helpers.first_and_last_of_list(group)
+
+    %GenLSP.Structures.FoldingRange{
+      start_line: start_line,
+      # We're not doing end_line - 1 on purpose.
+      # It seems weird to show the first _and_ last line of a comment block.
+      end_line: end_line,
+      kind: GenLSP.Enumerations.FoldingRangeKind.comment()
+    }
+  end
+end

apps/expert/lib/expert/provider/handlers/folding_range/helpers.ex

@@ -0,0 +1,13 @@
+defmodule Expert.Provider.Handlers.FoldingRange.Helpers do
+  @moduledoc false
+
+  def first_and_last_of_list([]), do: :empty_list
+
+  def first_and_last_of_list([head]), do: {head, head}
+
+  def first_and_last_of_list([head, last]), do: {head, last}
+
+  def first_and_last_of_list([head | tail]) do
+    {head, List.last(tail)}
+  end
+end

apps/expert/lib/expert/provider/handlers/folding_range/indentation.ex

@@ -0,0 +1,74 @@
+defmodule Expert.Provider.Handlers.FoldingRange.Indentation do
+  @moduledoc """
+  Code folding based on indentation level
+
+  Note that we trim trailing empty rows from regions.
+  """
+
+  import Forge.Document.Line
+
+  def provide_ranges(%{lines: lines}) do
+    ranges = lines
+      |> Enum.map(&extract_cell/1)
+      |> pair_cells()
+      |> pairs_to_ranges()
+
+    {:ok, ranges}
+  end
+
+  def extract_cell({line(line_number: line), indentation}), do: {line, indentation}
+
+  @doc """
+  Pairs cells into {start, end} tuples of regions
+  Public function for testing
+  """
+  def pair_cells(cells) do
+    do_pair_cells(cells, [], [], [])
+  end
+
+  # Base case
+  defp do_pair_cells([], _, _, pairs) do
+    pairs
+    |> Enum.map(fn
+      {cell1, cell2, []} -> {cell1, cell2}
+      {cell1, _, empties} -> {cell1, List.last(empties)}
+    end)
+    |> Enum.reject(fn {{r1, _}, {r2, _}} -> r1 + 1 >= r2 end)
+  end
+
+  # Empty row
+  defp do_pair_cells([{_, nil} = head | tail], stack, empties, pairs) do
+    do_pair_cells(tail, stack, [head | empties], pairs)
+  end
+
+  # Empty stack
+  defp do_pair_cells([head | tail], [], empties, pairs) do
+    do_pair_cells(tail, [head], empties, pairs)
+  end
+
+  # Non-empty stack: head is to the right of the top of the stack
+  defp do_pair_cells([{_, x} = head | tail], [{_, y} | _] = stack, _, pairs) when x > y do
+    do_pair_cells(tail, [head | stack], [], pairs)
+  end
+
+  # Non-empty stack: head is equal to or to the left of the top of the stack
+  defp do_pair_cells([{_, x} = head | tail], stack, empties, pairs) do
+    # If the head is <= to the top of the stack, then we need to pair it with
+    # everything on the stack to the right of it.
+    # The head can also start a new region, so it's pushed onto the stack.
+    {leftovers, new_tail_stack} = stack |> Enum.split_while(fn {_, y} -> x <= y end)
+    new_pairs = leftovers |> Enum.map(&{&1, head, empties})
+    do_pair_cells(tail, [head | new_tail_stack], [], new_pairs ++ pairs)
+  end
+
+  defp pairs_to_ranges(pairs) do
+    pairs
+    |> Enum.map(fn {{r1, _}, {r2, _}} ->
+      %GenLSP.Structures.FoldingRange{
+        start_line: r1,
+        end_line: r2 - 1,
+        kind: GenLSP.Enumerations.FoldingRangeKind.region()
+      }
+    end)
+  end
+end