Convert highlight into DocAST

Author: josevalim

lib/ex_doc/doc_ast.ex

@@ -1,3 +1,5 @@
+import Kernel, except: [to_string: 1]
+
 defmodule ExDoc.DocAST do
   # General helpers for dealing with the documentation AST
   # (which is the Markdown -> HTML AST).
@@ -198,41 +200,58 @@ defmodule ExDoc.DocAST do
   defp pivot([], acc, _headers), do: Enum.reverse(acc)
 
   def highlight(html, language, opts \\ []) do
-    highlight_info = language.highlight_info()
-
-    ## Html cannot be parsed with regex, but we try our best...
-    Regex.replace(
-      ~r/<pre(\s[^>]*)?><code(?:\s+class="([^"\s]*)")?>([^<]*)<\/code><\/pre>/,
-      html,
-      &highlight_code_block(&1, &2, &3, &4, highlight_info, opts)
-    )
+    do_highlight(html, language.highlight_info(), opts)
   end
 
-  defp highlight_code_block(full_block, pre_attr, lang, code, highlight_info, outer_opts) do
+  defp do_highlight(
+         {:pre, pre_attrs, [{:code, code_attrs, [code], code_meta}], pre_meta} = ast,
+         highlight_info,
+         opts
+       )
+       when is_binary(code) do
+    {lang, code_attrs} = Keyword.pop(code_attrs, :class, "")
+
     case pick_language_and_lexer(lang, highlight_info, code) do
-      {_language, nil, _opts} ->
-        full_block
+      {_lang, nil, _lexer_opts} ->
+        ast
 
-      {lang, lexer, opts} ->
+      {lang, lexer, lexer_opts} ->
         try do
-          render_code(pre_attr, lang, lexer, opts, code, outer_opts)
+          Makeup.highlight_inner_html(code,
+            lexer: lexer,
+            lexer_options: lexer_opts,
+            formatter_options: opts
+          )
         rescue
           exception ->
             ExDoc.Utils.warn(
               [
                 "crashed while highlighting #{lang} snippet:\n\n",
-                full_block,
+                ExDoc.DocAST.to_string(ast),
                 "\n\n",
                 Exception.format_banner(:error, exception, __STACKTRACE__)
               ],
               __STACKTRACE__
             )
 
-            full_block
+            ast
+        else
+          highlighted ->
+            code_attrs = [class: "makeup #{lang}", translate: "no"] ++ code_attrs
+            code_meta = Map.put(code_meta, :verbatim, true)
+            {:pre, pre_attrs, [{:code, code_attrs, [highlighted], code_meta}], pre_meta}
         end
     end
   end
 
+  defp do_highlight(list, highlight_info, opts) when is_list(list) do
+    Enum.map(list, &do_highlight(&1, highlight_info, opts))
+  end
+
+  defp do_highlight(other, _highlight_info, _opts) do
+    other
+  end
+
   defp pick_language_and_lexer("", _highlight_info, "$ " <> _) do
     {"shell", ExDoc.ShellLexer, []}
   end
@@ -251,36 +270,4 @@ defmodule ExDoc.DocAST do
       :error -> {lang, nil, []}
     end
   end
-
-  defp render_code(pre_attr, lang, lexer, lexer_opts, code, opts) do
-    highlight_tag = Keyword.get(opts, :highlight_tag, "span")
-
-    highlighted =
-      code
-      |> unescape_html()
-      |> IO.iodata_to_binary()
-      |> Makeup.highlight_inner_html(
-        lexer: lexer,
-        lexer_options: lexer_opts,
-        formatter_options: [highlight_tag: highlight_tag]
-      )
-
-    ~s(<pre#{pre_attr}><code class="makeup #{lang}" translate="no">#{highlighted}</code></pre>)
-  end
-
-  entities = [{"&amp;", ?&}, {"&lt;", ?<}, {"&gt;", ?>}, {"&quot;", ?"}, {"&#39;", ?'}]
-
-  for {encoded, decoded} <- entities do
-    defp unescape_html(unquote(encoded) <> rest) do
-      [unquote(decoded) | unescape_html(rest)]
-    end
-  end
-
-  defp unescape_html(<<c, rest::binary>>) do
-    [c | unescape_html(rest)]
-  end
-
-  defp unescape_html(<<>>) do
-    []
-  end
 end

lib/ex_doc/formatter/html.ex

@@ -144,8 +144,8 @@ defmodule ExDoc.Formatter.HTML do
   defp autolink_and_render(doc, language, autolink_opts, opts) do
     doc
     |> language.autolink_doc(autolink_opts)
-    |> ExDoc.DocAST.to_string()
     |> ExDoc.DocAST.highlight(language, opts)
+    |> ExDoc.DocAST.to_string()
   end
 
   defp output_setup(build, config) do

test/ex_doc/doc_ast_test.exs

@@ -158,54 +158,51 @@ defmodule ExDoc.DocASTTest do
 
   describe "highlight" do
     test "with default class" do
-      # Empty class
-      assert DocAST.highlight(
-               ~S[<pre><code class="">mix run --no-halt path/to/file.exs</code></pre>],
-               ExDoc.Language.Elixir
-             ) =~
+      # Four spaces
+      assert highlight("""
+                 mix run --no-halt path/to/file.exs
+             """) =~
                ~r{<pre><code class=\"makeup elixir\" translate="no">.*}
 
-      # Without class
-      assert DocAST.highlight(
-               "<pre><code>mix run --no-halt path/to/file.exs</code></pre>",
-               ExDoc.Language.Elixir
-             ) =~
+      # Code block without language
+      assert highlight("""
+             ```
+             mix run --no-halt path/to/file.exs</code></pre>
+             ```
+             """) =~
                ~r{<pre><code class=\"makeup elixir\" translate="no">.*}
 
-      # Pre class
-      assert DocAST.highlight(
-               ~S[<pre class="wrap"><code class="">mix run --no-halt path/to/file.exs</code></pre>],
-               ExDoc.Language.Elixir
-             ) =~
+      # Pre IAL
+      assert highlight("""
+             ```
+             mix run --no-halt path/to/file.exs</code></pre>
+             ```
+             {:class="wrap"}
+             """) =~
                ~r{<pre class="wrap"><code class=\"makeup elixir\" translate="no">.*}
 
-      # Pre id
-      assert DocAST.highlight(
-               ~S[<pre id="anchor"><code class="">mix run --no-halt path/to/file.exs</code></pre>],
-               ExDoc.Language.Elixir
-             ) =~
-               ~r{<pre id="anchor"><code class=\"makeup elixir\" translate="no">.*}
-
-      # Pre id and class
-      assert DocAST.highlight(
-               ~S[<pre id="anchor" class="wrap"><code class="">mix run --no-halt path/to/file.exs</code></pre>],
-               ExDoc.Language.Elixir
-             ) =~
-               ~r{<pre id="anchor" class="wrap"><code class=\"makeup elixir\" translate="no">.*}
-
-      # IEx highlight with empty class
-      assert DocAST.highlight(
-               ~S[<pre><code class="">iex&gt; max(4, 5)</code></pre>],
-               ExDoc.Language.Elixir
-             ) =~
-               ~r{<pre><code class=\"makeup elixir\" translate="no">.*}
+      # Code with language
+      assert highlight("""
+             ```html
+             <foo />
+             ```
+             """) =~
+               ~r{<pre><code class=\"makeup html\" translate="no">.*}
+
+      # Code with shell detection
+      assert highlight("""
+             ```
+             $ hello
+             ```
+             """) =~
+               ~r{<pre><code class=\"makeup shell\" translate="no"><span class="gp unselectable">\$.*}
+    end
 
-      # IEx highlight without class
-      assert DocAST.highlight(
-               ~S[<pre><code>iex&gt; max(4, 5)</code></pre>],
-               ExDoc.Language.Elixir
-             ) =~
-               ~r{<pre><code class=\"makeup elixir\" translate="no">.*}
+    defp highlight(markdown) do
+      markdown
+      |> ExDoc.DocAST.parse!("text/markdown")
+      |> ExDoc.DocAST.highlight(ExDoc.Language.Elixir)
+      |> ExDoc.DocAST.to_string()
     end
   end