feat: major paragraph spacing improvements and context-aware processing

This release introduces significant architectural improvements to address
paragraph spacing issues and enhances the overall quality of Markdown output.

Changes:
- Fixed paragraph spacing within containers (resolves issue #6)
- Added context-aware processing with block vs inline element detection
- Introduced Html2Markdown.ElementTypes module for element classification
- Enhanced list formatting with proper line breaks between items
- Improved code block whitespace preservation using Base64 encoding workaround
- Better language detection for syntax-highlighted code blocks
- Streamlined processing logic for cleaner, more readable output

Breaking changes:
- Paragraph spacing now uses standard double newlines (\n\n) instead of
  single newline + space (\n \n) for better Markdown compliance
- More comprehensive content preservation (includes navigation elements)

Author: cpursley

README.md

@@ -13,7 +13,7 @@ Add `html2markdown` to your list of dependencies in `mix.exs`:
 ```elixir
 def deps do
   [
-    {:html2markdown, "~> 0.2.1"}
+    {:html2markdown, "~> 0.3.0"}
   ]
 end
 ```

lib/html2markdown/converter.ex

@@ -23,7 +23,7 @@ defmodule Html2Markdown.Converter do
   - Preserves whitespace in code blocks while normalizing elsewhere
   """
 
-  alias Html2Markdown.{TableConverter, Options}
+  alias Html2Markdown.{TableConverter, Options, ElementTypes}
 
   @spec convert_to_markdown(list(Floki.html_node()), Options.t()) :: String.t()
   def convert_to_markdown(document, opts) do
@@ -226,10 +226,10 @@ defmodule Html2Markdown.Converter do
   defp process_node_to_iolist({"hr", _, _}, _opts), do: "\n\n---\n\n"
 
   defp process_node_to_iolist({"section", _, children}, opts),
-    do: ["\n", process_children_to_iolist(children, opts), "\n"]
+    do: process_children_with_context(children, opts, :block)
 
   defp process_node_to_iolist({"article", _, children}, opts),
-    do: ["\n", process_children_to_iolist(children, opts), "\n"]
+    do: process_children_with_context(children, opts, :block)
 
   defp process_node_to_iolist({"picture", _, children}, opts) do
     case Enum.find(children, fn
@@ -250,7 +250,20 @@ defmodule Html2Markdown.Converter do
   end
 
   defp process_node_to_iolist({"div", _, children}, opts),
-    do: [process_children_to_iolist(children, opts), "\n"]
+    do: process_children_with_context(children, opts, :block)
+
+  # Handle spans with preserved whitespace
+  defp process_node_to_iolist({"span", attrs, children}, opts) do
+    case List.keyfind(attrs, "data-ws", 0) do
+      {"data-ws", encoded} ->
+        # Decode preserved whitespace
+        Base.decode64!(encoded)
+
+      _ ->
+        # Normal span processing
+        process_children_to_iolist(children, opts)
+    end
+  end
 
   defp process_node_to_iolist({_, _, children}, opts),
     do: process_children_to_iolist(children, opts)
@@ -261,6 +274,7 @@ defmodule Html2Markdown.Converter do
       |> String.trim()
       |> normalize_whitespace()
     else
+      # When not normalizing whitespace (e.g., in code blocks), preserve text exactly as-is
       text
     end
   end
@@ -296,9 +310,18 @@ defmodule Html2Markdown.Converter do
   end
 
   defp detect_language(classes) do
-    case Regex.run(~r/language-(\w+)/, classes) do
-      [_, lang] -> lang
-      _ -> ""
+    cond do
+      # First check for standard language- prefix
+      match = Regex.run(~r/language-(\w+)/, classes) ->
+        elem(List.to_tuple(match), 1)
+
+      # Check for makeup syntax highlighting classes
+      match = Regex.run(~r/makeup (\w+)/, classes) ->
+        elem(List.to_tuple(match), 1)
+
+      # Default to empty string
+      true ->
+        ""
     end
   end
 
@@ -403,16 +426,79 @@ defmodule Html2Markdown.Converter do
   defp process_ordered_list_item_to_iolist(other, _index, opts),
     do: process_node_to_iolist(other, opts)
 
-  defp process_children_to_iolist(children, opts) do
+  # Context-aware processing for better spacing control  
+  defp process_children_with_context(children, opts, context) do
+    final_context = determine_context(children, context)
+
+    case final_context do
+      :block -> process_block_children(children, opts)
+      :inline -> process_inline_children(children, opts)
+    end
+  end
+
+  # Determine processing context based on children content
+  defp determine_context(children, :auto) do
+    has_block_elements =
+      Enum.any?(children, fn
+        {tag, _, _} when is_binary(tag) -> ElementTypes.block_element?(tag)
+        _ -> false
+      end)
+
+    if has_block_elements, do: :block, else: :inline
+  end
+
+  defp determine_context(_children, context), do: context
+
+  # Process block children with proper spacing between block elements
+  defp process_block_children(children, opts) do
+    children
+    |> Enum.filter(&ElementTypes.content_node?/1)
+    |> Enum.map(&process_node_to_iolist(&1, opts))
+    |> Enum.reject(&ElementTypes.empty_content?/1)
+    |> join_with_block_spacing()
+  end
+
+  # Process inline children with smart spacing (existing logic)
+  defp process_inline_children(children, opts) do
     iolist =
       children
       |> Enum.map(&process_node_to_iolist(&1, opts))
+
+    # Only apply smart spacing and trim when normalizing whitespace
+    if opts.normalize_whitespace do
+      iolist
       |> join_with_smart_spacing()
+      |> IO.iodata_to_binary()
+      |> String.trim()
+    else
+      # When not normalizing (e.g., in code blocks), just return the iolist as-is
+      iolist
+    end
+  end
+
+  # Join block elements with proper spacing (double newlines)
+  defp join_with_block_spacing([]), do: []
+
+  defp join_with_block_spacing([first | rest]) do
+    Enum.reduce(rest, [first], fn item, acc ->
+      [acc, "\n\n", item]
+    end)
+  end
+
+  # Legacy function maintained for backward compatibility
+  defp process_children_to_iolist(children, opts) do
+    iolist =
+      children
+      |> Enum.map(&process_node_to_iolist(&1, opts))
 
-    # Only trim if we're normalizing whitespace
+    # Only apply smart spacing and trim when normalizing whitespace
     if opts.normalize_whitespace do
-      iolist |> IO.iodata_to_binary() |> String.trim()
+      iolist
+      |> join_with_smart_spacing()
+      |> IO.iodata_to_binary()
+      |> String.trim()
     else
+      # When not normalizing (e.g., in code blocks), just return the iolist as-is
       iolist
     end
   end

lib/html2markdown/element_types.ex

@@ -0,0 +1,131 @@
+defmodule Html2Markdown.ElementTypes do
+  @moduledoc """
+  Provides element type classification for HTML to Markdown conversion.
+
+  This module categorizes HTML elements into block, inline, and other types
+  to enable proper spacing and formatting decisions during conversion.
+  """
+
+  @block_elements ~w[
+    p div h1 h2 h3 h4 h5 h6 ul ol li blockquote pre
+    article section aside header footer main nav
+    table tr td th thead tbody tfoot
+    dl dt dd figure figcaption
+    details summary
+  ]
+
+  @inline_elements ~w[
+    a em strong code span i b u del sup sub
+    img br hr abbr cite q time mark
+  ]
+
+  @list_elements ~w[ul ol li]
+
+  @heading_elements ~w[h1 h2 h3 h4 h5 h6]
+
+  @doc """
+  Determines if an HTML element is a block-level element.
+
+  Block elements typically start on a new line and take up the full width
+  available. They should be separated by blank lines in markdown.
+
+  ## Examples
+
+      iex> Html2Markdown.ElementTypes.block_element?("p")
+      true
+
+      iex> Html2Markdown.ElementTypes.block_element?("span")
+      false
+  """
+  @spec block_element?(String.t()) :: boolean()
+  def block_element?(tag) when tag in @block_elements, do: true
+  def block_element?(_), do: false
+
+  @doc """
+  Determines if an HTML element is an inline element.
+
+  Inline elements flow within text and don't create line breaks.
+  They should not have extra spacing added around them.
+
+  ## Examples
+
+      iex> Html2Markdown.ElementTypes.inline_element?("em")
+      true
+
+      iex> Html2Markdown.ElementTypes.inline_element?("p")
+      false
+  """
+  @spec inline_element?(String.t()) :: boolean()
+  def inline_element?(tag) when tag in @inline_elements, do: true
+  def inline_element?(_), do: false
+
+  @doc """
+  Determines if an HTML element is a list element.
+
+  ## Examples
+
+      iex> Html2Markdown.ElementTypes.list_element?("ul")
+      true
+
+      iex> Html2Markdown.ElementTypes.list_element?("li")
+      true
+
+      iex> Html2Markdown.ElementTypes.list_element?("p")
+      false
+  """
+  @spec list_element?(String.t()) :: boolean()
+  def list_element?(tag) when tag in @list_elements, do: true
+  def list_element?(_), do: false
+
+  @doc """
+  Determines if an HTML element is a heading element.
+
+  ## Examples
+
+      iex> Html2Markdown.ElementTypes.heading_element?("h1")
+      true
+
+      iex> Html2Markdown.ElementTypes.heading_element?("h7")
+      false
+  """
+  @spec heading_element?(String.t()) :: boolean()
+  def heading_element?(tag) when tag in @heading_elements, do: true
+  def heading_element?(_), do: false
+
+  @doc """
+  Determines if an element should be treated as content.
+
+  This is used to filter out empty text nodes, comments, etc.
+  """
+  @spec content_node?(Floki.html_node()) :: boolean()
+  def content_node?({:comment, _}), do: false
+  def content_node?({tag, _, _}) when is_binary(tag), do: true
+
+  def content_node?(text) when is_binary(text) do
+    String.trim(text) != ""
+  end
+
+  def content_node?(_), do: false
+
+  @doc """
+  Determines if processed content is empty.
+
+  Used to filter out elements that produce no markdown output.
+  """
+  @spec empty_content?(iodata()) :: boolean()
+  def empty_content?([]), do: true
+  def empty_content?(""), do: true
+
+  def empty_content?(content) when is_binary(content) do
+    String.trim(content) == ""
+  end
+
+  def empty_content?(content) when is_list(content) do
+    content
+    |> IO.iodata_to_binary()
+    |> String.trim()
+    |> then(&(&1 == ""))
+  end
+
+  def empty_content?(_), do: false
+end

lib/html2markdown/parser.ex

@@ -38,7 +38,40 @@ defmodule Html2Markdown.Parser do
   end
 
   defp prep_document(content) do
-    if is_html_document?(content), do: content, else: wrap_fragment(content)
+    content = if is_html_document?(content), do: content, else: wrap_fragment(content)
+
+    # Preserve whitespace in code blocks by replacing whitespace spans
+    # with placeholders that won't be stripped by Floki
+    preserve_code_whitespace(content)
+  end
+
+  # Replace whitespace-only spans in code blocks with placeholders
+  defp preserve_code_whitespace(content) do
+    # Match <pre> or <code> blocks and preserve whitespace within them
+    content
+    |> String.replace(~r/<(pre|code)([^>]*)>(.*?)<\/\1>/s, fn full_match ->
+      case Regex.run(~r/<(pre|code)([^>]*)>(.*?)<\/\1>/s, full_match) do
+        [_, tag, attrs, inner] ->
+          preserved_inner =
+            inner
+            |> String.replace(~r/<span([^>]*class="w"[^>]*)>([^<]*)<\/span>/, fn span_match ->
+              case Regex.run(~r/<span([^>]*class="w"[^>]*)>([^<]*)<\/span>/, span_match) do
+                [_, span_attrs, ws_content] ->
+                  # Encode whitespace content to preserve it
+                  encoded = Base.encode64(ws_content)
+                  ~s(<span#{span_attrs} data-ws="#{encoded}"></span>)
+
+                _ ->
+                  span_match
+              end
+            end)
+
+          "<#{tag}#{attrs}>#{preserved_inner}</#{tag}>"
+
+        _ ->
+          full_match
+      end
+    end)
   end
 
   defp is_html_document?(content) do

mix.exs

@@ -4,7 +4,7 @@ defmodule Html2Markdown.MixProject do
   def project do
     [
       app: :html2markdown,
-      version: "0.2.1",
+      version: "0.3.0",
       elixir: "~> 1.15",
       start_permanent: Mix.env() == :prod,
       description: description(),