feat: Implement advanced JSON repair patterns and pre-processing

This commit introduces a new pre-processing pipeline and several new repair modules to handle complex malformed JSON patterns, inspired by the json-repair Python library.

New Pre-processing Pipeline:

- A pre-processing stage has been added before the main layer pipeline to handle specific patterns that would otherwise be broken by subsequent layers.
- Multiple JSON Detection: A new MultipleJsonDetector utility now runs first to detect and aggregate consecutive JSON values (e.g., []{} becomes [[], {}]). This prevents Layer 1 from treating subsequent JSON as "wrapper text".
- Object Boundary Merging: The ObjectMerger now runs before Layer 1 to merge key-value pairs that appear after a premature closing brace (e.g., {"a":"b"},"c":"d"}).

New Layer 3 Filters:

Ellipsis Filter: A new module to remove unquoted ellipsis (...) placeholders, which are common in LLM-generated content.
Keyword Filter: A new module to remove unquoted, comment-like keywords such as COMMENT, DEBUG_INFO, and PLACEHOLDER.

Enhancements:

- Layer 1 (ContentCleaning): The trailing wrapper text removal logic is now smarter, checking if the trailing content is another valid JSON object before removing it.
- Tests: Previously skipped tests for patterns 1-4 (Multiple JSON, Object Merging, Ellipsis, and Keywords) have been enabled and are now passing.
- Cleanup: Removed temporary test scripts (test_boolean.exs, test_weiss.exs).

Author: nshkrdotcom

check_layer5_usage.exs

@@ -0,0 +1,21 @@
+# Test if Layer 5 is actually being called
+input = "[]{}"
+IO.puts("Input: #{input}")
+
+result = JsonRemedy.repair(input, logging: true)
+
+case result do
+  {:ok, data, repairs} ->
+    IO.puts("Result: #{inspect(data)}")
+    IO.puts("\nRepairs:")
+
+    Enum.each(repairs, fn r ->
+      IO.puts("  - Layer: #{r.layer}")
+    end)
+
+    layer5_used = Enum.any?(repairs, fn r -> r.layer == :tolerant_parsing end)
+    IO.puts("\nLayer 5 used? #{layer5_used}")
+
+  _ ->
+    IO.puts("Failed")
+end

lib/json_remedy.ex

@@ -7,8 +7,9 @@ defmodule JsonRemedy do
 
   This module provides the main API for JSON repair functionality. It supports
   multiple repair strategies and can handle various types of malformed JSON through
-  a four-layer processing pipeline:
+  a four-layer processing pipeline with intelligent preprocessing:
 
+  - **Preprocessing**: Multiple JSON detection (Pattern 1)
   - **Layer 1**: Content Cleaning (removes code fences, comments, extra text)
   - **Layer 2**: Structural Repair (fixes missing braces, brackets, etc.)
   - **Layer 3**: Syntax Normalization (quotes, booleans, commas, colons)
@@ -33,6 +34,7 @@ defmodule JsonRemedy do
   alias JsonRemedy.Layer2.StructuralRepair
   alias JsonRemedy.Layer3.SyntaxNormalization
   alias JsonRemedy.Layer4.Validation
+  alias JsonRemedy.Utils.MultipleJsonDetector
 
   # Type definitions
   @type json_value ::
@@ -321,14 +323,50 @@ defmodule JsonRemedy do
   # Private implementation functions
 
   defp process_through_pipeline(input, context, logging) do
+    # Pre-processing: Detect multiple JSON values (Pattern 1)
+    # Must run BEFORE layers because:
+    # - Layer 1 may remove additional JSON as "wrapper text"
+    # - Layer 3 adds commas between ]{ which breaks the pattern
+    enable_multiple_json =
+      Application.get_env(:json_remedy, :enable_multiple_json_aggregation, true)
+
+    if enable_multiple_json do
+      case MultipleJsonDetector.parse_multiple(input) do
+        {:ok, result} when is_list(result) and length(result) > 1 ->
+          # Multiple values detected
+          if logging do
+            {:ok, result, context.repairs}
+          else
+            {:ok, result}
+          end
+
+        _ ->
+          # Single value, continue normal pipeline
+          process_normal_pipeline(input, context, logging)
+      end
+    else
+      process_normal_pipeline(input, context, logging)
+    end
+  end
+
+  defp process_normal_pipeline(input, context, logging) do
+    # Pre-processing: Object boundary merging (before Layer 1 to prevent wrapper text removal)
+    {input_after_merge, _merge_repairs} =
+      if Application.get_env(:json_remedy, :enable_object_merging, true) do
+        JsonRemedy.Layer3.ObjectMerger.merge_object_boundaries(input)
+      else
+        {input, []}
+      end
+
     # Layer 1: Content Cleaning
-    with {:ok, output1, context1} <- ContentCleaning.process(input, context),
+    with {:ok, output1, context1} <- ContentCleaning.process(input_after_merge, context),
          # Layer 2: Structural Repair
          {:ok, output2, context2} <- StructuralRepair.process(output1, context1),
          # Layer 3: Syntax Normalization
          {:ok, output3, context3} <- SyntaxNormalization.process(output2, context2),
          # Layer 4: Validation
          {:ok, parsed, final_context} <- Validation.process(output3, context3) do
+      # Pipeline succeeded
       if logging do
         {:ok, parsed, final_context.repairs}
       else

lib/json_remedy/layer1/content_cleaning.ex

@@ -685,6 +685,15 @@ defmodule JsonRemedy.Layer1.ContentCleaning do
     find_balanced_end(rest, open, close, pos + 1, balance, in_string)
   end
 
+  # Check if a string is valid JSON (not just starts with valid char)
+  defp is_valid_json_start?(str) do
+    first_char = String.at(str, 0)
+
+    # Must start with JSON structure delimiter, not primitives
+    # This prevents "1 Volume(s) created" from being preserved
+    first_char in ["{", "["]
+  end
+
   # Remove trailing wrapper text after JSON
   defp remove_trailing_wrapper_text(input) do
     trimmed = String.trim(input)
@@ -724,23 +733,31 @@ defmodule JsonRemedy.Layer1.ContentCleaning do
 
         # Check if there's non-whitespace content after JSON ends
         after_json = String.slice(input, json_end, String.length(input))
+        after_json_trimmed = String.trim(after_json)
 
-        if String.trim(after_json) == "" do
-          # No significant trailing content
-          {input, []}
-        else
-          # Extract only the JSON portion
-          json_content = String.slice(input, 0, json_end)
+        cond do
+          after_json_trimmed == "" ->
+            # No significant trailing content
+            {input, []}
 
-          repair = %{
-            layer: :content_cleaning,
-            action: "removed trailing wrapper text",
-            position: json_end,
-            original: input,
-            replacement: json_content
-          }
+          is_valid_json_start?(after_json_trimmed) ->
+            # The "trailing" content is actually another JSON value (Pattern 1: Multiple JSON)
+            # Don't remove it - let it be handled by Layer 5
+            {input, []}
+
+          true ->
+            # Extract only the JSON portion, remove wrapper text
+            json_content = String.slice(input, 0, json_end)
+
+            repair = %{
+              layer: :content_cleaning,
+              action: "removed trailing wrapper text",
+              position: json_end,
+              original: input,
+              replacement: json_content
+            }
 
-          {json_content, [repair]}
+            {json_content, [repair]}
         end
     end
   end

lib/json_remedy/layer3/ellipsis_filter.ex

@@ -0,0 +1,71 @@
+defmodule JsonRemedy.Layer3.EllipsisFilter do
+  @moduledoc """
+  Filters unquoted ellipsis ("...") placeholders from JSON content.
+
+  LLMs and humans often use "..." as a placeholder to indicate truncated or
+  omitted content. This module detects and removes unquoted ellipsis while
+  preserving quoted "..." as valid string values.
+
+  Based on json_repair Python library (parse_array.py:34-37)
+  """
+
+  alias JsonRemedy.Layer3.SyntaxHelpers
+
+  @doc """
+  Remove unquoted ellipsis patterns from JSON content.
+  Returns {filtered_content, repairs}.
+  """
+  @spec filter_ellipsis(String.t() | nil) :: {String.t(), list()}
+  def filter_ellipsis(nil), do: {"", []}
+  def filter_ellipsis(input) when not is_binary(input), do: {inspect(input), []}
+
+  def filter_ellipsis(content) when is_binary(content) do
+    # Match unquoted ... (not inside quotes)
+    # Apply multiple passes to handle all variations
+    patterns = [
+      # Only ellipsis: [...] → []
+      {~r/\[\s*\.\.\.\s*\]/, "[]"},
+      # Trailing ellipsis: [1, 2, ...] → [1, 2]
+      {~r/,\s*\.\.\.\s*\]/, "]"},
+      # Leading ellipsis: [..., 1, 2] → [1, 2]
+      {~r/\[\s*\.\.\.\s*,/, "["},
+      # Middle ellipsis: [1, ..., 3] → [1, 3]
+      {~r/,\s*\.\.\.\s*,/, ","},
+      # Before closing brace/bracket: ..., } or ..., ]
+      {~r/,\s*\.\.\.\s*([}\]])/, "\\1"}
+    ]
+
+    {result, repairs} =
+      Enum.reduce(patterns, {content, []}, fn {pattern, replacement},
+                                              {acc_content, acc_repairs} ->
+        # Keep applying pattern until no more matches (handles multiple occurrences)
+        {final_content, match_count} =
+          apply_pattern_recursively(pattern, replacement, acc_content, 0)
+
+        if match_count > 0 do
+          repair =
+            SyntaxHelpers.create_repair(
+              "filtered ellipsis placeholder",
+              "Removed #{match_count} unquoted ... placeholder(s)",
+              0
+            )
+
+          {final_content, [repair | acc_repairs]}
+        else
+          {acc_content, acc_repairs}
+        end
+      end)
+
+    {result, Enum.reverse(repairs)}
+  end
+
+  # Apply pattern recursively until no more matches
+  defp apply_pattern_recursively(pattern, replacement, content, count) do
+    if Regex.match?(pattern, content) do
+      new_content = Regex.replace(pattern, content, replacement, global: false)
+      apply_pattern_recursively(pattern, replacement, new_content, count + 1)
+    else
+      {content, count}
+    end
+  end
+end

lib/json_remedy/layer3/keyword_filter.ex

@@ -0,0 +1,85 @@
+defmodule JsonRemedy.Layer3.KeywordFilter do
+  @moduledoc """
+  Filters comment-like keywords from JSON content.
+
+  LLMs and debug outputs sometimes include placeholder keywords like COMMENT,
+  SHOULD_NOT_EXIST, DEBUG_INFO, etc. This module detects and removes them.
+
+  Based on json_repair Python library (parse_string.py:450-456)
+  """
+
+  alias JsonRemedy.Layer3.SyntaxHelpers
+
+  # Common keywords that should be filtered
+  @filter_keywords ~w(
+    COMMENT
+    SHOULD_NOT_EXIST
+    DEBUG_INFO
+    DEBUG
+    TRACE_END
+    PLACEHOLDER
+    SEPARATOR
+    MARKER
+    HEADER
+    FOOTER
+    INVALID
+    TODO
+    FIXME
+    NOTE
+  )
+
+  @doc """
+  Remove comment-like keywords from JSON content.
+  Returns {filtered_content, repairs}.
+  """
+  @spec filter_keywords(String.t()) :: {String.t(), list()}
+  def filter_keywords(content) when is_binary(content) do
+    # Match patterns like: ", KEYWORD " or "{ KEYWORD " before next key
+    # We need to be careful to only match unquoted keywords
+
+    {result, repairs} =
+      Enum.reduce(@filter_keywords, {content, []}, fn keyword, {acc_content, acc_repairs} ->
+        # Pattern: keyword followed by a string delimiter (indicating next key/value)
+        # In objects: , KEYWORD "key" or { KEYWORD "key"
+        # In arrays: , KEYWORD value
+        patterns = [
+          # Between object pairs: , KEYWORD "
+          {~r/,\s+#{keyword}\s+"/i, ", \""},
+          # At start of object: { KEYWORD "
+          {~r/\{\s*#{keyword}\s+"/i, "{\""},
+          # In array with following value: , KEYWORD <value>
+          # This handles "PLACEHOLDER 3" -> "3"
+          {~r/,\s+#{keyword}\s+/i, ", "},
+          # At start of array: [ KEYWORD <value>
+          {~r/\[\s*#{keyword}\s+/i, "["},
+          # At end before closing: KEYWORD ]
+          {~r/\s+#{keyword}\s*\]/i, "]"},
+          # At end before closing: KEYWORD }
+          {~r/\s+#{keyword}\s*\}/i, "}"}
+        ]
+
+        Enum.reduce(patterns, {acc_content, acc_repairs}, fn {pattern, replacement},
+                                                             {content_acc, repairs_acc} ->
+          if Regex.match?(pattern, content_acc) do
+            new_content = Regex.replace(pattern, content_acc, replacement)
+
+            repair =
+              SyntaxHelpers.create_repair(
+                "filtered comment keyword",
+                "Removed #{keyword} placeholder",
+                0
+              )
+
+            {new_content, [repair | repairs_acc]}
+          else
+            {content_acc, repairs_acc}
+          end
+        end)
+      end)
+
+    {result, Enum.reverse(repairs)}
+  end
+
+  def filter_keywords(nil), do: {"", []}
+  def filter_keywords(input) when not is_binary(input), do: {inspect(input), []}
+end

lib/json_remedy/layer3/object_merger.ex

@@ -0,0 +1,74 @@
+defmodule JsonRemedy.Layer3.ObjectMerger do
+  @moduledoc """
+  Merges additional key-value pairs that appear after object closing braces.
+
+  Pattern: {"a":"b"},"c":"d"} should become {"a":"b","c":"d"}
+
+  This happens when objects are malformed with extra closing braces or when
+  additional pairs are erroneously placed outside the object.
+
+  Based on json_repair Python library (parse_object.py:123-143)
+  """
+
+  alias JsonRemedy.Layer3.SyntaxHelpers
+
+  @doc """
+  Merge additional key-value pairs after object closes.
+  Returns {merged_content, repairs}.
+  """
+  @spec merge_object_boundaries(String.t() | nil) :: {String.t(), list()}
+  def merge_object_boundaries(nil), do: {"", []}
+  def merge_object_boundaries(input) when not is_binary(input), do: {inspect(input), []}
+
+  def merge_object_boundaries(content) when is_binary(content) do
+    # Pattern: },"key":"value" → ,"key":"value"
+    # This removes the premature closing brace before additional pairs
+    # BUT only if there's an EXTRA closing brace (more } than {)
+
+    if should_merge?(content) do
+      pattern = ~r/}\s*,\s*"/
+      # Replace }, " with just , "
+      result_after_replace = Regex.replace(pattern, content, fn _match -> ", \"" end)
+
+      # Then remove any extra trailing } at the end
+      result = remove_extra_trailing_brace(result_after_replace)
+
+      repair =
+        SyntaxHelpers.create_repair(
+          "merged object boundary",
+          "Merged additional key-value pairs into object",
+          0
+        )
+
+      {result, [repair]}
+    else
+      {content, []}
+    end
+  end
+
+  # Check if we should merge - only if there are MORE closing braces than opening
+  defp should_merge?(content) do
+    opening = content |> String.graphemes() |> Enum.count(&(&1 == "{"))
+    closing = content |> String.graphemes() |> Enum.count(&(&1 == "}"))
+
+    # Only merge if there's an extra closing brace
+    closing > opening
+  end
+
+  # Remove extra trailing } that appears after merging
+  defp remove_extra_trailing_brace(content) do
+    # Count opening and closing braces to see if we have extra
+    opening = content |> String.graphemes() |> Enum.count(&(&1 == "{"))
+    closing = content |> String.graphemes() |> Enum.count(&(&1 == "}"))
+
+    if closing > opening do
+      # Remove one } from the end
+      content
+      |> String.reverse()
+      |> String.replace_prefix("}", "")
+      |> String.reverse()
+    else
+      content
+    end
+  end
+end