Add preprocessing to ensure blank lines around block content in tags

When lists or tables appear inside tags without blank lines, the CommonMark
parser uses lazy continuation to incorrectly merge tags with block content.
This caused closing tags to be absorbed into list items.

The fix adds preprocessing before Markdown parsing to insert blank lines:
- After opening tag lines when followed by block content (lists/tables)
- Before closing tag lines when preceded by block content

This ensures proper parsing and prevents tags from being merged with lists.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: jlevy

src/flowmark/linewrapping/markdown_filling.py

@@ -21,7 +21,7 @@
 )
 from flowmark.linewrapping.protocols import LineWrapper
 from flowmark.linewrapping.sentence_split_regex import split_sentences_regex
-from flowmark.linewrapping.tag_handling import TagWrapping
+from flowmark.linewrapping.tag_handling import TagWrapping, preprocess_tag_block_spacing
 from flowmark.linewrapping.text_filling import DEFAULT_WRAP_WIDTH
 from flowmark.transforms.doc_cleanups import doc_cleanups
 from flowmark.transforms.doc_transforms import rewrite_text_content
@@ -87,6 +87,11 @@ def fill_markdown(
 
     markdown_text = markdown_text.strip() + "\n"
 
+    # Preprocess: ensure proper blank lines around block content within tags.
+    # This must happen before parsing to prevent CommonMark lazy continuation
+    # from incorrectly merging tags with lists/tables.
+    markdown_text = preprocess_tag_block_spacing(markdown_text)
+
     # Parse and render.
     marko = flowmark_markdown(line_wrapper, list_spacing)
     document = marko.parse(markdown_text)

src/flowmark/linewrapping/tag_handling.py

@@ -266,6 +266,92 @@ def get_tag_coalescing_patterns(max_words: int = MAX_TAG_WORDS) -> list[tuple[st
     ]
 
 
+def _is_tag_only_line(line: str) -> bool:
+    """
+    Check if a line contains only a tag (opening or closing), not inline tags in content.
+
+    A tag-only line starts with a tag delimiter and ends with a tag delimiter,
+    with no substantial non-tag content. This distinguishes:
+    - `{% field %}` (tag-only line)
+    - `- [ ] Item {% #id %}` (content with inline tag - NOT tag-only)
+    """
+    stripped = line.strip()
+    if not stripped:
+        return False
+
+    # Check if it starts with a tag
+    starts_tag = (
+        stripped.startswith(JINJA_TAG_OPEN)
+        or stripped.startswith(JINJA_COMMENT_OPEN)
+        or stripped.startswith(JINJA_VAR_OPEN)
+        or stripped.startswith(HTML_COMMENT_OPEN)
+    )
+
+    # Check if it ends with a tag
+    ends_tag = (
+        stripped.endswith(JINJA_TAG_CLOSE)
+        or stripped.endswith(JINJA_COMMENT_CLOSE)
+        or stripped.endswith(JINJA_VAR_CLOSE)
+        or stripped.endswith(HTML_COMMENT_CLOSE)
+    )
+
+    return starts_tag and ends_tag
+
+
+def preprocess_tag_block_spacing(text: str) -> str:
+    """
+    Preprocess text to ensure proper blank lines around block content within tags.
+
+    When block content (lists, tables) appears directly after an opening tag or
+    directly before a closing tag, the CommonMark parser may use lazy continuation
+    to merge them incorrectly. This function inserts blank lines to prevent this.
+
+    This preprocessing must happen BEFORE Markdown parsing, as the parser's
+    structure cannot be fixed after the fact.
+
+    Example transformation:
+        {% field %}
+        - item 1
+        - item 2
+        {% /field %}
+
+    Becomes:
+        {% field %}
+
+        - item 1
+        - item 2
+
+        {% /field %}
+    """
+    lines = text.split("\n")
+    result_lines: list[str] = []
+
+    # Check if there are any tag-only lines in the text
+    has_tag_only_lines = any(_is_tag_only_line(line) for line in lines)
+    if not has_tag_only_lines:
+        return text
+
+    for i, line in enumerate(lines):
+        # Check if we need to add a blank line BEFORE this line
+        if i > 0:
+            prev_line = lines[i - 1]
+            prev_is_empty = prev_line.strip() == ""
+
+            # Case 1: Previous line is a tag-only line, current line is block content
+            # (need blank line after opening tag before list/table)
+            if not prev_is_empty and _is_tag_only_line(prev_line) and line_is_block_content(line):
+                result_lines.append("")
+
+            # Case 2: Previous line is block content, current line is a closing tag-only line
+            # (need blank line after list/table before closing tag)
+            if not prev_is_empty and line_is_block_content(prev_line) and _is_tag_only_line(line):
+                result_lines.append("")
+
+        result_lines.append(line)
+
+    return "\n".join(result_lines)
+
+
 def line_ends_with_tag(line: str) -> bool:
     """Check if a line ends with a Jinja/Markdoc tag or HTML comment."""
     stripped = line.rstrip()

tests/test_tag_formatting.py

@@ -507,3 +507,150 @@ def test_html_comment_multiline_closing():
 
     # Closing comment should be on its own line
     assert "-->\n<!-- /f:field -->" in result, f"HTML closing tag not split: {result}"
+
+
+def test_preprocess_tag_block_spacing_lists():
+    """
+    Test that blank lines are added around lists inside tags.
+
+    This prevents CommonMark lazy continuation from merging tags with lists.
+    """
+    from flowmark.linewrapping.tag_handling import (
+        preprocess_tag_block_spacing,
+    )
+
+    # Input without blank lines around list
+    text = dedent("""
+        {% field kind="single_select" id="choice" %}
+        - [ ] Option A {% #a %}
+        - [ ] Option B {% #b %}
+        {% /field %}
+        """).strip()
+
+    result = preprocess_tag_block_spacing(text)
+
+    # Should have blank line after opening tag
+    assert "{% field" in result
+    assert "%}\n\n-" in result, f"Missing blank line after opening tag: {result}"
+
+    # Should have blank line before closing tag
+    assert "\n\n{% /field" in result, f"Missing blank line before closing tag: {result}"
+
+
+def test_preprocess_tag_block_spacing_tables():
+    """Test that blank lines are added around tables inside tags."""
+    from flowmark.linewrapping.tag_handling import (
+        preprocess_tag_block_spacing,
+    )
+
+    # Input without blank lines around table
+    text = dedent("""
+        {% table id="data" %}
+        | A | B |
+        |---|---|
+        | 1 | 2 |
+        {% /table %}
+        """).strip()
+
+    result = preprocess_tag_block_spacing(text)
+
+    # Should have blank line after opening tag
+    assert "%}\n\n|" in result, f"Missing blank line after opening tag: {result}"
+
+    # Should have blank line before closing tag
+    assert "|\n\n{% /table" in result, f"Missing blank line before closing tag: {result}"
+
+
+def test_preprocess_tag_block_spacing_already_spaced():
+    """Test that already-spaced content is not double-spaced."""
+    from flowmark.linewrapping.tag_handling import (
+        preprocess_tag_block_spacing,
+    )
+
+    # Input already has proper blank lines
+    text = dedent("""
+        {% field kind="select" %}
+
+        - Option 1
+        - Option 2
+
+        {% /field %}
+        """).strip()
+
+    result = preprocess_tag_block_spacing(text)
+
+    # Should not add extra blank lines (no triple newlines)
+    assert "\n\n\n" not in result, f"Extra blank lines added: {result}"
+
+
+def test_preprocess_tag_block_spacing_inline_tags():
+    """Test that inline tags in list items don't trigger extra spacing."""
+    from flowmark.linewrapping.tag_handling import (
+        preprocess_tag_block_spacing,
+    )
+
+    # List items with inline tags - should NOT add blank lines between items
+    text = dedent("""
+        {% field %}
+
+        - Item 1 {% #item1 %}
+        - Item 2 {% #item2 %}
+
+        {% /field %}
+        """).strip()
+
+    result = preprocess_tag_block_spacing(text)
+
+    # Should NOT have blank lines between list items
+    assert "{% #item1 %}\n- Item 2" in result, f"Incorrectly added blank between items: {result}"
+
+
+def test_fill_markdown_with_list_in_tags():
+    """
+    Integration test: fill_markdown properly formats lists inside tags.
+
+    Regression test for the bug where closing tags were merged into list items
+    due to CommonMark lazy continuation.
+    """
+    text = dedent("""
+        {% field kind="single_select" id="rating" %}
+        - [ ] G {% #g %}
+        - [ ] PG {% #pg %}
+        - [x] R {% #r %}
+        {% /field %}
+        """).strip()
+
+    result = fill_markdown(text, semantic=True)
+
+    # Opening tag should be followed by blank line
+    assert "{% field" in result
+    lines = result.strip().split("\n")
+
+    # First line should be the opening tag
+    assert lines[0].startswith("{% field")
+
+    # Second line should be blank
+    assert lines[1] == "", f"Expected blank line after opening tag, got: {lines[1]}"
+
+    # List items should be together (no blank lines between them)
+    list_start = None
+    for i, line in enumerate(lines):
+        if line.startswith("- "):
+            list_start = i
+            break
+
+    assert list_start is not None
+    # Check consecutive list items
+    assert lines[list_start].startswith("- [ ] G")
+    assert lines[list_start + 1].startswith("- [ ] PG")
+    assert lines[list_start + 2].startswith("- ")
+
+    # There should be a blank line before closing tag
+    closing_idx = None
+    for i, line in enumerate(lines):
+        if line.strip() == "{% /field %}":
+            closing_idx = i
+            break
+
+    assert closing_idx is not None
+    assert lines[closing_idx - 1] == "", "Expected blank line before closing tag"

tests/testdocs/testdoc.expected.auto.md

@@ -1436,13 +1436,15 @@ Content should start on a new line.
 Closing tags should NOT be merged onto list item lines:
 
 {% field kind="single_select" id="choice" label="Choice" %}
+
 - [ ] Option A {% #option_a %}
 - [ ] Option B {% #option_b %}
 - [ ] Option C {% #option_c %}
 
 {% /field %}
 
 <!-- f:field kind="select" id="choice" -->
+
 - Option 1 <!-- #opt1 -->
 - Option 2 <!-- #opt2 -->
 - Option 3 <!-- #opt3 -->
@@ -1505,7 +1507,7 @@ lines):
 {% table_container id="pricing" %}
 
 | Plan | Price | Features |
-|------|-------|----------|
+| --- | --- | --- |
 | Free | $0 | Basic |
 | Pro | $10 | Advanced |
 
@@ -1516,7 +1518,7 @@ Tables inside HTML comment tags without blank lines:
 <!-- f:table id="comparison" -->
 
 | Feature | Product A | Product B |
-|---------|-----------|-----------|
+| --- | --- | --- |
 | Speed | Fast | Faster |
 | Cost | Low | Medium |
 
@@ -1527,7 +1529,7 @@ Tables inside Jinja variable tags (edge case):
 {{ table_header }}
 
 | Column 1 | Column 2 |
-|----------|----------|
+| --- | --- |
 | Data 1 | Data 2 |
 
 {{ table_footer }}
@@ -1594,15 +1596,15 @@ Self-closing tags with tables:
 {% divider style="double" /%}
 
 | Before Divider | After Divider |
-|----------------|---------------|
+| --- | --- |
 | A | B |
 
 {% spacer height="20" /%}
 
 <!-- separator -->
 
 | Item | Value |
-|------|-------|
+| --- | --- |
 | X | 1 |
 
 <!-- end-section -->
@@ -1638,6 +1640,7 @@ Your feedback is valuable to us.
 {% /description %}
 
 {% field kind="single_select" id="rating" label="Overall Rating" required=true %}
+
 - [ ] Excellent {% #excellent %}
 - [ ] Good {% #good %}
 - [ ] Fair {% #fair %}