Merge pull request #23 from jlevy/feature/benchmark-tooling

Fix performance regression and simplify tag handling

Author: jlevy

…e/plan-2026-01-14-atomic-tag-wrapping.md …e/plan-2026-01-14-atomic-tag-wrapping.mddocs/project/specs/active/plan-2026-01-14-atomic-tag-wrapping.md renamed to docs/project/specs/done/plan-2026-01-14-atomic-tag-wrapping.md docs/project/specs/active/plan-2026-01-14-atomic-tag-wrapping.md renamed to docs/project/specs/done/plan-2026-01-14-atomic-tag-wrapping.md

@@ -51,7 +51,6 @@
 from dataclasses import dataclass
 
 from flowmark.formats.flowmark_markdown import ListSpacing
-from flowmark.linewrapping.tag_handling import TagWrapping
 from flowmark.reformat_api import reformat_files
 
 
@@ -62,7 +61,6 @@ class Options:
     files: list[str]
     output: str
     width: int
-    tags: TagWrapping
     plaintext: bool
     semantic: bool
     cleanups: bool
@@ -111,14 +109,6 @@ def _parse_args(args: list[str] | None = None) -> Options:
     parser.add_argument(
         "-p", "--plaintext", action="store_true", help="Process as plaintext (no Markdown parsing)"
     )
-    parser.add_argument(
-        "--tags",
-        type=str,
-        choices=["atomic", "wrap"],
-        default="atomic",
-        help="How to handle template tags during line wrapping: 'atomic' (default) never breaks "
-        "tags across lines, 'wrap' allows tags to wrap like normal text",
-    )
     parser.add_argument(
         "-s",
         "--semantic",
@@ -190,7 +180,6 @@ def _parse_args(args: list[str] | None = None) -> Options:
         files=opts.files,
         output=opts.output,
         width=opts.width,
-        tags=TagWrapping(opts.tags),
         plaintext=opts.plaintext,
         semantic=opts.semantic,
         cleanups=opts.cleanups,
@@ -229,7 +218,6 @@ def main(args: list[str] | None = None) -> int:
             files=options.files,
             output=options.output,
             width=options.width,
-            tags=options.tags,
             inplace=options.inplace,
             nobackup=options.nobackup,
             plaintext=options.plaintext,

…/valid-2026-01-14-atomic-tag-wrapping.md …/valid-2026-01-14-atomic-tag-wrapping.mddocs/project/specs/active/valid-2026-01-14-atomic-tag-wrapping.md renamed to docs/project/specs/done/valid-2026-01-14-atomic-tag-wrapping.md docs/project/specs/active/valid-2026-01-14-atomic-tag-wrapping.md renamed to docs/project/specs/done/valid-2026-01-14-atomic-tag-wrapping.md

@@ -0,0 +1,189 @@
+"""
+Atomic pattern definitions for constructs that should not be broken during wrapping.
+
+Each AtomicPattern defines a regex for a specific type of construct (code span, link,
+template tag, etc.) that should be kept together as a single token during line wrapping.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class AtomicPattern:
+    """
+    Defines a regex pattern for an atomic construct that should not be broken.
+
+    For delimiter-based patterns (tags, comments), `open_delim`/`close_delim` store
+    the raw delimiters and `open_re`/`close_re` store regex-escaped versions.
+    For non-delimiter patterns, these are empty strings.
+    """
+
+    name: str
+    pattern: str
+    open_delim: str
+    close_delim: str
+    open_re: str
+    close_re: str
+
+
+def _make_paired_pattern(open_re: str, close_re: str, middle_char: str) -> str:
+    """
+    Generate a paired tag pattern: opening + closing kept together.
+
+    Uses `(?!\\s*/)` lookahead to ensure first tag is opening (not closing).
+    The middle_char is the character to exclude from middle content.
+    """
+    return (
+        rf"{open_re}(?!\s*/)[^{middle_char}]*{close_re}"
+        rf"\s*"
+        rf"{open_re}\s*/[^{middle_char}]*{close_re}"
+    )
+
+
+# Inline code spans with backticks (handles multi-backtick like ``code``)
+INLINE_CODE_SPAN = AtomicPattern(
+    name="inline_code_span",
+    pattern=r"(`+)(?:(?!\1).)+\1",
+    open_delim="",
+    close_delim="",
+    open_re="",
+    close_re="",
+)
+
+# Markdown links: [text](url) or [text][ref] or [text]
+MARKDOWN_LINK = AtomicPattern(
+    name="markdown_link",
+    pattern=r"\[[^\]]*\](?:\([^)]*\)|\[[^\]]*\])?",
+    open_delim="",
+    close_delim="",
+    open_re="",
+    close_re="",
+)
+
+# Jinja/Markdoc template tags: {% tag %}, {% /tag %}
+SINGLE_JINJA_TAG = AtomicPattern(
+    name="single_jinja_tag",
+    pattern=r"\{%.*?%\}",
+    open_delim="{%",
+    close_delim="%}",
+    open_re=r"\{%",
+    close_re=r"%\}",
+)
+
+PAIRED_JINJA_TAG = AtomicPattern(
+    name="paired_jinja_tag",
+    pattern=_make_paired_pattern(r"\{%", r"%\}", "%"),
+    open_delim="{%",
+    close_delim="%}",
+    open_re=r"\{%",
+    close_re=r"%\}",
+)
+
+# Jinja comments: {# comment #}
+SINGLE_JINJA_COMMENT = AtomicPattern(
+    name="single_jinja_comment",
+    pattern=r"\{#.*?#\}",
+    open_delim="{#",
+    close_delim="#}",
+    open_re=r"\{#",
+    close_re=r"#\}",
+)
+
+PAIRED_JINJA_COMMENT = AtomicPattern(
+    name="paired_jinja_comment",
+    pattern=_make_paired_pattern(r"\{#", r"#\}", "#"),
+    open_delim="{#",
+    close_delim="#}",
+    open_re=r"\{#",
+    close_re=r"#\}",
+)
+
+# Jinja variables: {{ variable }}
+SINGLE_JINJA_VAR = AtomicPattern(
+    name="single_jinja_var",
+    pattern=r"\{\{.*?\}\}",
+    open_delim="{{",
+    close_delim="}}",
+    open_re=r"\{\{",
+    close_re=r"\}\}",
+)
+
+PAIRED_JINJA_VAR = AtomicPattern(
+    name="paired_jinja_var",
+    pattern=_make_paired_pattern(r"\{\{", r"\}\}", "}"),
+    open_delim="{{",
+    close_delim="}}",
+    open_re=r"\{\{",
+    close_re=r"\}\}",
+)
+
+# HTML comments: <!-- comment -->
+SINGLE_HTML_COMMENT = AtomicPattern(
+    name="single_html_comment",
+    pattern=r"<!--.*?-->",
+    open_delim="<!--",
+    close_delim="-->",
+    open_re=r"<!--",
+    close_re=r"-->",
+)
+
+PAIRED_HTML_COMMENT = AtomicPattern(
+    name="paired_html_comment",
+    pattern=(
+        r"<!--(?!\s*/)[^-]*(?:-[^-]+)*-->"
+        r"\s*"
+        r"<!--\s*/[^-]*(?:-[^-]+)*-->"
+    ),
+    open_delim="<!--",
+    close_delim="-->",
+    open_re=r"<!--",
+    close_re=r"-->",
+)
+
+# HTML/XML tags: <tag>, </tag>
+HTML_OPEN_TAG = AtomicPattern(
+    name="html_open_tag",
+    pattern=r"<[a-zA-Z][^>]*>",
+    open_delim="",
+    close_delim="",
+    open_re="",
+    close_re="",
+)
+
+HTML_CLOSE_TAG = AtomicPattern(
+    name="html_close_tag",
+    pattern=r"</[a-zA-Z][^>]*>",
+    open_delim="",
+    close_delim="",
+    open_re="",
+    close_re="",
+)
+
+# All patterns in priority order (more specific patterns first).
+# Paired tag patterns must come before single tag patterns to match correctly.
+ATOMIC_PATTERNS: tuple[AtomicPattern, ...] = (
+    INLINE_CODE_SPAN,
+    MARKDOWN_LINK,
+    # Paired tags must come before single tags
+    PAIRED_JINJA_TAG,
+    PAIRED_JINJA_COMMENT,
+    PAIRED_JINJA_VAR,
+    PAIRED_HTML_COMMENT,
+    # Single tags
+    SINGLE_JINJA_TAG,
+    SINGLE_JINJA_COMMENT,
+    SINGLE_JINJA_VAR,
+    SINGLE_HTML_COMMENT,
+    # HTML tags
+    HTML_OPEN_TAG,
+    HTML_CLOSE_TAG,
+)
+
+# Compiled regex combining all patterns with alternation
+ATOMIC_CONSTRUCT_PATTERN: re.Pattern[str] = re.compile(
+    "|".join(p.pattern for p in ATOMIC_PATTERNS),
+    re.DOTALL,
+)

src/flowmark/cli.py

@@ -7,7 +7,6 @@
 from flowmark.linewrapping.protocols import LineWrapper
 from flowmark.linewrapping.sentence_split_regex import split_sentences_regex
 from flowmark.linewrapping.tag_handling import (
-    TagWrapping,
     add_tag_newline_handling,
     denormalize_adjacent_tags,
 )
@@ -79,7 +78,6 @@ def enhanced_wrapper(text: str, initial_indent: str, subsequent_indent: str) ->
 
 def line_wrap_to_width(
     width: int = DEFAULT_WRAP_WIDTH,
-    tags: TagWrapping = TagWrapping.atomic,
     len_fn: Callable[[str], int] = DEFAULT_LEN_FUNCTION,
     is_markdown: bool = False,
 ) -> LineWrapper:
@@ -95,14 +93,13 @@ def line_wrapper(text: str, initial_indent: str, subsequent_indent: str) -> str:
             subsequent_indent=subsequent_indent,
             len_fn=len_fn,
             is_markdown=is_markdown,
-            tags=tags,
         )
 
     if is_markdown:
         # Apply tag newline handling first, then hard break handling
         # Order matters: tag handling should operate on original newlines
         # before hard break handling normalizes explicit breaks
-        enhanced = add_tag_newline_handling(line_wrapper, tags=tags)
+        enhanced = add_tag_newline_handling(line_wrapper)
         return _add_markdown_hard_break_handling(enhanced)
     else:
         return line_wrapper
@@ -111,7 +108,6 @@ def line_wrapper(text: str, initial_indent: str, subsequent_indent: str) -> str:
 def line_wrap_by_sentence(
     split_sentences: SentenceSplitter = split_sentences_no_min_length,
     width: int = DEFAULT_WRAP_WIDTH,
-    tags: TagWrapping = TagWrapping.atomic,
     min_line_len: int = DEFAULT_MIN_LINE_LEN,
     len_fn: Callable[[str], int] = DEFAULT_LEN_FUNCTION,
     is_markdown: bool = False,
@@ -120,10 +116,6 @@ def line_wrap_by_sentence(
     Wrap lines of text to a given width but also keep sentences on their own lines.
     If the last line ends up shorter than `min_line_len`, it's combined with the
     next sentence.
-
-    The `tags` parameter controls template tag handling:
-    - `atomic`: Tags are treated as indivisible tokens (never broken across lines)
-    - `wrap`: Tags can wrap like normal text (legacy behavior with coalescing limits)
     """
 
     def line_wrapper(text: str, initial_indent: str, subsequent_indent: str) -> str:
@@ -152,7 +144,6 @@ def line_wrapper(text: str, initial_indent: str, subsequent_indent: str) -> str:
                 initial_column=current_column,
                 subsequent_offset=subsequent_indent_len,
                 is_markdown=is_markdown,
-                tags=tags,
             )
             # If last line is shorter than min_line_len, combine with next line.
             # Also handles if the first word doesn't fit.
@@ -182,7 +173,7 @@ def line_wrapper(text: str, initial_indent: str, subsequent_indent: str) -> str:
 
     if is_markdown:
         # Apply tag newline handling first, then hard break handling
-        enhanced = add_tag_newline_handling(line_wrapper, tags=tags)
+        enhanced = add_tag_newline_handling(line_wrapper)
         return _add_markdown_hard_break_handling(enhanced)
     else:
         return line_wrapper

src/flowmark/linewrapping/atomic_patterns.py

@@ -21,7 +21,7 @@
 )
 from flowmark.linewrapping.protocols import LineWrapper
 from flowmark.linewrapping.sentence_split_regex import split_sentences_regex
-from flowmark.linewrapping.tag_handling import TagWrapping, preprocess_tag_block_spacing
+from flowmark.linewrapping.tag_handling import preprocess_tag_block_spacing
 from flowmark.linewrapping.text_filling import DEFAULT_WRAP_WIDTH
 from flowmark.transforms.doc_cleanups import doc_cleanups
 from flowmark.transforms.doc_transforms import rewrite_text_content
@@ -37,7 +37,6 @@ def fill_markdown(
     markdown_text: str,
     dedent_input: bool = True,
     width: int = DEFAULT_WRAP_WIDTH,
-    tags: TagWrapping = TagWrapping.atomic,
     semantic: bool = False,
     cleanups: bool = False,
     smartquotes: bool = False,
@@ -61,19 +60,17 @@ def fill_markdown(
     With `semantic` enabled, the line breaks are wrapped approximately
     by sentence boundaries, to make diffs more readable.
 
-    The `tags` parameter controls how template tags (Markdoc, Jinja, HTML
-    comments) are handled during wrapping:
-    - `atomic` (default): Tags are never broken across lines
-    - `wrap`: Tags can wrap like normal text (legacy behavior)
+    Template tags (Markdoc, Jinja, HTML comments) are always treated atomically
+    and never broken across lines.
 
     Preserves YAML frontmatter (delimited by --- lines) if present at the
     beginning of the document.
     """
     if line_wrapper is None:
         if semantic:
-            line_wrapper = line_wrap_by_sentence(width=width, tags=tags, is_markdown=True)
+            line_wrapper = line_wrap_by_sentence(width=width, is_markdown=True)
         else:
-            line_wrapper = line_wrap_to_width(width=width, tags=tags, is_markdown=True)
+            line_wrapper = line_wrap_to_width(width=width, is_markdown=True)
 
     # Extract frontmatter before any processing
     frontmatter, content = split_frontmatter(markdown_text)