Merge pull request #16 from jlevy/feature/tag-formatting-edge-cases

Fix smart quotes to not apply inside Markdoc/Jinja/HTML tags

Author: jlevy

src/flowmark/linewrapping/tag_handling.py

@@ -47,6 +47,20 @@
 HTML_COMMENT_CLOSE_RE = r"-->"
 
 
+# Pattern to match complete template tags (for protecting content inside tags).
+# These tags can span multiple lines and may contain quotes in attribute values.
+# Uses DOTALL so . matches newlines within tags.
+# Note: In VERBOSE mode, # starts a comment, so we use [#] for literal hash.
+TEMPLATE_TAG_PATTERN: re.Pattern[str] = re.compile(
+    r"""
+    \{%.*?%\}             # Jinja/Markdoc template tags
+    | \{[#].*?[#]\}       # Jinja comments (use [#] to avoid VERBOSE comment)
+    | \{\{.*?\}\}         # Jinja variables
+    | <!--.*?-->          # HTML comments
+    """,
+    re.VERBOSE | re.DOTALL,
+)
+
 # Pattern to detect adjacent tags (closing tag immediately followed by opening tag)
 # This handles cases like %}{% or --><!-- where there's no space between
 _adjacent_tags_re: re.Pattern[str] = re.compile(

src/flowmark/typography/smartquotes.py

@@ -1,6 +1,8 @@
 import re
 from re import Match, Pattern
 
+from flowmark.linewrapping.tag_handling import TEMPLATE_TAG_PATTERN
+
 # Precompiled regex patterns
 PARAGRAPH_BREAK_PATTERN: Pattern[str] = re.compile(r"\n\s*\n")
 
@@ -19,49 +21,15 @@ def is_multi_paragraph(text: str) -> bool:
     return PARAGRAPH_BREAK_PATTERN.search(text) is not None
 
 
-def smart_quotes(text: str) -> str:
-    r"""
-    Replace straight ASCII quotes and apostrophes with typographic quotes and apostrophes
-    when this can be done safely. Aims to be conservative so it doesn't break code or
-    things that aren't language.
-
-    Text that is wrapped in single or double quotes is replaced with typographic quotes
-    if it has whitespace or a newline at the front and is followed by whitespace or
-    a [.,?!]. The content inside quotes must not contain any of the same type (single
-    or double). Quotes containing paragraph breaks (two newlines) are left unchanged.
-
-    Straight quotes are converted to apostrophes if they are the only straight quote
-    in the word, and have word characters on both sides:
-
-    I'm there with "George" -> I’m there with “George”
-    "Hello," he said. -> “Hello,” he said.
-    "I know!" -> “I know!”
-
-    Words in 'single quotes' work too -> Words in 'single quotes' work too
-
-    I'm there -> I’m there
-    I'll be there, don't worry -> I’ll be there, don’t worry
-    X is 'foo' -> X is ‘foo’
-
-    A few special rules to better help with English:
-
-    Jill's -> Jill’s
-    James' -> James’
-
-    Other patterns are unchanged:
-
-    x="foo" -> x="foo"
-    x='foo' -> x='foo'
-    Blah'blah'blah -> Blah'blah'blah
-    ""quotes"s -> ""quotes"s
-    \"escaped\" -> \"escaped\"
-    'apos -> 'apos
-    'apos'trophes -> 'apos'trophes
-    $James' -> $James'
+def _apply_smart_quotes_to_text(text: str) -> str:
+    """
+    Apply smart quote conversion to a text segment.
 
+    This is the core smart quotes logic, applied only to text that is NOT inside
+    template tags.
     """
 
-    # First handle quoted text - both single and double quotes
+    # Handle quoted text - both single and double quotes
     def replace_quotes(match: Match[str]) -> str:
         prefix = match.group(1)
         double_content = match.group(2)  # Content of double quotes
@@ -83,14 +51,11 @@ def replace_quotes(match: Match[str]) -> str:
 
     result = QUOTE_PATTERN.sub(replace_quotes, text)
 
-    # Now handle apostrophes/contractions
+    # Handle apostrophes/contractions
     # Only convert single quotes that are:
     # 1. The only quote in the word
     # 2. Have word characters on both sides OR are possessives at end of words ending in s/S
 
-    # Pattern for apostrophes: word char + ' + word char, where ' is the only quote in the word
-    # We need to be careful not to match words that have multiple quotes
-
     # Split by whitespace to process words individually
     words = re.split(r"(\s+)", result)
 
@@ -115,3 +80,80 @@ def replace_quotes(match: Match[str]) -> str:
                 words[i] = re.sub(r"\'", "\u2019", word)
 
     return "".join(words)
+
+
+def smart_quotes(text: str) -> str:
+    r"""
+    Replace straight ASCII quotes and apostrophes with typographic quotes and apostrophes
+    when this can be done safely. Aims to be conservative so it doesn't break code or
+    things that aren't language.
+
+    IMPORTANT: Quotes inside template tags (Jinja/Markdoc `{% %}`, `{# #}`, `{{ }}`,
+    and HTML comments `<!-- -->`) are NEVER converted, as this would break template
+    syntax.
+
+    Text that is wrapped in single or double quotes is replaced with typographic quotes
+    if it has whitespace or a newline at the front and is followed by whitespace or
+    a [.,?!]. The content inside quotes must not contain any of the same type (single
+    or double). Quotes containing paragraph breaks (two newlines) are left unchanged.
+
+    Straight quotes are converted to apostrophes if they are the only straight quote
+    in the word, and have word characters on both sides:
+
+    I'm there with "George" -> I’m there with “George”
+    "Hello," he said. -> “Hello,” he said.
+    "I know!" -> “I know!”
+
+    Words in 'single quotes' work too -> Words in 'single quotes' work too
+
+    I'm there -> I’m there
+    I'll be there, don't worry -> I’ll be there, don’t worry
+    X is 'foo' -> X is ‘foo’
+
+    A few special rules to better help with English:
+
+    Jill's -> Jill’s
+    James' -> James’
+
+    Other patterns are unchanged:
+
+    x="foo" -> x="foo"
+    x='foo' -> x='foo'
+    Blah'blah'blah -> Blah'blah'blah
+    ""quotes"s -> ""quotes"s
+    \"escaped\" -> \"escaped\"
+    'apos -> 'apos
+    'apos'trophes -> 'apos'trophes
+    $James' -> $James'
+
+    Template tag content is never modified:
+
+    {% field kind="string" %} -> {% field kind="string" %}
+    {{ variable }} -> {{ variable }}
+    {# comment "here" #} -> {# comment "here" #}
+    <!-- html kind="comment" --> -> <!-- html kind="comment" -->
+
+    """
+    # Split text into segments: template tags (protected) and regular text.
+    # We apply smart quotes only to regular text segments.
+    segments: list[str] = []
+    last_end = 0
+
+    for match in TEMPLATE_TAG_PATTERN.finditer(text):
+        start, end = match.span()
+
+        # Add the text before this tag (apply smart quotes to it)
+        if start > last_end:
+            before_text = text[last_end:start]
+            segments.append(_apply_smart_quotes_to_text(before_text))
+
+        # Add the tag itself unchanged
+        segments.append(match.group(0))
+        last_end = end
+
+    # Add any remaining text after the last tag
+    if last_end < len(text):
+        remaining = text[last_end:]
+        segments.append(_apply_smart_quotes_to_text(remaining))
+
+    return "".join(segments)