docs for quarto-markdown desugaring

crates/quarto-markdown-pandoc/src/pandoc/treesitter_utils/postprocess.rs

@@ -450,6 +450,45 @@ pub fn postprocess<E: ErrorCollector>(doc: Pandoc, error_collector: &mut E) -> R
                 )
             })
             .with_inlines(|inlines| {
+                // Combined filter: Handle Math + Attr pattern, then citation suffix pattern
+
+                // Step 1: Handle Math nodes followed by Attr
+                // Pattern: Math, Space (optional), Attr -> Span with "quarto-math-with-attribute" class
+                let mut math_processed = vec![];
+                let mut i = 0;
+
+                while i < inlines.len() {
+                    if let Inline::Math(math) = &inlines[i] {
+                        // Check if followed by Space then Attr, or just Attr
+                        let has_space = i + 1 < inlines.len() && matches!(inlines[i + 1], Inline::Space(_));
+                        let attr_idx = if has_space { i + 2 } else { i + 1 };
+
+                        if attr_idx < inlines.len() {
+                            if let Inline::Attr(attr) = &inlines[attr_idx] {
+                                // Found Math + (Space?) + Attr pattern
+                                // Wrap Math in a Span with the attribute
+                                let mut classes = vec!["quarto-math-with-attribute".to_string()];
+                                classes.extend(attr.1.clone());
+
+                                math_processed.push(Inline::Span(Span {
+                                    attr: (attr.0.clone(), classes, attr.2.clone()),
+                                    content: vec![Inline::Math(math.clone())],
+                                    source_info: empty_source_info(),
+                                }));
+
+                                // Skip the Math, optional Space, and Attr
+                                i = attr_idx + 1;
+                                continue;
+                            }
+                        }
+                    }
+
+                    // Not a Math + Attr pattern, add as is
+                    math_processed.push(inlines[i].clone());
+                    i += 1;
+                }
+
+                // Step 2: Handle citation suffix pattern on the math-processed result
                 let mut result = vec![];
                 // states in this state machine:
                 // 0. normal state, where we just collect inlines
@@ -461,7 +500,7 @@ pub fn postprocess<E: ErrorCollector>(doc: Pandoc, error_collector: &mut E) -> R
                 let mut state = 0;
                 let mut pending_cite: Option<crate::pandoc::inline::Cite> = None;
 
-                for inline in inlines {
+                for inline in math_processed {
                     match state {
                         0 => {
                             // Normal state - check if we see a valid cite
@@ -629,11 +668,40 @@ pub fn postprocess<E: ErrorCollector>(doc: Pandoc, error_collector: &mut E) -> R
                     if let Block::CaptionBlock(caption_block) = block {
                         // Look for a preceding Table
                         if let Some(Block::Table(table)) = result.last_mut() {
-                            // Attach caption to the table
+                            // Extract any trailing Inline::Attr from caption content
+                            let mut caption_content = caption_block.content.clone();
+                            let mut caption_attr: Option<Attr> = None;
+
+                            if let Some(Inline::Attr(attr)) = caption_content.last() {
+                                caption_attr = Some(attr.clone());
+                                caption_content.pop(); // Remove the Attr from caption content
+                            }
+
+                            // If we found attributes in the caption, merge them with the table's attr
+                            if let Some(caption_attr_value) = caption_attr {
+                                // Merge: caption attributes override table attributes
+                                // table.attr is (id, classes, key_values)
+                                // Merge key-value pairs from caption into table
+                                for (key, value) in caption_attr_value.2 {
+                                    table.attr.2.insert(key, value);
+                                }
+                                // Merge classes from caption into table
+                                for class in caption_attr_value.1 {
+                                    if !table.attr.1.contains(&class) {
+                                        table.attr.1.push(class);
+                                    }
+                                }
+                                // Use caption id if table doesn't have one
+                                if table.attr.0.is_empty() && !caption_attr_value.0.is_empty() {
+                                    table.attr.0 = caption_attr_value.0;
+                                }
+                            }
+
+                            // Attach caption to the table (with Attr removed from content)
                             table.caption = Caption {
                                 short: None,
                                 long: Some(vec![Block::Plain(Plain {
-                                    content: caption_block.content.clone(),
+                                    content: caption_content,
                                     source_info: caption_block.source_info.clone(),
                                 })]),
                             };

crates/quarto-markdown-pandoc/tests/snapshots/json/math-with-attr.qmd

@@ -0,0 +1,9 @@
+Inline math with attribute: $E = mc^2$ {#eq-einstein}
+
+Display math with attribute:
+
+$$
+\int_0^\infty e^{-x^2} dx = \frac{\sqrt{\pi}}{2}
+$$ {#eq-gaussian}
+
+Another inline example: $a^2 + b^2 = c^2$ {#eq-pythagorean}

crates/quarto-markdown-pandoc/tests/snapshots/json/math-with-attr.qmd.snapshot

@@ -0,0 +1 @@
+{"astContext":{"filenames":["tests/snapshots/json/math-with-attr.qmd"]},"blocks":[{"c":[{"c":"Inline","l":{"end":{"column":6,"offset":6,"row":0},"filenameIndex":0,"start":{"column":0,"offset":0,"row":0}},"t":"Str"},{"l":{"end":{"column":7,"offset":7,"row":0},"filenameIndex":0,"start":{"column":6,"offset":6,"row":0}},"t":"Space"},{"c":"math","l":{"end":{"column":11,"offset":11,"row":0},"filenameIndex":0,"start":{"column":7,"offset":7,"row":0}},"t":"Str"},{"l":{"end":{"column":12,"offset":12,"row":0},"filenameIndex":0,"start":{"column":11,"offset":11,"row":0}},"t":"Space"},{"c":"with","l":{"end":{"column":16,"offset":16,"row":0},"filenameIndex":0,"start":{"column":12,"offset":12,"row":0}},"t":"Str"},{"l":{"end":{"column":17,"offset":17,"row":0},"filenameIndex":0,"start":{"column":16,"offset":16,"row":0}},"t":"Space"},{"c":"attribute:","l":{"end":{"column":27,"offset":27,"row":0},"filenameIndex":0,"start":{"column":17,"offset":17,"row":0}},"t":"Str"},{"l":{"end":{"column":28,"offset":28,"row":0},"filenameIndex":0,"start":{"column":27,"offset":27,"row":0}},"t":"Space"},{"c":[["eq-einstein",["quarto-math-with-attribute"],[]],[{"c":[{"t":"InlineMath"},"E = mc^2"],"l":{"end":{"column":38,"offset":38,"row":0},"filenameIndex":0,"start":{"column":28,"offset":28,"row":0}},"t":"Math"}]],"l":{"end":{"column":0,"offset":0,"row":0},"filenameIndex":null,"start":{"column":0,"offset":0,"row":0}},"t":"Span"}],"l":{"end":{"column":0,"offset":54,"row":1},"filenameIndex":0,"start":{"column":0,"offset":0,"row":0}},"t":"Para"},{"c":[{"c":"Display","l":{"end":{"column":7,"offset":62,"row":2},"filenameIndex":0,"start":{"column":0,"offset":55,"row":2}},"t":"Str"},{"l":{"end":{"column":8,"offset":63,"row":2},"filenameIndex":0,"start":{"column":7,"offset":62,"row":2}},"t":"Space"},{"c":"math","l":{"end":{"column":12,"offset":67,"row":2},"filenameIndex":0,"start":{"column":8,"offset":63,"row":2}},"t":"Str"},{"l":{"end":{"column":13,"offset":68,"row":2},"filenameIndex":0,"start":{"column":12,"offset":67,"row":2}},"t":"Space"},{"c":"with","l":{"end":{"column":17,"offset":72,"row":2},"filenameIndex":0,"start":{"column":13,"offset":68,"row":2}},"t":"Str"},{"l":{"end":{"column":18,"offset":73,"row":2},"filenameIndex":0,"start":{"column":17,"offset":72,"row":2}},"t":"Space"},{"c":"attribute:","l":{"end":{"column":28,"offset":83,"row":2},"filenameIndex":0,"start":{"column":18,"offset":73,"row":2}},"t":"Str"}],"l":{"end":{"column":0,"offset":84,"row":3},"filenameIndex":0,"start":{"column":0,"offset":55,"row":2}},"t":"Para"},{"c":[{"c":[["eq-gaussian",["quarto-math-with-attribute"],[]],[{"c":[{"t":"DisplayMath"},"\n\\int_0^\\infty e^{-x^2} dx = \\frac{\\sqrt{\\pi}}{2}\n"],"l":{"end":{"column":2,"offset":139,"row":6},"filenameIndex":0,"start":{"column":0,"offset":85,"row":4}},"t":"Math"}]],"l":{"end":{"column":0,"offset":0,"row":0},"filenameIndex":null,"start":{"column":0,"offset":0,"row":0}},"t":"Span"}],"l":{"end":{"column":0,"offset":155,"row":7},"filenameIndex":0,"start":{"column":0,"offset":85,"row":4}},"t":"Para"},{"c":[{"c":"Another","l":{"end":{"column":7,"offset":163,"row":8},"filenameIndex":0,"start":{"column":0,"offset":156,"row":8}},"t":"Str"},{"l":{"end":{"column":8,"offset":164,"row":8},"filenameIndex":0,"start":{"column":7,"offset":163,"row":8}},"t":"Space"},{"c":"inline","l":{"end":{"column":14,"offset":170,"row":8},"filenameIndex":0,"start":{"column":8,"offset":164,"row":8}},"t":"Str"},{"l":{"end":{"column":15,"offset":171,"row":8},"filenameIndex":0,"start":{"column":14,"offset":170,"row":8}},"t":"Space"},{"c":"example:","l":{"end":{"column":23,"offset":179,"row":8},"filenameIndex":0,"start":{"column":15,"offset":171,"row":8}},"t":"Str"},{"l":{"end":{"column":24,"offset":180,"row":8},"filenameIndex":0,"start":{"column":23,"offset":179,"row":8}},"t":"Space"},{"c":[["eq-pythagorean",["quarto-math-with-attribute"],[]],[{"c":[{"t":"InlineMath"},"a^2 + b^2 = c^2"],"l":{"end":{"column":41,"offset":197,"row":8},"filenameIndex":0,"start":{"column":24,"offset":180,"row":8}},"t":"Math"}]],"l":{"end":{"column":0,"offset":0,"row":0},"filenameIndex":null,"start":{"column":0,"offset":0,"row":0}},"t":"Span"}],"l":{"end":{"column":0,"offset":216,"row":9},"filenameIndex":0,"start":{"column":0,"offset":156,"row":8}},"t":"Para"}],"meta":{},"pandoc-api-version":[1,23,1]}

crates/quarto-markdown-pandoc/tests/snapshots/json/table-caption-attr.qmd

@@ -0,0 +1,5 @@
+| Column 1 | Column 2 |
+|----------|----------|
+| Data 1   | Data 2   |
+
+: Table caption {tbl-colwidths="[30,70]"}

crates/quarto-markdown-pandoc/tests/snapshots/json/table-caption-attr.qmd.snapshot

@@ -0,0 +1 @@
+{"astContext":{"filenames":["tests/snapshots/json/table-caption-attr.qmd"]},"blocks":[{"c":[["",[],[["tbl-colwidths","[30,70]"]]],[null,[{"c":[{"c":"Table","l":{"end":{"column":7,"offset":80,"row":4},"filenameIndex":0,"start":{"column":2,"offset":75,"row":4}},"t":"Str"},{"l":{"end":{"column":8,"offset":81,"row":4},"filenameIndex":0,"start":{"column":7,"offset":80,"row":4}},"t":"Space"},{"c":"caption","l":{"end":{"column":15,"offset":88,"row":4},"filenameIndex":0,"start":{"column":8,"offset":81,"row":4}},"t":"Str"},{"l":{"end":{"column":16,"offset":89,"row":4},"filenameIndex":0,"start":{"column":15,"offset":88,"row":4}},"t":"Space"}],"l":{"end":{"column":0,"offset":115,"row":5},"filenameIndex":0,"start":{"column":0,"offset":72,"row":3}},"t":"Plain"}]],[[{"t":"AlignDefault"},{"t":"ColWidthDefault"}],[{"t":"AlignDefault"},{"t":"ColWidthDefault"}]],[["",[],[]],[[["",[],[]],[[["",[],[]],{"t":"AlignDefault"},1,1,[{"c":[{"c":"Column","l":{"end":{"column":8,"offset":8,"row":0},"filenameIndex":0,"start":{"column":2,"offset":2,"row":0}},"t":"Str"},{"l":{"end":{"column":9,"offset":9,"row":0},"filenameIndex":0,"start":{"column":8,"offset":8,"row":0}},"t":"Space"},{"c":"1","l":{"end":{"column":10,"offset":10,"row":0},"filenameIndex":0,"start":{"column":9,"offset":9,"row":0}},"t":"Str"}],"l":{"end":{"column":11,"offset":11,"row":0},"filenameIndex":0,"start":{"column":2,"offset":2,"row":0}},"t":"Plain"}]],[["",[],[]],{"t":"AlignDefault"},1,1,[{"c":[{"c":"Column","l":{"end":{"column":19,"offset":19,"row":0},"filenameIndex":0,"start":{"column":13,"offset":13,"row":0}},"t":"Str"},{"l":{"end":{"column":20,"offset":20,"row":0},"filenameIndex":0,"start":{"column":19,"offset":19,"row":0}},"t":"Space"},{"c":"2","l":{"end":{"column":21,"offset":21,"row":0},"filenameIndex":0,"start":{"column":20,"offset":20,"row":0}},"t":"Str"}],"l":{"end":{"column":22,"offset":22,"row":0},"filenameIndex":0,"start":{"column":13,"offset":13,"row":0}},"t":"Plain"}]]]]]],[[["",[],[]],0,[],[[["",[],[]],[[["",[],[]],{"t":"AlignDefault"},1,1,[{"c":[{"c":"Data","l":{"end":{"column":6,"offset":54,"row":2},"filenameIndex":0,"start":{"column":2,"offset":50,"row":2}},"t":"Str"},{"l":{"end":{"column":7,"offset":55,"row":2},"filenameIndex":0,"start":{"column":6,"offset":54,"row":2}},"t":"Space"},{"c":"1","l":{"end":{"column":8,"offset":56,"row":2},"filenameIndex":0,"start":{"column":7,"offset":55,"row":2}},"t":"Str"}],"l":{"end":{"column":11,"offset":59,"row":2},"filenameIndex":0,"start":{"column":2,"offset":50,"row":2}},"t":"Plain"}]],[["",[],[]],{"t":"AlignDefault"},1,1,[{"c":[{"c":"Data","l":{"end":{"column":17,"offset":65,"row":2},"filenameIndex":0,"start":{"column":13,"offset":61,"row":2}},"t":"Str"},{"l":{"end":{"column":18,"offset":66,"row":2},"filenameIndex":0,"start":{"column":17,"offset":65,"row":2}},"t":"Space"},{"c":"2","l":{"end":{"column":19,"offset":67,"row":2},"filenameIndex":0,"start":{"column":18,"offset":66,"row":2}},"t":"Str"}],"l":{"end":{"column":22,"offset":70,"row":2},"filenameIndex":0,"start":{"column":13,"offset":61,"row":2}},"t":"Plain"}]]]]]]],[["",[],[]],[]]],"l":{"end":{"column":0,"offset":72,"row":3},"filenameIndex":0,"start":{"column":0,"offset":0,"row":0}},"t":"Table"}],"meta":{},"pandoc-api-version":[1,23,1]}

docs/syntax/desugaring/definition-lists.qmd

@@ -0,0 +1,79 @@
+---
+title: "Definition Lists"
+---
+
+## Overview
+
+Quarto Markdown supports definition lists through a special div syntax with the `definition-list` class. During post-processing, divs meeting the structural requirements are transformed into Pandoc's native `DefinitionList` blocks.
+
+## Transformation
+
+A div with class `definition-list` containing a specific bullet list structure is converted to a `DefinitionList` block.
+
+### Required Structure
+
+```markdown
+::: {.definition-list}
+- Term 1
+  - Definition 1a
+  - Definition 1b
+- Term 2
+  - Definition 2
+:::
+```
+
+The structure must follow these rules:
+
+1. Div must have `definition-list` class
+2. Div contains exactly one bullet list
+3. Each list item has exactly two blocks:
+   - First: Plain or Paragraph (the term)
+   - Second: BulletList (the definitions)
+
+## Example
+
+### Input QMD
+
+```markdown
+::: {.definition-list}
+- **Markdown**
+  - A lightweight markup language
+  - Easy to read and write
+- **Pandoc**
+  - A universal document converter
+:::
+```
+
+### Output Structure
+
+Transforms to a `DefinitionList` block:
+
+```json
+{
+  "t": "DefinitionList",
+  "c": [
+    [
+      [{"t": "Strong", "c": [{"t": "Str", "c": "Markdown"}]}],
+      [
+        [[{"t": "Plain", "c": [{"t": "Str", "c": "A lightweight markup language"}]}]],
+        [[{"t": "Plain", "c": [{"t": "Str", "c": "Easy to read and write"}]}]]
+      ]
+    ],
+    [
+      [{"t": "Strong", "c": [{"t": "Str", "c": "Pandoc"}]}],
+      [
+        [[{"t": "Plain", "c": [{"t": "Str", "c": "A universal document converter"}]}]]
+      ]
+    ]
+  ]
+}
+```
+
+## Validation
+
+Invalid structures are left as regular divs. Common validation failures:
+
+- Div contains more than one bullet list
+- List items don't have exactly two blocks
+- First block is not Plain or Paragraph
+- Second block is not a BulletList

docs/syntax/desugaring/editorial-marks.qmd

@@ -0,0 +1,72 @@
+---
+title: "Editorial Marks"
+---
+
+## Overview
+
+Quarto Markdown's editorial marks (`[!! highlight]`, `[++ insert]`, `[-- delete]`, `[>> comment]`) are custom inline node types that don't exist in Pandoc's AST. During post-processing, these nodes are desugared into standard `Span` nodes with special classes.
+
+## Transformation
+
+All four editorial mark types follow the same desugaring pattern:
+
+| Original Node | Special Class | Example |
+|---------------|---------------|---------|
+| `Insert` | `quarto-insert` | `[++ text]` |
+| `Delete` | `quarto-delete` | `[-- text]` |
+| `Highlight` | `quarto-highlight` | `[!! text]` |
+| `EditComment` | `quarto-edit-comment` | `[>> text]` |
+
+The content is trimmed (leading/trailing spaces removed) before being placed in the Span.
+
+## Example
+
+### Input QMD
+
+```markdown
+This has [++ added text]{#my-add .important} and [!! highlighted]{.warn}.
+```
+
+### Output AST (simplified)
+
+```json
+[
+  {"t": "Str", "c": "This"},
+  {"t": "Space"},
+  {"t": "Str", "c": "has"},
+  {"t": "Space"},
+  {
+    "t": "Span",
+    "c": [
+      ["my-add", ["quarto-insert", "important"], []],
+      [{"t": "Str", "c": "added"}, {"t": "Space"}, {"t": "Str", "c": "text"}]
+    ]
+  },
+  {"t": "Space"},
+  {"t": "Str", "c": "and"},
+  {"t": "Space"},
+  {
+    "t": "Span",
+    "c": [
+      ["", ["quarto-highlight", "warn"], []],
+      [{"t": "Str", "c": "highlighted"}]
+    ]
+  }
+]
+```
+
+## Recognition
+
+Downstream tools can identify desugared editorial marks by checking for the special classes:
+
+```lua
+if span.classes:includes("quarto-insert") then
+  -- Handle insertion suggestion
+elseif span.classes:includes("quarto-delete") then
+  -- Handle deletion suggestion
+elseif span.classes:includes("quarto-highlight") then
+  -- Handle highlight
+elseif span.classes:includes("quarto-edit-comment") then
+  -- Handle editorial comment
+end
+```

docs/syntax/desugaring/index.qmd

@@ -0,0 +1,22 @@
+---
+title: "AST Desugaring"
+---
+
+Desugaring is the process of transforming extended syntax constructs into simpler, equivalent representations in the AST. Quarto Markdown includes several syntax features that don't have direct equivalents in Pandoc's standard AST. During the parsing and post-processing phases, `quarto-markdown-pandoc` transforms these extended constructs into standard Pandoc AST nodes with special attributes or classes, allowing downstream tools to recognize and process them appropriately.
+
+## Desugaring Transformations
+
+The following transformations are applied during AST post-processing:
+
+- [**Math with Attributes**](math-attributes.qmd) - Math expressions followed by attributes are wrapped in Span nodes with a special class
+- [**Editorial Marks**](editorial-marks.qmd) - Insert, Delete, Highlight, and EditComment nodes are converted to Span nodes with identifying classes
+- [**Table Caption Attributes**](table-captions.qmd) - Attributes in table captions are extracted and merged with the table's attribute field
+- [**Definition Lists**](definition-lists.qmd) - Divs with `definition-list` class are transformed into DefinitionList blocks
+- [**Note References**](note-references.qmd) - NoteReference nodes are converted to Span nodes with reference metadata
+- **Figures** - Single-image paragraphs are automatically promoted to Figure blocks with captions
+- **Shortcodes** - Shortcode nodes are transformed into Span nodes
+- **Citation Suffixes** - Citation followed by space and span are merged into citation with suffix
+
+## Implementation
+
+All desugaring transformations are implemented in `src/pandoc/treesitter_utils/postprocess.rs`. The transformations are applied using a filter-based traversal system that walks the AST and applies pattern-matching transformations.