table caption fixes (#52)

Author: cscheid

crates/quarto-markdown-pandoc/CLAUDE.md

@@ -19,10 +19,18 @@ need to be done with the traversal helpers in traversals.rs.
 - **IMPORTANT**: When making changes to the code, ALWAYS run both `cargo check` AND `cargo test` to ensure changes compile and don't affect behavior. The test suite is fast enough to run after each change. Never skip running `cargo test` - it must always be executed together with `cargo check`.
 - **CRITICAL**: Do NOT assume changes are safe if ANY tests fail, even if they seem unrelated. Some tests require pandoc to be properly installed to pass. Always ensure ALL tests pass before and after changes.
 
-## Environment setup
+## **CRITICAL**: HOW TO DO CODING WORK IN THIS REPO
 
-- Rust toolchain is installed at `/home/claude-sandbox/.cargo/bin`
-- Pandoc is installed at `/home/claude-sandbox/local/bin`
+Whenever you start working on a coding task, follow these steps:
+
+- Make a plan to yourself.
+  - The plan should include adding appropriate tests to the test suite.
+  - Before implementing the feature, write the test that you think should fail, and ensure that the test fails the way you expect to!
+- Work on the plan item by item.
+- You are not done until the test you wrote passes.
+- You are not done until the test you wrote is integrated to our test suite.
+- If you run out of ideas and still can't make the test pass, do not erase the test. Report back to me and we will work on it together.
+- If in the process of writing tests you run into an unexpected parse error, store it in a separate file and report it to me. We're still improving the parser and it's possible that you will run into bugs.
 
 # Error messages
 
@@ -44,6 +52,7 @@ After changing any of the resources/error-corpus/*.{json,qmd} files, run the scr
 
 The `quarto-markdown-pandoc` binary accepts the following options:
 - `-t, --to <TO>`: Output format (default: native)
+- `-f, --from <FROM>`: Input format (default: qmd)
 - `-v, --verbose`: Verbose output
 - `-i, --input <INPUT>`: Input file (default: stdin)
 - `--loose`: Loose parsing mode
@@ -52,8 +61,8 @@ The `quarto-markdown-pandoc` binary accepts the following options:
 
 ## Instructions
 
-- in this repository, "qmd" means "quarto markdown", the dialect of markdown we are developing. Although we aim to be largely compatible with Pandoc, it is not necessarily the case that a discrepancy in the behavior is a bug.
-- the qmd format only supports the inline syntax for a link [link](./target.html), and not the reference-style syntax [link][1].
+- In this repository, "qmd" means "quarto markdown", the dialect of markdown we are developing. Although we aim to be largely compatible with Pandoc, it is not necessarily the case that a discrepancy in the behavior is a bug.
+- The qmd format only supports the inline syntax for a link [link](./target.html), and not the reference-style syntax [link][1].
 - Always strive for test documents as small as possible. Prefer a large number of small test documents instead of small number of large documents.
 - When fixing bugs, always try to isolate and fix one bug at a time.
 - When fixing bugs using tests, run the failing test before attempting to fix issues. This helps ensuring that tests are exercising the failure as expected, and fixes actually fix the particular issue.

crates/quarto-markdown-pandoc/src/filters.rs

@@ -742,6 +742,12 @@ pub fn topdown_traverse_block(block: Block, filter: &mut Filter) -> Blocks {
                 },
             )]
         }
+        Block::CaptionBlock(_) => {
+            // CaptionBlock should have been removed by postprocessing
+            panic!(
+                "CaptionBlock found in filter - should have been processed during postprocessing"
+            )
+        }
     }
 }
 

crates/quarto-markdown-pandoc/src/pandoc/block.rs

@@ -35,6 +35,7 @@ pub enum Block {
     BlockMetadata(MetaBlock),
     NoteDefinitionPara(NoteDefinitionPara),
     NoteDefinitionFencedBlock(NoteDefinitionFencedBlock),
+    CaptionBlock(CaptionBlock),
 }
 
 pub type Blocks = Vec<Block>;
@@ -144,6 +145,12 @@ pub struct NoteDefinitionFencedBlock {
     pub source_info: SourceInfo,
 }
 
+#[derive(Debug, Clone, PartialEq)]
+pub struct CaptionBlock {
+    pub content: Inlines,
+    pub source_info: SourceInfo,
+}
+
 impl_source_location!(
     // blocks
     Plain,
@@ -163,7 +170,8 @@ impl_source_location!(
     // quarto extensions
     MetaBlock,
     NoteDefinitionPara,
-    NoteDefinitionFencedBlock
+    NoteDefinitionFencedBlock,
+    CaptionBlock
 );
 
 fn make_block_leftover(node: &tree_sitter::Node, input_bytes: &[u8]) -> Block {

crates/quarto-markdown-pandoc/src/pandoc/treesitter.rs

@@ -8,6 +8,7 @@ use crate::pandoc::treesitter_utils::attribute::process_attribute;
 use crate::pandoc::treesitter_utils::atx_heading::process_atx_heading;
 use crate::pandoc::treesitter_utils::backslash_escape::process_backslash_escape;
 use crate::pandoc::treesitter_utils::block_quote::process_block_quote;
+use crate::pandoc::treesitter_utils::caption::process_caption;
 use crate::pandoc::treesitter_utils::citation::process_citation;
 use crate::pandoc::treesitter_utils::code_fence_content::process_code_fence_content;
 use crate::pandoc::treesitter_utils::code_span::process_code_span;
@@ -86,6 +87,7 @@ fn get_block_source_info(block: &Block) -> &SourceInfo {
         Block::BlockMetadata(b) => &b.source_info,
         Block::NoteDefinitionPara(b) => &b.source_info,
         Block::NoteDefinitionFencedBlock(b) => &b.source_info,
+        Block::CaptionBlock(b) => &b.source_info,
     }
 }
 
@@ -733,7 +735,7 @@ fn native_visitor<T: Write>(
         }
         "pipe_table_delimiter_row" => process_pipe_table_delimiter_row(children, context),
         "pipe_table_cell" => process_pipe_table_cell(node, children, context),
-        "table_caption" => PandocNativeIntermediate::IntermediateInlines(native_inlines(children)),
+        "caption" => process_caption(node, children, context),
         "pipe_table" => process_pipe_table(node, children, context),
         "setext_h1_underline" => PandocNativeIntermediate::IntermediateSetextHeadingLevel(1),
         "setext_h2_underline" => PandocNativeIntermediate::IntermediateSetextHeadingLevel(2),

crates/quarto-markdown-pandoc/src/pandoc/treesitter_utils/caption.rs

@@ -0,0 +1,39 @@
+/*
+ * caption.rs
+ *
+ * Functions for processing caption nodes in the tree-sitter AST.
+ *
+ * Copyright (c) 2025 Posit, PBC
+ */
+
+use crate::pandoc::ast_context::ASTContext;
+use crate::pandoc::block::{Block, CaptionBlock};
+use crate::pandoc::inline::Inlines;
+use crate::pandoc::location::node_source_info_with_context;
+
+use super::pandocnativeintermediate::PandocNativeIntermediate;
+
+pub fn process_caption(
+    node: &tree_sitter::Node,
+    children: Vec<(String, PandocNativeIntermediate)>,
+    context: &ASTContext,
+) -> PandocNativeIntermediate {
+    let mut caption_inlines: Inlines = Vec::new();
+
+    for (node_name, child) in children {
+        if node_name == "inline" {
+            match child {
+                PandocNativeIntermediate::IntermediateInlines(inlines) => {
+                    caption_inlines.extend(inlines);
+                }
+                _ => panic!("Expected Inlines in caption, got {:?}", child),
+            }
+        }
+        // Skip other nodes like ":", blank_line, etc.
+    }
+
+    PandocNativeIntermediate::IntermediateBlock(Block::CaptionBlock(CaptionBlock {
+        content: caption_inlines,
+        source_info: node_source_info_with_context(node, context),
+    }))
+}

crates/quarto-markdown-pandoc/src/pandoc/treesitter_utils/mod.rs

@@ -7,6 +7,7 @@ pub mod attribute;
 pub mod atx_heading;
 pub mod backslash_escape;
 pub mod block_quote;
+pub mod caption;
 pub mod citation;
 pub mod code_fence_content;
 pub mod code_span;

crates/quarto-markdown-pandoc/src/pandoc/treesitter_utils/postprocess.rs

@@ -7,7 +7,7 @@ use crate::filters::{
     Filter, FilterReturn::FilterResult, FilterReturn::Unchanged, topdown_traverse,
 };
 use crate::pandoc::attr::{Attr, is_empty_attr};
-use crate::pandoc::block::{Block, DefinitionList, Div, Figure, Plain};
+use crate::pandoc::block::{Block, Blocks, DefinitionList, Div, Figure, Plain};
 use crate::pandoc::caption::Caption;
 use crate::pandoc::inline::{Inline, Inlines, Space, Span, Str, Superscript};
 use crate::pandoc::location::{Range, SourceInfo, empty_range, empty_source_info};
@@ -620,6 +620,42 @@ pub fn postprocess(doc: Pandoc) -> Result<Pandoc, Vec<String>> {
                     attr
                 ));
                 FilterResult(vec![], false)
+            })
+            .with_blocks(|blocks| {
+                // Process CaptionBlock nodes: attach to preceding tables or issue warnings
+                let mut result: Blocks = Vec::new();
+
+                for block in blocks {
+                    // Check if current block is a CaptionBlock
+                    if let Block::CaptionBlock(caption_block) = block {
+                        // Look for a preceding Table
+                        if let Some(Block::Table(table)) = result.last_mut() {
+                            // Attach caption to the table
+                            table.caption = Caption {
+                                short: None,
+                                long: Some(vec![Block::Plain(Plain {
+                                    content: caption_block.content.clone(),
+                                    source_info: caption_block.source_info.clone(),
+                                })]),
+                            };
+                            // Don't add the CaptionBlock to the result (it's now attached)
+                        } else {
+                            // TODO: Issue a warning/error when proper error infrastructure is ready
+                            // For now, print a warning to stderr
+                            eprintln!(
+                                "Warning: Caption found without a preceding table at {}:{}",
+                                caption_block.source_info.range.start.row + 1,
+                                caption_block.source_info.range.start.column + 1
+                            );
+                            // Remove the caption from the output (don't add to result)
+                        }
+                    } else {
+                        // Not a CaptionBlock, add it to result
+                        result.push(block);
+                    }
+                }
+
+                FilterResult(result, true)
             });
         topdown_traverse(doc, &mut filter)
     };

crates/quarto-markdown-pandoc/src/writers/json.rs

@@ -395,6 +395,11 @@ fn write_block(block: &Block) -> Value {
             "c": [refdef.id, write_blocks(&refdef.content)],
             "l": write_location(refdef),
         }),
+        Block::CaptionBlock(_) => {
+            panic!(
+                "CaptionBlock found in JSON writer - should have been processed during postprocessing"
+            )
+        }
     }
 }
 

crates/quarto-markdown-pandoc/src/writers/qmd.rs

@@ -680,6 +680,23 @@ fn write_table(table: &Table, buf: &mut dyn std::io::Write) -> std::io::Result<(
         }
     }
 
+    // Write caption if it exists
+    if let Some(ref long_caption) = table.caption.long {
+        if !long_caption.is_empty() {
+            writeln!(buf)?; // Blank line before caption
+            for block in long_caption {
+                // Extract inline content from Plain blocks in caption
+                if let Block::Plain(plain) = block {
+                    write!(buf, ": ")?;
+                    for inline in &plain.content {
+                        write_inline(inline, buf)?;
+                    }
+                    writeln!(buf)?;
+                }
+            }
+        }
+    }
+
     Ok(())
 }
 
@@ -1183,6 +1200,11 @@ fn write_block(block: &crate::pandoc::Block, buf: &mut dyn std::io::Write) -> st
         Block::NoteDefinitionFencedBlock(refdef) => {
             write_fenced_note_definition(refdef, buf)?;
         }
+        Block::CaptionBlock(_) => {
+            panic!(
+                "CaptionBlock found in QMD writer - should have been processed during postprocessing"
+            )
+        }
     }
     Ok(())
 }

crates/quarto-markdown-pandoc/tests/roundtrip_tests/qmd-json-qmd/table-caption.qmd

@@ -0,0 +1,6 @@
+| Name  | Age |
+|-------|-----|
+| Alice | 30  |
+| Bob   | 25  |
+
+: Sample table caption