test with code closer to main.rs, update snapshots and claude memory

Author: cscheid

CLAUDE.md

@@ -9,3 +9,5 @@ The main documentation for this repository is located at:
 - When fixing bugs using tests, run the failing test before attempting to fix issues. This helps ensuring that tests are exercising the failure as expected, and fixes actually fix the particular issue.
 - If you need to fix parser bugs, you will find use in running the application with "-v", which will provide a large amount of information from the tree-sitter parsing process, including a print of the concrete syntax tree out to stderr.
 - use "cargo run --" instead of trying to find the binary location, which will often be outside of this crate.
+- when calling shell scripts, be mindful of the current directory you're operating in. use `pwd` as necessary to avoid confusing yourself over commands that use relative paths.
+- use `jq` instead of `python3 -m json.tool` for pretty-printing. When processing JSON in a shell pipeline, prefer `jq` when possible.

crates/quarto-markdown-pandoc/tests/claude-examples/nested_meta.qmd

@@ -0,0 +1,11 @@
+---
+title: metadata1
+---
+
+::: hello
+
+---
+nested: meta
+---
+
+:::

crates/quarto-markdown-pandoc/tests/claude-examples/nested_meta_scoped.qmd

@@ -0,0 +1,12 @@
+---
+title: metadata1
+---
+
+::: hello
+
+---
+_scope: lexical
+nested: meta
+---
+
+:::

crates/quarto-markdown-pandoc/tests/scratch/008.qmd

@@ -0,0 +1,28 @@
+---
+title: abc
+format: pdf
+---
+# Test
+
+:::: {.callout-tip}
+To generate random numbers, you can use
+
+::: {.columns}
+::: {.column width="50%"}
+Standard python
+
+```py
+import python
+random.randint(0,100)
+```
+:::
+::: {.column width="50%"}
+Numpy
+
+```py
+import numpy as np
+np.random.default_rng().integers(0,100)
+```
+:::
+:::
+::::

crates/quarto-markdown-pandoc/tests/snapshots/json/001.qmd.snapshot

@@ -1 +1 @@
-{"blocks":[{"c":[{"c":"This","l":{"end":{"column":4,"offset":4,"row":0},"filename":null,"start":{"column":0,"offset":0,"row":0}},"t":"Str"},{"l":{"end":{"column":5,"offset":5,"row":0},"filename":null,"start":{"column":4,"offset":4,"row":0}},"t":"Space"},{"c":"is","l":{"end":{"column":7,"offset":7,"row":0},"filename":null,"start":{"column":5,"offset":5,"row":0}},"t":"Str"},{"l":{"end":{"column":8,"offset":8,"row":0},"filename":null,"start":{"column":7,"offset":7,"row":0}},"t":"Space"},{"c":"a","l":{"end":{"column":9,"offset":9,"row":0},"filename":null,"start":{"column":8,"offset":8,"row":0}},"t":"Str"},{"l":{"end":{"column":10,"offset":10,"row":0},"filename":null,"start":{"column":9,"offset":9,"row":0}},"t":"Space"},{"c":[{"c":"bold","l":{"end":{"column":16,"offset":16,"row":0},"filename":null,"start":{"column":12,"offset":12,"row":0}},"t":"Str"}],"l":{"end":{"column":18,"offset":18,"row":0},"filename":null,"start":{"column":10,"offset":10,"row":0}},"t":"Strong"},{"l":{"end":{"column":19,"offset":19,"row":0},"filename":null,"start":{"column":18,"offset":18,"row":0}},"t":"Space"},{"c":"test.","l":{"end":{"column":24,"offset":24,"row":0},"filename":null,"start":{"column":19,"offset":19,"row":0}},"t":"Str"}],"l":{"end":{"column":24,"offset":24,"row":0},"filename":null,"start":{"column":0,"offset":0,"row":0}},"t":"Para"}],"meta":{},"pandoc-api-version":[1,23,1]}
+{"blocks":[{"c":[{"c":"This","l":{"end":{"column":4,"offset":4,"row":0},"filename":null,"start":{"column":0,"offset":0,"row":0}},"t":"Str"},{"l":{"end":{"column":5,"offset":5,"row":0},"filename":null,"start":{"column":4,"offset":4,"row":0}},"t":"Space"},{"c":"is","l":{"end":{"column":7,"offset":7,"row":0},"filename":null,"start":{"column":5,"offset":5,"row":0}},"t":"Str"},{"l":{"end":{"column":8,"offset":8,"row":0},"filename":null,"start":{"column":7,"offset":7,"row":0}},"t":"Space"},{"c":"a","l":{"end":{"column":9,"offset":9,"row":0},"filename":null,"start":{"column":8,"offset":8,"row":0}},"t":"Str"},{"l":{"end":{"column":10,"offset":10,"row":0},"filename":null,"start":{"column":9,"offset":9,"row":0}},"t":"Space"},{"c":[{"c":"bold","l":{"end":{"column":16,"offset":16,"row":0},"filename":null,"start":{"column":12,"offset":12,"row":0}},"t":"Str"}],"l":{"end":{"column":18,"offset":18,"row":0},"filename":null,"start":{"column":10,"offset":10,"row":0}},"t":"Strong"},{"l":{"end":{"column":19,"offset":19,"row":0},"filename":null,"start":{"column":18,"offset":18,"row":0}},"t":"Space"},{"c":"test.","l":{"end":{"column":24,"offset":24,"row":0},"filename":null,"start":{"column":19,"offset":19,"row":0}},"t":"Str"}],"l":{"end":{"column":0,"offset":25,"row":1},"filename":null,"start":{"column":0,"offset":0,"row":0}},"t":"Para"}],"meta":{},"pandoc-api-version":[1,23,1]}

crates/quarto-markdown-pandoc/tests/snapshots/json/002.qmd

@@ -0,0 +1,11 @@
+---
+title: metadata1
+---
+
+::: hello
+
+---
+nested: meta
+---
+
+:::

crates/quarto-markdown-pandoc/tests/snapshots/json/002.qmd.snapshot

@@ -0,0 +1 @@
+{"blocks":[{"c":[["",["hello"],[]],[]],"l":{"end":{"column":0,"offset":63,"row":11},"filename":null,"start":{"column":0,"offset":26,"row":4}},"t":"Div"}],"meta":{"nested":{"c":[{"c":"meta","l":{"end":{"column":4,"offset":4,"row":0},"filename":null,"start":{"column":0,"offset":0,"row":0}},"t":"Str"}],"t":"MetaInlines"},"title":{"c":[{"c":"metadata1","l":{"end":{"column":9,"offset":9,"row":0},"filename":null,"start":{"column":0,"offset":0,"row":0}},"t":"Str"}],"t":"MetaInlines"}},"pandoc-api-version":[1,23,1]}

crates/quarto-markdown-pandoc/tests/snapshots/json/003.qmd

@@ -0,0 +1,12 @@
+---
+title: metadata1
+---
+
+::: hello
+
+---
+_scope: lexical
+nested: meta
+---
+
+:::

crates/quarto-markdown-pandoc/tests/snapshots/json/003.qmd.snapshot

@@ -0,0 +1 @@
+{"blocks":[{"c":[["",["hello"],[]],[{"c":{"_scope":{"c":[{"c":"lexical","l":{"end":{"column":7,"offset":7,"row":0},"filename":null,"start":{"column":0,"offset":0,"row":0}},"t":"Str"}],"t":"MetaInlines"},"nested":{"c":[{"c":"meta","l":{"end":{"column":4,"offset":4,"row":0},"filename":null,"start":{"column":0,"offset":0,"row":0}},"t":"Str"}],"t":"MetaInlines"}},"l":{"end":{"column":0,"offset":74,"row":10},"filename":null,"start":{"column":0,"offset":37,"row":6}},"t":"BlockMetadata"}]],"l":{"end":{"column":0,"offset":79,"row":12},"filename":null,"start":{"column":0,"offset":26,"row":4}},"t":"Div"}],"meta":{"title":{"c":[{"c":"metadata1","l":{"end":{"column":9,"offset":9,"row":0},"filename":null,"start":{"column":0,"offset":0,"row":0}},"t":"Str"}],"t":"MetaInlines"}},"pandoc-api-version":[1,23,1]}

crates/quarto-markdown-pandoc/tests/test.rs

@@ -6,8 +6,9 @@
 use glob::glob;
 use quarto_markdown_pandoc::errors::parse_is_good;
 use quarto_markdown_pandoc::pandoc::treesitter_to_pandoc;
+use quarto_markdown_pandoc::utils::output::VerboseOutput;
 use quarto_markdown_pandoc::{readers, writers};
-use std::io::Write;
+use std::io::{self, Write};
 use std::process::{Command, Stdio};
 use tree_sitter_qmd::MarkdownParser;
 
@@ -242,18 +243,20 @@ where
         match entry {
             Ok(path) => {
                 eprintln!("Opening file: {}", path.display());
-                let input = std::fs::read_to_string(&path).expect("Failed to read file");
                 let snapshot_path = path.with_extension("qmd.snapshot");
                 let mut buffer = Vec::new();
-
-                let pandoc = treesitter_to_pandoc(
-                    &mut std::io::sink(),
-                    &MarkdownParser::default()
-                        .parse(input.as_bytes(), None)
-                        .unwrap(),
+                let mut input = std::fs::read_to_string(&path).expect("Failed to read file");
+                if !input.ends_with("\n") {
+                    input.push('\n'); // ensure the input ends with a newline
+                }
+                let mut output_stream = VerboseOutput::Sink(io::sink());
+                let pandoc = readers::qmd::read(
                     input.as_bytes(),
-                )
-                .unwrap();
+                    false,
+                    &path.to_string_lossy(),
+                    &mut output_stream,
+                    None::<fn(&[u8], &quarto_markdown_pandoc::utils::tree_sitter_log_observer::TreeSitterLogObserver, &str) -> Vec<String>>,
+                ).unwrap();
 
                 writer(&pandoc, &mut buffer).unwrap();
                 let output = String::from_utf8(buffer).expect("Invalid UTF-8 in output");