set header autoid in readers

Author: cscheid

crates/quarto_markdown_pandoc/src/filters.rs

@@ -7,8 +7,6 @@ use crate::pandoc::MetaValue;
 use crate::pandoc::block::MetaBlock;
 use crate::pandoc::inline::AsInline;
 use crate::pandoc::meta::Meta;
-use crate::pandoc::meta::parse_metadata_strings;
-use crate::pandoc::meta::rawblock_to_meta;
 use crate::pandoc::{self, Block, Blocks, Inline, Inlines};
 
 // filters are destructive and take ownership of the input

crates/quarto_markdown_pandoc/src/pandoc/mod.rs

@@ -33,6 +33,5 @@ pub use crate::pandoc::table::{
     Alignment, Cell, ColWidth, Row, Table, TableBody, TableFoot, TableHead,
 };
 
-pub use crate::pandoc::location::{Location, Range};
 pub use crate::pandoc::meta::{Meta, MetaValue, rawblock_to_meta};
 pub use crate::pandoc::treesitter::treesitter_to_pandoc;

crates/quarto_markdown_pandoc/src/pandoc/treesitter.rs

@@ -24,6 +24,7 @@ use crate::pandoc::shortcode::{Shortcode, ShortcodeArg, shortcode_to_span};
 use crate::pandoc::table::{
     Alignment, Cell, ColSpec, ColWidth, Row, Table, TableBody, TableFoot, TableHead,
 };
+use crate::utils::autoid;
 use core::panic;
 use once_cell::sync::Lazy;
 use regex::Regex;
@@ -1816,15 +1817,22 @@ fn desugar(doc: Pandoc) -> Result<Pandoc, Vec<String>> {
                     .last()
                     .map_or(false, |v| matches!(v, Inline::Attr(_)));
                 if !is_last_attr {
-                    return Unchanged(header);
+                    let mut attr = header.attr.clone();
+                    if attr.0.is_empty() {
+                        attr.0 = autoid::auto_generated_id(&header.content);
+                        header.attr = attr;
+                        FilterResult(vec![Block::Header(header)], true)
+                    } else {
+                        Unchanged(header)
+                    }
+                } else {
+                    let Some(Inline::Attr(attr)) = header.content.pop() else {
+                        panic!("shouldn't happen, header should have an attribute at this point");
+                    };
+                    header.attr = attr;
+                    header.content = trim_inlines(header.content).0;
+                    FilterResult(vec![Block::Header(header)], true)
                 }
-
-                let Some(Inline::Attr(attr)) = header.content.pop() else {
-                    panic!("shouldn't happen, header should have an attribute at this point");
-                };
-                header.attr = attr;
-                header.content = trim_inlines(header.content).0;
-                FilterResult(vec![Block::Header(header)], true)
             })
             // attempt to desugar single-image paragraphs into figures
             .with_paragraph(|para| {

crates/quarto_markdown_pandoc/src/readers/qmd.rs

@@ -76,12 +76,8 @@ pub fn read<T: Write>(
     let mut result = pandoc::treesitter_to_pandoc(&mut output_stream, &tree, &input_bytes)?;
     let mut meta_from_parses = Meta::default();
 
-    eprintln!("Hello?");
-    eprintln!("{:?}", result);
-
     result = {
         let mut filter = Filter::new().with_raw_block(|rb| {
-            eprintln!("raw block: {:?}", rb);
             if rb.format != "quarto_minus_metadata" {
                 return Unchanged(rb);
             }
@@ -131,6 +127,5 @@ pub fn read<T: Write>(
     for (k, v) in meta_from_parses.into_iter() {
         result.meta.insert(k, v);
     }
-    eprintln!("result\n{:?}", result);
     Ok(result)
 }

crates/quarto_markdown_pandoc/src/writers/json.rs

@@ -5,7 +5,6 @@
 
 use crate::pandoc::{Attr, Block, Caption, CitationMode, Inline, Inlines, ListAttributes, Pandoc};
 use crate::utils::autoid;
-use crate::writers::json;
 use serde_json::{Value, json};
 
 fn write_location<T: crate::pandoc::location::SourceLocation>(item: &T) -> Value {
@@ -254,13 +253,9 @@ fn write_block(block: &Block) -> Value {
             "l": write_location(para),
         }),
         Block::Header(header) => {
-            let mut attr = header.attr.clone();
-            if attr.0.is_empty() {
-                attr.0 = autoid::auto_generated_id(&header.content);
-            }
             json!({
                 "t": "Header",
-                "c": [header.level, write_attr(&attr), write_inlines(&header.content)],
+                "c": [header.level, write_attr(&header.attr), write_inlines(&header.content)],
                 "l": write_location(header),
             })
         }

crates/quarto_markdown_pandoc/tests/pandoc-match-corpus/markdown/036.qmd

@@ -0,0 +1,5 @@
+---
+hello: world
+---
+
+## What

crates/quarto_markdown_pandoc/tests/test.rs

@@ -6,7 +6,7 @@
 use glob::glob;
 use quarto_markdown_pandoc::errors::parse_is_good;
 use quarto_markdown_pandoc::pandoc::treesitter_to_pandoc;
-use quarto_markdown_pandoc::writers;
+use quarto_markdown_pandoc::{readers, writers};
 use std::io::Write;
 use std::process::{Command, Stdio};
 use tree_sitter_qmd::MarkdownParser;
@@ -69,6 +69,12 @@ fn matches_canonical_pandoc_format(
     }
     let our_ast = canonicalize_pandoc_ast(ast, output_format, output_format);
     let pandoc_ast = canonicalize_pandoc_ast(markdown, pandoc_reader, output_format);
+    if our_ast != pandoc_ast {
+        eprintln!("Format: {} -> {}", pandoc_reader, output_format);
+        eprintln!("Input:\n{}", markdown);
+        eprintln!("Our AST:\n{}", our_ast);
+        eprintln!("Pandoc AST:\n{}", pandoc_ast);
+    }
     our_ast == pandoc_ast
 }
 
@@ -78,32 +84,18 @@ fn matches_pandoc_markdown_reader(input: &str) -> bool {
     }
     let mut buf1 = Vec::new();
     let mut buf2 = Vec::new();
-    writers::native::write(
-        &treesitter_to_pandoc(
-            &mut std::io::sink(),
-            &MarkdownParser::default()
-                .parse(input.as_bytes(), None)
-                .unwrap(),
-            input.as_bytes(),
-        )
-        .unwrap(),
-        &mut buf1,
-    )
-    .unwrap();
+
+    let doc = readers::qmd::read(input.as_bytes(), &mut std::io::sink()).unwrap();
+    writers::native::write(&doc, &mut buf1).unwrap();
     let native_output = String::from_utf8(buf1).expect("Invalid UTF-8 in output");
-    writers::json::write(
-        &treesitter_to_pandoc(
-            &mut std::io::sink(),
-            &MarkdownParser::default()
-                .parse(input.as_bytes(), None)
-                .unwrap(),
-            input.as_bytes(),
-        )
-        .unwrap(),
-        &mut buf2,
-    )
-    .unwrap();
+    writers::json::write(&doc, &mut buf2).unwrap();
     let json_output = String::from_utf8(buf2).expect("Invalid UTF-8 in output");
+
+    let mut our_value: serde_json::Value =
+        serde_json::from_str(&json_output).expect("Failed to parse our JSON");
+    remove_location_fields(&mut our_value);
+    let json_output = serde_json::to_string(&our_value).expect("Failed to serialize our JSON");
+
     matches_canonical_pandoc_format(input, &native_output, "markdown", "native")
         && matches_canonical_pandoc_format(input, &json_output, "markdown", "json")
 }

crates/quarto_markdown_pandoc/tests/test_meta.rs

@@ -1,4 +1,5 @@
-use quarto_markdown_pandoc::pandoc::{Location, MetaValue, Range, RawBlock, rawblock_to_meta};
+use quarto_markdown_pandoc::pandoc::location::{Location, Range};
+use quarto_markdown_pandoc::pandoc::{MetaValue, RawBlock, rawblock_to_meta};
 use std::fs;
 
 #[test]