improve cite processing on the front-end (#12)

Author: cscheid

crates/quarto-markdown-pandoc/src/pandoc/treesitter.rs

@@ -32,7 +32,7 @@ use crate::pandoc::treesitter_utils::pipe_table::{
     process_pipe_table, process_pipe_table_cell, process_pipe_table_delimiter_cell,
     process_pipe_table_delimiter_row, process_pipe_table_header_or_row,
 };
-use crate::pandoc::treesitter_utils::postprocess::{merge_strs, postprocess};
+use crate::pandoc::treesitter_utils::postprocess::{merge_strs, postprocess, split_cite_content_strings};
 use crate::pandoc::treesitter_utils::quoted_span::process_quoted_span;
 use crate::pandoc::treesitter_utils::raw_attribute::process_raw_attribute;
 use crate::pandoc::treesitter_utils::raw_specifier::process_raw_specifier;
@@ -753,5 +753,6 @@ pub fn treesitter_to_pandoc<T: Write>(
     };
     let result = postprocess(pandoc)?;
     let result = merge_strs(result);
+    let result = split_cite_content_strings(result);
     Ok(result)
 }

crates/quarto-markdown-pandoc/src/pandoc/treesitter_utils/postprocess.rs

@@ -412,6 +412,64 @@ pub fn postprocess(doc: Pandoc) -> Result<Pandoc, Vec<String>> {
                                     if let Some(mut cite) = pending_cite.take() {
                                         // Add span content to the citation's suffix
                                         cite.citations[0].suffix = span.content.clone();
+
+                                        // Update the content field to include the rendered suffix with brackets
+                                        // Pandoc breaks up the bracketed suffix text by spaces, with the opening
+                                        // bracket attached to the first word and closing bracket to the last word
+                                        // e.g., "@knuth [p. 33]" becomes: Str("@knuth"), Space, Str("[p."), Space, Str("33]")
+                                        cite.content.push(Inline::Space(Space {
+                                            source_info: SourceInfo::with_range(empty_range()),
+                                        }));
+
+                                        // The span content may have been merged into a single string, so we need to
+                                        // intelligently break it up to match Pandoc's behavior
+                                        let mut bracketed_content: Vec<Inline> = vec![];
+                                        for inline in &span.content {
+                                            if let Inline::Str(s) = inline {
+                                                // Split the string by spaces and create Str/Space inlines
+                                                let words: Vec<&str> = s.text.split(' ').collect();
+                                                for (i, word) in words.iter().enumerate() {
+                                                    if i > 0 {
+                                                        bracketed_content.push(Inline::Space(
+                                                            Space {
+                                                                source_info: SourceInfo::with_range(
+                                                                    empty_range(),
+                                                                ),
+                                                            },
+                                                        ));
+                                                    }
+                                                    if !word.is_empty() {
+                                                        bracketed_content.push(Inline::Str(Str {
+                                                            text: word.to_string(),
+                                                            source_info: s.source_info.clone(),
+                                                        }));
+                                                    }
+                                                }
+                                            } else {
+                                                bracketed_content.push(inline.clone());
+                                            }
+                                        }
+
+                                        // Now add brackets to the first and last Str elements
+                                        if !bracketed_content.is_empty() {
+                                            // Prepend "[" to the first Str element
+                                            if let Some(Inline::Str(first_str)) =
+                                                bracketed_content.first_mut()
+                                            {
+                                                first_str.text = format!("[{}", first_str.text);
+                                            }
+                                            // Append "]" to the last Str element (search from the end)
+                                            for i in (0..bracketed_content.len()).rev() {
+                                                if let Inline::Str(last_str) =
+                                                    &mut bracketed_content[i]
+                                                {
+                                                    last_str.text = format!("{}]", last_str.text);
+                                                    break;
+                                                }
+                                            }
+                                        }
+
+                                        cite.content.extend(bracketed_content);
                                         result.push(Inline::Cite(cite));
                                     }
                                     state = 0;
@@ -484,6 +542,57 @@ fn as_smart_str(s: String) -> String {
     }
 }
 
+/// Re-split Cite content strings after merge_strs to match Pandoc's behavior
+/// Pandoc breaks up citation suffix text by spaces
+pub fn split_cite_content_strings(pandoc: Pandoc) -> Pandoc {
+    topdown_traverse(
+        pandoc,
+        &mut Filter::new().with_cite(|mut cite| {
+            // Split any merged strings in the cite content back into separate Str/Space inlines
+            // Only split if the content contains spaces (i.e., hasn't been split yet)
+            let mut needs_split = false;
+            for inline in &cite.content {
+                if let Inline::Str(s) = inline {
+                    // Check for regular space or non-breaking space (U+00A0)
+                    if s.text.contains(' ') || s.text.contains('\u{00A0}') {
+                        needs_split = true;
+                        break;
+                    }
+                }
+            }
+
+            if !needs_split {
+                return Unchanged(cite);
+            }
+
+            let mut new_content: Vec<Inline> = vec![];
+            for inline in cite.content {
+                if let Inline::Str(s) = inline {
+                    // Split by regular spaces and non-breaking spaces
+                    let words: Vec<&str> = s.text.split(|c| c == ' ' || c == '\u{00A0}').collect();
+                    for (i, word) in words.iter().enumerate() {
+                        if i > 0 {
+                            new_content.push(Inline::Space(Space {
+                                source_info: SourceInfo::with_range(empty_range()),
+                            }));
+                        }
+                        if !word.is_empty() {
+                            new_content.push(Inline::Str(Str {
+                                text: word.to_string(),
+                                source_info: s.source_info.clone(),
+                            }));
+                        }
+                    }
+                } else {
+                    new_content.push(inline);
+                }
+            }
+            cite.content = new_content;
+            FilterResult(vec![Inline::Cite(cite)], true)
+        }),
+    )
+}
+
 /// Merge consecutive Str inlines and apply smart typography
 pub fn merge_strs(pandoc: Pandoc) -> Pandoc {
     topdown_traverse(