more capitalization and abbreviation fixes

Author: cscheid

crates/quarto-markdown-pandoc/src/pandoc/treesitter_utils/postprocess.rs

@@ -85,6 +85,39 @@ fn is_abbreviation(text: &str) -> bool {
     )
 }
 
+/// Check if a text string ends with a known abbreviation
+fn ends_with_abbreviation(text: &str) -> bool {
+    text.ends_with("Mr.")
+        || text.ends_with("Mrs.")
+        || text.ends_with("Ms.")
+        || text.ends_with("Capt.")
+        || text.ends_with("Dr.")
+        || text.ends_with("Prof.")
+        || text.ends_with("Gen.")
+        || text.ends_with("Gov.")
+        || text.ends_with("e.g.")
+        || text.ends_with("i.e.")
+        || text.ends_with("Sgt.")
+        || text.ends_with("St.")
+        || text.ends_with("vol.")
+        || text.ends_with("vs.")
+        || text.ends_with("Sen.")
+        || text.ends_with("Rep.")
+        || text.ends_with("Pres.")
+        || text.ends_with("Hon.")
+        || text.ends_with("Rev.")
+        || text.ends_with("Ph.D.")
+        || text.ends_with("M.D.")
+        || text.ends_with("M.A.")
+        || text.ends_with("p.")
+        || text.ends_with("pp.")
+        || text.ends_with("ch.")
+        || text.ends_with("chap.")
+        || text.ends_with("sec.")
+        || text.ends_with("cf.")
+        || text.ends_with("cp.")
+}
+
 /// Coalesce Str nodes that end with abbreviations with following words
 /// This matches Pandoc's behavior of keeping abbreviations with the next word
 /// Returns (result, did_coalesce) tuple
@@ -100,33 +133,23 @@ pub fn coalesce_abbreviations(inlines: Vec<Inline>) -> (Vec<Inline>, bool) {
             let mut end_info = str_inline.source_info.clone();
             let mut j = i + 1;
 
-            // Check if current text is an abbreviation
-            if is_abbreviation(&current_text) {
-                // Coalesce with following Space + Str until we hit a capital letter
+            // Check if current text ends with an abbreviation
+            if ends_with_abbreviation(&current_text) {
+                // Coalesce with following Space + Str
                 while j + 1 < inlines.len() {
                     if let (Inline::Space(_), Inline::Str(next_str)) =
                         (&inlines[j], &inlines[j + 1])
                     {
-                        // Stop before uppercase letters (potential sentence boundaries)
-                        if next_str
-                            .text
-                            .chars()
-                            .next()
-                            .map_or(false, |c| c.is_uppercase())
-                        {
-                            break;
-                        }
-
-                        // Coalesce
-                        current_text.push(' ');
+                        // Coalesce with non-breaking space (U+00A0) to match Pandoc
+                        current_text.push('\u{00A0}');
                         current_text.push_str(&next_str.text);
                         end_info = next_str.source_info.clone();
                         j += 2;
                         did_coalesce = true;
 
-                        // If this word is also an abbreviation, continue coalescing
+                        // If this word also ends with an abbreviation, continue coalescing
                         // Otherwise, stop after this word
-                        if !is_abbreviation(&next_str.text) {
+                        if !ends_with_abbreviation(&current_text) {
                             break;
                         }
                     } else {

crates/quarto-markdown-pandoc/tests/pandoc-match-corpus/markdown/046.qmd

@@ -0,0 +1 @@
+(e.g. this fails)

crates/quarto-markdown-pandoc/tests/pandoc-match-corpus/markdown/047.qmd

@@ -0,0 +1 @@
+Test with Dr. Smith today.

crates/quarto-markdown-pandoc/tests/snapshots/native/002.qmd.snapshot

@@ -1 +1 @@
-[ Para [Str "Blah", Space, Str "Blah", Space, Cite [Citation { citationId = "knuth1984", citationPrefix = [Str "see", Space], citationSuffix = [Str ",", Space, Str "pp. 33-35"], citationMode = NormalCitation, citationNoteNum = 1, citationHash = 0 }, Citation { citationId = "wickham2015", citationPrefix = [Space, Str "also", Space], citationSuffix = [Str ",", Space, Str "chap. 1"], citationMode = NormalCitation, citationNoteNum = 1, citationHash = 0 }] []] ]
+[ Para [Str "Blah", Space, Str "Blah", Space, Cite [Citation { citationId = "knuth1984", citationPrefix = [Str "see", Space], citationSuffix = [Str ",", Space, Str "pp. 33-35"], citationMode = NormalCitation, citationNoteNum = 1, citationHash = 0 }, Citation { citationId = "wickham2015", citationPrefix = [Space, Str "also", Space], citationSuffix = [Str ",", Space, Str "chap. 1"], citationMode = NormalCitation, citationNoteNum = 1, citationHash = 0 }] []] ]

crates/quarto-markdown-pandoc/tests/snapshots/native/003.qmd.snapshot

@@ -1 +1 @@
-[ Para [Str "Blah", Space, Str "Blah", Space, Cite [Citation { citationId = "knuth1984", citationPrefix = [], citationSuffix = [Str ",", Space, Str "pp. 33-35,", Space, Str "38-39", Space, Str "and", Space, Str "passim"], citationMode = NormalCitation, citationNoteNum = 1, citationHash = 0 }] []] ]
+[ Para [Str "Blah", Space, Str "Blah", Space, Cite [Citation { citationId = "knuth1984", citationPrefix = [], citationSuffix = [Str ",", Space, Str "pp. 33-35,", Space, Str "38-39", Space, Str "and", Space, Str "passim"], citationMode = NormalCitation, citationNoteNum = 1, citationHash = 0 }] []] ]

crates/quarto-markdown-pandoc/tests/snapshots/native/006.qmd.snapshot

@@ -1 +1 @@
-[ Para [Cite [Citation { citationId = "smith04", citationPrefix = [], citationSuffix = [Str "p. 33"], citationMode = AuthorInText, citationNoteNum = 1, citationHash = 0 }] [Str "@smith04"]] ]
+[ Para [Cite [Citation { citationId = "smith04", citationPrefix = [], citationSuffix = [Str "p. 33"], citationMode = AuthorInText, citationNoteNum = 1, citationHash = 0 }] [Str "@smith04"]] ]

crates/quarto-markdown-pandoc/tests/snapshots/native/014.qmd.snapshot

@@ -1 +1 @@
-[ Para [Str "i", Space, Str "think", Space, Str "e.g. this", Space, Str "is", Space, Str "good?", Space, Str "did", Space, Str "1–30", Space, Str "work?", Space, Str "wait—really—did", Space, Str "it?"] ]
+[ Para [Str "i", Space, Str "think", Space, Str "e.g. this", Space, Str "is", Space, Str "good?", Space, Str "did", Space, Str "1–30", Space, Str "work?", Space, Str "wait—really—did", Space, Str "it?"] ]