Fix whitespace trimming of spans when unicode present (#25)

Author: tesujimath

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "beancount-parser-lima"
-version = "0.10.0"
+version = "0.10.1-alpha.1"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 description = "A zero-copy parser for Beancount"

Cargo.toml

@@ -97,7 +97,8 @@ use std::{
     iter::once,
     path::{Path, PathBuf},
 };
-pub use types::*;
+
+pub use crate::{trim::trim_trailing_whitespace, types::*};
 
 /// Contains the content of the Beancount source file, and the content of
 /// the transitive closure of all the include'd source files.
@@ -306,32 +307,10 @@ impl BeancountSources {
 
         (
             source_id_str.to_string(),
-            self.adjusted_span(source_content, span),
+            trimmed_span(source_content, span),
         )
     }
 
-    // adjust the span to exclude trailing whitespace
-    fn adjusted_span(&self, content: &str, span: &Span) -> Span {
-        let mut chars = content.chars();
-        let content = chars.by_ref();
-        let mut i = span.start;
-        let mut final_non_whitespace = i;
-        for c in content.skip(span.start) {
-            if i >= span.end {
-                break;
-            }
-
-            if !c.is_whitespace() {
-                final_non_whitespace = i;
-            }
-
-            i += 1;
-        }
-        let mut adjusted_span = *span;
-        adjusted_span.end = final_non_whitespace + 1;
-        adjusted_span
-    }
-
     fn source_id_string(&self, source_id: SourceId) -> &str {
         self.source_id_strings[Into::<usize>::into(source_id)].as_str()
     }
@@ -869,6 +848,12 @@ fn end_of_input(source_id: SourceId, s: &str) -> Span {
     chumsky::span::Span::new(source_id, s.len()..s.len())
 }
 
+fn trimmed_span(source: &str, span: &Span) -> Span {
+    let mut trimmed = *span;
+    trimmed.end = trim_trailing_whitespace(source, span.start, span.end);
+    trimmed
+}
+
 #[cfg(test)]
 pub use lexer::bare_lex;
 mod format;
@@ -877,4 +862,5 @@ pub use options::Options;
 mod options;
 mod parsers;
 mod sort;
+mod trim;
 pub mod types;

src/lib.rs

@@ -0,0 +1,21 @@
+#[cfg(test)]
+use test_case::test_case;
+
+// adjust the span to exclude trailing whitespace
+pub fn trim_trailing_whitespace(content: &str, start: usize, end: usize) -> usize {
+    match &content[start..end].rfind(|c: char| !c.is_whitespace()) {
+        Some(i) => start + *i + 1,
+        None => end,
+    }
+}
+
+#[cfg(test)]
+#[test_case("abc", "abc"; "abc")]
+#[test_case("abc ", "abc"; "abc space")]
+#[test_case("abc\n", "abc"; "abc newline")]
+#[test_case("abc\ndef\n\n", "abc\ndef"; "abc def")]
+#[test_case("abc\n😱\ndef\n", "abc\n😱\ndef"; "abc scream def")]
+fn test_trim_trailing_whitespace(source: &str, expected: &str) {
+    let trimmed_end = trim_trailing_whitespace(source, 0, source.len());
+    assert_eq!(&source[..trimmed_end], expected);
+}