fix: bug with possible unicode bound fail

Author: swarnim-deepsource

cdracula/tests/test_capi.rs

@@ -37,6 +37,9 @@ mod python {
             let src = CString::from_vec_unchecked(
                 (String::from(
                     r#"
+# entp için anayzer
+if index == 10:
+    pass
 # skip this
 def python():
     """
@@ -47,7 +50,7 @@ def python():
                 ) + "\0")
                     .into(),
             );
-            assert_eq!(get_meaningful_line_count(src.as_ptr(), PYTHON_LANG, 0), 2);
+            assert_eq!(get_meaningful_line_count(src.as_ptr(), PYTHON_LANG, 0), 4);
         }
     }
 

src/parse.rs

@@ -279,7 +279,7 @@ pub struct Parser<'a, L: Language> {
     src: &'a str,
     index: usize,
     language_items: &'static [ParseItem],
-    _marker: PhantomData<L>
+    _marker: PhantomData<L>,
 }
 
 // most this is only used in tests atm!
@@ -290,7 +290,7 @@ impl<L: Language> Parser<'_, L> {
             src,
             language_items: L::PARSE_ITEMS,
             index: 0,
-            _marker: PhantomData::default()
+            _marker: PhantomData::default(),
         }
     }
 
@@ -304,18 +304,22 @@ impl<L: Language> Parser<'_, L> {
             .find_map(|i| Some((i, items[i].begin().matches(src)?)))
             .and_then(|(i, matches)| {
                 (matches[2].end..src.len()).find_map(|b| {
-                    Some((
-                        i,
-                        b,
-                        if items[i].is_key_matched() {
-                            items[i].end().matches_with_key(
-                                &src[b..],
-                                &src[matches[1].start..matches[1].end],
-                            )?
-                        } else {
-                            items[i].end().matches(&src[b..])?
-                        },
-                    ))
+                    if src.is_char_boundary(b) {
+                        Some((
+                            i,
+                            b,
+                            if items[i].is_key_matched() {
+                                items[i].end().matches_with_key(
+                                    &src[b..],
+                                    &src[matches[1].start..matches[1].end],
+                                )?
+                            } else {
+                                items[i].end().matches(&src[b..])?
+                            },
+                        ))
+                    } else {
+                        None
+                    }
                 })
             })
         {

src/tests.rs

@@ -39,7 +39,10 @@ mod simple_python {
     #[test]
     fn try_parse() {
         let parsed = Parser::<Python>::new(
-            r#"# some top level comments
+            r#"# entp için anayzer
+            if index == 10:
+                pass
+            # some top level comments
             def main():
                 print("s");"""
                 Multi-line Comments
@@ -53,6 +56,7 @@ mod simple_python {
         let mut line_count: usize = 0;
         let mut stack = vec![];
         for p in parsed {
+            eprintln!("{:?}", p);
             if matches!(p, ParseOutput::EOL(_) | ParseOutput::EOF) {
                 if stack.iter().any(|i| match i {
                     ParseOutput::Source(s) => Python::is_meaningful_src(s),
@@ -65,7 +69,7 @@ mod simple_python {
                 stack.push(p);
             }
         }
-        assert_eq!(line_count, 3)
+        assert_eq!(line_count, 5)
     }
 }