Fix a panic in cooked_byte on utf-8 chars

Don't want to slice on the wrong boundary!

Closes #54

Author: alexcrichton

src/stable.rs

@@ -904,7 +904,13 @@ fn cooked_byte(input: Cursor) -> PResult<()> {
     };
     if ok {
         match bytes.next() {
-            Some((offset, _)) => Ok((input.advance(offset), ())),
+            Some((offset, _)) => {
+                if input.chars().as_str().is_char_boundary(offset) {
+                    Ok((input.advance(offset), ()))
+                } else {
+                    Err(LexError)
+                }
+            }
             None => Ok((input.advance(input.len()), ())),
         }
     } else {

tests/test.rs

@@ -1,5 +1,7 @@
 extern crate proc_macro2;
 
+use std::str;
+
 use proc_macro2::{Term, Literal, TokenStream};
 
 #[cfg(procmacro2_semver_exempt)]
@@ -161,3 +163,10 @@ fn span_join() {
 
     assert_eq!(joined1.unwrap().source_file(), source1[0].span.source_file());
 }
+
+#[test]
+fn no_panic() {
+    let s = str::from_utf8(b"b\'\xc2\x86  \x00\x00\x00^\"").unwrap();
+    assert!(s.parse::<proc_macro2::TokenStream>().is_err());
+}
+