Merge pull request #354 from dtolnay/bom

Handle parsing input that starts with byte order mark

Author: dtolnay

src/fallback.rs

@@ -182,7 +182,13 @@ impl FromStr for TokenStream {
 
     fn from_str(src: &str) -> Result<TokenStream, LexError> {
         // Create a dummy file & add it to the source map
-        let cursor = get_cursor(src);
+        let mut cursor = get_cursor(src);
+
+        // Strip a byte order mark if present
+        const BYTE_ORDER_MARK: &str = "\u{feff}";
+        if cursor.starts_with(BYTE_ORDER_MARK) {
+            cursor = cursor.advance(BYTE_ORDER_MARK.len());
+        }
 
         parse::token_stream(cursor)
     }

src/parse.rs

@@ -14,7 +14,7 @@ pub(crate) struct Cursor<'a> {
 }
 
 impl<'a> Cursor<'a> {
-    fn advance(&self, bytes: usize) -> Cursor<'a> {
+    pub fn advance(&self, bytes: usize) -> Cursor<'a> {
         let (_front, rest) = self.rest.split_at(bytes);
         Cursor {
             rest,
@@ -23,7 +23,7 @@ impl<'a> Cursor<'a> {
         }
     }
 
-    fn starts_with(&self, s: &str) -> bool {
+    pub fn starts_with(&self, s: &str) -> bool {
         self.rest.starts_with(s)
     }
 

tests/test.rs

@@ -630,3 +630,16 @@ fn check_spans_internal(ts: TokenStream, lines: &mut &[(usize, usize, usize, usi
         }
     }
 }
+
+#[test]
+fn byte_order_mark() {
+    let string = "\u{feff}foo";
+    let tokens = string.parse::<TokenStream>().unwrap();
+    match tokens.into_iter().next().unwrap() {
+        TokenTree::Ident(ident) => assert_eq!(ident, "foo"),
+        _ => unreachable!(),
+    }
+
+    let string = "foo\u{feff}";
+    string.parse::<TokenStream>().unwrap_err();
+}