Strip a byte order mark if present

Author: dtolnay

src/fallback.rs

@@ -182,7 +182,13 @@ impl FromStr for TokenStream {
 
     fn from_str(src: &str) -> Result<TokenStream, LexError> {
         // Create a dummy file & add it to the source map
-        let cursor = get_cursor(src);
+        let mut cursor = get_cursor(src);
+
+        // Strip a byte order mark if present
+        const BYTE_ORDER_MARK: &str = "\u{feff}";
+        if cursor.starts_with(BYTE_ORDER_MARK) {
+            cursor = cursor.advance(BYTE_ORDER_MARK.len());
+        }
 
         parse::token_stream(cursor)
     }

src/parse.rs

@@ -14,7 +14,7 @@ pub(crate) struct Cursor<'a> {
 }
 
 impl<'a> Cursor<'a> {
-    fn advance(&self, bytes: usize) -> Cursor<'a> {
+    pub fn advance(&self, bytes: usize) -> Cursor<'a> {
         let (_front, rest) = self.rest.split_at(bytes);
         Cursor {
             rest,
@@ -23,7 +23,7 @@ impl<'a> Cursor<'a> {
         }
     }
 
-    fn starts_with(&self, s: &str) -> bool {
+    pub fn starts_with(&self, s: &str) -> bool {
         self.rest.starts_with(s)
     }