Add a test involving byte order mark

Currently fails with:

    thread 'byte_order_mark' panicked at 'called `Result::unwrap()` on
    an `Err` value: LexError { span: Span }', tests/test.rs:637:48

Author: dtolnay

tests/test.rs

@@ -630,3 +630,13 @@ fn check_spans_internal(ts: TokenStream, lines: &mut &[(usize, usize, usize, usi
         }
     }
 }
+
+#[test]
+fn byte_order_mark() {
+    let string = "\u{feff}foo";
+    let tokens = string.parse::<TokenStream>().unwrap();
+    match tokens.into_iter().next().unwrap() {
+        TokenTree::Ident(ident) => assert_eq!(ident, "foo"),
+        _ => unreachable!(),
+    }
+}