Add failure/low-level lexer tests

Author: mhallin

src/parser/lexer.rs

@@ -472,7 +472,7 @@ impl<'a> fmt::Display for Token<'a> {
             Token::Name(ref name) => write!(f, "{}", name),
             Token::Int(i) => write!(f, "{}", i),
             Token::Float(v) => write!(f, "{}", v),
-            Token::String(ref s) => write!(f, "{}", s),
+            Token::String(ref s) => write!(f, "\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\"")),
             Token::ExclamationMark => write!(f, "!"),
             Token::Dollar => write!(f, "$"),
             Token::ParenOpen => write!(f, "("),

src/parser/tests/lexer.rs

@@ -258,6 +258,18 @@ fn string_errors() {
         Spanning::zero_width(
             &SourcePosition::new(6, 0, 6),
             LexerError::UnknownEscapeSequence("\\uXXXF".to_owned())));
+
+    assert_eq!(
+        tokenize_error(r#""unterminated in string \""#),
+        Spanning::zero_width(
+            &SourcePosition::new(26, 0, 26),
+            LexerError::UnterminatedString));
+
+    assert_eq!(
+        tokenize_error(r#""unterminated \"#),
+        Spanning::zero_width(
+            &SourcePosition::new(15, 0, 15),
+            LexerError::UnterminatedString));
 }
 
 #[test]
@@ -276,6 +288,13 @@ fn numbers() {
             &SourcePosition::new(5, 0, 5),
             Token::Float(4.123)));
 
+    assert_eq!(
+        tokenize_single("4.0"),
+        Spanning::start_end(
+            &SourcePosition::new(0, 0, 0),
+            &SourcePosition::new(3, 0, 3),
+            Token::Float(4.0)));
+
     assert_eq!(
         tokenize_single("-4"),
         Spanning::start_end(
@@ -534,3 +553,45 @@ fn punctuation_error() {
             &SourcePosition::new(0, 0, 0),
             LexerError::UnknownCharacter('\u{200b}')));
 }
+
+#[test]
+fn display() {
+    assert_eq!(
+        format!("{}", Token::Name("identifier")),
+        "identifier"
+    );
+
+    assert_eq!(
+        format!("{}", Token::Int(123)),
+        "123"
+    );
+
+    assert_eq!(
+        format!("{}", Token::Float(4.123)),
+        "4.123"
+    );
+
+    assert_eq!(
+        format!("{}", Token::String("some string".to_owned())),
+        "\"some string\""
+    );
+
+    assert_eq!(
+        format!("{}", Token::String("string with \\ escape and \" quote".to_owned())),
+        "\"string with \\\\ escape and \\\" quote\""
+    );
+
+    assert_eq!(format!("{}", Token::ExclamationMark), "!");
+    assert_eq!(format!("{}", Token::Dollar), "$");
+    assert_eq!(format!("{}", Token::ParenOpen), "(");
+    assert_eq!(format!("{}", Token::ParenClose), ")");
+    assert_eq!(format!("{}", Token::BracketOpen), "[");
+    assert_eq!(format!("{}", Token::BracketClose), "]");
+    assert_eq!(format!("{}", Token::CurlyOpen), "{");
+    assert_eq!(format!("{}", Token::CurlyClose), "}");
+    assert_eq!(format!("{}", Token::Ellipsis), "...");
+    assert_eq!(format!("{}", Token::Colon), ":");
+    assert_eq!(format!("{}", Token::Equals), "=");
+    assert_eq!(format!("{}", Token::At), "@");
+    assert_eq!(format!("{}", Token::Pipe), "|");
+}