feat: split lexer test

Author: gengjiawen

lexer/Cargo.toml

@@ -1,26 +1,26 @@
-[package]
-name = "monkey-lexer"
-version = "0.6.0"
-description = "a lexer for monkey lang"
-homepage = "https://github.com/gengjiawen/monkey-rust"
-repository = "https://github.com/gengjiawen/monkey-rust"
-authors = ["gengjiawen <[email protected]>"]
-edition = "2018"
-license = "MIT"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[lib]
-name = "lexer"
-path= "lib.rs"
-
-[[bin]]
-name = "monkey-lexer"
-path = "main.rs"
-
-[dependencies]
-serde = {version = "1.0", features = ["derive"]}
-serde_json = "1.0"
-
-[dev-dependencies]
-insta = "1.7.1"
+[package]
+name = "monkey-lexer"
+version = "0.6.0"
+description = "a lexer for monkey lang"
+homepage = "https://github.com/gengjiawen/monkey-rust"
+repository = "https://github.com/gengjiawen/monkey-rust"
+authors = ["gengjiawen <[email protected]>"]
+edition = "2018"
+license = "MIT"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+name = "lexer"
+path= "lib.rs"
+
+[[bin]]
+name = "monkey-lexer"
+path = "main.rs"
+
+[dependencies]
+serde = {version = "1.0", features = ["derive"]}
+serde_json = "1.0"
+
+[dev-dependencies]
+insta = "1.7.1"

lexer/README.md

@@ -1,15 +1,15 @@
-# monkey-rust
-![Rust](https://github.com/gengjiawen/monkey-rust/workflows/Rust/badge.svg)
-[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/gengjiawen/monkey_rust)
-
-This is lexer for the Monkey programming language written in Rust
-
-![The Monkey Programming Language](https://cloud.githubusercontent.com/assets/1013641/22617482/9c60c27c-eb09-11e6-9dfa-b04c7fe498ea.png)
-
-## What’s Monkey?
-
-Monkey has a C-like syntax, supports **variable bindings**, **prefix** and **infix operators**, has **first-class** and **higher-order functions**, can handle **closures** with ease and has **integers**, **booleans**, **arrays** and **hashes** built-in.
-
-Official site is: https://monkeylang.org/. It's has various implementation languages :). 
-
-There is a book about learning how to make an interpreter: [Writing An Interpreter In Go](https://interpreterbook.com/#the-monkey-programming-language). This is where the Monkey programming language come from.
+# monkey-rust
+![Rust](https://github.com/gengjiawen/monkey-rust/workflows/Rust/badge.svg)
+[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/gengjiawen/monkey_rust)
+
+This is lexer for the Monkey programming language written in Rust
+
+![The Monkey Programming Language](https://cloud.githubusercontent.com/assets/1013641/22617482/9c60c27c-eb09-11e6-9dfa-b04c7fe498ea.png)
+
+## What’s Monkey?
+
+Monkey has a C-like syntax, supports **variable bindings**, **prefix** and **infix operators**, has **first-class** and **higher-order functions**, can handle **closures** with ease and has **integers**, **booleans**, **arrays** and **hashes** built-in.
+
+Official site is: https://monkeylang.org/. It's has various implementation languages :). 
+
+There is a book about learning how to make an interpreter: [Writing An Interpreter In Go](https://interpreterbook.com/#the-monkey-programming-language). This is where the Monkey programming language come from.

lexer/lexer_test.rs

@@ -0,0 +1,85 @@
+#[cfg(test)]
+mod tests {
+    use crate::Lexer;
+    use crate::token::{TokenKind, Token};
+    use insta::*;
+
+    fn test_token_set(l: &mut Lexer) -> Vec<Token> {
+        let mut token_vs: Vec<Token> = vec![];
+        loop {
+            let t = l.next_token();
+            if t.kind == TokenKind::EOF {
+                token_vs.push(t);
+                break;
+            } else {
+                token_vs.push(t);
+            }
+        }
+        token_vs
+    }
+
+    pub fn test_lexer_common(name: &str, input: &str) {
+        let mut l = Lexer::new(input);
+        let token_vs = test_token_set(&mut l);
+
+        assert_snapshot!(name, serde_json::to_string_pretty(&token_vs).unwrap(), input);
+    }
+
+    #[test]
+    fn test_lexer_simple() {
+        test_lexer_common("simple", "=+(){},:;");
+    }
+
+    #[test]
+    fn test_lexer_let() {
+        test_lexer_common("let", "let x=5");
+    }
+
+    #[test]
+    fn test_lexer_let_with_space() {
+        test_lexer_common("let_with_space", "let x = 5");
+    }
+
+    #[test]
+    fn test_lexer_string() {
+        test_lexer_common("string", r#""a""#);
+    }
+
+    #[test]
+    fn test_lexer_array() {
+        test_lexer_common("array", "[3]");
+    }
+
+    #[test]
+    fn test_lexer_hash() {
+        test_lexer_common("hash", r#"{"one": 1, "two": 2, "three": 3}"#);
+    }
+
+    #[test]
+    fn test_lexer_bool() {
+        test_lexer_common("bool", "let y=true");
+    }
+
+    #[test]
+    fn test_lexer_complex() {
+        test_lexer_common("complex", "let five = 5;
+let ten = 10;
+
+let add = fn(x, y) {
+  x + y;
+};
+
+let result = add(five, ten);
+!-/*5;
+5 < 10 > 5;
+
+if (5 < 10) {
+	return true;
+} else {
+	return false;
+}
+
+10 == 10;
+10 != 9;");
+    }
+}