Add # to tokenizer

Author: aceinetx

zenlang/src/tokenizer.rs

@@ -32,6 +32,7 @@ pub enum Token {
     Comma,
     Dot,
     Assign,
+    Hashtag,
     If,
     Elif,
     Else,
@@ -237,6 +238,9 @@ impl Tokenizer {
             } else if c == ']' {
                 self.pos += 1;
                 return Token::Rbracket;
+            } else if c == '#' {
+                self.pos += 1;
+                return Token::Hashtag;
             } else if c == '=' {
                 self.pos += 1;
                 if self.pos < self.code.len() {

zenlang/tests/tokenizer.rs

@@ -425,3 +425,19 @@ fn tokenizer_test_dot() {
     let token = tokenizer.next();
     assert!(matches!(token, Token::Number(1.0)));
 }
+
+#[test]
+fn tokenizer_test_hashtag() {
+    let mut tokenizer = Tokenizer::new("fn main {return 2 + 1 # 1 + 2; }".into());
+    tokenizer.next();
+    tokenizer.next();
+    tokenizer.next();
+    tokenizer.next();
+    tokenizer.next();
+    tokenizer.next();
+    tokenizer.next();
+    let token = tokenizer.next();
+    assert!(matches!(token, Token::Hashtag));
+    let token = tokenizer.next();
+    assert!(matches!(token, Token::Number(1.0)));
+}