fix: introducing lexical errors

Author: DeveloperC286

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "monkey_interpreter"
-version = "0.15.0"
+version = "0.15.1"
 description = "Implementation of an interpreter for the Monkey language written in Rust, currently under active development."
 authors = ["C <[email protected]>"]
 edition = "2021"
@@ -19,6 +19,9 @@ structopt = "0.3.25"
 log = "0.4.14"
 pretty_env_logger = "0.4.0"
 
+# For custom errors.
+thiserror = "1.0.30"
+
 
 [dev-dependencies]
 rstest = "0.11.0"

Cargo.toml

@@ -1,5 +1,5 @@
 # Monkey Interpreter
-[![pipeline status](https://img.shields.io/badge/Version-0.15.0-blue)](https://gitlab.com/DeveloperC/monkey_interpreter/commits/master) [![pipeline status](https://gitlab.com/DeveloperC/monkey_interpreter/badges/master/pipeline.svg)](https://gitlab.com/DeveloperC/monkey_interpreter/commits/master) [![Conventional Commits](https://img.shields.io/badge/Conventional%20Commits-1.0.0-yellow.svg)](https://conventionalcommits.org) [![License: AGPL v3](https://img.shields.io/badge/License-AGPLv3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
+[![pipeline status](https://img.shields.io/badge/Version-0.15.1-blue)](https://gitlab.com/DeveloperC/monkey_interpreter/commits/master) [![pipeline status](https://gitlab.com/DeveloperC/monkey_interpreter/badges/master/pipeline.svg)](https://gitlab.com/DeveloperC/monkey_interpreter/commits/master) [![Conventional Commits](https://img.shields.io/badge/Conventional%20Commits-1.0.0-yellow.svg)](https://conventionalcommits.org) [![License: AGPL v3](https://img.shields.io/badge/License-AGPLv3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
 
 
 ![The Monkey Programming Language Logo](https://cloud.githubusercontent.com/assets/1013641/22617482/9c60c27c-eb09-11e6-9dfa-b04c7fe498ea.png)

README.md

@@ -5,7 +5,7 @@ macro_rules! assert_expected_returned_object {
         insta::assert_debug_snapshot!(
             $snapshot_name,
             evaluator.evaluate(crate::syntax_analysis::SyntaxAnalysis::from(
-                crate::lexical_analysis::LexicalAnalysis::from($code)
+                crate::lexical_analysis::LexicalAnalysis::from($code).unwrap()
             ))
         );
     };

src/evaluator/tests/macros.rs

@@ -15,20 +15,26 @@ pub(crate) fn repl() {
     let mut evaluator = crate::evaluator::Evaluator::new();
 
     loop {
-        let tokens = LexicalAnalysis::from(&read());
-        let abstract_syntax_tree = SyntaxAnalysis::from(tokens);
-
-        if !abstract_syntax_tree.syntax_parsing_errors.is_empty() {
-            for error in abstract_syntax_tree
-                .syntax_parsing_errors
-                .iter()
-                .enumerate()
-            {
-                error!("{:?}", error);
+        match LexicalAnalysis::from(&read()) {
+            Ok(tokens) => {
+                let abstract_syntax_tree = SyntaxAnalysis::from(tokens);
+
+                if !abstract_syntax_tree.syntax_parsing_errors.is_empty() {
+                    for error in abstract_syntax_tree
+                        .syntax_parsing_errors
+                        .iter()
+                        .enumerate()
+                    {
+                        error!("{:?}", error);
+                    }
+                } else {
+                    let object = evaluator.evaluate(abstract_syntax_tree);
+                    println!("{:?}", object);
+                }
+            }
+            Err(error) => {
+                error!("{}", error);
             }
-        } else {
-            let object = evaluator.evaluate(abstract_syntax_tree);
-            println!("{:?}", object);
         }
     }
 }
@@ -41,7 +47,7 @@ fn read() -> String {
 
     match stdin().read_line(&mut buffer) {
         Ok(_) => {}
-        Err(error) => error!("Error reading user input: {}", error),
+        Err(_) => error!("Unable to read user input from standard input."),
     }
 
     buffer

src/interpreter/mod.rs

@@ -1,6 +1,7 @@
 use std::iter::{FromIterator, Peekable};
 use std::str::Chars;
 
+use crate::lexical_analysis::model::lexical_error::LexicalError;
 use crate::lexical_analysis::model::token::Token;
 use crate::lexical_analysis::utilities::*;
 
@@ -15,19 +16,19 @@ pub(crate) struct LexicalAnalysis<'a> {
 }
 
 impl<'a> LexicalAnalysis<'a> {
-    pub(crate) fn from(code: &str) -> Vec<Token> {
+    pub(crate) fn from(code: &str) -> Result<Vec<Token>, LexicalError> {
         let mut lexical_analysis = LexicalAnalysis {
             source_code: code.chars().peekable(),
         };
 
         lexical_analysis.get_tokens()
     }
 
-    fn get_tokens(&mut self) -> Vec<Token> {
+    fn get_tokens(&mut self) -> Result<Vec<Token>, LexicalError> {
         let mut tokens = Vec::new();
 
         loop {
-            let token = self.get_next_token();
+            let token = self.get_next_token()?;
             let end_of_file = token == Token::EndOfFile;
             tokens.push(token);
 
@@ -36,54 +37,52 @@ impl<'a> LexicalAnalysis<'a> {
             }
         }
 
-        tokens
+        Ok(tokens)
     }
 
-    fn get_next_token(&mut self) -> Token {
+    fn get_next_token(&mut self) -> Result<Token, LexicalError> {
         match self.get_next_character() {
             Some(character) => {
                 debug!("Matching the character '{}'.", character);
                 match character {
                     '!' => {
-                        check_next_character!(self.source_code, '=', Token::NotEquals);
-                        Token::Not
+                        check_next_character!(self.source_code, '=', Ok(Token::NotEquals));
+                        Ok(Token::Not)
                     }
-                    '-' => Token::Minus,
-                    '/' => Token::Divide,
-                    '*' => Token::Multiply,
-                    '>' => Token::GreaterThan,
-                    '<' => Token::LesserThan,
+                    '-' => Ok(Token::Minus),
+                    '/' => Ok(Token::Divide),
+                    '*' => Ok(Token::Multiply),
+                    '>' => Ok(Token::GreaterThan),
+                    '<' => Ok(Token::LesserThan),
                     '=' => {
-                        check_next_character!(self.source_code, '=', Token::Equals);
-                        Token::Assign
+                        check_next_character!(self.source_code, '=', Ok(Token::Equals));
+                        Ok(Token::Assign)
                     }
-                    '+' => Token::Plus,
-                    '(' => Token::OpeningRoundBracket,
-                    ')' => Token::ClosingRoundBracket,
-                    '{' => Token::OpeningCurlyBracket,
-                    '}' => Token::ClosingCurlyBracket,
-                    ',' => Token::Comma,
-                    ';' => Token::SemiColon,
+                    '+' => Ok(Token::Plus),
+                    '(' => Ok(Token::OpeningRoundBracket),
+                    ')' => Ok(Token::ClosingRoundBracket),
+                    '{' => Ok(Token::OpeningCurlyBracket),
+                    '}' => Ok(Token::ClosingCurlyBracket),
+                    ',' => Ok(Token::Comma),
+                    ';' => Ok(Token::SemiColon),
                     _ => {
                         if is_valid_identifier_character(character) {
                             debug!("Parsing word from characters.");
-                            return get_keyword_token(&self.get_word(character));
+                            return Ok(get_keyword_token(&self.get_word(character)));
                         }
 
                         if is_digit(character) {
                             debug!("Parsing integer from characters.");
-                            return Token::Integer {
+                            return Ok(Token::Integer {
                                 literal: self.get_integer(character),
-                            };
+                            });
                         }
 
-                        Token::Illegal {
-                            literal: character.to_string(),
-                        }
+                        Err(LexicalError::IllegalCharacter(character))
                     }
                 }
             }
-            None => Token::EndOfFile,
+            None => Ok(Token::EndOfFile),
         }
     }
 

src/lexical_analysis/mod.rs

@@ -0,0 +1,7 @@
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum LexicalError {
+    #[error("Illegal character provided for lexical analysis {0:?}.")]
+    IllegalCharacter(char),
+}

src/lexical_analysis/model/lexical_error/mod.rs

@@ -1 +1,2 @@
+pub(crate) mod lexical_error;
 pub(crate) mod token;

src/lexical_analysis/model/mod.rs

@@ -1,6 +1,5 @@
 #[derive(Debug, Clone, PartialEq)]
 pub(crate) enum Token {
-    Illegal { literal: String },
     EndOfFile,
 
     // Identifiers

src/lexical_analysis/model/token/mod.rs

@@ -108,3 +108,20 @@ fn test_lexical_analysis_tokenization_for_special_characters(code: &str, snapsho
 fn test_empty_code() {
     assert_expected_returned_tokens!("", "test_empty_code");
 }
+
+#[rstest(
+    code,
+    snapshot_name,
+    case("let x = `2`;", "test_lexical_analysis_producing_lexical_errors_case1"),
+    case(
+        "let x = 2;\n let z = &x;",
+        "test_lexical_analysis_producing_lexical_errors_case2"
+    ),
+    case(
+        "let x =| sum(2);",
+        "test_lexical_analysis_producing_lexical_errors_case3"
+    )
+)]
+fn test_lexical_analysis_producing_lexical_errors(code: &str, snapshot_name: &str) {
+    assert_expected_returned_tokens!(code, snapshot_name);
+}