Lexer and parser refactor

Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 tracy-client = { version = "0.17.3", default-features = false } # for tracy v0.11.1
+logos = "0.15"
 
 [profile.profiling]
 inherits = "release"

benches/benchmarks.rs

@@ -1,5 +1,6 @@
 use std::process::exit;
 
+use new_nu_parser::lexer::{lex, Tokens};
 use nu_protocol::engine::{EngineState, StateWorkingSet};
 use tango_bench::{benchmark_fn, tango_benchmarks, tango_main, Benchmark, IntoBenchmarks};
 
@@ -16,9 +17,11 @@ const BENCHMARKS: &[&str] = &[
     "combined10",
     "combined100",
     "combined1000",
+    "int100",
 ];
 
 enum Stage {
+    Lex,
     Parse,
     Resolve,
     Typecheck,
@@ -30,6 +33,7 @@ enum Stage {
 
 /// Stages of compilation we want to profile
 const STAGES: &[Stage] = &[
+    Stage::Lex,
     Stage::Parse,
     Stage::Resolve,
     Stage::Typecheck,
@@ -52,8 +56,15 @@ fn setup_compiler(
     let contents = std::fs::read(fname).map_err(|_| format!("Cannot find file {fname}"))?;
     compiler.add_file(&fname, &contents);
 
+    let (tokens, err) = lex(&contents, span_offset);
+    if let Err(e) = err {
+        tokens.eprint(&compiler.source);
+        eprintln!("Lexing error. Error: {:?}", e);
+        exit(1);
+    }
+
     if do_parse {
-        let parser = Parser::new(compiler, span_offset);
+        let parser = Parser::new(compiler, tokens);
         compiler = parser.parse();
 
         if !compiler.errors.is_empty() {
@@ -87,8 +98,8 @@ fn setup_compiler(
 }
 
 /// Parse only
-pub fn parse(mut compiler: Compiler, span_offset: usize) {
-    let parser = Parser::new(compiler, span_offset);
+pub fn parse(mut compiler: Compiler, tokens: Tokens) {
+    let parser = Parser::new(compiler, tokens);
     compiler = parser.parse();
 
     if !compiler.errors.is_empty() {
@@ -129,7 +140,14 @@ pub fn typecheck(mut compiler: Compiler, do_merge: bool) {
 
 /// Run all compiler stages
 pub fn compile(mut compiler: Compiler, span_offset: usize) {
-    let parser = Parser::new(compiler, span_offset);
+    let (tokens, err) = lex(&compiler.source, span_offset);
+    if let Err(e) = err {
+        tokens.eprint(&compiler.source);
+        eprintln!("Lexing error. Error: {:?}", e);
+        exit(1);
+    }
+
+    let parser = Parser::new(compiler, tokens);
     compiler = parser.parse();
 
     if !compiler.errors.is_empty() {
@@ -176,13 +194,36 @@ fn compiler_benchmarks() -> impl IntoBenchmarks {
             let bench_file = format!("benches/nu/{bench_name}.nu");
 
             let bench = match stage {
+                Stage::Lex => {
+                    let name = format!("{bench_name}_lex");
+                    benchmark_fn(name, move |b| {
+                        let contents = std::fs::read(&bench_file)
+                            .expect(&format!("Cannot find file {bench_file}"));
+                        b.iter(move || {
+                            let (tokens, err) = lex(&contents, 0);
+                            if let Err(e) = err {
+                                tokens.eprint(&contents);
+                                eprintln!("Lexing error. Error: {:?}", e);
+                                exit(1);
+                            }
+                        })
+                    })
+                }
                 Stage::Parse => {
                     let name = format!("{bench_name}_parse");
                     benchmark_fn(name, move |b| {
                         let (compiler_def_init, span_offset) =
                             setup_compiler(&bench_file, false, false, false)
                                 .expect("Error setting up compiler");
-                        b.iter(move || parse(compiler_def_init.clone(), span_offset))
+                        let contents = std::fs::read(&bench_file)
+                            .expect(&format!("Cannot find file {bench_file}"));
+                        let (tokens, err) = lex(&contents, span_offset);
+                        if let Err(e) = err {
+                            tokens.eprint(&contents);
+                            eprintln!("Lexing error. Error: {:?}", e);
+                            exit(1);
+                        }
+                        b.iter(move || parse(compiler_def_init.clone(), tokens.clone()))
                     })
                 }
                 Stage::Resolve => {