Lexer and parser refactor (#31)

The on-demand lexer is replaced with a standalone lexing pass using the
Logos crate. The lexing code is refactored away from the parser into a
separate data structure. The Parser is refactored and simplified
accordingly.

The new lexer correctly recognizes more syntactic shapes than the
previous one.

The new lexer+parser seems slightly faster (5-6 ms together on the
combined1000.nu benchmark) than the previous version, but the main
benefit is the lexer disentangled from the parser code. It is now
possible to change the lexer more easily, including turning it into an
on-demand lexer again if need be.

The main shortcomings are:
* Difficulty lexing string interpolation, e.g., `$"foo(1 + 2)bar"`. We'd
need to switch to another lexer. It might be easier to use on-demand
lexing for it.
* Unmatched delimiters. If the file ends with an unmatched delimiter, an
extremely unhelpful error is emitted. This should be solvable by using
[callbacks](https://logos.maciej.codes/callbacks.html).

Notable API changes:
* `self.peek()` -> `self.tokens.peek()`
* `self.next()` -> gone, use `self.tokens.advance()` to point at the
next tokens followed by `self.tokens.peek()`.

Author: kubouch

Cargo.lock

@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 tracy-client = { version = "0.17.3", default-features = false } # for tracy v0.11.1
+logos = "0.15"
 
 [profile.profiling]
 inherits = "release"

Cargo.toml

@@ -1,5 +1,6 @@
 use std::process::exit;
 
+use new_nu_parser::lexer::{lex, Tokens};
 use nu_protocol::engine::{EngineState, StateWorkingSet};
 use tango_bench::{benchmark_fn, tango_benchmarks, tango_main, Benchmark, IntoBenchmarks};
 
@@ -16,9 +17,11 @@ const BENCHMARKS: &[&str] = &[
     "combined10",
     "combined100",
     "combined1000",
+    "int100",
 ];
 
 enum Stage {
+    Lex,
     Parse,
     Resolve,
     Typecheck,
@@ -30,6 +33,7 @@ enum Stage {
 
 /// Stages of compilation we want to profile
 const STAGES: &[Stage] = &[
+    Stage::Lex,
     Stage::Parse,
     Stage::Resolve,
     Stage::Typecheck,
@@ -52,8 +56,15 @@ fn setup_compiler(
     let contents = std::fs::read(fname).map_err(|_| format!("Cannot find file {fname}"))?;
     compiler.add_file(&fname, &contents);
 
+    let (tokens, err) = lex(&contents, span_offset);
+    if let Err(e) = err {
+        tokens.eprint(&compiler.source);
+        eprintln!("Lexing error. Error: {:?}", e);
+        exit(1);
+    }
+
     if do_parse {
-        let parser = Parser::new(compiler, span_offset);
+        let parser = Parser::new(compiler, tokens);
         compiler = parser.parse();
 
         if !compiler.errors.is_empty() {
@@ -87,8 +98,8 @@ fn setup_compiler(
 }
 
 /// Parse only
-pub fn parse(mut compiler: Compiler, span_offset: usize) {
-    let parser = Parser::new(compiler, span_offset);
+pub fn parse(mut compiler: Compiler, tokens: Tokens) {
+    let parser = Parser::new(compiler, tokens);
     compiler = parser.parse();
 
     if !compiler.errors.is_empty() {
@@ -129,7 +140,14 @@ pub fn typecheck(mut compiler: Compiler, do_merge: bool) {
 
 /// Run all compiler stages
 pub fn compile(mut compiler: Compiler, span_offset: usize) {
-    let parser = Parser::new(compiler, span_offset);
+    let (tokens, err) = lex(&compiler.source, span_offset);
+    if let Err(e) = err {
+        tokens.eprint(&compiler.source);
+        eprintln!("Lexing error. Error: {:?}", e);
+        exit(1);
+    }
+
+    let parser = Parser::new(compiler, tokens);
     compiler = parser.parse();
 
     if !compiler.errors.is_empty() {
@@ -176,13 +194,36 @@ fn compiler_benchmarks() -> impl IntoBenchmarks {
             let bench_file = format!("benches/nu/{bench_name}.nu");
 
             let bench = match stage {
+                Stage::Lex => {
+                    let name = format!("{bench_name}_lex");
+                    benchmark_fn(name, move |b| {
+                        let contents = std::fs::read(&bench_file)
+                            .expect(&format!("Cannot find file {bench_file}"));
+                        b.iter(move || {
+                            let (tokens, err) = lex(&contents, 0);
+                            if let Err(e) = err {
+                                tokens.eprint(&contents);
+                                eprintln!("Lexing error. Error: {:?}", e);
+                                exit(1);
+                            }
+                        })
+                    })
+                }
                 Stage::Parse => {
                     let name = format!("{bench_name}_parse");
                     benchmark_fn(name, move |b| {
                         let (compiler_def_init, span_offset) =
                             setup_compiler(&bench_file, false, false, false)
                                 .expect("Error setting up compiler");
-                        b.iter(move || parse(compiler_def_init.clone(), span_offset))
+                        let contents = std::fs::read(&bench_file)
+                            .expect(&format!("Cannot find file {bench_file}"));
+                        let (tokens, err) = lex(&contents, span_offset);
+                        if let Err(e) = err {
+                            tokens.eprint(&contents);
+                            eprintln!("Lexing error. Error: {:?}", e);
+                            exit(1);
+                        }
+                        b.iter(move || parse(compiler_def_init.clone(), tokens.clone()))
                     })
                 }
                 Stage::Resolve => {