v2: add lexer (#431)

Part of the v2 implementation.

Going to prefix the crates with `squawk_` even though they aren't in the other repo. This should make it work with the existing crates which don't have a prefix.

Author: sbdchd

Cargo.lock

@@ -1,5 +1,12 @@
 [workspace]
 members = ["crates/*"]
+resolver = "2"
+
+[workspace.package]
+edition = "2021"
+rust-version = "1.81.0"
+authors = ["Squawk Team & Contributors"]
+license = "GPL-3.0"
 
 [workspace.dependencies]
 # third party
@@ -27,5 +34,15 @@ squawk-parser = { version = "0.0.0", path = "./crates/parser" }
 squawk-linter = { version = "0.0.0", path = "./crates/linter" }
 squawk-github = { version = "0.0.0", path = "./crates/github" }
 
+[workspace.lints.clippy]
+collapsible_else_if = "allow"
+collapsible_if = "allow"
+needless_return = "allow"
+doc_markdown = "deny"
+manual_let_else = "deny"
+
+[profile.dev]
+debug = 0
+
 [profile.dev.package]
 insta.opt-level = 3

Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "squawk-linter"
 version = "0.0.0"
-authors = ["Steve Dignam <[email protected]>"]
-edition = "2018"
-license = "GPL-3.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
 description = "Postgres SQL linter used in squawk"
 repository = "https://github.com/sbdchd/squawk"
 readme = "README.md"

crates/linter/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "lexer"
+version = "0.0.0"
+description = "TBD"
+
+authors.workspace = true
+edition.workspace = true
+license = "MIT"
+rust-version.workspace = true
+
+[lib]
+doctest = false
+
+[dependencies]
+
+[dev-dependencies]
+insta.workspace = true
+
+[lints]
+workspace = true

crates/squawk_lexer/Cargo.toml

@@ -0,0 +1,3 @@
+# lexer
+
+> Adapted from the Rust lexer.

crates/squawk_lexer/README.md

@@ -0,0 +1,25 @@
+from: https://github.com/rust-lang/rust/blob/176e5452095444815207be02c16de0b1487a1b53/LICENSE-MIT
+
+Permission is hereby granted, free of charge, to any
+person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the
+Software without restriction, including without
+limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software
+is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

crates/squawk_lexer/src/LICENSE-MIT

@@ -0,0 +1,65 @@
+use std::str::Chars;
+
+/// Peekable iterator over a char sequence.
+///
+/// Next characters can be peeked via `first` method,
+/// and position can be shifted forward via `bump` method.
+/// based on:
+/// - <https://github.com/rust-lang/rust/blob/d1b7355d3d7b4ead564dbecb1d240fcc74fff21b/compiler/rustc_lexer/src/cursor.rs>
+/// - <https://github.com/astral-sh/ruff/blob/d1079680bb29f6b797b5df15327195300f635f3c/crates/ruff_python_parser/src/lexer/cursor.rs>
+///
+pub(crate) struct Cursor<'a> {
+    /// Iterator over chars. Slightly faster than a &str.
+    chars: Chars<'a>,
+    len_remaining: usize,
+}
+
+pub(crate) const EOF_CHAR: char = '\0';
+
+impl<'a> Cursor<'a> {
+    pub(crate) fn new(input: &'a str) -> Cursor<'a> {
+        Cursor {
+            len_remaining: input.len(),
+            chars: input.chars(),
+        }
+    }
+
+    /// Peeks the next symbol from the input stream without consuming it.
+    /// If requested position doesn't exist, `EOF_CHAR` is returned.
+    /// However, getting `EOF_CHAR` doesn't always mean actual end of file,
+    /// it should be checked with `is_eof` method.
+    pub(crate) fn first(&self) -> char {
+        // `.next()` optimizes better than `.nth(0)`
+        self.chars.clone().next().unwrap_or(EOF_CHAR)
+    }
+
+    /// Checks if there is nothing more to consume.
+    pub(crate) fn is_eof(&self) -> bool {
+        self.chars.as_str().is_empty()
+    }
+
+    /// Returns amount of already consumed symbols.
+    pub(crate) fn pos_within_token(&self) -> u32 {
+        (self.len_remaining - self.chars.as_str().len()) as u32
+    }
+
+    /// Resets the number of bytes consumed to 0.
+    pub(crate) fn reset_pos_within_token(&mut self) {
+        self.len_remaining = self.chars.as_str().len();
+    }
+
+    /// Moves to the next character.
+    pub(crate) fn bump(&mut self) -> Option<char> {
+        let c = self.chars.next()?;
+        Some(c)
+    }
+
+    /// Eats symbols while predicate returns true or until the end of file is reached.
+    pub(crate) fn eat_while(&mut self, mut predicate: impl FnMut(char) -> bool) {
+        // It was tried making optimized version of this for eg. line comments, but
+        // LLVM can inline all of this and compile it down to fast iteration over bytes.
+        while predicate(self.first()) && !self.is_eof() {
+            self.bump();
+        }
+    }
+}