Optimize parser w/TagBit interning and simplified token access (#221)

Author: joshuadavidthomas

crates/djls-templates/src/ast.rs

@@ -59,7 +59,7 @@ impl Default for LineOffsets {
 pub enum Node<'db> {
     Tag {
         name: TagName<'db>,
-        bits: Vec<String>,
+        bits: Vec<TagBit<'db>>,
         span: Span,
     },
     Comment {
@@ -129,6 +129,11 @@ pub struct TagName<'db> {
     pub text: String,
 }
 
+#[salsa::interned(debug)]
+pub struct TagBit<'db> {
+    pub text: String,
+}
+
 #[salsa::interned(debug)]
 pub struct VariableName<'db> {
     pub text: String,

crates/djls-templates/src/lexer.rs

@@ -1,3 +1,4 @@
+use crate::ast::LineOffsets;
 use crate::db::Db as TemplateDb;
 use crate::tokens::Token;
 use crate::tokens::TokenContent;
@@ -14,7 +15,6 @@ pub struct Lexer<'db> {
     source: String,
     start: usize,
     current: usize,
-    line: usize,
 }
 
 impl<'db> Lexer<'db> {
@@ -25,12 +25,12 @@ impl<'db> Lexer<'db> {
             source: String::from(source),
             start: 0,
             current: 0,
-            line: 1,
         }
     }
 
-    pub fn tokenize(&mut self) -> Vec<Token<'db>> {
+    pub fn tokenize(&mut self) -> (Vec<Token<'db>>, LineOffsets) {
         let mut tokens = Vec::new();
+        let mut line_offsets = LineOffsets::default();
 
         while !self.is_at_end() {
             self.start = self.current;
@@ -53,9 +53,9 @@ impl<'db> Lexer<'db> {
             };
 
             match self.peek_previous() {
-                '\n' => self.line += 1,
+                '\n' => line_offsets.add_line(u32::try_from(self.current).unwrap_or(u32::MAX)),
                 '\r' => {
-                    self.line += 1;
+                    line_offsets.add_line(u32::try_from(self.current).unwrap_or(u32::MAX));
                     if self.peek() == '\n' {
                         self.current += 1;
                     }
@@ -68,7 +68,7 @@ impl<'db> Lexer<'db> {
 
         tokens.push(Token::Eof);
 
-        tokens
+        (tokens, line_offsets)
     }
 
     fn lex_django_construct(
@@ -255,7 +255,7 @@ mod tests {
         let db = TestDatabase::new();
         let source = r#"<div class="container" id="main" disabled></div>"#;
         let mut lexer = Lexer::new(&db, source);
-        let tokens = lexer.tokenize();
+        let (tokens, _) = lexer.tokenize();
         let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db);
         insta::assert_yaml_snapshot!(snapshot);
     }
@@ -265,7 +265,7 @@ mod tests {
         let db = TestDatabase::new();
         let source = "{{ user.name|default:\"Anonymous\"|title }}";
         let mut lexer = Lexer::new(&db, source);
-        let tokens = lexer.tokenize();
+        let (tokens, _) = lexer.tokenize();
         let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db);
         insta::assert_yaml_snapshot!(snapshot);
     }
@@ -275,7 +275,7 @@ mod tests {
         let db = TestDatabase::new();
         let source = "{% if user.is_staff %}Admin{% else %}User{% endif %}";
         let mut lexer = Lexer::new(&db, source);
-        let tokens = lexer.tokenize();
+        let (tokens, _) = lexer.tokenize();
         let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db);
         insta::assert_yaml_snapshot!(snapshot);
     }
@@ -294,7 +294,7 @@ mod tests {
     /* CSS comment */
 </style>";
         let mut lexer = Lexer::new(&db, source);
-        let tokens = lexer.tokenize();
+        let (tokens, _) = lexer.tokenize();
         let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db);
         insta::assert_yaml_snapshot!(snapshot);
     }
@@ -310,7 +310,7 @@ mod tests {
     console.log(x);
 </script>"#;
         let mut lexer = Lexer::new(&db, source);
-        let tokens = lexer.tokenize();
+        let (tokens, _) = lexer.tokenize();
         let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db);
         insta::assert_yaml_snapshot!(snapshot);
     }
@@ -325,7 +325,7 @@ mod tests {
     }
 </style>"#;
         let mut lexer = Lexer::new(&db, source);
-        let tokens = lexer.tokenize();
+        let (tokens, _) = lexer.tokenize();
         let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db);
         insta::assert_yaml_snapshot!(snapshot);
     }
@@ -339,7 +339,7 @@ mod tests {
 <!-- html comment -->
 <div>text</div>";
         let mut lexer = Lexer::new(&db, source);
-        let tokens = lexer.tokenize();
+        let (tokens, _) = lexer.tokenize();
         let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db);
         insta::assert_yaml_snapshot!(snapshot);
     }
@@ -378,7 +378,7 @@ mod tests {
 </body>
 </html>"#;
         let mut lexer = Lexer::new(&db, source);
-        let tokens = lexer.tokenize();
+        let (tokens, _) = lexer.tokenize();
         let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db);
         insta::assert_yaml_snapshot!(snapshot);
     }
@@ -388,7 +388,7 @@ mod tests {
         let db = TestDatabase::new();
         let source = "<style>body { color: blue; ";
         let mut lexer = Lexer::new(&db, source);
-        let tokens = lexer.tokenize();
+        let (tokens, _) = lexer.tokenize();
         let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db);
         insta::assert_yaml_snapshot!(snapshot);
     }

crates/djls-templates/src/lib.rs

@@ -75,14 +75,14 @@ use validation::TagValidator;
 #[salsa::tracked]
 fn lex_template(db: &dyn Db, file: SourceFile) -> TokenStream<'_> {
     if file.kind(db) != FileKind::Template {
-        return TokenStream::new(db, vec![]);
+        return TokenStream::new(db, vec![], LineOffsets::default());
     }
 
     let text_arc = djls_workspace::db::source_text(db, file);
     let text = text_arc.as_ref();
 
-    let tokens = Lexer::new(db, text).tokenize();
-    TokenStream::new(db, tokens)
+    let (tokens, line_offsets) = Lexer::new(db, text).tokenize();
+    TokenStream::new(db, tokens, line_offsets)
 }
 
 /// Parse tokens into an AST.