refactor parser a bit

Author: joshuadavidthomas

crates/djls-templates/src/ast.rs

@@ -1,43 +1,42 @@
-use crate::tokens::Token;
+use crate::tokens::{Token, TokenStream, TokenType};
 use serde::Serialize;
 use thiserror::Error;
 
-#[derive(Clone, Default, Debug, Serialize)]
-pub struct NodeList {
-    nodes: Vec<Node>,
+#[derive(Clone, Debug, Default, Serialize)]
+pub struct Ast {
+    nodelist: Vec<Node>,
     line_offsets: LineOffsets,
 }
 
-impl NodeList {
-    pub fn nodes(&self) -> &Vec<Node> {
-        &self.nodes
+impl Ast {
+    pub fn nodelist(&self) -> &Vec<Node> {
+        &self.nodelist
     }
 
     pub fn line_offsets(&self) -> &LineOffsets {
         &self.line_offsets
     }
 
     pub fn add_node(&mut self, node: Node) {
-        self.nodes.push(node);
+        self.nodelist.push(node);
     }
 
-    pub fn set_line_offsets(&mut self, line_offsets: LineOffsets) {
-        self.line_offsets = line_offsets
-    }
-
-    pub fn finalize(&mut self) -> NodeList {
-        self.clone()
+    pub fn set_line_offsets(&mut self, tokens: &TokenStream) {
+        for token in tokens.tokens() {
+            if let TokenType::Newline = token.token_type() {
+                if let Some(start) = token.start() {
+                    // Add offset for next line
+                    self.line_offsets.add_line(start + 1);
+                }
+            }
+        }
     }
 }
 
-#[derive(Clone, Default, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize)]
 pub struct LineOffsets(pub Vec<u32>);
 
 impl LineOffsets {
-    pub fn new() -> Self {
-        Self(vec![0])
-    }
-
     pub fn add_line(&mut self, offset: u32) {
         self.0.push(offset);
     }
@@ -67,6 +66,12 @@ impl LineOffsets {
     }
 }
 
+impl Default for LineOffsets {
+    fn default() -> Self {
+        Self(vec![0])
+    }
+}
+
 #[derive(Clone, Debug, Serialize)]
 pub enum Node {
     Tag {
@@ -155,13 +160,13 @@ mod tests {
 
         #[test]
         fn test_new_starts_at_zero() {
-            let offsets = LineOffsets::new();
+            let offsets = LineOffsets::default();
             assert_eq!(offsets.position_to_line_col(0), (1, 0)); // Line 1, column 0
         }
 
         #[test]
         fn test_start_of_lines() {
-            let mut offsets = LineOffsets::new();
+            let mut offsets = LineOffsets::default();
             offsets.add_line(10); // Line 2 starts at offset 10
             offsets.add_line(25); // Line 3 starts at offset 25
 
@@ -183,7 +188,7 @@ mod tests {
             assert!(errors.is_empty());
 
             // Find the variable node
-            let nodes = nodelist.nodes();
+            let nodes = nodelist.nodelist();
             let var_node = nodes
                 .iter()
                 .find(|n| matches!(n, Node::Variable { .. }))

crates/djls-templates/src/lib.rs

@@ -5,7 +5,7 @@ mod parser;
 mod tagspecs;
 mod tokens;
 
-use ast::NodeList;
+use ast::Ast;
 pub use error::{to_lsp_diagnostic, QuickFix, TemplateError};
 
 use lexer::Lexer;
@@ -18,7 +18,7 @@ pub use parser::{Parser, ParserError};
 ///
 /// Returns a `Result` containing a tuple of `(Ast, Vec<ParserError>)` on success,
 /// or a `ParserError` on failure.
-pub fn parse_template(source: &str) -> Result<(NodeList, Vec<TemplateError>), TemplateError> {
+pub fn parse_template(source: &str) -> Result<(Ast, Vec<TemplateError>), TemplateError> {
     let tokens = Lexer::new(source)
         .tokenize()
         .map_err(|e| TemplateError::Lexer(e.to_string()))?;

crates/djls-templates/src/parser.rs

@@ -1,4 +1,4 @@
-use crate::ast::{AstError, LineOffsets, Node, NodeList, Span};
+use crate::ast::{Ast, AstError, Node, Span};
 use crate::lexer::LexerError;
 use crate::tokens::{Token, TokenStream, TokenType};
 use thiserror::Error;
@@ -18,25 +18,14 @@ impl Parser {
         }
     }
 
-    pub fn parse(&mut self) -> Result<(NodeList, Vec<ParserError>), ParserError> {
-        let mut nodelist = NodeList::default();
-        let mut line_offsets = LineOffsets::new();
-
-        for token in self.tokens.tokens() {
-            if let TokenType::Newline = token.token_type() {
-                if let Some(start) = token.start() {
-                    // Add offset for next line
-                    line_offsets.add_line(start + 1);
-                }
-            }
-        }
-
-        self.current = 0;
+    pub fn parse(&mut self) -> Result<(Ast, Vec<ParserError>), ParserError> {
+        let mut ast = Ast::default();
+        ast.set_line_offsets(&self.tokens);
 
         while !self.is_at_end() {
             match self.next_node() {
                 Ok(node) => {
-                    nodelist.add_node(node);
+                    ast.add_node(node);
                 }
                 Err(err) => {
                     if !self.is_at_end() {
@@ -47,8 +36,7 @@ impl Parser {
             }
         }
 
-        nodelist.set_line_offsets(line_offsets);
-        Ok((nodelist.finalize(), std::mem::take(&mut self.errors)))
+        Ok((ast.clone(), std::mem::take(&mut self.errors)))
     }
 
     fn next_node(&mut self) -> Result<Node, ParserError> {

crates/djls-templates/src/snapshots/djls_templates__parser__tests__comments__parse_comments.snap

@@ -2,7 +2,7 @@
 source: crates/djls-templates/src/parser.rs
 expression: nodelist
 ---
-nodes:
+nodelist:
   - Text:
       content: "<!-- HTML comment -->"
       span:

crates/djls-templates/src/snapshots/djls_templates__parser__tests__django__parse_complex_if_elif.snap

@@ -2,7 +2,7 @@
 source: crates/djls-templates/src/parser.rs
 expression: nodelist
 ---
-nodes:
+nodelist:
   - Tag:
       name: if
       bits:

crates/djls-templates/src/snapshots/djls_templates__parser__tests__django__parse_django_for_block.snap

@@ -2,7 +2,7 @@
 source: crates/djls-templates/src/parser.rs
 expression: nodelist
 ---
-nodes:
+nodelist:
   - Tag:
       name: for
       bits:

crates/djls-templates/src/snapshots/djls_templates__parser__tests__django__parse_django_if_block.snap

@@ -2,7 +2,7 @@
 source: crates/djls-templates/src/parser.rs
 expression: nodelist
 ---
-nodes:
+nodelist:
   - Tag:
       name: if
       bits:

crates/djls-templates/src/snapshots/djls_templates__parser__tests__django__parse_django_tag_assignment.snap

@@ -2,7 +2,7 @@
 source: crates/djls-templates/src/parser.rs
 expression: nodelist
 ---
-nodes:
+nodelist:
   - Tag:
       name: url
       bits:

crates/djls-templates/src/snapshots/djls_templates__parser__tests__django__parse_django_variable.snap

@@ -2,7 +2,7 @@
 source: crates/djls-templates/src/parser.rs
 expression: nodelist
 ---
-nodes:
+nodelist:
   - Variable:
       var: user.name
       filters: []

crates/djls-templates/src/snapshots/djls_templates__parser__tests__django__parse_django_variable_with_filter.snap

@@ -2,7 +2,7 @@
 source: crates/djls-templates/src/parser.rs
 expression: nodelist
 ---
-nodes:
+nodelist:
   - Variable:
       var: user.name
       filters: