fix

Author: joshuadavidthomas

crates/djls-templates/src/lexer.rs

@@ -103,9 +103,16 @@ impl Lexer {
         let mut text = String::new();
         while !self.is_at_end() {
             let c = self.peek();
-            if c == '{' || c == '\n' {
+
+            if c == '{' {
+                let next = self.peek_next();
+                if next == '%' || next == '{' || next == '#' {
+                    break;
+                }
+            } else if c == '\n' {
                 break;
             }
+
             text.push(c);
             self.consume();
         }
@@ -125,23 +132,10 @@ impl Lexer {
     }
 
     fn peek_at(&self, offset: isize) -> char {
-        let index = if offset < 0 {
-            match self.current.checked_sub(offset.unsigned_abs()) {
-                Some(idx) => idx,
-                None => return '\0',
-            }
-        } else {
-            match self.current.checked_add(offset as usize) {
-                Some(idx) => idx,
-                None => return '\0',
-            }
+        let Some(index) = self.current.checked_add_signed(offset) else {
+            return '\0';
         };
-
-        if index >= self.chars.len() {
-            '\0'
-        } else {
-            self.chars[index]
-        }
+        self.chars.get(index).copied().unwrap_or('\0')
     }
 
     fn is_at_end(&self) -> bool {
@@ -307,4 +301,12 @@ mod tests {
         let tokens = lexer.tokenize();
         insta::assert_yaml_snapshot!(tokens);
     }
+
+    #[test]
+    fn test_tokenize_unclosed_style() {
+        let source = "<style>body { color: blue; ";
+        let mut lexer = Lexer::new(source);
+        let tokens = lexer.tokenize();
+        insta::assert_yaml_snapshot!(tokens);
+    }
 }

crates/djls-templates/src/parser.rs

@@ -13,7 +13,7 @@ use crate::ast::TextNode;
 use crate::ast::VariableName;
 use crate::ast::VariableNode;
 use crate::db::Db as TemplateDb;
-use crate::lexer::LexerError;
+
 use crate::tokens::Token;
 use crate::tokens::TokenStream;
 use crate::tokens::TokenType;
@@ -75,29 +75,18 @@ impl<'db> Parser<'db> {
         let token = self.consume()?;
 
         match token.token_type() {
-            TokenType::Comment(_, open, _) => self.parse_comment(open),
+            TokenType::Comment(_) => self.parse_comment(),
             TokenType::Eof => Err(ParserError::stream_error(StreamError::AtEnd)),
             TokenType::Block(_) => self.parse_django_block(),
             TokenType::Variable(_) => self.parse_django_variable(),
-            TokenType::HtmlTagClose(_)
-            | TokenType::HtmlTagOpen(_)
-            | TokenType::HtmlTagVoid(_)
-            | TokenType::Newline
-            | TokenType::ScriptTagClose(_)
-            | TokenType::ScriptTagOpen(_)
-            | TokenType::StyleTagClose(_)
-            | TokenType::StyleTagOpen(_)
+            TokenType::Error(_) => self.parse_error(),
+            TokenType::Newline
             | TokenType::Text(_)
             | TokenType::Whitespace(_) => self.parse_text(),
         }
     }
 
-    fn parse_comment(&mut self, open: &str) -> Result<Node<'db>, ParserError> {
-        // Only treat Django comments as Comment nodes
-        if open != "{#" {
-            return self.parse_text();
-        }
-
+    fn parse_comment(&mut self) -> Result<Node<'db>, ParserError> {
         let token = self.peek_previous()?;
 
         Ok(Node::Comment(CommentNode {
@@ -106,6 +95,23 @@ impl<'db> Parser<'db> {
         }))
     }
 
+    fn parse_error(&mut self) -> Result<Node<'db>, ParserError> {
+        let token = self.peek_previous()?;
+        
+        if let TokenType::Error(content) = token.token_type() {
+            let position = token.start().unwrap_or(0) as usize;
+            
+            Err(ParserError::MalformedConstruct {
+                position,
+                content: content.clone(),
+            })
+        } else {
+            Err(ParserError::InvalidSyntax {
+                context: "Expected Error token".to_string(),
+            })
+        }
+    }
+
     pub fn parse_django_block(&mut self) -> Result<Node<'db>, ParserError> {
         let token = self.peek_previous()?;
 
@@ -144,41 +150,60 @@ impl<'db> Parser<'db> {
     }
 
     fn parse_text(&mut self) -> Result<Node<'db>, ParserError> {
-        let token = self.peek_previous()?;
-
-        if token.token_type() == &TokenType::Newline {
-            return self.next_node();
-        }
-
-        let mut text = token.lexeme();
-
-        while let Ok(token) = self.peek() {
-            match token.token_type() {
-                TokenType::Block(_)
-                | TokenType::Variable(_)
-                | TokenType::Comment(_, _, _)
-                | TokenType::Newline
-                | TokenType::Eof => break,
-                _ => {
-                    let token_text = token.lexeme();
-                    text.push_str(&token_text);
+        // Keep looping until we find non-empty text or hit a significant token
+        loop {
+            let token = self.peek_previous()?;
+            
+            // Skip newlines directly without recursion
+            if token.token_type() == &TokenType::Newline {
+                if !self.is_at_end() {
                     self.consume()?;
+                    continue;
+                } else {
+                    return Err(ParserError::stream_error(StreamError::AtEnd));
                 }
             }
-        }
 
-        let content = match text.trim() {
-            "" => return self.next_node(),
-            trimmed => trimmed.to_string(),
-        };
+            let mut text = token.lexeme();
+
+            while let Ok(token) = self.peek() {
+                match token.token_type() {
+                    TokenType::Block(_)
+                    | TokenType::Variable(_)
+                    | TokenType::Comment(_)
+                    | TokenType::Error(_)
+                    | TokenType::Newline
+                    | TokenType::Eof => break,
+                    _ => {
+                        let token_text = token.lexeme();
+                        text.push_str(&token_text);
+                        self.consume()?;
+                    }
+                }
+            }
 
-        let start = token.start().unwrap_or(0);
-        let offset = u32::try_from(text.find(content.as_str()).unwrap_or(0))
-            .expect("Offset should fit in u32");
-        let length = u32::try_from(content.len()).expect("Content length should fit in u32");
-        let span = Span::new(start + offset, length);
+            let content = match text.trim() {
+                "" => {
+                    // Instead of recursing, continue the loop
+                    if !self.is_at_end() {
+                        self.consume()?;
+                        continue;
+                    } else {
+                        return Err(ParserError::stream_error(StreamError::AtEnd));
+                    }
+                }
+                trimmed => trimmed.to_string(),
+            };
+
+            // We have non-empty content, create the text node
+            let start = token.start().unwrap_or(0);
+            let offset = u32::try_from(text.find(content.as_str()).unwrap_or(0))
+                .expect("Offset should fit in u32");
+            let length = u32::try_from(content.len()).expect("Content length should fit in u32");
+            let span = Span::new(start + offset, length);
 
-        Ok(Node::Text(TextNode { content, span }))
+            return Ok(Node::Text(TextNode { content, span }));
+        }
     }
 
     fn peek(&self) -> Result<Token, ParserError> {
@@ -255,7 +280,7 @@ impl<'db> Parser<'db> {
         let sync_types = &[
             TokenType::Block(String::new()),
             TokenType::Variable(String::new()),
-            TokenType::Comment(String::new(), String::from("{#"), Some(String::from("#}"))),
+            TokenType::Comment(String::new()),
             TokenType::Eof,
         ];
 
@@ -314,8 +339,8 @@ pub enum ParseError {
     #[error("Empty tag")]
     EmptyTag,
 
-    #[error("Lexer error: {0}")]
-    Lexer(#[from] LexerError),
+    #[error("Malformed Django construct at position {position}: {content}")]
+    MalformedConstruct { position: usize, content: String },
 
     #[error("Stream error: {kind:?}")]
     StreamError { kind: StreamError },
@@ -392,7 +417,7 @@ mod tests {
     #[salsa::tracked]
     fn parse_test_template(db: &dyn TemplateDb, template: TestTemplate) -> NodeList<'_> {
         let source = template.source(db);
-        let tokens = Lexer::new(source).tokenize().unwrap();
+        let tokens = Lexer::new(source).tokenize();
         let token_stream = TokenStream::new(db, tokens);
         let mut parser = Parser::new(db, token_stream);
         let (ast, _) = parser.parse().unwrap();

crates/djls-templates/src/snapshots/djls_templates__lexer__tests__tokenize_comments.snap

@@ -3,27 +3,21 @@ source: crates/djls-templates/src/lexer.rs
 expression: tokens
 ---
 - token_type:
-    Comment:
-      - HTML comment
-      - "<!--"
-      - "-->"
+    Text: "<!-- HTML comment -->"
   line: 1
   start: 0
 - token_type: Newline
   line: 1
   start: 21
 - token_type:
-    Comment:
-      - Django comment
-      - "{#"
-      - "#}"
+    Comment: Django comment
   line: 2
   start: 22
 - token_type: Newline
   line: 2
   start: 42
 - token_type:
-    ScriptTagOpen: script
+    Text: "<script>"
   line: 3
   start: 43
 - token_type: Newline
@@ -34,10 +28,7 @@ expression: tokens
   line: 4
   start: 52
 - token_type:
-    Comment:
-      - JS single line comment
-      - //
-      - ~
+    Text: // JS single line comment
   line: 4
   start: 56
 - token_type: Newline
@@ -48,47 +39,52 @@ expression: tokens
   line: 5
   start: 82
 - token_type:
-    Comment:
-      - "JS multi-line\n       comment"
-      - /*
-      - "*/"
+    Text: /* JS multi-line
   line: 5
   start: 86
 - token_type: Newline
   line: 5
-  start: 120
+  start: 102
 - token_type:
-    HtmlTagClose: script
+    Whitespace: 7
   line: 6
-  start: 121
+  start: 103
+- token_type:
+    Text: comment */
+  line: 6
+  start: 110
 - token_type: Newline
   line: 6
-  start: 130
+  start: 120
 - token_type:
-    StyleTagOpen: style
+    Text: "</script>"
   line: 7
-  start: 131
+  start: 121
 - token_type: Newline
   line: 7
+  start: 130
+- token_type:
+    Text: "<style>"
+  line: 8
+  start: 131
+- token_type: Newline
+  line: 8
   start: 138
 - token_type:
     Whitespace: 4
-  line: 8
+  line: 9
   start: 139
 - token_type:
-    Comment:
-      - CSS comment
-      - /*
-      - "*/"
-  line: 8
+    Text: /* CSS comment */
+  line: 9
   start: 143
 - token_type: Newline
-  line: 8
+  line: 9
   start: 160
 - token_type:
-    HtmlTagClose: style
-  line: 9
+    Text: "</style>"
+  line: 10
   start: 161
 - token_type: Eof
-  line: 9
+  line: 10
   start: ~

crates/djls-templates/src/snapshots/djls_templates__lexer__tests__tokenize_django_block.snap

@@ -3,23 +3,23 @@ source: crates/djls-templates/src/lexer.rs
 expression: tokens
 ---
 - token_type:
-    DjangoBlock: if user.is_staff
+    Block: if user.is_staff
   line: 1
   start: 0
 - token_type:
     Text: Admin
   line: 1
   start: 22
 - token_type:
-    DjangoBlock: else
+    Block: else
   line: 1
   start: 27
 - token_type:
     Text: User
   line: 1
   start: 37
 - token_type:
-    DjangoBlock: endif
+    Block: endif
   line: 1
   start: 41
 - token_type: Eof

crates/djls-templates/src/snapshots/djls_templates__lexer__tests__tokenize_django_variable.snap

@@ -3,7 +3,7 @@ source: crates/djls-templates/src/lexer.rs
 expression: tokens
 ---
 - token_type:
-    DjangoVariable: "user.name|default:\"Anonymous\"|title"
+    Variable: "user.name|default:\"Anonymous\"|title"
   line: 1
   start: 0
 - token_type: Eof