wip

Author: joshuadavidthomas

crates/djls-templates/src/ast.rs

@@ -153,7 +153,7 @@ impl Span {
 
     #[must_use]
     pub fn from_token(token: &Token<'_>, db: &dyn TemplateDb) -> Self {
-        let start = token.start().unwrap_or(0);
+        let start = token.offset().unwrap_or(0);
         let length = token.length(db);
         Span::new(start, length)
     }

crates/djls-templates/src/lexer.rs

@@ -2,10 +2,16 @@ use crate::db::Db as TemplateDb;
 use crate::tokens::Token;
 use crate::tokens::TokenContent;
 
+const BLOCK_TAG_START: &str = "{%";
+const BLOCK_TAG_END: &str = "%}";
+const VARIABLE_TAG_START: &str = "{{";
+const VARIABLE_TAG_END: &str = "}}";
+const COMMENT_TAG_START: &str = "{#";
+const COMMENT_TAG_END: &str = "#}";
+
 pub struct Lexer<'db> {
     db: &'db dyn TemplateDb,
     source: String,
-    chars: Vec<char>,
     start: usize,
     current: usize,
     line: usize,
@@ -17,7 +23,6 @@ impl<'db> Lexer<'db> {
         Lexer {
             db,
             source: String::from(source),
-            chars: source.chars().collect(),
             start: 0,
             current: 0,
             line: 1,
@@ -32,22 +37,14 @@ impl<'db> Lexer<'db> {
 
             let token = match self.peek() {
                 '{' => match self.peek_next() {
-                    '%' => self.lex_django_construct("%}", |content, line, start| Token::Block {
-                        content,
-                        line,
-                        start,
+                    '%' => self.lex_django_construct(BLOCK_TAG_END, |content, offset| {
+                        Token::Block { content, offset }
                     }),
-                    '{' => {
-                        self.lex_django_construct("}}", |content, line, start| Token::Variable {
-                            content,
-                            line,
-                            start,
-                        })
-                    }
-                    '#' => self.lex_django_construct("#}", |content, line, start| Token::Comment {
-                        content,
-                        line,
-                        start,
+                    '{' => self.lex_django_construct(VARIABLE_TAG_END, |content, offset| {
+                        Token::Variable { content, offset }
+                    }),
+                    '#' => self.lex_django_construct(COMMENT_TAG_END, |content, offset| {
+                        Token::Comment { content, offset }
                     }),
                     _ => self.lex_text(),
                 },
@@ -69,49 +66,43 @@ impl<'db> Lexer<'db> {
             tokens.push(token);
         }
 
-        tokens.push(Token::Eof { line: self.line });
+        tokens.push(Token::Eof);
 
         tokens
     }
 
     fn lex_django_construct(
         &mut self,
         end: &str,
-        token_fn: impl FnOnce(TokenContent<'db>, usize, usize) -> Token<'db>,
+        token_fn: impl FnOnce(TokenContent<'db>, usize) -> Token<'db>,
     ) -> Token<'db> {
-        let line = self.line;
-        let start = self.start + 3;
+        let offset = self.start + 3;
 
         self.consume_n(2);
 
         match self.consume_until(end) {
             Ok(text) => {
                 self.consume_n(2);
                 let content = TokenContent::new(self.db, text);
-                token_fn(content, line, start)
+                token_fn(content, offset)
             }
             Err(err_text) => {
                 self.synchronize();
                 let content = TokenContent::new(self.db, err_text);
-                Token::Error {
-                    content,
-                    line,
-                    start,
-                }
+                Token::Error { content, offset }
             }
         }
     }
 
     fn lex_whitespace(&mut self, c: char) -> Token<'db> {
-        let line = self.line;
-        let start = self.start;
+        let offset = self.start;
 
         if c == '\n' || c == '\r' {
             self.consume(); // \r or \n
             if c == '\r' && self.peek() == '\n' {
                 self.consume(); // \n of \r\n
             }
-            Token::Newline { line, start }
+            Token::Newline { offset }
         } else {
             self.consume(); // Consume the first whitespace
             while !self.is_at_end() && self.peek().is_whitespace() {
@@ -121,67 +112,64 @@ impl<'db> Lexer<'db> {
                 self.consume();
             }
             let count = self.current - self.start;
-            Token::Whitespace { count, line, start }
+            Token::Whitespace { count, offset }
         }
     }
 
     fn lex_text(&mut self) -> Token<'db> {
-        let line = self.line;
-        let start = self.start;
+        let text_start = self.current;
 
-        let mut text = String::new();
         while !self.is_at_end() {
-            let c = self.peek();
-
-            if c == '{' {
-                let next = self.peek_next();
-                if next == '%' || next == '{' || next == '#' {
-                    break;
-                }
-            } else if c == '\n' {
+            if self.source[self.current..].starts_with(BLOCK_TAG_START)
+                || self.source[self.current..].starts_with(VARIABLE_TAG_START)
+                || self.source[self.current..].starts_with(COMMENT_TAG_START)
+                || self.source[self.current..].starts_with('\n')
+            {
                 break;
             }
-
-            text.push(c);
             self.consume();
         }
 
-        let content = TokenContent::new(self.db, text);
+        let text = &self.source[text_start..self.current];
+        let content = TokenContent::new(self.db, text.to_string());
         Token::Text {
             content,
-            line,
-            start,
+            offset: self.start,
         }
     }
 
+    #[inline]
     fn peek(&self) -> char {
-        self.peek_at(0)
+        self.source[self.current..].chars().next().unwrap_or('\0')
     }
 
     fn peek_next(&self) -> char {
-        self.peek_at(1)
+        let mut chars = self.source[self.current..].chars();
+        chars.next(); // Skip current
+        chars.next().unwrap_or('\0')
     }
 
     fn peek_previous(&self) -> char {
-        self.peek_at(-1)
-    }
-
-    fn peek_at(&self, offset: isize) -> char {
-        let Some(index) = self.current.checked_add_signed(offset) else {
+        if self.current == 0 {
             return '\0';
-        };
-        self.chars.get(index).copied().unwrap_or('\0')
+        }
+        let mut pos = self.current - 1;
+        while !self.source.is_char_boundary(pos) && pos > 0 {
+            pos -= 1;
+        }
+        self.source[pos..].chars().next().unwrap_or('\0')
     }
 
+    #[inline]
     fn is_at_end(&self) -> bool {
         self.current >= self.source.len()
     }
 
+    #[inline]
     fn consume(&mut self) {
-        if self.is_at_end() {
-            return;
+        if let Some(ch) = self.source[self.current..].chars().next() {
+            self.current += ch.len_utf8();
         }
-        self.current += 1;
     }
 
     fn consume_n(&mut self, count: usize) {
@@ -190,25 +178,24 @@ impl<'db> Lexer<'db> {
         }
     }
 
-    fn consume_until(&mut self, s: &str) -> Result<String, String> {
-        let start = self.current;
-        while !self.is_at_end() {
-            if self.chars[self.current..self.chars.len()]
-                .starts_with(s.chars().collect::<Vec<_>>().as_slice())
-            {
-                return Ok(self.source[start..self.current].trim().to_string());
+    fn consume_until(&mut self, delimiter: &str) -> Result<String, String> {
+        let offset = self.current;
+
+        while self.current < self.source.len() {
+            if self.source[self.current..].starts_with(delimiter) {
+                return Ok(self.source[offset..self.current].trim().to_string());
             }
             self.consume();
         }
-        Err(self.source[start..self.current].trim().to_string())
+
+        Err(self.source[offset..self.current].trim().to_string())
     }
 
     fn synchronize(&mut self) {
-        let sync_chars = &['{', '\n', '\r'];
+        const SYNC_POINTS: &[u8] = b"{\n\r";
 
         while !self.is_at_end() {
-            let current_char = self.peek();
-            if sync_chars.contains(&current_char) {
+            if SYNC_POINTS.contains(&self.source.as_bytes()[self.current]) {
                 return;
             }
             self.consume();

crates/djls-templates/src/parser.rs

@@ -37,7 +37,7 @@ impl<'db> Parser<'db> {
         let tokens = self.tokens.stream(self.db);
         for token in tokens {
             if matches!(token, Token::Newline { .. }) {
-                let start = token.start();
+                let start = token.offset();
                 if let Some(start) = start {
                     line_offsets.add_line(start + 1);
                 }
@@ -92,8 +92,7 @@ impl<'db> Parser<'db> {
 
         if let Token::Error {
             content,
-            line: _,
-            start,
+            offset: start,
             ..
         } = token
         {
@@ -152,7 +151,7 @@ impl<'db> Parser<'db> {
             return self.next_node();
         }
 
-        let start = first_token.start().unwrap_or(0);
+        let start = first_token.offset().unwrap_or(0);
         let mut end_position = start + first_token.length(self.db);
 
         while let Ok(token) = self.peek() {
@@ -164,7 +163,7 @@ impl<'db> Parser<'db> {
                 | Token::Eof { .. } => break, // Stop at Django constructs
                 Token::Text { .. } | Token::Whitespace { .. } | Token::Newline { .. } => {
                     // Update end position
-                    let token_start = token.start().unwrap_or(end_position);
+                    let token_start = token.offset().unwrap_or(end_position);
                     let token_length = token.length(self.db);
                     end_position = token_start + token_length;
                     self.consume()?;

crates/djls-templates/src/snapshots/djls_templates__lexer__tests__tokenize_comments.snap

@@ -4,86 +4,62 @@ expression: snapshot
 ---
 - Text:
     content: "<!-- HTML comment -->"
-    line: 1
-    start: 0
+    offset: 0
 - Newline:
-    line: 1
-    start: 21
+    offset: 21
 - Comment:
     content: Django comment
-    line: 2
-    start: 25
+    offset: 25
 - Newline:
-    line: 2
-    start: 42
+    offset: 42
 - Text:
     content: "<script>"
-    line: 3
-    start: 43
+    offset: 43
 - Newline:
-    line: 3
-    start: 51
+    offset: 51
 - Whitespace:
     count: 4
-    line: 4
-    start: 52
+    offset: 52
 - Text:
     content: // JS single line comment
-    line: 4
-    start: 56
+    offset: 56
 - Newline:
-    line: 4
-    start: 81
+    offset: 81
 - Whitespace:
     count: 4
-    line: 5
-    start: 82
+    offset: 82
 - Text:
     content: /* JS multi-line
-    line: 5
-    start: 86
+    offset: 86
 - Newline:
-    line: 5
-    start: 102
+    offset: 102
 - Whitespace:
     count: 7
-    line: 6
-    start: 103
+    offset: 103
 - Text:
     content: comment */
-    line: 6
-    start: 110
+    offset: 110
 - Newline:
-    line: 6
-    start: 120
+    offset: 120
 - Text:
     content: "</script>"
-    line: 7
-    start: 121
+    offset: 121
 - Newline:
-    line: 7
-    start: 130
+    offset: 130
 - Text:
     content: "<style>"
-    line: 8
-    start: 131
+    offset: 131
 - Newline:
-    line: 8
-    start: 138
+    offset: 138
 - Whitespace:
     count: 4
-    line: 9
-    start: 139
+    offset: 139
 - Text:
     content: /* CSS comment */
-    line: 9
-    start: 143
+    offset: 143
 - Newline:
-    line: 9
-    start: 160
+    offset: 160
 - Text:
     content: "</style>"
-    line: 10
-    start: 161
-- Eof:
-    line: 10
+    offset: 161
+- Eof