wip

Author: joshuadavidthomas

crates/djls-source/src/position.rs

@@ -32,6 +32,15 @@ impl Span {
         Self { start, length }
     }
 
+    /// Construct a span from integer bounds expressed as byte offsets.
+    #[must_use]
+    pub fn from_bounds(start: usize, end: usize) -> Self {
+        let start_u32 = u32::try_from(start).unwrap_or(u32::MAX);
+        let end_u32 = u32::try_from(end).unwrap_or(u32::MAX);
+        let length = end_u32.saturating_sub(start_u32);
+        Self::new(start_u32, length)
+    }
+
     #[must_use]
     pub fn start_offset(&self) -> ByteOffset {
         ByteOffset(self.start)

crates/djls-templates/src/lexer.rs

@@ -1,6 +1,9 @@
+use djls_source::Span;
+
 use crate::db::Db as TemplateDb;
 use crate::tokens::Token;
 use crate::tokens::TokenContent;
+use crate::tokens::TokenSpans;
 
 const BLOCK_TAG_START: &str = "{%";
 const BLOCK_TAG_END: &str = "%}";
@@ -35,14 +38,14 @@ impl<'db> Lexer<'db> {
 
             let token = match self.peek() {
                 '{' => match self.peek_next() {
-                    '%' => self.lex_django_construct(BLOCK_TAG_END, |content, offset| {
-                        Token::Block { content, offset }
+                    '%' => self.lex_django_construct(BLOCK_TAG_END, |content, spans| {
+                        Token::Block { content, spans }
                     }),
-                    '{' => self.lex_django_construct(VARIABLE_TAG_END, |content, offset| {
-                        Token::Variable { content, offset }
+                    '{' => self.lex_django_construct(VARIABLE_TAG_END, |content, spans| {
+                        Token::Variable { content, spans }
                     }),
-                    '#' => self.lex_django_construct(COMMENT_TAG_END, |content, offset| {
-                        Token::Comment { content, offset }
+                    '#' => self.lex_django_construct(COMMENT_TAG_END, |content, spans| {
+                        Token::Comment { content, spans }
                     }),
                     _ => self.lex_text(),
                 },
@@ -61,35 +64,45 @@ impl<'db> Lexer<'db> {
     fn lex_django_construct(
         &mut self,
         end: &str,
-        token_fn: impl FnOnce(TokenContent<'db>, usize) -> Token<'db>,
+        token_fn: impl FnOnce(TokenContent<'db>, TokenSpans) -> Token<'db>,
     ) -> Token<'db> {
-        let offset = self.start + 3;
+        let opening_len = 2;
+        let content_start = self.start + opening_len;
 
         self.consume_n(2);
 
         match self.consume_until(end) {
             Ok(text) => {
-                self.consume_n(2);
                 let content = TokenContent::new(self.db, text);
-                token_fn(content, offset)
+                let content_end = self.current;
+                let span = Span::from_bounds(content_start, content_end);
+                self.consume_n(end.len());
+                let full_end = self.current;
+                let full_span = Span::from_bounds(self.start, full_end);
+                token_fn(content, TokenSpans::new(span, full_span))
             }
             Err(err_text) => {
-                self.synchronize();
+                let content_end = self.current;
+                let span = Span::from_bounds(content_start, content_end);
+                let full_span = Span::from_bounds(self.start, content_end);
                 let content = TokenContent::new(self.db, err_text);
-                Token::Error { content, offset }
+                Token::Error {
+                    content,
+                    spans: TokenSpans::new(span, full_span),
+                }
             }
         }
     }
 
     fn lex_whitespace(&mut self, c: char) -> Token<'db> {
-        let offset = self.start;
-
         if c == '\n' || c == '\r' {
             self.consume(); // \r or \n
             if c == '\r' && self.peek() == '\n' {
                 self.consume(); // \n of \r\n
             }
-            Token::Newline { offset }
+            let span = Span::from_bounds(self.start, self.current);
+            let spans = TokenSpans::new(span, span);
+            Token::Newline { spans }
         } else {
             self.consume(); // Consume the first whitespace
             while !self.is_at_end() && self.peek().is_whitespace() {
@@ -98,8 +111,9 @@ impl<'db> Lexer<'db> {
                 }
                 self.consume();
             }
-            let count = self.current - self.start;
-            Token::Whitespace { count, offset }
+            let span = Span::from_bounds(self.start, self.current);
+            let spans = TokenSpans::new(span, span);
+            Token::Whitespace { spans }
         }
     }
 
@@ -119,10 +133,9 @@ impl<'db> Lexer<'db> {
 
         let text = &self.source[text_start..self.current];
         let content = TokenContent::new(self.db, text.to_string());
-        Token::Text {
-            content,
-            offset: self.start,
-        }
+        let span = Span::from_bounds(self.start, self.current);
+        let spans = TokenSpans::new(span, span);
+        Token::Text { content, spans }
     }
 
     #[inline]
@@ -156,26 +169,33 @@ impl<'db> Lexer<'db> {
 
     fn consume_until(&mut self, delimiter: &str) -> Result<String, String> {
         let offset = self.current;
+        let mut fallback: Option<usize> = None;
 
         while self.current < self.source.len() {
             if self.source[self.current..].starts_with(delimiter) {
-                return Ok(self.source[offset..self.current].trim().to_string());
+                return Ok(self.source[offset..self.current].to_string());
             }
-            self.consume();
-        }
 
-        Err(self.source[offset..self.current].trim().to_string())
-    }
-
-    fn synchronize(&mut self) {
-        const SYNC_POINTS: &[u8] = b"{\n\r";
+            if fallback.is_none()
+                && (self.source[self.current..].starts_with(BLOCK_TAG_START)
+                    || self.source[self.current..].starts_with(VARIABLE_TAG_START)
+                    || self.source[self.current..].starts_with(COMMENT_TAG_START))
+            {
+                fallback = Some(self.current);
+            }
 
-        while !self.is_at_end() {
-            if SYNC_POINTS.contains(&self.source.as_bytes()[self.current]) {
-                return;
+            let ch = self.peek();
+            if fallback.is_none() && matches!(ch, '\n' | '\r') {
+                fallback = Some(self.current);
             }
+
             self.consume();
         }
+
+        let end = fallback.unwrap_or(self.current);
+        let text = self.source[offset..end].to_string();
+        self.current = end;
+        Err(text)
     }
 }
 

crates/djls-templates/src/nodelist.rs

@@ -16,18 +16,22 @@ pub enum Node<'db> {
         name: TagName<'db>,
         bits: Vec<TagBit<'db>>,
         span: Span,
+        full_span: Span,
     },
     Comment {
         content: String,
         span: Span,
+        full_span: Span,
     },
     Text {
         span: Span,
+        full_span: Span,
     },
     Variable {
         var: VariableName<'db>,
         filters: Vec<FilterName<'db>>,
         span: Span,
+        full_span: Span,
     },
     Error {
         node: ErrorNode,
@@ -41,25 +45,19 @@ impl<'db> Node<'db> {
             Node::Tag { span, .. }
             | Node::Variable { span, .. }
             | Node::Comment { span, .. }
-            | Node::Text { span } => *span,
+            | Node::Text { span, .. } => *span,
             Node::Error { node, .. } => node.span,
         }
     }
 
     #[must_use]
     pub fn full_span(&self) -> Span {
         match self {
-            // account for delimiters
-            Node::Variable { span, .. }
-            | Node::Comment { span, .. }
-            | Node::Tag { span, .. } => {
-                Span {
-                    start: span.start.saturating_sub(3),
-                    length: span.length + 6,
-                }
-            }
-            Node::Error { node } => node.span,
-            Node::Text { span } => *span,
+            Node::Variable { full_span, .. }
+            | Node::Comment { full_span, .. }
+            | Node::Tag { full_span, .. }
+            | Node::Text { full_span, .. } => *full_span,
+            Node::Error { node } => node.full_span,
         }
     }
 
@@ -90,6 +88,7 @@ impl<'db> Node<'db> {
 pub struct ErrorNode {
     pub content: String,
     pub span: Span,
+    pub full_span: Span,
     pub error: ParseError,
 }