diff --git a/crates/djls-templates/src/ast.rs b/crates/djls-templates/src/ast.rs index 08210aef..9c9590dc 100644 --- a/crates/djls-templates/src/ast.rs +++ b/crates/djls-templates/src/ast.rs @@ -59,7 +59,7 @@ impl Default for LineOffsets { pub enum Node<'db> { Tag { name: TagName<'db>, - bits: Vec, + bits: Vec>, span: Span, }, Comment { @@ -129,6 +129,11 @@ pub struct TagName<'db> { pub text: String, } +#[salsa::interned(debug)] +pub struct TagBit<'db> { + pub text: String, +} + #[salsa::interned(debug)] pub struct VariableName<'db> { pub text: String, diff --git a/crates/djls-templates/src/lexer.rs b/crates/djls-templates/src/lexer.rs index e417c739..197ab1df 100644 --- a/crates/djls-templates/src/lexer.rs +++ b/crates/djls-templates/src/lexer.rs @@ -1,3 +1,4 @@ +use crate::ast::LineOffsets; use crate::db::Db as TemplateDb; use crate::tokens::Token; use crate::tokens::TokenContent; @@ -14,7 +15,6 @@ pub struct Lexer<'db> { source: String, start: usize, current: usize, - line: usize, } impl<'db> Lexer<'db> { @@ -25,12 +25,12 @@ impl<'db> Lexer<'db> { source: String::from(source), start: 0, current: 0, - line: 1, } } - pub fn tokenize(&mut self) -> Vec> { + pub fn tokenize(&mut self) -> (Vec>, LineOffsets) { let mut tokens = Vec::new(); + let mut line_offsets = LineOffsets::default(); while !self.is_at_end() { self.start = self.current; @@ -53,9 +53,9 @@ impl<'db> Lexer<'db> { }; match self.peek_previous() { - '\n' => self.line += 1, + '\n' => line_offsets.add_line(u32::try_from(self.current).unwrap_or(u32::MAX)), '\r' => { - self.line += 1; + line_offsets.add_line(u32::try_from(self.current).unwrap_or(u32::MAX)); if self.peek() == '\n' { self.current += 1; } @@ -68,7 +68,7 @@ impl<'db> Lexer<'db> { tokens.push(Token::Eof); - tokens + (tokens, line_offsets) } fn lex_django_construct( @@ -255,7 +255,7 @@ mod tests { let db = TestDatabase::new(); let source = r#"
"#; let mut lexer = Lexer::new(&db, source); - let tokens = lexer.tokenize(); + let (tokens, _) = lexer.tokenize(); let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db); insta::assert_yaml_snapshot!(snapshot); } @@ -265,7 +265,7 @@ mod tests { let db = TestDatabase::new(); let source = "{{ user.name|default:\"Anonymous\"|title }}"; let mut lexer = Lexer::new(&db, source); - let tokens = lexer.tokenize(); + let (tokens, _) = lexer.tokenize(); let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db); insta::assert_yaml_snapshot!(snapshot); } @@ -275,7 +275,7 @@ mod tests { let db = TestDatabase::new(); let source = "{% if user.is_staff %}Admin{% else %}User{% endif %}"; let mut lexer = Lexer::new(&db, source); - let tokens = lexer.tokenize(); + let (tokens, _) = lexer.tokenize(); let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db); insta::assert_yaml_snapshot!(snapshot); } @@ -294,7 +294,7 @@ mod tests { /* CSS comment */ "; let mut lexer = Lexer::new(&db, source); - let tokens = lexer.tokenize(); + let (tokens, _) = lexer.tokenize(); let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db); insta::assert_yaml_snapshot!(snapshot); } @@ -310,7 +310,7 @@ mod tests { console.log(x); "#; let mut lexer = Lexer::new(&db, source); - let tokens = lexer.tokenize(); + let (tokens, _) = lexer.tokenize(); let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db); insta::assert_yaml_snapshot!(snapshot); } @@ -325,7 +325,7 @@ mod tests { } "#; let mut lexer = Lexer::new(&db, source); - let tokens = lexer.tokenize(); + let (tokens, _) = lexer.tokenize(); let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db); insta::assert_yaml_snapshot!(snapshot); } @@ -339,7 +339,7 @@ mod tests {
text
"; let mut lexer = Lexer::new(&db, source); - let tokens = lexer.tokenize(); + let (tokens, _) = lexer.tokenize(); let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db); insta::assert_yaml_snapshot!(snapshot); } @@ -378,7 +378,7 @@ mod tests { "#; let mut lexer = Lexer::new(&db, source); - let tokens = lexer.tokenize(); + let (tokens, _) = lexer.tokenize(); let snapshot = TokenSnapshotVec(tokens).to_snapshot(&db); insta::assert_yaml_snapshot!(snapshot); } @@ -388,7 +388,7 @@ mod tests { let db = TestDatabase::new(); let source = "