Skip to content

Commit f0e2df2

Browse files
joshuadavidthomasjoshuadavidthomas
committed
new enum!
1 parent e60e879 commit f0e2df2

File tree

3 files changed

+76
-66
lines changed

3 files changed

+76
-66
lines changed

‎crates/djls-templates/src/lexer.rs‎

Lines changed: 32 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,9 @@
11
use djls_source::Span;
22

33
use crate::db::Db as TemplateDb;
4+
use crate::tokens::TagDelimiter;
45
use crate::tokens::Token;
56
use crate::tokens::TokenContent;
6-
use crate::tokens::BLOCK_TAG_END;
7-
use crate::tokens::BLOCK_TAG_START;
8-
use crate::tokens::COMMENT_TAG_END;
9-
use crate::tokens::COMMENT_TAG_START;
10-
use crate::tokens::DJANGO_TAG_LEN;
11-
use crate::tokens::VARIABLE_TAG_END;
12-
use crate::tokens::VARIABLE_TAG_START;
137

148
pub struct Lexer<'db> {
159
db: &'db dyn TemplateDb,
@@ -37,19 +31,25 @@ impl<'db> Lexer<'db> {
3731

3832
let token =
3933
match self.peek() {
40-
'{' => match self.peek_next() {
41-
'%' => self.lex_django_tag(BLOCK_TAG_END, |content, span| Token::Block {
42-
content,
43-
span,
44-
}),
45-
'{' => self.lex_django_tag(VARIABLE_TAG_END, |content, span| {
46-
Token::Variable { content, span }
47-
}),
48-
'#' => self.lex_django_tag(COMMENT_TAG_END, |content, span| {
49-
Token::Comment { content, span }
50-
}),
51-
_ => self.lex_text(),
52-
},
34+
'{' => {
35+
let remaining = &self.source[self.current..];
36+
37+
match TagDelimiter::from_input(remaining) {
38+
Some(TagDelimiter::Block) => self
39+
.lex_django_tag(TagDelimiter::Block, |content, span| {
40+
Token::Block { content, span }
41+
}),
42+
Some(TagDelimiter::Variable) => self
43+
.lex_django_tag(TagDelimiter::Variable, |content, span| {
44+
Token::Variable { content, span }
45+
}),
46+
Some(TagDelimiter::Comment) => self
47+
.lex_django_tag(TagDelimiter::Comment, |content, span| {
48+
Token::Comment { content, span }
49+
}),
50+
None => self.lex_text(),
51+
}
52+
}
5353
c if c.is_whitespace() => self.lex_whitespace(c),
5454
_ => self.lex_text(),
5555
};
@@ -64,18 +64,18 @@ impl<'db> Lexer<'db> {
6464

6565
fn lex_django_tag(
6666
&mut self,
67-
end: &str,
67+
delimiter: TagDelimiter,
6868
token_fn: impl FnOnce(TokenContent<'db>, Span) -> Token<'db>,
6969
) -> Token<'db> {
70-
let content_start = self.start + DJANGO_TAG_LEN as usize;
71-
self.consume_n(DJANGO_TAG_LEN as usize);
70+
let content_start = self.start + TagDelimiter::LENGTH;
71+
self.consume_n(TagDelimiter::LENGTH);
7272

73-
match self.consume_until(end) {
73+
match self.consume_until(delimiter.closer()) {
7474
Ok(text) => {
7575
let len = text.len();
7676
let content = TokenContent::new(self.db, text);
7777
let span = Span::from_parts(content_start, len);
78-
self.consume_n(end.len());
78+
self.consume_n(delimiter.closer().len());
7979
token_fn(content, span)
8080
}
8181
Err(err_text) => {
@@ -116,10 +116,10 @@ impl<'db> Lexer<'db> {
116116
let text_start = self.current;
117117

118118
while !self.is_at_end() {
119-
if self.source[self.current..].starts_with(BLOCK_TAG_START)
120-
|| self.source[self.current..].starts_with(VARIABLE_TAG_START)
121-
|| self.source[self.current..].starts_with(COMMENT_TAG_START)
122-
|| self.source[self.current..].starts_with('\n')
119+
let slice = &self.source[self.current..];
120+
if (self.peek() == '{' && TagDelimiter::from_input(slice).is_some())
121+
|| slice.starts_with('\n')
122+
|| slice.starts_with('\r')
123123
{
124124
break;
125125
}
@@ -137,13 +137,6 @@ impl<'db> Lexer<'db> {
137137
self.source[self.current..].chars().next().unwrap_or('\0')
138138
}
139139

140-
#[inline]
141-
fn peek_next(&self) -> char {
142-
let mut chars = self.source[self.current..].chars();
143-
chars.next(); // Skip current
144-
chars.next().unwrap_or('\0')
145-
}
146-
147140
#[inline]
148141
fn is_at_end(&self) -> bool {
149142
self.current >= self.source.len()
@@ -167,15 +160,12 @@ impl<'db> Lexer<'db> {
167160
let mut fallback: Option<usize> = None;
168161

169162
while self.current < self.source.len() {
170-
if self.source[self.current..].starts_with(delimiter) {
163+
let slice = &self.source[self.current..];
164+
if slice.starts_with(delimiter) {
171165
return Ok(self.source[offset..self.current].to_string());
172166
}
173167

174-
if fallback.is_none()
175-
&& (self.source[self.current..].starts_with(BLOCK_TAG_START)
176-
|| self.source[self.current..].starts_with(VARIABLE_TAG_START)
177-
|| self.source[self.current..].starts_with(COMMENT_TAG_START))
178-
{
168+
if fallback.is_none() && TagDelimiter::from_input(slice).is_some() {
179169
fallback = Some(self.current);
180170
}
181171

‎crates/djls-templates/src/nodelist.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use djls_source::Span;
22

33
use crate::db::Db as TemplateDb;
44
use crate::parser::ParseError;
5-
use crate::tokens::DJANGO_TAG_LEN;
5+
use crate::tokens::TagDelimiter;
66

77
#[salsa::tracked(debug)]
88
pub struct NodeList<'db> {
@@ -51,7 +51,7 @@ impl<'db> Node<'db> {
5151
pub fn full_span(&self) -> Span {
5252
match self {
5353
Node::Variable { span, .. } | Node::Comment { span, .. } | Node::Tag { span, .. } => {
54-
span.expand(DJANGO_TAG_LEN, DJANGO_TAG_LEN)
54+
span.expand(TagDelimiter::LENGTH_U32, TagDelimiter::LENGTH_U32)
5555
}
5656
Node::Text { span, .. } => *span,
5757
Node::Error { node } => node.full_span,

‎crates/djls-templates/src/tokens.rs‎

Lines changed: 42 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,42 @@ use djls_source::Span;
22

33
use crate::db::Db as TemplateDb;
44

5-
pub const BLOCK_TAG_START: &str = "{%";
6-
pub const BLOCK_TAG_END: &str = "%}";
7-
pub const VARIABLE_TAG_START: &str = "{{";
8-
pub const VARIABLE_TAG_END: &str = "}}";
9-
pub const COMMENT_TAG_START: &str = "{#";
10-
pub const COMMENT_TAG_END: &str = "#}";
5+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
6+
pub enum TagDelimiter {
7+
Block,
8+
Variable,
9+
Comment,
10+
}
11+
12+
impl TagDelimiter {
13+
pub const LENGTH: usize = 2;
14+
pub const LENGTH_U32: u32 = 2;
1115

12-
pub const DJANGO_TAG_LEN: u32 = 2;
16+
#[must_use]
17+
pub fn from_input(input: &str) -> Option<TagDelimiter> {
18+
[Self::Block, Self::Variable, Self::Comment]
19+
.into_iter()
20+
.find(|kind| input.starts_with(kind.opener()))
21+
}
22+
23+
#[must_use]
24+
pub fn opener(self) -> &'static str {
25+
match self {
26+
TagDelimiter::Block => "{%",
27+
TagDelimiter::Variable => "{{",
28+
TagDelimiter::Comment => "{#",
29+
}
30+
}
31+
32+
#[must_use]
33+
pub fn closer(self) -> &'static str {
34+
match self {
35+
TagDelimiter::Block => "%}",
36+
TagDelimiter::Variable => "}}",
37+
TagDelimiter::Comment => "#}",
38+
}
39+
}
40+
}
1341

1442
#[derive(Clone, Debug, PartialEq, Hash, salsa::Update)]
1543
pub enum Token<'db> {
@@ -98,7 +126,9 @@ impl<'db> Token<'db> {
98126
Token::Block { span, .. }
99127
| Token::Comment { span, .. }
100128
| Token::Error { span, .. }
101-
| Token::Variable { span, .. } => Some(span.start.saturating_sub(DJANGO_TAG_LEN)),
129+
| Token::Variable { span, .. } => {
130+
Some(span.start.saturating_sub(TagDelimiter::LENGTH_U32))
131+
}
102132
Token::Text { span, .. }
103133
| Token::Whitespace { span, .. }
104134
| Token::Newline { span, .. } => Some(span.start),
@@ -124,8 +154,10 @@ impl<'db> Token<'db> {
124154
match self {
125155
Token::Block { span, .. }
126156
| Token::Comment { span, .. }
127-
| Token::Variable { span, .. } => Some(span.expand(DJANGO_TAG_LEN, DJANGO_TAG_LEN)),
128-
Token::Error { span, .. } => Some(span.expand(DJANGO_TAG_LEN, 0)),
157+
| Token::Variable { span, .. } => {
158+
Some(span.expand(TagDelimiter::LENGTH_U32, TagDelimiter::LENGTH_U32))
159+
}
160+
Token::Error { span, .. } => Some(span.expand(TagDelimiter::LENGTH_U32, 0)),
129161
Token::Newline { span, .. }
130162
| Token::Text { span, .. }
131163
| Token::Whitespace { span, .. } => Some(*span),
@@ -240,15 +272,3 @@ pub struct TokenStream<'db> {
240272
#[returns(ref)]
241273
pub stream: Vec<Token<'db>>,
242274
}
243-
244-
impl<'db> TokenStream<'db> {
245-
/// Check if the token stream is empty
246-
pub fn is_empty(self, db: &'db dyn TemplateDb) -> bool {
247-
self.stream(db).is_empty()
248-
}
249-
250-
/// Get the number of tokens
251-
pub fn len(self, db: &'db dyn TemplateDb) -> usize {
252-
self.stream(db).len()
253-
}
254-
}

0 commit comments

Comments
 (0)