Skip to content

Commit a078faa

Browse files
joshuadavidthomasjoshuadavidthomas
committed
fix
1 parent b10df9b commit a078faa

14 files changed

+200
-204
lines changed

‎crates/djls-template-ast/src/ast.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ impl From<Token> for Span {
9797
let token_start = token.start().unwrap_or(0);
9898
match token.token_type() {
9999
TokenType::Comment(_, start, _) => token_start + start.len() as u32,
100-
TokenType::DjangoBlock(_) | TokenType::DjangoVariable(_) => token_start + 2,
100+
TokenType::DjangoBlock(_) | TokenType::DjangoVariable(_) => token_start + 3,
101101
_ => token_start,
102102
}
103103
};

‎crates/djls-template-ast/src/lexer.rs‎

Lines changed: 109 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -22,138 +22,134 @@ impl Lexer {
2222

2323
pub fn tokenize(&mut self) -> Result<TokenStream, LexerError> {
2424
let mut tokens = TokenStream::default();
25+
2526
while !self.is_at_end() {
26-
let token = self.next_token()?;
27-
tokens.add_token(token);
28-
}
29-
tokens.finalize(self.line);
30-
Ok(tokens)
31-
}
27+
self.start = self.current;
28+
29+
let token_type = match self.peek()? {
30+
'{' => match self.peek_next()? {
31+
'%' => {
32+
self.consume_n(2)?; // {%
33+
let content = self.consume_until("%}")?;
34+
self.consume_n(2)?; // %}
35+
TokenType::DjangoBlock(content)
36+
}
37+
'{' => {
38+
self.consume_n(2)?; // {{
39+
let content = self.consume_until("}}")?;
40+
self.consume_n(2)?; // }}
41+
TokenType::DjangoVariable(content)
42+
}
43+
'#' => {
44+
self.consume_n(2)?; // {#
45+
let content = self.consume_until("#}")?;
46+
self.consume_n(2)?; // #}
47+
TokenType::Comment(content, "{#".to_string(), Some("#}".to_string()))
48+
}
49+
_ => {
50+
self.consume()?; // {
51+
TokenType::Text(String::from("{"))
52+
}
53+
},
54+
55+
'<' => match self.peek_next()? {
56+
'/' => {
57+
self.consume_n(2)?; // </
58+
let tag = self.consume_until(">")?;
59+
self.consume()?; // >
60+
TokenType::HtmlTagClose(tag)
61+
}
62+
'!' if self.matches("<!--")? => {
63+
self.consume_n(4)?; // <!--
64+
let content = self.consume_until("-->")?;
65+
self.consume_n(3)?; // -->
66+
TokenType::Comment(content, "<!--".to_string(), Some("-->".to_string()))
67+
}
68+
_ => {
69+
self.consume()?; // consume <
70+
let tag = self.consume_until(">")?;
71+
self.consume()?; // consume >
72+
if tag.starts_with("script") {
73+
TokenType::ScriptTagOpen(tag)
74+
} else if tag.starts_with("style") {
75+
TokenType::StyleTagOpen(tag)
76+
} else if tag.ends_with("/") {
77+
TokenType::HtmlTagVoid(tag.trim_end_matches("/").to_string())
78+
} else {
79+
TokenType::HtmlTagOpen(tag)
80+
}
81+
}
82+
},
3283

33-
fn next_token(&mut self) -> Result<Token, LexerError> {
34-
self.start = self.current;
84+
'/' => match self.peek_next()? {
85+
'/' => {
86+
self.consume_n(2)?; // //
87+
let content = self.consume_until("\n")?;
88+
TokenType::Comment(content, "//".to_string(), None)
89+
}
90+
'*' => {
91+
self.consume_n(2)?; // /*
92+
let content = self.consume_until("*/")?;
93+
self.consume_n(2)?; // */
94+
TokenType::Comment(content, "/*".to_string(), Some("*/".to_string()))
95+
}
96+
_ => {
97+
self.consume()?;
98+
TokenType::Text("/".to_string())
99+
}
100+
},
35101

36-
let token_type = match self.peek()? {
37-
'{' => match self.peek_next()? {
38-
'%' => {
39-
self.consume_n(2)?; // {%
40-
let content = self.consume_until("%}")?;
41-
self.consume_n(2)?; // %}
42-
TokenType::DjangoBlock(content)
43-
}
44-
'{' => {
45-
self.consume_n(2)?; // {{
46-
let content = self.consume_until("}}")?;
47-
self.consume_n(2)?; // }}
48-
TokenType::DjangoVariable(content)
49-
}
50-
'#' => {
51-
self.consume_n(2)?; // {#
52-
let content = self.consume_until("#}")?;
53-
self.consume_n(2)?; // #}
54-
TokenType::Comment(content, "{#".to_string(), Some("#}".to_string()))
55-
}
56-
_ => {
57-
self.consume()?; // {
58-
TokenType::Text(String::from("{"))
59-
}
60-
},
61-
62-
'<' => match self.peek_next()? {
63-
'/' => {
64-
self.consume_n(2)?; // </
65-
let tag = self.consume_until(">")?;
66-
self.consume()?; // >
67-
TokenType::HtmlTagClose(tag)
68-
}
69-
'!' if self.matches("<!--")? => {
70-
self.consume_n(4)?; // <!--
71-
let content = self.consume_until("-->")?;
72-
self.consume_n(3)?; // -->
73-
TokenType::Comment(content, "<!--".to_string(), Some("-->".to_string()))
74-
}
75-
_ => {
76-
self.consume()?; // consume <
77-
let tag = self.consume_until(">")?;
78-
self.consume()?; // consume >
79-
if tag.starts_with("script") {
80-
TokenType::ScriptTagOpen(tag)
81-
} else if tag.starts_with("style") {
82-
TokenType::StyleTagOpen(tag)
83-
} else if tag.ends_with("/") {
84-
TokenType::HtmlTagVoid(tag.trim_end_matches("/").to_string())
102+
c if c.is_whitespace() => {
103+
if c == '\n' || c == '\r' {
104+
self.consume()?; // \r or \n
105+
if c == '\r' && self.peek()? == '\n' {
106+
self.consume()?; // \n of \r\n
107+
}
108+
TokenType::Newline
85109
} else {
86-
TokenType::HtmlTagOpen(tag)
110+
self.consume()?; // Consume the first whitespace
111+
while !self.is_at_end() && self.peek()?.is_whitespace() {
112+
if self.peek()? == '\n' || self.peek()? == '\r' {
113+
break;
114+
}
115+
self.consume()?;
116+
}
117+
let whitespace_count = self.current - self.start;
118+
TokenType::Whitespace(whitespace_count)
87119
}
88120
}
89-
},
90121

91-
'/' => match self.peek_next()? {
92-
'/' => {
93-
self.consume_n(2)?; // //
94-
let content = self.consume_until("\n")?;
95-
TokenType::Comment(content, "//".to_string(), None)
96-
}
97-
'*' => {
98-
self.consume_n(2)?; // /*
99-
let content = self.consume_until("*/")?;
100-
self.consume_n(2)?; // */
101-
TokenType::Comment(content, "/*".to_string(), Some("*/".to_string()))
102-
}
103122
_ => {
104-
self.consume()?;
105-
TokenType::Text("/".to_string())
106-
}
107-
},
108-
109-
c if c.is_whitespace() => {
110-
if c == '\n' || c == '\r' {
111-
self.consume()?; // \r or \n
112-
if c == '\r' && self.peek()? == '\n' {
113-
self.consume()?; // \n of \r\n
114-
}
115-
TokenType::Newline
116-
} else {
117-
self.consume()?; // Consume the first whitespace
118-
while !self.is_at_end() && self.peek()?.is_whitespace() {
119-
if self.peek()? == '\n' || self.peek()? == '\r' {
123+
let mut text = String::new();
124+
while !self.is_at_end() {
125+
let c = self.peek()?;
126+
if c == '{' || c == '<' || c == '\n' {
120127
break;
121128
}
129+
text.push(c);
122130
self.consume()?;
123131
}
124-
let whitespace_count = self.current - self.start;
125-
TokenType::Whitespace(whitespace_count)
132+
TokenType::Text(text)
126133
}
127-
}
134+
};
135+
136+
let token = Token::new(token_type, self.line, Some(self.start));
128137

129-
_ => {
130-
let mut text = String::new();
131-
while !self.is_at_end() {
132-
let c = self.peek()?;
133-
if c == '{' || c == '<' || c == '\n' {
134-
break;
138+
match self.peek_previous()? {
139+
'\n' => self.line += 1,
140+
'\r' => {
141+
self.line += 1;
142+
if self.peek()? == '\n' {
143+
self.current += 1;
135144
}
136-
text.push(c);
137-
self.consume()?;
138145
}
139-
TokenType::Text(text)
146+
_ => {}
140147
}
141-
};
142148

143-
let token = Token::new(token_type, self.line, Some(self.start));
144-
145-
match self.peek_previous()? {
146-
'\n' => self.line += 1,
147-
'\r' => {
148-
self.line += 1;
149-
if self.peek()? == '\n' {
150-
self.current += 1;
151-
}
152-
}
153-
_ => {}
149+
tokens.add_token(token);
154150
}
155-
156-
Ok(token)
151+
tokens.finalize(self.line);
152+
Ok(tokens)
157153
}
158154

159155
fn peek(&self) -> Result<char, LexerError> {

‎crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ nodes:
1111
- Comment:
1212
content: Django comment
1313
span:
14-
start: 21
14+
start: 23
1515
length: 14
1616
line_offsets:
1717
- 0

‎crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ nodes:
1313
- ">"
1414
- "0"
1515
span:
16-
start: 0
16+
start: 3
1717
length: 8
1818
tag_span:
19-
start: 0
19+
start: 3
2020
length: 8
2121
assignment: ~
2222
nodes:
@@ -35,10 +35,10 @@ nodes:
3535
- "<"
3636
- "0"
3737
span:
38-
start: 22
38+
start: 25
3939
length: 10
4040
tag_span:
41-
start: 22
41+
start: 25
4242
length: 10
4343
assignment: ~
4444
nodes:
@@ -54,10 +54,10 @@ nodes:
5454
bits:
5555
- else
5656
span:
57-
start: 46
57+
start: 49
5858
length: 4
5959
tag_span:
60-
start: 46
60+
start: 49
6161
length: 4
6262
assignment: ~
6363
nodes:
@@ -73,10 +73,10 @@ nodes:
7373
bits:
7474
- endif
7575
span:
76-
start: 60
76+
start: 63
7777
length: 5
7878
tag_span:
79-
start: 60
79+
start: 63
8080
length: 5
8181
assignment: ~
8282
assignments: ~

‎crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ nodes:
1313
- in
1414
- items
1515
span:
16-
start: 0
16+
start: 3
1717
length: 17
1818
tag_span:
19-
start: 0
19+
start: 3
2020
length: 17
2121
assignment: ~
2222
nodes:
@@ -34,10 +34,10 @@ nodes:
3434
bits:
3535
- empty
3636
span:
37-
start: 33
37+
start: 36
3838
length: 5
3939
tag_span:
40-
start: 33
40+
start: 36
4141
length: 5
4242
assignment: ~
4343
nodes:
@@ -53,10 +53,10 @@ nodes:
5353
bits:
5454
- endfor
5555
span:
56-
start: 52
56+
start: 55
5757
length: 6
5858
tag_span:
59-
start: 52
59+
start: 55
6060
length: 6
6161
assignment: ~
6262
assignments: ~

‎crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@ nodes:
1111
- if
1212
- user.is_authenticated
1313
span:
14-
start: 0
14+
start: 3
1515
length: 24
1616
tag_span:
17-
start: 0
17+
start: 3
1818
length: 24
1919
assignment: ~
2020
nodes:
@@ -30,10 +30,10 @@ nodes:
3030
bits:
3131
- endif
3232
span:
33-
start: 37
33+
start: 40
3434
length: 5
3535
tag_span:
36-
start: 37
36+
start: 40
3737
length: 5
3838
assignment: ~
3939
assignments: ~

‎crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_variable_with_filter.snap‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ nodes:
1111
- name: title
1212
args: []
1313
span:
14-
start: 4
15-
length: 15
14+
start: 10
15+
length: 5
1616
span:
1717
start: 3
1818
length: 15

0 commit comments

Comments
 (0)