Skip to content

Commit 8978fc3

Browse files
committed
Add support for UTF-8 literals
1 parent e88ba3d commit 8978fc3

File tree

5 files changed

+24680
-24531
lines changed

5 files changed

+24680
-24531
lines changed

corpus/literals.txt

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,50 @@ class A {
314314
(identifier)
315315
(equals_value_clause (string_literal (escape_sequence)))))))))
316316

317+
=======================================
318+
utf-8 string literals
319+
=======================================
320+
321+
var a = "lower"u8;
322+
var b = "upper"U8;
323+
var c = @"lower"u8;
324+
var d = @"upper"U8;
325+
---
326+
327+
(compilation_unit
328+
(global_statement
329+
(local_declaration_statement
330+
(variable_declaration
331+
(implicit_type)
332+
(variable_declarator
333+
(identifier)
334+
(equals_value_clause
335+
(string_literal))))))
336+
(global_statement
337+
(local_declaration_statement
338+
(variable_declaration
339+
(implicit_type)
340+
(variable_declarator
341+
(identifier)
342+
(equals_value_clause
343+
(string_literal))))))
344+
(global_statement
345+
(local_declaration_statement
346+
(variable_declaration
347+
(implicit_type)
348+
(variable_declarator
349+
(identifier)
350+
(equals_value_clause
351+
(verbatim_string_literal))))))
352+
(global_statement
353+
(local_declaration_statement
354+
(variable_declaration
355+
(implicit_type)
356+
(variable_declarator
357+
(identifier)
358+
(equals_value_clause
359+
(verbatim_string_literal)))))))
360+
317361
==================================================
318362
string literals containing newline
319363
==================================================

grammar.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1644,7 +1644,7 @@ module.exports = grammar({
16441644
$._string_literal_fragment,
16451645
$.escape_sequence
16461646
)),
1647-
'"'
1647+
choice('"', '"U8', '"u8')
16481648
),
16491649

16501650
_string_literal_fragment: $ => token.immediate(prec(1, /[^"\\\n]+/)),
@@ -1655,7 +1655,7 @@ module.exports = grammar({
16551655
/[^"]/,
16561656
'""',
16571657
)),
1658-
'"'
1658+
choice('"', '"U8', '"u8')
16591659
)),
16601660

16611661
// Comments

src/grammar.json

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9214,8 +9214,21 @@
92149214
}
92159215
},
92169216
{
9217-
"type": "STRING",
9218-
"value": "\""
9217+
"type": "CHOICE",
9218+
"members": [
9219+
{
9220+
"type": "STRING",
9221+
"value": "\""
9222+
},
9223+
{
9224+
"type": "STRING",
9225+
"value": "\"U8"
9226+
},
9227+
{
9228+
"type": "STRING",
9229+
"value": "\"u8"
9230+
}
9231+
]
92199232
}
92209233
]
92219234
},
@@ -9256,8 +9269,21 @@
92569269
}
92579270
},
92589271
{
9259-
"type": "STRING",
9260-
"value": "\""
9272+
"type": "CHOICE",
9273+
"members": [
9274+
{
9275+
"type": "STRING",
9276+
"value": "\""
9277+
},
9278+
{
9279+
"type": "STRING",
9280+
"value": "\"U8"
9281+
},
9282+
{
9283+
"type": "STRING",
9284+
"value": "\"u8"
9285+
}
9286+
]
92619287
}
92629288
]
92639289
}

src/node-types.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6269,6 +6269,14 @@
62696269
"type": "\"\"",
62706270
"named": false
62716271
},
6272+
{
6273+
"type": "\"U8",
6274+
"named": false
6275+
},
6276+
{
6277+
"type": "\"u8",
6278+
"named": false
6279+
},
62726280
{
62736281
"type": "$\"",
62746282
"named": false

0 commit comments

Comments
 (0)