Skip to content

Commit 7a6c21c

Browse files
steve-ssteve-s
committed
[GR-24682] Parse f-strings eagerly and enable more test_fstring tests.
PullRequest: graalpython/1102
2 parents f8a8628 + f10c5ed commit 7a6c21c

File tree

26 files changed

+1570
-564
lines changed

26 files changed

+1570
-564
lines changed

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/nodes/literal/FormatStringTests.java

Lines changed: 64 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,21 @@
4141

4242
package com.oracle.graal.python.nodes.literal;
4343

44+
import java.util.ArrayList;
45+
4446
import org.junit.Assert;
4547
import org.junit.Test;
4648

47-
import com.oracle.graal.python.runtime.PythonParser;
49+
import com.oracle.graal.python.PythonLanguage;
50+
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
51+
import com.oracle.graal.python.parser.sst.FormatStringParser;
52+
import com.oracle.graal.python.parser.sst.FormatStringParser.Token;
53+
import com.oracle.graal.python.runtime.PythonParser.ErrorType;
54+
import com.oracle.graal.python.runtime.PythonParser.ParserErrorCallback;
4855
import com.oracle.graal.python.test.parser.ParserTestBase;
49-
import com.oracle.truffle.api.Truffle;
50-
import com.oracle.truffle.api.frame.FrameDescriptor;
51-
import com.oracle.truffle.api.frame.VirtualFrame;
5256
import com.oracle.truffle.api.nodes.Node;
57+
import com.oracle.truffle.api.source.Source;
58+
import com.oracle.truffle.api.source.SourceSection;
5359

5460
public class FormatStringTests extends ParserTestBase {
5561

@@ -238,6 +244,11 @@ public void parser05() throws Exception {
238244
testFormatString("f'It {name} was'", "It +format((name))+ was");
239245
}
240246

247+
@Test
248+
public void strWithColon() throws Exception {
249+
testFormatString("f'{myarray[:1]}'", "format((myarray[:1]))");
250+
}
251+
241252
@Test
242253
public void str01() throws Exception {
243254
testFormatString("f'{name!s}'", "format(str((name)))");
@@ -255,37 +266,37 @@ public void ascii01() throws Exception {
255266

256267
@Test
257268
public void emptyExpression01() throws Exception {
258-
checkSyntaxError("f'{}'", FormatStringLiteralNode.ERROR_MESSAGE_EMPTY_EXPRESSION);
269+
checkSyntaxError("f'{}'", FormatStringParser.ERROR_MESSAGE_EMPTY_EXPRESSION);
259270
}
260271

261272
@Test
262273
public void emptyExpression02() throws Exception {
263-
checkSyntaxError("f'start{}end'", FormatStringLiteralNode.ERROR_MESSAGE_EMPTY_EXPRESSION);
274+
checkSyntaxError("f'start{}end'", FormatStringParser.ERROR_MESSAGE_EMPTY_EXPRESSION);
264275
}
265276

266277
@Test
267278
public void emptyExpression03() throws Exception {
268-
checkSyntaxError("f'start{}}end'", FormatStringLiteralNode.ERROR_MESSAGE_EMPTY_EXPRESSION);
279+
checkSyntaxError("f'start{}}end'", FormatStringParser.ERROR_MESSAGE_EMPTY_EXPRESSION);
269280
}
270281

271282
@Test
272283
public void emptyExpression04() throws Exception {
273-
checkSyntaxError("f'start{{{}}}end'", FormatStringLiteralNode.ERROR_MESSAGE_EMPTY_EXPRESSION);
284+
checkSyntaxError("f'start{{{}}}end'", FormatStringParser.ERROR_MESSAGE_EMPTY_EXPRESSION);
274285
}
275286

276287
@Test
277288
public void singleBracket01() throws Exception {
278-
checkSyntaxError("f'}'", FormatStringLiteralNode.ERROR_MESSAGE_SINGLE_BRACE);
289+
checkSyntaxError("f'}'", FormatStringParser.ERROR_MESSAGE_SINGLE_BRACE);
279290
}
280291

281292
@Test
282293
public void singleBracket02() throws Exception {
283-
checkSyntaxError("f'start}end'", FormatStringLiteralNode.ERROR_MESSAGE_SINGLE_BRACE);
294+
checkSyntaxError("f'start}end'", FormatStringParser.ERROR_MESSAGE_SINGLE_BRACE);
284295
}
285296

286297
@Test
287298
public void singleBracket03() throws Exception {
288-
checkSyntaxError("f'start{{}end'", FormatStringLiteralNode.ERROR_MESSAGE_SINGLE_BRACE);
299+
checkSyntaxError("f'start{{}end'", FormatStringParser.ERROR_MESSAGE_SINGLE_BRACE);
289300
}
290301

291302
@Test
@@ -315,73 +326,91 @@ public void missingSpecifier02() throws Exception {
315326

316327
@Test
317328
public void missingExpression01() throws Exception {
318-
checkSyntaxError("f'{!x}'", FormatStringLiteralNode.ERROR_MESSAGE_EMPTY_EXPRESSION);
329+
checkSyntaxError("f'{!x}'", FormatStringParser.ERROR_MESSAGE_EMPTY_EXPRESSION);
319330
}
320331

321332
@Test
322333
public void missingExpression02() throws Exception {
323-
checkSyntaxError("f'{ !x}'", FormatStringLiteralNode.ERROR_MESSAGE_EMPTY_EXPRESSION);
334+
checkSyntaxError("f'{ !x}'", FormatStringParser.ERROR_MESSAGE_EMPTY_EXPRESSION);
324335
}
325336

326337
@Test
327338
public void missingExpression03() throws Exception {
328-
checkSyntaxError("f'{ !xr:a}'", FormatStringLiteralNode.ERROR_MESSAGE_EMPTY_EXPRESSION);
339+
checkSyntaxError("f'{ !xr:a}'", FormatStringParser.ERROR_MESSAGE_EMPTY_EXPRESSION);
329340
}
330341

331342
@Test
332343
public void missingExpression04() throws Exception {
333-
checkSyntaxError("f'{:x'", FormatStringLiteralNode.ERROR_MESSAGE_EMPTY_EXPRESSION);
344+
checkSyntaxError("f'{:x'", FormatStringParser.ERROR_MESSAGE_EMPTY_EXPRESSION);
334345
}
335346

336347
@Test
337348
public void missingExpression05() throws Exception {
338-
checkSyntaxError("f'{!'", FormatStringLiteralNode.ERROR_MESSAGE_EMPTY_EXPRESSION);
349+
checkSyntaxError("f'{!'", FormatStringParser.ERROR_MESSAGE_EMPTY_EXPRESSION);
339350
}
340351

341352
@Test
342353
public void missingExpression06() throws Exception {
343-
checkSyntaxError("f'{10:{ }}'", FormatStringLiteralNode.ERROR_MESSAGE_EMPTY_EXPRESSION);
354+
checkSyntaxError("f'{10:{ }}'", FormatStringParser.ERROR_MESSAGE_EMPTY_EXPRESSION);
344355
}
345356

346-
private void checkSyntaxError(String text, String expectedMessage) throws Exception {
357+
private static void checkSyntaxError(String text, String expectedMessage) throws Exception {
347358
try {
348359
testFormatString(text, "Expected Error: " + expectedMessage);
349360
} catch (RuntimeException e) {
350361
Assert.assertEquals("SyntaxError: " + expectedMessage, e.getMessage());
351362
}
352363
}
353364

354-
private void testFormatString(String text, String expected) throws Exception {
355-
VirtualFrame frame = Truffle.getRuntime().createVirtualFrame(new Object[8], new FrameDescriptor());
356-
357-
Node parserResult = parse(text, "<fstringtest>", PythonParser.ParserMode.InlineEvaluation, frame);
358-
359-
Assert.assertTrue("The source has to be just fstring", parserResult instanceof FormatStringLiteralNode);
360-
FormatStringLiteralNode fsl = (FormatStringLiteralNode) parserResult;
361-
int[][] tokens = FormatStringLiteralNode.createTokens(fsl, fsl.getValues());
362-
FormatStringLiteralNode.StringPart[] fslParts = fsl.getValues();
363-
String[] expressions = FormatStringLiteralNode.createExpressionSources(fslParts, tokens, 0, tokens.length);
365+
private static void testFormatString(String fstring, String expected) throws Exception {
366+
assert fstring.startsWith("f'") && fstring.endsWith("'");
367+
// remove the f'...', to extract the text of the f-string
368+
String text = fstring.substring(2).substring(0, fstring.length() - 3);
369+
ArrayList<Token> tokens = new ArrayList<>();
370+
FormatStringParser.createTokens(tokens, new MockErrorCallback(), 0, text, 0);
371+
ArrayList<String> expressions = FormatStringParser.createExpressionSources(text, tokens, 0, tokens.size(), tokens.size());
364372
int expressionsIndex = 0;
365373
StringBuilder actual = new StringBuilder();
366374
boolean first = true;
367375
boolean wasLastString = true;
368-
for (int index = 0; index < tokens.length; index++) {
369-
int[] token = tokens[index];
376+
for (int index = 0; index < tokens.size(); index++) {
377+
Token token = tokens.get(index);
370378
if (first) {
371379
first = false;
372-
} else if (!(wasLastString && token[0] == FormatStringLiteralNode.TOKEN_TYPE_STRING)) {
380+
} else if (!(wasLastString && token.type == FormatStringParser.TOKEN_TYPE_STRING)) {
373381
actual.append("+");
374382
}
375-
if (token[0] == FormatStringLiteralNode.TOKEN_TYPE_STRING) {
376-
actual.append(fslParts[token[1]].getText().substring(token[2], token[3]));
383+
if (token.type == FormatStringParser.TOKEN_TYPE_STRING) {
384+
actual.append(text, token.startIndex, token.endIndex);
377385
wasLastString = true;
378386
} else {
379-
actual.append(expressions[expressionsIndex]);
380-
index += token[4];
387+
actual.append(expressions.get(expressionsIndex));
388+
index += token.formatTokensCount;
381389
wasLastString = false;
382390
}
383391
}
384392
Assert.assertEquals(expected, actual.toString());
385393
}
386394

395+
private static final class MockErrorCallback implements ParserErrorCallback {
396+
@Override
397+
public RuntimeException raise(PythonBuiltinClassType type, String message, Object... args) {
398+
throw new RuntimeException("SyntaxError: " + String.format(message, args));
399+
}
400+
401+
@Override
402+
public RuntimeException raiseInvalidSyntax(ErrorType type, Source source, SourceSection section, String message, Object... arguments) {
403+
throw new RuntimeException("SyntaxError: " + String.format(message, arguments));
404+
}
405+
406+
@Override
407+
public RuntimeException raiseInvalidSyntax(ErrorType type, Node location, String message, Object... arguments) {
408+
throw new RuntimeException("SyntaxError: " + String.format(message, arguments));
409+
}
410+
411+
@Override
412+
public PythonLanguage getLanguage() {
413+
return null;
414+
}
415+
}
387416
}

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/ParserTreePrinter.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -660,9 +660,9 @@ public boolean visit(Node node) {
660660
indent(level);
661661
sb.append("Values: ");
662662

663-
String[] values = new String[((FormatStringLiteralNode) node).getValues().length];
663+
String[] values = new String[((FormatStringLiteralNode) node).getParts().length];
664664
int index = 0;
665-
for (FormatStringLiteralNode.StringPart part : ((FormatStringLiteralNode) node).getValues()) {
665+
for (FormatStringLiteralNode.StringPart part : ((FormatStringLiteralNode) node).getParts()) {
666666
values[index++] = part.isFormatString() ? "<f>" + part.getText() : "<n>" + part.getText();
667667
}
668668
add(values);

‎graalpython/com.oracle.graal.python.test/src/tests/test_formatting.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,3 +185,24 @@ def __str__(self):
185185
assert "{0:10}".format(MyInt(2)) == " 2"
186186
assert format(MyInt(2), "") == "42"
187187
assert format(MyInt(2), "5") == " 2"
188+
189+
190+
def test_fstring():
191+
class CustomFormat:
192+
def __format__(self, format_spec):
193+
return format_spec
194+
195+
x = CustomFormat()
196+
assert f'{x:={1=}}' == "=1=1"
197+
# blank spaces after '=' are fine
198+
assert f'{42= :<10}' == '42= 42 '
199+
# curly braces in expressions
200+
assert f'{len({})}' == '0'
201+
assert f'{10:#{(len({1,2,3,4,5}))}}' == ' 10'
202+
# square brackets in expressions
203+
aligns = ['<', '>']
204+
align = 0
205+
assert f'{3:{aligns[align]}{5}}' == '3 '
206+
# this is not walrus but 'x' with a format specifier "=10"
207+
x = 20
208+
assert f'{x:=10}' == ' 20'

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_fstring.txt

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,48 @@
11
*graalpython.lib-python.3.test.test_fstring.TestCase.test__format__lookup
2+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_arguments
23
*graalpython.lib-python.3.test.test_fstring.TestCase.test_assignment
4+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_ast
35
*graalpython.lib-python.3.test.test_fstring.TestCase.test_ast_compile_time_concat
6+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_ast_line_numbers
7+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_ast_line_numbers_duplicate_expression
8+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_ast_line_numbers_multiline_fstring
9+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_ast_line_numbers_multiple_formattedvalues
10+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_ast_line_numbers_nested
411
*graalpython.lib-python.3.test.test_fstring.TestCase.test_backslash_char
512
*graalpython.lib-python.3.test.test_fstring.TestCase.test_call
13+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_closure
614
*graalpython.lib-python.3.test.test_fstring.TestCase.test_comments
715
*graalpython.lib-python.3.test.test_fstring.TestCase.test_compile_time_concat
816
*graalpython.lib-python.3.test.test_fstring.TestCase.test_compile_time_concat_errors
17+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_conversions
918
*graalpython.lib-python.3.test.test_fstring.TestCase.test_del
1019
*graalpython.lib-python.3.test.test_fstring.TestCase.test_dict
1120
*graalpython.lib-python.3.test.test_fstring.TestCase.test_docstring
1221
*graalpython.lib-python.3.test.test_fstring.TestCase.test_double_braces
1322
*graalpython.lib-python.3.test.test_fstring.TestCase.test_empty_format_specifier
1423
*graalpython.lib-python.3.test.test_fstring.TestCase.test_equal_equal
24+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_expressions_with_triple_quoted_strings
1525
*graalpython.lib-python.3.test.test_fstring.TestCase.test_if_conditional
26+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_invalid_string_prefixes
1627
*graalpython.lib-python.3.test.test_fstring.TestCase.test_leading_trailing_spaces
1728
*graalpython.lib-python.3.test.test_fstring.TestCase.test_literal
1829
*graalpython.lib-python.3.test.test_fstring.TestCase.test_literal_eval
1930
*graalpython.lib-python.3.test.test_fstring.TestCase.test_locals
2031
*graalpython.lib-python.3.test.test_fstring.TestCase.test_loop
32+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_many_expressions
33+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_mismatched_braces
2134
*graalpython.lib-python.3.test.test_fstring.TestCase.test_mismatched_parens
35+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_missing_expression
2236
*graalpython.lib-python.3.test.test_fstring.TestCase.test_missing_format_spec
2337
*graalpython.lib-python.3.test.test_fstring.TestCase.test_missing_variable
2438
*graalpython.lib-python.3.test.test_fstring.TestCase.test_multiple_vars
2539
*graalpython.lib-python.3.test.test_fstring.TestCase.test_nested_fstrings
40+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_newlines_in_expressions
41+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_no_escapes_for_braces
42+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_not_equal
2643
*graalpython.lib-python.3.test.test_fstring.TestCase.test_parens_in_expressions
2744
*graalpython.lib-python.3.test.test_fstring.TestCase.test_shadowed_global
2845
*graalpython.lib-python.3.test.test_fstring.TestCase.test_side_effect_order
2946
*graalpython.lib-python.3.test.test_fstring.TestCase.test_str_format_differences
3047
*graalpython.lib-python.3.test.test_fstring.TestCase.test_unterminated_string
48+
*graalpython.lib-python.3.test.test_fstring.TestCase.test_yield

‎graalpython/com.oracle.graal.python.test/testData/goldenFiles/FStringTests/topLevelParser01.tast

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,16 @@ ModuleRootNode Name: <module 'topLevelParser01'> SourceSection: [0,36]`name = 'P
1515
CachedDispatchFirst SourceSection: None
1616
FormatStringLiteralNode SourceSection: [20,35]`f"hello {name}"`
1717
Values: <f>hello {name}
18+
PythonCallUnary SourceSection: [0,14]`format((name))`
19+
CallUnaryMethodNodeGen SourceSection: None
20+
ReadNameNodeGen SourceSection: [0,6]`format`
21+
Identifier: format
22+
IsBuiltinClassProfile SourceSection: None
23+
CachedDispatchFirst SourceSection: None
24+
ReadNameNodeGen SourceSection: [8,12]`name`
25+
Identifier: name
26+
IsBuiltinClassProfile SourceSection: None
27+
CachedDispatchFirst SourceSection: None
1828
SideEffect:
1929
WriteNameNodeGen SourceSection: [0,13]`name = 'Pepa'`
2030
Identifier: name

‎graalpython/com.oracle.graal.python.test/testData/goldenFiles/FStringTests/topLevelParser02.tast

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,14 @@ ModuleRootNode Name: <module 'topLevelParser02'> SourceSection: [0,48]`''.join(f
3737
flagSlot: 0
3838
FormatStringLiteralNode SourceSection: [8,17]`f'{name}'`
3939
Values: <f>{name}
40+
PythonCallUnary SourceSection: [0,14]`format((name))`
41+
CallUnaryMethodNodeGen SourceSection: None
42+
ReadGlobalOrBuiltinNodeGen SourceSection: [0,6]`format`
43+
Identifier: format
44+
ReadAttributeFromObjectNotTypeNodeGen SourceSection: None
45+
ReadGeneratorFrameVariableNode SourceSection: [8,12]`name`
46+
Frame: [0,name,Illegal]
47+
ReadVariableFromFrameNodeGen SourceSection: None
4048
GeneratorAccessNode SourceSection: None
4149
WriteGeneratorFrameVariableNodeGen SourceSection: None
4250
Identifier: name

‎graalpython/com.oracle.graal.python.test/testData/goldenFiles/GeneratorAndCompForTests/fstring01.tast

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,14 @@ ModuleRootNode Name: <module 'fstring01'> SourceSection: [0,64]`def fn(someset):
7777
YieldNode SourceSection: [34,43]`f'{name}'`
7878
flagSlot: 0
7979
FormatStringLiteralNode SourceSection: [34,43]`f'{name}'`
80+
PythonCallUnary SourceSection: [0,14]`format((name))`
81+
CallUnaryMethodNodeGen SourceSection: None
82+
ReadGlobalOrBuiltinNodeGen SourceSection: [0,6]`format`
83+
Identifier: format
84+
ReadAttributeFromObjectNotTypeNodeGen SourceSection: None
85+
ReadGeneratorFrameVariableNode SourceSection: [8,12]`name`
86+
Frame: [0,name,Illegal]
87+
ReadVariableFromFrameNodeGen SourceSection: None
8088
GeneratorAccessNode SourceSection: None
8189
WriteGeneratorFrameVariableNodeGen SourceSection: None
8290
Identifier: name

0 commit comments

Comments
 (0)