Skip to content

Commit 29f57d9

Browse files
lukasstadlerlukasstadler
committed
do more of the number/float/string literal parsing during the source parsing step.
1 parent 4ec3302 commit 29f57d9

14 files changed

+486
-286
lines changed

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/literal/FormatStringLiteralNode.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public class FormatStringLiteralNode extends LiteralNode {
7272
protected static final int TOKEN_TYPE_EXPRESSION_REPR = 4;
7373
protected static final int TOKEN_TYPE_EXPRESSION_ASCII = 5;
7474

75-
public static class StringPart {
75+
public static final class StringPart {
7676
/**
7777
* Marks, whether the value is formatted string
7878
*/

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/PythonSSTNodeFactory.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
import com.oracle.graal.python.parser.sst.SSTNode;
7171
import com.oracle.graal.python.parser.sst.SimpleSSTNode;
7272
import com.oracle.graal.python.parser.sst.StarSSTNode;
73+
import com.oracle.graal.python.parser.sst.StringLiteralSSTNode;
7374
import com.oracle.graal.python.parser.sst.StringUtils;
7475
import com.oracle.graal.python.parser.sst.VarLookupSSTNode;
7576
import com.oracle.graal.python.parser.sst.WithSSTNode;
@@ -246,6 +247,10 @@ public YieldExpressionSSTNode createYieldExpressionSSTNode(SSTNode value, boolea
246247
return new YieldExpressionSSTNode(value, isFrom, startOffset, endOffset);
247248
}
248249

250+
public StringLiteralSSTNode createStringLiteral(String[] values, int startOffset, int endOffset) {
251+
return StringLiteralSSTNode.create(values, startOffset, endOffset, source, errors);
252+
}
253+
249254
public Node createParserResult(SSTNode parserSSTResult, PythonParser.ParserMode mode, Frame currentFrame) {
250255
Node result;
251256
boolean isGen = false;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3.g4

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1222,7 +1222,7 @@ factor returns [SSTNode result]
12221222
// solving cases like --2
12231223
$result = new UnarySSTNode(UnaryArithmetic.Neg, fResult, getStartIndex($ctx), getStopIndex($factor.stop));
12241224
} else {
1225-
((NumberLiteralSSTNode)fResult).setIsNegative(true);
1225+
((NumberLiteralSSTNode)fResult).negate();
12261226
fResult.setStartOffset($m.getStartIndex());
12271227
$result = fResult;
12281228
}
@@ -1318,34 +1318,34 @@ atom returns [SSTNode result]
13181318
| DECIMAL_INTEGER
13191319
{
13201320
String text = $DECIMAL_INTEGER.text;
1321-
$result = text != null ? new NumberLiteralSSTNode(text, 0, 10, $DECIMAL_INTEGER.getStartIndex(), $DECIMAL_INTEGER.getStopIndex() + 1) : null;
1321+
$result = text != null ? NumberLiteralSSTNode.create(text, 0, 10, $DECIMAL_INTEGER.getStartIndex(), $DECIMAL_INTEGER.getStopIndex() + 1) : null;
13221322
}
13231323
| OCT_INTEGER
13241324
{
13251325
String text = $OCT_INTEGER.text;
1326-
$result = text != null ? new NumberLiteralSSTNode(text, 2, 8, $OCT_INTEGER.getStartIndex(), $OCT_INTEGER.getStopIndex() + 1) : null;
1326+
$result = text != null ? NumberLiteralSSTNode.create(text, 2, 8, $OCT_INTEGER.getStartIndex(), $OCT_INTEGER.getStopIndex() + 1) : null;
13271327
}
13281328
| HEX_INTEGER
13291329
{
13301330
String text = $HEX_INTEGER.text;
1331-
$result = text != null ? new NumberLiteralSSTNode(text, 2, 16, $HEX_INTEGER.getStartIndex(), $HEX_INTEGER.getStopIndex() + 1) : null;
1331+
$result = text != null ? NumberLiteralSSTNode.create(text, 2, 16, $HEX_INTEGER.getStartIndex(), $HEX_INTEGER.getStopIndex() + 1) : null;
13321332
}
13331333
| BIN_INTEGER
13341334
{
13351335
String text = $BIN_INTEGER.text;
1336-
$result = text != null ? new NumberLiteralSSTNode(text, 2, 2, $BIN_INTEGER.getStartIndex(), $BIN_INTEGER.getStopIndex() + 1) : null;
1336+
$result = text != null ? NumberLiteralSSTNode.create(text, 2, 2, $BIN_INTEGER.getStartIndex(), $BIN_INTEGER.getStopIndex() + 1) : null;
13371337
}
13381338
| FLOAT_NUMBER
13391339
{
13401340
String text = $FLOAT_NUMBER.text;
1341-
$result = text != null ? new FloatLiteralSSTNode(text, false, $FLOAT_NUMBER.getStartIndex(), $FLOAT_NUMBER.getStopIndex() + 1) : null;
1341+
$result = text != null ? FloatLiteralSSTNode.create(text, false, $FLOAT_NUMBER.getStartIndex(), $FLOAT_NUMBER.getStopIndex() + 1) : null;
13421342
}
13431343
| IMAG_NUMBER
13441344
{
13451345
String text = $IMAG_NUMBER.text;
1346-
$result = text != null ? new FloatLiteralSSTNode(text, true, $IMAG_NUMBER.getStartIndex(), $IMAG_NUMBER.getStopIndex() + 1) : null;
1346+
$result = text != null ? FloatLiteralSSTNode.create(text, true, $IMAG_NUMBER.getStartIndex(), $IMAG_NUMBER.getStopIndex() + 1) : null;
13471347
}
1348-
| { int start = stringStart(); } ( STRING { pushString($STRING.text); } )+ { $result = new StringLiteralSSTNode(getStringArray(start), getStartIndex($ctx), getStopIndex($STRING)); }
1348+
| { int start = stringStart(); } ( STRING { pushString($STRING.text); } )+ { $result = factory.createStringLiteral(getStringArray(start), getStartIndex($ctx), getStopIndex($STRING)); }
13491349
| t='...' { int start = $t.getStartIndex(); $result = new SimpleSSTNode(SimpleSSTNode.Type.ELLIPSIS, start, start + 3);}
13501350
| t='None' { int start = $t.getStartIndex(); $result = new SimpleSSTNode(SimpleSSTNode.Type.NONE, start, start + 4);}
13511351
| t='True' { int start = $t.getStartIndex(); $result = new BooleanLiteralSSTNode(true, start, start + 4); }

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3Parser.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6307,7 +6307,7 @@ public final FactorContext factor() throws RecognitionException {
63076307
// solving cases like --2
63086308
_localctx.result = new UnarySSTNode(UnaryArithmetic.Neg, fResult, getStartIndex(_localctx), getStopIndex((_localctx.factor!=null?(_localctx.factor.stop):null)));
63096309
} else {
6310-
((NumberLiteralSSTNode)fResult).setIsNegative(true);
6310+
((NumberLiteralSSTNode)fResult).negate();
63116311
fResult.setStartOffset(_localctx.m.getStartIndex());
63126312
_localctx.result = fResult;
63136313
}
@@ -6777,7 +6777,7 @@ public final AtomContext atom() throws RecognitionException {
67776777
_localctx.DECIMAL_INTEGER = match(DECIMAL_INTEGER);
67786778

67796779
String text = (_localctx.DECIMAL_INTEGER!=null?_localctx.DECIMAL_INTEGER.getText():null);
6780-
_localctx.result = text != null ? new NumberLiteralSSTNode(text, 0, 10, _localctx.DECIMAL_INTEGER.getStartIndex(), _localctx.DECIMAL_INTEGER.getStopIndex() + 1) : null;
6780+
_localctx.result = text != null ? NumberLiteralSSTNode.create(text, 0, 10, _localctx.DECIMAL_INTEGER.getStartIndex(), _localctx.DECIMAL_INTEGER.getStopIndex() + 1) : null;
67816781

67826782
}
67836783
break;
@@ -6788,7 +6788,7 @@ public final AtomContext atom() throws RecognitionException {
67886788
_localctx.OCT_INTEGER = match(OCT_INTEGER);
67896789

67906790
String text = (_localctx.OCT_INTEGER!=null?_localctx.OCT_INTEGER.getText():null);
6791-
_localctx.result = text != null ? new NumberLiteralSSTNode(text, 2, 8, _localctx.OCT_INTEGER.getStartIndex(), _localctx.OCT_INTEGER.getStopIndex() + 1) : null;
6791+
_localctx.result = text != null ? NumberLiteralSSTNode.create(text, 2, 8, _localctx.OCT_INTEGER.getStartIndex(), _localctx.OCT_INTEGER.getStopIndex() + 1) : null;
67926792

67936793
}
67946794
break;
@@ -6799,7 +6799,7 @@ public final AtomContext atom() throws RecognitionException {
67996799
_localctx.HEX_INTEGER = match(HEX_INTEGER);
68006800

68016801
String text = (_localctx.HEX_INTEGER!=null?_localctx.HEX_INTEGER.getText():null);
6802-
_localctx.result = text != null ? new NumberLiteralSSTNode(text, 2, 16, _localctx.HEX_INTEGER.getStartIndex(), _localctx.HEX_INTEGER.getStopIndex() + 1) : null;
6802+
_localctx.result = text != null ? NumberLiteralSSTNode.create(text, 2, 16, _localctx.HEX_INTEGER.getStartIndex(), _localctx.HEX_INTEGER.getStopIndex() + 1) : null;
68036803

68046804
}
68056805
break;
@@ -6810,7 +6810,7 @@ public final AtomContext atom() throws RecognitionException {
68106810
_localctx.BIN_INTEGER = match(BIN_INTEGER);
68116811

68126812
String text = (_localctx.BIN_INTEGER!=null?_localctx.BIN_INTEGER.getText():null);
6813-
_localctx.result = text != null ? new NumberLiteralSSTNode(text, 2, 2, _localctx.BIN_INTEGER.getStartIndex(), _localctx.BIN_INTEGER.getStopIndex() + 1) : null;
6813+
_localctx.result = text != null ? NumberLiteralSSTNode.create(text, 2, 2, _localctx.BIN_INTEGER.getStartIndex(), _localctx.BIN_INTEGER.getStopIndex() + 1) : null;
68146814

68156815
}
68166816
break;
@@ -6821,7 +6821,7 @@ public final AtomContext atom() throws RecognitionException {
68216821
_localctx.FLOAT_NUMBER = match(FLOAT_NUMBER);
68226822

68236823
String text = (_localctx.FLOAT_NUMBER!=null?_localctx.FLOAT_NUMBER.getText():null);
6824-
_localctx.result = text != null ? new FloatLiteralSSTNode(text, false, _localctx.FLOAT_NUMBER.getStartIndex(), _localctx.FLOAT_NUMBER.getStopIndex() + 1) : null;
6824+
_localctx.result = text != null ? FloatLiteralSSTNode.create(text, false, _localctx.FLOAT_NUMBER.getStartIndex(), _localctx.FLOAT_NUMBER.getStopIndex() + 1) : null;
68256825

68266826
}
68276827
break;
@@ -6832,7 +6832,7 @@ public final AtomContext atom() throws RecognitionException {
68326832
_localctx.IMAG_NUMBER = match(IMAG_NUMBER);
68336833

68346834
String text = (_localctx.IMAG_NUMBER!=null?_localctx.IMAG_NUMBER.getText():null);
6835-
_localctx.result = text != null ? new FloatLiteralSSTNode(text, true, _localctx.IMAG_NUMBER.getStartIndex(), _localctx.IMAG_NUMBER.getStopIndex() + 1) : null;
6835+
_localctx.result = text != null ? FloatLiteralSSTNode.create(text, true, _localctx.IMAG_NUMBER.getStartIndex(), _localctx.IMAG_NUMBER.getStopIndex() + 1) : null;
68366836

68376837
}
68386838
break;
@@ -6855,7 +6855,7 @@ public final AtomContext atom() throws RecognitionException {
68556855
_errHandler.sync(this);
68566856
_la = _input.LA(1);
68576857
} while ( _la==STRING );
6858-
_localctx.result = new StringLiteralSSTNode(getStringArray(start), getStartIndex(_localctx), getStopIndex(_localctx.STRING));
6858+
_localctx.result = factory.createStringLiteral(getStringArray(start), getStartIndex(_localctx), getStopIndex(_localctx.STRING));
68596859
}
68606860
break;
68616861
case ELLIPSIS:

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/sst/FactorySSTVisitor.java

Lines changed: 38 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@
119119
import com.oracle.graal.python.parser.ScopeEnvironment;
120120
import com.oracle.graal.python.parser.ScopeInfo;
121121
import com.oracle.graal.python.parser.ScopeInfo.ScopeKind;
122+
import com.oracle.graal.python.parser.sst.NumberLiteralSSTNode.BigIntegerLiteralSSTNode;
123+
import com.oracle.graal.python.parser.sst.NumberLiteralSSTNode.IntegerLiteralSSTNode;
122124
import com.oracle.graal.python.runtime.PythonParser;
123125
import com.oracle.truffle.api.RootCallTarget;
124126
import com.oracle.truffle.api.Truffle;
@@ -221,7 +223,7 @@ public PNode visit(AnnAssignmentSSTNode node) {
221223
// the type is wrong
222224
// create simple SST tree for : __annotations__['var_name'] = type
223225
VarLookupSSTNode varLookupNode = (VarLookupSSTNode) node.lhs[0];
224-
SubscriptSSTNode getAnnotationSST = new SubscriptSSTNode(new VarLookupSSTNode(__ANNOTATIONS__, -1, -1), new StringLiteralSSTNode(new String[]{"'" + varLookupNode.name + "'"}, -1, -1), -1,
226+
SubscriptSSTNode getAnnotationSST = new SubscriptSSTNode(new VarLookupSSTNode(__ANNOTATIONS__, -1, -1), new StringLiteralSSTNode.RawStringLiteralSSTNode(varLookupNode.name, -1, -1), -1,
225227
-1);
226228
AssignmentSSTNode assignAnnotationSST = new AssignmentSSTNode(new SSTNode[]{getAnnotationSST}, node.type, -1, -1);
227229
PNode assignAnnotationNode = visit(assignAnnotationSST);
@@ -335,7 +337,7 @@ public PNode visit(BlockSSTNode node) {
335337

336338
@Override
337339
public PNode visit(BooleanLiteralSSTNode node) {
338-
ExpressionNode result = new com.oracle.graal.python.nodes.literal.BooleanLiteralNode(node.value);
340+
ExpressionNode result = new BooleanLiteralNode(node.value);
339341
result.assignSourceSection(createSourceSection(node.startOffset, node.endOffset));
340342
return result;
341343
}
@@ -679,12 +681,10 @@ public PNode visit(ExpressionStatementSSTNode node) {
679681
@Override
680682
public PNode visit(FloatLiteralSSTNode node) {
681683
ExpressionNode result;
682-
String value = node.value.indexOf('_') == -1 ? node.value : node.value.replace("_", "");
683684
if (node.imaginary) {
684-
double imag = Double.parseDouble(value.substring(0, value.length() - 1));
685-
result = new ComplexLiteralNode(new PComplex(0, imag));
685+
result = new ComplexLiteralNode(new PComplex(0, node.value));
686686
} else {
687-
result = new DoubleLiteralNode(Double.parseDouble(value));
687+
result = new DoubleLiteralNode(node.value);
688688
}
689689
result.assignSourceSection(createSourceSection(node.startOffset, node.endOffset));
690690
return result;
@@ -1020,51 +1020,24 @@ public PNode visit(NotSSTNode node) {
10201020
}
10211021

10221022
@Override
1023-
public PNode visit(NumberLiteralSSTNode node) {
1024-
final long max = node.negative ? Long.MIN_VALUE : -Long.MAX_VALUE;
1025-
final long moltmax = max / node.base;
1026-
int i = node.start;
1027-
long result = 0;
1028-
int lastD;
1029-
boolean overunder = false;
1030-
while (i < node.value.length()) {
1031-
lastD = digitValue(node.value.charAt(i));
1032-
1033-
long next = result;
1034-
if (next < moltmax) {
1035-
overunder = true;
1036-
} else {
1037-
next *= node.base;
1038-
if (next < (max + lastD)) {
1039-
overunder = true;
1040-
} else {
1041-
next -= lastD;
1042-
}
1043-
}
1044-
if (overunder) {
1045-
// overflow
1046-
BigInteger bigResult = BigInteger.valueOf(result);
1047-
BigInteger bigBase = BigInteger.valueOf(node.base);
1048-
while (i < node.value.length()) {
1049-
bigResult = bigResult.multiply(bigBase).subtract(BigInteger.valueOf(digitValue(node.value.charAt(i))));
1050-
i++;
1051-
}
1052-
if (!node.negative) {
1053-
bigResult = bigResult.negate();
1054-
}
1055-
PIntLiteralNode intLiteral = new PIntLiteralNode(bigResult);
1056-
intLiteral.assignSourceSection(createSourceSection(node.startOffset, node.endOffset));
1057-
return intLiteral;
1058-
}
1059-
result = next;
1060-
i++;
1061-
}
1023+
public PNode visit(IntegerLiteralSSTNode node) {
1024+
long result = node.value;
1025+
ExpressionNode intLiteral = Integer.MIN_VALUE <= result && result <= Integer.MAX_VALUE ? new IntegerLiteralNode((int) result) : new LongLiteralNode(result);
1026+
intLiteral.assignSourceSection(createSourceSection(node.startOffset, node.endOffset));
1027+
return intLiteral;
1028+
}
10621029

1063-
if (!node.negative) {
1064-
result = -1 * result;
1065-
}
1030+
private static final BigInteger MIN_LONG = BigInteger.valueOf(Long.MIN_VALUE);
10661031

1067-
ExpressionNode intLiteral = Integer.MIN_VALUE <= result && result <= Integer.MAX_VALUE ? new IntegerLiteralNode((int) result) : new LongLiteralNode(result);
1032+
@Override
1033+
public PNode visit(BigIntegerLiteralSSTNode node) {
1034+
SimpleLiteralNode intLiteral;
1035+
if (node.value.equals(MIN_LONG)) {
1036+
// this can happen because the numeric literals have to support negation
1037+
intLiteral = new LongLiteralNode(Long.MIN_VALUE);
1038+
} else {
1039+
intLiteral = new PIntLiteralNode(node.value);
1040+
}
10681041
intLiteral.assignSourceSection(createSourceSection(node.startOffset, node.endOffset));
10691042
return intLiteral;
10701043
}
@@ -1157,8 +1130,22 @@ public PNode visit(StarSSTNode node) {
11571130
}
11581131

11591132
@Override
1160-
public PNode visit(StringLiteralSSTNode node) {
1161-
PNode result = StringUtils.parseString(source, node, nodeFactory, errors);
1133+
public PNode visit(StringLiteralSSTNode.RawStringLiteralSSTNode node) {
1134+
PNode result = nodeFactory.createStringLiteral(node.value);
1135+
result.assignSourceSection(createSourceSection(node.startOffset, node.endOffset));
1136+
return result;
1137+
}
1138+
1139+
@Override
1140+
public PNode visit(StringLiteralSSTNode.BytesLiteralSSTNode node) {
1141+
PNode result = nodeFactory.createBytesLiteral(node.value);
1142+
result.assignSourceSection(createSourceSection(node.startOffset, node.endOffset));
1143+
return result;
1144+
}
1145+
1146+
@Override
1147+
public PNode visit(StringLiteralSSTNode.FormatStringLiteralSSTNode node) {
1148+
PNode result = nodeFactory.createFormatStringLiteral(node.value);
11621149
result.assignSourceSection(createSourceSection(node.startOffset, node.endOffset));
11631150
return result;
11641151
}
@@ -1269,17 +1256,6 @@ public PNode visit(YieldExpressionSSTNode node) {
12691256
return result;
12701257
}
12711258

1272-
private static int digitValue(char ch) {
1273-
if (ch >= '0' && ch <= '9') {
1274-
return ch - '0';
1275-
} else if (ch >= 'a' && ch <= 'f') {
1276-
return ch - 'a' + 10;
1277-
} else {
1278-
assert ch >= 'A' && ch <= 'f';
1279-
return ch - 'A' + 10;
1280-
}
1281-
}
1282-
12831259
protected SourceSection createSourceSection(int start, int stop) {
12841260
if (start < stop && source.getLength() > start && source.getLength() >= stop) {
12851261
return source.createSection(start, stop - start);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/sst/FloatLiteralSSTNode.java

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -42,15 +42,23 @@
4242
package com.oracle.graal.python.parser.sst;
4343

4444
public class FloatLiteralSSTNode extends SSTNode {
45-
protected final String value;
45+
protected final double value;
4646
protected final boolean imaginary;
4747

48-
public FloatLiteralSSTNode(String value, boolean imaginary, int startOffset, int endOffset) {
48+
public FloatLiteralSSTNode(double value, boolean imaginary, int startOffset, int endOffset) {
4949
super(startOffset, endOffset);
5050
this.value = value;
5151
this.imaginary = imaginary;
5252
}
5353

54+
public static FloatLiteralSSTNode create(String rawValue, boolean imaginary, int startOffset, int endOffset) {
55+
String value = rawValue.replace("_", "");
56+
if (imaginary) {
57+
value = value.substring(0, value.length() - 1);
58+
}
59+
return new FloatLiteralSSTNode(Double.parseDouble(value), imaginary, startOffset, endOffset);
60+
}
61+
5462
@Override
5563
public <T> T accept(SSTreeVisitor<T> visitor) {
5664
return visitor.visit(this);

0 commit comments

Comments
 (0)