Skip to content

Commit f8e2f7a

Browse files
otethalotethal
committed
AST conversion java->python
1 parent 18b81bb commit f8e2f7a

File tree

8 files changed

+1105
-17
lines changed

8 files changed

+1105
-17
lines changed

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

Lines changed: 55 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@
114114
import com.oracle.graal.python.builtins.modules.BuiltinFunctionsFactory.HexNodeFactory;
115115
import com.oracle.graal.python.builtins.modules.BuiltinFunctionsFactory.OctNodeFactory;
116116
import com.oracle.graal.python.builtins.modules.WarningsModuleBuiltins.WarnNode;
117+
import com.oracle.graal.python.builtins.modules.ast.AstModuleBuiltins;
117118
import com.oracle.graal.python.builtins.modules.io.IOModuleBuiltins;
118119
import com.oracle.graal.python.builtins.modules.io.IONodes;
119120
import com.oracle.graal.python.builtins.objects.PNone;
@@ -151,6 +152,8 @@
151152
import com.oracle.graal.python.builtins.objects.type.TypeBuiltins;
152153
import com.oracle.graal.python.builtins.objects.type.TypeNodes;
153154
import com.oracle.graal.python.builtins.objects.type.TypeNodes.IsTypeNode;
155+
import com.oracle.graal.python.compiler.Compiler;
156+
import com.oracle.graal.python.compiler.RaisePythonExceptionErrorCallback;
154157
import com.oracle.graal.python.lib.PyCallableCheckNode;
155158
import com.oracle.graal.python.lib.PyEvalGetGlobals;
156159
import com.oracle.graal.python.lib.PyMappingCheckNode;
@@ -228,6 +231,8 @@
228231
import com.oracle.graal.python.nodes.util.CastToJavaLongExactNode;
229232
import com.oracle.graal.python.nodes.util.CastToTruffleStringNode;
230233
import com.oracle.graal.python.pegparser.InputType;
234+
import com.oracle.graal.python.pegparser.Parser;
235+
import com.oracle.graal.python.pegparser.sst.ModTy;
231236
import com.oracle.graal.python.runtime.ExecutionContext.IndirectCallContext;
232237
import com.oracle.graal.python.runtime.PythonContext;
233238
import com.oracle.graal.python.runtime.PythonOptions;
@@ -854,7 +859,7 @@ protected boolean isAnyNone(Object object) {
854859
}
855860

856861
protected PCode createAndCheckCode(VirtualFrame frame, Object source) {
857-
PCode code = getCompileNode().execute(frame, source, T_STRING_SOURCE, getMode(), 0, false, -1);
862+
PCode code = getCompileNode().compile(frame, source, T_STRING_SOURCE, getMode(), -1);
858863
assertNoFreeVars(code);
859864
return code;
860865
}
@@ -1021,10 +1026,14 @@ public abstract static class CompileNode extends PythonBuiltinNode {
10211026
private static final int PyCF_MASK_OBSOLETE = CO_NESTED;
10221027

10231028
private static final int PyCF_DONT_IMPLY_DEDENT = 0x0200;
1024-
private static final int PyCF_ONLY_AST = 0x0400;
1025-
private static final int PyCF_TYPE_COMMENTS = 0x1000;
1029+
public static final int PyCF_ONLY_AST = 0x0400;
1030+
public static final int PyCF_TYPE_COMMENTS = 0x1000;
1031+
public static final int PyCF_ALLOW_TOP_LEVEL_AWAIT = 0x2000;
1032+
private static final int PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000;
1033+
private static final int PyCF_COMPILE_MASK = PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT;
10261034

10271035
private static final TruffleString T_SINGLE = tsLiteral("single");
1036+
private static final TruffleString T_FUNC_EVAL = tsLiteral("func_eval");
10281037

10291038
/**
10301039
* Decides whether this node should attempt to map the filename to a URI for the benefit of
@@ -1043,12 +1052,16 @@ public CompileNode() {
10431052
this.lstrip = false;
10441053
}
10451054

1046-
public abstract PCode execute(VirtualFrame frame, Object source, TruffleString filename, TruffleString mode, Object kwFlags, Object kwDontInherit, Object kwOptimize);
1055+
public final PCode compile(VirtualFrame frame, Object source, TruffleString filename, TruffleString mode, Object kwOptimize) {
1056+
return (PCode) executeInternal(frame, source, filename, mode, 0, false, kwOptimize);
1057+
}
1058+
1059+
protected abstract Object executeInternal(VirtualFrame frame, Object source, TruffleString filename, TruffleString mode, Object kwFlags, Object kwDontInherit, Object kwOptimize);
10471060

10481061
@SuppressWarnings("unused")
10491062
@Specialization
10501063
@TruffleBoundary
1051-
PCode compile(TruffleString expression, TruffleString filename, TruffleString mode, int kwFlags, Object kwDontInherit, int kwOptimize) {
1064+
Object compile(TruffleString expression, TruffleString filename, TruffleString mode, int kwFlags, Object kwDontInherit, int kwOptimize) {
10521065
checkFlags(kwFlags);
10531066
checkOptimize(kwOptimize, kwOptimize);
10541067
checkSource(expression);
@@ -1071,15 +1084,48 @@ PCode compile(TruffleString expression, TruffleString filename, TruffleString mo
10711084
pm = ParserMode.Eval;
10721085
} else if (mode.equalsUncached(T_SINGLE, TS_ENCODING)) {
10731086
pm = ParserMode.Statement;
1087+
} else if (mode.equalsUncached(T_FUNC_EVAL, TS_ENCODING)) {
1088+
if ((kwFlags & PyCF_ONLY_AST) == 0) {
1089+
throw raise(ValueError, ErrorMessages.COMPILE_MODE_FUNC_TYPE_REQUIED_FLAG_ONLY_AST);
1090+
}
1091+
pm = ParserMode.FuncType;
10741092
} else {
1075-
throw raise(ValueError, ErrorMessages.COMPILE_MUST_BE);
1093+
if ((kwFlags & PyCF_ONLY_AST) != 0) {
1094+
throw raise(ValueError, ErrorMessages.COMPILE_MODE_MUST_BE_AST_ONLY);
1095+
} else {
1096+
throw raise(ValueError, ErrorMessages.COMPILE_MODE_MUST_BE);
1097+
}
10761098
}
10771099
if (lstrip && !code.isEmpty()) {
10781100
int c = code.codePointAtIndexUncached(0, TS_ENCODING);
10791101
if (c == ' ' || c == '\t') {
10801102
code = code.substringUncached(1, code.codePointLengthUncached(TS_ENCODING) - 1, TS_ENCODING, true);
10811103
}
10821104
}
1105+
if ((kwFlags & PyCF_ONLY_AST) != 0) {
1106+
InputType type;
1107+
switch (pm) {
1108+
case File:
1109+
type = InputType.FILE;
1110+
break;
1111+
case Eval:
1112+
type = InputType.EVAL;
1113+
break;
1114+
case Statement:
1115+
type = InputType.SINGLE;
1116+
break;
1117+
case FuncType:
1118+
type = InputType.FUNCTION_TYPE;
1119+
break;
1120+
default:
1121+
throw CompilerDirectives.shouldNotReachHere();
1122+
}
1123+
Source source = PythonLanguage.newSource(context, code, filename, mayBeFromFile, PythonLanguage.MIME_TYPE);
1124+
RaisePythonExceptionErrorCallback errorCb = new RaisePythonExceptionErrorCallback(source, PythonOptions.isPExceptionWithJavaStacktrace(getLanguage()));
1125+
Parser parser = Compiler.createParser(code.toJavaStringUncached(), errorCb, type, false);
1126+
ModTy mod = (ModTy) parser.parse();
1127+
return AstModuleBuiltins.sst2Obj(getContext(), mod);
1128+
}
10831129
CallTarget ct;
10841130
TruffleString finalCode = code;
10851131
Supplier<CallTarget> createCode = () -> {
@@ -1113,7 +1159,7 @@ PCode compile(TruffleString expression, TruffleString filename, TruffleString mo
11131159
}
11141160

11151161
@Specialization(limit = "3")
1116-
PCode generic(VirtualFrame frame, Object wSource, Object wFilename, Object wMode, Object kwFlags, Object kwDontInherit, Object kwOptimize,
1162+
Object generic(VirtualFrame frame, Object wSource, Object wFilename, Object wMode, Object kwFlags, Object kwDontInherit, Object kwOptimize,
11171163
@CachedLibrary(limit = "3") PythonBufferAcquireLibrary acquireLib,
11181164
@CachedLibrary(limit = "3") PythonBufferAccessLibrary bufferLib,
11191165
@Cached CastToTruffleStringNode castStr,
@@ -1126,7 +1172,7 @@ PCode generic(VirtualFrame frame, Object wSource, Object wFilename, Object wMode
11261172
@Cached TruffleString.FromByteArrayNode fromByteArrayNode,
11271173
@Cached TruffleString.SwitchEncodingNode switchEncodingNode) {
11281174
if (wSource instanceof PCode) {
1129-
return (PCode) wSource;
1175+
return wSource;
11301176
}
11311177
TruffleString filename;
11321178
// TODO use PyUnicode_FSDecode
@@ -1186,7 +1232,7 @@ private void checkOptimize(int optimize, Object kwOptimize) throws PException {
11861232
}
11871233

11881234
private void checkFlags(int flags) {
1189-
if ((flags & ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST | PyCF_TYPE_COMMENTS)) > 0) {
1235+
if ((flags & ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_COMPILE_MASK)) != 0) {
11901236
throw raise(ValueError, ErrorMessages.UNRECOGNIZED_FLAGS);
11911237
}
11921238
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -750,7 +750,7 @@ PCode compile(VirtualFrame frame, Object codestr, TruffleString path, TruffleStr
750750
CallTarget callTarget = createCallTarget(source);
751751
return factory().createCode((RootCallTarget) callTarget);
752752
} else {
753-
return compileNode.execute(frame, codestr, path, mode, 0, false, 2);
753+
return compileNode.compile(frame, codestr, path, mode, 2);
754754
}
755755
}
756756

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ast/AstModuleBuiltins.java

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@
4040
*/
4141
package com.oracle.graal.python.builtins.modules.ast;
4242

43+
import static com.oracle.graal.python.builtins.modules.BuiltinFunctions.CompileNode.PyCF_ALLOW_TOP_LEVEL_AWAIT;
44+
import static com.oracle.graal.python.builtins.modules.BuiltinFunctions.CompileNode.PyCF_ONLY_AST;
45+
import static com.oracle.graal.python.builtins.modules.BuiltinFunctions.CompileNode.PyCF_TYPE_COMMENTS;
4346
import static com.oracle.graal.python.util.PythonUtils.EMPTY_OBJECT_ARRAY;
4447
import static com.oracle.graal.python.util.PythonUtils.tsLiteral;
4548

@@ -53,7 +56,10 @@
5356
import com.oracle.graal.python.builtins.objects.module.PythonModule;
5457
import com.oracle.graal.python.builtins.objects.tuple.PTuple;
5558
import com.oracle.graal.python.builtins.objects.type.PythonBuiltinClass;
59+
import com.oracle.graal.python.nodes.attributes.ReadAttributeFromObjectNode;
5660
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
61+
import com.oracle.graal.python.pegparser.sst.ModTy;
62+
import com.oracle.graal.python.runtime.PythonContext;
5763
import com.oracle.truffle.api.dsl.NodeFactory;
5864
import com.oracle.truffle.api.object.HiddenKey;
5965
import com.oracle.truffle.api.strings.TruffleString;
@@ -69,10 +75,6 @@ public final class AstModuleBuiltins extends PythonBuiltins {
6975
static final TruffleString T__FIELDS = tsLiteral("_fields");
7076
static final TruffleString T__ATTRIBUTES = tsLiteral("_attributes");
7177

72-
private static final int PyCF_ONLY_AST = 0x0400;
73-
private static final int PyCF_TYPE_COMMENTS = 0x1000;
74-
private static final int PyCF_ALLOW_TOP_LEVEL_AWAIT = 0x2000;
75-
7678
@Override
7779
protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFactories() {
7880
return new ArrayList<>();
@@ -101,4 +103,12 @@ public void postInitialize(Python3Core core) {
101103
AstState state = new AstState(astTypeFactory, core.lookupType(PythonBuiltinClassType.AST));
102104
astModule.setAttribute(AST_STATE_KEY, state);
103105
}
106+
107+
private static AstState getAstState(PythonContext context) {
108+
return (AstState) ReadAttributeFromObjectNode.getUncached().execute(context.lookupBuiltinModule(T__AST), AST_STATE_KEY);
109+
}
110+
111+
public static Object sst2Obj(PythonContext context, ModTy mod) {
112+
return mod.accept(new Sst2ObjVisitor(getAstState(context)));
113+
}
104114
}

0 commit comments

Comments
 (0)