Skip to content

Commit cd1579a

Browse files
lukasstadlerlukasstadler
committed
[GR-30401] All parsing via TruffleLanguage.Env.
PullRequest: graalpython/1730
2 parents 07a2e61 + 0ca92d3 commit cd1579a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+800
-926
lines changed

‎graalpython/com.oracle.graal.python.benchmarks/java/com/oracle/graal/python/benchmarks/parser/Deserializing.java

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@
4444
import com.oracle.graal.python.parser.PythonSSTNodeFactory;
4545
import com.oracle.graal.python.runtime.PythonParser;
4646
import com.oracle.truffle.api.source.Source;
47+
import com.oracle.truffle.api.source.SourceSection;
48+
4749
import java.util.ArrayList;
4850
import java.util.List;
4951
import org.openjdk.jmh.annotations.Benchmark;
@@ -53,11 +55,9 @@
5355
public class Deserializing extends ParserBenchRunner {
5456

5557
private static class Item {
56-
final Source source;
5758
final byte[] data;
5859

59-
public Item(Source source, byte[] serialization) {
60-
this.source = source;
60+
public Item(byte[] serialization) {
6161
this.data = serialization;
6262
}
6363
}
@@ -76,7 +76,7 @@ public void setup() {
7676
public void execute(Blackhole bh) {
7777
for (int n = 0; n < parsingCycles; n++) {
7878
for (Item serialization : serializations) {
79-
bh.consume(parser.deserialize(serialization.source, serialization.data));
79+
bh.consume(parser.deserialize(serialization.data));
8080
}
8181
}
8282
}
@@ -88,7 +88,8 @@ private List<Item> createSerializations() {
8888
try {
8989
PythonSSTNodeFactory sstFactory = new PythonSSTNodeFactory(core, source, parser);
9090
PythonParserImpl.CacheItem item = parser.parseWithANTLR(PythonParser.ParserMode.File, 0, core, sstFactory, source, null, null);
91-
result.add(new Item(source, PythonParserImpl.serialize(item.getAntlrResult(), item.getGlobalScope(), true)));
91+
SourceSection section = source.createSection(0, source.getLength());
92+
result.add(new Item(PythonParserImpl.serialize(section, item.getAntlrResult(), item.getGlobalScope(), true)));
9293
} catch (RuntimeException e) {
9394
// do nothing
9495
}

‎graalpython/com.oracle.graal.python.benchmarks/java/com/oracle/graal/python/benchmarks/parser/SSTTranslating.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public void execute(Blackhole bh) {
6666
PythonSSTNodeFactory sstFactory = new PythonSSTNodeFactory(core, item.getSource(), parser);
6767
sstFactory.getScopeEnvironment().setGlobalScope(item.getGlobalScope());
6868
try {
69-
bh.consume(sstFactory.createParserResult(item.getAntlrResult(), PythonParser.ParserMode.File, null));
69+
bh.consume(sstFactory.createParserResult(item.getAntlrResult(), PythonParser.ParserMode.File, null, null));
7070
} catch (Exception e) {
7171
// ignore it
7272
}

‎graalpython/com.oracle.graal.python.benchmarks/java/com/oracle/graal/python/benchmarks/parser/Serializing.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
package com.oracle.graal.python.benchmarks.parser;
4242

4343
import com.oracle.graal.python.parser.PythonParserImpl;
44+
import com.oracle.truffle.api.source.SourceSection;
45+
4446
import java.util.List;
4547
import org.openjdk.jmh.annotations.Benchmark;
4648
import org.openjdk.jmh.annotations.Setup;
@@ -62,7 +64,8 @@ public void setup() {
6264
public void execute(Blackhole bh) {
6365
for (int n = 0; n < parsingCycles; n++) {
6466
for (PythonParserImpl.CacheItem item : ssts) {
65-
bh.consume(PythonParserImpl.serialize(item.getAntlrResult(), item.getGlobalScope(), true));
67+
SourceSection section = item.getSource().createSection(0, item.getSource().getLength());
68+
bh.consume(PythonParserImpl.serialize(section, item.getAntlrResult(), item.getGlobalScope(), true));
6669
}
6770
}
6871
}

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/SSTSerializationTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1260,7 +1260,7 @@ public void checkSerialization(Source source) throws Exception {
12601260
byte[] serializeResult = serializer.serialize(parserResult);
12611261
Assert.assertNotNull("Serialized data are null", serializeResult);
12621262
// and get the tree from serialized data
1263-
RootNode deserialize = serializer.deserialize(source, serializeResult);
1263+
RootNode deserialize = serializer.deserialize(serializeResult);
12641264

12651265
Assert.assertNotNull("Deserialized result is null", parserResult);
12661266
// compare the tree from parser with the tree from serializer
@@ -1539,7 +1539,7 @@ public long[] getTimes(File file) throws Exception {
15391539
TruffleFile tFile = context.getEnv().getInternalTruffleFile(file.getAbsolutePath() + ".pyc");
15401540
byte[] desbytes = tFile.readAllBytes();
15411541
startMemory = System.nanoTime();
1542-
serializer.deserialize(source, desbytes);
1542+
serializer.deserialize(desbytes);
15431543
end = System.nanoTime();
15441544
result[2] = end - startMemory;
15451545
result[5] = end - startFile;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

Lines changed: 97 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,10 @@
2626
package com.oracle.graal.python;
2727

2828
import java.io.IOException;
29-
import java.lang.ref.WeakReference;
3029
import java.util.ArrayList;
3130
import java.util.Arrays;
3231
import java.util.concurrent.ConcurrentHashMap;
3332
import java.util.concurrent.Semaphore;
34-
import java.util.concurrent.atomic.AtomicReference;
3533
import java.util.logging.Level;
3634

3735
import org.graalvm.options.OptionDescriptors;
@@ -52,13 +50,11 @@
5250
import com.oracle.graal.python.nodes.BuiltinNames;
5351
import com.oracle.graal.python.nodes.HiddenAttributes;
5452
import com.oracle.graal.python.nodes.NodeFactory;
53+
import com.oracle.graal.python.nodes.PRootNode;
5554
import com.oracle.graal.python.nodes.call.InvokeNode;
5655
import com.oracle.graal.python.nodes.control.TopLevelExceptionHandler;
57-
import com.oracle.graal.python.nodes.expression.BinaryArithmetic;
5856
import com.oracle.graal.python.nodes.expression.ExpressionNode;
59-
import com.oracle.graal.python.nodes.expression.InplaceArithmetic;
60-
import com.oracle.graal.python.nodes.expression.TernaryArithmetic;
61-
import com.oracle.graal.python.nodes.expression.UnaryArithmetic;
57+
import com.oracle.graal.python.nodes.util.BadOPCodeNode;
6258
import com.oracle.graal.python.parser.PythonParserImpl;
6359
import com.oracle.graal.python.runtime.GilNode;
6460
import com.oracle.graal.python.runtime.PythonContext;
@@ -109,7 +105,11 @@
109105
@TruffleLanguage.Registration(id = PythonLanguage.ID, //
110106
name = PythonLanguage.NAME, //
111107
version = PythonLanguage.VERSION, //
112-
characterMimeTypes = PythonLanguage.MIME_TYPE, //
108+
characterMimeTypes = {PythonLanguage.MIME_TYPE,
109+
PythonLanguage.MIME_TYPE_COMPILE0, PythonLanguage.MIME_TYPE_COMPILE1, PythonLanguage.MIME_TYPE_COMPILE2,
110+
PythonLanguage.MIME_TYPE_EVAL0, PythonLanguage.MIME_TYPE_EVAL1, PythonLanguage.MIME_TYPE_EVAL2}, //
111+
byteMimeTypes = {PythonLanguage.MIME_TYPE_BYTECODE}, //
112+
defaultMimeType = PythonLanguage.MIME_TYPE, //
113113
dependentLanguages = {"nfi", "llvm"}, //
114114
interactive = true, internal = false, //
115115
contextPolicy = TruffleLanguage.ContextPolicy.SHARED, //
@@ -138,6 +138,15 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
138138
public static final int API_VERSION = 1013;
139139

140140
public static final String MIME_TYPE = "text/x-python";
141+
static final String MIME_TYPE_COMPILE0 = "text/x-python-compile0";
142+
static final String MIME_TYPE_COMPILE1 = "text/x-python-compile1";
143+
static final String MIME_TYPE_COMPILE2 = "text/x-python-compile2";
144+
static final String[] MIME_TYPE_COMPILE = {PythonLanguage.MIME_TYPE_COMPILE0, PythonLanguage.MIME_TYPE_COMPILE1, PythonLanguage.MIME_TYPE_COMPILE2};
145+
static final String[] MIME_TYPE_EVAL = {PythonLanguage.MIME_TYPE_EVAL0, PythonLanguage.MIME_TYPE_EVAL1, PythonLanguage.MIME_TYPE_EVAL2};
146+
static final String MIME_TYPE_EVAL0 = "text/x-python-eval0";
147+
static final String MIME_TYPE_EVAL1 = "text/x-python-eval1";
148+
static final String MIME_TYPE_EVAL2 = "text/x-python-eval2";
149+
public static final String MIME_TYPE_BYTECODE = "application/x-python-bytecode";
141150
public static final String EXTENSION = ".py";
142151
public static final String[] DEFAULT_PYTHON_EXTENSIONS = new String[]{EXTENSION, ".pyc"};
143152

@@ -152,16 +161,14 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
152161
public final Assumption singleThreadedAssumption = Truffle.getRuntime().createAssumption("Only a single thread is active");
153162

154163
private final NodeFactory nodeFactory;
155-
private final ConcurrentHashMap<String, RootCallTarget> builtinCallTargetCache = new ConcurrentHashMap<>();
164+
156165
/**
157-
* A thread-safe map that maps arithmetic operators (i.e.
158-
* {@link com.oracle.graal.python.nodes.expression.UnaryArithmetic},
159-
* {@link com.oracle.graal.python.nodes.expression.BinaryArithmetic},
160-
* {@link com.oracle.graal.python.nodes.expression.TernaryArithmetic}, and
161-
* {@link com.oracle.graal.python.nodes.expression.InplaceArithmetic}) to call targets. Use this
162-
* map to retrieve a singleton instance (per engine) such that proper AST sharing is possible.
166+
* A thread-safe map to retrieve (and cache) singleton instances of call targets, e.g., for
167+
* Arithmetic operations, wrappers, named cext functions, etc. This reduces the number of call
168+
* targets and allows AST sharing across contexts. The key in this map is either a single value
169+
* or a list of values.
163170
*/
164-
private final AtomicReference<ConcurrentHashMap<Object, WeakReference<RootCallTarget>>> arithmeticOpCallTargetCacheRef = new AtomicReference<>();
171+
private final ConcurrentHashMap<Object, RootCallTarget> cachedCallTargets = new ConcurrentHashMap<>();
165172

166173
private final Shape emptyShape = Shape.newBuilder().allowImplicitCastIntToDouble(false).allowImplicitCastIntToLong(true).shapeFlags(0).propertyAssumptions(true).build();
167174
@CompilationFinal(dimensions = 1) private final Shape[] builtinTypeInstanceShapes = new Shape[PythonBuiltinClassType.VALUES.length];
@@ -273,27 +280,72 @@ protected void initializeContext(PythonContext context) {
273280
context.initialize();
274281
}
275282

283+
public static String getCompileMimeType(int optimize) {
284+
if (optimize <= 0) {
285+
return MIME_TYPE_COMPILE0;
286+
} else if (optimize == 1) {
287+
return MIME_TYPE_COMPILE1;
288+
} else {
289+
return MIME_TYPE_COMPILE2;
290+
}
291+
}
292+
293+
public static String getEvalMimeType(int optimize) {
294+
if (optimize <= 0) {
295+
return MIME_TYPE_EVAL0;
296+
} else if (optimize == 1) {
297+
return MIME_TYPE_EVAL1;
298+
} else {
299+
return MIME_TYPE_EVAL2;
300+
}
301+
}
302+
276303
@Override
277304
protected CallTarget parse(ParsingRequest request) {
278305
PythonContext context = getCurrentContext(PythonLanguage.class);
279306
PythonCore core = context.getCore();
280307
Source source = request.getSource();
281-
CompilerDirectives.transferToInterpreter();
282-
if (core.isInitialized()) {
283-
context.initializeMainModule(source.getPath());
308+
if (source.getMimeType() == null || MIME_TYPE.equals(source.getMimeType())) {
309+
if (!request.getArgumentNames().isEmpty()) {
310+
return PythonUtils.getOrCreateCallTarget(parseWithArguments(request));
311+
}
312+
RootNode root = doParse(context, source, 0);
313+
if (root instanceof PRootNode) {
314+
((PRootNode) root).triggerDeprecationWarnings();
315+
}
316+
if (core.isInitialized()) {
317+
return PythonUtils.getOrCreateCallTarget(new TopLevelExceptionHandler(this, root, source));
318+
} else {
319+
return PythonUtils.getOrCreateCallTarget(root);
320+
}
284321
}
285322
if (!request.getArgumentNames().isEmpty()) {
286-
return PythonUtils.getOrCreateCallTarget(parseWithArguments(request));
323+
throw new IllegalStateException("parse with arguments is only allowed for " + MIME_TYPE + " mime type");
287324
}
288-
RootNode root = doParse(context, source);
289-
if (core.isInitialized()) {
290-
return PythonUtils.getOrCreateCallTarget(new TopLevelExceptionHandler(this, root));
291-
} else {
292-
return PythonUtils.getOrCreateCallTarget(root);
325+
326+
if (MIME_TYPE_BYTECODE.equals(source.getMimeType())) {
327+
byte[] bytes = source.getBytes().toByteArray();
328+
if (bytes.length == 0) {
329+
return createCachedCallTarget(l -> new BadOPCodeNode(l), BadOPCodeNode.class);
330+
}
331+
return PythonUtils.getOrCreateCallTarget(core.getSerializer().deserialize(bytes));
332+
}
333+
for (int optimize = 0; optimize < MIME_TYPE_EVAL.length; optimize++) {
334+
if (MIME_TYPE_EVAL[optimize].equals(source.getMimeType())) {
335+
assert !source.isInteractive();
336+
return PythonUtils.getOrCreateCallTarget((RootNode) core.getParser().parse(ParserMode.Eval, optimize, core, source, null, null));
337+
}
338+
}
339+
for (int optimize = 0; optimize < MIME_TYPE_COMPILE.length; optimize++) {
340+
if (MIME_TYPE_COMPILE[optimize].equals(source.getMimeType())) {
341+
assert !source.isInteractive();
342+
return PythonUtils.getOrCreateCallTarget((RootNode) core.getParser().parse(ParserMode.File, optimize, core, source, null, null));
343+
}
293344
}
345+
throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + source.getMimeType());
294346
}
295347

296-
private RootNode doParse(PythonContext context, Source source) {
348+
private RootNode doParse(PythonContext context, Source source, int optimize) {
297349
ParserMode mode;
298350
if (source.isInteractive()) {
299351
if (context.getOption(PythonOptions.TerminalIsInteractive)) {
@@ -311,7 +363,7 @@ private RootNode doParse(PythonContext context, Source source) {
311363
}
312364
PythonCore pythonCore = context.getCore();
313365
try {
314-
return (RootNode) pythonCore.getParser().parse(mode, 0, pythonCore, source, null, null);
366+
return (RootNode) pythonCore.getParser().parse(mode, optimize, pythonCore, source, null, null);
315367
} catch (PException e) {
316368
// handle PException during parsing (PIncompleteSourceException will propagate through)
317369
PythonUtils.getOrCreateCallTarget(new TopLevelExceptionHandler(this, e)).call();
@@ -617,7 +669,7 @@ public static TruffleLogger getCompatibilityLogger(Class<?> clazz) {
617669
return TruffleLogger.getLogger(ID, "compatibility." + clazz.getName());
618670
}
619671

620-
public static Source newSource(PythonContext ctxt, String src, String name, boolean mayBeFile) {
672+
public static Source newSource(PythonContext ctxt, String src, String name, boolean mayBeFile, String mime) {
621673
try {
622674
SourceBuilder sourceBuilder = null;
623675
if (mayBeFile) {
@@ -641,6 +693,9 @@ public static Source newSource(PythonContext ctxt, String src, String name, bool
641693
if (sourceBuilder == null) {
642694
sourceBuilder = Source.newBuilder(ID, src, name);
643695
}
696+
if (mime != null) {
697+
sourceBuilder.mimeType(mime);
698+
}
644699
return newSource(ctxt, sourceBuilder);
645700
} catch (IOException e) {
646701
throw new IllegalStateException(e);
@@ -720,10 +775,6 @@ protected void disposeThread(PythonContext context, Thread thread) {
720775
context.disposeThread(thread);
721776
}
722777

723-
public RootCallTarget getOrComputeBuiltinCallTarget(String key, Supplier<RootNode> supplier) {
724-
return builtinCallTargetCache.computeIfAbsent(key, (k) -> PythonUtils.getOrCreateCallTarget(supplier.get()));
725-
}
726-
727778
public Shape getEmptyShape() {
728779
return emptyShape;
729780
}
@@ -755,82 +806,6 @@ public Shape getBuiltinTypeInstanceShape(PythonBuiltinClassType type) {
755806
return shape;
756807
}
757808

758-
/**
759-
* Retrieve a call target for the given {@link UnaryArithmetic} operator. If the no such call
760-
* target exists yet, it will be created lazily. This method is thread-safe and should be used
761-
* for all contexts in this engine to enable AST sharing.
762-
*/
763-
@TruffleBoundary
764-
public RootCallTarget getOrCreateUnaryArithmeticCallTarget(UnaryArithmetic unaryOperator) {
765-
return getOrCreateArithmeticCallTarget(unaryOperator, unaryOperator::createCallTarget);
766-
}
767-
768-
/**
769-
* Retrieve a call target for the given {@link BinaryArithmetic} operator. If the no such call
770-
* target exists yet, it will be created lazily. This method is thread-safe and should be used
771-
* for all contexts in this engine to enable AST sharing.
772-
*/
773-
@TruffleBoundary
774-
public RootCallTarget getOrCreateBinaryArithmeticCallTarget(BinaryArithmetic unaryOperator) {
775-
return getOrCreateArithmeticCallTarget(unaryOperator, unaryOperator::createCallTarget);
776-
}
777-
778-
/**
779-
* Retrieve a call target for the given {@link TernaryArithmetic} operator. If the no such call
780-
* target exists yet, it will be created lazily. This method is thread-safe and should be used
781-
* for all contexts in this engine to enable AST sharing.
782-
*/
783-
@TruffleBoundary
784-
public RootCallTarget getOrCreateTernaryArithmeticCallTarget(TernaryArithmetic unaryOperator) {
785-
return getOrCreateArithmeticCallTarget(unaryOperator, unaryOperator::createCallTarget);
786-
}
787-
788-
/**
789-
* Retrieve a call target for the given {@link InplaceArithmetic} operator. If the no such call
790-
* target exists yet, it will be created lazily. This method is thread-safe and should be used
791-
* for all contexts in this engine to enable AST sharing.
792-
*/
793-
@TruffleBoundary
794-
public RootCallTarget getOrCreateInplaceArithmeticCallTarget(InplaceArithmetic unaryOperator) {
795-
return getOrCreateArithmeticCallTarget(unaryOperator, unaryOperator::createCallTarget);
796-
}
797-
798-
private RootCallTarget getOrCreateArithmeticCallTarget(Object arithmeticOperator, Function<PythonLanguage, RootCallTarget> supplier) {
799-
CompilerAsserts.neverPartOfCompilation();
800-
ConcurrentHashMap<Object, WeakReference<RootCallTarget>> arithmeticOpCallTargetCache = arithmeticOpCallTargetCacheRef.get();
801-
if (arithmeticOpCallTargetCache == null) {
802-
arithmeticOpCallTargetCache = arithmeticOpCallTargetCacheRef.updateAndGet((v) -> {
803-
// IMPORTANT: only create a new instance if we still see 'null'; otherwise we would
804-
// overwrite the update of a different thread
805-
if (v == null) {
806-
return new ConcurrentHashMap<>();
807-
}
808-
return v;
809-
});
810-
}
811-
812-
WeakReference<RootCallTarget> ctRef = arithmeticOpCallTargetCache.compute(arithmeticOperator, (k, v) -> {
813-
RootCallTarget cachedCallTarget = v != null ? v.get() : null;
814-
if (cachedCallTarget == null) {
815-
return new WeakReference<>(supplier.apply(this));
816-
}
817-
return v;
818-
});
819-
820-
RootCallTarget callTarget = ctRef.get();
821-
if (callTarget == null) {
822-
// Bad luck: we ensured that there is a mapping in the cache but the weak value got
823-
// collected before we could strongly reference it. Now, we need to be conservative and
824-
// create the call target eagerly, hold a strong reference to it until we've put it into
825-
// the map.
826-
final RootCallTarget callTargetToCache = supplier.apply(this);
827-
callTarget = callTargetToCache;
828-
arithmeticOpCallTargetCache.computeIfAbsent(arithmeticOperator, (k) -> new WeakReference<>(callTargetToCache));
829-
}
830-
assert callTarget != null;
831-
return callTarget;
832-
}
833-
834809
/**
835810
* Returns the shape used for the C API symbol cache.
836811
*/
@@ -852,4 +827,19 @@ public synchronized Shape getHPySymbolCacheShape() {
852827
}
853828
return hpySymbolCache;
854829
}
830+
831+
/**
832+
* Cache call targets that are created for every new context, based on a single key.
833+
*/
834+
public RootCallTarget createCachedCallTarget(Function<PythonLanguage, RootNode> rootNodeFunction, Object key) {
835+
CompilerAsserts.neverPartOfCompilation();
836+
return cachedCallTargets.computeIfAbsent(key, k -> PythonUtils.getOrCreateCallTarget(rootNodeFunction.apply(this)));
837+
}
838+
839+
/**
840+
* Cache call targets that are created for every new context, based on a list of keys.
841+
*/
842+
public RootCallTarget createCachedCallTarget(Function<PythonLanguage, RootNode> rootNodeFunction, Object... cacheKeys) {
843+
return createCachedCallTarget(rootNodeFunction, Arrays.asList(cacheKeys));
844+
}
855845
}

0 commit comments

Comments
 (0)