Skip to content

Commit a9fab60

Browse files
steve-ssteve-s
committed
[GR-33210] Force inner contexts to have their own language instance.
PullRequest: graalpython/1935
2 parents 9238a8b + f8ed5cf commit a9fab60

File tree

4 files changed

+108
-100
lines changed

4 files changed

+108
-100
lines changed

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

Lines changed: 23 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@
5858
import com.oracle.graal.python.parser.PythonParserImpl;
5959
import com.oracle.graal.python.runtime.GilNode;
6060
import com.oracle.graal.python.runtime.PythonContext;
61-
import com.oracle.graal.python.runtime.PythonContext.ChildContextData;
6261
import com.oracle.graal.python.runtime.PythonContext.PythonThreadState;
6362
import com.oracle.graal.python.runtime.PythonOptions;
6463
import com.oracle.graal.python.runtime.PythonParser.ParserMode;
@@ -100,7 +99,6 @@
10099
import com.oracle.truffle.api.object.Shape;
101100
import com.oracle.truffle.api.source.Source;
102101
import com.oracle.truffle.api.source.Source.SourceBuilder;
103-
import java.util.Map;
104102

105103
@TruffleLanguage.Registration(id = PythonLanguage.ID, //
106104
name = PythonLanguage.NAME, //
@@ -158,6 +156,9 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
158156

159157
public final Assumption singleContextAssumption = Truffle.getRuntime().createAssumption("Only a single context is active");
160158

159+
@CompilationFinal public boolean singleContext = true;
160+
private boolean firstContextInitialized;
161+
161162
/**
162163
* This assumption will be valid if all contexts are single-threaded. Hence, it will be
163164
* invalidated as soon as at least one context has been initialized for multi-threading.
@@ -198,6 +199,12 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
198199
* system is shut down, unless explicitly removed. We interpret this as meaning they all exist
199200
* globally per language instance, that is, they are shared between different Contexts in the
200201
* same engine.
202+
*
203+
* Top level contexts use this map to initialize their shared multiprocessing data. Inner
204+
* children contexts created for the multiprocessing module ignore this map in
205+
* {@link PythonLanguage} and instead inherit it in the shared multiprocessing data from their
206+
* parent context. This way, the child inner contexts do not have to run in the same engine
207+
* (have the same language instance), but can still share the named semaphores.
201208
*/
202209
public final ConcurrentHashMap<String, Semaphore> namedSemaphores = new ConcurrentHashMap<>();
203210

@@ -218,35 +225,6 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
218225

219226
private final MroShape mroShapeRoot = MroShape.createRoot();
220227

221-
private final Map<Long, Thread> childContextThreads = new ConcurrentHashMap<>();
222-
223-
private final Map<Long, ChildContextData> childContextData = new ConcurrentHashMap<>();
224-
225-
@TruffleBoundary
226-
public Thread getChildContextThread(long tid) {
227-
return childContextThreads.get(tid);
228-
}
229-
230-
@TruffleBoundary
231-
public void putChildContextThread(long id, Thread thread) {
232-
childContextThreads.put(id, thread);
233-
}
234-
235-
@TruffleBoundary
236-
public void removeChildContextThread(long id) {
237-
childContextThreads.remove(id);
238-
}
239-
240-
@TruffleBoundary
241-
public ChildContextData getChildContextData(long tid) {
242-
return childContextData.get(tid);
243-
}
244-
245-
@TruffleBoundary
246-
public void putChildContextData(long id, ChildContextData data) {
247-
childContextData.put(id, data);
248-
}
249-
250228
public static PythonLanguage get(Node node) {
251229
return REFERENCE.get(node);
252230
}
@@ -293,12 +271,16 @@ protected void finalizeContext(PythonContext context) {
293271

294272
@Override
295273
protected boolean areOptionsCompatible(OptionValues firstOptions, OptionValues newOptions) {
274+
if (singleContext) {
275+
return false;
276+
}
296277
return PythonOptions.areOptionsCompatible(firstOptions, newOptions);
297278
}
298279

299280
@Override
300281
protected boolean patchContext(PythonContext context, Env newEnv) {
301-
if (!areOptionsCompatible(context.getEnv().getOptions(), newEnv.getOptions())) {
282+
// We intentionally bypass the singleContext check in PythonLanguage#areOptionsCompatible
283+
if (!PythonOptions.areOptionsCompatible(context.getEnv().getOptions(), newEnv.getOptions())) {
302284
Python3Core.writeInfo("Cannot use preinitialized context.");
303285
return false;
304286
}
@@ -327,6 +309,7 @@ protected PythonContext createContext(Env env) {
327309
} else {
328310
assert areOptionsCompatible(options, PythonOptions.createEngineOptions(env)) : "invalid engine options";
329311
}
312+
firstContextInitialized = true;
330313
return context;
331314
}
332315

@@ -738,6 +721,14 @@ private static Source newSource(PythonContext ctxt, SourceBuilder srcBuilder) th
738721
@Override
739722
protected void initializeMultipleContexts() {
740723
super.initializeMultipleContexts();
724+
// We want to make sure that initializeMultipleContexts is always called before the first
725+
// context is created.
726+
// This would not be the case with inner contexts, but we achieve that by returning false
727+
// from areOptionsCompatible when it is invoked for the first inner context, then Truffle
728+
// creates a new PythonLanguage instance, calls initializeMultipleContexts on it, and only
729+
// then uses it to create the inner contexts.
730+
assert !firstContextInitialized;
731+
singleContext = false;
741732
singleContextAssumption.invalidate();
742733
}
743734

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MultiprocessingModuleBuiltins.java

Lines changed: 10 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -141,26 +141,16 @@ PSemLock construct(Object cls, Object kindObj, Object valueObj, Object maxvalueO
141141
// have to explicitly link it, so we do that here if we
142142
// must. CPython always uses O_CREAT | O_EXCL for creating named
143143
// semaphores, so a conflict raises.
144-
PythonLanguage lang = getLanguage();
145-
if (semaphoreExists(lang, name)) {
144+
SharedMultiprocessingData multiprocessing = getContext().getSharedMultiprocessingData();
145+
if (multiprocessing.getNamedSemaphore(name) != null) {
146146
throw raise(PythonBuiltinClassType.FileExistsError, ErrorMessages.SEMAPHORE_NAME_TAKEN, name);
147147
} else {
148-
semaphorePut(lang, semaphore, name);
148+
multiprocessing.putNamedSemaphore(name, semaphore);
149149
}
150150
}
151151
return factory().createSemLock(cls, name, kind, semaphore);
152152
}
153153

154-
@TruffleBoundary
155-
private static Object semaphorePut(PythonLanguage lang, Semaphore semaphore, String name) {
156-
return lang.namedSemaphores.put(name, semaphore);
157-
}
158-
159-
@TruffleBoundary
160-
private static boolean semaphoreExists(PythonLanguage lang, String name) {
161-
return lang.namedSemaphores.containsKey(name);
162-
}
163-
164154
@TruffleBoundary
165155
private static Semaphore newSemaphore(int value) {
166156
return new Semaphore(value);
@@ -172,17 +162,12 @@ private static Semaphore newSemaphore(int value) {
172162
abstract static class SemUnlink extends PythonUnaryBuiltinNode {
173163
@Specialization
174164
PNone doit(String name) {
175-
Semaphore prev = semaphoreRemove(name, getLanguage());
165+
Semaphore prev = getContext().getSharedMultiprocessingData().removeNamedSemaphore(name);
176166
if (prev == null) {
177167
throw raise(PythonBuiltinClassType.FileNotFoundError, ErrorMessages.NO_SUCH_FILE_OR_DIR, "semaphores", name);
178168
}
179169
return PNone.NONE;
180170
}
181-
182-
@TruffleBoundary
183-
private static Semaphore semaphoreRemove(String name, PythonLanguage lang) {
184-
return lang.namedSemaphores.remove(name);
185-
}
186171
}
187172

188173
@Builtin(name = "_spawn_context", minNumOfPositionalArgs = 3, parameterNames = {"fd", "sentinel", "keepFds"})
@@ -220,15 +205,15 @@ long getTid() {
220205
abstract static class WaitTidNode extends PythonBinaryBuiltinNode {
221206
@Specialization
222207
PTuple waittid(long id, @SuppressWarnings("unused") int options) {
223-
PythonLanguage lang = getLanguage();
224208
long tid = convertTid(id);
225209
// TODO implement for options - WNOHANG and 0
226-
Thread thread = lang.getChildContextThread(tid);
210+
final SharedMultiprocessingData multiprocessing = getContext().getSharedMultiprocessingData();
211+
Thread thread = multiprocessing.getChildContextThread(tid);
227212
if (thread != null && thread.isAlive()) {
228213
return factory().createTuple(new Object[]{0, 0, 0});
229214
}
230215

231-
PythonContext.ChildContextData data = lang.getChildContextData(tid);
216+
PythonContext.ChildContextData data = multiprocessing.getChildContextData(tid);
232217
return factory().createTuple(new Object[]{id, data.wasSignaled() ? data.getExitCode() : 0, data.getExitCode()});
233218
}
234219
}
@@ -239,10 +224,10 @@ abstract static class TerminateThreadNode extends PythonBinaryBuiltinNode {
239224
@Specialization
240225
@TruffleBoundary
241226
Object terminate(long id, PInt sig) {
242-
PythonLanguage language = getLanguage();
243-
Thread thread = language.getChildContextThread(convertTid(id));
227+
final SharedMultiprocessingData multiprocessing = getContext().getSharedMultiprocessingData();
228+
Thread thread = multiprocessing.getChildContextThread(convertTid(id));
244229
if (thread != null && thread.isAlive()) {
245-
PythonContext.ChildContextData data = language.getChildContextData(convertTid(id));
230+
PythonContext.ChildContextData data = multiprocessing.getChildContextData(convertTid(id));
246231
try {
247232
data.awaitRunning();
248233
TruffleContext truffleCtx = data.getTruffleContext();

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/thread/SemLockBuiltins.java

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -40,15 +40,16 @@
4040
*/
4141
package com.oracle.graal.python.builtins.objects.thread;
4242

43-
import com.oracle.graal.python.PythonLanguage;
44-
import com.oracle.graal.python.annotations.ArgumentClinic;
4543
import static com.oracle.graal.python.nodes.SpecialMethodNames.__ENTER__;
4644
import static com.oracle.graal.python.nodes.SpecialMethodNames.__EXIT__;
4745

4846
import java.util.List;
47+
import java.util.concurrent.Semaphore;
4948

49+
import com.oracle.graal.python.annotations.ArgumentClinic;
5050
import com.oracle.graal.python.builtins.Builtin;
5151
import com.oracle.graal.python.builtins.CoreFunctions;
52+
import com.oracle.graal.python.builtins.Python3Core;
5253
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
5354
import com.oracle.graal.python.builtins.PythonBuiltins;
5455
import com.oracle.graal.python.builtins.objects.PNone;
@@ -58,19 +59,18 @@
5859
import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
5960
import com.oracle.graal.python.nodes.function.builtins.PythonQuaternaryBuiltinNode;
6061
import com.oracle.graal.python.nodes.function.builtins.PythonTernaryBuiltinNode;
61-
import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
62-
import com.oracle.graal.python.builtins.Python3Core;
6362
import com.oracle.graal.python.nodes.function.builtins.PythonTernaryClinicBuiltinNode;
63+
import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
6464
import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider;
6565
import com.oracle.graal.python.nodes.util.CannotCastException;
6666
import com.oracle.graal.python.nodes.util.CastToJavaStringNode;
67+
import com.oracle.graal.python.runtime.PythonContext.SharedMultiprocessingData;
6768
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
6869
import com.oracle.truffle.api.dsl.Cached;
6970
import com.oracle.truffle.api.dsl.GenerateNodeFactory;
7071
import com.oracle.truffle.api.dsl.NodeFactory;
7172
import com.oracle.truffle.api.dsl.Specialization;
7273
import com.oracle.truffle.api.frame.VirtualFrame;
73-
import java.util.concurrent.Semaphore;
7474

7575
@CoreFunctions(extendClasses = {PythonBuiltinClassType.PSemLock})
7676
public class SemLockBuiltins extends PythonBuiltins {
@@ -214,28 +214,16 @@ Object doEnter(@SuppressWarnings("unused") Object handle, int kind, @SuppressWar
214214
throw raise(PythonBuiltinClassType.TypeError, ErrorMessages.ARG_D_MUST_BE_S_NOT_P, "_rebuild", 4, "str", nameObj);
215215
}
216216

217-
Semaphore semaphore;
218-
PythonLanguage lang = getLanguage();
219-
if (semaphoreExists(lang, name)) {
220-
semaphore = semaphoreGet(lang, name);
221-
} else {
217+
SharedMultiprocessingData multiprocessing = getContext().getSharedMultiprocessingData();
218+
Semaphore semaphore = multiprocessing.getNamedSemaphore(name);
219+
if (semaphore == null) {
222220
// TODO can this even happen? cpython simply creates a semlock object with the
223221
// provided handle
224222
semaphore = newSemaphore(0);
225223
}
226224
return factory().createSemLock(PythonBuiltinClassType.PSemLock, name, kind, semaphore);
227225
}
228226

229-
@TruffleBoundary
230-
private static Semaphore semaphoreGet(PythonLanguage lang, String name) {
231-
return lang.namedSemaphores.get(name);
232-
}
233-
234-
@TruffleBoundary
235-
private static boolean semaphoreExists(PythonLanguage lang, String name) {
236-
return lang.namedSemaphores.containsKey(name);
237-
}
238-
239227
@TruffleBoundary
240228
private static Semaphore newSemaphore(int value) {
241229
return new Semaphore(value);

0 commit comments

Comments
 (0)