Skip to content

Commit 4309944

Browse files
timfelcosminbasca
timfel
authored and
cosminbasca
committed
a bit of cleanup and flexibility when creating code objects
1 parent 1fabc50 commit 4309944

File tree

5 files changed

+114
-69
lines changed

5 files changed

+114
-69
lines changed

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeNodes.java

Lines changed: 86 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
import com.oracle.graal.python.PythonLanguage;
4444
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
4545
import com.oracle.graal.python.builtins.objects.PNone;
46-
import com.oracle.graal.python.builtins.objects.cell.PCell;
4746
import com.oracle.graal.python.builtins.objects.common.HashingStorageNodes;
4847
import com.oracle.graal.python.builtins.objects.dict.PDict;
4948
import com.oracle.graal.python.builtins.objects.function.PArguments;
@@ -70,10 +69,6 @@
7069
import com.oracle.truffle.api.RootCallTarget;
7170
import com.oracle.truffle.api.Truffle;
7271
import com.oracle.truffle.api.TruffleLanguage.ContextReference;
73-
import com.oracle.truffle.api.frame.FrameDescriptor;
74-
import com.oracle.truffle.api.frame.FrameSlot;
75-
import com.oracle.truffle.api.frame.FrameSlotKind;
76-
import com.oracle.truffle.api.frame.MaterializedFrame;
7772
import com.oracle.truffle.api.frame.VirtualFrame;
7873
import com.oracle.truffle.api.nodes.RootNode;
7974
import com.oracle.truffle.api.source.Source;
@@ -115,13 +110,12 @@ public PCode execute(VirtualFrame frame, LazyPythonClass cls, int argcount, int
115110
}
116111

117112
@TruffleBoundary
118-
private PCode createCode(LazyPythonClass cls, int argcount, int kwonlyargcount,
113+
private static PCode createCode(LazyPythonClass cls, int argcount, int kwonlyargcount,
119114
int nlocals, int stacksize, int flags,
120115
byte[] codestring, Object[] constants, Object[] names,
121116
Object[] varnames, Object[] freevars, Object[] cellvars,
122117
String filename, String name, int firstlineno,
123118
byte[] lnotab) {
124-
125119
PythonObjectFactory factory = PythonObjectFactory.getUncached();
126120
RootCallTarget callTarget = null;
127121

@@ -130,22 +124,11 @@ private PCode createCode(LazyPythonClass cls, int argcount, int kwonlyargcount,
130124
if (codestring.length > 0) {
131125
PythonCore core = PythonLanguage.getCore();
132126
if ((flags & PCode.FLAG_MODULE) == 0) {
133-
// we're looking for the function, not the module
134-
String funcdef = createFuncdef(codestring, freevars, name);
135-
MaterializedFrame frame = getFrameForFreeVars(freevars, argcount);
136-
rootNode = (RootNode) core.getParser().parse(ParserMode.File, core, Source.newBuilder(PythonLanguage.ID, funcdef, name).build(), frame);
137-
Object[] args = PArguments.create();
138-
PDict globals = factory.createDict();
139-
PArguments.setGlobals(args, globals);
140-
Object function = InvokeNode.invokeUncached(Truffle.getRuntime().createCallTarget(rootNode), args);
141-
if (function instanceof PFunction) {
142-
rootNode = ((PFunction) function).getFunctionRootNode();
143-
} else {
144-
throw PRaiseNode.getUncached().raise(PythonBuiltinClassType.ValueError, "got an invalid codestring trying to create a function code object");
145-
}
127+
String funcdef = createFuncdef(argcount, kwonlyargcount, flags, codestring, varnames, freevars, name);
128+
rootNode = getFunctionFromCode(core, factory, name, funcdef).getFunctionRootNode();
129+
rootNode = patchConstantsAndGlobalNames(rootNode, constants, names);
146130
} else {
147-
MaterializedFrame frame = getFrameForFreeVars(freevars, argcount);
148-
rootNode = (RootNode) core.getParser().parse(ParserMode.File, core, Source.newBuilder(PythonLanguage.ID, new String(codestring), name).build(), frame);
131+
rootNode = (RootNode) core.getParser().parse(ParserMode.File, core, Source.newBuilder(PythonLanguage.ID, new String(codestring), name).build(), null);
149132
assert rootNode instanceof ModuleRootNode;
150133
}
151134
callTarget = Truffle.getRuntime().createCallTarget(rootNode);
@@ -173,43 +156,95 @@ public boolean isPythonInternal() {
173156
return factory.createCode(cls, callTarget, signature, nlocals, stacksize, flags, codestring, constants, names, varnames, freevars, cellvars, filename, name, firstlineno, lnotab);
174157
}
175158

176-
private MaterializedFrame getFrameForFreeVars(Object[] freevars, int argcount) {
177-
MaterializedFrame frame = null;
178-
if (freevars.length > 0) {
179-
FrameDescriptor frameDescriptor = new FrameDescriptor();
180-
frame = Truffle.getRuntime().createMaterializedFrame(PArguments.create(argcount), frameDescriptor);
181-
for (int i = 0; i < freevars.length; i++) {
182-
Object ident = freevars[i];
183-
FrameSlot slot = frameDescriptor.addFrameSlot(ident);
184-
frameDescriptor.setFrameSlotKind(slot, FrameSlotKind.Object);
185-
frame.setObject(slot, new PCell(Truffle.getRuntime().createAssumption("cell is effectively final")));
186-
}
159+
private static RootNode patchConstantsAndGlobalNames(RootNode rootNode, Object[] constants, Object[] names) {
160+
// TODO: fill in updated constants, names, filename, and firstlineno
161+
for (int i = 0; i < constants.length; i++) {
162+
// TODO: replace constants if they changed with the new ones
163+
}
164+
for (int i = 0; i < names.length; i++) {
165+
// TODO: replace global names if they changed
187166
}
188-
return frame;
167+
return rootNode;
189168
}
190169

191-
private static String createFuncdef(byte[] codestring, Object[] freevars, String name) {
192-
CompilerAsserts.neverPartOfCompilation();
193-
String codeStr = new String(codestring);
194-
boolean isLambda = codeStr.trim().startsWith("lambda");
195-
// we build an outer function to provide the initial scoping
196-
String outernme = "_____" + System.nanoTime();
197-
StringBuilder sb = new StringBuilder();
198-
sb.append("def ").append(outernme).append("():\n");
199-
for (Object f : freevars) {
200-
String freevar = CastToJavaStringNode.getUncached().execute(f);
201-
if (freevar != null) {
202-
sb.append(" ").append(freevar).append(" = None\n");
170+
private static PFunction getFunctionFromCode(PythonCore core, PythonObjectFactory factory, String name, String funcdef) {
171+
RootNode rootNode = (RootNode) core.getParser().parse(ParserMode.File, core, Source.newBuilder(PythonLanguage.ID, funcdef, name).build(), null);
172+
Object[] args = PArguments.create();
173+
PDict globals = factory.createDict();
174+
PArguments.setGlobals(args, globals);
175+
Object returnValue = InvokeNode.invokeUncached(Truffle.getRuntime().createCallTarget(rootNode), args);
176+
if (!(returnValue instanceof PFunction)) {
177+
throw PRaiseNode.getUncached().raise(PythonBuiltinClassType.ValueError, "got an invalid codestring trying to create a function code object");
178+
}
179+
return (PFunction) returnValue;
180+
}
181+
182+
private static String createFuncdef(int argcount, int kwonlyargcount, int flags, byte[] codestring, Object[] varnames, Object[] freevars, String name) {
183+
String indent = " ";
184+
StringBuilder funcdef = new StringBuilder();
185+
long outerName = System.nanoTime();
186+
funcdef.append("def outer").append(outerName).append("():\n");
187+
for (Object freevar : freevars) {
188+
funcdef.append(indent).append(CastToJavaStringNode.getUncached().execute(freevar)).append(" = None\n");
189+
}
190+
191+
boolean isLambda = (flags & PCode.FLAG_LAMBDA) != 0;
192+
if (isLambda) {
193+
funcdef.append(indent).append("return lambda ");
194+
} else {
195+
funcdef.append(indent).append("def ").append(name).append("(");
196+
}
197+
int varnameIdx = 0;
198+
for (; varnameIdx < argcount; varnameIdx++) {
199+
funcdef.append(CastToJavaStringNode.getUncached().execute(varnames[varnameIdx])).append(",");
200+
}
201+
if ((flags & PCode.FLAG_VAR_ARGS) != 0) {
202+
// vararg name is after kwargs names
203+
funcdef.append("*").append(CastToJavaStringNode.getUncached().execute(varnames[varnameIdx + kwonlyargcount])).append(",");
204+
}
205+
for (; varnameIdx < kwonlyargcount; varnameIdx++) {
206+
funcdef.append(CastToJavaStringNode.getUncached().execute(varnames[varnameIdx])).append("=None,");
207+
}
208+
if ((flags & PCode.FLAG_VAR_KW_ARGS) != 0) {
209+
if ((flags & PCode.FLAG_VAR_ARGS) != 0) {
210+
varnameIdx++;
203211
}
212+
funcdef.append("**").append(CastToJavaStringNode.getUncached().execute(varnames[varnameIdx])).append(",");
204213
}
214+
205215
if (isLambda) {
206-
sb.append(" ").append("return ").append(codeStr);
216+
funcdef.append(": ").append(new String(codestring)).append("\n");
207217
} else {
208-
sb.append(" ").append(codeStr).append("\n");
209-
sb.append(" ").append("return ").append(name);
218+
funcdef.append("):\n");
219+
220+
String[] lines = new String(codestring).split("\n");
221+
String functionIndent = " ";
222+
String firstLineIndent = "";
223+
224+
// find indent inside the function to correctly insert nonlocal declarations
225+
for (String line : lines) {
226+
if (!line.trim().isEmpty()) {
227+
StringBuilder sb = new StringBuilder();
228+
int i = 0;
229+
char ch;
230+
while (Character.isWhitespace(ch = line.charAt(i++))) {
231+
sb.append(ch);
232+
}
233+
firstLineIndent = sb.toString();
234+
break;
235+
}
236+
}
237+
for (Object freevar : freevars) {
238+
funcdef.append(indent).append(functionIndent).append(firstLineIndent).append("nonlocal ").append(CastToJavaStringNode.getUncached().execute(freevar)).append("\n");
239+
}
240+
for (String line : lines) {
241+
funcdef.append(indent).append(functionIndent).append(line).append("\n");
242+
}
243+
funcdef.append(indent).append("return ").append(name).append("\n");
210244
}
211-
sb.append("\n\n").append(outernme).append("()");
212-
return sb.toString();
245+
246+
funcdef.append("outer").append(outerName).append("()");
247+
return funcdef.toString();
213248
}
214249

215250
private static Signature createSignature(int flags, int argcount, int kwonlyargcount, Object[] varnames) {
@@ -240,14 +275,6 @@ private static Signature createSignature(int flags, int argcount, int kwonlyargc
240275
return new Signature(PCode.takesVarKeywordArgs(flags), PCode.takesVarArgs(flags) ? argcount : -1, !PCode.takesVarArgs(flags) && kwonlyargcount > 0, paramNames, kwNames);
241276
}
242277

243-
private HashingStorageNodes.GetItemNode ensureGetItemNode() {
244-
if (getItemNode == null) {
245-
CompilerDirectives.transferToInterpreterAndInvalidate();
246-
getItemNode = insert(HashingStorageNodes.GetItemNode.create());
247-
}
248-
return getItemNode;
249-
}
250-
251278
private ContextReference<PythonContext> getContextRef() {
252279
if (contextRef == null) {
253280
CompilerDirectives.transferToInterpreterAndInvalidate();

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import com.oracle.graal.python.nodes.frame.FrameSlotIDs;
6161
import com.oracle.graal.python.nodes.frame.GlobalNode;
6262
import com.oracle.graal.python.nodes.function.FunctionDefinitionNode;
63+
import com.oracle.graal.python.nodes.function.FunctionRootNode;
6364
import com.oracle.graal.python.nodes.function.GeneratorExpressionNode;
6465
import com.oracle.graal.python.nodes.generator.GeneratorFunctionRootNode;
6566
import com.oracle.graal.python.nodes.literal.SimpleLiteralNode;
@@ -76,11 +77,11 @@
7677

7778
public final class PCode extends PythonBuiltinObject {
7879
static final String[] EMPTY_STRINGS = new String[0];
79-
static final long FLAG_GENERATOR = 32;
80-
static final long FLAG_VAR_ARGS = 0x0004;
81-
static final long FLAG_VAR_KW_ARGS = 0x0008;
82-
static final long FLAG_MODULE = 0x0040; // CO_NOFREE on CPython, we only set it on
83-
// modules
80+
static final long FLAG_VAR_ARGS = 0x4;
81+
static final long FLAG_VAR_KW_ARGS = 0x8;
82+
static final long FLAG_LAMBDA = 0x10; // CO_NESTED on CPython, not needed
83+
static final long FLAG_GENERATOR = 0x20;
84+
static final long FLAG_MODULE = 0x40; // CO_NOFREE on CPython, we use it on modules, it's redundant anyway
8485

8586
private final RootCallTarget callTarget;
8687
private final Signature signature;
@@ -300,6 +301,10 @@ private static int extractFlags(RootNode rootNode) {
300301
if (NodeUtil.findFirstNodeInstance(funcRootNode, ReadVarKeywordsNode.class) != null) {
301302
flags |= FLAG_VAR_KW_ARGS;
302303
}
304+
// 0x10 - lambda, not on CPython
305+
if (funcRootNode instanceof FunctionRootNode && ((FunctionRootNode) funcRootNode).isLambda()) {
306+
flags |= FLAG_LAMBDA;
307+
}
303308
}
304309
return flags;
305310
}
@@ -310,10 +315,15 @@ private static byte[] extractCodeString(RootNode rootNode) {
310315
if (rootNode instanceof GeneratorFunctionRootNode) {
311316
funcRootNode = ((GeneratorFunctionRootNode) rootNode).getFunctionRootNode();
312317
}
313-
SourceSection sourceSection = funcRootNode.getSourceSection();
314-
if (sourceSection != null) {
315-
return sourceSection.getCharacters().toString().getBytes();
318+
if (funcRootNode instanceof PClosureRootNode) {
319+
SourceSection sourceSection = funcRootNode.getSourceSection();
320+
if (sourceSection != null) {
321+
String src = sourceSection.getCharacters().toString();
322+
// strip the header, we're only interested in the actual source
323+
return src.replaceFirst("[^:]+:[ \\t]*", "").getBytes();
324+
}
316325
}
326+
// no code for non-user functions
317327
return new byte[0];
318328
}
319329

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/BuiltinNames.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@
4141
package com.oracle.graal.python.nodes;
4242

4343
public abstract class BuiltinNames {
44+
// special strings
45+
public static final String LAMBDA_NAME = "<lambda>";
46+
4447
// special arg names
4548
public static final String SELF = "self";
4649

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/function/FunctionRootNode.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.oracle.graal.python.builtins.objects.cell.PCell;
3030
import com.oracle.graal.python.builtins.objects.function.PArguments;
3131
import com.oracle.graal.python.builtins.objects.function.Signature;
32+
import com.oracle.graal.python.nodes.BuiltinNames;
3233
import com.oracle.graal.python.nodes.PClosureFunctionRootNode;
3334
import com.oracle.graal.python.nodes.expression.ExpressionNode;
3435
import com.oracle.graal.python.parser.ExecutionCellSlots;
@@ -86,6 +87,10 @@ public FunctionRootNode copyWithNewSignature(Signature newSignature) {
8687
return new FunctionRootNode(PythonLanguage.getCurrent(), getSourceSection(), functionName, isGenerator, isRewritten, getFrameDescriptor(), uninitializedBody, executionCellSlots, newSignature);
8788
}
8889

90+
public boolean isLambda() {
91+
return functionName.equals(BuiltinNames.LAMBDA_NAME);
92+
}
93+
8994
@Override
9095
public String getName() {
9196
return functionName;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/sst/FactorySSTVisitor.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141

4242
package com.oracle.graal.python.parser.sst;
4343

44+
import static com.oracle.graal.python.nodes.BuiltinNames.LAMBDA_NAME;
4445
import static com.oracle.graal.python.nodes.BuiltinNames.__BUILD_CLASS__;
4546
import static com.oracle.graal.python.nodes.SpecialAttributeNames.__ANNOTATIONS__;
4647
import static com.oracle.graal.python.nodes.SpecialAttributeNames.__CLASSCELL__;
@@ -903,8 +904,7 @@ public PNode visit(ImportSSTNode node) {
903904

904905
@Override
905906
public PNode visit(LambdaSSTNode node) {
906-
907-
String funcname = "anonymous";
907+
String funcname = LAMBDA_NAME;
908908
ScopeInfo oldScope = scopeEnvironment.getCurrentScope();
909909
scopeEnvironment.setCurrentScope(node.functionScope);
910910
/**

0 commit comments

Comments
 (0)