Skip to content

Commit 9e205b1

Browse files
msimacekmsimacek
committed
[GR-23234] Implement gi_yieldfrom
PullRequest: graalpython/1069
2 parents 4f7f717 + 446e900 commit 9e205b1

27 files changed

+356
-263
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ language
5252
Python3.g4.stamp
5353
*.orig
5454
/graalpython/com.oracle.graal.python.test/src/tests/cpyext/Py*.c
55+
/graalpython/com.oracle.graal.python.test/src/tests/cpyext/_Py*.c
5556
/graalpython/com.oracle.graal.python.test/src/tests/cpyext/Test*.c
5657
/graalpython/com.oracle.graal.python.test/src/tests/cpyext/*.sha256
5758
/graalpython/com.oracle.graal.python.test/src/tests/cpyext/PointerEquality_Primitive.c

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/ParserTreePrinter.java

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141

4242
package com.oracle.graal.python.test.parser;
4343

44+
import java.util.Set;
45+
4446
import com.oracle.graal.python.builtins.objects.function.Signature;
4547
import com.oracle.graal.python.nodes.ModuleRootNode;
4648
import com.oracle.graal.python.nodes.PClosureFunctionRootNode;
@@ -74,14 +76,14 @@
7476
import com.oracle.graal.python.nodes.statement.ImportFromNode;
7577
import com.oracle.graal.python.nodes.statement.ImportNode;
7678
import com.oracle.graal.python.parser.ExecutionCellSlots;
79+
import com.oracle.graal.python.parser.GeneratorInfo;
7780
import com.oracle.graal.python.runtime.ExecutionContext;
7881
import com.oracle.truffle.api.frame.FrameDescriptor;
7982
import com.oracle.truffle.api.frame.FrameSlot;
8083
import com.oracle.truffle.api.nodes.Node;
8184
import com.oracle.truffle.api.nodes.NodeVisitor;
8285
import com.oracle.truffle.api.nodes.RootNode;
8386
import com.oracle.truffle.api.source.SourceSection;
84-
import java.util.Set;
8587

8688
public class ParserTreePrinter implements NodeVisitor {
8789
private final int MAX_TEXT_LENGTH = 20;
@@ -195,13 +197,14 @@ public boolean visit(GeneratorFunctionDefinitionNode node) {
195197
level++;
196198
addFrameDescriptor(node.getFrameDescriptor());
197199
indent(level);
198-
sb.append("Active Flags: ").append(node.getNumOfActiveFlags());
200+
GeneratorInfo generatorInfo = node.getGeneratorInfo();
201+
sb.append("Active Flags: ").append(generatorInfo.getNumOfActiveFlags());
199202
newLine();
200203
indent(level);
201-
sb.append("For Nodes: ").append(node.getNumOfGeneratorForNode());
204+
sb.append("For Nodes: ").append(generatorInfo.getNumOfIteratorSlots());
202205
newLine();
203206
indent(level);
204-
sb.append("Block Nodes: ").append(node.getNumOfGeneratorBlockNode());
207+
sb.append("Block Nodes: ").append(generatorInfo.getNumOfBlockNodes());
205208
newLine();
206209
addFunctionDefinitionNode(node);
207210
level--;
@@ -350,13 +353,14 @@ public boolean visit(GeneratorExpressionNode node) {
350353
addFrameDescriptor(node.getEnclosingFrameDescriptor());
351354
level--;
352355
indent(level);
353-
sb.append("Active Flags: ").append(node.getNumOfActiveFlags());
356+
GeneratorInfo generatorInfo = node.getGeneratorInfo();
357+
sb.append("Active Flags: ").append(generatorInfo.getNumOfActiveFlags());
354358
newLine();
355359
indent(level);
356-
sb.append("For Nodes: ").append(node.getNumOfGeneratorForNode());
360+
sb.append("For Nodes: ").append(generatorInfo.getNumOfIteratorSlots());
357361
newLine();
358362
indent(level);
359-
sb.append("Block Nodes: ").append(node.getNumOfGeneratorBlockNode());
363+
sb.append("Block Nodes: ").append(generatorInfo.getNumOfBlockNodes());
360364
newLine();
361365
indent(level);
362366
sb.append("Is Enclosing Frame Generator: ").append(node.isEnclosingFrameGenerator());

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_generators.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@
1010
*graalpython.lib-python.3.test.test_generators.GeneratorTest.test_copy
1111
*graalpython.lib-python.3.test.test_generators.GeneratorTest.test_pickle
1212
*graalpython.lib-python.3.test.test_generators.SignalAndYieldFromTest.test_raise_and_yield_from
13+
*graalpython.lib-python.3.test.test_generators.YieldFromTests.test_generator_gi_yieldfrom

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/frame/FrameBuiltins.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,8 @@ int get(PFrame self) {
112112
@GenerateNodeFactory
113113
public abstract static class GetLastiNode extends PythonBuiltinNode {
114114
@Specialization
115-
int get(@SuppressWarnings("unused") PFrame self) {
116-
return -1;
115+
int get(PFrame self) {
116+
return self.getLasti();
117117
}
118118
}
119119

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/frame/PFrame.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ public final class PFrame extends PythonBuiltinObject {
6565
private Node location;
6666
private RootCallTarget callTarget;
6767
private int line = -2;
68+
private int lasti = -1;
6869

6970
private PFrame.Reference backref = null;
7071

@@ -309,4 +310,16 @@ public void setArguments(Object[] arguments2) {
309310
public void setLocation(Node location) {
310311
this.location = location;
311312
}
313+
314+
/**
315+
* Last bytecode instruction. Since we don't have bytecode this is -1 by default, but can be set
316+
* to a different value to distinguish started generators from unstarted
317+
*/
318+
public int getLasti() {
319+
return lasti;
320+
}
321+
322+
public void setLasti(int lasti) {
323+
this.lasti = lasti;
324+
}
312325
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/GeneratorBuiltins.java

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@
6868
import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
6969
import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode;
7070
import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
71+
import com.oracle.graal.python.nodes.generator.AbstractYieldNode;
72+
import com.oracle.graal.python.nodes.generator.YieldFromNode;
7173
import com.oracle.graal.python.nodes.object.IsBuiltinClassProfile;
7274
import com.oracle.graal.python.runtime.exception.PException;
7375
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
@@ -531,18 +533,40 @@ static Object getFrame(PGenerator self,
531533
MaterializedFrame generatorFrame = PArguments.getGeneratorFrame(self.getArguments());
532534
PDict locals = PArguments.getGeneratorFrameLocals(generatorFrame);
533535
Object[] arguments = PArguments.create();
534-
// TODO msimacek - this way, the generator will always have lineno == 1
535-
Node location = self.getCurrentCallTarget().getRootNode();
536+
Node location = self.getCurrentYieldNode();
537+
if (location == null) {
538+
location = self.getCurrentCallTarget().getRootNode();
539+
}
536540
PFrame frame = factory.createPFrame(PFrame.Reference.EMPTY, location, locals, false);
537541
PArguments.setGlobals(arguments, PArguments.getGlobals(self.getArguments()));
538542
PArguments.setClosure(arguments, PArguments.getClosure(self.getArguments()));
539543
PArguments.setGeneratorFrame(arguments, generatorFrame);
540544
frame.setArguments(arguments);
545+
if (self.isStarted()) {
546+
// Hack: Fake bytecode to make inspect.getgeneratorstate distinguish suspended
547+
// and unstarted generators
548+
frame.setLasti(10000);
549+
}
541550
return frame;
542551
}
543552
}
544553
}
545554

555+
@Builtin(name = "gi_yieldfrom", minNumOfPositionalArgs = 1, isGetter = true)
556+
@GenerateNodeFactory
557+
public abstract static class GetYieldFromNode extends PythonUnaryBuiltinNode {
558+
@Specialization
559+
static Object getYieldFrom(PGenerator self) {
560+
AbstractYieldNode currentYield = self.getCurrentYieldNode();
561+
if (currentYield instanceof YieldFromNode) {
562+
int iteratorSlot = ((YieldFromNode) currentYield).getIteratorSlot();
563+
return PArguments.getControlDataFromGeneratorArguments(self.getArguments()).getIteratorAt(iteratorSlot);
564+
} else {
565+
return PNone.NONE;
566+
}
567+
}
568+
}
569+
546570
@Builtin(name = __REPR__, minNumOfPositionalArgs = 1)
547571
@GenerateNodeFactory
548572
abstract static class ReprNode extends PythonUnaryBuiltinNode {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/GeneratorControlData.java

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
*/
2626
package com.oracle.graal.python.builtins.objects.generator;
2727

28+
import com.oracle.graal.python.parser.GeneratorInfo;
2829
import com.oracle.graal.python.runtime.exception.PException;
2930

3031
public final class GeneratorControlData {
@@ -38,11 +39,11 @@ public final class GeneratorControlData {
3839
private final PException[] activeExceptions;
3940
private int lastYieldIndex;
4041

41-
public GeneratorControlData(int numOfActiveFlags, int numOfGeneratorBlockNode, int numOfGeneratorForNode, int numOfTryNode) {
42-
this.activeFlags = new boolean[numOfActiveFlags];
43-
this.blockNodeIndices = new int[numOfGeneratorBlockNode];
44-
this.forNodeIterators = new Object[numOfGeneratorForNode];
45-
this.activeExceptions = new PException[numOfTryNode];
42+
public GeneratorControlData(GeneratorInfo generatorInfo) {
43+
this.activeFlags = new boolean[generatorInfo.getNumOfActiveFlags()];
44+
this.blockNodeIndices = new int[generatorInfo.getNumOfBlockNodes()];
45+
this.forNodeIterators = new Object[generatorInfo.getNumOfIteratorSlots()];
46+
this.activeExceptions = new PException[generatorInfo.getNumOfExceptionSlots()];
4647
}
4748

4849
public int getLastYieldIndex() {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/PGenerator.java

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@
3232
import com.oracle.graal.python.builtins.objects.function.PArguments;
3333
import com.oracle.graal.python.builtins.objects.iterator.PRangeIterator;
3434
import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject;
35+
import com.oracle.graal.python.nodes.generator.AbstractYieldNode;
3536
import com.oracle.graal.python.parser.ExecutionCellSlots;
37+
import com.oracle.graal.python.parser.GeneratorInfo;
3638
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
3739
import com.oracle.truffle.api.Assumption;
3840
import com.oracle.truffle.api.CompilerAsserts;
@@ -65,16 +67,17 @@ public final class PGenerator extends PythonBuiltinObject {
6567
private int currentCallTarget;
6668
private final Object iterator;
6769
private final boolean isPRangeIterator;
70+
private final GeneratorInfo generatorInfo;
6871
// running means it is currently on the stack, not just started
6972
private boolean running;
7073

7174
public static PGenerator create(String name, String qualname, RootCallTarget[] callTargets, FrameDescriptor frameDescriptor, Object[] arguments, PCell[] closure,
72-
ExecutionCellSlots cellSlots, int numOfActiveFlags, int numOfGeneratorBlockNode, int numOfGeneratorForNode, int numOfGeneratorTryNode, PythonObjectFactory factory,
75+
ExecutionCellSlots cellSlots, GeneratorInfo generatorInfo, PythonObjectFactory factory,
7376
Object iterator) {
7477
/*
7578
* Setting up the persistent frame in {@link #arguments}.
7679
*/
77-
GeneratorControlData generatorArgs = new GeneratorControlData(numOfActiveFlags, numOfGeneratorBlockNode, numOfGeneratorForNode, numOfGeneratorTryNode);
80+
GeneratorControlData generatorArgs = new GeneratorControlData(generatorInfo);
7881
Object[] generatorFrameArguments = PArguments.create();
7982
MaterializedFrame generatorFrame = Truffle.getRuntime().createMaterializedFrame(generatorFrameArguments, frameDescriptor);
8083
PArguments.setGeneratorFrame(arguments, generatorFrame);
@@ -96,7 +99,7 @@ public static PGenerator create(String name, String qualname, RootCallTarget[] c
9699
}
97100
assignCells(generatorFrame, cellVarSlots, cellVarAssumptions);
98101
PArguments.setGeneratorFrameLocals(generatorFrameArguments, factory.createDictLocals(generatorFrame));
99-
return new PGenerator(name, qualname, callTargets, frameDescriptor, arguments, closure, iterator);
102+
return new PGenerator(name, qualname, callTargets, generatorInfo, frameDescriptor, arguments, closure, iterator);
100103
}
101104

102105
@ExplodeLoop
@@ -115,11 +118,12 @@ private static void assignClosure(PCell[] closure, MaterializedFrame generatorFr
115118
}
116119
}
117120

118-
private PGenerator(String name, String qualname, RootCallTarget[] callTargets, FrameDescriptor frameDescriptor, Object[] arguments, PCell[] closure, Object iterator) {
121+
private PGenerator(String name, String qualname, RootCallTarget[] callTargets, GeneratorInfo generatorInfo, FrameDescriptor frameDescriptor, Object[] arguments, PCell[] closure, Object iterator) {
119122
super(PythonBuiltinClassType.PGenerator, PythonBuiltinClassType.PGenerator.newInstance());
120123
this.name = name;
121124
this.qualname = qualname;
122125
this.callTargets = callTargets;
126+
this.generatorInfo = generatorInfo;
123127
this.currentCallTarget = 0;
124128
this.frameDescriptor = frameDescriptor;
125129
this.arguments = arguments;
@@ -146,6 +150,15 @@ public RootCallTarget getCurrentCallTarget() {
146150
return callTargets[currentCallTarget];
147151
}
148152

153+
public AbstractYieldNode getCurrentYieldNode() {
154+
if (currentCallTarget == 0 || running || finished) {
155+
// Not stopped on a yield
156+
return null;
157+
}
158+
// Call target indices are yield indices + 1, see AbstractYieldNode
159+
return generatorInfo.getYieldNodes()[currentCallTarget - 1];
160+
}
161+
149162
public boolean isStarted() {
150163
return currentCallTarget != 0 && !running;
151164
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/NodeFactory.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@
104104
import com.oracle.graal.python.nodes.subscript.GetItemNode;
105105
import com.oracle.graal.python.nodes.subscript.SliceLiteralNode;
106106
import com.oracle.graal.python.parser.ExecutionCellSlots;
107+
import com.oracle.graal.python.parser.GeneratorInfo;
107108
import com.oracle.truffle.api.frame.FrameDescriptor;
108109
import com.oracle.truffle.api.frame.FrameSlot;
109110
import com.oracle.truffle.api.nodes.Node;
@@ -209,12 +210,12 @@ public StatementNode createBreakTarget(StatementNode forNode, StatementNode orel
209210
return new BreakTargetNode(forNode, orelse);
210211
}
211212

212-
public YieldNode createYield(ExpressionNode right) {
213-
return new YieldNode(right);
213+
public YieldNode createYield(ExpressionNode right, GeneratorInfo.Mutable generatorInfo) {
214+
return new YieldNode(right, generatorInfo);
214215
}
215216

216-
public YieldFromNode createYieldFrom(ExpressionNode right) {
217-
return new YieldFromNode(right);
217+
public YieldFromNode createYieldFrom(ExpressionNode right, GeneratorInfo.Mutable generatorInfo) {
218+
return new YieldFromNode(right, generatorInfo);
218219
}
219220

220221
public ExpressionNode createIntegerLiteral(int value) {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/function/GeneratorExpressionNode.java

Lines changed: 7 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -31,45 +31,37 @@
3131
import com.oracle.graal.python.nodes.generator.GeneratorFunctionRootNode;
3232
import com.oracle.graal.python.parser.DefinitionCellSlots;
3333
import com.oracle.graal.python.parser.ExecutionCellSlots;
34+
import com.oracle.graal.python.parser.GeneratorInfo;
3435
import com.oracle.graal.python.runtime.exception.PException;
3536
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
3637
import com.oracle.truffle.api.CompilerAsserts;
3738
import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
3839
import com.oracle.truffle.api.RootCallTarget;
3940
import com.oracle.truffle.api.frame.FrameDescriptor;
4041
import com.oracle.truffle.api.frame.VirtualFrame;
41-
import com.oracle.truffle.api.nodes.RootNode;
4242

4343
public final class GeneratorExpressionNode extends ExpressionDefinitionNode {
4444
private final String name;
4545
private final String qualname;
4646
private final RootCallTarget callTarget;
4747
@CompilationFinal(dimensions = 1) private RootCallTarget[] callTargets;
4848
private final FrameDescriptor frameDescriptor;
49-
private final int numOfActiveFlags;
50-
private final int numOfGeneratorBlockNode;
51-
private final int numOfGeneratorForNode;
52-
private final int numOfGeneratorTryNode;
49+
private final GeneratorInfo generatorInfo;
5350

5451
@CompilationFinal private FrameDescriptor enclosingFrameDescriptor;
5552
@CompilationFinal private boolean isEnclosingFrameGenerator;
56-
@CompilationFinal private boolean isOptimized;
5753
@Child private ExpressionNode getIterator;
5854
@Child private PythonObjectFactory factory = PythonObjectFactory.create();
5955

6056
public GeneratorExpressionNode(String name, String qualname, RootCallTarget callTarget, ExpressionNode getIterator, FrameDescriptor descriptor, DefinitionCellSlots definitionCellSlots,
61-
ExecutionCellSlots executionCellSlots,
62-
int numOfActiveFlags, int numOfGeneratorBlockNode, int numOfGeneratorForNode, int numOfGeneratorTryNode) {
57+
ExecutionCellSlots executionCellSlots, GeneratorInfo generatorInfo) {
6358
super(definitionCellSlots, executionCellSlots);
6459
this.name = name;
6560
this.qualname = qualname;
6661
this.callTarget = callTarget;
6762
this.getIterator = getIterator;
6863
this.frameDescriptor = descriptor;
69-
this.numOfActiveFlags = numOfActiveFlags;
70-
this.numOfGeneratorBlockNode = numOfGeneratorBlockNode;
71-
this.numOfGeneratorForNode = numOfGeneratorForNode;
72-
this.numOfGeneratorTryNode = numOfGeneratorTryNode;
64+
this.generatorInfo = generatorInfo;
7365
}
7466

7567
public String getName() {
@@ -101,32 +93,8 @@ public void setEnclosingFrameGenerator(boolean value) {
10193
isEnclosingFrameGenerator = value;
10294
}
10395

104-
public boolean isOptimized() {
105-
return isOptimized;
106-
}
107-
108-
public void setAsOptimized() {
109-
isOptimized = true;
110-
}
111-
112-
public int getNumOfActiveFlags() {
113-
return numOfActiveFlags;
114-
}
115-
116-
public int getNumOfGeneratorBlockNode() {
117-
return numOfGeneratorBlockNode;
118-
}
119-
120-
public int getNumOfGeneratorForNode() {
121-
return numOfGeneratorForNode;
122-
}
123-
124-
public int getNumOfGeneratorTryNode() {
125-
return numOfGeneratorTryNode;
126-
}
127-
128-
public RootNode getFunctionRootNode() {
129-
return callTarget.getRootNode();
96+
public GeneratorInfo getGeneratorInfo() {
97+
return generatorInfo;
13098
}
13199

132100
@Override
@@ -151,7 +119,7 @@ public Object execute(VirtualFrame frame) {
151119

152120
PCell[] closure = getClosureFromGeneratorOrFunctionLocals(frame);
153121
return factory.createGenerator(name, qualname, callTargets, frameDescriptor, arguments, closure, executionCellSlots,
154-
numOfActiveFlags, numOfGeneratorBlockNode, numOfGeneratorForNode, numOfGeneratorTryNode, iterator);
122+
generatorInfo, iterator);
155123
}
156124

157125
@Override

0 commit comments

Comments
 (0)