Skip to content

Commit f8a8628

Browse files
msimacekmsimacek
committed
[GR-23234] Fixes for test_generators
PullRequest: graalpython/1159
2 parents 118c89c + bee61df commit f8a8628

File tree

12 files changed

+339
-75
lines changed

12 files changed

+339
-75
lines changed

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/GeneratorBuiltins.java

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,9 @@ public abstract static class SendNode extends PythonBuiltinNode {
267267
Object send(VirtualFrame frame, PGenerator self, Object value,
268268
@Cached ResumeGeneratorNode resumeGeneratorNode) {
269269
checkResumable(this, self);
270+
if (!self.isStarted() && value != PNone.NONE) {
271+
throw raise(TypeError, ErrorMessages.SEND_NON_NONE_TO_UNSTARTED_GENERATOR);
272+
}
270273
return resumeGeneratorNode.execute(frame, self, value);
271274
}
272275
}
@@ -385,7 +388,9 @@ Object sendThrow(VirtualFrame frame, PGenerator self, Object typ, Object val, @S
385388
@Cached PrepareExceptionNode prepareExceptionNode,
386389
@Cached ResumeGeneratorNode resumeGeneratorNode,
387390
@Shared("language") @CachedLanguage PythonLanguage language) {
388-
checkResumable(this, self);
391+
if (self.isRunning()) {
392+
throw raise(ValueError, ErrorMessages.GENERATOR_ALREADY_EXECUTING);
393+
}
389394
PBaseException instance = prepareExceptionNode.execute(frame, typ, val);
390395
return doThrow(frame, resumeGeneratorNode, self, instance, language);
391396
}
@@ -395,15 +400,17 @@ Object sendThrow(VirtualFrame frame, PGenerator self, Object typ, Object val, PT
395400
@Cached PrepareExceptionNode prepareExceptionNode,
396401
@Cached ResumeGeneratorNode resumeGeneratorNode,
397402
@Shared("language") @CachedLanguage PythonLanguage language) {
398-
checkResumable(this, self);
403+
if (self.isRunning()) {
404+
throw raise(ValueError, ErrorMessages.GENERATOR_ALREADY_EXECUTING);
405+
}
399406
PBaseException instance = prepareExceptionNode.execute(frame, typ, val);
400407
instance.setTraceback(tb);
401408
return doThrow(frame, resumeGeneratorNode, self, instance, language);
402409
}
403410

404411
private Object doThrow(VirtualFrame frame, ResumeGeneratorNode resumeGeneratorNode, PGenerator self, PBaseException instance, PythonLanguage language) {
405412
instance.setContext(null); // Will be filled when caught
406-
if (self.isStarted()) {
413+
if (self.isStarted() && !self.isFinished()) {
407414
instance.ensureReified();
408415
// Pass it to the generator where it will be thrown by the last yield, the location
409416
// will be filled there

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/GeneratorControlData.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
package com.oracle.graal.python.builtins.objects.generator;
2727

2828
import com.oracle.graal.python.parser.GeneratorInfo;
29-
import com.oracle.graal.python.runtime.exception.PException;
3029

3130
public final class GeneratorControlData {
3231

@@ -36,14 +35,15 @@ public final class GeneratorControlData {
3635
private final boolean[] activeFlags;
3736
private final int[] blockNodeIndices; // See {@link GeneratorBlockNode}
3837
private final Object[] forNodeIterators; // See {@link GeneratorForNode}
39-
private final PException[] activeExceptions;
38+
private final RuntimeException[] activeExceptions; // See {@link GeneratorTryExceptNode}
39+
// {@link GeneratorTryFinallyNode}
4040
private int lastYieldIndex;
4141

4242
public GeneratorControlData(GeneratorInfo generatorInfo) {
4343
this.activeFlags = new boolean[generatorInfo.getNumOfActiveFlags()];
4444
this.blockNodeIndices = new int[generatorInfo.getNumOfBlockNodes()];
4545
this.forNodeIterators = new Object[generatorInfo.getNumOfIteratorSlots()];
46-
this.activeExceptions = new PException[generatorInfo.getNumOfExceptionSlots()];
46+
this.activeExceptions = new RuntimeException[generatorInfo.getNumOfExceptionSlots()];
4747
}
4848

4949
public int getLastYieldIndex() {
@@ -79,11 +79,11 @@ public void setIteratorAt(int slot, Object value) {
7979
forNodeIterators[slot] = value;
8080
}
8181

82-
public PException getActiveException(int slot) {
82+
public RuntimeException getActiveException(int slot) {
8383
return activeExceptions[slot];
8484
}
8585

86-
public void setActiveException(int slot, PException activeException) {
86+
public void setActiveException(int slot, RuntimeException activeException) {
8787
this.activeExceptions[slot] = activeException;
8888
}
8989
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/PGenerator.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ public boolean isPRangeIterator() {
193193

194194
@Override
195195
public String toString() {
196-
return "<generator object '" + name + "' at " + hashCode() + ">";
196+
return "<generator object " + name + " at " + hashCode() + ">";
197197
}
198198

199199
public static PGenerator require(Object value) {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/iterator/IteratorBuiltins.java

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@
5656
import com.oracle.graal.python.builtins.objects.tuple.PTuple;
5757
import com.oracle.graal.python.nodes.ErrorMessages;
5858
import com.oracle.graal.python.nodes.call.special.LookupAndCallBinaryNode;
59-
import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode;
6059
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
6160
import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode;
6261
import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
@@ -278,43 +277,47 @@ public int lengthHint(PBigRangeIterator self,
278277
}
279278

280279
@Specialization(guards = "!self.isExhausted()")
281-
public double lengthHint(PDoubleSequenceIterator self) {
280+
public int lengthHint(PDoubleSequenceIterator self) {
282281
int len = self.sequence.length() - self.getIndex();
283282
return len < 0 ? 0 : len;
284283
}
285284

286285
@Specialization(guards = "!self.isExhausted()")
287-
public long lengthHint(PLongSequenceIterator self) {
286+
public int lengthHint(PLongSequenceIterator self) {
288287
int len = self.sequence.length() - self.getIndex();
289288
return len < 0 ? 0 : len;
290289
}
291290

292291
@Specialization(guards = "!self.isExhausted()")
293-
public long lengthHint(PBaseSetIterator self,
292+
public int lengthHint(PBaseSetIterator self,
294293
@Cached ConditionProfile profile) {
295294
int size = self.getSize();
296295
if (profile.profile(self.getSet().size() != size)) {
297296
return 0;
298297
}
299-
return size - self.getIndex();
298+
int len = size - self.getIndex();
299+
return len < 0 ? 0 : len;
300300
}
301301

302302
@Specialization(guards = "!self.isExhausted()")
303-
public Object lengthHint(PStringIterator self) {
304-
return self.value.length() - self.getIndex();
303+
public int lengthHint(PStringIterator self) {
304+
int len = self.value.length() - self.getIndex();
305+
return len < 0 ? 0 : len;
305306
}
306307

307308
@Specialization(guards = {"!self.isExhausted()", "self.isPSequence()"})
308-
public Object lengthHint(PSequenceIterator self,
309+
public int lengthHint(PSequenceIterator self,
309310
@Cached SequenceNodes.LenNode lenNode) {
310-
return lenNode.execute(self.getPSequence()) - self.getIndex();
311+
int len = lenNode.execute(self.getPSequence()) - self.getIndex();
312+
return len < 0 ? 0 : len;
311313
}
312314

313-
@Specialization(guards = {"!self.isExhausted()", "!self.isPSequence()"})
314-
public Object lengthHint(VirtualFrame frame, PSequenceIterator self,
315-
@Cached("create(__LEN__)") LookupAndCallUnaryNode callLen,
316-
@Cached("create(__SUB__, __RSUB__)") LookupAndCallBinaryNode callSub) {
317-
return callSub.executeObject(frame, callLen.executeObject(frame, self.getObject()), self.getIndex());
315+
@Specialization(guards = {"!self.isExhausted()", "!self.isPSequence()"}, limit = "getCallSiteInlineCacheMaxDepth()")
316+
public int lengthHint(VirtualFrame frame, PSequenceIterator self,
317+
@CachedLibrary("self.getObject()") PythonObjectLibrary lib,
318+
@Cached ConditionProfile hasFrame) {
319+
int len = lib.lengthWithFrame(self.getObject(), hasFrame, frame) - self.getIndex();
320+
return len < 0 ? 0 : len;
318321
}
319322
}
320323

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/iterator/IteratorNodes.java

Lines changed: 51 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -40,20 +40,29 @@
4040
*/
4141
package com.oracle.graal.python.builtins.objects.iterator;
4242

43+
import static com.oracle.graal.python.builtins.PythonBuiltinClassType.TypeError;
44+
import static com.oracle.graal.python.nodes.SpecialMethodNames.__LENGTH_HINT__;
45+
import static com.oracle.graal.python.nodes.SpecialMethodNames.__LEN__;
46+
4347
import com.oracle.graal.python.builtins.objects.PNone;
48+
import com.oracle.graal.python.builtins.objects.PNotImplemented;
4449
import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
50+
import com.oracle.graal.python.nodes.ErrorMessages;
4551
import com.oracle.graal.python.nodes.PGuards;
4652
import com.oracle.graal.python.nodes.PNodeWithContext;
53+
import com.oracle.graal.python.nodes.PRaiseNode;
4754
import com.oracle.graal.python.nodes.SpecialMethodNames;
4855
import com.oracle.graal.python.nodes.attributes.LookupAttributeInMRONode;
4956
import com.oracle.graal.python.nodes.call.special.CallUnaryMethodNode;
50-
import com.oracle.graal.python.nodes.util.CastToJavaIntLossyNode;
57+
import com.oracle.graal.python.nodes.object.IsBuiltinClassProfile;
58+
import com.oracle.graal.python.runtime.exception.PException;
5159
import com.oracle.truffle.api.dsl.Cached;
5260
import com.oracle.truffle.api.dsl.GenerateNodeFactory;
5361
import com.oracle.truffle.api.dsl.ImportStatic;
5462
import com.oracle.truffle.api.dsl.Specialization;
5563
import com.oracle.truffle.api.frame.VirtualFrame;
5664
import com.oracle.truffle.api.library.CachedLibrary;
65+
import com.oracle.truffle.api.profiles.ConditionProfile;
5766

5867
public abstract class IteratorNodes {
5968

@@ -83,25 +92,53 @@ static int length(@SuppressWarnings({"unused"}) VirtualFrame frame, @SuppressWar
8392
int length(VirtualFrame frame, Object iterable,
8493
@CachedLibrary("iterable") PythonObjectLibrary plib,
8594
@CachedLibrary(limit = "3") PythonObjectLibrary toInt,
86-
@Cached CastToJavaIntLossyNode cast,
8795
@Cached("create(__LEN__)") LookupAttributeInMRONode lenNode,
8896
@Cached("create(__LENGTH_HINT__)") LookupAttributeInMRONode lenHintNode,
89-
@Cached CallUnaryMethodNode dispatchGetattribute) {
90-
Object len;
97+
@Cached CallUnaryMethodNode dispatchGetattribute,
98+
@Cached IsBuiltinClassProfile errorProfile,
99+
@Cached ConditionProfile hasLenProfile,
100+
@Cached ConditionProfile hasLenghtHintProfile,
101+
@Cached PRaiseNode raiseNode) {
91102
Object clazz = plib.getLazyPythonClass(iterable);
92103
Object attrLenObj = lenNode.execute(clazz);
93-
if (attrLenObj != PNone.NO_VALUE && attrLenObj != PNone.NONE) {
94-
len = dispatchGetattribute.executeObject(frame, attrLenObj, iterable);
95-
} else {
96-
Object attrLenHintObj = lenHintNode.execute(clazz);
97-
if (attrLenHintObj != PNone.NO_VALUE && attrLenHintObj != PNone.NONE) {
98-
len = dispatchGetattribute.executeObject(frame, attrLenHintObj, iterable);
99-
} else {
100-
return -1;
104+
if (hasLenProfile.profile(attrLenObj != PNone.NO_VALUE)) {
105+
Object len = null;
106+
try {
107+
len = dispatchGetattribute.executeObject(frame, attrLenObj, iterable);
108+
} catch (PException e) {
109+
e.expect(TypeError, errorProfile);
110+
}
111+
if (len != null && len != PNotImplemented.NOT_IMPLEMENTED) {
112+
if (toInt.canBeIndex(len)) {
113+
int intLen = toInt.asSize(len);
114+
if (intLen < 0) {
115+
throw raiseNode.raise(TypeError, ErrorMessages.LEN_SHOULD_RETURN_MT_ZERO);
116+
}
117+
return intLen;
118+
} else {
119+
throw raiseNode.raise(TypeError, ErrorMessages.MUST_BE_INTEGER, __LEN__, len);
120+
}
101121
}
102122
}
103-
if (toInt.canBeIndex(len)) {
104-
return cast.execute(toInt.asIndex(len));
123+
Object attrLenHintObj = lenHintNode.execute(clazz);
124+
if (hasLenghtHintProfile.profile(attrLenHintObj != PNone.NO_VALUE)) {
125+
Object len = null;
126+
try {
127+
len = dispatchGetattribute.executeObject(frame, attrLenHintObj, iterable);
128+
} catch (PException e) {
129+
e.expect(TypeError, errorProfile);
130+
}
131+
if (len != null && len != PNotImplemented.NOT_IMPLEMENTED) {
132+
if (toInt.canBeIndex(len)) {
133+
int intLen = toInt.asSize(len);
134+
if (intLen < 0) {
135+
throw raiseNode.raise(TypeError, ErrorMessages.LENGTH_HINT_SHOULD_RETURN_MT_ZERO);
136+
}
137+
return intLen;
138+
} else {
139+
throw raiseNode.raise(TypeError, ErrorMessages.MUST_BE_INTEGER, __LENGTH_HINT__, len);
140+
}
141+
}
105142
}
106143
return -1;
107144
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/reversed/ReversedBuiltins.java

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
import com.oracle.truffle.api.dsl.Specialization;
6060
import com.oracle.truffle.api.frame.VirtualFrame;
6161
import com.oracle.truffle.api.library.CachedLibrary;
62+
import com.oracle.truffle.api.profiles.ConditionProfile;
6263

6364
@CoreFunctions(extendClasses = PythonBuiltinClassType.PReverseIterator)
6465
public class ReversedBuiltins extends PythonBuiltins {
@@ -132,7 +133,7 @@ public int lengthHint(PStringReverseIterator self) {
132133
}
133134

134135
@Specialization(guards = {"!self.isExhausted()", "self.isPSequence()"})
135-
public Object lengthHint(PSequenceReverseIterator self,
136+
public int lengthHint(PSequenceReverseIterator self,
136137
@Cached SequenceNodes.LenNode lenNode) {
137138
int len = lenNode.execute(self.getPSequence());
138139
if (len == -1) {
@@ -144,15 +145,15 @@ public Object lengthHint(PSequenceReverseIterator self,
144145
return self.index + 1;
145146
}
146147

147-
@Specialization(guards = {"!self.isExhausted()", "!self.isPSequence()"})
148-
public Object lengthHint(VirtualFrame frame, PSequenceReverseIterator self,
149-
@Cached("create(__LEN__)") LookupAndCallUnaryNode callLen,
150-
@Cached("create(__ADD__, __RADD__)") LookupAndCallBinaryNode callAdd) {
151-
Object len = callLen.executeObject(frame, self.getObject());
152-
if (len == PNone.NO_VALUE || len == PNone.NONE) {
153-
throw raise(TypeError, OBJ_HAS_NO_LEN, self);
148+
@Specialization(guards = {"!self.isExhausted()", "!self.isPSequence()"}, limit = "getCallSiteInlineCacheMaxDepth()")
149+
public int lengthHint(VirtualFrame frame, PSequenceReverseIterator self,
150+
@CachedLibrary("self.getObject()") PythonObjectLibrary lib,
151+
@Cached ConditionProfile hasFrame) {
152+
int len = lib.lengthWithFrame(self.getObject(), hasFrame, frame);
153+
if (len < self.index) {
154+
return 0;
154155
}
155-
return callAdd.executeObject(frame, self.index, 1);
156+
return self.index + 1;
156157
}
157158
}
158159

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,7 @@ public abstract class ErrorMessages {
316316
public static final String LAZY_INITIALIZATION_FAILED = "lazy initialization of type %s failed";
317317
public static final String LEFT_BRACKET_WO_RIGHT_BRACKET_IN_ARG = "')' without '(' in argument parsing";
318318
public static final String LEN_SHOULD_RETURN_MT_ZERO = "__len__() should return >= 0";
319+
public static final String LENGTH_HINT_SHOULD_RETURN_MT_ZERO = "__length_hint__() should return >= 0";
319320
public static final String LIST_ASSIGMENT_INDEX_OUT_OF_RANGE = "list assignment index out of range";
320321
public static final String LIST_INDEX_OUT_OF_RANGE = "list index out of range";
321322
public static final String LIST_LENGTH_OUT_OF_RANGE = "list length out of range";
@@ -340,6 +341,7 @@ public abstract class ErrorMessages {
340341
public static final String MUST_BE_A_CELL = "%s must be a cell";
341342
public static final String MUST_BE_BYTE_STRING_LEGTH1_NOT_P = "must be a byte string of length 1, not %p";
342343
public static final String MUST_BE_EITHER_OR = "%s: '%s' must be either %s or %s";
344+
public static final String MUST_BE_INTEGER = "%s must be an integer, not %p";
343345
public static final String MUST_BE_NON_NEGATIVE = "%s must be non-negative";
344346
public static final String MUST_BE_NON_NEGATIVE_INTEGER = "%s must be non-negative integer";
345347
public static final String MUST_BE_NUMERIC = "must be numeric, not %p";
@@ -545,4 +547,5 @@ public abstract class ErrorMessages {
545547
public static final String NATIVE_ACCESS_NOT_ALLOWED = "Cannot run any C extensions because native access is not allowed.";
546548
public static final String HPY_LOAD_ERROR = "Could not load HPy C API from %s.";
547549
public static final String CANNOT_CONVERT_NEGATIVE_VALUE_TO_UNSIGNED_INT = "can't convert negative value to unsigned int";
550+
public static final String SEND_NON_NONE_TO_UNSTARTED_GENERATOR = "can't send non-None value to a just-started generator";
548551
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/SpecialMethodNames.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,6 @@ public abstract class SpecialMethodNames {
125125
public static final String __ITRUEDIV__ = "__itruediv__";
126126
public static final String __IFLOORDIV__ = "__ifloordiv__";
127127
public static final String __IMOD__ = "__imod__";
128-
public static final String __IDIVMOD__ = "__idivmod__";
129128
public static final String __IPOW__ = "__ipow__";
130129
public static final String __ILSHIFT__ = "__ilshift__";
131130
public static final String __IRSHIFT__ = "__irshift__";

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/generator/GeneratorAccessNode.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929

3030
import com.oracle.graal.python.builtins.objects.function.PArguments;
3131
import com.oracle.graal.python.builtins.objects.generator.GeneratorControlData;
32-
import com.oracle.graal.python.runtime.exception.PException;
3332
import com.oracle.graal.python.util.PythonUtils;
3433
import com.oracle.truffle.api.CompilerDirectives;
3534
import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
@@ -137,11 +136,11 @@ public void setIterator(VirtualFrame frame, int iteratorSlot, Object value) {
137136
getControlData(frame).setIteratorAt(iteratorSlot, value);
138137
}
139138

140-
public PException getActiveException(VirtualFrame frame, int slot) {
139+
public RuntimeException getActiveException(VirtualFrame frame, int slot) {
141140
return getControlData(frame).getActiveException(slot);
142141
}
143142

144-
public void setActiveException(VirtualFrame frame, int slot, PException ex) {
143+
public void setActiveException(VirtualFrame frame, int slot, RuntimeException ex) {
145144
getControlData(frame).setActiveException(slot, ex);
146145
}
147146

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/generator/GeneratorTryExceptNode.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,8 @@ private boolean catchExceptionInGeneratorFirstTime(VirtualFrame frame, PExceptio
143143
private void catchExceptionInGeneratorCached(VirtualFrame frame) {
144144
ExceptNode[] exceptNodes = getExceptNodes();
145145
CompilerAsserts.compilationConstant(exceptNodes);
146-
PException exception = gen.getActiveException(frame, activeExceptionIndex);
146+
// try-except only stores PExceptions
147+
PException exception = (PException) gen.getActiveException(frame, activeExceptionIndex);
147148
final int matchingExceptNodeIndex = gen.getIndex(frame, exceptIndex);
148149
assert matchingExceptNodeIndex <= exceptNodes.length;
149150
boolean wasHandled = false;

0 commit comments

Comments
 (0)