Skip to content

Commit 569beae

Browse files
steve-ssteve-s
committed
[GR-66406] Bytecode DSL: use custom yield to merge generator function and generator body; improve generators support.
PullRequest: graalpython/3990
2 parents 523431c + bf416fd commit 569beae

File tree

26 files changed

+720
-384
lines changed

26 files changed

+720
-384
lines changed

‎graalpython/com.oracle.graal.python.test/src/tests/test_code.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -184,7 +184,7 @@ def inner():
184184
assert "fn doc" not in code.co_consts
185185
for const in code.co_consts:
186186
if type(const) == types.CodeType:
187-
code = const
187+
code = const
188188
assert "fn doc" in code.co_consts
189189
assert "this is fun" not in code.co_consts
190190
for const in code.co_consts:
@@ -209,7 +209,7 @@ def inner():
209209
assert "gen doc" not in code.co_consts
210210
for const in code.co_consts:
211211
if type(const) == types.CodeType:
212-
code = const
212+
code = const
213213
assert "gen doc" in code.co_consts
214214
assert "this is fun" not in code.co_consts
215215
for const in code.co_consts:
@@ -223,3 +223,11 @@ def test_consts_do_not_leak_java_types():
223223
code = compile(codestr, '<test>', 'exec')
224224
for const in code.co_consts:
225225
assert isinstance(const, (str, tuple)) or const is None
226+
227+
228+
def test_generator_and_gen_body_code_are_equal():
229+
def g():
230+
yield 42
231+
232+
x = g()
233+
assert g.__code__ is x.gi_code

‎graalpython/com.oracle.graal.python.test/src/tests/test_exception.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,6 @@ def test_raise_none(self):
196196
except TypeError:
197197
pass
198198

199-
@unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: bug in comment above")
200199
def test_generator(self):
201200
def gen():
202201
try:
@@ -208,7 +207,6 @@ def gen():
208207
self.assertEqual(next(g), 1)
209208
self.assertRaises(ZeroDivisionError, lambda: next(g))
210209

211-
@unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: bug in comment above")
212210
def test_generator_nested(self):
213211
def gen():
214212
try:
@@ -448,7 +446,6 @@ def foo():
448446
self.assertEqual(e.__context__.__context__.args[0], "first")
449447
self.assertIsNone(e.__context__.__context__.__context__)
450448

451-
@unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: bug in comment above")
452449
def test_implicit_chaining_generator(self):
453450
def gen():
454451
try:
@@ -488,7 +485,6 @@ def gen():
488485
self.assertEqual(e.__context__.__context__.args[0], "first")
489486
self.assertIsNone(e.__context__.__context__.__context__)
490487

491-
@unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: bug in comment above")
492488
def test_implicit_chaining_generator_finally(self):
493489
def gen():
494490
try:

‎graalpython/com.oracle.graal.python.test/src/tests/test_generators.py‎

Lines changed: 134 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ class ExceptionTest(unittest.TestCase):
1111
# Tests for the issue #23353: check that the currently handled exception
1212
# is correctly saved/restored in PyEval_EvalFrameEx().
1313

14-
@unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: bug in comment above")
1514
def test_except_throw(self):
1615

1716
def store_raise_exc_generator():
@@ -49,7 +48,6 @@ def store_raise_exc_generator():
4948

5049
self.assertEqual(sys.exc_info(), (None, None, None))
5150

52-
@unittest.skipIf(os.environ.get('BYTECODE_DSL_INTERPRETER'), "TODO: bug in comment above")
5351
def test_except_next(self):
5452
def gen():
5553
self.assertEqual(sys.exc_info()[0], ValueError)
@@ -107,14 +105,14 @@ def gen():
107105
# yield
108106
# self.assertIsNone(sys.exc_info()[0])
109107
# yield "done"
110-
108+
111109
# g = gen()
112110
# next(g)
113111
# try:
114112
# raise ValueError
115113
# except Exception as exc:
116114
# g.throw(exc)
117-
115+
118116
# self.assertEqual(next(g), "done")
119117
# self.assertEqual(sys.exc_info(), (None, None, None))
120118

@@ -129,7 +127,7 @@ def gen(log):
129127
yield
130128
log.append(3)
131129
return
132-
130+
133131
log = []
134132
g = gen(log)
135133
next(g)
@@ -152,7 +150,7 @@ def gen(log):
152150
yield
153151
log.append(3)
154152
return
155-
153+
156154
log = []
157155
g = gen(log)
158156
next(g)
@@ -175,7 +173,7 @@ def gen(log):
175173
yield
176174
log.append(3)
177175
return
178-
176+
179177
log = []
180178
g = gen(log)
181179
next(g)
@@ -191,7 +189,7 @@ def test_gen_from_except(self):
191189
def gen():
192190
self.assertEqual(sys.exc_info()[0], None)
193191
yield
194-
192+
195193
try:
196194
raise TypeError
197195
except TypeError:
@@ -201,7 +199,7 @@ def gen():
201199
yield
202200
self.assertIsNone(sys.exc_info()[0])
203201
yield "done"
204-
202+
205203
try:
206204
raise ValueError
207205
except ValueError:
@@ -352,3 +350,130 @@ def illegal_state_expected_cell_got_list():
352350
]
353351

354352
assert len(illegal_state_expected_cell_got_list()) == 2
353+
354+
def test_generator_exceptions_finally():
355+
def get_exc_state():
356+
assert sys.exc_info()[1] == sys.exception()
357+
return sys.exception()
358+
359+
def generator():
360+
yield get_exc_state() # 1
361+
try:
362+
yield get_exc_state() # 2
363+
3 / 0
364+
except:
365+
yield get_exc_state() # 3
366+
yield get_exc_state() # 4
367+
finally:
368+
yield get_exc_state() # 5
369+
try:
370+
raise NameError()
371+
except:
372+
yield get_exc_state() # 6
373+
try:
374+
raise KeyError()
375+
except:
376+
yield get_exc_state() # 7
377+
yield get_exc_state() # 8
378+
yield get_exc_state() # 9
379+
yield get_exc_state() # 10
380+
381+
def run_test(check_caller_ex):
382+
g = generator()
383+
assert check_caller_ex(g.send(None)) # 1
384+
assert check_caller_ex(g.send(None)) # 2
385+
assert type(g.send(None)) == ZeroDivisionError # 3
386+
assert type(g.send(None)) == ZeroDivisionError # 4
387+
assert check_caller_ex(g.send(None)) # 5
388+
assert type(g.send(None)) == NameError # 6
389+
assert type(g.send(None)) == KeyError # 7
390+
assert type(g.send(None)) == NameError # 8
391+
assert check_caller_ex(g.send(None)) # 9
392+
assert check_caller_ex(g.send(None)) # 10
393+
394+
run_test(lambda x: x is None)
395+
try:
396+
raise NotImplementedError()
397+
except:
398+
run_test(lambda x: type(x) == NotImplementedError)
399+
400+
401+
def test_generator_exceptions_complex():
402+
def get_exc_state():
403+
assert sys.exc_info()[1] == sys.exception()
404+
return sys.exception()
405+
406+
def generator():
407+
yield get_exc_state() # 1
408+
try:
409+
yield get_exc_state() # 2
410+
3 / 0
411+
except:
412+
yield get_exc_state() # 3
413+
yield get_exc_state() # 4
414+
yield get_exc_state() # 5
415+
yield get_exc_state() # 6
416+
yield get_exc_state() # 7
417+
try:
418+
yield get_exc_state() # 8
419+
raise KeyError()
420+
except:
421+
yield get_exc_state() # 9
422+
yield get_exc_state() # 10
423+
yield get_exc_state() # 11
424+
try:
425+
raise NameError()
426+
except:
427+
yield get_exc_state() # 12
428+
try:
429+
raise NotImplementedError()
430+
except:
431+
yield get_exc_state() # 13
432+
yield get_exc_state() # 14
433+
yield get_exc_state() # 15
434+
yield get_exc_state() # 16
435+
436+
g = generator()
437+
try:
438+
raise AttributeError()
439+
except:
440+
assert type(g.send(None)) == AttributeError # 1
441+
assert type(g.send(None)) == AttributeError # 2
442+
assert type(g.send(None)) == ZeroDivisionError # 3
443+
assert type(g.send(None)) == ZeroDivisionError # 4
444+
assert type(g.send(None)) == AttributeError # 5
445+
assert g.send(None) is None # 6
446+
try:
447+
raise IndexError()
448+
except:
449+
assert type(g.send(None)) == IndexError # 7
450+
assert type(g.send(None)) == IndexError # 8
451+
assert type(g.send(None)) == KeyError # 9
452+
assert type(g.send(None)) == KeyError # 10
453+
assert type(g.send(None)) == IndexError # 11
454+
try:
455+
raise TypeError()
456+
except:
457+
assert type(g.send(None)) == NameError # 12
458+
assert type(g.send(None)) == NotImplementedError # 13
459+
assert type(g.send(None)) == NameError # 14
460+
assert type(g.send(None)) == TypeError # 15
461+
assert g.send(None) is None # 16
462+
463+
g = generator()
464+
assert g.send(None) is None # 1
465+
assert g.send(None) is None # 2
466+
assert type(g.send(None)) == ZeroDivisionError # 3
467+
assert type(g.send(None)) == ZeroDivisionError # 4
468+
assert g.send(None) is None # 5
469+
assert g.send(None) is None # 6
470+
assert g.send(None) is None # 7
471+
assert g.send(None) is None # 8
472+
assert type(g.send(None)) == KeyError # 9
473+
assert type(g.send(None)) == KeyError # 10
474+
assert g.send(None) is None # 11
475+
assert type(g.send(None)) == NameError # 12
476+
assert type(g.send(None)) == NotImplementedError # 13
477+
assert type(g.send(None)) == NameError # 14
478+
assert g.send(None) is None # 15
479+
assert g.send(None) is None # 16

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags_bytecode_dsl/test_generators.txt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,29 @@ DocTestCase.test.test_generators.__test__.refleaks @ linux-x86_64
66
DocTestCase.test.test_generators.__test__.syntax @ linux-x86_64
77
DocTestCase.test.test_generators.__test__.tut @ linux-x86_64
88
DocTestCase.test.test_generators.__test__.weakref @ linux-x86_64
9+
test.test_generators.ExceptionTest.test_except_gen_except @ linux-x86_64
10+
test.test_generators.ExceptionTest.test_except_next @ linux-x86_64
11+
test.test_generators.ExceptionTest.test_except_throw @ linux-x86_64
912
test.test_generators.ExceptionTest.test_except_throw_bad_exception @ linux-x86_64
13+
test.test_generators.ExceptionTest.test_except_throw_exception_context @ linux-x86_64
14+
test.test_generators.ExceptionTest.test_gen_3_arg_deprecation_warning @ linux-x86_64
15+
test.test_generators.ExceptionTest.test_nested_gen_except_loop @ linux-x86_64
1016
test.test_generators.ExceptionTest.test_return_stopiteration @ linux-x86_64
1117
test.test_generators.ExceptionTest.test_return_tuple @ linux-x86_64
1218
test.test_generators.ExceptionTest.test_stopiteration_error @ linux-x86_64
1319
test.test_generators.ExceptionTest.test_tutorial_stopiteration @ linux-x86_64
20+
test.test_generators.FinalizationTest.test_lambda_generator @ linux-x86_64
1421
test.test_generators.GeneratorStackTraceTest.test_send_with_yield_from @ linux-x86_64
1522
test.test_generators.GeneratorStackTraceTest.test_throw_with_yield_from @ linux-x86_64
1623
test.test_generators.GeneratorTest.test_copy @ linux-x86_64
24+
test.test_generators.GeneratorTest.test_cr_frame_f_back @ linux-x86_64
25+
test.test_generators.GeneratorTest.test_gi_frame_f_back @ linux-x86_64
26+
test.test_generators.GeneratorTest.test_issue103488 @ linux-x86_64
27+
test.test_generators.GeneratorTest.test_name @ linux-x86_64
1728
test.test_generators.GeneratorTest.test_pickle @ linux-x86_64
1829
test.test_generators.GeneratorTest.test_send_non_none_to_new_gen @ linux-x86_64
30+
test.test_generators.GeneratorThrowTest.test_exception_context_with_yield @ linux-x86_64
31+
test.test_generators.GeneratorThrowTest.test_exception_context_with_yield_from @ linux-x86_64
32+
test.test_generators.GeneratorThrowTest.test_exception_context_with_yield_from_with_context_cycle @ linux-x86_64
33+
test.test_generators.GeneratorThrowTest.test_exception_context_with_yield_inside_generator @ linux-x86_64
1934
test.test_generators.GeneratorThrowTest.test_throw_after_none_exc_type @ linux-x86_64

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags_bytecode_dsl/test_zipimport_support.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
savetest.test_zipimport_support.ZipSupportTests.test_doctest_issue4197 @ linux-x86_64
12
test.test_zipimport_support.ZipSupportTests.test_doctest_main_issue4197 @ linux-x86_64
23
test.test_zipimport_support.ZipSupportTests.test_inspect_getsource_issue4223 @ linux-x86_64
34
test.test_zipimport_support.ZipSupportTests.test_pdb_issue4201 @ linux-x86_64

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,6 @@
163163
import com.oracle.graal.python.runtime.sequence.storage.NativeSequenceStorage;
164164
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
165165
import com.oracle.graal.python.util.PythonUtils;
166-
import com.oracle.truffle.api.CallTarget;
167166
import com.oracle.truffle.api.CompilerAsserts;
168167
import com.oracle.truffle.api.CompilerDirectives;
169168
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
@@ -361,15 +360,15 @@ private void runFile(PythonContext context, TruffleString inputFilePath) {
361360
throw new PythonExitException(this, 2);
362361
}
363362
PythonLanguage language = context.getLanguage();
364-
CallTarget callTarget = context.getEnv().parsePublic(source);
363+
RootCallTarget callTarget = (RootCallTarget) context.getEnv().parsePublic(source);
365364
Object[] arguments = PArguments.create();
366365
PythonModule mainModule = context.getMainModule();
367366
PDict mainDict = GetOrCreateDictNode.executeUncached(mainModule);
368367
PArguments.setGlobals(arguments, mainModule);
369368
PArguments.setSpecialArgument(arguments, mainDict);
370369
PArguments.setException(arguments, PException.NO_EXCEPTION);
371370
context.initializeMainModule(inputFilePath);
372-
Object state = ExecutionContext.IndirectCalleeContext.enterIndirect(language, context, arguments);
371+
Object state = ExecutionContext.IndirectCalleeContext.enterIndirect(language, context, arguments, callTarget);
373372
try {
374373
callTarget.call(arguments);
375374
} finally {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeNodes.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,6 @@
5656
import com.oracle.graal.python.nodes.bytecode.PBytecodeGeneratorFunctionRootNode;
5757
import com.oracle.graal.python.nodes.bytecode.PBytecodeRootNode;
5858
import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit;
59-
import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLGeneratorFunctionRootNode;
60-
import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode;
6159
import com.oracle.graal.python.nodes.util.BadOPCodeNode;
6260
import com.oracle.graal.python.runtime.ExecutionContext.IndirectCallContext;
6361
import com.oracle.graal.python.runtime.IndirectCallData;
@@ -163,9 +161,6 @@ private RootCallTarget deserializeForBytecodeInterpreter(PythonLanguage language
163161
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
164162
BytecodeDSLCodeUnit code = (BytecodeDSLCodeUnit) codeUnit;
165163
rootNode = code.createRootNode(context, PythonUtils.createFakeSource());
166-
if (code.isGeneratorOrCoroutine()) {
167-
rootNode = new PBytecodeDSLGeneratorFunctionRootNode(language, rootNode.getFrameDescriptor(), (PBytecodeDSLRootNode) rootNode, code.name);
168-
}
169164
} else {
170165
BytecodeCodeUnit code = (BytecodeCodeUnit) codeUnit;
171166
if (cellvars != null && !Arrays.equals(code.cellvars, cellvars) || freevars != null && !Arrays.equals(code.freevars, freevars)) {

0 commit comments

Comments
 (0)