Skip to content

Commit 94ba18f

Browse files
author
Adam Hrbac
committed
[GR-28389] Implement async iterators and generators
PullRequest: graalpython/2681
2 parents e9c17e7 + 512b937 commit 94ba18f

37 files changed

+1842
-97
lines changed

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_asyncgen.txt

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,46 @@
1+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_aiter_bad_args
2+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_aiter_idempotent
3+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_anext_bad_args
4+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_aclose_after_exhaustion
5+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_aclose_compatible_with_get_stack
6+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_aclose_twice_with_different_coros
7+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_aiter
8+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_aiter_class
9+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_01
10+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_02
11+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_03
12+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_aclose_10
13+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_aclose_11
14+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_anext_04
15+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_anext_05
16+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_anext_06
17+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_anext_stopiteration
18+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_anext_tuple
19+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_asend_01
20+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_asend_03
21+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_athrow_01
22+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_athrow_02
23+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_athrow_03
24+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_athrow_stopiteration
25+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_athrow_tuple
26+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_shutdown_01
27+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_shutdown_02
28+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_shutdown_exception_01
29+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_await_same_aclose_coro_twice
30+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_await_same_anext_coro_twice
31+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_expression_01
32+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenAsyncioTest.test_asyncgen_nonstarted_hooks_are_cancellable
133
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenSyntaxTest.test_async_gen_syntax_01
234
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenSyntaxTest.test_async_gen_syntax_02
335
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenSyntaxTest.test_async_gen_syntax_03
436
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenSyntaxTest.test_async_gen_syntax_04
537
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenSyntaxTest.test_async_gen_syntax_05
38+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenTest.test_async_gen_exception_03
39+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenTest.test_async_gen_exception_04
40+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenTest.test_async_gen_exception_05
41+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenTest.test_async_gen_exception_07
42+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenTest.test_async_gen_exception_08
43+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenTest.test_async_gen_exception_09
44+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenTest.test_async_gen_exception_11
45+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenTest.test_async_gen_iteration_01
46+
*graalpython.lib-python.3.test.test_asyncgen.AsyncGenTest.test_async_gen_iteration_02

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_coroutines.txt

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,13 @@
88
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_1
99
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_10
1010
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_11
11+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_12
1112
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_13
13+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_14
14+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_15
1215
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_16
1316
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_2
17+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_3
1418
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_4
1519
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_5
1620
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_await_6
@@ -20,15 +24,51 @@
2024
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_comp_1
2125
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_comp_10
2226
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_comp_2
27+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_comp_3
28+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_comp_4
29+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_comp_4_2
30+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_comp_5
31+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_comp_6
32+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_comp_7
2333
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_comp_8
34+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_comp_9
35+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_copy
36+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_coro_wrapper_send_stop_iterator
37+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_coro_wrapper_send_tuple
38+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_cr_await
39+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_for_1
40+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_for_11
41+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_for_3
42+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_for_4
43+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_for_6
44+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_for_7
45+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_for_8
46+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_for_assign_raising_stop_async_iteration
47+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_for_assign_raising_stop_async_iteration_2
48+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_for_stop_iteration
49+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_for_tuple
50+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_1
51+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_10
52+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_11
53+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_15
54+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_16
55+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_17
56+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_18
2457
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_19
58+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_3
59+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_4
60+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_5
2561
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_6
2662
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_func_8
2763
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_gen_1
64+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_pickle
2865
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_stack_in_coroutine_throw
2966
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_with_1
3067
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_with_10
3168
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_with_12
69+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_with_2
70+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_with_3
71+
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_with_4
3272
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_with_5
3373
*graalpython.lib-python.3.test.test_coroutines.CoroutineTest.test_with_9
3474
*graalpython.lib-python.3.test.test_coroutines.TokenizerRegrTest.test_oneline_defs

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,9 @@
5555
import java.util.ServiceLoader;
5656
import java.util.logging.Level;
5757

58+
import com.oracle.graal.python.builtins.objects.asyncio.AsyncGenSendBuiltins;
59+
import com.oracle.graal.python.builtins.objects.asyncio.AsyncGenThrowBuiltins;
60+
import com.oracle.graal.python.builtins.objects.asyncio.AsyncGeneratorBuiltins;
5861
import org.graalvm.nativeimage.ImageInfo;
5962

6063
import com.oracle.graal.python.PythonLanguage;
@@ -713,7 +716,10 @@ private static PythonBuiltins[] initializeBuiltins(boolean nativeAccessAllowed)
713716
new GraalHPyDebugModuleBuiltins(),
714717

715718
// _asyncio
716-
new AsyncioModuleBuiltins()));
719+
new AsyncioModuleBuiltins(),
720+
new AsyncGeneratorBuiltins(),
721+
new AsyncGenSendBuiltins(),
722+
new AsyncGenThrowBuiltins()));
717723
if (hasProfilerTool) {
718724
builtins.add(new LsprofModuleBuiltins());
719725
builtins.add(LsprofModuleBuiltins.newProfilerBuiltins());

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,9 @@ public enum PythonBuiltinClassType implements TruffleObject {
173173
PList("list", J_BUILTINS, LIST_M_FLAGS),
174174
PMappingproxy("mappingproxy", Flags.PRIVATE_DERIVED_WODICT, MAPPINGPROXY_M_FLAGS),
175175
PMemoryView("memoryview", J_BUILTINS, Flags.PUBLIC_DERIVED_WODICT, MEMORYVIEW_M_FLAGS),
176+
PAsyncGenASend("async_generator_asend", Flags.PRIVATE_DERIVED_WODICT),
177+
PAsyncGenAThrow("async_generator_athrow", Flags.PRIVATE_DERIVED_WODICT),
178+
PAsyncGenAWrappedValue("async_generator_wrapped_value", Flags.PRIVATE_DERIVED_WODICT),
176179
PMethod("method", Flags.PRIVATE_DERIVED_WODICT),
177180
PMMap("mmap", "mmap", MMAP_M_FLAGS),
178181
PNone("NoneType", Flags.PRIVATE_DERIVED_WODICT, NONE_M_FLAGS),

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/AsyncioModuleBuiltins.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,7 @@ public Object getCurrentLoop() {
117117
public abstract static class InternalSetRunningLoop extends PythonUnaryBuiltinNode {
118118
@Specialization
119119
public Object setCurrentLoop(Object loop) {
120-
getContext().getThreadState(getLanguage()).setRunningEventLoop(loop);
121-
120+
getContext().getThreadState(getLanguage()).setRunningEventLoop(loop == PNone.NONE ? null : loop);
122121
return PNone.NONE;
123122
}
124123
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,7 @@
201201
import com.oracle.graal.python.nodes.attributes.SetAttributeNode;
202202
import com.oracle.graal.python.nodes.builtins.ListNodes;
203203
import com.oracle.graal.python.nodes.builtins.ListNodes.ConstructListNode;
204+
import com.oracle.graal.python.nodes.bytecode.GetAIterNode;
204205
import com.oracle.graal.python.nodes.bytecode.PBytecodeRootNode;
205206
import com.oracle.graal.python.nodes.call.CallDispatchNode;
206207
import com.oracle.graal.python.nodes.call.CallNode;
@@ -210,6 +211,7 @@
210211
import com.oracle.graal.python.nodes.call.special.LookupAndCallBinaryNode;
211212
import com.oracle.graal.python.nodes.call.special.LookupAndCallTernaryNode;
212213
import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode;
214+
import com.oracle.graal.python.nodes.call.special.LookupSpecialMethodSlotNode;
213215
import com.oracle.graal.python.nodes.classes.IsSubtypeNode;
214216
import com.oracle.graal.python.nodes.expression.BinaryArithmetic;
215217
import com.oracle.graal.python.nodes.expression.BinaryArithmetic.AddNode;
@@ -2584,4 +2586,35 @@ class InitializeBuildClass {
25842586
return cls;
25852587
}
25862588
}
2589+
2590+
@Builtin(name = "anext", minNumOfPositionalArgs = 1)
2591+
@GenerateNodeFactory
2592+
public abstract static class ANext extends PythonUnaryBuiltinNode {
2593+
@Specialization
2594+
public Object doGeneric(VirtualFrame frame, Object asyncIter,
2595+
@Bind("this") Node inliningTarget,
2596+
@Cached(parameters = "ANext") LookupSpecialMethodSlotNode getANext,
2597+
@Cached InlinedGetClassNode getAsyncIterType,
2598+
@Cached PRaiseNode.Lazy raiseNoANext,
2599+
@Cached CallUnaryMethodNode callANext,
2600+
@Cached TypeNodes.GetNameNode getName) {
2601+
// TODO: two argument anext
2602+
Object type = getAsyncIterType.execute(inliningTarget, asyncIter);
2603+
Object getter = getANext.execute(frame, type, asyncIter);
2604+
if (getter == NO_VALUE) {
2605+
throw raiseNoANext.get(inliningTarget).raise(PythonBuiltinClassType.TypeError, ErrorMessages.OBJECT_NOT_ASYNCGEN, getName.execute(type));
2606+
}
2607+
return callANext.executeObject(frame, getter, asyncIter);
2608+
}
2609+
}
2610+
2611+
@Builtin(name = "aiter", minNumOfPositionalArgs = 1)
2612+
@GenerateNodeFactory
2613+
public abstract static class AIter extends PythonUnaryBuiltinNode {
2614+
@Specialization
2615+
public Object doGeneric(VirtualFrame frame, Object arg,
2616+
@Cached(neverDefault = true) GetAIterNode aiter) {
2617+
return aiter.execute(frame, arg);
2618+
}
2619+
}
25872620
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ContextvarsModuleBuiltins.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -84,12 +84,12 @@ protected Object copyCtx() {
8484
@Builtin(name = "ContextVar", minNumOfPositionalArgs = 2, parameterNames = {"cls", "name", "default"}, constructsClass = PythonBuiltinClassType.ContextVar)
8585
@GenerateNodeFactory
8686
public abstract static class ContextVarNode extends PythonTernaryBuiltinNode {
87-
@Specialization
87+
@Specialization(guards = "isNoValue(def)")
8888
protected Object construct(Object cls, TruffleString name, @SuppressWarnings("unused") PNone def) {
8989
return constructDef(cls, name, PContextVar.NO_DEFAULT);
9090
}
9191

92-
@Specialization(guards = "!isPNone(def)")
92+
@Specialization(guards = "!isNoValue(def)")
9393
protected Object constructDef(@SuppressWarnings("unused") Object cls, TruffleString name, Object def) {
9494
return factory().createContextVar(name, def);
9595
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1136,6 +1136,12 @@ Object getProfile() {
11361136
abstract static class SetAsyncgenHooks extends PythonBuiltinNode {
11371137
@Specialization
11381138
Object setAsyncgenHooks(Object firstIter, Object finalizer) {
1139+
if (firstIter != PNone.NO_VALUE && firstIter != PNone.NONE) {
1140+
getContext().getThreadState(getLanguage()).setAsyncgenFirstIter(firstIter);
1141+
} else if (firstIter == PNone.NONE) {
1142+
getContext().getThreadState(getLanguage()).setAsyncgenFirstIter(null);
1143+
}
1144+
// Ignore finalizer, since we don't have a useful place to call it
11391145
return PNone.NONE;
11401146
}
11411147
}
@@ -1146,7 +1152,9 @@ abstract static class GetAsyncgenHooks extends PythonBuiltinNode {
11461152
@Specialization
11471153
Object setAsyncgenHooks() {
11481154
// TODO: use asyncgen_hooks object
1149-
return factory().createTuple(new Object[]{PNone.NONE, PNone.NONE});
1155+
PythonContext.PythonThreadState threadState = getContext().getThreadState(getLanguage());
1156+
Object firstiter = threadState.getAsyncgenFirstIter();
1157+
return factory().createTuple(new Object[]{firstiter == null ? PNone.NONE : firstiter, PNone.NONE});
11501158
}
11511159
}
11521160

0 commit comments

Comments
 (0)