Updates for generator changes

Summary:
Generators changed upstream to fix some issues with free-threading: python/cpython@08bc03f

This merges the interpreter loop changes that were present in our copied bytecodes, brings in `gen_try_set_executing` from ceval_macros.h and wraps gen_getyieldfrom which is now where the implementation of _PyGen_yf lives.

There's also some changes in `PyEval_GetAwaitable` which we current "borrow" by copying it. But the only thing we change is `_PyCoro_GetAwaitableIter` -> `JitCoro_GetAwaitableIter`. So this replaces it with a real borrow with ifdef's to rename things, that way we won't need to perform as many updates in the future.

Reviewed By: itamaro

Differential Revision: D90126048

fbshipit-source-id: 1e27d7697d4358b1f0b2e384a14e39f842f9665e

Author: DinoV

cinderx/Interpreter/3.15/Includes/ceval_macros.h

@@ -419,6 +419,32 @@ check_periodics(PyThreadState *tstate) {
     return 0;
 }
 
+// Mark the generator as executing. Returns true if the state was changed,
+// false if it was already executing or finished.
+static inline bool
+gen_try_set_executing(PyGenObject *gen)
+{
+#ifdef Py_GIL_DISABLED
+    if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) {
+        int8_t frame_state = _Py_atomic_load_int8_relaxed(&gen->gi_frame_state);
+        while (frame_state < FRAME_EXECUTING) {
+            if (_Py_atomic_compare_exchange_int8(&gen->gi_frame_state,
+                                                 &frame_state,
+                                                 FRAME_EXECUTING)) {
+                return true;
+            }
+        }
+    }
+#endif
+    // Use faster non-atomic modifications in the GIL-enabled build and when
+    // the object is uniquely referenced in the free-threaded build.
+    if (gen->gi_frame_state < FRAME_EXECUTING) {
+        gen->gi_frame_state = FRAME_EXECUTING;
+        return true;
+    }
+    return false;
+}
+
 // CO_NO_MONITORING_EVENTS indicates the code object is read-only and therefore
 // cannot have code-extra data added.
 #define CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE                            \

cinderx/Interpreter/3.15/Includes/generated_cases.c.h

@@ -9014,7 +9014,6 @@
                 PyGenObject* gen = _PyGen_GetGeneratorFromFrame(frame);
                 assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1);
                 assert(oparg == 0 || oparg == 1);
-                gen->gi_frame_state = FRAME_SUSPENDED + oparg;
                 _PyStackRef temp = retval;
                 stack_pointer += -1;
                 ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
@@ -9026,6 +9025,8 @@
                 frame = tstate->current_frame = frame->previous;
                 CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE
                 gen_frame->previous = NULL;
+                ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD;
+                FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg);
                 assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER);
                 #if TIER_ONE
                 assert(
@@ -12031,19 +12032,18 @@
             // _SEND
             {
                 v = stack_pointer[-1];
-                PyObject* receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
-                PyObject* retval_o;
+                PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
+                PyObject *retval_o;
                 assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
                 if (!IS_PEP523_HOOKED(tstate) &&
-                    (Py_TYPE(receiver_o) == &PyGen_Type ||
-                        Py_TYPE(receiver_o) == &PyCoro_Type) &&
-                    ((PyGenObject*)receiver_o)->gi_frame_state < FRAME_EXECUTING) {
-                    PyGenObject* gen = (PyGenObject*)receiver_o;
-                    _PyInterpreterFrame* gen_frame = &gen->gi_iframe;
+                    (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) &&
+                    gen_try_set_executing((PyGenObject *)receiver_o))
+                {
+                    PyGenObject *gen = (PyGenObject *)receiver_o;
+                    _PyInterpreterFrame *gen_frame = &gen->gi_iframe;
                     _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v));
                     stack_pointer += -1;
                     ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
-                    gen->gi_frame_state = FRAME_EXECUTING;
                     gen->gi_exc_state.previous_item = tstate->exc_info;
                     tstate->exc_info = &gen->gi_exc_state;
                     assert( 2u + oparg <= UINT16_MAX);
@@ -12056,10 +12056,12 @@
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                } else {
+                }
+                else {
                     _PyFrame_SetStackPointer(frame, stack_pointer);
-                    retval_o = PyObject_CallMethodOneArg(
-                        receiver_o, &_Py_ID(send), PyStackRef_AsPyObjectBorrow(v));
+                    retval_o = PyObject_CallMethodOneArg(receiver_o,
+                        &_Py_ID(send),
+                        PyStackRef_AsPyObjectBorrow(v));
                     stack_pointer = _PyFrame_GetStackPointer(frame);
                 }
                 if (retval_o == NULL) {
@@ -12077,7 +12079,8 @@
                     if (err == 0) {
                         assert(retval_o != NULL);
                         JUMPBY(oparg);
-                    } else {
+                    }
+                    else {
                         stack_pointer += -1;
                         ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
                         _PyFrame_SetStackPointer(frame, stack_pointer);
@@ -13788,7 +13791,6 @@
             PyGenObject* gen = _PyGen_GetGeneratorFromFrame(frame);
             assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1);
             assert(oparg == 0 || oparg == 1);
-            gen->gi_frame_state = FRAME_SUSPENDED + oparg;
             _PyStackRef temp = retval;
             stack_pointer += -1;
             ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
@@ -13800,6 +13802,8 @@
             frame = tstate->current_frame = frame->previous;
             CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE
             gen_frame->previous = NULL;
+            ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD;
+            FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg);
             assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER);
             #if TIER_ONE
             assert(

cinderx/Interpreter/3.15/borrowed-ceval.c.template

@@ -73,70 +73,16 @@
 // @Borrow function _PyEvalFramePushAndInit_Ex from Python/ceval.c
 
 // Cinder specific adapted functions
-static PyObject *
-Ci_PyEval_GetANext(PyObject *aiter)
-{
-    unaryfunc getter = NULL;
-    PyObject *next_iter = NULL;
-    PyTypeObject *type = Py_TYPE(aiter);
-    if (PyAsyncGen_CheckExact(aiter)) {
-        return type->tp_as_async->am_anext(aiter);
-    }
-    if (type->tp_as_async != NULL){
-        getter = type->tp_as_async->am_anext;
-    }
+#define _PyCoro_GetAwaitableIter JitCoro_GetAwaitableIter
+#define _PyEval_GetAwaitable Ci_PyEval_GetAwaitable
+#define _PyEval_GetANext Ci_PyEval_GetANext
 
-    if (getter != NULL) {
-        next_iter = (*getter)(aiter);
-        if (next_iter == NULL) {
-            return NULL;
-        }
-    }
-    else {
-        PyErr_Format(PyExc_TypeError,
-                        "'async for' requires an iterator with "
-                        "__anext__ method, got %.100s",
-                        type->tp_name);
-        return NULL;
-    }
+// @Borrow function _PyEval_GetANext from Python/ceval.c
+// @Borrow function _PyEval_GetAwaitable from Python/ceval.c
 
-    // CX: Changed from _PyCoro_GetAwaitableIter
-    PyObject *awaitable = JitCoro_GetAwaitableIter(next_iter);
-    if (awaitable == NULL) {
-        _PyErr_FormatFromCause(
-            PyExc_TypeError,
-            "'async for' received an invalid object "
-            "from __anext__: %.100s",
-            Py_TYPE(next_iter)->tp_name);
-    }
-    Py_DECREF(next_iter);
-    return awaitable;
-}
-
-static PyObject *
-Ci_PyEval_GetAwaitable(PyObject *iterable, int oparg)
-{
-    // CX: Changed from _PyCoro_GetAwaitableIter
-    PyObject *iter = JitCoro_GetAwaitableIter(iterable);
-
-    if (iter == NULL) {
-        _PyEval_FormatAwaitableError(PyThreadState_GET(),
-            Py_TYPE(iterable), oparg);
-    }
-    else if (PyCoro_CheckExact(iter)) {
-        PyObject *yf = _PyGen_yf((PyGenObject*)iter);
-        if (yf != NULL) {
-            /* `iter` is a coroutine object that is being
-                awaited, `yf` is a pointer to the current awaitable
-                being awaited on. */
-            Py_DECREF(yf);
-            Py_CLEAR(iter);
-            _PyErr_SetString(PyThreadState_GET(), PyExc_RuntimeError,
-                                "coroutine is being awaited already");
-        }
-    }
-    return iter;
-}
+#undef _PyCoro_GetAwaitableIter
+#undef _PyEval_GetAwaitable
+#undef _PyEval_GetANext
 
 #if Py_TAIL_CALL_INTERP
 #include "cinderx/Interpreter/cinderx_opcode_targets.h"

cinderx/Interpreter/3.15/ceval.h

@@ -1069,8 +1069,12 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func,
 }
 
 // Cinder specific adapted functions
-static PyObject *
-Ci_PyEval_GetANext(PyObject *aiter)
+#define _PyCoro_GetAwaitableIter JitCoro_GetAwaitableIter
+#define _PyEval_GetAwaitable Ci_PyEval_GetAwaitable
+#define _PyEval_GetANext Ci_PyEval_GetANext
+
+PyObject *
+_PyEval_GetANext(PyObject *aiter)
 {
     unaryfunc getter = NULL;
     PyObject *next_iter = NULL;
@@ -1096,8 +1100,7 @@ Ci_PyEval_GetANext(PyObject *aiter)
         return NULL;
     }
 
-    // CX: Changed from _PyCoro_GetAwaitableIter
-    PyObject *awaitable = JitCoro_GetAwaitableIter(next_iter);
+    PyObject *awaitable = _PyCoro_GetAwaitableIter(next_iter);
     if (awaitable == NULL) {
         _PyErr_FormatFromCause(
             PyExc_TypeError,
@@ -1108,32 +1111,32 @@ Ci_PyEval_GetANext(PyObject *aiter)
     Py_DECREF(next_iter);
     return awaitable;
 }
-
-static PyObject *
-Ci_PyEval_GetAwaitable(PyObject *iterable, int oparg)
+PyObject *
+_PyEval_GetAwaitable(PyObject *iterable, int oparg)
 {
-    // CX: Changed from _PyCoro_GetAwaitableIter
-    PyObject *iter = JitCoro_GetAwaitableIter(iterable);
+    PyObject *iter = _PyCoro_GetAwaitableIter(iterable);
 
     if (iter == NULL) {
         _PyEval_FormatAwaitableError(PyThreadState_GET(),
             Py_TYPE(iterable), oparg);
     }
     else if (PyCoro_CheckExact(iter)) {
-        PyObject *yf = _PyGen_yf((PyGenObject*)iter);
-        if (yf != NULL) {
-            /* `iter` is a coroutine object that is being
-                awaited, `yf` is a pointer to the current awaitable
-                being awaited on. */
-            Py_DECREF(yf);
+        PyCoroObject *coro = (PyCoroObject *)iter;
+        int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(coro->cr_frame_state);
+        if (frame_state == FRAME_SUSPENDED_YIELD_FROM) {
+            /* `iter` is a coroutine object that is being awaited. */
             Py_CLEAR(iter);
             _PyErr_SetString(PyThreadState_GET(), PyExc_RuntimeError,
-                                "coroutine is being awaited already");
+                             "coroutine is being awaited already");
         }
     }
     return iter;
 }
 
+#undef _PyCoro_GetAwaitableIter
+#undef _PyEval_GetAwaitable
+#undef _PyEval_GetANext
+
 #if Py_TAIL_CALL_INTERP
 #include "cinderx/Interpreter/cinderx_opcode_targets.h"
 #include "cinderx/Interpreter/3.14/Includes/generated_cases.c.h"

cinderx/Interpreter/3.15/cinder-bytecodes.c

@@ -455,46 +455,48 @@ static PyObject* dummy_func(
     }
 
     override op(_SEND, (receiver, v-- receiver, retval)) {
-      PyObject* receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
-      PyObject* retval_o;
+      PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
+      PyObject *retval_o;
       assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
       if (!IS_PEP523_HOOKED(tstate) &&
-          (Py_TYPE(receiver_o) == &PyGen_Type ||
-           Py_TYPE(receiver_o) == &PyCoro_Type) &&
-          ((PyGenObject*)receiver_o)->gi_frame_state < FRAME_EXECUTING) {
-        PyGenObject* gen = (PyGenObject*)receiver_o;
-        _PyInterpreterFrame* gen_frame = &gen->gi_iframe;
-        _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v));
-        DEAD(v);
-        SYNC_SP();
-        gen->gi_frame_state = FRAME_EXECUTING;
-        gen->gi_exc_state.previous_item = tstate->exc_info;
-        tstate->exc_info = &gen->gi_exc_state;
-        assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX);
-        frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg);
-        assert(gen_frame->previous == NULL);
-        gen_frame->previous = frame;
-        DISPATCH_INLINED(gen_frame);
+          (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) &&
+          gen_try_set_executing((PyGenObject *)receiver_o))
+      {
+          PyGenObject *gen = (PyGenObject *)receiver_o;
+          _PyInterpreterFrame *gen_frame = &gen->gi_iframe;
+          _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v));
+          DEAD(v);
+          SYNC_SP();
+          gen->gi_exc_state.previous_item = tstate->exc_info;
+          tstate->exc_info = &gen->gi_exc_state;
+          assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX);
+          frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg);
+          assert(gen_frame->previous == NULL);
+          gen_frame->previous = frame;
+          DISPATCH_INLINED(gen_frame);
       }
       if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) {
-        retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o);
-      } else {
-        retval_o = PyObject_CallMethodOneArg(
-            receiver_o, &_Py_ID(send), PyStackRef_AsPyObjectBorrow(v));
+          retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o);
+      }
+      else {
+          retval_o = PyObject_CallMethodOneArg(receiver_o,
+                                                &_Py_ID(send),
+                                                PyStackRef_AsPyObjectBorrow(v));
       }
       if (retval_o == NULL) {
-        int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
-        if (matches) {
-          _PyEval_MonitorRaise(tstate, frame, this_instr);
-        }
-        int err = _PyGen_FetchStopIterationValue(&retval_o);
-        if (err == 0) {
-          assert(retval_o != NULL);
-          JUMPBY(oparg);
-        } else {
-          PyStackRef_CLOSE(v);
-          ERROR_IF(true);
-        }
+          int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration);
+          if (matches) {
+              _PyEval_MonitorRaise(tstate, frame, this_instr);
+          }
+          int err = _PyGen_FetchStopIterationValue(&retval_o);
+          if (err == 0) {
+              assert(retval_o != NULL);
+              JUMPBY(oparg);
+          }
+          else {
+              PyStackRef_CLOSE(v);
+              ERROR_IF(true);
+          }
       }
       PyStackRef_CLOSE(v);
       retval = PyStackRef_FromPyObjectSteal(retval_o);
@@ -1410,7 +1412,6 @@ static PyObject* dummy_func(
       PyGenObject* gen = _PyGen_GetGeneratorFromFrame(frame);
       assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1);
       assert(oparg == 0 || oparg == 1);
-      gen->gi_frame_state = FRAME_SUSPENDED + oparg;
       _PyStackRef temp = retval;
       DEAD(retval);
       SAVE_STACK();
@@ -1424,6 +1425,8 @@ static PyObject* dummy_func(
       CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE
 
       gen_frame->previous = NULL;
+      ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD;
+      FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg);
       /* We don't know which of these is relevant here, so keep them equal */
       assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER);
 #if TIER_ONE

cinderx/UpstreamBorrow/borrowed-3.15.c.template

@@ -539,4 +539,9 @@ void Cix_gen_dealloc_with_custom_free(PyObject* obj) {
     gen_dealloc(obj);
 }
 
+// @Borrow function gen_getyieldfrom from Objects/genobject.c [3.15]
+PyObject * _PyGen_yf(PyGenObject *gen) {
+  return gen_getyieldfrom((PyObject *)gen, NULL);
+}
+
 // @Borrow function _PyTuple_MaybeUntrack from Objects/tupleobject.c [3.15]