[cppyy] fix usage of Python GIL to support threading

bindings/pyroot/cppyy/CPyCppyy/src/DispatchPtr.cxx

@@ -10,23 +10,25 @@
 //-----------------------------------------------------------------------------
 PyObject* CPyCppyy::DispatchPtr::Get(bool borrowed) const
 {
+    PyGILState_STATE state = PyGILState_Ensure();
+    PyObject* result = nullptr;
     if (fPyHardRef) {
         if (!borrowed) Py_INCREF(fPyHardRef);
-        return fPyHardRef;
-    }
-    if (fPyWeakRef) {
-        PyObject* disp = CPyCppyy_GetWeakRef(fPyWeakRef);
-        if (disp) {               // dispatcher object disappeared?
-            if (borrowed) Py_DECREF(disp);
-            return disp;
+        result = fPyHardRef;
+    } else if (fPyWeakRef) {
+        result = CPyCppyy_GetWeakRef(fPyWeakRef);
+        if (result) {               // dispatcher object disappeared?
+            if (borrowed) Py_DECREF(result);
         }
     }
-    return nullptr;
+    PyGILState_Release(state);
+    return result;
 }
 
 //-----------------------------------------------------------------------------
 CPyCppyy::DispatchPtr::DispatchPtr(PyObject* pyobj, bool strong) : fPyHardRef(nullptr)
 {
+    PyGILState_STATE state = PyGILState_Ensure();
     if (strong) {
         Py_INCREF(pyobj);
         fPyHardRef = pyobj;
@@ -36,15 +38,18 @@ CPyCppyy::DispatchPtr::DispatchPtr(PyObject* pyobj, bool strong) : fPyHardRef(nu
         fPyWeakRef = PyWeakref_NewRef(pyobj, nullptr);
     }
     ((CPPInstance*)pyobj)->SetDispatchPtr(this);
+    PyGILState_Release(state);
 }
 
 //-----------------------------------------------------------------------------
 CPyCppyy::DispatchPtr::DispatchPtr(const DispatchPtr& other, void* cppinst) : fPyWeakRef(nullptr)
 {
+    PyGILState_STATE state = PyGILState_Ensure();
     PyObject* pyobj = other.Get(false /* not borrowed */);
     fPyHardRef = pyobj ? (PyObject*)((CPPInstance*)pyobj)->Copy(cppinst) : nullptr;
     if (fPyHardRef) ((CPPInstance*)fPyHardRef)->SetDispatchPtr(this);
     Py_XDECREF(pyobj);
+    PyGILState_Release(state);
 }
 
 //-----------------------------------------------------------------------------
@@ -53,6 +58,7 @@ CPyCppyy::DispatchPtr::~DispatchPtr() {
 // of a dispatcher intermediate, then this delete is from the C++ side, and Python
 // is "notified" by nulling out the reference and an exception will be raised on
 // continued access
+    PyGILState_STATE state = PyGILState_Ensure();
     if (fPyWeakRef) {
         PyObject* pyobj = CPyCppyy_GetWeakRef(fPyWeakRef);
         if (pyobj && ((CPPScope*)Py_TYPE(pyobj))->fFlags & CPPScope::kIsPython)
@@ -63,11 +69,13 @@ CPyCppyy::DispatchPtr::~DispatchPtr() {
         ((CPPInstance*)fPyHardRef)->GetObjectRaw() = nullptr;
         Py_DECREF(fPyHardRef);
     }
+    PyGILState_Release(state);
 }
 
 //-----------------------------------------------------------------------------
 CPyCppyy::DispatchPtr& CPyCppyy::DispatchPtr::assign(const DispatchPtr& other, void* cppinst)
 {
+    PyGILState_STATE state = PyGILState_Ensure();
     if (this != &other) {
         Py_XDECREF(fPyWeakRef); fPyWeakRef = nullptr;
         Py_XDECREF(fPyHardRef);
@@ -76,6 +84,7 @@ CPyCppyy::DispatchPtr& CPyCppyy::DispatchPtr::assign(const DispatchPtr& other, v
         if (fPyHardRef) ((CPPInstance*)fPyHardRef)->SetDispatchPtr(this);
         Py_XDECREF(pyobj);
     }
+    PyGILState_Release(state);
     return *this;
 }
 
@@ -93,8 +102,10 @@ void CPyCppyy::DispatchPtr::PythonOwns()
 void CPyCppyy::DispatchPtr::CppOwns()
 {
 // C++ maintains the hardref, keeping the PyObject alive w/o outstanding ref
+    PyGILState_STATE state = PyGILState_Ensure();
     if (fPyWeakRef) {
         fPyHardRef = CPyCppyy_GetWeakRef(fPyWeakRef);
         Py_DECREF(fPyWeakRef); fPyWeakRef = nullptr;
     }
+    PyGILState_Release(state);
 }

bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx

@@ -41,7 +41,9 @@ static inline void InjectMethod(Cppyy::TCppMethod_t method, const std::string& m
 // possible crash
     code << "    PyObject* iself = (PyObject*)_internal_self;\n"
             "    if (!iself || iself == Py_None) {\n"
+            "      PyGILState_STATE state = PyGILState_Ensure();\n"
             "      PyErr_Warn(PyExc_RuntimeWarning, (char*)\"Call attempted on deleted python-side proxy\");\n"
+            "      PyGILState_Release(state);\n"
             "      return";
     if (retType != "void") {
         if (retType.back() != '*')
@@ -50,12 +52,13 @@ static inline void InjectMethod(Cppyy::TCppMethod_t method, const std::string& m
             code << " nullptr";
     }
     code << ";\n"
-            "    }\n"
-            "    Py_INCREF(iself);\n";
+            "    }\n";
 
 // start actual function body
     Utility::ConstructCallbackPreamble(retType, argtypes, code);
 
+    code << "    Py_INCREF(iself);\n";
+
 // perform actual method call
 #if PY_VERSION_HEX < 0x03000000
     code << "    PyObject* mtPyName = PyString_FromString(\"" << mtCppName << "\");\n" // TODO: intern?
@@ -238,6 +241,7 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct,
 // object goes before the C++ one, only __del__ is called)
     if (PyMapping_HasKeyString(dct, (char*)"__destruct__")) {
         code << "  virtual ~" << derivedName << "() {\n"
+                "PyGILState_STATE state = PyGILState_Ensure();\n"
                 "    PyObject* iself = (PyObject*)_internal_self;\n"
                 "    if (!iself || iself == Py_None)\n"
                 "      return;\n"      // safe, as destructor always returns void
@@ -250,6 +254,7 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct,
     // magic C++ exception ...
         code << "      if (!pyresult) PyErr_Print();\n"
                 "      else { Py_DECREF(pyresult); }\n"
+                "      PyGILState_Release(state);\n"
                 "  }\n";
     } else
         code << "  virtual ~" << derivedName << "() {}\n";
@@ -450,8 +455,11 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct,
 
 // provide an accessor to re-initialize after round-tripping from C++ (internal)
     code << "\n  static PyObject* _get_dispatch(" << derivedName << "* inst) {\n"
+            "    PyGILState_STATE state = PyGILState_Ensure();\n"
             "    PyObject* res = (PyObject*)inst->_internal_self;\n"
-            "    Py_XINCREF(res); return res;\n  }";
+            "    Py_XINCREF(res);\n"
+            "    PyGILState_Release(state);\n"
+            "    return res;\n  }";
 
 // finish class declaration
     code << "};\n}";

bindings/pyroot/cppyy/cppyy/test/test_concurrent.py

@@ -1,6 +1,6 @@
 import pytest
 from pytest import raises, skip, mark
 from support import IS_MAC_ARM
 
 
 class TestCONCURRENT:
@@ -26,8 +26,8 @@
     def test01_simple_threads(self):
         """Run basic Python threads"""
 
         import cppyy
         import threading
 
         cppyy.gbl.Workers.calc.__release_gil__ = True
 
@@ -63,8 +63,8 @@
         """Time-out with threads"""
 
         return
         import cppyy
         import threading, time
 
         cppyy.cppdef("""\
         namespace test12_timeout {
@@ -110,41 +110,19 @@
         };
 
         struct worker {
-            worker(consumer* c) : cons(c) {
-                // Get the main interpreter state state to spawn new thread states
-                PyThreadState* state = PyThreadState_Get();
-                interpreterState = state->interp;
-            }
+            worker(consumer* c) : cons(c) {}
             ~worker() { wait(); }
 
             void start() {
                 t = std::thread([this] {
                     int counter = 0;
-
-                    // Each thread needs a Python state object
-                    // Instead of using the higher-level PyGILState_Ensure and
-                    // PyGILState_Release functions, use the PyThreadState API
-                    // directly so that we only need to create one single
-                    // PyThreadState that can be restored and released in the
-                    // "hot loop".
-                    PyThreadState *pystate = PyThreadState_New(this->interpreterState);
-
                     while (counter++ < 10)
                         try {
-                            PyEval_RestoreThread(pystate);
                             cons->process(counter);
-                            PyEval_SaveThread();
                         } catch (CPyCppyy::PyException& e) {
                             err_msg = e.what();
-                            PyEval_SaveThread();
                             return;
                         }
-
-                    PyEval_RestoreThread(pystate);
-                    PyThreadState_Clear(pystate);
-                    PyEval_SaveThread();
-
-                    PyThreadState_Delete(pystate);
                 });
             }
 
@@ -153,7 +131,6 @@
                     t.join();
             }
 
-            PyInterpreterState* interpreterState = nullptr;
             std::thread t;
             consumer* cons = nullptr;
             std::string err_msg;
@@ -229,11 +206,7 @@
                 for (int i = 0; i < channels; ++i)
                     data[i] = new float[samples];
 
-                // Set Python thread because we call back into Python
-                PyGILState_STATE gstate;
-                gstate = PyGILState_Ensure();
                 p.process(data, channels, samples);
-                PyGILState_Release(gstate);
 
                 for (int i = 0; i < channels; ++i)
                     delete[] data[i];
@@ -265,8 +238,8 @@
     def test06_overload_reuse_in_threads(self):
         """Threads reuse overload objects; check for clashes"""
 
         import cppyy
         import threading
 
         cppyy.cppdef("""\
         namespace CPPOverloadReuse {
@@ -296,8 +269,8 @@
           # the culprit here is occasional std::system_error if a thread can not be joined
             skip("JIT exceptions can not be caught in JITed code on Mac ARM")
 
         import cppyy
         import threading
 
         cppyy.cppdef("""\
         namespace CPPOverloadReuse {