[cppyy] fix usage of Python GIL to support threading

Author: Vipul-Cariappa

bindings/pyroot/cppyy/CPyCppyy/src/Dispatcher.cxx

@@ -41,7 +41,9 @@ static inline void InjectMethod(Cppyy::TCppMethod_t method, const std::string& m
 // possible crash
     code << "    PyObject* iself = (PyObject*)_internal_self;\n"
             "    if (!iself || iself == Py_None) {\n"
+            "      PyGILState_STATE state = PyGILState_Ensure();\n"
             "      PyErr_Warn(PyExc_RuntimeWarning, (char*)\"Call attempted on deleted python-side proxy\");\n"
+            "      PyGILState_Release(state);\n"
             "      return";
     if (retType != "void") {
         if (retType.back() != '*')
@@ -238,6 +240,7 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct,
 // object goes before the C++ one, only __del__ is called)
     if (PyMapping_HasKeyString(dct, (char*)"__destruct__")) {
         code << "  virtual ~" << derivedName << "() {\n"
+                "PyGILState_STATE state = PyGILState_Ensure();\n"
                 "    PyObject* iself = (PyObject*)_internal_self;\n"
                 "    if (!iself || iself == Py_None)\n"
                 "      return;\n"      // safe, as destructor always returns void
@@ -250,6 +253,7 @@ bool CPyCppyy::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct,
     // magic C++ exception ...
         code << "      if (!pyresult) PyErr_Print();\n"
                 "      else { Py_DECREF(pyresult); }\n"
+                "      PyGILState_Release(state);\n"
                 "  }\n";
     } else
         code << "  virtual ~" << derivedName << "() {}\n";

bindings/pyroot/cppyy/cppyy/test/test_concurrent.py

@@ -110,41 +110,19 @@ def test04_cpp_threading_with_exceptions(self):
         };
 
         struct worker {
-            worker(consumer* c) : cons(c) {
-                // Get the main interpreter state state to spawn new thread states
-                PyThreadState* state = PyThreadState_Get();
-                interpreterState = state->interp;
-            }
+            worker(consumer* c) : cons(c) {}
             ~worker() { wait(); }
 
             void start() {
                 t = std::thread([this] {
                     int counter = 0;
-
-                    // Each thread needs a Python state object
-                    // Instead of using the higher-level PyGILState_Ensure and
-                    // PyGILState_Release functions, use the PyThreadState API
-                    // directly so that we only need to create one single
-                    // PyThreadState that can be restored and released in the
-                    // "hot loop".
-                    PyThreadState *pystate = PyThreadState_New(this->interpreterState);
-
                     while (counter++ < 10)
                         try {
-                            PyEval_RestoreThread(pystate);
                             cons->process(counter);
-                            PyEval_SaveThread();
                         } catch (CPyCppyy::PyException& e) {
                             err_msg = e.what();
-                            PyEval_SaveThread();
                             return;
                         }
-
-                    PyEval_RestoreThread(pystate);
-                    PyThreadState_Clear(pystate);
-                    PyEval_SaveThread();
-
-                    PyThreadState_Delete(pystate);
                 });
             }
 
@@ -153,7 +131,6 @@ def test04_cpp_threading_with_exceptions(self):
                     t.join();
             }
 
-            PyInterpreterState* interpreterState = nullptr;
             std::thread t;
             consumer* cons = nullptr;
             std::string err_msg;
@@ -229,11 +206,7 @@ def test05_float2d_callback(self):
                 for (int i = 0; i < channels; ++i)
                     data[i] = new float[samples];
 
-                // Set Python thread because we call back into Python
-                PyGILState_STATE gstate;
-                gstate = PyGILState_Ensure();
                 p.process(data, channels, samples);
-                PyGILState_Release(gstate);
 
                 for (int i = 0; i < channels; ++i)
                     delete[] data[i];