Merge pull request #46 from mrkn/gvl

Release GVL while the Python interpreter is running [Fix #45]

Author: mrkn

.travis.yml

@@ -39,8 +39,10 @@ matrix:
     compiler: clang
     rvm: 2.4.1
     env: PYENV_VERSION=miniconda3-4.3.11
-  allow_failure:
+  allow_failures:
   - os: osx
+  - rvm: 2.2.8
+  - rvm: 2.1.10
 
 before_install:
 - gem update --system

CHANGES.md

@@ -6,6 +6,10 @@
 
   *Naoto Takai*
 
+* Release GVL while the Python interpreter is running
+
+* Drop support Ruby 2.2.x and 2.1.x
+
 ## 1.0.3
 
 * Fix anaconda support to define the environment variable `PYTHONHOME`.

Rakefile

@@ -17,6 +17,8 @@ Rake::ExtensionTask.new('pycall', gem_spec) do |ext|
   end
 end
 
+Rake::ExtensionTask.new('pycall/spec_helper')
+
 desc "Compile binaries for mingw platform using rake-compiler-dock"
 task 'build:mingw' do
   require 'rake_compiler_dock'

appveyor.yml

@@ -1,71 +1,37 @@
 ---
 environment:
   matrix:
-  # Ruby 2.1 (32bit)
-  - ruby_version: "21"
+  # Ruby 2.4 (32bit)
+  - ruby_version: "24"
     PYTHONDIR: "C:\\Python27"
     PYTHON: "C:\\Python27\\python.exe"
 
-  - ruby_version: "21"
+  - ruby_version: "24"
     PYTHONDIR: "C:\\Python34"
     PYTHON: "C:\\Python34\\python.exe"
 
-  - ruby_version: "21"
+  - ruby_version: "24"
     PYTHONDIR: "C:\\Python35"
     PYTHON: "C:\\Python35\\python.exe"
 
-  - ruby_version: "21"
+  - ruby_version: "24"
     PYTHONDIR: "C:\\Python36"
     PYTHON: "C:\\Python36\\python.exe"
 
-  # Ruby 2.1 (64bit)
-  - ruby_version: "21-x64"
+  # Ruby 2.4 (64bit)
+  - ruby_version: "24-x64"
     PYTHONDIR: "C:\\Python27-x64"
     PYTHON: "C:\\Python27-x64\\python.exe"
 
-  - ruby_version: "21-x64"
+  - ruby_version: "24-x64"
     PYTHONDIR: "C:\\Python34-x64"
     PYTHON: "C:\\Python34-x64\\python.exe"
 
-  - ruby_version: "21-x64"
+  - ruby_version: "24-x64"
     PYTHONDIR: "C:\\Python35-x64"
     PYTHON: "C:\\Python35-x64\\python.exe"
 
-  - ruby_version: "21-x64"
-    PYTHONDIR: "C:\\Python36-x64"
-    PYTHON: "C:\\Python36-x64\\python.exe"
-
-  # Ruby 2.2 (32bit)
-  - ruby_version: "22"
-    PYTHONDIR: "C:\\Python27"
-    PYTHON: "C:\\Python27\\python.exe"
-
-  - ruby_version: "22"
-    PYTHONDIR: "C:\\Python34"
-    PYTHON: "C:\\Python34\\python.exe"
-
-  - ruby_version: "22"
-    PYTHONDIR: "C:\\Python35"
-    PYTHON: "C:\\Python35\\python.exe"
-
-  - ruby_version: "22"
-    PYTHONDIR: "C:\\Python36"
-    PYTHON: "C:\\Python36\\python.exe"
-
-  # Ruby 2.2 (64bit)
-  - ruby_version: "22-x64"
-    PYTHONDIR: "C:\\Python27-x64"
-    PYTHON: "C:\\Python27-x64\\python.exe"
-
-  - ruby_version: "22-x64"
-    PYTHONDIR: "C:\\Python34-x64"
-    PYTHON: "C:\\Python34-x64\\python.exe"
-
-  - ruby_version: "22-x64"
-    PYTHONDIR: "C:\\Python35-x64"
-    PYTHON: "C:\\Python35-x64\\python.exe"
-
-  - ruby_version: "22-x64"
+  - ruby_version: "24-x64"
     PYTHONDIR: "C:\\Python36-x64"
     PYTHON: "C:\\Python36-x64\\python.exe"
 

ext/pycall/libpython.c

@@ -154,12 +154,16 @@ pycall_init_libpython_api_table(VALUE libpython_handle)
 
   INIT_API_TABLE_ENTRY(PyIter_Next, required);
 
+  INIT_API_TABLE_ENTRY(PyEval_ThreadsInitialized, required);
+  INIT_API_TABLE_ENTRY(PyEval_InitThreads, required);
+
   INIT_API_TABLE_ENTRY(PyErr_Occurred, required);
   INIT_API_TABLE_ENTRY(PyErr_Fetch, required);
   INIT_API_TABLE_ENTRY(PyErr_Restore, required);
   INIT_API_TABLE_ENTRY(PyErr_Clear, required);
   INIT_API_TABLE_ENTRY(PyErr_SetString, required);
   INIT_API_TABLE_ENTRY(PyErr_Format, required);
+  INIT_API_TABLE_ENTRY(PyErr_SetInterrupt, required);
 
   INIT_API_TABLE_ENTRY(PyImport_ImportModule, required);
   INIT_API_TABLE_ENTRY(PyImport_ImportModuleLevel, required);

ext/pycall/pycall.c

@@ -858,6 +858,42 @@ pycall_extract_kwargs_from_ruby_hash(VALUE key, VALUE value, VALUE arg)
   return ST_CONTINUE;
 }
 
+static void
+pycall_interrupt_python_thread(void *ptr)
+{
+  Py_API(PyErr_SetInterrupt)();
+}
+
+struct call_pyobject_call_params {
+  PyObject *pycallable;
+  PyObject *args;
+  PyObject *kwargs;
+};
+
+PyObject *
+call_pyobject_call(struct call_pyobject_call_params *params)
+{
+  PyObject *res;
+  res = Py_API(PyObject_Call)(params->pycallable, params->args, params->kwargs); /* New reference */
+  return res;
+}
+
+PyObject *
+pyobject_call_without_gvl(PyObject *pycallable, PyObject *args, PyObject *kwargs)
+{
+  PyObject *res;
+  struct call_pyobject_call_params params;
+  params.pycallable = pycallable;
+  params.args = args;
+  params.kwargs = kwargs;
+
+  res = (PyObject *)rb_thread_call_without_gvl(
+      (void * (*)(void *))call_pyobject_call, (void *)&params,
+      (rb_unblock_function_t *)pycall_interrupt_python_thread, NULL);
+
+  return res;
+}
+
 static VALUE
 pycall_call_python_callable(PyObject *pycallable, int argc, VALUE *argv)
 {
@@ -905,7 +941,7 @@ pycall_call_python_callable(PyObject *pycallable, int argc, VALUE *argv)
     }
   }
 
-  res = Py_API(PyObject_Call)(pycallable, args, kwargs); /* New reference */
+  res = pyobject_call_without_gvl(pycallable, args, kwargs); /* New reference */
   if (!res) {
     pycall_pyerror_fetch_and_raise("PyObject_Call in pycall_call_python_callable");
   }
@@ -1947,6 +1983,10 @@ init_python(void)
   Py_API(Py_InitializeEx)(0);
   Py_API(PySys_SetArgvEx)(0, (char **)argv, 0);
 
+  if (!Py_API(PyEval_ThreadsInitialized)()) {
+    Py_API(PyEval_InitThreads)();
+  }
+
   /* check the availability of stackless extension */
   python_has_stackless_extension = (Py_API(PyImport_ImportModule)("stackless") != NULL);
   if (!python_has_stackless_extension) {

ext/pycall/pycall_internal.h

@@ -10,6 +10,7 @@ extern "C" {
 
 #include <ruby.h>
 #include <ruby/encoding.h>
+#include <ruby/thread.h>
 #include <assert.h>
 #include <inttypes.h>
 #include <limits.h>
@@ -34,6 +35,9 @@ pycall_integer_type_p(VALUE obj)
 #endif
 
 void ruby_debug_breakpoint();
+int ruby_thread_has_gvl_p(void);
+
+#define CALL_WITH_GVL(func, data) rb_thread_call_with_gvl((void * (*)(void *))(func), (void *)(data))
 
 /* ==== python ==== */
 
@@ -581,12 +585,16 @@ typedef struct {
 
   PyObject * (* PyIter_Next)(PyObject *);
 
+  int (* PyEval_ThreadsInitialized)(void);
+  void (* PyEval_InitThreads)(void);
+
   PyObject * (* PyErr_Occurred)(void);
   void (* PyErr_Fetch)(PyObject **, PyObject **, PyObject **);
   void (* PyErr_Restore)(PyObject *, PyObject *, PyObject *);
   void (* PyErr_Clear)(void);
   void (* PyErr_SetString)(PyObject *, const char *);   /* decoded from utf-8 */
   void (* PyErr_Format)(PyObject *, const char *, ...); /* ASCII-encoded string  */
+  void (* PyErr_SetInterrupt)(void);
 
   PyObject * (* PyImport_ImportModule)(char const*);
   PyObject * (* PyImport_ImportModuleLevel)(char const*, PyObject *, PyObject *, PyObject *, int);