always pydecref on the main thread

Author: marius311

src/PyCall.jl

@@ -43,6 +43,25 @@ Python executable used by PyCall in the current process.
 current_python() = _current_python[]
 const _current_python = Ref(pyprogramname)
 
+
+# PyCall isn't really thread-safe. We can ask the user to only use
+# PyCall from the main thread, but we can't prevent GC from running on
+# other threads. To prevent errors in such cases, we use a
+# synchronized queue to ensure all calls to pydecref during GC happen
+# on the main thread.
+const _pydecref_queue = []
+const _pydecref_queue_lock = Threads.SpinLock()
+function queue_pydecref(o)
+    lock(_pydecref_queue_lock)
+    try
+        push!(_pydecref_queue, o)
+    finally
+        unlock(_pydecref_queue_lock)
+    end
+    return o
+end
+
+
 #########################################################################
 
 # Mirror of C PyObject struct (for non-debugging Python builds).
@@ -76,7 +95,7 @@ mutable struct PyObject
     o::PyPtr # the actual PyObject*
     function PyObject(o::PyPtr)
         po = new(o)
-        finalizer(pydecref, po)
+        finalizer(queue_pydecref, po)
         return po
     end
 end

src/pybuffer.jl

@@ -32,7 +32,7 @@ mutable struct PyBuffer
         b = new(Py_buffer(C_NULL, PyPtr_NULL, 0, 0,
                           0, 0, C_NULL, C_NULL, C_NULL, C_NULL,
                           C_NULL, C_NULL, C_NULL))
-        finalizer(pydecref, b)
+        finalizer(queue_pydecref, b)
         return b
     end
 end

src/pyinit.jl

@@ -190,6 +190,24 @@ function __init__()
         Py_SetPythonHome(libpy_handle, pyversion, pyhome)
         Py_SetProgramName(libpy_handle, pyversion, current_python())
         ccall((@pysym :Py_InitializeEx), Cvoid, (Cint,), 0)
+
+        # _pydecref_queue used to pydecref only on the main thread:
+        @async begin
+            while !_finalized[]
+                if !isempty(_pydecref_queue)
+                    lock(_pydecref_queue_lock)
+                    o = try
+                        pop!(_pydecref_queue, o)
+                    finally
+                        unlock(_pydecref_queue_lock)
+                    end
+                    pydecref(o)
+                else
+                    yield()
+                end
+            end
+        end
+
     end
 
     # Will get reinitialized properly on first use