stop using Requires.jl (use extensions instead)

Author: Christopher Doris

Project.toml

@@ -10,33 +10,41 @@ Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 UnsafePointers = "e17b2a0c-0bdf-430a-bd0c-3a23cae4ff39"
 
 [compat]
 Aqua = "0 - 999"
+CategoricalArrays = "0.10, 1"
 CondaPkg = "0.2.30"
 Dates = "1"
 Libdl = "1"
 MacroTools = "0.5"
 Markdown = "1"
 Pkg = "1"
 PyCall = "1"
-Requires = "1"
 Serialization = "1"
 Tables = "1"
 Test = "1"
 TestItemRunner = "0 - 999"
 UnsafePointers = "1"
 julia = "1.9"
 
+[extensions]
+PyCallExt = "PyCall"
+CategoricalArraysExt = "CategoricalArrays"
+
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
 PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
 
 [targets]
 test = ["Aqua", "PyCall", "Test", "TestItemRunner"]
+
+[weakdeps]
+CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
+PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"

ext/CategoricalArraysExt.jl

@@ -0,0 +1,14 @@
+module CategoricalArraysExt
+
+using PythonCall
+
+using CategoricalArrays: CategoricalArrays
+
+function PythonCall.Compat.aspandasvector(x::CategoricalArrays.CategoricalArray)
+    codes = map(x -> x === missing ? -1 : Int(CategoricalArrays.levelcode(x)) - 1, x)
+    cats = CategoricalArrays.levels(x)
+    ordered = x.pool.ordered
+    pyimport("pandas").Categorical.from_codes(codes, cats, ordered = ordered)
+end
+
+end

ext/PyCallExt.jl

@@ -0,0 +1,46 @@
+module PyCallExt
+
+using PythonCall
+using PythonCall.Core
+using PythonCall.C
+
+using PyCall: PyCall
+
+import PythonCall: Py
+
+# true if PyCall and PythonCall are using the same interpreter
+const SAME = Ref{Bool}(false)
+
+function __init__()
+    # see if PyCall and PythonCall are using the same interpreter by checking if a couple of memory addresses are the same
+    ptr1 = C.Py_GetVersion()
+    ptr2 = ccall(PyCall.@pysym(:Py_GetVersion), Ptr{Cchar}, ())
+    SAME[] = ptr1 == ptr2
+    if PythonCall.C.CTX.which == :PyCall
+        @assert SAME[]
+    end
+end
+
+# allow explicit conversion between PythonCall.Py and PyCall.PyObject
+# provided they are using the same interpretr
+const ERRMSG = """
+Conversion between `PyCall.PyObject` and `PythonCall.Py` is only possible when using the same Python interpreter.
+
+There are two ways to achieve this:
+- Set the environment variable `JULIA_PYTHONCALL_EXE` to `"@PyCall"`. This forces PythonCall to use the same
+  interpreter as PyCall, but PythonCall loses the ability to manage its own dependencies.
+- Set the environment variable `PYTHON` to `PythonCall.python_executable_path()` and rebuild PyCall. This forces
+  PyCall to use the same interpreter as PythonCall, but needs to be repeated whenever you switch Julia environment.
+"""
+
+function Py(x::PyCall.PyObject)
+    SAME[] || error(ERRMSG)
+    return pynew(C.PyPtr(PyCall.pyreturn(x)))
+end
+
+function PyCall.PyObject(x::Py)
+    SAME[] || error(ERRMSG)
+    return PyCall.PyObject(PyCall.PyPtr(getptr(incref(x))))
+end
+
+end

src/C/C.jl

@@ -9,11 +9,11 @@ using Base: @kwdef
 using UnsafePointers: UnsafePtr
 using CondaPkg: CondaPkg
 using Pkg: Pkg
-using Requires: @require
 using Libdl:
     dlpath, dlopen, dlopen_e, dlclose, dlsym, dlsym_e, RTLD_LAZY, RTLD_DEEPBIND, RTLD_GLOBAL
 
-import ..PythonCall: python_executable_path, python_library_path, python_library_handle, python_version
+import ..PythonCall:
+    python_executable_path, python_library_path, python_library_handle, python_version
 
 
 include("consts.jl")

src/C/context.jl

@@ -17,7 +17,6 @@ A handle to a loaded instance of libpython, its interpreter, function pointers,
     pyhome_w::Any = missing
     which::Symbol = :unknown # :CondaPkg, :PyCall, :embedded or :unknown
     version::Union{VersionNumber,Missing} = missing
-    matches_pycall::Union{Bool,Missing} = missing
 end
 
 const CTX = Context()
@@ -141,15 +140,9 @@ function init_context()
         # Get function pointers from the library
         init_pointers()
 
-        # Compare libpath with PyCall
-        @require PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall)
-
         # Initialize the interpreter
         CTX.is_preinitialized = Py_IsInitialized() != 0
-        if CTX.is_preinitialized
-            @assert CTX.which == :PyCall || CTX.matches_pycall isa Bool
-        else
-            @assert CTX.which != :PyCall
+        if !CTX.is_preinitialized
             # Find ProgramName and PythonHome
             script = if Sys.iswindows()
                 """
@@ -243,13 +236,3 @@ const PYTHONCALL_PKGID = Base.PkgId(PYTHONCALL_UUID, "PythonCall")
 
 const PYCALL_UUID = Base.UUID("438e738f-606a-5dbb-bf0a-cddfbfd45ab0")
 const PYCALL_PKGID = Base.PkgId(PYCALL_UUID, "PyCall")
-
-function init_pycall(PyCall::Module)
-    # see if PyCall and PythonCall are using the same interpreter by checking if a couple of memory addresses are the same
-    ptr1 = Py_GetVersion()
-    ptr2 = @eval PyCall ccall(@pysym(:Py_GetVersion), Ptr{Cchar}, ())
-    CTX.matches_pycall = ptr1 == ptr2
-    if CTX.which == :PyCall
-        @assert CTX.matches_pycall
-    end
-end

src/Compat/Compat.jl

@@ -13,7 +13,6 @@ using ..Wrap
 
 using Serialization: Serialization, AbstractSerializer, serialize, deserialize
 using Tables: Tables
-using Requires: @require
 
 import ..PythonCall: event_loop_on, event_loop_off, fix_qt_plugin_path, pytable
 
@@ -22,13 +21,10 @@ include("ipython.jl")
 include("multimedia.jl")
 include("serialization.jl")
 include("tables.jl")
-include("pycall.jl")
 
 function __init__()
     init_gui()
     init_pyshow()
-    init_tables()
-    @require PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall)
 end
 
 end

src/Compat/pycall.jl

@@ -63,14 +63,3 @@ function _pytable_pandas(src, cols = Tables.columns(src); opts...)
         opts...,
     )
 end
-
-function init_tables()
-    @require CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" @eval begin
-        aspandasvector(x::CategoricalArrays.CategoricalArray) = begin
-            codes = map(x -> x === missing ? -1 : Int(CategoricalArrays.levelcode(x)) - 1, x)
-            cats = CategoricalArrays.levels(x)
-            ordered = x.pool.ordered
-            pyimport("pandas").Categorical.from_codes(codes, cats, ordered = ordered)
-        end
-    end
-end

src/Compat/tables.jl

@@ -39,11 +39,10 @@ end
 end
 
 @testitem "PyCall.jl" setup = [PyCall] begin
+    @test Base.get_extension(PythonCall, :PyCallExt).SAME[]
     x1 = pylist()
     x2 = PyCall.PyObject(x1)
     x3 = Py(x2)
-    @test PythonCall.C.CTX.matches_pycall
-    @test pyisinstance(x3, pybuiltins.list)
     @test pyis(x3, x1)
 end