diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4e41bb25..212044eb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ on: jobs: julia: - name: Test Julia (${{ matrix.jlversion }}, ${{ matrix.os }}) + name: Test Julia (${{ matrix.jlversion }}, ${{ matrix.os }}, ${{ matrix.pythonexe }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -20,6 +20,12 @@ jobs: arch: [x64] # x86 unsupported by MicroMamba os: [ubuntu-latest, windows-latest, macos-latest] jlversion: ['1','1.9'] + pythonexe: ['@CondaPkg'] + include: + - arch: x64 + os: ubuntu-latest + jlversion: '1' + pythonexe: python steps: - uses: actions/checkout@v5 @@ -34,12 +40,16 @@ jobs: - name: Build package uses: julia-actions/julia-buildpkg@v1 + env: + PYTHON: python - name: Run tests uses: julia-actions/julia-runtest@v1 env: JULIA_DEBUG: PythonCall JULIA_NUM_THREADS: '2' + PYTHON: python + JULIA_PYTHONCALL_EXE: ${{ matrix.pythonexe }} - name: Process coverage uses: julia-actions/julia-processcoverage@v1 diff --git a/Project.toml b/Project.toml index 064df3be..748bdbe5 100644 --- a/Project.toml +++ b/Project.toml @@ -10,13 +10,13 @@ Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -Requires = "ae029012-a4dd-5104-9daa-d747884805df" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" UnsafePointers = "e17b2a0c-0bdf-430a-bd0c-3a23cae4ff39" [compat] Aqua = "0 - 999" +CategoricalArrays = "0.10, 1" CondaPkg = "0.2.30" Dates = "1" Libdl = "1" @@ -24,7 +24,6 @@ MacroTools = "0.5" Markdown = "1" Pkg = "1" PyCall = "1" -Requires = "1" Serialization = "1" Tables = "1" Test = "1" @@ -32,11 +31,20 @@ TestItemRunner = "0 - 999" UnsafePointers = "1" julia = "1.9" +[extensions] +PyCallExt = "PyCall" +CategoricalArraysExt = "CategoricalArrays" + [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" [targets] test = ["Aqua", "PyCall", "Test", "TestItemRunner"] + +[weakdeps] +CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" +PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index 5393d710..05caf14f 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -1,5 +1,9 @@ # Release Notes +## Unreleased +* Bug fixes. +* Internal: switch from Requires.jl to package extensions. + ## 0.9.27 (2025-08-19) * Internal: Use heap-allocated types (PyType_FromSpec) to improve ABI compatibility. * Minimum supported Python version is now 3.9. diff --git a/ext/CategoricalArraysExt.jl b/ext/CategoricalArraysExt.jl new file mode 100644 index 00000000..49c4fc03 --- /dev/null +++ b/ext/CategoricalArraysExt.jl @@ -0,0 +1,14 @@ +module CategoricalArraysExt + +using PythonCall + +using CategoricalArrays: CategoricalArrays + +function PythonCall.Compat.aspandasvector(x::CategoricalArrays.CategoricalArray) + codes = map(x -> x === missing ? -1 : Int(CategoricalArrays.levelcode(x)) - 1, x) + cats = CategoricalArrays.levels(x) + ordered = x.pool.ordered + pyimport("pandas").Categorical.from_codes(codes, cats, ordered = ordered) +end + +end diff --git a/ext/PyCallExt.jl b/ext/PyCallExt.jl new file mode 100644 index 00000000..752b744d --- /dev/null +++ b/ext/PyCallExt.jl @@ -0,0 +1,46 @@ +module PyCallExt + +using PythonCall +using PythonCall.Core +using PythonCall.C + +using PyCall: PyCall + +import PythonCall: Py + +# true if PyCall and PythonCall are using the same interpreter +const SAME = Ref{Bool}(false) + +function __init__() + # see if PyCall and PythonCall are using the same interpreter by checking if a couple of memory addresses are the same + ptr1 = C.Py_GetVersion() + ptr2 = ccall(PyCall.@pysym(:Py_GetVersion), Ptr{Cchar}, ()) + SAME[] = ptr1 == ptr2 + if PythonCall.C.CTX.which == :PyCall + @assert SAME[] + end +end + +# allow explicit conversion between PythonCall.Py and PyCall.PyObject +# provided they are using the same interpretr +const ERRMSG = """ +Conversion between `PyCall.PyObject` and `PythonCall.Py` is only possible when using the same Python interpreter. + +There are two ways to achieve this: +- Set the environment variable `JULIA_PYTHONCALL_EXE` to `"@PyCall"`. This forces PythonCall to use the same + interpreter as PyCall, but PythonCall loses the ability to manage its own dependencies. +- Set the environment variable `PYTHON` to `PythonCall.python_executable_path()` and rebuild PyCall. This forces + PyCall to use the same interpreter as PythonCall, but needs to be repeated whenever you switch Julia environment. +""" + +function Py(x::PyCall.PyObject) + SAME[] || error(ERRMSG) + return pynew(C.PyPtr(PyCall.pyreturn(x))) +end + +function PyCall.PyObject(x::Py) + SAME[] || error(ERRMSG) + return PyCall.PyObject(PyCall.PyPtr(getptr(incref(x)))) +end + +end diff --git a/src/C/C.jl b/src/C/C.jl index c27ffe01..a8e7d5f8 100644 --- a/src/C/C.jl +++ b/src/C/C.jl @@ -9,11 +9,11 @@ using Base: @kwdef using UnsafePointers: UnsafePtr using CondaPkg: CondaPkg using Pkg: Pkg -using Requires: @require using Libdl: dlpath, dlopen, dlopen_e, dlclose, dlsym, dlsym_e, RTLD_LAZY, RTLD_DEEPBIND, RTLD_GLOBAL -import ..PythonCall: python_executable_path, python_library_path, python_library_handle, python_version +import ..PythonCall: + python_executable_path, python_library_path, python_library_handle, python_version include("consts.jl") diff --git a/src/C/context.jl b/src/C/context.jl index 07135409..5d3bf92e 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -17,7 +17,6 @@ A handle to a loaded instance of libpython, its interpreter, function pointers, pyhome_w::Any = missing which::Symbol = :unknown # :CondaPkg, :PyCall, :embedded or :unknown version::Union{VersionNumber,Missing} = missing - matches_pycall::Union{Bool,Missing} = missing end const CTX = Context() @@ -141,15 +140,9 @@ function init_context() # Get function pointers from the library init_pointers() - # Compare libpath with PyCall - @require PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall) - # Initialize the interpreter CTX.is_preinitialized = Py_IsInitialized() != 0 - if CTX.is_preinitialized - @assert CTX.which == :PyCall || CTX.matches_pycall isa Bool - else - @assert CTX.which != :PyCall + if !CTX.is_preinitialized # Find ProgramName and PythonHome script = if Sys.iswindows() """ @@ -243,13 +236,3 @@ const PYTHONCALL_PKGID = Base.PkgId(PYTHONCALL_UUID, "PythonCall") const PYCALL_UUID = Base.UUID("438e738f-606a-5dbb-bf0a-cddfbfd45ab0") const PYCALL_PKGID = Base.PkgId(PYCALL_UUID, "PyCall") - -function init_pycall(PyCall::Module) - # see if PyCall and PythonCall are using the same interpreter by checking if a couple of memory addresses are the same - ptr1 = Py_GetVersion() - ptr2 = @eval PyCall ccall(@pysym(:Py_GetVersion), Ptr{Cchar}, ()) - CTX.matches_pycall = ptr1 == ptr2 - if CTX.which == :PyCall - @assert CTX.matches_pycall - end -end diff --git a/src/Compat/Compat.jl b/src/Compat/Compat.jl index db8bd566..d9b8a148 100644 --- a/src/Compat/Compat.jl +++ b/src/Compat/Compat.jl @@ -13,7 +13,6 @@ using ..Wrap using Serialization: Serialization, AbstractSerializer, serialize, deserialize using Tables: Tables -using Requires: @require import ..PythonCall: event_loop_on, event_loop_off, fix_qt_plugin_path, pytable @@ -22,13 +21,10 @@ include("ipython.jl") include("multimedia.jl") include("serialization.jl") include("tables.jl") -include("pycall.jl") function __init__() init_gui() init_pyshow() - init_tables() - @require PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall) end end diff --git a/src/Compat/pycall.jl b/src/Compat/pycall.jl deleted file mode 100644 index 7cc54a47..00000000 --- a/src/Compat/pycall.jl +++ /dev/null @@ -1,21 +0,0 @@ -function init_pycall(PyCall::Module) - # allow explicit conversion between PythonCall.Py and PyCall.PyObject - # provided they are using the same interpretr - errmsg = """ - Conversion between `PyCall.PyObject` and `PythonCall.Py` is only possible when using the same Python interpreter. - - There are two ways to achieve this: - - Set the environment variable `JULIA_PYTHONCALL_EXE` to `"@PyCall"`. This forces PythonCall to use the same - interpreter as PyCall, but PythonCall loses the ability to manage its own dependencies. - - Set the environment variable `PYTHON` to `PythonCall.C.CTX.exe_path` and rebuild PyCall. This forces PyCall - to use the same interpreter as PythonCall, but needs to be repeated whenever you switch Julia environment. - """ - @eval function PythonCall.Py(x::$PyCall.PyObject) - C.CTX.matches_pycall::Bool || error($errmsg) - return pynew(C.PyPtr($PyCall.pyreturn(x))) - end - @eval function PyCall.PyObject(x::Py) - C.CTX.matches_pycall::Bool || error($errmsg) - return $PyCall.PyObject($PyCall.PyPtr(getptr(incref(x)))) - end -end diff --git a/src/Compat/tables.jl b/src/Compat/tables.jl index dc4fef84..db12df49 100644 --- a/src/Compat/tables.jl +++ b/src/Compat/tables.jl @@ -63,14 +63,3 @@ function _pytable_pandas(src, cols = Tables.columns(src); opts...) opts..., ) end - -function init_tables() - @require CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" @eval begin - aspandasvector(x::CategoricalArrays.CategoricalArray) = begin - codes = map(x -> x === missing ? -1 : Int(CategoricalArrays.levelcode(x)) - 1, x) - cats = CategoricalArrays.levels(x) - ordered = x.pool.ordered - pyimport("pandas").Categorical.from_codes(codes, cats, ordered = ordered) - end - end -end diff --git a/test/Compat.jl b/test/Compat.jl index 3753dd40..79037f29 100644 --- a/test/Compat.jl +++ b/test/Compat.jl @@ -38,13 +38,23 @@ end # TODO end -@testitem "PyCall.jl" setup = [PyCall] begin - x1 = pylist() - x2 = PyCall.PyObject(x1) - x3 = Py(x2) - @test PythonCall.C.CTX.matches_pycall - @test pyisinstance(x3, pybuiltins.list) - @test pyis(x3, x1) +@testitem "PyCall.jl" begin + if (get(ENV, "CI", "") != "") && (ENV["JULIA_PYTHONCALL_EXE"] == "python") + # Only run this test when we can guarantee PyCall and PythonCall are using the + # same Python. Currently this only runs in CI, and if PythonCall is using the + # system Python installation. + using PyCall + # Check they are indeed using the same Python. + @test Base.get_extension(PythonCall, :PyCallExt).SAME[] + # Check we can round-trip and object PythonCall -> PyCall -> PythonCall and + # have the same identical Python object afterward. + x1 = pylist()::Py + x2 = PyCall.PyObject(x1)::PyCall.PyObject + x3 = Py(x2)::Py + @test pyis(x3, x1) + else + @warn "Skipping PyCall.jl tests." + end end @testitem "Serialization.jl" begin diff --git a/test/runtests.jl b/test/runtests.jl index a4e2b920..b9e874db 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,12 +1,3 @@ using TestItemRunner @run_package_tests - -@testmodule PyCall begin - using PythonCall: PythonCall - using Pkg: Pkg - ENV["PYTHON"] = PythonCall.python_executable_path() - @info "Building PyCall..." ENV["PYTHON"] - Pkg.build("PyCall") - using PyCall -end