Switch from PyCall to PythonCall for testing (#176)

* Switch from PyCall to PythonCall

* enable other tests

* show julia version info

* reduce changes

Author: nhz2

.github/workflows/CI.yml

@@ -20,23 +20,15 @@ jobs:
           - macOS-latest
           - windows-latest
         arch:
-          - x64
+          - 'default'
     steps:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - uses: actions/cache@v4
-        env:
-          cache-name: cache-artifacts
-        with:
-          path: ~/.julia/artifacts
-          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-test-${{ env.cache-name }}-
-            ${{ runner.os }}-test-
-            ${{ runner.os }}-
+          show-versioninfo: true
+      - uses: julia-actions/cache@v2
       - uses: julia-actions/julia-buildpkg@v1
         env:
           PYTHON:

.gitignore

@@ -1,3 +1,4 @@
 Manifest.toml
 docs/build
 *.zarr
+.CondaPkg

Project.toml

@@ -33,13 +33,3 @@ OpenSSL = "1"
 URIs = "1"
 ZipArchives = "2"
 julia = "1.2"
-
-[extras]
-Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
-PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
-Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[targets]
-test = ["Test", "Conda", "PyCall", "Sockets", "Random"]

test/CondaPkg.toml

@@ -0,0 +1,3 @@
+[deps]
+zarr = ">=2.13,<3"
+python = ">=3.7,<4"

test/Project.toml

@@ -1,11 +1,15 @@
 [deps]
-Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
+CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Minio = "4281f0d9-7ae0-406e-9172-b7277c1efa20"
 Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
+PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99"
+
+[sources]
+Zarr = {path = ".."}

test/python.jl

@@ -6,10 +6,9 @@
 @testset "Python zarr implementation" begin
 
 import Mmap
-using PyCall
-import PyCall: @py_str
+using PythonCall
 #If we are on conda, import zarr
-pyimport_conda("zarr","zarr")
+zarr = pyimport("zarr")
 
 #Create some directories
 proot = tempname()
@@ -82,113 +81,114 @@ end
 
 # Test reading in python
 for julia_path in (pjulia, pjulia*".zip")
-py"""
-import zarr
-import numcodecs
-g = zarr.open_group($julia_path)
-gatts = g.attrs
-"""
+    g = zarr.open_group(julia_path)
+    gatts = pyconvert(Any, g.attrs)
 
-#Test group attributes
-@test py"gatts['String attribute']" == "One"
-@test py"gatts['Int attribute']" == 5
-@test py"gatts['Float attribute']" == 10.5
+    #Test group attributes
+    @test gatts["String attribute"] == "One"
+    @test gatts["Int attribute"] == 5
+    @test gatts["Float attribute"] == 10.5
 
-dtypesp = ("uint8","uint16","uint32","uint64",
-    "int8","int16","int32","int64",
-    "float16","float32","float64",
-    "complex64", "complex128","bool","S10","U10", "O")
+    dtypesp = ("uint8","uint16","uint32","uint64",
+        "int8","int16","int32","int64",
+        "float16","float32","float64",
+        "complex64", "complex128","bool","S10","U10", "O")
 
-#Test accessing arrays from python and reading data
-for i=1:length(dtypes), co in compressors
-    compstr,comp = co
-    t = dtypes[i]
-    tp = dtypesp[i]
-    arname = string("a",t,compstr)
-    py"""
-    ar=g[$arname]
-    """
+    #Test accessing arrays from python and reading data
+    for i=1:length(dtypes), co in compressors
+        compstr,comp = co
+        t = dtypes[i]
+        tp = dtypesp[i]
+        arname = string("a",t,compstr)
+        ar=g[arname]
 
-    @test py"ar.attrs['This is a nested attribute']" == Dict("a"=>5)
-    @test py"ar.dtype==$tp"
-    @test py"ar.shape" == (2,6,10)
-    if t<:MaxLengthString
-      pyar = py"ar[:,:,:]"
-      jar = [get(get(get(pyar,k-1),j-1),i-1) for i in 1:10, j in 1:6, k in 1:2]
-      @test jar == testarrays[t]
-    else
-      @test py"ar[:,:,:]" == permutedims(testarrays[t],(3,2,1))
+        @test pyconvert(Any, ar.attrs["This is a nested attribute"]) == Dict("a"=>5)
+        @test pyeq(Bool, ar.dtype, tp)
+        @test pyconvert(Tuple, ar.shape) == (2,6,10)
+        if t<:MaxLengthString || t<:String
+            jar = [
+                if tp == "S10"
+                    pyconvert(String, ar[k, j, i].decode())
+                else
+                    pyconvert(String, ar[k, j, i])
+                end
+                for i in 0:9, j in 0:5, k in 0:1
+            ]
+            @test jar == testarrays[t]
+        else
+            @test PyArray(ar[pybuiltins.Ellipsis]) == permutedims(testarrays[t],(3,2,1))
+        end
     end
-end
 
-# Test reading filtered arrays from python
-for (filterstr, filter) in filters
-    t = eltype(filter) == Any ? Float64 : eltype(filter)
-    arname = string("filter_",filterstr)
-    try
-        py"""
-        ar=g[$arname]
-        """
-    catch e
-        @error "Error loading group with filter $filterstr" exception=(e,catch_backtrace())
-        @test false # test failed.
+    # Test reading filtered arrays from python
+    for (filterstr, filter) in filters
+        t = eltype(filter) == Any ? Float64 : eltype(filter)
+        arname = string("filter_",filterstr)
+        local ar
+        try
+            ar=g[arname]
+        catch e
+            @error "Error loading group with filter $filterstr" exception=(e,catch_backtrace())
+            @test false # test failed.
+        end
+        
+        @test pyconvert(Any, ar.attrs["Filter test attribute"]) == Dict("b"=>6)
+        @test pyconvert(Tuple, ar.shape) == (2,6,10)
+        
+        # Test zero-dimensional filtered array
+        arname = string("filter_zerodim_",filterstr) 
+        ar_zero=g[arname]
+        @test pyconvert(Tuple, ar_zero.shape) == ()
     end
-    
-    @test py"ar.attrs['Filter test attribute']" == Dict("b"=>6)
-    @test py"ar.shape" == (2,6,10)
-    
-    # Test zero-dimensional filtered array
-    arname = string("filter_zerodim_",filterstr) 
-    py"""
-    ar_zero=g[$arname]
-    """
-    @test py"ar_zero.shape" == ()
-end
 
-for i=1:length(dtypes), co in compressors
-    compstr,comp = co
-    t = dtypes[i]
-    tp = dtypesp[i]
-    if t == UInt64
-        continue
-        # need to exclude UInt64:
-        # need explicit conversion because of https://github.com/JuliaPy/PyCall.jl/issues/744
-        # but explicit conversion uses PyLong_AsLongLongAndOverflow, which converts everything
-        # to a signed 64-bit integer, which can error out if the UInt64 is too large.
-        # Adding an overload to PyCall for unsigned ints doesn't work with NumPy scalars because
-        # they are not subtypes of integer: https://stackoverflow.com/a/58816671
-    end
+    for i=1:length(dtypes), co in compressors
+        compstr,comp = co
+        t = dtypes[i]
+        tp = dtypesp[i]
+        if t == UInt64
+            continue
+            # need to exclude UInt64:
+            # need explicit conversion because of https://github.com/JuliaPy/PyCall.jl/issues/744
+            # but explicit conversion uses PyLong_AsLongLongAndOverflow, which converts everything
+            # to a signed 64-bit integer, which can error out if the UInt64 is too large.
+            # Adding an overload to PyCall for unsigned ints doesn't work with NumPy scalars because
+            # they are not subtypes of integer: https://stackoverflow.com/a/58816671
+        end
 
-    arname = string("azerodim",t,compstr)
-    py"""
-    ar=g[$arname]
-    """
+        arname = string("azerodim",t,compstr)
+        ar=g[arname]
 
-    @test py"ar.dtype==$tp"
-    @test py"ar.shape" == ()
-    @test convert(t, py"ar[()]") == testzerodimarrays[t]
-end
-py"""
-g.store.close()
-"""
+        @test pyeq(Bool, ar.dtype, tp)
+        @test pyconvert(Tuple, ar.shape) == ()
+        if t<:MaxLengthString || t<:String
+            local x = if tp == "S10"
+                pyconvert(String, ar[()].decode())
+            else
+                pyconvert(String, ar[()])
+            end
+            @test x == testzerodimarrays[t]
+        else
+            @test pyconvert(Any, ar[()])[] == testzerodimarrays[t]
+        end
+    end
+    g.store.close()
 end
 
 ## Now the other way around, we create a zarr array using the python lib and read back into julia
 data = rand(Int32,2,6,10)
-py"""
-import numcodecs
-import numpy as np
-g = zarr.group($ppython)
+
+numpy = pyimport("numpy")
+numcodecs = pyimport("numcodecs")
+g = zarr.group(ppython)
 g.attrs["groupatt"] = "Hi"
-z1 = g.create_dataset("a1", shape=(2,6,10),chunks=(1,2,3), dtype='i4')
-z1[:,:,:]=$data
-z1.attrs["test"]={"b": 6}
-z2 = g.create_dataset("a2", shape=(5,),chunks=(5,), dtype='S1', compressor=numcodecs.Zlib())
-z2[:]=[k for k in 'hallo']
-z3 = g.create_dataset('a3', shape=(2,), dtype=str)
-z3[:]=np.asarray(['test1', 'test234'], dtype='O')
-zarr.consolidate_metadata($ppython)
-"""
+z1 = g.create_dataset("a1", shape=(2,6,10),chunks=(1,2,3), dtype="i4")
+z1[pybuiltins.Ellipsis] = numpy.array(data)
+z1.attrs["test"] = pydict(Dict("b"=>6))
+z2 = g.create_dataset("a2", shape=(5,),chunks=(5,), dtype="S1", compressor=numcodecs.Zlib())
+z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"])
+z3 = g.create_dataset("a3", shape=(2,), dtype=pybuiltins.str)
+z3[pybuiltins.Ellipsis]=numpy.asarray(["test1", "test234"], dtype="O")
+zarr.consolidate_metadata(ppython)
 
 #Open in Julia
 g = zopen(ppython)
@@ -225,21 +225,17 @@ a1[:,1,1] = 1:10
 
 # Test zip file can be read
 ppythonzip = ppython*".zip"
-py"""
-import numcodecs
-import numpy as np
-store = zarr.ZipStore($ppythonzip, mode="w")
+store = zarr.ZipStore(ppythonzip, mode="w")
 g = zarr.group(store=store)
 g.attrs["groupatt"] = "Hi"
-z1 = g.create_dataset("a1", shape=(2,6,10),chunks=(1,2,3), dtype='i4')
-z1[:,:,:]=$data
-z1.attrs["test"]={"b": 6}
-z2 = g.create_dataset("a2", shape=(5,),chunks=(5,), dtype='S1', compressor=numcodecs.Zlib())
-z2[:]=[k for k in 'hallo']
-z3 = g.create_dataset('a3', shape=(2,), dtype=str)
-z3[:]=np.asarray(['test1', 'test234'], dtype='O')
+z1 = g.create_dataset("a1", shape=(2,6,10),chunks=(1,2,3), dtype="i4")
+z1[pybuiltins.Ellipsis] = numpy.array(data)
+z1.attrs["test"] = pydict(Dict("b"=>6))
+z2 = g.create_dataset("a2", shape=(5,),chunks=(5,), dtype="S1", compressor=numcodecs.Zlib())
+z2[pybuiltins.Ellipsis] = pylist([k for k in "hallo"])
+z3 = g.create_dataset("a3", shape=(2,), dtype=pybuiltins.str)
+z3[pybuiltins.Ellipsis] = numpy.asarray(["test1", "test234"], dtype="O")
 store.close()
-"""
 
 g = zopen(Zarr.ZipStore(Mmap.mmap(ppythonzip)))
 @test g isa Zarr.ZGroup
@@ -282,16 +278,15 @@ for pt in [Week, Day, Hour, Minute, Second,
 end
 
 zarr = pyimport("zarr")
-np = pyimport("numpy")
-
+numpy = pyimport("numpy")
 g_julia = zopen(p)
 g_python = zarr.open(p)
 
 for unit in ["Week", "Day", "Hour", "Minute", "Second", 
         "Millisecond"]
-    @test_py np.datetime64(g_julia[unit][1] |> DateTime |> string) == get(getproperty(g_python,unit),0)
-    @test_py np.datetime64(g_julia[unit][10] |> DateTime |> string) ==  get(getproperty(g_python,unit),9)
-    @test_py np.datetime64(g_julia[unit][100] |> DateTime |> string) == get(getproperty(g_python,unit),99)
+    for i in [0, 9, 99]
+        @test pyeq(Bool, numpy.datetime64(g_julia[unit][i+1] |> DateTime |> string), g_python[unit][i])
+    end
 end
 
 end

test/runtests.jl

@@ -2,14 +2,11 @@ using Test
 using Zarr
 using JSON
 using Pkg
-using PyCall
+using PythonCall
+using CondaPkg
 using Dates
 
-macro test_py(ex)
-    quote
-        @test convert(Bool, $(esc(ex)))
-    end
-end
+CondaPkg.add("zarr"; version="2.*")
 
 @testset "Zarr" begin