updates

Author: CarloLucibello

.github/dependabot.yml

@@ -0,0 +1,7 @@
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/workflows/CI.yml

@@ -19,7 +19,7 @@ jobs:
       matrix:
         version:
           - '1.9'
-          - '1' # add back when 1.10 is out
+          - '1'
           - 'nightly'
         os:
           - ubuntu-latest

.github/workflows/CompatHelper.yml

@@ -15,7 +15,7 @@ jobs:
         run: which julia
         continue-on-error: true
       - name: Install Julia, but only if it is not already available in the PATH
-        uses: julia-actions/setup-julia@v1
+        uses: julia-actions/setup-julia@v2
         with:
           version: '1'
           arch: ${{ runner.arch }}
@@ -41,5 +41,10 @@ jobs:
         shell: julia --color=yes {0}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # This repo uses Documenter, so we can reuse our [Documenter SSH key](https://documenter.juliadocs.org/stable/man/hosting/walkthrough/).
+          # If we didn't have one of those setup, we could configure a dedicated ssh deploy key `COMPATHELPER_PRIV` following https://juliaregistries.github.io/CompatHelper.jl/dev/#Creating-SSH-Key.
+          # Either way, we need an SSH key if we want the PRs that CompatHelper creates to be able to trigger CI workflows themselves.
+          # That is because GITHUB_TOKEN's can't trigger other workflows (see https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow).
+          # Check if you have a deploy key setup using these docs: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/reviewing-your-deploy-keys.
           COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }}
           # COMPATHELPER_PRIV: ${{ secrets.COMPATHELPER_PRIV }}

.github/workflows/TagBot.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
     inputs:
       lookback:
-        default: 3
+        default: "3"
 permissions:
   actions: read
   checks: read

CondaPkg.toml

@@ -4,7 +4,7 @@ channels = ["conda-forge"]
 # h5py = ""
 # pillow = ">=9.1, <10"
 # pyarrow = "==6.0.0"
-datasets = ">=2.12, <3"
-numpy = ">=1.20, <2"
+datasets = ">=3.0, <4"
+numpy = ">=2.0, <3"
 pillow = ""
 

Project.toml

@@ -12,7 +12,7 @@ PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
 
 [compat]
 CondaPkg = "0.2"
-DLPack = "0.1"
+DLPack = "0.3"
 ImageCore = "0.9, 0.10"
 MLUtils = "0.4.1"
 PythonCall = "0.9"

README.md

@@ -23,31 +23,31 @@ HuggingFaceDatasets.jl provides wrappers around types from the `datasets` python
 Check out the [examples/](https://github.com/JuliaGenAI/HuggingFaceDatasets.jl/tree/main/examples) folder for usage examples.
 
 ```julia
+julia> using HuggingFaceDatasets
+
 julia> train_data = load_dataset("mnist", split = "train")
 Dataset({
     features: ['image', 'label'],
     num_rows: 60000
 })
 
-# Indexing starts with 1. 
-# Python types are returned by default.
 julia> train_data[1]
-Python: {'image': <PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x7F04DE661CD0>, 'label': 5}
+Python: {'image': <PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x3340B0290>, 'label': 5}
 
 julia> length(train_data)
 60000
 
-# Now we set the julia format
 julia> train_data = load_dataset("mnist", split = "train").with_format("julia");
 
-# Returned observations are now julia objects
+       # Returned observations are now julia objects
+
 julia> train_data[1]
 Dict{String, Any} with 2 entries:
   "label" => 5
-  "image" => Gray{N0f8}[Gray{N0f8}(0.0) Gray{N0f8}(0.0) … Gray{N0f8}(0.0) Gray{N0f8}(0.0); Gray{N0f8}(0.0) Gray{N0f8}(0.0) … Gray{N0f8}(0.0) Gray{N0f8}(0.0); … ; Gray{N0f8}(0.0) Gray{N0f8}(0.0) ……
+  "image" => Gray{N0f8}[0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
 
 julia> train_data[1:2]
 Dict{String, Vector} with 2 entries:
   "label" => [5, 0]
-  "image" => ReinterpretArray{Gray{N0f8}, 2, UInt8, Matrix{UInt8}, false}[[Gray{N0f8}(0.0) Gray{N0f8}(0.0) … Gray{N0f8}(0.0) Gray{N0f8}(0.0); Gray{N0f8}(0.0) Gray{N0f8}(0.0) … Gray{N0f8}(0.0) Gra…
+  "image" => ReinterpretArray{Gray{N0f8}, 2, UInt8, Matrix{UInt8}, false}[[0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0…
 ```

docs/Manifest.toml

@@ -1,10 +1,60 @@
 
-# See https://github.com/cjdoris/PythonCall.jl/issues/172.
-function _pyconvert(x::Py)
+# # See https://github.com/cjdoris/PythonCall.jl/issues/172.
+# function _pyconvert(x::Py)
+#     @show x
+#     if pyisinstance(x, datasets.Dataset)
+#         return Dataset(x)
+#     elseif pyisinstance(x, datasets.DatasetDict)
+#         return DatasetDict(x)
+#     elseif pyisinstance(x, PIL.PngImagePlugin.PngImageFile) || pyisinstance(x, PIL.JpegImagePlugin.JpegImageFile)
+#         @show x
+#         a = numpy2jl(np.array(x))
+#         if ndims(a) == 3 && size(a, 1) == 3
+#             return colorview(RGB{N0f8}, a)
+#         elseif ndims(a) == 2
+#             return reinterpret(Gray{N0f8}, a)
+#         else
+#             error("Unknown image format")
+#         end
+#     elseif pyisinstance(x, np.ndarray)
+#         return numpy2jl(x)
+#     else
+#         return pyconvert(Any, x)
+#     end
+# end
+
+# # # Do nothing on a non-Py object.
+# # _pyconvert(x) = x
+
+"""
+    py2jl(x)
+
+Convert Python types to Julia types applying `pyconvert` recursively.
+"""
+py2jl
+
+# py2jl recurses through pycanonicalize and converts through _pyconvert
+py2jl(x) = pyconvert(Any, x)
+
+function py2jl(x::Py)
+    # handle datasets
     if pyisinstance(x, datasets.Dataset)
         return Dataset(x)
     elseif pyisinstance(x, datasets.DatasetDict)
         return DatasetDict(x)
+    # handle list, tuple, dict, and set
+    elseif pyisinstance(x, pytype(pylist()))
+        return [py2jl(x) for x in x]
+    elseif pyisinstance(x, pytype(pytuple()))
+        return tuple(py2jl(x) for x in x)
+    elseif pyisinstance(x, pytype(pydict()))
+        return Dict(py2jl(k) => py2jl(v) for (k, v) in x.items())
+    elseif pyisinstance(x, pytype(pyset()))
+        return Set(py2jl(x) for x in x)
+    # handle numpy arrays   
+    elseif pyisinstance(x, np.ndarray)
+        return numpy2jl(x)
+    # handle PIL images
     elseif pyisinstance(x, PIL.PngImagePlugin.PngImageFile) || pyisinstance(x, PIL.JpegImagePlugin.JpegImageFile)
         a = numpy2jl(np.array(x))
         if ndims(a) == 3 && size(a, 1) == 3
@@ -14,30 +64,12 @@ function _pyconvert(x::Py)
         else
             error("Unknown image format")
         end
-    elseif pyisinstance(x, np.ndarray)
-        return numpy2jl(x)
+    # handle other types
     else
         return pyconvert(Any, x)
     end
 end
 
-# Do nothing on a non-Py object.
-_pyconvert(x) = x
-
-"""
-    py2jl(x)
-
-Convert Python types to Julia types applying `pyconvert` recursively.
-"""
-py2jl
-
-# py2jl recurses through pycanonicalize and converts through _pyconvert
-py2jl(x) = pycanonicalize(_pyconvert(x))
-
-pycanonicalize(x) = x
-
-pycanonicalize(x::PyList) = [py2jl(x) for x in x]
-pycanonicalize(x::PyDict) = Dict(py2jl(k) => py2jl(v) for (k, v) in pairs(x))
 
 """
     numpy2jl(x)

src/transforms.jl

@@ -1,5 +1,3 @@
-using HuggingFaceDatasets, ImageShow, Test, ImageCore
-
 @testset "image classification" begin 
     @testset "cifar10" begin
         ds = load_dataset("cifar10", split = "test").with_format("julia")
@@ -25,8 +23,12 @@ end
         ds = load_dataset("cppe-5", split = "test").with_format("julia")
         @test ds[1]["image"] isa AbstractMatrix{RGB{N0f8}}
         @test ds[1]["objects"] isa Dict{String, Vector}
-
-        @test ds[1:2]["image"] isa Vector{<:AbstractMatrix{RGB{N0f8}}}
+        imgs = ds[1:2]["image"]
+        @test imgs isa Vector{<:AbstractArray}
+        @test imgs isa Vector{<:AbstractMatrix} broken=true
+        @test imgs isa Vector{<:AbstractMatrix{RGB{N0f8}}} broken=true
+        @test size(imgs[1]) == (1920, 1088)
+        @test imgs[1] isa AbstractMatrix{RGB{N0f8}}
         @test ds[1:2]["objects"] isa Vector{Dict{String, Vector}}
     end
 end