cleanup

Author: CarloLucibello

README.md

@@ -39,9 +39,7 @@ julia> length(train_data)
 
 julia> train_data = load_dataset("mnist", split = "train").with_format("julia");
 
-       # Returned observations are now julia objects
-
-julia> train_data[1]
+julia> train_data[1] # Returned observations are now julia objects
 Dict{String, Any} with 2 entries:
   "label" => 5
   "image" => Gray{N0f8}[0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]

docs/make.jl

@@ -15,6 +15,7 @@ makedocs(;
     ),
     pages=[
         "Home" => "index.md",
+        "API" => "api.md",
     ],
 )
 

docs/src/api.md

@@ -1,12 +1,9 @@
-# API
-
-## Index
-
-```@index
-Pages   = ["api.md"]
+```@meta
+CurrentModule = HuggingFaceDatasets
+CollapsedDocStrings = true
 ```
 
-## Docs
+# API
 
 ```@autodocs
 Modules = [HuggingFaceDatasets]

docs/src/index.md

@@ -26,26 +26,30 @@ HuggingFaceDatasets.jl provides wrappers around types from the `datasets` python
 Check out the `examples/` folder for usage examples.
 
 ```julia
+# Returned observations are now julia objects
+julia> using HuggingFaceDatasets
+
 julia> train_data = load_dataset("mnist", split = "train")
-Dataset(<py Dataset({
+Dataset({
     features: ['image', 'label'],
     num_rows: 60000
-})>, identity)
+})
 
-# Indexing starts with 1. 
-# By defaul, python types are returned.
 julia> train_data[1]
-Python dict: {'image': <PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x2B64E2E90>, 'label': 5}
+Python: {'image': <PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x3340B0290>, 'label': 5}
 
-julia> set_format!(train_data, "julia")
-Dataset(<py Dataset({
-    features: ['image', 'label'],
-    num_rows: 60000
-})>, HuggingFaceDatasets.py2jl)
+julia> length(train_data)
+60000
 
-# Now we have julia types
-julia> train_data[1]
+julia> train_data = load_dataset("mnist", split = "train").with_format("julia");
+
+julia> train_data[1] # Returned observations are now julia objects
 Dict{String, Any} with 2 entries:
   "label" => 5
-  "image" => UInt8[0x00 0x00 … 0x00 0x00; 0x00 0x00 … 0x00 0x00; … ; 0x00 0x00 … 0x00 0x00; 0x00 0x00 … 0x00 0x00]
+  "image" => Gray{N0f8}[0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]
+
+julia> train_data[1:2]
+Dict{String, Vector} with 2 entries:
+  "label" => [5, 0]
+  "image" => ReinterpretArray{Gray{N0f8}, 2, UInt8, Matrix{UInt8}, false}[[0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0…
 ```

src/HuggingFaceDatasets.jl

@@ -3,7 +3,7 @@ module HuggingFaceDatasets
 using PythonCall
 using MLUtils: getobs, numobs
 import MLUtils
-using DLPack
+using DLPack: DLPack
 using ImageCore
 
 const datasets = PythonCall.pynew()

src/transforms.jl

@@ -1,39 +1,11 @@
 
-# # See https://github.com/cjdoris/PythonCall.jl/issues/172.
-# function _pyconvert(x::Py)
-#     @show x
-#     if pyisinstance(x, datasets.Dataset)
-#         return Dataset(x)
-#     elseif pyisinstance(x, datasets.DatasetDict)
-#         return DatasetDict(x)
-#     elseif pyisinstance(x, PIL.PngImagePlugin.PngImageFile) || pyisinstance(x, PIL.JpegImagePlugin.JpegImageFile)
-#         @show x
-#         a = numpy2jl(np.array(x))
-#         if ndims(a) == 3 && size(a, 1) == 3
-#             return colorview(RGB{N0f8}, a)
-#         elseif ndims(a) == 2
-#             return reinterpret(Gray{N0f8}, a)
-#         else
-#             error("Unknown image format")
-#         end
-#     elseif pyisinstance(x, np.ndarray)
-#         return numpy2jl(x)
-#     else
-#         return pyconvert(Any, x)
-#     end
-# end
-
-# # # Do nothing on a non-Py object.
-# # _pyconvert(x) = x
-
 """
     py2jl(x)
 
-Convert Python types to Julia types applying `pyconvert` recursively.
+Convert Python types to Julia types. It will recursively traverse built-in python
+containers such as lists, tuples, dicts, and sets, and convert all nested objects.
+On the leaves, it will call either `pyconvert(Any, x)` or [`numpy2jl`](@ref).
 """
-py2jl
-
-# py2jl recurses through pycanonicalize and converts through _pyconvert
 py2jl(x) = pyconvert(Any, x)
 
 function py2jl(x::Py)
@@ -74,21 +46,26 @@ end
 """
     numpy2jl(x)
 
-Convert a numpy array to a Julia array using DLPack.
+Convert a numpy array to a Julia array using DLPack.jl.
 The conversion is copyless, and mutations to the Julia array are reflected in the numpy array.
+For row major python arrays, the returned Julia array has permuted dimensions.
+
+This function is called by [`py2jl`](@ref).
+See also [`jl2numpy`](@ref).
 """
 function numpy2jl(x::Py)
-    # pyconvert(Any, x)
-    # PyArray(x, copy=false)
-    if Bool(x.dtype.type == np.str_)
-        return PyArray(x, copy=false)
-    else
-        return DLPack.wrap(x, x -> x.__dlpack__())
-    end
+    return DLPack.from_dlpack(x)
 end
 
-## TODO this doesn't work yet.
-## https://github.com/pabloferz/DLPack.jl/issues/32
-# function jl2numpy(x::AbstractArray)
-#     return DLPack.share(x, np.from_dlpack)
-# end
+"""
+    jl2numpy(x)
+
+Convert a Julia array to a numpy array using DLPack.jl.
+The conversion is copyless, and mutations to the numpy array are reflected in the Julia array.
+The returned numpy array has permuted dimensions with respect to the input Julia array.
+
+See also [`numpy2jl`](@ref).
+"""
+function jl2numpy(x::AbstractArray)
+    return DLPack.share(x, np.from_dlpack)
+end