lazy_import and require_import

Author: CarloLucibello

src/MLDatasets.jl

@@ -3,7 +3,7 @@ module MLDatasets
 using FixedPointNumbers
 using SparseArrays
 using Tables
-using Glob
+# using Glob
 # using DataFrames
 # import ImageCore
 using DataDeps
@@ -23,10 +23,7 @@ using FileIO
 using DelimitedFiles: readdlm
 ##########
 
-export getobs, numobs
-export convert2image
-
-
+export getobs, numobs # From MLUtils.jl
 
 include("abstract_datasets.jl")
 # export AbstractDataset, 

src/datasets/misc/boston_housing.jl

@@ -78,7 +78,7 @@ function BostonHousing(; as_df = true, dir = nothing)
     @assert dir === nothing "custom `dir` is not supported at the moment."
     path = joinpath(@__DIR__, "..", "..", "..", "data", "boston_housing.csv")
     df = read_csv(path)
-    DFs = checked_import(idDataFrames)
+    DFs = require_import(:DataFrames)
     features = df[!, DFs.Not(:MEDV)]
     targets = df[!, [:MEDV]]
 

src/datasets/misc/iris.jl

@@ -80,7 +80,7 @@ end
 function Iris(; dir = nothing, as_df = true)
     path = datafile("Iris", "iris.data", dir)
     df = read_csv(path, header=0)
-    DFs = checked_import(idDataFrames)
+    DFs = require_import(:DataFrames)
     DFs.rename!(df, ["sepallength", "sepalwidth", "petallength", "petalwidth", "class"])
 
     features = df[!, DFs.Not(:class)]

src/datasets/misc/mutagenesis.jl

@@ -85,7 +85,7 @@ function Mutagenesis(split::Symbol; dir=nothing)
     metadata = read_json(metadata_path)
     labelkey = metadata["label"]
     targets = map(i -> i[labelkey], samples)
-    features = map(x->delete!(copy(x), Symbol(labelkey)), samples)
+    features = map(x -> delete!(copy(x), Symbol(labelkey)), samples)
     val_num = metadata["val_samples"]
     test_num = metadata["test_samples"]
     train_idxs = 1:length(samples)-val_num-test_num

src/datasets/misc/titanic.jl

@@ -62,7 +62,7 @@ function Titanic(; as_df = true, dir = nothing)
     @assert dir === nothing "custom `dir` is not supported at the moment."
     path = joinpath(@__DIR__, "..", "..", "..", "data", "titanic.csv")
     df = read_csv(path)
-    DFs = checked_import(idDataFrames)
+    DFs = require_import(:DataFrames)
 
     features = df[!, DFs.Not(:Survived)]
     targets = df[!, [:Survived]]

src/datasets/vision/cifar10.jl

@@ -165,8 +165,8 @@ convert2image(::Type{<:CIFAR10}, x::AbstractArray{<:Integer}) =
 function convert2image(::Type{<:CIFAR10}, x::AbstractArray{T,N}) where {T,N}
     @assert N == 3 || N == 4
     x = permutedims(x, (3, 2, 1, 4:N...))
-    checked_import(idImageCore).colorview(RGB, x)
-    # return  ImageCore.colorview(RGB, x)
+    ImageCore = lazy_import(:ImageCore)
+    return ImageCore.colorview(RGB, x)
 end
 
 

src/datasets/vision/mnist.jl

@@ -141,7 +141,8 @@ convert2image(::Type{<:MNIST}, x::AbstractArray{<:Integer}) =
 function convert2image(::Type{<:MNIST}, x::AbstractArray{T,N}) where {T,N}
     @assert N == 2 || N == 3
     x = permutedims(x, (2, 1, 3:N...))
-    checked_import(idImageCore).colorview(Gray, x)
+    ImageCore = lazy_import(:ImageCore)
+    return ImageCore.colorview(Gray, x)
 end
 
 # DEPRECATED INTERFACE, REMOVE IN v0.7 (or 0.6.x)

src/imports.jl

@@ -1,24 +1,26 @@
+# Check that packages like JSON3 are already imported by the user.
+
+function require_import(m::Symbol)
+    pkgid = Base.identify_package(string(m))
+    if Base.root_module_exists(pkgid)
+        return Base.root_module(pkgid)
+    else
+        error("Add `import $m` or `using $m` to your code to unlock this functionality.")
+    end
+end
 
-const idDataFrames = Base.PkgId(Base.UUID("a93c6f00-e57d-5684-b7b6-d8193f3e46c0"), "DataFrames")
-const idCSV = Base.PkgId(Base.UUID("336ed68f-0bac-5ca0-87d4-7b16caf5d00b"), "CSV")
-const idImageCore = Base.PkgId(Base.UUID("a09fc81d-aa75-5fe9-8630-4744c3626534"), "ImageCore")
-const idPickle = Base.PkgId(Base.UUID("fbb45041-c46e-462f-888f-7c521cafbc2c"), "Pickle")
-const idHDF5 = Base.PkgId(Base.UUID("f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f"), "HDF5")
-const idMAT = Base.PkgId(Base.UUID("23992714-dd62-5051-b70f-ba57cb901cac"), "MAT")
-const idJSON3 = Base.PkgId(Base.UUID("0f8b85d8-7281-11e9-16c2-39a750bddbf1"), "JSON3")
-# ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
-# JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
 
 const load_locker = Threads.ReentrantLock()
 
-function checked_import(pkgid)
+function lazy_import(m::Symbol)
+    pkgid = Base.identify_package(string(m))
     mod = if Base.root_module_exists(pkgid)
-            Base.root_module(pkgid)
-        else
-            lock(load_locker) do
-                Base.require(pkgid)
-            end
-        end
+              Base.root_module(pkgid)
+          else
+              lock(load_locker) do
+                  Base.require(pkgid)
+              end
+          end
 
     return ImportedModule(mod)
 end
@@ -36,4 +38,4 @@ function Base.getproperty(m::ImportedModule, s::Symbol)
         end
         return f
     end
-end
+end

src/io.jl

@@ -1,3 +1,8 @@
+# IMPORTS 
+# Use `lazy_import(:SomePkg)` whenever the returned types are "native" types,
+#  i.e. not defined in SomePkg itself, otherwise use `require_import(:SomePkg)`.
+
+
 function read_csv(path; kws...)
     return read_csv_asdf(path; kws...)
 end
@@ -8,34 +13,39 @@ end
 # end
 
 function read_csv(path, sink::Type{A}; kws...) where A <: AbstractMatrix
-    A(read_csv(path; kws...))
+    return A(read_csv(path; kws...))
 end
 
 function read_csv_asdf(path; kws...)
-    DF = checked_import(idDataFrames).mod.DataFrame
-    return checked_import(idCSV).read(path, DF; kws...)
+    DataFrames = require_import(:DataFrames)
+    CSV = lazy_import(:CSV)
+    return CSV.read(path, DataFrames.DataFrame; kws...)
 end
 
 function read_npy(path)
-    return FileIO.load(File{format"NPY"}(path))
+    return FileIO.load(File{format"NPY"}(path)) # FileIO does lazy import of NPZ.jl
 end
 
 function read_npz(path)
-    return FileIO.load(File{format"NPZ"}(path))
+    return FileIO.load(File{format"NPZ"}(path)) # FileIO does lazy import of NPZ.jl
 end
 
 function read_pytorch(path)
-    Base.invokelatest(checked_import(idPickle).mod.Torch.THload, path)
+    Pickle = lazy_import(:Pickle)
+    return Pickle.Torch.THload(path)
 end
 
 function read_pickle(path)
-    checked_import(idPickle).npyload(path)
+    Pickle = lazy_import(:Pickle)
+    return Pickle.npyload(path)
 end
 
 function read_mat(path)
-    checked_import(idMAT).matread(path)
+    MAT = lazy_import(:MAT)
+    return MAT.matread(path)
 end
 
 function read_json(path)
-    Base.invokelatest(open, checked_import(idJSON3).mod.read, path)
+    JSON3 = require_import(:JSON3)
+    return open(JSON3.read, path)
 end

test/datasets/misc.jl

@@ -63,7 +63,6 @@ end
     @test val_x isa Vector{<:Dict}
 
     x, y = dtrain[1:2]
-    @show typeof(x) typeof(y)
     @test x isa Vector{Dict{Symbol, Any}}
     @test length(x) == 2
     @test y isa Vector{Int}