Add some docstrings and test for FileDataset

Author: darsnack

Project.toml

@@ -13,6 +13,7 @@ FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
 FixedPointNumbers = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
 GZip = "92fee26a-97fe-5a0c-ad85-20a5f3185b63"
+Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 MAT = "23992714-dd62-5051-b70f-ba57cb901cac"
 MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54"
@@ -23,11 +24,12 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
 [compat]
 BinDeps = "1"
-ColorTypes = "0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 0.11"
 CSV = "0.10.2"
+ColorTypes = "0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 0.11"
 DataDeps = "0.3, 0.4, 0.5, 0.6, 0.7"
 DataFrames = "1.3"
 FixedPointNumbers = "0.3, 0.4, 0.5, 0.6, 0.7, 0.8"
+Glob = "1.3"
 GZip = "0.5"
 ImageCore = "0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8"
 JSON3 = "1"

src/MLDatasets.jl

@@ -10,6 +10,7 @@ using SparseArrays
 using DataFrames, CSV, Tables
 using FilePathsBase
 using FilePathsBase: AbstractPath
+using Glob
 
 import MLUtils
 using MLUtils: getobs, numobs, AbstractDataContainer

src/containers/filedataset.jl

@@ -1,4 +1,7 @@
-# FileDataset
+matches(re::Regex) = f -> matches(re, f)
+matches(re::Regex, f) = !isnothing(match(re, f))
+const RE_IMAGEFILE = r".*\.(gif|jpe?g|tiff?|png|webp|bmp)$"i
+isimagefile(f) = matches(RE_IMAGEFILE, f)
 
 """
     rglob(filepattern, dir = pwd(), depth = 4)
@@ -28,11 +31,19 @@ function loadfile(file::String)
 end
 loadfile(file::AbstractPath) = loadfile(string(file))
 
+"""
+    FileDataset(paths)
+    FileDataset(dir, pattern = "*", depth = 4)
+
+Wrap a set of file `paths` as a dataset (traversed in the same order as `paths`).
+Alternatively, specify a `dir` and collect all paths that match a glob `pattern`
+(recursively globbing by `depth`). The glob order determines the traversal order.
+"""
 struct FileDataset{T} <: AbstractDataContainer
     paths::T
 end
 
-FileDataset(dir, pattern = "*", depth = 4) = rglob(pattern, string(dir), depth)
+FileDataset(dir, pattern = "*", depth = 4) = FileDataset(rglob(pattern, string(dir), depth))
 
 MLUtils.getobs(dataset::FileDataset, i) = loadfile(dataset.paths[i])
 MLUtils.numobs(dataset::FileDataset) = length(dataset.paths)

src/containers/tabledataset.jl

@@ -1,3 +1,11 @@
+"""
+    TableDataset(table)
+    TableDataset(path::Union{AbstractPath, AbstractString})
+
+Wrap a Tables.jl-compatible `table` as a dataset container.
+Alternatively, specify the `path` to a CSV file directly
+to load it with CSV.jl + DataFrames.jl.
+"""
 struct TableDataset{T} <: AbstractDataContainer
     table::T
 

test/containers/filedataset.jl

@@ -0,0 +1,36 @@
+function setup_filedataset_test()
+    files = [
+        "root/p1/f1.csv",
+        "root/p2/f2.csv",
+        "root/p2/p2p1/f2.csv",
+        "root/p3/p3p1/f1.csv"
+    ]
+
+    for (i, file) in enumerate(files)
+        paths = splitpath(file)[1:(end - 1)]
+        root = ""
+        for p in paths
+            fullp = joinpath(root, p)
+            isdir(fullp) || mkdir(fullp)
+            root = fullp
+        end
+
+        open(file, "w") do io
+            write(io, "a,b,c\n")
+            write(io, join(i .* [1, 2, 3], ","))
+        end
+    end
+
+    return files
+end
+
+@testset "FileDataset" begin
+    files = setup_filedataset_test()
+    dataset = FileDataset("root", "*.csv")
+    @test numobs(dataset) == length(files)
+    for (i, file) in enumerate(files)
+        true_obs = MLDatasets.loadfile(file)
+        @test getobs(dataset, i) == true_obs
+    end
+    rm("root"; recursive = true)
+end

test/runtests.jl

@@ -28,6 +28,7 @@ dataset_tests = [
 ]
 
 container_tests = [
+    "containers/filedataset.jl",
     "containers/tabledataset.jl",
 ]
 
@@ -37,8 +38,10 @@ container_tests = [
     end
 end
 
-@testset "Containers" for t in container_tests
-    include(t)
+@testset "Containers" begin
+    for t in container_tests
+        include(t)
+    end
 end
 
 #temporary to not stress CI