add FashionMNIST dataset

Author: CarloLucibello

src/CIFAR10.jl

@@ -3,7 +3,7 @@ module CIFAR10
 
 using BinDeps
 
-const defdir = joinpath(Pkg.dir("MLDatasets"), "datasets/cifar10")
+const defdir = joinpath(Pkg.dir("MLDatasets"), "datasets", "cifar10")
 
 function getdata(dir)
     mkpath(dir)
@@ -25,7 +25,7 @@ function readdata(data::Vector{UInt8})
 end
 
 function traindata(dir=defdir)
-    files = ["$(dir)/cifar-10-batches-bin/data_batch_$(i).bin" for i=1:5]
+    files = [joinpath(dir,"cifar-10-batches-bin","data_batch_$i.bin") for i=1:5]
     all(isfile, files) || getdata(dir)
     data = UInt8[]
     for file in files
@@ -35,7 +35,7 @@ function traindata(dir=defdir)
 end
 
 function testdata(dir=defdir)
-    file = "$(dir)/cifar-10-batches-bin/test_batch.bin"
+    file = joinpath(dir,"cifar-10-batches-bin","test_batch.bin")
     isfile(file) || getdata(dir)
     readdata(open(read,file))
 end

src/CIFAR100.jl

@@ -3,7 +3,7 @@ module CIFAR100
 
 using BinDeps
 
-const defdir = joinpath(Pkg.dir("MLDatasets"), "datasets/cifar100")
+const defdir = joinpath(Pkg.dir("MLDatasets"), "datasets","cifar100")
 
 function getdata(dir)
     mkpath(dir)
@@ -25,13 +25,13 @@ function readdata(data::Vector{UInt8})
 end
 
 function traindata(dir=defdir)
-    file = joinpath(dir, "cifar-100-binary/train.bin")
+    file = joinpath(dir, "cifar-100-binary","train.bin")
     isfile(file) || getdata(dir)
     readdata(open(read,file))
 end
 
 function testdata(dir=defdir)
-    file = joinpath(dir, "cifar-100-binary/test.bin")
+    file = joinpath(dir, "cifar-100-binary","test.bin")
     isfile(file) || getdata(dir)
     readdata(open(read,file))
 end

src/FashionMNIST/FashionMNIST.jl

@@ -0,0 +1,30 @@
+export FashionMNIST
+module FashionMNIST
+    using ImageCore
+    using ColorTypes
+
+    export
+
+        traintensor,
+        testtensor,
+
+        trainlabels,
+        testlabels,
+
+        traindata,
+        testdata,
+
+        convert2image,
+        convert2features,
+
+        download_helper
+
+    const DEFAULT_DIR = abspath(joinpath(dirname(@__FILE__), "..", "..", "datasets", "fashion_mnist"))
+
+    include(joinpath("Reader","Reader.jl"))
+    import .Reader.download_helper
+    include("interface.jl")
+    include(joinpath("..", "MNIST", "utils.jl"))
+
+    Reader.download_helper(; nargs...) = Reader.download_helper(DEFAULT_DIR; nargs...)
+end

src/FashionMNIST/Reader/Reader.jl

@@ -0,0 +1,30 @@
+module Reader
+    using GZip
+    using BinDeps
+
+    export
+
+        readtrainimages,
+        readtestimages,
+        readtrainimages,
+        readtestlabels,
+
+        download_helper
+
+    # Constants
+
+    const IMAGEOFFSET = 16
+    const LABELOFFSET = 8
+
+    const TRAINIMAGES = "train-images-idx3-ubyte.gz"
+    const TRAINLABELS = "train-labels-idx1-ubyte.gz"
+    const TESTIMAGES  = "t10k-images-idx3-ubyte.gz"
+    const TESTLABELS  = "t10k-labels-idx1-ubyte.gz"
+
+    # Includes
+
+    include("readheader.jl")
+    include("readimages.jl")
+    include("readlabels.jl")
+    include("download.jl")
+end

src/FashionMNIST/Reader/download.jl

@@ -0,0 +1,69 @@
+msg_notfound(dir, filename) = "The FashionMNIST file \"$filename\" was not found in \"$dir\".
+                                You can download the dataset at https://github.com/zalandoresearch/fashion-mnist,
+                                or alternatively use FashionMNIST.download_helper(directory) to do it for you."
+
+msg_prompt(dir, files) = """
+Interactive session detected. FashionMNIST.download_helper initiated.
+
+Dataset: THE FashionMNIST DATABASE of fashion products
+Authors: Han Xiao, Kashif Rasul, Roland Vollgraf
+Website: https://github.com/zalandoresearch/fashion-mnist
+
+Paper: Han Xiao, Kashif Rasul, Roland Vollgraf "Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms."
+
+The specified directory \"$dir\" is missing the files $(join(map(f->"\"$f\"", files), ", ", " and ")) of the full data set.
+
+The files are available for download at the offical website linked above.
+We can download these files for you if you wish.
+You want to download the dataset to \"$dir\"? [y/n] """
+
+function downloaded_file(dir, filename)
+    path = joinpath(dir, filename)
+    if !isfile(path)
+        if isinteractive()
+            warn(msg_notfound(dir, filename))
+            download_helper(dir)
+        else
+            error(msg_notfound(dir, filename))
+        end
+    end
+    path
+end
+
+"""
+    download_helper(dir; i_accept_the_terms_of_use = true)
+
+Check if the FashionMNIST dataset is contained in the specified `dir`,
+or if any of the four files are missing. If `dir` is omitted it
+will default to `MLDatasets/datasets/fashion_mnist`.
+
+In the case that any of the four files is missing the user will be presented
+with the option to download it to the specified `dir`.
+"""
+function download_helper(dir; i_accept_the_terms_of_use = true)
+    files = filter(file->!isfile(joinpath(dir, file)),
+                   [TRAINIMAGES, TRAINLABELS, TESTIMAGES, TESTLABELS])
+    if !isempty(files)
+        if !i_accept_the_terms_of_use && isinteractive()
+            print(msg_prompt(dir, files))
+            answer = first(readline())
+            if answer == 'y'
+                i_accept_the_terms_of_use = true
+            end
+        end
+        if i_accept_the_terms_of_use
+            mkpath(dir)
+            for file in files
+                url = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/$file"
+                path = joinpath(dir, file)
+                info("downloading $file from $url to $dir")
+                run(download_cmd(url, path))
+            end
+        else
+            error("Unable to download the dataset. Please visit the website at https://github.com/zalandoresearch/fashion-mnist and download the files manually.")
+        end
+    else
+        info("Nothing to do.")
+    end
+    nothing
+end

src/FashionMNIST/Reader/readheader.jl

@@ -0,0 +1,56 @@
+"""
+    readimageheader(io::IO)
+
+Reads four 32 bit integers at the current position of `io` and
+interprets them as a FashionMNIST-image-file header, which is described
+in detail in the table below
+
+            ║     First    │  Second  │  Third  │   Fourth
+    ════════╬══════════════╪══════════╪═════════╪════════════
+    offset  ║         0000 │     0004 │    0008 │       0012
+    descr   ║ magic number │ # images │  # rows │  # columns
+
+These four numbers are returned as a Tuple in the same storage order
+"""
+function readimageheader(io::IO)
+    magic_number = bswap(read(io, UInt32))
+    total_items  = bswap(read(io, UInt32))
+    nrows = bswap(read(io, UInt32))
+    ncols = bswap(read(io, UInt32))
+    UInt32(magic_number), Int(total_items), Int(nrows), Int(ncols)
+end
+
+"""
+    readimageheader(file::AbstractString)
+
+Opens and reads the first four 32 bits values of `file` and
+returns them interpreted as an FashionMNIST-image-file header
+"""
+function readimageheader(file::AbstractString)
+    gzopen(readimageheader, file, "r")::Tuple{UInt32,Int,Int,Int}
+end
+
+"""
+    readlabelheader(io::IO)
+
+Reads two 32 bit integers at the current position of `io` and
+interprets them as a FashionMNIST-label-file header, which consists of a
+*magic number* and the *total number of labels* stored in the
+file. These two numbers are returned as a Tuple in the same
+storage order.
+"""
+function readlabelheader(io::IO)
+    magic_number = bswap(read(io, UInt32))
+    total_items  = bswap(read(io, UInt32))
+    UInt32(magic_number), Int(total_items)
+end
+
+"""
+    readlabelheader(file::AbstractString)
+
+Opens and reads the first two 32 bits values of `file` and
+returns them interpreted as an FashionMNIST-label-file header
+"""
+function readlabelheader(file::AbstractString)
+    gzopen(readlabelheader, file, "r")::Tuple{UInt32,Int}
+end

src/FashionMNIST/Reader/readimages.jl

@@ -0,0 +1,112 @@
+function readimages!(buffer::Matrix{UInt8}, io::IO, index::Integer, nrows::Integer, ncols::Integer)
+    seek(io, IMAGEOFFSET + nrows * ncols * (index - 1))
+    read!(io, buffer)
+end
+
+"""
+    readimages(io::IO, index::Integer, nrows::Integer, ncols::Integer)
+
+Jumps to the position of `io` where the bytes for the `index`'th
+image are located and reads the next `nrows` * `ncols` bytes. The
+read bytes are returned as a `Matrix{UInt8}` of size `(nrows, ncols)`.
+"""
+function readimages(io::IO, index::Integer, nrows::Integer, ncols::Integer)
+    buffer = Array{UInt8}(nrows, ncols)
+    readimages!(buffer, io, index, nrows, ncols)
+end
+
+"""
+    readimages(io::IO, indices::AbstractVector, nrows::Integer, ncols::Integer)
+
+Reads the first `nrows` * `ncols` bytes for each image index in
+`indices` and stores them in a `Array{UInt8,3}` of size `(nrows,
+ncols, length(indices))` in the same order as denoted by
+`indices`.
+"""
+function readimages(io::IO, indices::AbstractVector, nrows::Integer, ncols::Integer)
+    images = Array{UInt8}(nrows, ncols, length(indices))
+    buffer = Array{UInt8}(nrows, ncols)
+    dst_index = 1
+    for src_index in indices
+        readimages!(buffer, io, src_index, nrows, ncols)
+        copy!(images, 1 + nrows * ncols * (dst_index - 1), buffer, 1, nrows * ncols)
+        dst_index += 1
+    end
+    images
+end
+
+"""
+    readimages(file, [indices])
+
+Reads the images denoted by `indices` from `file`. The given
+`file` can either be specified using an IO-stream or a string
+that denotes the fully qualified path. The conent of `file` is
+assumed to be in the FashionMNIST image-file format, as it is described
+on the official homepage at http://yann.lecun.com/exdb/FashionMNIST/
+
+- if `indices` is an `Integer`, the single image is returned as
+  `Matrix{UInt8}` in horizontal major layout, which means that
+  the first dimension denotes the pixel *rows* (x), and the
+  second dimension denotes the pixel *columns* (y) of the image.
+
+- if `indices` is a `AbstractVector`, the images are returned as
+  a 3D array (i.e. a `Array{UInt8,3}`), in which the first
+  dimension corresponds to the pixel *rows* (x) of the image, the
+  second dimension to the pixel *columns* (y) of the image, and
+  the third dimension denotes the index of the image.
+
+- if `indices` is ommited all images are returned
+  (as 3D array described above)
+"""
+function readimages(io::IO, indices)
+    _, nimages, nrows, ncols = readimageheader(io)
+    @assert minimum(indices) >= 1 && maximum(indices) <= nimages
+    readimages(io, indices, nrows, ncols)
+end
+
+function readimages(file::AbstractString, index::Integer)
+    gzopen(file, "r") do io
+        readimages(io, index)
+    end::Matrix{UInt8}
+end
+
+function readimages(file::AbstractString, indices::AbstractVector)
+    gzopen(file, "r") do io
+        readimages(io, indices)
+    end::Array{UInt8,3}
+end
+
+function readimages(file::AbstractString)
+    gzopen(file, "r") do io
+        _, nimages, nrows, ncols = readimageheader(io)
+        readimages(io, 1:nimages, nrows, ncols)
+    end::Array{UInt8,3}
+end
+
+"""
+    readtrainimages(dir::AbstractString, [indices]) -> Array{UInt8}
+
+Reads the images of the given `indices` from the file
+\"$TRAINIMAGES\" using the function [`readimages`](@ref), and
+returns them as a multi-dimensional array of `UInt8`.
+
+The file is expected to be located in the given directory `dir`.
+In the case that the file does not yet exist the function will
+provide you with download instructions.
+"""
+readtrainimages(dir) = readimages(downloaded_file(dir, TRAINIMAGES))
+readtrainimages(dir, indices) = readimages(downloaded_file(dir, TRAINIMAGES), indices)
+
+"""
+    readtestimages(dir::AbstractString, [indices]) -> Array{UInt8}
+
+Reads the images of the given `indices` from the file
+\"$TESTIMAGES\" using the function [`readimages`](@ref), and
+returns them as a multi-dimensional array of `UInt8`.
+
+The file is expected to be located in the given directory `dir`.
+In the case that the file does not yet exist the function will
+provide you with download instructions.
+"""
+readtestimages(dir) = readimages(downloaded_file(dir, TESTIMAGES))
+readtestimages(dir, indices) = readimages(downloaded_file(dir, TESTIMAGES), indices)