Skip to content

Commit 842e0ab

Browse files
EvizeroEvizero
committed
outline SVHN format 2 dataset
1 parent 142ee84 commit 842e0ab

File tree

6 files changed

+328
-0
lines changed

6 files changed

+328
-0
lines changed

‎README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ Dataset | Classes | `traintensor` | `trainlabels` | `testtensor` | `testlabels`
7070
[**FashionMNIST**](https://juliaml.github.io/MLDatasets.jl/latest/datasets/FashionMNIST/) | 10 | 28x28x60000 | 60000 | 28x28x10000 | 10000
7171
[**CIFAR-10**](https://juliaml.github.io/MLDatasets.jl/latest/datasets/CIFAR10/) | 10 | 32x32x3x50000 | 50000 | 32x32x3x10000 | 10000
7272
[**CIFAR-100**](https://juliaml.github.io/MLDatasets.jl/latest/datasets/CIFAR100/) | 100 (20) | 32x32x3x50000 | 50000 (x2) | 32x32x3x10000 | 10000 (x2)
73+
[**SVHN-2**](https://juliaml.github.io/MLDatasets.jl/latest/datasets/SVHN2/)(*) | 10 | 32x32x3x73257 | 73257 | 32x32x3x26032 | 26032
74+
75+
(*) Note that the SVHN-2 dataset provides an additional 531131 observations aside from the training- and testset
7376

7477
### Language Modeling
7578

‎REQUIRE

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ ColorTypes 0.4
55
DataDeps
66
GZip
77
BinDeps
8+
MAT

‎src/MLDatasets.jl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ include("CIFAR10/CIFAR10.jl")
1616
include("CIFAR100/CIFAR100.jl")
1717
include("MNIST/MNIST.jl")
1818
include("FashionMNIST/FashionMNIST.jl")
19+
include("SVHN2/SVHN2.jl")
1920
include("PTBLM/PTBLM.jl")
2021
include("UD_English/UD_English.jl")
2122

‎src/SVHN2/SVHN2.jl

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
export SVHN2
2+
3+
"""
4+
The Street View House Numbers (SVHN) Dataset
5+
6+
Authors: Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Bo Wu, Andrew Y. Ng
7+
Website: http://ufldl.stanford.edu/housenumbers
8+
9+
SVHN was obtained from house numbers in Google Street View
10+
images. As such they are quite diverse in terms of orientation
11+
and image background. Similar to MNIST, SVHN has 10 classes (the
12+
digits 0-9), but unlike MNIST there is more data and the images
13+
are a little bigger (32x32 instead of 28x28) with an additional
14+
RGB color channel. The dataset is split up into three subsets:
15+
73257 digits for training, 26032 digits for testing, and 531131
16+
additional to use as extra training data.
17+
18+
## Interface
19+
20+
- [SVHN2.traindata](@ref)
21+
- [SVHN2.testdata](@ref)
22+
- [SVHN2.extradata](@ref)
23+
24+
## Utilities
25+
26+
- [SVHN2.convert2features](@ref)
27+
- [SVHN2.convert2image](@ref)
28+
"""
29+
module SVHN2
30+
using DataDeps
31+
using MAT
32+
using ImageCore
33+
using ColorTypes
34+
using FixedPointNumbers
35+
using ..bytes_to_type
36+
using ..datafile
37+
using ..download_dep
38+
using ..download_docstring
39+
40+
export
41+
42+
traindata,
43+
testdata,
44+
extradata,
45+
46+
convert2image,
47+
convert2features,
48+
49+
download
50+
51+
const DEPNAME = "SVHN2"
52+
const TRAINDATA = "train_32x32.mat"
53+
const TESTDATA = "test_32x32.mat"
54+
const EXTRADATA = "extra_32x32.mat"
55+
const CLASSES = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0]
56+
57+
download(args...; kw...) = download_dep(DEPNAME, args...; kw...)
58+
59+
include("interface.jl")
60+
include("utils.jl")
61+
62+
function __init__()
63+
RegisterDataDep(
64+
DEPNAME,
65+
"""
66+
Dataset: The Street View House Numbers (SVHN) Dataset
67+
Authors: Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Bo Wu, Andrew Y. Ng
68+
Website: http://ufldl.stanford.edu/housenumbers
69+
Format: Cropped Digits (Format 2 on the website)
70+
Note: for non-commercial use only
71+
72+
[Netzer et al., 2011]
73+
Yuval Netzer, Tao Wang, Adam Coates, Alessandro Bissacco, Bo Wu, Andrew Y. Ng
74+
"Reading Digits in Natural Images with Unsupervised Feature Learning"
75+
NIPS Workshop on Deep Learning and Unsupervised Feature Learning 2011
76+
77+
The dataset is split up into three subsets: 73257
78+
digits for training, 26032 digits for testing, and
79+
531131 additional to use as extra training data.
80+
81+
The files are available for download at the official
82+
website linked above. Note that using the data
83+
responsibly and respecting copyright remains your
84+
responsibility. For example the website mentions that
85+
the data is for non-commercial use only. Please read
86+
the website to make sure you want to download the
87+
dataset.
88+
""",
89+
"http://ufldl.stanford.edu/housenumbers/" .* [TRAINDATA, TESTDATA, EXTRADATA],
90+
)
91+
end
92+
end

‎src/SVHN2/interface.jl

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
"""
2+
classnames() -> Vector{Int}
3+
4+
Return the 10 digits for the SVHN classes as a vector of integers.
5+
"""
6+
classnames() = CLASSES
7+
8+
"""
9+
traindata([T = N0f8], [indices]; [dir]) -> images, labels
10+
11+
Returns the SVHN **trainset** corresponding to the given
12+
`indices` as a two-element tuple. If `indices` is omitted the
13+
full trainset is returned. The first element of the return values
14+
will be the images as a multi-dimensional array, and the second
15+
element the corresponding labels as integers.
16+
17+
The image(s) is/are returned in the native vertical-major memory
18+
layout as a single numeric array of eltype `T`. If `T <:
19+
Integer`, then all values will be within `0` and `255`, otherwise
20+
the values are scaled to be between `0` and `1`. You can use the
21+
utility function [`convert2image`](@ref) to convert an SVHN array
22+
into a Julia image with the appropriate `RGB` eltype. The integer
23+
values of the labels correspond 1-to-1 the digit that they
24+
represent with the exception of 0 which is encoded as `10`.
25+
26+
Note that because of the nature of how the dataset is stored on
27+
disk, `SVHN2.traindata` will always load the full trainset,
28+
regardless of which observations are requested. In the case
29+
`indices` are provided by the user, it will simply result in a
30+
sub-setting. This option is just provided for convenience.
31+
32+
```julia
33+
train_x, train_y = SVHN2.traindata() # full dataset
34+
train_x, train_y = SVHN2.traindata(2) # only second observation
35+
train_x, train_y = SVHN2.traindata(dir="./SVHN") # custom folder
36+
```
37+
38+
$(download_docstring("SVHN", DEPNAME))
39+
"""
40+
function traindata(args...; dir = nothing)
41+
traindata(N0f8, args...; dir = dir)
42+
end
43+
44+
function traindata(::Type{T}; dir = nothing) where T
45+
path = datafile(DEPNAME, TRAINDATA, dir)
46+
vars = matread(path)
47+
images, labels = vars["X"], vars["y"]
48+
bytes_to_type(T, images), Vector{Int}(vec(labels))
49+
end
50+
51+
function traindata(::Type{T}, indices; dir = nothing) where T
52+
images, labels = traindata(T, dir = dir)
53+
images[:,:,:,indices], labels[indices]
54+
end
55+
56+
"""
57+
testdata([T = N0f8], [indices]; [dir]) -> images, labels
58+
59+
Returns the SVHN **testset** corresponding to the given
60+
`indices` as a two-element tuple. If `indices` is omitted the
61+
full testset is returned. The first element of the return
62+
values will be the images as a multi-dimensional array, and the
63+
second element the corresponding labels as integers.
64+
65+
The image(s) is/are returned in the native vertical-major memory
66+
layout as a single numeric array of eltype `T`. If `T <:
67+
Integer`, then all values will be within `0` and `255`, otherwise
68+
the values are scaled to be between `0` and `1`. You can use the
69+
utility function [`convert2image`](@ref) to convert an SVHN array
70+
into a Julia image with the appropriate `RGB` eltype. The integer
71+
values of the labels correspond 1-to-1 the digit that they
72+
represent with the exception of 0 which is encoded as `10`.
73+
74+
Note that because of the nature of how the dataset is stored on
75+
disk, `SVHN2.testdata` will always load the full testset,
76+
regardless of which observations are requested. In the case
77+
`indices` are provided by the user, it will simply result in a
78+
sub-setting. This option is just provided for convenience.
79+
80+
```julia
81+
test_x, test_y = SVHN2.testdata() # full dataset
82+
test_x, test_y = SVHN2.testdata(2) # only second observation
83+
test_x, test_y = SVHN2.testdata(dir="./SVHN") # custom folder
84+
```
85+
86+
$(download_docstring("SVHN", DEPNAME))
87+
"""
88+
function testdata(args...; dir = nothing)
89+
testdata(N0f8, args...; dir = dir)
90+
end
91+
92+
function testdata(::Type{T}; dir = nothing) where T
93+
path = datafile(DEPNAME, TESTDATA, dir)
94+
vars = matread(path)
95+
images, labels = vars["X"], vars["y"]
96+
bytes_to_type(T, images), Vector{Int}(vec(labels))
97+
end
98+
99+
function testdata(::Type{T}, indices; dir = nothing) where T
100+
images, labels = testdata(T, dir = dir)
101+
images[:,:,:,indices], labels[indices]
102+
end
103+
104+
"""
105+
extradata([T = N0f8], [indices]; [dir]) -> images, labels
106+
107+
Returns the SVHN **extra trainset** corresponding to the given
108+
`indices` as a two-element tuple. If `indices` is omitted the
109+
full dataset is returned. The first element of the return values
110+
will be the images as a multi-dimensional array, and the second
111+
element the corresponding labels as integers.
112+
113+
The image(s) is/are returned in the native vertical-major memory
114+
layout as a single numeric array of eltype `T`. If `T <:
115+
Integer`, then all values will be within `0` and `255`, otherwise
116+
the values are scaled to be between `0` and `1`. You can use the
117+
utility function [`convert2image`](@ref) to convert an SVHN array
118+
into a Julia image with the appropriate `RGB` eltype. The integer
119+
values of the labels correspond 1-to-1 the digit that they
120+
represent with the exception of 0 which is encoded as `10`.
121+
122+
Note that because of the nature of how the dataset is stored on
123+
disk, `SVHN2.extradata` will always load the full extra trainset,
124+
regardless of which observations are requested. In the case
125+
`indices` are provided by the user, it will simply result in a
126+
sub-setting. This option is just provided for convenience.
127+
128+
```julia
129+
extra_x, extra_y = SVHN2.extradata() # full dataset
130+
extra_x, extra_y = SVHN2.extradata(2) # only second observation
131+
extra_x, extra_y = SVHN2.extradata(dir="./SVHN") # custom folder
132+
```
133+
134+
$(download_docstring("SVHN", DEPNAME))
135+
"""
136+
function extradata(args...; dir = nothing)
137+
extradata(N0f8, args...; dir = dir)
138+
end
139+
140+
function extradata(::Type{T}; dir = nothing) where T
141+
path = datafile(DEPNAME, EXTRADATA, dir)
142+
vars = matread(path)
143+
images, labels = vars["X"], vars["y"]
144+
bytes_to_type(T, images), Vector{Int}(vec(labels))
145+
end
146+
147+
function extradata(::Type{T}, indices; dir = nothing) where T
148+
images, labels = extradata(T, dir = dir)
149+
images[:,:,:,indices], labels[indices]
150+
end

‎src/SVHN2/utils.jl

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
"""
2+
convert2features(array)
3+
4+
Convert the given SVHN tensor to a feature matrix (or feature
5+
vector in the case of a single image). The purpose of this
6+
function is to drop the spatial dimensions such that traditional
7+
ML algorithms can process the dataset.
8+
9+
```julia
10+
julia> SVHN2.convert2features(SVHN2.traindata(Float32)[1]) # full training data
11+
3072×50000 Array{Float32,2}:
12+
[...]
13+
14+
julia> SVHN2.convert2features(SVHN2.traindata(Float32)[1][:,:,:,1]) # first observation
15+
3072-element Array{Float32,1}:
16+
[...]
17+
```
18+
"""
19+
function convert2features(array::AbstractArray{<:Number,3})
20+
nrows, ncols, nchan = size(array)
21+
@assert nchan == 3 "the given array should have the RGB channel in the third dimension"
22+
vec(array)
23+
end
24+
25+
function convert2features(array::AbstractArray{<:Number,4})
26+
nrows, ncols, nchan, nimages = size(array)
27+
@assert nchan == 3 "the given array should have the RGB channel in the third dimension"
28+
reshape(array, (nrows * ncols * nchan, nimages))
29+
end
30+
31+
convert2features(array::AbstractArray{<:RGB,2}) =
32+
convert2features(permutedims(channelview(array), (3,1,2)))
33+
34+
convert2features(array::AbstractArray{<:RGB,3}) =
35+
convert2features(permutedims(channelview(array), (3,1,2,4)))
36+
37+
"""
38+
convert2image(array) -> Array{RGB}
39+
40+
Convert the given SVHN tensor (or feature vector/matrix) to a
41+
`RGB` array.
42+
43+
```julia
44+
julia> SVHN2.convert2image(SVHN2.traindata()[1]) # full training dataset
45+
32×32×50000 Array{RGB{N0f8},3}:
46+
[...]
47+
48+
julia> SVHN2.convert2image(SVHN2.traindata()[1][:,:,:,1]) # first training image
49+
32×32 Array{RGB{N0f8},2}:
50+
[...]
51+
```
52+
"""
53+
function convert2image(array::AbstractVector{<:Number})
54+
@assert length(array) % 3072 == 0
55+
if length(array) == 3072
56+
convert2image(reshape(array, 32, 32, 3))
57+
else
58+
n = Int(length(array) / 3072)
59+
convert2image(reshape(array, 32, 32, 3, n))
60+
end
61+
end
62+
63+
function convert2image(array::AbstractMatrix{<:Number})
64+
@assert size(array, 1) == 3072
65+
convert2image(reshape(array, 32, 32, 3, size(array, 2)))
66+
end
67+
68+
function convert2image(array::AbstractArray{<:Number,3})
69+
nrows, ncols, nchan = size(array)
70+
@assert nchan == 3 "the given array should have the RGB channel in the third dimension"
71+
colorview(RGB, permutedims(_norm_array(array), (3,1,2)))
72+
end
73+
74+
function convert2image(array::AbstractArray{<:Number,4})
75+
nrows, ncols, nchan, nimages = size(array)
76+
@assert nchan == 3 "the given array should have the RGB channel in the third dimension"
77+
colorview(RGB, permutedims(_norm_array(array), (3,1,2,4)))
78+
end
79+
80+
_norm_array(array::AbstractArray) = array
81+
_norm_array(array::AbstractArray{<:Integer}) = reinterpret(N0f8, convert(Array{UInt8}, array))

0 commit comments

Comments
 (0)