rework the NLP datasets dor DataDeps

Author: Evizero

src/io/CoNLL.jl renamed to src/CoNLL.jl

@@ -9,14 +9,14 @@ bytes_to_type(::Type{T}, A::Array{UInt8}) where T<:Integer = convert(Array{T}, A
 bytes_to_type(::Type{T}, A::Array{UInt8}) where T<:AbstractFloat = A ./ T(255)
 bytes_to_type(::Type{T}, A::Array{UInt8}) where T<:Number  = convert(Array{T}, reinterpret(N0f8, A))
 
-include("io/download.jl")
-include("io/CoNLL.jl")
+include("download.jl")
+include("CoNLL.jl")
 
 include("CIFAR10/CIFAR10.jl")
 include("CIFAR100/CIFAR100.jl")
 include("MNIST/MNIST.jl")
 include("FashionMNIST/FashionMNIST.jl")
-include("PTBLM.jl")
-include("UD_English.jl")
+include("PTBLM/PTBLM.jl")
+include("UD_English/UD_English.jl")
 
 end

src/MLDatasets.jl

@@ -0,0 +1,81 @@
+export PTBLM
+module PTBLM
+
+    using DataDeps
+    using ..datafile
+    using ..download_dep
+
+    export
+
+        traindata,
+        testdata,
+
+        download
+
+    const DEPNAME = "PTBLM"
+    const TRAINFILE = "ptb.train.txt"
+    const TESTFILE  = "ptb.test.txt"
+
+    download(args...; kw...) = download_dep(DEPNAME, args...; kw...)
+
+    traindata(; dir = nothing) = traindata(dir)
+    testdata(; dir = nothing) = testdata(dir)
+
+    function traindata(dir)
+        path = datafile(DEPNAME, TRAINFILE, dir)
+        xs = readdata(path)
+        ys = makeys(xs)
+        xs, ys
+    end
+
+    function testdata(dir)
+        path = datafile(DEPNAME, TESTFILE, dir)
+        xs = readdata(path)
+        ys = makeys(xs)
+        xs, ys
+    end
+
+    function readdata(path)
+        lines = open(readlines, path)
+        map(l -> Vector{String}(split(chomp(l))), lines)
+    end
+
+    function makeys(xs::Vector{Vector{String}})
+        map(xs) do x
+            y = copy(x)
+            shift!(y)
+            push!(y, "<eos>")
+        end
+    end
+
+    function __init__()
+        RegisterDataDep(
+            DEPNAME,
+            """
+            Dataset: Penn Treebank sentences for language modeling
+            Website: https://github.com/tomsercu/lstm
+
+            -----------------------------------------------------
+            WARNING: EXPERIMENTAL STATUS
+            Please be aware that this dataset is from a secondary
+            source. The provided interface by this package is not
+            as developed as those for other datasets. We would
+            welcome any contribution to provide this dataset in a
+            more mature manner.
+            ------------------------------------------------------
+
+            The PTBLM dataset consists of Penn Treebank sentences
+            for language modeling, available from tomsercu/lstm.
+            The unknown words are replaced with <unk> so that the
+            total vocaburary size becomes 10000.
+
+            The files are available for download at the github
+            repository linked above. Note that using the data
+            responsibly and respecting copyright remains your
+            responsibility.
+            """,
+            "https://raw.githubusercontent.com/tomsercu/lstm/master/data/" .* [TRAINFILE, TESTFILE],
+            "218f4e6c7288bb5efeb03cc4cb8ae9c04ecd8462ebfba8e13e3549fab69dc25f",
+        )
+    end
+end

src/PTBLM.jl

@@ -0,0 +1,77 @@
+export UD_English
+module UD_English
+
+    using DataDeps
+    using ..MLDatasets
+    using ..datafile
+    using ..download_dep
+
+    export
+
+        traindata,
+        testdata,
+
+        download
+
+    const DEPNAME = "UD_English"
+    const TRAINFILE = "en-ud-train.conllu"
+    const DEVFILE   = "en-ud-dev.conllu"
+    const TESTFILE  = "en-ud-test.conllu"
+
+    download(args...; kw...) = download_dep(DEPNAME, args...; kw...)
+
+    traindata(; dir = nothing) = traindata(dir)
+    devdata(; dir = nothing)   = devdata(dir)
+    testdata(; dir = nothing)  = testdata(dir)
+
+    traindata(dir) = readdata(dir, TRAINFILE)
+    devdata(dir)   = readdata(dir, DEVFILE)
+    testdata(dir)  = readdata(dir, TESTFILE)
+
+    function readdata(dir, filename)
+        path = datafile(DEPNAME, filename, dir)
+        CoNLL.read(path)
+    end
+
+    function __init__()
+        RegisterDataDep(
+            DEPNAME,
+            """
+            Dataset: Universal Dependencies - English Dependency Treebank Universal Dependencies English Web Treebank
+            Authors: Natalia Silveira and Timothy Dozat and
+                     Marie-Catherine de Marneffe and Samuel
+                     Bowman and Miriam Connor and John Bauer and
+                     Christopher D. Manning
+            Website: https://github.com/UniversalDependencies/UD_English
+
+            A Gold Standard Universal Dependencies Corpus for
+            English, built over the source material of the
+            English Web Treebank LDC2012T13
+            (https://catalog.ldc.upenn.edu/LDC2012T13).
+
+            You are encouraged to cite this paper if you use the
+            Universal Dependencies English Web Treebank:
+
+                @inproceedings{silveira14gold,
+                    year = {2014},
+                    author = {Natalia Silveira and Timothy Dozat
+                        and Marie-Catherine de Marneffe and Samuel
+                        Bowman and Miriam Connor and John Bauer and
+                        Christopher D. Manning},
+                    title = {A Gold Standard Dependency Corpus for {E}nglish},
+                    booktitle = {Proceedings of the Ninth
+                        International Conference on Language Resources
+                        and Evaluation (LREC-2014)}
+                }
+
+            The files are available for download at the github
+            repository linked above. Note that using the data
+            responsibly and respecting copyright remains your
+            responsibility. Copyright and License is discussed in
+            detail on the Website.
+            """,
+            "https://raw.githubusercontent.com/UniversalDependencies/UD_English/master/" .* [TRAINFILE, DEVFILE, TESTFILE],
+            "2311e260488453d5ba170cfd94e58ac4bd536263ea9545c7b25f0804e87b28a2",
+        )
+    end
+end

src/PTBLM/PTBLM.jl

@@ -32,8 +32,13 @@ function datafile(depname, filename, dir = nothing; recurse = true, kw...)
     path = joinpath(datadir(depname, dir; kw...), filename)
     if !isfile(path)
         warn("The file \"$path\" does not exist, even though the dataset-specific folder does. This is an unusual situation that may have been caused by a manual creation of an empty folder, or manual deletion of the given file \"$filename\".")
-        info("Retriggering DataDeps.jl for \"$depname\" to \"$dir\".")
-        download_dep(depname, dir; kw...)
+        if dir == nothing
+            info("Retriggering DataDeps.jl for \"$depname\"")
+            download_dep(depname; kw...)
+        else
+            info("Retriggering DataDeps.jl for \"$depname\" to \"$dir\".")
+            download_dep(depname, dir; kw...)
+        end
         if recurse
             datafile(depname, filename, dir; recurse = false, kw...)
         else