add Cora Dataset (#69)

* add Cora Dataset

Author: CarloLucibello

README.md

@@ -38,10 +38,8 @@ Find below a list of available datasets and links to their documentation.
   - [UD_English](https://juliaml.github.io/MLDatasets.jl/latest/datasets/UD_English/)
 
 #### Graphs
-  - To be added.
+  - [Cora](https://juliaml.github.io/MLDatasets.jl/latest/datasets/Cora/)
 
-#### Audio
-  - To be added.
 
 
 ## Installation
@@ -55,6 +53,11 @@ import Pkg
 Pkg.add("MLDatasets")
 ```
 
+## Contributing to MLDatasets
+
+New dataset contributions are warmly welcome. See `src/Cora/Cora.jl` for an example
+of a minimal implementation. 
+
 ## License
 
 This code is free to use under the terms of the MIT license.

docs/make.jl

@@ -1,6 +1,7 @@
 using Documenter, MLDatasets
 
-DocMeta.setdocmeta!(MLDatasets, :DocTestSetup, :(using MLDatasets); recursive=true)
+## Commented out since gives warning
+# DocMeta.setdocmeta!(MLDatasets, :DocTestSetup, :(using MLDatasets); recursive=true)
 
 # Build documentation.
 # ====================
@@ -38,9 +39,14 @@ makedocs(
                 "UD_English" => "datasets/UD_English.md",
             ],
 
+            "Graphs" => Any[
+                "Cora" => "datasets/Cora.md",
+            ],
+
         ],
         "LICENSE.md",
-    ]
+    ],
+    strict = true
 )
 
 

docs/src/datasets/Cora.md

@@ -0,0 +1,11 @@
+# Cora
+
+```@docs
+Cora
+```
+
+## API reference
+
+```@docs
+Cora.alldata
+```

src/Cora/Cora.jl

@@ -0,0 +1,69 @@
+export Cora
+
+"""
+    Cora
+
+The full Cora citation network dataset from the
+`"Deep Gaussian Embedding of Graphs: Unsupervised Inductive Learning via
+Ranking" <https://arxiv.org/abs/1707.03815>`_ paper.
+Nodes represent documents and edges represent citation links.
+
+## Interface
+
+- [`Cora.alldata`](@ref)
+"""
+module Cora
+
+using DataDeps
+using ..MLDatasets: datafile
+using DelimitedFiles: readdlm
+
+const DEPNAME = "Cora"
+const LINK = "http://nrvis.com/download/data/labeled/cora.zip"
+const DOCS = "http://networkrepository.com/cora.php"
+
+
+function __init__()
+    register(DataDep(
+        DEPNAME,
+        """
+        Dataset: The $DEPNAME dataset.
+        Website: $DOCS
+        """,
+        LINK,
+        "a3e3a37c34c9385fe8089bbc7c17ef78ecc3bdf8a4b03b80d02aaa080d9501c8",  # if checksum omitted, will be generated by DataDeps
+        post_fetch_method = unpack
+    ))
+end
+
+"""
+    alldata(; dir=nothing)
+
+Retrieve the Cora dataset. The output is a named tuple with fields
+
+- `edges`
+- `node_labels`
+- `directed`
+
+## Usage Examples
+```juliarepl
+julia> using MLDatasets: Cora
+
+julia> data = Cora.alldata()
+(edges = [1 9; 1 436; … ; 2708 1390; 2708 2345], node_labels = [3, 6, 5, 5, 4, 4, 7, 3, 3, 7  …  4, 4, 4, 3, 2, 2, 2, 2, 1, 3], directed = true)
+```
+"""
+function alldata(; dir=nothing)
+    edges = readdlm(datafile(DEPNAME, "cora.edges", dir), ',', Int)
+    @assert all(edges[:,3] .== 1)
+    edges = edges[:,1:2]
+    
+    node_labels = readdlm(datafile(DEPNAME, "cora.node_labels", dir), ',', Int)
+    node_labels = node_labels[:,2] # first column is just 1:n
+    
+    return (; edges=edges, 
+              node_labels=node_labels, 
+              directed=true)
+end
+
+end

src/EMNIST/EMNIST.jl

@@ -9,19 +9,6 @@ using ..MNIST: convert2image
 
     const DEPNAME = "EMNIST"
 
-    """
-        download([dir]; [i_accept_the_terms_of_use])
-
-    Trigger the (interactive) download of the full dataset into
-    "`dir`". If no `dir` is provided the dataset will be
-    downloaded into "~/.julia/datadeps/$DEPNAME".
-
-    This function will display an interactive dialog unless
-    either the keyword parameter `i_accept_the_terms_of_use` or
-    the environment variable `DATADEPS_ALWAY_ACCEPT` is set to
-    `true`. Note that using the data responsibly and respecting
-    copyright/terms-of-use remains your responsibility.
-    """
     download(args...; kw...) = download_dep(DEPNAME, args...; kw...)
 
     function __init__()

src/MLDatasets.jl

@@ -25,17 +25,27 @@ end
 
 
 include("download.jl")
-include("CoNLL.jl")
+
+
+# Misc.
 include("BostonHousing/BostonHousing.jl")
 include("Iris/Iris.jl")
+
+# Vision
 include("CIFAR10/CIFAR10.jl")
 include("CIFAR100/CIFAR100.jl")
 include("MNIST/MNIST.jl")
 include("FashionMNIST/FashionMNIST.jl")
 include("SVHN2/SVHN2.jl")
+include("EMNIST/EMNIST.jl")
+
+# Text
+include("CoNLL.jl")
 include("PTBLM/PTBLM.jl")
 include("UD_English/UD_English.jl")
-include("EMNIST/EMNIST.jl")
+
+# Graphs
+include("Cora/Cora.jl")
 
 function __init__()
     # initialize optional dependencies

src/download.jl

@@ -32,7 +32,7 @@ function datafile(depname, filename, dir = nothing; recurse = true, kw...)
     path = joinpath(datadir(depname, dir; kw...), filename)
     if !isfile(path)
         @warn "The file \"$path\" does not exist, even though the dataset-specific folder does. This is an unusual situation that may have been caused by a manual creation of an empty folder, or manual deletion of the given file \"$filename\"."
-        if dir == nothing
+        if dir === nothing
             @info "Retriggering DataDeps.jl for \"$depname\""
             download_dep(depname; kw...)
         else

test/runtests.jl

@@ -1,6 +1,10 @@
 using Test
 using MLDatasets
 using ImageCore
+using DataDeps
+
+
+ENV["DATADEPS_ALWAYS_ACCEPT"] = true
 
 tests = [
     "tst_iris.jl",
@@ -11,6 +15,7 @@ tests = [
     "tst_fashion_mnist.jl",
     "tst_svhn2.jl",
     "tst_emnist.jl",
+    "tst_cora.jl"
 ]
 
 for t in tests

test/tst_boston_housing.jl

@@ -4,7 +4,7 @@ using DataDeps
 using MLDatasets
 
 
-@testset "Iris" begin
+@testset "Boston Housing" begin
     X  = BostonHousing.features()
     Y  = BostonHousing.targets()
     names = BostonHousing.feature_names()

test/tst_cora.jl

@@ -0,0 +1,14 @@
+data_dir = withenv("DATADEPS_ALWAY_ACCEPT"=>"true") do
+    datadep"Cora"
+end
+
+@testset "Cora" begin
+    data  = Cora.alldata()
+    @test data isa NamedTuple
+
+    @test data.edges isa Matrix{Int}
+    @test size(data.edges) == (5429, 2)
+    @test data.node_labels isa Vector{Int}
+    @test size(data.node_labels) == (2708,)
+    @test data.directed
+end