Merge pull request #79 from CTUAvastLab/master

Added mutagenesis dataset

Author: CarloLucibello

Project.toml

@@ -9,6 +9,7 @@ DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 FixedPointNumbers = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
 GZip = "92fee26a-97fe-5a0c-ad85-20a5f3185b63"
+JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 MAT = "23992714-dd62-5051-b70f-ba57cb901cac"
 PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
@@ -20,6 +21,7 @@ DataDeps = "0.3, 0.4, 0.5, 0.6, 0.7"
 FixedPointNumbers = "0.3, 0.4, 0.5, 0.6, 0.7, 0.8"
 GZip = "0.5"
 ImageCore = "0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8"
+JSON3 = "1"
 MAT = "0.7, 0.8, 0.9, 0.10"
 PyCall = "1"
 Requires = "1"

README.md

@@ -30,6 +30,7 @@ Find below a list of available datasets and links to their documentation.
 #### Miscellaneous
   - [BostonHousing](https://juliaml.github.io/MLDatasets.jl/dev/datasets/BostonHousing/)
   - [Iris](https://juliaml.github.io/MLDatasets.jl/dev/datasets/Iris/)
+  - [Mutagenesis](https://relational.fit.cvut.cz/dataset/Mutagenesis)
 
 #### Text
   - [PTBLM](https://juliaml.github.io/MLDatasets.jl/dev/datasets/PTBLM/)

docs/make.jl

@@ -33,6 +33,7 @@ makedocs(
             "Miscellaneous" => Any[
                 "Iris" => "datasets/Iris.md",
                 "Boston Housing" => "datasets/BostonHousing.md",
+                "Mutagenesis" => "datasets/Mutagenesis.md",
             ],
 
             "Text" => Any[

docs/src/datasets/Mutagenesis.md

@@ -0,0 +1,5 @@
+# Mutagenesis
+
+```@docs
+Mutagenesis
+```

src/MLDatasets.jl

@@ -39,6 +39,7 @@ include("download.jl")
 # Misc.
 include("BostonHousing/BostonHousing.jl")
 include("Iris/Iris.jl")
+include("Mutagenesis/Mutagenesis.jl")
 
 # Vision
 include("CIFAR10/CIFAR10.jl")

src/Mutagenesis/Mutagenesis.jl

@@ -0,0 +1,98 @@
+export Mutagenesis
+
+"""
+Mutagenesis
+
+Website: https://relational.fit.cvut.cz/dataset/Mutagenesis
+License: CC0
+
+The `Mutagenesis` dataset comprises 188 molecules trialed for mutagenicity on Salmonella typhimurium, available from
+ [relational.fit.cvut.cz](https://relational.fit.cvut.cz/dataset/Mutagenesis) and
+ [CTUAvastLab/datasets](https://github.com/CTUAvastLab/datasets/tree/main/mutagenesis).
+
+Train, test and validation data can be loaded using following code.
+The `withenv("DATADEPS_ALWAYS_ACCEPT"=>"true")` disables the accept download prompt.
+```jldoctest
+julia> using MLDatasets: Mutagenesis
+
+julia> train_x, train_y = withenv("DATADEPS_ALWAYS_ACCEPT"=>"true") do; Mutagenesis.traindata(); end;
+
+julia> test_x, test_y = Mutagenesis.testdata();
+
+julia> val_x, val_y = Mutagenesis.valdata();
+
+julia> train_x[1]
+JSON3.Object{Base.CodeUnits{UInt8, String}, SubArray{UInt64, 1, Vector{UInt64}, Tuple{UnitRange{Int64}}, true}} with 6 entries:
+  :ind1      => 1
+  :inda      => 0
+  :logp      => 4.23
+  :lumo      => -1.246
+  :mutagenic => 1
+  :atoms     => JSON3.Object[{…
+
+julia> train_y[1]
+1
+```
+"""
+module Mutagenesis
+
+using DataDeps, JSON3
+using ..MLDatasets: datafile
+
+const DEPNAME = "Mutagenesis"
+const DATA = "data.json"
+const METADATA = "meta.json"
+
+function __init__()
+    ORIGINAL_LINK = "https://relational.fit.cvut.cz/dataset/Mutagenesis"
+    DATA_LINK = "https://raw.githubusercontent.com/CTUAvastLab/datasets/main/mutagenesis"
+
+    register(DataDep(
+        DEPNAME,
+        """
+        Dataset: The $DEPNAME dataset.
+        Website: $ORIGINAL_LINK
+        License: CC0
+        """,
+        "$DATA_LINK/" .* [DATA, METADATA],
+        "80ec1716217135e1f2e0b5a61876c65184e2014e64551103c41e174775ca207c"
+    ))
+end
+
+traindata(; dir = nothing) = traindata(dir)
+testdata(; dir = nothing) = testdata(dir)
+valdata(; dir = nothing) = valdata(dir)
+
+function traindata(dir)
+    samples, targets, train_idxs, val_idxs, test_idxs = load_data(dir)
+    samples[train_idxs], targets[train_idxs]
+end
+
+function testdata(dir)
+    samples, targets, train_idxs, val_idxs, test_idxs = load_data(dir)
+    samples[test_idxs], targets[test_idxs]
+end
+
+function valdata(dir)
+    samples, targets, train_idxs, val_idxs, test_idxs = load_data(dir)
+    samples[val_idxs], targets[val_idxs]
+end
+
+function load_data(dir)
+    data_path = datafile(DEPNAME, DATA, dir)
+    metadata_path = datafile(DEPNAME, METADATA, dir)
+    samples = read_data(data_path)
+    metadata = read_data(metadata_path)
+    labelkey = metadata["label"]
+    targets = map(i -> i[labelkey], samples)
+    val_num = metadata["val_samples"]
+    test_num = metadata["test_samples"]
+    train_idxs = 1:length(samples)-val_num-test_num
+    val_idxs = length(samples)-val_num-test_num+1:length(samples)-test_num
+    test_idxs = length(samples)-test_num+1:length(samples)
+    samples, targets, train_idxs, val_idxs, test_idxs
+end
+
+read_data(path) = open(JSON3.read, path)
+
+end # module

test/runtests.jl

@@ -10,6 +10,7 @@ tests = [
     # misc
     "tst_iris.jl",
     "tst_boston_housing.jl",
+    "tst_mutagenesis.jl",
     # vision
     "tst_cifar10.jl",
     "tst_cifar100.jl",

test/tst_mutagenesis.jl

@@ -0,0 +1,13 @@
+data_dir = withenv("DATADEPS_ALWAY_ACCEPT"=>"true") do
+    datadep"Mutagenesis"
+end
+
+@testset "Mutagenesis" begin
+    train_x, train_y = Mutagenesis.traindata()
+    test_x, test_y = Mutagenesis.testdata()
+    val_x, val_y = Mutagenesis.valdata()
+
+    @test length(train_x) == length(train_y) == 100
+    @test length(test_x) == length(test_y) == 44
+    @test length(val_x) == length(val_y) == 44
+end