save downloaded arff file as artifact

Author: jbrea

Project.toml

@@ -8,6 +8,7 @@ ARFFFiles = "da404889-ca92-49ff-9e8b-0aa6b4d38dc8"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 ScientificTypes = "321657f4-b219-11e9-178b-2701a2544e81"
 
 [compat]

src/OpenML.jl

@@ -5,6 +5,7 @@ using JSON
 import ARFFFiles
 import ScientificTypes: Continuous, Count, Textual, Multiclass, coerce, autotype
 using Markdown
+using Pkg.Artifacts
 
 export OpenML
 

src/data.jl

@@ -5,10 +5,6 @@ const API_URL = "https://www.openml.org/api/v1/json"
 # https://github.com/openml/OpenML/tree/master/openml_OS/views/pages/api_new/v1/xsd
 # https://www.openml.org/api_docs#!/data/get_data_id
 
-# TODO:
-# - Use e.g. DataDeps to cache data locally
-# - Put the ARFF parser to a separate package or use ARFFFiles when
-#   https://github.com/cjdoris/ARFFFiles.jl/issues/4 is fixed.
 
 """
 Returns information about a dataset. The information includes the name,
@@ -58,9 +54,17 @@ df = DataFrame(table);
 ```
 """
 function load(id::Int; parser = :arff)
-    response = load_Dataset_Description(id)
-    arff_file = HTTP.request("GET", response["data_set_description"]["url"])
-    data = ARFFFiles.load(IOBuffer(arff_file.body))
+    dir = first(Artifacts.artifacts_dirs())
+    toml = joinpath(dir, "OpenMLArtifacts.toml")
+    hash = artifact_hash(string(id), toml)
+    if hash === nothing || !artifact_exists(hash)
+        hash = Artifacts.create_artifact() do artifact_dir
+            url = load_Dataset_Description(id)["data_set_description"]["url"]
+            download(url, joinpath(artifact_dir, "$id.arff"))
+        end
+        bind_artifact!(toml, string(id), hash)
+    end
+    data = ARFFFiles.load(joinpath(artifact_path(hash), "$id.arff"))
     if parser == :auto
         return coerce(data, autotype(data))
     else

test/data.jl

@@ -4,6 +4,7 @@ using Test
 using HTTP
 using OpenML
 import Tables.istable
+using Pkg.Artifacts
 
 response_test = OpenML.load_Dataset_Description(61)
 ntp_test = OpenML.load(61)
@@ -40,5 +41,13 @@ end
     @test length(filters_test["data"]["dataset"][1]) == offset
 end
 
+@testset "artifacts" begin
+    dir = first(Artifacts.artifacts_dirs())
+    toml = joinpath(dir, "OpenMLArtifacts.toml")
+    hash = artifact_hash("61", toml)
+    @test artifact_exists(hash)
+end
+
+
 end
 true