Merge pull request #8 from jbrea/caching

save downloaded arff file as artifact

Author: ablaom

Project.toml

@@ -8,6 +8,7 @@ ARFFFiles = "da404889-ca92-49ff-9e8b-0aa6b4d38dc8"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 ScientificTypes = "321657f4-b219-11e9-178b-2701a2544e81"
 
 [compat]

src/OpenML.jl

@@ -5,6 +5,9 @@ using JSON
 import ARFFFiles
 import ScientificTypes: Continuous, Count, Textual, Multiclass, coerce, autotype
 using Markdown
+if VERSION > v"1.3.0"
+    using Pkg.Artifacts
+end
 
 export OpenML
 

src/data.jl

@@ -5,10 +5,6 @@ const API_URL = "https://www.openml.org/api/v1/json"
 # https://github.com/openml/OpenML/tree/master/openml_OS/views/pages/api_new/v1/xsd
 # https://www.openml.org/api_docs#!/data/get_data_id
 
-# TODO:
-# - Use e.g. DataDeps to cache data locally
-# - Put the ARFF parser to a separate package or use ARFFFiles when
-#   https://github.com/cjdoris/ARFFFiles.jl/issues/4 is fixed.
 
 """
 Returns information about a dataset. The information includes the name,
@@ -58,9 +54,24 @@ df = DataFrame(table);
 ```
 """
 function load(id::Int; parser = :arff)
-    response = load_Dataset_Description(id)
-    arff_file = HTTP.request("GET", response["data_set_description"]["url"])
-    data = ARFFFiles.load(IOBuffer(arff_file.body))
+    if VERSION > v"1.3.0"
+        dir = first(Artifacts.artifacts_dirs())
+        toml = joinpath(dir, "OpenMLArtifacts.toml")
+        hash = artifact_hash(string(id), toml)
+        if hash === nothing || !artifact_exists(hash)
+            hash = Artifacts.create_artifact() do artifact_dir
+                url = load_Dataset_Description(id)["data_set_description"]["url"]
+                download(url, joinpath(artifact_dir, "$id.arff"))
+            end
+            bind_artifact!(toml, string(id), hash)
+        end
+        filename = joinpath(artifact_path(hash), "$id.arff")
+    else
+        url = load_Dataset_Description(id)["data_set_description"]["url"]
+        filename = tempname()
+        download(url, filename)
+    end
+    data = ARFFFiles.load(filename)
     if parser == :auto
         return coerce(data, autotype(data))
     else

test/data.jl

@@ -40,5 +40,15 @@ end
     @test length(filters_test["data"]["dataset"][1]) == offset
 end
 
+if VERSION > v"1.3.0"
+    using Pkg.Artifacts
+    @testset "artifacts" begin
+        dir = first(Artifacts.artifacts_dirs())
+        toml = joinpath(dir, "OpenMLArtifacts.toml")
+        hash = artifact_hash("61", toml)
+        @test artifact_exists(hash)
+    end
+end
+
 end
 true