Flexible string-based data addressing with URL-like "dataspec" (#33)

This parses the argument to dataset() with a URL-like scheme, enabling
more flexible naming of resources within a given dataset. Like URLs, we
use the & and # as separators for query and fragment sections.

Like URLs, this provides for a compact string representation which can
be easily communicated between users of a shared dataset to address
subresources without the need to create an entirely new and largely
duplicated dataset configuration.

As in URLs, the query section is to be used during resource resolution,
for example a version number for the dataset can be specified.  We
choose the standard & and = separators for key value pairs:

    name/of/dataset?version=v1

As in URLs, the fragment section refers to a subresource and its
interpretation depends on the type of the dataset. (In analogy, the
meaning of the fragment of a URL depends on the mime type received from
the server.) For example, a subtree of a BlobTree can be addressed using

tree_name#path/to/subtree

The use of fragment for indexing a BlobTree is implemented here as one
handy use case of this syntax.

Author: c42f

Project.toml

@@ -1,7 +1,7 @@
 name = "DataSets"
 uuid = "c9661210-8a83-48f0-b833-72e62abce419"
 authors = ["Chris Foster <[email protected]> and contributors"]
-version = "0.2.5"
+version = "0.2.6"
 
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
@@ -17,7 +17,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 [compat]
 AbstractTrees = "0.3"
 ReplMaker = "0.2"
-ResourceContexts = "0.1"
+ResourceContexts = "0.1,0.2"
 TOML = "1"
 julia = "1.5"
 

src/DataSets.jl

@@ -119,6 +119,20 @@ function Base.getproperty(d::DataSet, name::Symbol)
     end
 end
 
+Base.getindex(d::DataSet, name::AbstractString) = getindex(d.conf, name)
+Base.haskey(d::DataSet, name::AbstractString) = haskey(d.conf, name)
+
+# Split the fragment section as a '/' separated RelPath
+function dataspec_fragment_as_path(d::DataSet)
+    if haskey(d, "dataspec")
+        fragment = get(d.dataspec, "fragment", nothing)
+        if !isnothing(fragment)
+            return RelPath(split(fragment, '/'))
+        end
+    end
+    return nothing
+end
+
 function Base.show(io::IO, d::DataSet)
     print(io, DataSet, "(name=$(repr(d.name)), uuid=$(repr(d.uuid)), #= … =#)")
 end
@@ -188,21 +202,83 @@ To open a directory as a browsable tree object,
 open(BlobTree, dataset("a_tree_example"))
 ```
 """
-function dataset(proj::AbstractDataProject, name::AbstractString)
-    # Non-fancy search... for now :)
-    # In the future, we can consider parsing `name` into a dataset prefix and a
-    # data selector / resource section. Eg a path for BlobTree which gives us a
-    # SubDataSet
-    #
-    # The URN RFC8141 has some good design inspiration here, in particular the
-    # distinction between r-component and q-component seems relevant:
-    # * Some parameters may need to be passed to the "resolver" (ie, the data
-    #   storage backend)
-    # * Some parameters may need to be passed to the dataset itself (eg, a
-    #   relative path within the dataset)
-    #
-    # See https://datatracker.ietf.org/doc/html/rfc8141#page-12
-    return proj[name]
+function dataset(proj::AbstractDataProject, spec::AbstractString)
+    namestr, query, fragmentstr = _split_dataspec(spec)
+
+    if isnothing(namestr)
+        throw(ArgumentError("Invalid dataset specification: $spec"))
+    end
+
+    dataset = proj[namestr]
+
+    if isnothing(query) && isnothing(fragmentstr)
+        return dataset
+    end
+
+    # Enhance dataset with "dataspec" holding URL-like fragment & query
+    dataspec = Dict()
+    if !isnothing(query)
+        dataspec["query"] = Dict{String,Any}(query)
+    end
+    if !isnothing(fragmentstr)
+        dataspec["fragment"] = fragmentstr
+    end
+
+    # We need to take care here with copy() to avoid modifying the original
+    # dataset configuration.
+    conf = copy(dataset.conf)
+    conf["dataspec"] = dataspec
+
+    return DataSet(conf)
+end
+
+
+# Percent-decode a string according to the URI escaping rules.
+# Vendored from URIs.jl for now to avoid depending on that entire package for
+# this one function.
+function _unescapeuri(str)
+    occursin("%", str) || return str
+    out = IOBuffer()
+    i = 1
+    io = IOBuffer(str)
+    while !eof(io)
+        c = read(io, Char)
+        if c == '%'
+            c1 = read(io, Char)
+            c = read(io, Char)
+            write(out, parse(UInt8, string(c1, c); base=16))
+        else
+            write(out, c)
+        end
+    end
+    return String(take!(out))
+end
+
+function _split_dataspec(spec::AbstractString)
+    # Parse as a suffix of URI syntax
+    # name/of/dataset?param1=value1&param2=value2#fragment
+    m = match(r"
+        ^
+        ((?:[[:alpha:]][[:alnum:]_]*/?)+)  # name     - a/b/c
+        (?:\?([^#]*))?                     # query    - a=b&c=d
+        (?:\#(.*))?                        # fragment - ...
+        $"x,
+        spec)
+    if isnothing(m)
+        return nothing, nothing, nothing
+    end
+    namestr = m[1]
+    query = m[2]
+    fragmentstr = m[3]
+
+    if !isnothing(query)
+        query = [_unescapeuri(x)=>_unescapeuri(y) for (x,y) in split.(split(query, '&'), '=')]
+    end
+    if !isnothing(fragmentstr)
+        fragmentstr = _unescapeuri(fragmentstr)
+    end
+
+    namestr, query, fragmentstr
 end
 
 function Base.haskey(proj::AbstractDataProject, name::AbstractString)

src/filesystem.jl

@@ -271,7 +271,7 @@ end
 #--------------------------------------------------
 
 # Filesystem storage driver
-function connect_filesystem(f, config, _)
+function connect_filesystem(f, config, dataset)
     path = config["path"]
     type = config["type"]
     if type == "Blob"
@@ -280,6 +280,10 @@ function connect_filesystem(f, config, _)
     elseif type == "BlobTree"
         isdir(path)  || throw(ArgumentError("$(repr(path)) should be a directory"))
         storage = BlobTree(FileSystemRoot(path))
+        path = dataspec_fragment_as_path(dataset)
+        if !isnothing(path)
+            storage = storage[path]
+        end
     else
         throw(ArgumentError("DataSet type $type not supported on the filesystem"))
     end

test/runtests.jl

@@ -57,6 +57,12 @@ end
         tree = @! open(tree_data)
         @test readdir(tree) == ["1.csv", "2.csv"]
     end
+
+    blob_in_tree_data = dataset(proj, "a_tree_example#1.csv")
+    @test open(blob_in_tree_data) isa Blob
+    @context begin
+        @test @!(open(String, blob_in_tree_data)) == """Name,Age\n"Aaron",23\n"Harry",42\n"""
+    end
 end
 
 #-------------------------------------------------------------------------------
@@ -101,6 +107,26 @@ end
     @test_throws ErrorException DataSets.check_dataset_name("/a/b")
 end
 
+@testset "URL-like dataspec parsing" begin
+    proj = DataSets.load_project("Data.toml")
+
+    @test !haskey(dataset(proj, "a_text_file"), "dataspec")
+
+    # URL-like query
+    @test dataset(proj, "a_text_file?x=1&yy=2")["dataspec"]["query"] == Dict("x"=>"1", "yy"=>"2")
+    @test dataset(proj, "a_text_file?y%20y=x%20x")["dataspec"]["query"] == Dict("y y"=>"x x")
+    @test dataset(proj, "a_text_file?x=%3d&y=%26")["dataspec"]["query"] == Dict("x"=>"=", "y"=>"&")
+
+    # URL-like fragment
+    @test dataset(proj, "a_text_file#a/b")["dataspec"]["fragment"] == "a/b"
+    @test dataset(proj, "a_text_file#x%20x")["dataspec"]["fragment"] == "x x"
+    @test dataset(proj, "a_text_file#x%ce%b1x")["dataspec"]["fragment"] == "xαx"
+
+    # Combined query and fragment
+    @test dataset(proj, "a_text_file?x=1&yy=2#frag")["dataspec"]["query"] == Dict("x"=>"1", "yy"=>"2")
+    @test dataset(proj, "a_text_file?x=1&yy=2#frag")["dataspec"]["fragment"] == "frag"
+end
+
 #-------------------------------------------------------------------------------
 # Trees
 @testset "Temporary trees" begin