Merge pull request #7 from JuliaGeo/as/dataset_from_store

Add methods to construct a ZarrDataset from Zarr.jl objects

Author: Alexander-Barth

Project.toml

@@ -1,7 +1,7 @@
 name = "ZarrDatasets"
 uuid = "519a4cdf-1362-424a-9ea1-b1d782dbb24b"
 authors = ["Alexander Barth <[email protected]> and contributors"]
-version = "0.1.2"
+version = "0.1.3"
 
 [deps]
 CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157"

README.md

@@ -49,3 +49,14 @@ NCDataset("$(dataset_id)_selection.nc","c") do ds_nc
     write(ds_nc,ds_sub)
 end
 ```
+
+## Using a pre-existing Zarr array or store
+
+It's also simple to wrap an existing Zarr array, or a manually constructed Zarr store, in a `ZarrDataset`:
+
+```julia
+zg = zopen("/path/to/zarr")
+zd = ZarrDataset(zg)
+```
+
+and you can pass a constructed `Zarr.AbstractStore` similarly.

src/dataset.jl

@@ -83,6 +83,7 @@ CDM.maskingvalue(ds::ZarrDataset) = ds.maskingvalue
     ds = ZarrDataset(url::AbstractString,mode = "r";
                      _omitcode = [404,403],
                      maskingvalue = missing)
+    ZarrDataset(zg::Zarr.ZGroup; _omitcode, maskingvalue)
     ZarrDataset(f::Function,url::AbstractString,mode = "r";
                      maskingvalue = missing)
 
@@ -135,33 +136,54 @@ function ZarrDataset(url::AbstractString,mode = "r";
                      )
 
     dimensions = OrderedDict{Symbol,Int}()
-    iswritable = false
 
-    if mode == "r"
+    zg = if mode == "r"
         zg = Zarr.zopen(url,mode)
-        if (zg.storage isa Zarr.HTTPStore) ||
-            (zg.storage isa Zarr.ConsolidatedStore{Zarr.HTTPStore})
+    elseif mode == "c"
+        store = Zarr.DirectoryStore(url)
+        zg = zgroup(store, "",attrs = Dict{String,Any}(attrib))
+    end
+    ZarrDataset(zg; mode, parentdataset, _omitcode, maskingvalue, attrib)
+end
+
+function ZarrDataset(store::Zarr.AbstractStore,mode = "r";
+                     parentdataset = nothing,
+                     _omitcode = [404,403],
+                     maskingvalue = missing,
+                     attrib = Dict(),
+                     )
+    return ZarrDataset(zopen(store, mode); mode, parentdataset, _omitcode, maskingvalue, attrib)
+end
+
+function ZarrDataset(zg::Zarr.ZGroup;
+                     mode = "r",
+                     parentdataset = nothing,
+                     _omitcode = [404,403],
+                     maskingvalue = missing,
+                     attrib = Dict(),
+                     )
+
+    dimensions = ZarrDatasets.OrderedDict{Symbol,Int}()
+    iswritable = false
+    if (zg.storage isa Zarr.HTTPStore) ||
+        (zg.storage isa Zarr.ConsolidatedStore{Zarr.HTTPStore})
             @debug "omit chunks on HTTP error" _omitcode
             Zarr.missing_chunk_return_code!(zg.storage,_omitcode)
         end
 
         for (varname,zarray) in zg.arrays
             for (dimname,dimlen) in zip(reverse(zarray.attrs["_ARRAY_DIMENSIONS"]),size(zarray))
-
                 dn = Symbol(dimname)
                 if haskey(dimensions,dn)
                     @assert dimensions[dn] == dimlen
                 else
                     dimensions[dn] = dimlen
                 end
-            end
         end
-    elseif mode == "c"
-        store = Zarr.DirectoryStore(url)
-        zg = zgroup(store, "",attrs = Dict{String,Any}(attrib))
-        iswritable = true
     end
-    ZarrDataset(parentdataset,zg,dimensions,iswritable,maskingvalue)
+
+    return ZarrDataset(parentdataset, zg, dimensions, mode == "r" ? false : zg.writeable, maskingvalue)
+
 end
 
 

test/runtests.jl

@@ -7,5 +7,6 @@ using ZarrDatasets
     include("test_write.jl")
     include("test_groups.jl")
     include("test_fillvalue.jl")
+    include("test_preexisting.jl")
     include("test_aqua.jl")
 end

test/test_preexisting.jl

@@ -0,0 +1,49 @@
+using ZarrDatasets
+using Test
+using Zarr
+
+# Test opening a ZarrDataset using a pre-existing Zarr store or group
+
+# First, 
+fname = tempname()
+mkdir(fname)
+gattrib = Dict("title" => "this is the title")
+ds = ZarrDataset(fname,"c",attrib = gattrib)
+
+ds.attrib["number"] = 1
+defDim(ds,"lon",3)
+defDim(ds,"lat",5)
+
+attrib = Dict(
+    "units" => "m/s",
+    "long_name" => "test",
+)
+
+
+varname = "var2"
+dimensionnames = ("lon","lat")
+vtype = Int32
+
+zv = defVar(ds,varname,vtype,dimensionnames, attrib = attrib)
+zv[:,:] = data = rand(Int32,3,5)
+
+zv.attrib["number"] = 12
+zv.attrib["standard_name"] = "test"
+ds.attrib["history"] = "test"
+close(ds)
+
+for ds in ZarrDataset.((Zarr.storefromstring(fname)[1], Zarr.zopen(fname), ))
+
+    zv = ds[varname]
+
+    @test zv.attrib["number"] == 12
+    @test zv.attrib["standard_name"] == "test"
+    @test ds.attrib["history"] == "test"
+
+    @test zv[:,:] == data
+
+    io = IOBuffer()
+    show(io,ds)
+    str = String(take!(io))
+    @test occursin("Global",str)
+end