add support for symbols as keys

docs/src/s3examples.md

@@ -86,7 +86,7 @@ the overhead of repeatedly requesting many metadata files:
 g = zopen(store.zstore[1], consolidated=true)
 ````
 
-You can access the meta-information through `g.attrs` or for example read the first
+You can access the meta-information through `attributes(g)` or for example read the first
 time slice through
 
 ````julia

src/ZArray.jl

@@ -53,7 +53,7 @@ function Base.show(io::IO,::MIME"text/plain",z::ZArray)
   print(io, "ZArray{", eltype(z) ,"} of size ",join(string.(size(z)), " x "))
 end
 
-zname(z::ZArray) = zname(z.path)
+zname(z::ZArray) = zname(path(z))
 
 function zname(s::String)
   spl = split(rstrip(s,'/'),'/')
@@ -66,7 +66,7 @@ storagesize(z::ZArray)
 
 Returns the size of the compressed data stored in the ZArray `z` in bytes
 """
-storagesize(z::ZArray) = storagesize(z.storage,z.path)
+storagesize(z::ZArray) = storagesize(storage(z),path(z))
 
 """
 storageratio(z::ZArray)
@@ -84,18 +84,18 @@ nobytes(z::ZArray{<:String}) = "unknown"
 zinfo(z::ZArray) = zinfo(stdout,z)
 function zinfo(io::IO,z::ZArray)
   ninit = sum(chunkindices(z)) do i
-    isinitialized(z.storage,z.path,i)
+    isinitialized(storage(z),path(z),i)
   end
   allinfos = [
   "Type" => "ZArray",
   "Data type" => eltype(z),
   "Shape" => size(z),
   "Chunk Shape" => z.metadata.chunks,
   "Order" => z.metadata.order,
-  "Read-Only" => !z.writeable,
+  "Read-Only" => !iswriteable(z),
   "Compressor" => z.metadata.compressor,
   "Filters" => z.metadata.filters, 
-  "Store type" => z.storage,
+  "Store type" => storage(z),
   "No. bytes"  => nobytes(z),
   "No. bytes stored" => storagesize(z),
   "Storage ratio" => storageratio(z),
@@ -159,10 +159,10 @@ function readblock!(aout::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes
   #bufferdict = IdDict((current_task()=>getchunkarray(z),))
   a = getchunkarray(z)
   # Now loop through the chunks
-  c = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage))
+  c = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(storage(z)))
 
   task = @async begin
-    read_items!($z.storage,c, $z.path, $blockr)
+    read_items!($storage(z),c, $path(z), $blockr)
   end
   bind(c,task)
 
@@ -186,7 +186,7 @@ end
 
 function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::CartesianIndices{N}) where {N}
 
-  z.writeable || error("Can not write to read-only ZArray")
+  iswriteable(z) || error("Can not write to read-only ZArray")
 
   input_base_offsets = map(i->first(i)-1,r.indices)
   # Determines which chunks are affected
@@ -195,17 +195,17 @@ function writeblock!(ain::AbstractArray{<:Any,N}, z::ZArray{<:Any, N}, r::Cartes
   #bufferdict = IdDict((current_task()=>getchunkarray(z),))
   a = getchunkarray(z)
   # Now loop through the chunks
-  readchannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage))
+  readchannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(storage(z)))
 
   readtask = @async begin 
-    read_items!(z.storage,readchannel, z.path, blockr)
+    read_items!(storage(z),readchannel, path(z), blockr)
   end
   bind(readchannel,readtask)
 
-  writechannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(z.storage))
+  writechannel = Channel{Pair{eltype(blockr),Union{Nothing,Vector{UInt8}}}}(channelsize(storage(z)))
 
   writetask = @async begin
-    write_items!(z.storage,writechannel,z.path,blockr)
+    write_items!(storage(z),writechannel,path(z),blockr)
   end
   bind(writechannel,writetask)
 
@@ -394,9 +394,9 @@ function zzeros(T,dims...;kwargs...)
   z = zcreate(T,dims...;kwargs...)
   as = zeros(T, z.metadata.chunks...)
   data_encoded = compress_raw(as,z)
-  p = z.path
+  p = path(z)
   for i in chunkindices(z)
-    z.storage[p,i] = data_encoded
+    storage(z)[p,i] = data_encoded
   end
   z
 end
@@ -414,9 +414,9 @@ function Base.resize!(z::ZArray{T,N}, newsize::NTuple{N}) where {T,N}
   z.metadata.shape[] = newsize
   #Check if array was shrunk
   if any(map(<,newsize, oldsize))
-    prune_oob_chunks(z.storage,z.path,oldsize,newsize, z.metadata.chunks)
+    prune_oob_chunks(storage(z),path(z),oldsize,newsize, z.metadata.chunks)
   end
-  writemetadata(z.storage, z.path, z.metadata)
+  writemetadata(storage(z), path(z), z.metadata)
   nothing
 end
 Base.resize!(z::ZArray, newsize::Integer...) = resize!(z,newsize)

src/ZGroup.jl

@@ -6,6 +6,14 @@ struct ZGroup{S<:AbstractStore}
     attrs::Dict
     writeable::Bool
 end
+const ZArrayOrGroup = Union{ZArray, ZGroup}
+storage(a::ZArrayOrGroup)=getfield(a,:storage)
+path(a::ZArrayOrGroup)=getfield(a,:path)
+attributes(a::ZArrayOrGroup)=getfield(a,:attrs)
+iswriteable(a::ZArrayOrGroup)=getfield(a,:writeable)
+arrays(g::ZGroup)=getfield(g,:arrays)
+groups(g::ZGroup)=getfield(g,:groups)
+export attributes
 
 # path can also be a SubString{String}
 ZGroup(storage, path::AbstractString, arrays, groups, attrs, writeable) =
@@ -58,27 +66,51 @@ function zopen_noerr(s::AbstractStore, mode="r";
 end
 
 function Base.show(io::IO, g::ZGroup)
-    print(io, "ZarrGroup at ", g.storage, " and path ", g.path)
-    !isempty(g.arrays) && print(io, "\nVariables: ", map(i -> string(zname(i), " "), values(g.arrays))...)
-    !isempty(g.groups) && print(io, "\nGroups: ", map(i -> string(zname(i), " "), values(g.groups))...)
+    print(io, "ZarrGroup at ", storage(g), " and path ", path(g))
+    !isempty(arrays(g)) && print(io, "\nVariables: ", map(i -> string(zname(i), " "), values(arrays(g)))...)
+    !isempty(groups(g)) && print(io, "\nGroups: ", map(i -> string(zname(i), " "), values(groups(g)))...)
     nothing
 end
-Base.haskey(g::ZGroup,k)= haskey(g.groups,k) || haskey(g.arrays,k)
+Base.haskey(g::ZGroup,k)= haskey(groups(g),k) || haskey(arrays(g),k)
 
 
 const SymbolOrString = Union{Symbol,AbstractString}
 Base.haskey(x::Dict{String,ZGroup}, key::Symbol) = haskey(x, "$key")
 Base.haskey(x::Dict{String,ZArray}, key::Symbol) = haskey(x, "$key")
 
-function Base.getindex(g::ZGroup, k::SymbolOrString)
-    if haskey(g.groups, k)
-        return g.groups[k]
-    elseif haskey(g.arrays, k)
-       return g.arrays[k]
+function Base.getindex(g::ZGroup, k::AbstractString)
+    if haskey(groups(g), k)
+        return groups(g)[k]
+    elseif haskey(arrays(g), k)
+       return arrays(g)[k]
     else
        throw(KeyError("Zarr Dataset does not contain $k"))
     end
 end
+Base.getindex(g::ZGroup,k)=getindex(g,string(k))
+function Base.propertynames(g::ZGroup,private::Bool=false)
+  p = if private
+    Symbol[:attrs]
+  else
+    Symbol[]
+  end
+  for k in keys(groups(g))
+    push!(p,Symbol(k))
+  end
+  for k in keys(arrays(g))
+    push!(p,Symbol(k))
+  end
+  p
+end
+
+function Base.getproperty(g::ZGroup, k::Symbol)
+  if k === :attrs
+    @warn "Accessing attributes through `.attrs` is not recommended anymore. Please use `attributes(g)` instead."
+    return getfield(g,:attrs)
+  else
+    return g[k]
+  end
+end
 
 """
     zopen(s::AbstractStore, mode="r"; consolidated = false, path = "", lru = 0)
@@ -146,21 +178,21 @@ zgroup(s::String;kwargs...)=zgroup(storefromstring(s, true)...;kwargs...)
 
 "Create a subgroup of the group g"
 function zgroup(g::ZGroup, name; attrs=Dict()) 
-  g.writeable || throw(IOError("Zarr group is not writeable. Please re-open in write mode to create an array"))
-  g.groups[name] = zgroup(g.storage,_concatpath(g.path,name),attrs=attrs)
+  iswriteable(g) || throw(IOError("Zarr group is not writeable. Please re-open in write mode to create an array"))
+  groups(g)[name] = zgroup(storage(g),_concatpath(path(g),name),attrs=attrs)
 end
 
 "Create a new subarray of the group g"
 function zcreate(::Type{T},g::ZGroup, name::AbstractString, addargs...; kwargs...) where T
-  g.writeable || throw(IOError("Zarr group is not writeable. Please re-open in write mode to create an array"))
+  iswriteable(g) || throw(IOError("Zarr group is not writeable. Please re-open in write mode to create an array"))
   name = string(name)
-  z = zcreate(T, g.storage, addargs...; path = _concatpath(g.path,name), kwargs...)
-  g.arrays[name] = z
+  z = zcreate(T, storage(g), addargs...; path = _concatpath(path(g),name), kwargs...)
+  arrays(g)[name] = z
   return z
 end
 
-HTTP.serve(s::Union{ZArray,ZGroup}, args...; kwargs...) = HTTP.serve(s.storage, s.path, args...; kwargs...)
+HTTP.serve(s::Union{ZArray,ZGroup}, args...; kwargs...) = HTTP.serve(storage(s), path(s), args...; kwargs...)
 function consolidate_metadata(z::Union{ZArray,ZGroup}) 
-  z.writeable || throw(Base.IOError("Zarr group is not writeable. Please re-open in write mode to create an array",0))
-  consolidate_metadata(z.storage,z.path)
+  iswriteable(z) || throw(Base.IOError("Zarr group is not writeable. Please re-open in write mode to create an array",0))
+  consolidate_metadata(storage(z),path(z))
 end

test/python.jl

@@ -132,11 +132,11 @@ zarr.consolidate_metadata($ppython)
 #Open in Julia
 g = zopen(ppython)
 @test g isa Zarr.ZGroup
-@test g.attrs["groupatt"] == "Hi"
+@test attributes(g)["groupatt"] == "Hi"
 a1 = g["a1"]
 @test a1 isa ZArray
 @test a1[:,:,:]==permutedims(data,(3,2,1))
-@test a1.attrs["test"]==Dict("b"=>6)
+@test attributes(a1)["test"]==Dict("b"=>6)
 # Test reading the string array
 @test String(g["a2"][:])=="hallo"
 @test g["a3"] == ["test1", "test234"]
@@ -149,13 +149,13 @@ rm(joinpath(ppython,"a1",".zarray"))
 rm(joinpath(ppython,"a2",".zarray"))
 g = zopen(ppython, "w", consolidated=true)
 @test g isa Zarr.ZGroup
-@test g.attrs["groupatt"] == "Hi"
+@test attributes(g)["groupatt"] == "Hi"
 a1 = g["a1"]
 @test a1 isa ZArray
 @test a1[:,:,:]==permutedims(data,(3,2,1))
-@test a1.attrs["test"]==Dict("b"=>6)
+@test attributes(a1)["test"]==Dict("b"=>6)
 @test storagesize(a1) == 960
-@test sort(Zarr.subkeys(a1.storage,"a1"))[1:5] == ["0.0.0","0.0.1","0.0.2","0.0.3","0.1.0"]
+@test sort(Zarr.subkeys(Zarr.storage(a1),"a1"))[1:5] == ["0.0.0","0.0.1","0.0.2","0.0.3","0.1.0"]
 a1[:,1,1] = 1:10
 @test a1[:,1,1] == 1:10
 # Test reading the string array

test/runtests.jl

@@ -4,6 +4,7 @@ using JSON
 using Pkg
 using PyCall
 using Dates
+using Zarr: storage
 
 macro test_py(ex)
     quote
@@ -20,9 +21,9 @@ end
         @test z isa ZArray{Int64, 2, Zarr.BloscCompressor,
             Zarr.DictStore}
 
-        @test length(z.storage.a) === 3
-        @test length(z.storage.a["0.0"]) === 64
-        @test eltype(z.storage.a["0.0"]) === UInt8
+        @test length(storage(z).a) === 3
+        @test length(storage(z).a["0.0"]) === 64
+        @test eltype(storage(z).a["0.0"]) === UInt8
         @test z.metadata.shape[] === (2, 3)
         @test z.metadata.order === 'C'
         @test z.metadata.chunks === (2, 3)
@@ -32,8 +33,8 @@ end
         @test z.metadata.compressor.clevel === 5
         @test z.metadata.compressor.cname === "lz4"
         @test z.metadata.compressor.shuffle === 1
-        @test z.attrs == Dict{Any, Any}()
-        @test z.writeable === true
+        @test attributes(z) == Dict{Any, Any}()
+        @test Zarr.iswriteable(z) === true
         @test_throws ArgumentError zzeros(Int64,2,3, chunks = (0,1))
         @test_throws ArgumentError zzeros(Int64,0,-1)
         @test_throws ArgumentError Zarr.Metadata(zeros(2,2), (2,2), zarr_format = 3)
@@ -63,7 +64,7 @@ end
                 compressor=Zarr.NoCompressor())
 
             @test z.metadata.compressor === Zarr.NoCompressor()
-            @test z.storage === Zarr.DirectoryStore("$dir/$name")
+            @test storage(z) === Zarr.DirectoryStore("$dir/$name")
             @test isdir("$dir/$name")
             @test ispath("$dir/$name/.zarray")
             @test ispath("$dir/$name/.zattrs")
@@ -92,9 +93,23 @@ end
     g2 = zgroup(g,"asubgroup",attrs = Dict("a1"=>5))
     @test Zarr.is_zgroup(store,"mygroup")
     @test Zarr.is_zgroup(store,"mygroup/asubgroup")
-    @test g2.attrs["a1"]==5
+    @test attributes(g2)["a1"]==5
     @test isdir(joinpath(store.folder,"mygroup"))
     @test isdir(joinpath(store.folder,"mygroup","asubgroup"))
+
+    #Another test for indexing and getproperty
+    a = zgroup(Zarr.DictStore(),attrs=Dict("a"=>5))
+
+    zzeros(Float64,a,"a",3,3)
+    zzeros(Int,a,"b",5,4,2)
+
+    @test a["a"] isa ZArray
+    @test a[:a] isa ZArray
+    @test a.b isa ZArray
+    @test issetequal(propertynames(a),(:a,:b))
+    @test issetequal(propertynames(a,true),(:a,:b,:attrs))
+    @test @test_warn "Accessing attributes" a.attrs["a"]==5
+    @test attributes(a) == Dict("a"=>5)
 end
 
 @testset "Metadata" begin
@@ -177,7 +192,7 @@ end
   @test all(ismissing,amiss[:,2])
   @test all(i->isequal(i...),zip(amiss[1:3,4],[1,missing,3]))
   # Test that chunk containing only missings is not initialized
-  @test !Zarr.isinitialized(amiss.storage,Zarr.citostring(CartesianIndex((1,5))))
+  @test !Zarr.isinitialized(Zarr.storage(amiss),Zarr.citostring(CartesianIndex((1,5))))
   #
   amiss = zcreate(Int64, 10,10,chunks=(5,2), fill_value=-1, fill_as_missing=false)
   amiss[:,1] = 1:10
@@ -189,7 +204,7 @@ end
   @test all(==(-1),amiss[:,2])
   @test all(i->isequal(i...),zip(amiss[1:3,4],[1,-1,3]))
   # Test that chunk containing only fill values is not initialized
-  @test !Zarr.isinitialized(amiss.storage,Zarr.citostring(CartesianIndex((1,5))))
+  @test !Zarr.isinitialized(Zarr.storage(amiss),Zarr.citostring(CartesianIndex((1,5))))
 end
 
 @testset "resize" begin

test/storage.jl

@@ -105,7 +105,7 @@ end
   @test storagesize(S3,p) == 0
   @test Zarr.is_zgroup(S3,p) == true
   S3group = zopen(S3,path=p)
-  S3Array = S3group.groups["bar"].arrays["baz"]
+  S3Array = Zarr.arrays(Zarr.groups(S3group)["bar"])["baz"]
   @test eltype(S3Array) == Zarr.ASCIIChar
   @test storagesize(S3Array) == 69
   @test String(S3Array[:]) == "Hello from the cloud!"
@@ -142,8 +142,8 @@ end
   ip,port = getsockname(server)
   @async HTTP.serve(g,ip,port,server=server)
   g2 = zopen("http://$ip:$port")
-  @test g2.attrs == Dict("groupatt"=>5)
-  @test g2["a1"].attrs == Dict("arratt"=>2.5)
+  @test attributes(g2) == Dict("groupatt"=>5)
+  @test attributes(g2["a1"]) == Dict("arratt"=>2.5)
   @test g2["a1"][:,:] == reshape(1:200,10,20)
   close(server)
 end