diff --git a/docs/src/index.md b/docs/src/index.md index 2aadba3..90ae4b9 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,7 +1,5 @@ - ## ZarrDatasets - See the [documentation of JuliaGeo/CommonDataModel.jl](https://juliageo.org/CommonDataModel.jl/stable/) for the full documentation of the API. As a quick reference, here is an example how to create and read a Zarr file store as a quick reference. ### Create a Zarr file store @@ -39,15 +37,10 @@ data_units = ds["varname"].attrib["units"] ``` - ```@autodocs Modules = [ZarrDatasets] ``` - - - - ### Differences between Zarr and NetCDF files * All metadata (in particular attributes) is stored in JSON files for the Zarr format with the following implications: diff --git a/src/ZarrDatasets.jl b/src/ZarrDatasets.jl index d13dd57..039d3a6 100644 --- a/src/ZarrDatasets.jl +++ b/src/ZarrDatasets.jl @@ -31,13 +31,16 @@ import DiskArrays: haschunks import CommonDataModel as CDM +import JSON + using DataStructures using Zarr -import JSON + +export ZarrDataset +export defDim, defVar, defGroup include("types.jl") include("dataset.jl") include("variable.jl") -export ZarrDataset end diff --git a/src/dataset.jl b/src/dataset.jl index e618fb2..cf955cc 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -1,14 +1,20 @@ +# Base interface -CDM.name(v::ZarrDataset) = Zarr.zname(v.zgroup) Base.keys(ds::ZarrDataset) = keys(ds.zgroup.arrays) -Base.haskey(ds::ZarrDataset,varname::SymbolOrString) = haskey(ds.zgroup.arrays,String(varname)) +Base.haskey(ds::ZarrDataset, varname::SymbolOrString) = haskey(ds.zgroup.arrays, String(varname)) + +# CommonDataModel.jl interface -function CDM.variable(ds::ZarrDataset,varname::SymbolOrString) +CDM.name(v::ZarrDataset) = Zarr.zname(v.zgroup) +function CDM.variable(ds::ZarrDataset, varname::SymbolOrString) zarray = ds.zgroup.arrays[String(varname)] - ZarrVariable{eltype(zarray),ndims(zarray),typeof(zarray),typeof(ds)}(zarray,ds) + ZarrVariable{eltype(zarray),ndims(zarray),typeof(zarray),typeof(ds)}(zarray, ds) end - CDM.dimnames(ds::ZarrDataset) = Tuple(String.(keys(ds.dimensions))) +CDM.dim(ds::ZarrDataset, dimname::SymbolOrString) = ds.dimensions[Symbol(dimname)] +CDM.varnames(ds::ZarrDataset) = keys(ds.zgroup.arrays) +CDM.attribnames(ds::ZarrDataset) = keys(ds.zgroup.attrs) +CDM.attrib(ds::ZarrDataset, name::SymbolOrString) = ds.zgroup.attrs[String(name)] # function CDM.unlimited(ds::ZarrDataset) # ul = ds.unlimited @@ -18,85 +24,59 @@ CDM.dimnames(ds::ZarrDataset) = Tuple(String.(keys(ds.dimensions))) # return ul # end -function _dim(ds::ZarrDataset,dimname::SymbolOrString) - dimlen = get(ds.dimensions,Symbol(dimname),nothing) - - if !isnothing(dimlen) - return dimlen - end - - if ds.parentdataset !== nothing - return _dim(ds.parentdataset,dimname) - end - - error("dimension $dimname is not defined") -end - -CDM.dim(ds::ZarrDataset,dimname::SymbolOrString) = ds.dimensions[Symbol(dimname)] - -function CDM.defDim(ds::ZarrDataset,dimname::SymbolOrString,dimlen) +function CDM.defDim(ds::ZarrDataset, dimname::SymbolOrString, dimlen) dn = Symbol(dimname) - @assert !haskey(ds.dimensions,dn) + @assert !haskey(ds.dimensions, dn) ds.dimensions[dn] = dimlen end -CDM.varnames(ds::ZarrDataset) = keys(ds.zgroup.arrays) - -CDM.attribnames(ds::ZarrDataset) = keys(ds.zgroup.attrs) -CDM.attrib(ds::ZarrDataset,name::SymbolOrString) = ds.zgroup.attrs[String(name)] - -function CDM.defAttrib(ds::ZarrDataset,name::SymbolOrString,value) +function CDM.defAttrib(ds::ZarrDataset, name::SymbolOrString, value) @assert iswritable(ds) ds.zgroup.attrs[String(name)] = value storage = ds.zgroup.storage io = IOBuffer() JSON.print(io, ds.zgroup.attrs) - storage[ds.zgroup.path,".zattrs"] = take!(io) + storage[ds.zgroup.path, ".zattrs"] = take!(io) end # groups - -function CDM.defGroup(ds::ZarrDataset,groupname::SymbolOrString; attrib = Dict()) +function CDM.defGroup(ds::ZarrDataset, groupname::SymbolOrString; attrib=Dict()) _attrib = Dict{String,Any}(attrib) - zg = zgroup(ds.zgroup,String(groupname),attrs = _attrib) + zg = zgroup(ds.zgroup, String(groupname), attrs=_attrib) dimensions = OrderedDict{Symbol,Int}() - return ZarrDataset(ds,zg,dimensions,ds.iswritable,ds.maskingvalue) + return ZarrDataset(ds, zg, dimensions, ds.iswritable, ds.maskingvalue) end - CDM.groupnames(ds::ZarrDataset) = keys(ds.zgroup.groups) - -function CDM.group(ds::ZarrDataset,groupname::SymbolOrString) +function CDM.group(ds::ZarrDataset, groupname::SymbolOrString) dimensions = OrderedDict{Symbol,Int}() zg = ds.zgroup.groups[String(groupname)] - return ZarrDataset(ds,zg,dimensions,ds.iswritable,ds.maskingvalue) + return ZarrDataset(ds, zg, dimensions, ds.iswritable, ds.maskingvalue) end - - CDM.parentdataset(ds::ZarrDataset) = ds.parentdataset CDM.iswritable(ds::ZarrDataset) = ds.iswritable CDM.maskingvalue(ds::ZarrDataset) = ds.maskingvalue - """ - ds = ZarrDataset(url::AbstractString,mode = "r"; - _omitcode = [404,403], - maskingvalue = missing) - ZarrDataset(zg::Zarr.ZGroup; _omitcode, maskingvalue) - ZarrDataset(f::Function,url::AbstractString,mode = "r"; - maskingvalue = missing) + ZarrDataset(url::AbstractString, mode = "r"; kw...) + ZarrDataset(zg::Zarr.ZGroup; kw...) + ZarrDataset(f::Function, url::AbstractString, mode = "r"; kw...) Open the zarr dataset at the url or path `url`. The mode can only be `"r"` (read-only) or `"c"` (create). `ds` supports the API of the [JuliaGeo/CommonDataModel.jl](https://github.com/JuliaGeo/CommonDataModel.jl). -The experimental `_omitcode` allows to define which HTTP error code should be used -for missing chunks. For compatibility with python's Zarr, the HTTP error 403 -(permission denied) is also used to missing chunks in addition to 404 (not -found). -The parameter `maskingvalue` allows to define which special value should be used -as replacement for fill values. The default is `missing`. +# Keywords + +- `_omitcode`: *experimental*, allows to define which HTTP error code should be used + for missing chunks. For compatibility with python's Zarr, the HTTP error 403 + (permission denied) is also used to missing chunks in addition to 404 (not + found). Default [404, 403]. +- `maskingvalue`: The parameter `maskingvalue` allows to define which special + value should be used as replacement for fill values. The default is `missing`. + Defaults to `missing``. +- `attrib`: Attributes, defualts to `Dict()` Example: @@ -128,70 +108,59 @@ zos1 = ZarrDataset(url) do ds end # implicit call to close(ds) ``` """ -function ZarrDataset(url::AbstractString,mode = "r"; - parentdataset = nothing, - _omitcode = [404,403], - maskingvalue = missing, - attrib = Dict(), - ) - - dimensions = OrderedDict{Symbol,Int}() - +function ZarrDataset(url::AbstractString, mode="r"; + parentdataset=nothing, + _omitcode=[404, 403], + maskingvalue=missing, + attrib=Dict(), +) zg = if mode == "r" - zg = Zarr.zopen(url,mode) + zg = Zarr.zopen(url, mode) elseif mode == "c" store = Zarr.DirectoryStore(url) - zg = zgroup(store, "",attrs = Dict{String,Any}(attrib)) + zg = zgroup(store, "", attrs=Dict{String,Any}(attrib)) end ZarrDataset(zg; mode, parentdataset, _omitcode, maskingvalue, attrib) end - -function ZarrDataset(store::Zarr.AbstractStore,mode = "r"; - parentdataset = nothing, - _omitcode = [404,403], - maskingvalue = missing, - attrib = Dict(), - ) +function ZarrDataset(store::Zarr.AbstractStore, mode="r"; + parentdataset=nothing, + _omitcode=[404, 403], + maskingvalue=missing, + attrib=Dict(), +) return ZarrDataset(zopen(store, mode); mode, parentdataset, _omitcode, maskingvalue, attrib) end - function ZarrDataset(zg::Zarr.ZGroup; - mode = "r", - parentdataset = nothing, - _omitcode = [404,403], - maskingvalue = missing, - attrib = Dict(), - ) - + mode="r", + parentdataset=nothing, + _omitcode=[404, 403], + maskingvalue=missing, + attrib=Dict(), +) dimensions = ZarrDatasets.OrderedDict{Symbol,Int}() - iswritable = false if (zg.storage isa Zarr.HTTPStore) || - (zg.storage isa Zarr.ConsolidatedStore{Zarr.HTTPStore}) - @debug "omit chunks on HTTP error" _omitcode - Zarr.missing_chunk_return_code!(zg.storage,_omitcode) - end + (zg.storage isa Zarr.ConsolidatedStore{Zarr.HTTPStore}) + @debug "omit chunks on HTTP error" _omitcode + Zarr.missing_chunk_return_code!(zg.storage, _omitcode) + end - for (varname,zarray) in zg.arrays - for (dimname,dimlen) in zip(reverse(zarray.attrs["_ARRAY_DIMENSIONS"]),size(zarray)) - dn = Symbol(dimname) - if haskey(dimensions,dn) - @assert dimensions[dn] == dimlen - else - dimensions[dn] = dimlen - end + for (varname, zarray) in zg.arrays + for (dimname, dimlen) in zip(reverse(zarray.attrs["_ARRAY_DIMENSIONS"]), size(zarray)) + dn = Symbol(dimname) + if haskey(dimensions, dn) + @assert dimensions[dn] == dimlen + else + dimensions[dn] = dimlen + end end end return ZarrDataset(parentdataset, zg, dimensions, mode == "r" ? false : zg.writeable, maskingvalue) end - - -ZarrDataset(fnames::AbstractArray{<:AbstractString,N}, args...; kwargs...) where N = - MFDataset(ZarrDataset,fnames, args...; kwargs...) - - -function ZarrDataset(f::Function,args...; kwargs...) +ZarrDataset(fnames::AbstractArray{<:AbstractString,N}, args...; kwargs...) where {N} = + MFDataset(ZarrDataset, fnames, args...; kwargs...) +function ZarrDataset(f::Function, args...; kwargs...) ds = ZarrDataset(args...; kwargs...) try f(ds) @@ -200,7 +169,16 @@ function ZarrDataset(f::Function,args...; kwargs...) end end -export ZarrDataset -export defDim -export defVar -export defGroup +function _dim(ds::ZarrDataset, dimname::SymbolOrString) + dimlen = get(ds.dimensions, Symbol(dimname), nothing) + + if !isnothing(dimlen) + return dimlen + end + + if ds.parentdataset !== nothing + return _dim(ds.parentdataset, dimname) + end + + error("dimension $dimname is not defined") +end \ No newline at end of file diff --git a/src/variable.jl b/src/variable.jl index d14622e..f0ebc24 100644 --- a/src/variable.jl +++ b/src/variable.jl @@ -1,39 +1,45 @@ - -Base.getindex(v::ZarrVariable,ij::Union{Integer,Colon,AbstractVector{<:Integer}}...) = v.zarray[ij...] -CDM.load!(v::ZarrVariable,buffer,ij...) = buffer .= view(v.zarray,ij...) - -function Base.setindex!(v::ZarrVariable,data,ij::Union{Integer,Colon,AbstractVector{<:Integer}}...) - v.zarray[ij...] = data +# Base interface + +Base.getindex(v::ZarrVariable, I::Union{Integer,Colon,AbstractVector{<:Integer}}...) = + v.zarray[I...] +function Base.setindex!( + v::ZarrVariable, data, I::Union{Integer,Colon,AbstractVector{<:Integer}}... +) + v.zarray[I...] = data end Base.size(v::ZarrVariable) = size(v.zarray) + +# DiskArrays.jl interface + +# TODO make ZarrVariable a real disk array +eachchunk(v::ZarrVariable) = eachchunk(v.zarray) +haschunks(v::ZarrVariable) = haschunks(v.zarray) +eachchunk(v::CFVariable{T,N,<:ZarrVariable}) where {T,N} = eachchunk(v.var) +haschunks(v::CFVariable{T,N,<:ZarrVariable}) where {T,N} = haschunks(v.var) + +# CommonDataModel.jl interface + +CDM.load!(v::ZarrVariable, buffer, ij...) = buffer .= view(v.zarray, ij...) CDM.name(v::ZarrVariable) = Zarr.zname(v.zarray) CDM.dimnames(v::ZarrVariable) = Tuple(reverse(v.zarray.attrs["_ARRAY_DIMENSIONS"])) CDM.dataset(v::ZarrVariable) = v.parentdataset -function _iscoordvar(v) - dn = dimnames(v) - if length(dn) == 0 - return false - end - return name(v) == first(dn) -end - function CDM.attribnames(v::ZarrVariable) - names = filter(!=("_ARRAY_DIMENSIONS"),keys(v.zarray.attrs)) + names = filter(!=("_ARRAY_DIMENSIONS"), keys(v.zarray.attrs)) if !isnothing(v.zarray.metadata.fill_value) && !_iscoordvar(v) - push!(names,"_FillValue") + push!(names, "_FillValue") end return names end -function CDM.attrib(v::ZarrVariable{T},name::SymbolOrString) where T +function CDM.attrib(v::ZarrVariable{T}, name::SymbolOrString) where {T} if String(name) == "_FillValue" && !isnothing(v.zarray.metadata.fill_value) return T(v.zarray.metadata.fill_value) end return v.zarray.attrs[String(name)] end -function CDM.defAttrib(v::ZarrVariable,name::SymbolOrString,value) +function CDM.defAttrib(v::ZarrVariable, name::SymbolOrString, value) @assert iswritable(dataset(v)) @assert String(name) !== "_FillValue" @@ -42,39 +48,45 @@ function CDM.defAttrib(v::ZarrVariable,name::SymbolOrString,value) storage = v.zarray.storage io = IOBuffer() JSON.print(io, v.zarray.attrs) - storage[v.zarray.path,".zattrs"] = take!(io) + storage[v.zarray.path, ".zattrs"] = take!(io) end - -# DiskArray methods -eachchunk(v::ZarrVariable) = eachchunk(v.zarray) -haschunks(v::ZarrVariable) = haschunks(v.zarray) -eachchunk(v::CFVariable{T,N,<:ZarrVariable}) where {T,N} = eachchunk(v.var) -haschunks(v::CFVariable{T,N,<:ZarrVariable}) where {T,N} = haschunks(v.var) +function _iscoordvar(v) + dn = dimnames(v) + if length(dn) == 0 + return false + end + return name(v) == first(dn) +end """ + defVar(ds::ZarrDataset, name::SymbolOrString, vtype::DataType, dimensionnames; + chunksizes=nothing, attrib=Dict(), fillvalue=nothing) - defVar(ds::ZarrDataset,name::SymbolOrString,vtype::DataType,dimensionnames; chunksizes=nothing, attrib = Dict(), fillvalue = nothing) - +Create a variable `name` in the dataset `ds` with the type +`vtype` and the dimension `dimensionnames`. -Create a variable `name` in the dataset `ds` with the type `vtype` and the dimension `dimensionnames`. - -For coordinate variables, fill values will be used a background value of undefined chunks and not as missing value as coordinate variables cannot have the `_FillValues` in the CF convension as in Zarr v2 format a `fill_value` does not necessarily indicate a missing value. +For coordinate variables, fill values will be used a background value of +undefined chunks and not as missing value as coordinate variables cannot +have the `_FillValues` in the CF convension as in Zarr v2 format a `fill_value` +does not necessarily indicate a missing value. See also `CommonDataModel.defVar` for more information. """ -function CDM.defVar(ds::ZarrDataset,name::SymbolOrString,vtype::DataType,dimensionnames; chunksizes=nothing, attrib = Dict(), fillvalue = nothing, kwargs...) +function CDM.defVar(ds::ZarrDataset, name::SymbolOrString, vtype::DataType, dimensionnames; + chunksizes=nothing, attrib=Dict(), fillvalue=nothing, kwargs... +) @assert iswritable(ds) if isnothing(fillvalue) - fillvalue = get(attrib,"_FillValue",nothing) + fillvalue = get(attrib, "_FillValue", nothing) end _attrib = Dict{String,Any}(attrib) _attrib["_ARRAY_DIMENSIONS"] = reverse(dimensionnames) _size = ntuple(length(dimensionnames)) do i - _dim(ds,dimensionnames[i]) + _dim(ds, dimensionnames[i]) end if isnothing(chunksizes) @@ -83,14 +95,14 @@ function CDM.defVar(ds::ZarrDataset,name::SymbolOrString,vtype::DataType,dimensi zarray = zcreate( vtype, ds.zgroup, name, _size...; - chunks = chunksizes, - attrs = _attrib, - fill_value = fillvalue, + chunks=chunksizes, + attrs=_attrib, + fill_value=fillvalue, kwargs... ) - zv = ZarrVariable{vtype,ndims(zarray),typeof(zarray),typeof(ds)}( - zarray,ds) + # TODO zv is not used? + zv = ZarrVariable{vtype,ndims(zarray),typeof(zarray),typeof(ds)}(zarray, ds) return ds[name] -end +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index a49e3d0..76c8032 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,4 +9,4 @@ using ZarrDatasets include("test_fillvalue.jl") include("test_preexisting.jl") include("test_aqua.jl") -end +end \ No newline at end of file diff --git a/test/test_aqua.jl b/test/test_aqua.jl index bdaab43..16b6bf3 100644 --- a/test/test_aqua.jl +++ b/test/test_aqua.jl @@ -1,7 +1,6 @@ using Aqua using ZarrDatasets - Aqua.test_ambiguities(ZarrDatasets) # some internal ambiguities in DiskArray 0.3 probably fixed in 0.4 -Aqua.test_all(ZarrDatasets, ambiguities = false) +Aqua.test_all(ZarrDatasets, ambiguities=false) diff --git a/test/test_cdm.jl b/test/test_cdm.jl index 7b8956a..d951454 100644 --- a/test/test_cdm.jl +++ b/test/test_cdm.jl @@ -16,12 +16,12 @@ mkpath(fname) nczarr_name = "file://" * fname * "#mode=zarr" @debug "filenames " nczarr_name fname -v = randn(2,3) -ds = NCDataset(nczarr_name,"c") -defVar(ds,"var",v,("lon","lat"),attrib = Dict( +v = randn(2, 3) +ds = NCDataset(nczarr_name, "c") +defVar(ds, "var", v, ("lon", "lat"), attrib=Dict( "foo" => "bar", "int_attribute" => 1, - "float_attribute" => 1., + "float_attribute" => 1.0, "scale_factor" => 1.23)) ds.attrib["title"] = "test file" close(ds) @@ -31,29 +31,29 @@ dsz = ZarrDataset(fname) @test Set(dimnames(dsz)) == Set(dimnames(ds)) -for (name,len) in ds.dim +for (name, len) in ds.dim @test dsz.dim[name] == len end -for (varname,v) in ds - @test haskey(dsz,varname) +for (varname, v) in ds + @test haskey(dsz, varname) v2 = dsz[varname] @test Array(v2) == Array(v) - for (attribname,attribval) in v.attrib + for (attribname, attribval) in v.attrib @test v2.attrib[attribname] == attribval end end -for (attribname,attribval) in ds.attrib +for (attribname, attribval) in ds.attrib @test dsz.attrib[attribname] == attribval end io = IOBuffer() -show(io,dsz) +show(io, dsz) str = String(take!(io)) -@test occursin("title",str) +@test occursin("title", str) @test !iswritable(dsz) @test "title" in attribnames(dsz) @@ -68,8 +68,8 @@ end @test zvar == Array(ds["var"]) v = dsz["var"].var -buffer = zeros(eltype(v),size(v)) -load!(v,buffer,:,:) +buffer = zeros(eltype(v), size(v)) +load!(v, buffer, :, :) @test buffer == Array(ds["var"].var) diff --git a/test/test_fillvalue.jl b/test/test_fillvalue.jl index 524f0b6..4337df4 100644 --- a/test/test_fillvalue.jl +++ b/test/test_fillvalue.jl @@ -11,8 +11,8 @@ store = Zarr.DirectoryStore(fname) zg = zgroup(store, "") zarray = zcreate( Int, zg, "lon", 3; - fill_value = 9999, - attrs = Dict("_ARRAY_DIMENSIONS" => ("lon",))) + fill_value=9999, + attrs=Dict("_ARRAY_DIMENSIONS" => ("lon",))) ds = ZarrDataset(fname) @test eltype(ds["lon"]) == Int @@ -20,12 +20,12 @@ ds = ZarrDataset(fname) fname = tempname() mkdir(fname) -ds = ZarrDataset(fname,"c") +ds = ZarrDataset(fname, "c") # variable which is not a CF coordinate variable -v2 = defVar(ds,"foo",Int,(),fillvalue=9999) +v2 = defVar(ds, "foo", Int, (), fillvalue=9999) @test eltype(v2) == Union{Missing,Int} -v3 = defVar(ds,"bar",Int16[2,3,4],("time",),fillvalue=9999) +v3 = defVar(ds, "bar", Int16[2, 3, 4], ("time",), fillvalue=9999) @test eltype(v3) == Union{Missing,Int16} close(ds) diff --git a/test/test_groups.jl b/test/test_groups.jl index 99f8e91..c1c4647 100644 --- a/test/test_groups.jl +++ b/test/test_groups.jl @@ -1,9 +1,9 @@ using ZarrDatasets using Test -data = rand(Int32,3,5) -data2 = rand(Int8,3,5) -data3 = rand(Int16,4) +data = rand(Int32, 3, 5) +data2 = rand(Int8, 3, 5) +data3 = rand(Int16, 4) fname = tempname() mkdir(fname) @@ -13,9 +13,9 @@ TDS = ZarrDataset #using NCDatasets #TDS = NCDataset -ds = TDS(fname,"c") -defDim(ds,"lon",3) -defDim(ds,"lat",5) +ds = TDS(fname, "c") +defDim(ds, "lon", 3) +defDim(ds, "lat", 5) attrib = Dict( "units" => "m/s", @@ -23,11 +23,11 @@ attrib = Dict( ) varname = "var2" -dimensionnames = ("lon","lat") +dimensionnames = ("lon", "lat") vtype = Int32 -zv = defVar(ds,varname,vtype,dimensionnames, attrib = attrib) -zv[:,:] = data +zv = defVar(ds, varname, vtype, dimensionnames, attrib=attrib) +zv[:, :] = data zv.attrib["number"] = 12 ds.attrib["history"] = "test" @@ -35,34 +35,34 @@ ds.attrib["history"] = "test" group_name = "sub-group" attrib = Dict("int_attrib" => 42) -dsg = defGroup(ds,group_name; attrib = attrib) +dsg = defGroup(ds, group_name; attrib=attrib) -defDim(dsg,"time",length(data3)) +defDim(dsg, "time", length(data3)) -zvg = defVar(dsg,"data2",eltype(data2),("lon","lat")) -zvg[:,:] = data2 +zvg = defVar(dsg, "data2", eltype(data2), ("lon", "lat")) +zvg[:, :] = data2 zvg.attrib["standard_name"] = "test" -zv3 = defVar(dsg,"data3",eltype(data3),("time",)) +zv3 = defVar(dsg, "data3", eltype(data3), ("time",)) zv3[:] = data3 -@test_throws Exception defVar(dsg,"data4",Int8,("dimension_does_not_exists",)) +@test_throws Exception defVar(dsg, "data4", Int8, ("dimension_does_not_exists",)) io = IOBuffer() -show(io,ds) +show(io, ds) s = String(take!(io)) -@test occursin("sub-group",s) +@test occursin("sub-group", s) close(ds) # load data from group -ds = TDS(fname,"r") +ds = TDS(fname, "r") -@test ds["var2"][:,:] == data -@test ds.group[group_name]["data2"][:,:] == data2 +@test ds["var2"][:, :] == data +@test ds.group[group_name]["data2"][:, :] == data2 @test ds.group[group_name]["data2"].attrib["standard_name"] == "test" @test ds.group[group_name]["data3"][:] == data3 diff --git a/test/test_multifile.jl b/test/test_multifile.jl index 5afc3b5..d5f394b 100644 --- a/test/test_multifile.jl +++ b/test/test_multifile.jl @@ -12,66 +12,66 @@ using ZarrDatasets fnames = [tempname(), tempname()] -v = randn(2,3,length(fnames)) +v = randn(2, 3, length(fnames)) nczarr_names = ["file://" * fname * "#mode=zarr" for fname in fnames] for i = 1:length(fnames) local ds mkpath(fnames[i]) - ds = NCDataset(nczarr_names[i],"c") - defVar(ds,"var",v[:,:,i:i],("lon","lat","time"),attrib = Dict( + ds = NCDataset(nczarr_names[i], "c") + defVar(ds, "var", v[:, :, i:i], ("lon", "lat", "time"), attrib=Dict( "foo" => "bar", "int_attribute" => 1, - "float_attribute" => 1., + "float_attribute" => 1.0, "scale_factor" => 1.23)) ds.attrib["title"] = "test file" close(ds) end -ds = NCDataset(nczarr_names,aggdim="time") -dsz = ZarrDataset(fnames,aggdim="time") +ds = NCDataset(nczarr_names, aggdim="time") +dsz = ZarrDataset(fnames, aggdim="time") @test Set(dimnames(dsz)) == Set(dimnames(ds)) -for (name,len) in ds.dim +for (name, len) in ds.dim @test dsz.dim[name] == len end -for (varname,v) in ds - @test haskey(dsz,varname) +for (varname, v) in ds + @test haskey(dsz, varname) v2 = dsz[varname] @test Array(v2) == Array(v) - for (attribname,attribval) in v.attrib + for (attribname, attribval) in v.attrib @test v2.attrib[attribname] == attribval end end -for (attribname,attribval) in ds.attrib +for (attribname, attribval) in ds.attrib @test dsz.attrib[attribname] == attribval end io = IOBuffer() -show(io,dsz) +show(io, dsz) str = String(take!(io)) -@test occursin("title",str) +@test occursin("title", str) @test !iswritable(dsz) @test "title" in attribnames(dsz) @test isnothing(parentdataset(dsz)) -zvar = ZarrDataset(fnames,aggdim = "time") do ds3 +zvar = ZarrDataset(fnames, aggdim="time") do ds3 Array(ds3["var"]) end @test zvar == Array(ds["var"]) v = dsz["var"].var -buffer = zeros(eltype(v),size(v)) -load!(v,buffer,:,:,:) +buffer = zeros(eltype(v), size(v)) +load!(v, buffer, :, :, :) @test buffer == Array(ds["var"].var) diff --git a/test/test_preexisting.jl b/test/test_preexisting.jl index b4cce4e..628e822 100644 --- a/test/test_preexisting.jl +++ b/test/test_preexisting.jl @@ -8,11 +8,11 @@ using Zarr fname = tempname() mkdir(fname) gattrib = Dict("title" => "this is the title") -ds = ZarrDataset(fname,"c",attrib = gattrib) +ds = ZarrDataset(fname, "c", attrib=gattrib) ds.attrib["number"] = 1 -defDim(ds,"lon",3) -defDim(ds,"lat",5) +defDim(ds, "lon", 3) +defDim(ds, "lat", 5) attrib = Dict( "units" => "m/s", @@ -21,18 +21,18 @@ attrib = Dict( varname = "var2" -dimensionnames = ("lon","lat") +dimensionnames = ("lon", "lat") vtype = Int32 -zv = defVar(ds,varname,vtype,dimensionnames, attrib = attrib) -zv[:,:] = data = rand(Int32,3,5) +zv = defVar(ds, varname, vtype, dimensionnames, attrib=attrib) +zv[:, :] = data = rand(Int32, 3, 5) zv.attrib["number"] = 12 zv.attrib["standard_name"] = "test" ds.attrib["history"] = "test" close(ds) -for ds in ZarrDataset.((Zarr.storefromstring(fname)[1], Zarr.zopen(fname), )) +for ds in ZarrDataset.((Zarr.storefromstring(fname)[1], Zarr.zopen(fname),)) zv = ds[varname] @@ -40,10 +40,10 @@ for ds in ZarrDataset.((Zarr.storefromstring(fname)[1], Zarr.zopen(fname), )) @test zv.attrib["standard_name"] == "test" @test ds.attrib["history"] == "test" - @test zv[:,:] == data + @test zv[:, :] == data io = IOBuffer() - show(io,ds) + show(io, ds) str = String(take!(io)) - @test occursin("Global",str) + @test occursin("Global", str) end diff --git a/test/test_write.jl b/test/test_write.jl index c1056dd..74b9012 100644 --- a/test/test_write.jl +++ b/test/test_write.jl @@ -4,16 +4,16 @@ using ZarrDatasets: defDim, defVar -data = rand(Int32,3,5) +data = rand(Int32, 3, 5) fname = tempname() mkdir(fname) gattrib = Dict("title" => "this is the title") -ds = ZarrDataset(fname,"c",attrib = gattrib) +ds = ZarrDataset(fname, "c", attrib=gattrib) ds.attrib["number"] = 1 -defDim(ds,"lon",3) -defDim(ds,"lat",5) +defDim(ds, "lon", 3) +defDim(ds, "lat", 5) attrib = Dict( "units" => "m/s", @@ -22,11 +22,11 @@ attrib = Dict( varname = "var2" -dimensionnames = ("lon","lat") +dimensionnames = ("lon", "lat") vtype = Int32 -zv = defVar(ds,varname,vtype,dimensionnames, attrib = attrib) -zv[:,:] = data +zv = defVar(ds, varname, vtype, dimensionnames, attrib=attrib) +zv[:, :] = data zv.attrib["number"] = 12 zv.attrib["standard_name"] = "test" ds.attrib["history"] = "test" @@ -40,23 +40,23 @@ zv = ds[varname] @test zv.attrib["standard_name"] == "test" @test ds.attrib["history"] == "test" -@test zv[:,:] == data +@test zv[:, :] == data io = IOBuffer() -show(io,ds) +show(io, ds) str = String(take!(io)) -@test occursin("Global",str) +@test occursin("Global", str) # fill value fname = tempname() -ds = ZarrDataset(fname,"c") -defDim(ds,"lon",100) -v = defVar(ds,"lon",Float32,("lon",),fillvalue = 9999.) +ds = ZarrDataset(fname, "c") +defDim(ds, "lon", 100) +v = defVar(ds, "lon", Float32, ("lon",), fillvalue=9999.0) v .= 1 close(ds) ds = ZarrDataset(fname) @test eltype(ds["lon"]) == Float32 -@test eltype(cfvariable(ds,"lon",fillvalue=nothing)) == Float32 +@test eltype(cfvariable(ds, "lon", fillvalue=nothing)) == Float32