Merge branch 'main' of github.com:JuliaGeo/CommonDataModel.jl

Author: Alexander-Barth

README.md

@@ -14,6 +14,7 @@ This package contains abstracts type definition for loading and manipulating GRI
 | GRIB    | [`GRIBDatasets`](https://github.com/JuliaGeo/GRIBDatasets.jl)        |            ✔ |             - |
 | geoTIFF | [`TIFFDatasets`](https://github.com/Alexander-Barth/TIFFDatasets.jl) |            ✔ |             - |
 | Zarr    | [`ZarrDatasets`](https://github.com/JuliaGeo/ZarrDatasets.jl)        |            ✔ |             ✔ |
+|         | [`MetopDatasets`](https://github.com/eumetsat/MetopDatasets.jl)        |            ✔ |             - |
 
 
 Features include:

src/cfvariable.jl

@@ -11,20 +11,21 @@ Return a tuple of integers with the size of the variable `var`.
     Note that the size of a variable can change, i.e. for a variable with an
     unlimited dimension.
 """
-Base.size(v::CFVariable) = size(v.var)
+Base.size(v::CFVariable) = size(parent(v))
+Base.parent(v::CFVariable) = v.var
 
-name(v::CFVariable) = name(v.var)
-dataset(v::CFVariable) = dataset(v.var)
+name(v::CFVariable) = name(parent(v))
+dataset(v::CFVariable) = dataset(parent(v))
 
 
-# be aware that for GRIBDatasets v.attrib is different from v.var.attrib
+# be aware that for GRIBDatasets v.attrib is different from parent(v).attrib
 attribnames(v::CFVariable) = keys(v.attrib)
 attrib(v::CFVariable,name::SymbolOrString) = v.attrib[name]
 defAttrib(v::CFVariable,name,value) = v.attrib[name] = value
 delAttrib(v::CFVariable,name) = delete!(v,name)
 
-dimnames(v::CFVariable) = dimnames(v.var)
-dim(v::CFVariable,name::SymbolOrString) = dim(v.var,name)
+dimnames(v::CFVariable) = dimnames(parent(v))
+dim(v::CFVariable,name::SymbolOrString) = dim(parent(v),name)
 
 # necessary for IJulia if showing a variable from a closed file
 Base.show(io::IO,::MIME"text/plain",v::AbstractVariable) = show(io,v)
@@ -215,7 +216,7 @@ missing_values(v::CFVariable) = v._storage_attrib.missing_values
 
 # collect all possible fill values
 function fill_and_missing_values(v::CFVariable)
-    T = eltype(v.var)
+    T = eltype(parent(v))
     fv = ()
     if !isnothing(fillvalue(v))
         fv = (fillvalue(v),)
@@ -448,29 +449,29 @@ end
 #@inline CFinvtransformdata(data::Char,fv,scale_factor,add_offset,time_origin,time_factor,DT) = CFtransform_replace_missing(data,fv)
 
 function Base.getindex(v::CFVariable, indexes::TIndices...)
-    data = v.var[indexes...]
+    data = parent(v)[indexes...]
     return CFtransformdata(data,fill_and_missing_values(v),scale_factor(v),add_offset(v),
                            time_origin(v),time_factor(v),maskingvalue(v),eltype(v))
 end
 
 function Base.setindex!(v::CFVariable,data::Array{Missing,N},indexes::TIndices...) where N
-    v.var[indexes...] = fill(fillvalue(v),size(data))
+    parent(v)[indexes...] = fill(fillvalue(v),size(data))
 end
 
 function Base.setindex!(v::CFVariable,data::Missing,indexes::TIndices...)
-    v.var[indexes...] = fillvalue(v)
+    parent(v)[indexes...] = fillvalue(v)
 end
 
 function Base.setindex!(v::CFVariable,data::Union{T,Array{T}},indexes::TIndices...) where T <: Union{AbstractCFDateTime,DateTime,Missing}
 
     if calendar(v) !== nothing
         # can throw an convertion error if calendar attribute already exists and
         # is incompatible with the provided data
-        v.var[indexes...] = CFinvtransformdata(
+        parent(v)[indexes...] = CFinvtransformdata(
             data,fill_and_missing_values(v),scale_factor(v),add_offset(v),
             time_origin(v),time_factor(v),
             maskingvalue(v),
-            eltype(v.var))
+            eltype(parent(v)))
         return data
     end
 
@@ -479,12 +480,12 @@ end
 
 
 function Base.setindex!(v::CFVariable,data,indexes::TIndices...)
-    v.var[indexes...] = CFinvtransformdata(
+    parent(v)[indexes...] = CFinvtransformdata(
         data,fill_and_missing_values(v),
         scale_factor(v),add_offset(v),
         time_origin(v),time_factor(v),
         maskingvalue(v),
-        eltype(v.var))
+        eltype(parent(v)))
 
     return data
 end
@@ -551,20 +552,20 @@ end
 
 Return a tuple of strings with the dimension names of the variable `v`.
 """
-dimnames(v::Union{CFVariable,MFCFVariable}) = dimnames(v.var)
+dimnames(v::Union{CFVariable,MFCFVariable}) = dimnames(parent(v))
 
-name(v::Union{CFVariable,MFCFVariable}) = name(v.var)
-chunking(v::CFVariable,storage,chunksize) = chunking(v.var,storage,chunksize)
-chunking(v::CFVariable) = chunking(v.var)
+name(v::Union{CFVariable,MFCFVariable}) = name(parent(v))
+chunking(v::CFVariable,storage,chunksize) = chunking(parent(v),storage,chunksize)
+chunking(v::CFVariable) = chunking(parent(v))
 
-deflate(v::CFVariable,shuffle,dodeflate,deflate_level) = deflate(v.var,shuffle,dodeflate,deflate_level)
-deflate(v::CFVariable) = deflate(v.var)
+deflate(v::CFVariable,shuffle,dodeflate,deflate_level) = deflate(parent(v),shuffle,dodeflate,deflate_level)
+deflate(v::CFVariable) = deflate(parent(v))
 
-checksum(v::CFVariable,checksummethod) = checksum(v.var,checksummethod)
-checksum(v::CFVariable) = checksum(v.var)
+checksum(v::CFVariable,checksummethod) = checksum(parent(v),checksummethod)
+checksum(v::CFVariable) = checksum(parent(v))
 
 
-fillmode(v::CFVariable) = fillmode(v.var)
+fillmode(v::CFVariable) = fillmode(parent(v))
 
 
 ############################################################
@@ -585,7 +586,7 @@ end
 
 Loads a NetCDF (or other format) variables `ncvar` in-place and puts the result in `data` (an
 array of `eltype(ncvar)`) along the specified `indices`. `buffer` is a temporary
- array of the same size as data but the type should be `eltype(ncv.var)`, i.e.
+ array of the same size as data but the type should be `eltype(parent(ncv))`, i.e.
 the corresponding type in the files (before applying `scale_factor`,
 `add_offset` and masking fill values). Scaling and masking will be applied to
 the array `data`.
@@ -608,17 +609,17 @@ ds = Dataset("file.nc")
 ncv = ds["vgos"];
 # data and buffer must have the right shape and type
 data = zeros(eltype(ncv),size(ncv)); # here Vector{Float64}
-buffer = zeros(eltype(ncv.var),size(ncv)); # here Vector{Int16}
+buffer = zeros(eltype(parent(ncv)),size(ncv)); # here Vector{Int16}
 NCDatasets.load!(ncv,data,buffer,:,:,:)
 close(ds)
 ```
 """
 @inline function load!(v::Union{CFVariable{T,N},MFCFVariable{T,N},SubVariable{T,N}}, data, buffer, indices::TIndices...) where {T,N}
 
-    if v.var == nothing
+    if parent(v) == nothing
         return load!(v,indices...)
     else
-        load!(v.var,buffer,indices...)
+        load!(parent(v),buffer,indices...)
         fmv = fill_and_missing_values(v)
         return CFtransformdata!(data,buffer,fmv,scale_factor(v),add_offset(v),
                                 time_origin(v),time_factor(v),

src/defer.jl

@@ -61,16 +61,16 @@ function DeferDataset(TDS,r::Resource,groupname::String,data::OrderedDict)
    return dds
 end
 
-function DeferDataset(TDS,filename::AbstractString,mode::AbstractString,info::OrderedDict)
-    r = Resource(filename,mode,info)
+function DeferDataset(TDS,filename::AbstractString,mode::AbstractString,info::OrderedDict; kwargs...)
+    r = Resource(filename,mode,Dict(kwargs),info)
     groupname = "/"
     return DeferDataset(TDS,r,groupname,info)
 end
 
-function DeferDataset(TDS,filename::AbstractString,mode = "r")
+function DeferDataset(TDS,filename::AbstractString,mode = "r"; kwargs...)
     TDS(filename,mode) do ds
         info = metadata(ds)
-        r = Resource(filename,mode,info)
+        r = Resource(filename,mode,Dict(kwargs),info)
         groupname = "/"
         return DeferDataset(TDS,r,groupname,info)
     end
@@ -83,8 +83,14 @@ groupname(dds::DeferDataset) = dds.groupname
 path(dds::DeferDataset) = dds.r.filename
 varnames(dds::DeferDataset) = collect(keys(dds.data[:var]))
 
+function maskingvalue(dds::DeferDataset{TDS}) where TDS
+    TDS(dds.r.filename,dds.r.mode; dds.r.args...) do ds
+        maskingvalue(ds)
+    end
+end
+
 function Variable(f::Function, dv::DeferVariable{T,N,TDS}) where {T,N,TDS}
-    TDS(dv.r.filename,dv.r.mode) do ds
+    TDS(dv.r.filename,dv.r.mode; dv.r.args...) do ds
         f(variable(ds,dv.varname))
     end
 end
@@ -103,7 +109,7 @@ end
 variable(dds::DeferDataset,varname::Symbol) = variable(dds,string(varname))
 
 dataset(dv::DeferVariable{T,N,TDS}) where {T,N,TDS} =
-    DeferDataset(TDS,dv.r.filename,dv.r.mode)
+    DeferDataset(TDS,dv.r.filename,dv.r.mode; dv.r.args...)
 
 function Base.getindex(dv::DeferVariable,indexes::Union{Int,Colon,AbstractRange{<:Integer}}...)
     Variable(dv) do v

src/memory_dataset.jl

@@ -68,12 +68,12 @@ function grow_unlimited_dimension(ds,dname,len)
     end
 end
 
-Base.getindex(v::MemoryVariable,ij::TIndices...) = v.data[ij...]
-CDM.load!(v::MemoryVariable,buffer,ij...) = buffer .= view(v.data,ij...)
-
+Base.parent(v::MemoryVariable) = v.data
+Base.size(v::MemoryVariable) = size(parent(v))
+Base.getindex(v::MemoryVariable,ij::TIndices...) = parent(v)[ij...]
 function Base.setindex!(v::MemoryVariable,data,ij...)
-    sz = size(v.data)
-    v.data[ij...] = data
+    sz = size(v)
+    parent(v)[ij...] = data
 
     root = _root(v)
     for idim = findall(size(v) .> sz)
@@ -82,7 +82,8 @@ function Base.setindex!(v::MemoryVariable,data,ij...)
     end
     return data
 end
-Base.size(v::MemoryVariable) = size(v.data)
+
+CDM.load!(v::MemoryVariable,buffer,ij...) = buffer .= view(parent(v),ij...)
 CDM.name(v::Union{MemoryVariable,MemoryDataset}) = v.name
 CDM.dimnames(v::MemoryVariable) = v.dimnames
 CDM.dataset(v::MemoryVariable) = v.parent_dataset

src/multifile.jl

@@ -72,8 +72,6 @@ function parentdataset(mfds::MFDataset)
     end
 end
 
-Base.Array(v::MFVariable) = Array(v.var)
-
 iswritable(mfds::MFDataset) = iswritable(mfds.ds[1])
 
 function MFDataset(ds,aggdim,isnewdim,constvars)
@@ -94,6 +92,7 @@ function MFDataset(TDS,fnames::AbstractArray{<:AbstractString,N},mode = "r";
                    _aggdimconstant = false,
                    isnewdim = false,
                    constvars = Union{Symbol,String}[],
+                   kwargs...
                    ) where N
     if !(mode == "r" || mode == "a")
         throw(ArgumentError("""Unsupported mode for multi-file dataset (mode = $(mode)). Mode must be "r" or "a"."""))
@@ -105,20 +104,20 @@ function MFDataset(TDS,fnames::AbstractArray{<:AbstractString,N},mode = "r";
         if _aggdimconstant
             # load only metadata from master
             master_index = 1
-            ds_master = TDS(fnames[master_index],mode);
+            ds_master = TDS(fnames[master_index],mode; kwargs...);
             data_master = metadata(ds_master)
             ds = Vector{Union{TDS,DeferDataset}}(undef,length(fnames))
             #ds[master_index] = ds_master
             for (i,fname) in enumerate(fnames)
                 #if i !== master_index
-                ds[i] = DeferDataset(TDS,fname,mode,data_master)
+                ds[i] = DeferDataset(TDS,fname,mode,data_master; kwargs...)
                 #end
             end
         else
-            ds = DeferDataset.(TDS,fnames,mode)
+            ds = DeferDataset.(TDS,fnames,mode; kwargs...)
         end
     else
-        ds = TDS.(fnames,mode);
+        ds = TDS.(fnames,mode,kwargs...);
     end
 
     if (aggdim == nothing) && !isnewdim
@@ -135,7 +134,7 @@ function MFDataset(ds::AbstractVector{<:AbstractDataset};
                    deferopen = true,
                    _aggdimconstant = false,
                    isnewdim = false,
-                   constvars = Union{Symbol,String}[],
+                   constvars = Union{Symbol,String}[]
                    )
 
     MFDataset(ds,aggdim,isnewdim,Symbol.(constvars))
@@ -176,18 +175,26 @@ function varnames(mfds::MFDataset)
     end
 end
 
-Base.getindex(v::MFVariable,indexes::TIndices...) = getindex(v.var,indexes...)
-Base.setindex!(v::MFVariable,data,indexes::TIndices...) = setindex!(v.var,data,indexes...)
+maskingvalue(mfds::MFDataset) = maskingvalue(mfds.ds[1])
 
 
-load!(v::MFVariable,buffer,indexes...) = CatArrays.load!(v.var,buffer,indexes...)
+Base.parent(v::MFVariable) = v.var
+Base.parent(v::MFCFVariable) = v.var
+Base.Array(v::MFVariable) = Array(parent(v))
+Base.getindex(v::MFVariable,indexes::TIndices...) = getindex(parent(v),indexes...)
+Base.setindex!(v::MFVariable,data,indexes::TIndices...) = setindex!(parent(v),data,indexes...)
+Base.size(v::MFVariable) = size(parent(v))
+Base.size(v::MFCFVariable) = size(parent(v))
+Base.getindex(v::MFCFVariable,ind::TIndices...) = v.cfvar[ind...]
+Base.setindex!(v::MFCFVariable,data,ind::TIndices...) = v.cfvar[ind...] = data
+function Base.cat(vs::AbstractVariable...; dims::Integer)
+    CatArrays.CatArray(dims,vs...)
+end
 
-Base.size(v::MFVariable) = size(v.var)
-Base.size(v::MFCFVariable) = size(v.var)
+load!(v::MFVariable,buffer,indexes...) = CatArrays.load!(parent(v),buffer,indexes...)
 dimnames(v::MFVariable) = v.dimnames
 name(v::MFVariable) = v.varname
 
-
 function variable(mfds::MFDataset,varname::SymbolOrString)
     if mfds.isnewdim
         if Symbol(varname) in mfds.constvars
@@ -268,15 +275,6 @@ end
 
 dataset(v::Union{MFVariable,MFCFVariable}) = v.ds
 
-
-Base.getindex(v::MFCFVariable,ind::TIndices...) = v.cfvar[ind...]
-Base.setindex!(v::MFCFVariable,data,ind::TIndices...) = v.cfvar[ind...] = data
-
-
-function Base.cat(vs::AbstractVariable...; dims::Integer)
-    CatArrays.CatArray(dims,vs...)
-end
-
 """
     storage,chunksizes = chunking(v::MFVariable)
     storage,chunksizes = chunking(v::MFCFVariable)
@@ -299,6 +297,6 @@ end
 deflate(v::MFVariable) = deflate(v.ds.ds[1][name(v)])
 checksum(v::MFVariable) = checksum(v.ds.ds[1][name(v)])
 
-chunking(v::MFCFVariable) = chunking(v.var)
-deflate(v::MFCFVariable) = deflate(v.var)
-checksum(v::MFCFVariable) = checksum(v.var)
+chunking(v::MFCFVariable) = chunking(parent(v))
+deflate(v::MFCFVariable) = deflate(parent(v))
+checksum(v::MFCFVariable) = checksum(parent(v))