Merge branch 'main' of github.com:JuliaGeo/AbstractDatasets.jl

Author: Alexander-Barth

docs/src/index.md

@@ -1,7 +1,13 @@
 
 ## Abstract types
 
-Overview of methods
+In order to implement a new dataset based `CommonDataModel.jl`
+one has to create two types derived from:
+1 .`AbstractVariable`: a variable with named dimension and metadata
+2. `AbstractDataset`: a collection of variable with named dimension, metadata and sub-groups. The sub-groups are also `AbstractDataset`.
+
+
+Overview of methods:
 
 |            | get names     | get value  | write / set value |
 |------------|---------------|------------|-------------------|

src/CommonDataModel.jl

@@ -14,13 +14,23 @@ NetCDF or GRIB file)
 
 A data set `ds` of a type derived from `AbstractDataset` should implemented at minimum:
 
-* `Base.key(ds)`: return a list of variable names a strings
-* `Base.getindex(ds,varname::String)`: return an array-like data structure (derived from `AbstractVariable`) of the variables corresponding to `varname`. This array-like data structure should follow the CF semantics.
-* `attribnames(ds)`: should be an iterable with all attribute names
-* `attrib(ds,name)`: attribute value corresponding to name
+* `Base.key(ds)`: return a list of variable names as strings
+* `variable(ds,varname::String)`: return an array-like data structure (derived from `AbstractVariable`) of the variables corresponding to `varname`. This array-like data structure should follow the CF semantics.
 * `dimnames(ds)`: should be an iterable with all dimension names in the data set  `ds`
 * `dim(ds,name)`: dimension value corresponding to name
 
+Optionally a data set can have attributes and groups:
+
+* `attribnames(ds)`: should be an iterable with all attribute names
+* `attrib(ds,name)`: attribute value corresponding to name
+* `groupnames(ds)`: should be an iterable with all group names
+* `group(ds,name)`: group corresponding to the name
+
+For a writable-dataset, one should also implement:
+* `defDim`: define a dimension
+* `defAttrib`: define a attribute
+* `defVar`: define a variable
+* `defGroup`: define a group
 """
 abstract type AbstractDataset
 end
@@ -29,9 +39,21 @@ end
 """
 `AbstractVariable{T,N}` is a subclass of `AbstractArray{T, N}`. A variable `v` of a type derived from `AbstractVariable` should implement:
 
-* `v.attrib`: should be a Dict-like data structure with all variable attribute names as keys and the corresponding value
-* `name(v)`: should be the name of variable within the dataset
+* `name(v)`: should be the name of variable within the data set
 * `dimnames(v)`: should be a iterable data structure with all dimension names of the variable `v`
+* `dataset(v)`: the parent dataset containing `v`
+* `Base.size(v)`: the size of the variable
+* `Base.getindex(v,indices...)`: get the data of `v` at the provided indices
+
+Optionally a variable can have attributes:
+
+* `attribnames(v)`: should be an iterable with all attribute names
+* `attrib(v,name)`: attribute value corresponding to name
+
+For a writable-dataset, one should also implement:
+* `defAttrib`: define a attribute
+* `Base.setindex(v,data,indices...)`: set the data in `v` at the provided indices
+
 """
 abstract type AbstractVariable{T,N} <: AbstractArray{T, N}
 end
@@ -44,3 +66,7 @@ include("attribute.jl")
 include("dimension.jl")
 
 end # module CommonDataModel
+
+#  LocalWords:  AbstractDataset NetCDF GRIB ds AbstractVariable
+#  LocalWords:  varname dimnames iterable attribnames attrib dataset
+#  LocalWords:  groupnames AbstractArray

src/attribute.jl

@@ -15,6 +15,11 @@ function attrib(ds::Union{AbstractDataset,AbstractVariable},attribname::Abstract
     error("no attributes $attribname in $(path(ds))")
 end
 
+"""
+    CommonDatamodel.defAttrib(ds::Union{AbstractDataset,AbstractVariable},name::AbstractString,data)
+
+Create an attribute with the name `attrib` in the data set or variable `ds`.
+"""
 function defAttrib(ds::AbstractDataset,name::AbstractString,data)
     error("unimplemnted for abstract type")
 end

src/cfvariable.jl

@@ -22,6 +22,8 @@ Return a tuple of integers with the size of the variable `var`.
 Base.size(v::CFVariable) = size(v.var)
 
 name(v::CFVariable) = name(v.var)
+dataset(v::CFVariable) = dataset(v.var)
+
 
 attribnames(v::CFVariable) = keys(v.attrib)
 attrib(v::CFVariable,name::AbstractString) = v.attrib[name]

src/dataset.jl

@@ -35,6 +35,12 @@ function group(ds::AbstractDataset,groupname::AbstractString)
     error("no group $groupname in $(path(ds))")
 end
 
+"""
+    group = CommonDatamodel.defGroup(ds::AbstractDataset,name::AbstractString)
+
+Create an empty sub-group with the name `name` in the data set `ds`.
+The `group` is a sub-type of `AbstractDataset`.
+"""
 function defGroup(ds::AbstractDataset,name::AbstractString)
     error("unimplemented for abstract type")
 end
@@ -110,6 +116,9 @@ function Base.show(io::IO,ds::AbstractDataset)
 end
 
 
+Base.getindex(ds::AbstractDataset,varname) = cfvariable(ds,varname)
+
+
 for (item_color,default) in (
     (:section_color, :red),
     (:attribute_color, :cyan),

src/dimension.jl

@@ -13,13 +13,22 @@ dimnames(ds::Union{AbstractDataset,AbstractVariable}) = ()
 Return the length of the dimension `dimname` in the data set `ds`.
 """
 function dim(v::AbstractVariable,dimname::AbstractString)
-    error("no dimension $dimname in $(name(v))")
+    if !(dimname in dimnames(v))
+        error("$dimname is not among the dimensions of $(name(v))")
+    end
+    return dim(dataset(v),dimname)
 end
 
 function dim(ds::AbstractDataset,dimname::AbstractString)
     error("no dimension $dimname in $(path(ds))")
 end
 
+"""
+    CommonDatamodel.defDim(ds::AbstractDataset,name::AbstractString,len)
+
+Create dimension with the name `name` in the data set `ds` with the length `len`.
+`len` can be `Inf` for unlimited dimensions.
+"""
 function defDim(ds::AbstractDataset,name::AbstractString,len)
     error("unimplemnted for abstract type")
 end

src/variable.jl

@@ -1,28 +1,28 @@
 """
-    CommonDatamodel.name(v::AbstractVariable)
+    CommonDataModel.name(v::AbstractVariable)
 
 Return the name of the variable `v` as a string.
 """
 name(v::AbstractVariable) = ""
 
 
 """
-    CommonDatamodel.dimnames(v::AbstractVariable)
+    CommonDataModel.dimnames(v::AbstractVariable)
 
 Return an iterable of the dimension names of the variable `v`.
 """
 dimnames(av::AbstractVariable) = ()
 
 
 """
-    CommonDatamodel.varnames(ds::AbstractDataset)
+    CommonDataModel.varnames(ds::AbstractDataset)
 
 Return an iterable of all the variable name.
 """
 varnames(ds::AbstractDataset) = ()
 
 """
-    CommonDatamodel.variable(ds::AbstractDataset,variablename::AbstractString)
+    CommonDataModel.variable(ds::AbstractDataset,variablename::AbstractString)
 
 Return the variable with the name `variablename` from the data set `ds`.
 """
@@ -31,10 +31,17 @@ function variable(ds::AbstractDataset,variablename::AbstractString)
 end
 
 function defVar(ds::AbstractDataset,name::AbstractString,type,dimnames)
-    error("unimplemnted for abstract type")
+    error("unimplemented for abstract type")
 end
 
+"""
+    ds = CommonDataModel.dataset(v::AbstractVariable)
 
+Return the data set `ds` to which a the variable `v` belongs to.
+"""
+function dataset(v::AbstractVariable)
+    error("unimplemented for abstract type")
+end
 
 
 function Base.show(io::IO,v::AbstractVariable)

test/memory_dataset.jl

@@ -20,14 +20,13 @@ end
 
 Base.getindex(v::MemoryVariable,ij...) = v.data[ij...]
 Base.setindex!(v::MemoryVariable,data,ij...) = v.data[ij...] = data
+Base.size(v::MemoryVariable) = size(v.data)
 CDM.name(v::MemoryVariable) = v.name
 CDM.dimnames(v::MemoryVariable) = v.dimnames
-Base.size(v::MemoryVariable) = size(v.data)
-CDM.dim(v::MemoryVariable,name::AbstractString) = v.parent_dataset.dim[name]
+CDM.dataset(v::MemoryVariable) = v.parent_dataset
 
 Base.keys(md::MemoryDataset) = keys(md.variables)
 CDM.variable(md::MemoryDataset,varname::AbstractString) = md.variables[varname]
-Base.getindex(md::MemoryDataset,varname::AbstractString) = CDM.cfvariable(md,varname)
 CDM.dimnames(md::MemoryDataset) = keys(md.dim)
 CDM.dim(md::MemoryDataset,name::AbstractString) = md.dim[name]
 CDM.attribnames(md::Union{MemoryDataset,MemoryVariable}) = keys(md.attrib)

test/test_conversion.jl

@@ -2,6 +2,7 @@ import NCDatasets
 import GRIBDatasets
 import CommonDataModel as CDM
 using Test
+using DataStructures
 
 datadir = joinpath(dirname(pathof(GRIBDatasets)),"..","test","sample-data")
 filename = joinpath(datadir,"era5-levels-members.grib")
@@ -28,9 +29,9 @@ NCDatasets.write(tmp_filename,ds)
 dsnc = NCDatasets.Dataset(tmp_filename)
 @test ds["number"][:] == dsnc["number"][:]
 
-@test CDM.dims(ds) == CDM.dims(dsnc)
-@test CDM.attribs(ds) == CDM.attribs(dsnc)
-@test CDM.groups(ds) == CDM.groups(dsnc)
+@test OrderedDict(CDM.dims(ds)) == OrderedDict(CDM.dims(dsnc))
+@test OrderedDict(CDM.attribs(ds)) == OrderedDict(CDM.attribs(dsnc))
+@test OrderedDict(CDM.groups(ds)) == OrderedDict(CDM.groups(dsnc))
 
 
 close(dsnc)

test/test_scaling.jl

@@ -122,4 +122,6 @@ time_origin = DateTime(2000,1,1)
 md["data"][1,2] = DateTime(2000,2,1)
 @test md["data"].var[1,2] == Dates.value(md["data"][1,2] - time_origin) ÷ (24*60*60*1000)
 
+
+@test CDM.dataset(md["data"]) == md
 close(md)