allow the construction of the multi-file data based on a list of abstract datasets

Author: Alexander-Barth

src/memory_dataset.jl

@@ -118,7 +118,7 @@ CDM.attribnames(md::Union{MemoryDataset,MemoryVariable}) = keys(md._attrib)
 CDM.attrib(md::Union{MemoryDataset,MemoryVariable},name::SymbolOrString) = md._attrib[String(name)]
 
 
-CDM.groupnames(md::MemoryDataset) = keys(md._group)
+CDM.groupnames(md::MemoryDataset) = collect(keys(md._group))
 CDM.group(md::MemoryDataset,name::SymbolOrString) = md._group[String(name)]
 
 function CDM.defDim(md::MemoryDataset,name::SymbolOrString,len)

src/multifile.jl

@@ -52,10 +52,12 @@ end
 
 unlimited(ds::MFDataset) = unique(reduce(vcat,unlimited.(ds.ds)))
 
-groupnames(ds::MFDataset) = groupnames(ds.ds[1])
+groupnames(ds::MFDataset) = unique(reduce(vcat,groupnames.(ds.ds)))
 
 function group(mfds::MFDataset,name::SymbolOrString)
-    ds = group.(mfds.ds,name)
+    ds_sel = filter(ds -> name in groupnames(ds),mfds.ds)
+
+    ds = group.(ds_sel,name)
     constvars = Symbol[]
     return MFDataset(ds,mfds.aggdim,mfds.isnewdim,constvars)
 end
@@ -84,7 +86,10 @@ function MFDataset(ds,aggdim,isnewdim,constvars)
 end
 
 
-function MFDataset(TDS,fnames::AbstractArray{<:AbstractString,N},mode = "r"; aggdim = nothing,
+# when aggdim is "", the variables list of each file is
+# merged and no variable is concatanted
+function MFDataset(TDS,fnames::AbstractArray{<:AbstractString,N},mode = "r";
+                   aggdim = nothing,
                    deferopen = true,
                    _aggdimconstant = false,
                    isnewdim = false,
@@ -125,6 +130,25 @@ function MFDataset(TDS,fnames::AbstractArray{<:AbstractString,N},mode = "r"; agg
     return mfds
 end
 
+"""
+    mfds = MFDataset(ds)
+
+Virtually concatente a vector of AbstractDataset (`ds`) by merging the list
+of variables. Every variable which happend to be repeated in several datasets
+is assumed to be constant accorss all datasets.
+"""
+function MFDataset(ds::AbstractVector{<:AbstractDataset};
+                   aggdim = nothing,
+                   deferopen = true,
+                   _aggdimconstant = false,
+                   isnewdim = false,
+                   constvars = Union{Symbol,String}[],
+                   )
+
+    MFDataset(ds,aggdim,isnewdim,Symbol.(constvars))
+end
+
+
 
 function close(mfds::MFDataset)
     close.(mfds.ds)
@@ -192,8 +216,9 @@ function variable(mfds::MFDataset,varname::SymbolOrString)
 
         if (dim != nothing)
             v = CatArrays.CatArray(dim,vars...)
+            # GRIBdatasets.dimnames can be a vector of String
             return MFVariable(mfds,v,
-                          dimnames(vars[1]),String(varname))
+                          Tuple(dimnames(vars[1])),String(varname))
         else
             return vars[1]
         end

test/test_multifile.jl

@@ -267,6 +267,9 @@ ds = TDS(fnames,aggdim = "", deferopen = false);
 
 @test sort(keys(ds)) == ["ampl", "lat", "lon", "time", "vel"]
 
+ds_all = TDS.(fnames)
+mfds = CDM.MFDataset(ds_all,aggdim = "")
+@test sort(keys(mfds)) == ["ampl", "lat", "lon", "time", "vel"]
 
 # save a merged file
 fname_merged = tempname()