experimental MemoryDataset

Author: Alexander-Barth

src/CommonDataModel.jl

@@ -7,6 +7,7 @@ using Printf
 using Preferences
 using DataStructures
 import Base:
+    checkbounds,
     close,
     collect,
     display,
@@ -20,6 +21,7 @@ import Base:
     show,
     size,
     write
+
 import Statistics
 import Statistics:
     maximum,
@@ -48,6 +50,7 @@ include("select.jl")
 include("aggregation.jl")
 include("groupby.jl")
 include("rolling.jl")
+include("memory_dataset.jl")
 
 end # module CommonDataModel
 

test/memory_dataset.jl renamed to src/memory_dataset.jl

@@ -1,32 +1,4 @@
-import Base: size, getindex, setindex!, checkbounds
 import CommonDataModel as CDM
-#import CommonDataModel: SymbolOrString
-using DataStructures
-import CommonDataModel: defVar, unlimited, name, dimnames, dataset, variable, dim, attribnames, attrib, defDim, defAttrib, delAttrib, MFDataset, iswritable, SymbolOrString, parentdataset, load!, maskingvalue
-
-mutable struct ResizableArray{T,N} <: AbstractArray{T,N}
-    A::AbstractArray{T,N}
-    fillvalue::T
-end
-
-struct MemoryVariable{T,N,TP,TA <: AbstractArray{T,N}} <: CDM.AbstractVariable{T,N}
-    parent_dataset::TP
-    name::String
-    dimnames::NTuple{N,String}
-    data::TA
-    _attrib::OrderedDict{String,Any}
-end
-
-struct MemoryDataset{TP,Tmasingvalue} <: CDM.AbstractDataset
-    parent_dataset::TP
-    name::String # "/" for root group
-    dimensions::OrderedDict{String,Int}
-    variables::OrderedDict{String,MemoryVariable}
-    _attrib::OrderedDict{String,Any}
-    unlimited::Vector{String}
-    _group::OrderedDict{String,Any}
-    maskingvalue::Tmasingvalue
-end
 
 Base.size(RA::ResizableArray) = size(RA.A)
 Base.getindex(RA::ResizableArray,inds...) = getindex(RA.A,inds...)
@@ -233,7 +205,7 @@ end
 
 
 MemoryDataset(keys::AbstractArray{<:AbstractString,N}, args...; kwargs...) where N =
-    MFDataset(MemoryDataset,fnames, args...; kwargs...)
+    MFDataset(MemoryDataset,keys, args...; kwargs...)
 
 
 function MemoryDataset(f::Function,args...; kwargs...)

src/types.jl

@@ -171,3 +171,31 @@ end
 
 
 const Iterable = Union{Attributes,Dimensions,Groups,AbstractDataset}
+
+
+# memory dataset
+
+mutable struct ResizableArray{T,N} <: AbstractArray{T,N}
+    A::AbstractArray{T,N}
+    fillvalue::T
+end
+
+struct MemoryVariable{T,N,TP,TA <: AbstractArray{T,N}} <: AbstractVariable{T,N}
+    parent_dataset::TP
+    name::String
+    dimnames::NTuple{N,String}
+    data::TA
+    _attrib::OrderedDict{String,Any}
+end
+
+struct MemoryDataset{TP,Tmasingvalue} <: AbstractDataset
+    parent_dataset::TP
+    name::String # "/" for root group
+    dimensions::OrderedDict{String,Int}
+    variables::OrderedDict{String,MemoryVariable}
+    _attrib::OrderedDict{String,Any}
+    unlimited::Vector{String}
+    _group::OrderedDict{String,Any}
+    maskingvalue::Tmasingvalue
+end
+

test/runtests.jl

@@ -1,8 +1,7 @@
 using CommonDataModel
+using CommonDataModel: MemoryDataset
 using Test
 
-include("memory_dataset.jl")
-
 @testset "CommonDataModel" begin
     #include("test_conversion.jl")
     include("test_empty.jl")

test/test_multifile.jl

@@ -12,10 +12,14 @@ import CommonDataModel:
     Dimensions,
     checksum,
     chunking,
+    dataset,
     defGroup,
     deflate,
+    dimnames,
     fillvalue,
-    sync
+    parentdataset,
+    sync,
+    variable
 
 function example_file(TDS,i,array, fname = tempname();
     varname = "var")

test/test_select.jl

@@ -4,7 +4,16 @@ using Dates
 using IntervalSets
 using Test
 using Printf
-import CommonDataModel: AbstractVariable, dimnames, coordinate_value, coordinate_names, Near, split_by_and, SubDataset, @select, select
+import CommonDataModel:
+    @select,
+    AbstractVariable,
+    Near,
+    SubDataset,
+    coordinate_names,
+    coordinate_value,
+    dimnames,
+    select,
+    split_by_and
 
 
 struct SelectableVariable{T,N,NT,TA} <: AbstractArray{T,N} where NT <: NTuple where TA <: AbstractArray{T,N}
@@ -367,7 +376,7 @@ end
 
 fname_subset = tempname()
 times = DateTime(1992,1,1):Dates.Day(1):DateTime(1992,1,3)
-ds = MFDataset(TDS,url.(TDS,times),aggdim = "time",deferopen=false);
+ds = TDS(url.(TDS,times),aggdim = "time",deferopen=false);
 ds_subset = @select(ds,30 <= lon <= 60 && 40 <= lat <= 90)
 TDS(fname_subset,"c")  do ds_dest
     write(ds_dest,ds_subset)

test/test_variable.jl

@@ -5,6 +5,10 @@ using Printf
 #using NCDatasets: NetCDFError, load!
 using DataStructures
 using CFTime
+using CommonDataModel:
+    MemoryDataset,
+    name,
+    load!
 
 sz = (4,5)
 filename = tempname()