test defVar for data

Author: Alexander-Barth

src/variable.jl

@@ -43,7 +43,7 @@ function defVar(ds::AbstractDataset,
                 name::SymbolOrString,
                 data::AbstractArray{Union{Missing,T},N},
                 dimnames;
-                kwargs...) where T <: Union{Int8,UInt8,Int16,Int32,Int64,Float32,Float64} where N
+                kwargs...) where T <: Union{Int8,UInt8,Int16,Int32,Int64,Float32,Float64,Char,String} where N
     _defVar(ds::AbstractDataset,name,data,T,dimnames; kwargs...)
 end
 
@@ -55,7 +55,7 @@ function defVar(ds::AbstractDataset,
                 data::AbstractArray{<:Union{Missing,T},N},
                 dimnames;
                 kwargs...) where T <: Union{DateTime,AbstractCFDateTime} where N
-    _defVar(ds::AbstractDataset,name,data,Float64,dimnames; kwargs...)
+    _defVar(ds,name,data,Float64,dimnames; kwargs...)
 end
 
 function defVar(ds::AbstractDataset,name::SymbolOrString,data,dimnames; kwargs...)
@@ -65,15 +65,16 @@ function defVar(ds::AbstractDataset,name::SymbolOrString,data,dimnames; kwargs..
     else
         nctype = eltype(data)
     end
-    _defVar(ds::AbstractDataset,name,data,nctype,dimnames; kwargs...)
+
+    _defVar(ds,name,data,nctype,dimnames; kwargs...)
 end
 
 function _defVar(ds::AbstractDataset,name::SymbolOrString,data,nctype,vardimnames; attrib = [], kwargs...)
     # define the dimensions if necessary
     for (i,dimname) in enumerate(vardimnames)
         if !(dimname in dimnames(ds))
             defDim(ds,dimname,size(data,i))
-        elseif !(dimname in unlimited(ds.dim))
+        elseif !(dimname in unlimited(ds))
             dimlen = dim(ds,dimname)
 
             if (dimlen != size(data,i))
@@ -84,14 +85,15 @@ function _defVar(ds::AbstractDataset,name::SymbolOrString,data,nctype,vardimname
     end
 
     T = eltype(data)
-    attrib = collect(attrib)
+    # we should preserve the order
+    # value type is promoted to Any as we add values of different type
+    attrib = convert(OrderedDict{String,Any},OrderedDict(attrib))
 
     if T <: Union{TimeType,Missing}
-        dattrib = Dict(attrib)
-        if !haskey(dattrib,"units")
+        if !haskey(attrib,"units")
             push!(attrib,"units" => CFTime.DEFAULT_TIME_UNITS)
         end
-        if !haskey(dattrib,"calendar")
+        if !haskey(attrib,"calendar")
             # these dates cannot be converted to the standard calendar
             if T <: Union{DateTime360Day,Missing}
                 push!(attrib,"calendar" => "360_day")
@@ -103,18 +105,14 @@ function _defVar(ds::AbstractDataset,name::SymbolOrString,data,nctype,vardimname
         end
     end
 
-    v =
-        if Missing <: T
-            # make sure a fill value is set (it might be overwritten by kwargs...)
-            defVar(ds,name,nctype,vardimnames;
-                   fillvalue = fillvalue(nctype),
-                   attrib = attrib,
-                   kwargs...)
-        else
-            defVar(ds,name,nctype,vardimnames;
+    # make sure a fill value is set
+    if (Missing <: T) && !haskey(attrib,"_FillValue")
+        push!(attrib,"_FillValue" => fillvalue(nctype))
+    end
+
+    v = defVar(ds,name,nctype,vardimnames;
                    attrib = attrib,
                    kwargs...)
-        end
 
     v[:] = data
     return v

test/memory_dataset.jl

@@ -1,7 +1,8 @@
 import Base
 import CommonDataModel as CDM
+#import CommonDataModel: SymbolOrString
 using DataStructures
-
+import CommonDataModel: defVar, unlimited, name, dimnames, dataset, variable, dim, attribnames, attrib, defDim, defAttrib
 
 struct MemoryVariable{T,N,TP} <: CDM.AbstractVariable{T,N}
     parent_dataset::TP
@@ -42,12 +43,13 @@ function CDM.defDim(md::MemoryDataset,name::AbstractString,len)
     end
 end
 
-function CDM.defVar(md::MemoryDataset,name::AbstractString,T,dimnames;
+function CDM.defVar(md::MemoryDataset,name::AbstractString,T::DataType,dimnames;
                     attrib = OrderedDict{String,Any}(),
                     )
     sz = ntuple(i -> CDM.dim(md,dimnames[i]),length(dimnames))
     data = Array{T,length(dimnames)}(undef,sz...)
-    mv = MemoryVariable(md,name,(dimnames...,), data, attrib)
+    mv = MemoryVariable(md,name,(dimnames...,), data,
+                        convert(OrderedDict{String,Any},attrib))
     md.variables[name] = mv
 
     return md[name] # return CFVariable

test/test_scaling.jl

@@ -47,7 +47,7 @@ for sample_data = ( -100:100,
     local io, data, fill_value, mv, md, add_offset, scale_factor
 
     fill_value = sample_data[1]
-    data = rand(sample_data[2:end],30,31)
+    data = rand(sample_data[2:end],3,4)
 
     md = MemoryDataset()
     CDM.defDim(md,"lon",size(data,1))
@@ -56,7 +56,7 @@ for sample_data = ( -100:100,
     scale_factor = 10
 
     mv = CDM.defVar(md,"data",eltype(data),("lon","lat"), attrib =
-        OrderedDict{String,Any}(
+        OrderedDict(
             "_FillValue" => fill_value,
             "add_offset" => add_offset,
             "scale_factor" => scale_factor,
@@ -89,13 +89,23 @@ for sample_data = ( -100:100,
         mv[3,3] = 'y'
         @test mv.var[3,3] == 'y'
     end
+
+    # defVar(ds,name,data,dimnames)
+
+    data2 = replace(data,fill_value => missing)
+    mv = CDM.defVar(md,"data2",data2,("lon","lat"), attrib =
+        OrderedDict(
+            "_FillValue" => fill_value
+        ))
+
+    @test all(mv[:,:] .=== data2)
 end
 
 
 # time
 
 sample_data =  -100:100
-data = rand(sample_data,30,31)
+data = rand(sample_data,3,4)
 
 md = MemoryDataset()
 CDM.defDim(md,"lon",size(data,1))
@@ -128,6 +138,4 @@ md["data"][1,2] = DateTime(2000,2,1)
 
 
 @test CDM.dataset(md["data"]) == md
-
-
 close(md)