Merge pull request #576 from JuliaRobotics/feat/20Q3/small_CRUD

Small Data Types and in-memory CRUD

Author: Affie

attic/InMemoryDataStore.jl

@@ -0,0 +1,61 @@
+"""
+    $(TYPEDEF)
+Simple in-memory data store with a specified data type and a specified key type.
+"""
+struct InMemoryDataStore{T, E <: AbstractDataEntry} <: AbstractDataStore{T}
+    data::Dict{Symbol, T}
+    entries::Dict{Symbol, E}
+end
+
+"""
+    $(SIGNATURES)
+Create an in-memory store using a specific data type.
+"""
+function InMemoryDataStore{T, E}() where {T, E <: AbstractDataEntry}
+    return InMemoryDataStore{T, E}(Dict{Symbol, T}(), Dict{Symbol, E}())
+end
+
+"""
+    $(SIGNATURES)
+Create an in-memory store using binary data (UInt8) as a type.
+"""
+function InMemoryDataStore()
+    return InMemoryDataStore{Vector{UInt8}, GeneralDataEntry}()
+end
+
+
+##==============================================================================
+## InMemoryDataStore CRUD
+##==============================================================================
+function getDataBlob(store::InMemoryDataStore{T, E}, entry::E)::Union{T, Nothing} where {T, E <: AbstractDataEntry}
+    !haskey(store.data, entry.storeKey) && return nothing
+    return store.data[entry.storeKey]
+end
+
+function addDataBlob!(store::InMemoryDataStore{T, E}, entry::E, data::T)::T where {T, E <: AbstractDataEntry}
+    haskey(store.entries, entry.storeKey) && @warn "Key '$(entry.storeKey)' already exists in the data store, overwriting!"
+    store.entries[entry.storeKey] = entry
+    # Update timestamp
+    entry.lastUpdatedTimestamp = now()
+    return store.data[entry.storeKey] = data
+end
+
+function updateDataBlob!(store::InMemoryDataStore{T, E}, entry::E, data::T)::Union{T, Nothing} where {T, E <: AbstractDataEntry}
+    !haskey(store.entries, entry.storeKey) && (@error "Key '$(entry.storeKey)' doesn't exist in the data store!"; return nothing)
+    store.entries[entry.storeKey] = entry
+    # Update timestamp
+    entry.lastUpdatedTimestamp = now()
+    return store.data[entry.storeKey] = data
+end
+
+function deleteDataBlob!(store::InMemoryDataStore{T, E}, entry::E)::T where {T, E <: AbstractDataEntry}
+    data = getDataBlob(store, entry)
+    data == nothing && return nothing
+    delete!(store.data, entry.storeKey)
+    delete!(store.entries, entry.storeKey)
+    return data
+end
+
+function listDataBlobs(store::InMemoryDataStore{T, E})::Vector{E} where {T, E <: AbstractDataEntry}
+    return collect(values(store.entries))
+end

src/Deprecated.jl

@@ -85,7 +85,6 @@ listDataBlobs(store::AbstractDataStore) = error("AbstractDataStore $(typeof(stor
 
 
 
-
 """
 $(SIGNATURES)
 Add Big Data Entry to distributed factor graph.
@@ -188,4 +187,20 @@ function Base.getproperty(x::InferenceVariable, f::Symbol)
     end
   end
   return getfield(x,f)
-end
+end
+
+
+
+# SmallData is now a Symbol Dict
+function createSymbolDict(d::Dict{String, String})
+  newsmalld = Dict{Symbol,SmallDataTypes}() 
+  for p in pairs(d)
+    push!(newsmalld, Symbol(p.first) => p.second)
+  end
+  return newsmalld
+end
+
+@deprecate setSmallData!(v::DFGVariable, smallData::Dict{String, String}) setSmallData!(v, createSymbolDict(smallData)) 
+
+
+

src/DistributedFactorGraphs.jl

@@ -123,6 +123,8 @@ export getSolverData
 
 export getVariableType
 
+# Small Data CRUD
+export SmallDataTypes, getSmallData, addSmallData!, updateSmallData!, deleteSmallData!, listSmallData, emptySmallData!
 export getSmallData, setSmallData!
 
 

src/entities/DFGVariable.jl

@@ -217,6 +217,8 @@ getEstimateFields(::MeanMaxPPE) = [:suggested, :max, :mean]
 ## DFG Variables
 ##==============================================================================
 
+const SmallDataTypes = Union{Int, Float64, String, Bool, Vector{Int}, Vector{Float64}, Vector{String}, Vector{Bool}}
+
 ##------------------------------------------------------------------------------
 ## DFGVariable lv2
 ##------------------------------------------------------------------------------
@@ -248,7 +250,7 @@ struct DFGVariable{T<:InferenceVariable} <: AbstractDFGVariable
     solverDataDict::Dict{Symbol, VariableNodeData{T}}
     """Dictionary of small data associated with this variable.
     Accessors: [`getSmallData`](@ref), [`setSmallData!`](@ref)"""
-    smallData::Dict{String, String}#Ref{Dict{String, String}} #why was Ref here?
+    smallData::Dict{Symbol, SmallDataTypes}
     """Dictionary of large data associated with this variable.
     Accessors: [`addDataEntry!`](@ref), [`getDataEntry`](@ref), [`updateDataEntry!`](@ref), and [`deleteDataEntry!`](@ref)"""
     dataDict::Dict{Symbol, AbstractDataEntry}
@@ -270,7 +272,7 @@ function DFGVariable(label::Symbol, softtype::T;
             tags::Set{Symbol}=Set{Symbol}(),
             estimateDict::Dict{Symbol, <: AbstractPointParametricEst}=Dict{Symbol, MeanMaxPPE}(),
             solverDataDict::Dict{Symbol, VariableNodeData{T}}=Dict{Symbol, VariableNodeData{T}}(),
-            smallData::Dict{String, String}=Dict{String, String}(),
+            smallData::Dict{Symbol, SmallDataTypes}=Dict{Symbol, SmallDataTypes}(),
             dataDict::Dict{Symbol, AbstractDataEntry}=Dict{Symbol,AbstractDataEntry}(),
             solvable::Int=1) where {T <: InferenceVariable}
 
@@ -287,7 +289,7 @@ function DFGVariable(label::Symbol,
             nstime::Nanosecond = Nanosecond(0),
             tags::Set{Symbol}=Set{Symbol}(),
             estimateDict::Dict{Symbol, <: AbstractPointParametricEst}=Dict{Symbol, MeanMaxPPE}(),
-            smallData::Dict{String, String}=Dict{String, String}(),
+            smallData::Dict{Symbol, SmallDataTypes}=Dict{Symbol, SmallDataTypes}(),
             dataDict::Dict{Symbol, AbstractDataEntry}=Dict{Symbol,AbstractDataEntry}(),
             solvable::Int=1) where {T <: InferenceVariable}
     if timestamp isa DateTime

src/services/AbstractDFG.jl

@@ -166,7 +166,6 @@ end
 ##==============================================================================
 
 #NOTE with API standardization this should become something like:
-# JT, I however do not feel we should force it, as I prever dot notation
 getUserData(dfg::AbstractDFG, key::Symbol)::String = dfg.userData[key]
 getRobotData(dfg::AbstractDFG, key::Symbol)::String = dfg.robotData[key]
 getSessionData(dfg::AbstractDFG, key::Symbol)::String = dfg.sessionData[key]

src/services/DFGVariable.jl

@@ -272,19 +272,88 @@ setSolverData!(v::DFGVariable, data::VariableNodeData, key::Symbol=:default) = v
 """
     $(SIGNATURES)
 Get the small data for a variable.
+Note: Rather use SmallData CRUD
 """
-getSmallData(v::DFGVariable)::Dict{String, String} = v.smallData
+getSmallData(v::DFGVariable) = v.smallData
 
 """
     $(SIGNATURES)
 Set the small data for a variable.
 This will overwrite old smallData.
+Note: Rather use SmallData CRUD
 """
-function setSmallData!(v::DFGVariable, smallData::Dict{String, String})::Dict{String, String}
+function setSmallData!(v::DFGVariable, smallData::Dict{Symbol, SmallDataTypes})
     empty!(v.smallData)
     merge!(v.smallData, smallData)
 end
 
+# Generic SmallData CRUD
+# TODO optimize for difference in in-memory by extending in other drivers. 
+
+"""
+    $(SIGNATURES)
+Get the small data entry at `key` for variable `label` in `dfg`
+"""
+function getSmallData(dfg::AbstractDFG, label::Symbol, key::Symbol) 
+    getVariable(dfg, label).smallData[key]
+end
+
+"""
+    $(SIGNATURES)
+Add a small data pair `key=>value` for variable `label` in `dfg`
+"""
+function addSmallData!(dfg::AbstractDFG, label::Symbol, pair::Pair{Symbol, <:SmallDataTypes})
+    v = getVariable(dfg, label)
+    haskey(v.smallData, pair.first) && error("$(pair.first) already exists.")
+    push!(v.smallData, pair)
+    updateVariable!(dfg, v)
+    return v.smallData #or pair TODO
+end
+
+"""
+    $(SIGNATURES)
+Update a small data pair `key=>value` for variable `label` in `dfg`
+"""
+function updateSmallData!(dfg::AbstractDFG, label::Symbol, pair::Pair{Symbol, <:SmallDataTypes})
+    v = getVariable(dfg, label)
+    !haskey(v.smallData, pair.first) && @warn("$(pair.first) does not exist, adding.")
+    push!(v.smallData, pair)
+    updateVariable!(dfg, v)
+    return v.smallData #or pair TODO
+end
+
+"""
+    $(SIGNATURES)
+Delete a small data entry at `key` for variable `label` in `dfg`
+"""
+function deleteSmallData!(dfg::AbstractDFG, label::Symbol, key::Symbol)
+    v = getVariable(dfg, label)
+    rval = pop!(v.smallData, key)
+    updateVariable!(dfg, v)
+    return rval
+end
+
+"""
+    $(SIGNATURES)
+List all small data keys for a variable `label` in `dfg`
+"""
+function listSmallData(dfg::AbstractDFG, label::Symbol)
+    v = getVariable(dfg, label)
+    return collect(keys(v.smallData)) #or pair TODO
+end
+
+"""
+    $(SIGNATURES)
+Empty all small data from variable `label` in `dfg`
+"""
+function emptySmallData!(dfg::AbstractDFG, label::Symbol)
+    v = getVariable(dfg, label)
+    empty!(v.smallData)
+    updateVariable!(dfg, v)
+    return v.smallData #or pair TODO
+end
+
+
 ##------------------------------------------------------------------------------
 ## Data Entries and Blobs
 ##------------------------------------------------------------------------------

src/services/Serialization.jl

@@ -39,7 +39,7 @@ function unpackVariable(dfg::G,
         tags = Symbol.(packedProps["tags"])
     end
     ppeDict = unpackPPEs ? JSON2.read(packedProps["ppeDict"], Dict{Symbol, MeanMaxPPE}) : Dict{Symbol, MeanMaxPPE}()
-    smallData = JSON2.read(packedProps["smallData"], Dict{String, String})
+    smallData = JSON2.read(packedProps["smallData"], Dict{Symbol, SmallDataTypes})
 
     softtypeString = packedProps["softtype"]
     softtype = getTypeFromSerializationModule(dfg, Symbol(softtypeString))

test/interfaceTests.jl

@@ -56,7 +56,7 @@ end
     @test printFactor(fac1) == nothing
 
     @test printVariable(iobuf, var1, skipfields=[:timestamp, :solver, :ppe, :nstime]) == nothing
-    @test String(take!(iobuf)) == "DFGVariable{TestSofttype1}\nlabel:\n:a\ntags:\nSet([:VARIABLE, :POSE])\nsmallData:\nDict(\"small\"=>\"data\")\ndataDict:\nDict{Symbol,AbstractDataEntry}()\nsolvable:\n0\n"
+    @test String(take!(iobuf)) == "DFGVariable{TestSofttype1}\nlabel:\n:a\ntags:\nSet([:VARIABLE, :POSE])\nsmallData:\nDict{Symbol,Union{Bool, Float64, Int64, Array{Bool,1}, Array{Float64,1}, Array{Int64,1}, Array{String,1}, String}}(:small=>\"data\")\ndataDict:\nDict{Symbol,AbstractDataEntry}()\nsolvable:\n0\n"
 
     @test printVariable(iobuf, var1, short=true) == nothing
     @test String(take!(iobuf)) == "DFGVariable{TestSofttype1}\nlabel: a\ntags: Set([:VARIABLE, :POSE])\nsize marginal samples: (1, 1)\nkde bandwidths: [0.0]\nNo PPEs\n"
@@ -108,6 +108,10 @@ end
     VSDTestBlock!(fg1, var1)
 end
 
+@testset "SmallData CRUD" begin
+    smallDataTestBlock!(fg1)
+end
+
 @testset "Data Entries and Blobs" begin
     if typeof(fg1) <: InMemoryDFGTypes
         DataEntriesTestBlock!(fg1, var2)

test/testBlocks.jl

@@ -238,7 +238,7 @@ function DFGVariableSCA()
 
     v1_lbl = :a
     v1_tags = Set([:VARIABLE, :POSE])
-    small = Dict("small"=>"data")
+    small = Dict{Symbol, SmallDataTypes}(:small=>"data")
     testTimestamp = now(localzone())
     # Constructors
     v1 = DFGVariable(v1_lbl, TestSofttype1(), tags=v1_tags, solvable=0, solverDataDict=Dict(:default=>VariableNodeData{TestSofttype1}()))
@@ -262,7 +262,7 @@ function DFGVariableSCA()
 
     @test getPPEDict(v1) == v1.ppeDict
 
-    @test getSmallData(v1) == Dict{String,String}()
+    @test getSmallData(v1) == Dict{Symbol,SmallDataTypes}()
 
     @test getSofttype(v1) == TestSofttype1()
 
@@ -767,6 +767,42 @@ function  VSDTestBlock!(fg, v1)
 
 end
 
+function smallDataTestBlock!(fg)
+
+    @test listSmallData(fg, :a) == Symbol[:small]
+    @test listSmallData(fg, :b) == Symbol[]
+    @test small = getSmallData(fg, :a, :small) == "data"
+
+    @test addSmallData!(fg, :a, :a=>5) == getVariable(fg, :a).smallData
+    @test addSmallData!(fg, :a, :b=>10.0) == getVariable(fg, :a).smallData
+    @test addSmallData!(fg, :a, :c=>true) == getVariable(fg, :a).smallData
+    @test addSmallData!(fg, :a, :d=>"yes") == getVariable(fg, :a).smallData
+    @test addSmallData!(fg, :a, :e=>[1, 2, 3])  == getVariable(fg, :a).smallData
+    @test addSmallData!(fg, :a, :f=>[1.4, 2.5, 3.6]) == getVariable(fg, :a).smallData
+    @test addSmallData!(fg, :a, :g=>["yes", "maybe"]) == getVariable(fg, :a).smallData
+    @test addSmallData!(fg, :a, :h=>[true, false]) == getVariable(fg, :a).smallData
+
+    @test_throws ErrorException addSmallData!(fg, :a, :a=>3)
+    @test updateSmallData!(fg, :a, :a=>3) == getVariable(fg, :a).smallData
+    
+    @test_throws MethodError addSmallData!(fg, :a, :no=>0x01)
+    @test_throws MethodError addSmallData!(fg, :a, :no=>1f0)
+    @test_throws MethodError addSmallData!(fg, :a, :no=>Nanosecond(3))
+    @test_throws MethodError addSmallData!(fg, :a, :no=>[0x01])
+    @test_throws MethodError addSmallData!(fg, :a, :no=>[1f0])
+    @test_throws MethodError addSmallData!(fg, :a, :no=>[Nanosecond(3)])
+    
+    @test deleteSmallData!(fg, :a, :a) == 3
+    @test updateSmallData!(fg, :a, :a=>3) == getVariable(fg, :a).smallData
+    @test length(listSmallData(fg, :a)) == 9
+    emptySmallData!(fg, :a)
+    @test length(listSmallData(fg, :a)) == 0
+
+end
+
+
+
+
 function  DataEntriesTestBlock!(fg, v2)
     # "Data Entries"