Merge pull request #893 from JuliaRobotics/22Q3/enh/dsrzvargql

deseriaize variable on dicts or vectors

Author: dehann

Project.toml

@@ -17,6 +17,7 @@ ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb"
 Neo4j = "d2adbeaf-5838-5367-8a2f-e46d570981db"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"

src/DistributedFactorGraphs.jl

@@ -36,6 +36,9 @@ import ManifoldsBase
 import ManifoldsBase: AbstractManifold, manifold_dimension
 export AbstractManifold, manifold_dimension
 
+import RecursiveArrayTools: ArrayPartition
+export ArrayPartition
+
 import Base: getindex
 
 

src/Neo4jDFG/Neo4jDFG.jl

@@ -87,7 +87,9 @@ import ...DistributedFactorGraphs:  setSolverParams!,
                                     addDataEntry!,
                                     updateDataEntry!,
                                     deleteDataEntry!,
-                                    _getDFGVersion
+                                    _getDFGVersion,
+                                    _unpackPPE,
+                                    _unpackVariableNodeData
 
 using Neo4j
 using Base64

src/Neo4jDFG/services/Neo4jDFG.jl

@@ -530,32 +530,13 @@ end
 
 ### PPE CRUD
 
-"""
-$(SIGNATURES)
-Unpack a Dict{String, Any} into a PPE.
-"""
-function _unpackPPE(dfg::G, packedPPE::Dict{String, Any})::AbstractPointParametricEst where G <: AbstractDFG
-    # Cleanup Zoned timestamp, which is always UTC
-    if packedPPE["lastUpdatedTimestamp"][end] == 'Z'
-        packedPPE["lastUpdatedTimestamp"] = packedPPE["lastUpdatedTimestamp"][1:end-1]
-    end
-
-    !haskey(packedPPE, "_type") && error("Cannot find type key '_type' in packed PPE data")
-    type = pop!(packedPPE, "_type")
-    (type === nothing || type == "") && error("Cannot deserialize PPE, type key is empty")
-    ppe = Unmarshal.unmarshal(
-            DistributedFactorGraphs.getTypeFromSerializationModule(dfg, Symbol(type)),
-            packedPPE)
-    return ppe
-end
-
 function listPPEs(dfg::Neo4jDFG, variablekey::Symbol; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::Vector{Symbol}
     return _listVarSubnodesForType(dfg, variablekey, MeanMaxPPE, "solveKey"; currentTransaction=currentTransaction)
 end
 
 function getPPE(dfg::Neo4jDFG, variablekey::Symbol, ppekey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::AbstractPointParametricEst
     properties = _getVarSubnodeProperties(dfg, variablekey, MeanMaxPPE, ppekey; currentTransaction=currentTransaction)
-    return _unpackPPE(dfg, properties)
+    return _unpackPPE(properties)
 end
 
 function _generateAdditionalProperties(variableType::ST, ppe::P)::Dict{String, String} where {P <: AbstractPointParametricEst, ST <: InferenceVariable}
@@ -594,7 +575,7 @@ function addPPE!(dfg::Neo4jDFG,
     variableType = getVariableType(dfg, variablekey)
     # Add additional properties for the PPE
     addProps = _generateAdditionalProperties(variableType, ppe)
-    return _unpackPPE(dfg, _matchmergeVariableSubnode!(
+    return _unpackPPE(_matchmergeVariableSubnode!(
         dfg,
         variablekey,
         _getLabelsForInst(dfg, ppe, parentKey=variablekey),
@@ -617,7 +598,7 @@ function updatePPE!(
     variableType = getVariableType(dfg, variablekey, currentTransaction=currentTransaction)
     # Add additional properties for the PPE
     addProps = _generateAdditionalProperties(variableType, ppe)
-    return _unpackPPE(dfg, _matchmergeVariableSubnode!(
+    return _unpackPPE(_matchmergeVariableSubnode!(
         dfg,
         variablekey,
         _getLabelsForInst(dfg, ppe, parentKey=variablekey),
@@ -649,7 +630,7 @@ function deletePPE!(dfg::Neo4jDFG, variablekey::Symbol, ppekey::Symbol=:default;
         _getLabelsForType(dfg, MeanMaxPPE, parentKey=variablekey),
         ppekey,
         currentTransaction=currentTransaction)
-    return _unpackPPE(dfg, props)
+    return _unpackPPE(props)
 end
 
 ## DataEntry CRUD
@@ -747,15 +728,6 @@ end
 
 ## VariableSolverData CRUD
 
-"""
-$(SIGNATURES)
-Unpack a Dict{String, Any} into a PPE.
-"""
-function _unpackVariableNodeData(dfg::G, packedDict::Dict{String, Any})::VariableNodeData where G <: AbstractDFG
-    packedVND = Unmarshal.unmarshal(PackedVariableNodeData, packedDict)
-    return unpackVariableNodeData(dfg, packedVND)
-end
-
 function listVariableSolverData(dfg::Neo4jDFG, variablekey::Symbol; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::Vector{Symbol}
     return _listVarSubnodesForType(dfg, variablekey, VariableNodeData, "solveKey"; currentTransaction=currentTransaction)
 end

src/entities/DFGVariable.jl

@@ -235,13 +235,13 @@ abstract type AbstractPointParametricEst end
 
 Data container to store Parameteric Point Estimate (PPE) for mean and max.
 """
-struct MeanMaxPPE <: AbstractPointParametricEst
+Base.@kwdef struct MeanMaxPPE <: AbstractPointParametricEst
     # repeat key value internally (from a design request by Sam)
     solveKey::Symbol
     suggested::Vector{Float64}
     max::Vector{Float64}
     mean::Vector{Float64}
-    lastUpdatedTimestamp::DateTime
+    lastUpdatedTimestamp::DateTime = now(Dates.UTC)
 end
 
 ##------------------------------------------------------------------------------

src/services/DFGVariable.jl

@@ -86,7 +86,7 @@ See documentation in [Manifolds.jl on making your own](https://juliamanifolds.gi
 
 Example:
 ```
-DFG.@defVariable Pose2 SpecialEuclidean(2) ProductRepr([0;0.0],[1 0; 0 1.0])
+DFG.@defVariable Pose2 SpecialEuclidean(2) ArrayPartition([0;0.0],[1 0; 0 1.0])
 ```
 """
 macro defVariable(structname, manifold, point_identity)

src/services/Serialization.jl

@@ -79,10 +79,11 @@ end
 
 typeModuleName(varT::Type{<:InferenceVariable}) = typeModuleName(varT())
 
-function getTypeFromSerializationModule(variableTypeString::String)
+function getTypeFromSerializationModule(_typeString::AbstractString)
+    @debug "DFG converting type string to Julia type" _typeString
     try
         # split the type at last `.`
-        split_st = split(variableTypeString, r"\.(?!.*\.)")
+        split_st = split(_typeString, r"\.(?!.*\.)")
         #if module is specified look for the module in main, otherwise use Main        
         if length(split_st) == 2
             m = getfield(Main, Symbol(split_st[1]))
@@ -102,7 +103,7 @@ function getTypeFromSerializationModule(variableTypeString::String)
         return ret 
 
     catch ex
-        @error "Unable to deserialize soft type $(variableTypeString)"
+        @error "Unable to deserialize type $(_typeString)"
         io = IOBuffer()
         showerror(io, ex, catch_backtrace())
         err = String(take!(io))
@@ -111,6 +112,25 @@ function getTypeFromSerializationModule(variableTypeString::String)
     nothing
 end
 
+"""
+  $(SIGNATURES)
+Get a type from the serialization module inside DFG.
+"""
+function getTypeFromSerializationModule(dfg::G, moduleType::Symbol) where G <: AbstractDFG
+    @warn "Deprecating getTypeFromSerializationModule(dfg,symbol), use getTypeFromSerializationModule(string) instead." maxlog=10
+    st = nothing
+    try
+        st = getfield(Main, Symbol(moduleType))
+    catch ex
+        @error "Unable to deserialize packed variableType $(moduleType)"
+        io = IOBuffer()
+        showerror(io, ex, catch_backtrace())
+        err = String(take!(io))
+        @error(err)
+    end
+    return st
+end
+
 ##==============================================================================
 ## Variable Packing and unpacking
 ##==============================================================================
@@ -131,12 +151,69 @@ function packVariable(dfg::AbstractDFG, v::DFGVariable)
     return props::Dict{String, Any}
 end
 
+"""
+$(SIGNATURES)
+
+Common unpack a Dict{String, Any} into a PPE.
+"""
+function _unpackPPE(
+        packedPPE::Dict{String, Any};
+        _type = pop!(packedPPE, "_type") # required for generic use
+    )
+    #
+    # Cleanup Zoned timestamp, which is always UTC
+    if packedPPE["lastUpdatedTimestamp"][end] == 'Z'
+        packedPPE["lastUpdatedTimestamp"] = packedPPE["lastUpdatedTimestamp"][1:end-1]
+    end
+
+    # !haskey(packedPPE, "_type") && error("Cannot find type key '_type' in packed PPE data")
+    if (_type === nothing || _type == "")
+        @warn "Cannot deserialize PPE, unknown type key, trying DistributedFactorGraphs.MeanMaxPPE" _type
+        _type = "DistributedFactorGraphs.MeanMaxPPE"
+    end
+    ppeType = getTypeFromSerializationModule(_type)
+    
+    ppe = Unmarshal.unmarshal(
+        ppeType,
+        packedPPE
+    )
+    # _pk = Symbol(packedPPE["solveKey"])
+    # ppe = MeanMaxPPE(;
+    #             solveKey=_pk,
+    #             suggested=float.(pd["suggested"]),
+    #             max=float.(pd["max"]),
+    #             mean=float.(pd["mean"]),
+    #             lastUpdatedTimestamp=DateTime(string(pd["lastUpdatedTimestamp"]))
+    #         )
+
+    return ppe
+end
+
+"""
+$(SIGNATURES)
+
+Unpack a Dict{String, Any} into a PPE.
+
+Notes:
+- returns `::VariableNodeData`
+"""
+function _unpackVariableNodeData(
+        dfg::AbstractDFG, 
+        packedDict::Dict{String, Any}
+    )
+    #
+    packedVND = Unmarshal.unmarshal(PackedVariableNodeData, packedDict)
+    return unpackVariableNodeData(dfg, packedVND)
+end
+
 # returns a DFGVariable
-function unpackVariable(dfg::G,
+function unpackVariable(dfg::AbstractDFG,
                         packedProps::Dict{String, Any};
                         unpackPPEs::Bool=true,
                         unpackSolverData::Bool=true,
-                        unpackBigData::Bool=true) where G <: AbstractDFG
+                        unpackBigData::Bool = haskey(packedProps,"dataEntryType") && haskey(packedProps, "dataEntry")
+    )
+    #
     @debug "Unpacking variable:\r\n$packedProps"
     # Version checking.
     _versionCheck(packedProps)
@@ -146,32 +223,51 @@ function unpackVariable(dfg::G,
     # Parse it
     timestamp = ZonedDateTime(packedProps["timestamp"])
     nstime = Nanosecond(get(packedProps, "nstime", 0))
-    # Supporting string serialization using packVariable and CGDFG serialization (Vector{String})
-    if packedProps["tags"] isa String
-        tags = JSON2.read(packedProps["tags"], Vector{Symbol})
+
+    # FIXME, drop nested packing, see DFG #867
+    #   string serialization using packVariable and CGDFG serialization (Vector{String})
+    tags = if packedProps["tags"] isa String
+        JSON2.read(packedProps["tags"], Vector{Symbol})
     else
-        tags = Symbol.(packedProps["tags"])
+        Symbol.(packedProps["tags"])
     end
-    ppeDict = unpackPPEs ? JSON2.read(packedProps["ppeDict"], Dict{Symbol, MeanMaxPPE}) : Dict{Symbol, MeanMaxPPE}()
-    smallData = JSON2.read(packedProps["smallData"], Dict{Symbol, SmallDataTypes})
 
-    variableTypeString = if haskey(packedProps, "softtype")
-        # TODO Deprecate, remove in v0.12
-        @warn "Packed field `softtype` is deprecated and replaced with `variableType`"
-        packedProps["softtype"]
+    # FIXME, drop nested packing, see DFG #867
+    ppeDict = if unpackPPEs && haskey(packedProps,"ppesDict")
+        JSON2.read(packedProps["ppeDict"], Dict{Symbol, MeanMaxPPE})
+    elseif unpackPPEs && haskey(packedProps,"ppes") && packedProps["ppes"] isa AbstractVector
+        # these different cases are not well covered in tests, but first fix #867
+        # TODO dont hardcode the ppeType (which is already discovered for each entry in _updatePPE)
+        ppedict = Dict{Symbol, MeanMaxPPE}()
+        for pd in packedProps["ppes"]
+            _type = get(pd, "_type", "DistributedFactorGraphs.MeanMaxPPE")
+            ppedict[Symbol(pd["solveKey"])] = _unpackPPE(pd; _type)
+        end
+        ppedict
     else
-        packedProps["variableType"]
+        Dict{Symbol, MeanMaxPPE}()
     end
 
+    smallData = JSON2.read(packedProps["smallData"], Dict{Symbol, SmallDataTypes})
+
+    variableTypeString = packedProps["variableType"]
+
     variableType = getTypeFromSerializationModule(variableTypeString)
     isnothing(variableType) && error("Cannot deserialize variableType '$variableTypeString' in variable '$label'")
     pointType = getPointType(variableType)
 
-    if unpackSolverData
+    # FIXME, drop nested packing, see DFG #867
+    solverData = if unpackSolverData && haskey(packedProps, "solverDataDict")
         packed = JSON2.read(packedProps["solverDataDict"], Dict{String, PackedVariableNodeData})
-        solverData = Dict{Symbol, VariableNodeData{variableType, pointType}}(Symbol.(keys(packed)) .=> map(p -> unpackVariableNodeData(dfg, p), values(packed)))
+        Dict{Symbol, VariableNodeData{variableType, pointType}}(Symbol.(keys(packed)) .=> map(p -> unpackVariableNodeData(dfg, p), values(packed)))
+    elseif unpackPPEs && haskey(packedProps,"solverData") && packedProps["solverData"] isa AbstractVector
+        solverdict = Dict{Symbol, VariableNodeData{variableType, pointType}}()
+        for sd in packedProps["solverData"]
+            solverdict[Symbol(sd["solveKey"])] = _unpackVariableNodeData(dfg, sd)
+        end
+        solverdict
     else
-        solverData = Dict{Symbol, VariableNodeData{variableType, pointType}}()
+        Dict{Symbol, VariableNodeData{variableType, pointType}}()
     end
     # Rebuild DFGVariable using the first solver variableType in solverData
     # @info "dbg Serialization 171" variableType Symbol(packedProps["label"]) timestamp nstime ppeDict solverData smallData Dict{Symbol,AbstractDataEntry}() Ref(packedProps["solvable"])
@@ -191,23 +287,12 @@ function unpackVariable(dfg::G,
     # Now rehydrate complete DataEntry type.
     if unpackBigData
         #TODO Deprecate - for backward compatibility between v0.8 and v0.9, remove in v0.10
-        if haskey(packedProps, "bigDataElemType")
-            @warn "`bigDataElemType` is deprecate, please save data again with new version that uses `dataEntryType`"
-            dataElemTypes = JSON2.read(packedProps["bigDataElemType"], Dict{Symbol, Symbol})
-        else
-            dataElemTypes = JSON2.read(packedProps["dataEntryType"], Dict{Symbol, Symbol})
-            for (k,name) in dataElemTypes 
-                dataElemTypes[k] = Symbol(split(string(name), '.')[end])
-            end
+        dataElemTypes = JSON2.read(packedProps["dataEntryType"], Dict{Symbol, Symbol})
+        for (k,name) in dataElemTypes 
+            dataElemTypes[k] = Symbol(split(string(name), '.')[end])
         end
 
-        #TODO Deprecate - for backward compatibility between v0.8 and v0.9, remove in v0.10
-        if haskey(packedProps, "bigData")
-            @warn "`bigData` is deprecate, please save data again with new version"
-            dataIntermed = JSON2.read(packedProps["bigData"], Dict{Symbol, String})
-        else
-            dataIntermed = JSON2.read(packedProps["dataEntry"], Dict{Symbol, String})
-        end
+        dataIntermed = JSON2.read(packedProps["dataEntry"], Dict{Symbol, String})
 
         for (k,bdeInter) in dataIntermed
             interm = JSON.parse(bdeInter)
@@ -452,20 +537,4 @@ end
 ## Serialization
 ##==============================================================================
 
-"""
-  $(SIGNATURES)
-Get a type from the serialization module inside DFG.
-"""
-function getTypeFromSerializationModule(dfg::G, moduleType::Symbol) where G <: AbstractDFG
-    st = nothing
-    try
-        st = getfield(Main, Symbol(moduleType))
-    catch ex
-        @error "Unable to deserialize packed variableType $(moduleType)"
-        io = IOBuffer()
-        showerror(io, ex, catch_backtrace())
-        err = String(take!(io))
-        @error(err)
-    end
-    return st
-end
+