Merge pull request #997 from JuliaRobotics/23Q1/maint/packvar_updates

Packed[Variable/Factor] as a DFGType and nstime as string

Author: Affie

docs/src/variable_factor_serialization.md

@@ -26,7 +26,7 @@ prior = addFactor!(dfg, [:x0], PriorPose2( MvNormal([10; 10; 1.0/8.0], Matrix(Di
 
 # Now serialize them:
 pVariable = packVariable(dfg, x0)
-pFactor = packFactor(dfg, prior)
+pFactor = packFactor(prior)
 
 # And we can deserialize them
 upVariable = unpackVariable(dfg, pVariable)
@@ -60,7 +60,7 @@ for v in getVariables(dfg)
 end
 # Factors
 for f in getFactors(dfg)
-    fPacked = packFactor(dfg, f)
+    fPacked = packFactor(f)
     io = open("$folder/factors/$(f.label).json", "w")
     JSON3.write(io, fPacked)
     close(io)

src/DataBlobs/services/HelpersDataWrapEntryBlob.jl

@@ -156,6 +156,7 @@ addData!(
     kwargs...
 )
 
+#FIXME id used wrong
 function addData!(
     dfg::AbstractDFG, 
     blobstore::AbstractBlobStore, 

src/Deprecated.jl

@@ -2,7 +2,7 @@
 ## ================================================================================
 ## Remove in v0.21
 ##=================================================================================
-
+@deprecate packFactor(dfg::AbstractDFG, f::DFGFactor) packFactor(f::DFGFactor)
 
 # #TODO check this one
 # function addData!(

src/DistributedFactorGraphs.jl

@@ -173,6 +173,7 @@ export getPPEDict, getVariablePPEDict, getVariablePPE
 
 # CRUD & SET
 export getPPE,
+       getPPEs,
        getVariablePPE,
        addPPE!,
        updatePPE!,

src/entities/DFGFactor.jl

@@ -83,13 +83,13 @@ FunctionNodeData(args...; kw...) = FunctionNodeData{typeof(args[4])}(args...; kw
 # | DFGFactor         |   X   |   X  |     X     |     X    |      X     |
 
 # Packed Factor
-Base.@kwdef struct PackedFactor
+Base.@kwdef struct PackedFactor <: AbstractDFGFactor
     id::Union{UUID, Nothing} = nothing
     label::Symbol
     tags::Vector{Symbol}
     _variableOrderSymbols::Vector{Symbol}
     timestamp::ZonedDateTime
-    nstime::Int
+    nstime::String
     fnctype::String
     solvable::Int
     data::String
@@ -280,8 +280,8 @@ StructTypes.omitempties(::Type{SkeletonDFGFactor}) = (:id,)
 ##==============================================================================
 ## Define factor levels
 ##==============================================================================
-const FactorDataLevel0 = Union{DFGFactor, DFGFactorSummary, SkeletonDFGFactor}
-const FactorDataLevel1 = Union{DFGFactor, DFGFactorSummary}
+const FactorDataLevel0 = Union{DFGFactor, DFGFactorSummary, PackedFactor, SkeletonDFGFactor}
+const FactorDataLevel1 = Union{DFGFactor, DFGFactorSummary, PackedFactor}
 const FactorDataLevel2 = Union{DFGFactor}
 
 ##==============================================================================

src/entities/DFGVariable.jl

@@ -178,12 +178,12 @@ getEstimateFields(::MeanMaxPPE) = [:suggested, :max, :mean]
 ##==============================================================================
 
 # Packed Variable
-Base.@kwdef struct PackedVariable
+Base.@kwdef struct PackedVariable  <: AbstractDFGVariable
     id::Union{UUID, Nothing} = nothing
     label::Symbol
     tags::Vector{Symbol} = Symbol[]
     timestamp::ZonedDateTime = now(tz"UTC")
-    nstime::Int = 0
+    nstime::String = "0"
     ppes::Vector{MeanMaxPPE} = MeanMaxPPE[]
     blobEntries::Vector{BlobEntry} = BlobEntry[]
     variableType::String
@@ -193,6 +193,33 @@ Base.@kwdef struct PackedVariable
     solverData::Vector{PackedVariableNodeData} = PackedVariableNodeData[]
 end
 
+#IIF like contruction helper for packed variable
+function PackedVariable(
+    label::Symbol,
+    variableType::String;
+    tags::Vector{Symbol} = Symbol[],
+    timestamp::ZonedDateTime = now(tz"UTC"),
+    solvable::Int = 1,
+    nanosecondtime::Int64 = 0,
+    smalldata::Dict{Symbol, SmallDataTypes} = Dict{Symbol, SmallDataTypes}(),
+    kwargs...
+)
+    union!(tags, [:VARIABLE])
+
+    pacvar = PackedVariable(;
+        label,
+        variableType,
+        nstime = string(nanosecondtime),
+        solvable,
+        tags,
+        metadata = base64encode(JSON3.write(smalldata)),
+        timestamp,
+        kwargs...
+    )
+
+    return pacvar
+end
+
 StructTypes.StructType(::Type{PackedVariable}) = StructTypes.UnorderedStruct()
 StructTypes.idproperty(::Type{PackedVariable}) = :id
 StructTypes.omitempties(::Type{PackedVariable}) = (:id,)
@@ -265,7 +292,7 @@ DFGVariable(label::Symbol,
             solverDataDict::Dict{Symbol, VariableNodeData{T,P}}=Dict{Symbol, VariableNodeData{T,getPointType(T)}}(),
             kw...) where {T <: InferenceVariable, P} = DFGVariable(label, T; solverDataDict=solverDataDict, kw...)
 #
-@deprecate DFGVariable(label::Symbol, T_::Type{<:InferenceVariable},w...; timestamp::DateTime=now(),kw...) DFGVariable(label, T_, w...; timestamp=ZonedDateTime(timestamp), kw...)
+# @deprecate DFGVariable(label::Symbol, T_::Type{<:InferenceVariable},w...; timestamp::DateTime=now(),kw...) DFGVariable(label, T_, w...; timestamp=ZonedDateTime(timestamp), kw...)
 #
 
 
@@ -397,8 +424,8 @@ StructTypes.omitempties(::Type{SkeletonDFGVariable}) = (:id,)
 ##==============================================================================
 # Define variable levels
 ##==============================================================================
-const VariableDataLevel0 = Union{DFGVariable, DFGVariableSummary, SkeletonDFGVariable}
-const VariableDataLevel1 = Union{DFGVariable, DFGVariableSummary}
+const VariableDataLevel0 = Union{DFGVariable, DFGVariableSummary, PackedVariable, SkeletonDFGVariable}
+const VariableDataLevel1 = Union{DFGVariable, DFGVariableSummary, PackedVariable}
 const VariableDataLevel2 = Union{DFGVariable}
 
 ##==============================================================================

src/services/AbstractDFG.jl

@@ -952,7 +952,7 @@ function copyGraph!(destDFG::AbstractDFG,
     # And then all factors to the destDFG.
     for factor in sourceFactors
         # Get the original factor variables (we need them to create it)
-        sourceFactorVariableIds = getNeighbors(sourceDFG, factor)
+        sourceFactorVariableIds = collect(factor._variableOrderSymbols)
         # Find the labels and associated variables in our new subgraph
         factVariableIds = Symbol[]
         for variable in sourceFactorVariableIds

src/services/DFGFactor.jl

@@ -86,6 +86,9 @@ setTimestamp(f::DFGFactor, ts::ZonedDateTime) = DFGFactor(f.label, ts, f.nstime,
 setTimestamp(f::DFGFactorSummary, ts::ZonedDateTime) = DFGFactorSummary(f.id, f.label, f.tags, f._variableOrderSymbols, ts)
 setTimestamp(f::DFGFactorSummary, ts::DateTime) = DFGFactorSummary(f, ZonedDateTime(ts, localzone()))
 
+function setTimestamp(v::PackedFactor, timestamp::ZonedDateTime)
+  return PackedFactor(;(key => getproperty(v, key) for key in fieldnames(PackedFactor))..., timestamp)
+end
 
 ##------------------------------------------------------------------------------
 ## solvable

src/services/DFGVariable.jl

@@ -344,6 +344,9 @@ function setTimestamp(v::DFGVariableSummary, ts::ZonedDateTime; verbose::Bool=tr
     return DFGVariableSummary(v.id, v.label, ts, v.tags, v.ppeDict, v.variableTypeName, v.dataDict)
 end
 
+function setTimestamp(v::PackedVariable, timestamp::ZonedDateTime; verbose::Bool=true)
+    return PackedVariable(;(key => getproperty(v, key) for key in fieldnames(PackedVariable))..., timestamp)
+end
 
 ##------------------------------------------------------------------------------
 ## solvable
@@ -386,6 +389,13 @@ function getPPE(vari::VariableDataLevel1, solveKey::Symbol=:default)
     # return haskey(ppeDict, solveKey) ? ppeDict[solveKey] : nothing
 end
 
+"""
+    $SIGNATURES
+
+Get all the parametric point estimate (PPE) for a variable in the factor graph.
+"""
+function getPPEs end
+
 # afew more aliases on PPE, brought back from deprecated DF
 
 """

src/services/Serialization.jl

@@ -6,10 +6,16 @@ export _packSolverData
 const TYPEKEY = "_type"
 
 ## Version checking
-# FIXME return VersionNumber
+#NOTE fixed really bad function but kept similar as fallback #TODO upgrade to use pkgversion(m::Module)
 function _getDFGVersion()
-    if haskey(Pkg.dependencies(), Base.UUID("b5cc3c7e-6572-11e9-2517-99fb8daf2f04"))
-        return string(Pkg.dependencies()[Base.UUID("b5cc3c7e-6572-11e9-2517-99fb8daf2f04")].version) |> VersionNumber
+
+    pkgorigin = get(Base.pkgorigins, Base.PkgId(DistributedFactorGraphs), nothing) 
+    if !isnothing(pkgorigin) 
+        return pkgorigin.version
+    end
+    dep = get(Pkg.dependencies(), Base.UUID("b5cc3c7e-6572-11e9-2517-99fb8daf2f04"), nothing)
+    if !isnothing(dep)
+        return dep.version
     else
         # This is arguably slower, but needed for Travis.
         return Pkg.TOML.parse(read(joinpath(dirname(pathof(@__MODULE__)), "..", "Project.toml"), String))["version"] |> VersionNumber
@@ -18,7 +24,7 @@ end
 
 function _versionCheck(node::Union{<:PackedVariable, <:PackedFactor})
     if VersionNumber(node._version) < _getDFGVersion()
-        @warn "This data was serialized using DFG $(props["_version"]) but you have $(_getDFGVersion()) installed, there may be deserialization issues." maxlog=10
+        @warn "This data was serialized using DFG $(node._version) but you have $(_getDFGVersion()) installed, there may be deserialization issues." maxlog=10
     end
 end
 
@@ -151,7 +157,7 @@ function packVariable(v::AbstractDFGVariable; includePPEs::Bool=true, includeSol
         id=v.id,
         label = v.label,
         timestamp = v.timestamp,
-        nstime = v.nstime.value,
+        nstime = string(v.nstime.value),
         tags = collect(v.tags), # Symbol.()
         ppes = collect(values(v.ppeDict)),
         solverData = packVariableNodeData.(collect(values(v.solverDataDict))),
@@ -216,15 +222,15 @@ function _packSolverData(
 end
 
 # returns PackedFactor
-function packFactor(dfg::AbstractDFG, f::DFGFactor)
+function packFactor(f::DFGFactor)
     fnctype = getSolverData(f).fnc.usrfnc!
     return PackedFactor(;    
         id = f.id,
         label = f.label,
         tags = collect(f.tags),
         _variableOrderSymbols = f._variableOrderSymbols,
         timestamp = f.timestamp,
-        nstime = f.nstime.value,
+        nstime = string(f.nstime.value),
         fnctype = String(_getname(fnctype)),
         solvable = getSolvable(f),
         metadata = base64encode(JSON3.write(f.smallData)),