More metadata for FileDFG and restore as saved (#1038)

* More metadata support for file dfg
* getBlobEntryFirst for Variable

Author: Affie

src/DataBlobs/services/BlobEntry.jl

@@ -114,6 +114,14 @@ function getBlobEntryFirst(var::AbstractDFGVariable, key::Regex)
     )
 end
 
+function getBlobEntryFirst(var::Variable, key::Regex)   
+    firstIdx = findfirst(x->contains(string(x.label), key), var.blobEntries)
+    if isnothing(firstIdx)
+        throw(KeyError("$key"))
+    end
+    return var.blobEntries[firstIdx]
+end
+
 getBlobEntry(var::AbstractDFGVariable, key::AbstractString) = getBlobEntry(var,Regex(key))
 
 #TODO split betweeen getfirstBlobEntry and getBlobEntry

src/DistributedFactorGraphs.jl

@@ -347,6 +347,12 @@ include("entities/AbstractDFGSummary.jl")
 
 include("services/AbstractDFG.jl")
 
+#Blobs
+include("DataBlobs/services/BlobEntry.jl")
+include("DataBlobs/services/BlobStores.jl")
+include("DataBlobs/services/BlobPacking.jl")
+include("DataBlobs/services/HelpersDataWrapEntryBlob.jl")
+
 # In Memory Types
 include("GraphsDFG/GraphsDFG.jl")
 @reexport using .GraphsDFGs
@@ -362,11 +368,6 @@ include("services/DFGVariable.jl")
 include("services/DFGFactor.jl")
 include("Deprecated.jl")
 include("services/CompareUtils.jl")
-#Blobs
-include("DataBlobs/services/BlobEntry.jl")
-include("DataBlobs/services/BlobStores.jl")
-include("DataBlobs/services/BlobPacking.jl")
-include("DataBlobs/services/HelpersDataWrapEntryBlob.jl")
 
 # Include the FilesDFG API.
 include("FileDFG/FileDFG.jl")

src/FileDFG/services/FileDFG.jl

@@ -149,7 +149,6 @@ function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString; overwriteDFGMet
     end
 
     # extract the factor graph from fileDFG folder
-    variables = DFGVariable[]
     factors = DFGFactor[]
     varFolder = "$folder/variables"
     factorFolder = "$folder/factors"
@@ -160,43 +159,62 @@ function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString; overwriteDFGMet
 
     varFiles = sort(readdir(varFolder; sort=false); lt=natural_lt)
     factorFiles = sort(readdir(factorFolder; sort=false); lt=natural_lt)
-    @showprogress 1 "loading variables" for varFile in varFiles
-        jstr = read("$varFolder/$varFile", String)
-        try
-            packedData = JSON3.read(jstr, PackedVariable)
-            push!(variables, unpackVariable(packedData))
-        catch ex
-            @error("JSON3 is having trouble reading $varFolder/$varFile into a PackedVariable")
-            @show jstr
-            throw(ex)
+
+    if isa(dfgLoadInto, GraphsDFG) && GraphsDFGs._variablestype(dfgLoadInto) == Variable
+        variables = @showprogress 1 "loading variables" map(varFiles) do varFile
+            jstr = read("$varFolder/$varFile", String)
+            return JSON3.read(jstr, PackedVariable)
+        end
+    else
+        variables = DFGVariable[]
+        @showprogress 1 "loading variables" for varFile in varFiles
+            jstr = read("$varFolder/$varFile", String)
+            try
+                packedData = JSON3.read(jstr, PackedVariable)
+                push!(variables, unpackVariable(packedData))
+            catch ex
+                @error("JSON3 is having trouble reading $varFolder/$varFile into a PackedVariable")
+                @show jstr
+                throw(ex)
+            end
         end
     end
-    @info "Loaded $(length(variables)) variables - $(map(v->v.label, variables))"
+
+    @info "Loaded $(length(variables)) variables"#- $(map(v->v.label, variables))"
     @info "Inserting variables into graph..."
     # Adding variables
     map(v->addVariable!(dfgLoadInto, v), variables)
 
-    @showprogress 1 "loading factors" for factorFile in factorFiles
-        jstr = read("$factorFolder/$factorFile", String)
-        try
-            packedData = JSON3.read(jstr, PackedFactor)
-            push!(factors, unpackFactor(dfgLoadInto, packedData))
-        catch ex
-            @error("JSON3 is having trouble reading $factorFolder/$factorFile into a PackedFactor")
-            @show jstr
-            throw(ex)
+    if isa(dfgLoadInto, GraphsDFG) && GraphsDFGs._factorstype(dfgLoadInto) == PackedFactor
+        factors = @showprogress 1 "loading factors" map(factorFiles) do factorFile
+            jstr = read("$factorFolder/$factorFile", String)
+            return JSON3.read(jstr, PackedFactor)
+        end
+    else
+        @showprogress 1 "loading factors" for factorFile in factorFiles
+            jstr = read("$factorFolder/$factorFile", String)
+            try
+                packedData = JSON3.read(jstr, PackedFactor)
+                push!(factors, unpackFactor(dfgLoadInto, packedData))
+            catch ex
+                @error("JSON3 is having trouble reading $factorFolder/$factorFile into a PackedFactor")
+                @show jstr
+                throw(ex)
+            end
         end
     end
-    @info "Loaded $(length(variables)) factors - $(map(f->f.label, factors))"
+    @info "Loaded $(length(factors)) factors"# - $(map(f->f.label, factors))"
     @info "Inserting factors into graph..."
     # # Adding factors
     map(f->addFactor!(dfgLoadInto, f), factors)
 
-    # Finally, rebuild the CCW's for the factors to completely reinflate them
-    # NOTE CREATES A NEW DFGFactor IF  CCW TYPE CHANGES
-    @info "Rebuilding CCW's for the factors..."
-    @showprogress 1 "build factor operational memory" for factor in factors
-        rebuildFactorMetadata!(dfgLoadInto, factor)
+    if isa(dfgLoadInto, GraphsDFG) && GraphsDFGs._factorstype(dfgLoadInto) != PackedFactor
+        # Finally, rebuild the CCW's for the factors to completely reinflate them
+        # NOTE CREATES A NEW DFGFactor IF  CCW TYPE CHANGES
+        @info "Rebuilding CCW's for the factors..."
+        @showprogress 1 "build factor operational memory" for factor in factors
+            rebuildFactorMetadata!(dfgLoadInto, factor)
+        end
     end
 
     # remove the temporary unzipped file

src/GraphsDFG/entities/GraphsDFG.jl

@@ -21,7 +21,7 @@ mutable struct GraphsDFG{T <: AbstractParams, V <: AbstractDFGVariable, F <:Abst
     sessionBlobEntries::OrderedDict{Symbol, BlobEntry}
     addHistory::Vector{Symbol} #TODO: Discuss more - is this an audit trail?
     solverParams::T # Solver parameters
-    blobStores::Dict{Symbol, <:AbstractBlobStore}
+    blobStores::Dict{Symbol, AbstractBlobStore}
 end
 
 """
@@ -46,7 +46,7 @@ function GraphsDFG{T,V,F}(
     sessionBlobEntries::OrderedDict{Symbol, BlobEntry} = OrderedDict{Symbol, BlobEntry}(),
     addHistory::Vector{Symbol} = Symbol[],
     solverParams::T=T(),
-    blobstores=Dict{Symbol, AbstractBlobStore}(),
+    blobStores=Dict{Symbol, AbstractBlobStore}(),
 ) where {T <: AbstractParams, V <:AbstractDFGVariable, F<:AbstractDFGFactor}
     # Validate the userLabel, robotLabel, and sessionLabel
     !isValidLabel(userLabel) && error("'$userLabel' is not a valid User label")
@@ -67,7 +67,7 @@ function GraphsDFG{T,V,F}(
         sessionBlobEntries,
         addHistory,
         solverParams,
-        blobstores
+        blobStores
     )
 end
 
@@ -92,7 +92,7 @@ GraphsDFG(
     robotData::Dict{Symbol, SmallDataTypes},
     sessionData::Dict{Symbol, SmallDataTypes},
     solverParams::AbstractParams,
-    blobstores=Dict{Symbol, AbstractBlobStore}()
+    blobStores=Dict{Symbol, AbstractBlobStore}()
 ) = GraphsDFG{typeof(solverParams),DFGVariable,DFGFactor}(
         FactorGraph{Int,DFGVariable,DFGFactor}();
         description,
@@ -103,7 +103,7 @@ GraphsDFG(
         robotData,
         sessionData,
         solverParams,
-        blobstores
+        blobStores
     )
 
 
@@ -116,7 +116,7 @@ function GraphsDFG{T,V,F}(
     robotData::Dict{Symbol, SmallDataTypes},
     sessionData::Dict{Symbol, SmallDataTypes},
     solverParams::T,
-    blobstores=Dict{Symbol, AbstractBlobStore}()
+    blobStores=Dict{Symbol, AbstractBlobStore}()
 ) where {T <: AbstractParams, V <:AbstractDFGVariable, F<:AbstractDFGFactor}
     return GraphsDFG{T,V,F}(
         FactorGraph{Int,V,F}();
@@ -128,6 +128,6 @@ function GraphsDFG{T,V,F}(
         robotData,
         sessionData,
         solverParams,
-        blobstores
+        blobStores
     )
 end

src/GraphsDFG/services/GraphsDFG.jl

@@ -1,4 +1,10 @@
 
+function getDFGMetadata(fg::GraphsDFG) 
+    metafields = Set(fieldnames(GraphsDFG))
+    setdiff!(metafields, [:g, :solverParams])
+    metaprops = NamedTuple(k => getproperty(fg, k) for k in metafields)
+    return metaprops
+end
 
 function exists(dfg::GraphsDFG{P,V,F}, node::V) where {P <: AbstractParams, V <: AbstractDFGVariable, F <: AbstractDFGFactor}
     return haskey(dfg.g.variables, node.label)

src/GraphsDFG/services/GraphsDFGSerialization.jl

@@ -14,8 +14,10 @@ using InteractiveUtils
     addHistory::Vector{Symbol}
     solverParams::T
     solverParams_type::String = string(typeof(solverParams))
-    #TODO
-    # blobStores::Dict{Symbol, AbstractBlobStore}
+    # TODO remove Union.Nothing in DFG v0.24
+    typePackedVariable::Union{Nothing,Bool} = false # Are variables packed or full
+    typePackedFactor::Union{Nothing,Bool} = false # Are factors packed or full
+    blobStores::Union{Nothing, Dict{Symbol, FolderStore{Vector{UInt8}}}}
 end
 
 StructTypes.StructType(::Type{PackedGraphsDFG}) = StructTypes.AbstractType()
@@ -27,25 +29,60 @@ function StructTypes.subtypes(::Type{PackedGraphsDFG})
     NamedTuple(map(s->Symbol(s)=>PackedGraphsDFG{s}, subs))
 end
 
+_variablestype(fg::GraphsDFG{<:AbstractParams,T,<:AbstractDFGFactor}) where T = T
+_factorstype(fg::GraphsDFG{<:AbstractParams,<:AbstractDFGVariable, T}) where T = T
+
 ##
 """
     $(SIGNATURES)
 Packing function to serialize DFG metadata from.
 """
 function packDFGMetadata(fg::GraphsDFG)
     commonfields = intersect(fieldnames(PackedGraphsDFG), fieldnames(GraphsDFG))
+    
+    setdiff!(commonfields, [:blobStores])
+    blobStores = Dict{Symbol, FolderStore{Vector{UInt8}}}()
+    foreach(values(fg.blobStores)) do store
+        if store isa FolderStore{Vector{UInt8}}
+            blobStores[store.key] = store
+        else
+            @warn "BlobStore $(store.key) of type $(typeof(store)) is not supported yet and will not be saved"
+        end
+    end
+
     props = (k => getproperty(fg, k) for k in commonfields)
-    return PackedGraphsDFG(;props...)
+    return PackedGraphsDFG(;
+        typePackedVariable = _variablestype(fg) == Variable,
+        typePackedFactor = _factorstype(fg) == PackedFactor,
+        blobStores,
+        props...
+    )
 end
 
 function unpackDFGMetadata(packed::PackedGraphsDFG)
     commonfields = intersect(fieldnames(GraphsDFG), fieldnames(PackedGraphsDFG))
+    
+    #FIXME Deprecate remove in DFG v0.24
+    setdiff!(commonfields, [:blobStores])
+    blobStores = Dict{Symbol, AbstractBlobStore}()
+    !isnothing(packed.blobStores) && merge!(blobStores, packed.blobStores)
+    
     props = (k => getproperty(packed, k) for k in commonfields)
-    GraphsDFG(;props...)
+
+    VT = isnothing(packed.typePackedVariable) || !packed.typePackedVariable ? DFGVariable : Variable
+    FT = isnothing(packed.typePackedFactor) || !packed.typePackedFactor ? DFGFactor : PackedFactor
+    # VT = isnothing(packed.typePackedVariable) || packed.typePackedVariable ? Variable : DFGVariable 
+    # FT = isnothing(packed.typePackedFactor) || packed.typePackedFactor ? PackedFactor : DFGFactor
+    GraphsDFG{typeof(packed.solverParams), VT, FT}(;blobStores, props...)
 end
 
 function unpackDFGMetadata!(dfg::GraphsDFG, packed::PackedGraphsDFG)
     commonfields = intersect(fieldnames(GraphsDFG), fieldnames(PackedGraphsDFG))
+
+    #FIXME Deprecate remove Nothing union in DFG v0.24
+    setdiff!(commonfields, [:blobStores])
+    !isnothing(packed.blobStores) && merge!(dfg.blobStores, packed.blobStores)
+
     props = (k => getproperty(packed, k) for k in commonfields)
     foreach(props) do (k,v)
         setproperty!(dfg, k, v)

src/services/AbstractDFG.jl

@@ -1266,7 +1266,7 @@ function mergeVariableData!(dfg::AbstractDFG, sourceVariable::AbstractDFGVariabl
     :solverDataDict in fieldnames(typeof(var)) && mergeVariableSolverData!(var, sourceVariable)
 
     #update if its not a InMemoryDFGTypes, otherwise it was a reference
-    # if satelite nodes are used it can be updated seprarately
+    # if satelite nodes are used it can be updated separately
     # !(isa(dfg, InMemoryDFGTypes)) && updateVariable!(dfg, var)
 
     return var

src/services/DFGFactor.jl

@@ -118,6 +118,7 @@ Get the variable ordering for this factor.
 Should be equivalent to listNeighbors unless something was deleted in the graph.
 """
 getVariableOrder(fct::DFGFactor) = fct._variableOrderSymbols::Vector{Symbol}
+getVariableOrder(fct::PackedFactor) = fct._variableOrderSymbols::Vector{Symbol}
 getVariableOrder(dfg::AbstractDFG, fct::Symbol) = getVariableOrder(getFactor(dfg, fct))
 
 ##------------------------------------------------------------------------------

src/services/DFGVariable.jl

@@ -450,6 +450,8 @@ Note: Rather use SmallData CRUD
 """
 getSmallData(v::DFGVariable) = v.smallData
 
+getSmallData(v::Variable) = JSON3.read(base64decode(v.metadata))
+
 """
     $(SIGNATURES)
 Set the small data for a variable.

src/services/Serialization.jl

@@ -162,7 +162,7 @@ end
 ## Variable Packing and unpacking
 ##==============================================================================
 
-function packVariable(v::AbstractDFGVariable; includePPEs::Bool=true, includeSolveData::Bool=true, includeDataEntries::Bool=true)
+function packVariable(v::DFGVariable; includePPEs::Bool=true, includeSolveData::Bool=true, includeDataEntries::Bool=true)
     return PackedVariable(;
         id=v.id,
         label = v.label,
@@ -177,7 +177,11 @@ function packVariable(v::AbstractDFGVariable; includePPEs::Bool=true, includeSol
         blobEntries = collect(values(v.dataDict)),
         _version = string(DFG._getDFGVersion()))
 end
-  
+
+function packVariable(v::Variable; includePPEs::Bool=true, includeSolveData::Bool=true, includeDataEntries::Bool=true)
+    return v
+end
+
 function unpackVariable(variable::PackedVariable; skipVersionCheck::Bool=false)
     !skipVersionCheck && _versionCheck(variable)
 
@@ -205,8 +209,9 @@ function unpackVariable(variable::PackedVariable; skipVersionCheck::Bool=false)
         smallData=metadata, 
         dataDict=dataDict, 
         solvable=variable.solvable )
-    end
+end
 
+DFGVariable(v::Variable) = unpackVariable(v)
 
 ##==============================================================================
 ## Factor Packing and unpacking
@@ -250,6 +255,8 @@ function packFactor(f::DFGFactor)
     return props
 end
 
+packFactor(f::PackedFactor) = f
+
 function reconstFactorData end
 
 function decodePackedType(dfg::AbstractDFG, varOrder::AbstractVector{Symbol}, ::Type{T}, packeddata::GenericFunctionNodeData{PT}) where {T<:FactorOperationalMemory, PT}