Fixing tests with simple factors, and changing solver update call to be mergeUpdateVariableSolverData!.

Author: GearsAD

src/CloudGraphsDFG/CloudGraphsDFG.jl

@@ -18,7 +18,7 @@ export getAddHistory, getDescription, getLabelDict
 export addVariable!, addFactor!
 export ls, lsf, getVariables, getFactors, getVariableIds, getFactorIds
 export getVariable, getFactor
-export updateVariable!, updateFactor!, updateVariableSolverData!
+export updateVariable!, updateFactor!, mergeUpdateVariableSolverData!
 export deleteVariable!, deleteFactor!
 export getAdjacencyMatrix
 export getNeighbors

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -318,13 +318,17 @@ end
     $(SIGNATURES)
 Update solver and estimate data for a variable (variable can be from another graph).
 """
-function updateVariableSolverData!(dfg::CloudGraphsDFG, sourceVariable::DFGVariable)::DFGVariable
+function mergeUpdateVariableSolverData!(dfg::CloudGraphsDFG, sourceVariable::DFGVariable)::DFGVariable
     if !exists(dfg, sourceVariable)
         error("Source variable '$(sourceVariable.label)' doesn't exist in the graph.")
     end
-    nodeId = _tryGetNeoNodeIdFromNodeLabel(dfg.neo4jInstance, dfg.userId, dfg.robotId, dfg.sessionId, sourceVariable.label)
-    Neo4j.setnodeproperty(dfg.neo4jInstance.graph, nodeId, "estimateDict", JSON2.write(sourceVariable.estimateDict))
-    Neo4j.setnodeproperty(dfg.neo4jInstance.graph, nodeId, "solverDataDict", JSON2.write(Dict(keys(sourceVariable.solverDataDict) .=> map(vnd -> pack(dfg, vnd), values(sourceVariable.solverDataDict)))))
+    var = getVariable(dfg, sourceVariable.label)
+    newEsts = merge!(var.estimateDict, deepcopy(sourceVariable.estimateDict))
+    newSolveData = merge!(var.solverDataDict, sourceVariable.solverDataDict)
+    Neo4j.setnodeproperty(dfg.neo4jInstance.graph, var._internalId, "estimateDict",
+        JSON2.write(newEsts))
+    Neo4j.setnodeproperty(dfg.neo4jInstance.graph, var._internalId, "solverDataDict",
+        JSON2.write(Dict(keys(newSolveData) .=> map(vnd -> pack(dfg, vnd), values(newSolveData)))))
     return sourceVariable
 end
 

src/DistributedFactorGraphs.jl

@@ -30,7 +30,7 @@ export pushUserData!, pushRobotData!, pushSessionData!, popUserData!, popRobotDa
 
 # Services/AbstractDFG Exports
 export hasFactor, hasVariable, isInitialized, getFactorFunction, isVariable, isFactor
-export updateGraphSolverData!
+export mergeUpdateVariableSolverData!, mergeUpdateGraphSolverData!
 
 # Solver (IIF) Exports
 export VariableNodeData, PackedVariableNodeData, VariableEstimate
@@ -50,7 +50,8 @@ export saveDFG, loadDFG
 export getSummary, getSummaryGraph
 
 # Comparisons
-export compareField,
+export
+    compareField,
     compareFields,
     compareAll,
     compareAllSpecial,
@@ -85,14 +86,17 @@ using .SymbolDFGs
 include("LightDFG/LightDFG.jl")
 @reexport using .LightDFGs
 
+#supported in Memory fg types
+const InMemoryDFGTypes = Union{GraphsDFG, LightDFG}
+export InMemoryDFGTypes
+
 function __init__()
     @info "Looking for @require modules"
-    #FIXME still can't figure @require out
-    # @require Neo4j="d2adbeaf-5838-5367-8a2f-e46d570981db" begin
-    #     @info "Including CloudGraphsDFG"
-    #     # Include the Cloudgraphs API
-    #     include("CloudGraphsDFG/CloudGraphsDFG.jl")
-    # end
+    @require Neo4j="d2adbeaf-5838-5367-8a2f-e46d570981db" begin
+        @info "Including CloudGraphsDFG"
+        # Include the Cloudgraphs API
+        include("CloudGraphsDFG/CloudGraphsDFG.jl")
+    end
 
     @require GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231" begin
         @info "Including Plots"
@@ -102,10 +106,6 @@ function __init__()
 
 end
 
-#FIXME still can't figure @require out
-# Include the Cloudgraphs API
-include("CloudGraphsDFG/CloudGraphsDFG.jl")
-
 # To be moved as necessary.
 include("Common.jl")
 

src/services/AbstractDFG.jl

@@ -415,11 +415,11 @@ end
 
 """
     $(SIGNATURES)
-Update solver and estimate data for a variable (variable can be from another graph).
+Merges and updates solver and estimate data for a variable (variable can be from another graph).
 Note: Makes a copy of the estimates and solver data so that there is no coupling
 between graphs.
 """
-function updateVariableSolverData!(dfg::AbstractDFG, sourceVariable::AbstractDFGVariable)::AbstractDFGVariable
+function mergeUpdateVariableSolverData!(dfg::AbstractDFG, sourceVariable::AbstractDFGVariable)::AbstractDFGVariable
     if !exists(dfg, sourceVariable)
         error("Source variable '$(sourceVariable.label)' doesn't exist in the graph.")
     end
@@ -436,11 +436,11 @@ end
 Common function to update all solver data and estimates from one graph to another.
 This should be used to push local solve data back into a cloud graph, for example.
 """
-function updateGraphSolverData!(sourceDFG::G, destDFG::H, varSyms::Vector{Symbol})::Nothing where {G <: AbstractDFG, H <: AbstractDFG}
+function mergeUpdateGraphSolverData!(sourceDFG::G, destDFG::H, varSyms::Vector{Symbol})::Nothing where {G <: AbstractDFG, H <: AbstractDFG}
     # Update all variables in the destination
     # (For now... we may change this soon)
     for variableId in varSyms
-        updateVariableSolverData!(destDFG, getVariable(sourceDFG, variableId))
+        mergeUpdateVariableSolverData!(destDFG, getVariable(sourceDFG, variableId))
     end
 end
 

src/services/CompareUtils.jl

@@ -1,5 +1,8 @@
 
 function compareField(Allc, Bllc, syms)::Bool
+  (!isdefined(Allc, syms) && !isdefined(Bllc, syms)) && return true
+  !isdefined(Allc, syms) && return false
+  !isdefined(Bllc, syms) && return false
   return eval(:($Allc.$syms == $Bllc.$syms))
 end
 
@@ -101,6 +104,9 @@ function compareAll(Al::T, Bl::T; show::Bool=true, skip::Vector{Symbol}=Symbol[]
   for field in fieldnames(T)
     field in skip && continue
     @debug("  Checking field: $field")
+    (!isdefined(Al, field) && !isdefined(Al, field)) && return true
+    !isdefined(Al, field) && return false
+    !isdefined(Bl, field) && return false
     Ad = eval(:($Al.$field))
     Bd = eval(:($Bl.$field))
     !compareAll(Ad, Bd, show=show, skip=skip) && return false
@@ -126,8 +132,8 @@ function compareVariable(A::DFGVariable,
   union!(skiplist, skip)
   TP = TP && compareAll(A.solverDataDict, B.solverDataDict, skip=skiplist, show=show)
 
-  Ad = getData(A)
-  Bd = getData(B)
+  Ad = solverData(A)
+  Bd = solverData(B)
 
   # TP = TP && compareAll(A.attributes, B.attributes, skip=[:softtype;], show=show)
   varskiplist = union(varskiplist, [:softtype;:_internalId])
@@ -161,23 +167,22 @@ function compareFactor(A::DFGFactor,
                        skipsamples::Bool=true,
                        skipcompute::Bool=true  )
   #
-  @info show
   TP =  compareAll(A, B, skip=union([:attributes;:data;:_variableOrderSymbols;:_internalId],skip), show=show)
   # TP = TP & compareAll(A.attributes, B.attributes, skip=[:data;], show=show)
-  TP = TP & compareAllSpecial(getData(A), getData(B), skip=union([:fnc;:_internalId], skip), show=show)
-  TP = TP & compareAllSpecial(getData(A).fnc, getData(B).fnc, skip=union([:cpt;:measurement;:params;:varidx;:threadmodel], skip), show=show)
-  TP = TP & (skipsamples || compareAll(getData(A).fnc.measurement, getData(B).fnc.measurement, show=show, skip=skip))
-  TP = TP & (skipcompute || compareAll(getData(A).fnc.params, getData(B).fnc.params, show=show, skip=skip))
-  TP = TP & (skipcompute || compareAll(getData(A).fnc.varidx, getData(B).fnc.varidx, show=show, skip=skip))
+  TP = TP & compareAllSpecial(solverData(A), solverData(B), skip=union([:fnc;:_internalId], skip), show=show)
+  TP = TP & compareAllSpecial(solverData(A).fnc, solverData(B).fnc, skip=union([:cpt;:measurement;:params;:varidx;:threadmodel], skip), show=show)
+  TP = TP & (skipsamples || compareAll(solverData(A).fnc.measurement, solverData(B).fnc.measurement, show=show, skip=skip))
+  TP = TP & (skipcompute || compareAll(solverData(A).fnc.params, solverData(B).fnc.params, show=show, skip=skip))
+  TP = TP & (skipcompute || compareAll(solverData(A).fnc.varidx, solverData(B).fnc.varidx, show=show, skip=skip))
 
   return TP
 end
-  # Ad = getData(A)
-  # Bd = getData(B)
+  # Ad = solverData(A)
+  # Bd = solverData(B)
   # TP =  compareAll(A, B, skip=[:attributes;:data], show=show)
   # TP &= compareAll(A.attributes, B.attributes, skip=[:data;], show=show)
-  # TP &= compareAllSpecial(getData(A).fnc, getData(B).fnc, skip=[:cpt;], show=show)
-  # TP &= compareAll(getData(A).fnc.cpt, getData(B).fnc.cpt, show=show)
+  # TP &= compareAllSpecial(solverData(A).fnc, solverData(B).fnc, skip=[:cpt;], show=show)
+  # TP &= compareAll(solverData(A).fnc.cpt, solverData(B).fnc.cpt, show=show)
 
 
 """

src/services/DFGFactor.jl

@@ -79,12 +79,4 @@ Equality check for DFGFactor.
 """
 function ==(a::DFGFactor, b::DFGFactor)::Bool
     return compareFactor(a, b)
-  a.label != b.label && @debug("label is not equal")==nothing && return false
-  a.tags != b.tags && @debug("tags is not equal")==nothing && return false
-  a.data != b.data && @debug("data is not equal")==nothing && return false
-  a.ready != b.ready && @debug("ready is not equal")==nothing && return false
-  a.backendset != b.backendset && @debug("backendset is not equal")==nothing && return false
-  a._internalId != b._internalId && @debug("_internalId is not equal")==nothing && return false
-  a._variableOrderSymbols != b._variableOrderSymbols && @debug("_variableOrderSymbols is not equal")==nothing && return false
-  return true
 end

test/LightDFGSummaryTypes.jl

@@ -129,7 +129,7 @@ end
         :mean => VariableEstimate(:default, :mean, [50.0]),
         :modefit => VariableEstimate(:default, :modefit, [75.0]))
     #update
-    updateVariableSolverData!(dfg, newvar)
+    mergeUpdateVariableSolverData!(dfg, newvar)
     #TODO maybe implement ==; @test newvar==var
     Base.:(==)(varest1::VariableEstimate, varest2::VariableEstimate) = begin
         varest1.lastUpdatedTimestamp == varest2.lastUpdatedTimestamp || return false
@@ -150,7 +150,7 @@ end
         :ppe => VariableEstimate(:default, :ppe, [7.0]))
 
     # Persist to the original variable.
-    updateVariableSolverData!(dfg, newvar)
+    mergeUpdateVariableSolverData!(dfg, newvar)
     # At this point newvar will have only :second, and var should have both (it is the reference)
     @test symdiff(collect(keys(estimates(var))), [:default, :second]) == Symbol[]
     @test symdiff(collect(keys(estimates(newvar))), [:second]) == Symbol[]

test/iifInterfaceTests.jl

@@ -1,6 +1,6 @@
 # using GraphPlot
 # using Neo4j
-# using DistributedFactorGraphs
+using DistributedFactorGraphs
 # using IncrementalInference
 # using Test
 # dfg = apis[2]
@@ -174,60 +174,65 @@ end
     @test !isInitialized(v2, key=:second)
 
     # Session, robot, and user small data tests
-    smallUserData = Dict{Symbol, String}(:a => "42", :b => "Hello")
-    smallRobotData = Dict{Symbol, String}(:a => "43", :b => "Hello")
-    smallSessionData = Dict{Symbol, String}(:a => "44", :b => "Hello")
-    setUserData(dfg, deepcopy(smallUserData))
-    setRobotData(dfg, deepcopy(smallRobotData))
-    setSessionData(dfg, deepcopy(smallSessionData))
-    @test getUserData(dfg) == smallUserData
-    @test getRobotData(dfg) == smallRobotData
-    @test getSessionData(dfg) == smallSessionData
-
+    # NOTE: CloudGraphDFG isnt supporting this yet.
+    if !(typeof(dfg) <: CloudGraphsDFG)
+        smallUserData = Dict{Symbol, String}(:a => "42", :b => "Hello")
+        smallRobotData = Dict{Symbol, String}(:a => "43", :b => "Hello")
+        smallSessionData = Dict{Symbol, String}(:a => "44", :b => "Hello")
+        setUserData(dfg, deepcopy(smallUserData))
+        setRobotData(dfg, deepcopy(smallRobotData))
+        setSessionData(dfg, deepcopy(smallSessionData))
+        @test getUserData(dfg) == smallUserData
+        @test getRobotData(dfg) == smallRobotData
+        @test getSessionData(dfg) == smallSessionData
+    end
 end
 
 @testset "BigData" begin
-    oid = zeros(UInt8,12); oid[12] = 0x01
-    de1 = MongodbBigDataEntry(:key1, NTuple{12,UInt8}(oid))
-
-    oid = zeros(UInt8,12); oid[12] = 0x02
-    de2 = MongodbBigDataEntry(:key2, NTuple{12,UInt8}(oid))
-
-    oid = zeros(UInt8,12); oid[12] = 0x03
-    de2_update = MongodbBigDataEntry(:key2, NTuple{12,UInt8}(oid))
-
-    #add
-    v1 = getVariable(dfg, :a)
-    @test addBigDataEntry!(v1, de1)
-    @test addBigDataEntry!(dfg, :a, de2)
-    @test addBigDataEntry!(v1, de1)
-
-    #get
-    @test deepcopy(de1) == getBigDataEntry(v1, :key1)
-    @test deepcopy(de2) == getBigDataEntry(dfg, :a, :key2)
-    @test_throws Any getBigDataEntry(v2, :key1)
-    @test_throws Any getBigDataEntry(dfg, :b, :key1)
-
-    #update
-    @test updateBigDataEntry!(dfg, :a, de2_update)
-    @test deepcopy(de2_update) == getBigDataEntry(dfg, :a, :key2)
-    @test !updateBigDataEntry!(dfg, :b, de2_update)
-
-    #list
-    entries = getBigDataEntries(dfg, :a)
-    @test length(entries) == 2
-    @test symdiff(map(e->e.key, entries), [:key1, :key2]) == Symbol[]
-    @test length(getBigDataEntries(dfg, :b)) == 0
-
-    @test symdiff(getBigDataKeys(dfg, :a), [:key1, :key2]) == Symbol[]
-    @test getBigDataKeys(dfg, :b) == Symbol[]
-
-    #delete
-    @test deepcopy(de1) == deleteBigDataEntry!(v1, :key1)
-    @test getBigDataKeys(v1) == Symbol[:key2]
-    #delete from dfg
-    @test deepcopy(de2_update) == deleteBigDataEntry!(dfg, :a, :key2)
-    @test getBigDataKeys(v1) == Symbol[]
+    # NOTE: CloudGraphDFG isnt supporting this yet.
+    if !(typeof(dfg) <: CloudGraphsDFG)
+        oid = zeros(UInt8,12); oid[12] = 0x01
+        de1 = MongodbBigDataEntry(:key1, NTuple{12,UInt8}(oid))
+
+        oid = zeros(UInt8,12); oid[12] = 0x02
+        de2 = MongodbBigDataEntry(:key2, NTuple{12,UInt8}(oid))
+
+        oid = zeros(UInt8,12); oid[12] = 0x03
+        de2_update = MongodbBigDataEntry(:key2, NTuple{12,UInt8}(oid))
+
+        #add
+        v1 = getVariable(dfg, :a)
+        @test addBigDataEntry!(v1, de1)
+        @test addBigDataEntry!(dfg, :a, de2)
+        @test addBigDataEntry!(v1, de1)
+
+        #get
+        @test deepcopy(de1) == getBigDataEntry(v1, :key1)
+        @test deepcopy(de2) == getBigDataEntry(dfg, :a, :key2)
+        @test_throws Any getBigDataEntry(v2, :key1)
+        @test_throws Any getBigDataEntry(dfg, :b, :key1)
+
+        #update
+        @test updateBigDataEntry!(dfg, :a, de2_update)
+        @test deepcopy(de2_update) == getBigDataEntry(dfg, :a, :key2)
+        @test !updateBigDataEntry!(dfg, :b, de2_update)
+
+        #list
+        entries = getBigDataEntries(dfg, :a)
+        @test length(entries) == 2
+        @test symdiff(map(e->e.key, entries), [:key1, :key2]) == Symbol[]
+        @test length(getBigDataEntries(dfg, :b)) == 0
+
+        @test symdiff(getBigDataKeys(dfg, :a), [:key1, :key2]) == Symbol[]
+        @test getBigDataKeys(dfg, :b) == Symbol[]
+
+        #delete
+        @test deepcopy(de1) == deleteBigDataEntry!(v1, :key1)
+        @test getBigDataKeys(v1) == Symbol[:key2]
+        #delete from dfg
+        @test deepcopy(de2_update) == deleteBigDataEntry!(dfg, :a, :key2)
+        @test getBigDataKeys(v1) == Symbol[]
+    end
 end
 
 @testset "Updating Nodes and Estimates" begin
@@ -241,7 +246,7 @@ end
         :mean => VariableEstimate(:default, :mean, [50.0]),
         :modefit => VariableEstimate(:default, :modefit, [75.0]))
     #update
-    updateVariableSolverData!(dfg, newvar)
+    mergeUpdateVariableSolverData!(dfg, newvar)
 
     #Check if variable is updated
     var = getVariable(dfg, :a)
@@ -256,7 +261,7 @@ end
     # Confirm they're different
     @test estimates(newvar) != estimates(var)
     # Persist it.
-    updateVariableSolverData!(dfg, newvar)
+    mergeUpdateVariableSolverData!(dfg, newvar)
     # Get the latest
     var = getVariable(dfg, :a)
     @test symdiff(collect(keys(estimates(var))), [:default, :second]) == Symbol[]
@@ -270,7 +275,7 @@ end
     #confirm delete
     @test symdiff(collect(keys(estimates(newvar))), [:second]) == Symbol[]
     # Persist it.
-    updateVariableSolverData!(dfg, newvar)
+    mergeUpdateVariableSolverData!(dfg, newvar)
 
     # Get the latest and confirm they're the same, :second
     var = getVariable(dfg, :a)