Refactoring for solver parameters, cleanup

Author: GearsAD

.travis.yml

@@ -3,6 +3,9 @@ language: julia
 os:
     - linux
 
+services:
+    - neo4j
+
 julia:
     - 1.0
     - 1.1

Project.toml

@@ -23,6 +23,8 @@ julia = "0.7, 1"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+IncrementalInference = "904591bb-b899-562f-9e6f-b8df64c7d480"
+RoME = "91fb55c2-4c03-5a59-ba21-f4ea956187b8"
 
 [targets]
 test = ["Test"]

src/CloudGraphsDFG/entities/CloudGraphsDFG.jl

@@ -5,7 +5,7 @@ mutable struct Neo4jInstance
   graph::Neo4j.Graph
 end
 
-mutable struct CloudGraphsDFG <: AbstractDFG
+mutable struct CloudGraphsDFG{T <: AbstractParams} <: AbstractDFG
     neo4jInstance::Neo4jInstance
     description::String
     userId::String
@@ -18,7 +18,7 @@ mutable struct CloudGraphsDFG <: AbstractDFG
     variableCache::Dict{Symbol, DFGVariable}
     factorCache::Dict{Symbol, DFGFactor}
     addHistory::Vector{Symbol} #TODO: Discuss more - is this an audit trail?
-    solverParams::Any # Solver parameters
+    solverParams::T # Solver parameters
     useCache::Bool
 end
 

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -41,18 +41,18 @@ end
     $(SIGNATURES)
 Create a new CloudGraphs-based DFG factor graph using a Neo4j.Connection.
 """
-function CloudGraphsDFG(neo4jConnection::Neo4j.Connection, userId::String, robotId::String, sessionId::String, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc; description::String="CloudGraphs DFG", solverParams::Any=nothing, useCache::Bool=false)
+function CloudGraphsDFG{T}(neo4jConnection::Neo4j.Connection, userId::String, robotId::String, sessionId::String, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc; description::String="CloudGraphs DFG", solverParams::T=NoSolverParams(), useCache::Bool=false) where T <: AbstractParams
     graph = Neo4j.getgraph(neo4jConnection)
     neo4jInstance = Neo4jInstance(neo4jConnection, graph)
-    return CloudGraphsDFG(neo4jInstance, description, userId, robotId, sessionId, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc, Dict{Symbol, Int64}(), Dict{Symbol, DFGVariable}(), Dict{Symbol, DFGFactor}(), Symbol[], solverParams, useCache)
+    return CloudGraphsDFG{T}(neo4jInstance, description, userId, robotId, sessionId, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc, Dict{Symbol, Int64}(), Dict{Symbol, DFGVariable}(), Dict{Symbol, DFGFactor}(), Symbol[], solverParams, useCache)
 end
 """
     $(SIGNATURES)
 Create a new CloudGraphs-based DFG factor graph by specifying the Neo4j connection information.
 """
-function CloudGraphsDFG(host::String, port::Int, dbUser::String, dbPassword::String, userId::String, robotId::String, sessionId::String, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc; description::String="CloudGraphs DFG", solverParams::Any=nothing, useCache::Bool=false)
+function CloudGraphsDFG{T}(host::String, port::Int, dbUser::String, dbPassword::String, userId::String, robotId::String, sessionId::String, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc; description::String="CloudGraphs DFG", solverParams::T=NoSolverParams(), useCache::Bool=false) where T <: AbstractParams
     neo4jConnection = Neo4j.Connection(host, port=port, user=dbUser, password=dbPassword);
-    return CloudGraphsDFG(neo4jConnection, userId, robotId, sessionId, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc, description=description, solverParams=solverParams, useCache=useCache)
+    return CloudGraphsDFG{T}(neo4jConnection, userId, robotId, sessionId, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc, description=description, solverParams=solverParams, useCache=useCache)
 end
 
 """
@@ -68,7 +68,7 @@ function _getDuplicatedEmptyDFG(dfg::CloudGraphsDFG)::CloudGraphsDFG
         length(_getLabelsFromCyphonQuery(dfg.neo4jInstance, "(node:$(dfg.userId):$(dfg.robotId):$(sessionId))")) == 0 && break
     end
     @debug "Unique+empty copy session name: $sessionId"
-    return CloudGraphsDFG(dfg.neo4jInstance.connection, dfg.userId, dfg.robotId, sessionId, dfg.encodePackedTypeFunc, dfg.getPackedTypeFunc, dfg.decodePackedTypeFunc, solverParams=deepcopy(dfg.solverParams), description="(Copy of) $(dfg.description)", useCache=dfg.useCache)
+    return CloudGraphsDFG{typeof(dfg.solverParams)}(dfg.neo4jInstance.connection, dfg.userId, dfg.robotId, sessionId, dfg.encodePackedTypeFunc, dfg.getPackedTypeFunc, dfg.decodePackedTypeFunc, solverParams=deepcopy(dfg.solverParams), description="(Copy of) $(dfg.description)", useCache=dfg.useCache)
 end
 
 # Accessors
@@ -77,7 +77,9 @@ getDescription(dfg::CloudGraphsDFG) = dfg.description
 setDescription(dfg::CloudGraphsDFG, description::String) = dfg.description = description
 getAddHistory(dfg::CloudGraphsDFG) = dfg.addHistory
 getSolverParams(dfg::CloudGraphsDFG) = dfg.solverParams
-setSolverParams(dfg::CloudGraphsDFG, solverParams::Any) = dfg.solverParams = solverParams
+function setSolverParams(dfg::CloudGraphsDFG, solverParams::T) where T <: AbstractParams
+    dfg.solverParams = solverParams
+end
 
 """
     $(SIGNATURES)
@@ -440,16 +442,12 @@ function updateFactor!(dfg::CloudGraphsDFG, variables::Vector{DFGVariable}, fact
     # Update the body
     factor = updateFactor!(dfg, factor)
 
-    @show "HERE WITH $(factor.label)!"
-    @show map(v->v.label, variables)
-
     # Now update the relationships
-    @show existingNeighbors = getNeighbors(dfg, factor)
+    existingNeighbors = getNeighbors(dfg, factor)
     if symdiff(existingNeighbors, map(v->v.label, variables)) == []
         # Done, otherwise we need to remake the edges.
         return factor
     end
-    @show "HERE WITH $(factor.label)!"
     # Delete existing relationships
     fNode = Neo4j.getnode(dfg.neo4jInstance.graph, factor._internalId)
     for relationship in Neo4j.getrels(fNode)
@@ -707,39 +705,8 @@ function ls(dfg::CloudGraphsDFG, label::Symbol)::Vector{Symbol} where T <: DFGNo
     return getNeighbors(dfg, label)
 end
 
-function _copyIntoGraph!(sourceDFG::CloudGraphsDFG, destDFG::CloudGraphsDFG, variableFactorLabels::Vector{Symbol}, includeOrphanFactors::Bool=false)::Nothing
-    @show variableFactorLabels
-    # Split into variables and factors
-    sourceVariables = map(vId->getVariable(sourceDFG, vId), intersect(getVariableIds(sourceDFG), variableFactorLabels))
-    sourceFactors = map(fId->getFactor(sourceDFG, fId), intersect(getFactorIds(sourceDFG), variableFactorLabels))
-    if length(sourceVariables) + length(sourceFactors) != length(variableFactorLabels)
-        rem = symdiff(map(v->v.label, sourceVariables), variableFactorLabels)
-        rem = symdiff(map(f->f.label, sourceFactors), variableFactorLabels)
-        error("Cannot copy because cannot find the following nodes in the source graph: $rem")
-    end
-
-    # Now we have to add all variables first,
-    for variable in sourceVariables
-        addVariable!(destDFG, deepcopy(variable))
-    end
-    # And then all factors to the destDFG.
-    for factor in sourceFactors
-        # Get the original factor variables (we need them to create it)
-        sourceFactorVariableIds = getNeighbors(sourceDFG, factor)
-        # Find the labels and associated variables in our new subgraph
-        factVariableIds = Symbol[]
-        for variable in sourceFactorVariableIds
-            if exists(destDFG, variable)
-                push!(factVariableIds, variable)
-            end
-        end
-        # Only if we have all of them should we add it (otherwise strange things may happen on evaluation)
-        if includeOrphanFactors || length(factVariableIds) == length(sourceFactorVariableIds)
-            addFactor!(destDFG, factVariableIds, deepcopy(factor))
-        end
-    end
-    return nothing
-end
+## This is moved to services/AbstractDFG.jl
+# function _copyIntoGraph!(sourceDFG::CloudGraphsDFG, destDFG::CloudGraphsDFG, variableFactorLabels::Vector{Symbol}, includeOrphanFactors::Bool=false)::Nothing
 
 """
     $(SIGNATURES)
@@ -809,8 +776,8 @@ function getAdjacencyMatrix(dfg::CloudGraphsDFG)::Matrix{Union{Nothing, Symbol}}
         error(string(nodes.errors))
     end
     # Add in the relationships
-    @show varRel = Symbol.(map(node -> node["row"][1], nodes.results[1]["data"]))
-    @show factRel = Symbol.(map(node -> node["row"][2], nodes.results[1]["data"]))
+    varRel = Symbol.(map(node -> node["row"][1], nodes.results[1]["data"]))
+    factRel = Symbol.(map(node -> node["row"][2], nodes.results[1]["data"]))
     for i = 1:length(varRel)
         adjMat[fDict[factRel[i]], vDict[varRel[i]]] = factRel[i]
     end

src/GraphsDFG/entities/GraphsDFG.jl

@@ -8,12 +8,7 @@ mutable struct GraphsNode
 end
 const FGType = Graphs.GenericIncidenceList{GraphsNode,Graphs.Edge{GraphsNode},Dict{Int,GraphsNode},Dict{Int,Array{Graphs.Edge{GraphsNode},1}}}
 
-abstract type AbstractParams end
-
-mutable struct NoSolverParams <: AbstractParams
-end
-
-mutable struct GraphsDFG{T<:AbstractParams} <: AbstractDFG
+mutable struct GraphsDFG{T <: AbstractParams} <: AbstractDFG
     g::FGType
     description::String
     nodeCounter::Int64

src/entities/AbstractTypes.jl

@@ -12,3 +12,8 @@ Abstract parent struct for a DFG graph.
 """
 abstract type AbstractDFG
 end
+
+abstract type AbstractParams end
+
+mutable struct NoSolverParams <: AbstractParams
+end

src/entities/DFGVariable.jl

@@ -15,7 +15,8 @@ mutable struct VariableNodeData
   partialinit::Bool
   ismargin::Bool
   dontmargin::Bool
-  VariableNodeData() = new()
+  # A valid, packable default constructor is needed.
+  VariableNodeData() = new(zeros(1,1), zeros(1,1), Symbol[], Int[], 0, false, :NOTHING, Symbol[], "", false, false, false, false)
   VariableNodeData(x1::Array{Float64,2},
                    x2::Array{Float64,2},
                    x3::Vector{Symbol},

src/services/AbstractDFG.jl

@@ -24,3 +24,41 @@ function getSerializationModule(dfg::G)::Module where G <: AbstractDFG
     @warn "Retrieving serialization module from AbstractDFG - override this in the '$(typeof(dfg)) structure! This is returning Main"
     return Main
 end
+
+"""
+    $(SIGNATURES)
+Common function for copying nodes from one graph into another graph.
+This is overridden in specialized implementations for performance.
+"""
+function _copyIntoGraph!(sourceDFG::G, destDFG::H, variableFactorLabels::Vector{Symbol}, includeOrphanFactors::Bool=false)::Nothing where {G <: AbstractDFG, H <: AbstractDFG}
+    # Split into variables and factors
+    sourceVariables = map(vId->getVariable(sourceDFG, vId), intersect(getVariableIds(sourceDFG), variableFactorLabels))
+    sourceFactors = map(fId->getFactor(sourceDFG, fId), intersect(getFactorIds(sourceDFG), variableFactorLabels))
+    if length(sourceVariables) + length(sourceFactors) != length(variableFactorLabels)
+        rem = symdiff(map(v->v.label, sourceVariables), variableFactorLabels)
+        rem = symdiff(map(f->f.label, sourceFactors), variableFactorLabels)
+        error("Cannot copy because cannot find the following nodes in the source graph: $rem")
+    end
+
+    # Now we have to add all variables first,
+    for variable in sourceVariables
+        addVariable!(destDFG, deepcopy(variable))
+    end
+    # And then all factors to the destDFG.
+    for factor in sourceFactors
+        # Get the original factor variables (we need them to create it)
+        sourceFactorVariableIds = getNeighbors(sourceDFG, factor)
+        # Find the labels and associated variables in our new subgraph
+        factVariableIds = Symbol[]
+        for variable in sourceFactorVariableIds
+            if exists(destDFG, variable)
+                push!(factVariableIds, variable)
+            end
+        end
+        # Only if we have all of them should we add it (otherwise strange things may happen on evaluation)
+        if includeOrphanFactors || length(factVariableIds) == length(sourceFactorVariableIds)
+            addFactor!(destDFG, factVariableIds, deepcopy(factor))
+        end
+    end
+    return nothing
+end

src/services/DFGVariable.jl

@@ -8,7 +8,7 @@ function pack(dfg::G, d::VariableNodeData)::PackedVariableNodeData where G <: Ab
                                 d.BayesNetOutVertIDs,
                                 d.dimIDs, d.dims, d.eliminated,
                                 d.BayesNetVertID, d.separator,
-                                string(d.softtype), d.initialized, d.partialinit, d.ismargin, d.dontmargin)
+                                d.softtype != nothing ? string(d.softtype) : nothing, d.initialized, d.partialinit, d.ismargin, d.dontmargin)
 end
 
 function unpack(dfg::G, d::PackedVariableNodeData)::VariableNodeData where G <: AbstractDFG
@@ -26,10 +26,12 @@ function unpack(dfg::G, d::PackedVariableNodeData)::VariableNodeData where G <:
   mainmod = getSerializationModule(dfg)
   mainmod == nothing && error("Serialization module is null - please call setSerializationNamespace!(\"Main\" => Main) in your main program.")
   try
-      unpackedTypeName = split(d.softtype, "(")[1]
-      unpackedTypeName = split(unpackedTypeName, '.')[end]
-      @debug "DECODING Softtype = $unpackedTypeName"
-      st = getfield(mainmod, Symbol(unpackedTypeName))()
+      if d.softtype != ""
+          unpackedTypeName = split(d.softtype, "(")[1]
+          unpackedTypeName = split(unpackedTypeName, '.')[end]
+          @debug "DECODING Softtype = $unpackedTypeName"
+          st = getfield(mainmod, Symbol(unpackedTypeName))()
+      end
   catch ex
       @error "Unable to deserialize soft type $(d.softtype)"
       io = IOBuffer()