Merge pull request #210 from JuliaRobotics/feature/4Q19/v051_cleanup

Feature - 4Q19 - v0.51 cleanup

Author: GearsAD

Project.toml

@@ -13,7 +13,6 @@ Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 JSON2 = "2535ab7d-5cd8-5a07-80ac-9b1792aadce3"
 LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-MetaGraphs = "626554b9-1ddb-594c-aa3c-2596fe9399a5"
 Neo4j = "d2adbeaf-5838-5367-8a2f-e46d570981db"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
@@ -27,7 +26,6 @@ GraphPlot = "0.3.1, 0.4"
 Graphs = "0.10.2, 0.11, 1"
 JSON2 = "0.3.1"
 LightGraphs = "1.2, 1.3"
-MetaGraphs = "^0.6.3"
 Neo4j = "2"
 Reexport = "0.2, 0.3, 0.4, 0.5, 1"
 Requires = "0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 1"

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -91,16 +91,7 @@ function addVariable!(dfg::CloudGraphsDFG, variable::DFGVariable)::Bool
     if exists(dfg, variable)
         error("Variable '$(variable.label)' already exists in the factor graph")
     end
-    props = Dict{String, Any}()
-    props["label"] = string(variable.label)
-    props["timestamp"] = string(variable.timestamp)
-    props["tags"] = JSON2.write(variable.tags)
-    props["estimateDict"] = JSON2.write(variable.estimateDict)
-    props["solverDataDict"] = JSON2.write(Dict(keys(variable.solverDataDict) .=> map(vnd -> pack(dfg, vnd), values(variable.solverDataDict))))
-    props["smallData"] = JSON2.write(variable.smallData)
-    props["ready"] = variable.solvable
-    props["backendset"] = variable.solveInProgress
-    # Don't handle big data at the moment.
+    props = packVariable(dfg, variable)
 
     neo4jNode = Neo4j.createnode(dfg.neo4jInstance.graph, props);
     variable._internalId = neo4jNode.id
@@ -132,20 +123,7 @@ function addFactor!(dfg::CloudGraphsDFG, variables::Vector{DFGVariable}, factor:
     factor._variableOrderSymbols = map(v->v.label, variables)
 
     # Construct the properties to save
-    props = Dict{String, Any}()
-    props["label"] = string(factor.label)
-    props["tags"] = JSON2.write(factor.tags)
-    # Pack the node data
-    fnctype = factor.data.fnc.usrfnc!
-    packtype = getfield(_getmodule(fnctype), Symbol("Packed$(_getname(fnctype))"))
-    packed = convert(PackedFunctionNodeData{packtype}, factor.data)
-    props["data"] = JSON2.write(packed)
-    # Include the type
-    props["fnctype"] = String(_getname(fnctype))
-    props["_variableOrderSymbols"] = JSON2.write(factor._variableOrderSymbols)
-    props["backendset"] = factor.solveInProgress
-    props["ready"] = factor.solvable
-    # Don't handle big data at the moment.
+    props = packFactor(dfg, factor)
 
     neo4jNode = Neo4j.createnode(dfg.neo4jInstance.graph, props);
     factor._internalId = neo4jNode.id
@@ -182,27 +160,8 @@ Get a DFGVariable from a DFG using its underlying integer ID.
 """
 function getVariable(dfg::CloudGraphsDFG, variableId::Int64)::DFGVariable
     props = getnodeproperties(dfg.neo4jInstance.graph, variableId)
-    # Time to do deserialization
-    # props["label"] = Symbol(variable.label)
-    timestamp = DateTime(props["timestamp"])
-    tags =  JSON2.read(props["tags"], Vector{Symbol})
-    #TODO this will work for some time, but unpacking in an <: AbstractPointParametricEst would be lekker.
-    estimateDict = JSON2.read(props["estimateDict"], Dict{Symbol, MeanMaxPPE})
-    smallData = nothing
-    smallData = JSON2.read(props["smallData"], Dict{String, String})
-
-    packed = JSON2.read(props["solverDataDict"], Dict{String, PackedVariableNodeData})
-    solverData = Dict(Symbol.(keys(packed)) .=> map(p -> unpack(dfg, p), values(packed)))
-
-    # Rebuild DFGVariable
-    variable = DFGVariable(Symbol(props["label"]), variableId)
-    variable.timestamp = timestamp
-    variable.tags = tags
-    variable.estimateDict = estimateDict
-    variable.solverDataDict = solverData
-    variable.smallData = smallData
-    variable.solvable = props["ready"]
-    variable.solveInProgress = props["backendset"]
+    variable = unpackVariable(dfg, props)
+    variable._internalId = variableId
 
     # Add to cache
     push!(dfg.variableCache, variable.label=>variable)
@@ -235,35 +194,14 @@ Get a DFGFactor from a DFG using its underlying integer ID.
 """
 function getFactor(dfg::CloudGraphsDFG, factorId::Int64)::DFGFactor
     props = getnodeproperties(dfg.neo4jInstance.graph, factorId)
-
-    label = props["label"]
-    tags = JSON2.read(props["tags"], Vector{Symbol})
-
-    data = props["data"]
-    datatype = props["fnctype"]
-    # fulltype = getfield(Main, Symbol(datatype))
-    packtype = getfield(Main, Symbol("Packed"*datatype))
-    packed = JSON2.read(data, GenericFunctionNodeData{packtype,String})
-    fullFactor = dfg.decodePackedTypeFunc(dfg, packed)
-
-    # Include the type
-    _variableOrderSymbols = JSON2.read(props["_variableOrderSymbols"], Vector{Symbol})
-    backendset = props["backendset"]
-    ready = props["ready"]
-
-    # Rebuild DFGVariable
-    factor = DFGFactor{typeof(fullFactor.fnc), Symbol}(Symbol(label), factorId)
-    factor.tags = tags
-    factor.data = fullFactor
-    factor._variableOrderSymbols = _variableOrderSymbols
-    factor.solvable = ready
-    factor.solveInProgress = backendset
+    factor = unpackFactor(dfg, props, getSerializationModule(dfg))
+    factor._internalId = factorId
 
     # Lastly, rebuild the metadata
     factor = dfg.rebuildFactorMetadata!(dfg, factor)
     # GUARANTEED never to bite us in the future...
     # ... TODO: refactor if changed: https://github.com/JuliaRobotics/IncrementalInference.jl/issues/350
-    solverData(factor).fncargvID = _variableOrderSymbols
+    solverData(factor).fncargvID = factor._variableOrderSymbols
 
     # Add to cache
     push!(dfg.factorCache, factor.label=>factor)
@@ -302,17 +240,7 @@ function updateVariable!(dfg::CloudGraphsDFG, variable::DFGVariable)::DFGVariabl
     variable._internalId = nodeId
 
     neo4jNode = Neo4j.getnode(dfg.neo4jInstance.graph, nodeId)
-    props = getnodeproperties(dfg.neo4jInstance.graph, nodeId)
-
-    props["label"] = string(variable.label)
-    props["timestamp"] = string(variable.timestamp)
-    props["tags"] = JSON2.write(variable.tags)
-    props["estimateDict"] = JSON2.write(variable.estimateDict)
-    props["solverDataDict"] = JSON2.write(Dict(keys(variable.solverDataDict) .=> map(vnd -> pack(dfg, vnd), values(variable.solverDataDict))))
-    props["smallData"] = JSON2.write(variable.smallData)
-    props["ready"] = variable.solvable
-    props["backendset"] = variable.solveInProgress
-    # Don't handle big data at the moment.
+    props = packVariable(dfg, variable)
 
     Neo4j.updatenodeproperties(neo4jNode, props)
     Neo4j.updatenodelabels(neo4jNode, union([string(variable.label), "VARIABLE", dfg.userId, dfg.robotId, dfg.sessionId], variable.tags))
@@ -349,21 +277,7 @@ function updateFactor!(dfg::CloudGraphsDFG, factor::DFGFactor)::DFGFactor
     # Update the _internalId
     factor._internalId = nodeId
     neo4jNode = Neo4j.getnode(dfg.neo4jInstance.graph, nodeId)
-    props = getnodeproperties(dfg.neo4jInstance.graph, nodeId)
-
-    props["label"] = string(factor.label)
-    props["tags"] = JSON2.write(factor.tags)
-    # Pack the node data
-    fnctype = factor.data.fnc.usrfnc!
-    packtype = getfield(_getmodule(fnctype), Symbol("Packed$(_getname(fnctype))"))
-    packed = convert(PackedFunctionNodeData{packtype}, factor.data)
-    props["data"] = JSON2.write(packed)
-    # Include the type
-    props["fnctype"] = String(_getname(fnctype))
-    props["_variableOrderSymbols"] = JSON2.write(factor._variableOrderSymbols)
-    props["backendset"] = factor.solveInProgress
-    props["ready"] = factor.solvable
-    # Don't handle big data at the moment.
+    props = packFactor(dfg, factor)
 
     Neo4j.updatenodeproperties(neo4jNode, props)
     Neo4j.updatenodelabels(neo4jNode, union([string(factor.label), "FACTOR", dfg.userId, dfg.robotId, dfg.sessionId], factor.tags))
@@ -597,10 +511,10 @@ function getNeighbors(dfg::CloudGraphsDFG, node::T; ready::Union{Nothing, Int}=n
     query = "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(node.label))--(node) where (node:VARIABLE or node:FACTOR) "
     if ready != nothing || backendset != nothing
         if ready != nothing
-            query = query * " and node.ready >= $(ready)"
+            query = query * " and node.solvable >= $(ready)"
         end
         if backendset != nothing
-            query = query * " and node.backendset = $(backendset)"
+            query = query * " and node.solveInProgress = $(backendset)"
         end
     end
     @debug "[Query] $query"
@@ -620,10 +534,10 @@ function getNeighbors(dfg::CloudGraphsDFG, label::Symbol; ready::Union{Nothing,
     query = "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(label))--(node) where (node:VARIABLE or node:FACTOR) "
     if ready != nothing || backendset != nothing
         if ready != nothing
-            query = query * " and node.ready >= $(ready)"
+            query = query * " and node.solvable >= $(ready)"
         end
         if backendset != nothing
-            query = query * " and node.backendset = $(backendset)"
+            query = query * " and node.solveInProgress = $(backendset)"
         end
     end
     @debug "[Query] $query"
@@ -756,32 +670,3 @@ function getAdjacencyMatrixSparse(dfg::CloudGraphsDFG)::Tuple{SparseMatrixCSC, V
 
     return adjMat, varLabels, factLabels
 end
-
-# """
-#     $(SIGNATURES)
-# Produces a dot-format of the graph for visualization.
-# """
-# function toDot(dfg::CloudGraphsDFG)::String
-#     m = PipeBuffer()
-#     write(m,Graphs.to_dot(dfg.g))
-#     data = take!(m)
-#     close(m)
-#     return String(data)
-# end
-#
-# """
-#     $(SIGNATURES)
-# Produces a dot file of the graph for visualization.
-# Download XDot to see the data
-#
-# Note
-# - Default location "/tmp/dfg.dot" -- MIGHT BE REMOVED
-# - Can be viewed with the `xdot` system application.
-# - Based on graphviz.org
-# """
-# function toDotFile(dfg::CloudGraphsDFG, fileName::String="/tmp/dfg.dot")::Nothing
-#     open(fileName, "w") do fid
-#         write(fid,Graphs.to_dot(dfg.g))
-#     end
-#     return nothing
-# end

src/DFGPlots/DFGPlots.jl

@@ -2,7 +2,6 @@ module DFGPlots
 
 using Colors
 using LightGraphs
-using MetaGraphs
 using GraphPlot
 using DocStringExtensions
 import GraphPlot: gplot
@@ -84,39 +83,6 @@ using Compose
 draw(PDF("/tmp/graph.pdf", 16cm, 16cm), dfgplot(fg))
 ```
 
-More information at [GraphPlot.jl](https://github.com/JuliaGraphs/GraphPlot.jl)
-"""
-function dfgplot(dfg::DistributedFactorGraphs.MetaGraphsDFG, p::DFGPlotProps = DFGPlotProps())
-    @info "Deprecated, please use GraphsDFG or LightDFG."
-    nodesize = [has_prop(dfg.g,i,:factor) ?  p.nodesize.fac : p.nodesize.var for i=vertices(dfg.g)]
-    if p.drawlabels
-        nodelabel = [has_prop(dfg.g,i,:factor) ? "" : string(get_prop(dfg.g,i,:label)) for i=vertices(dfg.g)]
-    else
-        nodelabel = nothing
-    end
-    nodefillc = [has_prop(dfg.g,i,:factor) ? p.nodefillc.fac : p.nodefillc.var for i=vertices(dfg.g)]
-
-    gplot(dfg.g, nodelabel=nodelabel, nodesize=nodesize, nodefillc=nodefillc, layout=p.layout)
-
-end
-
-"""
-    $(SIGNATURES)
-Plots the structure of the factor graph. GraphPlot must be imported before DistributedFactoGraphs for these functions to be available.
-Returns the plot context.
-
-E.g.
-```
-using GraphPlot
-using DistributedFactorGraphs, DistributedFactorGraphs.DFGPlots
-# ... Make graph...
-# Using GraphViz plotting
-dfgplot(fg)
-# Save to PDFG
-using Compose
-draw(PDF("/tmp/graph.pdf", 16cm, 16cm), dfgplot(fg))
-```
-
 More information at [GraphPlot.jl](https://github.com/JuliaGraphs/GraphPlot.jl)
 """
 function dfgplot(dfg::AbstractDFG, p::DFGPlotProps = DFGPlotProps())
@@ -128,11 +94,6 @@ function dfgplot(dfg::AbstractDFG, p::DFGPlotProps = DFGPlotProps())
     dfgplot(ldfg, p)
 end
 
-function gplot(dfg::DistributedFactorGraphs.MetaGraphsDFG; keyargs...)
-    @info "Deprecated, please use GraphsDFG or LightDFG."
-    gplot(dfg.g; keyargs...)
-end
-
 function gplot(dfg::LightDFG; keyargs...)
     gplot(dfg.g; keyargs...)
 end

src/DistributedFactorGraphs.jl

@@ -9,6 +9,8 @@ using Reexport
 using JSON2
 using LinearAlgebra
 using SparseArrays
+# This is used in the definition of getAdjacencyMatrixSparse
+using LightGraphs
 
 # Entities
 include("entities/AbstractDFG.jl")
@@ -87,9 +89,6 @@ include("GraphsDFG/GraphsDFG.jl")
 # Include the FilesDFG API.
 include("FileDFG/FileDFG.jl")
 
-# Include the LightGraphs.jl (MetaGraphs.jl) API.
-include("MetaGraphsDFG/MetaGraphsDFG.jl")
-
 include("SymbolDFG/SymbolDFG.jl")
 using .SymbolDFGs
 

src/SymbolDFG/services/SymbolDFG.jl

@@ -459,38 +459,6 @@ function getAdjacencyMatrixSparse(dfg::SymbolDFG; solvable::Int=0)::Tuple{LightG
 	end
 	return adjMat, varLabels, factLabels
 end
-#=
-"""
-    $(SIGNATURES)
-Produces a dot-format of the graph for visualization.
-"""
-function toDot(dfg::SymbolDFG)::String
-	@error "toDot(dfg::SymbolDFG) is not sopported yet, see https://github.com/JuliaGraphs/MetaGraphs.jl/issues/86"
-    m = PipeBuffer()
-    MetaGraphs.savedot(m, dfg.g)
-    data = take!(m)
-    close(m)
-    return String(data)
-end
-
-"""
-    $(SIGNATURES)
-Produces a dot file of the graph for visualization.
-Download XDot to see the data
-
-Note
-- Default location "/tmp/dfg.dot" -- MIGHT BE REMOVED
-- Can be viewed with the `xdot` system application.
-- Based on graphviz.org
-"""
-function toDotFile(dfg::SymbolDFG, fileName::String="/tmp/dfg.dot")::Nothing
-	@error "toDotFile(dfg::SymbolDFG,filename) is not sopported yet, see https://github.com/JuliaGraphs/MetaGraphs.jl/issues/86"
-    open(fileName, "w") do fid
-        MetaGraphs.savedot(fid, dfg.g)
-    end
-    return nothing
-end
-=#
 
 """
     $(SIGNATURES)

src/attic/MetaGraphsDFG/DFGPlots.jl

@@ -0,0 +1,37 @@
+"""
+    $(SIGNATURES)
+Plots the structure of the factor graph. GraphPlot must be imported before DistributedFactoGraphs for these functions to be available.
+Returns the plot context.
+
+E.g.
+```
+using GraphPlot
+using DistributedFactorGraphs, DistributedFactorGraphs.DFGPlots
+# ... Make graph...
+# Using GraphViz plotting
+dfgplot(fg)
+# Save to PDFG
+using Compose
+draw(PDF("/tmp/graph.pdf", 16cm, 16cm), dfgplot(fg))
+```
+
+More information at [GraphPlot.jl](https://github.com/JuliaGraphs/GraphPlot.jl)
+"""
+function dfgplot(dfg::DistributedFactorGraphs.MetaGraphsDFG, p::DFGPlotProps = DFGPlotProps())
+    @info "Deprecated, please use GraphsDFG or LightDFG."
+    nodesize = [has_prop(dfg.g,i,:factor) ?  p.nodesize.fac : p.nodesize.var for i=vertices(dfg.g)]
+    if p.drawlabels
+        nodelabel = [has_prop(dfg.g,i,:factor) ? "" : string(get_prop(dfg.g,i,:label)) for i=vertices(dfg.g)]
+    else
+        nodelabel = nothing
+    end
+    nodefillc = [has_prop(dfg.g,i,:factor) ? p.nodefillc.fac : p.nodefillc.var for i=vertices(dfg.g)]
+
+    gplot(dfg.g, nodelabel=nodelabel, nodesize=nodesize, nodefillc=nodefillc, layout=p.layout)
+
+end
+
+function gplot(dfg::DistributedFactorGraphs.MetaGraphsDFG; keyargs...)
+    @info "Deprecated, please use GraphsDFG or LightDFG."
+    gplot(dfg.g; keyargs...)
+end

src/MetaGraphsDFG/MetaGraphsDFG.jl renamed to src/attic/MetaGraphsDFG/MetaGraphsDFG.jl

@@ -4,6 +4,7 @@ using MetaGraphs
 # Imports
 include("entities/MetaGraphsDFG.jl")
 include("services/MetaGraphsDFG.jl")
+include("DFGPlots.jl")
 
 # Exports
 # Deprecated -