Merge branch 'feature/summaryfunction' of https://github.com/JuliaRobotics/DistributedFactorGraphs.jl.git into jt/develop

Author: Affie

Project.toml

@@ -4,10 +4,10 @@ version = "0.4.0"
 
 [deps]
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
-DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231"
 Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 JSON2 = "2535ab7d-5cd8-5a07-80ac-9b1792aadce3"
 LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d"
@@ -29,6 +29,7 @@ julia = "0.7, 1"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231"
 
 [targets]
-test = ["Test"]
+test = ["Test", "GraphPlot"]

docs/src/func_ref.md

@@ -58,13 +58,15 @@ getSubgraphAroundNode
 getSubgraph
 ```
 
-### Visualization
+### Summaries
 ```@docs
-toDot
-toDotFile
+getSummary
+getSummaryGraph
 ```
 
-### DataFrame Extension Functions
+### Visualization and Plotting
 ```@docs
-getAdjacencyMatrixDataFrame
+toDot
+toDotFile
+dfgplot
 ```

src/CloudGraphsDFG/CloudGraphsDFG.jl

@@ -17,10 +17,9 @@ export getAddHistory, getDescription, getLabelDict
 export addVariable!, addFactor!
 export ls, lsf, getVariables, getFactors, getVariableIds, getFactorIds
 export getVariable, getFactor
-export updateVariable!, updateFactor!
+export updateVariable!, updateFactor!, updateVariableSolverData!
 export deleteVariable!, deleteFactor!
 export getAdjacencyMatrix
-export getAdjacencyMatrixDataFrame
 export getNeighbors
 export getSubgraphAroundNode
 export getSubgraph

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -63,8 +63,8 @@ getDescription(dfg::CloudGraphsDFG) = dfg.description
 setDescription(dfg::CloudGraphsDFG, description::String) = dfg.description = description
 getAddHistory(dfg::CloudGraphsDFG) = dfg.addHistory
 getSolverParams(dfg::CloudGraphsDFG) = dfg.solverParams
-function setSolverParams(dfg::CloudGraphsDFG, solverParams::T) where T <: AbstractParams
-    dfg.solverParams = solverParams
+function setSolverParams(dfg::CloudGraphsDFG, solverParams::T)::T where T <: AbstractParams
+    return dfg.solverParams = solverParams
 end
 
 """
@@ -255,7 +255,7 @@ function getVariable(dfg::CloudGraphsDFG, variableId::Int64)::DFGVariable
     # props["label"] = Symbol(variable.label)
     timestamp = DateTime(props["timestamp"])
     tags =  JSON2.read(props["tags"], Vector{Symbol})
-    estimateDict = JSON2.read(props["estimateDict"], Dict{Symbol, VariableEstimate})
+    estimateDict = JSON2.read(props["estimateDict"], Dict{Symbol, Dict{Symbol, VariableEstimate}})
     smallData = nothing
     smallData = JSON2.read(props["smallData"], Dict{String, String})
 
@@ -389,6 +389,20 @@ function updateVariable!(dfg::CloudGraphsDFG, variable::DFGVariable)::DFGVariabl
     return variable
 end
 
+"""
+    $(SIGNATURES)
+Update solver and estimate data for a variable (variable can be from another graph).
+"""
+function updateVariableSolverData!(dfg::CloudGraphsDFG, sourceVariable::DFGVariable)::DFGVariable
+    if !exists(dfg, sourceVariable)
+        error("Source variable '$(sourceVariable.label)' doesn't exist in the graph.")
+    end
+    nodeId = _tryGetNeoNodeIdFromNodeLabel(dfg.neo4jInstance, dfg.userId, dfg.robotId, dfg.sessionId, sourceVariable.label)
+    Neo4j.setnodeproperty(dfg.neo4jInstance.graph, nodeId, "estimateDict", JSON2.write(sourceVariable.estimateDict))
+    Neo4j.setnodeproperty(dfg.neo4jInstance.graph, nodeId, "solverDataDict", JSON2.write(Dict(keys(sourceVariable.solverDataDict) .=> map(vnd -> pack(dfg, vnd), values(sourceVariable.solverDataDict)))))
+    return sourceVariable
+end
+
 """
     $(SIGNATURES)
 Update a complete DFGFactor in the DFG.
@@ -704,25 +718,11 @@ Optionally provide a distance to specify the number of edges should be followed.
 Optionally provide an existing subgraph addToDFG, the extracted nodes will be copied into this graph. By default a new subgraph will be created.
 Note: By default orphaned factors (where the subgraph does not contain all the related variables) are not returned. Set includeOrphanFactors to return the orphans irrespective of whether the subgraph contains all the variables.
 """
-function getSubgraphAroundNode(dfg::CloudGraphsDFG, node::DFGNode, distance::Int64=1, includeOrphanFactors::Bool=false, addToDFG::Union{Nothing, CloudGraphsDFG}=nothing)::CloudGraphsDFG
+function getSubgraphAroundNode(dfg::CloudGraphsDFG, node::DFGNode, distance::Int64=1, includeOrphanFactors::Bool=false, addToDFG::AbstractDFG=_getDuplicatedEmptyDFG(dfg))::AbstractDFG
     distance < 1 && error("getSubgraphAroundNode() only works for distance > 0")
 
     # Making a copy session if not specified
-    if addToDFG == nothing
-        addToDFG = _getDuplicatedEmptyDFG(dfg)
-    end
-
-    # Thank you Neo4j for 0..* awesomeness!!
-    neighborList = _getLabelsFromCyphonQuery(dfg.neo4jInstance, "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(node.label))-[FACTORGRAPH*0..$distance]-(node:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId))")
-
-    # Copy the section of graph we want
-    _copyIntoGraph!(dfg, addToDFG, neighborList, includeOrphanFactors)
-    return addToDFG
-end
-
-function getSubgraphAroundNode(dfg::CloudGraphsDFG{<:AbstractParams}, node::DFGNode, distance::Int64, includeOrphanFactors::Bool, addToDFG::MetaGraphsDFG{AbstractParams})::AbstractDFG
-    distance < 1 && error("getSubgraphAroundNode() only works for distance > 0")
-
+    #moved to parameter addToDFG::AbstractDFG=_getDuplicatedEmptyDFG(dfg)
 
     # Thank you Neo4j for 0..* awesomeness!!
     neighborList = _getLabelsFromCyphonQuery(dfg.neo4jInstance, "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(node.label))-[FACTORGRAPH*0..$distance]-(node:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId))")
@@ -739,11 +739,11 @@ Get a deep subgraph copy from the DFG given a list of variables and factors.
 Optionally provide an existing subgraph addToDFG, the extracted nodes will be copied into this graph. By default a new subgraph will be created.
 Note: By default orphaned factors (where the subgraph does not contain all the related variables) are not returned. Set includeOrphanFactors to return the orphans irrespective of whether the subgraph contains all the variables.
 """
-function getSubgraph(dfg::CloudGraphsDFG, variableFactorLabels::Vector{Symbol}, includeOrphanFactors::Bool=false, addToDFG::Union{Nothing, CloudGraphsDFG}=nothing)::CloudGraphsDFG
+function getSubgraph(dfg::CloudGraphsDFG,
+                     variableFactorLabels::Vector{Symbol},
+                     includeOrphanFactors::Bool=false,
+                     addToDFG::G=_getDuplicatedEmptyDFG(dfg) )::G where {G <: AbstractDFG}
     # Making a copy session if not specified
-    if addToDFG == nothing
-        addToDFG = _getDuplicatedEmptyDFG(dfg)
-    end
 
     _copyIntoGraph!(dfg, addToDFG, variableFactorLabels, includeOrphanFactors)
 
@@ -787,6 +787,33 @@ function getAdjacencyMatrix(dfg::CloudGraphsDFG)::Matrix{Union{Nothing, Symbol}}
     return adjMat
 end
 
+function getAdjacencyMatrixSparse(dfg::CloudGraphsDFG)::Tuple{SparseMatrixCSC, Vector{Symbol}, Vector{Symbol}}
+    varLabels = getVariableIds(dfg)
+    factLabels = sort(getFactorIds(dfg))
+    vDict = Dict(varLabels .=> [1:length(varLabels)...].+1)
+    fDict = Dict(factLabels .=> [1:length(factLabels)...].+1)
+
+    adjMat = spzeros(Int, length(factLabels)+1, length(varLabels)+1)
+
+    # Now ask for all relationships for this session graph
+    loadtx = transaction(dfg.neo4jInstance.connection)
+    query = "START n=node(*) MATCH (n:VARIABLE:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId))-[r:FACTORGRAPH]-(m:FACTOR:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId)) RETURN n.label as variable, m.label as factor;"
+    nodes = loadtx(query; submit=true)
+    # Have to finish the transaction
+    commit(loadtx)
+    if length(nodes.errors) > 0
+        error(string(nodes.errors))
+    end
+    # Add in the relationships
+    varRel = Symbol.(map(node -> node["row"][1], nodes.results[1]["data"]))
+    factRel = Symbol.(map(node -> node["row"][2], nodes.results[1]["data"]))
+    for i = 1:length(varRel)
+        adjMat[fDict[factRel[i]], vDict[varRel[i]]] = 1
+    end
+
+    return adjMat, varLabels, factLabels
+end
+
 # """
 #     $(SIGNATURES)
 # Produces a dot-format of the graph for visualization.
@@ -815,30 +842,3 @@ end
 #     end
 #     return nothing
 # end
-#
-# function __init__()
-#     @require DataFrames="a93c6f00-e57d-5684-b7b6-d8193f3e46c0" begin
-#         if isdefined(Main, :DataFrames)
-#             """
-#                 $(SIGNATURES)
-#             Get an adjacency matrix for the DFG as a DataFrame.
-#             Rows are all factors, columns are all variables, and each cell contains either nothing or the symbol of the relating factor.
-#             The first column is the factor headings.
-#             """
-#             function getAdjacencyMatrixDataFrame(dfg::CloudGraphsDFG)::Main.DataFrames.DataFrame
-#                 varLabels = sort(map(v->v.label, getVariables(dfg)))
-#                 factLabels = sort(map(f->f.label, getFactors(dfg)))
-#                 adjDf = DataFrames.DataFrame(:Factor => Union{Missing, Symbol}[])
-#                 for varLabel in varLabels
-#                     adjDf[varLabel] = Union{Missing, Symbol}[]
-#                 end
-#                 for (i, factLabel) in enumerate(factLabels)
-#                     push!(adjDf, [factLabel, DataFrames.missings(length(varLabels))...])
-#                     factVars = getNeighbors(dfg, getFactor(dfg, factLabel))
-#                     map(vLabel -> adjDf[vLabel][i] = factLabel, factVars)
-#                 end
-#                 return adjDf
-#             end
-#         end
-#     end
-# end

src/Common.jl

@@ -1,12 +1,19 @@
 
-export sortVarNested
+export sortVarNested, sortDFG
 export isPrior, lsfPriors
-export getData
 export getVariableType, getSofttype
 export getFactorType, getfnctype
 export lsTypes, lsfTypes
 export lsWho, lsfWho
 
+## Utility functions for getting type names and modules (from IncrementalInference)
+function _getmodule(t::T) where T
+  T.name.module
+end
+function _getname(t::T) where T
+  T.name.name
+end
+
 """
     $(SIGNATURES)
 Test if all elements of the string is a number:  Ex, "123" is true, "1_2" is false.
@@ -85,10 +92,20 @@ end
 """
     $SIGNATURES
 
-Retrieve data structure stored in a node.
+Sort variable (factor) lists in a meaningful way, for example `[:april;:x1_3;:x1_6;]`
+
+Notes
+- Not fool proof, but does better than native sort.
+
+Example
+
+`sortDFG(ls(dfg))`
+
+Related
+
+ls, lsf
 """
-getData(v::DFGFactor)::GenericFunctionNodeData = v.data
-getData(v::DFGVariable; solveKey::Symbol=:default)::VariableNodeData = v.solverDataDict[solveKey]
+sortDFG(vars::Vector{Symbol})::Vector{Symbol} = sortVarNested(vars)
 
 """
     $SIGNATURES
@@ -232,10 +249,38 @@ function lsTypes(dfg::G)::Dict{Symbol, Vector{String}} where G <: AbstractDFG
 end
 
 
+function ls(dfg::G, ::Type{T}; solveKey::Symbol=:default) where {G <: AbstractDFG, T <: InferenceVariable}
+  xx = getVariables(dfg)
+  mask = getVariableType.(xx, solveKey=solveKey) .|> typeof .== T
+  vxx = view(xx, mask)
+  map(x->x.label, vxx)
+end
+
+
+function ls(dfg::G, ::Type{T}) where {G <: AbstractDFG, T <: FunctorInferenceType}
+  xx = getFactors(dfg)
+  names = getfield.(typeof.(getFactorType.(xx)), :name) .|> Symbol
+  vxx = view(xx, names .== Symbol(T))
+  map(x->x.label, vxx)
+end
+
+function lsf(dfg::G, ::Type{T}) where {G <: AbstractDFG, T <: FunctorInferenceType}
+  ls(dfg, T)
+end
+
+
 """
     $(SIGNATURES)
 Gives back all factor labels that fit the bill:
 	lsWho(dfg, :Pose3)
+
+Dev Notes
+- Cloud versions will benefit from less data transfer
+ - `ls(dfg::C, ::T) where {C <: CloudDFG, T <: ..}`
+
+Related
+
+ls, lsf, lsfPriors
 """
 function lsWho(dfg::AbstractDFG, type::Symbol; solveKey::Symbol=:default)::Vector{Symbol}
     vars = getVariables(dfg)
@@ -252,6 +297,14 @@ end
     $(SIGNATURES)
 Gives back all factor labels that fit the bill:
 	lsfWho(dfg, :Point2Point2)
+
+Dev Notes
+- Cloud versions will benefit from less data transfer
+ - `ls(dfg::C, ::T) where {C <: CloudDFG, T <: ..}`
+
+Related
+
+ls, lsf, lsfPriors
 """
 function lsfWho(dfg::AbstractDFG, type::Symbol)::Vector{Symbol}
 	facs = getFactors(dfg)

src/DFGPlots/DFGPlots.jl

@@ -4,6 +4,7 @@ using Colors
 using LightGraphs
 using MetaGraphs
 using GraphPlot
+using DocStringExtensions
 import GraphPlot: gplot
 
 using ...DistributedFactorGraphs
@@ -28,6 +29,26 @@ DFGPlotProps() = DFGPlotProps(  (var=colorant"seagreen", fac=colorant"cyan3"),
                                 true)
 
 
+
+"""
+    $(SIGNATURES)
+Plots the structure of the factor graph. GraphPlot must be imported before DistributedFactoGraphs for these functions to be available.
+Returns the plot context.
+
+E.g.
+```
+using GraphPlot
+using DistributedFactorGraphs, DistributedFactorGraphs.DFGPlots
+# ... Make graph...
+# Using GraphViz plotting
+dfgplot(fg)
+# Save to PDFG
+using Compose
+draw(PDF("/tmp/graph.pdf", 16cm, 16cm), dfgplot(fg))
+```
+
+More information at [GraphPlot.jl](https://github.com/JuliaGraphs/GraphPlot.jl)
+"""
 function dfgplot(dfg::LightDFG, p::DFGPlotProps = DFGPlotProps())
 
     nodetypes = [haskey(dfg.g.variables, s) for s in dfg.g.labels]
@@ -46,6 +67,25 @@ function dfgplot(dfg::LightDFG, p::DFGPlotProps = DFGPlotProps())
 
 end
 
+"""
+    $(SIGNATURES)
+Plots the structure of the factor graph. GraphPlot must be imported before DistributedFactoGraphs for these functions to be available.
+Returns the plot context.
+
+E.g.
+```
+using GraphPlot
+using DistributedFactorGraphs, DistributedFactorGraphs.DFGPlots
+# ... Make graph...
+# Using GraphViz plotting
+dfgplot(fg)
+# Save to PDFG
+using Compose
+draw(PDF("/tmp/graph.pdf", 16cm, 16cm), dfgplot(fg))
+```
+
+More information at [GraphPlot.jl](https://github.com/JuliaGraphs/GraphPlot.jl)
+"""
 function dfgplot(dfg::MetaGraphsDFG, p::DFGPlotProps = DFGPlotProps())
 
     nodesize = [has_prop(dfg.g,i,:factor) ?  p.nodesize.fac : p.nodesize.var for i=vertices(dfg.g)]
@@ -60,14 +100,32 @@ function dfgplot(dfg::MetaGraphsDFG, p::DFGPlotProps = DFGPlotProps())
 
 end
 
+"""
+    $(SIGNATURES)
+Plots the structure of the factor graph. GraphPlot must be imported before DistributedFactoGraphs for these functions to be available.
+Returns the plot context.
+
+E.g.
+```
+using GraphPlot
+using DistributedFactorGraphs, DistributedFactorGraphs.DFGPlots
+# ... Make graph...
+# Using GraphViz plotting
+dfgplot(fg)
+# Save to PDFG
+using Compose
+draw(PDF("/tmp/graph.pdf", 16cm, 16cm), dfgplot(fg))
+```
+
+More information at [GraphPlot.jl](https://github.com/JuliaGraphs/GraphPlot.jl)
+"""
 function dfgplot(dfg::AbstractDFG, p::DFGPlotProps = DFGPlotProps())
     # TODO implement convert functions
     @warn "TODO Implement convert"
     ldfg = MetaGraphsDFG{AbstractParams}()
     DistributedFactorGraphs._copyIntoGraph!(dfg, ldfg, union(getVariableIds(dfg), getFactorIds(dfg)), true)
 
     dfgplot(ldfg, p)
-
 end
 
 function gplot(dfg::MetaGraphsDFG; keyargs...)