Merge branch 'master' into 2Q20/378_cloudsolving

Author: Affie

Project.toml

@@ -31,7 +31,7 @@ LightGraphs = "1.2, 1.3"
 Neo4j = "2"
 Reexport = "0.2, 0.3, 0.4, 0.5, 1"
 Requires = "0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 1"
-Unmarshal = "0.3"
+Unmarshal = "0.3, 0.4"
 julia = "1.2"
 
 [extras]

src/CloudGraphsDFG/entities/CloudGraphsDFG.jl

@@ -10,6 +10,7 @@ mutable struct CloudGraphsDFG{T <: AbstractParams} <: AbstractDFG
     userId::String
     robotId::String
     sessionId::String
+    description::String
     encodePackedTypeFunc
     getPackedTypeFunc
     decodePackedTypeFunc
@@ -26,38 +27,41 @@ function CloudGraphsDFG{T}(neo4jConnection::Neo4j.Connection,
                             userId::String,
                             robotId::String,
                             sessionId::String,
+                            description::String,
                             encodePackedTypeFunc,
                             getPackedTypeFunc,
                             decodePackedTypeFunc,
                             rebuildFactorMetadata!;
                             solverParams::T=NoSolverParams()) where T <: AbstractParams
     graph = Neo4j.getgraph(neo4jConnection)
     neo4jInstance = Neo4jInstance(neo4jConnection, graph)
-    return CloudGraphsDFG{T}(neo4jInstance, userId, robotId, sessionId, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc, rebuildFactorMetadata!, Symbol[], solverParams)
+    return CloudGraphsDFG{T}(neo4jInstance, userId, robotId, sessionId, description, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc, rebuildFactorMetadata!, Symbol[], solverParams)
 end
 """
     $(SIGNATURES)
 Create a new CloudGraphs-based DFG factor graph by specifying the Neo4j connection information.
 """
 function CloudGraphsDFG{T}(host::String,
-                            port::Int,
-                            dbUser::String,
-                            dbPassword::String,
-                            userId::String,
-                            robotId::String,
-                            sessionId::String,
-                            encodePackedTypeFunc,
-                            getPackedTypeFunc,
-                            decodePackedTypeFunc,
-                            rebuildFactorMetadata!;
-                            solverParams::T=NoSolverParams()) where T <: AbstractParams
+                           port::Int,
+                           dbUser::String,
+                           dbPassword::String,
+                           userId::String,
+                           robotId::String,
+                           sessionId::String,
+                           description::String,
+                           encodePackedTypeFunc,
+                           getPackedTypeFunc,
+                           decodePackedTypeFunc,
+                           rebuildFactorMetadata!;
+                           solverParams::T=NoSolverParams()) where T <: AbstractParams
     neo4jConnection = Neo4j.Connection(host, port=port, user=dbUser, password=dbPassword);
-    return CloudGraphsDFG{T}(neo4jConnection, userId, robotId, sessionId, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc, rebuildFactorMetadata!, solverParams=solverParams)
+    return CloudGraphsDFG{T}(neo4jConnection, userId, robotId, sessionId, description, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc, rebuildFactorMetadata!, solverParams=solverParams)
 end
 
 
 function show(io::IO, c::CloudGraphsDFG)
     println(io, "CloudGraphsDFG:")
     println(io, " - Neo4J instance: $(c.neo4jInstance.connection.host)")
     println(io, " - Session: $(c.userId):$(c.robotId):$(c.sessionId)")
+    println(io, " - Description: ", c.description)
 end

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -17,6 +17,7 @@ function _getDuplicatedEmptyDFG(dfg::CloudGraphsDFG)::CloudGraphsDFG
         dfg.userId,
         dfg.robotId,
         sessionId,
+        dfg.description,
         dfg.encodePackedTypeFunc,
         dfg.getPackedTypeFunc,
         dfg.decodePackedTypeFunc,
@@ -32,11 +33,13 @@ function getDescription(dfg::CloudGraphsDFG)
         "description")
 end
 function setDescription!(dfg::CloudGraphsDFG, description::String)
-    _setNodeProperty(
-        dfg.neo4jInstance,
-        [dfg.sessionId, dfg.robotId, dfg.userId, "SESSION"],
-        "description",
-        description)
+    count = _setNodeProperty(dfg.neo4jInstance,
+                            [dfg.sessionId, dfg.robotId, dfg.userId, "SESSION"],
+                            "description",
+                            description)
+    @assert(count == 1)
+    dfg.description = description
+    return getDescription(dfg)
 end
 
 function getSerializationModule(dfg::CloudGraphsDFG)::Module where G <: AbstractDFG
@@ -53,43 +56,89 @@ function getUserData(dfg::CloudGraphsDFG)::Dict{Symbol, String}
     propVal = _getNodeProperty(dfg.neo4jInstance, _getLabelsForType(dfg, User), "data")
     return JSON2.read(String(base64decode(propVal)), Dict{Symbol, String})
 end
-function setUserData!(dfg::CloudGraphsDFG, data::Dict{Symbol, String})::Bool
+function setUserData!(dfg::CloudGraphsDFG, data::Dict{Symbol, String})::Dict{Symbol, String}
     count = _setNodeProperty(dfg.neo4jInstance, [dfg.userId, "USER"], "data", base64encode(JSON2.write(data)))
-    return count == 1
+    @assert(count == 1)
+    return  getUserData(dfg)
 end
+
 function getRobotData(dfg::CloudGraphsDFG)::Dict{Symbol, String}
     propVal = _getNodeProperty(dfg.neo4jInstance, [dfg.userId, dfg.robotId, "ROBOT"], "data")
     return JSON2.read(String(base64decode(propVal)), Dict{Symbol, String})
 end
-function setRobotData!(dfg::CloudGraphsDFG, data::Dict{Symbol, String})::Bool
+function setRobotData!(dfg::CloudGraphsDFG, data::Dict{Symbol, String})::Dict{Symbol, String}
     count = _setNodeProperty(dfg.neo4jInstance, [dfg.userId, dfg.robotId, "ROBOT"], "data", base64encode(JSON2.write(data)))
-    return count == 1
+    @assert(count == 1)
+    return  getRobotData(dfg)
 end
+
 function getSessionData(dfg::CloudGraphsDFG)::Dict{Symbol, String}
     propVal = _getNodeProperty(dfg.neo4jInstance, [dfg.userId, dfg.robotId, dfg.sessionId, "SESSION"], "data")
     return JSON2.read(String(base64decode(propVal)), Dict{Symbol, String})
 end
-function setSessionData!(dfg::CloudGraphsDFG, data::Dict{Symbol, String})::Bool
+function setSessionData!(dfg::CloudGraphsDFG, data::Dict{Symbol, String})::Dict{Symbol, String}
     count = _setNodeProperty(dfg.neo4jInstance, [dfg.userId, dfg.robotId, dfg.sessionId, "SESSION"], "data", base64encode(JSON2.write(data)))
-    return count == 1
+    @assert(count == 1)
+    return  getSessionData(dfg)
 end
 
 # New API
-getUserData(dfg::CloudGraphsDFG, key::Symbol)::String = error("Not supported yet")
-getRobotData(dfg::CloudGraphsDFG, key::Symbol)::String = error("Not supported yet")
-getSessionData(dfg::CloudGraphsDFG, key::Symbol)::String = error("Not supported yet")
+getUserData(dfg::CloudGraphsDFG, key::Symbol) = getUserData(dfg::CloudGraphsDFG)[key]
+getRobotData(dfg::CloudGraphsDFG, key::Symbol)::String = getRobotData(dfg::CloudGraphsDFG)[key]
+getSessionData(dfg::CloudGraphsDFG, key::Symbol)::String = getSessionData(dfg::CloudGraphsDFG)[key]
+
+function updateUserData!(dfg::CloudGraphsDFG, pair::Pair{Symbol,String})
+    data = getUserData(dfg::CloudGraphsDFG)
+    push!(data, pair)
+    setUserData!(dfg, data)
+end
+
+function updateRobotData!(dfg::CloudGraphsDFG, pair::Pair{Symbol,String})
+    data = getRobotData(dfg::CloudGraphsDFG)
+    push!(data, pair)
+    setRobotData!(dfg, data)
+end
+
+function updateSessionData!(dfg::CloudGraphsDFG, pair::Pair{Symbol,String})
+    data = getSessionData(dfg::CloudGraphsDFG)
+    push!(data, pair)
+    setSessionData!(dfg, data)
+end
+
+function deleteUserData!(dfg::CloudGraphsDFG, key::Symbol)
+    data = getUserData(dfg::CloudGraphsDFG)
+    delval = pop!(data, key)
+    setUserData!(dfg, data)
+    return delval
+end
+
+function deleteRobotData!(dfg::CloudGraphsDFG, key::Symbol)
+    data = getRobotData(dfg::CloudGraphsDFG)
+    delval = pop!(data, key)
+    setRobotData!(dfg, data)
+    return delval
+end
 
-updateUserData!(dfg::CloudGraphsDFG, pair::Pair{Symbol,String}) = error("Not supported yet")
-updateRobotData!(dfg::CloudGraphsDFG, pair::Pair{Symbol,String}) = error("Not supported yet")
-updateSessionData!(dfg::CloudGraphsDFG, pair::Pair{Symbol,String}) = error("Not supported yet")
+function deleteSessionData!(dfg::CloudGraphsDFG, key::Symbol)
+    data = getSessionData(dfg::CloudGraphsDFG)
+    delval = pop!(data, key)
+    setSessionData!(dfg, data)
+    return delval
+end
+
+
+function emptyUserData!(dfg::CloudGraphsDFG)
+    return setUserData!(dfg, Dict{Symbol, String}())
+end
 
-deleteUserData!(dfg::CloudGraphsDFG, key::Symbol) = error("Not supported yet")
-deleteRobotData!(dfg::CloudGraphsDFG, key::Symbol) = error("Not supported yet")
-deleteSessionData!(dfg::CloudGraphsDFG, key::Symbol) = error("Not supported yet")
+function emptyRobotData!(dfg::CloudGraphsDFG)
+    return setRobotData!(dfg, Dict{Symbol, String}())
+end
+
+function emptySessionData!(dfg::CloudGraphsDFG)
+    return setSessionData!(dfg, Dict{Symbol, String}())
+end
 
-emptyUserData!(dfg::CloudGraphsDFG) = error("Not supported yet")
-emptyRobotData!(dfg::CloudGraphsDFG) = error("Not supported yet")
-emptySessionData!(dfg::CloudGraphsDFG) = error("Not supported yet")
 
 ##==============================================================================
 ## CRUD Interfaces
@@ -132,17 +181,15 @@ function addVariable!(dfg::CloudGraphsDFG, variable::DFGVariable)::DFGVariable
     return variable
 end
 
-function addFactor!(dfg::CloudGraphsDFG, factor::DFGFactor)
-    addFactor!(dfg, factor._variableOrderSymbols, factor)
-end
 
-function addFactor!(dfg::CloudGraphsDFG, variables::Vector{<:DFGVariable}, factor::DFGFactor)::DFGFactor
+function addFactor!(dfg::CloudGraphsDFG, factor::DFGFactor)::DFGFactor
     if exists(dfg, factor)
         error("Factor '$(factor.label)' already exists in the factor graph")
     end
 
-    # Update the variable ordering
-    factor._variableOrderSymbols = map(v->v.label, variables)
+    variableIds = factor._variableOrderSymbols
+    variables = map(vId -> getVariable(dfg, vId), variableIds)
+
 
     # Construct the properties to save
     props = packFactor(dfg, factor)
@@ -163,16 +210,11 @@ function addFactor!(dfg::CloudGraphsDFG, variables::Vector{<:DFGVariable}, facto
     return factor
 end
 
-function addFactor!(dfg::CloudGraphsDFG, variableIds::Vector{Symbol}, factor::DFGFactor)::DFGFactor
-    variables = map(vId -> getVariable(dfg, vId), variableIds)
-    return addFactor!(dfg, variables, factor)
-end
-
 function getVariable(dfg::CloudGraphsDFG, label::Union{Symbol, String})::DFGVariable
     if typeof(label) == String
         label = Symbol(label)
     end
-    nodeId = _tryGetNeoNodeIdFromNodeLabel(dfg.neo4jInstance, dfg.userId, dfg.robotId, dfg.sessionId, label)
+    nodeId = _tryGetNeoNodeIdFromNodeLabel(dfg.neo4jInstance, dfg.userId, dfg.robotId, dfg.sessionId, label, nodeType="VARIABLE")
     if nodeId == nothing
         error("Unable to retrieve the ID for variable '$label'. Please check your connection to the database and that the variable exists.")
     end
@@ -191,7 +233,7 @@ function getFactor(dfg::CloudGraphsDFG, label::Union{Symbol, String})::DFGFactor
     if typeof(label) == String
         label = Symbol(label)
     end
-    nodeId = _tryGetNeoNodeIdFromNodeLabel(dfg.neo4jInstance, dfg.userId, dfg.robotId, dfg.sessionId, label)
+    nodeId = _tryGetNeoNodeIdFromNodeLabel(dfg.neo4jInstance, dfg.userId, dfg.robotId, dfg.sessionId, label, nodeType="FACTOR")
     if nodeId == nothing
         error("Unable to retrieve the ID for factor '$label'. Please check your connection to the database and that the factor exists.")
     end

src/CloudGraphsDFG/services/CommonFunctions.jl

@@ -160,10 +160,15 @@ end
 $(SIGNATURES)
 Try get a Neo4j node ID from a node label.
 """
-function _tryGetNeoNodeIdFromNodeLabel(neo4jInstance::Neo4jInstance, userId::String, robotId::String, sessionId::String, nodeLabel::Symbol)::Union{Nothing, Int}
+function _tryGetNeoNodeIdFromNodeLabel(neo4jInstance::Neo4jInstance, userId::String, robotId::String, sessionId::String, nodeLabel::Symbol; nodeType::Union{Nothing, String}=nothing)::Union{Nothing, Int}
     @debug "Looking up symbolic node ID where n.label = '$nodeLabel'..."
-    nodes = _getNeoNodesFromCyphonQuery(neo4jInstance, "(node:$userId:$robotId:$sessionId) where exists(node.label) and node.label = \"$(string(nodeLabel))\"")
-    if(length(nodes) != 1)
+    if nodeType == nothing
+        nodes = _getNeoNodesFromCyphonQuery(neo4jInstance, "(node:$userId:$robotId:$sessionId) where exists(node.label) and node.label = \"$(string(nodeLabel))\"")
+    else
+        nodes = _getNeoNodesFromCyphonQuery(neo4jInstance, "(node:$nodeType:$userId:$robotId:$sessionId) where exists(node.label) and node.label = \"$(string(nodeLabel))\"")
+    end
+
+    if length(nodes) != 1
         return nothing
     end
     nodeId = nodes[1].id

src/Deprecated.jl

@@ -38,12 +38,7 @@ Base.getproperty(x::DFGFactor,f::Symbol) = begin
   elseif f == :_internalId
       getfield(x,:_dfgNodeParams)._internalId
   elseif f == :data
-
-      if !(@isdefined getFactorDataWarnOnce)
-          @warn "get: data field is deprecated, use getSolverData. Further warnings are suppressed"
-          global getFactorDataWarnOnce = true
-      end
-
+      Base.depwarn("DFGFactor get: data field is deprecated, use getSolverData", :getproperty)
       getfield(x, :solverData)
   else
       getfield(x,f)
@@ -57,12 +52,7 @@ Base.setproperty!(x::DFGFactor,f::Symbol, val) = begin
     elseif f == :_internalId
         getfield(x,:_dfgNodeParams)._internalId = val
     elseif f == :data
-
-        if !(@isdefined setFactorDataWarnOnce)
-            @warn "set: data field is deprecated, use ...TODO? Further warnings are suppressed"
-            global setFactorDataWarnOnce = true
-        end
-
+        Base.depwarn("DFGFactor set: data field is deprecated, use getSolverData", :getproperty)
         setfield!(x,:solverData, val)
     else
         setfield!(x,f,val)

src/DistributedFactorGraphs.jl

@@ -249,6 +249,7 @@ include("services/AbstractDFG.jl")
 
 # In Memory Types
 include("GraphsDFG/GraphsDFG.jl")
+@reexport using .GraphsDFGs
 include("LightDFG/LightDFG.jl")
 @reexport using .LightDFGs
 #supported in Memory fg types

src/GraphsDFG/GraphsDFG.jl

@@ -1,4 +1,49 @@
+module GraphsDFGs
+
 using Graphs
+using DocStringExtensions
+
+using ...DistributedFactorGraphs
+
+# import DFG functions to extend
+import ...DistributedFactorGraphs:  setSolverParams!,
+                                    getFactor,
+                                    setDescription!,
+                                    # getLabelDict,
+                                    getUserData,
+                                    setUserData!,
+                                    getRobotData,
+                                    setRobotData!,
+                                    getSessionData,
+                                    setSessionData!,
+                                    addVariable!,
+                                    getVariable,
+                                    getAddHistory,
+                                    addFactor!,
+                                    getSolverParams,
+                                    exists,
+                                    isVariable,
+                                    isFactor,
+                                    getDescription,
+                                    updateVariable!,
+                                    updateFactor!,
+                                    deleteVariable!,
+                                    deleteFactor!,
+                                    getVariables,
+                                    listVariables,
+                                    ls,
+                                    getFactors,
+                                    listFactors,
+                                    lsf,
+                                    isFullyConnected,
+                                    hasOrphans,
+                                    getNeighbors,
+                                    buildSubgraph,
+                                    copyGraph!,
+                                    getBiadjacencyMatrix,
+                                    _getDuplicatedEmptyDFG,
+                                    toDot,
+                                    toDotFile
 
 # Imports
 include("entities/GraphsDFG.jl")
@@ -7,4 +52,4 @@ include("services/GraphsDFG.jl")
 # Exports
 export GraphsDFG
 
-export toDot, toDotFile
+end

src/GraphsDFG/services/GraphsDFG.jl

@@ -121,7 +121,9 @@ function getVariable(dfg::GraphsDFG, label::Union{Symbol, String})::DFGVariable
     if !haskey(dfg.labelDict, label)
         error("Variable label '$(label)' does not exist in the factor graph")
     end
-    return dfg.g.vertices[dfg.labelDict[label]].dfgNode
+    node = dfg.g.vertices[dfg.labelDict[label]].dfgNode
+    !isa(node, AbstractDFGVariable) && error("Node with label '$(label)' is not a variable")
+    return node
 end
 
 function getFactor(dfg::GraphsDFG, factorId::Int64)::DFGFactor
@@ -140,7 +142,9 @@ function getFactor(dfg::GraphsDFG, label::Union{Symbol, String})::DFGFactor
     if !haskey(dfg.labelDict, label)
         error("Factor label '$(label)' does not exist in the factor graph")
     end
-    return dfg.g.vertices[dfg.labelDict[label]].dfgNode
+    node = dfg.g.vertices[dfg.labelDict[label]].dfgNode
+    !isa(node, AbstractDFGFactor) && error("Node with label '$(label)' is not a factor")
+    return node
 end
 
 function updateVariable!(dfg::GraphsDFG, variable::DFGVariable)::DFGVariable
@@ -262,3 +266,10 @@ Produces a dot-format of the graph for visualization.
 function toDot(dfg::GraphsDFG)::String
     return Graphs.to_dot(dfg.g)
 end
+
+function toDotFile(dfg::GraphsDFG, fileName::String="/tmp/dfg.dot")::Nothing
+    open(fileName, "w") do fid
+        write(fid, Graphs.to_dot(dfg.g))
+    end
+    return nothing
+end