Merge pull request #901 from JuliaRobotics/master

release v0.18.6-rc1

Author: dehann

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.18.5"
+version = "0.18.6"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

src/DataBlobs/entities/AbstractDataEntries.jl

@@ -48,3 +48,50 @@ struct MongodbDataEntry <: AbstractDataEntry
     #flags::Bool ready, valid, locked, permissions
     #MIMEType::String
 end
+
+
+##==============================================================================
+## FileDataEntryBlob Types
+##==============================================================================
+export FileDataEntry
+"""
+    $(TYPEDEF)
+Data Entry in a file.
+"""
+struct FileDataEntry <: AbstractDataEntry
+    label::Symbol
+    id::UUID
+    folder::String
+    hash::String #using bytes2hex or perhaps Vector{Uint8}?
+    createdTimestamp::ZonedDateTime
+
+    function FileDataEntry(label, id, folder, hash, timestamp)
+        if !isdir(folder)
+            @info "Folder '$folder' doesn't exist - creating."
+            # create new folder
+            mkpath(folder)
+        end
+        return new(label, id, folder, hash, timestamp)
+    end
+end
+
+
+##==============================================================================
+## InMemoryDataEntry Types
+##==============================================================================
+export InMemoryDataEntry
+"""
+    $(TYPEDEF)
+Store data temporary in memory.
+NOTE: Neither Entry nor Blob will be persisted.
+"""
+struct InMemoryDataEntry{T} <: AbstractDataEntry
+    label::Symbol
+    id::UUID
+    #hash::String #Is this needed?
+    createdTimestamp::ZonedDateTime
+    data::T
+end
+
+# getHash(::InMemoryDataEntry) = UInt8[]
+assertHash(de::InMemoryDataEntry, db; hashfunction = sha256) = true

src/DataBlobs/services/AbstractDataEntries.jl

@@ -29,14 +29,22 @@ buildSourceString(dfg::AbstractDFG, label::Symbol) =
 Get data entry
 """
 function getDataEntry(var::AbstractDFGVariable, key::Symbol)
-    !hasDataEntry(var, key) && error("Data entry $(key) does not exist in variable $(getLabel(var))")
+    !hasDataEntry(var, key) && error("No dataEntry label $(key) found in variable $(getLabel(var))")
     return var.dataDict[key]
 end
 
-function getDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol)
-    return getDataEntry(getVariable(dfg, label), key)
+function getDataEntry(var::AbstractDFGVariable, blobId::UUID)
+    for (k,v) in var.dataDict
+        if v.id == blobId
+            return v
+        end
+    end
+    error("No dataEntry with blobId $(blobId) found in variable $(getLabel(var))")
 end
 
+getDataEntry(dfg::AbstractDFG, label::Symbol, key::Union{Symbol,UUID}) = getDataEntry(getVariable(dfg, label), key)
+
+
 """
     $(SIGNATURES)
 Add Data Entry to a DFG variable

src/DataBlobs/services/BlobStores.jl

@@ -63,7 +63,13 @@ end
 ## Store and Entry Data CRUD
 ##==============================================================================
 
-function getData(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol, key::Symbol; hashfunction = sha256)
+function getData(
+    dfg::AbstractDFG, 
+    blobstore::AbstractBlobStore, 
+    label::Symbol, 
+    key::Union{Symbol,UUID}; 
+    hashfunction = sha256
+)
     de = getDataEntry(dfg, label, key)
     db = getDataBlob(blobstore, de)
     assertHash(de, db, hashfunction=hashfunction)

src/DataBlobs/services/DataEntryBlob.jl

@@ -111,8 +111,13 @@ end
 ## DFG Data CRUD
 ##==============================================================================
 
-function getData(dfg::AbstractDFG, label::Symbol, key::Symbol; hashfunction = sha256)
-    de = getDataEntry(dfg, label, key)
+function getData(
+    dfg::AbstractDFG, 
+    vlabel::Symbol, 
+    key::Union{Symbol,UUID}; 
+    hashfunction = sha256
+)
+    de = getDataEntry(dfg, vlabel, key)
     db = getDataBlob(dfg, de)
 
     assertHash(de, db, hashfunction=hashfunction)

src/DataBlobs/services/FileDataEntryBlob.jl

@@ -1,27 +1,4 @@
-##==============================================================================
-## FileDataEntryBlob Types
-##==============================================================================
-export FileDataEntry
-"""
-    $(TYPEDEF)
-Data Entry in a file.
-"""
-struct FileDataEntry <: AbstractDataEntry
-    label::Symbol
-    id::UUID
-    folder::String
-    hash::String #using bytes2hex or perhaps Vector{Uint8}?
-    createdTimestamp::ZonedDateTime
 
-    function FileDataEntry(label, id, folder, hash, timestamp)
-        if !isdir(folder)
-            @info "Folder '$folder' doesn't exist - creating."
-            # create new folder
-            mkpath(folder)
-        end
-        return new(label, id, folder, hash, timestamp)
-    end
-end
 
 # @generated function ==(x::FileDataEntry, y::FileDataEntry)
 #     mapreduce(n -> :(x.$n == y.$n), (a,b)->:($a && $b), fieldnames(x))

src/DataBlobs/services/InMemoryDataEntryBlob.jl

@@ -1,22 +1,3 @@
-##==============================================================================
-## InMemoryDataEntry Types
-##==============================================================================
-export InMemoryDataEntry
-"""
-    $(TYPEDEF)
-Store data temporary in memory.
-NOTE: Neither Entry nor Blob will be persisted.
-"""
-struct InMemoryDataEntry{T} <: AbstractDataEntry
-    label::Symbol
-    id::UUID
-    #hash::String #Is this needed?
-    createdTimestamp::ZonedDateTime
-    data::T
-end
-
-# getHash(::InMemoryDataEntry) = UInt8[]
-assertHash(de::InMemoryDataEntry, db; hashfunction = sha256) = true
 
 
 ##==============================================================================

src/Neo4jDFG/entities/Neo4jDFG.jl

@@ -33,18 +33,10 @@ mutable struct Neo4jDFG{T <: AbstractParams} <: AbstractDFG{T}
         !isValidLabel(robotId) && error("'$robotId' is not a valid Robot ID")
         !isValidLabel(sessionId) && error("'$sessionId' is not a valid Session ID")
 
-        # neo4jConnection = Neo4j.Connection(host, port=port, user=dbUser, password=dbPassword);
-        # graph = Neo4j.getgraph(neo4jConnection)
-        # neo4jInstance = Neo4jInstance(neo4jConnection, graph)
-
         dfg = new{T}(neo4jInstance, userId, robotId, sessionId, description, addHistory, solverParams, blobStores)
         # Create the session if it doesn't already exist
         if createSessionNodes
             createDfgSessionIfNotExist(dfg)
-            setUserData!(dfg, userData)
-            setRobotData!(dfg, robotData)
-            setSessionData!(dfg, sessionData)
-            setDescription!(dfg, description)
         end
 
         return dfg

src/Neo4jDFG/services/CGStructure.jl

@@ -6,7 +6,6 @@ end
 
 # Fastest way I can think to convert the data into a dict
 #TODO: Probably should be made more efficient...definitely should be made more efficient
-#  return ::Dict{String, Any}
 function _convertNodeToDict(abstractNode::N) where N <: AbstractCGNode
     cp = deepcopy(abstractNode)
     data = length(cp.data) != 0 ? JSON2.write(cp.data) : "{}"
@@ -16,8 +15,6 @@ function _convertNodeToDict(abstractNode::N) where N <: AbstractCGNode
     return ser
 end
 
-#TODO: Refactor, #HACK :D (but it works!)
-# return ::Session
 function _convertDictToSession(dict::Dict{String, Any})
     data = JSON2.read(String(base64decode(dict["data"])), Dict{Symbol, String})
     session = Session(
@@ -31,8 +28,7 @@ function _convertDictToSession(dict::Dict{String, Any})
         dict["lastUpdatedTimestamp"])
     return session
 end
-#TODO: Refactor, #HACK :D (but it works!)
-# returns ::Robot
+
 function _convertDictToRobot(dict::Dict{String, Any})
     data = JSON2.read(String(base64decode(dict["data"])), Dict{Symbol, String})
     robot = Robot(
@@ -45,8 +41,7 @@ function _convertDictToRobot(dict::Dict{String, Any})
         dict["lastUpdatedTimestamp"])
     return robot
 end
-#TODO: Refactor, #HACK :D (but it works!)
-# returns ::User
+
 function _convertDictToUser(dict::Dict{String, Any})
     data = JSON2.read(String(base64decode(dict["data"])), Dict{Symbol, String})
     user = User(
@@ -59,81 +54,33 @@ function _convertDictToUser(dict::Dict{String, Any})
     return user
 end
 
-# returns ::User
-function createUser(dfg::Neo4jDFG, user::User)
-    Symbol(dfg.userId) != user.id && error("DFG user ID must match user's ID")
-    !isValidLabel(user) && error("Node cannot have an ID '$(user.id)'.")
-
-    props = _convertNodeToDict(user)
-    # TODO: Switch to _queryNeo4j
-    retNode = _createNode(dfg.neo4jInstance, ["USER", String(user.id)], props, nothing)
-    return user
-end
-
-# returns ::Robot
-function createRobot(dfg::Neo4jDFG, robot::Robot)
-    Symbol(dfg.robotId) != robot.id && error("DFG robot ID must match robot's ID")
-    Symbol(dfg.userId) != robot.userId && error("DFG user ID must match robot's user ID")
-    !isValidLabel(robot) && error("Node cannot have an ID '$(robot.id)'.")
-
-    # Find the parent
-    parents = _getNeoNodesFromCyphonQuery(dfg.neo4jInstance, "(node:USER:`$(dfg.userId)`)")
-    length(parents) == 0 && error("Cannot find user '$(dfg.userId)'")
-    length(parents) > 1 && error("Found multiple users '$(dfg.userId)'")
-
-    # Already exists?
-    length(_getNeoNodesFromCyphonQuery(dfg.neo4jInstance, "(node:ROBOT:`$(dfg.userId)`:`$(robot.id)`)")) != 0 &&
-        error("Robot '$(robot.id)' already exists for user '$(robot.userId)'")
-
-    props = _convertNodeToDict(robot)
-    # TODO: Switch to _queryNeo4j
-    retNode = _createNode(dfg.neo4jInstance, ["ROBOT", String(robot.userId), String(robot.id)], props, parents[1], :ROBOT)
-    return robot
-end
-
-# returns ::Session
-function createSession(dfg::Neo4jDFG, session::Session)
-    Symbol(dfg.robotId) != session.robotId && error("DFG robot ID must match session's robot ID")
-    Symbol(dfg.userId) != session.userId && error("DFG user ID must match session's->robot's->user ID")
-    !isValidLabel(session) && error("Node cannot have an ID '$(session.id)'.")
-
-    # Find the parent
-    parents = _getNeoNodesFromCyphonQuery(dfg.neo4jInstance, "(node:ROBOT:`$(dfg.robotId)`:`$(dfg.userId)`)")
-    length(parents) == 0 && error("Cannot find robot '$(dfg.robotId)' for user '$(dfg.userId)'")
-    length(parents) > 1 && error("Found multiple robots '$(dfg.robotId)' for user '$(dfg.userId)'")
-
-    # Already exists?
-    length(_getNeoNodesFromCyphonQuery(dfg.neo4jInstance, "(node:SESSION:`$(session.userId)`:`$(session.robotId)`:`$(session.id)`)")) != 0 &&
-        error("Session '$(session.id)' already exists for robot '$(session.robotId)' and user '$(session.userId)'")
-
-    props = _convertNodeToDict(session)
-    # TODO: Switch to _queryNeo4j
-    retNode = _createNode(dfg.neo4jInstance, ["SESSION", String(session.userId), String(session.robotId), String(session.id)], props, parents[1], :SESSION)
-    return session
-end
-
 """
 $(SIGNATURES)
-Shortcut method to create the user, robot, and session if it doesn't already exist.
-
-Notes
-- return `::Session`
+Efficient shortcut method to create the user, robot, and session if it doesn't already exist.
 """
 function createDfgSessionIfNotExist(dfg::Neo4jDFG)
     strip(dfg.userId) == "" && error("User ID is not populated in DFG.")
     strip(dfg.robotId) == "" && error("Robot ID is not populated in DFG.")
     strip(dfg.sessionId) == "" && error("Session ID is not populated in DFG.")
+    !isValidLabel(dfg.userId) && error("Node cannot have an ID '$(dfg.userId)'.")
+    !isValidLabel(dfg.robotId) && error("Node cannot have an ID '$(dfg.robotId)'.")
+    !isValidLabel(dfg.sessionId) && error("Node cannot have an ID '$(dfg.sessionId)'.")
+
     user = User(Symbol(dfg.userId), dfg.userId, "Description for $(dfg.userId)", Dict{Symbol, String}())
     robot = Robot(Symbol(dfg.robotId), Symbol(dfg.userId), dfg.robotId, "Description for $(dfg.userId):$(dfg.robotId)", Dict{Symbol, String}())
     session = Session(Symbol(dfg.sessionId), Symbol(dfg.robotId), Symbol(dfg.userId), dfg.sessionId, dfg.description, Dict{Symbol, String}())
 
-    _getNodeCount(dfg.neo4jInstance, [dfg.userId, "USER"]) == 0 && createUser(dfg, user)
-    _getNodeCount(dfg.neo4jInstance, [dfg.userId, dfg.robotId, "ROBOT"]) == 0 && createRobot(dfg, robot)
-    if _getNodeCount(dfg.neo4jInstance, [dfg.userId, dfg.robotId, dfg.sessionId, "SESSION"]) == 0
-        return createSession(dfg, session)
-    else
-        return getSession(dfg)
-    end
+    # NOTE that this doesn't get updated then, you need to use the set* (e.g. setSessionData) functions yourself.
+    query = """
+    MERGE (u:USER:`$(String(user.id))`) ON CREATE SET $(join(["u.$k = '$(v)'" for (k,v) in _convertNodeToDict(user)], ", "))\r\n
+    MERGE (r:ROBOT:`$(String(user.id))`:`$(String(robot.id))`) ON CREATE SET $(join(["r.$k = '$(v)'" for (k,v) in _convertNodeToDict(robot)], ", "))\r\n
+    MERGE (s:SESSION:`$(String(user.id))`:`$(String(robot.id))`:`$(String(session.id))`) ON CREATE SET $(join(["s.$k = '$(v)'" for (k,v) in _convertNodeToDict(session)], ", "))
+    MERGE (u)-[:ROBOT]->(r) 
+    MERGE (r)-[:SESSION]->(s) 
+    """
+
+    _queryNeo4j(dfg.neo4jInstance, query)
+    return nothing
 end
 
 """
@@ -320,7 +267,7 @@ Notes
 - Returns `::Neo4jDFG `
 """
 function copySession!(sourceDFG::Neo4jDFG, destDFG::Union{Nothing, <:Neo4jDFG})
-    if destDFG == nothing
+    if destDFG === nothing
         destDFG = _getDuplicatedEmptyDFG(sourceDFG)
     end
     _copyIntoGraph!(sourceDFG, destDFG, union(listVariables(sourceDFG), listFactors(sourceDFG)), true)

src/services/CustomPrinting.jl

@@ -191,9 +191,9 @@ function Base.show(io::IO, dfg::AbstractDFG)
     println(io, "  Description: ", dfg.description)
     println(io, "  Nr variables: ", length(ls(dfg)))
     println(io, "  Nr factors: ",length(lsf(dfg)))
-    println(io, "  User Data: ", keys(dfg.userData))
-    println(io, "  Robot Data: ", keys(dfg.robotData))
-    println(io, "  Session Data: ", keys(dfg.sessionData))
+    println(io, "  User Data: ", keys(getUserData(dfg)))
+    println(io, "  Robot Data: ", keys(getRobotData(dfg)))
+    println(io, "  Session Data: ", keys(getSessionData(dfg)))
 end
 
 Base.show(io::IO, ::MIME"text/plain", dfg::AbstractDFG) = show(io, dfg)