Merge pull request #167 from JuliaRobotics/jt/develop_merge

megamerge - trying to tie up loose ends

Author: GearsAD

.travis.yml

@@ -1,4 +1,6 @@
 language: julia
+sudo: required
+dist: trusty
 
 os:
     - linux
@@ -13,37 +15,35 @@ julia:
     - 1.3
     - nightly
 
-notifications:
-    email: false
-
-matrix:
-  allow_failures:
-    - julia: nightly
-
-# script:
-#   - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
-#   - julia --project --check-bounds=yes -e 'using UUIDs; write("Project.toml", replace(read("Project.toml", String), r"uuid = .*?\n" =>"uuid = \"$(uuid4())\"\n"));
-# import Pkg; Pkg.build("DistributedFactorGraphs"); Pkg.test("DistributedFactorGraphs"; coverage=true)'
-
-# script:
-#   - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
-#   - julia --check-bounds=yes -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("DistributedFactorGraphs"); Pkg.test("DistributedFactorGraphs"; coverage=true)'
-
-# can be used if master of package is required
-# script:
-#     - julia --project --color=yes --check-bounds=yes -e 'using Pkg; Pkg.add(PackageSpec(name="MetaGraphs", rev="master")); Pkg.build(); Pkg.test(coverage=true)'
-
-after_success:
-    - julia -e 'using Pkg; cd(Pkg.dir("DistributedFactorGraphs")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder()); Coveralls.submit(process_folder())'
+env:
+    - IIF_TEST=false
 
 jobs:
   include:
+    - julia: 1.2
+      env: IIF_TEST=true
     - stage: "Documentation"
       julia: 1.0
       os: linux
       script:
-        - julia -e 'import Pkg; Pkg.add("Documenter"); Pkg.add("DataFrames")'
+        - julia -e 'import Pkg; Pkg.add("Documenter"); Pkg.add("Neo4j"); Pkg.add("GraphPlot")'
         - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd()));
                                                Pkg.instantiate()'
         - julia --project=docs/ docs/make.jl
       after_success: skip
+
+notifications:
+    email: false
+
+matrix:
+  allow_failures:
+  - julia: nightly
+  - env: IIF_TEST=true
+
+# Set the password for Neo4j to neo4j:test
+before_script:
+    - sleep 10
+    - curl -v POST http://neo4j:neo4j@localhost:7474/user/neo4j/password -d"password=test"
+
+after_success:
+    - julia -e 'using Pkg; cd(Pkg.dir("DistributedFactorGraphs")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder()); Coveralls.submit(process_folder())'

Project.toml

@@ -30,6 +30,7 @@ julia = "0.7, 1"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231"
 Neo4j = "d2adbeaf-5838-5367-8a2f-e46d570981db"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [targets]
-test = ["Test", "GraphPlot", "Neo4j"]
+test = ["Test", "GraphPlot", "Neo4j", "Pkg"]

src/BigData.jl

@@ -0,0 +1,139 @@
+
+# TODO move to ...
+
+
+# """
+#     $(TYPEDEF)
+# Abstract parent struct for big data entry.
+# """
+# abstract type AbstractBigDataEntry end
+
+# --- AbstractBigData Interfaces ---
+# fields to implement:
+# - key::Symbol
+# available methods:
+# - addBigDataEntry!
+# - getBigDataEntry
+# - updateBigDataEntry!
+# - deleteBigDataEntry!
+# - getBigDataEntries
+# - getBigDataKeys
+
+export  addBigDataEntry!,
+        getBigDataEntry,
+        updateBigDataEntry!,
+        deleteBigDataEntry!,
+        getBigDataEntries,
+        getBigDataKeys,
+        MongodbBigDataEntry,
+        FileBigDataEntry
+
+# Methods
+
+#TODO should this return Bool or the modified Variable?
+"""
+    $(SIGNATURES)
+Add Big Data Entry to a DFG variable
+"""
+function addBigDataEntry!(var::AbstractDFGVariable, bde::AbstractBigDataEntry)::Bool
+    haskey(var.bigData,bde.key) && @warn "$(bde.key) already exists in variable, overwriting!"
+    var.bigData[bde.key] = bde
+    return true
+end
+
+"""
+    $(SIGNATURES)
+Add Big Data Entry to distrubuted factor graph.
+Should be extended if DFG variable is not returned by by reference.
+"""
+function addBigDataEntry!(dfg::AbstractDFG, label::Symbol, bde::AbstractBigDataEntry)::Bool
+    return addBigDataEntry!(getVariable(dfg, label), bde)
+end
+
+"""
+    $(SIGNATURES)
+Get big data entry
+"""
+function getBigDataEntry(var::AbstractDFGVariable, key::Symbol)::AbstractBigDataEntry
+    return var.bigData[key]
+end
+function getBigDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol)::AbstractBigDataEntry
+    return getBigDataEntry(getVariable(dfg, label), key)
+end
+
+"""
+    $(SIGNATURES)
+Update big data entry
+"""
+function updateBigDataEntry!(var::AbstractDFGVariable,  bde::AbstractBigDataEntry)::Bool#TODO should this return Bool?
+    !haskey(var.bigData,bde.key) && (@error "$(bde.key) does not exist in variable!"; return false)
+    var.bigData[bde.key] = bde
+    return true
+end
+function updateBigDataEntry!(dfg::AbstractDFG, label::Symbol,  bde::AbstractBigDataEntry)::Bool
+    updateBigDataEntry!(getVariable(dfg, label), bde)
+end
+
+"""
+    $(SIGNATURES)
+Delete big data entry
+"""
+function deleteBigDataEntry!(var::AbstractDFGVariable, key::Symbol)::AbstractBigDataEntry #users responsibility to delete big data in db before deleting entry
+    bde = getBigDataEntry(var, key)
+    delete!(var.bigData, key)
+    return bde
+end
+
+function deleteBigDataEntry!(dfg::AbstractDFG, label::Symbol, key::Symbol)::AbstractBigDataEntry #users responsibility to delete big data in db before deleting entry
+    deleteBigDataEntry!(getVariable(dfg, label), key)
+end
+
+"""
+    $(SIGNATURES)
+Get big data entries, Vector{AbstractBigDataEntry}
+"""
+function getBigDataEntries(var::AbstractDFGVariable)::Vector{AbstractBigDataEntry}
+    #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractBigDataEntry}}?
+    collect(values(var.bigData))
+end
+function getBigDataEntries(dfg::AbstractDFG, label::Symbol)::Vector{AbstractBigDataEntry}
+    #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractBigDataEntry}}?
+    getBigDataEntries(getVariable(dfg, label))
+end
+
+
+"""
+    $(SIGNATURES)
+getBigDataKeys
+"""
+function getBigDataKeys(var::AbstractDFGVariable)::Vector{Symbol}
+    collect(keys(var.bigData))
+end
+function getBigDataKeys(dfg::AbstractDFG, label::Symbol)::Vector{Symbol}
+    getBigDataKeys(getVariable(dfg, label))
+end
+
+
+
+# Types <: AbstractBigDataEntry
+"""
+    $(TYPEDEF)
+BigDataEntry in MongoDB.
+"""
+struct MongodbBigDataEntry <: AbstractBigDataEntry
+    key::Symbol
+    oid::NTuple{12, UInt8} #mongodb object id
+    #maybe other fields such as:
+    #flags::Bool ready, valid, locked, permissions
+    #MIMEType::Symbol
+end
+
+
+"""
+    $(TYPEDEF)
+BigDataEntry in a file.
+"""
+struct FileBigDataEntry <: AbstractBigDataEntry
+    key::Symbol
+    filename::String
+end

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -54,7 +54,7 @@ function _getDuplicatedEmptyDFG(dfg::CloudGraphsDFG)::CloudGraphsDFG
         length(_getLabelsFromCyphonQuery(dfg.neo4jInstance, "(node:$(dfg.userId):$(dfg.robotId):$(sessionId))")) == 0 && break
     end
     @debug "Unique+empty copy session name: $sessionId"
-    return CloudGraphsDFG{typeof(dfg.solverParams)}(dfg.neo4jInstance.connection, dfg.userId, dfg.robotId, sessionId, dfg.encodePackedTypeFunc, dfg.getPackedTypeFunc, dfg.decodePackedTypeFunc, solverParams=deepcopy(dfg.solverParams), description="(Copy of) $(dfg.description)", useCache=dfg.useCache)
+    return CloudGraphsDFG{typeof(dfg.solverParams)}(dfg.neo4jInstance.connection, dfg.userId, dfg.robotId, sessionId, dfg.encodePackedTypeFunc, dfg.getPackedTypeFunc, dfg.decodePackedTypeFunc, dfg.rebuildFactorMetadata!, solverParams=deepcopy(dfg.solverParams), description="(Copy of) $(dfg.description)", useCache=dfg.useCache)
 end
 
 # Accessors
@@ -67,6 +67,11 @@ function setSolverParams(dfg::CloudGraphsDFG, solverParams::T)::T where T <: Abs
     return dfg.solverParams = solverParams
 end
 
+function getSerializationModule(dfg::CloudGraphsDFG)::Module where G <: AbstractDFG
+    # TODO: If we need to specialize this for RoME etc, here is where we'll change it.
+    return Main
+end
+
 """
     $(SIGNATURES)
 True if the variable or factor exists in the graph.
@@ -158,9 +163,7 @@ Add a DFGVariable to a DFG.
 """
 function addVariable!(dfg::CloudGraphsDFG, variable::DFGVariable)::Bool
     if exists(dfg, variable)
-        @warn "Variable '$(variable.label)' already exists in the graph, so updating it."
-        updateVariable!(dfg, variable)
-        return true
+        error("Variable '$(variable.label)' already exists in the factor graph")
     end
     props = Dict{String, Any}()
     props["label"] = string(variable.label)
@@ -192,9 +195,7 @@ Add a DFGFactor to a DFG.
 """
 function addFactor!(dfg::CloudGraphsDFG, variables::Vector{DFGVariable}, factor::DFGFactor)::Bool
     if exists(dfg, factor)
-        @warn "Factor '$(factor.label)' already exist in the graph, so updating it."
-        updateFactor!(dfg, variables, factor)
-        return true
+        error("Factor '$(factor.label)' already exists in the factor graph")
     end
 
     # Update the variable ordering
@@ -363,9 +364,7 @@ Update a complete DFGVariable in the DFG.
 """
 function updateVariable!(dfg::CloudGraphsDFG, variable::DFGVariable)::DFGVariable
     if !exists(dfg, variable)
-        @warn "Variable '$(variable.label)' doesn't exist in the graph, so adding it."
-        addVariable!(dfg, variable)
-        return variable
+        error("Variable label '$(variable.label)' does not exist in the factor graph")
     end
     nodeId = _tryGetNeoNodeIdFromNodeLabel(dfg.neo4jInstance, dfg.userId, dfg.robotId, dfg.sessionId, variable.label)
     # Update the node ID
@@ -409,9 +408,7 @@ Update a complete DFGFactor in the DFG.
 """
 function updateFactor!(dfg::CloudGraphsDFG, factor::DFGFactor)::DFGFactor
     if !exists(dfg, factor)
-        @warn "Factor '$(factor.label)' doesn't exist in the graph, so adding it."
-        addFactor!(dfg, factor)
-        return factor
+        error("Factor label '$(factor.label)' does not exist in the factor graph")
     end
     nodeId = _tryGetNeoNodeIdFromNodeLabel(dfg.neo4jInstance, dfg.userId, dfg.robotId, dfg.sessionId, factor.label)
     # Update the _internalId
@@ -552,9 +549,15 @@ deleteFactor!(dfg::CloudGraphsDFG, factor::DFGFactor)::DFGFactor = deleteFactor!
 List the DFGVariables in the DFG.
 Optionally specify a label regular expression to retrieves a subset of the variables.
 """
-function getVariables(dfg::CloudGraphsDFG, regexFilter::Union{Nothing, Regex}=nothing)::Vector{DFGVariable}
+function getVariables(dfg::CloudGraphsDFG, regexFilter::Union{Nothing, Regex}=nothing; tags::Vector{Symbol}=Symbol[])::Vector{DFGVariable}
     variableIds = getVariableIds(dfg, regexFilter)
-    return map(vId->getVariable(dfg, vId), variableIds)
+    # TODO: Optimize to use tags in query here!
+    variables = map(vId->getVariable(dfg, vId), variableIds)
+    if length(tags) > 0
+        mask = map(v -> length(intersect(v.tags, tags)) > 0, variables )
+        return variables[mask]
+    end
+    return variables
 end
 
 """
@@ -629,15 +632,19 @@ function isFullyConnected(dfg::CloudGraphsDFG)::Bool
     # Total nodes
     varIds = getVariableIds(dfg)
     factIds = getFactorIds(dfg)
-    totalNodes = length(varIds) + length(factIds)
-    if length(varIds) == 0
-        return false
-    end
+    length(varIds) + length(factIds) == 0 && return false
 
     # Total connected nodes - thank you Neo4j for 0..* awesomeness!!
-    connectedList = _getLabelsFromCyphonQuery(dfg.neo4jInstance, "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(varIds[1]))-[FACTORGRAPH*]-(node:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId))")
-
-    return length(connectedList) == totalNodes
+    query = """
+        match (n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(varIds[1]))-[FACTORGRAPH*]-(node:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId))
+        WHERE n:VARIABLE OR n:FACTOR OR node:VARIABLE OR node:FACTOR
+        WITH collect(n)+collect(node) as nodelist
+        unwind nodelist as nodes
+        return count(distinct nodes)"""
+    @debug "[Querying] $query"
+    result = _queryNeo4j(dfg.neo4jInstance, query)
+    # Neo4j.jl data structure sometimes feels brittle... like below
+    return result.results[1]["data"][1]["row"][1] == length(varIds) + length(factIds)
 end
 
 #Alias
@@ -655,10 +662,10 @@ function getNeighbors(dfg::CloudGraphsDFG, node::T; ready::Union{Nothing, Int}=n
     query = "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(node.label))--(node) where node:VARIABLE or node:FACTOR "
     if ready != nothing || backendset != nothing
         if ready != nothing
-            query = query + "and node.ready = $(ready)"
+            query = query * "and node.ready = $(ready)"
         end
         if backendset != nothing
-            query = query + "and node.backendset = $(backendset)"
+            query = query * "and node.backendset = $(backendset)"
         end
     end
     neighbors = _getLabelsFromCyphonQuery(dfg.neo4jInstance, query)
@@ -677,10 +684,10 @@ function getNeighbors(dfg::CloudGraphsDFG, label::Symbol; ready::Union{Nothing,
     query = "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(label))--(node) where node:VARIABLE or node:FACTOR "
     if ready != nothing || backendset != nothing
         if ready != nothing
-            query = query + "and node.ready = $(ready)"
+            query = query * "and node.ready = $(ready)"
         end
         if backendset != nothing
-            query = query + "and node.backendset = $(backendset)"
+            query = query * "and node.backendset = $(backendset)"
         end
     end
     neighbors = _getLabelsFromCyphonQuery(dfg.neo4jInstance, query)

src/CloudGraphsDFG/services/CommonFunctions.jl

@@ -1,5 +1,21 @@
 alphaOnlyMatchRegex = r"^[a-zA-Z0-9_]*$"
 
+"""
+$(SIGNATURES)
+Returns the transaction for a given query.
+NOTE: Must commit(transaction) after you're done.
+"""
+function _queryNeo4j(neo4jInstance::Neo4jInstance, query::String)
+    loadtx = transaction(neo4jInstance.connection)
+    result = loadtx(query; submit=true)
+    if length(result.errors) > 0
+        error(string(result.errors))
+    end
+    # Have to finish the transaction
+    commit(loadtx)
+    return result
+end
+
 """
 $(SIGNATURES)
 Returns the list of CloudGraph nodes that matches the Cyphon query.
@@ -11,16 +27,9 @@ If orderProperty is not ==, then 'order by n.{orderProperty} will be appended to
 So can make orderProperty = label or id.
 """
 function _getLabelsFromCyphonQuery(neo4jInstance::Neo4jInstance, matchCondition::String, orderProperty::String="")::Vector{Symbol}
-    # 2. Perform the transaction
-    loadtx = transaction(neo4jInstance.connection)
     query = "match $matchCondition return distinct(node.label) $(orderProperty != "" ? "order by node.$orderProperty" : "")";
-    nodes = loadtx(query; submit=true)
-    if length(nodes.errors) > 0
-        error(string(nodes.errors))
-    end
-    nodeIds = map(node -> node["row"][1], nodes.results[1]["data"])
-    # Have to finish the transaction
-    commit(loadtx)
+    result = _queryNeo4j(neo4jInstance, query)
+    nodeIds = map(node -> node["row"][1], result.results[1]["data"])
     return Symbol.(nodeIds)
 end