Merge branch 'master' into feature/149_DFGVariableSofttype

Author: Affie

.travis.yml

@@ -1,4 +1,6 @@
 language: julia
+sudo: required
+dist: trusty
 
 os:
     - linux
@@ -13,37 +15,35 @@ julia:
     - 1.3
     - nightly
 
-notifications:
-    email: false
-
-matrix:
-  allow_failures:
-    - julia: nightly
-
-# script:
-#   - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
-#   - julia --project --check-bounds=yes -e 'using UUIDs; write("Project.toml", replace(read("Project.toml", String), r"uuid = .*?\n" =>"uuid = \"$(uuid4())\"\n"));
-# import Pkg; Pkg.build("DistributedFactorGraphs"); Pkg.test("DistributedFactorGraphs"; coverage=true)'
-
-# script:
-#   - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
-#   - julia --check-bounds=yes -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("DistributedFactorGraphs"); Pkg.test("DistributedFactorGraphs"; coverage=true)'
-
-# can be used if master of package is required
-# script:
-#     - julia --project --color=yes --check-bounds=yes -e 'using Pkg; Pkg.add(PackageSpec(name="MetaGraphs", rev="master")); Pkg.build(); Pkg.test(coverage=true)'
-
-after_success:
-    - julia -e 'using Pkg; cd(Pkg.dir("DistributedFactorGraphs")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder()); Coveralls.submit(process_folder())'
+env:
+    - IIF_TEST=false
 
 jobs:
   include:
+    - julia: 1.2
+      env: IIF_TEST=true
     - stage: "Documentation"
       julia: 1.0
       os: linux
       script:
-        - julia -e 'import Pkg; Pkg.add("Documenter"); Pkg.add("DataFrames")'
+        - julia -e 'import Pkg; Pkg.add("Documenter"); Pkg.add("Neo4j"); Pkg.add("GraphPlot")'
         - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd()));
                                                Pkg.instantiate()'
         - julia --project=docs/ docs/make.jl
       after_success: skip
+
+notifications:
+    email: false
+
+matrix:
+  allow_failures:
+  - julia: nightly
+  - env: IIF_TEST=true
+
+# Set the password for Neo4j to neo4j:test
+before_script:
+    - sleep 10
+    - curl -v POST http://neo4j:neo4j@localhost:7474/user/neo4j/password -d"password=test"
+
+after_success:
+    - julia -e 'using Pkg; cd(Pkg.dir("DistributedFactorGraphs")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder()); Coveralls.submit(process_folder())'

Project.toml

@@ -25,13 +25,12 @@ MetaGraphs = "≥ 0.6.4"
 Reexport = "≥ 0.2"
 Requires = "≥ 0.5"
 julia = "0.7, 1"
-IncrementalInference = "0.7.7"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231"
 Neo4j = "d2adbeaf-5838-5367-8a2f-e46d570981db"
-IncrementalInference = "904591bb-b899-562f-9e6f-b8df64c7d480"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [targets]
-test = ["Test", "GraphPlot", "Neo4j", "IncrementalInference"]
+test = ["Test", "GraphPlot", "Neo4j", "Pkg"]

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -67,6 +67,11 @@ function setSolverParams(dfg::CloudGraphsDFG, solverParams::T)::T where T <: Abs
     return dfg.solverParams = solverParams
 end
 
+function getSerializationModule(dfg::CloudGraphsDFG)::Module where G <: AbstractDFG
+    # TODO: If we need to specialize this for RoME etc, here is where we'll change it.
+    return Main
+end
+
 """
     $(SIGNATURES)
 True if the variable or factor exists in the graph.
@@ -627,15 +632,19 @@ function isFullyConnected(dfg::CloudGraphsDFG)::Bool
     # Total nodes
     varIds = getVariableIds(dfg)
     factIds = getFactorIds(dfg)
-    totalNodes = length(varIds) + length(factIds)
-    if length(varIds) == 0
-        return false
-    end
+    length(varIds) + length(factIds) == 0 && return false
 
     # Total connected nodes - thank you Neo4j for 0..* awesomeness!!
-    connectedList = _getLabelsFromCyphonQuery(dfg.neo4jInstance, "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(varIds[1]))-[FACTORGRAPH*]-(node:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId))")
-
-    return length(connectedList) == totalNodes
+    query = """
+        match (n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(varIds[1]))-[FACTORGRAPH*]-(node:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId))
+        WHERE n:VARIABLE OR n:FACTOR OR node:VARIABLE OR node:FACTOR
+        WITH collect(n)+collect(node) as nodelist
+        unwind nodelist as nodes
+        return count(distinct nodes)"""
+    @debug "[Querying] $query"
+    result = _queryNeo4j(dfg.neo4jInstance, query)
+    # Neo4j.jl data structure sometimes feels brittle... like below
+    return result.results[1]["data"][1]["row"][1] == length(varIds) + length(factIds)
 end
 
 #Alias

src/CloudGraphsDFG/services/CommonFunctions.jl

@@ -1,5 +1,21 @@
 alphaOnlyMatchRegex = r"^[a-zA-Z0-9_]*$"
 
+"""
+$(SIGNATURES)
+Returns the transaction for a given query.
+NOTE: Must commit(transaction) after you're done.
+"""
+function _queryNeo4j(neo4jInstance::Neo4jInstance, query::String)
+    loadtx = transaction(neo4jInstance.connection)
+    result = loadtx(query; submit=true)
+    if length(result.errors) > 0
+        error(string(result.errors))
+    end
+    # Have to finish the transaction
+    commit(loadtx)
+    return result
+end
+
 """
 $(SIGNATURES)
 Returns the list of CloudGraph nodes that matches the Cyphon query.
@@ -11,16 +27,9 @@ If orderProperty is not ==, then 'order by n.{orderProperty} will be appended to
 So can make orderProperty = label or id.
 """
 function _getLabelsFromCyphonQuery(neo4jInstance::Neo4jInstance, matchCondition::String, orderProperty::String="")::Vector{Symbol}
-    # 2. Perform the transaction
-    loadtx = transaction(neo4jInstance.connection)
     query = "match $matchCondition return distinct(node.label) $(orderProperty != "" ? "order by node.$orderProperty" : "")";
-    nodes = loadtx(query; submit=true)
-    if length(nodes.errors) > 0
-        error(string(nodes.errors))
-    end
-    nodeIds = map(node -> node["row"][1], nodes.results[1]["data"])
-    # Have to finish the transaction
-    commit(loadtx)
+    result = _queryNeo4j(neo4jInstance, query)
+    nodeIds = map(node -> node["row"][1], result.results[1]["data"])
     return Symbol.(nodeIds)
 end
 

src/LightDFG/services/LightDFG.jl

@@ -257,7 +257,7 @@ end
 Checks if the graph is fully connected, returns true if so.
 """
 function isFullyConnected(dfg::LightDFG)::Bool
-    return length(connected_components(dfg.g)) == 1
+    return length(LightGraphs.connected_components(dfg.g)) == 1
 end
 
 function _isready(dfg::LightDFG, label::Symbol, ready::Int)::Bool

src/SymbolDFG/services/SymbolDFG.jl

@@ -264,7 +264,7 @@ end
 Checks if the graph is fully connected, returns true if so.
 """
 function isFullyConnected(dfg::SymbolDFG)::Bool
-    return length(connected_components(dfg.g)) == 1
+    return length(LightGraphs.connected_components(dfg.g)) == 1
 end
 
 #Alias

src/services/AbstractDFG.jl

@@ -569,7 +569,6 @@ showFactor(fgl::G, fsym::Symbol) where G <: AbstractDFG = @show getFactor(fgl,fs
 Produces a dot-format of the graph for visualization.
 """
 function toDot(dfg::AbstractDFG)::String
-    @warn "Falling Back to convert to GraphsDFG"
     #TODO implement convert
     graphsdfg = GraphsDFG{AbstractParams}()
     DistributedFactorGraphs._copyIntoGraph!(dfg, graphsdfg, union(getVariableIds(dfg), getFactorIds(dfg)), true)
@@ -589,7 +588,6 @@ Note
 - Based on graphviz.org
 """
 function toDotFile(dfg::AbstractDFG, fileName::String="/tmp/dfg.dot")::Nothing
-    @warn "Falling Back to convert to GraphsDFG"
     #TODO implement convert
     graphsdfg = GraphsDFG{AbstractParams}()
     DistributedFactorGraphs._copyIntoGraph!(dfg, graphsdfg, union(getVariableIds(dfg), getFactorIds(dfg)), true)

src/services/DFGVariable.jl

@@ -77,31 +77,80 @@ function unpack(dfg::G, d::PackedVariableNodeData)::VariableNodeData where G <:
   end
   @debug "Net conversion result: $st"
 
+  if st == nothing
+      error("The variable doesn't seem to have a softtype. It needs to set up with an InferenceVariable from IIF. This will happen if you use DFG to add serialized variables directly and try use them. Please use IncrementalInference.addVariable().")
+  end
+
   return VariableNodeData(M3,M4, d.BayesNetOutVertIDs,
     d.dimIDs, d.dims, d.eliminated, d.BayesNetVertID, d.separator,
     st, d.initialized, d.inferdim, d.ismargin, d.dontmargin )
 end
 
 function compare(a::VariableNodeData, b::VariableNodeData)
-    TP = true
-    TP = TP && a.val == b.val
-    TP = TP && a.bw == b.bw
-    TP = TP && a.BayesNetOutVertIDs == b.BayesNetOutVertIDs
-    TP = TP && a.dimIDs == b.dimIDs
-    TP = TP && a.dims == b.dims
-    TP = TP && a.eliminated == b.eliminated
-    TP = TP && a.BayesNetVertID == b.BayesNetVertID
-    TP = TP && a.separator == b.separator
-    TP = TP && abs(a.inferdim - b.inferdim) < 1e-14
-    TP = TP && a.ismargin == b.ismargin
-    TP = TP && a.softtype == b.softtype
-    return TP
+    a.val != b.val && @debug("val is not equal")==nothing && return false
+    a.bw != b.bw && @debug("bw is not equal")==nothing && return false
+    a.BayesNetOutVertIDs != b.BayesNetOutVertIDs && @debug("BayesNetOutVertIDs is not equal")==nothing && return false
+    a.dimIDs != b.dimIDs && @debug("dimIDs is not equal")==nothing && return false
+    a.dims != b.dims && @debug("dims is not equal")==nothing && return false
+    a.eliminated != b.eliminated && @debug("eliminated is not equal")==nothing && return false
+    a.BayesNetVertID != b.BayesNetVertID && @debug("BayesNetVertID is not equal")==nothing && return false
+    a.separator != b.separator && @debug("separator is not equal")==nothing && return false
+    a.initialized != b.initialized && @debug("initialized is not equal")==nothing && return false
+    abs(a.inferdim - b.inferdim) > 1e-14 && @debug("inferdim is not equal")==nothing && return false
+    a.ismargin != b.ismargin && @debug("ismargin is not equal")==nothing && return false
+    a.dontmargin != b.dontmargin && @debug("dontmargin is not equal")==nothing && return false
+    typeof(a.softtype) != typeof(b.softtype) && @debug("softtype is not equal")==nothing && return false
+    return true
 end
 
+"""
+    $(SIGNATURES)
+Equality check for VariableNodeData.
+"""
 function ==(a::VariableNodeData,b::VariableNodeData, nt::Symbol=:var)
   return DistributedFactorGraphs.compare(a,b)
 end
 
+"""
+    $(SIGNATURES)
+Equality check for VariableEstimate.
+"""
+function ==(a::VariableEstimate, b::VariableEstimate)::Bool
+  a.solverKey != b.solverKey && @debug("solverKey are not equal")==nothing && return false
+  a.ppeType != b.ppeType && @debug("type is not equal")==nothing && return false
+  a.estimate != b.estimate && @debug("estimate are not equal")==nothing && return false
+  a.lastUpdatedTimestamp != b.lastUpdatedTimestamp && @debug("lastUpdatedTimestamp is not equal")==nothing && return false
+  return true
+end
+
+"""
+    $(SIGNATURES)
+Equality check for DFGVariable.
+"""
+function ==(a::DFGVariable, b::DFGVariable)::Bool
+  a.label != b.label && @debug("label is not equal")==nothing && return false
+  a.timestamp != b.timestamp && @debug("timestamp is not equal")==nothing && return false
+  a.tags != b.tags && @debug("tags is not equal")==nothing && return false
+  symdiff(keys(a.estimateDict), keys(b.estimateDict)) != Set(Symbol[]) && @debug("estimateDict keys are not equal")==nothing && return false
+  for k in keys(a.estimateDict)
+    a.estimateDict[k] != b.estimateDict[k] && @debug("estimateDict[$k] is not equal")==nothing && return false
+  end
+  symdiff(keys(a.solverDataDict), keys(b.solverDataDict)) != Set(Symbol[]) && @debug("solverDataDict keys are not equal")==nothing && return false
+  for k in keys(a.solverDataDict)
+    a.solverDataDict[k] != b.solverDataDict[k] && @debug("solverDataDict[$k] is not equal")==nothing && return false
+  end
+  a.smallData != b.smallData && @debug("smallData is not equal")==nothing && return false
+  a.bigData != b.bigData && @debug("bigData is not equal")==nothing && return false
+  a.ready != b.ready && @debug("ready is not equal")==nothing && return false
+  a.backendset != b.backendset && @debug("backendset is not equal")==nothing && return false
+  a._internalId != b._internalId && @debug("_internalId is not equal")==nothing && return false
+  return true
+end
+
+"""
+    $(SIGNATURES)
+Convert a DFGVariable to a DFGVariableSummary.
+"""
 function convert(::Type{DFGVariableSummary}, v::DFGVariable)
     return DFGVariableSummary(v.label, v.timestamp, deepcopy(v.tags), deepcopy(v.estimateDict), Symbol(typeof(getSofttype(v))), v._internalId)
 end