Merge branch feature/IIFTests_cgdfg into jt/develop_merge without tests

Author: Affie

.travis.yml

@@ -1,4 +1,6 @@
 language: julia
+sudo: required
+dist: trusty
 
 sudo: required
 dist: trusty
@@ -23,21 +25,10 @@ matrix:
   allow_failures:
     - julia: nightly
 
+# Set the password for Neo4j to neo4j:test
 before_script:
     - sleep 10
     - curl -v POST http://neo4j:neo4j@localhost:7474/user/neo4j/password -d"password=test"
-# script:
-#   - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
-#   - julia --project --check-bounds=yes -e 'using UUIDs; write("Project.toml", replace(read("Project.toml", String), r"uuid = .*?\n" =>"uuid = \"$(uuid4())\"\n"));
-# import Pkg; Pkg.build("DistributedFactorGraphs"); Pkg.test("DistributedFactorGraphs"; coverage=true)'
-
-# script:
-#   - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
-#   - julia --check-bounds=yes -e 'using Pkg; Pkg.clone(pwd()); Pkg.build("DistributedFactorGraphs"); Pkg.test("DistributedFactorGraphs"; coverage=true)'
-
-# can be used if master of package is required
-# script:
-#     - julia --project --color=yes --check-bounds=yes -e 'using Pkg; Pkg.add(PackageSpec(name="MetaGraphs", rev="master")); Pkg.build(); Pkg.test(coverage=true)'
 
 after_success:
     - julia -e 'using Pkg; cd(Pkg.dir("DistributedFactorGraphs")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder()); Coveralls.submit(process_folder())'
@@ -53,3 +44,10 @@ jobs:
                                                Pkg.instantiate()'
         - julia --project=docs/ docs/make.jl
       after_success: skip
+
+    - stage: "IIF Driver Tests"
+      julia: 1.2
+      os: linux
+      script:
+        - julia -e 'import Pkg; Pkg.add("IncrementalInference"); Pkg.test("DistributedFactorGraphs")'
+      after_success: skip

Project.toml

@@ -25,13 +25,11 @@ MetaGraphs = "≥ 0.6.4"
 Reexport = "≥ 0.2"
 Requires = "≥ 0.5"
 julia = "0.7, 1"
-IncrementalInference = "0.7.7"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231"
 Neo4j = "d2adbeaf-5838-5367-8a2f-e46d570981db"
-IncrementalInference = "904591bb-b899-562f-9e6f-b8df64c7d480"
 
 [targets]
-test = ["Test", "GraphPlot", "Neo4j", "IncrementalInference"]
+test = ["Test", "GraphPlot", "Neo4j"]

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -67,6 +67,11 @@ function setSolverParams(dfg::CloudGraphsDFG, solverParams::T)::T where T <: Abs
     return dfg.solverParams = solverParams
 end
 
+function getSerializationModule(dfg::CloudGraphsDFG)::Module where G <: AbstractDFG
+    # TODO: If we need to specialize this for RoME etc, here is where we'll change it.
+    return Main
+end
+
 """
     $(SIGNATURES)
 True if the variable or factor exists in the graph.
@@ -627,15 +632,19 @@ function isFullyConnected(dfg::CloudGraphsDFG)::Bool
     # Total nodes
     varIds = getVariableIds(dfg)
     factIds = getFactorIds(dfg)
-    totalNodes = length(varIds) + length(factIds)
-    if length(varIds) == 0
-        return false
-    end
+    length(varIds) + length(factIds) == 0 && return false
 
     # Total connected nodes - thank you Neo4j for 0..* awesomeness!!
-    connectedList = _getLabelsFromCyphonQuery(dfg.neo4jInstance, "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(varIds[1]))-[FACTORGRAPH*]-(node:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId))")
-
-    return length(connectedList) == totalNodes
+    query = """
+        match (n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(varIds[1]))-[FACTORGRAPH*]-(node:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId))
+        WHERE n:VARIABLE OR n:FACTOR OR node:VARIABLE OR node:FACTOR
+        WITH collect(n)+collect(node) as nodelist
+        unwind nodelist as nodes
+        return count(distinct nodes)"""
+    @debug "[Querying] $query"
+    result = _queryNeo4j(dfg.neo4jInstance, query)
+    # Neo4j.jl data structure sometimes feels brittle... like below
+    return result.results[1]["data"][1]["row"][1] == length(varIds) + length(factIds)
 end
 
 #Alias

src/CloudGraphsDFG/services/CommonFunctions.jl

@@ -1,5 +1,21 @@
 alphaOnlyMatchRegex = r"^[a-zA-Z0-9_]*$"
 
+"""
+$(SIGNATURES)
+Returns the transaction for a given query.
+NOTE: Must commit(transaction) after you're done.
+"""
+function _queryNeo4j(neo4jInstance::Neo4jInstance, query::String)
+    loadtx = transaction(neo4jInstance.connection)
+    result = loadtx(query; submit=true)
+    if length(result.errors) > 0
+        error(string(result.errors))
+    end
+    # Have to finish the transaction
+    commit(loadtx)
+    return result
+end
+
 """
 $(SIGNATURES)
 Returns the list of CloudGraph nodes that matches the Cyphon query.
@@ -11,16 +27,9 @@ If orderProperty is not ==, then 'order by n.{orderProperty} will be appended to
 So can make orderProperty = label or id.
 """
 function _getLabelsFromCyphonQuery(neo4jInstance::Neo4jInstance, matchCondition::String, orderProperty::String="")::Vector{Symbol}
-    # 2. Perform the transaction
-    loadtx = transaction(neo4jInstance.connection)
     query = "match $matchCondition return distinct(node.label) $(orderProperty != "" ? "order by node.$orderProperty" : "")";
-    nodes = loadtx(query; submit=true)
-    if length(nodes.errors) > 0
-        error(string(nodes.errors))
-    end
-    nodeIds = map(node -> node["row"][1], nodes.results[1]["data"])
-    # Have to finish the transaction
-    commit(loadtx)
+    result = _queryNeo4j(neo4jInstance, query)
+    nodeIds = map(node -> node["row"][1], result.results[1]["data"])
     return Symbol.(nodeIds)
 end
 

src/services/AbstractDFG.jl

@@ -569,7 +569,6 @@ showFactor(fgl::G, fsym::Symbol) where G <: AbstractDFG = @show getFactor(fgl,fs
 Produces a dot-format of the graph for visualization.
 """
 function toDot(dfg::AbstractDFG)::String
-    @warn "Falling Back to convert to GraphsDFG"
     #TODO implement convert
     graphsdfg = GraphsDFG{AbstractParams}()
     DistributedFactorGraphs._copyIntoGraph!(dfg, graphsdfg, union(getVariableIds(dfg), getFactorIds(dfg)), true)
@@ -589,7 +588,6 @@ Note
 - Based on graphviz.org
 """
 function toDotFile(dfg::AbstractDFG, fileName::String="/tmp/dfg.dot")::Nothing
-    @warn "Falling Back to convert to GraphsDFG"
     #TODO implement convert
     graphsdfg = GraphsDFG{AbstractParams}()
     DistributedFactorGraphs._copyIntoGraph!(dfg, graphsdfg, union(getVariableIds(dfg), getFactorIds(dfg)), true)

src/services/DFGVariable.jl

@@ -77,31 +77,80 @@ function unpack(dfg::G, d::PackedVariableNodeData)::VariableNodeData where G <:
   end
   @debug "Net conversion result: $st"
 
+  if st == nothing
+      error("The variable doesn't seem to have a softtype. It needs to set up with an InferenceVariable from IIF. This will happen if you use DFG to add serialized variables directly and try use them. Please use IncrementalInference.addVariable().")
+  end
+
   return VariableNodeData(M3,M4, d.BayesNetOutVertIDs,
     d.dimIDs, d.dims, d.eliminated, d.BayesNetVertID, d.separator,
     st, d.initialized, d.inferdim, d.ismargin, d.dontmargin )
 end
 
 function compare(a::VariableNodeData, b::VariableNodeData)
-    TP = true
-    TP = TP && a.val == b.val
-    TP = TP && a.bw == b.bw
-    TP = TP && a.BayesNetOutVertIDs == b.BayesNetOutVertIDs
-    TP = TP && a.dimIDs == b.dimIDs
-    TP = TP && a.dims == b.dims
-    TP = TP && a.eliminated == b.eliminated
-    TP = TP && a.BayesNetVertID == b.BayesNetVertID
-    TP = TP && a.separator == b.separator
-    TP = TP && abs(a.inferdim - b.inferdim) < 1e-14
-    TP = TP && a.ismargin == b.ismargin
-    TP = TP && a.softtype == b.softtype
-    return TP
+    a.val != b.val && @debug("val is not equal")==nothing && return false
+    a.bw != b.bw && @debug("bw is not equal")==nothing && return false
+    a.BayesNetOutVertIDs != b.BayesNetOutVertIDs && @debug("BayesNetOutVertIDs is not equal")==nothing && return false
+    a.dimIDs != b.dimIDs && @debug("dimIDs is not equal")==nothing && return false
+    a.dims != b.dims && @debug("dims is not equal")==nothing && return false
+    a.eliminated != b.eliminated && @debug("eliminated is not equal")==nothing && return false
+    a.BayesNetVertID != b.BayesNetVertID && @debug("BayesNetVertID is not equal")==nothing && return false
+    a.separator != b.separator && @debug("separator is not equal")==nothing && return false
+    a.initialized != b.initialized && @debug("initialized is not equal")==nothing && return false
+    abs(a.inferdim - b.inferdim) > 1e-14 && @debug("inferdim is not equal")==nothing && return false
+    a.ismargin != b.ismargin && @debug("ismargin is not equal")==nothing && return false
+    a.dontmargin != b.dontmargin && @debug("dontmargin is not equal")==nothing && return false
+    typeof(a.softtype) != typeof(b.softtype) && @debug("softtype is not equal")==nothing && return false
+    return true
 end
 
+"""
+    $(SIGNATURES)
+Equality check for VariableNodeData.
+"""
 function ==(a::VariableNodeData,b::VariableNodeData, nt::Symbol=:var)
   return DistributedFactorGraphs.compare(a,b)
 end
 
+"""
+    $(SIGNATURES)
+Equality check for VariableEstimate.
+"""
+function ==(a::VariableEstimate, b::VariableEstimate)::Bool
+  a.solverKey != b.solverKey && @debug("solverKey are not equal")==nothing && return false
+  a.type != b.type && @debug("type is not equal")==nothing && return false
+  a.estimate != b.estimate && @debug("estimate are not equal")==nothing && return false
+  a.lastUpdatedTimestamp != b.lastUpdatedTimestamp && @debug("lastUpdatedTimestamp is not equal")==nothing && return false
+  return true
+end
+
+"""
+    $(SIGNATURES)
+Equality check for DFGVariable.
+"""
+function ==(a::DFGVariable, b::DFGVariable)::Bool
+  a.label != b.label && @debug("label is not equal")==nothing && return false
+  a.timestamp != b.timestamp && @debug("timestamp is not equal")==nothing && return false
+  a.tags != b.tags && @debug("tags is not equal")==nothing && return false
+  symdiff(keys(a.estimateDict), keys(b.estimateDict)) != Set(Symbol[]) && @debug("estimateDict keys are not equal")==nothing && return false
+  for k in keys(a.estimateDict)
+    a.estimateDict[k] != b.estimateDict[k] && @debug("estimateDict[$k] is not equal")==nothing && return false
+  end
+  symdiff(keys(a.solverDataDict), keys(b.solverDataDict)) != Set(Symbol[]) && @debug("solverDataDict keys are not equal")==nothing && return false
+  for k in keys(a.solverDataDict)
+    a.solverDataDict[k] != b.solverDataDict[k] && @debug("solverDataDict[$k] is not equal")==nothing && return false
+  end
+  a.smallData != b.smallData && @debug("smallData is not equal")==nothing && return false
+  a.bigData != b.bigData && @debug("bigData is not equal")==nothing && return false
+  a.ready != b.ready && @debug("ready is not equal")==nothing && return false
+  a.backendset != b.backendset && @debug("backendset is not equal")==nothing && return false
+  a._internalId != b._internalId && @debug("_internalId is not equal")==nothing && return false
+  return true
+end
+
+"""
+    $(SIGNATURES)
+Convert a DFGVariable to a DFGVariableSummary.
+"""
 function convert(::Type{DFGVariableSummary}, v::DFGVariable)
     return DFGVariableSummary(v.label, v.timestamp, deepcopy(v.tags), deepcopy(v.estimateDict), Symbol(typeof(getSofttype(v))), v._internalId)
 end

test/Project.toml

@@ -0,0 +1,18 @@
+[deps]
+Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
+DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231"
+Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
+IncrementalInference = "904591bb-b899-562f-9e6f-b8df64c7d480"
+JSON2 = "2535ab7d-5cd8-5a07-80ac-9b1792aadce3"
+LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MetaGraphs = "626554b9-1ddb-594c-aa3c-2596fe9399a5"
+Neo4j = "d2adbeaf-5838-5367-8a2f-e46d570981db"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
+Requires = "ae029012-a4dd-5104-9daa-d747884805df"
+SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

test/obsolete_nowinRoME/cloudGraphsDFGTests.jl

@@ -1,10 +1,4 @@
 ## Sandboxing for CloudgraphsDFG
-
-## Requires local Neo4j with user/pass neo4j:test
-# To run the Docker image
-# Install: docker pull neo4j
-# Run: sudo docker run --publish=7474:7474 --publish=7687:7687 --env NEO4J_AUTH=neo4j/test neo4j
-
 using Revise
 using Neo4j
 using DistributedFactorGraphs

test/runtests.jl

@@ -2,7 +2,13 @@ using Test
 using GraphPlot # For plotting tests
 using Neo4j
 using DistributedFactorGraphs
-using IncrementalInference
+using Pkg
+
+## To run the IIF tests, you need a local Neo4j with user/pass neo4j:test
+# To run a Docker image
+# Install: docker pull neo4j
+# Run: docker run --publish=7474:7474 --publish=7687:7687 --env NEO4J_AUTH=neo4j/test neo4j
+##
 
 # Instantiate the APIs that you would like to test here
 # Can do duplicates with different parameters.
@@ -48,6 +54,43 @@ end
     @test_throws UndefVarError MetaGraphsDFG{NoSolverParams}()
 end
 
+if haskey(Pkg.installed(), "IncrementalInference")
+    @info "------------------------------------------------------------------------"
+    @info "These tests are using IncrementalInference to do additional driver tests"
+    @info "------------------------------------------------------------------------"
+
+    using IncrementalInference
+
+    apis = [
+        # GraphsDFG{NoSolverParams}(),
+        # LightDFG{NoSolverParams}(),
+        # MetaGraphsDFG{NoSolverParams}(),
+        # SymbolDFG{NoSolverParams}(),
+        CloudGraphsDFG{SolverParams}("localhost", 7474, "neo4j", "test",
+                                    "testUser", "testRobot", "testSession",
+                                    nothing,
+                                    nothing,
+                                    IncrementalInference.decodePackedType,
+                                    IncrementalInference.rebuildFactorMetadata!,
+                                    solverParams=SolverParams())
+            ]
+    for api in apis
+        @testset "Testing Driver: $(typeof(api))" begin
+            @info "Testing Driver: $(api)"
+            global dfg = deepcopy(api)
+            include("iifInterfaceTests.jl")
+        end
+    end
+else
+    @warn "Skipping IncrementalInference driver tests"
+end
+
+Test that we don't export LightDFG and MetaGraphsDFG
+@testset "Deprecated Drivers Test" begin
+    @test_throws UndefVarError SymbolDFG{NoSolverParams}()
+    @test_throws UndefVarError MetaGraphsDFG{NoSolverParams}()
+end
+
 # Test special cases
 
 @testset "Plotting Tests" begin