Standardize and test cloud graph variable and factor crud (#412)

* apt-get neo4j:3.5
* Variable and Factor CRUD and test (un)packing
* decodePackedType only when not running iif

Author: Affie

.travis.yml

@@ -1,6 +1,5 @@
 language: julia
 sudo: required
-# dist: trusty
 
 os:
     - linux
@@ -12,7 +11,6 @@ services:
     - neo4j
 
 julia:
-    - 1.2
     - 1.4
     - nightly
 
@@ -21,17 +19,12 @@ env:
 
 jobs:
   include:
-    - julia: 1.3
-      dist: trusty
+    - julia: 1.4
       env: IIF_TEST=true
       if: NOT branch =~ ^release.*$
-      # Set the password for Neo4j to neo4j:test
-      before_script:
-        - sleep 10
-        - curl -v POST http://neo4j:neo4j@localhost:7474/user/neo4j/password -d"password=test"
     - arch: arm64
     - stage: "Documentation"
-      julia: 1.3
+      julia: 1.4
       os: linux
       script:
         - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
@@ -41,26 +34,24 @@ jobs:
   allow_failures:
       - julia: nightly
       - arch: arm64
-      #- env: IIF_TEST=true
 
 notifications:
     email: false
 
-
 # Attempt to install neo4j on Xenial and set the password for Neo4j to neo4j:test
-# Cloud tests errors with authentication error
-#before_script:
-#   - sudo add-apt-repository -y ppa:openjdk-r/ppa
-#   - sudo apt-get update
-#   - wget -O - https://debian.neo4j.com/neotechnology.gpg.key | sudo apt-key add -
-#   - echo 'deb https://debian.neo4j.com stable latest' | sudo tee -a /etc/apt/sources.list.d/neo4j.list
-#   - sudo apt-get update
-#   - apt list -a neo4j
-#   - sudo apt-get install neo4j=1:4.0.0
-#   - sudo service neo4j start
-#   - sleep 10
-#   - sudo neo4j-admin set-initial-password test
-#   - curl -I http://localhost:7474/
+before_script:
+  - sudo add-apt-repository -y ppa:openjdk-r/ppa
+  - sudo apt-get update
+  - wget -O - https://debian.neo4j.com/neotechnology.gpg.key | sudo apt-key add -
+  - echo 'deb https://debian.neo4j.com stable 3.5' | sudo tee /etc/apt/sources.list.d/neo4j.list
+  - sudo apt-get update
+  - apt list -a neo4j
+  - sudo apt-get install neo4j
+  - sudo service neo4j start
+  - sleep 10
+  - curl -v POST http://neo4j:neo4j@localhost:7474/user/neo4j/password -d"password=test"
+# - sudo neo4j-admin set-initial-password test
+  - curl -I http://localhost:7474/
 
 after_success:
     - julia -e 'using Pkg; cd(Pkg.dir("DistributedFactorGraphs")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())'

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -400,7 +400,7 @@ function getNeighbors(dfg::CloudGraphsDFG, node::T; solvable::Int=0)::Vector{Sym
     query = "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(node.label))--(node) where (node:VARIABLE or node:FACTOR) and node.solvable >= $solvable"
     @debug "[Query] $query"
     neighbors = _getLabelsFromCyphonQuery(dfg.neo4jInstance, query)
-    # If factor, need to do variable ordering
+    # If factor, need to do variable ordering TODO, Do we? does it matter if we always use _variableOrderSymbols in calculations?
     if T <: DFGFactor
         neighbors = intersect(node._variableOrderSymbols, neighbors)
     end
@@ -411,7 +411,7 @@ function getNeighbors(dfg::CloudGraphsDFG, label::Symbol; solvable::Int=0)::Vect
     query = "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(label))--(node) where (node:VARIABLE or node:FACTOR) and node.solvable >= $solvable"
     @debug "[Query] $query"
     neighbors = _getLabelsFromCyphonQuery(dfg.neo4jInstance, query)
-    # If factor, need to do variable ordering
+    # If factor, need to do variable ordering TODO, Do we? does it matter if we always use _variableOrderSymbols in calculations?
     if isFactor(dfg, label)
         # Server is authority
         serverOrder = Symbol.(JSON2.read(_getNodeProperty(dfg.neo4jInstance, [dfg.userId, dfg.robotId, dfg.sessionId, String(label)], "_variableOrderSymbols")))

src/entities/DFGFactor.jl

@@ -7,6 +7,14 @@ abstract type PackedInferenceType end
 
 abstract type FunctorInferenceType <: Function end
 
+# DF suggestion1 (this is a first guess)  `ConvolutionObject` --> `FactorInsituObject`
+# DF second guess,  ConvolutionObject <: FactorInsituObject
+# JT Maybe second guess as intermediate step where ConvolutionObject is moved to IIF.
+# then it can be removed to CommonConvWrapper <: FactorInsituObject
+
+# DF, Convolution is IIF idea, but DFG should know about "FactorOperationalMemory"
+# DF, IIF.CommonConvWrapper <: FactorOperationalMemory # FIXME
+# MAYBE rename "FactorOperationalMemory"
 abstract type ConvolutionObject <: Function end
 
 abstract type FunctorSingleton <: FunctorInferenceType end
@@ -19,13 +27,22 @@ abstract type FunctorPairwiseMinimize <: FunctorInferenceType end
 
 """
 $(TYPEDEF)
+
+Notes
+- S::Symbol
+
+Designing (WIP)
+- T <: Union{FactorOperationalMemory, PackedInferenceType}
+# in IIF.CCW{T <: DFG.InferenceType}
+# in IIF.FunctorPairwiseMinimize <: InferenceType # DFG whatever, something, we'll figure it out
+# in Main/User, SomeFactor <: FunctorPairwiseMinimize
 """
-mutable struct GenericFunctionNodeData{T, S}
+mutable struct GenericFunctionNodeData{T, S} #{T<:Union{PackedInferenceType, FunctorInferenceType, ConvolutionObject}, S<:Union{Symbol, AbstractString}}
     fncargvID::Vector{Symbol}
     eliminated::Bool
     potentialused::Bool
     edgeIDs::Array{Int,1}
-    frommodule::S #Union{Symbol, AbstractString}
+    frommodule::S # JT TODO remove frommodule, not used at all as far as i can tell
     fnc::T
     multihypo::Array{Float64} # likely to moved when GenericWrapParam is refactored
     certainhypo::Vector{Int}

src/services/Serialization.jl

@@ -146,10 +146,11 @@ function unpackFactor(dfg::G, packedProps::Dict{String, Any})::DFGFactor where G
     solvable = packedProps["solvable"]
 
     # Rebuild DFGFactor
+    #TODO use constuctor to create factor
     factor = DFGFactor{typeof(fullFactor.fnc), Symbol}(Symbol(label), 0, timestamp)
 
     union!(factor.tags, tags)
-    # factor.data = fullFactor #TODO 
+    # factor.data = fullFactor #TODO
     setSolverData!(factor, fullFactor)
     factor._variableOrderSymbols = _variableOrderSymbols
     setSolvable!(factor, solvable)

test/consolInterfaceDev.jl

@@ -5,7 +5,7 @@ using DistributedFactorGraphs
 using Pkg
 using Dates
 
-using IncrementalInference
+# using IncrementalInference
 
 include("testBlocks.jl")
 
@@ -19,8 +19,8 @@ function DFG.CloudGraphsDFG(; params=NoSolverParams())
                                   "description",
                                   nothing,
                                   nothing,
-                                  IncrementalInference.decodePackedType,#(dfg,f)->f,
-                                  IncrementalInference.rebuildFactorMetadata!,#(dfg,f)->f,
+                                  (dfg,f)->f,#IncrementalInference.decodePackedType,#(dfg,f)->f,
+                                  (dfg,f)->f,#ncrementalInference.rebuildFactorMetadata!,#(dfg,f)->f,
                                   solverParams=params)
     createDfgSessionIfNotExist(cgfg)
     return cgfg
@@ -49,8 +49,8 @@ function DFG.CloudGraphsDFG(description::String,
                                                 description,
                                                 nothing,
                                                 nothing,
-                                                IncrementalInference.decodePackedType,#(dfg,f)->f,
-                                                IncrementalInference.rebuildFactorMetadata!,#(dfg,f)->f,
+                                                (dfg,f)->f,#IncrementalInference.decodePackedType,#(dfg,f)->f,
+                                                (dfg,f)->f,#IncrementalInference.rebuildFactorMetadata!,#(dfg,f)->f,
                                                 solverParams=solverParams)
 
     createDfgSessionIfNotExist(cdfg)
@@ -82,25 +82,15 @@ end
 end
 
 # DFGVariable structure construction and accessors
-# @testset "DFG Variable" begin
-#     global var1, var2, var3, v1_tags
-#     var1, var2, var3, v1_tags = DFGVariableSCA()
-# end
-newfg = initfg()
-var1 = addVariable!(newfg, :a, ContinuousScalar, labels=[:POSE])
-var2 = addVariable!(newfg, :b, ContinuousScalar, labels=[:LANDMARK])
-var3 = addVariable!(newfg, :c, ContinuousScalar)
-v1_tags = Set([:VARIABLE, :POSE])
-
+@testset "DFG Variable" begin
+    global var1, var2, var3, v1_tags, vorphan
+    var1, var2, var3, vorphan, v1_tags = DFGVariableSCA()
+end
 
 # DFGFactor structure construction and accessors
-# @testset "DFG Factor" begin
-#     global fac0, fac1, fac2 = DFGFactorSCA()
-# end
-
-fac0 = addFactor!(newfg, [:a], Prior(Normal()))
-fac1 = addFactor!(newfg, [:a, :b], LinearConditional(Normal()))
-fac2 = addFactor!(newfg, [:b, :c], LinearConditional(Normal()))
+@testset "DFG Factor" begin
+    global fac0, fac1, fac2 = DFGFactorSCA()
+end
 
 @testset "Variables and Factors CRUD and SET" begin
     VariablesandFactorsCRUD_SET!(fg1, var1, var2, var3, fac0, fac1, fac2)
@@ -109,6 +99,66 @@ end
 @testset "tags" begin
     tagsTestBlock!(fg1, var1, v1_tags)
 end
+
+# @testset "Parametric Point Estimates" begin
+#     PPETestBlock!(fg1, var1)
+# end
+
+@testset "Variable Solver Data" begin
+    VSDTestBlock!(fg1, var1)
+end
+
+@testset "BigData Entries" begin
+    @test_skip BigDataEntriesTestBlock!(fg1, var2)
+end
+
+# @testset "TODO Sorteer groep" begin
+#     testGroup!(fg1, var1, var2, fac0, fac1)
+# end
+
+
+@testset "Adjacency Matrices" begin
+    fg = testDFGAPI()
+    clearRobot!!(fg)
+
+    DFGVariable(:a, TestSofttype1())
+    addVariable!(fg, var1)
+    setSolvable!(fg, :a, 1)
+    addVariable!(fg, var2)
+    addFactor!(fg, fac1)
+    addVariable!(fg, vorphan)
+
+    AdjacencyMatricesTestBlock(fg)
+end
+
+
+@testset "Getting Neighbors" begin
+    GettingNeighbors(testDFGAPI)
+end
+
+
+@testset "Getting Subgraphs" begin
+    GettingSubgraphs(testDFGAPI)
+end
+
+# @testset "Building Subgraphs" begin
+#     BuildingSubgraphs(testDFGAPI)
+# end
+#
+# #TODO Summaries and Summary Graphs
+# @testset "Summaries and Summary Graphs" begin
+#     Summaries(testDFGAPI)
+# end
+#
+# @testset "Producing Dot Files" begin
+#     ProducingDotFiles(testDFGAPI)
+# end
+#
+# @testset "Connectivity Test" begin
+#      ConnectivityTest(testDFGAPI)
+# end
+
+
 #
 #
 # fg = fg1

test/interfaceTests.jl

@@ -14,7 +14,7 @@ end
 # DFGVariable structure construction and accessors
 @testset "DFG Variable" begin
     global var1, var2, var3, v1_tags
-    var1, var2, var3, v1_tags = DFGVariableSCA()
+    var1, var2, var3, vorphan, v1_tags = DFGVariableSCA()
 end
 
 # DFGFactor structure construction and accessors

test/runtests.jl

@@ -40,6 +40,22 @@ for api in apis
     end
 end
 
+if get(ENV, "IIF_TEST", "") != "true"
+    @testset "Consolidation WIP Testing Driver: CloudGraphsDFG" begin
+        global decodePackedType
+        function decodePackedType(dfg::G, packeddata::GenericFunctionNodeData{PT,<:AbstractString}) where {PT, G <: AbstractDFG}
+          # usrtyp = convert(FunctorInferenceType, packeddata.fnc)
+          # Also look at parentmodule
+          usrtyp = getfield(PT.name.module, Symbol(string(PT.name.name)[7:end]))
+          fulltype = DFG.FunctionNodeData{TestCCW{usrtyp}}
+          factor = convert(fulltype, packeddata)
+          return factor
+        end
+        @info "Testing Driver: CloudGraphsDFG"
+        global testDFGAPI = CloudGraphsDFG
+        include("consolInterfaceDev.jl")
+    end
+end
 
 # Test special cases
 @testset "Plotting Tests" begin
@@ -76,12 +92,6 @@ if get(ENV, "IIF_TEST", "") == "true"
 
     using IncrementalInference
 
-    @testset "Consolidation WIP Testing Driver: CloudGraphsDFG" begin
-        @info "Testing Driver: CloudGraphsDFG"
-        global testDFGAPI = CloudGraphsDFG
-        include("consolInterfaceDev.jl")
-    end
-
     apis = [
         GraphsDFG{SolverParams}(),
         LightDFG{SolverParams}(),