Merge branch 'master' of https://github.com/JuliaRobotics/DistributedFactorGraphs.jl into 1Q20/cgdfg_ppes

Author: GearsAD

.github/workflows/TagBot.yml

@@ -0,0 +1,11 @@
+name: TagBot
+on:
+  schedule:
+    - cron: 0 * * * *
+jobs:
+  TagBot:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: JuliaRegistries/TagBot@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

.travis.yml

@@ -7,33 +7,33 @@ os:
 
 arch:
     - amd64
-#FIXME#    - arm64
 
 services:
     - neo4j
 
 julia:
-#FIXME#    - 1.0
+    - 1.0
     - 1.2
-#FIXME#    - 1.3
-#FIXME#    - nightly
+    - 1.3
+    - nightly
 
 env:
-    - IIF_TEST=true
-
-#FIXME#jobs:
-#FIXME#  include:
-#FIXME#    - julia: 1.2
-#FIXME#      env: IIF_TEST=true
-#FIXME#      if: NOT branch =~ ^release.*$
-#FIXME#    - stage: "Documentation"
-#FIXME#      julia: 1.2
-#FIXME#      os: linux
-#FIXME#      script:
-#FIXME#        - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
-#FIXME#        - julia --project=docs/ docs/make.jl
-#FIXME#      after_success: skip
-#FIXME#  fast_finish: true
+    - IIF_TEST=false
+
+jobs:
+  include:
+    - julia: 1.2
+      env: IIF_TEST=true
+      if: NOT branch =~ ^release.*$
+    - arch: arm64
+    - stage: "Documentation"
+      julia: 1.2
+      os: linux
+      script:
+        - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
+        - julia --project=docs/ docs/make.jl
+      after_success: skip
+  fast_finish: true
 
 notifications:
     email: false

docs/Project.toml

@@ -1,5 +1,6 @@
 [deps]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231"
 LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d"
 
 [compat]

docs/make.jl

@@ -1,4 +1,5 @@
 using Documenter
+using GraphPlot
 using DistributedFactorGraphs
 
 makedocs(

docs/src/BuildingGraphs.md

@@ -111,16 +111,16 @@ Reading, updating, and deleting all use DFG functions (as opposed to adding,
 where using the IncrementalInference functions are recommended).
 
 Each variable and factor is uniquely identified by its label. The list of
-variable and factor labels can be retrieved with the `ls`/`getVariableIds` and
-`lsf`/`getFactorIds` functions:
+variable and factor labels can be retrieved with the `ls`/`listVariables` and
+`lsf`/`listFactors` functions:
 
 ```@docs
-getVariableIds
+listVariables
 ls
 ```
 
 ```@docs
-getFactorIds
+listFactors
 lsf
 ```
 

docs/src/GraphData.md

@@ -43,27 +43,24 @@ Labels are the principle identifier of a variable or factor.
 getLabel
 ```
 
-```@docs
-getLabel
-```
-
 #### Timestamps
 
 Each variable or factor can have a timestamp associated with it.
 
 ```@docs
 getTimestamp
-setTimestamp!
+setTimestamp
 ```
 
 #### Tags
 
 Tags are a set of symbols that contain identifiers for the variable or factor.
 
 ```@docs
-addTag
+listTags
 mergeTags!
-deleteTags!
+removeTags!
+emptyTags!
 ```
 
 ### Solvable
@@ -95,7 +92,6 @@ For each PPE structure, there are accessors for getting individual values:
 getMaxPPE
 getMeanPPE
 getSuggestedPPE
-getPPE
 ```
 
 Related functions for getting, adding/updating, and deleting PPE structures:
@@ -106,6 +102,7 @@ getPPE
 addPPE!
 updatePPE!
 deletePPE!
+mergePPEs!
 ```
 
 Example of PPE operations:
@@ -139,6 +136,7 @@ getVariableSolverData
 addVariableSolverData!
 updateVariableSolverData!
 deleteVariableSolverData!
+mergeVariableSolverData!
 ```
 
 Example of solver data operations:
@@ -163,7 +161,7 @@ itself, large entries will slow the graph down, so if data should exceed a
 few bytes/kb, it should rather be saved in bigData.
 
 ```@docs
-getSmallData,
+getSmallData
 setSmallData!
 ```
 

docs/src/TraversingAndQuerying.md

@@ -4,6 +4,5 @@
 getNeighbors
 getSubgraphAroundNode
 getSubgraph
-getIncidenceMatrix
-getIncidenceMatrixSparse
+getBiadjacencyMatrix
 ```

docs/src/func_ref.md

@@ -7,67 +7,6 @@ Pages = [
 Depth = 3
 ```
 
-### Graph Types
-```@docs
-GraphsDFG
-```
-
-### Creating DFG Factor Graphs
-```@docs
-addVariable!
-addFactor!
-```
-
-### Getting Factor Graph Nodes
-```@docs
-getVariables
-getFactors
-getVariable
-getFactor
-getNeighbors
-ls
-lsf
-```
-
-### Updating Factor Graph Nodes
-```@docs
-updateVariable!
-updateFactor!
-```
-
-### Deleting Factor Graph Nodes
-```@docs
-deleteVariable!
-deleteFactor!
-```
-
-### Getting Adjacency Matrix
-```@docs
-getIncidenceMatrix
-getIncidenceMatrixSparse
-```
-
-### Validating Factor Graph Connectivity
-```@docs
-isFullyConnected
-hasOrphans
-```
-
-### Copying Subgraphs
-```@docs
-getSubgraphAroundNode
-getSubgraph
-```
-
-### Summaries
-```@docs
-getSummary
-getSummaryGraph
-```
-
-### Visualization and Plotting
-```@docs
-toDot
-toDotFile
-dfgplot
+```@autodocs
+Modules = [DistributedFactorGraphs, LightDFGs, DFGPlots]
 ```

src/BigData/services/AbstractBigDataEntries.jl

@@ -15,3 +15,96 @@ end
 function ==(a::FileBigDataEntry, b::FileBigDataEntry)
     return a.key == b.key && a.filename == b.filename
 end
+
+"""
+    $(SIGNATURES)
+Add Big Data Entry to a DFG variable
+"""
+function addBigDataEntry!(var::AbstractDFGVariable, bde::AbstractBigDataEntry)::AbstractBigDataEntry
+    haskey(var.bigData,bde.key) && error("BigData entry $(bde.key) already exists in variable")
+    var.bigData[bde.key] = bde
+    return bde
+end
+
+"""
+    $(SIGNATURES)
+Add Big Data Entry to distributed factor graph.
+Should be extended if DFG variable is not returned by reference.
+"""
+function addBigDataEntry!(dfg::AbstractDFG, label::Symbol, bde::AbstractBigDataEntry)::AbstractBigDataEntry
+    return addBigDataEntry!(getVariable(dfg, label), bde)
+end
+
+"""
+    $(SIGNATURES)
+Get big data entry
+"""
+function getBigDataEntry(var::AbstractDFGVariable, key::Symbol)::Union{Nothing, AbstractBigDataEntry}
+    !haskey(var.bigData, key) && (error("BigData entry $(key) does not exist in variable"); return nothing)
+    return var.bigData[key]
+end
+
+function getBigDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol)::Union{Nothing, AbstractBigDataEntry}
+    return getBigDataEntry(getVariable(dfg, label), key)
+end
+
+"""
+    $(SIGNATURES)
+Update big data entry
+"""
+function updateBigDataEntry!(var::AbstractDFGVariable,  bde::AbstractBigDataEntry)::Union{Nothing, AbstractBigDataEntry}
+    !haskey(var.bigData,bde.key) && (@warn "$(bde.key) does not exist in variable, adding")
+    var.bigData[bde.key] = bde
+    return bde
+end
+function updateBigDataEntry!(dfg::AbstractDFG, label::Symbol,  bde::AbstractBigDataEntry)::Union{Nothing, AbstractBigDataEntry}
+    # !isVariable(dfg, label) && return nothing
+    return updateBigDataEntry!(getVariable(dfg, label), bde)
+end
+
+"""
+    $(SIGNATURES)
+Delete big data entry from the factor graph.
+Note this doesn't remove it from any data stores.
+"""
+function deleteBigDataEntry!(var::AbstractDFGVariable, key::Symbol)::Union{Nothing, AbstractDFGVariable} #users responsibility to delete big data in db before deleting entry
+    bde = getBigDataEntry(var, key)
+    bde == nothing && return nothing
+    delete!(var.bigData, key)
+    return var
+end
+function deleteBigDataEntry!(dfg::AbstractDFG, label::Symbol, key::Symbol)::Union{Nothing, AbstractDFGVariable} #users responsibility to delete big data in db before deleting entry
+    !isVariable(dfg, label) && return nothing
+    return deleteBigDataEntry!(getVariable(dfg, label), key)
+end
+
+function deleteBigDataEntry!(var::AbstractDFGVariable, entry::AbstractBigDataEntry)::Union{Nothing, AbstractDFGVariable} #users responsibility to delete big data in db before deleting entry
+    return deleteBigDataEntry!(var, entry.key)
+end
+
+"""
+    $(SIGNATURES)
+Get big data entries, Vector{AbstractBigDataEntry}
+"""
+function getBigDataEntries(var::AbstractDFGVariable)::Vector{AbstractBigDataEntry}
+    #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractBigDataEntry}}?
+    collect(values(var.bigData))
+end
+function getBigDataEntries(dfg::AbstractDFG, label::Symbol)::Union{Nothing, Vector{AbstractBigDataEntry}}
+    !isVariable(dfg, label) && return nothing
+    #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractBigDataEntry}}?
+    getBigDataEntries(getVariable(dfg, label))
+end
+
+
+"""
+    $(SIGNATURES)
+getBigDataKeys
+"""
+function getBigDataKeys(var::AbstractDFGVariable)::Vector{Symbol}
+    collect(keys(var.bigData))
+end
+function getBigDataKeys(dfg::AbstractDFG, label::Symbol)::Union{Nothing, Vector{Symbol}}
+    !isVariable(dfg, label) && return nothing
+    getBigDataKeys(getVariable(dfg, label))
+end

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -306,8 +306,9 @@ function getFactors(dfg::CloudGraphsDFG, regexFilter::Union{Nothing, Regex}=noth
     return map(vId->getFactor(dfg, vId), factorIds)
 end
 
-function listFactors(dfg::CloudGraphsDFG, regexFilter::Union{Nothing, Regex}=nothing; solvable::Int=0)::Vector{Symbol}
+function listFactors(dfg::CloudGraphsDFG, regexFilter::Union{Nothing, Regex}=nothing; tags::Vector{Symbol}=Symbol[], solvable::Int=0)::Vector{Symbol}
     # Optimized for DB call
+    length(tags) > 0 && (@error "Filter on tags not implemented for CloudGraphsDFG")
     if regexFilter == nothing
         return _getLabelsFromCyphonQuery(dfg.neo4jInstance, "(node:$(join(_getLabelsForType(dfg, DFGFactor),':'))) where node.solvable >= $solvable")
     else