Merging master

Author: GearsAD

.travis.yml

@@ -15,12 +15,30 @@ julia:
     - 1.3
     - nightly
 
+env:
+    - IIF_TEST=false
+
+jobs:
+  include:
+    - julia: 1.2
+      env: IIF_TEST=true
+    - stage: "Documentation"
+      julia: 1.0
+      os: linux
+      script:
+        - julia -e 'import Pkg; Pkg.add("Documenter"); Pkg.add("Neo4j"); Pkg.add("GraphPlot")'
+        - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd()));
+                                               Pkg.instantiate()'
+        - julia --project=docs/ docs/make.jl
+      after_success: skip
+
 notifications:
     email: false
 
 matrix:
   allow_failures:
-    - julia: nightly
+  - julia: nightly
+  - env: IIF_TEST=true
 
 # Set the password for Neo4j to neo4j:test
 before_script:
@@ -29,22 +47,3 @@ before_script:
 
 after_success:
     - julia -e 'using Pkg; cd(Pkg.dir("DistributedFactorGraphs")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder()); Coveralls.submit(process_folder())'
-
-jobs:
-  include:
-    - stage: "Documentation"
-      julia: 1.0
-      os: linux
-      script:
-        - julia -e 'import Pkg; Pkg.add("Documenter"); Pkg.add("DataFrames")'
-        - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd()));
-                                               Pkg.instantiate()'
-        - julia --project=docs/ docs/make.jl
-      after_success: skip
-
-    - stage: "IIF Driver Tests"
-      julia: 1.2
-      os: linux
-      script:
-        - julia -e 'import Pkg; Pkg.add("IncrementalInference"); Pkg.test("DistributedFactorGraphs")'
-      after_success: skip

src/BigData.jl

@@ -0,0 +1,139 @@
+
+# TODO move to ...
+
+
+# """
+#     $(TYPEDEF)
+# Abstract parent struct for big data entry.
+# """
+# abstract type AbstractBigDataEntry end
+
+# --- AbstractBigData Interfaces ---
+# fields to implement:
+# - key::Symbol
+# available methods:
+# - addBigDataEntry!
+# - getBigDataEntry
+# - updateBigDataEntry!
+# - deleteBigDataEntry!
+# - getBigDataEntries
+# - getBigDataKeys
+
+export  addBigDataEntry!,
+        getBigDataEntry,
+        updateBigDataEntry!,
+        deleteBigDataEntry!,
+        getBigDataEntries,
+        getBigDataKeys,
+        MongodbBigDataEntry,
+        FileBigDataEntry
+
+# Methods
+
+#TODO should this return Bool or the modified Variable?
+"""
+    $(SIGNATURES)
+Add Big Data Entry to a DFG variable
+"""
+function addBigDataEntry!(var::AbstractDFGVariable, bde::AbstractBigDataEntry)::Bool
+    haskey(var.bigData,bde.key) && @warn "$(bde.key) already exists in variable, overwriting!"
+    var.bigData[bde.key] = bde
+    return true
+end
+
+"""
+    $(SIGNATURES)
+Add Big Data Entry to distrubuted factor graph.
+Should be extended if DFG variable is not returned by by reference.
+"""
+function addBigDataEntry!(dfg::AbstractDFG, label::Symbol, bde::AbstractBigDataEntry)::Bool
+    return addBigDataEntry!(getVariable(dfg, label), bde)
+end
+
+"""
+    $(SIGNATURES)
+Get big data entry
+"""
+function getBigDataEntry(var::AbstractDFGVariable, key::Symbol)::AbstractBigDataEntry
+    return var.bigData[key]
+end
+function getBigDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol)::AbstractBigDataEntry
+    return getBigDataEntry(getVariable(dfg, label), key)
+end
+
+"""
+    $(SIGNATURES)
+Update big data entry
+"""
+function updateBigDataEntry!(var::AbstractDFGVariable,  bde::AbstractBigDataEntry)::Bool#TODO should this return Bool?
+    !haskey(var.bigData,bde.key) && (@error "$(bde.key) does not exist in variable!"; return false)
+    var.bigData[bde.key] = bde
+    return true
+end
+function updateBigDataEntry!(dfg::AbstractDFG, label::Symbol,  bde::AbstractBigDataEntry)::Bool
+    updateBigDataEntry!(getVariable(dfg, label), bde)
+end
+
+"""
+    $(SIGNATURES)
+Delete big data entry
+"""
+function deleteBigDataEntry!(var::AbstractDFGVariable, key::Symbol)::AbstractBigDataEntry #users responsibility to delete big data in db before deleting entry
+    bde = getBigDataEntry(var, key)
+    delete!(var.bigData, key)
+    return bde
+end
+
+function deleteBigDataEntry!(dfg::AbstractDFG, label::Symbol, key::Symbol)::AbstractBigDataEntry #users responsibility to delete big data in db before deleting entry
+    deleteBigDataEntry!(getVariable(dfg, label), key)
+end
+
+"""
+    $(SIGNATURES)
+Get big data entries, Vector{AbstractBigDataEntry}
+"""
+function getBigDataEntries(var::AbstractDFGVariable)::Vector{AbstractBigDataEntry}
+    #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractBigDataEntry}}?
+    collect(values(var.bigData))
+end
+function getBigDataEntries(dfg::AbstractDFG, label::Symbol)::Vector{AbstractBigDataEntry}
+    #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractBigDataEntry}}?
+    getBigDataEntries(getVariable(dfg, label))
+end
+
+
+"""
+    $(SIGNATURES)
+getBigDataKeys
+"""
+function getBigDataKeys(var::AbstractDFGVariable)::Vector{Symbol}
+    collect(keys(var.bigData))
+end
+function getBigDataKeys(dfg::AbstractDFG, label::Symbol)::Vector{Symbol}
+    getBigDataKeys(getVariable(dfg, label))
+end
+
+
+
+# Types <: AbstractBigDataEntry
+"""
+    $(TYPEDEF)
+BigDataEntry in MongoDB.
+"""
+struct MongodbBigDataEntry <: AbstractBigDataEntry
+    key::Symbol
+    oid::NTuple{12, UInt8} #mongodb object id
+    #maybe other fields such as:
+    #flags::Bool ready, valid, locked, permissions
+    #MIMEType::Symbol
+end
+
+
+"""
+    $(TYPEDEF)
+BigDataEntry in a file.
+"""
+struct FileBigDataEntry <: AbstractBigDataEntry
+    key::Symbol
+    filename::String
+end

src/Common.jl

@@ -129,7 +129,7 @@ function getfnctype(data::GenericFunctionNodeData)
   return data.fnc.usrfnc!
 end
 function getfnctype(fact::DFGFactor; solveKey::Symbol=:default)
-  data = getData(fact) # TODO , solveKey=solveKey)
+  data = solverData(fact) # TODO , solveKey=solveKey)
   return getfnctype(data)
 end
 function getfnctype(dfg::T, lbl::Symbol; solveKey::Symbol=:default) where T <: AbstractDFG
@@ -145,7 +145,7 @@ Notes
 - Replaces older `getfnctype`.
 """
 getFactorType(data::GenericFunctionNodeData) = data.fnc.usrfnc!
-getFactorType(fct::DFGFactor) = getFactorType(getData(fct))
+getFactorType(fct::DFGFactor) = getFactorType(solverData(fct))
 function getFactorType(dfg::G, lbl::Symbol) where G <: AbstractDFG
   getFactorType(getFactor(dfg, lbl))
 end
@@ -190,7 +190,7 @@ function getSofttype(vnd::VariableNodeData)
   return vnd.softtype
 end
 function getSofttype(v::DFGVariable; solveKey::Symbol=:default)
-  return getSofttype(getData(v, solveKey=solveKey))
+  return getSofttype(solverData(v, solveKey))
 end
 
 """

src/DistributedFactorGraphs.jl

@@ -21,9 +21,13 @@ export AbstractParams, NoSolverParams
 export DFGNode, DFGVariable, DFGFactor
 export InferenceType, PackedInferenceType, FunctorInferenceType, InferenceVariable, ConvolutionObject
 export FunctorSingleton, FunctorPairwise, FunctorPairwiseMinimize
-export label, timestamp, tags, estimates, estimate, data, solverData, getData, solverDataDict, setSolverData, internalId, smallData, bigData
+export label, timestamp, tags, estimates, estimate, data, softtype, solverData, getData, solverDataDict, setSolverData, internalId, smallData, bigData
 export DFGVariableSummary, DFGFactorSummary, AbstractDFGSummary
 
+#graph small data
+export getUserData, setUserData, getRobotData, setRobotData, getSessionData, setSessionData
+export pushUserData!, pushRobotData!, pushSessionData!, popUserData!, popRobotData!, popSessionData!
+
 # Services/AbstractDFG Exports
 export hasFactor, hasVariable, isInitialized, getFactorFunction, isVariable, isFactor
 export updateGraphSolverData!
@@ -36,7 +40,6 @@ export pack, unpack
 # Resolve with above
 export packVariable, unpackVariable, packFactor, unpackFactor
 
-
 #Interfaces
 export getAdjacencyMatrixSparse
 
@@ -51,6 +54,8 @@ include("services/AbstractDFG.jl")
 include("services/DFGVariable.jl")
 include("services/DFGFactor.jl")
 
+include("BigData.jl")
+
 # Include the Graphs.jl API.
 include("GraphsDFG/GraphsDFG.jl")
 
@@ -61,24 +66,32 @@ include("FileDFG/FileDFG.jl")
 include("MetaGraphsDFG/MetaGraphsDFG.jl")
 
 include("SymbolDFG/SymbolDFG.jl")
-# @reexport using .SymbolDFGs
+using .SymbolDFGs
 
 include("LightDFG/LightDFG.jl")
 @reexport using .LightDFGs
 
 function __init__()
-    @require Neo4j="d2adbeaf-5838-5367-8a2f-e46d570981db" begin
-        # Include the Cloudgraphs API
-        include("CloudGraphsDFG/CloudGraphsDFG.jl")
-    end
+    @info "Looking for @require modules"
+    #FIXME still can't figure @require out
+    # @require Neo4j="d2adbeaf-5838-5367-8a2f-e46d570981db" begin
+    #     @info "Including CloudGraphsDFG"
+    #     # Include the Cloudgraphs API
+    #     include("CloudGraphsDFG/CloudGraphsDFG.jl")
+    # end
 
     @require GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231" begin
+        @info "Including Plots"
         include("DFGPlots/DFGPlots.jl")
         @reexport using .DFGPlots
     end
 
 end
 
+#FIXME still can't figure @require out
+# Include the Cloudgraphs API
+include("CloudGraphsDFG/CloudGraphsDFG.jl")
+
 # To be moved as necessary.
 include("Common.jl")
 

src/GraphsDFG/entities/GraphsDFG.jl

@@ -14,6 +14,9 @@ mutable struct GraphsDFG{T <: AbstractParams} <: AbstractDFG
     userId::String
     robotId::String
     sessionId::String
+    userData::Dict{Symbol, String}
+    robotData::Dict{Symbol, String}
+    sessionData::Dict{Symbol, String}
     nodeCounter::Int64
     labelDict::Dict{Symbol, Int64}
     addHistory::Vector{Symbol} #TODO: Discuss more - is this an audit trail?
@@ -28,4 +31,7 @@ GraphsDFG{T}(   g::FGType=Graphs.incdict(GraphsNode,is_directed=false),
                 userId::String = "UserID",
                 robotId::String = "robotID",
                 sessionId::String = "sessionID",
-                params::T=NoSolverParams()) where T <: AbstractParams = GraphsDFG{T}(g, d, userId, robotId, sessionId, n, l, a, params)
+                userData::Dict{Symbol, String} = Dict{Symbol, String}(),
+                robotData::Dict{Symbol, String} = Dict{Symbol, String}(),
+                sessionData::Dict{Symbol, String} = Dict{Symbol, String}(),
+                params::T=NoSolverParams()) where T <: AbstractParams = GraphsDFG{T}(g, d, userId, robotId, sessionId, userData, robotData, sessionData, n, l, a, params)

src/GraphsDFG/services/GraphsDFG.jl

@@ -26,6 +26,23 @@ function setSolverParams(dfg::GraphsDFG, solverParams::T) where T <: AbstractPar
     dfg.solverParams = solverParams
 end
 
+# Get user, robot, and session "small" data.
+getUserData(dfg::GraphsDFG)::Dict{Symbol, String} = return dfg.userData
+function setUserData(dfg::GraphsDFG, data::Dict{Symbol, String})::Bool
+	dfg.userData = data
+	return true
+end
+getRobotData(dfg::GraphsDFG)::Dict{Symbol, String} = return dfg.robotData
+function setRobotData(dfg::GraphsDFG, data::Dict{Symbol, String})::Bool
+	dfg.robotData = data
+	return true
+end
+getSessionData(dfg::GraphsDFG)::Dict{Symbol, String} = return dfg.sessionData
+function setSessionData(dfg::GraphsDFG, data::Dict{Symbol, String})::Bool
+	dfg.sessionData = data
+	return true
+end
+
 """
     $(SIGNATURES)
 Gets an empty and unique CloudGraphsDFG derived from an existing DFG.

src/LightDFG/LightDFG.jl

@@ -11,6 +11,12 @@ import ...DistributedFactorGraphs:  setSolverParams,
                                     getFactor,
                                     setDescription,
                                     getLabelDict,
+                                    getUserData,
+                                    setUserData,
+                                    getRobotData,
+                                    setRobotData,
+                                    getSessionData,
+                                    setSessionData,
                                     addVariable!,
                                     getVariable,
                                     getAddHistory,

src/LightDFG/entities/LightDFG.jl

@@ -13,6 +13,9 @@ mutable struct LightDFG{T <: AbstractParams, V <: AbstractDFGVariable, F <:Abstr
     userId::String
     robotId::String
     sessionId::String
+    userData::Dict{Symbol, String}
+    robotData::Dict{Symbol, String}
+    sessionData::Dict{Symbol, String}
     addHistory::Vector{Symbol} #TODO: Discuss more - is this an audit trail?
     solverParams::T # Solver parameters
 end
@@ -30,9 +33,12 @@ function LightDFG{T,V,F}(g::FactorGraph{Int,V,F}=FactorGraph{Int,V,F}();
                            userId::String="User ID",
                            robotId::String="Robot ID",
                            sessionId::String="Session ID",
+                           userData::Dict{Symbol, String} = Dict{Symbol, String}(),
+                           robotData::Dict{Symbol, String} = Dict{Symbol, String}(),
+                           sessionData::Dict{Symbol, String} = Dict{Symbol, String}(),
                            params::T=NoSolverParams()) where {T <: AbstractParams, V <:AbstractDFGVariable, F<:AbstractDFGFactor}
 
-    LightDFG{T,V,F}(g, description, userId, robotId, sessionId, Symbol[], params)
+    LightDFG{T,V,F}(g, description, userId, robotId, sessionId, userData, robotData, sessionData, Symbol[], params)
 end
 
 # LightDFG{T}(; kwargs...) where T <: AbstractParams = LightDFG{T,DFGVariable,DFGFactor}(;kwargs...)

src/LightDFG/services/LightDFG.jl

@@ -13,6 +13,23 @@ function setSolverParams(dfg::LightDFG, solverParams::P) where P <: AbstractPara
   dfg.solverParams = solverParams
 end
 
+# Get user, robot, and session "small" data.
+getUserData(dfg::LightDFG)::Dict{Symbol, String} = return dfg.userData
+function setUserData(dfg::LightDFG, data::Dict{Symbol, String})::Bool
+	dfg.userData = data
+	return true
+end
+getRobotData(dfg::LightDFG)::Dict{Symbol, String} = return dfg.robotData
+function setRobotData(dfg::LightDFG, data::Dict{Symbol, String})::Bool
+	dfg.robotData = data
+	return true
+end
+getSessionData(dfg::LightDFG)::Dict{Symbol, String} = return dfg.sessionData
+function setSessionData(dfg::LightDFG, data::Dict{Symbol, String})::Bool
+	dfg.sessionData = data
+	return true
+end
+
 """
     $(SIGNATURES)
 True if the variable or factor exists in the graph.