Merge branch 'master' into enhancement/2Q20/nodelabelvalidation

Author: Affie

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.7.5"
+version = "0.8.0"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

src/BigData/services/AbstractBigDataEntries.jl

@@ -41,6 +41,14 @@ addDataEntry!(x...) = addBigDataEntry!(x...)
 $(SIGNATURES)
 Add Big Data Entry to distributed factor graph.
 Should be extended if DFG variable is not returned by reference.
+
+Example
+
+See docs for `getDataEntryElement`.
+
+Related
+
+addData!, getDataEntryElement, fetchData
 """
 function addDataEntry!(dfg::AbstractDFG,
                        lbl::Symbol,
@@ -51,12 +59,13 @@ function addDataEntry!(dfg::AbstractDFG,
   #
   node = isVariable(dfg, lbl) ? getVariable(dfg, lbl) : getFactor(dfg, lbl)
   # Make a big data entry in the graph - use JSON2 to just write this
-  element = GeneralBigDataEntry(dfg, node, descr, mimeType=mimeType)
+  entry = GeneralBigDataEntry(dfg, node, descr, mimeType=mimeType)
   # Set it in the store
-  addBigData!(datastore, element, data)
+  addBigData!(datastore, entry, data)
   # Add the entry to the graph
-  addBigDataEntry!(node, element)
+  addBigDataEntry!(node, entry)
 end
+const addData! = addDataEntry!
 
 """
     $SIGNATURES
@@ -79,9 +88,68 @@ function getBigDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol)
     return getBigDataEntry(getVariable(dfg, label), key)
 end
 
+"""
+$SIGNATURES
+Get both the entry and raw data element from datastore returning as a tuple.
+
+Notes:
+- This is the counterpart to `addDataEntry!`.
+- Data is identified by the node in the DFG object `dfglabel::Symbol` as well as `datalabel::Symbol`.
+- The data should have been stored along with a `entry.mimeType::String` which describes the format of the data.
+- ImageMagick.jl is useful for storing images in png or jpg compressed format.
+
+Example
+
+```julia
+# some dfg object
+fg = initfg()
+addVariable!(fg, :x0, IIF.ContinuousScalar) # using IncrementalInference
+# FileDataStore (as example)
+datastore = FileDataStore(joinLogPath(fg,"datastore"))
+
+# now some data comes in
+mydata = Dict(:soundBite => randn(Float32, 10000), :meta => "something about lazy foxes and fences.")
+
+# store the data
+addDataEntry!( fg, :x0, datastore, :SOUND_DATA, "application/json", Vector{UInt8}(JSON2.write( mydata ))  )
+
+# get/fetch the data
+entry, rawData = fetchData(fg, :x0, datastore, :SOUND_DATA)
+
+# unpack data to original format (this must be done by the user)
+@show entry.mimeType # "applicatio/json"
+userFormat = JSON2.read(IOBuffer(rawData))
+```
+
+Related
+
+addDataEntry!, addData!, fetchData, fetchDataEntryElement
+"""
+function getDataEntryElement(dfg::AbstractDFG,
+                             dfglabel::Symbol,
+                             datastore::Union{FileDataStore, InMemoryDataStore},
+                             datalabel::Symbol)
+  #
+  vari = getVariable(dfg, dfglabel)
+  if !hasDataEntry(vari, datalabel)
+    @error "missing data entry $datalabel in $dfglabel"
+    return nothing, nothing
+  end
+  entry = getBigDataEntry(vari, datalabel)
+  element = getBigData(datastore, entry)
+  return entry, element
+end
+const fetchDataEntryElement = getDataEntryElement
+const fetchData = getDataEntryElement
+
+
+
 """
     $(SIGNATURES)
 Update big data entry
+
+DevNote
+- DF, unclear if `update` verb is applicable in this case, see #404
 """
 function updateBigDataEntry!(var::AbstractDFGVariable,  bde::AbstractBigDataEntry)
     !haskey(var.bigData,bde.key) && (@warn "$(bde.key) does not exist in variable, adding")

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -304,22 +304,27 @@ function updateFactor!(dfg::CloudGraphsDFG, factor::DFGFactor)::DFGFactor
     return factor
 end
 
-function deleteVariable!(dfg::CloudGraphsDFG, label::Symbol)::DFGVariable
+function deleteVariable!(dfg::CloudGraphsDFG, label::Symbol)#::Tuple{AbstractDFGVariable, Vector{<:AbstractDFGFactor}}
     variable = getVariable(dfg, label)
     if variable == nothing
         error("Unable to retrieve the ID for variable '$label'. Please check your connection to the database and that the variable exists.")
     end
 
+    deleteNeighbors = true # reserved, orphaned factors are not supported at this time
+    if deleteNeighbors
+        neigfacs = map(l->deleteFactor!(dfg, l), getNeighbors(dfg, label))
+    end
+
     # Perform detach+deletion
     _getNeoNodesFromCyphonQuery(dfg.neo4jInstance, "(node:$label:$(join(_getLabelsForType(dfg, DFGVariable),':'))) detach delete node ")
 
     # Clearing history
     dfg.addHistory = symdiff(dfg.addHistory, [label])
-    return variable
+    return variable, neigfacs
 end
 
 #Alias
-deleteVariable!(dfg::CloudGraphsDFG, variable::DFGVariable)::DFGVariable = deleteVariable!(dfg, variable.label)
+deleteVariable!(dfg::CloudGraphsDFG, variable::DFGVariable) = deleteVariable!(dfg, variable.label)
 
 function deleteFactor!(dfg::CloudGraphsDFG, label::Symbol)::DFGFactor
     factor = getFactor(dfg, label)
@@ -375,7 +380,7 @@ function listFactors(dfg::CloudGraphsDFG, regexFilter::Union{Nothing, Regex}=not
     end
 end
 
-function isFullyConnected(dfg::CloudGraphsDFG)::Bool
+function isConnected(dfg::CloudGraphsDFG)::Bool
     # If the total number of nodes == total number of distinct connected nodes, then it is fully connected
     # Total nodes
     varIds = listVariables(dfg)
@@ -397,7 +402,7 @@ function isFullyConnected(dfg::CloudGraphsDFG)::Bool
 end
 
 function getNeighbors(dfg::CloudGraphsDFG, node::T; solvable::Int=0)::Vector{Symbol}  where T <: DFGNode
-    query = "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(node.label))--(node) where (node:VARIABLE or node:FACTOR) and node.solvable >= $solvable"
+    query = "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(node.label))-[r:FACTORGRAPH]-(node) where (node:VARIABLE or node:FACTOR) and node.solvable >= $solvable"
     @debug "[Query] $query"
     neighbors = _getLabelsFromCyphonQuery(dfg.neo4jInstance, query)
     # If factor, need to do variable ordering TODO, Do we? does it matter if we always use _variableOrderSymbols in calculations?
@@ -408,7 +413,7 @@ function getNeighbors(dfg::CloudGraphsDFG, node::T; solvable::Int=0)::Vector{Sym
 end
 
 function getNeighbors(dfg::CloudGraphsDFG, label::Symbol; solvable::Int=0)::Vector{Symbol}
-    query = "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(label))--(node) where (node:VARIABLE or node:FACTOR) and node.solvable >= $solvable"
+    query = "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(label))-[r:FACTORGRAPH]-(node) where (node:VARIABLE or node:FACTOR) and node.solvable >= $solvable"
     @debug "[Query] $query"
     neighbors = _getLabelsFromCyphonQuery(dfg.neo4jInstance, query)
     # If factor, need to do variable ordering TODO, Do we? does it matter if we always use _variableOrderSymbols in calculations?

src/Deprecated.jl

@@ -106,6 +106,10 @@ function Base.convert(::Type{SkeletonDFGFactor}, f::FactorDataLevel1)
     return SkeletonDFGFactor(f)
 end
 
+
+@deprecate hasOrphans(dfg) !isConnected(dfg)
+@deprecate isFullyConnected(dfg) isConnected(dfg)
+
 ##==============================================================================
 ## WIP on consolidated subgraph functions, aim to remove in 0.8
 ##==============================================================================

src/DistributedFactorGraphs.jl

@@ -71,7 +71,7 @@ export exists,
        getVariables, getFactors,
        isVariable, isFactor
 
-export isFullyConnected, hasOrphans
+export isConnected
 
 export getBiadjacencyMatrix
 
@@ -165,15 +165,19 @@ export getNeighborhood, getNeighbors, _getDuplicatedEmptyDFG
 export copyGraph!, deepcopyGraph, deepcopyGraph!, buildSubgraph, mergeGraph!
 # Big Data
 ##------------------------------------------------------------------------------
-export addBigDataEntry!, addDataEntry!, getBigDataEntry, updateBigDataEntry!, deleteBigDataEntry!, getBigDataEntries, getBigDataKeys, hasDataEntry, hasBigDataEntry
+export addBigDataEntry!, getBigDataEntry, updateBigDataEntry!, deleteBigDataEntry!, getBigDataEntries, getBigDataKeys, hasDataEntry, hasBigDataEntry
+# convenience wrappers
+export addDataEntry!, getDataEntryElement
+# aliases
+export addData!, fetchData, fetchDataEntryElement
 
 
 ##------------------------------------------------------------------------------
 # Factors
 ##------------------------------------------------------------------------------
 # Factor Data
 export GenericFunctionNodeData
-export InferenceType, PackedInferenceType, FunctorInferenceType, ConvolutionObject
+export InferenceType, PackedInferenceType, FunctorInferenceType, FactorOperationalMemory
 export FunctorSingleton, FunctorPairwise, FunctorPairwiseMinimize
 
 # accessors
@@ -228,7 +232,8 @@ export
 
 
 ## Deprecated exports should be listed in Deprecated.jl if possible, otherwise here
-
+#TODO remove export in DFG v0.8.0
+export ConvolutionObject
 
 ## needsahome.jl
 import Base: print

src/GraphsDFG/GraphsDFG.jl

@@ -36,8 +36,7 @@ import ...DistributedFactorGraphs:  setSolverParams!,
                                     getFactors,
                                     listFactors,
                                     lsf,
-                                    isFullyConnected,
-                                    hasOrphans,
+                                    isConnected,
                                     getNeighbors,
                                     buildSubgraph,
                                     copyGraph!,

src/GraphsDFG/services/GraphsDFG.jl

@@ -165,14 +165,21 @@ function updateFactor!(dfg::GraphsDFG, factor::DFGFactor)::DFGFactor
     return factor
 end
 
-function deleteVariable!(dfg::GraphsDFG, label::Symbol)::DFGVariable
+function deleteVariable!(dfg::GraphsDFG, label::Symbol)#::Tuple{AbstractDFGVariable, Vector{<:AbstractDFGFactor}}
     if !haskey(dfg.labelDict, label)
         error("Variable label '$(label)' does not exist in the factor graph")
     end
+
+    deleteNeighbors = true # reserved, orphaned factors are not supported at this time
+    if deleteNeighbors
+        neigfacs = map(l->deleteFactor!(dfg, l), getNeighbors(dfg, label))
+    end
+
+
     variable = dfg.g.vertices[dfg.labelDict[label]].dfgNode
     delete_vertex!(dfg.g.vertices[dfg.labelDict[label]], dfg.g)
     delete!(dfg.labelDict, label)
-    return variable
+    return variable, neigfacs
 end
 
 function deleteFactor!(dfg::GraphsDFG, label::Symbol)::DFGFactor
@@ -220,7 +227,7 @@ function getFactors(dfg::GraphsDFG, regexFilter::Union{Nothing, Regex}=nothing;
     return factors
 end
 
-function isFullyConnected(dfg::GraphsDFG)::Bool
+function isConnected(dfg::GraphsDFG)::Bool
     return length(Graphs.connected_components(dfg.g)) == 1
 end
 

src/LightDFG/LightDFG.jl

@@ -38,8 +38,7 @@ import ...DistributedFactorGraphs:  setSolverParams!,
                                     getFactors,
                                     listFactors,
                                     lsf,
-                                    isFullyConnected,
-                                    hasOrphans,
+                                    isConnected,
                                     getNeighbors,
                                     buildSubgraph,
                                     copyGraph!,

src/LightDFG/services/LightDFG.jl

@@ -130,14 +130,19 @@ function updateFactor!(dfg::LightDFG, factor::F)::F where F <: AbstractDFGFactor
     return factor
 end
 
-function deleteVariable!(dfg::LightDFG, label::Symbol)::AbstractDFGVariable
+function deleteVariable!(dfg::LightDFG, label::Symbol)#::Tuple{AbstractDFGVariable, Vector{<:AbstractDFGFactor}}
     if !haskey(dfg.g.variables, label)
         error("Variable label '$(label)' does not exist in the factor graph")
     end
+
+    deleteNeighbors = true # reserved, orphaned factors are not supported at this time
+    if deleteNeighbors
+        neigfacs = map(l->deleteFactor!(dfg, l), getNeighbors(dfg, label))
+    end
     variable = dfg.g.variables[label]
     rem_vertex!(dfg.g, dfg.g.labels[label])
 
-    return variable
+    return variable, neigfacs
 end
 
 function deleteFactor!(dfg::LightDFG, label::Symbol)::AbstractDFGFactor
@@ -210,8 +215,9 @@ function listFactors(dfg::LightDFG, regexFilter::Union{Nothing, Regex}=nothing;
     return factors
 end
 
-function isFullyConnected(dfg::LightDFG)::Bool
-    return length(LightGraphs.connected_components(dfg.g)) == 1
+function isConnected(dfg::LightDFG)::Bool
+    return LightGraphs.is_connected(dfg.g)
+    # return length(LightGraphs.connected_components(dfg.g)) == 1
 end
 
 function _isSolvable(dfg::LightDFG, label::Symbol, ready::Int)::Bool

src/entities/DFGFactor.jl

@@ -7,15 +7,12 @@ abstract type PackedInferenceType end
 
 abstract type FunctorInferenceType <: Function end
 
-# DF suggestion1 (this is a first guess)  `ConvolutionObject` --> `FactorInsituObject`
-# DF second guess,  ConvolutionObject <: FactorInsituObject
-# JT Maybe second guess as intermediate step where ConvolutionObject is moved to IIF.
-# then it can be removed to CommonConvWrapper <: FactorInsituObject
-
-# DF, Convolution is IIF idea, but DFG should know about "FactorOperationalMemory"
-# DF, IIF.CommonConvWrapper <: FactorOperationalMemory # FIXME
-# MAYBE rename "FactorOperationalMemory"
-abstract type ConvolutionObject <: Function end
+# NOTE DF, Convolution is IIF idea, but DFG should know about "FactorOperationalMemory"
+# DF, IIF.CommonConvWrapper <: FactorOperationalMemory #
+abstract type FactorOperationalMemory <: Function end
+# TODO to be removed from DFG,
+# we can add to IIF or have IIF.CommonConvWrapper <: FactorOperationalMemory directly
+abstract type ConvolutionObject <: FactorOperationalMemory end
 
 abstract type FunctorSingleton <: FunctorInferenceType end
 abstract type FunctorPairwise <: FunctorInferenceType end
@@ -37,7 +34,7 @@ Designing (WIP)
 # in IIF.FunctorPairwiseMinimize <: InferenceType # DFG whatever, something, we'll figure it out
 # in Main/User, SomeFactor <: FunctorPairwiseMinimize
 """
-mutable struct GenericFunctionNodeData{T, S} #{T<:Union{PackedInferenceType, FunctorInferenceType, ConvolutionObject}, S<:Union{Symbol, AbstractString}}
+mutable struct GenericFunctionNodeData{T, S} #{T<:Union{PackedInferenceType, FunctorInferenceType, FactorOperationalMemory}, S<:Union{Symbol, AbstractString}}
     fncargvID::Vector{Symbol}
     eliminated::Bool
     potentialused::Bool
@@ -62,7 +59,7 @@ end
 const PackedFunctionNodeData{T} = GenericFunctionNodeData{T, <: AbstractString}
 PackedFunctionNodeData(x1, x2, x3, x4, x5::S, x6::T, multihypo::Vector{Float64}=[], certainhypo::Vector{Int}=Int[], x9::Int=0) where {T <: PackedInferenceType, S <: AbstractString} = GenericFunctionNodeData(x1, x2, x3, x4, x5, x6, multihypo, certainhypo, x9)
 const FunctionNodeData{T} = GenericFunctionNodeData{T, Symbol}
-FunctionNodeData(x1, x2, x3, x4, x5::Symbol, x6::T, multihypo::Vector{Float64}=[], certainhypo::Vector{Int}=Int[], x9::Int=0) where {T <: Union{FunctorInferenceType, ConvolutionObject}}= GenericFunctionNodeData{T, Symbol}(x1, x2, x3, x4, x5, x6, multihypo, certainhypo, x9)
+FunctionNodeData(x1, x2, x3, x4, x5::Symbol, x6::T, multihypo::Vector{Float64}=[], certainhypo::Vector{Int}=Int[], x9::Int=0) where {T <: Union{FunctorInferenceType, FactorOperationalMemory}}= GenericFunctionNodeData{T, Symbol}(x1, x2, x3, x4, x5, x6, multihypo, certainhypo, x9)
 
 ##==============================================================================
 ## Factors
@@ -104,7 +101,7 @@ mutable struct DFGFactor{T, S} <: AbstractDFGFactor
     """Mutable parameters for the variable. We suggest using accessors to get to this data.
     Accessors: [`getSolvable`](@ref), [`setSolvable!`](@ref)"""
     _dfgNodeParams::DFGNodeParams
-    """Internal cache of the ordering of the neighbor variables. Rather use getNeighbors to get the list as this is an internal value.
+    """Internal cache of the ordering of the neighbor variables. Rather use getVariableOrder to get the list as this is an internal value.
     Accessors: [`getVariableOrder`](@ref)"""
     _variableOrderSymbols::Vector{Symbol}
 end