Merge branch 'master' into feature/177_sentinels

Author: GearsAD

.travis.yml

@@ -22,23 +22,23 @@ jobs:
   include:
     - julia: 1.2
       env: IIF_TEST=true
+      if: NOT branch =~ ^release.*$
     - stage: "Documentation"
       julia: 1.0
       os: linux
       script:
-        - julia -e 'import Pkg; Pkg.add("Documenter"); Pkg.add("Neo4j"); Pkg.add("GraphPlot")'
-        - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd()));
-                                               Pkg.instantiate()'
+        - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
         - julia --project=docs/ docs/make.jl
       after_success: skip
+  fast_finish: true
 
 notifications:
     email: false
 
 matrix:
   allow_failures:
   - julia: nightly
-  - env: IIF_TEST=true
+# - env: IIF_TEST=true
 
 # Set the password for Neo4j to neo4j:test
 before_script:

docs/Project.toml

@@ -1,6 +1,5 @@
 [deps]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
-DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 
 [compat]
-Documenter = "~0.21"
+Documenter = "0.23"

docs/make.jl

@@ -1,5 +1,4 @@
 using Documenter
-using DataFrames
 using DistributedFactorGraphs
 
 makedocs(

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -186,7 +186,8 @@ function getVariable(dfg::CloudGraphsDFG, variableId::Int64)::DFGVariable
     # props["label"] = Symbol(variable.label)
     timestamp = DateTime(props["timestamp"])
     tags =  JSON2.read(props["tags"], Vector{Symbol})
-    estimateDict = JSON2.read(props["estimateDict"], Dict{Symbol, Dict{Symbol, VariableEstimate}})
+    #TODO this will work for some time, but unpacking in an <: AbstractPointParametricEst would be lekker.
+    estimateDict = JSON2.read(props["estimateDict"], Dict{Symbol, MeanMaxPPE})
     smallData = nothing
     smallData = JSON2.read(props["smallData"], Dict{String, String})
 

src/DistributedFactorGraphs.jl

@@ -18,12 +18,15 @@ include("entities/AbstractDFGSummary.jl")
 
 export AbstractDFG
 export AbstractParams, NoSolverParams
-export DFGNode, DFGVariable, DFGFactor
+export DFGNode, DFGVariable, DFGFactor, AbstractDFGVariable, AbstractDFGFactor
 export InferenceType, PackedInferenceType, FunctorInferenceType, InferenceVariable, ConvolutionObject
 export FunctorSingleton, FunctorPairwise, FunctorPairwiseMinimize
 export label, timestamp, tags, estimates, estimate, data, softtype, solverData, getData, solverDataDict, setSolverData, internalId, smallData, bigData
 export DFGVariableSummary, DFGFactorSummary, AbstractDFGSummary
 
+#Skeleton types
+export SkeletonDFGVariable, SkeletonDFGFactor
+
 #graph small data
 export getUserData, setUserData, getRobotData, setRobotData, getSessionData, setSessionData
 export pushUserData!, pushRobotData!, pushSessionData!, popUserData!, popRobotData!, popSessionData!
@@ -33,13 +36,16 @@ export hasFactor, hasVariable, isInitialized, getFactorFunction, isVariable, isF
 export mergeUpdateVariableSolverData!, mergeUpdateGraphSolverData!
 
 # Solver (IIF) Exports
-export VariableNodeData, PackedVariableNodeData, VariableEstimate
+export VariableNodeData, PackedVariableNodeData
 export GenericFunctionNodeData#, FunctionNodeData
 export getSerializationModule, setSerializationModule!
 export pack, unpack
 # Resolve with above
 export packVariable, unpackVariable, packFactor, unpackFactor
 
+#PPE exports
+export MeanMaxPPE
+
 #Interfaces
 export getAdjacencyMatrixSparse
 

src/FileDFG/services/FileDFG.jl

@@ -30,7 +30,7 @@ function saveDFG(dfg::G, folder::String) where G <: AbstractDFG
     end
 end
 
-function loadDFG(folder::String, iifModule, dfgLoadInto::G=GraphsDFG{NoSolverParams}()) where G <: AbstractDFG
+function loadDFG(folder::String, iifModule, dfgLoadInto::G) where G <: AbstractDFG
     variables = DFGVariable[]
     factors = DFGFactor[]
     varFolder = "$folder/variables"
@@ -70,9 +70,8 @@ function loadDFG(folder::String, iifModule, dfgLoadInto::G=GraphsDFG{NoSolverPar
 
     # PATCH - To update the fncargvID for factors, it's being cleared somewhere in rebuildFactorMetadata.
     # TEMPORARY
-    # TODO: Remove
+    # TODO: Remove in future
     map(f->solverData(f).fncargvID = f._variableOrderSymbols, getFactors(dfgLoadInto))
 
-
     return dfgLoadInto
 end

src/LightDFG/services/LightDFG.jl

@@ -88,8 +88,9 @@ function addFactor!(dfg::LightDFG{<:AbstractParams, V, F}, variables::Vector{V},
 
 	variableLabels = map(v->v.label, variables)
 
-    factor._variableOrderSymbols = copy(variableLabels)
-
+	resize!(factor._variableOrderSymbols, length(variableLabels))
+	factor._variableOrderSymbols .= variableLabels
+    # factor._variableOrderSymbols = copy(variableLabels)
 
     return FactorGraphs.addFactor!(dfg.g, variableLabels, factor)
 end
@@ -104,12 +105,23 @@ function addFactor!(dfg::LightDFG{<:AbstractParams, <:AbstractDFGVariable, F}, v
         error("Factor '$(factor.label)' already exists in the factor graph")
     end
 
-    factor._variableOrderSymbols = variableLabels
+	resize!(factor._variableOrderSymbols, length(variableLabels))
+	factor._variableOrderSymbols .= variableLabels
+	# factor._variableOrderSymbols = copy(variableLabels)
 
     return FactorGraphs.addFactor!(dfg.g, variableLabels, factor)
 end
 
 
+function addFactor!(dfg::LightDFG{<:AbstractParams, <:AbstractDFGVariable, F}, factor::F)::Bool where F <: AbstractDFGFactor
+	#TODO should this be an error
+	if haskey(dfg.g.factors, factor.label)
+        error("Factor '$(factor.label)' already exists in the factor graph")
+    end
+
+    return FactorGraphs.addFactor!(dfg.g, variableLabels, factor)
+end
+
 """
     $(SIGNATURES)
 Get a DFGVariable from a DFG using its label.

src/entities/DFGFactor.jl

@@ -103,3 +103,19 @@ label(f::DFGFactorSummary) = f.label
 data(f::DFGFactorSummary) = f.data
 tags(f::DFGFactorSummary) = f.tags
 internalId(f::DFGFactorSummary) = f._internalId
+
+
+# SKELETON DFG
+"""
+	$(TYPEDEF)
+Skeleton variable with essentials.
+"""
+struct SkeletonDFGVariable <: AbstractDFGVariable
+	label::Symbol
+	tags::Vector{Symbol}
+end
+
+SkeletonDFGVariable(label::Symbol) = SkeletonDFGVariable(label, Symbol[])
+
+label(v::SkeletonDFGVariable) = v.label
+tags(v::SkeletonDFGVariable) = v.tags

src/entities/DFGVariable.jl

@@ -63,26 +63,28 @@ mutable struct PackedVariableNodeData
                          x15::Bool ) = new(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15)
 end
 
-
-abstract type AbstractVariableEstimate end
+# AbstractPointParametricEst interface
+abstract type AbstractPointParametricEst end
 """
     $TYPEDEF
 
-Data container to store Parameteric Point Estimate (PPE) from a variety of types.
-
-Notes
-- `ppeType` is something like `:max/:mean/:modefit` etc.
-- `solveKey` is from super-solve concept, starting with `:default`,
-- `estimate` is the actual numerical estimate value,
-- Additional information such as how the data is represented (ie softtype) is stored alongside this data container in the `DFGVariableSummary` container.
+Data container to store Parameteric Point Estimate (PPE) for mean and max.
 """
-struct VariableEstimate <: AbstractVariableEstimate
-  solverKey::Symbol
-  ppeType::Symbol
-  estimate::Vector{Float64}
-  lastUpdatedTimestamp::DateTime
+struct MeanMaxPPE <: AbstractPointParametricEst
+    solverKey::Symbol #repeated because of Sam's request
+    max::Vector{Float64}
+    mean::Vector{Float64}
+    lastUpdatedTimestamp::DateTime
 end
-VariableEstimate(solverKey::Symbol, type::Symbol, estimate::Vector{Float64}) = VariableEstimate(solverKey, type, estimate, now())
+MeanMaxPPE(solverKey::Symbol,max::Vector{Float64},mean::Vector{Float64}) = MeanMaxPPE(solverKey, max, mean, now())
+
+getMaxPPE(est::AbstractPointParametricEst) = est.max
+getMeanPPE(est::AbstractPointParametricEst) = est.mean
+getLastUpdatedTimestamp(est::AbstractPointParametricEst) = est.lastUpdatedTimestamp
+
+
+VariableEstimate(params...) = errror("VariableEstimate is depreciated, please use MeanMaxPPE")
+
 
 """
     $(TYPEDEF)
@@ -93,7 +95,7 @@ mutable struct DFGVariable <: AbstractDFGVariable
     label::Symbol
     timestamp::DateTime
     tags::Vector{Symbol}
-    estimateDict::Dict{Symbol, Dict{Symbol, <: AbstractVariableEstimate}}
+    estimateDict::Dict{Symbol, <: AbstractPointParametricEst}
     solverDataDict::Dict{Symbol, VariableNodeData}
     smallData::Dict{String, String}
     bigData::Dict{Symbol, AbstractBigDataEntry}
@@ -107,10 +109,10 @@ end
 DFGVariable constructors.
 """
 DFGVariable(label::Symbol, _internalId::Int64) =
-        DFGVariable(label, now(), Symbol[], Dict{Symbol, Dict{Symbol, VariableEstimate}}(), Dict{Symbol, VariableNodeData}(:default => VariableNodeData()), Dict{String, String}(), Dict{Symbol,AbstractBigDataEntry}(), 0, 0, _internalId)
+        DFGVariable(label, now(), Symbol[], Dict{Symbol, MeanMaxPPE}(), Dict{Symbol, VariableNodeData}(:default => VariableNodeData()), Dict{String, String}(), Dict{Symbol,AbstractBigDataEntry}(), 0, 0, _internalId)
 
 DFGVariable(label::Symbol) =
-        DFGVariable(label, now(), Symbol[], Dict{Symbol, VariableEstimate}(), Dict{Symbol, VariableNodeData}(:default => VariableNodeData()), Dict{String, String}(), Dict{Symbol,AbstractBigDataEntry}(), 0, 0, 0)
+        DFGVariable(label, now(), Symbol[], Dict{Symbol, MeanMaxPPE}(), Dict{Symbol, VariableNodeData}(:default => VariableNodeData()), Dict{String, String}(), Dict{Symbol,AbstractBigDataEntry}(), 0, 0, 0)
 
 # Accessors
 label(v::DFGVariable) = v.label
@@ -138,12 +140,7 @@ solverData(v::DFGVariable, key::Symbol=:default) = haskey(v.solverDataDict, key)
 Retrieve data structure stored in a variable.
 """
 function getData(v::DFGVariable; solveKey::Symbol=:default)::VariableNodeData
-  #FIXME but back in later, it just slows everything down
-  if !(@isdefined getDataWarnOnce)
-    @warn "getData is deprecated, please use solverData(), future warnings in getData is suppressed"
-    global getDataWarnOnce = true
-  end
-  # @warn "getData is deprecated, please use solverData()"
+  @warn "getData is deprecated, please use solverData()"
   return v.solverDataDict[solveKey]
 end
 """
@@ -166,7 +163,7 @@ mutable struct DFGVariableSummary <: AbstractDFGVariable
     label::Symbol
     timestamp::DateTime
     tags::Vector{Symbol}
-    estimateDict::Dict{Symbol, Dict{Symbol, <:AbstractVariableEstimate}}
+    estimateDict::Dict{Symbol, <:AbstractPointParametricEst}
     softtypename::Symbol
     _internalId::Int64
 end
@@ -177,3 +174,22 @@ estimates(v::DFGVariableSummary) = v.estimateDict
 estimate(v::DFGVariableSummary, key::Symbol=:default) = haskey(v.estimateDict, key) ? v.estimateDict[key] : nothing
 softtype(v::DFGVariableSummary)::Symbol = v.softtypename
 internalId(v::DFGVariableSummary) = v._internalId
+
+
+
+# SKELETON DFG
+"""
+	$(TYPEDEF)
+Skeleton factor with essentials.
+"""
+struct SkeletonDFGFactor <: AbstractDFGFactor
+    label::Symbol
+	tags::Vector{Symbol}
+	_variableOrderSymbols::Vector{Symbol}
+end
+
+#NOTE I feel like a want to force a variableOrderSymbols
+SkeletonDFGFactor(label::Symbol, variableOrderSymbols::Vector{Symbol} = Symbol[]) = SkeletonDFGFactor(label, Symbol[], variableOrderSymbols)
+
+label(f::SkeletonDFGFactor) = f.label
+tags(f::SkeletonDFGFactor) = f.tags

src/services/DFGVariable.jl

@@ -17,7 +17,8 @@ function unpackVariable(dfg::G, packedProps::Dict{String, Any})::DFGVariable whe
     label = Symbol(packedProps["label"])
     timestamp = DateTime(packedProps["timestamp"])
     tags =  JSON2.read(packedProps["tags"], Vector{Symbol})
-    estimateDict = JSON2.read(packedProps["estimateDict"], Dict{Symbol, Dict{Symbol, VariableEstimate}})
+    #TODO this will work for some time, but unpacking in an <: AbstractPointParametricEst would be lekker.
+    estimateDict = JSON2.read(packedProps["estimateDict"], Dict{Symbol, MeanMaxPPE})
     smallData = nothing
     smallData = JSON2.read(packedProps["smallData"], Dict{String, String})
 
@@ -112,15 +113,15 @@ function ==(a::VariableNodeData,b::VariableNodeData, nt::Symbol=:var)
 end
 
 """
-    $(SIGNATURES)
-Equality check for VariableEstimate.
+    ==(x::T, y::T) where T <: AbstractPointParametricEst
+Equality check for AbstractPointParametricEst.
 """
-function ==(a::VariableEstimate, b::VariableEstimate)::Bool
-  a.solverKey != b.solverKey && @debug("solverKey are not equal")==nothing && return false
-  a.ppeType != b.ppeType && @debug("ppeType is not equal")==nothing && return false
-  a.estimate != b.estimate && @debug("estimate are not equal")==nothing && return false
-  a.lastUpdatedTimestamp != b.lastUpdatedTimestamp && @debug("lastUpdatedTimestamp is not equal")==nothing && return false
-  return true
+@generated function ==(x::T, y::T) where T <: AbstractPointParametricEst
+    mapreduce(n -> :(x.$n == y.$n), (a,b)->:($a && $b), fieldnames(x))
+end
+
+@generated function Base.:(==)(x::T, y::T) where T <: Union{DFGFactorSummary, DFGVariableSummary, SkeletonDFGVariable, SkeletonDFGFactor}
+    mapreduce(n -> :(x.$n == y.$n), (a,b)->:($a && $b), fieldnames(x))
 end
 
 """