Standardize more types, filters, and more cleanup (#1163)

Author: Affie

.gitignore

@@ -7,3 +7,4 @@ docs/build
 Manifest.toml
 dev
 test/sandbox.jl
+lcov.info

NEWS.md

@@ -2,13 +2,19 @@ Listing news on any major breaking changes in DFG.  For regular changes, see int
 # v0.28
 - Reading or deserialzing of factor graphs created prior to v0.25 are no longer suppoted with the complete removal of User/Robot/Session
 - Deprecated AbstractRelativeMinimize and AbstractManifoldsMinimize
-
-#TODO pending: 
-- AbstractPrior -> AbstractPriorObservation/PriorObservation
-- AbstractRelative -> RelativeObservation/RelativeObservation
-- InferenceType -> AbstractPackedFactorObservation
-- InferenceVariable -> [Abstract]VariableStateType/AbstractVarstateType
-- PackedSamplableBelief -> [Abstract]PackedBelief
+- Rename `VariableState` -> `State`
+
+Abstract Types Standardized, see #1153: 
+- AbstractParams -> [Abstract]DFGParams
+- DFGNode -> [Abstract]GraphNode
+- AbstractDFGVariable -> [Abstract]GraphVariable
+- AbstractDFGFactor -> [Abstract]GraphFactor
+- AbstractPackedFactorObservation -> [Abstract]PackedObservation
+- AbstractFactorObservation -> [Abstract]Observation
+- AbstractPrior -> [Abstract]PriorObservation
+- AbstractRelative -> [Abstract]RelativeObservation
+- FactorSolverCache -> [Abstract]FactorCache
+- VariableStateType -> [Abstract]StateType
 
 # v0.27
 - `delete` returns number of nodes deleted and no longer the object that was deleted.

docs/src/DrawingGraphs.md

@@ -64,8 +64,8 @@ More information at [GraphMakie.jl](https://github.com/MakieOrg/GraphMakie.jl)
 Dot files are a standard format for visualizing graphs and applications such as
 xdot are available to view the files. Dot plotting does not require `GraphMakie`
 and can be drawn by either:
-- Calling [`toDot`](@ref) on any graph to produce a string of the graph
-- Calling [`toDotFile`](@ref) on any graph to save it directly to a dotfile
+- Calling [`DFG.toDot`](@ref) on any graph to produce a string of the graph
+- Calling [`DFG.toDotFile`](@ref) on any graph to save it directly to a dotfile
 
 ```julia
 using DistributedFactorGraphs

docs/src/GraphData.md

@@ -15,7 +15,7 @@ The following is a guideline to using these parameters.
 
 **NOTE**: Adds in general throw an error if the element already exists. Update will update the element if it exists, otherwise it will add it.
 
-**NOTE**: In general these functions will return an error if the respective element is not found. This is to avoid returning, say, nothing, which will be horribly confusing if you tried `getVariableState(dfg, :a, :b)` and it returned nothing - which was missing, :a or :b, or was there a communication issue? We recommend coding defensively and trapping errors in critical portions of your user code.
+**NOTE**: In general these functions will return an error if the respective element is not found. This is to avoid returning, say, nothing, which will be horribly confusing if you tried `getState(dfg, :a, :b)` and it returned nothing - which was missing, :a or :b, or was there a communication issue? We recommend coding defensively and trapping errors in critical portions of your user code.
 
 **NOTE**: All data is passed by reference, so if you update the returned structure it will update in the graph. The database driver is an exception, and once the variable or factor is updated you need to call update* to persist the changes to the graph.
 
@@ -53,7 +53,7 @@ Each variable or factor can have a timestamp associated with it.
 
 Tags are a set of symbols that contain identifiers for the variable or factor.
 
-- [`listTags`](@ref)
+- [`getTags`](@ref)
 - [`mergeTags!`](@ref)
 - [`removeTags!`](@ref)
 - [`emptyTags!`](@ref)
@@ -118,41 +118,41 @@ updatePPE!(dfg, :x0, ppe, :default)
 updatePPE!(dfg, [x0], :default)
 ```
 
-#### Solver Data
+#### Variable States (Solver Data)
 
-Solver data is used by IncrementalInference/RoME/Caesar solver to produce the above PPEs.
+Variable `State`s are used by the IncrementalInference/RoME/Caesar solver.
 
 Related functions:
 
 
-- [`listVariableStates`](@ref)
-- [`getVariableState`](@ref)
-- [`addVariableState!`](@ref)
-- [`mergeVariableState!`](@ref)
-- [`deleteVariableState!`](@ref)
-- [`mergeVariableState!`](@ref)
+- [`listStates`](@ref)
+- [`getState`](@ref)
+- [`addState!`](@ref)
+- [`mergeState!`](@ref)
+- [`deleteState!`](@ref)
+- [`mergeState!`](@ref)
 
 
-Example of solver data operations:
+Example of variable `State` operations:
 
 ```julia
 # Add new VND of type ContinuousScalar to :x0
-# Could also do VariableState(ContinuousScalar())
-vnd = VariableState{ContinuousScalar}()
-addVariableState!(dfg, :x0, vnd, :parametric)
-@show listVariableStates(dfg, :x0)
+# Could also do State(ContinuousScalar())
+state = State{ContinuousScalar}()
+addState!(dfg, :x0, state, :parametric)
+@show listStates(dfg, :x0)
 # Get the data back - note that this is a reference to above.
-vndBack = getVariableState(dfg, :x0, :parametric)
+stateBack = getState(dfg, :x0, :parametric)
 # Delete it
-deleteVariableState!(dfg, :x0, :parametric)
+deleteState!(dfg, :x0, :parametric)
 ```
 
-#### Small Data
+#### Metadata
 
-Small data allows you to assign a dictionary to variables. It is a useful way to
-keep small amounts of string data in a variable. As it is stored in the graph
+Metadata (small data) allows you to assign a dictionary to variables. It is a useful way to
+keep small amounts of primative (Strings, Integers, Floats, Bool) data in a variable. As it is stored in the graph
 itself, large entries will slow the graph down, so if data should exceed a
-few bytes/kb, it should rather be saved in bigData.
+few bytes/kb, it should rather be saved in Blobs.
 
 
 - [`getMetadata`](@ref)

src/Common.jl

@@ -3,10 +3,10 @@
 _getmodule(t::T) where {T} = T.name.module
 _getname(t::T) where {T} = T.name.name
 
-function convertPackedType(t::Union{T, Type{T}}) where {T <: AbstractFactorObservation}
+function convertPackedType(t::Union{T, Type{T}}) where {T <: AbstractObservation}
     return getfield(_getmodule(t), Symbol("Packed$(_getname(t))"))
 end
-function convertStructType(::Type{PT}) where {PT <: AbstractPackedFactorObservation}
+function convertStructType(::Type{PT}) where {PT <: AbstractPackedObservation}
     # see #668 for expanded reasoning.  PT may be ::UnionAll if the type is of template type.
     ptt = PT isa DataType ? PT.name.name : PT
     moduleName = PT isa DataType ? PT.name.module : Main
@@ -64,11 +64,68 @@ Related
 
 ls, lsf
 """
-function sortDFG(vars::Vector{<:DFGNode}; by = getTimestamp, kwargs...)
+function sortDFG(vars::Vector{<:AbstractGraphNode}; by = getTimestamp, kwargs...)
     return sort(vars; by = by, kwargs...)
 end
 sortDFG(vars::Vector{Symbol}; lt = natural_lt, kwargs...) = sort(vars; lt = lt, kwargs...)
 
+##==============================================================================
+## Filtering
+##==============================================================================
+
+# std_numeric_predicates = [==, <, <=, >, >=, in]
+# std_string_predicates = [==, in, contains, startswith, endswith]
+
+# # full list
+# tags_includes(tag::Symbol) = Base.Fix1(in, tag)
+# solvable_eq(x::Int) = ==(x)
+# solvable_in(x::Vector{Int}) = in(x)
+# solvable_lt(x::Int) = <(x)
+# solvable_lte(x::Int) = <=(x)
+# solvable_gt(x::Int) = >(x)
+# solvable_gte(x::Int) = >=(x)
+# label_in(x::Vector{Symbol}) = in(x)
+# label_contains(x::String) = contains(x)
+# label_startswith(x::String) = startswith(x)
+# label_endswith(x::String) = endswith(x)
+# type_eq(x::AbstractStateType) = ==(x)
+# type_in(x::Vector{<:AbstractStateType}) = in(x)
+# type_contains(x::String) = contains(x)
+# type_startswith(x::String) = startswith(x)
+# type_endswith(x::String) = endswith(x)
+
+# Set predicates
+# collection_includes(item) = Base.Fix1(in, item) # collection includes item (item in collection)
+
+# not supported helper
+# collection_overlap(collection) = !isdisjoint(collection) # collection overlaps with another collection
+
+"""
+    $SIGNATURES
+Filter nodes in a DFG based on a predicate function.
+This function modifies the input `nodes` vector in place, removing nodes that do not satisfy the predicate.
+
+- **For cross-backend compatibility:**  
+  Use only the standard predicates (`==`, `<`, `<=`, `>`, `>=`, `in`, `contains`, `startswith`, `endswith`) 
+  when you need your code to work with both in-memory and database-backed DFGs. 
+  These are likely to be supported by database query languages and are defined in `std_numeric_predicates` and `std_string_predicates`.
+
+- **For in-memory only operations:**  
+  You can use any Julia predicate, since you have full access to the data and Julia's capabilities.
+  This is more flexible but will not work if you later switch to a database backend or another programming language.
+
+Standard predicates
+- Numeric predicates: `==`, `<`, `<=`, `>`, `>=`, `in`
+- String predicates: `==`, `contains`, `startswith`, `endswith`, `in`
+"""
+function filterDFG! end
+
+filterDFG!(nodes, predicate::Nothing, by::Function = identity) = nodes
+# function filterDFG!(nodes, predicate::Base.Fix2, by=identity)
+function filterDFG!(nodes, predicate::Function, by::Function = identity)
+    return filter!(predicate ∘ by, nodes)
+end
+
 ##==============================================================================
 ## Validation of session, robot, and user labels.
 ##==============================================================================
@@ -83,10 +140,8 @@ $(SIGNATURES)
 Returns true if the label is valid for node.
 """
 function isValidLabel(id::Union{Symbol, String})
-    if typeof(id) == Symbol
-        id = String(id)
-    end
-    return !in(uppercase(id), _invalidIds) && !isnothing(match(_validLabelRegex, id))
+    _id = string(id)
+    return occursin(_validLabelRegex, _id) && !in(uppercase(_id), _invalidIds)
 end
 
 """

src/DataBlobs/services/BlobEntry.jl

@@ -65,7 +65,7 @@ Get data entry
 
 Also see: [`addBlobentry!`](@ref), [`getBlob`](@ref), [`listBlobentries`](@ref)
 """
-function getBlobentry(var::AbstractDFGVariable, key::Symbol)
+function getBlobentry(var::AbstractGraphVariable, key::Symbol)
     if !hasBlobentry(var, key)
         throw(LabelNotFoundError("Blobentry", key, collect(keys(var.dataDict))))
     end
@@ -85,7 +85,7 @@ Finds and returns the first blob entry that matches the filter.
 
 Also see: [`getBlobentry`](@ref)
 """
-function getfirstBlobentry(var::AbstractDFGVariable, blobId::UUID)
+function getfirstBlobentry(var::AbstractGraphVariable, blobId::UUID)
     for (k, v) in var.dataDict
         if blobId == v.blobId
             return v
@@ -98,7 +98,7 @@ function getfirstBlobentry(dfg::AbstractDFG, label::Symbol, blobId::UUID)
     return getfirstBlobentry(getVariable(dfg, label), blobId)
 end
 
-function getfirstBlobentry(var::AbstractDFGVariable, key::Regex)
+function getfirstBlobentry(var::AbstractGraphVariable, key::Regex)
     for (k, v) in var.dataDict
         if occursin(key, string(v.label))
             return v
@@ -152,7 +152,7 @@ Should be extended if DFG variable is not returned by reference.
 
 Also see: [`getBlobentry`](@ref), [`addBlob!`](@ref), [`mergeBlobentries!`](@ref)
 """
-function addBlobentry!(var::AbstractDFGVariable, entry::Blobentry)
+function addBlobentry!(var::AbstractGraphVariable, entry::Blobentry)
     # see https://github.com/JuliaRobotics/DistributedFactorGraphs.jl/issues/985
     # blobId::Union{UUID,Nothing} = (isnothing(entry.blobId) ? entry.id : entry.blobId),
     # blobSize::Int = (hasfield(Blobentry, :size) ? entry.size : -1)
@@ -182,7 +182,7 @@ Update a Blobentry in the factor graph.
 If the Blobentry does not exist, it will be added.
 Notes:
 """
-function mergeBlobentry!(var::AbstractDFGVariable, bde::Blobentry)
+function mergeBlobentry!(var::AbstractGraphVariable, bde::Blobentry)
     if !haskey(var.dataDict, bde.label)
         addBlobentry!(var, bde)
     else
@@ -203,7 +203,7 @@ Note this doesn't remove it from any data stores.
 Notes:
 - users responsibility to delete data in db before deleting entry
 """
-function deleteBlobentry!(var::AbstractDFGVariable, key::Symbol)
+function deleteBlobentry!(var::AbstractGraphVariable, key::Symbol)
     pop!(var.dataDict, key)
     return 1
 end
@@ -226,7 +226,7 @@ function deleteBlobentry!(dfg::AbstractDFG, label::Symbol, key::Symbol)
     return deleteBlobentry!(getVariable(dfg, label), key)
 end
 
-function deleteBlobentry!(var::AbstractDFGVariable, entry::Blobentry)
+function deleteBlobentry!(var::AbstractGraphVariable, entry::Blobentry)
     #users responsibility to delete data in db before deleting entry
     return deleteBlobentry!(var, entry.label)
 end
@@ -240,7 +240,8 @@ end
 
 Does a blob entry exist with `blobLabel`.
 """
-hasBlobentry(var::AbstractDFGVariable, blobLabel::Symbol) = haskey(var.dataDict, blobLabel)
+hasBlobentry(var::AbstractGraphVariable, blobLabel::Symbol) =
+    haskey(var.dataDict, blobLabel)
 
 function hasBlobentry(var::VariableDFG, label::Symbol)
     return label in getproperty.(var.blobEntries, :label)
@@ -251,7 +252,7 @@ end
 
 Get blob entries, Vector{Blobentry}
 """
-function getBlobentries(var::AbstractDFGVariable)
+function getBlobentries(var::AbstractGraphVariable)
     #or should we return the iterator, Base.ValueIterator{Dict{Symbol,Blobentry}}?
     return collect(values(var.dataDict))
 end
@@ -311,7 +312,7 @@ end
     $(SIGNATURES)
 List the blob entries associated with a particular variable.
 """
-function listBlobentries(var::AbstractDFGVariable)
+function listBlobentries(var::AbstractGraphVariable)
     return collect(keys(var.dataDict))
 end
 

src/DataBlobs/services/BlobStores.jl

@@ -7,6 +7,8 @@ Get the data blob for the specified blobstore or dfg.
 
 Related
 [`getBlobentry`](@ref)
+Implement 
+`getBlob(store::AbstractBlobstore, blobId::UUID)`
 
 $(METHODLIST)
 """
@@ -17,67 +19,31 @@ Adds a blob to the blob store or dfg with the blobId.
 
 Related
 [`addBlobentry!`](@ref)
-
+Implement
+`addBlob!(store::AbstractBlobstore, blobId::UUID, data)`
 $(METHODLIST)
 """
 function addBlob! end
 
-"""
-Update a blob to the blob store or dfg with the given entry.
-Related
-[`mergeBlobentry!`](@ref)
-
-$(METHODLIST)
-
-DevNotes
-- TODO TBD update verb on data since data blobs and entries are restricted to immutable only.
-"""
-function updateBlob! end
-
 """
 Delete a blob from the blob store or dfg with the given entry.
 
 Related
 [`deleteBlobentry!`](@ref)
-
+Implement
+`deleteBlob!(store::AbstractBlobstore, blobId::UUID)`
 $(METHODLIST)
 """
 function deleteBlob! end
 
 """
     $(SIGNATURES)
-List all ids in the blob store.
+List all `blobId`s in the blob store.
+Implement
+`listBlobs(store::AbstractBlobstore)`
 """
 function listBlobs end
 
-##==============================================================================
-## AbstractBlobstore CRUD Interface
-##==============================================================================
-
-function getBlob(store::AbstractBlobstore, ::UUID)
-    return error("$(typeof(store)) doesn't override 'getBlob'.")
-end
-
-function addBlob!(store::AbstractBlobstore{T}, ::UUID, ::T) where {T}
-    return error("$(typeof(store)) doesn't override 'addBlob!'.")
-end
-
-function updateBlob!(store::AbstractBlobstore{T}, ::UUID, ::T) where {T}
-    return error("$(typeof(store)) doesn't override 'updateBlob!'.")
-end
-
-function deleteBlob!(store::AbstractBlobstore, ::UUID)
-    return error("$(typeof(store)) doesn't override 'deleteBlob!'.")
-end
-
-function listBlobs(store::AbstractBlobstore)
-    return error("$(typeof(store)) doesn't override 'listBlobs'.")
-end
-
-function hasBlob(store::AbstractBlobstore, ::UUID)
-    return error("$(typeof(store)) doesn't override 'hasBlob'.")
-end
-
 ##==============================================================================
 ## AbstractBlobstore derived CRUD for Blob 
 ##==============================================================================