More standardization and improvements towards v1.0 (#1165)

* Standardize / check some of the core functions

* improve lsPriors 

* getBlobentriesVariables -> gatherBlobentries

* standard getfirstBlobentry

Author: Affie

Project.toml

@@ -75,16 +75,18 @@ Test = "1.10"
 TimeZones = "1.3.1"
 UUIDs = "1.10"
 julia = "1.10"
+LieGroups = "0.1"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 GraphMakie = "1ecd5474-83a3-4783-bb4f-06765db800d2"
+LieGroups = "6774de46-80ba-43f8-ba42-e41071ccfc5f"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Manifolds = "1cead3c2-87b3-11e9-0ccd-23c62b72b94e"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Aqua", "Test", "DataStructures", "GraphMakie", "LinearAlgebra", "Manifolds", "Pkg", "Statistics"]
+test = ["Aqua", "Test", "DataStructures", "GraphMakie", "LinearAlgebra", "Manifolds", "Pkg", "Statistics", "LieGroups"]

src/DataBlobs/entities/BlobStores.jl

@@ -29,3 +29,4 @@ Design goal: all `Blobstore`s with the same `label` can contain the same `blobId
 
 """
 abstract type AbstractBlobstore{T} end
+const Blobstore = AbstractBlobstore

src/DataBlobs/services/BlobEntry.jl

@@ -1,14 +1,3 @@
-
-##==============================================================================
-## Blobentry - compare
-##==============================================================================
-
-import Base: ==
-
-@generated function ==(x::T, y::T) where {T <: Blobentry}
-    return mapreduce(n -> :(x.$n == y.$n), (a, b) -> :($a && $b), fieldnames(x))
-end
-
 ##==============================================================================
 ## Blobentry - common
 ##==============================================================================
@@ -85,49 +74,26 @@ Finds and returns the first blob entry that matches the filter.
 
 Also see: [`getBlobentry`](@ref)
 """
-function getfirstBlobentry(var::AbstractGraphVariable, blobId::UUID)
-    for (k, v) in var.dataDict
-        if blobId == v.blobId
-            return v
-        end
-    end
-    throw(KeyError("No blobEntry with blobId $(blobId) found in variable $(getLabel(var))"))
-end
-
-function getfirstBlobentry(dfg::AbstractDFG, label::Symbol, blobId::UUID)
-    return getfirstBlobentry(getVariable(dfg, label), blobId)
-end
-
-function getfirstBlobentry(var::AbstractGraphVariable, key::Regex)
-    for (k, v) in var.dataDict
-        if occursin(key, string(v.label))
-            return v
-        end
-    end
-    throw(
-        KeyError(
-            "No blobEntry with label matching regex $(key) found in variable $(getLabel(var))",
-        ),
-    )
-end
-
-function getfirstBlobentry(var::VariableDFG, key::Regex)
-    firstIdx = findfirst(x -> contains(string(x.label), key), var.blobEntries)
-    if isnothing(firstIdx)
-        throw(KeyError("$key"))
+function getfirstBlobentry(
+    v::AbstractGraphVariable;
+    labelFilter::Union{Nothing, Function} = nothing,
+    blobIdFilter::Union{Nothing, Function} = nothing,
+)
+    entries = getBlobentries(v; labelFilter, blobIdFilter)
+    if isempty(entries)
+        return nothing
+    else
+        return entries[1]
     end
-    return var.blobEntries[firstIdx]
 end
 
-function getfirstBlobentry(dfg::AbstractDFG, label::Symbol, key::Regex)
-    els = listBlobentries(dfg, label)
-    firstIdx = findfirst(contains(key), string.(els))
-    isnothing(firstIdx) && throw(
-        KeyError(
-            "No blobEntry with label matching regex $(key) found in variable $(label)",
-        ),
-    )
-    return getBlobentry(dfg, label, els[firstIdx])
+function getfirstBlobentry(
+    dfg::AbstractDFG,
+    label::Symbol;
+    labelFilter::Union{Nothing, Function} = nothing,
+    blobIdFilter::Union{Nothing, Function} = nothing,
+)
+    return getfirstBlobentry(getVariable(dfg, label); labelFilter, blobIdFilter)
 end
 
 # TODO Consider autogenerating all methods of the form:
@@ -140,22 +106,18 @@ end
 #     @eval DistributedFactorGraphs $met(dfg::AbstractDFG, label::Symbol, args...; kwargs...) = $met(getVariable(dfg, label), args...; kwargs...)
 # end
 
-function getBlobentry(dfg::AbstractDFG, label::Symbol, key::Symbol)
-    return getBlobentry(getVariable(dfg, label), key)
+function getBlobentry(dfg::AbstractDFG, varLabel::Symbol, label::Symbol)
+    return getBlobentry(getVariable(dfg, varLabel), label)
 end
-# getBlobentry(dfg::AbstractDFG, label::Symbol, key::Symbol) = getBlobentry(getVariable(dfg, label), key)
 
 """
     $(SIGNATURES)
-Add Data Entry to a DFG variable
+Add a `Blobentry` to a variable
 Should be extended if DFG variable is not returned by reference.
 
-Also see: [`getBlobentry`](@ref), [`addBlob!`](@ref), [`mergeBlobentries!`](@ref)
+Also see: [`getBlobentry`](@ref), [`addBlob!`](@ref), [`mergeBlobentry!`](@ref)
 """
-function addBlobentry!(var::AbstractGraphVariable, entry::Blobentry)
-    # see https://github.com/JuliaRobotics/DistributedFactorGraphs.jl/issues/985
-    # blobId::Union{UUID,Nothing} = (isnothing(entry.blobId) ? entry.id : entry.blobId),
-    # blobSize::Int = (hasfield(Blobentry, :size) ? entry.size : -1)
+function addBlobentry!(var::VariableCompute, entry::Blobentry)
     haskey(var.dataDict, entry.label) && throw(LabelExistsError("Blobentry", entry.label))
     var.dataDict[entry.label] = entry
     return entry
@@ -197,37 +159,28 @@ end
 
 """
     $(SIGNATURES)
-Delete a blob entry from the factor graph.
-Note this doesn't remove it from any data stores.
+Delete a `Blobentry` from the factor graph variable.
 
 Notes:
-- users responsibility to delete data in db before deleting entry
+- This doesn't remove the associated `Blob` from any Blobstores.
 """
-function deleteBlobentry!(var::AbstractGraphVariable, key::Symbol)
-    pop!(var.dataDict, key)
+function deleteBlobentry!(var::VariableCompute, key::Symbol)
+    !hasBlobentry(var, key) && throw(LabelNotFoundError("Blobentry", key))
+    delete!(var.dataDict, key)
     return 1
 end
 
 function deleteBlobentry!(var::VariableDFG, key::Symbol)
-    if !hasBlobentry(var, key)
-        throw(
-            KeyError(
-                "No dataEntry label $(key) found in variable $(getLabel(var)). Available keys: $(keys(var.dataDict))",
-            ),
-        )
-    end
+    !hasBlobentry(var, key) && throw(LabelNotFoundError("Blobentry", key))
     deleteat!(var.blobEntries, findfirst(x -> x.label == key, var.blobEntries))
     return 1
 end
 
 function deleteBlobentry!(dfg::AbstractDFG, label::Symbol, key::Symbol)
-    #users responsibility to delete data in db before deleting entry
-    # !isVariable(dfg, label) && return nothing
     return deleteBlobentry!(getVariable(dfg, label), key)
 end
 
 function deleteBlobentry!(var::AbstractGraphVariable, entry::Blobentry)
-    #users responsibility to delete data in db before deleting entry
     return deleteBlobentry!(var, entry.label)
 end
 
@@ -240,73 +193,66 @@ end
 
 Does a blob entry exist with `blobLabel`.
 """
-hasBlobentry(var::AbstractGraphVariable, blobLabel::Symbol) =
-    haskey(var.dataDict, blobLabel)
+hasBlobentry(v::VariableCompute, blobLabel::Symbol) = haskey(v.dataDict, blobLabel)
 
-function hasBlobentry(var::VariableDFG, label::Symbol)
-    return label in getproperty.(var.blobEntries, :label)
+function hasBlobentry(v::VariableDFG, label::Symbol)
+    return label in getproperty.(v.blobEntries, :label)
 end
 
 """
     $(SIGNATURES)
 
-Get blob entries, Vector{Blobentry}
+Get blob entries, returns a `Vector{Blobentry}`.
 """
-function getBlobentries(var::AbstractGraphVariable)
-    #or should we return the iterator, Base.ValueIterator{Dict{Symbol,Blobentry}}?
-    return collect(values(var.dataDict))
+function getBlobentries(v::VariableCompute)
+    return collect(values(v.dataDict))
 end
 
-function getBlobentries(var::VariableDFG)
-    return var.blobEntries
+function getBlobentries(v::VariableDFG)
+    return copy(v.blobEntries)
 end
 
-function getBlobentries(dfg::AbstractDFG, label::Symbol)
-    # !isVariable(dfg, label) && return nothing
-    #or should we return the iterator, Base.ValueIterator{Dict{Symbol,Blobentry}}?
-    return getBlobentries(getVariable(dfg, label))
-end
-
-function getBlobentries(dfg::AbstractDFG, label::Symbol, regex::Regex)
-    entries = getBlobentries(dfg, label)
-    return filter(entries) do e
-        return occursin(regex, string(e.label))
-    end
+function getBlobentries(
+    v::AbstractGraphVariable;
+    labelFilter::Union{Nothing, Function} = nothing,
+    blobIdFilter::Union{Nothing, Function} = nothing,
+)
+    entries = getBlobentries(v)
+    filterDFG!(entries, labelFilter, x -> string(x.label))
+    filterDFG!(entries, blobIdFilter, x -> string(x.blobId))
+    return entries
 end
 
 function getBlobentries(
     dfg::AbstractDFG,
-    label::Symbol,
-    skey::Union{Symbol, <:AbstractString},
+    variableLabel::Symbol;
+    labelFilter::Union{Nothing, Function} = nothing,
+    blobIdFilter::Union{Nothing, Function} = nothing,
 )
-    return getBlobentries(dfg, label, Regex(string(skey)))
+    return getBlobentries(getVariable(dfg, variableLabel); labelFilter, blobIdFilter)
 end
 
-"""
-    $(SIGNATURES)
-
-Get all blob entries matching a Regex pattern over variables
-
-Notes
-- Use `dropEmpties=true` to not include empty lists in result.
-- Use keyword `varList` for which variables to search through.
-"""
-function getBlobentriesVariables(
-    dfg::AbstractDFG,
-    bLblPattern::Regex;
-    varList::AbstractVector{Symbol} = sort(listVariables(dfg); lt = natural_lt),
-    dropEmpties::Bool = false,
+function gatherBlobentries(
+    dfg::AbstractDFG;
+    labelFilter::Union{Nothing, Function} = nothing,
+    blobIdFilter::Union{Nothing, Function} = nothing,
+    solvableFilter::Union{Nothing, Function} = nothing,
+    tagsFilter::Union{Nothing, Function} = nothing,
+    typeFilter::Union{Nothing, Function} = nothing,
+    variableLabelFilter::Union{Nothing, Function} = nothing,
 )
-    RETLIST = Vector{Vector{Blobentry}}()
-    @showprogress "Get entries matching $bLblPattern" for vl in varList
-        bes = filter(s -> occursin(bLblPattern, string(s.label)), listBlobentries(dfg, vl))
-        # only push to list if there are entries on this variable
-        (!dropEmpties || 0 < length(bes)) ? nothing : continue
-        push!(RETLIST, bes)
+    vls = listVariables(
+        dfg;
+        solvableFilter,
+        tagsFilter,
+        typeFilter,
+        labelFilter = variableLabelFilter,
+    )
+    return map(vls) do vl
+        return vl => getBlobentries(dfg, vl; labelFilter, blobIdFilter)
     end
-
-    return RETLIST
 end
+const collectBlobentries = gatherBlobentries
 
 """
     $(SIGNATURES)
@@ -321,7 +267,6 @@ function listBlobentries(var::VariableDFG)
 end
 
 function listBlobentries(dfg::AbstractDFG, label::Symbol)
-    # !isVariable(dfg, label) && return nothing
     return listBlobentries(getVariable(dfg, label))
 end
 
@@ -357,66 +302,6 @@ function listBlobentrySequence(
     return ents_[findall(entMsk)] |> _sort
 end
 
-"""
-    $SIGNATURES
-
-Add a blob entry into the destination variable which already exists 
-in a source variable.
-
-See also: [`addBlobentry!`](@ref), [`getBlobentry`](@ref), [`listBlobentries`](@ref), [`getBlob`](@ref)
-"""
-function mergeBlobentries!(
-    dst::AbstractDFG,
-    dlbl::Symbol,
-    src::AbstractDFG,
-    slbl::Symbol,
-    bllb::Union{Symbol, UUID, <:AbstractString, Regex},
-)
-    #
-    _makevec(s) = [s;]
-    _makevec(s::AbstractVector) = s
-    des_ = getBlobentry(src, slbl, bllb)
-    des = _makevec(des_)
-    # don't add data entries that already exist 
-    dde = listBlobentries(dst, dlbl)
-    # HACK, verb list should just return vector of Symbol. NCE36
-    _getid(s) = s
-    _getid(s::Blobentry) = s.id
-    uids = _getid.(dde) # (s->s.id).(dde)
-    filter!(s -> !(_getid(s) in uids), des)
-    # add any data entries not already in the destination variable, by uuid
-    return addBlobentry!.(dst, dlbl, des)
-end
-
-function mergeBlobentries!(
-    dst::AbstractDFG,
-    dlbl::Symbol,
-    src::AbstractDFG,
-    slbl::Symbol,
-    ::Colon = :,
-)
-    des = listBlobentries(src, slbl)
-    # don't add data entries that already exist 
-    uids = listBlobentries(dst, dlbl)
-    # verb list should just return vector of Symbol. NCE36
-    filter!(s -> !(s in uids), des)
-    if 0 < length(des)
-        union(((s -> mergeBlobentries!(dst, dlbl, src, slbl, s)).(des))...)
-    end
-end
-
-function mergeBlobentries!(
-    dest::AbstractDFG,
-    src::AbstractDFG,
-    w...;
-    varList::AbstractVector = listVariables(dest) |> sortDFG,
-)
-    @showprogress 1 "merging data entries" for vl in varList
-        mergeBlobentries!(dest, vl, src, vl, w...)
-    end
-    return varList
-end
-
 """
     $SIGNATURES
 

src/DataBlobs/services/BlobStores.jl

@@ -275,10 +275,6 @@ function getBlob(store::LinkStore, blobId::UUID)
     return read(fname)
 end
 
-function addBlob!(store::LinkStore, entry::Blobentry, linkfile::String)
-    return addBlob!(store, entry.blobId, linkfile)
-end
-
 function addBlob!(store::LinkStore, blobId::UUID, linkfile::String)
     if haskey(store.cache, blobId)
         throw(IdExistsError("Blob", blobId))