Merge pull request #1001 from JuliaRobotics/master

v0.20.2-rc1

Author: Affie

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.20.1"
+version = "0.20.2"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

src/DataBlobs/entities/BlobEntry.jl

@@ -30,6 +30,4 @@ StructTypes.StructType(::Type{BlobEntry}) = StructTypes.UnorderedStruct()
 StructTypes.idproperty(::Type{BlobEntry}) = :id
 StructTypes.omitempties(::Type{BlobEntry}) = (:id,)
 
-
-
 _fixtimezone(cts::NamedTuple) = ZonedDateTime(cts.utc_datetime*"+00")

src/DataBlobs/services/BlobEntry.jl

@@ -46,7 +46,7 @@ function assertHash(de::BlobEntry, db; hashfunction::Function = sha256)
 end
 
 
-function Base.show(io::IO, entry::BlobEntry) 
+function Base.show(io::IO, ::MIME"text/plain", entry::BlobEntry)
     println(io, "_type=BlobEntry {")
     println(io, "  id:            ", entry.id) 
     println(io, "  blobId:        ", entry.blobId) 
@@ -62,10 +62,6 @@ function Base.show(io::IO, entry::BlobEntry)
     println(io, "}") 
 end 
 
-Base.show(io::IO, ::MIME"text/plain", entry::BlobEntry) = show(io, entry)
-
-
-
 ##==============================================================================
 ## BlobEntry - CRUD
 ##==============================================================================
@@ -200,7 +196,8 @@ hasBlobEntry(var::AbstractDFGVariable, blobLabel::Symbol) = haskey(var.dataDict,
 
 """
     $(SIGNATURES)
-Get data entries, Vector{BlobEntry}
+
+Get blob entries, Vector{BlobEntry}
 """
 function getBlobEntries(var::AbstractDFGVariable)
     #or should we return the iterator, Base.ValueIterator{Dict{Symbol,BlobEntry}}?
@@ -212,6 +209,36 @@ function getBlobEntries(dfg::AbstractDFG, label::Symbol)
     getBlobEntries(getVariable(dfg, label))
 end
 
+"""
+    $(SIGNATURES)
+
+Get all blob entries matching a Regex pattern over variables
+
+Notes
+- Use `dropEmpties=true` to not include empty lists in result.
+- Use keyword `varList` for which variables to search through.
+"""
+function getBlobEntriesVariables(
+    dfg::AbstractDFG,
+    bLblPattern::Regex;
+    varList::AbstractVector{Symbol} = sort(listVariables(dfg); lt=natural_lt),
+    dropEmpties::Bool = false
+)
+    RETLIST = Vector{Vector{BlobEntry}}()
+    @showprogress "Get entries matching $bLblPattern" for vl in varList
+        bes = filter(
+            s->occursin(bLblPattern,string(s.label)),
+            listBlobEntries(dfg,vl)
+        )
+        # only push to list if there are entries on this variable
+        (!dropEmpties || 0 < length(bes)) ? nothing : continue
+        push!(RETLIST, bes)
+    end
+    
+    return RETLIST
+end
+
+
 """
     $(SIGNATURES)
 List the blob entries associated with a particular variable.

src/DataBlobs/services/HelpersDataWrapEntryBlob.jl

@@ -6,7 +6,7 @@
 # NOTE this is the convenience wrappers for entry and blob.
 
 """
-Get the data entry and blob for the specified blobstore or dfg retured as a tuple.
+Get the blob entry and blob for the specified blobstore or dfg retured as a tuple.
 Related
 [`getBlobEntry`](@ref)
 
@@ -241,12 +241,12 @@ end
 deleteData!(
     dfg::AbstractDFG, 
     blobstore::AbstractBlobStore, 
-    label::Symbol, 
+    vLbl::Symbol, 
     entry::BlobEntry
 ) = deleteBlob!(
     dfg, 
     blobstore, 
-    label, 
+    vLbl, 
     entry.label
 )
 

src/DistributedFactorGraphs.jl

@@ -200,6 +200,7 @@ export hasBlobEntry,getBlobEntry,addBlobEntry!,updateBlobEntry!,deleteBlobEntry!
 export incrDataLabelSuffix
 
 export getBlobEntries, listDataEntries, hasDataEntry, hasDataEntry
+export getBlobEntriesVariables
 export listDataEntrySequence
 # convenience wrappers
 export mergeDataEntries!

src/FileDFG/services/FileDFG.jl

@@ -40,12 +40,12 @@ function saveDFG(folder::AbstractString, dfg::AbstractDFG)
     map(f -> rm("$varFolder/$f"), readdir(varFolder))
     map(f -> rm("$factorFolder/$f"), readdir(factorFolder))
     # Variables
-    for v in variables
+    @showprogress "saving variables" for v in variables
         vPacked = packVariable(v)
         JSON3.write("$varFolder/$(v.label).json", vPacked)
     end
     # Factors
-    for f in factors
+    @showprogress "saving factors" for f in factors
         fPacked = packFactor(dfg, f)
         JSON3.write("$factorFolder/$(f.label).json", fPacked)
     end
@@ -56,9 +56,9 @@ function saveDFG(folder::AbstractString, dfg::AbstractDFG)
     destfile = joinpath(savedir, savename*".tar.gz")
     # FIXME, switch to Tar.jl and Transcode Zlib / Codec, see #351
     if length(savedir) != 0
-      run( pipeline(`tar -zcf - -C $savedir $savename`, stdout="$destfile"))
+        run( pipeline(`tar -zcf - -C $savedir $savename`, stdout="$destfile"))
     else
-      run( pipeline(`tar -zcf - $savename`, stdout="$destfile"))
+        run( pipeline(`tar -zcf - $savename`, stdout="$destfile"))
     end
     Base.rm(joinpath(savedir,savename), recursive=true)
 end

src/entities/DFGFactor.jl

@@ -48,6 +48,12 @@ Base.@kwdef mutable struct GenericFunctionNodeData{T<:Union{<:AbstractPackedFact
     inflation::Float64 = 0.0
 end
 
+# TODO should we move non FactorOperationalMemory to DFGFactor: 
+# fnc, multihypo, nullhypo, inflation ?
+# that way we split solverData <: FactorOperationalMemory and constants
+# TODO see if above ever changes?
+
+
 ## Constructors
 
 
@@ -78,8 +84,7 @@ FunctionNodeData(args...; kw...) = FunctionNodeData{typeof(args[4])}(args...; kw
 
 # Packed Factor
 Base.@kwdef struct PackedFactor
-    # NOTE: This has to match the order of the JSON deserializer as we're using OrderedStructs.
-    id::Union{UUID, Nothing}
+    id::Union{UUID, Nothing} = nothing
     label::Symbol
     tags::Vector{Symbol}
     _variableOrderSymbols::Vector{Symbol}
@@ -89,7 +94,7 @@ Base.@kwdef struct PackedFactor
     solvable::Int
     data::String
     metadata::String
-    _version::String
+    _version::String = string(_getDFGVersion())
   end
 
 StructTypes.StructType(::Type{PackedFactor}) = StructTypes.UnorderedStruct()

src/entities/DFGVariable.jl

@@ -91,7 +91,7 @@ VariableNodeData(variableType::InferenceVariable; kwargs...) = VariableNodeData{
 
 """
 $(TYPEDEF)
-Packed VariabeNodeData structure for serializing DFGVariables.
+Packed VariableNodeData structure for serializing DFGVariables.
 
   ---
 Fields:
@@ -179,19 +179,18 @@ getEstimateFields(::MeanMaxPPE) = [:suggested, :max, :mean]
 
 # Packed Variable
 Base.@kwdef struct PackedVariable
-    # NOTE: This has to match the order of the JSON deserializer as we're using OrderedStructs.
-    id::Union{UUID, Nothing}
+    id::Union{UUID, Nothing} = nothing
     label::Symbol
-    tags::Vector{Symbol}
-    timestamp::ZonedDateTime
-    nstime::Int
-    ppes::Vector{MeanMaxPPE}
-    blobEntries::Vector{BlobEntry}
+    tags::Vector{Symbol} = Symbol[]
+    timestamp::ZonedDateTime = now(tz"UTC")
+    nstime::Int = 0
+    ppes::Vector{MeanMaxPPE} = MeanMaxPPE[]
+    blobEntries::Vector{BlobEntry} = BlobEntry[]
     variableType::String
-    _version::String
-    metadata::String
-    solvable::Int
-    solverData::Vector{PackedVariableNodeData}
+    _version::String = string(_getDFGVersion())
+    metadata::String = "e30="
+    solvable::Int = 1
+    solverData::Vector{PackedVariableNodeData} = PackedVariableNodeData[]
 end
 
 StructTypes.StructType(::Type{PackedVariable}) = StructTypes.UnorderedStruct()
@@ -344,6 +343,27 @@ Base.@kwdef struct DFGVariableSummary <: AbstractDFGVariable
     dataDict::Dict{Symbol, BlobEntry}
 end
 
+function DFGVariableSummary(
+    id,
+    label,
+    timestamp,
+    tags,
+    ::Nothing,
+    variableTypeName,
+    ::Nothing,
+)
+    return DFGVariableSummary(
+        id,
+        label,
+        timestamp,
+        tags,
+        Dict{Symbol, MeanMaxPPE}(),
+        variableTypeName,
+        Dict{Symbol, BlobEntry}(),
+    )
+end
+
+StructTypes.names(::Type{DFGVariableSummary}) = ((:variableTypeName, :variableType),)
 
 ##------------------------------------------------------------------------------
 ## SkeletonDFGVariable.jl

src/services/DFGVariable.jl

@@ -16,8 +16,8 @@ getPPEMean(fg::AbstractDFG, varlabel::Symbol, solveKey::Symbol=:default) =
 
 "$(SIGNATURES)"
 getPPESuggested(est::AbstractPointParametricEst) = est.suggested
-getPPESuggested(fg::AbstractDFG, varlabel::Symbol, solveKey::Symbol=:default) =
-                getPPE(fg, varlabel, solveKey) |> getPPESuggested
+getPPESuggested(var::DFGVariable, solveKey::Symbol=:default) = getPPE(var, solveKey) |> getPPESuggested
+getPPESuggested(dfg::AbstractDFG, varlabel::Symbol, solveKey::Symbol=:default) = getPPE(getVariable(dfg, varlabel), solveKey) |> getPPESuggested
 
 "$(SIGNATURES)"
 getLastUpdatedTimestamp(est::AbstractPointParametricEst) = est.lastUpdatedTimestamp
@@ -570,7 +570,7 @@ Get variable solverdata for a given solve key.
 """
 function getVariableSolverData(dfg::AbstractDFG, variablekey::Symbol, solvekey::Symbol=:default)
     v = getVariable(dfg, variablekey)
-    !haskey(v.solverDataDict, solvekey) && error("Solve key '$solvekey' not found in variable '$variablekey'")
+    !haskey(v.solverDataDict, solvekey) && throw(KeyError("Solve key '$solvekey' not found in variable '$variablekey'"))
     return v.solverDataDict[solvekey]
 end
 
@@ -736,7 +736,7 @@ function deleteVariableSolverData!(dfg::AbstractDFG, variablekey::Symbol, solveK
     var = getVariable(dfg, variablekey)
 
     if !haskey(var.solverDataDict, solveKey)
-        error("VariableNodeData '$(solveKey)' does not exist")
+        throw(KeyError("VariableNodeData '$(solveKey)' does not exist"))
     end
     vnd = pop!(var.solverDataDict, solveKey)
     return vnd
@@ -793,14 +793,13 @@ Notes
 - Defaults on keywords `solveKey` and `method`
 
 Related
-getMeanPPE, getMaxPPE, getKDEMean, getKDEFit, getPPEs, getVariablePPEs
+[`getMeanPPE`](@ref), [`getMaxPPE`](@ref), [`updatePPE!`](@ref), getKDEMean, getKDEFit, getPPEs, getVariablePPEs
 """
-function getPPE(dfg::AbstractDFG, variablekey::Symbol, ppekey::Symbol=:default)
-    v = getVariable(dfg, variablekey)
-    !haskey(v.ppeDict, ppekey) && error("PPE key '$ppekey' not found in variable '$variablekey'")
+function getPPE(v::DFGVariable, ppekey::Symbol=:default)
+    !haskey(v.ppeDict, ppekey) && throw(KeyError("PPE key '$ppekey' not found in variable '$(getLabel(v))'"))
     return v.ppeDict[ppekey]
 end
-
+getPPE(dfg::AbstractDFG, variablekey::Symbol, ppekey::Symbol=:default) = getPPE(getVariable(dfg, variablekey), ppekey)
 # Not the most efficient call but it at least reuses above (in memory it's probably ok)
 getPPE(dfg::AbstractDFG, sourceVariable::VariableDataLevel1, ppekey::Symbol=:default) = getPPE(dfg, sourceVariable.label, ppekey)
 
@@ -870,7 +869,7 @@ function deletePPE!(dfg::AbstractDFG, variablekey::Symbol, ppekey::Symbol=:defau
     var = getVariable(dfg, variablekey)
 
     if !haskey(var.ppeDict, ppekey)
-        error("VariableNodeData '$(ppekey)' does not exist")
+        throw(KeyError("VariableNodeData '$(ppekey)' does not exist"))
     end
     vnd = pop!(var.ppeDict, ppekey)
     return vnd

test/testBlocks.jl

@@ -590,9 +590,9 @@ function  PPETestBlock!(fg, v1)
     @test_throws ErrorException addPPE!(fg, :a, ppe)
 
     @test listPPEs(fg, :a) == [:default]
-    # Get the data back - note that this is a reference to above.
-    @test getPPE(fg, :a, :default) == ppe
 
+    # Get the data back - note that this is a reference to above.
+    @test getPPE(getVariable(fg, :a), :default) == ppe
     @test getPPE(fg, :a, :default) == ppe
     @test getPPEMean(fg, :a, :default) == ppe.mean
     @test getPPEMax(fg, :a, :default) == ppe.max
@@ -601,7 +601,7 @@ function  PPETestBlock!(fg, v1)
     # Delete it
     @test deletePPE!(fg, :a, :default) == ppe
 
-    @test_throws ErrorException getPPE(fg, :a, :default)
+    @test_throws KeyError getPPE(fg, :a, :default)
     # Update add it
     @test @test_logs (:warn, Regex("'$(ppe.solveKey)' does not exist")) match_mode=:any updatePPE!(fg, :a, ppe) == ppe
     # Update update it
@@ -702,7 +702,7 @@ function  VSDTestBlock!(fg, v1)
     #  - `updateVariableSolverData!`
     #  - `deleteVariableSolverData!`
     #
-    # > - `getVariableSolverDataAll` #TODO Data is already plural so maybe Variables, All or Dict
+    # > - `getVariableSolverDataAll` #TODO Data is already plural so maybe Variables, All or Dict, or use Datum for singular
     # > - `getVariablesSolverData`
     #
     # **Set like**
@@ -761,7 +761,7 @@ function  VSDTestBlock!(fg, v1)
     # Delete parametric from v1
     @test deleteVariableSolverData!(fg, :a, :parametric) == vnd
 
-    @test_throws ErrorException getVariableSolverData(fg, :a, :parametric)
+    @test_throws KeyError getVariableSolverData(fg, :a, :parametric)
 
     #FIXME copied from lower
     @test getSolverData(v1) === v1.solverDataDict[:default]