Merge branch 'master' into 23Q1/enh/pcktojustvar

Author: Affie

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.20.1"
+version = "0.20.2"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

docs/src/variable_factor_serialization.md

@@ -26,7 +26,7 @@ prior = addFactor!(dfg, [:x0], PriorPose2( MvNormal([10; 10; 1.0/8.0], Matrix(Di
 
 # Now serialize them:
 pVariable = packVariable(dfg, x0)
-pFactor = packFactor(dfg, prior)
+pFactor = packFactor(prior)
 
 # And we can deserialize them
 upVariable = unpackVariable(dfg, pVariable)
@@ -60,7 +60,7 @@ for v in getVariables(dfg)
 end
 # Factors
 for f in getFactors(dfg)
-    fPacked = packFactor(dfg, f)
+    fPacked = packFactor(f)
     io = open("$folder/factors/$(f.label).json", "w")
     JSON3.write(io, fPacked)
     close(io)

src/DataBlobs/entities/BlobEntry.jl

@@ -30,6 +30,4 @@ StructTypes.StructType(::Type{BlobEntry}) = StructTypes.UnorderedStruct()
 StructTypes.idproperty(::Type{BlobEntry}) = :id
 StructTypes.omitempties(::Type{BlobEntry}) = (:id,)
 
-
-
 _fixtimezone(cts::NamedTuple) = ZonedDateTime(cts.utc_datetime*"+00")

src/DataBlobs/services/BlobEntry.jl

@@ -46,7 +46,7 @@ function assertHash(de::BlobEntry, db; hashfunction::Function = sha256)
 end
 
 
-function Base.show(io::IO, entry::BlobEntry) 
+function Base.show(io::IO, ::MIME"text/plain", entry::BlobEntry)
     println(io, "_type=BlobEntry {")
     println(io, "  id:            ", entry.id) 
     println(io, "  blobId:        ", entry.blobId) 
@@ -62,10 +62,6 @@ function Base.show(io::IO, entry::BlobEntry)
     println(io, "}") 
 end 
 
-Base.show(io::IO, ::MIME"text/plain", entry::BlobEntry) = show(io, entry)
-
-
-
 ##==============================================================================
 ## BlobEntry - CRUD
 ##==============================================================================
@@ -200,7 +196,8 @@ hasBlobEntry(var::AbstractDFGVariable, blobLabel::Symbol) = haskey(var.dataDict,
 
 """
     $(SIGNATURES)
-Get data entries, Vector{BlobEntry}
+
+Get blob entries, Vector{BlobEntry}
 """
 function getBlobEntries(var::AbstractDFGVariable)
     #or should we return the iterator, Base.ValueIterator{Dict{Symbol,BlobEntry}}?
@@ -212,6 +209,36 @@ function getBlobEntries(dfg::AbstractDFG, label::Symbol)
     getBlobEntries(getVariable(dfg, label))
 end
 
+"""
+    $(SIGNATURES)
+
+Get all blob entries matching a Regex pattern over variables
+
+Notes
+- Use `dropEmpties=true` to not include empty lists in result.
+- Use keyword `varList` for which variables to search through.
+"""
+function getBlobEntriesVariables(
+    dfg::AbstractDFG,
+    bLblPattern::Regex;
+    varList::AbstractVector{Symbol} = sort(listVariables(dfg); lt=natural_lt),
+    dropEmpties::Bool = false
+)
+    RETLIST = Vector{Vector{BlobEntry}}()
+    @showprogress "Get entries matching $bLblPattern" for vl in varList
+        bes = filter(
+            s->occursin(bLblPattern,string(s.label)),
+            listBlobEntries(dfg,vl)
+        )
+        # only push to list if there are entries on this variable
+        (!dropEmpties || 0 < length(bes)) ? nothing : continue
+        push!(RETLIST, bes)
+    end
+    
+    return RETLIST
+end
+
+
 """
     $(SIGNATURES)
 List the blob entries associated with a particular variable.

src/DataBlobs/services/HelpersDataWrapEntryBlob.jl

@@ -6,7 +6,7 @@
 # NOTE this is the convenience wrappers for entry and blob.
 
 """
-Get the data entry and blob for the specified blobstore or dfg retured as a tuple.
+Get the blob entry and blob for the specified blobstore or dfg retured as a tuple.
 Related
 [`getBlobEntry`](@ref)
 
@@ -156,6 +156,7 @@ addData!(
     kwargs...
 )
 
+#FIXME id used wrong
 function addData!(
     dfg::AbstractDFG, 
     blobstore::AbstractBlobStore, 
@@ -241,12 +242,12 @@ end
 deleteData!(
     dfg::AbstractDFG, 
     blobstore::AbstractBlobStore, 
-    label::Symbol, 
+    vLbl::Symbol, 
     entry::BlobEntry
 ) = deleteBlob!(
     dfg, 
     blobstore, 
-    label, 
+    vLbl, 
     entry.label
 )
 

src/Deprecated.jl

@@ -2,7 +2,7 @@
 ## ================================================================================
 ## Remove in v0.21
 ##=================================================================================
-
+@deprecate packFactor(dfg::AbstractDFG, f::DFGFactor) packFactor(f::DFGFactor)
 
 # #TODO check this one
 # function addData!(

src/DistributedFactorGraphs.jl

@@ -173,6 +173,7 @@ export getPPEDict, getVariablePPEDict, getVariablePPE
 
 # CRUD & SET
 export getPPE,
+       getPPEs,
        getVariablePPE,
        addPPE!,
        updatePPE!,
@@ -200,6 +201,7 @@ export hasBlobEntry,getBlobEntry,addBlobEntry!,updateBlobEntry!,deleteBlobEntry!
 export incrDataLabelSuffix
 
 export getBlobEntries, listDataEntries, hasDataEntry, hasDataEntry
+export getBlobEntriesVariables
 export listDataEntrySequence
 # convenience wrappers
 export mergeDataEntries!

src/FileDFG/services/FileDFG.jl

@@ -40,12 +40,12 @@ function saveDFG(folder::AbstractString, dfg::AbstractDFG)
     map(f -> rm("$varFolder/$f"), readdir(varFolder))
     map(f -> rm("$factorFolder/$f"), readdir(factorFolder))
     # Variables
-    for v in variables
+    @showprogress "saving variables" for v in variables
         vPacked = packVariable(v)
         JSON3.write("$varFolder/$(v.label).json", vPacked)
     end
     # Factors
-    for f in factors
+    @showprogress "saving factors" for f in factors
         fPacked = packFactor(dfg, f)
         JSON3.write("$factorFolder/$(f.label).json", fPacked)
     end
@@ -56,9 +56,9 @@ function saveDFG(folder::AbstractString, dfg::AbstractDFG)
     destfile = joinpath(savedir, savename*".tar.gz")
     # FIXME, switch to Tar.jl and Transcode Zlib / Codec, see #351
     if length(savedir) != 0
-      run( pipeline(`tar -zcf - -C $savedir $savename`, stdout="$destfile"))
+        run( pipeline(`tar -zcf - -C $savedir $savename`, stdout="$destfile"))
     else
-      run( pipeline(`tar -zcf - $savename`, stdout="$destfile"))
+        run( pipeline(`tar -zcf - $savename`, stdout="$destfile"))
     end
     Base.rm(joinpath(savedir,savename), recursive=true)
 end

src/entities/DFGFactor.jl

@@ -48,6 +48,12 @@ Base.@kwdef mutable struct GenericFunctionNodeData{T<:Union{<:AbstractPackedFact
     inflation::Float64 = 0.0
 end
 
+# TODO should we move non FactorOperationalMemory to DFGFactor: 
+# fnc, multihypo, nullhypo, inflation ?
+# that way we split solverData <: FactorOperationalMemory and constants
+# TODO see if above ever changes?
+
+
 ## Constructors
 
 
@@ -77,19 +83,18 @@ FunctionNodeData(args...; kw...) = FunctionNodeData{typeof(args[4])}(args...; kw
 # | DFGFactor         |   X   |   X  |     X     |     X    |      X     |
 
 # Packed Factor
-Base.@kwdef struct PackedFactor
-    # NOTE: This has to match the order of the JSON deserializer as we're using OrderedStructs.
-    id::Union{UUID, Nothing}
+Base.@kwdef struct PackedFactor <: AbstractDFGFactor
+    id::Union{UUID, Nothing} = nothing
     label::Symbol
     tags::Vector{Symbol}
     _variableOrderSymbols::Vector{Symbol}
     timestamp::ZonedDateTime
-    nstime::Int
+    nstime::String
     fnctype::String
     solvable::Int
     data::String
     metadata::String
-    _version::String
+    _version::String = string(_getDFGVersion())
   end
 
 StructTypes.StructType(::Type{PackedFactor}) = StructTypes.UnorderedStruct()
@@ -275,8 +280,8 @@ StructTypes.omitempties(::Type{SkeletonDFGFactor}) = (:id,)
 ##==============================================================================
 ## Define factor levels
 ##==============================================================================
-const FactorDataLevel0 = Union{DFGFactor, DFGFactorSummary, SkeletonDFGFactor}
-const FactorDataLevel1 = Union{DFGFactor, DFGFactorSummary}
+const FactorDataLevel0 = Union{DFGFactor, DFGFactorSummary, PackedFactor, SkeletonDFGFactor}
+const FactorDataLevel1 = Union{DFGFactor, DFGFactorSummary, PackedFactor}
 const FactorDataLevel2 = Union{DFGFactor}
 
 ##==============================================================================

src/entities/DFGVariable.jl

@@ -91,7 +91,7 @@ VariableNodeData(variableType::InferenceVariable; kwargs...) = VariableNodeData{
 
 """
 $(TYPEDEF)
-Packed VariabeNodeData structure for serializing DFGVariables.
+Packed VariableNodeData structure for serializing DFGVariables.
 
   ---
 Fields:
@@ -177,27 +177,54 @@ getEstimateFields(::MeanMaxPPE) = [:suggested, :max, :mean]
 ## DFG Variables
 ##==============================================================================
 
+
 export Variable
 
 """
     $(TYPEDEF)
 
 The Variable information packed in a way that accomdates multi-lang using json.
 """
-Base.@kwdef struct Variable
-    # NOTE: This has to match the order of the JSON deserializer as we're using OrderedStructs.
-    id::Union{UUID, Nothing}
+Base.@kwdef struct Variable <: AbstractDFGVariable
+    id::Union{UUID, Nothing} = nothing
     label::Symbol
-    tags::Vector{Symbol}
-    timestamp::ZonedDateTime
-    nstime::Int
-    ppes::Vector{MeanMaxPPE}
-    blobEntries::Vector{BlobEntry}
+    tags::Vector{Symbol} = Symbol[]
+    timestamp::ZonedDateTime = now(tz"UTC")
+    nstime::String = "0"
+    ppes::Vector{MeanMaxPPE} = MeanMaxPPE[]
+    blobEntries::Vector{BlobEntry} = BlobEntry[]
     variableType::String
-    _version::String
-    metadata::String
-    solvable::Int
-    solverData::Vector{PackedVariableNodeData}
+    _version::String = string(_getDFGVersion())
+    metadata::String = "e30="
+    solvable::Int = 1
+    solverData::Vector{PackedVariableNodeData} = PackedVariableNodeData[]
+end
+
+#IIF like contruction helper for packed variable
+function PackedVariable(
+    label::Symbol,
+    variableType::String;
+    tags::Vector{Symbol} = Symbol[],
+    timestamp::ZonedDateTime = now(tz"UTC"),
+    solvable::Int = 1,
+    nanosecondtime::Int64 = 0,
+    smalldata::Dict{Symbol, SmallDataTypes} = Dict{Symbol, SmallDataTypes}(),
+    kwargs...
+)
+    union!(tags, [:VARIABLE])
+
+    pacvar = PackedVariable(;
+        label,
+        variableType,
+        nstime = string(nanosecondtime),
+        solvable,
+        tags,
+        metadata = base64encode(JSON3.write(smalldata)),
+        timestamp,
+        kwargs...
+    )
+
+    return pacvar
 end
 const PackedVariable = Variable
 
@@ -273,7 +300,7 @@ DFGVariable(label::Symbol,
             solverDataDict::Dict{Symbol, VariableNodeData{T,P}}=Dict{Symbol, VariableNodeData{T,getPointType(T)}}(),
             kw...) where {T <: InferenceVariable, P} = DFGVariable(label, T; solverDataDict=solverDataDict, kw...)
 #
-@deprecate DFGVariable(label::Symbol, T_::Type{<:InferenceVariable},w...; timestamp::DateTime=now(),kw...) DFGVariable(label, T_, w...; timestamp=ZonedDateTime(timestamp), kw...)
+# @deprecate DFGVariable(label::Symbol, T_::Type{<:InferenceVariable},w...; timestamp::DateTime=now(),kw...) DFGVariable(label, T_, w...; timestamp=ZonedDateTime(timestamp), kw...)
 #
 
 
@@ -351,6 +378,27 @@ Base.@kwdef struct DFGVariableSummary <: AbstractDFGVariable
     dataDict::Dict{Symbol, BlobEntry}
 end
 
+function DFGVariableSummary(
+    id,
+    label,
+    timestamp,
+    tags,
+    ::Nothing,
+    variableTypeName,
+    ::Nothing,
+)
+    return DFGVariableSummary(
+        id,
+        label,
+        timestamp,
+        tags,
+        Dict{Symbol, MeanMaxPPE}(),
+        variableTypeName,
+        Dict{Symbol, BlobEntry}(),
+    )
+end
+
+StructTypes.names(::Type{DFGVariableSummary}) = ((:variableTypeName, :variableType),)
 
 ##------------------------------------------------------------------------------
 ## SkeletonDFGVariable.jl
@@ -384,8 +432,8 @@ StructTypes.omitempties(::Type{SkeletonDFGVariable}) = (:id,)
 ##==============================================================================
 # Define variable levels
 ##==============================================================================
-const VariableDataLevel0 = Union{DFGVariable, DFGVariableSummary, SkeletonDFGVariable}
-const VariableDataLevel1 = Union{DFGVariable, DFGVariableSummary}
+const VariableDataLevel0 = Union{DFGVariable, DFGVariableSummary, PackedVariable, SkeletonDFGVariable}
+const VariableDataLevel1 = Union{DFGVariable, DFGVariableSummary, PackedVariable}
 const VariableDataLevel2 = Union{DFGVariable}
 
 ##==============================================================================