Merge master into 22Q3/perf/var_point_param

Author: Affie

NEWS.md

@@ -2,6 +2,9 @@ Listing news on any major breaking changes in DFG.  For regular changes, see int
 
 # v0.20
 
+- Throw `KeyError` if `getBlobEntry` is not found, previously was `ErrorException`.
+- Change return type on `addData!` convenience wrappers to only return new `BlobEntry`.
+- Fix `addBlob!` calls for `FolderStore` and `InMemoryBlobStore` to use `BlobEntry.originId` and not previous bug `entry.id`.
 - Close long running serialization redo (#590) using only JSON3.jl and StructTypes.jl going forward.
 - Standardize BlobEntry=>Blob naming of functions, and keeping convenience wrappers `{get,add,update,delete}Data[!]`.
 - Consolidate to only one `BlobEntry` definition, dropping use of `AbstractBlobEntry`.

Project.toml

@@ -1,9 +1,10 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.20.0"
+version = "0.20.2"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"

attic/DataBlobs/FileDataEntryBlob.jl

@@ -42,7 +42,9 @@ function addBlob!(dfg::AbstractDFG, entry::BlobEntry, data::Vector{UInt8})
         open(entryfilename(entry), "w") do f
             JSON.print(f, entry)
         end
-        return getBlob(dfg, entry)::Vector{UInt8}
+        # FIXME update for entry.blobId vs entry.originId
+        return UUID(entry.id)
+        # return getBlob(dfg, entry)::Vector{UInt8}
     end
 end
 
@@ -72,7 +74,8 @@ end
 function addData!(::Type{BlobEntry}, dfg::AbstractDFG, label::Symbol, key::Symbol, folder::String, blob::Vector{UInt8}, timestamp=now(localzone());
                   id::UUID = uuid4(), hashfunction = sha256)
     fde = BlobEntry(key, id, folder, bytes2hex(hashfunction(blob)), timestamp)
-    de = addBlobEntry!(dfg, label, fde)
-    db = addBlob!(dfg, fde, blob)
-    return de=>db
+    blobId = addBlob!(dfg, fde, blob) |> UUID
+    newEntry = BlobEntry(fde; id=blobId, blobId)
+    de = addBlobEntry!(dfg, label, newEntry)
+    return de # de=>db
 end

docs/extra/CoreAPI.fods

@@ -26,7 +26,7 @@ prior = addFactor!(dfg, [:x0], PriorPose2( MvNormal([10; 10; 1.0/8.0], Matrix(Di
 
 # Now serialize them:
 pVariable = packVariable(dfg, x0)
-pFactor = packFactor(dfg, prior)
+pFactor = packFactor(prior)
 
 # And we can deserialize them
 upVariable = unpackVariable(dfg, pVariable)
@@ -60,7 +60,7 @@ for v in getVariables(dfg)
 end
 # Factors
 for f in getFactors(dfg)
-    fPacked = packFactor(dfg, f)
+    fPacked = packFactor(f)
     io = open("$folder/factors/$(f.label).json", "w")
     JSON3.write(io, fPacked)
     close(io)

docs/src/variable_factor_serialization.md

@@ -30,6 +30,4 @@ StructTypes.StructType(::Type{BlobEntry}) = StructTypes.UnorderedStruct()
 StructTypes.idproperty(::Type{BlobEntry}) = :id
 StructTypes.omitempties(::Type{BlobEntry}) = (:id,)
 
-
-
 _fixtimezone(cts::NamedTuple) = ZonedDateTime(cts.utc_datetime*"+00")

src/DataBlobs/DataBlobs.jl

@@ -46,7 +46,7 @@ function assertHash(de::BlobEntry, db; hashfunction::Function = sha256)
 end
 
 
-function Base.show(io::IO, entry::BlobEntry) 
+function Base.show(io::IO, ::MIME"text/plain", entry::BlobEntry)
     println(io, "_type=BlobEntry {")
     println(io, "  id:            ", entry.id) 
     println(io, "  blobId:        ", entry.blobId) 
@@ -62,10 +62,6 @@ function Base.show(io::IO, entry::BlobEntry)
     println(io, "}") 
 end 
 
-Base.show(io::IO, ::MIME"text/plain", entry::BlobEntry) = show(io, entry)
-
-
-
 ##==============================================================================
 ## BlobEntry - CRUD
 ##==============================================================================
@@ -77,13 +73,15 @@ Get data entry
 Also see: [`addBlobEntry`](@ref), [`getBlob`](@ref), [`listBlobEntries`](@ref)
 """
 function getBlobEntry(var::AbstractDFGVariable, key::Symbol)
-    !hasBlobEntry(var, key) && error("No dataEntry label $(key) found in variable $(getLabel(var))")
+    if !hasBlobEntry(var, key)
+        throw(KeyError("No dataEntry label $(key) found in variable $(getLabel(var)). Available keys: $(keys(var.dataDict))"))
+    end
     return var.dataDict[key]
 end
 
 function getBlobEntry(var::AbstractDFGVariable, blobId::UUID)
     for (k,v) in var.dataDict
-        if v.id == blobId
+        if blobId in [v.originId, v.blobId]
             return v
         end
     end
@@ -92,9 +90,15 @@ function getBlobEntry(var::AbstractDFGVariable, blobId::UUID)
     )
 end
 
-#TODO consider changing this to use `first` as in julia's findfirst - as in findfirstBlobEntry->label::Symbol
-# or getBlobEntryFirst, btw this returns a vector in some other places
-function getBlobEntry(var::AbstractDFGVariable, key::Regex)
+@deprecate getBlobEntry(var::AbstractDFGVariable, key::Regex) getBlobEntryFirst(var, key)
+
+"""
+    $(SIGNATURES)
+Finds and returns the first blob entry that matches the regex.
+
+Also see: [`getBlobEntry`](@ref)
+"""
+function getBlobEntryFirst(var::AbstractDFGVariable, key::Regex)
     for (k,v) in var.dataDict
         if occursin(key, string(v.label))
             return v
@@ -107,7 +111,7 @@ end
 
 getBlobEntry(var::AbstractDFGVariable, key::AbstractString) = getBlobEntry(var,Regex(key))
 
-
+#TODO split betweeen getfirstBlobEntry and getBlobEntry
 getBlobEntry(dfg::AbstractDFG, label::Symbol, key::Union{Symbol, UUID, <:AbstractString, Regex}) = getBlobEntry(getVariable(dfg, label), key)
 # getBlobEntry(dfg::AbstractDFG, label::Symbol, key::Symbol) = getBlobEntry(getVariable(dfg, label), key)
 
@@ -117,16 +121,35 @@ getBlobEntry(dfg::AbstractDFG, label::Symbol, key::Union{Symbol, UUID, <:Abstrac
 Add Data Entry to a DFG variable
 Should be extended if DFG variable is not returned by reference.
 
-Also see: [`getBlobEntry`](@ref), [`addBlob`](@ref), [`mergeBlobEntries!`](@ref)
+Also see: [`getBlobEntry`](@ref), [`addBlob!`](@ref), [`mergeBlobEntries!`](@ref)
 """
-function addBlobEntry!(var::AbstractDFGVariable, bde::BlobEntry)
-    haskey(var.dataDict, bde.label) && error("blobEntry $(bde.label) already exists on variable $(getLabel(var))")
-    var.dataDict[bde.label] = bde
-    return bde
+function addBlobEntry!(
+    var::AbstractDFGVariable, 
+    entry::BlobEntry;
+)
+    # see https://github.com/JuliaRobotics/DistributedFactorGraphs.jl/issues/985
+    # blobId::Union{UUID,Nothing} = (isnothing(entry.blobId) ? entry.id : entry.blobId),
+    # blobSize::Int = (hasfield(BlobEntry, :size) ? entry.size : -1)
+    haskey(var.dataDict, entry.label) && error("blobEntry $(entry.label) already exists on variable $(getLabel(var))")
+    var.dataDict[entry.label] = entry
+    return entry
 end
 
-function addBlobEntry!(dfg::AbstractDFG, label::Symbol, bde::BlobEntry)
-    return addBlobEntry!(getVariable(dfg, label), bde)
+function addBlobEntry!(
+    var::PackedVariable, 
+    entry::BlobEntry,
+)   
+    entry.label in getproperty.(var.blobEntries,:label) && error("blobEntry $(entry.label) already exists on variable $(getLabel(var))")
+    push!(var.blobEntries, entry)
+    return entry
+end
+
+function addBlobEntry!(
+    dfg::AbstractDFG, 
+    vLbl::Symbol, 
+    entry::BlobEntry;
+)
+    return addBlobEntry!(getVariable(dfg, vLbl), entry)
 end
 
 
@@ -186,18 +209,62 @@ hasBlobEntry(var::AbstractDFGVariable, blobLabel::Symbol) = haskey(var.dataDict,
 
 """
     $(SIGNATURES)
-Get data entries, Vector{BlobEntry}
+
+Get blob entries, Vector{BlobEntry}
 """
 function getBlobEntries(var::AbstractDFGVariable)
     #or should we return the iterator, Base.ValueIterator{Dict{Symbol,BlobEntry}}?
     collect(values(var.dataDict))
 end
+
+function getBlobEntries(var::PackedVariable)
+    var.blobEntries
+end
+
 function getBlobEntries(dfg::AbstractDFG, label::Symbol)
     # !isVariable(dfg, label) && return nothing
     #or should we return the iterator, Base.ValueIterator{Dict{Symbol,BlobEntry}}?
     getBlobEntries(getVariable(dfg, label))
 end
 
+function getBlobEntries(dfg::AbstractDFG, label::Symbol, regex::Regex)
+    entries = getBlobEntries(dfg, label)
+    return filter(entries) do e
+        occursin(regex, string(e.label))
+    end        
+end
+
+
+"""
+    $(SIGNATURES)
+
+Get all blob entries matching a Regex pattern over variables
+
+Notes
+- Use `dropEmpties=true` to not include empty lists in result.
+- Use keyword `varList` for which variables to search through.
+"""
+function getBlobEntriesVariables(
+    dfg::AbstractDFG,
+    bLblPattern::Regex;
+    varList::AbstractVector{Symbol} = sort(listVariables(dfg); lt=natural_lt),
+    dropEmpties::Bool = false
+)
+    RETLIST = Vector{Vector{BlobEntry}}()
+    @showprogress "Get entries matching $bLblPattern" for vl in varList
+        bes = filter(
+            s->occursin(bLblPattern,string(s.label)),
+            listBlobEntries(dfg,vl)
+        )
+        # only push to list if there are entries on this variable
+        (!dropEmpties || 0 < length(bes)) ? nothing : continue
+        push!(RETLIST, bes)
+    end
+    
+    return RETLIST
+end
+
+
 """
     $(SIGNATURES)
 List the blob entries associated with a particular variable.