Merge pull request #912 from JuliaRobotics/master

v0.18.8-rc1

Author: dehann

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.18.7"
+version = "0.18.8"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

src/DataBlobs/services/AbstractDataEntries.jl

@@ -214,5 +214,7 @@ function mergeDataEntries!(
     dde = listDataEntries(dst, dlbl)
     uids = (s->s.id).(dde)
     filter!(s -> !(s.id in uids), des)
-    union(((s->mergeDataEntries!(dst, dlbl, src, slbl, s.id)).(des))...)
+    if 0  < length(des)
+        union(((s->mergeDataEntries!(dst, dlbl, src, slbl, s.id)).(des))...)
+    end
 end

src/DataBlobs/services/BlobStores.jl

@@ -2,8 +2,26 @@
 ## AbstractBlobStore CRUD Interface
 ##==============================================================================
 
-getDataBlob(dfg::AbstractDFG, entry::BlobStoreEntry) =
-        getDataBlob(getBlobStore(dfg, entry.blobstore), entry)
+function getDataBlob(dfg::AbstractDFG, entry::BlobStoreEntry)
+    # cannot use entry.blobstore because the blob can be in any one of the blobstores
+    stores = getBlobStores(dfg)
+    blob = UInt8[]
+    for (k,store) in stores
+        try
+            blob = getDataBlob(store, entry)
+            return blob
+        catch err
+            if !(err isa KeyError)
+                throw(err)
+            end
+        end
+    end
+    throw(
+        KeyError(
+            "could not find $(entry.label), uuid $(entry.id)) in any of the listed blobstores:\n $((s->getKey(s)).(stores))"
+        )
+    )
+end
 
 function getDataBlob(store::AbstractBlobStore, entry::BlobStoreEntry)
     error("$(typeof(store)) doesn't override 'getDataBlob'.")
@@ -68,16 +86,17 @@ function getData(
     blobstore::AbstractBlobStore, 
     label::Symbol, 
     key::Union{Symbol,UUID}; 
-    hashfunction = sha256
+    hashfunction = sha256,
+    checkhash::Bool=true
 )
     de = getDataEntry(dfg, label, key)
     db = getDataBlob(blobstore, de)
-    assertHash(de, db, hashfunction=hashfunction)
+    checkhash && assertHash(de, db; hashfunction)
     return de=>db
 end
 
 function addData!(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol, entry::AbstractDataEntry, blob::Vector{UInt8}; hashfunction = sha256)
-    assertHash(entry, blob, hashfunction=hashfunction)
+    assertHash(entry, blob; hashfunction)
     de = addDataEntry!(dfg, label, entry)
     db = addDataBlob!(blobstore, de, blob)
     return de=>db
@@ -122,7 +141,7 @@ function addData!(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol,
                            "$(dfg.userId)|$(dfg.robotId)|$(dfg.sessionId)|$(label)",
                            description, mimeType, timestamp)
 
-    addData!(dfg, blobstore, label, entry, blob; hashfunction = hashfunction)
+    addData!(dfg, blobstore, label, entry, blob; hashfunction)
 end
 
 
@@ -157,7 +176,7 @@ function getDataBlob(store::FolderStore{T}, entry::BlobStoreEntry) where T
             return read(f)
         end
     else
-        error("Could not find file '$(blobfilename)'.")
+        throw(KeyError("Could not find file '$(blobfilename)'."))
         # return nothing
     end
 end

src/DataBlobs/services/DataEntryBlob.jl

@@ -115,30 +115,52 @@ function getData(
     dfg::AbstractDFG, 
     vlabel::Symbol, 
     key::Union{Symbol,UUID}; 
-    hashfunction = sha256
+    hashfunction = sha256,
+    checkhash::Bool=true
 )
-    de = getDataEntry(dfg, vlabel, key)
+    de_ = getDataEntry(dfg, vlabel, key)
+    _first(s) = s
+    _first(s::AbstractVector) = s[1]
+    de = _first(de_)
     db = getDataBlob(dfg, de)
 
-    assertHash(de, db, hashfunction=hashfunction)
+    checkhash && assertHash(de, db, hashfunction=hashfunction)
     return de=>db
 end
 
-function addData!(dfg::AbstractDFG, label::Symbol, entry::AbstractDataEntry, blob::Vector{UInt8}; hashfunction = sha256)
-    assertHash(entry, blob, hashfunction=hashfunction)
+function addData!(
+    dfg::AbstractDFG, 
+    label::Symbol, 
+    entry::AbstractDataEntry, 
+    blob::Vector{UInt8}; 
+    hashfunction = sha256,
+    checkhash::Bool=true
+)
+    checkhash && assertHash(entry, blob, hashfunction=hashfunction)
     de = addDataEntry!(dfg, label, entry)
     db = addDataBlob!(dfg, de, blob)
     return de=>db
 end
 
-function updateData!(dfg::AbstractDFG, label::Symbol,  entry::AbstractDataEntry,  blob::Vector{UInt8}; hashfunction = sha256)
-    assertHash(entry, blob, hashfunction=hashfunction)
+function updateData!(
+    dfg::AbstractDFG, 
+    label::Symbol, 
+    entry::AbstractDataEntry, 
+    blob::Vector{UInt8};
+    hashfunction = sha256,
+    checkhash::Bool=true
+)
+    checkhash && assertHash(entry, blob, hashfunction=hashfunction)
     de = updateDataEntry!(dfg, label, entry)
     db = updateDataBlob!(dfg, de, blob)
     return de=>db
 end
 
-function deleteData!(dfg::AbstractDFG, label::Symbol, key::Symbol)
+function deleteData!(
+    dfg::AbstractDFG, 
+    label::Symbol, 
+    key::Symbol
+)
     de = deleteDataEntry!(dfg, label, key)
     db = deleteDataBlob!(dfg, de)
     return de=>db

src/DataBlobs/services/FileDataEntryBlob.jl

@@ -25,7 +25,7 @@ function getDataBlob(dfg::AbstractDFG, entry::FileDataEntry)
             return read(f)
         end
     else
-        error("Could not find file '$(blobfilename(entry))'.")
+        throw(KeyError("Could not find file '$(blobfilename(entry))'."))
         # return nothing
     end
 end

src/GraphsDFG/entities/GraphsDFG.jl

@@ -18,7 +18,7 @@ mutable struct GraphsDFG{T <: AbstractParams, V <: AbstractDFGVariable, F <:Abst
     sessionData::Dict{Symbol, String}
     addHistory::Vector{Symbol} #TODO: Discuss more - is this an audit trail?
     solverParams::T # Solver parameters
-    blobStores::Dict{Symbol, AbstractBlobStore}
+    blobStores::Dict{Symbol, <:AbstractBlobStore}
 end
 
 """
@@ -63,7 +63,7 @@ end
 
 function GraphsDFG(g::FactorGraph{Int,DFGVariable,DFGFactor}=FactorGraph{Int,DFGVariable,DFGFactor}();
                   solverParams::T=NoSolverParams(), kwargs...) where T
-    return GraphsDFG{T,DFGVariable,DFGFactor}(g; solverParams=solverParams, kwargs...)
+    return GraphsDFG{T,DFGVariable,DFGFactor}(g; solverParams, kwargs...)
 end
 
 

src/Neo4jDFG/services/CommonFunctions.jl

@@ -300,7 +300,7 @@ function _matchmergeVariableSubnode!(
         relationshipKey::Symbol;
         addProps::Dict{String, String}=Dict{String, String}(),
         currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing) where
-        {N <: DFGNode, APPE <: AbstractPointParametricEst, ABDE <: AbstractDataEntry, PVND <: PackedVariableNodeData}
+        {APPE <: AbstractPointParametricEst, ABDE <: AbstractDataEntry, PVND <: PackedVariableNodeData}
 
     # Defensively wrap the node labels.
     nodeLabels = [!startswith(n, "`") && !endswith(n, "`") ? "`$(n)`" : n for n in nodeLabels]    

src/services/AbstractDFG.jl

@@ -191,6 +191,7 @@ abstract type AbstractBlobStore{T} end
 # AbstractBlobStore should have key or overwrite getKey
 getKey(store::AbstractBlobStore) = store.key
 
+getBlobStores(dfg::AbstractDFG) = dfg.blobStores
 getBlobStore(dfg::AbstractDFG, key::Symbol) = dfg.blobStores[key]
 addBlobStore!(dfg::AbstractDFG, bs::AbstractBlobStore) = push!(dfg.blobStores, getKey(bs)=>bs)
 updateBlobStore!(dfg::AbstractDFG, bs::AbstractBlobStore) = push!(dfg.blobStores, getKey(bs)=>bs)

src/services/DFGVariable.jl

@@ -417,7 +417,7 @@ Retrieve solver data structure stored in a variable.
 """
 function getSolverData(v::DFGVariable, key::Symbol=:default)
     #TODO this does not fit in with some of the other error behaviour. but its used so added @error
-    vnd =  haskey(v.solverDataDict, key) ? v.solverDataDict[key] : (@error "Variable does not have solver data $(key)"; nothing)
+    vnd =  haskey(v.solverDataDict, key) ? v.solverDataDict[key] : (@error "Variable $(getLabel(v)) does not have solver data $(key)"; nothing)
     return vnd
 end
 

src/services/Serialization.jl

@@ -193,16 +193,18 @@ function _unpackVariableNodeData(
 end
 
 # returns a DFGVariable
-function unpackVariable(dfg::AbstractDFG,
-                        packedProps::Dict{String, Any};
-                        unpackPPEs::Bool=true,
-                        unpackSolverData::Bool=true,
-                        unpackBigData::Bool = haskey(packedProps,"dataEntryType") && haskey(packedProps, "dataEntry")
-    )
+function unpackVariable(
+    dfg::AbstractDFG,
+    packedProps::Dict{String, Any};
+    unpackPPEs::Bool=true,
+    unpackSolverData::Bool=true,
+    unpackBigData::Bool = haskey(packedProps,"dataEntryType") && haskey(packedProps, "dataEntry"),
+    skipVersionCheck::Bool=false,
+)
     #
     @debug "Unpacking variable:\r\n$packedProps"
     # Version checking.
-    _versionCheck(packedProps)
+    !skipVersionCheck && _versionCheck(packedProps)
     label = Symbol(packedProps["label"])
     # Make sure that the timestamp is correctly formatted with subseconds
     packedProps["timestamp"] = getStandardZDTString(packedProps["timestamp"])