Skip to content

Commit f145c5b

Browse files
AffieAffie
authored
Merge pull request #593 from JuliaRobotics/feat/20Q3/in_mem_store
add InMemoryBlobStore, more tests, and small bugfixes
2 parents ee1a256 + a6ec78f commit f145c5b

File tree

6 files changed

+168
-115
lines changed

6 files changed

+168
-115
lines changed

‎attic/InMemoryDataStore.jl

Lines changed: 0 additions & 61 deletions
This file was deleted.

‎src/CloudGraphsDFG/services/CloudGraphsDFG.jl

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -239,9 +239,9 @@ function getVariable(dfg::CloudGraphsDFG, label::Union{Symbol, String})
239239
for solveKey in listVariableSolverData(dfg, label)
240240
variable.solverDataDict[solveKey] = getVariableSolverData(dfg, label, solveKey)
241241
end
242-
dataDict = getDataEntries(dfg, label)
243-
for (k,v) in dataDict
244-
variable.dataDict[k] = v
242+
dataSet = getDataEntries(dfg, label)
243+
for v in dataSet
244+
variable.dataDict[v.label] = v
245245
end
246246

247247
return variable
@@ -258,7 +258,7 @@ function mergeVariableData!(dfg::CloudGraphsDFG, sourceVariable::DFGVariable; cu
258258
updateVariableSolverData!(dfg, getLabel(sourceVariable), v, currentTransaction=currentTransaction)
259259
end
260260
for (k,v) in sourceVariable.dataDict
261-
updateDatEntry!(dfg, getLabel(sourceVariable), v, currentTransaction=currentTransaction)
261+
updateDataEntry!(dfg, getLabel(sourceVariable), v, currentTransaction=currentTransaction)
262262
end
263263
return sourceVariable
264264
end
@@ -643,11 +643,9 @@ end
643643
function getDataEntries(dfg::CloudGraphsDFG, label::Symbol; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)
644644
entries = Dict{Symbol, BlobStoreEntry}()
645645
# TODO: Optimize if necessary.
646-
for key in listDataEntries(dfg, label, currentTransaction=currentTransaction)
647-
entry = getDataEntry(dfg, label, key, currentTransaction=currentTransaction)
648-
entries[entry.label] = entry
649-
end
650-
return entries
646+
delist = listDataEntries(dfg, label, currentTransaction=currentTransaction)
647+
return getDataEntry.(dfg, label, delist; currentTransaction=currentTransaction)
648+
651649
end
652650

653651
function listDataEntries(dfg::CloudGraphsDFG, label::Symbol; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)
@@ -665,6 +663,10 @@ function getDataEntry(dfg::CloudGraphsDFG, label::Symbol, key::Symbol; currentTr
665663
BlobStoreEntry,
666664
key;
667665
currentTransaction=currentTransaction)
666+
667+
#FIXME
668+
properties["createdTimestamp"] = DistributedFactorGraphs.getStandardZDTString(properties["createdTimestamp"])
669+
668670
return Unmarshal.unmarshal(
669671
BlobStoreEntry,
670672
properties)
@@ -681,6 +683,10 @@ function addDataEntry!(dfg::CloudGraphsDFG, label::Symbol, bde::BlobStoreEntry;
681683
bde,
682684
:DATA,
683685
currentTransaction=currentTransaction)
686+
687+
#FIXME
688+
packed["createdTimestamp"] = DistributedFactorGraphs.getStandardZDTString(packed["createdTimestamp"])
689+
684690
return Unmarshal.unmarshal(
685691
BlobStoreEntry,
686692
packed)
@@ -697,6 +703,10 @@ function updateDataEntry!(dfg::CloudGraphsDFG, label::Symbol, bde::BlobStoreEnt
697703
bde,
698704
:DATA,
699705
currentTransaction=currentTransaction)
706+
707+
#FIXME
708+
packed["createdTimestamp"] = DistributedFactorGraphs.getStandardZDTString(packed["createdTimestamp"])
709+
700710
return Unmarshal.unmarshal(
701711
BlobStoreEntry,
702712
packed)
@@ -710,6 +720,10 @@ function deleteDataEntry!(dfg::CloudGraphsDFG, label::Symbol, key::Symbol; curre
710720
_getLabelsForType(dfg, BlobStoreEntry, parentKey=label),
711721
key,
712722
currentTransaction=currentTransaction)
723+
724+
#FIXME
725+
props["createdTimestamp"] = DistributedFactorGraphs.getStandardZDTString(props["createdTimestamp"])
726+
713727
return Unmarshal.unmarshal(
714728
BlobStoreEntry,
715729
props)

‎src/DataBlobs/services/BlobStores.jl

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ addData!(dfg::AbstractDFG, blobstorekey::Symbol, label::Symbol, key::Symbol, blo
109109
kwargs...)
110110

111111
function addData!(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol, key::Symbol,
112-
blob::Vector{UInt8}, timestamp=now(localzone()); description="", mimeType = "", id::UUID = uuid4(), hashfunction = sha256)
112+
blob::Vector{UInt8}, timestamp=now(localzone()); description="", mimeType = "application/octet-stream", id::UUID = uuid4(), hashfunction = sha256)
113113

114114

115115
entry = BlobStoreEntry(key, id, blobstore.key, bytes2hex(hashfunction(blob)),
@@ -196,9 +196,43 @@ end
196196
function deleteDataBlob!(store::FolderStore{T}, entry::BlobStoreEntry) where T
197197
blobfilename = joinpath(store.folder,"$(entry.id).dat")
198198
entryfilename = joinpath(store.folder,"$(entry.id).json")
199-
199+
200200
data = getDataBlob(store, entry)
201201
rm(blobfilename)
202202
rm(entryfilename)
203203
return data
204204
end
205+
206+
##==============================================================================
207+
## InMemoryBlobStore
208+
##==============================================================================
209+
export InMemoryBlobStore
210+
struct InMemoryBlobStore{T} <: AbstractBlobStore{T}
211+
key::Symbol
212+
blobs::Dict{UUID, T}
213+
end
214+
215+
InMemoryBlobStore{T}(storeKey::Symbol) where T = InMemoryBlobStore{Vector{UInt8}}(storeKey, Dict{UUID, T}())
216+
InMemoryBlobStore(storeKey::Symbol=:default_inmemory_store) = InMemoryBlobStore{Vector{UInt8}}(storeKey)
217+
218+
function getDataBlob(store::InMemoryBlobStore{T}, entry::BlobStoreEntry) where T
219+
return store.blobs[entry.id]
220+
end
221+
222+
function addDataBlob!(store::InMemoryBlobStore{T}, entry::BlobStoreEntry, data::T) where T
223+
if haskey(store.blobs, entry.id)
224+
error("Key '$(entry.id)' blob already exists.")
225+
end
226+
return store.blobs[entry.id] = data
227+
end
228+
229+
function updateDataBlob!(store::InMemoryBlobStore{T}, entry::BlobStoreEntry, data::T) where T
230+
if haskey(store.blobs, entry.id)
231+
@warn "Key '$(entry.id)' doesn't exist."
232+
end
233+
return store.blobs[entry.id] = data
234+
end
235+
236+
function deleteDataBlob!(store::InMemoryBlobStore{T}, entry::BlobStoreEntry) where T
237+
return pop!(store.blobs, entry.id)
238+
end

‎test/consol_DataEntryBlobTests.jl

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,29 @@ dfs = FolderStore("/tmp/defaultfolderstore")
9797
@test dfs.key == :default_folder_store
9898
@test dfs isa FolderStore{Vector{UInt8}}
9999

100+
##==============================================================================
101+
## InMemoryBlobStore
102+
##==============================================================================
103+
104+
# Create a data store and add it to DFG
105+
ds = InMemoryBlobStore()
106+
addBlobStore!(dfg, ds)
107+
108+
ade,adb = addData!(dfg, :default_inmemory_store, :x1, :random, dataset1)
109+
gde,gdb = getData(dfg, :x1, :random)
110+
dde,ddb = deleteData!(dfg, :x1, :random)
111+
112+
@test ade == gde == dde
113+
@test adb == gdb == ddb
114+
115+
ade2,adb2 = addData!(dfg, :x2, deepcopy(ade), dataset1)
116+
# ade3,adb3 = updateData!(dfg, :x2, deepcopy(ade), dataset1)
117+
118+
@test ade == ade2# == ade3
119+
@test adb == adb2# == adb3
120+
121+
deleteData!(dfg, :x2, :random)
122+
100123
##==============================================================================
101124
## Unimplemented store
102125
##==============================================================================

‎test/interfaceTests.jl

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,10 @@ end
115115
@testset "Data Entries and Blobs" begin
116116
if typeof(fg1) <: InMemoryDFGTypes
117117
DataEntriesTestBlock!(fg1, var2)
118-
else
119-
@test_skip DataEntriesTestBlock!(fg1, var2)
120118
end
121-
122-
# New data blob API
123-
blobsTestBlock!(fg1, var2)
119+
@testset "Data blob tests" begin
120+
blobsStoresTestBlock!(fg1)
121+
end
124122
end
125123

126124
@testset "TODO Sorteer groep" begin

‎test/testBlocks.jl

Lines changed: 83 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -865,46 +865,91 @@ function DataEntriesTestBlock!(fg, v2)
865865
#delete from dfg
866866
@test deleteDataEntry!(fg, :a, :key2) == de2_update
867867
@test listDataEntries(v1) == Symbol[]
868+
deleteDataEntry!(fg, :b, :key2)
868869
end
869870

870-
function blobsTestBlock!(fg, v1)
871-
@testset "Data blob tests" begin
872-
# Blobstore functions
873-
fs = FolderStore("/tmp/$(string(uuid4())[1:8])")
874-
# Adding
875-
addBlobStore!(fg, fs)
876-
# Listing
877-
@test listBlobStores(fg) == [fs.key]
878-
# Getting
879-
@test getBlobStore(fg, fs.key) == fs
880-
# Deleting
881-
@test deleteBlobStore!(fg, fs.key) == fs
882-
# Updating
883-
updateBlobStore!(fg, fs)
884-
@test listBlobStores(fg) == [fs.key]
885-
# Emptying
886-
emptyBlobStore!(fg)
887-
@test listBlobStores(fg) == []
888-
# Add it back
889-
addBlobStore!(fg, fs)
890-
891-
# Data functions
892-
testData = rand(UInt8, 50)
893-
# Adding
894-
newData = addData!(fg, fs.key, getLabel(v1), :testing, testData)
895-
# Listing
896-
@test :testing in listDataEntries(fg, getLabel(v1))
897-
# Getting
898-
data = getData(fg, fs, getLabel(v1), :testing)
899-
@test data[1].hash == newData[1].hash
900-
@test data[2] == newData[2]
901-
# Updating
902-
updateData = updateData!(fg, fs, getLabel(v1), newData[1], rand(UInt8, 50))
903-
@test updateData[1].hash != data[1].hash
904-
@test updateData[2] != data[2]
905-
# Deleting
906-
retData = deleteData!(fg, getLabel(v1), :testing)
907-
end
871+
function blobsStoresTestBlock!(fg)
872+
873+
de1 = BlobStoreEntry(:label1,uuid4(), :store1, "AAAA","origin1","description1","mimetype1",now(localzone()))
874+
de2 = BlobStoreEntry(:label2,uuid4(), :store2, "FFFF","origin2","description2","mimetype2",ZonedDateTime("2020-08-12T12:00:00.000+00:00"))
875+
de2_update = BlobStoreEntry(:label2,uuid4(), :store2, "0123","origin2","description2","mimetype2",ZonedDateTime("2020-08-12T12:00:01.000+00:00"))
876+
@test getLabel(de1) == de1.label
877+
@test getId(de1) == de1.id
878+
@test getHash(de1) == hex2bytes(de1.hash)
879+
@test getCreatedTimestamp(de1) == de1.createdTimestamp
880+
881+
#add
882+
var1 = getVariable(fg, :a)
883+
@test addDataEntry!(var1, de1) == de1
884+
updateVariable!(fg, var1)
885+
@test addDataEntry!(fg, :a, de2) == de2
886+
@test_throws ErrorException addDataEntry!(var1, de1)
887+
@test de2 in getDataEntries(fg, var1.label)
888+
889+
#get
890+
@test deepcopy(de1) == getDataEntry(var1, :label1)
891+
@test deepcopy(de2) == getDataEntry(fg, :a, :label2)
892+
@test_throws ErrorException getDataEntry(v2, :label1)
893+
@test_throws ErrorException getDataEntry(fg, :b, :label1)
894+
895+
#update
896+
@test updateDataEntry!(fg, :a, de2_update) == de2_update
897+
@test deepcopy(de2_update) == getDataEntry(fg, :a, :label2)
898+
@test @test_logs (:warn, r"does not exist") updateDataEntry!(fg, :b, de2_update) == de2_update
899+
900+
#list
901+
entries = getDataEntries(fg, :a)
902+
@test length(entries) == 2
903+
@test issetequal(map(e->e.label, entries), [:label1, :label2])
904+
@test length(getDataEntries(fg, :b)) == 1
905+
906+
@test issetequal(listDataEntries(fg, :a), [:label1, :label2])
907+
@test listDataEntries(fg, :b) == Symbol[:label2]
908+
909+
#delete
910+
@test deleteDataEntry!(fg, var1.label, de1.label) == de1
911+
@test listDataEntries(fg, var1.label) == Symbol[:label2]
912+
#delete from dfg
913+
@test deleteDataEntry!(fg, :a, :label2) == de2_update
914+
var1 = getVariable(fg, :a)
915+
@test listDataEntries(var1) == Symbol[]
916+
917+
# Blobstore functions
918+
fs = FolderStore("/tmp/$(string(uuid4())[1:8])")
919+
# Adding
920+
addBlobStore!(fg, fs)
921+
# Listing
922+
@test listBlobStores(fg) == [fs.key]
923+
# Getting
924+
@test getBlobStore(fg, fs.key) == fs
925+
# Deleting
926+
@test deleteBlobStore!(fg, fs.key) == fs
927+
# Updating
928+
updateBlobStore!(fg, fs)
929+
@test listBlobStores(fg) == [fs.key]
930+
# Emptying
931+
emptyBlobStore!(fg)
932+
@test listBlobStores(fg) == []
933+
# Add it back
934+
addBlobStore!(fg, fs)
935+
936+
# Data functions
937+
testData = rand(UInt8, 50)
938+
# Adding
939+
newData = addData!(fg, fs.key, :a, :testing, testData)
940+
# Listing
941+
@test :testing in listDataEntries(fg, :a)
942+
# Getting
943+
data = getData(fg, fs, :a, :testing)
944+
@test data[1].hash == newData[1].hash
945+
@test data[2] == newData[2]
946+
# Updating
947+
updateData = updateData!(fg, fs, :a, newData[1], rand(UInt8, 50))
948+
@test updateData[1].hash != data[1].hash
949+
@test updateData[2] != data[2]
950+
# Deleting
951+
retData = deleteData!(fg, :a, :testing)
952+
908953
end
909954

910955

0 commit comments

Comments
 (0)