Skip to content

Commit 559fd62

Browse files
AffieAffie
committed
add blobStores to LightDFG and tests
1 parent 469bd5c commit 559fd62

File tree

8 files changed

+86
-52
lines changed

8 files changed

+86
-52
lines changed

‎src/DataBlobs/DataBlobStores.jl

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
export BlobStoreEntry
2+
23
"""
34
$(TYPEDEF)
45
Genaral Data Store Entry.
@@ -14,30 +15,35 @@ struct BlobStoreEntry <: AbstractDataEntry
1415
createdTimestamp::ZonedDateTime # of when the entry was created
1516
end
1617

17-
abstract type AbstractBlobStore{T} end
18-
19-
# AbstractBlobStore should have key or overwrite getKey
20-
getKey(store::AbstractBlobStore) = store.key
21-
22-
#TODO consider adding this to dfg
23-
const BlobStores = Dict{Symbol, AbstractBlobStore}
24-
2518

2619
##==============================================================================
2720
## AbstractBlobStore CRUD Interface
2821
##==============================================================================
22+
23+
getDataBlob(dfg::AbstractDFG, entry::BlobStoreEntry) =
24+
getDataBlob(getBlobStore(dfg, entry.blobstore), entry)
25+
2926
function getDataBlob(store::AbstractBlobStore, entry::BlobStoreEntry)
3027
error("$(typeof(store)) doesn't override 'getDataBlob'.")
3128
end
3229

30+
addDataBlob!(dfg::AbstractDFG, entry::BlobStoreEntry, data::T) where T =
31+
addDataBlob!(getBlobStore(dfg, entry.blobstore), entry, data)
32+
3333
function addDataBlob!(store::AbstractBlobStore{T}, entry::BlobStoreEntry, data::T) where T
3434
error("$(typeof(store)) doesn't override 'addDataBlob!'.")
3535
end
3636

37+
updateDataBlob!(dfg::AbstractDFG, entry::BlobStoreEntry, data::T) where T =
38+
updateDataBlob!(getBlobStore(dfg, entry.blobstore), entry, data)
39+
3740
function updateDataBlob!(store::AbstractBlobStore{T}, entry::BlobStoreEntry, data::T) where T
3841
error("$(typeof(store)) doesn't override 'updateDataBlob!'.")
3942
end
4043

44+
deleteDataBlob!(dfg::AbstractDFG, entry::BlobStoreEntry) =
45+
deleteDataBlob!(getBlobStore(dfg, entry.blobstore), entry)
46+
4147
function deleteDataBlob!(store::AbstractBlobStore, entry::BlobStoreEntry)
4248
error("$(typeof(store)) doesn't override 'deleteDataBlob!'.")
4349
end
@@ -80,13 +86,23 @@ function deleteData!(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symb
8086
return de=>db
8187
end
8288

89+
##==============================================================================
90+
91+
addData!(dfg::AbstractDFG, blobstorekey::Symbol, label::Symbol, key::Symbol, blob::Vector{UInt8},
92+
timestamp=now(localzone()); kwargs...) = addData!(dfg,
93+
getBlobStore(dfg, blobstorekey),
94+
label,
95+
key,
96+
blob,
97+
timestamp;
98+
kwargs...)
8399

84100
function addData!(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol, key::Symbol,
85101
blob::Vector{UInt8}, timestamp=now(localzone()); description="", mimeType = "", id::UUID = uuid4(), hashfunction = sha256)
86102

87103

88104
entry = BlobStoreEntry(key, id, blobstore.key, bytes2hex(hashfunction(blob)),
89-
"$(dfg.userId)|$(dfg.robotId)|$(dfg.sessionId)|$(dfg.label)",
105+
"$(dfg.userId)|$(dfg.robotId)|$(dfg.sessionId)|$(label)",
90106
description, mimeType, timestamp)
91107

92108
addData!(dfg, blobstore, label, entry, blob; hashfunction = hashfunction)

‎src/DataBlobs/DataEntryBlob.jl

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,23 +17,23 @@ end
1717
# deleteDataBlob!(dfg::AbstractDFG, entry::AbstractDataEntry)
1818

1919
function getDataBlob(dfg::AbstractDFG, entry::AbstractDataEntry)
20-
error("$(typeof(store)) doesn't override 'getDataBlob'.")
20+
error("$(typeof(dfg)) doesn't override 'getDataBlob'.")
2121
end
2222

23-
function addDataBlob!(dfg::AbstractDFG, entry::BlobStoreEntry, data::T) where T
24-
error("$(typeof(store)) doesn't override 'addDataBlob!'.")
23+
function addDataBlob!(dfg::AbstractDFG, entry::AbstractDataEntry, data::T) where T
24+
error("$(typeof(dfg)) doesn't override 'addDataBlob!'.")
2525
end
2626

27-
function updateDataBlob!(dfg::AbstractDFG, entry::BlobStoreEntry, data::T) where T
28-
error("$(typeof(store)) doesn't override 'updateDataBlob!'.")
27+
function updateDataBlob!(dfg::AbstractDFG, entry::AbstractDataEntry, data::T) where T
28+
error("$(typeof(dfg)) doesn't override 'updateDataBlob!'.")
2929
end
3030

31-
function deleteDataBlob!(dfg::AbstractDFG, entry::BlobStoreEntry)
32-
error("$(typeof(store)) doesn't override 'deleteDataBlob!'.")
31+
function deleteDataBlob!(dfg::AbstractDFG, entry::AbstractDataEntry)
32+
error("$(typeof(dfg)) doesn't override 'deleteDataBlob!'.")
3333
end
3434

3535
function listDataBlobs(dfg::AbstractDFG)
36-
error("$(typeof(store)) doesn't override 'listDataBlobs'.")
36+
error("$(typeof(dfg)) doesn't override 'listDataBlobs'.")
3737
end
3838

3939
##==============================================================================
@@ -55,7 +55,7 @@ function addData!(dfg::AbstractDFG, label::Symbol, entry::AbstractDataEntry, blo
5555
return de=>db
5656
end
5757

58-
function updateData!(dfg::AbstractDFG, label::Symbol, entry::AbstractDataEntry, blob::Vector{UInt8})
58+
function updateData!(dfg::AbstractDFG, label::Symbol, entry::AbstractDataEntry, blob::Vector{UInt8}; hashfunction = sha256)
5959
assertHash(entry, blob, hashfunction=hashfunction)
6060
de = updateDataEntry!(dfg, label, entry)
6161
db = updateDataBlob!(dfg, de, blob)

‎src/DistributedFactorGraphs.jl

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ export DFG
3939
##------------------------------------------------------------------------------
4040
export AbstractDFG
4141
export AbstractParams, NoSolverParams
42+
export AbstractBlobStore
4243

4344
# accessors & crud
4445
export getUserId, getRobotId, getSessionId
@@ -49,6 +50,11 @@ export getDescription, setDescription!,
4950
getRobotData, setRobotData!,
5051
getSessionData, setSessionData!,
5152
getAddHistory
53+
export getBlobStore,
54+
addBlobStore!,
55+
updateBlobStore!,
56+
deleteBlobStore!,
57+
emptyBlobStore!
5258

5359
# TODO Not sure these are needed or should work everywhere, implement in cloud?
5460
export updateUserData!, updateRobotData!, updateSessionData!, deleteUserData!, deleteRobotData!, deleteSessionData!

‎src/LightDFG/entities/LightDFG.jl

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ mutable struct LightDFG{T <: AbstractParams, V <: AbstractDFGVariable, F <:Abstr
1818
sessionData::Dict{Symbol, String}
1919
addHistory::Vector{Symbol} #TODO: Discuss more - is this an audit trail?
2020
solverParams::T # Solver parameters
21+
blobStores::Dict{Symbol, AbstractBlobStore}
2122
end
2223

2324
"""
@@ -36,12 +37,13 @@ function LightDFG{T,V,F}(g::FactorGraph{Int,V,F}=FactorGraph{Int,V,F}();
3637
userData::Dict{Symbol, String} = Dict{Symbol, String}(),
3738
robotData::Dict{Symbol, String} = Dict{Symbol, String}(),
3839
sessionData::Dict{Symbol, String} = Dict{Symbol, String}(),
39-
solverParams::T=T()) where {T <: AbstractParams, V <:AbstractDFGVariable, F<:AbstractDFGFactor}
40+
solverParams::T=T(),
41+
blobstores=Dict{Symbol, AbstractBlobStore}()) where {T <: AbstractParams, V <:AbstractDFGVariable, F<:AbstractDFGFactor}
4042
# Validate the userId, robotId, and sessionId
4143
!isValidLabel(userId) && error("'$userId' is not a valid User ID")
4244
!isValidLabel(robotId) && error("'$robotId' is not a valid Robot ID")
4345
!isValidLabel(sessionId) && error("'$sessionId' is not a valid Session ID")
44-
return LightDFG{T,V,F}(g, description, userId, robotId, sessionId, userData, robotData, sessionData, Symbol[], solverParams)
46+
return LightDFG{T,V,F}(g, description, userId, robotId, sessionId, userData, robotData, sessionData, Symbol[], solverParams, blobstores)
4547
end
4648

4749
# LightDFG{T}(; kwargs...) where T <: AbstractParams = LightDFG{T,DFGVariable,DFGFactor}(;kwargs...)
@@ -84,8 +86,9 @@ LightDFG(description::String,
8486
userData::Dict{Symbol, String},
8587
robotData::Dict{Symbol, String},
8688
sessionData::Dict{Symbol, String},
87-
solverParams::AbstractParams) =
88-
LightDFG(FactorGraph{Int,DFGVariable,DFGFactor}(), description, userId, robotId, sessionId, userData, robotData, sessionData, Symbol[], solverParams)
89+
solverParams::AbstractParams,
90+
blobstores=Dict{Symbol, AbstractBlobStore}()) =
91+
LightDFG(FactorGraph{Int,DFGVariable,DFGFactor}(), description, userId, robotId, sessionId, userData, robotData, sessionData, Symbol[], solverParams, blobstores)
8992

9093

9194
LightDFG{T,V,F}(description::String,
@@ -95,5 +98,6 @@ LightDFG{T,V,F}(description::String,
9598
userData::Dict{Symbol, String},
9699
robotData::Dict{Symbol, String},
97100
sessionData::Dict{Symbol, String},
98-
solverParams::T) where {T <: AbstractParams, V <:AbstractDFGVariable, F<:AbstractDFGFactor} =
99-
LightDFG(FactorGraph{Int,V,F}(), description, userId, robotId, sessionId, userData, robotData, sessionData, Symbol[], solverParams)
101+
solverParams::T,
102+
blobstores=Dict{Symbol, AbstractBlobStore}()) where {T <: AbstractParams, V <:AbstractDFGVariable, F<:AbstractDFGFactor} =
103+
LightDFG(FactorGraph{Int,V,F}(), description, userId, robotId, sessionId, userData, robotData, sessionData, Symbol[], solverParams, blobstores)

‎src/services/AbstractDFG.jl

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ Base.Broadcast.broadcastable(dfg::AbstractDFG) = Ref(dfg)
2121
# - `sessionData::Dict{Symbol, String}`
2222
# - `solverParams::T<:AbstractParams`
2323
# - `addHistory::Vector{Symbol}`
24+
# - `blobStores::Dict{Symbol, AbstractBlobStore}`
2425
# AbstractDFG Accessors
2526

2627
##------------------------------------------------------------------------------
@@ -184,7 +185,18 @@ emptySessionData!(dfg::AbstractDFG) = empty!(dfg.sessionData)
184185

185186
#TODO add__Data!?
186187

187-
188+
##==============================================================================
189+
## AbstractBlobStore CRUD
190+
##==============================================================================
191+
abstract type AbstractBlobStore{T} end
192+
# AbstractBlobStore should have key or overwrite getKey
193+
getKey(store::AbstractBlobStore) = store.key
194+
195+
getBlobStore(dfg::AbstractDFG, key::Symbol) = dfg.blobStores[key]
196+
addBlobStore!(dfg::AbstractDFG, bs::AbstractBlobStore) = push!(dfg.blobStores, getKey(bs)=>bs)
197+
updateBlobStore!(dfg::AbstractDFG, bs::AbstractBlobStore) = push!(dfg.blobStores, getKey(bs)=>bs)
198+
deleteBlobStore!(dfg::AbstractDFG, key::Symbol) = pop!(dfg.blobStores, key)
199+
emptyBlobStore!(dfg::AbstractDFG) = empty!(dfg.blobStores)
188200

189201
##==============================================================================
190202
## CRUD Interfaces

‎test/consol_DataEntryBlobTests.jl

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,14 +51,26 @@ dde,ddb = deleteData!(dfg, :x1, :random)
5151
## FolderStore
5252
##==============================================================================
5353

54-
# Create a data store and a dataset
54+
# Create a data store and add it to DFG
5555
ds = FolderStore{Vector{UInt8}}(:filestore, "/tmp/dfgFilestore")
56+
addBlobStore!(dfg, ds)
5657

57-
entry1 = BlobStoreEntry(:random, uuid4(), :filestore, bytes2hex(sha256(dataset1)), "","","", now(localzone()))
58-
59-
ade,adb = addData!(dfg, ds, :x1, entry1, dataset1)
60-
gde,gdb = getData(dfg, ds, :x1, :random)
61-
dde,ddb = deleteData!(dfg, ds, :x1, :random)
58+
ade,adb = addData!(dfg, :filestore, :x1, :random, dataset1)
59+
gde,gdb = getData(dfg, :x1, :random)
60+
dde,ddb = deleteData!(dfg, :x1, :random)
6261

6362
@test ade == gde == dde
6463
@test adb == gdb == ddb
64+
65+
##==============================================================================
66+
## Unimplemented store
67+
##==============================================================================
68+
struct TestStore{T} <: DFG.AbstractBlobStore{T} end
69+
70+
store = TestStore{Int}()
71+
72+
@test_throws ErrorException getDataBlob(store, ade)
73+
@test_throws ErrorException addDataBlob!(store, ade, 1)
74+
@test_throws ErrorException updateDataBlob!(store, ade, 1)
75+
@test_throws ErrorException deleteDataBlob!(store, ade)
76+
@test_throws ErrorException listDataBlobs(store)

‎test/interfaceTests.jl

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -188,27 +188,11 @@ end
188188

189189
@testset "File Save Functions" begin
190190
rand(7)
191-
filename = "/tmp/fileDFG"
192-
193-
dfg, vars, facs = connectivityTestGraph(testDFGAPI)
194-
195-
saveDFG(dfg, filename)
196-
197-
copyDfg = DistributedFactorGraphs._getDuplicatedEmptyDFG(dfg)
198-
199-
@info "Going to load $filename"
200-
201-
loadDFG!(copyDfg, filename)
202-
203-
for var in vars
204-
@test getVariable(dfg, var.label) == getVariable(copyDfg, var.label)
205-
end
206-
for fac in facs
207-
@test getFactor(dfg, fac.label) == getFactor(copyDfg, fac.label)
208-
# @test getFactor(dfg, fac.label) == fac
191+
if testDFGAPI <: InMemoryDFGTypes
192+
FileDFGTestBlock(testDFGAPI)
193+
else
194+
@test_skip FileDFGTestBlock(testDFGAPI)
209195
end
210-
211-
FileDFGTestBlock(testDFGAPI)
212196
end
213197
#=
214198
fg = fg1

‎test/testBlocks.jl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1418,7 +1418,7 @@ function FileDFGTestBlock(testDFGAPI; kwargs...)
14181418
# Save and load the graph to test.
14191419
saveDFG(dfg, filename)
14201420

1421-
retDFG = testDFGAPI()
1421+
retDFG = testDFGAPI(userId="testUserId")
14221422
@info "Going to load $filename"
14231423

14241424
@test_throws AssertionError loadDFG!(retDFG,"badfilename")

0 commit comments

Comments
 (0)