Skip to content

Commit 3d2414e

Browse files
AffieAffie
committed
DataEntry and BlobStore Options
1 parent 14ef9b1 commit 3d2414e

11 files changed

+425
-330
lines changed

‎src/DataBlobs/DataBlobStores.jl

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
export BlobStoreEntry
2+
"""
3+
$(TYPEDEF)
4+
Genaral Data Store Entry.
5+
"""
6+
struct BlobStoreEntry <: AbstractDataEntry
7+
label::Symbol
8+
id::UUID
9+
blobstore::Symbol
10+
hash::String # Probably https://docs.julialang.org/en/v1/stdlib/SHA
11+
origin::String # E.g. user|robot|session|varlabel
12+
description::String
13+
mimeType::String
14+
createdTimestamp::ZonedDateTime # of when the entry was created
15+
end
16+
17+
abstract type AbstractBlobStore{T} end
18+
19+
# AbstractBlobStore should have key or overwrite getKey
20+
getKey(store::AbstractBlobStore) = store.key
21+
22+
#TODO consider adding this to dfg
23+
const BlobStores = Dict{Symbol, AbstractBlobStore}
24+
25+
26+
##==============================================================================
27+
## AbstractBlobStore CRUD Interface
28+
##==============================================================================
29+
function getDataBlob(store::AbstractBlobStore, entry::BlobStoreEntry)
30+
error("$(typeof(store)) doesn't override 'getDataBlob'.")
31+
end
32+
33+
function addDataBlob!(store::AbstractBlobStore{T}, entry::BlobStoreEntry, data::T) where T
34+
error("$(typeof(store)) doesn't override 'addDataBlob!'.")
35+
end
36+
37+
function updateDataBlob!(store::AbstractBlobStore{T}, entry::BlobStoreEntry, data::T) where T
38+
error("$(typeof(store)) doesn't override 'updateDataBlob!'.")
39+
end
40+
41+
function deleteDataBlob!(store::AbstractBlobStore, entry::BlobStoreEntry)
42+
error("$(typeof(store)) doesn't override 'deleteDataBlob!'.")
43+
end
44+
45+
function listDataBlobs(store::AbstractBlobStore)
46+
error("$(typeof(store)) doesn't override 'listDataBlobs'.")
47+
end
48+
49+
##==============================================================================
50+
## Store and Entry Data CRUD
51+
##==============================================================================
52+
53+
function getData(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol, key::Symbol; hashfunction = sha256)
54+
de = getDataEntry(dfg, label, key)
55+
db = getDataBlob(blobstore, de)
56+
assertHash(de, db, hashfunction=hashfunction)
57+
return de=>db
58+
end
59+
60+
function addData!(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol, entry::AbstractDataEntry, blob::Vector{UInt8}; hashfunction = sha256)
61+
assertHash(entry, blob, hashfunction=hashfunction)
62+
de = addDataEntry!(dfg, label, entry)
63+
db = addDataBlob!(blobstore, de, blob)
64+
return de=>db
65+
end
66+
67+
function updateData!(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol, entry::AbstractDataEntry, blob::Vector{UInt8})
68+
assertHash(entry, blob, hashfunction=hashfunction)
69+
de = updateDataEntry!(dfg, label, entry)
70+
db = updateDataBlob!(blobstore, de, blob)
71+
return de=>db
72+
end
73+
74+
deleteData!(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol, entry::AbstractDataEntry) =
75+
deleteData!(dfg, blobstore, label, entry.label)
76+
77+
function deleteData!(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol, key::Symbol)
78+
de = deleteDataEntry!(dfg, label, key)
79+
db = deleteDataBlob!(blobstore, de)
80+
return de=>db
81+
end
82+
83+
84+
function addData!(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol, key::Symbol,
85+
blob::Vector{UInt8}, timestamp=now(localzone()); description="", mimeType = "", id::UUID = uuid4(), hashfunction = sha256)
86+
87+
88+
entry = BlobStoreEntry(key, id, blobstore.key, bytes2hex(hashfunction(blob)),
89+
"$(dfg.userId)|$(dfg.robotId)|$(dfg.sessionId)|$(dfg.label)",
90+
description, mimeType, timestamp)
91+
92+
addData!(dfg, blobstore, label, entry, blob; hashfunction = hashfunction)
93+
end
94+
95+
96+
##==============================================================================
97+
## FolderStore
98+
##==============================================================================
99+
export FolderStore
100+
101+
struct FolderStore{T} <: AbstractBlobStore{T}
102+
key::Symbol
103+
folder::String
104+
end
105+
106+
blobfilename(store::FolderStore, entry::FileDataEntry) = joinpath(store.folder,"$(entry.id).dat")
107+
entryfilename(store::FolderStore, entry::FileDataEntry) = joinpath(store.folder,"$(entry.id).json")
108+
109+
function getDataBlob(store::FolderStore{T}, entry::BlobStoreEntry) where T
110+
blobfilename = joinpath(store.folder,"$(entry.id).dat")
111+
# entryfilename = "$(store.folder)/$(entry.id).json"
112+
if isfile(blobfilename)
113+
open(blobfilename) do f
114+
return read(f)
115+
end
116+
else
117+
error("Could not find file '$(blobfilename)'.")
118+
# return nothing
119+
end
120+
end
121+
122+
function addDataBlob!(store::FolderStore{T}, entry::BlobStoreEntry, data::T) where T
123+
blobfilename = joinpath(store.folder,"$(entry.id).dat")
124+
entryfilename = joinpath(store.folder,"$(entry.id).json")
125+
if isfile(blobfilename)
126+
error("Key '$(id)' blob already exists.")
127+
elseif isfile(entryfilename)
128+
error("Key '$(id)' entry already exists, but no blob.")
129+
else
130+
open(blobfilename, "w") do f
131+
write(f, data)
132+
end
133+
open(entryfilename, "w") do f
134+
JSON.print(f, entry)
135+
end
136+
return data
137+
end
138+
end
139+
140+
function deleteDataBlob!(store::FolderStore{T}, entry::BlobStoreEntry) where T
141+
blobfilename = joinpath(store.folder,"$(entry.id).dat")
142+
entryfilename = joinpath(store.folder,"$(entry.id).json")
143+
144+
data = getDataBlob(store, entry)
145+
rm(blobfilename)
146+
rm(entryfilename)
147+
return data
148+
end

‎src/DataBlobs/DataBlobs.jl

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@ include("services/AbstractDataEntries.jl")
1313
include("services/InMemoryDataStore.jl")
1414
include("services/FileDataStore.jl")
1515

16-
include("services/FileDataEntryBlob.jl")
16+
include("FileDataEntryBlob.jl")
17+
include("InMemoryDataEntryBlob.jl")
18+
include("DataBlobStores.jl")
19+
include("DataEntryBlob.jl")
1720

1821
export AbstractDataStore
1922

‎src/DataBlobs/DataEntryBlob.jl

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
2+
function assertHash(de::AbstractDataEntry, db; hashfunction = sha256)
3+
if hashfunction(db) == getHash(de)
4+
return true #or nothing?
5+
else
6+
error("Stored hash and data blob hash do not match")
7+
end
8+
end
9+
10+
##==============================================================================
11+
## DFG DataBlob CRUD
12+
##==============================================================================
13+
14+
#
15+
# addDataBlob!(dfg::AbstractDFG, entry::AbstractDataEntry, blob)
16+
# updateDataBlob!(dfg::AbstractDFG, entry::AbstractDataEntry, blob)
17+
# deleteDataBlob!(dfg::AbstractDFG, entry::AbstractDataEntry)
18+
19+
function getDataBlob(dfg::AbstractDFG, entry::AbstractDataEntry)
20+
error("$(typeof(store)) doesn't override 'getDataBlob'.")
21+
end
22+
23+
function addDataBlob!(dfg::AbstractDFG, entry::BlobStoreEntry, data::T) where T
24+
error("$(typeof(store)) doesn't override 'addDataBlob!'.")
25+
end
26+
27+
function updateDataBlob!(dfg::AbstractDFG, entry::BlobStoreEntry, data::T) where T
28+
error("$(typeof(store)) doesn't override 'updateDataBlob!'.")
29+
end
30+
31+
function deleteDataBlob!(dfg::AbstractDFG, entry::BlobStoreEntry)
32+
error("$(typeof(store)) doesn't override 'deleteDataBlob!'.")
33+
end
34+
35+
function listDataBlobs(dfg::AbstractDFG)
36+
error("$(typeof(store)) doesn't override 'listDataBlobs'.")
37+
end
38+
39+
##==============================================================================
40+
## DFG Data CRUD
41+
##==============================================================================
42+
43+
function getData(dfg::AbstractDFG, label::Symbol, key::Symbol; hashfunction = sha256)
44+
de = getDataEntry(dfg, label, key)
45+
db = getDataBlob(dfg, de)
46+
47+
assertHash(de, db, hashfunction=hashfunction)
48+
return de=>db
49+
end
50+
51+
function addData!(dfg::AbstractDFG, label::Symbol, entry::AbstractDataEntry, blob::Vector{UInt8}; hashfunction = sha256)
52+
assertHash(entry, blob, hashfunction=hashfunction)
53+
de = addDataEntry!(dfg, label, entry)
54+
db = addDataBlob!(dfg, de, blob)
55+
return de=>db
56+
end
57+
58+
function updateData!(dfg::AbstractDFG, label::Symbol, entry::AbstractDataEntry, blob::Vector{UInt8})
59+
assertHash(entry, blob, hashfunction=hashfunction)
60+
de = updateDataEntry!(dfg, label, entry)
61+
db = updateDataBlob!(dfg, de, blob)
62+
return de=>db
63+
end
64+
65+
function deleteData!(dfg::AbstractDFG, label::Symbol, key::Symbol)
66+
de = deleteDataEntry!(dfg, label, key)
67+
db = deleteDataBlob!(dfg, de)
68+
return de=>db
69+
end

‎src/DataBlobs/FileDataEntryBlob.jl

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
##==============================================================================
2+
## FileDataEntryBlob Types
3+
##==============================================================================
4+
export FileDataEntry
5+
"""
6+
$(TYPEDEF)
7+
Data Entry in a file.
8+
"""
9+
struct FileDataEntry <: AbstractDataEntry
10+
label::Symbol
11+
id::UUID
12+
folder::String
13+
hash::String #using bytes2hex or perhaps Vector{Uint8}?
14+
createdTimestamp::ZonedDateTime
15+
16+
function FileDataEntry(label, id, folder, hash, timestamp)
17+
if !isdir(folder)
18+
@warn "Folder '$folder' doesn't exist - creating."
19+
# create new folder
20+
mkpath(folder)
21+
end
22+
return new(label, id, folder, hash, timestamp)
23+
end
24+
end
25+
#
26+
# getHash(entry::AbstractDataEntry) = hex2bytes(entry.hash)
27+
28+
29+
##==============================================================================
30+
## FileDataEntry Common
31+
##==============================================================================
32+
blobfilename(entry::FileDataEntry) = joinpath(entry.folder,"$(entry.id).dat")
33+
entryfilename(entry::FileDataEntry) = joinpath(entry.folder,"$(entry.id).json")
34+
35+
36+
##==============================================================================
37+
## FileDataEntry Blob CRUD
38+
##==============================================================================
39+
40+
function getDataBlob(dfg::AbstractDFG, entry::FileDataEntry)
41+
if isfile(blobfilename(entry))
42+
open(blobfilename(entry)) do f
43+
return read(f)
44+
end
45+
else
46+
error("Could not find file '$(blobfilename(entry))'.")
47+
# return nothing
48+
end
49+
end
50+
51+
function addDataBlob!(dfg::AbstractDFG, entry::FileDataEntry, data::Vector{UInt8})
52+
if isfile(blobfilename(entry))
53+
error("Key '$(entry.id)' blob already exists.")
54+
elseif isfile(entryfilename(entry))
55+
error("Key '$(entry.id)' entry already exists, but no blob.")
56+
else
57+
open(blobfilename(entry), "w") do f
58+
write(f, data)
59+
end
60+
open(entryfilename(entry), "w") do f
61+
JSON.print(f, entry)
62+
end
63+
return getDataBlob(dfg, entry)::Vector{UInt8}
64+
end
65+
end
66+
67+
function updateDataBlob!(dfg::AbstractDFG, entry::FileDataEntry, data::Vector{UInt8})
68+
if !isfile(blobfilename(entry))
69+
@warn "Entry '$(entry.id)' does not exist, adding."
70+
return addDataBlob!(entry, data)
71+
else
72+
# perhaps add an explicit force update flag and error otherwise
73+
@warn "Key '$(entry.id)' already exists, data will be overwritten."
74+
deleteDataBlob!(entry::AbstractDataEntry)
75+
return addDataBlob!(entry, data)
76+
end
77+
end
78+
79+
function deleteDataBlob!(dfg::AbstractDFG, entry::FileDataEntry)
80+
data = getDataBlob(dfg, entry)
81+
rm(blobfilename(entry))
82+
rm(entryfilename(entry))
83+
return data
84+
end
85+
86+
##==============================================================================
87+
## FileDataEntry CRUD Helpers
88+
##==============================================================================
89+
90+
function addData!(::Type{FileDataEntry}, dfg::AbstractDFG, label::Symbol, key::Symbol, folder::String, blob::Vector{UInt8}, timestamp=now(localzone());
91+
id::UUID = uuid4(), hashfunction = sha256)
92+
fde = FileDataEntry(key, id, folder, bytes2hex(hashfunction(blob)), timestamp)
93+
de = addDataEntry!(dfg, label, fde)
94+
db = addDataBlob!(dfg, fde, blob)
95+
return de=>db
96+
end

‎src/DataBlobs/InMemoryDataEntryBlob.jl

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
##==============================================================================
2+
## InMemoryDataEntry Types
3+
##==============================================================================
4+
export InMemoryDataEntry
5+
"""
6+
$(TYPEDEF)
7+
Store data temporary in memory.
8+
NOTE: Neigher Entry nor Blob will be persisted.
9+
"""
10+
struct InMemoryDataEntry{T} <: AbstractDataEntry
11+
label::Symbol
12+
id::UUID
13+
#hash::String #Is this needed?
14+
createdTimestamp::ZonedDateTime
15+
data::T
16+
end
17+
18+
# getHash(::InMemoryDataEntry) = UInt8[]
19+
assertHash(de::InMemoryDataEntry, db; hashfunction = sha256) = true
20+
21+
22+
##==============================================================================
23+
## InMemoryDataEntry Blob CRUD - dit lyk nie of dit gebruik moet word nie
24+
##==============================================================================
25+
26+
getDataBlob(dfg::AbstractDFG, entry::InMemoryDataEntry) = entry.data
27+
28+
# addDataBlob!(dfg::AbstractDFG, entry::InMemoryDataEntry, blob) = error("Not suported")#entry.blob
29+
# updateDataBlob!(dfg::AbstractDFG, entry::InMemoryDataEntry, blob) = error("Not suported")#entry.blob
30+
deleteDataBlob!(dfg::AbstractDFG, entry::InMemoryDataEntry) = entry.data
31+
32+
function addData!(dfg::AbstractDFG, label::Symbol, entry::AbstractDataEntry; hashfunction = sha256)
33+
# assertHash(entry, entry.data, hashfunction=hashfunction)
34+
de = addDataEntry!(dfg, label, entry)
35+
db = getDataBlob(dfg, entry)
36+
return de=>db
37+
end
38+
39+
function updateData!(dfg::AbstractDFG, label::Symbol, entry::AbstractDataEntry)
40+
# assertHash(entry, entry.data, hashfunction=hashfunction)
41+
de = updateDataEntry!(dfg, label, entry)
42+
db = getDataBlob(dfg, entry)
43+
return de=>db
44+
end
45+
##==============================================================================
46+
## InMemoryDataEntry CRUD Helpers
47+
##==============================================================================
48+
49+
# function addData!(dfg::AbstractDFG, label::Symbol, key::Symbol, folder::String, blob::Vector{UInt8}, timestamp=now(localzone());
50+
function addData!(::Type{InMemoryDataEntry}, dfg::AbstractDFG, label::Symbol, key::Symbol, blob, timestamp=now(localzone());
51+
id::UUID = uuid4(), hashfunction = sha256)
52+
fde = InMemoryDataEntry(key, id, timestamp, blob)
53+
de = addDataEntry!(dfg, label, fde)
54+
return de=>blob
55+
end

0 commit comments

Comments
 (0)