fix helpers addData! and entry.originId

Author: dehann

NEWS.md

@@ -2,6 +2,8 @@ Listing news on any major breaking changes in DFG.  For regular changes, see int
 
 # v0.20
 
+- Change return type on `addData!` convenience wrappers to only return new `BlobEntry`.
+- Fix `addBlob!` calls for `FolderStore` and `InMemoryBlobStore` to use `BlobEntry.originId` and not previous bug `entry.id`.
 - Close long running serialization redo (#590) using only JSON3.jl and StructTypes.jl going forward.
 - Standardize BlobEntry=>Blob naming of functions, and keeping convenience wrappers `{get,add,update,delete}Data[!]`.
 - Consolidate to only one `BlobEntry` definition, dropping use of `AbstractBlobEntry`.

attic/DataBlobs/FileDataEntryBlob.jl

@@ -42,7 +42,9 @@ function addBlob!(dfg::AbstractDFG, entry::BlobEntry, data::Vector{UInt8})
         open(entryfilename(entry), "w") do f
             JSON.print(f, entry)
         end
-        return getBlob(dfg, entry)::Vector{UInt8}
+        # FIXME update for entry.blobId vs entry.originId
+        return UUID(entry.id)
+        # return getBlob(dfg, entry)::Vector{UInt8}
     end
 end
 
@@ -72,7 +74,8 @@ end
 function addData!(::Type{BlobEntry}, dfg::AbstractDFG, label::Symbol, key::Symbol, folder::String, blob::Vector{UInt8}, timestamp=now(localzone());
                   id::UUID = uuid4(), hashfunction = sha256)
     fde = BlobEntry(key, id, folder, bytes2hex(hashfunction(blob)), timestamp)
-    de = addBlobEntry!(dfg, label, fde)
-    db = addBlob!(dfg, fde, blob)
-    return de=>db
+    blobId = addBlob!(dfg, fde, blob) |> UUID
+    newEntry = BlobEntry(fde; id=blobId, blobId)
+    de = addBlobEntry!(dfg, label, newEntry)
+    return de # de=>db
 end

src/DataBlobs/services/BlobStores.jl

@@ -165,30 +165,32 @@ function getBlob(store::FolderStore{T}, entry::BlobEntry) where T
 end
 
 function addBlob!(store::FolderStore{T}, entry::BlobEntry, data::T) where T
-    blobfilename = joinpath(store.folder,"$(entry.id).dat")
-    entryfilename = joinpath(store.folder,"$(entry.id).json")
+    blobfilename = joinpath(store.folder,"$(entry.originId).dat")
+    entryfilename = joinpath(store.folder,"$(entry.originId).json")
     if isfile(blobfilename)
-        error("Key '$(entry.id)' blob already exists.")
+        error("Key '$(entry.originId)' blob already exists.")
     elseif isfile(entryfilename)
-        error("Key '$(entry.id)' entry already exists, but no blob.")
+        error("Key '$(entry.originId)' entry already exists, but no blob.")
     else
         open(blobfilename, "w") do f
             write(f, data)
         end
         open(entryfilename, "w") do f
             JSON3.write(f, entry)
         end
-        return data
+        # return data
+        # FIXME update for entry.blobId vs. entry.originId
+        return UUID(entry.originId)
     end
 end
 
 function updateBlob!(store::FolderStore{T},  entry::BlobEntry, data::T) where T
-    blobfilename = joinpath(store.folder,"$(entry.id).dat")
-    entryfilename = joinpath(store.folder,"$(entry.id).json")
+    blobfilename = joinpath(store.folder,"$(entry.originId).dat")
+    entryfilename = joinpath(store.folder,"$(entry.originId).json")
     if !isfile(blobfilename)
-        @warn "Key '$(entry.id)' doesn't exist."
+        @warn "Key '$(entry.originId)' doesn't exist."
     elseif !isfile(entryfilename)
-        @warn "Key '$(entry.id)' doesn't exist."
+        @warn "Key '$(entry.originId)' doesn't exist."
     else
         open(blobfilename, "w") do f
             write(f, data)
@@ -228,19 +230,21 @@ function getBlob(store::InMemoryBlobStore{T}, entry::BlobEntry) where T
 end
 
 function addBlob!(store::InMemoryBlobStore{T}, entry::BlobEntry, data::T) where T
-    if haskey(store.blobs, entry.id)
-        error("Key '$(entry.id)' blob already exists.")
+    if haskey(store.blobs, entry.originId)
+        error("Key '$(entry.originId)' blob already exists.")
     end
-    return store.blobs[entry.id] = data
+    # FIXME update for entry.originId vs .blobId
+    store.blobs[entry.originId] = data
+    return UUID(entry.originId)
 end
 
 function updateBlob!(store::InMemoryBlobStore{T},  entry::BlobEntry, data::T) where T
-    if haskey(store.blobs, entry.id)
-        @warn "Key '$(entry.id)' doesn't exist."
+    if haskey(store.blobs, entry.originId)
+        @warn "Key '$(entry.originId)' doesn't exist."
     end
-    return store.blobs[entry.id] = data
+    return store.blobs[entry.originId] = data
 end
 
 function deleteBlob!(store::InMemoryBlobStore{T}, entry::BlobEntry) where T
-    return pop!(store.blobs, entry.id)
+    return pop!(store.blobs, entry.originId)
 end

src/DataBlobs/services/HelpersDataWrapEntryBlob.jl

@@ -114,35 +114,43 @@ function addData!(
     entry::BlobEntry, 
     blob::Vector{UInt8}; 
     hashfunction = sha256,
-    checkhash::Bool=true
+    checkhash::Bool=false
 )
     checkhash && assertHash(entry, blob, hashfunction=hashfunction)
-    de = addBlobEntry!(dfg, label, entry)
-    db = addBlob!(dfg, de, blob)
-    return de=>db
+    blobId = addBlob!(dfg, entry, blob) |> UUID
+    newEntry = BlobEntry(entry; id=blobId, blobId) #, size=length(blob))
+    addBlobEntry!(dfg, label, newEntry)
 end
 
-function addData!(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol, entry::BlobEntry, blob::Vector{UInt8}; hashfunction = sha256)
-    assertHash(entry, blob; hashfunction)
-    de = addBlobEntry!(dfg, label, entry)
-    db = addBlob!(blobstore, de, blob)
-    return de=>db
+function addData!(
+    dfg::AbstractDFG, 
+    blobstore::AbstractBlobStore, 
+    label::Symbol, 
+    entry::BlobEntry, 
+    blob::Vector{UInt8}; 
+    hashfunction = sha256,
+    checkhash::Bool=false,
+)
+    checkhash && assertHash(entry, blob; hashfunction)
+    blobId = addBlob!(blobstore, entry, blob) |> UUID
+    newEntry = BlobEntry(entry; id=blobId, blobId) #, size=length(blob))
+    addBlobEntry!(dfg, label, newEntry)
 end
 
 
 addData!(
     dfg::AbstractDFG, 
     blobstorekey::Symbol, 
-    label::Symbol, 
-    key::Symbol, 
+    vLbl::Symbol, 
+    bLbl::Symbol, 
     blob::Vector{UInt8},
     timestamp=now(localzone()); 
     kwargs...
 ) = addData!(
     dfg,
     getBlobStore(dfg, blobstorekey),
-    label,
-    key,
+    vLbl,
+    bLbl,
     blob,
     timestamp;
     kwargs...
@@ -151,31 +159,32 @@ addData!(
 function addData!(
     dfg::AbstractDFG, 
     blobstore::AbstractBlobStore, 
-    label::Symbol, 
-    key::Symbol,
-    blob::AbstractVector{UInt8}, 
+    vLbl::Symbol, 
+    bLbl::Symbol,
+    blob::Vector{UInt8}, 
     timestamp=now(localzone()); 
     description="",
     metadata = "",
-    mimeType = "application/octet-stream", 
-    id::UUID = uuid4(), 
+    mimeType::String = "application/octet-stream", 
+    id::Union{UUID,Nothing} = nothing, #only assign if blobstore issued you an id
+    originId::UUID = uuid4(),
     hashfunction = sha256
 )
     #
-    @warn "ID's and origin IDs should be reconciled here."
+    @warn "ID's and origin IDs should be reconciled here in DFG.addData!." maxlog=50
     entry = BlobEntry(;
-        id = id, 
-        originId = id,
-        label = key, 
+        id, 
+        originId,
+        label = bLbl, 
         blobstore = blobstore.key, 
-        hash = bytes2hex(hashfunction(blob)),
-        origin = buildSourceString(dfg, label),
+        hash = string(bytes2hex(hashfunction(blob))),
+        origin = buildSourceString(dfg, vLbl),
         description = description, 
-        mimeType = mimeType, 
+        mimeType, 
         metadata, 
-        timestamp = timestamp)
+        timestamp)
 
-    addData!(dfg, blobstore, label, entry, blob; hashfunction)
+    addData!(dfg, blobstore, vLbl, entry, blob; hashfunction)
 end
 
 
@@ -188,6 +197,7 @@ function updateData!(
     checkhash::Bool=true
 )
     checkhash && assertHash(entry, blob; hashfunction)
+    # order of ops with unknown new blobId not tested
     de = updateBlobEntry!(dfg, label, entry)
     db = updateBlob!(dfg, de, blob)
     return de=>db
@@ -203,6 +213,7 @@ function updateData!(
     hashfunction = sha256
 )
     # Recalculate the hash - NOTE Assuming that this is going to be a BlobEntry. TBD.
+    # order of operations with unknown new blobId not tested
     newEntry = BlobEntry(
         entry; # and kwargs to override new values
         blobstore = blobstore.key, 

test/consol_DataEntryBlobTests.jl

@@ -91,8 +91,8 @@ dataset2 = rand(UInt8, 1000)
 ds = FolderStore{Vector{UInt8}}(:filestore, "/tmp/dfgFolderStore")
 addBlobStore!(dfg, ds)
 
-ade,adb = addData!(dfg, :filestore, :x1, :random, dataset1)
-_,_     = addData!(dfg, :filestore, :x1, :another_1, dataset1)
+ade = addData!(dfg, :filestore, :x1, :random, dataset1)
+_   = addData!(dfg, :filestore, :x1, :another_1, dataset1)
 _,_ = getData(dfg, :x1, "random")
 _,_ = getData(dfg, :x1, r"rando")
 gde,gdb = getData(dfg, :x1, :random)
@@ -108,13 +108,13 @@ dde,ddb = deleteData!(dfg, :x1, :random)
 _,_     = deleteData!(dfg, :x1, :another_1)
 
 @test ade == gde == dde
-@test adb == gdb == ddb
+@test dataset1 == gdb == ddb
 
-ade2,adb2 = addData!(dfg, :x2, deepcopy(ade), dataset1)
+ade2 = addData!(dfg, :x2, deepcopy(ade), dataset1)
 # ade3,adb3 = updateBlob!(dfg, :x2, deepcopy(ade), dataset1)
 
 @test ade == ade2# == ade3
-@test adb == adb2# == adb3
+# @test adb == adb2# == adb3
 
 deleteData!(dfg, :x2, :random)
 
@@ -132,18 +132,18 @@ dfs = FolderStore("/tmp/defaultfolderstore")
 ds = InMemoryBlobStore()
 addBlobStore!(dfg, ds)
 
-ade,adb = addData!(dfg, :default_inmemory_store, :x1, :random, dataset1)
+ade = addData!(dfg, :default_inmemory_store, :x1, :random, dataset1)
 gde,gdb = getData(dfg, :x1, :random)
 dde,ddb = deleteData!(dfg, :x1, :random)
 
 @test ade == gde == dde
-@test adb == gdb == ddb
+@test dataset1 == gdb == ddb
 
-ade2,adb2 = addData!(dfg, :x2, deepcopy(ade), dataset1)
+ade2 = addData!(dfg, :x2, deepcopy(ade), dataset1)
 # ade3,adb3 = updateBlob!(dfg, :x2, deepcopy(ade), dataset1)
 
 @test ade == ade2# == ade3
-@test adb == adb2# == adb3
+# @test adb == adb2# == adb3
 
 deleteData!(dfg, :x2, :random)
 

test/testBlocks.jl

@@ -1050,10 +1050,10 @@ function blobsStoresTestBlock!(fg)
     @test :testing in listBlobEntries(fg, :a)
     # Getting
     data = getData(fg, fs, :a, :testing) # convenience wrapper over getBlob
-    @test data[1].hash == newData[1].hash
-    @test data[2] == newData[2]
+    @test data[1].hash == newData.hash #[1]
+    # @test data[2] == newData[2]
     # Updating
-    updateData = updateData!(fg, fs, :a, newData[1], rand(UInt8, 50)) # convenience wrapper around updateBlob!
+    updateData = updateData!(fg, fs, :a, newData, rand(UInt8, 50)) # convenience wrapper around updateBlob!
     @test updateData[1].hash != data[1].hash
     @test updateData[2] != data[2]
     # Deleting