WIP - Datablob concepts for discussion

Author: Affie

src/DataBlobs/DataBlobs.jl

@@ -1,3 +1,8 @@
+# using DataFrames
+# using CSV
+using JSON
+using SHA
+
 include("entities/AbstractDataStore.jl")
 include("entities/AbstractDataEntries.jl")
 include("entities/InMemoryDataStore.jl")
@@ -8,6 +13,8 @@ include("services/AbstractDataEntries.jl")
 include("services/InMemoryDataStore.jl")
 include("services/FileDataStore.jl")
 
+include("services/FileDataEntryBlob.jl")
+
 export AbstractDataStore
 
 export AbstractDataEntry, GeneralDataEntry, MongodbDataEntry, FileDataEntry

src/DataBlobs/entities/AbstractDataEntries.jl

@@ -4,40 +4,95 @@ GeneralDataEntry is a generic multipurpose data entry that creates a unique
 reproducible key using userId_robotId_sessionId_variableId_key.
 """
 mutable struct GeneralDataEntry <: AbstractDataEntry
-    key::Symbol
-    storeKey::Symbol # Could swap this to string, but using it as an index later, so better as a symbol I believe.
+    label::Symbol
+    id::UUID # Could swap this to string, but using it as an index later, so better as a symbol I believe.
     createdTimestamp::DateTime
     lastUpdatedTimestamp::DateTime
     mimeType::String
 end
 
+#TODO Deprecation - Remove in v0.10
+Base.getproperty(x::GeneralDataEntry,f::Symbol) = begin
+    if f == :key
+        Base.depwarn("GeneralDataEntry field key is deprecated, use `label` instead", :getproperty)
+        getfield(x,:label)
+    elseif f == :storeKey
+        Base.depwarn("GeneralDataEntry field storeKey is deprecated, use `id` instead", :getproperty)
+        getfield(x,:id)
+    else
+        getfield(x,f)
+    end
+end
+
+#TODO Deprecation - Remove in v0.10
+Base.setproperty!(x::GeneralDataEntry, f::Symbol, val) = begin
+    if f == :key
+        Base.depwarn("GeneralDataEntry field `key` is deprecated, use `label` instead", :setproperty!)
+        setfield(x, :label)
+    elseif f == :storeKey
+        Base.depwarn("GeneralDataEntry field `storeKey` is deprecated, use `id` instead", :setproperty!)
+        setfield(x, :id)
+    else
+        setfield!(x,f,val)
+    end
+end
+
+
 """
     $(SIGNATURES)
-Internal function to generate a unique key for the entry - userId_robotId_sessionId_variable_key.
-Simple symbol.
+Function to generate source string - userId|robotId|sessionId|varLabel
 """
-function _uniqueKey(dfg::G, v::V, key::Symbol)::Symbol where {G <: AbstractDFG, V <: AbstractDFGVariable}
-    key = join(String.([dfg.userId, dfg.robotId, dfg.sessionId, getLabel(v), String(key)]), "_")
-    return Symbol(key)
-end
+buildSourceString(dfg::AbstractDFG, label::Symbol) =
+    "$(dfg.userId)|$(dfg.robotId)|$(dfg.sessionId)|$label"
+
+
+_uniqueKey(dfg::AbstractDFG, v::AbstractDFGVariable, key::Symbol)::Symbol =
+    error("_uniqueKey is deprecated")
 
 
-GeneralDataEntry(key::Symbol, storeKey::Symbol;
+GeneralDataEntry(key::Symbol, storeKey::Symbol; mimeType::String="") = error("storeKey Deprecated, use UUID")
+
+GeneralDataEntry(label::Symbol, id::UUID;
                     mimeType::String="application/octet-stream") =
-                    GeneralDataEntry(key, storeKey, now(), now(), mimeType)
+                    GeneralDataEntry(label, uuid4(), now(UTC), now(UTC), mimeType)
+
+function GeneralDataEntry(dfg::AbstractDFG, var::AbstractDFGVariable, key::Symbol;
+                             mimeType::String="application/octet-stream")
+    return GeneralDataEntry(key, uuid4(), mimeType=mimeType)
+end
+
 
-function GeneralDataEntry(dfg::G, var::V, key::Symbol;
-                             mimeType::String="application/octet-stream") where {G <: AbstractDFG, V <: AbstractDFGVariable}
-    return GeneralDataEntry(key, _uniqueKey(dfg, var, key), mimeType=mimeType)
+"""
+    $(TYPEDEF)
+Genaral Data Store Entry.
+"""
+struct DataStoreEntry <: AbstractDataEntry
+    label::Symbol
+    id::UUID
+    datastorekey::Symbol #TODO
+    hash::String # Probably https://docs.julialang.org/en/v1/stdlib/SHA
+    source::String # E.g. user|robot|session|varlabel
+    description::String
+    mimeType::String
+    createdTimestamp::DateTime
+    # lastUpdatedTimestamp::DateTime # don't think this makes sense?
 end
 
 """
     $(TYPEDEF)
 Data Entry in MongoDB.
 """
 struct MongodbDataEntry <: AbstractDataEntry
-    key::Symbol
+    label::Symbol
+    id::UUID
     oid::NTuple{12, UInt8} #mongodb object id
+    hash::String
+
+    # mongodb
+    # client::String
+    # database::String
+    # collection::String
+
     #maybe other fields such as:
     #flags::Bool ready, valid, locked, permissions
     #MIMEType::String
@@ -49,6 +104,18 @@ end
 Data Entry in a file.
 """
 struct FileDataEntry <: AbstractDataEntry
-    key::Symbol
-    filename::String
+    label::Symbol
+    id::UUID
+    folder::String
+    hash::String #using bytes2hex or perhaps Vector{Uint8}?
+    timestamp::DateTime
+
+    function FileDataEntry(label, id, folder, hash, timestamp)
+        if !isdir(folder)
+            @warn "Folder '$folder' doesn't exist - creating."
+            # create new folder
+            mkpath(folder)
+        end
+        return new(label, id, folder, hash, timestamp)
+      end
 end

src/DataBlobs/entities/FileDataStore.jl

@@ -2,13 +2,26 @@
     $(TYPEDEF)
 Simple file data store with a specified data type and a specified key type.
 """
-struct FileDataStore <: AbstractDataStore{UInt8}
+struct FileDataStore{T} <: AbstractDataStore{T}
     folder::String
-    FileDataStore(folder::String) = begin
-        if !isdir(folder)
-            @warn "Folder '$folder' doesn't exist - creating."
-            mkpath(folder)
-        end
-        return new(folder)
+end
+
+function FileDataStore(folder::String)
+    if !isdir(folder)
+        @warn "Folder '$folder' doesn't exist - creating."
+        # create new folder
+        mkpath(folder)
     end
+
+    return FileDataStore{UInt8}(folder)
 end
+
+# if !isfile(joinpath(folder,"datatable.csv"))
+#     # create new datatable.csv
+#     @info "Creating new datatable.csv file"
+#     df = DataFrame(userId = String[], robotId = String[], sessionId = String[],
+#                    varLabel = Symbol[], dataLabel = Symbol[], dataId = UUID[])
+#     CSV.write(joinpath(folder,"datatable.csv"), df)
+# else
+#     @info "Existing datatable.csv found."
+# end

src/DataBlobs/services/AbstractDataEntries.jl

@@ -1,28 +1,35 @@
+##
+
+getHash(entry::AbstractDataEntry) = hex2bytes(entry.hash)
+
+
+##
+
 import Base: ==
 
 function ==(a::GeneralDataEntry, b::GeneralDataEntry)
-    return a.key == b.key &&
+    return a.label == b.label &&
         a.storeKey == b.storeKey &&
         a.mimeType == b.mimeType &&
         Dates.value(a.createdTimestamp - b.createdTimestamp) < 1000 &&
         Dates.value(a.lastUpdatedTimestamp - b.lastUpdatedTimestamp) < 1000 #1 second
 end
 
 function ==(a::MongodbDataEntry, b::MongodbDataEntry)
-    return a.key == b.key && a.oid == b.oid
+    return a.label == b.label && a.oid == b.oid
 end
 
 function ==(a::FileDataEntry, b::FileDataEntry)
-    return a.key == b.key && a.filename == b.filename
+    return a.label == b.label && a.filename == b.filename
 end
 
 """
     $(SIGNATURES)
 Add Big Data Entry to a DFG variable
 """
 function addDataEntry!(var::AbstractDFGVariable, bde::AbstractDataEntry)
-    haskey(var.dataDict, bde.key) && error("Data entry $(bde.key) already exists in variable")
-    var.dataDict[bde.key] = bde
+    haskey(var.dataDict, bde.label) && error("Data entry $(bde.label) already exists in variable")
+    var.dataDict[bde.label] = bde
     return bde
 end
 
@@ -79,7 +86,7 @@ hasDataEntry(var::DFGVariable, key::Symbol) = haskey(var.dataDict, key)
 Get big data entry
 """
 function getDataEntry(var::AbstractDFGVariable, key::Symbol)
-    !hasDataEntry(var, key) && (error("Data entry $(key) does not exist in variable"); return nothing)
+    !hasDataEntry(var, key) && error("Data entry $(key) does not exist in variable")
     return var.dataDict[key]
 end
 
@@ -152,8 +159,8 @@ DevNote
 - DF, unclear if `update` verb is applicable in this case, see #404
 """
 function updateDataEntry!(var::AbstractDFGVariable,  bde::AbstractDataEntry)
-    !haskey(var.dataDict, bde.key) && (@warn "$(bde.key) does not exist in variable, adding")
-    var.dataDict[bde.key] = bde
+    !haskey(var.dataDict, bde.label) && (@warn "$(bde.label) does not exist in variable, adding")
+    var.dataDict[bde.label] = bde
     return bde
 end
 function updateDataEntry!(dfg::AbstractDFG, label::Symbol,  bde::AbstractDataEntry)
@@ -170,20 +177,17 @@ Notes:
 - users responsibility to delete big data in db before deleting entry
 """
 function deleteDataEntry!(var::AbstractDFGVariable, key::Symbol)
-    bde = getDataEntry(var, key)
-    bde == nothing && return nothing
-    delete!(var.dataDict, key)
-    return var
+    return pop!(var.dataDict, key)
 end
 function deleteDataEntry!(dfg::AbstractDFG, label::Symbol, key::Symbol)
     #users responsibility to delete big data in db before deleting entry
-    !isVariable(dfg, label) && return nothing
+    # !isVariable(dfg, label) && return nothing
     return deleteDataEntry!(getVariable(dfg, label), key)
 end
 
 function deleteDataEntry!(var::AbstractDFGVariable, entry::AbstractDataEntry)
     #users responsibility to delete big data in db before deleting entry
-    return deleteDataEntry!(var, entry.key)
+    return deleteDataEntry!(var, entry.label)
 end
 
 """
@@ -195,7 +199,7 @@ function getDataEntries(var::AbstractDFGVariable)
     collect(values(var.dataDict))
 end
 function getDataEntries(dfg::AbstractDFG, label::Symbol)
-    !isVariable(dfg, label) && return nothing
+    # !isVariable(dfg, label) && return nothing
     #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractDataEntry}}?
     getDataEntries(getVariable(dfg, label))
 end
@@ -210,6 +214,6 @@ function listDataEntries(var::AbstractDFGVariable)
 end
 
 function listDataEntries(dfg::AbstractDFG, label::Symbol)
-    !isVariable(dfg, label) && return nothing
+    # !isVariable(dfg, label) && return nothing
     listDataEntries(getVariable(dfg, label))
 end