Reorder and deprecate

Author: Affie

src/DataBlobs/DataBlobs.jl

@@ -3,22 +3,16 @@
 using JSON
 using SHA
 
-include("entities/AbstractDataStore.jl")
 include("entities/AbstractDataEntries.jl")
-include("entities/InMemoryDataStore.jl")
-include("entities/FileDataStore.jl")
+# include("entities/BlobStores.jl")
 
-include("services/AbstractDataStore.jl")
 include("services/AbstractDataEntries.jl")
-include("services/InMemoryDataStore.jl")
-include("services/FileDataStore.jl")
+include("services/DataEntryBlob.jl")
+include("services/BlobStores.jl")
 
-include("FileDataEntryBlob.jl")
-include("InMemoryDataEntryBlob.jl")
-include("DataBlobStores.jl")
-include("DataEntryBlob.jl")
+include("services/FileDataEntryBlob.jl")
+include("services/InMemoryDataEntryBlob.jl")
 
-export AbstractDataStore
 
 export AbstractDataEntry, GeneralDataEntry, MongodbDataEntry, FileDataEntry
 export InMemoryDataStore, FileDataStore

src/DataBlobs/entities/AbstractDataEntries.jl

@@ -1,67 +1,37 @@
-"""
-    $(TYPEDEF)
-GeneralDataEntry is a generic multipurpose data entry that creates a unique
-reproducible key using userId_robotId_sessionId_variableId_key.
-"""
-mutable struct GeneralDataEntry <: AbstractDataEntry
-    label::Symbol
-    id::UUID
-    createdTimestamp::ZonedDateTime
-    lastUpdatedTimestamp::ZonedDateTime
-    mimeType::String
-end
+##==============================================================================
+## AbstractDataEntry - Defined in src/entities/AbstractDFG.jl
+##==============================================================================
+# Fields to be implemented
+# label
+# id
 
-#TODO Deprecation - Remove in v0.10
-Base.getproperty(x::GeneralDataEntry,f::Symbol) = begin
-    if f == :key
-        Base.depwarn("GeneralDataEntry field key is deprecated, use `label` instead", :getproperty)
-        getfield(x,:label)
-    elseif f == :storeKey
-        Base.depwarn("GeneralDataEntry field storeKey is deprecated, use `id` instead", :getproperty)
-        getfield(x,:id)
-    else
-        getfield(x,f)
-    end
-end
+getLabel(entry::AbstractDataEntry) = entry.label
+getId(entry::AbstractDataEntry) = entry.id
+getHash(entry::AbstractDataEntry) = hex2bytes(entry.hash)
+getCreatedTimestamp(entry::AbstractDataEntry) = entry.createdTimestamp
 
-#TODO Deprecation - Remove in v0.10
-Base.setproperty!(x::GeneralDataEntry, f::Symbol, val) = begin
-    if f == :key
-        Base.depwarn("GeneralDataEntry field `key` is deprecated, use `label` instead", :setproperty!)
-        setfield(x, :label)
-    elseif f == :storeKey
-        Base.depwarn("GeneralDataEntry field `storeKey` is deprecated, use `id` instead", :setproperty!)
-        setfield(x, :id)
-    else
-        setfield!(x,f,val)
-    end
-end
 
+##==============================================================================
+## BlobStoreEntry
+##==============================================================================
+export BlobStoreEntry
 
 """
-    $(SIGNATURES)
-Function to generate source string - userId|robotId|sessionId|varLabel
+    $(TYPEDEF)
+Genaral Data Store Entry.
 """
-buildSourceString(dfg::AbstractDFG, label::Symbol) =
-    "$(dfg.userId)|$(dfg.robotId)|$(dfg.sessionId)|$label"
-
-
-_uniqueKey(dfg::AbstractDFG, v::AbstractDFGVariable, key::Symbol)::Symbol =
-    error("_uniqueKey is deprecated")
-
-
-GeneralDataEntry(key::Symbol, storeKey::Symbol; mimeType::String="") = error("storeKey Deprecated, use UUID")
-
-GeneralDataEntry(label::Symbol, id::UUID=uuid4();
-                    mimeType::String="application/octet-stream") =
-                    GeneralDataEntry(label, id, now(localzone()), now(localzone()), mimeType)
-
-function GeneralDataEntry(dfg::AbstractDFG, var::AbstractDFGVariable, key::Symbol;
-                             mimeType::String="application/octet-stream")
-    return GeneralDataEntry(key, uuid4(), mimeType=mimeType)
+struct BlobStoreEntry <: AbstractDataEntry
+    label::Symbol
+    id::UUID
+    blobstore::Symbol
+    hash::String # Probably https://docs.julialang.org/en/v1/stdlib/SHA
+    origin::String # E.g. user|robot|session|varlabel
+    description::String
+    mimeType::String
+    createdTimestamp::ZonedDateTime # of when the entry was created
 end
 
-
+# TODO
 """
     $(TYPEDEF)
 Data Entry in MongoDB.

src/DataBlobs/entities/AbstractDataStore.jl

@@ -1,81 +1,28 @@
-##
-
-getHash(entry::AbstractDataEntry) = hex2bytes(entry.hash)
-
-
-##
+##==============================================================================
+## AbstractDataEntry - compare
+##==============================================================================
 
 import Base: ==
 
-function ==(a::GeneralDataEntry, b::GeneralDataEntry)
-    return a.label == b.label &&
-        a.storeKey == b.storeKey &&
-        a.mimeType == b.mimeType &&
-        Dates.value(a.createdTimestamp - b.createdTimestamp) < 1000 &&
-        Dates.value(a.lastUpdatedTimestamp - b.lastUpdatedTimestamp) < 1000 #1 second
-end
-
-@generated function ==(x::MongodbDataEntry, y::MongodbDataEntry)
+@generated function ==(x::T, y::T) where T <: AbstractDataEntry
     mapreduce(n -> :(x.$n == y.$n), (a,b)->:($a && $b), fieldnames(x))
 end
 
-"""
-    $(SIGNATURES)
-Add Big Data Entry to a DFG variable
-"""
-function addDataEntry!(var::AbstractDFGVariable, bde::AbstractDataEntry)
-    haskey(var.dataDict, bde.label) && error("Data entry $(bde.label) already exists in variable")
-    var.dataDict[bde.label] = bde
-    return bde
-end
-
+##==============================================================================
+## AbstractDataEntry - common
+##==============================================================================
 
 """
     $(SIGNATURES)
-Add Big Data Entry to distributed factor graph.
-Should be extended if DFG variable is not returned by reference.
-"""
-function addDataEntry!(dfg::AbstractDFG, label::Symbol, bde::AbstractDataEntry)
-    return addDataEntry!(getVariable(dfg, label), bde)
-end
-
-
+Function to generate source string - userId|robotId|sessionId|varLabel
 """
-$(SIGNATURES)
-Add Big Data Entry to distributed factor graph.
-Should be extended if DFG variable is not returned by reference.
-
-Example
+buildSourceString(dfg::AbstractDFG, label::Symbol) =
+    "$(dfg.userId)|$(dfg.robotId)|$(dfg.sessionId)|$label"
 
-See docs for `getDataEntryElement`.
 
-Related
-
-addData!, getDataEntryElement, fetchData
-"""
-function addData!(dfg::AbstractDFG,
-                  lbl::Symbol,
-                  datastore::Union{FileDataStore, InMemoryDataStore},
-                  descr::Symbol,
-                  mimeType::AbstractString,
-                  data::Vector{UInt8} )
-  #
-  node = isVariable(dfg, lbl) ? getVariable(dfg, lbl) : getFactor(dfg, lbl)
-  # Make a big data entry in the graph - use JSON2 to just write this
-  entry = GeneralDataEntry(dfg, node, descr, mimeType=mimeType)
-  # Set it in the store
-  addDataBlob!(datastore, entry, data)
-  # Add the entry to the graph
-  addDataEntry!(node, entry)
-end
-# const addDataEntry! = addData!
-
-"""
-    $SIGNATURES
-
-Does a data entry (element) exist at `key`.
-"""
-hasDataEntry(var::DFGVariable, key::Symbol) = haskey(var.dataDict, key)
+##==============================================================================
+## AbstractDataEntry - CRUD
+##==============================================================================
 
 """
     $(SIGNATURES)
@@ -91,61 +38,25 @@ function getDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol)
 end
 
 """
-$SIGNATURES
-Get both the entry and raw data element from datastore returning as a tuple.
+    $(SIGNATURES)
+Add Big Data Entry to a DFG variable
+"""
+function addDataEntry!(var::AbstractDFGVariable, bde::AbstractDataEntry)
+    haskey(var.dataDict, bde.label) && error("Data entry $(bde.label) already exists in variable")
+    var.dataDict[bde.label] = bde
+    return bde
+end
 
-Notes:
-- This is the counterpart to `addDataEntry!`.
-- Data is identified by the node in the DFG object `dfglabel::Symbol` as well as `datalabel::Symbol`.
-- The data should have been stored along with a `entry.mimeType::String` which describes the format of the data.
-- ImageMagick.jl is useful for storing images in png or jpg compressed format.
-
-Example
-
-```julia
-# some dfg object
-fg = initfg()
-addVariable!(fg, :x0, IIF.ContinuousScalar) # using IncrementalInference
-# FileDataStore (as example)
-datastore = FileDataStore(joinLogPath(fg,"datastore"))
-
-# now some data comes in
-mydata = Dict(:soundBite => randn(Float32, 10000), :meta => "something about lazy foxes and fences.")
-
-# store the data
-addDataEntry!( fg, :x0, datastore, :SOUND_DATA, "application/json", Vector{UInt8}(JSON2.write( mydata ))  )
-
-# get/fetch the data
-entry, rawData = fetchData(fg, :x0, datastore, :SOUND_DATA)
-
-# unpack data to original format (this must be done by the user)
-@show entry.mimeType # "applicatio/json"
-userFormat = JSON2.read(IOBuffer(rawData))
-```
-
-Related
-
-addDataEntry!, addData!, fetchData, fetchDataEntryElement
-"""
-function getDataEntryBlob(dfg::AbstractDFG,
-                          dfglabel::Symbol,
-                          datastore::Union{FileDataStore, InMemoryDataStore},
-                          datalabel::Symbol)
-  #
-  vari = getVariable(dfg, dfglabel)
-  if !hasDataEntry(vari, datalabel)
-      # current standards is to fail hard
-    error("missing data entry $datalabel in $dfglabel")
-    # return nothing, nothing
-  end
-  entry = getDataEntry(vari, datalabel)
-  element = getDataBlob(datastore, entry)
-  return entry, element
+
+"""
+    $(SIGNATURES)
+Add Big Data Entry to distributed factor graph.
+Should be extended if DFG variable is not returned by reference.
+"""
+function addDataEntry!(dfg::AbstractDFG, label::Symbol, bde::AbstractDataEntry)
+    return addDataEntry!(getVariable(dfg, label), bde)
 end
-const fetchDataEntryElement = getDataEntryBlob
-const fetchData = getDataEntryBlob
 
-#
 
 """
     $(SIGNATURES)
@@ -186,6 +97,18 @@ function deleteDataEntry!(var::AbstractDFGVariable, entry::AbstractDataEntry)
     return deleteDataEntry!(var, entry.label)
 end
 
+##==============================================================================
+## AbstractDataEntry - Helper functions, Lists, etc
+##==============================================================================
+
+"""
+    $SIGNATURES
+
+Does a data entry (element) exist at `key`.
+"""
+hasDataEntry(var::DFGVariable, key::Symbol) = haskey(var.dataDict, key)
+
+
 """
     $(SIGNATURES)
 Get big data entries, Vector{AbstractDataEntry}
@@ -200,7 +123,6 @@ function getDataEntries(dfg::AbstractDFG, label::Symbol)
     getDataEntries(getVariable(dfg, label))
 end
 
-
 """
     $(SIGNATURES)
 listDataEntries