Merge pull request #519 from JuliaRobotics/maint/20Q2/bigdata_blob

BigData rename

Author: Affie

src/BigData/BigData.jl

@@ -1,17 +1,20 @@
 include("entities/AbstractDataStore.jl")
+#TODO Rename to AbstractDataEntries
 include("entities/AbstractBigDataEntries.jl")
 include("entities/InMemoryDataStore.jl")
 include("entities/FileDataStore.jl")
 
 include("services/AbstractDataStore.jl")
+#TODO Rename to AbstractDataEntries
 include("services/AbstractBigDataEntries.jl")
 include("services/InMemoryDataStore.jl")
 include("services/FileDataStore.jl")
 
 export AbstractDataStore
 
-export AbstractBigDataEntry, GeneralBigDataEntry, MongodbBigDataEntry, FileBigDataEntry
+export AbstractDataEntry, GeneralDataEntry, MongodbDataEntry, FileDataEntry
 export InMemoryDataStore, FileDataStore
 
-export getBigData, addBigData!, updateBigData!, deleteBigData!, listStoreEntries
+export getData, addData!, updateData!, deleteData!, listStoreEntries
+export getDataBlob, addDataBlob!, updateDataBlob!, deleteDataBlob!, listDataBlobs
 export copyStore

src/BigData/entities/AbstractBigDataEntries.jl

@@ -1,9 +1,9 @@
 """
     $(TYPEDEF)
-GeneralBigDataEntry is a generic multipurpose data entry that creates a unique
+GeneralDataEntry is a generic multipurpose data entry that creates a unique
 reproducible key using userId_robotId_sessionId_variableId_key.
 """
-mutable struct GeneralBigDataEntry <: AbstractBigDataEntry
+mutable struct GeneralDataEntry <: AbstractDataEntry
     key::Symbol
     storeKey::Symbol # Could swap this to string, but using it as an index later, so better as a symbol I believe.
     createdTimestamp::DateTime
@@ -21,32 +21,34 @@ function _uniqueKey(dfg::G, v::V, key::Symbol)::Symbol where {G <: AbstractDFG,
     return Symbol(key)
 end
 
-GeneralBigDataEntry(key::Symbol, storeKey::Symbol;
+
+GeneralDataEntry(key::Symbol, storeKey::Symbol;
                     mimeType::String="application/octet-stream") =
-                    GeneralBigDataEntry(key, storeKey, now(), now(), mimeType)
+                    GeneralDataEntry(key, storeKey, now(), now(), mimeType)
 
-function GeneralBigDataEntry(dfg::G, var::V, key::Symbol;
+function GeneralDataEntry(dfg::G, var::V, key::Symbol;
                              mimeType::String="application/octet-stream") where {G <: AbstractDFG, V <: AbstractDFGVariable}
-    return GeneralBigDataEntry(key, _uniqueKey(dfg, var, key), mimeType=mimeType)
+    return GeneralDataEntry(key, _uniqueKey(dfg, var, key), mimeType=mimeType)
 end
 
 """
     $(TYPEDEF)
-BigDataEntry in MongoDB.
+Data Entry in MongoDB.
 """
-struct MongodbBigDataEntry <: AbstractBigDataEntry
+struct MongodbDataEntry <: AbstractDataEntry
     key::Symbol
     oid::NTuple{12, UInt8} #mongodb object id
     #maybe other fields such as:
     #flags::Bool ready, valid, locked, permissions
-    #MIMEType::Symbol
+    #MIMEType::String
 end
 
+
 """
     $(TYPEDEF)
-BigDataEntry in a file.
+Data Entry in a file.
 """
-struct FileBigDataEntry <: AbstractBigDataEntry
+struct FileDataEntry <: AbstractDataEntry
     key::Symbol
     filename::String
 end

src/BigData/entities/InMemoryDataStore.jl

@@ -2,7 +2,7 @@
     $(TYPEDEF)
 Simple in-memory data store with a specified data type and a specified key type.
 """
-struct InMemoryDataStore{T, E <: AbstractBigDataEntry} <: AbstractDataStore{T}
+struct InMemoryDataStore{T, E <: AbstractDataEntry} <: AbstractDataStore{T}
     data::Dict{Symbol, T}
     entries::Dict{Symbol, E}
 end
@@ -11,7 +11,7 @@ end
     $(SIGNATURES)
 Create an in-memory store using a specific data type.
 """
-function InMemoryDataStore{T, E}() where {T, E <: AbstractBigDataEntry}
+function InMemoryDataStore{T, E}() where {T, E <: AbstractDataEntry}
     return InMemoryDataStore{T, E}(Dict{Symbol, T}(), Dict{Symbol, E}())
 end
 
@@ -20,5 +20,5 @@ end
 Create an in-memory store using binary data (UInt8) as a type.
 """
 function InMemoryDataStore()
-    return InMemoryDataStore{Vector{UInt8}, GeneralBigDataEntry}()
+    return InMemoryDataStore{Vector{UInt8}, GeneralDataEntry}()
 end

src/BigData/services/AbstractBigDataEntries.jl

@@ -1,41 +1,41 @@
 import Base: ==
 
-function ==(a::GeneralBigDataEntry, b::GeneralBigDataEntry)
+function ==(a::GeneralDataEntry, b::GeneralDataEntry)
     return a.key == b.key &&
         a.storeKey == b.storeKey &&
         a.mimeType == b.mimeType &&
         Dates.value(a.createdTimestamp - b.createdTimestamp) < 1000 &&
         Dates.value(a.lastUpdatedTimestamp - b.lastUpdatedTimestamp) < 1000 #1 second
 end
 
-function ==(a::MongodbBigDataEntry, b::MongodbBigDataEntry)
+function ==(a::MongodbDataEntry, b::MongodbDataEntry)
     return a.key == b.key && a.oid == b.oid
 end
 
-function ==(a::FileBigDataEntry, b::FileBigDataEntry)
+function ==(a::FileDataEntry, b::FileDataEntry)
     return a.key == b.key && a.filename == b.filename
 end
 
 """
     $(SIGNATURES)
 Add Big Data Entry to a DFG variable
 """
-function addBigDataEntry!(var::AbstractDFGVariable, bde::AbstractBigDataEntry)
-    haskey(var.bigData,bde.key) && error("BigData entry $(bde.key) already exists in variable")
-    var.bigData[bde.key] = bde
+function addDataEntry!(var::AbstractDFGVariable, bde::AbstractDataEntry)
+    haskey(var.dataDict, bde.key) && error("Data entry $(bde.key) already exists in variable")
+    var.dataDict[bde.key] = bde
     return bde
 end
 
+
 """
     $(SIGNATURES)
 Add Big Data Entry to distributed factor graph.
 Should be extended if DFG variable is not returned by reference.
 """
-function addBigDataEntry!(dfg::AbstractDFG, label::Symbol, bde::AbstractBigDataEntry)
-    return addBigDataEntry!(getVariable(dfg, label), bde)
+function addDataEntry!(dfg::AbstractDFG, label::Symbol, bde::AbstractDataEntry)
+    return addDataEntry!(getVariable(dfg, label), bde)
 end
 
-addDataEntry!(x...) = addBigDataEntry!(x...)
 
 """
 $(SIGNATURES)
@@ -50,42 +50,41 @@ Related
 
 addData!, getDataEntryElement, fetchData
 """
-function addDataEntry!(dfg::AbstractDFG,
-                       lbl::Symbol,
-                       datastore::Union{FileDataStore, InMemoryDataStore},
-                       descr::Symbol,
-                       mimeType::AbstractString,
-                       data::Vector{UInt8} )
+function addData!(dfg::AbstractDFG,
+                  lbl::Symbol,
+                  datastore::Union{FileDataStore, InMemoryDataStore},
+                  descr::Symbol,
+                  mimeType::AbstractString,
+                  data::Vector{UInt8} )
   #
   node = isVariable(dfg, lbl) ? getVariable(dfg, lbl) : getFactor(dfg, lbl)
   # Make a big data entry in the graph - use JSON2 to just write this
-  entry = GeneralBigDataEntry(dfg, node, descr, mimeType=mimeType)
+  entry = GeneralDataEntry(dfg, node, descr, mimeType=mimeType)
   # Set it in the store
-  addBigData!(datastore, entry, data)
+  addDataBlob!(datastore, entry, data)
   # Add the entry to the graph
-  addBigDataEntry!(node, entry)
+  addDataEntry!(node, entry)
 end
-const addData! = addDataEntry!
+# const addDataEntry! = addData!
 
 """
     $SIGNATURES
 
 Does a data entry (element) exist at `key`.
 """
-hasDataEntry(var::DFGVariable, key::Symbol) = haskey(var.bigData, key)
-const hasBigDataEntry = hasDataEntry
+hasDataEntry(var::DFGVariable, key::Symbol) = haskey(var.dataDict, key)
 
 """
     $(SIGNATURES)
 Get big data entry
 """
-function getBigDataEntry(var::AbstractDFGVariable, key::Symbol)
-    !hasDataEntry(var, key) && (error("BigData entry $(key) does not exist in variable"); return nothing)
-    return var.bigData[key]
+function getDataEntry(var::AbstractDFGVariable, key::Symbol)
+    !hasDataEntry(var, key) && (error("Data entry $(key) does not exist in variable"); return nothing)
+    return var.dataDict[key]
 end
 
-function getBigDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol)
-    return getBigDataEntry(getVariable(dfg, label), key)
+function getDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol)
+    return getDataEntry(getVariable(dfg, label), key)
 end
 
 """
@@ -125,24 +124,25 @@ Related
 
 addDataEntry!, addData!, fetchData, fetchDataEntryElement
 """
-function getDataEntryElement(dfg::AbstractDFG,
-                             dfglabel::Symbol,
-                             datastore::Union{FileDataStore, InMemoryDataStore},
-                             datalabel::Symbol)
+function getDataEntryBlob(dfg::AbstractDFG,
+                          dfglabel::Symbol,
+                          datastore::Union{FileDataStore, InMemoryDataStore},
+                          datalabel::Symbol)
   #
   vari = getVariable(dfg, dfglabel)
   if !hasDataEntry(vari, datalabel)
-    @error "missing data entry $datalabel in $dfglabel"
-    return nothing, nothing
+      # current standards is to fail hard
+    error("missing data entry $datalabel in $dfglabel")
+    # return nothing, nothing
   end
-  entry = getBigDataEntry(vari, datalabel)
-  element = getBigData(datastore, entry)
+  entry = getDataEntry(vari, datalabel)
+  element = getDataBlob(datastore, entry)
   return entry, element
 end
-const fetchDataEntryElement = getDataEntryElement
-const fetchData = getDataEntryElement
-
+const fetchDataEntryElement = getDataEntryBlob
+const fetchData = getDataEntryBlob
 
+#
 
 """
     $(SIGNATURES)
@@ -151,14 +151,14 @@ Update big data entry
 DevNote
 - DF, unclear if `update` verb is applicable in this case, see #404
 """
-function updateBigDataEntry!(var::AbstractDFGVariable,  bde::AbstractBigDataEntry)
-    !haskey(var.bigData,bde.key) && (@warn "$(bde.key) does not exist in variable, adding")
-    var.bigData[bde.key] = bde
+function updateDataEntry!(var::AbstractDFGVariable,  bde::AbstractDataEntry)
+    !haskey(var.dataDict, bde.key) && (@warn "$(bde.key) does not exist in variable, adding")
+    var.dataDict[bde.key] = bde
     return bde
 end
-function updateBigDataEntry!(dfg::AbstractDFG, label::Symbol,  bde::AbstractBigDataEntry)
+function updateDataEntry!(dfg::AbstractDFG, label::Symbol,  bde::AbstractDataEntry)
     # !isVariable(dfg, label) && return nothing
-    return updateBigDataEntry!(getVariable(dfg, label), bde)
+    return updateDataEntry!(getVariable(dfg, label), bde)
 end
 
 """
@@ -169,46 +169,47 @@ Note this doesn't remove it from any data stores.
 Notes:
 - users responsibility to delete big data in db before deleting entry
 """
-function deleteBigDataEntry!(var::AbstractDFGVariable, key::Symbol)
-    bde = getBigDataEntry(var, key)
+function deleteDataEntry!(var::AbstractDFGVariable, key::Symbol)
+    bde = getDataEntry(var, key)
     bde == nothing && return nothing
-    delete!(var.bigData, key)
+    delete!(var.dataDict, key)
     return var
 end
-function deleteBigDataEntry!(dfg::AbstractDFG, label::Symbol, key::Symbol)
+function deleteDataEntry!(dfg::AbstractDFG, label::Symbol, key::Symbol)
     #users responsibility to delete big data in db before deleting entry
     !isVariable(dfg, label) && return nothing
-    return deleteBigDataEntry!(getVariable(dfg, label), key)
+    return deleteDataEntry!(getVariable(dfg, label), key)
 end
 
-function deleteBigDataEntry!(var::AbstractDFGVariable, entry::AbstractBigDataEntry)
+function deleteDataEntry!(var::AbstractDFGVariable, entry::AbstractDataEntry)
     #users responsibility to delete big data in db before deleting entry
-    return deleteBigDataEntry!(var, entry.key)
+    return deleteDataEntry!(var, entry.key)
 end
 
 """
     $(SIGNATURES)
-Get big data entries, Vector{AbstractBigDataEntry}
+Get big data entries, Vector{AbstractDataEntry}
 """
-function getBigDataEntries(var::AbstractDFGVariable)
-    #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractBigDataEntry}}?
-    collect(values(var.bigData))
+function getDataEntries(var::AbstractDFGVariable)
+    #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractDataEntry}}?
+    collect(values(var.dataDict))
 end
-function getBigDataEntries(dfg::AbstractDFG, label::Symbol)
+function getDataEntries(dfg::AbstractDFG, label::Symbol)
     !isVariable(dfg, label) && return nothing
-    #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractBigDataEntry}}?
-    getBigDataEntries(getVariable(dfg, label))
+    #or should we return the iterator, Base.ValueIterator{Dict{Symbol,AbstractDataEntry}}?
+    getDataEntries(getVariable(dfg, label))
 end
 
 
 """
     $(SIGNATURES)
-getBigDataKeys
+listDataEntries
 """
-function getBigDataKeys(var::AbstractDFGVariable)
-    collect(keys(var.bigData))
+function listDataEntries(var::AbstractDFGVariable)
+    collect(keys(var.dataDict))
 end
-function getBigDataKeys(dfg::AbstractDFG, label::Symbol)
+
+function listDataEntries(dfg::AbstractDFG, label::Symbol)
     !isVariable(dfg, label) && return nothing
-    getBigDataKeys(getVariable(dfg, label))
+    listDataEntries(getVariable(dfg, label))
 end