Merge pull request #1011 from JuliaRobotics/23Q2/docs/blobentry

add optional timestamp size fields BlobEntry, (un)packBlob, docs

Author: dehann

LICENSE

@@ -1,32 +1,4 @@
-Copyright (c) 2017-2022 NavAbility
-
-The source code in this repository is separated into two parts 
-that are licensed separately.  The Neo4jDFG library/driver
-is licensed separately.  For more details see: 
-https://github.com/JuliaRobotics/DistributedFactorGraphs.jl/blob/develop/src/Neo4jDFG/LICENSE
-
-DistributedFactorGraphs' Neo4jDFG Driver is not licensed for hosted, webservice or cloud 
-solutions (public or private).  Please contact NavAbility via email at 
-<[email protected]> for clarification or additional licensing options.
-NavAbility reserves all rights regarding, and to deny use of the software 
-for hosted webservice or cloud offerings or deployments; over and above 
-any conflicts that may arise with the licenses herein.
-
-Other general use or modification of the software beyond that of 
-webservices or clouds (private or public) is permitted 
-according to terms defined by Apache 2.0 or other library specifc licenses.
-
-   Copyright 2017-2022 NavAbility
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-   
+Copyright (c) 2017-2023 NavAbility Contributors
 
                                  Apache License
                            Version 2.0, January 2004

Project.toml

@@ -9,6 +9,7 @@ Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
@@ -37,6 +38,7 @@ CSV = "0.10"
 Colors = "0.10, 0.11, 0.12"
 Distributions = "0.23, 0.24, 0.25"
 DocStringExtensions = "0.8, 0.9"
+FileIO = "1"
 GraphPlot = "0.5.0"
 Graphs = "1.4"
 JSON3 = "1"

src/DataBlobs/entities/BlobEntry.jl

@@ -5,27 +5,50 @@
 
 """
     $(TYPEDEF)
-General Data Store Entry.
+
+A `BlobEntry` is a small about of structured data that holds reference information to find an actual blob. Many `BlobEntry`s 
+can exist on different graph nodes spanning Robots, and Sessions which can all reference the same `Blob`.  A `BlobEntry` 
+is also a equivalent to a bridging entry between local `.originId` and a remotely assigned `.blobIds`.
+
+Notes:
+- All `.blobId`s are unique across the entire distributed system and are immutable.  The `.originId` should be globally unique except for stochastic `uuid4` collisions that cannot be checked from a main reference owing to practical limitations such as network connectivity.
 """
 @Base.kwdef struct BlobEntry
-    """ This is created by server-side GraphQL """
-    id::Union{UUID, Nothing}=nothing 
-    """ This is the forced server generated blobId, or the filesystem blobId. """
-    blobId::Union{UUID, Nothing}=nothing 
-    """ This is the ID at creation at the edge, do whatever you want with this, but make sure you populate it. """
-    originId::UUID
+    """ Remotely assigned and globally unique identifier for the `BlobEntry` itself (not the `.blobId`). """
+    id::Union{UUID, Nothing} = nothing 
+    """ Machine friendly and globally unique identifier of the 'Blob', usually assigned from a common point in the system.  This can be used to guarantee unique retrieval of the large data blob. """
+    blobId::Union{UUID, Nothing} = nothing 
+    """ Machine friendly and locally assigned identifier of the 'Blob'.  `.originId`s are mandatory upon first creation at the origin regardless of network access.  Separate from `.blobId` since some architectures do not allow edge processes to assign a uuid4 to data store elements. """
+    originId::UUID = uuid4()
+    """ Human friendly label of the `Blob` and also used as unique identifier per node on which a `BlobEntry` is added.  E.g. do "LEFTCAM_1", "LEFTCAM_2", ... of you need to repeat a label on the same variable. """
     label::Symbol
+    """ A hint about where the `Blob` itself might be stored.  Remember that a Blob may be duplicated over multiple blobstores. """
     blobstore::Symbol
+    """ A hash value to ensure data consistency which must correspond to the stored hash upon retrieval.  [Legacy: some usage functions allow the check to be skipped if needed.] """
     hash::String # Probably https://docs.julialang.org/en/v1/stdlib/SHA
-    origin::String # E.g. user|robot|session|varlabel
-    description::String
-    mimeType::String
+    """ Context from which a BlobEntry=>Blob was first created. E.g. user|robot|session|varlabel. """
+    origin::String 
+    """ number of bytes in blob """
+    size::Union{Int, Nothing} = nothing
+    """ Additional information that can help a different user of the Blob. """
+    description::String = ""
+    """ MIME description describing the format of binary data in the `Blob`, e.g. 'image/png' or 'application/json; _type=CameraModel'. """
+    mimeType::String = "application/octet-stream"
+    """ Additional storage for functional metadata used in some scenarios, e.g. to support advanced features such as `parsejson(base64decode(entry.metadata))['time_sync']`. """
     metadata::String = ""
+    """ When the Blob itself was first created. """
     timestamp::ZonedDateTime = now(localzone())
+    """ When the BlobEntry was created. """
+    createdTimestamp::Union{ZonedDateTime,Nothing} = nothing
+    """ Use carefully, but necessary to support advanced usage such as time synchronization over Blob data. """
+    lastUpdatedTimestamp::Union{ZonedDateTime,Nothing} = nothing
+    """ Self type declaration for when duck-typing happens. """
     _type::String = "BlobEntry"
-    _version::String = string(_getDFGVersion()) # TBD consider upgrading to ::VersionNumber
+    """ Type version of this BlobEntry. TBD.jl consider upgrading to `::VersionNumber`. """
+    _version::String = string(_getDFGVersion())
 end
 
+
 StructTypes.StructType(::Type{BlobEntry}) = StructTypes.UnorderedStruct()
 StructTypes.idproperty(::Type{BlobEntry}) = :id
 StructTypes.omitempties(::Type{BlobEntry}) = (:id,)

src/DataBlobs/services/BlobPacking.jl

@@ -0,0 +1,95 @@
+# using FileIO
+# using ImageIO
+# using LasIO
+# using BSON
+# using OrderedCollections
+
+# 2 types for now with MIME type
+# 1. JSON     - application/octet-stream/json
+# 2. FileIO   - application/octet-stream 
+#             - application/bson 
+#             - image/jpeg
+#             - image/png
+
+const _MIMETypes = OrderedDict{MIME, DataType}()
+push!(_MIMETypes, MIME("application/octet-stream/json")=>format"JSON")
+push!(_MIMETypes, MIME("application/bson")=>format"BSON")
+push!(_MIMETypes, MIME("image/png")=>format"PNG")
+push!(_MIMETypes, MIME("image/jpeg")=>format"JPG")
+push!(_MIMETypes, MIME("application/octet-stream; ext=las")=>format"LAS")
+
+"""
+    packBlob
+Convert a file (JSON, JPG, PNG, BSON, LAS) to Vector{UInt8} for use as a Blob.  
+Returns the blob and MIME type.
+"""
+function packBlob end
+"""
+    unpackBlob
+Convert a Blob back to the origanal typ using the MIME type or DataFormat type.
+"""
+function unpackBlob end
+
+unpackBlob(mime::String, blob) = unpackBlob(MIME(mime), blob)
+
+function unpackBlob(T::MIME, blob)
+    dataformat = get(_MIMETypes, T, nothing)
+    isnothing(dataformat) && error("Format not found for MIME type $(T)")
+    return unpackBlob(dataformat, blob)
+end
+
+
+# 1. JSON strings are saved as is
+function packBlob(::Type{format"JSON"}, json_str::String)
+    mimetype = findfirst(==(format"JSON"), _MIMETypes)
+    # blob = codeunits(json_str)
+    blob = Vector{UInt8}(json_str)
+    return blob, mimetype
+end
+
+function unpackBlob(::Type{format"JSON"}, blob::Vector{UInt8})
+    return String(copy(blob))
+end
+
+
+# 2/ FileIO
+function packBlob(::Type{T}, data::Any; kwargs...) where T <: DataFormat
+    io = IOBuffer()
+    save(Stream{T}(io), data; kwargs...)
+    blob = take!(io)
+    mimetype = findfirst(==(T), _MIMETypes)
+    if isnothing(mimetype)
+        @warn "No MIME type found for format $T"
+        mimetype = MIME"application/octet-stream"
+    end
+    return blob, mimetype
+end
+
+function unpackBlob(::Type{T}, blob::Vector{UInt8}) where T <: DataFormat
+    io = IOBuffer(blob)
+    return load(Stream{T}(io))
+end
+
+
+
+
+# if false
+# json_str = "{\"name\":\"John\"}"
+# blob, mimetype = packBlob(format"JSON", json_str)
+# @assert json_str == unpackBlob(format"JSON", blob)
+# @assert json_str == unpackBlob(MIME("application/octet-stream/json"), blob)
+# @assert json_str == unpackBlob("application/octet-stream/json", blob)
+
+
+# blob,mime = packBlob(format"PNG", img)
+# up_img = unpackBlob(format"PNG", blob)
+
+# #TODO BSON does not work yet, can extend [un]packBlob(::Type{format"BSON"}, ...)
+# packBlob(format"BSON", Dict("name"=>"John"))
+# unpackBlob(format"BSON", Dict("name"=>"John"))
+
+
+# end
+
+
+

src/DataBlobs/services/HelpersDataWrapEntryBlob.jl

@@ -51,11 +51,14 @@ BlobEntry(
     label::Symbol = entry.label, 
     blobstore::Symbol = entry.blobstore, 
     hash::String = entry.hash,
+    size::Union{Int,Nothing} = entry.size,
     origin::String = entry.origin,
     description::String = entry.description, 
     mimeType::String = entry.mimeType, 
     metadata::String = entry.metadata, 
     timestamp::ZonedDateTime = entry.timestamp, 
+    createdTimestamp = entry.createdTimestamp,
+    lastUpdatedTimestamp = entry.lastUpdatedTimestamp,
     _type::String = entry._type, 
     _version::String = entry._version,
 ) = BlobEntry(;
@@ -66,10 +69,13 @@ BlobEntry(
     blobstore,
     hash,
     origin,
+    size,
     description,
     mimeType,
     metadata,
     timestamp,
+    createdTimestamp,
+    lastUpdatedTimestamp,
     _type,
     _version
 )

src/Deprecated.jl

@@ -1,3 +1,11 @@
+
+
+## ================================================================================
+## Remove in v0.23
+##=================================================================================
+
+@deprecate BlobEntry(id, blobId, originId::UUID, label::Symbol, blobstore::Symbol, hash::String, origin::String, description::String, mimeType::String, metadata::String, timestamp::ZonedDateTime, _type::String, _version::String) BlobEntry(id, blobId, originId, label, blobstore, hash, origin, -1, description, mimeType, metadata, timestamp, nothing, nothing, _type, _version)
+
 ## ================================================================================
 ## Remove in v0.22
 ##=================================================================================

src/DistributedFactorGraphs.jl

@@ -30,6 +30,8 @@ using Pkg
 using TensorCast
 using ProgressMeter
 using SHA
+using OrderedCollections
+using FileIO
 
 using CSV
 
@@ -210,6 +212,7 @@ export listDataEntrySequence
 export mergeDataEntries!
 # aliases
 export addBlob!
+export packBlob, unpackBlob
 
 ##------------------------------------------------------------------------------
 # Factors
@@ -338,6 +341,7 @@ include("services/CompareUtils.jl")
 #Blobs
 include("DataBlobs/services/BlobEntry.jl")
 include("DataBlobs/services/BlobStores.jl")
+include("DataBlobs/services/BlobPacking.jl")
 include("DataBlobs/services/HelpersDataWrapEntryBlob.jl")
 
 # Include the FilesDFG API.