saveDFG and loadDFG with metadata and Tar.jl (#1025)

* saveDFG and loadDFG with metadata and Tar.jl
* Allow loadDFG with old format
* save/loadDFG SolverParams

Author: Affie

Project.toml

@@ -5,12 +5,14 @@ version = "0.22.0"
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb"
@@ -23,6 +25,7 @@ SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 TensorCast = "02d47bb6-7ce6-556a-be16-bb1710789e2b"
 TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
@@ -35,6 +38,7 @@ DFGPlots = "GraphPlot"
 
 [compat]
 CSV = "0.10"
+CodecZlib = "0.7"
 Colors = "0.10, 0.11, 0.12"
 Distributions = "0.23, 0.24, 0.25"
 DocStringExtensions = "0.8, 0.9"
@@ -50,6 +54,7 @@ RecursiveArrayTools = "2"
 Reexport = "1"
 StaticArrays = "1"
 StructTypes = "1"
+Tar = "1.9"
 TensorCast = "0.3.3, 0.4"
 TimeZones = "1.3.1"
 julia = "1.9"

src/DistributedFactorGraphs.jl

@@ -31,6 +31,9 @@ using TensorCast
 using ProgressMeter
 using SHA
 using FileIO
+import Tar
+import CodecZlib
+
 using OrderedCollections
 export OrderedDict
 
@@ -70,7 +73,24 @@ export getDescription, setDescription!,
        getSessionData, setSessionData!,
        getAddHistory
 
-export getSessionBlobEntry, addSessionBlobEntry!
+export  getSessionBlobEntry,
+        getSessionBlobEntries,
+        addSessionBlobEntry!,
+        addSessionBlobEntries!,
+        updateSessionBlobEntry!,
+        deleteSessionBlobEntry!,
+        getRobotBlobEntry,
+        getRobotBlobEntries,
+        addRobotBlobEntry!,
+        addRobotBlobEntries!,
+        updateRobotBlobEntry!,
+        deleteRobotBlobEntry!,
+        getUserBlobEntry,
+        getUserBlobEntries,
+        addUserBlobEntry!,
+        addUserBlobEntries!,
+        updateUserBlobEntry!,
+        deleteUserBlobEntry!
 
 export getBlobStore,
        addBlobStore!,

src/FileDFG/services/FileDFG.jl

@@ -17,7 +17,7 @@ v1 = addVariable!(dfg, :a, ContinuousScalar, tags = [:POSE], solvable=0)
 saveDFG(dfg, "/tmp/saveDFG.tar.gz")
 ```
 """
-function saveDFG(folder::AbstractString, dfg::AbstractDFG)
+function saveDFG(folder::AbstractString, dfg::AbstractDFG; saveMetadata::Bool=true)
 
     # TODO: Deprecate the folder functionality
 
@@ -46,29 +46,40 @@ function saveDFG(folder::AbstractString, dfg::AbstractDFG)
     end
     # Factors
     @showprogress "saving factors" for f in factors
-        fPacked = packFactor(dfg, f)
+        fPacked = packFactor(f)
         JSON3.write("$factorFolder/$(f.label).json", fPacked)
     end
+    #GraphsDFG metadata
+    if saveMetadata
+        @assert isa(dfg, GraphsDFG) "only metadata for GraphsDFG are supported"
+        @info "saving dfg metadata"
+        fgPacked = GraphsDFGs.packDFGMetadata(dfg)
+        JSON3.write("$savepath/dfg.json", fgPacked)
+    end
 
     savedir = dirname(savepath) # is this a path of just local name? #344 -- workaround with unique names
     savename = basename(string(savepath))
     @assert savename != ""
     destfile = joinpath(savedir, savename*".tar.gz")
-    # FIXME, switch to Tar.jl and Transcode Zlib / Codec, see #351
-    if length(savedir) != 0
-        run( pipeline(`tar -zcf - -C $savedir $savename`, stdout="$destfile"))
-    else
-        run( pipeline(`tar -zcf - $savename`, stdout="$destfile"))
-    end
+    
+    #create Tarbal using Tar.jl #351
+    tar_gz = open(destfile, write=true)
+    tar = CodecZlib.GzipCompressorStream(tar_gz)
+    Tar.create(joinpath(savedir,savename), tar)
+    close(tar)
+    #not compressed version
+    # Tar.create(joinpath(savedir,savename), destfile)
+
     Base.rm(joinpath(savedir,savename), recursive=true)
 end
 # support both argument orders, #581
 saveDFG(dfg::AbstractDFG, folder::AbstractString) = saveDFG(folder, dfg)
 
+#TODO  loadDFG(dst::AbstractString) to load an equivalent dfg, but defined in IIF
+
 """
     $(SIGNATURES)
-Load a DFG from a saved folder. Always provide the IIF module as the second
-parameter.
+Load a DFG from a saved folder.
 
 # Example
 ```julia
@@ -81,7 +92,7 @@ loadDFG!(dfg, "/tmp/savedgraph.tar.gz")
 ls(dfg)
 ```
 """
-function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString)
+function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString; overwriteDFGMetadata::Bool=true, useDeprExtract::Bool=false)
 
     #
     # loaddir gets deleted so needs to be unique
@@ -112,8 +123,31 @@ function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString)
         @info "loadDFG! detected a gzip $dstname -- unpacking via $loaddir now..."
         Base.rm(folder, recursive=true, force=true)
         # unzip the tar file
-        run(`tar -zxf $dstname -C $loaddir`)
+
+        # TODO deprecated, remove. Kept for legacy support if older tarbals
+        if useDeprExtract
+            @warn "Old FileDFG compressed tar files are deprecated, load with useDeprExtract=true and use saveDFG again to update"
+            run(`tar -zxf $dstname -C $loaddir`)
+        else
+            tar_gz = open(dstname)
+            tar = CodecZlib.GzipDecompressorStream(tar_gz)
+            Tar.extract(tar, folder)
+            close(tar)
+        end
+
+        #or for non-compressed
+        # Tar.extract(dstname, folder)
     end
+
+    #GraphsDFG metadata
+    if overwriteDFGMetadata 
+        @assert isa(dfgLoadInto, GraphsDFG) "Only GraphsDFG metadata are supported"
+        @info "loading dfg metadata"
+        jstr = read("$folder/dfg.json", String)
+        fgPacked = JSON3.read(jstr, GraphsDFGs.PackedGraphsDFG)
+        GraphsDFGs.unpackDFGMetadata!(dfgLoadInto, fgPacked)
+    end
+
     # extract the factor graph from fileDFG folder
     variables = DFGVariable[]
     factors = DFGFactor[]
@@ -175,5 +209,31 @@ function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString)
     return dfgLoadInto
 end
 
-# to be extended by users with particular choices in dispatch.
-function loadDFG end
+function loadDFG(file::AbstractString)
+
+     # add if doesn't have .tar.gz extension
+    if !contains(basename(file), ".tar.gz")
+        file *= ".tar.gz"
+    end
+    # check the file actually exists
+    @assert isfile(file) "cannot find file $file"
+    
+    # only extract dfg.json to rebuild DFG object
+    tar_gz = open(file)
+    tar = CodecZlib.GzipDecompressorStream(tar_gz)
+    loaddir = Tar.extract(hdr -> contains(hdr.path, "dfg.json"), tar)
+    close(tar)
+    
+    #Only GraphsDFG metadata supported
+    jstr = read("$loaddir/dfg.json", String)
+    fgPacked = JSON3.read(jstr, GraphsDFGs.PackedGraphsDFG)
+    dfg = GraphsDFGs.unpackDFGMetadata(fgPacked)
+
+
+    @debug "DFG.loadDFG! is deleting a temp folder created during unzip, $loaddir"
+    # cleanup temporary folder
+    Base.rm(loaddir, recursive=true, force=true)
+
+    return loadDFG!(dfg, file)
+
+end

src/GraphsDFG/GraphsDFG.jl

@@ -5,6 +5,7 @@ using DocStringExtensions
 using UUIDs
 using JSON3
 using OrderedCollections
+using StructTypes
 
 using ...DistributedFactorGraphs
 
@@ -59,6 +60,7 @@ using .FactorGraphs
 # Imports
 include("entities/GraphsDFG.jl")
 include("services/GraphsDFG.jl")
+include("services/GraphsDFGSerialization.jl")
 
 # Exports
 export GraphsDFG

src/GraphsDFG/services/GraphsDFGSerialization.jl

@@ -0,0 +1,54 @@
+using InteractiveUtils
+
+@kwdef struct PackedGraphsDFG{T<:AbstractParams}
+    description::String
+    userLabel::String
+    robotLabel::String
+    sessionLabel::String
+    userData::Dict{Symbol, SmallDataTypes}
+    robotData::Dict{Symbol, SmallDataTypes}
+    sessionData::Dict{Symbol, SmallDataTypes}
+    userBlobEntries::OrderedDict{Symbol, BlobEntry}
+    robotBlobEntries::OrderedDict{Symbol, BlobEntry}
+    sessionBlobEntries::OrderedDict{Symbol, BlobEntry}
+    addHistory::Vector{Symbol}
+    solverParams::T
+    solverParams_type::String = string(typeof(solverParams))
+    #TODO
+    # blobStores::Dict{Symbol, AbstractBlobStore}
+end
+
+StructTypes.StructType(::Type{PackedGraphsDFG}) = StructTypes.AbstractType()
+StructTypes.subtypekey(::Type{PackedGraphsDFG}) = :solverParams_type
+#TODO look at StructTypes.@register_struct_subtype when new StructTypes.jl is tagged (for type field)
+
+function StructTypes.subtypes(::Type{PackedGraphsDFG}) 
+    subs = subtypes(AbstractParams)
+    NamedTuple(map(s->Symbol(s)=>PackedGraphsDFG{s}, subs))
+end
+
+##
+"""
+    $(SIGNATURES)
+Packing function to serialize DFG metadata from.
+"""
+function packDFGMetadata(fg::GraphsDFG)
+    commonfields = intersect(fieldnames(PackedGraphsDFG), fieldnames(GraphsDFG))
+    props = (k => getproperty(fg, k) for k in commonfields)
+    return PackedGraphsDFG(;props...)
+end
+
+function unpackDFGMetadata(packed::PackedGraphsDFG)
+    commonfields = intersect(fieldnames(GraphsDFG), fieldnames(PackedGraphsDFG))
+    props = (k => getproperty(packed, k) for k in commonfields)
+    GraphsDFG(;props...)
+end
+
+function unpackDFGMetadata!(dfg::GraphsDFG, packed::PackedGraphsDFG)
+    commonfields = intersect(fieldnames(GraphsDFG), fieldnames(PackedGraphsDFG))
+    props = (k => getproperty(packed, k) for k in commonfields)
+    foreach(props) do (k,v)
+        setproperty!(dfg, k, v)
+    end    
+    return dfg
+end

src/entities/AbstractDFG.jl

@@ -37,9 +37,12 @@ abstract type AbstractDFG{T<:AbstractParams} end
 $(TYPEDEF)
 Empty structure for solver parameters.
 """
-struct NoSolverParams <: AbstractParams
+@kwdef struct NoSolverParams <: AbstractParams
+    d::Int = 0#FIXME JSON3.jl error MethodError: no method matching read(::StructTypes.SingletonType, ...
 end
 
+StructTypes.StructType(::NoSolverParams) = StructTypes.Struct()
+
 """
 Types valid for small data.
 """

test/data/0_23_0.tar.gz

@@ -42,6 +42,39 @@ using UUIDs
         #test multihypo
         addFactor!(dfg, [:x1, :x2, :x3], LinearRelative(Normal(50.0,2.0)), multihypo = [1, 0.3, 0.7])
 
+
+        #test user/robot/session metadata
+
+        
+        #test user/robot/session blob entries
+        be = BlobEntry(
+            nothing, 
+            nothing, 
+            uuid4(), 
+            :testing2, 
+            :store, 
+            "", 
+            "", 
+            "", 
+            "", 
+            "", 
+            ZonedDateTime(2023, 2, 3, 20, tz"UTC+1"), 
+            "BlobEntry", 
+            string(DFG._getDFGVersion())
+        )
+
+        addSessionBlobEntry!(dfg, be)
+        #TODO addRobotBlobEntry!(dfg, be)
+        #TODO addUserBlobEntry!(dfg, be)
+        smallUserData = Dict{Symbol, SmallDataTypes}(:a => "42", :b => "small_user")
+        smallRobotData = Dict{Symbol, SmallDataTypes}(:a => "43", :b => "small_robot")
+        smallSessionData = Dict{Symbol, SmallDataTypes}(:a => "44", :b => "small_session")
+
+        setUserData!(dfg, smallUserData)
+        setRobotData!(dfg, smallRobotData)
+        setSessionData!(dfg, smallSessionData)
+    
+
         # Save and load the graph to test.
         saveDFG(filename, dfg)