compat fixes on loading fileDFG

Author: dehann

Project.toml

@@ -11,6 +11,7 @@ DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 JSON2 = "2535ab7d-5cd8-5a07-80ac-9b1792aadce3"
+JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"

src/DistributedFactorGraphs.jl

@@ -25,6 +25,7 @@ using Reexport
 using JSON
 using Unmarshal
 using JSON2 # JSON2 requires all properties to be in correct sequence, can't guarantee that from DB.
+# using JSON3
 using LinearAlgebra
 using SparseArrays
 using UUIDs

src/FileDFG/services/FileDFG.jl

@@ -129,7 +129,7 @@ function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString)
 
     varFiles = readdir(varFolder)
     factorFiles = readdir(factorFolder)
-    for varFile in varFiles
+    @showprogress 1 "loading variables" for varFile in varFiles
         packedData = JSON.parsefile("$varFolder/$varFile"; dicttype=Dict{String, Any})
         # open("$varFolder/$varFile") do io
         #     packedData = JSON.parse(io; dicttype=Dict{String, Any})
@@ -141,7 +141,7 @@ function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString)
     # Adding variables
     map(v->addVariable!(dfgLoadInto, v), variables)
 
-    for factorFile in factorFiles
+    @showprogress 1 "loading factors" for factorFile in factorFiles
         open("$factorFolder/$factorFile") do io
             packedData = JSON2.read(io, Dict{String, Any})
             push!(factors, unpackFactor(dfgLoadInto, packedData))
@@ -155,7 +155,7 @@ function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString)
     # Finally, rebuild the CCW's for the factors to completely reinflate them
     # NOTE CREATES A NEW DFGFactor IF  CCW TYPE CHANGES
     @info "Rebuilding CCW's for the factors..."
-    for factor in factors
+    @showprogress 1 "build factor operational memory" for factor in factors
         rebuildFactorMetadata!(dfgLoadInto, factor)
     end
 

src/services/Serialization.jl

@@ -307,7 +307,11 @@ function unpackVariable(
 
     # FIXME, drop nested packing, see DFG #867
     #   string serialization using packVariable and CGDFG serialization (Vector{String})
-    tags = Symbol.(packedProps["tags"])
+    tags_ = if packedProps["tags"] isa String
+        Symbol.(JSON2.read(packedProps["tags"], Vector{String}))
+    else
+        Symbol.(packedProps["tags"])
+    end
 
     # FIXME, drop nested packing, see DFG #867
     ppeDict = if unpackPPEs && haskey(packedProps,"ppesDict")
@@ -345,15 +349,24 @@ function unpackVariable(
     isnothing(variableType) && error("Cannot deserialize variableType '$variableTypeString' in variable '$label'")
     pointType = getPointType(variableType)
 
+    _ensureid!(s::Dict) = begin s["id"] = haskey(s, "id") ? s["id"] : nothing end
+    _ensureid!(s::PackedVariableNodeData) = s
+
     # FIXME, drop nested packing, see DFG #867
     solverData = if unpackSolverData && haskey(packedProps, "solverDataDict")
         packed = if packedProps["solverDataDict"] isa String
-            JSON2.read(packedProps["solverDataDict"], Dict{String, PackedVariableNodeData})
+            # JSON2.read(packedProps["solverDataDict"], Dict{String, PackedVariableNodeData})
+            # JSON3.read(packedProps["solverDataDict"], Dict{String, PackedVariableNodeData})
+            jdc = JSON.parse(packedProps["solverDataDict"])
+            jpvd = Dict{String,PackedVariableNodeData}()
+            for (k,v) in jdc
+                _ensureid!(v)
+                jpvd[k] = transcodeType(PackedVariableNodeData, v)
+            end
+            jpvd
         else
             packedProps["solverDataDict"]
         end
-        _ensureid!(s::Dict) = begin s["id"]=nothing; end
-        _ensureid!(s::PackedVariableNodeData) = s
         packedvals = values(packed)
         _ensureid!.(packedvals)
         # TODO deprecate, this is for DFG18 compat only
@@ -365,17 +378,20 @@ function unpackVariable(
     # Rebuild DFGVariable using the first solver variableType in solverData
     # @info "dbg Serialization 171" variableType Symbol(packedProps["label"]) timestamp nstime ppeDict solverData smallData Dict{Symbol,AbstractDataEntry}() Ref(packedProps["solvable"])
     # variable = DFGVariable{variableType}(Symbol(packedProps["label"]), timestamp, nstime, Set(tags), ppeDict, solverData,  smallData, Dict{Symbol,AbstractDataEntry}(), Ref(packedProps["solvable"]))
-    variable = DFGVariable( id = id,
-                            Symbol(packedProps["label"]), 
-                            variableType, 
-                            timestamp=timestamp, 
-                            nstime=nstime, 
-                            tags=Set{Symbol}(tags), 
-                            estimateDict=ppeDict, 
-                            solverDataDict=solverData,  
-                            smallData=smallData, 
-                            dataDict=Dict{Symbol,AbstractDataEntry}(), 
-                            solvable=packedProps["solvable"] )
+    
+    variable = DFGVariable{variableType}(;
+        id=id,
+        label = Symbol(packedProps["label"]), 
+        # variableType = variableType, 
+        timestamp = timestamp, 
+        nstime = nstime, 
+        tags = Set{Symbol}(tags_), 
+        ppeDict = ppeDict, 
+        solverDataDict = solverData,  
+        smallData= smallData, 
+        dataDict = Dict{Symbol,AbstractDataEntry}(), 
+        solvable = Ref(Int(packedProps["solvable"])) 
+    )
     #
 
     # Now rehydrate complete DataEntry type.