Skip to content

Commit 0a5eab9

Browse files
dehanndehann
authored
Merge pull request #933 from JuliaRobotics/23Q1/refac/vartopjson
serialize DFGVariable top level JSON only
2 parents 8f98091 + f7e899f commit 0a5eab9

File tree

4 files changed

+122
-26
lines changed

4 files changed

+122
-26
lines changed

‎Project.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name = "DistributedFactorGraphs"
22
uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
3-
version = "0.18.10"
3+
version = "0.19.0"
44

55
[deps]
66
Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

‎src/DataBlobs/entities/AbstractDataEntries.jl

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,33 @@ struct BlobStoreEntry <: AbstractDataEntry
3131
createdTimestamp::ZonedDateTime # of when the entry was created
3232
end
3333

34+
_fixtimezone(cts::NamedTuple) = ZonedDateTime(cts.utc_datetime*"+00")
35+
36+
# needed for deserialization from JSON during DFG v0.19 transition, see #867
37+
function BlobStoreEntry(;
38+
label,
39+
id,
40+
blobstore,
41+
hash,
42+
origin,
43+
description,
44+
mimeType,
45+
createdTimestamp,
46+
kwargs... # drop excessive fields
47+
)
48+
#
49+
BlobStoreEntry(
50+
Symbol(label),
51+
UUID(id),
52+
Symbol(blobstore),
53+
hash,
54+
origin,
55+
description,
56+
mimeType,
57+
_fixtimezone(createdTimestamp),
58+
)
59+
end
60+
3461
# TODO
3562
"""
3663
$(TYPEDEF)

‎src/FileDFG/services/FileDFG.jl

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,14 +43,14 @@ function saveDFG(folder::AbstractString, dfg::AbstractDFG)
4343
for v in variables
4444
vPacked = packVariable(dfg, v)
4545
io = open("$varFolder/$(v.label).json", "w")
46-
JSON2.write(io, vPacked)
46+
println(io,JSON.json(vPacked))
4747
close(io)
4848
end
4949
# Factors
5050
for f in factors
5151
fPacked = packFactor(dfg, f)
5252
io = open("$factorFolder/$(f.label).json", "w")
53-
JSON2.write(io, fPacked)
53+
println(io,JSON.json(fPacked))
5454
close(io)
5555
end
5656

@@ -130,10 +130,11 @@ function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString)
130130
varFiles = readdir(varFolder)
131131
factorFiles = readdir(factorFolder)
132132
for varFile in varFiles
133-
open("$varFolder/$varFile") do io
134-
packedData = JSON2.read(io, Dict{String, Any})
133+
packedData = JSON.parsefile("$varFolder/$varFile"; dicttype=Dict{String, Any})
134+
# open("$varFolder/$varFile") do io
135+
# packedData = JSON.parse(io; dicttype=Dict{String, Any})
135136
push!(variables, unpackVariable(dfgLoadInto, packedData))
136-
end
137+
# end
137138
end
138139
@info "Loaded $(length(variables)) variables - $(map(v->v.label, variables))"
139140
@info "Inserting variables into graph..."

‎src/services/Serialization.jl

Lines changed: 88 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ end
5353

5454
# Corrects any `::ZonedDateTime` fields of T in corresponding `interm::Dict` as `dateformat"yyyy-mm-ddTHH:MM:SS.ssszzz"`
5555
function standardizeZDTStrings!(T, interm::Dict)
56+
5657
for (name, typ) in zip(fieldnames(T), T.types)
5758
if typ <: ZonedDateTime
5859
namestr = string(name)
@@ -125,16 +126,16 @@ function packVariable(dfg::AbstractDFG, v::DFGVariable)
125126
props["label"] = string(v.label)
126127
props["timestamp"] = Dates.format(v.timestamp, "yyyy-mm-ddTHH:MM:SS.ssszzz")
127128
props["nstime"] = string(v.nstime.value)
128-
props["tags"] = JSON2.write(v.tags)
129-
props["ppeDict"] = JSON2.write(v.ppeDict)
130-
props["solverDataDict"] = JSON2.write(Dict(keys(v.solverDataDict) .=> map(vnd -> packVariableNodeData(dfg, vnd), values(v.solverDataDict))))
131-
props["smallData"] = JSON2.write(v.smallData)
129+
props["tags"] = v.tags # JSON2.write(v.tags)
130+
props["ppeDict"] = v.ppeDict # JSON2.write(v.ppeDict)
131+
props["solverDataDict"] = (Dict(keys(v.solverDataDict) .=> map(vnd -> packVariableNodeData(dfg, vnd), values(v.solverDataDict)))) # JSON2.write
132+
props["smallData"] = v.smallData # JSON2.write(v.smallData)
132133
props["solvable"] = v.solvable
133134
props["variableType"] = typeModuleName(getVariableType(v))
134-
props["dataEntry"] = JSON2.write(Dict(keys(v.dataDict) .=> map(bde -> JSON.json(bde), values(v.dataDict))))
135-
props["dataEntryType"] = JSON2.write(Dict(keys(v.dataDict) .=> map(bde -> typeof(bde), values(v.dataDict))))
135+
props["dataEntry"] = (Dict(keys(v.dataDict) .=> values(v.dataDict))) # map(bde -> JSON.json(bde), values(v.dataDict)))) #JSON2.write
136+
props["dataEntryType"] = (Dict(keys(v.dataDict) .=> map(bde -> typeof(bde), values(v.dataDict)))) #JSON2.write
136137
props["_version"] = _getDFGVersion()
137-
return props::Dict{String, Any}
138+
return props #::Dict{String, Any}
138139
end
139140

140141
"""
@@ -192,7 +193,14 @@ function _unpackVariableNodeData(
192193
return unpackVariableNodeData(dfg, packedVND)
193194
end
194195

195-
# returns a DFGVariable
196+
"""
197+
$SIGNATURES
198+
Returns a DFGVariable.
199+
200+
DevNotes
201+
- v0.19 packVariable fixed nested JSON bug on these fields, see #867:
202+
- `tags`, `ppeDict`, `solverDataDict`, `smallData`, `dataEntry`, `dataEntryType`
203+
"""
196204
function unpackVariable(
197205
dfg::AbstractDFG,
198206
packedProps::Dict{String, Any};
@@ -236,7 +244,19 @@ function unpackVariable(
236244
Dict{Symbol, MeanMaxPPE}()
237245
end
238246

239-
smallData = JSON2.read(packedProps["smallData"], Dict{Symbol, SmallDataTypes})
247+
smallData = if haskey(packedProps, "smallData")
248+
if packedProps["smallData"] isa String
249+
JSON2.read(packedProps["smallData"], Dict{Symbol, SmallDataTypes})
250+
elseif packedProps["smallData"] isa Dict
251+
Dict{Symbol, SmallDataTypes}( Symbol.(keys(packedProps["smallData"])) .=> values(packedProps["smallData"]) )
252+
# packedProps["smallData"]
253+
else
254+
@warn "unknown smallData deserialization on $label, type $(typeof(packedProps["smallData"]))" maxlog=10
255+
Dict{Symbol, SmallDataTypes}()
256+
end
257+
else
258+
Dict{Symbol, SmallDataTypes}()
259+
end
240260

241261
variableTypeString = packedProps["variableType"]
242262

@@ -246,7 +266,11 @@ function unpackVariable(
246266

247267
# FIXME, drop nested packing, see DFG #867
248268
solverData = if unpackSolverData && haskey(packedProps, "solverDataDict")
249-
packed = JSON2.read(packedProps["solverDataDict"], Dict{String, PackedVariableNodeData})
269+
packed = if packedProps["solverDataDict"] isa String
270+
JSON2.read(packedProps["solverDataDict"], Dict{String, PackedVariableNodeData})
271+
else
272+
packedProps["solverDataDict"]
273+
end
250274
Dict{Symbol, VariableNodeData{variableType, pointType}}(Symbol.(keys(packed)) .=> map(p -> unpackVariableNodeData(dfg, p), values(packed)))
251275
elseif unpackPPEs && haskey(packedProps,"solverData") && packedProps["solverData"] isa AbstractVector
252276
solverdict = Dict{Symbol, VariableNodeData{variableType, pointType}}()
@@ -275,16 +299,42 @@ function unpackVariable(
275299
# Now rehydrate complete DataEntry type.
276300
if unpackBigData
277301
#TODO Deprecate - for backward compatibility between v0.8 and v0.9, remove in v0.10
278-
dataElemTypes = JSON2.read(packedProps["dataEntryType"], Dict{Symbol, Symbol})
302+
dataElemTypes = if packedProps["dataEntryType"] isa String
303+
JSON2.read(packedProps["dataEntryType"], Dict{Symbol, String})
304+
else
305+
# packedProps["dataEntryType"]
306+
Dict{Symbol, String}( Symbol.(keys(packedProps["dataEntryType"])) .=> values(packedProps["dataEntryType"]) )
307+
end
279308
for (k,name) in dataElemTypes
280-
dataElemTypes[k] = Symbol(split(string(name), '.')[end])
309+
val = split(string(name), '.')[end]
310+
dataElemTypes[k] = val
311+
end
312+
313+
dataIntermed = if packedProps["dataEntry"] isa String
314+
JSON2.read(packedProps["dataEntry"], Dict{Symbol, String})
315+
elseif packedProps["dataEntry"] isa NamedTuple
316+
# case where JSON2 did unpacking of all fields as hard types (no longer String)
317+
# Dict{Symbol, String}( Symbol.(keys(packedProps["dataEntry"])) .=> values(packedProps["dataEntry"]) )
318+
for i in 1:length(packedProps["dataEntry"])
319+
k = keys(packedProps["dataEntry"])[i]
320+
bdeInter = values(packedProps["dataEntry"])[i]
321+
objType = getfield(DistributedFactorGraphs, Symbol(dataElemTypes[k]))
322+
# standardizeZDTStrings!(objType, bdeInter)
323+
# fullVal = Unmarshal.unmarshal(objType, bdeInter)
324+
variable.dataDict[k] = objType(;bdeInter...)
325+
end
326+
# forcefully skip, since variabe.dataDict already populated here
327+
Dict{Symbol,String}()
328+
else
329+
Dict( Symbol.(keys(packedProps["dataEntry"])) .=> values(packedProps["dataEntry"]) )
281330
end
282331

283-
dataIntermed = JSON2.read(packedProps["dataEntry"], Dict{Symbol, String})
332+
_doparse(s) = s
333+
_doparse(s::String) = JSON.parse(s)
284334

285335
for (k,bdeInter) in dataIntermed
286-
interm = JSON.parse(bdeInter)
287-
objType = getfield(DistributedFactorGraphs, dataElemTypes[k])
336+
interm = _doparse(bdeInter) # JSON.parse(bdeInter) # bdeInter
337+
objType = getfield(DistributedFactorGraphs, Symbol(dataElemTypes[k]))
288338
standardizeZDTStrings!(objType, interm)
289339
fullVal = Unmarshal.unmarshal(objType, interm)
290340
variable.dataDict[k] = fullVal
@@ -329,7 +379,7 @@ function packVariableNodeData(::G, d::VariableNodeData{T}) where {G <: AbstractD
329379
d.solveKey)
330380
end
331381

332-
function unpackVariableNodeData(dfg::G, d::PackedVariableNodeData) where G <: AbstractDFG
382+
function unpackVariableNodeData(dfg::G, d::Union{<:PackedVariableNodeData,<:NamedTuple}) where G <: AbstractDFG
333383
@debug "Dispatching conversion packed variable -> variable for type $(string(d.variableType))"
334384
# Figuring out the variableType
335385
# TODO deprecated remove in v0.11 - for backward compatibility for saved variableTypes.
@@ -352,13 +402,31 @@ function unpackVariableNodeData(dfg::G, d::PackedVariableNodeData) where G <: Ab
352402
BW = reshape(d.vecbw,r4,c4)
353403

354404
#
355-
return VariableNodeData{T, getPointType(T)}(vals, BW, d.BayesNetOutVertIDs,
356-
d.dimIDs, d.dims, d.eliminated, d.BayesNetVertID, d.separator,
357-
T(), d.initialized, d.infoPerCoord, d.ismargin, d.dontmargin,
358-
d.solveInProgress, d.solvedCount, d.solveKey,
405+
return VariableNodeData{T, getPointType(T)}(
406+
vals,
407+
BW,
408+
Symbol.(d.BayesNetOutVertIDs),
409+
d.dimIDs,
410+
d.dims,
411+
d.eliminated,
412+
Symbol(d.BayesNetVertID),
413+
Symbol.(d.separator),
414+
T(),
415+
d.initialized,
416+
d.infoPerCoord,
417+
d.ismargin,
418+
d.dontmargin,
419+
d.solveInProgress,
420+
d.solvedCount,
421+
Symbol(d.solveKey),
359422
Dict{Symbol,Threads.Condition}() )
360423
end
361424

425+
unpackVariableNodeData(
426+
dfg::AbstractDFG,
427+
d::Dict) = unpackVariableNodeData(dfg, Unmarshal.unmarshal(PackedVariableNodeData, d))
428+
429+
362430
##==============================================================================
363431
## Factor Packing and unpacking
364432
##==============================================================================

0 commit comments

Comments
 (0)