Merge pull request #937 from JuliaRobotics/23Q1/srz/transcodetypeunmarshal

add transcodeType util for super unmarshal

Author: dehann

Project.toml

@@ -51,6 +51,7 @@ Unmarshal = "0.4"
 julia = "1.6"
 
 [extras]
+DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Manifolds = "1cead3c2-87b3-11e9-0ccd-23c62b72b94e"
@@ -59,4 +60,4 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test", "GraphPlot", "LinearAlgebra", "Manifolds", "Pkg", "Statistics"]
+test = ["Test", "DataStructures", "GraphPlot", "LinearAlgebra", "Manifolds", "Pkg", "Statistics"]

src/services/Serialization.jl

@@ -117,6 +117,106 @@ function getTypeFromSerializationModule(_typeString::AbstractString)
     nothing
 end
 
+## =============================================================================
+## Transcoding / Unmarshal types helper
+## =============================================================================
+
+"""
+    $SIGNATURES
+Should be a highly reusable function for any transcoding of intermediate type (or dict) to a desired output type.
+
+examples
+```julia
+Base.@kwdef struct HardType
+    name::String
+    time::DateTime = now(UTC)
+    val::Float64 = 0.0
+end
+
+# slight human overhead for each type to ignore extraneous field construction
+# TODO, devnote drop this requirement with filter of _names in transcodeType
+HardType(;
+    name::String,
+    time::DateTime = now(UTC),
+    val::Float64 = 0.0,
+    ignorekws...
+) = HardType(name,time,val)
+
+# somehow one gets an intermediate type
+imt = IntermediateType(
+    v"1.0",
+    "NotUsedYet",
+    "test",
+    now(UTC),
+    1.0
+)
+# or dict (testing string keys)
+imd = Dict(
+    "_version" => v"1.0",
+    "_type" => "NotUsedYet",
+    "name" => "test",
+    "time" => now(UTC),
+    "val" => 1.0
+)
+# ordered dict (testing symbol keys)
+iod = OrderedDict(
+    :_version => v"1.0",
+    :_type => "NotUsedYet",
+    :name => "test",
+    :time => now(UTC),
+    # :val => 1.0
+)
+
+# do the transcoding to a slighly different hard type
+T1 = transcodeType(HardType, imt)
+T2 = transcodeType(HardType, imd)
+T3 = transcodeType(HardType, iod)
+```
+"""
+function transcodeType(
+    ::Type{T},
+    inObj
+) where T
+    #
+    # specializations as inner functions (don't have to be inners)
+    # these few special cases came up with examples below, note recursions
+    _instance(S::Type, x) = S(x)
+    _instance(_::Type{S}, x::S) where S = x # if ambiguous, delete and do alternative `_instance(S::Type, x) = S===Any ? x : S(x)`
+    _instance(S::Type{I}, x::AbstractString) where I <: Number = Base.parse(I, x)
+    _instance(S::Type{E}, x::AbstractVector) where E <: AbstractVector = _instance.(eltype(E),x)
+    _instance(S::Type{<:AbstractDict{K,V}}, x::AbstractDict) where {K,V} = (tup=(Symbol.(keys(x)) .=> _instance.(V,values(x)) ) ; S(tup...) )
+
+    # what the struct wants
+    _types = fieldtypes(T)
+    _names = fieldnames(T)
+    # (closure) resolve random ordering problem
+    _getIdx(s::Symbol) = findfirst(x->x==s, _names)
+    # (closure) create an instance of a field
+    makething(k::Symbol, v) = begin 
+        idx = _getIdx(k)
+        if !isnothing(idx)
+            # this field is in the output type and should be included
+            k => _instance(_types[_getIdx(k)], v)
+        else
+            # this field should be ignored in the output type
+            Symbol(:VOID_,rand(1:1000000)) => nothing
+        end
+    end
+    # zip together keys/fields and values for either dict or intermediate type
+    _srckeys(s::AbstractDict) = keys(s)
+    _srckeys(s) = fieldnames(typeof(s))
+    _srcvals(s::AbstractDict) = values(s)
+    _srcvals(s) = map(k->getproperty(s,k), _srckeys(s))  
+    # NOTE, improvement, filter extraneous fields not in _names
+    arr = [makething(Symbol(k),v) for (k,v) in zip(_srckeys(inObj),_srcvals(inObj))]
+    filter!(s->s[1] in _names, arr)
+    # create dict provided fields into a NamedTuple as a type stable "pre-struct"
+    nt = (;arr...)
+    # use keyword constructors provided by Base.@kwdef to resolve random ordering, incomplete dicts, and defaults
+    T(;nt...)
+end
+
+
 
 ##==============================================================================
 ## Variable Packing and unpacking
@@ -128,7 +228,7 @@ function packVariable(v::DFGVariable)
     props["nstime"] = string(v.nstime.value)
     props["tags"] = v.tags # JSON2.write(v.tags)
     props["ppeDict"] = v.ppeDict # JSON2.write(v.ppeDict)
-    props["solverDataDict"] = (Dict(keys(v.solverDataDict) .=> map(vnd -> packVariableNodeData(dfg, vnd), values(v.solverDataDict)))) # JSON2.write
+    props["solverDataDict"] = (Dict(keys(v.solverDataDict) .=> map(vnd -> packVariableNodeData(vnd), values(v.solverDataDict)))) # JSON2.write
     props["smallData"] = v.smallData # JSON2.write(v.smallData)
     props["solvable"] = v.solvable
     props["variableType"] = typeModuleName(getVariableType(v))

test/runtests.jl

@@ -21,6 +21,7 @@ using UUIDs
 # logger = SimpleLogger(stdout, Logging.Debug)
 # global_logger(logger)
 
+include("testTranscodeTypeUnmarshaling.jl")
 include("test_defVariable.jl")
 
 include("testBlocks.jl")

test/testTranscodeTypeUnmarshaling.jl

@@ -0,0 +1,107 @@
+
+# test transcoding and unmarshal util
+
+
+using Test
+using DistributedFactorGraphs
+using DataStructures: OrderedDict
+using Dates
+
+##
+
+
+Base.@kwdef struct HardType
+    name::String
+    time::DateTime = now(UTC)
+    val::Float64 = 0.0
+end
+  # # slight human overhead for each type to ignore extraneous field construction
+  # # TODO, devnote drop this requirement with filter of _names in transcodeType
+  # HardType(;
+  #     name::String,
+  #     time::DateTime = now(UTC),
+  #     val::Float64 = 0.0,
+  #     ignorekws...
+  # ) = HardType(name,time,val)
+
+@testset "Test transcoding of Intermediate, Dict, OrderedDict to a HardType" begin
+
+struct IntermediateType
+  _version
+  _type
+  name
+  time
+  val
+end
+
+# somehow one gets an intermediate type
+imt = IntermediateType(
+    v"1.0",
+    "NotUsedYet",
+    "test",
+    now(UTC),
+    1.0
+)
+# or dict (testing string keys)
+imd = Dict(
+    "_version" => v"1.0",
+    "_type" => "NotUsedYet",
+    "name" => "test",
+    "time" => now(UTC),
+    "val" => 1.0
+)
+# ordered dict (testing symbol keys)
+iod = OrderedDict(
+    :_version => v"1.0",
+    :_type => "NotUsedYet",
+    :name => "test",
+    :time => now(UTC),
+    # :val => 1.0
+)
+
+# do the transcoding to a slighly different hard type
+T1 = DistributedFactorGraphs.transcodeType(HardType, imt)
+T2 = DistributedFactorGraphs.transcodeType(HardType, imd)
+T3 = DistributedFactorGraphs.transcodeType(HardType, iod)
+
+end
+
+Base.@kwdef struct MyType{T <: Real}
+  tags::Vector{Symbol} = Symbol[]
+  count::Int
+  funfun::Complex{T} = 1 + 5im
+  somedata::Dict{Symbol,Any} = Dict{Symbol, Any}()
+  data::Vector{Float64} = zeros(0)
+  binary::Vector{UInt8} = Vector{UInt8}()
+end
+
+@testset "More super unmarshaling tests of various test dicts" begin
+
+d = Dict("count" => 3)
+DistributedFactorGraphs.transcodeType(MyType, d)
+
+d2 = Dict("count" => 3, "tags" => Any["hi", "okay"])
+DistributedFactorGraphs.transcodeType(MyType, d2)
+
+d3 = Dict("count" => "3", "tags" => String["hi", "okay"])
+DistributedFactorGraphs.transcodeType(MyType, d3)
+
+d4 = Dict("count" => 3.0, "funfun" => "8 - 3im", "tags" => Any["hi", "okay"])
+DistributedFactorGraphs.transcodeType(MyType{Float32}, d4)
+
+d5 = Dict("count" => 3, "somedata" => Dict{String,Any}("calibration"=>[1.1;2.2], "description"=>"this is a test"))
+DistributedFactorGraphs.transcodeType(MyType{Float64}, d5)
+
+d6 = Dict("count" => 3.0, "data" => Any[10, 60])
+DistributedFactorGraphs.transcodeType(MyType, d6)
+
+d7 = Dict("count" => 3.0, "data" => String["10", "60"])
+DistributedFactorGraphs.transcodeType(MyType, d7)
+
+d8 = Dict("count" => 4, "binary" => take!(IOBuffer("hello world")))
+DistributedFactorGraphs.transcodeType(MyType, d8)
+
+d9 = Dict("count" => 4, "somedata" => Dict{Symbol,Any}(:test => "no ambiguity"))
+DistributedFactorGraphs.transcodeType(MyType, d9)
+
+end