Skip to content

Commit 180e54d

Browse files
GearsADGearsAD
committed
Efficient PPE calls for CGDFG
1 parent 35d13a3 commit 180e54d

File tree

6 files changed

+146
-21
lines changed

6 files changed

+146
-21
lines changed

‎Project.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,15 @@ Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
1010
DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
1111
GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231"
1212
Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
13+
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
1314
JSON2 = "2535ab7d-5cd8-5a07-80ac-9b1792aadce3"
1415
LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d"
1516
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
1617
Neo4j = "d2adbeaf-5838-5367-8a2f-e46d570981db"
1718
Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
1819
Requires = "ae029012-a4dd-5104-9daa-d747884805df"
1920
SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
21+
Unmarshal = "cbff2730-442d-58d7-89d1-8e530c41eb02"
2022

2123
[compat]
2224
Colors = "0.8, 0.9, 0.10, 0.11"

‎src/CloudGraphsDFG/services/CloudGraphsDFG.jl

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,3 +453,101 @@ function getBiadjacencyMatrix(dfg::CloudGraphsDFG; solvable::Int=0)::NamedTuple{
453453

454454
return (B=adjMat, varLabels=varLabels, facLabels=factLabels)
455455
end
456+
457+
### PPEs with DB calls
458+
459+
function listPPE(dfg::CloudGraphsDFG, variablekey::Symbol; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::Vector{Symbol}
460+
query = "match (ppe:PPE:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$variablekey) return ppe.solverKey"
461+
@debug "[CGDFG] PPE read query:\r\n$query"
462+
result = nothing
463+
if currentTransaction != nothing
464+
result = currentTransaction(query; submit=true)
465+
else
466+
tx = transaction(dfg.neo4jInstance.connection)
467+
tx(query)
468+
result = commit(tx)
469+
end
470+
length(result.errors) > 0 && error(string(result.errors))
471+
vals = map(d -> d["row"][1], result.results[1]["data"])
472+
return Symbol.(vals)
473+
end
474+
475+
function getPPE(dfg::CloudGraphsDFG, variablekey::Symbol, ppekey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::AbstractPointParametricEst
476+
query = "match (ppe:PPE:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$variablekey:$(ppekey)) return properties(ppe)"
477+
@debug "[CGDFG] PPE read query:\r\n$query"
478+
result = nothing
479+
if currentTransaction != nothing
480+
result = currentTransaction(query; submit=true)
481+
else
482+
tx = transaction(dfg.neo4jInstance.connection)
483+
tx(query)
484+
result = commit(tx)
485+
end
486+
length(result.errors) > 0 && error(string(result.errors))
487+
length(result.results[1]["data"]) != 1 && error("Cannot find PPE '$ppekey' for variable '$variablekey'")
488+
length(result.results[1]["data"][1]["row"]) != 1 && error("Cannot find PPE '$ppekey' for variable '$variablekey'")
489+
return unpackPPE(dfg, result.results[1]["data"][1]["row"][1])
490+
end
491+
492+
function addPPE!(dfg::CloudGraphsDFG, variablekey::Symbol, ppe::P, ppekey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::AbstractPointParametricEst where P <: AbstractPointParametricEst
493+
if ppekey in listPPE(dfg, variablekey, ppekey, currentTransaction=currentTransaction)
494+
error("PPE '$(ppekey)' already exists")
495+
end
496+
return updatePPE!(dfg, variablekey, ppe, ppekey, currentTransaction=currentTransaction)
497+
end
498+
499+
function updatePPE!(dfg::CloudGraphsDFG, variablekey::Symbol, ppe::P, ppekey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::P where P <: AbstractPointParametricEst
500+
packed = packPPE(dfg, ppe)
501+
query = """
502+
MATCH (var:VARIABLE:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$variablekey)
503+
MERGE (ppe:PPE:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$variablekey:$(ppe.solverKey))
504+
SET ppe = $(_dictToNeo4jProps(packed))
505+
CREATE UNIQUE (var)-[:PPE]->(ppe)
506+
RETURN properties(ppe)"""
507+
@debug "[CGDFG] PPE update query:\r\n$query"
508+
result = nothing
509+
if currentTransaction != nothing
510+
result = currentTransaction(query; submit=true) # TODO: Maybe we should submit (; submit = true) for the results to fail early?
511+
else
512+
tx = transaction(dfg.neo4jInstance.connection)
513+
tx(query)
514+
result = commit(tx)
515+
end
516+
length(result.errors) > 0 && error(string(result.errors))
517+
length(result.results[1]["data"]) != 1 && error("Cannot find PPE '$(ppe.solverKey)' for variable '$variablekey'")
518+
length(result.results[1]["data"][1]["row"]) != 1 && error("Cannot find PPE '$(ppe.solverKey)' for variable '$variablekey'")
519+
return unpackPPE(dfg, result.results[1]["data"][1]["row"][1])
520+
end
521+
522+
function updatePPE!(dfg::CloudGraphsDFG, sourceVariables::Vector{<:DFGVariable}, ppekey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)
523+
tx = currentTransaction == nothing ? transaction(dfg.neo4jInstance.connection) : currentTransaction
524+
for var in sourceVariables
525+
updatePPE!(dfg, var.label, getPPE(dfg, var, ppekey), ppekey, currentTransaction=tx)
526+
end
527+
if currentTransaction == nothing
528+
result = commit(tx)
529+
end
530+
return nothing
531+
end
532+
533+
function deletePPE!(dfg::CloudGraphsDFG, variablekey::Symbol, ppekey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::AbstractPointParametricEst
534+
query = """
535+
match (ppe:PPE:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$variablekey:$(ppekey))
536+
with ppe, properties(ppe) as props
537+
detach delete ppe
538+
return props
539+
"""
540+
@debug "[CGDFG] PPE delete query:\r\n$query"
541+
result = nothing
542+
if currentTransaction != nothing
543+
result = currentTransaction(query; submit=true) # TODO: Maybe we should submit (; submit = true) for the results to fail early?
544+
else
545+
tx = transaction(dfg.neo4jInstance.connection)
546+
tx(query)
547+
result = commit(tx)
548+
end
549+
length(result.errors) > 0 && error(string(result.errors))
550+
length(result.results[1]["data"]) != 1 && error("Cannot find PPE '$ppekey' for variable '$variablekey'")
551+
length(result.results[1]["data"][1]["row"]) != 1 && error("Cannot find PPE '$ppekey' for variable '$variablekey'")
552+
return unpackPPE(dfg, @show result.results[1]["data"][1]["row"][1])
553+
end

‎src/CloudGraphsDFG/services/CommonFunctions.jl

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -478,3 +478,14 @@ function _addGraphChangeNotification(userId::String, robotId::String, sessionId:
478478
Main.App.NaviConfig.addGraphChange(conn, gc)
479479
return true
480480
end
481+
482+
"""
483+
$(SIGNATURES)
484+
485+
Build a Cypher-compliant set of properies from a JSON dictionary.
486+
Note individual values are serialized if they are not already.
487+
"""
488+
function _dictToNeo4jProps(dict::Dict{String, Any})::String
489+
# TODO: Use an IO buffer/stream for this
490+
return "{" * join(map((k) -> "$k: $(JSON.json(dict[k]))", collect(keys(dict))), ", ")*"}"
491+
end

‎src/DistributedFactorGraphs.jl

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ using Requires
66
using Dates
77
using Distributions
88
using Reexport
9-
using JSON2
9+
using JSON
10+
using Unmarshal
11+
using JSON2 # JSON2 requires all properties to be in correct sequence, can't guarantee that from DB.
1012
using LinearAlgebra
1113
using SparseArrays
1214

‎src/LightDFG/entities/LightDFG.jl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ Create an in-memory LightDFG with the following parameters:
3030
"""
3131
function LightDFG{T,V,F}(g::FactorGraph{Int,V,F}=FactorGraph{Int,V,F}();
3232
description::String="LightGraphs.jl implementation",
33-
userId::String="User ID",
34-
robotId::String="Robot ID",
35-
sessionId::String="Session ID",
33+
userId::String="UserID",
34+
robotId::String="RobotID",
35+
sessionId::String="SessionID",
3636
userData::Dict{Symbol, String} = Dict{Symbol, String}(),
3737
robotData::Dict{Symbol, String} = Dict{Symbol, String}(),
3838
sessionData::Dict{Symbol, String} = Dict{Symbol, String}(),

‎src/services/DFGVariable.jl

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import Base: ==, convert
22

3+
# For all types that pack their type into their own structure (e.g. PPE)
4+
const TYPEKEY = "_type"
5+
36
function packVariable(dfg::G, v::DFGVariable)::Dict{String, Any} where G <: AbstractDFG
47
props = Dict{String, Any}()
58
props["label"] = string(v.label)
@@ -82,23 +85,32 @@ function unpackVariableNodeData(dfg::G, d::PackedVariableNodeData)::VariableNode
8285
st(), d.initialized, d.inferdim, d.ismargin, d.dontmargin, d.solveInProgress, d.solvedCount)
8386
end
8487

85-
# function compare(a::VariableNodeData, b::VariableNodeData)
86-
# a.val != b.val && @debug("val is not equal")==nothing && return false
87-
# a.bw != b.bw && @debug("bw is not equal")==nothing && return false
88-
# a.BayesNetOutVertIDs != b.BayesNetOutVertIDs && @debug("BayesNetOutVertIDs is not equal")==nothing && return false
89-
# a.dimIDs != b.dimIDs && @debug("dimIDs is not equal")==nothing && return false
90-
# a.dims != b.dims && @debug("dims is not equal")==nothing && return false
91-
# a.eliminated != b.eliminated && @debug("eliminated is not equal")==nothing && return false
92-
# a.BayesNetVertID != b.BayesNetVertID && @debug("BayesNetVertID is not equal")==nothing && return false
93-
# a.separator != b.separator && @debug("separator is not equal")==nothing && return false
94-
# a.initialized != b.initialized && @debug("initialized is not equal")==nothing && return false
95-
# abs(a.inferdim - b.inferdim) > 1e-14 && @debug("inferdim is not equal")==nothing && return false
96-
# a.ismargin != b.ismargin && @debug("ismargin is not equal")==nothing && return false
97-
# a.dontmargin != b.dontmargin && @debug("dontmargin is not equal")==nothing && return false
98-
# a.solveInProgress != b.solveInProgress && @debug("solveInProgress is not equal")==nothing && return false
99-
# typeof(a.softtype) != typeof(b.softtype) && @debug("softtype is not equal")==nothing && return false
100-
# return true
101-
# end
88+
"""
89+
$(SIGNATURES)
90+
91+
Pack a PPE into a Dict{String, Any}.
92+
"""
93+
function packPPE(dfg::G, ppe::P)::Dict{String, Any} where {G <: AbstractDFG, P <: AbstractPointParametricEst}
94+
packedPPE = JSON.parse(JSON.json(ppe)) #TODO: Maybe better way to serialize as dictionary?
95+
packedPPE[TYPEKEY] = string(typeof(ppe)) # Append the type
96+
return packedPPE
97+
end
98+
99+
"""
100+
$(SIGNATURES)
101+
102+
Unpack a Dict{String, Any} into a PPE.
103+
"""
104+
function unpackPPE(dfg::G, packedPPE::Dict{String, Any})::AbstractPointParametricEst where G <: AbstractDFG
105+
!haskey(packedPPE, TYPEKEY) && error("Cannot find type key '$TYPEKEY' in packed PPE data")
106+
type = pop!(packedPPE, TYPEKEY)
107+
(type == nothing || type == "") && error("Cannot deserialize PPE, type key is empty")
108+
ppe = Unmarshal.unmarshal(
109+
DistributedFactorGraphs.getTypeFromSerializationModule(dfg, Symbol(type)),
110+
packedPPE)
111+
return ppe
112+
end
113+
102114

103115
#FIXME
104116
# """

0 commit comments

Comments
 (0)