Skip to content

Commit 9ac3d94

Browse files
AffieAffie
authored
Add User/Robot/Session BlobEntries and Blob fix (#1020)
* addBlob! store data name fallback * Session Blob Entries
1 parent a3178cc commit 9ac3d94

File tree

11 files changed

+232
-84
lines changed

11 files changed

+232
-84
lines changed

‎Project.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name = "DistributedFactorGraphs"
22
uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
3-
version = "0.21.2"
3+
version = "0.22.0"
44

55
[deps]
66
Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

‎src/DataBlobs/services/BlobStores.jl

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ function getBlob(dfg::AbstractDFG, entry::BlobEntry)
9898
end
9999
throw(
100100
KeyError(
101-
"could not find $(entry.label), uuid $blobId) in any of the listed blobstores:\n $([s->getKey(s) for (s,v) in stores]))"
101+
"could not find $(entry.label), uuid $(entry.blobId) in any of the listed blobstores:\n $([s->getKey(s) for (s,v) in stores]))"
102102
)
103103
)
104104
end
@@ -125,6 +125,9 @@ addBlob!(store::AbstractBlobStore, data) =
125125
addBlob!(store::AbstractBlobStore, blobId::UUID, data, ::String) =
126126
addBlob!(store, blobId, data)
127127

128+
addBlob!(store::AbstractBlobStore, data, ::String) =
129+
addBlob!(store, uuid4(), data)
130+
128131
#update
129132
updateBlob!(dfg::AbstractDFG, entry::BlobEntry, data::T) where {T} =
130133
updateBlob!(getBlobStore(dfg, entry.blobstore), entry, data)

‎src/DistributedFactorGraphs.jl

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,9 @@ using Pkg
3030
using TensorCast
3131
using ProgressMeter
3232
using SHA
33-
using OrderedCollections
3433
using FileIO
34+
using OrderedCollections
35+
export OrderedDict
3536

3637
using CSV
3738

@@ -68,6 +69,9 @@ export getDescription, setDescription!,
6869
getRobotData, setRobotData!,
6970
getSessionData, setSessionData!,
7071
getAddHistory
72+
73+
export getSessionBlobEntry, addSessionBlobEntry!
74+
7175
export getBlobStore,
7276
addBlobStore!,
7377
updateBlobStore!,

‎src/FileDFG/services/FileDFG.jl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,8 @@ function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString)
124124
!isdir(varFolder) && error("Can't load DFG graph - folder '$varFolder' doesn't exist")
125125
!isdir(factorFolder) && error("Can't load DFG graph - folder '$factorFolder' doesn't exist")
126126

127-
varFiles = readdir(varFolder)
128-
factorFiles = readdir(factorFolder)
127+
varFiles = sort(readdir(varFolder; sort=false); lt=natural_lt)
128+
factorFiles = sort(readdir(factorFolder; sort=false); lt=natural_lt)
129129
@showprogress 1 "loading variables" for varFile in varFiles
130130
jstr = read("$varFolder/$varFile", String)
131131
try

‎src/GraphsDFG/GraphsDFG.jl

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ module GraphsDFGs
33
using Graphs
44
using DocStringExtensions
55
using UUIDs
6+
using JSON3
7+
using OrderedCollections
68

79
using ...DistributedFactorGraphs
810

@@ -44,7 +46,11 @@ import ...DistributedFactorGraphs: setSolverParams!,
4446
_getDuplicatedEmptyDFG,
4547
toDot,
4648
toDotFile,
47-
findShortestPathDijkstra
49+
findShortestPathDijkstra,
50+
getSessionBlobEntry,
51+
getSessionBlobEntries,
52+
addSessionBlobEntry!,
53+
addSessionBlobEntries!
4854

4955
include("FactorGraphs/FactorGraphs.jl")
5056
using .FactorGraphs

‎src/GraphsDFG/entities/GraphsDFG.jl

Lines changed: 76 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,12 @@ mutable struct GraphsDFG{T <: AbstractParams, V <: AbstractDFGVariable, F <:Abst
1313
userLabel::String
1414
robotLabel::String
1515
sessionLabel::String
16-
userData::Dict{Symbol, String}
17-
robotData::Dict{Symbol, String}
18-
sessionData::Dict{Symbol, String}
16+
userData::Dict{Symbol, SmallDataTypes}
17+
robotData::Dict{Symbol, SmallDataTypes}
18+
sessionData::Dict{Symbol, SmallDataTypes}
19+
userBlobEntries::OrderedDict{Symbol, BlobEntry}
20+
robotBlobEntries::OrderedDict{Symbol, BlobEntry}
21+
sessionBlobEntries::OrderedDict{Symbol, BlobEntry}
1922
addHistory::Vector{Symbol} #TODO: Discuss more - is this an audit trail?
2023
solverParams::T # Solver parameters
2124
blobStores::Dict{Symbol, <:AbstractBlobStore}
@@ -35,46 +38,40 @@ function GraphsDFG{T,V,F}(
3538
userLabel::String="DefaultUser",
3639
robotLabel::String="DefaultRobot",
3740
sessionLabel::String="Session_$(string(uuid4())[1:6])",
38-
userData::Dict{Symbol, String} = Dict{Symbol, String}(),
39-
robotData::Dict{Symbol, String} = Dict{Symbol, String}(),
40-
sessionData::Dict{Symbol, String} = Dict{Symbol, String}(),
41+
userData::Dict{Symbol, SmallDataTypes} = Dict{Symbol, SmallDataTypes}(),
42+
robotData::Dict{Symbol, SmallDataTypes} = Dict{Symbol, SmallDataTypes}(),
43+
sessionData::Dict{Symbol, SmallDataTypes} = Dict{Symbol, SmallDataTypes}(),
44+
userBlobEntries::OrderedDict{Symbol, BlobEntry} = OrderedDict{Symbol, BlobEntry}(),
45+
robotBlobEntries::OrderedDict{Symbol, BlobEntry} = OrderedDict{Symbol, BlobEntry}(),
46+
sessionBlobEntries::OrderedDict{Symbol, BlobEntry} = OrderedDict{Symbol, BlobEntry}(),
47+
addHistory::Vector{Symbol} = Symbol[],
4148
solverParams::T=T(),
4249
blobstores=Dict{Symbol, AbstractBlobStore}(),
43-
# deprecating
44-
userId::Union{Nothing, String} = nothing,
45-
robotId::Union{Nothing, String} = nothing,
46-
sessionId::Union{Nothing, String} = nothing,
4750
) where {T <: AbstractParams, V <:AbstractDFGVariable, F<:AbstractDFGFactor}
4851
# Validate the userLabel, robotLabel, and sessionLabel
49-
if userId !== nothing
50-
@error "Obsolete use of userId::String with GraphsDFG, use userLabel::String instead" maxlog=10
51-
userLabel = userId
52-
end
53-
if robotId !== nothing
54-
@error "Obsolete use of robotId::String with GraphsDFG, use robotLabel::String instead" maxlog=10
55-
robotLabel = robotId
56-
end
57-
if sessionId !== nothing
58-
@error "Obsolete use of sessionId::String with GraphsDFG, use sessionLabel::String instead" maxlog=10
59-
sessionLabel = sessionId
60-
end
61-
6252
!isValidLabel(userLabel) && error("'$userLabel' is not a valid User label")
6353
!isValidLabel(robotLabel) && error("'$robotLabel' is not a valid Robot label")
6454
!isValidLabel(sessionLabel) && error("'$sessionLabel' is not a valid Session label")
65-
return GraphsDFG{T,V,F}(g, description, userLabel, robotLabel, sessionLabel, userData, robotData, sessionData, Symbol[], solverParams, blobstores)
55+
56+
return GraphsDFG{T,V,F}(
57+
g,
58+
description,
59+
userLabel,
60+
robotLabel,
61+
sessionLabel,
62+
userData,
63+
robotData,
64+
sessionData,
65+
userBlobEntries,
66+
robotBlobEntries,
67+
sessionBlobEntries,
68+
addHistory,
69+
solverParams,
70+
blobstores
71+
)
6672
end
6773

6874
# GraphsDFG{T}(; kwargs...) where T <: AbstractParams = GraphsDFG{T,DFGVariable,DFGFactor}(;kwargs...)
69-
70-
"""
71-
$(SIGNATURES)
72-
73-
Create an in-memory GraphsDFG with the following parameters:
74-
- T: Solver parameters (defaults to `NoSolverParams()`)
75-
- V: Variable type
76-
- F: Factor type
77-
"""
7875
function GraphsDFG{T}(g::FactorGraph{Int,DFGVariable,DFGFactor}=FactorGraph{Int,DFGVariable,DFGFactor}();
7976
kwargs...) where T <: AbstractParams
8077
return GraphsDFG{T,DFGVariable,DFGFactor}(g; kwargs...)
@@ -86,25 +83,51 @@ function GraphsDFG(g::FactorGraph{Int,DFGVariable,DFGFactor}=FactorGraph{Int,DFG
8683
end
8784

8885

89-
GraphsDFG(description::String,
90-
userLabel::String,
91-
robotLabel::String,
92-
sessionLabel::String,
93-
userData::Dict{Symbol, String},
94-
robotData::Dict{Symbol, String},
95-
sessionData::Dict{Symbol, String},
96-
solverParams::AbstractParams,
97-
blobstores=Dict{Symbol, AbstractBlobStore}()) =
98-
GraphsDFG(FactorGraph{Int,DFGVariable,DFGFactor}(), description, userLabel, robotLabel, sessionLabel, userData, robotData, sessionData, Symbol[], solverParams, blobstores)
86+
GraphsDFG(
87+
description::String,
88+
userLabel::String,
89+
robotLabel::String,
90+
sessionLabel::String,
91+
userData::Dict{Symbol, SmallDataTypes},
92+
robotData::Dict{Symbol, SmallDataTypes},
93+
sessionData::Dict{Symbol, SmallDataTypes},
94+
solverParams::AbstractParams,
95+
blobstores=Dict{Symbol, AbstractBlobStore}()
96+
) = GraphsDFG{typeof(solverParams),DFGVariable,DFGFactor}(
97+
FactorGraph{Int,DFGVariable,DFGFactor}();
98+
description,
99+
userLabel,
100+
robotLabel,
101+
sessionLabel,
102+
userData,
103+
robotData,
104+
sessionData,
105+
solverParams,
106+
blobstores
107+
)
99108

100109

101-
GraphsDFG{T,V,F}(description::String,
102-
userLabel::String,
103-
robotLabel::String,
104-
sessionLabel::String,
105-
userData::Dict{Symbol, String},
106-
robotData::Dict{Symbol, String},
107-
sessionData::Dict{Symbol, String},
108-
solverParams::T,
109-
blobstores=Dict{Symbol, AbstractBlobStore}()) where {T <: AbstractParams, V <:AbstractDFGVariable, F<:AbstractDFGFactor} =
110-
GraphsDFG(FactorGraph{Int,V,F}(), description, userLabel, robotLabel, sessionLabel, userData, robotData, sessionData, Symbol[], solverParams, blobstores)
110+
function GraphsDFG{T,V,F}(
111+
description::String,
112+
userLabel::String,
113+
robotLabel::String,
114+
sessionLabel::String,
115+
userData::Dict{Symbol, SmallDataTypes},
116+
robotData::Dict{Symbol, SmallDataTypes},
117+
sessionData::Dict{Symbol, SmallDataTypes},
118+
solverParams::T,
119+
blobstores=Dict{Symbol, AbstractBlobStore}()
120+
) where {T <: AbstractParams, V <:AbstractDFGVariable, F<:AbstractDFGFactor}
121+
return GraphsDFG{T,V,F}(
122+
FactorGraph{Int,V,F}();
123+
description,
124+
userLabel,
125+
robotLabel,
126+
sessionLabel,
127+
userData,
128+
robotData,
129+
sessionData,
130+
solverParams,
131+
blobstores
132+
)
133+
end

‎src/GraphsDFG/services/GraphsDFG.jl

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -480,3 +480,54 @@ function traverseGraphTopologicalSort(fg::GraphsDFG, s::Symbol, fs_tree=bfs_tree
480480
return symlist
481481
end
482482

483+
484+
# FG blob entries
485+
# session blob entries
486+
function getBlobEntry(var::AbstractDFGVariable, key::Symbol)
487+
if !hasBlobEntry(var, key)
488+
throw(KeyError("No dataEntry label $(key) found in variable $(getLabel(var)). Available keys: $(keys(var.dataDict))"))
489+
end
490+
return var.dataDict[key]
491+
end
492+
493+
function getSessionBlobEntry(fg::GraphsDFG, label::Symbol)
494+
return fg.sessionBlobEntries[label]
495+
end
496+
497+
function getSessionBlobEntries(fg::GraphsDFG, startwith::Union{Nothing,String}=nothing)
498+
entries = collect(values(fg.sessionBlobEntries))
499+
!isnothing(startwith) && filter!(e->startswith(string(e.label), startwith), entries)
500+
return entries
501+
end
502+
503+
function addSessionBlobEntry!(fg::GraphsDFG, entry::BlobEntry)
504+
if haskey(fg.sessionBlobEntries, entry.label)
505+
error("BlobEntry '$(entry.label)' already exists in the factor graph's session blob entries.")
506+
end
507+
push!(fg.sessionBlobEntries, entry.label=>entry)
508+
return entry
509+
end
510+
511+
function addSessionBlobEntries!(fg::GraphsDFG, entries::Vector{BlobEntry})
512+
return map(entries) do entry
513+
addSessionBlobEntry!(fg, entry)
514+
end
515+
end
516+
517+
# function getSessionBlobEntry(fg::GraphsDFG, label::Symbol)
518+
# return JSON3.read(fg.sessionData[label], BlobEntry)
519+
# end
520+
521+
# function getSessionBlobEntries(fg::GraphsDFG, startwith::Union{Nothing,String}=nothing)
522+
# entries = map(values(fg.sessionData)) do entry
523+
# JSON3.read(entry, BlobEntry)
524+
# end
525+
# !isnothing(startwith) && filter!(e->startswith(string(e.label), startwith), entries)
526+
# return entries
527+
# end
528+
529+
# function addSessionBlobEntries!(fg::GraphsDFG, entries::Vector{BlobEntry})
530+
# return map(entries) do entry
531+
# push!(fg.sessionData, entry.label=>JSON3.write(entry))
532+
# end
533+
# end

‎src/services/AbstractDFG.jl

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -100,14 +100,14 @@ $SIGNATURES
100100
101101
Get the user data associated with the graph.
102102
"""
103-
getUserData(dfg::AbstractDFG) = return dfg.userData
103+
getUserData(dfg::AbstractDFG) = dfg.userData
104104

105105
"""
106106
$SIGNATURES
107107
108108
Set the user data associated with the graph.
109109
"""
110-
function setUserData!(dfg::AbstractDFG, data::Dict{Symbol, String})
110+
function setUserData!(dfg::AbstractDFG, data::Dict{Symbol, SmallDataTypes})
111111
dfg.userData = data #TODO keep memory? use clear and then add
112112
return dfg.userData
113113
end
@@ -117,14 +117,14 @@ $SIGNATURES
117117
118118
Get the robot data associated with the graph.
119119
"""
120-
getRobotData(dfg::AbstractDFG)::Union{Nothing, Dict{Symbol, String}} = return dfg.robotData
120+
getRobotData(dfg::AbstractDFG) = dfg.robotData
121121

122122
"""
123123
$SIGNATURES
124124
125125
Set the robot data associated with the graph.
126126
"""
127-
function setRobotData!(dfg::AbstractDFG, data::Dict{Symbol, String})
127+
function setRobotData!(dfg::AbstractDFG, data::Dict{Symbol, SmallDataTypes})
128128
dfg.robotData = data
129129
return dfg.robotData
130130
end
@@ -134,14 +134,14 @@ $SIGNATURES
134134
135135
Get the session data associated with the graph.
136136
"""
137-
getSessionData(dfg::AbstractDFG)::Dict{Symbol, String} = return dfg.sessionData
137+
getSessionData(dfg::AbstractDFG) = dfg.sessionData
138138

139139
"""
140140
$SIGNATURES
141141
142142
Set the session data associated with the graph.
143143
"""
144-
function setSessionData!(dfg::AbstractDFG, data::Dict{Symbol, String})
144+
function setSessionData!(dfg::AbstractDFG, data::Dict{Symbol, SmallDataTypes})
145145
dfg.sessionData = data
146146
return dfg.sessionData
147147
end
@@ -151,9 +151,9 @@ end
151151
##==============================================================================
152152

153153
#NOTE with API standardization this should become something like:
154-
getUserData(dfg::AbstractDFG, key::Symbol)::String = dfg.userData[key]
155-
getRobotData(dfg::AbstractDFG, key::Symbol)::String = dfg.robotData[key]
156-
getSessionData(dfg::AbstractDFG, key::Symbol)::String = dfg.sessionData[key]
154+
getUserData(dfg::AbstractDFG, key::Symbol) = dfg.userData[key]
155+
getRobotData(dfg::AbstractDFG, key::Symbol) = dfg.robotData[key]
156+
getSessionData(dfg::AbstractDFG, key::Symbol) = dfg.sessionData[key]
157157

158158
updateUserData!(dfg::AbstractDFG, pair::Pair{Symbol,String}) = push!(dfg.userData, pair)
159159
updateRobotData!(dfg::AbstractDFG, pair::Pair{Symbol,String}) = push!(dfg.robotData, pair)
@@ -169,6 +169,30 @@ emptySessionData!(dfg::AbstractDFG) = empty!(dfg.sessionData)
169169

170170
#TODO add__Data!?
171171

172+
##==============================================================================
173+
## User/Robot/Session Blob Entries CRUD
174+
##==============================================================================
175+
function getSessionBlobEntry end
176+
function getSessionBlobEntries end
177+
function addSessionBlobEntry! end
178+
function addSessionBlobEntries! end
179+
function updateSessionBlobEntry! end
180+
function deleteSessionBlobEntry! end
181+
182+
function getRobotBlobEntry end
183+
function getRobotBlobEntries end
184+
function addRobotBlobEntry! end
185+
function addRobotBlobEntries! end
186+
function updateRobotBlobEntry! end
187+
function deleteRobotBlobEntry! end
188+
189+
function getUserBlobEntry end
190+
function getUserBlobEntries end
191+
function addUserBlobEntry! end
192+
function addUserBlobEntries! end
193+
function updateUserBlobEntry! end
194+
function deleteUserBlobEntry! end
195+
172196
##==============================================================================
173197
## AbstractBlobStore CRUD
174198
##==============================================================================

0 commit comments

Comments
 (0)