Skip to content

Commit 93c9973

Browse files
authored
Merge pull request #431 from JuliaRobotics/master
v0.7.6-rc1
2 parents 72d150d + fb7b781 commit 93c9973

23 files changed

+487
-157
lines changed

.travis.yml

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
language: julia
22
sudo: required
3-
# dist: trusty
43

54
os:
65
- linux
@@ -12,26 +11,30 @@ services:
1211
- neo4j
1312

1413
julia:
15-
- 1.2
1614
- 1.4
1715
- nightly
1816

1917
env:
2018
- IIF_TEST=false
2119

20+
branches:
21+
only:
22+
- master
23+
- develop
24+
- /^release-.*$/
25+
2226
jobs:
2327
include:
24-
- julia: 1.3
25-
dist: trusty
26-
env: IIF_TEST=true
28+
- julia: 1.4
29+
env:
30+
- IIF_TEST=true
31+
- SKIP_CGDFG_TESTS=true
2732
if: NOT branch =~ ^release.*$
28-
# Set the password for Neo4j to neo4j:test
29-
before_script:
30-
- sleep 10
31-
- curl -v POST http://neo4j:neo4j@localhost:7474/user/neo4j/password -d"password=test"
3233
- arch: arm64
34+
env: SKIP_CGDFG_TESTS=true
35+
before_script:
3336
- stage: "Documentation"
34-
julia: 1.3
37+
julia: 1.4
3538
os: linux
3639
script:
3740
- julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
@@ -41,26 +44,24 @@ jobs:
4144
allow_failures:
4245
- julia: nightly
4346
- arch: arm64
44-
#- env: IIF_TEST=true
4547

4648
notifications:
4749
email: false
4850

49-
5051
# Attempt to install neo4j on Xenial and set the password for Neo4j to neo4j:test
51-
# Cloud tests errors with authentication error
52-
#before_script:
53-
# - sudo add-apt-repository -y ppa:openjdk-r/ppa
54-
# - sudo apt-get update
55-
# - wget -O - https://debian.neo4j.com/neotechnology.gpg.key | sudo apt-key add -
56-
# - echo 'deb https://debian.neo4j.com stable latest' | sudo tee -a /etc/apt/sources.list.d/neo4j.list
57-
# - sudo apt-get update
58-
# - apt list -a neo4j
59-
# - sudo apt-get install neo4j=1:4.0.0
60-
# - sudo service neo4j start
61-
# - sleep 10
62-
# - sudo neo4j-admin set-initial-password test
63-
# - curl -I http://localhost:7474/
52+
before_script:
53+
- sudo add-apt-repository -y ppa:openjdk-r/ppa
54+
- sudo apt-get update
55+
- wget -O - https://debian.neo4j.com/neotechnology.gpg.key | sudo apt-key add -
56+
- echo 'deb https://debian.neo4j.com stable 3.5' | sudo tee /etc/apt/sources.list.d/neo4j.list
57+
- sudo apt-get update
58+
- apt list -a neo4j
59+
- sudo apt-get install neo4j
60+
- sudo service neo4j start
61+
- sleep 10
62+
- curl -v POST http://neo4j:neo4j@localhost:7474/user/neo4j/password -d"password=test"
63+
# - sudo neo4j-admin set-initial-password test
64+
- curl -I http://localhost:7474/
6465

6566
after_success:
6667
- julia -e 'using Pkg; cd(Pkg.dir("DistributedFactorGraphs")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())'

Project.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name = "DistributedFactorGraphs"
22
uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
3-
version = "0.7.5"
3+
version = "0.7.6"
44

55
[deps]
66
Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

src/BigData/services/AbstractBigDataEntries.jl

Lines changed: 72 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,21 +41,31 @@ addDataEntry!(x...) = addBigDataEntry!(x...)
4141
$(SIGNATURES)
4242
Add Big Data Entry to distributed factor graph.
4343
Should be extended if DFG variable is not returned by reference.
44+
45+
Example
46+
47+
See docs for `getDataEntryElement`.
48+
49+
Related
50+
51+
addData!, getDataEntryElement, fetchData
4452
"""
4553
function addDataEntry!(dfg::AbstractDFG,
4654
lbl::Symbol,
55+
datastore::Union{FileDataStore, InMemoryDataStore},
4756
descr::Symbol,
4857
mimeType::AbstractString,
4958
data::Vector{UInt8} )
5059
#
5160
node = isVariable(dfg, lbl) ? getVariable(dfg, lbl) : getFactor(dfg, lbl)
5261
# Make a big data entry in the graph - use JSON2 to just write this
53-
element = GeneralBigDataEntry(dfg, node, descr, mimeType=mimeType)
62+
entry = GeneralBigDataEntry(dfg, node, descr, mimeType=mimeType)
5463
# Set it in the store
55-
addBigData!(fec.datastore, element, data)
64+
addBigData!(datastore, entry, data)
5665
# Add the entry to the graph
57-
addBigDataEntry!(node, element)
66+
addBigDataEntry!(node, entry)
5867
end
68+
const addData! = addDataEntry!
5969

6070
"""
6171
$SIGNATURES
@@ -78,9 +88,68 @@ function getBigDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol)
7888
return getBigDataEntry(getVariable(dfg, label), key)
7989
end
8090

91+
"""
92+
$SIGNATURES
93+
Get both the entry and raw data element from datastore returning as a tuple.
94+
95+
Notes:
96+
- This is the counterpart to `addDataEntry!`.
97+
- Data is identified by the node in the DFG object `dfglabel::Symbol` as well as `datalabel::Symbol`.
98+
- The data should have been stored along with a `entry.mimeType::String` which describes the format of the data.
99+
- ImageMagick.jl is useful for storing images in png or jpg compressed format.
100+
101+
Example
102+
103+
```julia
104+
# some dfg object
105+
fg = initfg()
106+
addVariable!(fg, :x0, IIF.ContinuousScalar) # using IncrementalInference
107+
# FileDataStore (as example)
108+
datastore = FileDataStore(joinLogPath(fg,"datastore"))
109+
110+
# now some data comes in
111+
mydata = Dict(:soundBite => randn(Float32, 10000), :meta => "something about lazy foxes and fences.")
112+
113+
# store the data
114+
addDataEntry!( fg, :x0, datastore, :SOUND_DATA, "application/json", Vector{UInt8}(JSON2.write( mydata )) )
115+
116+
# get/fetch the data
117+
entry, rawData = fetchData(fg, :x0, datastore, :SOUND_DATA)
118+
119+
# unpack data to original format (this must be done by the user)
120+
@show entry.mimeType # "applicatio/json"
121+
userFormat = JSON2.read(IOBuffer(rawData))
122+
```
123+
124+
Related
125+
126+
addDataEntry!, addData!, fetchData, fetchDataEntryElement
127+
"""
128+
function getDataEntryElement(dfg::AbstractDFG,
129+
dfglabel::Symbol,
130+
datastore::Union{FileDataStore, InMemoryDataStore},
131+
datalabel::Symbol)
132+
#
133+
vari = getVariable(dfg, dfglabel)
134+
if !hasDataEntry(vari, datalabel)
135+
@error "missing data entry $datalabel in $dfglabel"
136+
return nothing, nothing
137+
end
138+
entry = getBigDataEntry(vari, datalabel)
139+
element = getBigData(datastore, entry)
140+
return entry, element
141+
end
142+
const fetchDataEntryElement = getDataEntryElement
143+
const fetchData = getDataEntryElement
144+
145+
146+
81147
"""
82148
$(SIGNATURES)
83149
Update big data entry
150+
151+
DevNote
152+
- DF, unclear if `update` verb is applicable in this case, see #404
84153
"""
85154
function updateBigDataEntry!(var::AbstractDFGVariable, bde::AbstractBigDataEntry)
86155
!haskey(var.bigData,bde.key) && (@warn "$(bde.key) does not exist in variable, adding")

src/CloudGraphsDFG/entities/CGStructure.jl

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,34 @@ mutable struct User <: AbstractCGNode
1010
id::Symbol
1111
name::String
1212
description::String
13-
# labels::Vector{Symbol}
1413
data::Dict{Symbol, String}
14+
createdTimestamp::String
15+
lastUpdatedTimestamp::String
16+
User(id::Symbol,
17+
name::String,
18+
description::String,
19+
data::Dict{Symbol, String},
20+
createdTimestamp::String=string(now(UTC)),
21+
lastUpdatedTimestamp::String=string(now(UTC))) =
22+
new(id, name, description, data, createdTimestamp, lastUpdatedTimestamp)
1523
end
1624

1725
mutable struct Robot <: AbstractCGNode
1826
id::Symbol
1927
userId::Symbol
2028
name::String
2129
description::String
22-
# labels::Vector{Symbol}
2330
data::Dict{Symbol, String}
31+
createdTimestamp::String
32+
lastUpdatedTimestamp::String
33+
Robot(id::Symbol,
34+
userId::Symbol,
35+
name::String,
36+
description::String,
37+
data::Dict{Symbol, String},
38+
createdTimestamp::String=string(now(UTC)),
39+
lastUpdatedTimestamp::String=string(now(UTC))) =
40+
new(id, userId, name, description, data, createdTimestamp, lastUpdatedTimestamp)
2441
end
2542

2643
mutable struct Session <: AbstractCGNode
@@ -29,6 +46,17 @@ mutable struct Session <: AbstractCGNode
2946
userId::Symbol
3047
name::String
3148
description::String
32-
# labels::Vector{Symbol}
3349
data::Dict{Symbol, String}
50+
createdTimestamp::String
51+
lastUpdatedTimestamp::String
52+
Session(id::Symbol,
53+
robotId::Symbol,
54+
userId::Symbol,
55+
name::String,
56+
description::String,
57+
data::Dict{Symbol, String},
58+
createdTimestamp::String=string(now(UTC)),
59+
lastUpdatedTimestamp::String=string(now(UTC))) =
60+
new(id, robotId,userId, name, description, data, createdTimestamp, lastUpdatedTimestamp)
61+
3462
end

src/CloudGraphsDFG/entities/CloudGraphsDFG.jl

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,14 @@ function CloudGraphsDFG{T}(neo4jConnection::Neo4j.Connection,
3232
getPackedTypeFunc,
3333
decodePackedTypeFunc,
3434
rebuildFactorMetadata!;
35-
solverParams::T=NoSolverParams()) where T <: AbstractParams
35+
solverParams::T=NoSolverParams(),
36+
createSessionNodes::Bool=true) where T <: AbstractParams
3637
graph = Neo4j.getgraph(neo4jConnection)
3738
neo4jInstance = Neo4jInstance(neo4jConnection, graph)
38-
return CloudGraphsDFG{T}(neo4jInstance, userId, robotId, sessionId, description, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc, rebuildFactorMetadata!, Symbol[], solverParams)
39+
dfg = CloudGraphsDFG{T}(neo4jInstance, userId, robotId, sessionId, description, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc, rebuildFactorMetadata!, Symbol[], solverParams)
40+
# Create the session if it doesn't already exist
41+
createSessionNodes && createDfgSessionIfNotExist(dfg)
42+
return dfg
3943
end
4044
"""
4145
$(SIGNATURES)
@@ -53,9 +57,10 @@ function CloudGraphsDFG{T}(host::String,
5357
getPackedTypeFunc,
5458
decodePackedTypeFunc,
5559
rebuildFactorMetadata!;
56-
solverParams::T=NoSolverParams()) where T <: AbstractParams
60+
solverParams::T=NoSolverParams(),
61+
createSessionNodes::Bool=true) where T <: AbstractParams
5762
neo4jConnection = Neo4j.Connection(host, port=port, user=dbUser, password=dbPassword);
58-
return CloudGraphsDFG{T}(neo4jConnection, userId, robotId, sessionId, description, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc, rebuildFactorMetadata!, solverParams=solverParams)
63+
return CloudGraphsDFG{T}(neo4jConnection, userId, robotId, sessionId, description, encodePackedTypeFunc, getPackedTypeFunc, decodePackedTypeFunc, rebuildFactorMetadata!, solverParams=solverParams, createSessionNodes=createSessionNodes)
5964
end
6065

6166

src/CloudGraphsDFG/services/CGStructure.jl

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@ function _convertDictToSession(dict::Dict{String, Any})::Session
3232
Symbol(dict["userId"]),
3333
dict["name"],
3434
dict["description"],
35-
data)
35+
data,
36+
dict["createdTimestamp"],
37+
dict["lastUpdatedTimestamp"])
3638
return session
3739
end
3840
#TODO: Refactor, #HACK :D (but it works!)
@@ -43,7 +45,9 @@ function _convertDictToRobot(dict::Dict{String, Any})::Robot
4345
Symbol(dict["userId"]),
4446
dict["name"],
4547
dict["description"],
46-
data)
48+
data,
49+
dict["createdTimestamp"],
50+
dict["lastUpdatedTimestamp"])
4751
return robot
4852
end
4953
#TODO: Refactor, #HACK :D (but it works!)
@@ -53,7 +57,9 @@ function _convertDictToUser(dict::Dict{String, Any})::User
5357
Symbol(dict["id"]),
5458
dict["name"],
5559
dict["description"],
56-
data)
60+
data,
61+
dict["createdTimestamp"],
62+
dict["lastUpdatedTimestamp"])
5763
return user
5864
end
5965

@@ -114,7 +120,7 @@ function createDfgSessionIfNotExist(dfg::CloudGraphsDFG)::Session
114120
strip(dfg.sessionId) == "" && error("Session ID is not populated in DFG.")
115121
user = User(Symbol(dfg.userId), dfg.userId, "Description for $(dfg.userId)", Dict{Symbol, String}())
116122
robot = Robot(Symbol(dfg.robotId), Symbol(dfg.userId), dfg.robotId, "Description for $(dfg.userId):$(dfg.robotId)", Dict{Symbol, String}())
117-
session = Session(Symbol(dfg.sessionId), Symbol(dfg.robotId), Symbol(dfg.userId), dfg.sessionId, "Description for $(dfg.userId):$(dfg.robotId):$(dfg.sessionId)", Dict{Symbol, String}())
123+
session = Session(Symbol(dfg.sessionId), Symbol(dfg.robotId), Symbol(dfg.userId), dfg.sessionId, dfg.description, Dict{Symbol, String}())
118124

119125
_getNodeCount(dfg.neo4jInstance, [dfg.userId, "USER"]) == 0 && createUser(dfg, user)
120126
_getNodeCount(dfg.neo4jInstance, [dfg.userId, dfg.robotId, "ROBOT"]) == 0 && createRobot(dfg, robot)

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ function _getDuplicatedEmptyDFG(dfg::CloudGraphsDFG)::CloudGraphsDFG
99
while true #do..while loop
1010
count += 1
1111
sessionId = dfg.sessionId*"_$count"
12-
length(_getLabelsFromCyphonQuery(dfg.neo4jInstance, "(node:$(dfg.userId):$(dfg.robotId):$(sessionId))")) == 0 && break
12+
_getNodeCount(dfg.neo4jInstance, [dfg.userId, dfg.robotId, sessionId]) == 0 && break
1313
end
1414
@debug "Unique+empty copy session name: $sessionId"
1515
return CloudGraphsDFG{typeof(dfg.solverParams)}(
@@ -375,7 +375,7 @@ function listFactors(dfg::CloudGraphsDFG, regexFilter::Union{Nothing, Regex}=not
375375
end
376376
end
377377

378-
function isFullyConnected(dfg::CloudGraphsDFG)::Bool
378+
function isConnected(dfg::CloudGraphsDFG)::Bool
379379
# If the total number of nodes == total number of distinct connected nodes, then it is fully connected
380380
# Total nodes
381381
varIds = listVariables(dfg)
@@ -400,7 +400,7 @@ function getNeighbors(dfg::CloudGraphsDFG, node::T; solvable::Int=0)::Vector{Sym
400400
query = "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(node.label))--(node) where (node:VARIABLE or node:FACTOR) and node.solvable >= $solvable"
401401
@debug "[Query] $query"
402402
neighbors = _getLabelsFromCyphonQuery(dfg.neo4jInstance, query)
403-
# If factor, need to do variable ordering
403+
# If factor, need to do variable ordering TODO, Do we? does it matter if we always use _variableOrderSymbols in calculations?
404404
if T <: DFGFactor
405405
neighbors = intersect(node._variableOrderSymbols, neighbors)
406406
end
@@ -411,7 +411,7 @@ function getNeighbors(dfg::CloudGraphsDFG, label::Symbol; solvable::Int=0)::Vect
411411
query = "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(label))--(node) where (node:VARIABLE or node:FACTOR) and node.solvable >= $solvable"
412412
@debug "[Query] $query"
413413
neighbors = _getLabelsFromCyphonQuery(dfg.neo4jInstance, query)
414-
# If factor, need to do variable ordering
414+
# If factor, need to do variable ordering TODO, Do we? does it matter if we always use _variableOrderSymbols in calculations?
415415
if isFactor(dfg, label)
416416
# Server is authority
417417
serverOrder = Symbol.(JSON2.read(_getNodeProperty(dfg.neo4jInstance, [dfg.userId, dfg.robotId, dfg.sessionId, String(label)], "_variableOrderSymbols")))
@@ -520,10 +520,11 @@ function addPPE!(dfg::CloudGraphsDFG, variablekey::Symbol, ppe::P, ppekey::Symbo
520520
if ppekey in listPPEs(dfg, variablekey, currentTransaction=currentTransaction)
521521
error("PPE '$(ppekey)' already exists")
522522
end
523-
return updatePPE!(dfg, variablekey, ppe, ppekey, currentTransaction=currentTransaction)
523+
return _matchmergePPE!(dfg, variablekey, ppe, ppekey, currentTransaction=currentTransaction)
524524
end
525525

526-
function updatePPE!(dfg::CloudGraphsDFG, variablekey::Symbol, ppe::P, ppekey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::P where P <: AbstractPointParametricEst
526+
#TODO clean this, just to match api for update
527+
function _matchmergePPE!(dfg::CloudGraphsDFG, variablekey::Symbol, ppe::P, ppekey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::P where P <: AbstractPointParametricEst
527528
packed = packPPE(dfg, ppe)
528529
query = """
529530
MATCH (var:$variablekey:$(join(_getLabelsForType(dfg, DFGVariable, parentKey=variablekey),':')))
@@ -546,10 +547,17 @@ function updatePPE!(dfg::CloudGraphsDFG, variablekey::Symbol, ppe::P, ppekey::Sy
546547
return unpackPPE(dfg, result.results[1]["data"][1]["row"][1])
547548
end
548549

550+
function updatePPE!(dfg::CloudGraphsDFG, variablekey::Symbol, ppe::P, ppekey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::P where P <: AbstractPointParametricEst
551+
if !(ppekey in listPPEs(dfg, variablekey, currentTransaction=currentTransaction))
552+
@warn "PPE '$(ppekey)' does not exist, adding"
553+
end
554+
return _matchmergePPE!(dfg, variablekey, ppe, ppekey, currentTransaction=currentTransaction)
555+
end
556+
549557
function updatePPE!(dfg::CloudGraphsDFG, sourceVariables::Vector{<:DFGVariable}, ppekey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)
550558
tx = currentTransaction == nothing ? transaction(dfg.neo4jInstance.connection) : currentTransaction
551559
for var in sourceVariables
552-
updatePPE!(dfg, var.label, getPPE(dfg, var, ppekey), ppekey, currentTransaction=tx)
560+
updatePPE!(dfg, var.label, getPPE(var, ppekey), ppekey, currentTransaction=tx)
553561
end
554562
if currentTransaction == nothing
555563
result = commit(tx)

0 commit comments

Comments
 (0)