Merge branch 'master' of https://github.com/JuliaRobotics/DistributedFactorGraphs.jl into 3Q20/542/saving_serialized_version

Author: GearsAD

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.9.0"
+version = "0.10.0"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

test/DataStoreTests.jl renamed to attic/DataStoreTests.jl

@@ -79,7 +79,3 @@ sleep(0.1)
 #TODO
     #listStoreEntries(ds)
 end
-
-1320*4
-
-3600+2200

attic/InMemoryDataStore.jl

@@ -133,18 +133,16 @@ Modules = [CloudGraphsDFGs]
 
 
 
-## Big Data
+## Data Entries and Blobs
 
 ```@autodocs
 Modules = [DistributedFactorGraphs]
 
-Pages = ["entities/AbstractBigDataEntries.jl",
-        "entities/AbstractDataStore.jl",
-        "entities/FileDataStore.jl",
-        "entities/InMemoryDataStore.jl",
-        "services/AbstractBigDataEntries.jl",
-        "services/AbstractDataStore.jl",
-        "services/FileDataStore.jl",
-        "services/InMemoryDataStore.jl"]
+Pages = ["entities/AbstractDataEntries.jl",
+        "services/AbstractDataEntries.jl",
+        "services/BlobStores.jl",
+        "services/DataEntryBlob.jl",
+        "services/FileDataEntryBlob.jl",
+        "services/InMemoryDataEntryBlob.jl"]
 
 ```

docs/src/func_ref.md

@@ -155,7 +155,7 @@ isFactor(dfg::CloudGraphsDFG, sym::Symbol)::Bool =
 function getSofttype(dfg::CloudGraphsDFG, lbl::Symbol; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)
     st = _getNodeProperty(dfg.neo4jInstance, union(_getLabelsForType(dfg, DFGVariable), [String(lbl)]), "softtype", currentTransaction=currentTransaction)
     @debug "Trying to find softtype: $st"
-    softType = getTypeFromSerializationModule(dfg, Symbol(st))
+    softType = getTypeFromSerializationModule(st)
     return softType()
 end
 
@@ -177,8 +177,12 @@ function updateVariable!(dfg::CloudGraphsDFG, variable::DFGVariable; skipAddErro
     # Create/update the base variable
     # NOTE: We are not merging the variable.tags into the labels anymore. We can index by that but not
     # going to pollute the graph with unnecessary (and potentially dangerous) labels.
+<<<<<<< HEAD
     addProps = Dict(
         "softtype" => "\"$(string(typeof(getSofttype(variable))))\"")
+=======
+    addProps = Dict("softtype" => "\"$(DistributedFactorGraphs.typeModuleName(getSofttype(variable)))\"")
+>>>>>>> ff56034195104ff1cd5cdee9b10f7d271eb1d5f8
     query = """
     MATCH (session:$(join(_getLabelsForType(dfg, Session), ":")))
     MERGE (node:$(join(_getLabelsForInst(dfg, variable), ":")))
@@ -237,12 +241,12 @@ function getVariable(dfg::CloudGraphsDFG, label::Union{Symbol, String})
     for ppe in listPPEs(dfg, label)
         variable.ppeDict[ppe] = getPPE(dfg, label, ppe)
     end
-    for solverKey in listVariableSolverData(dfg, label)
-        variable.solverDataDict[solverKey] = getVariableSolverData(dfg, label, solverKey)
+    for solveKey in listVariableSolverData(dfg, label)
+        variable.solverDataDict[solveKey] = getVariableSolverData(dfg, label, solveKey)
     end
-    dataDict = getDataEntries(dfg, label)
-    for (k,v) in dataDict
-        variable.dataDict[k] = v
+    dataSet = getDataEntries(dfg, label)
+    for v in dataSet
+        variable.dataDict[v.label] = v
     end
 
     return variable
@@ -259,7 +263,7 @@ function mergeVariableData!(dfg::CloudGraphsDFG, sourceVariable::DFGVariable; cu
         updateVariableSolverData!(dfg, getLabel(sourceVariable), v, currentTransaction=currentTransaction)
     end
     for (k,v) in sourceVariable.dataDict
-        updateDatEntry!(dfg, getLabel(sourceVariable), v, currentTransaction=currentTransaction)
+        updateDataEntry!(dfg, getLabel(sourceVariable), v, currentTransaction=currentTransaction)
     end
     return sourceVariable
 end
@@ -545,7 +549,7 @@ function _unpackPPE(dfg::G, packedPPE::Dict{String, Any})::AbstractPointParametr
 end
 
 function listPPEs(dfg::CloudGraphsDFG, variablekey::Symbol; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::Vector{Symbol}
-    return _listVarSubnodesForType(dfg, variablekey, MeanMaxPPE, "solverKey"; currentTransaction=currentTransaction)
+    return _listVarSubnodesForType(dfg, variablekey, MeanMaxPPE, "solveKey"; currentTransaction=currentTransaction)
 end
 
 function getPPE(dfg::CloudGraphsDFG, variablekey::Symbol, ppekey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::AbstractPointParametricEst
@@ -580,8 +584,8 @@ function addPPE!(dfg::CloudGraphsDFG,
                  ppe::P;
                  currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::AbstractPointParametricEst where
                  {P <: AbstractPointParametricEst}
-    if ppe.solverKey in listPPEs(dfg, variablekey, currentTransaction=currentTransaction)
-        error("PPE '$(ppe.solverKey)' already exists")
+    if ppe.solveKey in listPPEs(dfg, variablekey, currentTransaction=currentTransaction)
+        error("PPE '$(ppe.solveKey)' already exists")
     end
     softType = getSofttype(dfg, variablekey)
     # Add additional properties for the PPE
@@ -602,8 +606,8 @@ function updatePPE!(
         ppe::P;
         currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::P where
         {P <: AbstractPointParametricEst}
-    if !(ppe.solverKey in listPPEs(dfg, variablekey, currentTransaction=currentTransaction))
-        @warn "PPE '$(ppe.solverKey)' does not exist, adding"
+    if !(ppe.solveKey in listPPEs(dfg, variablekey, currentTransaction=currentTransaction))
+        @warn "PPE '$(ppe.solveKey)' does not exist, adding"
     end
     softType = getSofttype(dfg, variablekey, currentTransaction=currentTransaction)
     # Add additional properties for the PPE
@@ -645,11 +649,9 @@ end
 function getDataEntries(dfg::CloudGraphsDFG, label::Symbol; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)
     entries = Dict{Symbol, BlobStoreEntry}()
     # TODO: Optimize if necessary.
-    for key in listDataEntries(dfg, label, currentTransaction=currentTransaction)
-        entry = getDataEntry(dfg, label, key, currentTransaction=currentTransaction)
-        entries[entry.label] = entry
-    end
-    return entries
+    delist = listDataEntries(dfg, label, currentTransaction=currentTransaction)
+    return getDataEntry.(dfg, label, delist; currentTransaction=currentTransaction)
+
 end
 
 function listDataEntries(dfg::CloudGraphsDFG, label::Symbol; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)
@@ -667,6 +669,10 @@ function getDataEntry(dfg::CloudGraphsDFG, label::Symbol, key::Symbol; currentTr
         BlobStoreEntry, 
         key; 
         currentTransaction=currentTransaction)
+    
+    #FIXME
+    properties["createdTimestamp"] = DistributedFactorGraphs.getStandardZDTString(properties["createdTimestamp"])
+    
     return Unmarshal.unmarshal(
         BlobStoreEntry,
         properties)
@@ -683,6 +689,10 @@ function addDataEntry!(dfg::CloudGraphsDFG, label::Symbol, bde::BlobStoreEntry;
         bde,
         :DATA,
         currentTransaction=currentTransaction)
+    
+    #FIXME
+    packed["createdTimestamp"] = DistributedFactorGraphs.getStandardZDTString(packed["createdTimestamp"])
+
     return Unmarshal.unmarshal(
             BlobStoreEntry,
             packed)
@@ -699,6 +709,10 @@ function updateDataEntry!(dfg::CloudGraphsDFG, label::Symbol,  bde::BlobStoreEnt
         bde,
         :DATA,
         currentTransaction=currentTransaction)
+
+    #FIXME
+    packed["createdTimestamp"] = DistributedFactorGraphs.getStandardZDTString(packed["createdTimestamp"])
+
     return Unmarshal.unmarshal(
             BlobStoreEntry,
             packed)
@@ -712,6 +726,10 @@ function deleteDataEntry!(dfg::CloudGraphsDFG, label::Symbol, key::Symbol; curre
         _getLabelsForType(dfg, BlobStoreEntry, parentKey=label),
         key,
         currentTransaction=currentTransaction)
+    
+    #FIXME
+    props["createdTimestamp"] = DistributedFactorGraphs.getStandardZDTString(props["createdTimestamp"])
+
     return Unmarshal.unmarshal(
         BlobStoreEntry,
         props)
@@ -729,20 +747,20 @@ function _unpackVariableNodeData(dfg::G, packedDict::Dict{String, Any})::Variabl
 end
 
 function listVariableSolverData(dfg::CloudGraphsDFG, variablekey::Symbol; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::Vector{Symbol}
-    return _listVarSubnodesForType(dfg, variablekey, VariableNodeData, "solverKey"; currentTransaction=currentTransaction)
+    return _listVarSubnodesForType(dfg, variablekey, VariableNodeData, "solveKey"; currentTransaction=currentTransaction)
 end
 
-function getVariableSolverData(dfg::CloudGraphsDFG, variablekey::Symbol, solverKey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::VariableNodeData
-    properties = _getVarSubnodeProperties(dfg, variablekey, VariableNodeData, solverKey; currentTransaction=currentTransaction)
+function getVariableSolverData(dfg::CloudGraphsDFG, variablekey::Symbol, solveKey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::VariableNodeData
+    properties = _getVarSubnodeProperties(dfg, variablekey, VariableNodeData, solveKey; currentTransaction=currentTransaction)
     return _unpackVariableNodeData(dfg, properties)
 end
 
 function addVariableSolverData!(dfg::CloudGraphsDFG,
                                 variablekey::Symbol,
                                 vnd::VariableNodeData;
                                 currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::VariableNodeData
-    if vnd.solverKey in listVariableSolverData(dfg, variablekey, currentTransaction=currentTransaction)
-        error("Solver data '$(vnd.solverKey)' already exists")
+    if vnd.solveKey in listVariableSolverData(dfg, variablekey, currentTransaction=currentTransaction)
+        error("Solver data '$(vnd.solveKey)' already exists")
     end
     retPacked = _matchmergeVariableSubnode!(
         dfg,
@@ -760,8 +778,8 @@ function updateVariableSolverData!(dfg::CloudGraphsDFG,
                                 useCopy::Bool=true,
                                 fields::Vector{Symbol}=Symbol[];
                                 currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::VariableNodeData
-    if !(vnd.solverKey in listVariableSolverData(dfg, variablekey, currentTransaction=currentTransaction))
-        @warn "Solver data '$(vnd.solverKey)' does not exist, adding rather than updating."
+    if !(vnd.solveKey in listVariableSolverData(dfg, variablekey, currentTransaction=currentTransaction))
+        @warn "Solver data '$(vnd.solveKey)' does not exist, adding rather than updating."
     end
     # TODO: Update this to use the selective parameters from fields.
     retPacked = _matchmergeVariableSubnode!(
@@ -774,13 +792,13 @@ function updateVariableSolverData!(dfg::CloudGraphsDFG,
     return _unpackVariableNodeData(dfg, retPacked)
 end
 
-function deleteVariableSolverData!(dfg::CloudGraphsDFG, variablekey::Symbol, solverKey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::VariableNodeData
+function deleteVariableSolverData!(dfg::CloudGraphsDFG, variablekey::Symbol, solveKey::Symbol=:default; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)::VariableNodeData
     retPacked = _deleteVarSubnode!(
         dfg,
         variablekey,
         :SOLVERDATA,
         _getLabelsForType(dfg, VariableNodeData, parentKey=variablekey),
-        solverKey,
+        solveKey,
         currentTransaction=currentTransaction)
     return _unpackVariableNodeData(dfg, retPacked)
 end
@@ -841,12 +859,12 @@ function updateVariableSolverData!(dfg::CloudGraphsDFG,
 end
 
 function mergeVariableSolverData!(dfg::CloudGraphsDFG, sourceVariable::DFGVariable)
-    for solverKey in listVariableSolverData(dfg, sourceVariable.label)
+    for solveKey in listVariableSolverData(dfg, sourceVariable.label)
         updateVariableSolverData!(
             dfg,
             sourceVariable.label,
-            getVariableSolverData(dfg, sourceVariable, solverKey),
-            solverKey)
+            getVariableSolverData(dfg, sourceVariable, solveKey),
+            solveKey)
     end
 end
 

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -146,6 +146,7 @@ function _structToNeo4jProps(inst::Union{User, Robot, Session, PVND, N, APPE, AB
         # Neo4j type conversion if possible - keep timestamps timestamps, etc.
         if field isa ZonedDateTime
             val = "datetime(\"$(string(field))\")"
+            # val = "datetime(\"$(Dates.format(field, "yyyy-mm-ddTHH:MM:SS.ssszzz"))\")"
         end
         if field isa UUID
             val = "\"$(string(field))\""
@@ -167,7 +168,7 @@ function _structToNeo4jProps(inst::Union{User, Robot, Session, PVND, N, APPE, AB
                 val = field.value
             end
             if fieldname == :softtype
-                val = string(typeof(getSofttype(inst)))
+                val = DistributedFactorGraphs.typeModuleName(getSofttype(inst))
             end
             # Factors
             # TODO: Consolidate with packFactor in Serialization.jl - https://github.com/JuliaRobotics/DistributedFactorGraphs.jl/issues/525
@@ -250,8 +251,8 @@ function _getLabelsForInst(dfg::CloudGraphsDFG,
     labels = _getLabelsForType(dfg, typeof(inst), parentKey=parentKey)
     typeof(inst) <: DFGVariable && push!(labels, String(getLabel(inst)))
     typeof(inst) <: DFGFactor && push!(labels, String(getLabel(inst)))
-    typeof(inst) <: AbstractPointParametricEst && push!(labels, String(inst.solverKey))
-    typeof(inst) <: VariableNodeData && push!(labels, String(inst.solverKey))
+    typeof(inst) <: AbstractPointParametricEst && push!(labels, String(inst.solveKey))
+    typeof(inst) <: VariableNodeData && push!(labels, String(inst.solveKey))
     typeof(inst) <: AbstractDataEntry && push!(labels, String(inst.label))
     return labels
 end
@@ -317,8 +318,8 @@ function _matchmergeVariableSubnode!(
         result = commit(tx)
     end
     length(result.errors) > 0 && error(string(result.errors))
-    length(result.results[1]["data"]) != 1 && error("Cannot find subnode '$(ppe.solverKey)' for variable '$variablekey'")
-    length(result.results[1]["data"][1]["row"]) != 1 && error("Cannot find subnode '$(ppe.solverKey)' for variable '$variablekey'")
+    length(result.results[1]["data"]) != 1 && error("Cannot find subnode '$(ppe.solveKey)' for variable '$variablekey'")
+    length(result.results[1]["data"][1]["row"]) != 1 && error("Cannot find subnode '$(ppe.solveKey)' for variable '$variablekey'")
     return result.results[1]["data"][1]["row"][1]
 end
 

src/CloudGraphsDFG/services/CommonFunctions.jl

@@ -15,7 +15,6 @@ include("services/InMemoryDataEntryBlob.jl")
 
 
 export AbstractDataEntry, GeneralDataEntry, MongodbDataEntry, FileDataEntry
-export InMemoryDataStore, FileDataStore
 
 export getData, addData!, updateData!, deleteData!, listStoreEntries
 export getDataBlob, addDataBlob!, updateDataBlob!, deleteDataBlob!, listDataBlobs