Merge pull request #259 from JuliaRobotics/master

v0.5.3-rc1

Author: dehann

.github/workflows/CompatHelper.yml

@@ -0,0 +1,24 @@
+name: CompatHelper
+
+on:
+  schedule:
+    - cron: '00 00 * * *'
+
+jobs:
+  CompatHelper:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        julia-version: [1.2.0]
+        julia-arch: [x86]
+        os: [ubuntu-latest]
+    steps:
+      - uses: julia-actions/setup-julia@latest
+        with:
+          version: ${{ matrix.julia-version }}
+      - name: Pkg.add("CompatHelper")
+        run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
+      - name: CompatHelper.main()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: julia -e 'using CompatHelper; CompatHelper.main()'

.travis.yml

@@ -5,12 +5,15 @@ dist: trusty
 os:
     - linux
 
+arch:
+    - amd64
+    - arm64
+
 services:
     - neo4j
 
 julia:
     - 1.0
-    - 1.1
     - 1.2
     - 1.3
     - nightly
@@ -24,7 +27,7 @@ jobs:
       env: IIF_TEST=true
       if: NOT branch =~ ^release.*$
     - stage: "Documentation"
-      julia: 1.0
+      julia: 1.2
       os: linux
       script:
         - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
@@ -38,6 +41,7 @@ notifications:
 matrix:
   allow_failures:
   - julia: nightly
+  - arch: arm64
 # - env: IIF_TEST=true
 
 # Set the password for Neo4j to neo4j:test

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.5.2"
+version = "0.5.3"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

src/BigData/entities/AbstractBigDataEntries.jl

@@ -17,7 +17,7 @@ Internal function to generate a unique key for the entry - userId_robotId_sessio
 Simple symbol.
 """
 function _uniqueKey(dfg::G, v::V, key::Symbol)::Symbol where {G <: AbstractDFG, V <: AbstractDFGVariable}
-    key = join(String.([dfg.userId, dfg.robotId, dfg.sessionId, label(v), String(key)]), "_")
+    key = join(String.([dfg.userId, dfg.robotId, dfg.sessionId, getLabel(v), String(key)]), "_")
     return Symbol(key)
 end
 

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -256,9 +256,9 @@ function mergeUpdateVariableSolverData!(dfg::CloudGraphsDFG, sourceVariable::DFG
         error("Source variable '$(sourceVariable.label)' doesn't exist in the graph.")
     end
     var = getVariable(dfg, sourceVariable.label)
-    newEsts = merge!(var.estimateDict, deepcopy(sourceVariable.estimateDict))
+    newEsts = merge!(var.ppeDict, deepcopy(sourceVariable.ppeDict))
     newSolveData = merge!(var.solverDataDict, sourceVariable.solverDataDict)
-    Neo4j.setnodeproperty(dfg.neo4jInstance.graph, var._internalId, "estimateDict",
+    Neo4j.setnodeproperty(dfg.neo4jInstance.graph, var._internalId, "ppeDict",
         JSON2.write(newEsts))
     Neo4j.setnodeproperty(dfg.neo4jInstance.graph, var._internalId, "solverDataDict",
         JSON2.write(Dict(keys(newSolveData) .=> map(vnd -> pack(dfg, vnd), values(newSolveData)))))

src/CommonAccessors.jl

@@ -20,18 +20,6 @@ Return the label for a variable or factor.
 """
 getLabel(v::DataLevel0) = v.label
 
-"""
-$SIGNATURES
-
-Return the label for a variable or factor.
-
-DEPRECATED label -> getLabel
-"""
-function label(v::DataLevel0)
-  @warn "Deprecated label, use getLabel instead."
-  getLabel(v)
-end
-
 
 """
 $SIGNATURES
@@ -41,18 +29,6 @@ Return the tags for a variable.
 getTags(v::DataLevel0) = v.tags
 
 
-"""
-$SIGNATURES
-
-Return the tags for a variable.
-
-DEPRECATED, tags -> getTags
-"""
-function tags(v::DataLevel0)
-  @warn "tags deprecated, use getTags instead"
-  getTags(v)
-end
-
 """
 $SIGNATURES
 

src/Deprecated.jl

@@ -0,0 +1,133 @@
+# deprecation staging area
+
+## quick deprecation handle
+import Base: propertynames, getproperty
+
+Base.propertynames(x::VariableDataLevel1, private::Bool=false) = private ? (:estimateDict, :ppeDict) : (:ppeDict,)
+
+Base.getproperty(x::VariableDataLevel1,f::Symbol) = begin
+    if f == :estimateDict
+      @warn "estimateDict is deprecated, use ppeDict instead"
+      getfield(x, :ppeDict)
+    else
+      getfield(x,f)
+    end
+  end
+
+
+
+"""
+$SIGNATURES
+
+Return the estimates for a variable.
+"""
+function getEstimates(v::VariableDataLevel1)
+  @warn "Deprecated getEstimates, use getVariablePPE/getPPE instead."
+  getVariablePPEs(vari)
+end
+
+"""
+  $SIGNATURES
+
+Return the estimates for a variable.
+
+DEPRECATED, estimates -> getVariablePPEs/getPPEs
+"""
+function estimates(v::VariableDataLevel1)
+  @warn "Deprecated estimates, use getVariablePPEs/getPPE instead."
+  getVariablePPEs(v)
+end
+
+"""
+  $SIGNATURES
+
+Return a keyed estimate (default is :default) for a variable.
+
+DEPRECATED use getVariablePPE/getPPE instead.
+"""
+function getEstimate(v::VariableDataLevel1, key::Symbol=:default)
+@warn "Deprecated getEstimate, use getVariablePPE/getPPE instead."
+getVariablePPE(v, key)
+end
+
+"""
+$SIGNATURES
+
+Return a keyed estimate (default is :default) for a variable.
+"""
+function estimate(v::VariableDataLevel1, key::Symbol=:default)
+  @warn "DEPRECATED estimate, use getVariablePPE/getPPE instead."
+  getVariablePPE(v, key)
+end
+
+
+"""
+$SIGNATURES
+
+Return the softtype for a variable.
+
+DEPRECATED, softtype -> getSofttype
+"""
+function softtype(v::VariableDataLevel1)
+    @warn "Deprecated softtype, use getSofttype instead."
+    getSofttype(v)
+end
+
+
+"""
+$SIGNATURES
+
+Return the label for a variable or factor.
+
+DEPRECATED label -> getLabel
+"""
+function label(v::DataLevel0)
+  @warn "Deprecated label, use getLabel instead."
+  getLabel(v)
+end
+
+
+"""
+$SIGNATURES
+
+Return the tags for a variable.
+
+DEPRECATED, tags -> getTags
+"""
+function tags(v::DataLevel0)
+  @warn "tags deprecated, use getTags instead"
+  getTags(v)
+end
+
+
+"""
+    $SIGNATURES
+
+Retrieve data structure stored in a variable.
+"""
+function getData(v::DFGVariable; solveKey::Symbol=:default)::VariableNodeData
+  @warn "getData is deprecated, please use solverData()"
+  return v.solverDataDict[solveKey]
+end
+
+
+"""
+    $SIGNATURES
+
+Set solver data structure stored in a variable.
+"""
+function setSolverData(v::DFGVariable, data::VariableNodeData, key::Symbol=:default)
+    @warn "Deprecated setSolverData, use setSolverData! instead."
+    setSolverData!(v, data, key)
+end
+
+
+"""
+    $SIGNATURES
+
+Retrieve solver data structure stored in a factor.
+"""
+function data(f::DFGFactor)::GenericFunctionNodeData
+  @warn "data() is deprecated, please use solverData()"
+  return f.data
+end

src/DistributedFactorGraphs.jl

@@ -21,7 +21,7 @@ export AbstractParams, NoSolverParams
 export DFGNode, DFGVariable, DFGFactor, AbstractDFGVariable, AbstractDFGFactor
 export InferenceType, PackedInferenceType, FunctorInferenceType, InferenceVariable, ConvolutionObject
 export FunctorSingleton, FunctorPairwise, FunctorPairwiseMinimize
-export getMaxPPE, getMeanPPE, getSuggestedPPE
+export getMaxPPE, getMeanPPE, getSuggestedPPE, getVariablePPE, getPPE, getVariablePPEs, getPPEs #, getEstimates
 export timestamp # DEPRECATED
 export label, getTimestamp, setTimestamp!, tags, setTags!, estimates, estimate, data, softtype, solverData, getData, solverDataDict, setSolverData, setSolverData!, internalId, smallData, setSmallData!, bigData
 export DFGVariableSummary, DFGFactorSummary, AbstractDFGSummary
@@ -80,6 +80,7 @@ include("services/AbstractDFG.jl")
 include("services/DFGVariable.jl")
 include("services/DFGFactor.jl")
 include("CommonAccessors.jl")
+include("Deprecated.jl")
 include("services/CompareUtils.jl")
 
 # Include the Graphs.jl API.

src/FileDFG/services/FileDFG.jl

@@ -14,7 +14,7 @@ v1 = addVariable!(dfg, :a, ContinuousScalar, labels = [:POSE], solvable=0)
 saveDFG(dfg, "/tmp/saveDFG")
 ```
 """
-function saveDFG(dfg::G, folder::String) where G <: AbstractDFG
+function saveDFG(dfg::AbstractDFG, folder::String; compress::Symbol=:gzip)
     variables = getVariables(dfg)
     factors = getFactors(dfg)
     varFolder = "$folder/variables"
@@ -43,6 +43,16 @@ function saveDFG(dfg::G, folder::String) where G <: AbstractDFG
         JSON2.write(io, fPacked)
         close(io)
     end
+
+    # compress newly saved folder, skip if not supported format
+    !(compress in [:gzip]) && return
+    savepath = folder[end] == '/' ? folder[1:end-1] : folder
+    savedir = dirname(savepath)
+    savename = splitpath(string(savepath))[end]
+    @assert savename != ""
+    # temporarily change working directory to get correct zipped path
+    run( pipeline(`tar -zcf - -C $savedir $savename`, stdout="$savepath.tar.gz") )
+    Base.rm(joinpath(savedir,savename), recursive=true)
 end
 
 """
@@ -56,20 +66,46 @@ using DistributedFactorGraphs, IncrementalInference
 # Create a DFG - can make one directly, e.g. GraphsDFG{NoSolverParams}() or use IIF:
 dfg = initfg()
 # Load the graph
-loadDFG("/tmp/savedgraph", IncrementalInference, dfg)
+loadDFG("/tmp/savedgraph.tar.gz", IncrementalInference, dfg)
+loadDFG("/tmp/savedgraph", IncrementalInference, dfg) # alternative
 # Use the DFG as you do normally.
 ls(dfg)
 ```
 """
-function loadDFG(folder::String, iifModule, dfgLoadInto::G) where G <: AbstractDFG
+function loadDFG(dst::String, iifModule, dfgLoadInto::G; loaddir=joinpath("/","tmp","caesar","random")) where G <: AbstractDFG
+    # Check if zipped destination (dst) by first doing fuzzy search from user supplied dst
+    folder = dst  # working directory for fileDFG variable and factor operations
+    dstname = dst # path name could either be legacy FileDFG dir or .tar.gz file of FileDFG files.
+    unzip = false
+    # add if doesn't have .tar.gz extension
+    lastdirname = splitpath(dstname)[end]
+    if !isdir(dst)
+        unzip = true
+        sdst = split(lastdirname, '.')
+        if sdst[end] != "gz" #  length(sdst) == 1 &&
+            dstname *= ".tar.gz"
+            lastdirname *= ".tar.gz"
+        end
+    end
+    # do actual unzipping
+    if unzip
+        @show sfolder = split(dstname, '.')
+        Base.mkpath(loaddir)
+        folder = joinpath(loaddir, lastdirname[1:(end-length(".tar.gz"))]) #splitpath(string(sfolder[end-2]))[end]
+        @info "loadDFG detected a gzip $dstname -- unpacking via $loaddir now..."
+        Base.rm(folder, recursive=true, force=true)
+        # unzip the tar file
+        run(`tar -zxf $dstname -C $loaddir`)
+    end
+    # extract the factor graph from fileDFG folder
     variables = DFGVariable[]
     factors = DFGFactor[]
     varFolder = "$folder/variables"
     factorFolder = "$folder/factors"
     # Folder preparations
     !isdir(folder) && error("Can't load DFG graph - folder '$folder' doesn't exist")
-    !isdir(varFolder) && error("Can't load DFG graph - folder '$folder' doesn't exist")
-    !isdir(factorFolder) && error("Can't load DFG graph - folder '$folder' doesn't exist")
+    !isdir(varFolder) && error("Can't load DFG graph - folder '$varFolder' doesn't exist")
+    !isdir(factorFolder) && error("Can't load DFG graph - folder '$factorFolder' doesn't exist")
 
     varFiles = readdir(varFolder)
     factorFiles = readdir(factorFolder)

src/LightDFG/entities/LightDFG.jl

@@ -58,10 +58,10 @@ Base.propertynames(x::LightDFG, private::Bool=false) =
 
 Base.getproperty(x::LightDFG,f::Symbol) = begin
     if f == :nodeCounter
-        @error "Field nodeCounter depreciated. returning number of nodes"
+        @error "Field nodeCounter deprecated. returning number of nodes"
         nv(x.g)
     elseif f == :labelDict
-        @error "Field labelDict depreciated. Consider using exists(dfg,label) or getLabelDict(dfg) instead. Returning internals copy"
+        @error "Field labelDict deprecated. Consider using exists(dfg,label) or getLabelDict(dfg) instead. Returning internals copy"
         #TODO: https://github.com/JuliaRobotics/DistributedFactorGraphs.jl/issues/111
         copy(x.g.labels.sym_int)
     else