Merge branch 'master' into feat/4Q19/showvar

Author: dehann

.github/workflows/CompatHelper.yml

@@ -0,0 +1,24 @@
+name: CompatHelper
+
+on:
+  schedule:
+    - cron: '00 00 * * *'
+
+jobs:
+  CompatHelper:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        julia-version: [1.2.0]
+        julia-arch: [x86]
+        os: [ubuntu-latest]
+    steps:
+      - uses: julia-actions/setup-julia@latest
+        with:
+          version: ${{ matrix.julia-version }}
+      - name: Pkg.add("CompatHelper")
+        run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
+      - name: CompatHelper.main()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: julia -e 'using CompatHelper; CompatHelper.main()'

.travis.yml

@@ -5,12 +5,15 @@ dist: trusty
 os:
     - linux
 
+arch:
+    - amd64
+    - arm64
+
 services:
     - neo4j
 
 julia:
     - 1.0
-    - 1.1
     - 1.2
     - 1.3
     - nightly
@@ -24,7 +27,7 @@ jobs:
       env: IIF_TEST=true
       if: NOT branch =~ ^release.*$
     - stage: "Documentation"
-      julia: 1.0
+      julia: 1.2
       os: linux
       script:
         - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
@@ -38,6 +41,7 @@ notifications:
 matrix:
   allow_failures:
   - julia: nightly
+  - arch: arm64
 # - env: IIF_TEST=true
 
 # Set the password for Neo4j to neo4j:test

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.5.2"
+version = "0.5.4"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

src/BigData/entities/AbstractBigDataEntries.jl

@@ -17,7 +17,7 @@ Internal function to generate a unique key for the entry - userId_robotId_sessio
 Simple symbol.
 """
 function _uniqueKey(dfg::G, v::V, key::Symbol)::Symbol where {G <: AbstractDFG, V <: AbstractDFGVariable}
-    key = join(String.([dfg.userId, dfg.robotId, dfg.sessionId, label(v), String(key)]), "_")
+    key = join(String.([dfg.userId, dfg.robotId, dfg.sessionId, getLabel(v), String(key)]), "_")
     return Symbol(key)
 end
 

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -256,9 +256,9 @@ function mergeUpdateVariableSolverData!(dfg::CloudGraphsDFG, sourceVariable::DFG
         error("Source variable '$(sourceVariable.label)' doesn't exist in the graph.")
     end
     var = getVariable(dfg, sourceVariable.label)
-    newEsts = merge!(var.estimateDict, deepcopy(sourceVariable.estimateDict))
+    newEsts = merge!(var.ppeDict, deepcopy(sourceVariable.ppeDict))
     newSolveData = merge!(var.solverDataDict, sourceVariable.solverDataDict)
-    Neo4j.setnodeproperty(dfg.neo4jInstance.graph, var._internalId, "estimateDict",
+    Neo4j.setnodeproperty(dfg.neo4jInstance.graph, var._internalId, "ppeDict",
         JSON2.write(newEsts))
     Neo4j.setnodeproperty(dfg.neo4jInstance.graph, var._internalId, "solverDataDict",
         JSON2.write(Dict(keys(newSolveData) .=> map(vnd -> pack(dfg, vnd), values(newSolveData)))))

src/CommonAccessors.jl

@@ -20,18 +20,6 @@ Return the label for a variable or factor.
 """
 getLabel(v::DataLevel0) = v.label
 
-"""
-$SIGNATURES
-
-Return the label for a variable or factor.
-
-DEPRECATED label -> getLabel
-"""
-function label(v::DataLevel0)
-  @warn "Deprecated label, use getLabel instead."
-  getLabel(v)
-end
-
 
 """
 $SIGNATURES
@@ -41,18 +29,6 @@ Return the tags for a variable.
 getTags(v::DataLevel0) = v.tags
 
 
-"""
-$SIGNATURES
-
-Return the tags for a variable.
-
-DEPRECATED, tags -> getTags
-"""
-function tags(v::DataLevel0)
-  @warn "tags deprecated, use getTags instead"
-  getTags(v)
-end
-
 """
 $SIGNATURES
 

src/DFGPlots/DFGPlots.jl

@@ -88,7 +88,7 @@ More information at [GraphPlot.jl](https://github.com/JuliaGraphs/GraphPlot.jl)
 function dfgplot(dfg::AbstractDFG, p::DFGPlotProps = DFGPlotProps())
     # TODO implement convert functions
     @warn "TODO Implement convert"
-    ldfg = LightDFG{AbstractParams}()
+    ldfg = LightDFG{NoSolverParams}()
     DistributedFactorGraphs._copyIntoGraph!(dfg, ldfg, union(getVariableIds(dfg), getFactorIds(dfg)), true, copyGraphMetadata=false)
 
     dfgplot(ldfg, p)

src/Deprecated.jl

@@ -0,0 +1,133 @@
+# deprecation staging area
+
+## quick deprecation handle
+import Base: propertynames, getproperty
+
+Base.propertynames(x::VariableDataLevel1, private::Bool=false) = private ? (:estimateDict, :ppeDict) : (:ppeDict,)
+
+Base.getproperty(x::VariableDataLevel1,f::Symbol) = begin
+    if f == :estimateDict
+      @warn "estimateDict is deprecated, use ppeDict instead"
+      getfield(x, :ppeDict)
+    else
+      getfield(x,f)
+    end
+  end
+
+
+
+"""
+$SIGNATURES
+
+Return the estimates for a variable.
+"""
+function getEstimates(v::VariableDataLevel1)
+  @warn "Deprecated getEstimates, use getVariablePPE/getPPE instead."
+  getVariablePPEs(vari)
+end
+
+"""
+  $SIGNATURES
+
+Return the estimates for a variable.
+
+DEPRECATED, estimates -> getVariablePPEs/getPPEs
+"""
+function estimates(v::VariableDataLevel1)
+  @warn "Deprecated estimates, use getVariablePPEs/getPPE instead."
+  getVariablePPEs(v)
+end
+
+"""
+  $SIGNATURES
+
+Return a keyed estimate (default is :default) for a variable.
+
+DEPRECATED use getVariablePPE/getPPE instead.
+"""
+function getEstimate(v::VariableDataLevel1, key::Symbol=:default)
+@warn "Deprecated getEstimate, use getVariablePPE/getPPE instead."
+getVariablePPE(v, key)
+end
+
+"""
+$SIGNATURES
+
+Return a keyed estimate (default is :default) for a variable.
+"""
+function estimate(v::VariableDataLevel1, key::Symbol=:default)
+  @warn "DEPRECATED estimate, use getVariablePPE/getPPE instead."
+  getVariablePPE(v, key)
+end
+
+
+"""
+$SIGNATURES
+
+Return the softtype for a variable.
+
+DEPRECATED, softtype -> getSofttype
+"""
+function softtype(v::VariableDataLevel1)
+    @warn "Deprecated softtype, use getSofttype instead."
+    getSofttype(v)
+end
+
+
+"""
+$SIGNATURES
+
+Return the label for a variable or factor.
+
+DEPRECATED label -> getLabel
+"""
+function label(v::DataLevel0)
+  @warn "Deprecated label, use getLabel instead."
+  getLabel(v)
+end
+
+
+"""
+$SIGNATURES
+
+Return the tags for a variable.
+
+DEPRECATED, tags -> getTags
+"""
+function tags(v::DataLevel0)
+  @warn "tags deprecated, use getTags instead"
+  getTags(v)
+end
+
+
+"""
+    $SIGNATURES
+
+Retrieve data structure stored in a variable.
+"""
+function getData(v::DFGVariable; solveKey::Symbol=:default)::VariableNodeData
+  @warn "getData is deprecated, please use solverData()"
+  return v.solverDataDict[solveKey]
+end
+
+
+"""
+    $SIGNATURES
+
+Set solver data structure stored in a variable.
+"""
+function setSolverData(v::DFGVariable, data::VariableNodeData, key::Symbol=:default)
+    @warn "Deprecated setSolverData, use setSolverData! instead."
+    setSolverData!(v, data, key)
+end
+
+
+"""
+    $SIGNATURES
+
+Retrieve solver data structure stored in a factor.
+"""
+function data(f::DFGFactor)::GenericFunctionNodeData
+  @warn "data() is deprecated, please use solverData()"
+  return f.data
+end

src/DistributedFactorGraphs.jl

@@ -21,11 +21,13 @@ export AbstractParams, NoSolverParams
 export DFGNode, DFGVariable, DFGFactor, AbstractDFGVariable, AbstractDFGFactor
 export InferenceType, PackedInferenceType, FunctorInferenceType, InferenceVariable, ConvolutionObject
 export FunctorSingleton, FunctorPairwise, FunctorPairwiseMinimize
-export getMaxPPE, getMeanPPE, getSuggestedPPE
+export getMaxPPE, getMeanPPE, getSuggestedPPE, getVariablePPE, getPPE, getVariablePPEs, getPPEs #, getEstimates
 export timestamp # DEPRECATED
+export getSolvedCount, isSolved, setSolvedCount!
 export label, getTimestamp, setTimestamp!, tags, setTags!, estimates, estimate, data, softtype, solverData, getData, solverDataDict, setSolverData, setSolverData!, internalId, smallData, setSmallData!, bigData
 export DFGVariableSummary, DFGFactorSummary, AbstractDFGSummary
 export addBigDataEntry!, getBigDataEntry, updateBigDataEntry!, deleteBigDataEntry!, getBigDataEntries, getBigDataKeys
+export getNeighborhood, getSubgraph, getSubgraphAroundNode
 export printFactor, printVariable
 
 #Skeleton types
@@ -81,6 +83,7 @@ include("services/AbstractDFG.jl")
 include("services/DFGVariable.jl")
 include("services/DFGFactor.jl")
 include("CommonAccessors.jl")
+include("Deprecated.jl")
 include("services/CompareUtils.jl")
 
 # Include the Graphs.jl API.

src/FileDFG/services/FileDFG.jl

@@ -14,7 +14,7 @@ v1 = addVariable!(dfg, :a, ContinuousScalar, labels = [:POSE], solvable=0)
 saveDFG(dfg, "/tmp/saveDFG")
 ```
 """
-function saveDFG(dfg::G, folder::String) where G <: AbstractDFG
+function saveDFG(dfg::AbstractDFG, folder::String; compress::Symbol=:gzip)
     variables = getVariables(dfg)
     factors = getFactors(dfg)
     varFolder = "$folder/variables"
@@ -43,6 +43,16 @@ function saveDFG(dfg::G, folder::String) where G <: AbstractDFG
         JSON2.write(io, fPacked)
         close(io)
     end
+
+    # compress newly saved folder, skip if not supported format
+    !(compress in [:gzip]) && return
+    savepath = folder[end] == '/' ? folder[1:end-1] : folder
+    savedir = dirname(savepath)
+    savename = splitpath(string(savepath))[end]
+    @assert savename != ""
+    # temporarily change working directory to get correct zipped path
+    run( pipeline(`tar -zcf - -C $savedir $savename`, stdout="$savepath.tar.gz") )
+    Base.rm(joinpath(savedir,savename), recursive=true)
 end
 
 """
@@ -56,20 +66,48 @@ using DistributedFactorGraphs, IncrementalInference
 # Create a DFG - can make one directly, e.g. GraphsDFG{NoSolverParams}() or use IIF:
 dfg = initfg()
 # Load the graph
-loadDFG("/tmp/savedgraph", IncrementalInference, dfg)
+loadDFG("/tmp/savedgraph.tar.gz", IncrementalInference, dfg)
+loadDFG("/tmp/savedgraph", IncrementalInference, dfg) # alternative
 # Use the DFG as you do normally.
 ls(dfg)
 ```
 """
-function loadDFG(folder::String, iifModule, dfgLoadInto::G) where G <: AbstractDFG
+function loadDFG(dst::String, iifModule, dfgLoadInto::G; loaddir=joinpath("/","tmp","caesar","random")) where G <: AbstractDFG
+    # Check if zipped destination (dst) by first doing fuzzy search from user supplied dst
+    folder = dst  # working directory for fileDFG variable and factor operations
+    dstname = dst # path name could either be legacy FileDFG dir or .tar.gz file of FileDFG files.
+    unzip = false
+    # add if doesn't have .tar.gz extension
+    lastdirname = splitpath(dstname)[end]
+    if !isdir(dst)
+        unzip = true
+        sdst = split(lastdirname, '.')
+        if sdst[end] != "gz" #  length(sdst) == 1 &&
+            dstname *= ".tar.gz"
+            lastdirname *= ".tar.gz"
+        end
+    end
+    # TODO -- what if it is not a tar.gz but classic folder instead?
+    # do actual unzipping
+    filename = lastdirname[1:(end-length(".tar.gz"))] |> string
+    if unzip
+        @show sfolder = split(dstname, '.')
+        Base.mkpath(loaddir)
+        folder = joinpath(loaddir, filename) #splitpath(string(sfolder[end-2]))[end]
+        @info "loadDFG detected a gzip $dstname -- unpacking via $loaddir now..."
+        Base.rm(folder, recursive=true, force=true)
+        # unzip the tar file
+        run(`tar -zxf $dstname -C $loaddir`)
+    end
+    # extract the factor graph from fileDFG folder
     variables = DFGVariable[]
     factors = DFGFactor[]
     varFolder = "$folder/variables"
     factorFolder = "$folder/factors"
     # Folder preparations
     !isdir(folder) && error("Can't load DFG graph - folder '$folder' doesn't exist")
-    !isdir(varFolder) && error("Can't load DFG graph - folder '$folder' doesn't exist")
-    !isdir(factorFolder) && error("Can't load DFG graph - folder '$folder' doesn't exist")
+    !isdir(varFolder) && error("Can't load DFG graph - folder '$varFolder' doesn't exist")
+    !isdir(factorFolder) && error("Can't load DFG graph - folder '$factorFolder' doesn't exist")
 
     varFiles = readdir(varFolder)
     factorFiles = readdir(factorFolder)
@@ -104,5 +142,12 @@ function loadDFG(folder::String, iifModule, dfgLoadInto::G) where G <: AbstractD
     # TODO: Remove in future
     map(f->solverData(f).fncargvID = f._variableOrderSymbols, getFactors(dfgLoadInto))
 
+    # remove the temporary unzipped file
+    if unzip
+      @info "DFG.loadDFG is deleting a temp folder created during unzip, $folder"
+      # need this because the number of files created in /tmp/caesar/random is becoming redonkulous.
+      Base.rm(folder, recursive=true)
+    end
+
     return dfgLoadInto
 end