Fixing tar file saving, comparator for Graphs.jl, and the FileDFG tests

Author: GearsAD

src/FileDFG/services/FileDFG.jl

@@ -11,18 +11,25 @@ dfg = initfg()
 # ... Add stuff to graph using either IIF or DFG:
 v1 = addVariable!(dfg, :a, ContinuousScalar, labels = [:POSE], solvable=0)
 # Now save it:
-saveDFG(dfg, "/tmp/saveDFG")
+saveDFG(dfg, "/tmp/saveDFG.tar.gz")
 ```
 """
-function saveDFG(dfg::AbstractDFG, folder::String; compress::Symbol=:gzip)
+function saveDFG(dfg::AbstractDFG, folder::String)
+
+    # TODO: Deprecate the folder functionality in v0.6.1
+
+    # Clean up save path if a file is specified
+    savepath = folder[end] == '/' ? folder[1:end-1] : folder
+    savepath = splitext(splitext(savepath)[1])[1] # In case of .tar.gz
+
     variables = getVariables(dfg)
     factors = getFactors(dfg)
-    varFolder = "$folder/variables"
-    factorFolder = "$folder/factors"
+    varFolder = "$savepath/variables"
+    factorFolder = "$savepath/factors"
     # Folder preparations
-    if !isdir(folder)
-        @info "Folder '$folder' doesn't exist, creating..."
-        mkpath(folder)
+    if !isdir(savepath)
+        @info "Folder '$savepath' doesn't exist, creating..."
+        mkpath(savepath)
     end
     !isdir(varFolder) && mkpath(varFolder)
     !isdir(factorFolder) && mkpath(factorFolder)
@@ -39,19 +46,16 @@ function saveDFG(dfg::AbstractDFG, folder::String; compress::Symbol=:gzip)
     # Factors
     for f in factors
         fPacked = packFactor(dfg, f)
-        io = open("$folder/factors/$(f.label).json", "w")
+        io = open("$factorFolder/$(f.label).json", "w")
         JSON2.write(io, fPacked)
         close(io)
     end
 
-    # compress newly saved folder, skip if not supported format
-    !(compress in [:gzip]) && return
-    savepath = folder[end] == '/' ? folder[1:end-1] : folder
     savedir = dirname(savepath)
-    savename = splitpath(string(savepath))[end]
+    savename = basename(string(savepath))
     @assert savename != ""
-    # temporarily change working directory to get correct zipped path
-    run( pipeline(`tar -zcf - -C $savedir $savename`, stdout="$savepath.tar.gz") )
+    destfile = joinpath(savedir, savename*".tar.gz")
+    run( pipeline(`tar -zcf - -C $savedir $savename`, stdout="$destfile"))
     Base.rm(joinpath(savedir,savename), recursive=true)
 end
 
@@ -67,7 +71,6 @@ using DistributedFactorGraphs, IncrementalInference
 dfg = initfg()
 # Load the graph
 loadDFG("/tmp/savedgraph.tar.gz", IncrementalInference, dfg)
-loadDFG("/tmp/savedgraph", IncrementalInference, dfg) # alternative
 # Use the DFG as you do normally.
 ls(dfg)
 ```

src/services/CompareUtils.jl

@@ -24,7 +24,8 @@ const GeneratedCompareUnion = Union{MeanMaxPPE, VariableNodeData, DFGNodeParams,
                               DFGFactor, DFGFactorSummary, SkeletonDFGFactor}
 
 @generated function ==(x::T, y::T) where T <: GeneratedCompareUnion
-    mapreduce(n -> :(x.$n == y.$n), (a,b)->:($a && $b), fieldnames(x))
+    ignored = [:_internalId]
+    mapreduce(n -> :(x.$n == y.$n), (a,b)->:($a && $b), setdiff(fieldnames(x), ignored))
 end
 
 

test/fileDFGTests.jl

@@ -1,58 +1,59 @@
 using DistributedFactorGraphs
 using IncrementalInference
 using Test
-dfg = LightDFG{SolverParams}()
 
 @testset "FileDFG Tests" begin
-
-    if typeof(dfg) <: CloudGraphsDFG
-        @warn "TEST: Nuking all data for user '$(dfg.userId)', robot '$(dfg.robotId)'!"
-        clearRobot!!(dfg)
-    end
-
-    # Same graph as iifInterfaceTests.jl
-    numNodes = 10
-
-    #change ready and solvable for x7,x8 for improved tests on x7x8f1
-    verts = map(n -> addVariable!(dfg, Symbol("x$n"), ContinuousScalar, labels = [:POSE]), 1:numNodes)
-    #TODO fix this to use accessors
-    verts[7].solvable = 1
-    verts[8].solvable = 0
-    getSolverData(verts[8]).solveInProgress = 1
-    setSolvedCount!(verts[1], 5)
-    #call update to set it on cloud
-    updateVariable!(dfg, verts[7])
-    updateVariable!(dfg, verts[8])
-
-    # Add some bigData to x1, x2
-    addBigDataEntry!(verts[1], GeneralBigDataEntry(:testing, :testing; mimeType="application/nuthin!"))
-    addBigDataEntry!(verts[2], FileBigDataEntry(:testing2, "/dev/null"))
-    #call update to set it on cloud
-    updateVariable!(dfg, verts[1])
-    updateVariable!(dfg, verts[2])
-
-    facts = map(n -> addFactor!(dfg, [verts[n], verts[n+1]], LinearConditional(Normal(50.0,2.0))), 1:(numNodes-1))
-
-    # Save and load the graph to test.
-    saveFolder = "/tmp/fileDFG"
-    saveDFG(dfg, saveFolder)
-
-    copyDfg = DistributedFactorGraphs._getDuplicatedEmptyDFG(dfg)
-    @info "Going to load $saveFolder"
-    retDFG = loadDFG(saveFolder*".tar.gz", Main, copyDfg, loaddir="/tmp")
-
-    @test issetequal(ls(dfg), ls(retDFG))
-    @test issetequal(lsf(dfg), lsf(retDFG))
-    for var in ls(dfg)
-        @test getVariable(dfg, var) == getVariable(retDFG, var)
+    for filename in ["/tmp/fileDFG", "/tmp/FileDFGExtension.tar.gz"]
+        global dfg
+        dfg = DistributedFactorGraphs._getDuplicatedEmptyDFG(dfg)
+
+        if typeof(dfg) <: CloudGraphsDFG
+            @warn "TEST: Nuking all data for user '$(dfg.userId)', robot '$(dfg.robotId)'!"
+            clearRobot!!(dfg)
+        end
+
+        # Same graph as iifInterfaceTests.jl
+        numNodes = 5
+
+        #change ready and solvable for x7,x8 for improved tests on x7x8f1
+        verts = map(n -> addVariable!(dfg, Symbol("x$n"), ContinuousScalar, labels = [:POSE]), 1:numNodes)
+        #TODO fix this to use accessors
+        verts[3].solvable = 1
+        verts[4].solvable = 0
+        getSolverData(verts[4]).solveInProgress = 1
+        #call update to set it on cloud
+        updateVariable!(dfg, verts[3])
+        updateVariable!(dfg, verts[4])
+
+        setSolvedCount!(verts[1], 5)
+        # Add some bigData to x1, x2
+        addBigDataEntry!(verts[1], GeneralBigDataEntry(:testing, :testing; mimeType="application/nuthin!"))
+        addBigDataEntry!(verts[2], FileBigDataEntry(:testing2, "/dev/null"))
+        #call update to set it on cloud
+        updateVariable!(dfg, verts[1])
+        updateVariable!(dfg, verts[2])
+
+        facts = map(n -> addFactor!(dfg, [verts[n], verts[n+1]], LinearConditional(Normal(50.0,2.0))), 1:(numNodes-1))
+
+        # Save and load the graph to test.
+        saveDFG(dfg, filename)
+
+        copyDfg = DistributedFactorGraphs._getDuplicatedEmptyDFG(dfg)
+        @info "Going to load $filename"
+        retDFG = loadDFG(filename, Main, copyDfg, loaddir="/tmp")
+
+        @test issetequal(ls(dfg), ls(retDFG))
+        @test issetequal(lsf(dfg), lsf(retDFG))
+        for var in ls(dfg)
+            @test getVariable(dfg, var) == getVariable(retDFG, var)
+        end
+        for fact in lsf(dfg)
+            @test getFactor(dfg, fact) == getFactor(retDFG, fact)
+        end
+
+        @test length(getBigDataEntries(getVariable(retDFG, :x1))) == 1
+        @test typeof(getBigDataEntry(getVariable(retDFG, :x1),:testing)) == GeneralBigDataEntry
+        @test length(getBigDataEntries(getVariable(retDFG, :x2))) == 1
+        @test typeof(getBigDataEntry(getVariable(retDFG, :x2),:testing2)) == FileBigDataEntry
     end
-    for fact in lsf(dfg)
-        @test getFactor(dfg, fact) == getFactor(retDFG, fact)
-    end
-
-    @test length(getBigDataEntries(getVariable(retDFG, :x1))) == 1
-    @test typeof(getBigDataEntry(getVariable(retDFG, :x1),:testing)) == GeneralBigDataEntry
-    @test length(getBigDataEntries(getVariable(retDFG, :x2))) == 1
-    @test typeof(getBigDataEntry(getVariable(retDFG, :x2),:testing2)) == FileBigDataEntry
-
 end