Restore IIF tests and some fixes (#972)

* Restore IIF tests and some fixes
* update deprecated addDataEntry
* Fix compare and update fildDFG regression tests
* Skip more in compare tests

Author: Affie

.github/workflows/ci.yml

@@ -58,7 +58,7 @@ jobs:
       
       - name: Run tests
         env:
-          IIF_TEST: false
+          IIF_TEST: true
           DO_CGDFG_TESTS: false
           DFG_USE_CGDFG: false
         uses: julia-actions/julia-runtest@latest

src/Deprecated.jl

@@ -11,19 +11,37 @@ Abstract parent struct for big data entry.
 abstract type AbstractBlobEntry end
 
 # should be deprecated by v0.21
-export BlobStoreEntry
-const BlobStoreEntry = BlobEntry
+
+@deprecate BlobStoreEntry(
+    label::Symbol,
+    id::UUID,
+    blobstore::Symbol,
+    hash::String,
+    origin::String,
+    description::String,
+    mimeType::String,
+    createdTimestamp::ZonedDateTime) BlobEntry(;
+        originId=id,
+        label,
+        blobstore,
+        hash,
+        origin,
+        description,
+        mimeType,
+    )
+
 
 @deprecate hasDataEntry(w...;kw...) hasBlobEntry(w...;kw...)
-@deprecate getBlobEntry(w...;kw...) getBlobEntry(w...;kw...)
+@deprecate getDataEntry(w...;kw...) getBlobEntry(w...;kw...)
+@deprecate getDataEntries(w...;kw...) getBlobEntries(w...;kw...)
 @deprecate addDataEntry!(w...;kw...) addBlobEntry!(w...;kw...)
 @deprecate updateDataEntry!(w...;kw...) updateBlobEntry!(w...;kw...)
 @deprecate deleteDataEntry!(w...;kw...) deleteBlobEntry!(w...;kw...)
 @deprecate listDataEntry(w...;kw...) listBlobEntry(w...;kw...)
 @deprecate listDataEntrySequence(w...;kw...) listBlobEntrySequence(w...;kw...)
 @deprecate mergeDataEntry!(w...;kw...) mergeBlobEntry!(w...;kw...)
 
-@deprecate getData(w...;kw...) getBlob(w...;kw...)
+# @deprecate getData(w...;kw...) getBlob(w...;kw...)
 @deprecate getDataBlob(w...;kw...) getBlob(w...;kw...)
 @deprecate addDataBlob!(w...;kw...) addBlob!(w...;kw...)
 @deprecate updateDataBlob!(w...;kw...) updateBlob!(w...;kw...)

src/services/CompareUtils.jl

@@ -37,7 +37,13 @@ function compareField(Allc, Bllc, syms)::Bool
     (!isdefined(Allc, syms) && !isdefined(Bllc, syms)) && return true
     !isdefined(Allc, syms) && return false
     !isdefined(Bllc, syms) && return false
-    return eval(:($Allc.$syms == $Bllc.$syms))
+    a = getproperty(Allc, syms)
+    b = getproperty(Bllc, syms)
+    if a isa Base.RefValue
+      return a[] == b[]
+    else
+      return a == b
+    end
 end
 
 """
@@ -57,6 +63,7 @@ function compareFields( Al::T1,
     (field in skip) && continue
     tp = compareField(Al, Bl, field)
     show && @debug("  $tp : $field") === nothing
+    show && !tp && (@debug "  $field" a=getproperty(Al,field) b=getproperty(Bl,field))
     !tp && return false
   end
   return true
@@ -414,7 +421,7 @@ function compareFactorGraphs( fgA::G1,
 
   TP = compareAll(fgA, fgB, skip=skiplist, show=show)
   TP = TP && compareSimilarVariables(fgA, fgB, skipsamples=skipsamples, show=show, skip=skiplist )
-  TP = TP && compareSimilarFactors(fgA, fgB, skipsamples=skipsamples, skipcompute=skipcompute, show=show )
+  TP = TP && compareSimilarFactors(fgA, fgB, skipsamples=skipsamples, skipcompute=skipcompute, show=show, skip=skiplist )
   TP = TP && compareAll(fgA.solverParams, fgB.solverParams, skip=skiplist)
 
   return TP

src/services/Serialization.jl

@@ -159,7 +159,7 @@ function packVariable(v::AbstractDFGVariable; includePPEs::Bool=true, includeSol
         solvable = v.solvable,
         variableType = DFG.typeModuleName(DFG.getVariableType(v)),
         dataEntries = collect(values(v.dataDict)),
-        _version = DFG._getDFGVersion())
+        _version = string(DFG._getDFGVersion()))
 end
 
 function unpackVariable(variable::PackedVariable; skipVersionCheck::Bool=false)

test/data/0_20_0.tar.gz

@@ -2,7 +2,7 @@ using DistributedFactorGraphs
 using IncrementalInference
 using Test
 using TimeZones
-
+using UUIDs
 ##
 
 @testset "FileDFG Tests" begin
@@ -20,12 +20,12 @@ using TimeZones
         map(v -> setSolvedCount!(v, Int(round(10*rand()))), verts)
 
         # Add some data entries
-        map(v -> addDataEntry!(v, BlobStoreEntry(:testing, uuid4(), :store, "", "", "", "", now(localzone()))), verts)
-        map(v -> addDataEntry!(v, BlobStoreEntry(:testing2, uuid4(), :store, "", "", "", "", ZonedDateTime(2014, 5, 30, 21, tz"UTC-4"))), verts)
+        map(v -> addBlobEntry!(v, BlobEntry(nothing, nothing, uuid4(), :testing, :store, "", "", "", "", "", now(localzone()), "BlobEntry", string(DFG._getDFGVersion()))), verts)
+        map(v -> addBlobEntry!(v, BlobEntry(nothing, nothing, uuid4(), :testing2, :store, "", "", "", "", "", ZonedDateTime(2014, 5, 30, 21, tz"UTC-4"), "BlobEntry", string(DFG._getDFGVersion()))), verts)
 
         # Add some PPEs
         ppe1 = MeanMaxPPE(:default, [1.0], [0.2], [3.456])
-        ppe2 = MeanMaxPPE(:other, [1.0], [0.2], [3.456], ZonedDateTime(2014, 5, 30, 21, tz"UTC-4"))
+        ppe2 = MeanMaxPPE(;solveKey=:other, suggested=[1.0], mean=[0.2], max=[3.456], createdTimestamp=ZonedDateTime(2014, 5, 30, 21, tz"UTC-4"))
         map(v -> addPPE!(dfg, getLabel(v), deepcopy(ppe1)), verts)
         map(v -> addPPE!(dfg, getLabel(v), deepcopy(ppe2)), verts)
 
@@ -67,7 +67,7 @@ using TimeZones
 
         # Check data entries
         for v in ls(dfg)
-            @test getDataEntries(getVariable(dfg, v)) == getDataEntries(getVariable(retDFG, v))
+            @test getBlobEntries(getVariable(dfg, v)) == getBlobEntries(getVariable(retDFG, v))
             @test issetequal(listPPEs(dfg, v), listPPEs(retDFG, v))
             for ppe in listPPEs(dfg, v)
                 @test getPPE(dfg, v, ppe) == getPPE(retDFG, v, ppe)
@@ -87,11 +87,11 @@ end
         loadFile = joinpath(@__DIR__, "data", file)
         global dfg
         dfgCopy = DistributedFactorGraphs._getDuplicatedEmptyDFG(dfg)
-        retDFG = loadDFG!(dfgCopy, loadFile)        
+        retDFG = loadDFG!(dfgCopy, loadFile)
         # It should have failed if there were any issues.
         # Trivial check as well
         @test issetequal(ls(retDFG), [:x1, :x2, :x3, :x4, :x5])
-        @test issetequal(lsf(retDFG), [:x3x4f1, :x4x5f1, :x1x2f1, :x2x3f1])
+        @test issetequal(lsf(retDFG), [:x3x4f1, :x4x5f1, :x1x2f1, :x2x3f1, :x1x2x3f1])
     end
 end
 

test/data/0_12_0.tar.gz renamed to test/data/archive/0_12_0.tar.gz

@@ -52,17 +52,17 @@ using Test
 
     initAll!(fg2)
 
-    @test compareSimilarVariables(fg, fg2, skipsamples=true, skip=Symbol[:initialized;:inferdim;:ppeDict;:solvedCount])
+    @test compareSimilarVariables(fg, fg2, skipsamples=true, skip=Symbol[:infoPerCoord;:initialized;:inferdim;:ppeDict;:solvedCount])
     # fg2 has been solved, so it should fail on the estimate dictionary
     @test !compareSimilarVariables(fg, fg2, skipsamples=true, skip=Symbol[:initialized;:inferdim])
 
     tree = buildTreeReset!(fg2)
 
-    @test compareSimilarFactors(fg, fg2, skipsamples=true, skipcompute=true)
+    @test compareSimilarFactors(fg, fg2, skipsamples=true, skipcompute=true, skip=[:fullvariables])
 
     @test !compareSimilarFactors(fg, fg2, skipsamples=true, skipcompute=false)
 
-    @test compareFactorGraphs(fg, fg2, skipsamples=true, skipcompute=true, skip=[:initialized;:inferdim;:ppeDict; :solvedCount])
+    @test compareFactorGraphs(fg, fg2, skipsamples=true, skipcompute=true, skip=[:fullvariables; :infoPerCoord; :initialized; :inferdim; :ppeDict; :solvedCount])
 
 end
 

test/fileDFGTests.jl

@@ -212,48 +212,70 @@ end
 end
 
 @testset "Data Entries" begin
-    oid = zeros(UInt8,12); oid[12] = 0x01
-    de1 = MongodbDataEntry(:key1, uuid4(), NTuple{12,UInt8}(oid), "", now(localzone()))
-
-    oid = zeros(UInt8,12); oid[12] = 0x02
-    de2 = MongodbDataEntry(:key2, uuid4(), NTuple{12,UInt8}(oid), "", now(localzone()))
-
-    oid = zeros(UInt8,12); oid[12] = 0x03
-    de2_update = MongodbDataEntry(:key2, uuid4(), NTuple{12,UInt8}(oid), "", now(localzone()))
+    
+    de1 = BlobEntry(
+        originId = uuid4(),
+        label = :key1,
+        blobstore = :test,
+        hash = "",
+        origin = "",
+        description = "",
+        mimeType = ""
+    )
+
+    de2 = BlobEntry(
+        originId = uuid4(),
+        label = :key2,
+        blobstore = :test,
+        hash = "",
+        origin = "",
+        description = "",
+        mimeType = ""
+    )
+
+    de2_update = BlobEntry(
+        originId = uuid4(),
+        label = :key2,
+        blobstore = :test,
+        hash = "",
+        origin = "",
+        description = "",
+        mimeType = "image/jpg"
+    )
 
     #add
     v1 = getVariable(dfg, :a)
-    @test addDataEntry!(v1, de1) == de1
-    @test addDataEntry!(dfg, :a, de2) == de2
-    @test_throws ErrorException addDataEntry!(v1, de1)
-    @test de2 in getDataEntries(v1)
+    @test addBlobEntry!(v1, de1) == de1
+    @test addBlobEntry!(dfg, :a, de2) == de2
+    @test_throws ErrorException addBlobEntry!(v1, de1)
+    @test de2 in getBlobEntries(v1)
 
     #get
-    @test deepcopy(de1) == getDataEntry(v1, :key1)
-    @test deepcopy(de2) == getDataEntry(dfg, :a, :key2)
-    @test_throws ErrorException getDataEntry(v2, :key1)
-    @test_throws ErrorException getDataEntry(dfg, :b, :key1)
+    @test deepcopy(de1) == getBlobEntry(v1, :key1)
+    @test deepcopy(de2) == getBlobEntry(dfg, :a, :key2)
+    @test_throws ErrorException getBlobEntry(v2, :key1)
+    @test_throws ErrorException getBlobEntry(dfg, :b, :key1)
 
     #update
-    @test updateDataEntry!(dfg, :a, de2_update) == de2_update
-    @test deepcopy(de2_update) == getDataEntry(dfg, :a, :key2)
-    @test @test_logs (:warn, r"does not exist") match_mode=:any updateDataEntry!(dfg, :b, de2_update) == de2_update
+    @test updateBlobEntry!(dfg, :a, de2_update) == de2_update
+    @test deepcopy(de2_update) == getBlobEntry(dfg, :a, :key2)
+    @test @test_logs (:warn, r"does not exist") match_mode=:any updateBlobEntry!(dfg, :b, de2_update) == de2_update
 
     #list
-    entries = getDataEntries(dfg, :a)
+    entries = getBlobEntries(dfg, :a)
     @test length(entries) == 2
     @test issetequal(map(e->e.label, entries), [:key1, :key2])
-    @test length(getDataEntries(dfg, :b)) == 1
+    @test length(getBlobEntries(dfg, :b)) == 1
 
-    @test issetequal(listDataEntries(dfg, :a), [:key1, :key2])
-    @test listDataEntries(dfg, :b) == Symbol[:key2]
+    @test issetequal(listBlobEntries(dfg, :a), [:key1, :key2])
+    @test listBlobEntries(dfg, :b) == Symbol[:key2]
 
     #delete
-    @test deleteDataEntry!(v1, :key1) == de1
-    @test listDataEntries(v1) == Symbol[:key2]
+    @test deleteBlobEntry!(v1, :key1) == de1
+    @test listBlobEntries(v1) == Symbol[:key2]
     #delete from ddfg
-    @test deleteDataEntry!(dfg, :a, :key2) == de2_update
-    @test listDataEntries(v1) == Symbol[]
+    @test deleteBlobEntry!(dfg, :a, :key2) == de2_update
+    @test listBlobEntries(v1) == Symbol[]
 end
 
 @testset "Updating Nodes and Estimates" begin