Merge branch 'master' into 22Q4/api/packvarsimpler

Author: dehann

.github/workflows/ci.yml

@@ -20,6 +20,7 @@ jobs:
         version:
           - '1.6'
           - '1.8'
+          - '~1.9.0-0'
           - 'nightly'
         os:
           - ubuntu-latest
@@ -81,7 +82,6 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.6'
           - '1.8'
         os:
           - ubuntu-latest

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.18.8"
+version = "0.19.0"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -17,6 +17,7 @@ ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb"
 Neo4j = "d2adbeaf-5838-5367-8a2f-e46d570981db"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
@@ -40,6 +41,7 @@ ManifoldsBase = "0.11, 0.12, 0.13"
 Neo4j = "2"
 OrderedCollections = "1.4"
 Pkg = "1.4, 1.5"
+ProgressMeter = "1"
 RecursiveArrayTools = "2"
 Reexport = "1"
 Requires = "1"

src/Common.jl

@@ -105,3 +105,11 @@ DevNotes
 
 """
 getVariableLabelNumber(vs::Symbol, prefix=string(vs)[1]) = parse(Int, string(vs)[(length(prefix)+1):end])
+                               
+                                
+## =================================
+## Additional Downstream dispatches
+## =================================
+
+function solveGraph! end
+function solveGraphParametric! end

src/DataBlobs/entities/AbstractDataEntries.jl

@@ -31,6 +31,33 @@ struct BlobStoreEntry <: AbstractDataEntry
     createdTimestamp::ZonedDateTime # of when the entry was created
 end
 
+_fixtimezone(cts::NamedTuple) = ZonedDateTime(cts.utc_datetime*"+00")
+
+# needed for deserialization from JSON during DFG v0.19 transition, see #867
+function BlobStoreEntry(;
+    label,
+    id,
+    blobstore,
+    hash,
+    origin,
+    description,
+    mimeType,
+    createdTimestamp,
+    kwargs... # drop excessive fields
+)
+    #
+    BlobStoreEntry(
+        Symbol(label),
+        UUID(id),
+        Symbol(blobstore),
+        hash,
+        origin,
+        description,
+        mimeType,
+        _fixtimezone(createdTimestamp),
+    )
+end
+
 # TODO
 """
     $(TYPEDEF)

src/DataBlobs/services/AbstractDataEntries.jl

@@ -41,11 +41,25 @@ function getDataEntry(var::AbstractDFGVariable, blobId::UUID)
             return v
         end
     end
-    error("No dataEntry with blobId $(blobId) found in variable $(getLabel(var))")
+    throw(
+        KeyError("No dataEntry with blobId $(blobId) found in variable $(getLabel(var))")
+    )
 end
+function getDataEntry(var::AbstractDFGVariable, key::Regex)
+    for (k,v) in var.dataDict
+        if occursin(key, string(v.label))
+            return v
+        end
+    end
+    throw(
+        KeyError("No dataEntry with label matching regex $(key) found in variable $(getLabel(var))")
+    )
+end
+getDataEntry(var::AbstractDFGVariable, key::AbstractString) = getDataEntry(var,Regex(key))
+
 
-getDataEntry(dfg::AbstractDFG, label::Symbol, key::UUID) = getDataEntry(getVariable(dfg, label), key)
-getDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol) = getDataEntry(getVariable(dfg, label), key)
+getDataEntry(dfg::AbstractDFG, label::Symbol, key::Union{Symbol, UUID, <:AbstractString, Regex}) = getDataEntry(getVariable(dfg, label), key)
+# getDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol) = getDataEntry(getVariable(dfg, label), key)
 
 
 """
@@ -196,8 +210,11 @@ function mergeDataEntries!(
     des = _makevec(des_)
     # don't add data entries that already exist 
     dde = listDataEntries(dst, dlbl)
-    uids = (s->s.id).(dde)
-    filter!(s -> !(s.id in uids), des)
+        # HACK, verb list should just return vector of Symbol. NCE36
+        _getid(s) = s
+        _getid(s::AbstractDataEntry) = s.id
+    uids = _getid.(dde) # (s->s.id).(dde)
+    filter!(s -> !(_getid(s) in uids), des)
     # add any data entries not already in the destination variable, by uuid
     addDataEntry!.(dst, dlbl, des)
 end
@@ -212,9 +229,67 @@ function mergeDataEntries!(
     des = listDataEntries(src, slbl)
     # don't add data entries that already exist 
     dde = listDataEntries(dst, dlbl)
-    uids = (s->s.id).(dde)
-    filter!(s -> !(s.id in uids), des)
+        # HACK, verb list should just return vector of Symbol. NCE36
+        _getid(s) = s
+        _getid(s::AbstractDataEntry) = s.id    
+    uids = _getid.(dde) # (s->s.id).(dde)
+    filter!(s -> !(_getid(s) in uids), des)
     if 0  < length(des)
         union(((s->mergeDataEntries!(dst, dlbl, src, slbl, s.id)).(des))...)
     end
-end
+end
+
+function mergeDataEntries!(
+    dest::AbstractDFG,
+    src::AbstractDFG,
+    w...;
+    varList::AbstractVector = listVariables(dest) |> sortDFG
+)
+    @showprogress 1 "merging data entries" for vl in varList
+        mergeDataEntries!(dest, vl, src, vl, w...)
+    end
+    varList
+end
+
+"""
+    $SIGNATURES
+
+If the blob label `datalabel` already exists, then this function will return the name `datalabel_1`.
+If the blob label `datalabel_1` already exists, then this function will return the name `datalabel_2`.
+"""
+function incrDataLabelSuffix(
+    dfg::AbstractDFG,
+    vla,
+    bllb::S; 
+    datalabel=Ref("")
+) where {S <: Union{Symbol, <:AbstractString}}
+    count = 1
+    hasund = false
+    len = 0
+    try
+        de,_ = getData(dfg, Symbol(vla), bllb)
+        bllb = string(bllb)
+        # bllb *= bllb[end] != '_' ? "_" : ""
+        datalabel[] = string(de.label)
+        dlb = match(r"\d*", reverse(datalabel[]))
+        # slightly complicated search if blob name already has an underscore number suffix, e.g. `_4`
+        count, hasund, len = if occursin(Regex(dlb.match*"_"), reverse(datalabel[]))
+            parse(Int, dlb.match |> reverse)+1, true, length(dlb.match)
+        else
+            1, datalabel[][end] == '_', 0
+        end
+    catch err
+        # append latest count
+        if !(err isa KeyError)
+        throw(err)
+        end
+    end
+    # the piece from old label without the suffix count number
+    bllb = datalabel[][1:(end-len)]
+    if !hasund || bllb[end] != '_'
+        bllb *= "_"
+    end
+    bllb *= string(count)
+
+    S(bllb)
+end

src/DataBlobs/services/BlobStores.jl

@@ -17,7 +17,7 @@ function getDataBlob(dfg::AbstractDFG, entry::BlobStoreEntry)
     end
     throw(
         KeyError(
-            "could not find $(entry.label), uuid $(entry.id)) in any of the listed blobstores:\n $((s->getKey(s)).(stores))"
+            "could not find $(entry.label), uuid $(entry.id)) in any of the listed blobstores:\n $([s->getKey(s) for (s,v) in stores]))"
         )
     )
 end
@@ -84,7 +84,7 @@ function getData(
     dfg::AbstractDFG, 
     blobstore::AbstractBlobStore, 
     label::Symbol, 
-    key::Union{Symbol,UUID}; 
+    key::Union{Symbol,UUID, <:AbstractString, Regex}; 
     hashfunction = sha256,
     checkhash::Bool=true
 )

src/DataBlobs/services/DataEntryBlob.jl

@@ -114,7 +114,7 @@ end
 function getData(
     dfg::AbstractDFG, 
     vlabel::Symbol, 
-    key::Union{Symbol,UUID}; 
+    key::Union{Symbol,UUID, <:AbstractString, Regex}; 
     hashfunction = sha256,
     checkhash::Bool=true
 )

src/DistributedFactorGraphs.jl

@@ -30,6 +30,7 @@ using SparseArrays
 using UUIDs
 using Pkg
 using TensorCast
+using ProgressMeter
 
 # used for @defVariable
 import ManifoldsBase
@@ -192,12 +193,14 @@ export getSolvedCount, isSolved, setSolvedCount!, isInitialized, isMarginalized,
 export getNeighborhood, getNeighbors, _getDuplicatedEmptyDFG
 export findFactorsBetweenNaive
 export copyGraph!, deepcopyGraph, deepcopyGraph!, buildSubgraph, mergeGraph!
-# Big Data
+
+# Entry Blob Data
 ##------------------------------------------------------------------------------
 export addDataEntry!, getDataEntry, updateDataEntry!, deleteDataEntry!, getDataEntries, listDataEntries, hasDataEntry, hasDataEntry
 export listDataEntrySequence
 # convenience wrappers
 export addDataEntry!, mergeDataEntries!
+export incrDataLabelSuffix
 # aliases
 export addData!
 

src/FileDFG/services/FileDFG.jl

@@ -43,14 +43,14 @@ function saveDFG(folder::AbstractString, dfg::AbstractDFG)
     for v in variables
         vPacked = packVariable(dfg, v)
         io = open("$varFolder/$(v.label).json", "w")
-        JSON2.write(io, vPacked)
+        println(io,JSON.json(vPacked))
         close(io)
     end
     # Factors
     for f in factors
         fPacked = packFactor(dfg, f)
         io = open("$factorFolder/$(f.label).json", "w")
-        JSON2.write(io, fPacked)
+        println(io,JSON.json(fPacked))
         close(io)
     end
 
@@ -130,10 +130,11 @@ function loadDFG!(dfgLoadInto::AbstractDFG, dst::AbstractString)
     varFiles = readdir(varFolder)
     factorFiles = readdir(factorFolder)
     for varFile in varFiles
-        open("$varFolder/$varFile") do io
-            packedData = JSON2.read(io, Dict{String, Any})
+        packedData = JSON.parsefile("$varFolder/$varFile"; dicttype=Dict{String, Any})
+        # open("$varFolder/$varFile") do io
+        #     packedData = JSON.parse(io; dicttype=Dict{String, Any})
             push!(variables, unpackVariable(dfgLoadInto, packedData))
-        end
+        # end
     end
     @info "Loaded $(length(variables)) variables - $(map(v->v.label, variables))"
     @info "Inserting variables into graph..."

src/entities/DFGFactor.jl

@@ -8,7 +8,7 @@ abstract type AbstractPackedFactor end
 
 abstract type AbstractPrior <: AbstractFactor end
 abstract type AbstractRelative <: AbstractFactor end
-abstract type AbstractRelativeRoots <: AbstractRelative end
+abstract type AbstractRelativeRoots <: AbstractRelative end    # NOTE Cannot be used for partials
 abstract type AbstractRelativeMinimize <: AbstractRelative end
 abstract type AbstractManifoldMinimize <: AbstractRelative end # FIXME move here from IIF