Merge branch 'master' into 22Q4/enh/moregetppe

Author: dehann

.github/workflows/ci.yml

@@ -18,7 +18,6 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.6'
           - '1.8'
           - '~1.9.0-0'
           - 'nightly'
@@ -58,7 +57,7 @@ jobs:
       
       - name: Run tests
         env:
-          IIF_TEST: false
+          IIF_TEST: true
           DO_CGDFG_TESTS: false
           DFG_USE_CGDFG: false
         uses: julia-actions/julia-runtest@latest
@@ -72,83 +71,6 @@ jobs:
         with:
           file: lcov.info
 
-  test-cfg:
-    # if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release**'
-    name: w/ LGPL ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
-    runs-on: ${{ matrix.os }}
-    env:
-      JULIA_PKG_SERVER: ""
-    strategy:
-      fail-fast: false
-      matrix:
-        version:
-          - '1.8'
-        os:
-          - ubuntu-latest
-        arch:
-          - x64
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Setup Julia
-        uses: julia-actions/setup-julia@v1
-        with:
-          version: ${{ matrix.version }}
-          arch: ${{ matrix.arch }}
-
-      - name: Install neo4j
-        run: |
-          sudo add-apt-repository -y ppa:openjdk-r/ppa
-          sudo apt-get update
-          wget -O - https://debian.neo4j.com/neotechnology.gpg.key | sudo apt-key add -
-          echo 'deb https://debian.neo4j.com stable 3.5' | sudo tee /etc/apt/sources.list.d/neo4j.list
-          sudo apt-get update
-          apt list -a neo4j
-          sudo apt-get install neo4j
-          sudo service neo4j start
-          sleep 10
-          curl -v POST http://neo4j:neo4j@localhost:7474/user/neo4j/password -d"password=test"
-          # sudo neo4j-admin set-initial-password test
-          curl -I http://localhost:7474/
-
-      - name: Cache Artifacts
-        uses: actions/cache@v1
-        env:
-          cache-name: cache-artifacts
-        with:
-          path: ~/.julia/artifacts
-          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-test-${{ env.cache-name }}-
-            ${{ runner.os }}-test-
-            ${{ runner.os }}-
-
-      - name: Julia Build Pkg
-        env:
-          DFG_USE_CGDFG: true
-        uses: julia-actions/julia-buildpkg@latest
-      
-      - name: Git Test Credentials
-        run: |
-          git config --global user.name Tester
-          git config --global user.email [email protected]
-      
-      - name: Run tests
-        env:
-          IIF_TEST: false
-          DO_CGDFG_TESTS: true
-          DFG_USE_CGDFG: true
-        uses: julia-actions/julia-runtest@latest
-      
-      - name: Process Coverage
-        uses: julia-actions/julia-processcoverage@v1
-        
-      - name: Code Coverage
-        uses: codecov/codecov-action@v1
-        with:
-          file: lcov.info
-
   docs:
     needs: test
     name: Documentation

NEWS.md

@@ -1,5 +1,28 @@
 Listing news on any major breaking changes in DFG.  For regular changes, see integrated Github.com project milestones for DFG.
 
+# v0.20
+
+- Throw `KeyError` if `getBlobEntry` is not found, previously was `ErrorException`.
+- Change return type on `addData!` convenience wrappers to only return new `BlobEntry`.
+- Fix `addBlob!` calls for `FolderStore` and `InMemoryBlobStore` to use `BlobEntry.originId` and not previous bug `entry.id`.
+- Close long running serialization redo (#590) using only JSON3.jl and StructTypes.jl going forward.
+- Standardize BlobEntry=>Blob naming of functions, and keeping convenience wrappers `{get,add,update,delete}Data[!]`.
+- Consolidate to only one `BlobEntry` definition, dropping use of `AbstractBlobEntry`.
+- Include type field `VariableNodeData.covar`.
+- Drop minimum Julia compat to 1.8.
+
+# v0.19
+
+- Add ids and metadata to data types.
+- Use `Base.@kwdef` on stuct types for default values and serialization.
+- Dropped dependency on Unmarshal.jl.
+- Note src/Serialization.jl was refactored and currently contains lots of legacy code for DFG v0.18 compat, and much will be deleted in DFG v0.20 to standardize serialization around JSON3.jl, see #590.
+- `Neo4jDFG` has been removed.
+- `LightDFG` has been removed, and `GraphsDFG` is not the standard in-memory driver for alias `LocalDFG`.
+- Standardize all timestamp fields to `ZonedDateTime` from previous `DateTime` so that time zones will always be available.
+- internal `getDFGVersion()` function now returns a `::VersionNumber`.
+- Use `userLabel, robotLabel, sessionLabel` instead of legacy `userId, robotId, sessionId`.
+
 # v0.18.0
 
 - Unpack of GenericFactorNodeData with `reconstrFactorData` now gets `dfg::AbstractDFG` and `varOrder::Vector{Symbol}`, deprecating previous use of `convert` without the graph context (#832).

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.18.9"
+version = "0.20.1"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -9,12 +9,9 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
-JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
-JSON2 = "2535ab7d-5cd8-5a07-80ac-9b1792aadce3"
-LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d"
+JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb"
-Neo4j = "d2adbeaf-5838-5367-8a2f-e46d570981db"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
@@ -23,34 +20,32 @@ Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
 TensorCast = "02d47bb6-7ce6-556a-be16-bb1710789e2b"
 TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
-Unmarshal = "cbff2730-442d-58d7-89d1-8e530c41eb02"
 
 [compat]
 Colors = "0.10, 0.11, 0.12"
 Distributions = "0.23, 0.24, 0.25"
 DocStringExtensions = "0.8, 0.9"
 GraphPlot = "0.5.0"
 Graphs = "1.4"
-JSON = "0.21"
-JSON2 = "0.3.1"
-LightGraphs = "1.2, 1.3"
-ManifoldsBase = "0.11, 0.12, 0.13"
-Neo4j = "2"
+JSON3 = "1"
+ManifoldsBase = "0.12, 0.13, 0.14"
 OrderedCollections = "1.4"
 Pkg = "1.4, 1.5"
 ProgressMeter = "1"
 RecursiveArrayTools = "2"
 Reexport = "1"
 Requires = "1"
+StructTypes = "1"
 TensorCast = "0.3.3, 0.4"
 TimeZones = "1.3.1"
-Unmarshal = "0.4"
-julia = "1.6"
+julia = "1.8"
 
 [extras]
+DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Manifolds = "1cead3c2-87b3-11e9-0ccd-23c62b72b94e"
@@ -59,4 +54,4 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test", "GraphPlot", "LinearAlgebra", "Manifolds", "Pkg", "Statistics"]
+test = ["Test", "DataStructures", "GraphPlot", "LinearAlgebra", "Manifolds", "Pkg", "Statistics"]

README.md

@@ -34,7 +34,7 @@ add DistributedFactorGraphs
 
 # Usage
 
-The in-memory implementation is the default, using LightGraphs.jl.
+The in-memory implementation is the default, using Graphs.jl.
 
 It is recommended to use `IncrementalInference` to create factor graphs as they will be solvable. 
 ```julia

src/DataBlobs/services/FileDataEntryBlob.jl renamed to attic/DataBlobs/FileDataEntryBlob.jl

@@ -1,25 +1,25 @@
 
 
-# @generated function ==(x::FileDataEntry, y::FileDataEntry)
+# @generated function ==(x::BlobEntry, y::BlobEntry)
 #     mapreduce(n -> :(x.$n == y.$n), (a,b)->:($a && $b), fieldnames(x))
 # end
 
 #
-# getHash(entry::AbstractDataEntry) = hex2bytes(entry.hash)
+# getHash(entry::AbstractBlobEntry) = hex2bytes(entry.hash)
 
 
 ##==============================================================================
-## FileDataEntry Common
+## BlobEntry Common
 ##==============================================================================
-blobfilename(entry::FileDataEntry) = joinpath(entry.folder,"$(entry.id).dat")
-entryfilename(entry::FileDataEntry) = joinpath(entry.folder,"$(entry.id).json")
+blobfilename(entry::BlobEntry) = joinpath(entry.folder,"$(entry.id).dat")
+entryfilename(entry::BlobEntry) = joinpath(entry.folder,"$(entry.id).json")
 
 
 ##==============================================================================
-## FileDataEntry Blob CRUD
+## BlobEntry Blob CRUD
 ##==============================================================================
 
-function getDataBlob(dfg::AbstractDFG, entry::FileDataEntry)
+function getBlob(dfg::AbstractDFG, entry::BlobEntry)
     if isfile(blobfilename(entry))
         open(blobfilename(entry)) do f
             return read(f)
@@ -30,7 +30,7 @@ function getDataBlob(dfg::AbstractDFG, entry::FileDataEntry)
     end
 end
 
-function addDataBlob!(dfg::AbstractDFG, entry::FileDataEntry, data::Vector{UInt8})
+function addBlob!(dfg::AbstractDFG, entry::BlobEntry, data::Vector{UInt8})
     if isfile(blobfilename(entry))
         error("Key '$(entry.id)' blob already exists.")
     elseif isfile(entryfilename(entry))
@@ -42,37 +42,40 @@ function addDataBlob!(dfg::AbstractDFG, entry::FileDataEntry, data::Vector{UInt8
         open(entryfilename(entry), "w") do f
             JSON.print(f, entry)
         end
-        return getDataBlob(dfg, entry)::Vector{UInt8}
+        # FIXME update for entry.blobId vs entry.originId
+        return UUID(entry.id)
+        # return getBlob(dfg, entry)::Vector{UInt8}
     end
 end
 
-function updateDataBlob!(dfg::AbstractDFG, entry::FileDataEntry, data::Vector{UInt8})
+function updateBlob!(dfg::AbstractDFG, entry::BlobEntry, data::Vector{UInt8})
     if !isfile(blobfilename(entry))
         @warn "Entry '$(entry.id)' does not exist, adding."
-        return addDataBlob!(dfg, entry, data)
+        return addBlob!(dfg, entry, data)
     else
         # perhaps add an explicit force update flag and error otherwise
         @warn "Key '$(entry.id)' already exists, data will be overwritten."
-        deleteDataBlob!(dfg, entry)
-        return addDataBlob!(dfg, entry, data)
+        deleteBlob!(dfg, entry)
+        return addBlob!(dfg, entry, data)
     end
 end
 
-function deleteDataBlob!(dfg::AbstractDFG, entry::FileDataEntry)
-    data = getDataBlob(dfg, entry)
+function deleteBlob!(dfg::AbstractDFG, entry::BlobEntry)
+    data = getBlob(dfg, entry)
     rm(blobfilename(entry))
     rm(entryfilename(entry))
     return data
 end
 
 ##==============================================================================
-## FileDataEntry CRUD Helpers
+## BlobEntry CRUD Helpers
 ##==============================================================================
 
-function addData!(::Type{FileDataEntry}, dfg::AbstractDFG, label::Symbol, key::Symbol, folder::String, blob::Vector{UInt8}, timestamp=now(localzone());
+function addData!(::Type{BlobEntry}, dfg::AbstractDFG, label::Symbol, key::Symbol, folder::String, blob::Vector{UInt8}, timestamp=now(localzone());
                   id::UUID = uuid4(), hashfunction = sha256)
-    fde = FileDataEntry(key, id, folder, bytes2hex(hashfunction(blob)), timestamp)
-    de = addDataEntry!(dfg, label, fde)
-    db = addDataBlob!(dfg, fde, blob)
-    return de=>db
+    fde = BlobEntry(key, id, folder, bytes2hex(hashfunction(blob)), timestamp)
+    blobId = addBlob!(dfg, fde, blob) |> UUID
+    newEntry = BlobEntry(fde; id=blobId, blobId)
+    de = addBlobEntry!(dfg, label, newEntry)
+    return de # de=>db
 end