diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ac87c07c..8990bd8c 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -2,6 +2,12 @@ name: Coverage on: [push, pull_request] +env: + # It's really `--all-features`, but not adding `persistence`, we expect the + # persistence feature to go away again in the future (but if we add it + # unconditionally it changes the code that's run significantly) + ALMOST_ALL_FEATURES: --features "with-serde with-csv with-nexmark" + jobs: check: name: Coverage @@ -29,7 +35,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: llvm-cov - args: --all-features --workspace --lcov --output-path coverage.info --ignore-filename-regex tests.rs + args: ${{ env.ALMOST_ALL_FEATURES }} --workspace --lcov --output-path coverage.info --ignore-filename-regex tests.rs - name: Upload to codecov.io uses: codecov/codecov-action@v3 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 28c1ba82..d671fa90 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,6 +8,10 @@ env: CARGO_NET_RETRY: 10 RUST_BACKTRACE: short RUSTUP_MAX_RETRIES: 10 + # It's really `--all-features`, but not adding `persistence`, we expect the + # persistence feature to go away again in the future (but if we add it + # unconditionally it changes the code that's run significantly) + ALMOST_ALL_FEATURES: --features "with-serde with-csv with-nexmark" jobs: tests: @@ -82,7 +86,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - args: --all-features --target ${{ matrix.target }} + args: ${{ env.ALMOST_ALL_FEATURES }} --target ${{ matrix.target }} # miri: # name: Miri @@ -116,7 +120,7 @@ jobs: # MIRIFLAGS: "-Zmiri-tag-raw-pointers -Zmiri-disable-isolation -Zmiri-preemption-rate=0" # with: # command: miri - # args: test --all-features + # args: test ${{ env.ALMOST_ALL_FEATURES }} test-sanitizers: name: Sanitizer Tests @@ -181,13 +185,18 @@ jobs: RUSTDOCFLAGS: "-Z sanitizer=${{ matrix.sanitizer }}" RUSTFLAGS: "-Z sanitizer=${{ matrix.sanitizer }}" ASAN_OPTIONS: detect_stack_use_after_return=1,detect_leaks=1 + # Ensure the C++ code (rocksdb etc.) also gets compiled with the correct sanitizer arguments + CC: "clang" + CCFLAGS: "-fsanitize=${{ matrix.sanitizer }}" + CXX: "clang++" + CXXFLAGS: "-fsanitize=${{ matrix.sanitizer }}" + ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer-14" # Backtraces sometimes mess with sanitizers RUST_BACKTRACE: 0 with: command: test - # leak sanitizer is crashing on exchange tests and is very slow on - # many of the proptests. - args: --all-features --target ${{ matrix.target }} -Z build-std -- --skip 'exchange' --skip 'proptest' + # leak sanitizer is crashing on exchange tests. + args: ${{ env.ALMOST_ALL_FEATURES }} --target ${{ matrix.target }} -Z build-std -- --skip 'exchange' --skip 'proptest' --skip 'persistent' clippy: name: Clippy @@ -258,7 +267,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: rustdoc - args: --all-features -- -D warnings --cfg docsrs + args: ${{ env.ALMOST_ALL_FEATURES }} -- -D warnings --cfg docsrs udeps: name: Unused Dependencies diff --git a/Cargo.lock b/Cargo.lock index 7ed6375d..bdde89b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,9 +33,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.19" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] @@ -48,15 +48,26 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anyhow" -version = "1.0.65" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" +checksum = "c794e162a5eff65c72ef524dfe393eb923c354e350bb78b9c7383df13f3bc142" [[package]] name = "arcstr" version = "1.1.4" +source = "git+https://github.com/gz/arcstr.git?rev=b43120c#b43120cd13db16a1c1f61a12d1637b047b9bcd89" +dependencies = [ + "bincode", +] + +[[package]] +name = "arrayvec" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5ebd6061b8b70d59b36bc58d2337540bd7cc1f23ab2bea356f452bc7fe3505" +checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" +dependencies = [ + "nodrop", +] [[package]] name = "arrayvec" @@ -76,9 +87,9 @@ version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.43", + "quote 1.0.21", + "syn 1.0.99", ] [[package]] @@ -116,6 +127,44 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a32fd6af2b5827bce66c29053ba0e7c42b9dcab01835835058558c10851a46b" +[[package]] +name = "bincode" +version = "2.0.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb50c5a2ef4b9b1e7ae73e3a73b52ea24b20312d629f9c4df28260b7ad2c3c4" +dependencies = [ + "bincode_derive", + "serde", +] + +[[package]] +name = "bincode_derive" +version = "2.0.0-rc.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a45a23389446d2dd25dc8e73a7a3b3c43522b630cac068927f0649d43d719d2" +dependencies = [ + "virtue", +] + +[[package]] +name = "bindgen" +version = "0.60.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2 1.0.43", + "quote 1.0.21", + "regex", + "rustc-hash", + "shlex", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -151,9 +200,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" dependencies = [ "generic-array", ] @@ -172,9 +221,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" [[package]] name = "byteorder" @@ -236,8 +285,8 @@ checksum = "751f7f4e7a091545e7f6c65bacc404eaee7e87bfb1f9ece234a1caa173dc16f2" dependencies = [ "cached_proc_macro_types", "darling", - "quote", - "syn", + "quote 1.0.21", + "syn 1.0.99", ] [[package]] @@ -261,6 +310,15 @@ dependencies = [ "jobserver", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -303,11 +361,22 @@ dependencies = [ "generic-array", ] +[[package]] +name = "clang-sys" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" -version = "3.2.22" +version = "3.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" +checksum = "a3dbbb6653e7c55cc8595ad3e1f7be8f32aba4eb7ff7f0fd1163d4f3d137c0a9" dependencies = [ "atty", "bitflags", @@ -322,15 +391,15 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.2.18" +version = "3.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" +checksum = "9ba52acd3b0a5c33aeada5cdaa3267cdc7c594a98731d4268cdc1532f4264cb4" dependencies = [ "heck", "proc-macro-error", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.43", + "quote 1.0.21", + "syn 1.0.99", ] [[package]] @@ -344,13 +413,13 @@ dependencies = [ [[package]] name = "console" -version = "0.15.2" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c050367d967ced717c04b65d8c619d863ef9292ce0c5760028655a2fb298718c" +checksum = "89eab4d20ce20cea182308bca13088fecea9c05f6776cf287205d41a0ed3c847" dependencies = [ "encode_unicode", - "lazy_static", "libc", + "once_cell", "terminal_size", "unicode-width", "winapi", @@ -380,9 +449,9 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.2.5" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" dependencies = [ "libc", ] @@ -469,14 +538,15 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.11" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348" +checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", "memoffset", + "once_cell", "scopeguard", ] @@ -492,11 +562,12 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.12" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" +checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" dependencies = [ "cfg-if", + "once_cell", ] [[package]] @@ -555,10 +626,10 @@ checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", - "proc-macro2", - "quote", + "proc-macro2 1.0.43", + "quote 1.0.21", "strsim", - "syn", + "syn 1.0.99", ] [[package]] @@ -568,20 +639,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core", - "quote", - "syn", + "quote 1.0.21", + "syn 1.0.99", ] [[package]] name = "dashmap" -version = "5.4.0" +version = "5.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +checksum = "3495912c9c1ccf2e18976439f4443f3fee0fd61f424ff99fde6a66b15ecb448f" dependencies = [ "cfg-if", "hashbrown", "lock_api", - "once_cell", "parking_lot_core", ] @@ -592,6 +662,7 @@ dependencies = [ "anyhow", "arcstr", "ascii_table", + "bincode", "bitvec", "cached", "clap", @@ -612,10 +683,12 @@ dependencies = [ "petgraph", "priority-queue", "proptest", + "proptest-derive", "rand", "rand_xoshiro", "regex", "reqwest", + "rocksdb", "rstest", "rust_decimal", "serde", @@ -624,6 +697,7 @@ dependencies = [ "textwrap", "time", "typedmap", + "uuid", "xxhash-rust", "zip", "zstd", @@ -631,9 +705,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.5" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ "block-buffer", "crypto-common", @@ -642,9 +716,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" [[package]] name = "encode_unicode" @@ -721,10 +795,11 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" dependencies = [ + "matches", "percent-encoding", ] @@ -736,9 +811,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.24" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" +checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" dependencies = [ "futures-channel", "futures-core", @@ -751,9 +826,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.24" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" dependencies = [ "futures-core", "futures-sink", @@ -761,15 +836,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.24" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" [[package]] name = "futures-executor" -version = "0.3.24" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" dependencies = [ "futures-core", "futures-task", @@ -778,32 +853,32 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.24" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" [[package]] name = "futures-macro" -version = "0.3.24" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.43", + "quote 1.0.21", + "syn 1.0.99", ] [[package]] name = "futures-sink" -version = "0.3.24" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" [[package]] name = "futures-task" -version = "0.3.24" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" [[package]] name = "futures-timer" @@ -813,9 +888,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.24" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" dependencies = [ "futures-channel", "futures-core", @@ -850,11 +925,17 @@ dependencies = [ "wasi", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "h2" -version = "0.3.14" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" dependencies = [ "bytes", "fnv", @@ -916,7 +997,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", - "itoa 1.0.4", + "itoa 1.0.3", ] [[package]] @@ -932,9 +1013,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" [[package]] name = "httpdate" @@ -957,7 +1038,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.4", + "itoa 1.0.3", "pin-project-lite", "socket2", "tokio", @@ -987,10 +1068,11 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.3.0" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" dependencies = [ + "matches", "unicode-bidi", "unicode-normalization", ] @@ -1001,9 +1083,9 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.43", + "quote 1.0.21", + "syn 1.0.99", ] [[package]] @@ -1018,9 +1100,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfddc9561e8baf264e0e45e197fd7696320026eb10a8180340debc27b18f535b" +checksum = "fcc42b206e70d86ec03285b123e65a5458c92027d1fb2ae3555878b8113b3ddf" dependencies = [ "console", "number_prefix", @@ -1059,24 +1141,24 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.4" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" [[package]] name = "jobserver" -version = "0.1.25" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" dependencies = [ "wasm-bindgen", ] @@ -1087,17 +1169,58 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" -version = "0.2.135" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b" + +[[package]] +name = "libloading" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c" +checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "librocksdb-sys" +version = "0.8.0+7.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", +] + +[[package]] +name = "libz-sys" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ "autocfg", "scopeguard", @@ -1112,6 +1235,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "memchr" version = "2.5.0" @@ -1143,11 +1272,17 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" -version = "0.5.4" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" dependencies = [ "adler", ] @@ -1182,6 +1317,22 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + +[[package]] +name = "nom" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num" version = "0.4.0" @@ -1218,12 +1369,12 @@ dependencies = [ [[package]] name = "num-format" -version = "0.4.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b862ff8df690cf089058c98b183676a7ed0f974cc08b426800093227cbff3b" +checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" dependencies = [ - "arrayvec", - "itoa 1.0.4", + "arrayvec 0.4.12", + "itoa 0.4.8", ] [[package]] @@ -1295,9 +1446,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.15.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" +checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0" [[package]] name = "oorandom" @@ -1313,9 +1464,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.42" +version = "0.10.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" +checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" dependencies = [ "bitflags", "cfg-if", @@ -1332,9 +1483,9 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.43", + "quote 1.0.21", + "syn 1.0.99", ] [[package]] @@ -1345,9 +1496,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.76" +version = "0.9.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce" +checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" dependencies = [ "autocfg", "cc", @@ -1358,9 +1509,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "3.2.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "129d36517b53c461acc6e1580aeb919c8ae6708a4b1eae61c4463a615d4f0411" +checksum = "96bcbab4bfea7a59c2c0fe47211a1ac4e3e96bea6eb446d704f310bc5c732ae2" dependencies = [ "num-traits", "serde", @@ -1368,9 +1519,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.3.0" +version = "6.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" +checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4" [[package]] name = "parking_lot_core" @@ -1414,11 +1565,17 @@ dependencies = [ "sha2", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "petgraph" @@ -1450,9 +1607,9 @@ checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "plotters" -version = "0.3.4" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +checksum = "9428003b84df1496fb9d6eeee9c5f8145cb41ca375eb0dad204328888832811f" dependencies = [ "num-traits", "plotters-backend", @@ -1469,9 +1626,9 @@ checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" [[package]] name = "plotters-svg" -version = "0.3.3" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +checksum = "e0918736323d1baff32ee0eade54984f6f201ad7e97d5cfb5d6ab4a358529615" dependencies = [ "plotters-backend", ] @@ -1499,9 +1656,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.43", + "quote 1.0.21", + "syn 1.0.99", "version_check", ] @@ -1511,16 +1668,25 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.43", + "quote 1.0.21", "version_check", ] [[package]] name = "proc-macro2" -version = "1.0.47" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "proc-macro2" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" dependencies = [ "unicode-ident", ] @@ -1545,6 +1711,17 @@ dependencies = [ "tempfile", ] +[[package]] +name = "proptest-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b46295382dc76166cb7cf2bb4a97952464e4b7ed5a43e6cd34e1fec3349ddc" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -1557,13 +1734,22 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.43", ] [[package]] @@ -1595,9 +1781,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ "getrandom", ] @@ -1687,9 +1873,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.12" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" +checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" dependencies = [ "base64", "bytes", @@ -1703,10 +1889,10 @@ dependencies = [ "hyper-tls", "ipnet", "js-sys", + "lazy_static", "log", "mime", "native-tls", - "once_cell", "percent-encoding", "pin-project-lite", "serde", @@ -1722,6 +1908,16 @@ dependencies = [ "winreg", ] +[[package]] +name = "rocksdb" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc" +dependencies = [ + "libc", + "librocksdb-sys", +] + [[package]] name = "rstest" version = "0.15.0" @@ -1741,10 +1937,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5015e68a0685a95ade3eee617ff7101ab6a3fc689203101ca16ebc16f2b89c66" dependencies = [ "cfg-if", - "proc-macro2", - "quote", + "proc-macro2 1.0.43", + "quote 1.0.21", "rustc_version", - "syn", + "syn 1.0.99", ] [[package]] @@ -1753,11 +1949,17 @@ version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee9164faf726e4f3ece4978b25ca877ddc6802fa77f38cdccb32c7f805ecd70c" dependencies = [ - "arrayvec", + "arrayvec 0.7.2", "num-traits", "serde", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.0" @@ -1812,9 +2014,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "security-framework" -version = "2.7.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" +checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" dependencies = [ "bitflags", "core-foundation", @@ -1835,37 +2037,37 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.14" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +checksum = "93f6841e709003d68bb2deee8c343572bf446003ec20a583e76f7b15cebf3711" [[package]] name = "serde" -version = "1.0.145" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" +checksum = "53e8e5d5b70924f74ff5c6d64d9a5acd91422117c60f48c4e07855238a254553" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.145" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" +checksum = "d3d8e8de557aee63c26b85b947f5e59b690d0454c753f3adeb5cd7835ab88391" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.43", + "quote 1.0.21", + "syn 1.0.99", ] [[package]] name = "serde_json" -version = "1.0.86" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074" +checksum = "38dd04e3c8279e75b31ef29dbdceebfe5ad89f4d0937213c53f7d49d01b3d5a7" dependencies = [ - "itoa 1.0.4", + "itoa 1.0.3", "ryu", "serde", ] @@ -1877,16 +2079,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.4", + "itoa 1.0.3", "ryu", "serde", ] [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "c77f4e7f65455545c2153c1253d25056825e77ee2533f0e41deb65a93a34852f" dependencies = [ "cfg-if", "cpufeatures", @@ -1895,20 +2097,26 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" dependencies = [ "cfg-if", "cpufeatures", "digest", ] +[[package]] +name = "shlex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" + [[package]] name = "size-of" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c81bb88893bf92e254271c696cd396254edc79cce57c1ed7728084c36e416987" +checksum = "8b6cf33b5dcf8bba2e7b0af3987126dfb5099544412d98cade4f797b21c741b3" dependencies = [ "arcstr", "rust_decimal", @@ -1922,9 +2130,9 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a15926c1c296ac124170d7f42a03f815286de1d4b8fe0525021eba09b5622ad6" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.43", + "quote 1.0.21", + "syn 1.0.99", ] [[package]] @@ -1938,9 +2146,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.10.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" [[package]] name = "smawk" @@ -1950,9 +2158,9 @@ checksum = "f67ad224767faa3c7d8b6d91985b78e70a1324408abcb1cfcc2be4c06bc06043" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" dependencies = [ "libc", "winapi", @@ -1972,12 +2180,23 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.102" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid", +] + +[[package]] +name = "syn" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1" +checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.43", + "quote 1.0.21", "unicode-ident", ] @@ -2033,9 +2252,9 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.15.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" +checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" dependencies = [ "smawk", "unicode-linebreak", @@ -2044,42 +2263,52 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "f5f6586b7f764adc0231f4c79be7b920e766bb2f3e51b3661cdb263828f19994" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "12bafc5b54507e0149cdf1b145a5d80ab80a90bcd9275df43d4fff68460f6c21" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.43", + "quote 1.0.21", + "syn 1.0.99", ] [[package]] name = "time" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d634a985c4d4238ec39cacaed2e7ae552fbd3c476b552c1deac3021b7d7eaf0c" +checksum = "0fab5c8b9980850e06d92ddbe3ab839c062c801f3927c0fb8abd6fc8e918fbca" dependencies = [ - "itoa 1.0.4", + "itoa 1.0.3", "libc", "num_threads", "serde", + "time-core", "time-macros", ] +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + [[package]] name = "time-macros" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" +checksum = "65bb801831d812c562ae7d2bfb531f26e66e4e1f6b17307ba4149c5064710e5b" +dependencies = [ + "time-core", +] [[package]] name = "tinytemplate" @@ -2108,9 +2337,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.21.2" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" dependencies = [ "autocfg", "bytes", @@ -2118,6 +2347,7 @@ dependencies = [ "memchr", "mio", "num_cpus", + "once_cell", "pin-project-lite", "socket2", "tokio-macros", @@ -2130,9 +2360,9 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.43", + "quote 1.0.21", + "syn 1.0.99", ] [[package]] @@ -2147,9 +2377,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" dependencies = [ "bytes", "futures-core", @@ -2167,9 +2397,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" dependencies = [ "cfg-if", "pin-project-lite", @@ -2178,9 +2408,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" dependencies = [ "once_cell", ] @@ -2214,46 +2444,61 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf" [[package]] name = "unicode-linebreak" -version = "0.1.4" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5faade31a542b8b35855fff6e8def199853b2da8da256da52f52f1316ee3137" +checksum = "3a52dcaab0c48d931f7cc8ef826fa51690a08e1ea55117ef26f89864f532383f" dependencies = [ - "hashbrown", "regex", ] [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" dependencies = [ "tinyvec", ] [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" [[package]] name = "url" -version = "2.3.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" dependencies = [ "form_urlencoded", "idna", + "matches", "percent-encoding", ] +[[package]] +name = "uuid" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f" +dependencies = [ + "getrandom", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -2266,6 +2511,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "virtue" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b60dcd6a64dd45abf9bd426970c9843726da7fc08f44cd6fcebf68c21220a63" + [[package]] name = "wait-timeout" version = "0.2.0" @@ -2304,9 +2555,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2314,24 +2565,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.43", + "quote 1.0.21", + "syn 1.0.99", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad" dependencies = [ "cfg-if", "js-sys", @@ -2341,38 +2592,38 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" dependencies = [ - "quote", + "quote 1.0.21", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.43", + "quote 1.0.21", + "syn 1.0.99", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 621cfd88..a22db3d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,16 +4,19 @@ version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" - [package.metadata.docs.rs] - all-features = true - rustdoc-args = ["--cfg", "docsrs"] +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] [features] +# Note: If you add a feature, adjust the ALMOST_ALL_FEATURES environment variable in +# main.yml and coverage.yml: default = ["with-serde"] +persistence = ["rocksdb", "uuid"] with-serde = ["serde"] with-csv = ["csv"] with-nexmark = [ - "arcstr", + "arcstr/bincode", "cached", "clap", "csv", @@ -49,6 +52,9 @@ paste = { version = "1.0.9", optional = true } bitvec = "1.0.1" xxhash-rust = { version = "0.8.6", features = ["xxh3"] } crossbeam = "0.8.2" +rocksdb = { version = "0.19", default-features = false, features = ["multi-threaded-cf"], optional = true } +bincode = { version = "2.0.0-rc.2", features = ["serde"] } +uuid = { version = "1.1.2", features = ["v4"], optional = true } # TODO: Remove these dependencies rand = { version = "0.8", optional = true } @@ -66,6 +72,7 @@ rstest = "0.15" cached = "0.38.0" proptest = "1.0.0" criterion = "0.4.0" +proptest-derive = "0.3.0" rand_xoshiro = "0.6.0" indicatif = "0.17.0-rc.11" mimalloc-rust-sys = "1.7.2" @@ -109,3 +116,7 @@ harness = false [[bench]] name = "column_leaf" harness = false + +[patch.crates-io] +# Waiting for bincode 2.0.0 to be released (https://github.com/thomcc/arcstr/pull/45) +arcstr = { git = "https://github.com/gz/arcstr.git", features = ["bincode"], rev = "b43120c", optional = true } diff --git a/benches/fraud.rs b/benches/fraud.rs index f799d47f..a13529c5 100644 --- a/benches/fraud.rs +++ b/benches/fraud.rs @@ -69,6 +69,7 @@ mod mimalloc; use anyhow::Result; +use bincode::{Decode, Encode}; use clap::Parser; use crossbeam::channel::bounded; use csv::Reader as CsvReader; @@ -100,7 +101,7 @@ const DEFAULT_BATCH_SIZE: &'static str = "10000"; const DAY_IN_SECONDS: i64 = 24 * 3600; -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, SizeOf)] +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Decode, Encode, SizeOf)] struct QueryResult { // day: Weekday, // age: u32, @@ -119,7 +120,9 @@ struct QueryResult { is_fraud: u32, } -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Deserialize, SizeOf)] +#[derive( + Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Deserialize, Decode, Encode, SizeOf, +)] struct Demographics { cc_num: F64, first: u32, @@ -132,11 +135,15 @@ struct Demographics { long: F64, city_pop: u32, job: u32, + #[bincode(with_serde)] dob: Date, } -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Deserialize, SizeOf)] +#[derive( + Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Deserialize, Decode, Encode, SizeOf, +)] struct Transaction { + #[bincode(with_serde)] #[serde(deserialize_with = "primitive_date_time_from_str")] trans_date_trans_time: PrimitiveDateTime, cc_num: F64, diff --git a/src/algebra/floats.rs b/src/algebra/floats.rs index 41a5b5e9..48d25e8a 100644 --- a/src/algebra/floats.rs +++ b/src/algebra/floats.rs @@ -396,9 +396,120 @@ impl From for F32 { } } -#[test] -fn fromstr() { - assert_eq!(Ok(F32::new(10.0)), F32::from_str("10")); - assert_eq!(Ok(F64::new(-10.0)), F64::from_str("-10")); - assert!(F32::from_str("what").is_err()); +impl bincode::Encode for F64 { + fn encode( + &self, + encoder: &mut E, + ) -> core::result::Result<(), bincode::error::EncodeError> { + bincode::Encode::encode(&self.0 .0, encoder)?; + Ok(()) + } +} + +impl bincode::Decode for F64 { + fn decode( + decoder: &mut D, + ) -> Result { + let f: f64 = bincode::Decode::decode(decoder)?; + Ok(Self::new(f)) + } +} + +impl<'de> bincode::BorrowDecode<'de> for F64 { + fn borrow_decode>( + decoder: &mut D, + ) -> Result { + let f: f64 = bincode::BorrowDecode::borrow_decode(decoder)?; + Ok(Self::new(f)) + } +} + +impl bincode::Encode for F32 { + fn encode( + &self, + encoder: &mut E, + ) -> core::result::Result<(), bincode::error::EncodeError> { + bincode::Encode::encode(&self.0 .0, encoder)?; + Ok(()) + } +} + +impl bincode::Decode for F32 { + fn decode( + decoder: &mut D, + ) -> Result { + let f: f32 = bincode::Decode::decode(decoder)?; + Ok(Self::new(f)) + } +} + +impl<'de> bincode::BorrowDecode<'de> for F32 { + fn borrow_decode>( + decoder: &mut D, + ) -> Result { + let f: f32 = bincode::BorrowDecode::borrow_decode(decoder)?; + Ok(Self::new(f)) + } +} + +#[cfg(test)] +mod tests { + use super::{F32, F64}; + use std::str::FromStr; + + #[test] + fn fromstr() { + assert_eq!(Ok(F32::new(10.0)), F32::from_str("10")); + assert_eq!(Ok(F64::new(-10.0)), F64::from_str("-10")); + assert!(F32::from_str("what").is_err()); + } + + #[test] + fn f64_decode_encode() { + let mut slice = [0u8; 12]; + + for input in [ + F64::new(-1.0), + F64::new(0.0), + F64::new(1.0), + F64::new(f64::MAX), + F64::new(f64::MIN), + F64::new(f64::NAN), + F64::new(f64::INFINITY), + ] + .into_iter() + { + let _length = + bincode::encode_into_slice(&input, &mut slice, bincode::config::standard()) + .unwrap(); + let decoded: F64 = bincode::decode_from_slice(&slice, bincode::config::standard()) + .unwrap() + .0; + assert_eq!(decoded, input); + } + } + + #[test] + fn f32_decode_encode() { + let mut slice = [0u8; 12]; + for input in [ + F32::new(-1.0), + F32::new(0.0), + F32::new(1.0), + F32::new(f32::MAX), + F32::new(f32::MIN), + F32::new(f32::NAN), + F32::new(f32::INFINITY), + ] + .into_iter() + { + let _length = + bincode::encode_into_slice(&input, &mut slice, bincode::config::standard()) + .unwrap(); + let decoded: F32 = bincode::decode_from_slice(&slice, bincode::config::standard()) + .unwrap() + .0; + assert_eq!(decoded, input); + } + } } diff --git a/src/algebra/present.rs b/src/algebra/present.rs index e0b5add9..e3a49e23 100644 --- a/src/algebra/present.rs +++ b/src/algebra/present.rs @@ -3,7 +3,20 @@ use size_of::SizeOf; use std::ops::{Add, AddAssign, Mul, MulAssign, Neg, Sub, SubAssign}; /// A zero-sized weight that indicates a value is present -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, SizeOf, Default)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + SizeOf, + Default, + bincode::Decode, + bincode::Encode, +)] pub struct Present; impl HasZero for Present { diff --git a/src/nexmark/model.rs b/src/nexmark/model.rs index 0a03d66b..c9f89d6b 100644 --- a/src/nexmark/model.rs +++ b/src/nexmark/model.rs @@ -3,13 +3,14 @@ //! Based on the equivalent [Nexmark Flink Java model classes](https://github.com/nexmark/nexmark/blob/v0.2.0/nexmark-flink/src/main/java/com/github/nexmark/flink/model). use arcstr::ArcStr; +use bincode::{Decode, Encode}; use size_of::SizeOf; /// The Nexmark Person model based on the [Nexmark Java Person class](https://github.com/nexmark/nexmark/blob/v0.2.0/nexmark-flink/src/main/java/com/github/nexmark/flink/model/Person.java). /// /// Note that Rust can simply derive the equivalent methods on the Java /// class. -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, SizeOf)] +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, SizeOf, Encode, Decode)] pub struct Person { pub id: u64, pub name: ArcStr, @@ -25,7 +26,7 @@ pub struct Person { /// /// Note that Rust can simply derive the equivalent methods on the Java /// class. -#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, SizeOf)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, SizeOf, Encode, Decode)] pub struct Auction { pub id: u64, pub item_name: ArcStr, @@ -43,7 +44,7 @@ pub struct Auction { /// /// Note that Rust can simply derive the equivalent methods on the Java /// class. -#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, SizeOf)] +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd, SizeOf, Encode, Decode)] pub struct Bid { /// Id of auction this bid is for. pub auction: u64, @@ -64,7 +65,7 @@ pub struct Bid { /// An event in the auction system, either a (new) `Person`, a (new) `Auction`, /// or a `Bid`. -#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, SizeOf)] +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, SizeOf, Encode, Decode)] pub enum Event { Person(Person), Auction(Auction), diff --git a/src/nexmark/queries/q14.rs b/src/nexmark/queries/q14.rs index c91c0988..687031f0 100644 --- a/src/nexmark/queries/q14.rs +++ b/src/nexmark/queries/q14.rs @@ -4,6 +4,7 @@ use arcstr::ArcStr; use rust_decimal::Decimal; use size_of::SizeOf; use std::hash::Hash; +use std::ops::Deref; /// Query 14: Calculation (Not in original suite) /// @@ -42,18 +43,73 @@ use std::hash::Hash; /// WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000; /// ``` -#[derive(Eq, Clone, Debug, Hash, PartialEq, PartialOrd, Ord, SizeOf)] +#[derive(Eq, Clone, Debug, Hash, PartialEq, PartialOrd, Ord, SizeOf, bincode::Decode, bincode::Encode)] +pub struct Q14Output(u64, u64, BincodeDecimal, BidTimeType, u64, ArcStr, usize); + +type Q14Stream = Stream, OrdZSet>; + +/// Wrapper type for `Decimal` that implements Decode and Encode. +/// +/// # Note +/// Since the query doesn't use spine we don't actually end up +/// serializing/deserializing Decimals but we still need to implement it to +/// satisfy trait constraints. +/// +/// For the future we can submit a PR to `rust_decimal` to implement `Decode` +/// and `Encode` for `Decimal` directly or if we end up using rykv anyways +/// `rust_decimal` has support for it already. +#[derive(Eq, Clone, Debug, Hash, PartialEq, PartialOrd, Ord)] +struct BincodeDecimal(Decimal); + +impl bincode::Encode for BincodeDecimal { + fn encode( + &self, + encoder: &mut E, + ) -> core::result::Result<(), bincode::error::EncodeError> { + bincode::Encode::encode(&self.0.to_string(), encoder)?; + Ok(()) + } +} + +impl bincode::Decode for BincodeDecimal { + fn decode( + decoder: &mut D, + ) -> Result { + let s: String = bincode::Decode::decode(decoder)?; + Ok(Self(Decimal::from_str_exact(&s).unwrap())) + } +} + +impl<'de> bincode::BorrowDecode<'de> for BincodeDecimal { + fn borrow_decode>( + decoder: &mut D + ) -> Result { + let s: String = bincode::BorrowDecode::borrow_decode(decoder)?; + Ok(Self(Decimal::from_str_exact(&s).unwrap())) + } +} + +impl Deref for BincodeDecimal { + type Target = Decimal; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl SizeOf for BincodeDecimal { + fn size_of_children(&self, context: &mut size_of::Context) { + self.0.size_of_children(context); + } +} + +#[derive(Eq, Clone, Debug, Hash, PartialEq, PartialOrd, Ord, SizeOf, bincode::Decode, bincode::Encode)] enum BidTimeType { Day, Night, Other, } -#[derive(Eq, Clone, Debug, Hash, PartialEq, PartialOrd, Ord, SizeOf)] -pub struct Q14Output(u64, u64, Decimal, BidTimeType, u64, ArcStr, usize); - -type Q14Stream = Stream, OrdZSet>; - // This is used because we can't currently use chrono.Utc, which would simply // be Utc.timestamp_millis(b.date_time as i64).hour(), as it's waiting on a // release to fix a security issue. @@ -70,7 +126,7 @@ pub fn q14(input: NexmarkStream) -> Q14Stream { Some(Q14Output( b.auction, b.bidder, - new_price, + BincodeDecimal(new_price), match hour_for_millis(b.date_time) { 8..=18 => BidTimeType::Day, 20..=23 | 0..=6 => BidTimeType::Night, @@ -98,14 +154,14 @@ mod tests { use rstest::rstest; #[rstest] - #[case::decimal_price_converted(2_000_000, 0, "", zset![Q14Output(1, 1, Decimal::new(1_816_000_000, 3), BidTimeType::Night, 0, ArcStr::new(), 0) => 1])] + #[case::decimal_price_converted(2_000_000, 0, "", zset![Q14Output(1, 1, BincodeDecimal(Decimal::new(1_816_000_000, 3)), BidTimeType::Night, 0, ArcStr::new(), 0) => 1])] #[case::decimal_price_converted_outside_range(1_000_000, 0, "", zset![])] #[case::decimal_price_converted_on_exclusive_boundary(1_000_000, 0, "", zset![])] - #[case::date_time_is_nighttime(2_000_000, 20*60*60*1000 + 1, "", zset![Q14Output(1, 1, Decimal::new(1_816_000_000, 3), BidTimeType::Night, 20*60*60*1000 + 1, ArcStr::new(), 0) => 1])] - #[case::date_time_is_daytime(2_000_000, 8*60*60*1000 + 1, "", zset![Q14Output(1, 1, Decimal::new(1_816_000_000, 3), BidTimeType::Day, 8*60*60*1000 + 1, ArcStr::new(), 0) => 1])] - #[case::date_time_is_daytime_2022(2_000_000, 52*366*24*60*60*1000 + 8*60*60*1000 + 1, "", zset![Q14Output(1, 1, Decimal::new(1_816_000_000, 3), BidTimeType::Day, 52*366*24*60*60*1000 + 8*60*60*1000 + 1, ArcStr::new(), 0) => 1])] - #[case::date_time_is_othertime(2_000_000, 8*60*60*1000 - 1, "", zset![Q14Output(1, 1, Decimal::new(1_816_000_000, 3), BidTimeType::Other, 8*60*60*1000 - 1, ArcStr::new(), 0) => 1])] - #[case::counts_cs_in_extra(2_000_000, 0, "cause I can't calculate has four of them.", zset![Q14Output(1, 1, Decimal::new(1_816_000_000, 3), BidTimeType::Night, 0, String::from("cause I can't calculate has four of them.").into(), 4) => 1])] + #[case::date_time_is_nighttime(2_000_000, 20*60*60*1000 + 1, "", zset![Q14Output(1, 1, BincodeDecimal(Decimal::new(1_816_000_000, 3)), BidTimeType::Night, 20*60*60*1000 + 1, ArcStr::new(), 0) => 1])] + #[case::date_time_is_daytime(2_000_000, 8*60*60*1000 + 1, "", zset![Q14Output(1, 1, BincodeDecimal(Decimal::new(1_816_000_000, 3)), BidTimeType::Day, 8*60*60*1000 + 1, ArcStr::new(), 0) => 1])] + #[case::date_time_is_daytime_2022(2_000_000, 52*366*24*60*60*1000 + 8*60*60*1000 + 1, "", zset![Q14Output(1, 1, BincodeDecimal(Decimal::new(1_816_000_000, 3)), BidTimeType::Day, 52*366*24*60*60*1000 + 8*60*60*1000 + 1, ArcStr::new(), 0) => 1])] + #[case::date_time_is_othertime(2_000_000, 8*60*60*1000 - 1, "", zset![Q14Output(1, 1, BincodeDecimal(Decimal::new(1_816_000_000, 3)), BidTimeType::Other, 8*60*60*1000 - 1, ArcStr::new(), 0) => 1])] + #[case::counts_cs_in_extra(2_000_000, 0, "cause I can't calculate has four of them.", zset![Q14Output(1, 1, BincodeDecimal(Decimal::new(1_816_000_000, 3)), BidTimeType::Night, 0, String::from("cause I can't calculate has four of them.").into(), 4) => 1])] fn test_q14( #[case] price: usize, #[case] date_time: u64, diff --git a/src/nexmark/queries/q15.rs b/src/nexmark/queries/q15.rs index ef2cab7f..076780b4 100644 --- a/src/nexmark/queries/q15.rs +++ b/src/nexmark/queries/q15.rs @@ -56,7 +56,7 @@ use time::{ /// GROUP BY DATE_FORMAT(dateTime, 'yyyy-MM-dd'); /// ``` -#[derive(Eq, Clone, SizeOf, Debug, Default, Hash, PartialEq, PartialOrd, Ord)] +#[derive(Eq, Clone, SizeOf, Debug, Default, Hash, PartialEq, PartialOrd, Ord, bincode::Decode, bincode::Encode)] pub struct Q15Output { day: String, total_bids: usize, diff --git a/src/nexmark/queries/q16.rs b/src/nexmark/queries/q16.rs index 123f5459..16351143 100644 --- a/src/nexmark/queries/q16.rs +++ b/src/nexmark/queries/q16.rs @@ -68,7 +68,19 @@ use time::{ /// GROUP BY channel, DATE_FORMAT(dateTime, 'yyyy-MM-dd'); /// ``` -#[derive(Eq, Clone, Debug, Default, Hash, PartialEq, PartialOrd, Ord, SizeOf)] +#[derive( + Eq, + Clone, + Debug, + Default, + Hash, + PartialEq, + PartialOrd, + Ord, + SizeOf, + bincode::Encode, + bincode::Decode, +)] pub struct Q16Output { channel: ArcStr, day: ArcStr, @@ -89,7 +101,19 @@ pub struct Q16Output { type Q16Stream = Stream, OrdZSet>; -#[derive(Clone, Debug, Default, Eq, Hash, PartialEq, PartialOrd, Ord, SizeOf)] +#[derive( + Clone, + Debug, + Default, + Eq, + Hash, + PartialEq, + PartialOrd, + Ord, + SizeOf, + bincode::Decode, + bincode::Encode, +)] pub struct Q16Intermediate1( isize, (u8, u8), @@ -104,7 +128,19 @@ pub struct Q16Intermediate1( isize, ); -#[derive(Clone, Debug, Default, Eq, Hash, PartialEq, PartialOrd, Ord, SizeOf)] +#[derive( + Clone, + Debug, + Default, + Eq, + Hash, + PartialEq, + PartialOrd, + Ord, + SizeOf, + bincode::Decode, + bincode::Encode, +)] pub struct Q16Intermediate2( isize, (u8, u8), diff --git a/src/nexmark/queries/q9.rs b/src/nexmark/queries/q9.rs index 4b26c124..cee20b9f 100644 --- a/src/nexmark/queries/q9.rs +++ b/src/nexmark/queries/q9.rs @@ -56,7 +56,7 @@ use size_of::SizeOf; /// WHERE rownum <= 1; /// ``` -#[derive(Eq, Clone, Debug, Hash, PartialEq, PartialOrd, Ord, SizeOf)] +#[derive(Eq, Clone, Debug, Hash, PartialEq, PartialOrd, Ord, SizeOf, bincode::Decode, bincode::Encode)] pub struct Q9Output( u64, ArcStr, diff --git a/src/operator/aggregate/average.rs b/src/operator/aggregate/average.rs index cc6bdb73..7c47a5ee 100644 --- a/src/operator/aggregate/average.rs +++ b/src/operator/aggregate/average.rs @@ -63,6 +63,35 @@ impl Avg { } } +impl bincode::Encode for Avg +where + T: bincode::Encode + bincode::Decode, + R: bincode::Encode + bincode::Decode, +{ + fn encode( + &self, + encoder: &mut E, + ) -> core::result::Result<(), bincode::error::EncodeError> { + bincode::Encode::encode(&self.sum, encoder)?; + bincode::Encode::encode(&self.count, encoder)?; + Ok(()) + } +} + +impl bincode::Decode for Avg +where + T: bincode::Encode + bincode::Decode, + R: bincode::Encode + bincode::Decode, +{ + fn decode( + decoder: &mut D, + ) -> Result { + let sum: T = bincode::Decode::decode(decoder)?; + let count: R = bincode::Decode::decode(decoder)?; + Ok(Self::new(sum, count)) + } +} + impl HasZero for Avg where T: HasZero, @@ -296,4 +325,25 @@ mod tests { let output = apply_average(input); assert_eq!(output, expected); } + + #[test] + fn avg_decode_encode() { + let mut slice = [0u8; 20]; + for input in [ + Avg::new(usize::MIN, isize::MIN), + Avg::new(0, 0), + Avg::new(usize::MAX, isize::MAX), + ] + .into_iter() + { + let _length = + bincode::encode_into_slice(&input, &mut slice, bincode::config::standard()) + .unwrap(); + let decoded: Avg = + bincode::decode_from_slice(&slice, bincode::config::standard()) + .unwrap() + .0; + assert_eq!(decoded, input); + } + } } diff --git a/src/operator/aggregate/mod.rs b/src/operator/aggregate/mod.rs index ba3bcdbc..ad1fd0a3 100644 --- a/src/operator/aggregate/mod.rs +++ b/src/operator/aggregate/mod.rs @@ -20,8 +20,7 @@ use crate::{ time::Timestamp, trace::{ cursor::{Cursor, CursorGroup}, - spine_fueled::Spine, - Batch, BatchReader, Builder, + Batch, BatchReader, Builder, Spine, }, DBData, DBTimestamp, DBWeight, OrdIndexedZSet, OrdZSet, }; @@ -799,6 +798,7 @@ mod test { proptest! { #[test] + #[cfg_attr(feature = "persistence", ignore = "takes a long time?")] fn proptest_aggregate_test_st(inputs in test_input()) { let iterations = inputs.len(); let circuit = Circuit::build(|circuit| aggregate_test_circuit(circuit, inputs)).unwrap().0; @@ -809,6 +809,7 @@ mod test { } #[test] + #[cfg_attr(feature = "persistence", ignore = "takes a long time?")] fn proptest_aggregate_test_mt(inputs in test_input(), workers in (2..=16usize)) { let iterations = inputs.len(); let mut circuit = Runtime::init_circuit(workers, |circuit| aggregate_test_circuit(circuit, inputs)).unwrap().0; diff --git a/src/operator/communication/shard.rs b/src/operator/communication/shard.rs index 8f5c29b2..a6673b0c 100644 --- a/src/operator/communication/shard.rs +++ b/src/operator/communication/shard.rs @@ -7,7 +7,7 @@ use crate::{ circuit::GlobalNodeId, circuit_cache_key, default_hash, - trace::{cursor::Cursor, spine_fueled::Spine, Batch, BatchReader, Builder, Trace}, + trace::{cursor::Cursor, Batch, BatchReader, Builder, Spine, Trace}, Circuit, Runtime, Stream, }; use std::{hash::Hash, panic::Location}; diff --git a/src/operator/filter_map.rs b/src/operator/filter_map.rs index 53aa4b63..04a993aa 100644 --- a/src/operator/filter_map.rs +++ b/src/operator/filter_map.rs @@ -693,17 +693,17 @@ mod test { #[test] fn filter_map_test() { let circuit = Circuit::build(move |circuit| { - let mut input: vec::IntoIter> = - vec![zset! { (1, "1") => 1, (-1, "-1") => 1, (5, "5 foo") => 1, (-2, "-2") => 1 }].into_iter(); + let mut input: vec::IntoIter> = + vec![zset! { (1, "1".to_string()) => 1, (-1, "-1".to_string()) => 1, (5, "5 foo".to_string()) => 1, (-2, "-2".to_string()) => 1 }].into_iter(); let mut filter_output = vec![zset! { 1 => 1, 5 => 1 }].into_iter(); let mut i_filter_output = - vec![indexed_zset! { 5 => {"5 foo" => 1} }].into_iter(); + vec![indexed_zset! { 5 => {"5 foo".to_string() => 1} }].into_iter(); let mut indexed_output = vec![indexed_zset! { 1 => {1 => 1}, -1 => {-1 => 1}, 5 => {5 => 1}, -2 => {-2 => 1} }].into_iter(); let mut i_indexed_output = - vec![indexed_zset! { 2 => {"1" => 1}, -2 => {"-1" => 1}, 10 => {"5 foo" => 1}, -4 => {"-2" => 1} }].into_iter(); + vec![indexed_zset! { 2 => {"1".to_string() => 1}, -2 => {"-1".to_string() => 1}, 10 => {"5 foo".to_string() => 1}, -4 => {"-2".to_string() => 1} }].into_iter(); let mut times2_output = vec![zset! { 2 => 1, -2 => 1, 10 => 1, -4 => 1 }].into_iter(); let mut i_times2_output = @@ -723,11 +723,11 @@ mod test { let mut abs_output = vec![zset! { 1 => 2, 5 => 1, 2 => 1 }].into_iter(); let mut i_abs_output = - vec![zset! { (1, "1") => 1, (1, "-1") => 1, (5, "5 foo") => 1, (2, "-2") => 1 }].into_iter(); + vec![zset! { (1, "1".to_string()) => 1, (1, "-1".to_string()) => 1, (5, "5 foo".to_string()) => 1, (2, "-2".to_string()) => 1 }].into_iter(); let mut abs_pos_output = vec![zset! { 1 => 1, 5 => 1 }].into_iter(); let mut i_abs_pos_output = - vec![zset! { (1, "1") => 1, (5, "5 foo") => 1 }].into_iter(); + vec![zset! { (1, "1".to_string()) => 1, (5, "5 foo".to_string()) => 1 }].into_iter(); let mut sqr_output = vec![zset! { 1 => 2, 25 => 1, 4 => 1 }].into_iter(); let mut i_sqr_output = @@ -739,7 +739,7 @@ mod test { let mut sqr_pos_indexed_output = vec![indexed_zset! { 1 => {1 => 1}, 25 => {5 => 1} }].into_iter(); let mut i_sqr_pos_indexed_output = - vec![indexed_zset! { 1 => {"1" => 1}, 25 => {"5 foo" => 1} }].into_iter(); + vec![indexed_zset! { 1 => {"1".to_string() => 1}, 25 => {"5 foo".to_string() => 1} }].into_iter(); let input = circuit.add_source(Generator::new(move || input.next().unwrap())); @@ -759,16 +759,16 @@ mod test { let sqr_pos_indexed = input_ints.flat_map_index(|&n| if n > 0 { Some((n * n, n)) } else { None }); let i_filter_pos = input_indexed.filter(|(&n, s)| n > 0 && s.contains("foo")); - let i_indexed = input_indexed.map_index(|(&n, &s)| (2 * n, s)); + let i_indexed = input_indexed.map_index(move |(&n, s)| (2 * n, s.clone())); let i_times2 = input_indexed.map(|(&n, _)| n * 2); let i_times2_pos = input_indexed.flat_map(|(&n, s)| if n > 0 && s.contains("foo") { Some(n * 2) } else { None }); let i_neg = input_indexed.map(|(n, _)| -n); let i_neg_pos = input_indexed.flat_map(|(&n, s)| if n > 0 && s.contains("foo") { Some(-n) } else { None }); - let i_abs = input_indexed.map(|(n, &s)| (n.abs(), s)); - let i_abs_pos = input_indexed.flat_map(|(&n, &s)| if n > 0 { Some((n.abs(), s)) } else { None }); + let i_abs = input_indexed.map(|(n, s)| (n.abs(), s.clone())); + let i_abs_pos = input_indexed.flat_map(|(&n, s)| if n > 0 { Some((n.abs(), s.clone())) } else { None }); let i_sqr = input_indexed.map(|(n, _)| n * n); let i_sqr_pos = input_indexed.flat_map(|(&n, s)| if n > 0 && s.contains("foo") { Some(n * n) } else { None }); - let i_sqr_pos_indexed = input_indexed.flat_map_index(|(&n, &s)| if n > 0 { Some((n * n, s)) } else { None }); + let i_sqr_pos_indexed = input_indexed.flat_map_index(|(&n, s)| if n > 0 { Some((n * n, s.clone())) } else { None }); filter_pos.inspect(move |n| { assert_eq!(*n, filter_output.next().unwrap()); diff --git a/src/operator/index.rs b/src/operator/index.rs index 53496b38..bb26e039 100644 --- a/src/operator/index.rs +++ b/src/operator/index.rs @@ -242,22 +242,22 @@ mod test { fn index_test() { let circuit = Circuit::build(move |circuit| { let mut inputs = vec![ - zset!{ (1, "a") => 1 - , (1, "b") => 1 - , (2, "a") => 1 - , (2, "c") => 1 - , (1, "a") => 2 - , (1, "b") => -1 + zset!{ (1, 'a') => 1 + , (1, 'b') => 1 + , (2, 'a') => 1 + , (2, 'c') => 1 + , (1, 'a') => 2 + , (1, 'b') => -1 }, - zset!{ (1, "d") => 1 - , (1, "e") => 1 - , (2, "a") => -1 - , (3, "a") => 2 + zset!{ (1, 'd') => 1 + , (1, 'e') => 1 + , (2, 'a') => -1 + , (3, 'a') => 2 }, ].into_iter(); let mut outputs = vec![ - indexed_zset!{ 1 => {"a" => 3}, 2 => {"a" => 1, "c" => 1}}, - indexed_zset!{ 1 => {"a" => 3, "d" => 1, "e" => 1}, 2 => {"c" => 1}, 3 => {"a" => 2}}, + indexed_zset!{ 1 => {'a' => 3}, 2 => {'a' => 1, 'c' => 1}}, + indexed_zset!{ 1 => {'a' => 3, 'd' => 1, 'e' => 1}, 2 => {'c' => 1}, 3 => {'a' => 2}}, ].into_iter(); circuit.add_source(Generator::new(move || inputs.next().unwrap() )) .index() @@ -275,22 +275,22 @@ mod test { fn index_with_test() { let circuit = Circuit::build(move |circuit| { let mut inputs = vec![ - zset!{ (1, "a") => 1 - , (1, "b") => 1 - , (2, "a") => 1 - , (2, "c") => 1 - , (1, "a") => 2 - , (1, "b") => -1 + zset!{ (1, 'a') => 1 + , (1, 'b') => 1 + , (2, 'a') => 1 + , (2, 'c') => 1 + , (1, 'a') => 2 + , (1, 'b') => -1 }, - zset!{ (1, "d") => 1 - , (1, "e") => 1 - , (2, "a") => -1 - , (3, "a") => 2 + zset!{ (1, 'd') => 1 + , (1, 'e') => 1 + , (2, 'a') => -1 + , (3, 'a') => 2 }, ].into_iter(); let mut outputs = vec![ - indexed_zset!{ 1 => {"a" => 3}, 2 => {"a" => 1, "c" => 1}}, - indexed_zset!{ 1 => {"a" => 3, "d" => 1, "e" => 1}, 2 => {"c" => 1}, 3 => {"a" => 2}}, + indexed_zset!{ 1 => {'a' => 3}, 2 => {'a' => 1, 'c' => 1}}, + indexed_zset!{ 1 => {'a' => 3, 'd' => 1, 'e' => 1}, 2 => {'c' => 1}, 3 => {'a' => 2}}, ].into_iter(); circuit.add_source(Generator::new(move || inputs.next().unwrap() )) .index_with(|&(k, v)| (k, v)) diff --git a/src/operator/join.rs b/src/operator/join.rs index 5c17ab73..b06b8c01 100644 --- a/src/operator/join.rs +++ b/src/operator/join.rs @@ -9,10 +9,7 @@ use crate::{ }, circuit_cache_key, time::Timestamp, - trace::{ - cursor::Cursor as TraceCursor, spine_fueled::Spine, Batch, BatchReader, Batcher, Builder, - Trace, - }, + trace::{cursor::Cursor as TraceCursor, Batch, BatchReader, Batcher, Builder, Spine, Trace}, DBData, DBTimestamp, OrdIndexedZSet, OrdZSet, }; use size_of::{Context, SizeOf}; @@ -839,30 +836,30 @@ mod test { let circuit = Circuit::build(move |circuit| { let mut input1 = vec![ zset! { - (1, "a") => 1, - (1, "b") => 2, - (2, "c") => 3, - (2, "d") => 4, - (3, "e") => 5, - (3, "f") => -2, + (1, "a".to_string()) => 1, + (1, "b".to_string()) => 2, + (2, "c".to_string()) => 3, + (2, "d".to_string()) => 4, + (3, "e".to_string()) => 5, + (3, "f".to_string()) => -2, }, - zset! {(1, "a") => 1}, - zset! {(1, "a") => 1}, - zset! {(4, "n") => 2}, - zset! {(1, "a") => 0}, + zset! {(1, "a".to_string()) => 1}, + zset! {(1, "a".to_string()) => 1}, + zset! {(4, "n".to_string()) => 2}, + zset! {(1, "a".to_string()) => 0}, ] .into_iter(); let mut input2 = vec![ zset! { - (2, "g") => 3, - (2, "h") => 4, - (3, "i") => 5, - (3, "j") => -2, - (4, "k") => 5, - (4, "l") => -2, + (2, "g".to_string()) => 3, + (2, "h".to_string()) => 4, + (3, "i".to_string()) => 5, + (3, "j".to_string()) => -2, + (4, "k".to_string()) => 5, + (4, "l".to_string()) => -2, }, - zset! {(1, "b") => 1}, - zset! {(4, "m") => 1}, + zset! {(1, "b".to_string()) => 1}, + zset! {(4, "m".to_string()) => 1}, zset! {}, zset! {}, ] @@ -915,7 +912,7 @@ mod test { let mut inc_outputs = inc_outputs_vec.clone().into_iter(); let mut inc_outputs2 = inc_outputs_vec.into_iter(); - let index1: Stream<_, OrdIndexedZSet> = circuit + let index1: Stream<_, OrdIndexedZSet> = circuit .add_source(Generator::new(move || { if Runtime::worker_index() == 0 { input1.next().unwrap() @@ -924,7 +921,7 @@ mod test { } })) .index(); - let index2: Stream<_, OrdIndexedZSet> = circuit + let index2: Stream<_, OrdIndexedZSet> = circuit .add_source(Generator::new(move || { if Runtime::worker_index() == 0 { input2.next().unwrap() @@ -1063,7 +1060,19 @@ mod test { } } - #[derive(Clone, Debug, Default, Hash, Ord, PartialOrd, Eq, PartialEq, SizeOf)] + #[derive( + Clone, + Debug, + Default, + Hash, + Ord, + PartialOrd, + Eq, + PartialEq, + SizeOf, + bincode::Decode, + bincode::Encode, + )] struct Label(pub usize, pub u16); impl Display for Label { @@ -1072,7 +1081,19 @@ mod test { } } - #[derive(Clone, Debug, Default, Ord, PartialOrd, Hash, Eq, PartialEq, SizeOf)] + #[derive( + Clone, + Debug, + Default, + Ord, + PartialOrd, + Hash, + Eq, + PartialEq, + SizeOf, + bincode::Decode, + bincode::Encode, + )] struct Edge(pub usize, pub usize); impl Display for Edge { diff --git a/src/operator/join_range.rs b/src/operator/join_range.rs index aec020ca..9bcd6b46 100644 --- a/src/operator/join_range.rs +++ b/src/operator/join_range.rs @@ -205,63 +205,63 @@ mod test { let circuit = Circuit::build(move |circuit| { let mut input1 = vec![ zset! { - (1, "a") => 1, - (1, "b") => 2, - (2, "c") => 3, - (2, "d") => 4, - (3, "e") => 5, - (3, "f") => -2, + (1, 'a') => 1, + (1, 'b') => 2, + (2, 'c') => 3, + (2, 'd') => 4, + (3, 'e') => 5, + (3, 'f') => -2, }, - zset! {(1, "a") => 1}, - zset! {(1, "a") => 1}, - zset! {(4, "n") => 2}, - zset! {(1, "a") => 0}, + zset! {(1, 'a') => 1}, + zset! {(1, 'a') => 1}, + zset! {(4, 'n') => 2}, + zset! {(1, 'a') => 0}, ] .into_iter(); let mut input2 = vec![ zset! { - (2, "g") => 3, - (2, "h") => 4, - (3, "i") => 5, - (3, "j") => -2, - (4, "k") => 5, - (4, "l") => -2, + (2, 'g') => 3, + (2, 'h') => 4, + (3, 'i') => 5, + (3, 'j') => -2, + (4, 'k') => 5, + (4, 'l') => -2, }, - zset! {(1, "b") => 1}, - zset! {(4, "m") => 1}, + zset! {(1, 'b') => 1}, + zset! {(4, 'm') => 1}, zset! {}, zset! {}, ] .into_iter(); let mut outputs = vec![ zset! { - ((1, "a"), (2, "g")) => 3, - ((1, "a"), (2, "h")) => 4, - ((1, "b"), (2, "g")) => 6, - ((1, "b"), (2, "h")) => 8, - ((2, "c"), (2, "g")) => 9, - ((2, "c"), (2, "h")) => 12, - ((2, "c"), (3, "i")) => 15, - ((2, "c"), (3, "j")) => -6, - ((2, "d"), (2, "g")) => 12, - ((2, "d"), (2, "h")) => 16, - ((2, "d"), (3, "i")) => 20, - ((2, "d"), (3, "j")) => -8, - ((3, "e"), (2, "g")) => 15, - ((3, "e"), (2, "h")) => 20, - ((3, "e"), (3, "i")) => 25, - ((3, "e"), (3, "j")) => -10, - ((3, "e"), (4, "k")) => 25, - ((3, "e"), (4, "l")) => -10, - ((3, "f"), (2, "g")) => -6, - ((3, "f"), (2, "h")) => -8, - ((3, "f"), (3, "i")) => -10, - ((3, "f"), (3, "j")) => 4, - ((3, "f"), (4, "k")) => -10, - ((3, "f"), (4, "l")) => 4, + ((1, 'a'), (2, 'g')) => 3, + ((1, 'a'), (2, 'h')) => 4, + ((1, 'b'), (2, 'g')) => 6, + ((1, 'b'), (2, 'h')) => 8, + ((2, 'c'), (2, 'g')) => 9, + ((2, 'c'), (2, 'h')) => 12, + ((2, 'c'), (3, 'i')) => 15, + ((2, 'c'), (3, 'j')) => -6, + ((2, 'd'), (2, 'g')) => 12, + ((2, 'd'), (2, 'h')) => 16, + ((2, 'd'), (3, 'i')) => 20, + ((2, 'd'), (3, 'j')) => -8, + ((3, 'e'), (2, 'g')) => 15, + ((3, 'e'), (2, 'h')) => 20, + ((3, 'e'), (3, 'i')) => 25, + ((3, 'e'), (3, 'j')) => -10, + ((3, 'e'), (4, 'k')) => 25, + ((3, 'e'), (4, 'l')) => -10, + ((3, 'f'), (2, 'g')) => -6, + ((3, 'f'), (2, 'h')) => -8, + ((3, 'f'), (3, 'i')) => -10, + ((3, 'f'), (3, 'j')) => 4, + ((3, 'f'), (4, 'k')) => -10, + ((3, 'f'), (4, 'l')) => 4, }, zset! { - ((1, "a"), (1, "b")) => 1, + ((1, 'a'), (1, 'b')) => 1, }, zset! {}, zset! {}, diff --git a/src/operator/recursive.rs b/src/operator/recursive.rs index 39b7eb0d..b8194716 100644 --- a/src/operator/recursive.rs +++ b/src/operator/recursive.rs @@ -4,7 +4,7 @@ use crate::{ algebra::{ZRingValue, ZSet}, circuit::{schedule::Error as SchedulerError, Circuit, Stream}, operator::DelayedFeedback, - trace::spine_fueled::Spine, + trace::Spine, }; use impl_trait_for_tuples::impl_for_tuples; use size_of::SizeOf; @@ -119,6 +119,10 @@ impl RecursiveStreams for Tuple { } } +// We skip formatting this until +// https://github.com/rust-lang/rustfmt/issues/5420 is resolved +// (or we can run this doctest with persistence enabled) +#[rustfmt::skip] impl Circuit

{ /// Create a nested circuit that computes one or more mutually recursive /// streams of Z-sets. @@ -199,9 +203,9 @@ impl Circuit

{ /// // Initial labeling of the graph. /// let mut init_labels = vec![ /// // Start with a single label on node 1. - /// zset_set! { (1, "l1") }, + /// zset_set! { (1, "l1".to_string()) }, /// // Add a label to node 2. - /// zset_set! { (2, "l2") }, + /// zset_set! { (2, "l2".to_string()) }, /// zset! { }, /// ] /// .into_iter(); @@ -211,13 +215,13 @@ impl Circuit

{ /// /// // Expected _changes_ to the output graph labeling after each clock cycle. /// let mut expected_outputs = vec![ - /// zset! { (1, "l1") => 1, (2, "l1") => 1, (3, "l1") => 1, (4, "l1") => 1 }, - /// zset! { (1, "l2") => 1, (2, "l2") => 1, (3, "l2") => 1, (4, "l2") => 1, (5, "l1") => 1, (5, "l2") => 1 }, - /// zset! { (2, "l1") => -1, (3, "l1") => -1, (4, "l1") => -1, (5, "l1") => -1 }, + /// zset! { (1, "l1".to_string()) => 1, (2, "l1".to_string()) => 1, (3, "l1".to_string()) => 1, (4, "l1".to_string()) => 1 }, + /// zset! { (1, "l2".to_string()) => 1, (2, "l2".to_string()) => 1, (3, "l2".to_string()) => 1, (4, "l2".to_string()) => 1, (5, "l1".to_string()) => 1, (5, "l2".to_string()) => 1 }, + /// zset! { (2, "l1".to_string()) => -1, (3, "l1".to_string()) => -1, (4, "l1".to_string()) => -1, (5, "l1".to_string()) => -1 }, /// ] /// .into_iter(); /// - /// let labels = circuit.recursive(|child, labels: Stream<_, OrdZSet<(usize, &'static str), isize>>| { + /// let labels = circuit.recursive(|child, labels: Stream<_, OrdZSet<(usize, String), isize>>| { /// // Import `edges` and `init_labels` relations from the parent circuit. /// let edges = edges.delta0(child); /// let init_labels = init_labels.delta0(child); @@ -227,7 +231,7 @@ impl Circuit

{ /// let result = labels.index() /// .join::( /// &edges.index(), - /// |_from, l, to| (*to, *l), + /// |_from, l, to| (*to, l.clone()), /// ) /// .plus(&init_labels); /// Ok(result) diff --git a/src/operator/time_series/radix_tree/mod.rs b/src/operator/time_series/radix_tree/mod.rs index b524cf7c..664924d7 100644 --- a/src/operator/time_series/radix_tree/mod.rs +++ b/src/operator/time_series/radix_tree/mod.rs @@ -350,6 +350,33 @@ pub struct Prefix { prefix_len: u32, } +impl bincode::Encode for Prefix +where + TS: bincode::Encode + bincode::Decode, +{ + fn encode( + &self, + encoder: &mut E, + ) -> core::result::Result<(), bincode::error::EncodeError> { + bincode::Encode::encode(&self.key, encoder)?; + bincode::Encode::encode(&self.prefix_len, encoder)?; + Ok(()) + } +} + +impl bincode::Decode for Prefix +where + TS: bincode::Encode + bincode::Decode, +{ + fn decode( + decoder: &mut D, + ) -> Result { + let key: TS = bincode::Decode::decode(decoder)?; + let prefix_len: u32 = bincode::Decode::decode(decoder)?; + Ok(Self { key, prefix_len }) + } +} + impl Display for Prefix where TS: PrimInt + Debug, @@ -495,6 +522,38 @@ struct ChildPtr { child_agg: A, } +impl bincode::Encode for ChildPtr +where + TS: bincode::Encode + bincode::Decode, + A: bincode::Encode + bincode::Decode, +{ + fn encode( + &self, + encoder: &mut E, + ) -> core::result::Result<(), bincode::error::EncodeError> { + bincode::Encode::encode(&self.child_prefix, encoder)?; + bincode::Encode::encode(&self.child_agg, encoder)?; + Ok(()) + } +} + +impl bincode::Decode for ChildPtr +where + TS: bincode::Encode + bincode::Decode, + A: bincode::Encode + bincode::Decode, +{ + fn decode( + decoder: &mut D, + ) -> Result { + let child_prefix: Prefix = bincode::Decode::decode(decoder)?; + let child_agg: A = bincode::Decode::decode(decoder)?; + Ok(Self { + child_prefix, + child_agg, + }) + } +} + impl Display for ChildPtr where TS: PrimInt + Debug, @@ -532,6 +591,33 @@ pub struct TreeNode { children: [Option>; RADIX], } +impl bincode::Encode for TreeNode +where + TS: bincode::Encode + bincode::Decode, + A: bincode::Encode + bincode::Decode, +{ + fn encode( + &self, + encoder: &mut E, + ) -> core::result::Result<(), bincode::error::EncodeError> { + bincode::Encode::encode(&self.children, encoder)?; + Ok(()) + } +} + +impl bincode::Decode for TreeNode +where + TS: bincode::Encode + bincode::Decode + 'static, + A: bincode::Encode + bincode::Decode + 'static, +{ + fn decode( + decoder: &mut D, + ) -> Result { + let children: [Option>; RADIX] = bincode::Decode::decode(decoder)?; + Ok(Self { children }) + } +} + impl Display for TreeNode where TS: PrimInt + Debug, @@ -870,4 +956,62 @@ pub(super) mod test { ); assert_eq!(node.aggregate::>(), Some(130)); } + + #[test] + fn prefix_decode_encode() { + let mut slice = [0u8; 20]; + for input in [ + Prefix::new(0xffff_ffff_0000_0000u64, 32), + Prefix::new(0x1234_5678_0000_1111u64, 64), + Prefix::new(u64::MAX, 64), + ] + .into_iter() + { + let _length = + bincode::encode_into_slice(&input, &mut slice, bincode::config::standard()) + .unwrap(); + let decoded: Prefix = + bincode::decode_from_slice(&slice, bincode::config::standard()) + .unwrap() + .0; + assert_eq!(decoded, input); + } + } + + #[test] + fn childptr_decode_encode() { + let mut slice = [0u8; 20]; + for input in [ + ChildPtr::from_timestamp(u64::MIN, -1), + ChildPtr::from_timestamp(0x1000_0000_0000_0000u64, 10), + ChildPtr::from_timestamp(u64::MAX, 3), + ] + .into_iter() + { + let _length = + bincode::encode_into_slice(&input, &mut slice, bincode::config::standard()) + .unwrap(); + let decoded: ChildPtr = + bincode::decode_from_slice(&slice, bincode::config::standard()) + .unwrap() + .0; + assert_eq!(decoded, input); + } + } + + #[test] + fn treenode_decode_encode() { + let mut slice = [0u8; 28]; + + let mut input = TreeNode::new(); + *input.slot_mut(1) = Some(ChildPtr::from_timestamp(0x1000_0000_0000_0000u64, 10)); + + let _length = + bincode::encode_into_slice(&input, &mut slice, bincode::config::standard()).unwrap(); + let decoded: TreeNode = + bincode::decode_from_slice(&slice, bincode::config::standard()) + .unwrap() + .0; + assert_eq!(decoded, input); + } } diff --git a/src/operator/time_series/radix_tree/partitioned_tree_aggregate.rs b/src/operator/time_series/radix_tree/partitioned_tree_aggregate.rs index fa3fa9a6..0ee97adc 100644 --- a/src/operator/time_series/radix_tree/partitioned_tree_aggregate.rs +++ b/src/operator/time_series/radix_tree/partitioned_tree_aggregate.rs @@ -13,7 +13,7 @@ use crate::{ trace::{DelayedTraceId, IntegrateTraceId, UntimedTraceAppend, Z1Trace}, Aggregator, }, - trace::{spine_fueled::Spine, Builder, Cursor}, + trace::{Builder, Cursor, Spine}, Circuit, DBData, DBWeight, OrdIndexedZSet, Stream, }; use num::PrimInt; diff --git a/src/operator/time_series/radix_tree/tree_aggregate.rs b/src/operator/time_series/radix_tree/tree_aggregate.rs index 45daa817..52c23226 100644 --- a/src/operator/time_series/radix_tree/tree_aggregate.rs +++ b/src/operator/time_series/radix_tree/tree_aggregate.rs @@ -10,7 +10,7 @@ use crate::{ trace::{DelayedTraceId, IntegrateTraceId, UntimedTraceAppend, Z1Trace}, Aggregator, }, - trace::{spine_fueled::Spine, Batch, BatchReader, Builder}, + trace::{Batch, BatchReader, Builder, Spine}, Circuit, NumEntries, OrdIndexedZSet, Stream, }; use num::PrimInt; diff --git a/src/operator/time_series/rolling_aggregate.rs b/src/operator/time_series/rolling_aggregate.rs index 21f74282..595780b8 100644 --- a/src/operator/time_series/rolling_aggregate.rs +++ b/src/operator/time_series/rolling_aggregate.rs @@ -16,7 +16,7 @@ use crate::{ trace::{DelayedTraceId, IntegrateTraceId, UntimedTraceAppend, Z1Trace}, Aggregator, }, - trace::{spine_fueled::Spine, Builder, Cursor}, + trace::{Builder, Cursor, Spine}, Circuit, DBData, DBWeight, Stream, }; use num::PrimInt; @@ -643,6 +643,7 @@ mod test { proptest! { #[test] + #[cfg_attr(feature = "persistence", ignore = "takes a long time?")] fn proptest_partitioned_over_range_sparse(trace in input_trace(5, 1_000_000, 20, 20)) { let (mut circuit, mut input) = partition_rolling_aggregate_circuit(); @@ -655,6 +656,7 @@ mod test { } #[test] + #[cfg_attr(feature = "persistence", ignore = "takes a long time?")] fn proptest_partitioned_over_range_dense(trace in input_trace(5, 1_000, 50, 20)) { let (mut circuit, mut input) = partition_rolling_aggregate_circuit(); diff --git a/src/operator/time_series/window.rs b/src/operator/time_series/window.rs index 7df7605b..f304c817 100644 --- a/src/operator/time_series/window.rs +++ b/src/operator/time_series/window.rs @@ -6,7 +6,7 @@ use crate::{ operator_traits::{Operator, TernaryOperator}, Circuit, OwnershipPreference, Scope, Stream, }, - trace::{cursor::Cursor, ord::OrdZSet, spine_fueled::Spine, Batch, BatchReader}, + trace::{cursor::Cursor, ord::OrdZSet, Batch, BatchReader, Spine}, }; use std::{borrow::Cow, cmp::max, marker::PhantomData}; @@ -225,23 +225,23 @@ mod test { let mut input = vec![ zset! { // old value before the first window, should never appear in the output. - (800, "800") => 1, (900, "900") => 1, (950, "950") => 1, (999, "999") => 1, + (800, "800".to_string()) => 1, (900, "900".to_string()) => 1, (950, "950".to_string()) => 1, (999, "999".to_string()) => 1, // will appear in the next window - (1000, "1000") => 1 + (1000, "1000".to_string()) => 1 }, zset! { // old value before the first window - (700, "700") => 1, + (700, "700".to_string()) => 1, // too late, the window already moved forward - (900, "900") => 1, - (901, "901") => 1, - (999, "999") => 1, - (1000, "1000") => 1, - (1001, "1001") => 1, // will appear in the next window - (1002, "1002") => 1, // will appear two windows later - (1003, "1003") => 1, // will appear three windows later + (900, "900".to_string()) => 1, + (901, "901".to_string()) => 1, + (999, "999".to_string()) => 1, + (1000, "1000".to_string()) => 1, + (1001, "1001".to_string()) => 1, // will appear in the next window + (1002, "1002".to_string()) => 1, // will appear two windows later + (1003, "1003".to_string()) => 1, // will appear three windows later }, - zset! { (1004, "1004") => 1 }, // no new values in this window + zset! { (1004, "1004".to_string()) => 1 }, // no new values in this window zset! {}, zset! {}, zset! {}, @@ -249,12 +249,12 @@ mod test { .into_iter(); let mut output = vec![ - zset! { "900" => 1 , "950" => 1 , "999" => 1 }, - zset! { "900" => -1 , "901" => 1 , "999" => 1 , "1000" => 2 }, - zset! { "901" => -1 , "1001" => 1 }, - zset! { "1002" => 1 }, - zset! { "1003" => 1 }, - zset! { "1004" => 1 }, + zset! { "900".to_string() => 1 , "950".to_string() => 1 , "999".to_string() => 1 }, + zset! { "900".to_string() => -1 , "901".to_string() => 1 , "999".to_string() => 1 , "1000".to_string() => 2 }, + zset! { "901".to_string() => -1 , "1001".to_string() => 1 }, + zset! { "1002".to_string() => 1 }, + zset! { "1003".to_string() => 1 }, + zset! { "1004".to_string() => 1 }, ] .into_iter(); @@ -265,7 +265,7 @@ mod test { res })); - let index1: Stream<_, OrdIndexedZSet> = circuit + let index1: Stream<_, OrdIndexedZSet> = circuit .add_source(Generator::new(move || input.next().unwrap())) .index(); index1 @@ -287,34 +287,34 @@ mod test { let mut input = vec![ // window: 995..1000 - zset! { (700, "700") => 1 , (995, "995") => 1 , (996, "996") => 1 , (999, "999") => 1 , (1000, "1000") => 1 }, - zset! { (995, "995") => 1 , (1000, "1000") => 1 , (1001, "1001") => 1 }, - zset! { (999, "999") => 1 }, - zset! { (1002, "1002") => 1 }, - zset! { (1003, "1003") => 1 }, + zset! { (700, "700".to_string()) => 1 , (995, "995".to_string()) => 1 , (996, "996".to_string()) => 1 , (999, "999".to_string()) => 1 , (1000, "1000".to_string()) => 1 }, + zset! { (995, "995".to_string()) => 1 , (1000, "1000".to_string()) => 1 , (1001, "1001".to_string()) => 1 }, + zset! { (999, "999".to_string()) => 1 }, + zset! { (1002, "1002".to_string()) => 1 }, + zset! { (1003, "1003".to_string()) => 1 }, // window: 1000..1005 - zset! { (996, "996") => 1 }, // no longer within window - zset! { (999, "999") => 1 }, - zset! { (1004, "1004") => 1 }, - zset! { (1005, "1005") => 1 }, // next window - zset! { (1010, "1010") => 1 }, + zset! { (996, "996".to_string()) => 1 }, // no longer within window + zset! { (999, "999".to_string()) => 1 }, + zset! { (1004, "1004".to_string()) => 1 }, + zset! { (1005, "1005".to_string()) => 1 }, // next window + zset! { (1010, "1010".to_string()) => 1 }, // window: 1005..1010 - zset! { (1005, "1005") => 1 }, + zset! { (1005, "1005".to_string()) => 1 }, ] .into_iter(); let mut output = vec![ - zset! { "995" => 1 , "996" => 1 , "999" => 1 }, - zset! { "995" => 1 }, - zset! { "999" => 1 }, + zset! { "995".to_string() => 1 , "996".to_string() => 1 , "999".to_string() => 1 }, + zset! { "995".to_string() => 1 }, + zset! { "999".to_string() => 1 }, zset! {}, zset! {}, - zset! { "1000" => 2 , "1001" => 1 , "1002" => 1 , "1003" => 1 , "995" => -2 , "996" => -1 , "999" => -2 }, + zset! { "1000".to_string() => 2 , "1001".to_string() => 1 , "1002".to_string() => 1 , "1003".to_string() => 1 , "995".to_string() => -2 , "996".to_string() => -1 , "999".to_string() => -2 }, zset! {}, - zset! { "1004" => 1 }, + zset! { "1004".to_string() => 1 }, zset! {}, zset! {}, - zset! { "1000" => -2 , "1001" => -1 , "1002" => -1 , "1003" => -1 , "1004" => -1 , "1005" => 2 }, + zset! { "1000".to_string() => -2 , "1001".to_string() => -1 , "1002".to_string() => -1 , "1003".to_string() => -1 , "1004".to_string() => -1 , "1005".to_string() => 2 }, ] .into_iter(); @@ -328,7 +328,7 @@ mod test { res })); - let index1: Stream<_, OrdIndexedZSet> = circuit + let index1: Stream<_, OrdIndexedZSet> = circuit .add_source(Generator::new(move || input.next().unwrap())) .index(); index1 @@ -349,30 +349,30 @@ mod test { let mut input = vec![ zset! { - (800, "800") => 1, - (900, "900") => 1, - (950, "950") => 1, - (990, "990") => 1, - (999, "999") => 1, - (1000, "1000") => 1 + (800, "800".to_string()) => 1, + (900, "900".to_string()) => 1, + (950, "950".to_string()) => 1, + (990, "990".to_string()) => 1, + (999, "999".to_string()) => 1, + (1000, "1000".to_string()) => 1 }, zset! { - (700, "700") => 1, - (900, "900") => 1, - (901, "901") => 1, - (915, "915") => 1, - (940, "940") => 1, - (985, "985") => 1, - (999, "999") => 1, - (1000, "1000") => 1, - (1001, "1001") => 1, - (1002, "1002") => 1, - (1003, "1003") => 1, + (700, "700".to_string()) => 1, + (900, "900".to_string()) => 1, + (901, "901".to_string()) => 1, + (915, "915".to_string()) => 1, + (940, "940".to_string()) => 1, + (985, "985".to_string()) => 1, + (999, "999".to_string()) => 1, + (1000, "1000".to_string()) => 1, + (1001, "1001".to_string()) => 1, + (1002, "1002".to_string()) => 1, + (1003, "1003".to_string()) => 1, }, - zset! { (1004, "1004") => 1, - (1010, "1010") => 1, - (1020, "1020") => 1, - (1039, "1039") => 1 }, + zset! { (1004, "1004".to_string()) => 1, + (1010, "1010".to_string()) => 1, + (1020, "1020".to_string()) => 1, + (1039, "1039".to_string()) => 1 }, zset! {}, zset! {}, zset! {}, @@ -380,12 +380,12 @@ mod test { .into_iter(); let mut output = vec![ - zset! { "900" => 1 , "950" => 1 , "990" => 1, "999" => 1 }, - zset! { "900" => -1 , "915" => 1 , "940" => 1 , "985" => 1, "990" => -1, "999" => -1 }, - zset! { "915" => -1 , "985" => -1 }, - zset! { "1000" => 2, "1001" => 1, "1002" => 1, "1003" => 1, "1004" => 1, "1010" => 1, "1020" => 1, "1039" => 1, "985" => 1, "990" => 1, "999" => 2 }, - zset! { "1039" => -1, "940" => -1 }, - zset! { "1020" => -1, "950" => -1 }, + zset! { "900".to_string() => 1 , "950".to_string() => 1 , "990".to_string() => 1, "999".to_string() => 1 }, + zset! { "900".to_string() => -1 , "915".to_string() => 1 , "940".to_string() => 1 , "985".to_string() => 1, "990".to_string() => -1, "999".to_string() => -1 }, + zset! { "915".to_string() => -1 , "985".to_string() => -1 }, + zset! { "1000".to_string() => 2, "1001".to_string() => 1, "1002".to_string() => 1, "1003".to_string() => 1, "1004".to_string() => 1, "1010".to_string() => 1, "1020".to_string() => 1, "1039".to_string() => 1, "985".to_string() => 1, "990".to_string() => 1, "999".to_string() => 2 }, + zset! { "1039".to_string() => -1, "940".to_string() => -1 }, + zset! { "1020".to_string() => -1, "950".to_string() => -1 }, ] .into_iter(); @@ -407,7 +407,7 @@ mod test { })); - let index1: Stream<_, OrdIndexedZSet> = circuit + let index1: Stream<_, OrdIndexedZSet> = circuit .add_source(Generator::new(move || input.next().unwrap())) .index(); index1 diff --git a/src/operator/trace.rs b/src/operator/trace.rs index fc242015..08629c12 100644 --- a/src/operator/trace.rs +++ b/src/operator/trace.rs @@ -5,7 +5,7 @@ use crate::{ Circuit, ExportId, ExportStream, GlobalNodeId, OwnershipPreference, Scope, Stream, }, circuit_cache_key, - trace::{cursor::Cursor, spine_fueled::Spine, Batch, BatchReader, Builder, Trace}, + trace::{cursor::Cursor, Batch, BatchReader, Builder, Spine, Trace}, Timestamp, }; use size_of::SizeOf; diff --git a/src/operator/upsert.rs b/src/operator/upsert.rs index 63bb28b6..8634cc6c 100644 --- a/src/operator/upsert.rs +++ b/src/operator/upsert.rs @@ -6,8 +6,7 @@ use crate::{ }, operator::trace::{DelayedTraceId, TraceAppend, TraceId, Z1Trace}, trace::{ - consolidation::consolidate, cursor::Cursor, spine_fueled::Spine, Batch, BatchReader, - Builder, Trace, + consolidation::consolidate, cursor::Cursor, Batch, BatchReader, Builder, Spine, Trace, }, utils::VecExt, Circuit, DBData, DBTimestamp, Stream, Timestamp, diff --git a/src/time/nested_ts32.rs b/src/time/nested_ts32.rs index dfbd5c77..2661f0c5 100644 --- a/src/time/nested_ts32.rs +++ b/src/time/nested_ts32.rs @@ -17,9 +17,34 @@ const EPOCH_MASK: u32 = 0x80000000; /// This representation precisely captures the nested clock value, but only /// distinguishes the latest parent clock cycle, or "epoch", (higher-order bit /// set to `1`) from all previous epochs (higher order bit is `0`). -#[derive(Clone, SizeOf, Default, Eq, PartialEq, Debug, Hash, PartialOrd, Ord)] +#[derive( + Clone, + SizeOf, + Default, + Eq, + PartialEq, + Debug, + Hash, + PartialOrd, + Ord, + bincode::Encode, + bincode::Decode, +)] pub struct NestedTimestamp32(u32); +#[cfg(test)] +impl proptest::arbitrary::Arbitrary for NestedTimestamp32 { + type Parameters = (); + type Strategy = proptest::prelude::BoxedStrategy; + + fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy { + use proptest::prelude::*; + (any::(), 0u32..10) + .prop_map(|(epoch, x)| NestedTimestamp32::new(epoch, x)) + .boxed() + } +} + impl Display for NestedTimestamp32 { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> { write!( diff --git a/src/time/product.rs b/src/time/product.rs index 6a3b5ba2..bf2b7664 100644 --- a/src/time/product.rs +++ b/src/time/product.rs @@ -9,7 +9,19 @@ use size_of::SizeOf; use std::fmt::{Debug, Display, Formatter}; /// A nested pair of timestamps, one outer and one inner. -#[derive(Copy, Clone, Hash, Eq, PartialEq, Default, Ord, PartialOrd, SizeOf)] +#[derive( + Copy, + Clone, + Hash, + Eq, + PartialEq, + Default, + Ord, + PartialOrd, + SizeOf, + bincode::Decode, + bincode::Encode, +)] pub struct Product { /// Outer timestamp. pub outer: TOuter, diff --git a/src/trace/cursor/mod.rs b/src/trace/cursor/mod.rs index 809ee064..47839e62 100644 --- a/src/trace/cursor/mod.rs +++ b/src/trace/cursor/mod.rs @@ -177,6 +177,31 @@ pub trait CursorDebug<'s, K: Clone, V: Clone, T: Clone, R: Clone>: Cursor<'s, K, } out } + + /// Returns values with time, weights for a given cursor. + /// + /// Starts wherever the current cursor is pointing to and walks to the end + /// of the values for the current key. + /// + /// Should only be called with `key_valid() == true`. + /// + /// # Panics + /// - Panics (in debug mode) if the key is not valid. + fn val_to_vec(&mut self) -> Vec<(V, Vec<(T, R)>)> { + debug_assert!(self.key_valid()); + let mut vs = Vec::new(); + while self.val_valid() { + let mut weights = Vec::new(); + self.map_times(|ts, r| { + weights.push((ts.clone(), r.clone())); + }); + + vs.push((self.val().clone(), weights)); + self.step_val(); + } + + vs + } } impl<'s, C, K: Clone, V: Clone, T: Clone, R: Clone> CursorDebug<'s, K, V, T, R> for C where diff --git a/src/trace/mod.rs b/src/trace/mod.rs index 3a115cdd..5bd3fe6a 100644 --- a/src/trace/mod.rs +++ b/src/trace/mod.rs @@ -14,9 +14,16 @@ pub mod consolidation; pub mod cursor; pub mod layers; pub mod ord; +#[cfg(feature = "persistence")] +pub mod persistent; +// We export the `spine_fueled` module only for testing (so we can explicitly +// choose the in-memory DS). For regular logic, since we replace it with the +// persistent feature sometimes, one should import the re-exported path e.g., +// `crate::trace::Spine`. +#[cfg(test)] pub mod spine_fueled; - -pub use cursor::{Consumer, Cursor, ValueConsumer}; +#[cfg(not(test))] +mod spine_fueled; use crate::{ algebra::{HasZero, MonoidValue}, @@ -24,7 +31,14 @@ use crate::{ time::{AntichainRef, Timestamp}, NumEntries, }; +#[cfg(feature = "persistence")] +use bincode::{Decode, Encode}; +pub use cursor::{Consumer, Cursor, ValueConsumer}; +#[cfg(feature = "persistence")] +pub use persistent::PersistentTrace as Spine; use size_of::SizeOf; +#[cfg(not(feature = "persistence"))] +pub use spine_fueled::Spine; use std::{fmt::Debug, hash::Hash}; /// Trait for data stored in batches. @@ -34,7 +48,22 @@ use std::{fmt::Debug, hash::Hash}; /// must be generic over any relational data, it is sufficient to impose /// `DBData` as a trait bound on types. Conversely, a trait bound of the form /// `B: BatchReader` implies `B::Key: DBData` and `B::Val: DBData`. +#[cfg(feature = "persistence")] +pub trait DBData: + Clone + Eq + Ord + Hash + SizeOf + Send + Debug + Decode + Encode + 'static +{ +} + +#[cfg(not(feature = "persistence"))] pub trait DBData: Clone + Eq + Ord + Hash + SizeOf + Send + Debug + 'static {} + +#[cfg(feature = "persistence")] +impl DBData for T where + T: Clone + Eq + Ord + Hash + SizeOf + Send + Debug + Decode + Encode + 'static +{ +} + +#[cfg(not(feature = "persistence"))] impl DBData for T where T: Clone + Eq + Ord + Hash + SizeOf + Send + Debug + 'static {} /// Trait for data types used as weights. @@ -363,6 +392,8 @@ pub mod rc_blanket_impls { } /// The number of updates in the batch. + /// + /// This won't be accurate on all implementations either. fn len(&self) -> usize { B::len(self) } diff --git a/src/trace/ord/mod.rs b/src/trace/ord/mod.rs index 537e73bd..c2ed7b86 100644 --- a/src/trace/ord/mod.rs +++ b/src/trace/ord/mod.rs @@ -34,7 +34,7 @@ pub use key_batch::OrdKeyBatch; pub use val_batch::OrdValBatch; pub use zset_batch::OrdZSet; -use crate::trace::spine_fueled::Spine; +use crate::trace::Spine; /// A trace implementation using a spine of ordered lists. pub type OrdValSpine = Spine>; diff --git a/src/trace/persistent/cursor.rs b/src/trace/persistent/cursor.rs new file mode 100644 index 00000000..6a9822fb --- /dev/null +++ b/src/trace/persistent/cursor.rs @@ -0,0 +1,299 @@ +//! Implements the cursor for the persistent trace. +//! +//! The cursor is a wrapper around the RocksDB iterator and some custom logic +//! that ensures it behaves the same as our other cursors. + +use std::sync::Arc; + +use bincode::decode_from_slice; +use rocksdb::{BoundColumnFamily, DBRawIterator}; + +use super::trace::PersistedValue; +use super::{ReusableEncodeBuffer, Values, BINCODE_CONFIG, ROCKS_DB_INSTANCE}; +use crate::algebra::PartialOrder; +use crate::trace::{Batch, Cursor}; + +/// The cursor for the persistent trace. +pub struct PersistentTraceCursor<'s, B: Batch + 's> { + /// Iterator of the underlying RocksDB instance. + db_iter: DBRawIterator<'s>, + + /// Current key this will always be Some(key) if the iterator is valid. + /// + /// Once we reached the end (or seeked to the end) will be set to None. + cur_key: Option, + /// Values for the `cur_key`. + /// + /// This will always be `Some(values)` if `cur_key` is something/valid or + /// `None` otherwise. + cur_vals: Option>, + /// Current index of iterator in `cur_vals`. + /// + /// Value will be `0 <= val_idx < cur_vals.len()`. + val_idx: usize, + + /// Temporary storage to hold the last key of the cursor. + /// + /// A reference to this is handed out by [`Cursor::last_key`]. + last_key: Option, + + /// Temporary storage serializing seeked keys. + tmp_key: ReusableEncodeBuffer, +} + +impl<'s, B> PersistentTraceCursor<'s, B> +where + B: Batch + 's, +{ + /// Loads the current key and its values from RocksDB and stores them in the + /// [`PersistentTraceCursor`] struct. + /// + /// # Panics + /// - In case the `db_iter` is invalid. + fn update_current_key_weight(&mut self) { + assert!(self.db_iter.valid()); + let (key, values) = PersistentTraceCursor::<'s, B>::read_key_val_weights(&mut self.db_iter); + + self.val_idx = 0; + self.cur_key = key; + self.cur_vals = values; + } + + /// Loads the current key and its values from RocksDB. + /// + /// # Returns + /// - The key and its values. + /// - `None` if the iterator is invalid. + #[allow(clippy::type_complexity)] + fn read_key_val_weights( + iter: &mut DBRawIterator<'s>, + ) -> (Option, Option>) { + loop { + if !iter.valid() { + return (None, None); + } + + if let (Some(k), Some(v)) = (iter.key(), iter.value()) { + let (key, _) = + decode_from_slice(k, BINCODE_CONFIG).expect("Can't decode current key"); + let (values, _): (PersistedValue, usize) = + decode_from_slice(v, BINCODE_CONFIG).expect("Can't decode current value"); + match values { + PersistedValue::Values(vals) => { + return (Some(key), Some(vals)); + } + PersistedValue::Tombstone => { + // Skip tombstones: + // they will be deleted by the compaction filter at a later point in time + iter.next(); + continue; + } + } + } else { + // This holds according to the RocksDB C++ API docs + unreachable!("db_iter.valid() implies Some(key)") + } + } + } +} + +impl<'s, B: Batch> PersistentTraceCursor<'s, B> { + /// Creates a new [`PersistentTraceCursor`], requires to pass a handle to + /// the column family of the trace. + pub(super) fn new(cf: &Arc) -> Self { + let mut db_iter = ROCKS_DB_INSTANCE.raw_iterator_cf(cf); + db_iter.seek_to_first(); + let (cur_key, cur_vals) = + PersistentTraceCursor::<'s, B>::read_key_val_weights(&mut db_iter); + + PersistentTraceCursor { + db_iter, + val_idx: 0, + cur_key, + cur_vals, + last_key: None, + tmp_key: ReusableEncodeBuffer(Vec::new()), + } + } +} + +impl<'s, B: Batch> Cursor<'s, B::Key, B::Val, B::Time, B::R> for PersistentTraceCursor<'s, B> { + #[inline] + fn key_valid(&self) -> bool { + self.cur_key.is_some() + } + + #[inline] + fn val_valid(&self) -> bool { + // A value is valid if `cur_vals` is set and we have not iterated past + // the length of the current values. + self.cur_vals.is_some() && self.val_idx < self.cur_vals.as_ref().unwrap().len() + } + + #[inline] + fn key(&self) -> &B::Key { + self.cur_key.as_ref().unwrap() + } + + #[inline] + fn val(&self) -> &B::Val { + &self.cur_vals.as_ref().unwrap()[self.val_idx].0 + } + + #[inline] + fn map_times(&mut self, mut logic: L) { + if self.val_valid() { + for (time, val) in self.cur_vals.as_ref().unwrap()[self.val_idx].1.iter() { + logic(time, val); + } + } + } + + fn map_times_through(&mut self, mut logic: L, upper: &B::Time) { + if self.key_valid() && self.val_valid() { + // Note that `map_times_through` uses `less_equal` to determine if + // `logic` should be called on a given `(time, diff)`. However, + // `cur_vals` is sorted by `Ord` hence we have to go through all the + // values to determine how many times `logic` should be called. + for (time, val) in self.cur_vals.as_ref().unwrap()[self.val_idx].1.iter() { + if time.less_equal(upper) { + logic(time, val); + } + } + } + } + + #[inline] + fn weight(&mut self) -> B::R + where + B::Time: PartialEq<()>, + { + // This is super ugly because the RocksDB storage format is generic and + // needs to support all the different data-structures, but if we can + // call weight we can basically just access the first elements in the + // list of lists of lists of lists... + self.cur_vals.as_ref().unwrap()[self.val_idx].1[0].1.clone() + } + + #[inline] + fn step_key(&mut self) { + if self.db_iter.valid() { + // Note: RocksDB only allows to call `next` on a `valid` cursor + self.db_iter.next(); + + if self.db_iter.valid() { + self.update_current_key_weight(); + } else { + self.cur_key = None; + self.cur_vals = None; + } + } else { + self.cur_key = None; + self.cur_vals = None; + } + } + + /// Returns the last key in the cursor or `None` if the cursor is empty. + /// + /// This method should not mutate the cursor position/state. So we seek to + /// the end and back again, since seeking is generally efficient in RocksDB + /// due to indexes it shouldn't be a huge problem aside from having to + /// serialize. + fn last_key(&mut self) -> Option<&B::Key> { + self.db_iter.seek_to_last(); + + if self.db_iter.valid() { + let (key, _) = decode_from_slice(self.db_iter.key().unwrap(), BINCODE_CONFIG) + .expect("Can't decode current key"); + self.last_key = Some(key); + + // Revert iterator + match &self.cur_key { + Some(key) => { + // TODO: Check if it's better to call db_iter.key() in the + // beginning of the function rather than encoding it here... + let encoded_key = self.tmp_key.encode(key).expect("Can't encode `key`"); + self.db_iter.seek(encoded_key); + } + None => { + self.db_iter.next(); + debug_assert!(!self.db_iter.valid()); + } + } + } else { + self.last_key = None; + } + + self.last_key.as_ref() + } + + #[inline] + fn seek_key(&mut self, key: &B::Key) { + if self.cur_key.is_none() { + // We are at the end of the cursor. + return; + } + if let Some(cur_key) = self.cur_key.as_ref() { + if cur_key >= key { + // The rocksdb seek call will start from the beginning, whereas + // dbsp cursors will start to seek from the current position. We + // can fix the discrepancy here since we know the batch is + // ordered: If we're seeking something that's behind us, we just + // skip the seek call: + self.val_idx = 0; + return; + } + } + + let encoded_key = self.tmp_key.encode(key).expect("Can't encode `key`"); + self.db_iter.seek(encoded_key); + self.cur_key = Some(key.clone()); + + if self.db_iter.valid() { + self.update_current_key_weight(); + } else { + self.cur_key = None; + self.cur_vals = None; + self.val_idx = 0; + } + } + + #[inline] + fn step_val(&mut self) { + self.val_idx += 1; + } + + #[inline] + fn seek_val(&mut self, val: &B::Val) { + while self.val_valid() && self.val() < val { + self.step_val(); + } + } + + #[inline] + fn seek_val_with

(&mut self, predicate: P) + where + P: Fn(&B::Val) -> bool, + { + while self.val_valid() && !predicate(self.val()) { + self.step_val(); + } + } + + #[inline] + fn rewind_keys(&mut self) { + self.db_iter.seek_to_first(); + if self.db_iter.valid() { + self.update_current_key_weight(); + } else { + self.cur_key = None; + self.cur_vals = None; + self.val_idx = 0; + } + } + + #[inline] + fn rewind_vals(&mut self) { + self.val_idx = 0; + } +} diff --git a/src/trace/persistent/mod.rs b/src/trace/persistent/mod.rs new file mode 100644 index 00000000..64fcc84b --- /dev/null +++ b/src/trace/persistent/mod.rs @@ -0,0 +1,138 @@ +//! This module implements logic and datastructures to provide a trace that is +//! using on-disk storage with the help of RocksDB. + +use std::cmp::Ordering; + +use bincode::config::{BigEndian, Fixint}; +use bincode::enc::write::Writer; +use bincode::error::EncodeError; +use bincode::{decode_from_slice, Decode, Encode}; +use once_cell::sync::Lazy; +use rocksdb::{Cache, DBCompressionType, Options, DB}; +use uuid::Uuid; + +mod cursor; +#[cfg(test)] +mod tests; +mod trace; + +/// A single value with many time and weight tuples. +type ValueTimeWeights = (V, Vec<(T, R)>); + +/// A collection of values with time and weight tuples, this is the type that we +/// persist in RocksDB under values. +type Values = Vec>; + +/// The cursor for the persistent trace. +pub use cursor::PersistentTraceCursor; +/// The persistent trace itself, it should be equivalent to the [`Spine`]. +pub use trace::PersistentTrace; + +/// DB in-memory cache size [bytes]. +/// +/// # TODO +/// Set to 1 GiB for now, in the future we should probably make this +/// configurable or determine it based on the system parameters. +const DB_DRAM_CACHE_SIZE: usize = 1024 * 1024 * 1024; + +/// Path of the RocksDB database file on disk. +/// +/// # TODO +/// Eventually should be supplied as a command-line argument or provided in a +/// config file. +static DB_PATH: Lazy = Lazy::new(|| format!("/tmp/{}.db", Uuid::new_v4())); + +/// Options for the RocksDB database +static DB_OPTS: Lazy = Lazy::new(|| { + let cache = Cache::new_lru_cache(DB_DRAM_CACHE_SIZE).expect("Can't create cache for DB"); + let mut global_opts = Options::default(); + // Create the database file if it's missing (the default behavior) + global_opts.create_if_missing(true); + // Disable compression for now, this will help with benchmarking but + // probably not ideal for a production setting + global_opts.set_compression_type(DBCompressionType::None); + // Ensure we use a shared cache for all column families + global_opts.set_row_cache(&cache); + // RocksDB doesn't like to close files by default, if we set this it limits + // the number of open files by closing them again (should be set in + // accordance with ulimit) + global_opts.set_max_open_files(9000); + // Some options (that seem to hurt more than help -- needs more + // experimentation): + //global_opts.increase_parallelism(2); + //global_opts.set_max_background_jobs(2); + //global_opts.set_max_write_buffer_number(2); + //global_opts.set_write_buffer_size(1024*1024*4); + //global_opts.set_target_file_size_base(1024*1024*8); + + global_opts +}); + +/// The RocksDB instance that holds all traces (in different columns). +static ROCKS_DB_INSTANCE: Lazy = Lazy::new(|| { + // Open the database (or create it if it doesn't exist + DB::open(&DB_OPTS, DB_PATH.clone()).unwrap() +}); + +/// Configuration we use for encodings/decodings to/from RocksDB data. +static BINCODE_CONFIG: bincode::config::Configuration = + bincode::config::standard() + .with_fixed_int_encoding() + .with_big_endian(); + +/// Wrapper function for doing key comparison in RockDB. +/// +/// It works by deserializing the keys and then comparing it (as opposed to the +/// byte-wise comparison which is the default in RocksDB). +pub(self) fn rocksdb_key_comparator(a: &[u8], b: &[u8]) -> Ordering { + let (key_a, _) = decode_from_slice::(a, BINCODE_CONFIG).expect("Can't decode_from_slice"); + let (key_b, _) = decode_from_slice::(b, BINCODE_CONFIG).expect("Can't decode_from_slice"); + key_a.cmp(&key_b) +} + +/// A buffer that holds an encoded value. +/// +/// Useful to keep around in code where serialization happens repeatedly as it +/// can avoid repeated [`Vec`] allocations. +#[derive(Default)] +struct ReusableEncodeBuffer(Vec); + +impl ReusableEncodeBuffer { + /// Creates a buffer with initial capacity of `cap` bytes. + fn with_capacity(cap: usize) -> Self { + ReusableEncodeBuffer(Vec::with_capacity(cap)) + } + + /// Encodes `val` into the buffer owned by this struct. + /// + /// # Returns + /// - An error if encoding failed. + /// - A reference to the buffer where `val` was encoded into. Makes sure the + /// buffer won't change until the reference out-of-scope again. + fn encode(&mut self, val: &T) -> Result<&[u8], EncodeError> { + self.0.clear(); + bincode::encode_into_writer(val, &mut *self, BINCODE_CONFIG)?; + Ok(&self.0) + } +} + +/// We can get the internal storage if we don't need the ReusableEncodeBuffer +/// anymore with the `From` trait. +impl From for Vec { + fn from(r: ReusableEncodeBuffer) -> Vec { + r.0 + } +} + +impl Writer for &mut ReusableEncodeBuffer { + /// Allows bincode to write into the buffer. + /// + /// # Note + /// Client needs to ensure that the buffer is cleared in advance if we store + /// something new. When possible use the [`Self::encode`] method instead + /// which takes care of that. + fn write(&mut self, bytes: &[u8]) -> Result<(), EncodeError> { + self.0.extend(bytes); + Ok(()) + } +} diff --git a/src/trace/persistent/tests/mod.rs b/src/trace/persistent/tests/mod.rs new file mode 100644 index 00000000..8a83b7b0 --- /dev/null +++ b/src/trace/persistent/tests/mod.rs @@ -0,0 +1,380 @@ +//! Tests to ensure correctness of the persistent trace implementation. +//! +//! This module has: +//! - a few unit test that check interesting cases (mostly found by proptests) +//! - proptests that check that the persistent trace behaves like a spine + +#[cfg(test)] +mod proptests; + +use super::PersistentTrace; +use crate::time::NestedTimestamp32; +use crate::trace::cursor::Cursor; +use crate::trace::ord::{OrdIndexedZSet, OrdKeyBatch, OrdValBatch, OrdZSet}; +use crate::trace::{Batch, BatchReader, Batcher, Trace}; +use proptests::{spine_ptrace_are_equal, ComplexKey}; + +#[test] +fn vals_are_sorted() { + let mut val_builder = as Batch>::Batcher::new_batcher(()); + let mut b = vec![((String::from(""), 9777), 1)]; + val_builder.push_batch(&mut b); + let vset1 = val_builder.seal(); + + let mut val_builder = as Batch>::Batcher::new_batcher(()); + let mut b = vec![((String::from(""), 0), 1), ((String::from(""), 0), 1)]; + val_builder.push_batch(&mut b); + let vset2 = val_builder.seal(); + + let mut spine = + crate::trace::spine_fueled::Spine::>::new(None); + spine.insert(vset1.clone()); + spine.insert(vset2.clone()); + let mut fuel = 10000; + spine.apply_fuel(&mut fuel); + + let mut ptrace = PersistentTrace::>::new(None); + ptrace.insert(vset1.clone()); + ptrace.insert(vset2.clone()); + + assert!(spine_ptrace_are_equal(&spine, &ptrace)); +} + +#[test] +fn empty_batch_ptrace() { + let val_builder = as Batch>::Batcher::new_batcher(0); + let vset1 = val_builder.seal(); + let mut spine = + crate::trace::spine_fueled::Spine::>::new(None); + spine.insert(vset1.clone()); + let mut ptrace = PersistentTrace::>::new(None); + ptrace.insert(vset1.clone()); + + assert!(spine_ptrace_are_equal(&spine, &ptrace)); +} + +#[test] +fn insert_bug() { + let mut val_builder = + as Batch>::Batcher::new_batcher(0); + let mut b = vec![((String::from("a"), String::from("b")), 1)]; + val_builder.push_batch(&mut b); + let vset1 = val_builder.seal(); + + let mut val_builder = + as Batch>::Batcher::new_batcher(0); + let mut b = vec![ + ((String::from("a"), String::from("b")), 1), + ((String::from("a"), String::from("c")), 1), + ]; + val_builder.push_batch(&mut b); + let vset2 = val_builder.seal(); + + let mut spine = + crate::trace::spine_fueled::Spine::>::new(None); + spine.insert(vset1.clone()); + spine.insert(vset2.clone()); + let mut fuel = 10000; + spine.apply_fuel(&mut fuel); + + let mut ptrace = PersistentTrace::>::new(None); + ptrace.insert(vset1.clone()); + ptrace.insert(vset2.clone()); + + assert!(spine_ptrace_are_equal(&spine, &ptrace)); +} + +#[test] +fn times_are_sorted() { + let mut val_builder = + as Batch>::Batcher::new_batcher(473291538); + let mut b = vec![((String::from("y"), String::from("")), 1)]; + val_builder.push_batch(&mut b); + let vset1 = val_builder.seal(); + + let mut val_builder = + as Batch>::Batcher::new_batcher(0); + let mut b = vec![((String::from("y"), String::from("")), 1)]; + val_builder.push_batch(&mut b); + let vset2 = val_builder.seal(); + + let mut spine = + crate::trace::spine_fueled::Spine::>::new(None); + spine.insert(vset1.clone()); + spine.insert(vset2.clone()); + let mut fuel = 10000; + spine.apply_fuel(&mut fuel); + + let mut ptrace = PersistentTrace::>::new(None); + ptrace.insert(vset1.clone()); + ptrace.insert(vset2.clone()); + + assert!(spine_ptrace_are_equal(&spine, &ptrace)); +} + +#[test] +fn recede_to_bug() { + let mut val_builder = + as Batch>::Batcher::new_batcher(2); + let mut b = vec![((String::from("a"), 1), 1)]; + val_builder.push_batch(&mut b); + let vset1 = val_builder.seal(); + + let mut val_builder = + as Batch>::Batcher::new_batcher(0); + let mut b = vec![((String::from("a"), 1), 1)]; + val_builder.push_batch(&mut b); + let vset2 = val_builder.seal(); + + let mut spine = + crate::trace::spine_fueled::Spine::>::new(None); + spine.insert(vset1.clone()); + spine.insert(vset2.clone()); + spine.recede_to(&1); + + let mut ptrace = PersistentTrace::>::new(None); + ptrace.insert(vset1.clone()); + ptrace.insert(vset2.clone()); + ptrace.recede_to(&1); + + assert!(spine_ptrace_are_equal(&spine, &ptrace)); +} + +#[test] +fn weights_cancellation() { + // [(ComplexKey { _a: 0, ord: "" }, -8)])), Insert(((), [(ComplexKey { _a: 0, + // ord: "" }, 8)]))] + + let mut val_builder = + as Batch>::Batcher::new_batcher(0); + let mut b = vec![((String::from("a"), 1), -8)]; + val_builder.push_batch(&mut b); + let vset1 = val_builder.seal(); + + let mut val_builder = + as Batch>::Batcher::new_batcher(0); + let mut b = vec![((String::from("a"), 1), 8)]; + val_builder.push_batch(&mut b); + let vset2 = val_builder.seal(); + + let mut spine = + crate::trace::spine_fueled::Spine::>::new(None); + spine.insert(vset1.clone()); + spine.insert(vset2.clone()); + let mut fuel = 10000; + spine.apply_fuel(&mut fuel); + + let mut ptrace = PersistentTrace::>::new(None); + ptrace.insert(vset1.clone()); + ptrace.insert(vset2.clone()); + + assert!(spine_ptrace_are_equal(&spine, &ptrace)); +} + +#[test] +fn timestamp_aggregation() { + // actions = [ + // Insert((NestedTimestamp32(2147483649), [(ComplexKey { _a: 0, ord: "" }, + // 1)])), Insert((NestedTimestamp32(2), [(ComplexKey { _a: 0, ord: "" }, + // 1)])), Insert((NestedTimestamp32(1), [(ComplexKey { _a: 0, ord: "" }, + // 1)])), RecedeTo(NestedTimestamp32(2)) + //] + + let mut val_builder = + as Batch>::Batcher::new_batcher( + NestedTimestamp32::new(true, 1), + ); + let mut b = vec![( + ComplexKey { + _a: 0, + ord: String::from(""), + }, + 1, + )]; + val_builder.push_batch(&mut b); + let vset1 = val_builder.seal(); + + let mut val_builder = + as Batch>::Batcher::new_batcher( + NestedTimestamp32::new(false, 2), + ); + let mut b = vec![( + ComplexKey { + _a: 0, + ord: String::from(""), + }, + 1, + )]; + val_builder.push_batch(&mut b); + let vset2 = val_builder.seal(); + + let mut val_builder = + as Batch>::Batcher::new_batcher( + NestedTimestamp32::new(false, 1), + ); + let mut b = vec![( + ComplexKey { + _a: 0, + ord: String::from(""), + }, + 1, + )]; + val_builder.push_batch(&mut b); + let vset3 = val_builder.seal(); + + let mut spine = crate::trace::spine_fueled::Spine::< + OrdKeyBatch, + >::new(None); + spine.insert(vset1.clone()); + let mut fuel = isize::MAX; + spine.exert(&mut fuel); + spine.insert(vset2.clone()); + let mut fuel = isize::MAX; + spine.exert(&mut fuel); + spine.insert(vset3.clone()); + let mut fuel = isize::MAX; + spine.exert(&mut fuel); + spine.recede_to(&NestedTimestamp32::new(false, 2)); + + let mut ptrace = + PersistentTrace::>::new(None); + ptrace.insert(vset1.clone()); + ptrace.insert(vset2.clone()); + ptrace.insert(vset3.clone()); + ptrace.recede_to(&NestedTimestamp32::new(false, 2)); + + assert!(spine_ptrace_are_equal(&spine, &ptrace)); +} + +#[test] +fn weights_cancellation2() { + // [ + // Insert((NestedTimestamp32(0), [(ComplexKey { _a: 0, ord: "" }, -1)])), + // Insert((NestedTimestamp32(4), [(ComplexKey { _a: 0, ord: "" }, -9)])), + // Insert((NestedTimestamp32(4), [(ComplexKey { _a: 0, ord: "" }, 9)])), + // ] + let mut val_builder = + as Batch>::Batcher::new_batcher( + NestedTimestamp32::new(false, 0), + ); + let mut b = vec![( + ComplexKey { + _a: 0, + ord: String::from(""), + }, + -1, + )]; + val_builder.push_batch(&mut b); + let vset1 = val_builder.seal(); + + let mut val_builder = + as Batch>::Batcher::new_batcher( + NestedTimestamp32::new(false, 4), + ); + let mut b = vec![( + ComplexKey { + _a: 0, + ord: String::from(""), + }, + -9, + )]; + val_builder.push_batch(&mut b); + let vset2 = val_builder.seal(); + + let mut val_builder = + as Batch>::Batcher::new_batcher( + NestedTimestamp32::new(false, 4), + ); + let mut b = vec![( + ComplexKey { + _a: 0, + ord: String::from(""), + }, + 9, + )]; + val_builder.push_batch(&mut b); + let vset3 = val_builder.seal(); + + let mut spine = crate::trace::spine_fueled::Spine::< + OrdKeyBatch, + >::new(None); + spine.insert(vset1.clone()); + let mut fuel = isize::MAX; + spine.exert(&mut fuel); + spine.insert(vset2.clone()); + let mut fuel = isize::MAX; + spine.exert(&mut fuel); + spine.insert(vset3.clone()); + let mut fuel = isize::MAX; + spine.exert(&mut fuel); + spine.recede_to(&NestedTimestamp32::new(false, 2)); + + let mut ptrace = + PersistentTrace::>::new(None); + ptrace.insert(vset1.clone()); + ptrace.insert(vset2.clone()); + ptrace.insert(vset3.clone()); + ptrace.recede_to(&NestedTimestamp32::new(false, 2)); + + assert!(spine_ptrace_are_equal(&spine, &ptrace)); +} + +#[test] +fn cursor_weight() { + // Use a unit-test for weight() -- in proptests we don't call it it's + // specific for things with () timestamps + + let mut val_builder = as Batch>::Batcher::new_batcher(()); + let mut b = vec![(String::from("a"), -8)]; + val_builder.push_batch(&mut b); + let vset1 = val_builder.seal(); + + let mut spine = crate::trace::spine_fueled::Spine::>::new(None); + spine.insert(vset1.clone()); + let mut fuel = 10000; + spine.apply_fuel(&mut fuel); + + let mut ptrace = PersistentTrace::>::new(None); + ptrace.insert(vset1.clone()); + + let mut ptrace_cursor = ptrace.cursor(); + let mut spine_cursor = spine.cursor(); + + assert_eq!(ptrace_cursor.weight(), -8); + assert_eq!(ptrace_cursor.weight(), spine_cursor.weight()); +} + +#[test] +fn cursor_weight_multiple_values_bug() { + // Use a unit-test for weight() -- in proptests we don't call it it's + // specific for things with () timestamps + // + // Checks that if we have multiple values with weights we need to report the + // right weights when stepping through them + + let mut val_builder = as Batch>::Batcher::new_batcher(()); + let mut b = vec![ + ((String::from("k"), String::from("v")), 1), + ((String::from("k"), String::from("v2")), 2), + ]; + + val_builder.push_batch(&mut b); + let vset1 = val_builder.seal(); + + let mut spine = + crate::trace::spine_fueled::Spine::>::new(None); + spine.insert(vset1.clone()); + let mut fuel = 10000; + spine.apply_fuel(&mut fuel); + + let mut ptrace = PersistentTrace::>::new(None); + ptrace.insert(vset1.clone()); + + let mut ptrace_cursor = ptrace.cursor(); + let mut spine_cursor = spine.cursor(); + + assert_eq!(ptrace_cursor.weight(), spine_cursor.weight()); + ptrace_cursor.step_val(); + spine_cursor.step_val(); + assert_eq!(ptrace_cursor.weight(), spine_cursor.weight()); +} diff --git a/src/trace/persistent/tests/proptests.rs b/src/trace/persistent/tests/proptests.rs new file mode 100644 index 00000000..df53ebc3 --- /dev/null +++ b/src/trace/persistent/tests/proptests.rs @@ -0,0 +1,577 @@ +//! Property based testing that ensure correctness of the persistent trace +//! implementation. +//! +//! We define "correct" by testing if it behaves the same as Spine. + +use std::fmt::Debug; +use std::ops::Range; +use std::vec::Vec; + +use bincode::{Decode, Encode}; +use proptest::prelude::*; +use proptest_derive::Arbitrary; +use size_of::SizeOf; + +use crate::algebra::{HasZero, MonoidValue}; +use crate::time::NestedTimestamp32; +use crate::trace::cursor::Cursor; +use crate::trace::ord::{OrdIndexedZSet, OrdKeyBatch, OrdValBatch, OrdZSet}; +use crate::trace::persistent::{cursor::PersistentTraceCursor, PersistentTrace}; +use crate::trace::spine_fueled::{Spine, SpineCursor}; +use crate::trace::{Batch, BatchReader, Builder, Trace}; + +/// Maximum number of actions to perform on a trace. +const TRACE_ACTIONS_RANGE: Range = 0..8; + +/// Range for possible numbers of entries in batches. +const ENTRIES_RANGE: Range = 0..24; + +/// How many actions to execute on the cursors. +const ACTIONS_RANGE: Range = 0..64; + +/// The sample range for `R` weights. +/// +/// The bound would ideally be unconstrained +/// - but neither Spine nor PersistentTrace handles integer overflows. +/// - and by having a very small range we force more "interesting cases" where +/// weights cancel each other out. +const WEIGHT_RANGE: Range = -10..10; + +/// This is a "complex" key for testing. +/// +/// It's "complex" because it defines a custom ordering logic & has a heap +/// allocated [`String`] inside of it. +/// +/// The tests ensure that the RocksDB based data-structure adhere to the same +/// ordering as the DRAM based version which is defined through the [`Ord`] +/// trait. +#[derive(Clone, Hash, Debug, Encode, Decode, Arbitrary, SizeOf)] +pub(super) struct ComplexKey { + /// We ignore this type for ordering. + pub(super) _a: isize, + /// We use this to define the order of `Self`. + pub(super) ord: String, +} + +impl std::fmt::Display for ComplexKey { + fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { + write!(f, "{}", self.ord) + } +} + +impl PartialEq for ComplexKey { + fn eq(&self, other: &Self) -> bool { + self.ord.eq(&other.ord) + } +} + +impl Eq for ComplexKey {} + +impl PartialOrd for ComplexKey { + fn partial_cmp(&self, other: &Self) -> Option { + self.ord.partial_cmp(&other.ord) + } +} + +impl Ord for ComplexKey { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.ord.cmp(&other.ord) + } +} + +/// Makes sure that everything we find in `spine` is also in `ptrace` (and not +/// more). +pub(super) fn spine_ptrace_are_equal(spine: &Spine, ptrace: &PersistentTrace) -> bool +where + B: Batch + 'static, + B::Key: Clone + Debug + Eq + Ord + Decode + Encode, + B::Val: Clone + Debug + Eq + Ord + Decode + Encode, + B::R: Clone + Debug + Eq + Ord + Decode + Encode, + B::Time: Encode + Decode + Ord, +{ + let mut spine_cursor = spine.cursor(); + let mut ptcursor = ptrace.cursor(); + while spine_cursor.key_valid() { + if !ptcursor.key_valid() { + eprintln!("spine_cursor key_valid(true) != ptcursor key_valid(false)"); + return false; + } + + if spine_cursor.key() != ptcursor.key() { + eprintln!( + "spine_cursor key({:?}) != ptcursor key({:?})", + spine_cursor.key(), + ptcursor.key() + ); + return false; + } + + while spine_cursor.val_valid() { + if !ptcursor.val_valid() { + eprintln!( + "spine_cursor val_valid({}, val={:?}) != ptcursor val_valid({})", + spine_cursor.val_valid(), + spine_cursor.val(), + ptcursor.val_valid() + ); + return false; + } + + if spine_cursor.val() != ptcursor.val() { + eprintln!( + "spine_cursor val({:?}) != ptcursor val({:?})", + spine_cursor.val(), + ptcursor.val() + ); + return false; + } + + let mut model_invocations = Vec::new(); + let mut test_invocation = Vec::new(); + spine_cursor.map_times(|v, t| { + model_invocations.push((v.clone(), t.clone())); + }); + ptcursor.map_times(|v, t| { + test_invocation.push((v.clone(), t.clone())); + }); + assert_eq!(model_invocations, test_invocation, "time->weights mismatch"); + + spine_cursor.step_val(); + ptcursor.step_val(); + } + + debug_assert!(!spine_cursor.val_valid()); + if ptcursor.val_valid() { + eprintln!("spine_cursor !val_valid != ptcursor !val_valid"); + return false; + } + + spine_cursor.step_key(); + ptcursor.step_key(); + } + + debug_assert!(!spine_cursor.key_valid()); + if ptcursor.key_valid() { + eprintln!( + "spine_cursor !key_valid but ptcursor key_valid(key = {:?})", + ptcursor.key() + ); + return false; + } + + true +} + +/// Models calls we can make on the data-structure once we implement the +/// [`Cursor`] trait. +#[derive(Debug, Clone)] +enum CursorAction< + K: Arbitrary + Clone + 'static, + V: Arbitrary + Clone + 'static, + T: Arbitrary + Clone + 'static, +> { + StepKey, + StepVal, + SeekKey(K), + SeekVal(V), + SeekValWith(V), + RewindKeys, + RewindVals, + Key, + LastKey, + Val, + MapTimes, + MapTimesThrough(T), +} + +/// Generates a random action to perform on a cursor. +fn action< + K: Arbitrary + Clone + 'static, + V: Arbitrary + Clone + 'static, + T: Arbitrary + Clone + 'static, +>() -> impl Strategy> { + prop_oneof![ + Just(CursorAction::StepKey), + Just(CursorAction::StepVal), + Just(CursorAction::RewindKeys), + Just(CursorAction::RewindVals), + Just(CursorAction::Key), + Just(CursorAction::LastKey), + Just(CursorAction::Val), + Just(CursorAction::MapTimes), + any::().prop_map(CursorAction::MapTimesThrough), + any::().prop_map(CursorAction::SeekKey), + any::().prop_map(CursorAction::SeekVal), + any::().prop_map(CursorAction::SeekValWith), + ] +} + +/// Generates a sequence of actions to perform on a cursor. +fn actions< + K: Arbitrary + Clone + 'static, + V: Arbitrary + Clone + 'static, + T: Arbitrary + Clone + 'static, +>() -> impl Strategy>> { + prop::collection::vec(action::(), ACTIONS_RANGE) +} + +/// Performs a random series of actions on a [`Spine`] cursor and a +/// [`PersistentTrace`] cursor, and makes sure they produce the same results. +fn cursor_trait( + mut data: Vec<(B::Item, B::R)>, + ops: Vec>, +) where + I: Ord + Clone + 'static, + B: Batch + 'static, + B::Key: Arbitrary + Ord + Clone + Encode + Decode, + B::Val: Arbitrary + Ord + Clone + Encode + Decode, + B::R: Arbitrary + Ord + Clone + MonoidValue + Encode + Decode, + B::Time: Arbitrary + Clone + Encode + Decode + Default, +{ + // Builder interface wants sorted, unique(?) keys: + data.sort_unstable(); + data.dedup_by(|a, b| a.0.eq(&b.0)); + + // Instantiate a Batch + let mut batch_builder = B::Builder::new_builder(B::Time::default()); + for data_tuple in data.into_iter() { + batch_builder.push(data_tuple); + } + let batch = batch_builder.done(); + + let mut model = Spine::::new(None); + model.insert(batch.clone()); + let mut model_cursor = model.cursor(); + + let mut ptrace = PersistentTrace::::new(None); + ptrace.insert(batch); + let mut totest_cursor = ptrace.cursor(); + + // We check the non-mutating cursor interface after every + // command/mutation. + fn check_eq_invariants( + step: usize, + model_cursor: &SpineCursor, + totest_cursor: &PersistentTraceCursor, + ) where + B::Key: Ord + Clone + Encode + Decode, + B::Val: Ord + Clone + Encode + Decode, + B::Time: Encode + Decode, + B::R: MonoidValue + Encode + Decode, + { + assert_eq!( + model_cursor.key_valid(), + totest_cursor.key_valid(), + "key_valid() mismatch in step {}", + step + ); + assert_eq!( + model_cursor.val_valid(), + totest_cursor.val_valid(), + "val_valid() mismatch in step {}", + step + ); + } + + //assert_eq!(ptrace.len(), model.len()); + for (i, action) in ops.iter().enumerate() { + match action { + CursorAction::StepKey => { + model_cursor.step_key(); + totest_cursor.step_key(); + check_eq_invariants(i, &model_cursor, &totest_cursor); + } + CursorAction::StepVal => { + model_cursor.step_val(); + totest_cursor.step_val(); + check_eq_invariants(i, &model_cursor, &totest_cursor); + } + CursorAction::SeekKey(k) => { + model_cursor.seek_key(&k); + totest_cursor.seek_key(&k); + check_eq_invariants(i, &model_cursor, &totest_cursor); + } + CursorAction::SeekVal(v) => { + model_cursor.seek_val(&v); + totest_cursor.seek_val(&v); + check_eq_invariants(i, &model_cursor, &totest_cursor); + } + CursorAction::SeekValWith(v) => { + model_cursor.seek_val_with(|cmp_with| cmp_with >= v); + totest_cursor.seek_val_with(|cmp_with| cmp_with >= v); + check_eq_invariants(i, &model_cursor, &totest_cursor); + } + CursorAction::RewindKeys => { + model_cursor.rewind_keys(); + totest_cursor.rewind_keys(); + check_eq_invariants(i, &model_cursor, &totest_cursor); + } + CursorAction::RewindVals => { + model_cursor.rewind_vals(); + totest_cursor.rewind_vals(); + check_eq_invariants(i, &model_cursor, &totest_cursor); + } + CursorAction::Key => { + if model_cursor.key_valid() { + assert_eq!(model_cursor.key(), totest_cursor.key()); + } + check_eq_invariants(i, &model_cursor, &totest_cursor); + } + CursorAction::LastKey => { + assert_eq!(model_cursor.last_key(), totest_cursor.last_key()); + check_eq_invariants(i, &model_cursor, &totest_cursor); + } + CursorAction::Val => { + if model_cursor.val_valid() { + assert_eq!(model_cursor.val(), totest_cursor.val()); + } + check_eq_invariants(i, &model_cursor, &totest_cursor); + } + CursorAction::MapTimes => { + let mut model_invocations = Vec::new(); + let mut test_invocation = Vec::new(); + model_cursor.map_times(|v, t| { + model_invocations.push((v.clone(), t.clone())); + }); + totest_cursor.map_times(|v, t| { + test_invocation.push((v.clone(), t.clone())); + }); + assert_eq!(model_invocations, test_invocation); + } + CursorAction::MapTimesThrough(upper) => { + let mut model_invocations = Vec::new(); + let mut test_invocation = Vec::new(); + model_cursor.map_times_through( + |v, t| { + model_invocations.push((v.clone(), t.clone())); + }, + &upper, + ); + totest_cursor.map_times_through( + |v, t| { + test_invocation.push((v.clone(), t.clone())); + }, + &upper, + ); + assert_eq!(model_invocations, test_invocation); + } + } + } +} + +/// Generates a `(K, V)` tuple for insertion into batches that store values. +fn kv_tuple() -> impl Strategy { + (any::(), any::()) +} + +/// Generates a vector of `((K, V), R)` tuples for insertion into batches that +/// store values and weights. +fn kvr_tuples( + rs: impl Strategy, +) -> impl Strategy> { + prop::collection::vec(((any::(), any::()), rs), ENTRIES_RANGE) +} + +/// Generates a vector of `(K, R)` tuples for insertion into batches that don't +/// store values. +fn kr_tuples( + rs: impl Strategy, +) -> impl Strategy> { + prop::collection::vec((any::(), rs), ENTRIES_RANGE) +} + +/// Mutable calls we can perform on something that implements the [`Trace`] +/// trait. +#[derive(Clone, Debug)] +enum TraceAction +where + I: Ord + Clone + 'static, + R: Ord + Clone + MonoidValue + Encode + Decode, + T: Clone + 'static, +{ + ClearDirtyFlag, + RecedeTo(T), + Insert((T, Vec<(I, R)>)), + // Exert -- ignored because it's a nop in the PersistentTrace + // Consolidate -- consumes self so we should just do it once at the end of the test +} + +/// Generate one action to perform on a [`Trace`]. +fn trace_action( + is: impl Strategy, + rs: impl Strategy, +) -> impl Strategy> +where + T: Arbitrary + Debug + Clone + 'static, + I: Debug + Ord + Clone + 'static, + R: Debug + Ord + Clone + MonoidValue + Encode + Decode, +{ + prop_oneof![ + Just(TraceAction::ClearDirtyFlag), + any::().prop_map(TraceAction::RecedeTo), + (any::(), prop::collection::vec((is, rs), ENTRIES_RANGE)).prop_map(TraceAction::Insert), + ] +} + +/// Generate a sequence of actions to perform on a [`Trace`]. +fn trace_actions( + is: impl Strategy, + rs: impl Strategy, +) -> impl Strategy>> +where + T: Arbitrary + Debug + Clone + 'static, + I: Arbitrary + Debug + Ord + Clone + 'static, + R: Arbitrary + Debug + Ord + Clone + MonoidValue + Encode + Decode, +{ + prop::collection::vec(trace_action::(is, rs), TRACE_ACTIONS_RANGE) +} + +/// Performs a random series of actions on a [`Spine`] and a +/// [`PersistentTrace`], and makes sure they produce the same results. +fn trace_trait(actions: Vec>) +where + I: Ord + Clone + 'static, + B: Batch + 'static, + B::Key: Arbitrary + Ord + Clone + Encode + Decode, + B::Val: Arbitrary + Ord + Clone + Encode + Decode, + B::R: Arbitrary + Ord + Clone + MonoidValue + Encode + Decode, + B::Time: Arbitrary + Clone + Encode + Decode + Default, +{ + let mut model = Spine::::new(None); + let mut ptrace = PersistentTrace::::new(None); + + // We check the non-mutating functions after every + // command/mutation. + fn check_eq_invariants( + step: usize, + model: &Spine, + totest: &PersistentTrace, + ) where + B::Key: Debug + Ord + Clone + Encode + Decode, + B::Val: Debug + Ord + Clone + Encode + Decode, + B::Time: Encode + Decode, + B::R: Debug + MonoidValue + Encode + Decode, + { + assert_eq!( + model.dirty(), + totest.dirty(), + "dirty() mismatch in step {}", + step + ); + // Not comparable, no accurate counts with rocksdb: + //assert_eq!( + // model.key_count(), + // totest.key_count(), + // "key_count() mismatch in step {}", + // step + //); + //assert_eq!(model.len(), totest.len(), "len() mismatch in step {}", step); + //assert_eq!( + //model.is_empty(), + //totest.is_empty(), + //"is_empty() mismatch in step {}", + //step + //); + assert_eq!( + model.lower(), + totest.lower(), + "lower() mismatch in step {}", + step + ); + assert_eq!( + model.upper(), + totest.upper(), + "upper() mismatch in step {}", + step + ); + } + + for (step, action) in actions.into_iter().enumerate() { + match action { + TraceAction::RecedeTo(t) => { + // TODO: implement recede_to + model.recede_to(&t); + ptrace.recede_to(&t); + check_eq_invariants(step, &model, &ptrace); + assert!(spine_ptrace_are_equal(&model, &ptrace)); + } + TraceAction::ClearDirtyFlag => { + ptrace.clear_dirty_flag(); + model.clear_dirty_flag(); + check_eq_invariants(step, &model, &ptrace); + assert!(spine_ptrace_are_equal(&model, &ptrace)); + } + TraceAction::Insert((time, mut batch_tuples)) => { + // Builder interface wants sorted, unique(?) keys: + batch_tuples.sort_unstable(); + batch_tuples.dedup_by(|a, b| a.0.eq(&b.0)); + + // Instantiate a Batch + let mut batch_builder = B::Builder::new_builder(time); + for tuple in batch_tuples.into_iter() { + if !tuple.1.is_zero() { + batch_builder.push(tuple); + } + } + let batch = batch_builder.done(); + + model.insert(batch.clone()); + let mut fuel = isize::MAX; + model.exert(&mut fuel); + + ptrace.insert(batch); + + check_eq_invariants(step, &model, &ptrace); + assert!(spine_ptrace_are_equal(&model, &ptrace)); + } + } + } +} + +proptest! { + // Verify that our [`Cursor`] implementation for the persistent trace + // behaves the same as spine (non-persistent traces): + + #[test] + fn cursor_val_batch_ptrace_behaves_like_spine(ks in kvr_tuples::(WEIGHT_RANGE), ops in actions::()) { + cursor_trait::, (ComplexKey, String)>(ks, ops); + } + + #[test] + fn cursor_key_batch_ptrace_behaves_like_spine(ks in kr_tuples::(WEIGHT_RANGE), ops in actions::()) { + cursor_trait::, ComplexKey>(ks, ops); + } + + #[test] + fn cursor_zset_ptrace_behaves_like_spine(ks in kr_tuples::(WEIGHT_RANGE), ops in actions::()) { + cursor_trait::, ComplexKey>(ks, ops); + } + + #[test] + fn cursor_indexed_zset_ptrace_behaves_like_spine(ks in kvr_tuples::(WEIGHT_RANGE), ops in actions::()) { + cursor_trait::, (ComplexKey, usize)>(ks, ops); + } + + // Verify that our [`Trace`] implementation behaves the same as the spine + // trace implementation: + + #[test] + fn val_batch_trace_behaves_like_spine(actions in trace_actions::(kv_tuple::(), WEIGHT_RANGE)) { + trace_trait::, (ComplexKey, String)>(actions); + } + + #[test] + fn key_batch_trace_behaves_like_spine(actions in trace_actions::(any::(), WEIGHT_RANGE)) { + trace_trait::, ComplexKey>(actions); + } + + #[test] + fn zset_trace_behaves_like_spine(actions in trace_actions::<(), ComplexKey, isize>(any::(), WEIGHT_RANGE)) { + trace_trait::, ComplexKey>(actions); + } + + #[test] + fn indexed_zset_trace_behaves_like_spine(actions in trace_actions::<(), (ComplexKey, usize), isize>(kv_tuple::(), WEIGHT_RANGE)) { + trace_trait::, (ComplexKey, usize)>(actions); + } +} diff --git a/src/trace/persistent/trace.rs b/src/trace/persistent/trace.rs new file mode 100644 index 00000000..a9adaac8 --- /dev/null +++ b/src/trace/persistent/trace.rs @@ -0,0 +1,686 @@ +//! The implementation of the persistent trace. +use std::collections::HashMap; +use std::fmt::{Debug, Formatter}; +use std::marker::PhantomData; +use std::sync::Arc; + +use bincode::{decode_from_slice, Decode, Encode}; +use rocksdb::compaction_filter::Decision; +use rocksdb::{BoundColumnFamily, MergeOperands, Options, WriteBatch}; +use size_of::SizeOf; +use uuid::Uuid; + +use super::{rocksdb_key_comparator, PersistentTraceCursor, ReusableEncodeBuffer, Values}; +use super::{BINCODE_CONFIG, ROCKS_DB_INSTANCE}; +use crate::algebra::AddAssignByRef; +use crate::circuit::Activator; +use crate::time::{Antichain, Timestamp}; +use crate::trace::cursor::Cursor; +use crate::trace::{ + AntichainRef, Batch, BatchReader, Builder, Consumer, DBData, DBTimestamp, DBWeight, HasZero, + Trace, ValueConsumer, +}; +use crate::NumEntries; + +/// A persistent trace implementation. +/// +/// - It mimics the (external) behavior of a `Spine`, but internally it uses a +/// RocksDB ColumnFamily to store it's data. +/// +/// - It also relies on merging and compaction of the RocksDB key-value store +/// rather than controlling these aspects itself. +#[derive(SizeOf)] +pub struct PersistentTrace +where + B: Batch, +{ + lower: Antichain, + upper: Antichain, + dirty: bool, + approximate_len: usize, + + /// Where all the dataz is. + #[size_of(skip)] + cf: Arc>, + cf_name: String, + #[size_of(skip)] + _cf_options: Options, + + _phantom: std::marker::PhantomData, +} + +impl Default for PersistentTrace +where + B: Batch, +{ + fn default() -> Self { + PersistentTrace::new(None) + } +} + +impl Drop for PersistentTrace +where + B: Batch, +{ + /// Deletes the RocksDB column family. + fn drop(&mut self) { + ROCKS_DB_INSTANCE + .drop_cf(&self.cf_name) + .expect("Can't delete CF?"); + } +} + +impl Debug for PersistentTrace +where + B: Batch, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> { + let mut cursor: PersistentTraceCursor = self.cursor(); + writeln!(f, "PersistentTrace:")?; + writeln!(f, " rocksdb column {}:", self.cf_name)?; + while cursor.key_valid() { + writeln!(f, "{:?}:", cursor.key())?; + while cursor.val_valid() { + writeln!(f, "{:?}:", cursor.val())?; + cursor.map_times(|t, w| { + writeln!( + f, + "{}", + textwrap::indent(format!("{t:?} -> {w:?}").as_str(), " ") + ) + .expect("can't write out"); + }); + + cursor.step_val(); + } + cursor.step_key(); + } + writeln!(f)?; + Ok(()) + } +} + +impl Clone for PersistentTrace +where + B: Batch, +{ + fn clone(&self) -> Self { + unimplemented!("PersistentTrace::clone") + } +} + +impl NumEntries for PersistentTrace +where + B: Batch, +{ + const CONST_NUM_ENTRIES: Option = None; + + fn num_entries_shallow(&self) -> usize { + self.key_count() + } + + fn num_entries_deep(&self) -> usize { + // Same as Spine implementation: + self.num_entries_shallow() + } +} + +pub struct PersistentConsumer +where + B: Batch, +{ + __type: PhantomData, +} + +impl Consumer for PersistentConsumer +where + B: Batch, +{ + type ValueConsumer<'a> = PersistentTraceValueConsumer<'a, B> + where + Self: 'a; + + fn key_valid(&self) -> bool { + todo!() + } + + fn peek_key(&self) -> &B::Key { + todo!() + } + + fn next_key(&mut self) -> (B::Key, Self::ValueConsumer<'_>) { + todo!() + } + + fn seek_key(&mut self, _key: &B::Key) + where + B::Key: Ord, + { + todo!() + } +} + +pub struct PersistentTraceValueConsumer<'a, B> { + __type: PhantomData<&'a B>, +} + +impl<'a, B> ValueConsumer<'a, B::Val, B::R, B::Time> for PersistentTraceValueConsumer<'a, B> +where + B: Batch, +{ + fn value_valid(&self) -> bool { + todo!() + } + + fn next_value(&mut self) -> (B::Val, B::R, B::Time) { + todo!() + } + + fn remaining_values(&self) -> usize { + todo!() + } +} + +impl BatchReader for PersistentTrace +where + B: Batch, +{ + type Key = B::Key; + type Val = B::Val; + type Time = B::Time; + type R = B::R; + + type Cursor<'s> = PersistentTraceCursor<'s, B>; + type Consumer = PersistentConsumer; + + fn consumer(self) -> Self::Consumer { + PersistentConsumer { + __type: std::marker::PhantomData, + } + } + + /// The number of keys in the batch. + /// + /// This is an estimate as there is no way to get an exact count from + /// RocksDB. + fn key_count(&self) -> usize { + ROCKS_DB_INSTANCE + .property_int_value_cf(&self.cf, rocksdb::properties::ESTIMATE_NUM_KEYS) + .expect("Can't get key count estimate") + .map_or_else(|| 0, |c| c as usize) + } + + /// The number of updates in the batch. + /// + /// This is an estimate, not an accurate count. + fn len(&self) -> usize { + self.approximate_len + } + + fn lower(&self) -> AntichainRef { + self.lower.as_ref() + } + + fn upper(&self) -> AntichainRef { + self.upper.as_ref() + } + + fn cursor(&self) -> Self::Cursor<'_> { + PersistentTraceCursor::new(&self.cf) + } +} + +/// The data-type that is persisted as the value in RocksDB. +#[derive(Debug)] +pub(super) enum PersistedValue +where + V: DBData, + T: DBTimestamp, + R: DBWeight, +{ + /// Values with key-weight pairs. + Values(Values), + /// A tombstone for a key which had its values deleted (during merges). + /// + /// It signifies that the key shouldn't exist anymore. See also + /// [`tombstone_compaction`] which gets rid of Tombstones during compaction. + Tombstone, +} + +/// Decode for [`Self`] is currently implemented manually thanks to a bug in +/// bincode (enum+generics seems to break things). +/// +/// See also: https://github.com/bincode-org/bincode/issues/537 +impl Decode for PersistedValue +where + V: DBData, + T: DBTimestamp, + R: DBWeight, +{ + fn decode( + decoder: &mut D, + ) -> core::result::Result { + let typ: u8 = bincode::Decode::decode(decoder)?; + + match typ { + 0 => Ok(Self::Tombstone), + 1 => Ok(Self::Values(bincode::Decode::decode(decoder)?)), + _ => panic!("Unknown PersistedValue type"), + } + } +} + +/// Encode for [`Self`] is currently implemented manually thanks to a bug in +/// bincode (enum+generics seems to break things). +/// +/// See also: https://github.com/bincode-org/bincode/issues/537 +impl Encode for PersistedValue +where + V: DBData, + T: DBTimestamp, + R: DBWeight, +{ + fn encode( + &self, + encoder: &mut E, + ) -> core::result::Result<(), bincode::error::EncodeError> { + match self { + Self::Tombstone => { + bincode::Encode::encode(&0u8, encoder)?; + } + Self::Values(v) => { + bincode::Encode::encode(&1u8, encoder)?; + bincode::Encode::encode(v, encoder)?; + } + } + Ok(()) + } +} + +/// A merge-op is what [`PersistentTrace`] supplies to the RocksDB instance to +/// indicate how to update the values. +#[derive(Clone, Debug)] +enum MergeOp +where + V: DBData, + T: DBTimestamp, + R: DBWeight, +{ + /// A recede-to command to reset times of values. + RecedeTo(T), + /// An insertion of a new value or update of an existing value. + Insert(Values), +} + +/// Decode for [`Self`] is currently implemented manually thanks to a bug in +/// bincode (enum+generics seems to break things). +/// +/// See also: https://github.com/bincode-org/bincode/issues/537 +impl Decode for MergeOp +where + V: DBData, + T: DBTimestamp, + R: DBWeight, +{ + fn decode( + decoder: &mut D, + ) -> core::result::Result { + let typ: u8 = bincode::Decode::decode(decoder)?; + + match typ { + 0 => Ok(Self::RecedeTo(bincode::Decode::decode(decoder)?)), + 1 => Ok(Self::Insert(bincode::Decode::decode(decoder)?)), + _ => panic!("Unknown MergeOp type"), + } + } +} + +/// Encode for [`Self`] is currently implemented manually thanks to a bug in +/// bincode (enum+generics seems to break things). +/// +/// See also: https://github.com/bincode-org/bincode/issues/537 +impl Encode for MergeOp +where + V: DBData, + T: DBTimestamp, + R: DBWeight, +{ + fn encode( + &self, + encoder: &mut E, + ) -> core::result::Result<(), bincode::error::EncodeError> { + match self { + Self::RecedeTo(t) => { + bincode::Encode::encode(&0u8, encoder)?; + bincode::Encode::encode(t, encoder)?; + } + Self::Insert(v) => { + bincode::Encode::encode(&1u8, encoder)?; + bincode::Encode::encode(v, encoder)?; + } + } + Ok(()) + } +} + +/// The implementation of the merge operator for the RocksDB instance. +/// +/// This essentially re-implements the core-logic of [`PersistentTrace`] in how +/// values, times and weights are updated. +/// +/// # TODO +/// Probably lots of efficiency improvements to be had here: We're sorting +/// several times when we probably can be smarter etc. -- not clear it matters +/// without benchmarking though. +fn rocksdb_concat_merge( + new_key: &[u8], + existing_val: Option<&[u8]>, + operands: &MergeOperands, +) -> Option> +where + K: DBData, + V: DBData, + R: DBWeight, + T: DBTimestamp, +{ + let (_key, _) = + decode_from_slice::(new_key, BINCODE_CONFIG).expect("Can't decode_from_slice"); + + let mut vals: Values = if let Some(val) = existing_val { + let (decoded_val, _): (PersistedValue, usize) = + decode_from_slice(val, BINCODE_CONFIG).expect("Can't decode current value"); + match decoded_val { + PersistedValue::Values(vals) => vals, + PersistedValue::Tombstone => Vec::new(), + } + } else { + Vec::new() + }; + + for op in operands { + let (decoded_update, _): (MergeOp, usize) = + decode_from_slice(op, BINCODE_CONFIG).expect("Can't decode current value"); + + match decoded_update { + MergeOp::Insert(new_vals) => { + for (v, tws) in new_vals { + let mut found_v = false; + let mut found_t = false; + let mut delete_zero_w = false; + for (existing_v, ref mut existing_tw) in vals.iter_mut() { + if existing_v == &v { + for (t, w) in &tws { + for (existing_t, ref mut existing_w) in existing_tw.iter_mut() { + if existing_t == t { + existing_w.add_assign_by_ref(w); + found_t = true; + if existing_w.is_zero() { + delete_zero_w = true; + } + } + } + if !found_t { + existing_tw.push((t.clone(), w.clone())); + // TODO: May be better if push (above) inserts at the + // right place instead of paying the cost of sorting + // everything: + existing_tw.sort_unstable_by(|(t1, _), (t2, _)| t1.cmp(t2)); + break; + } else if delete_zero_w { + existing_tw.retain(|(_, w)| !w.is_zero()); + } + } + found_v = true; + break; + } + } + + if !found_v { + //tws.sort_unstable_by(|(t1, _), (t2, _)| t1.cmp(t2)); + vals.push((v, tws)); + } else { + // Delete values which ended up with zero weights + vals.retain(|(_ret_v, ret_tws)| { + ret_tws + .iter() + .filter(|(_ret_t, ret_w)| !ret_w.is_zero()) + .count() + != 0 + }); + } + } + } + MergeOp::RecedeTo(frontier) => { + for (_existing_v, ref mut existing_tw) in vals.iter_mut() { + let mut modified_t = false; + for (ref mut existing_t, _existing_w) in existing_tw.iter_mut() { + // I think due to this being sorted by Ord we have to + // walk all of them (see also `map_batches_through`): + if !existing_t.less_equal(&frontier) { + // example: times [1,2,3,4], frontier 3 -> [1,2,3,3] + *existing_t = existing_t.meet(&frontier); + modified_t = true; + } + } + + if modified_t { + // I think due to this being sorted by `Ord` we can't + // rely on `recede_to(x)` having only affected + // consecutive elements, so we create a new (t, w) + // vector with a hashmap that we sort again. + let mut new_tw = HashMap::with_capacity(existing_tw.len()); + for (cur_t, cur_w) in &*existing_tw { + new_tw + .entry(cur_t.clone()) + .and_modify(|w: &mut R| w.add_assign_by_ref(cur_w)) + .or_insert_with(|| cur_w.clone()); + } + let new_tw_vec: Vec<(T, R)> = new_tw.into_iter().collect(); + *existing_tw = new_tw_vec; + } + existing_tw.sort_unstable_by(|(t1, _), (t2, _)| t1.cmp(t2)); + } + + // Delete values which ended up with zero weights + vals.retain(|(_ret_v, ret_tws)| { + ret_tws + .iter() + .filter(|(_ret_t, ret_w)| !ret_w.is_zero()) + .count() + != 0 + }); + } + } + } + + // TODO: We can probably avoid re-sorting in some cases (see if found_v above)? + vals.sort_unstable_by(|v1, v2| v1.0.cmp(&v2.0)); + + let vals = if !vals.is_empty() { + PersistedValue::Values(vals) + } else { + PersistedValue::Tombstone + }; + + let mut buf = ReusableEncodeBuffer::with_capacity(existing_val.map(|v| v.len()).unwrap_or(0)); + buf.encode(&vals).expect("Can't encode `vals`"); + Some(buf.into()) +} + +/// Throw away keys that no longer have values and should've been deleted (but +/// can't because RocksDB doesn't support deletions during merge operations). +fn tombstone_compaction(_level: u32, _key: &[u8], val: &[u8]) -> Decision +where + V: DBData, + R: DBWeight, + T: DBTimestamp, +{ + // TODO: Ideally we shouldn't have to pay the price of decoding the whole + // Vec<(V, Vec<(T, R)>)> as we only care about what the enum variant is. + let (decoded_val, _): (PersistedValue, usize) = + decode_from_slice(val, BINCODE_CONFIG).expect("Can't decode current value"); + match decoded_val { + PersistedValue::Values(_vals) => Decision::Keep, + PersistedValue::Tombstone => Decision::Remove, + } +} + +impl Trace for PersistentTrace +where + B: Batch + Clone + 'static, + B::Time: DBTimestamp, +{ + type Batch = B; + + /// Create a new PersistentTrace. + /// + /// It works by creating a new column-family with a random name and + /// configuring it with the right custom functions for comparison, merge, + /// and compaction. + /// + /// # Arguments + /// - `activator`: This is not used, None should be supplied. + fn new(_activator: Option) -> Self { + // Create a new column family for the Trace + let cf_name = Uuid::new_v4().to_string(); + let mut cf_options = Options::default(); + cf_options.set_comparator("Rust type compare", rocksdb_key_comparator::); + cf_options.set_merge_operator_associative( + "Trace value merge function", + rocksdb_concat_merge::, + ); + cf_options.set_compaction_filter( + "Remove empty vals", + tombstone_compaction::, + ); + cf_options.create_if_missing(true); + + ROCKS_DB_INSTANCE + .create_cf(cf_name.as_str(), &cf_options) + .expect("Can't create column family?"); + let cf = ROCKS_DB_INSTANCE + .cf_handle(cf_name.as_str()) + .expect("Can't find just created column family?"); + + Self { + lower: Antichain::from_elem(B::Time::minimum()), + upper: Antichain::new(), + approximate_len: 0, + dirty: false, + cf, + cf_name, + _cf_options: cf_options, + _phantom: std::marker::PhantomData, + } + } + + /// Recede to works by sending a `RecedeTo` command to every key in the + /// trace. + fn recede_to(&mut self, frontier: &B::Time) { + let mut tmp_key = ReusableEncodeBuffer::default(); + let mut tmp_val = ReusableEncodeBuffer::default(); + + let mut cursor = self.cursor(); + while cursor.key_valid() { + let key = cursor.key(); + let encoded_key = tmp_key.encode(&key).expect("Can't encode `key`"); + + let update: MergeOp = MergeOp::RecedeTo(frontier.clone()); + let encoded_update = tmp_val.encode(&update).expect("Can't encode `vals`"); + + ROCKS_DB_INSTANCE + .merge_cf(&self.cf, encoded_key, encoded_update) + .expect("Can't merge recede update"); + cursor.step_key(); + } + } + + fn exert(&mut self, _effort: &mut isize) { + // This is a no-op for the persistent trace as RocksDB will decide when + // to apply the merge / compaction operators etc. + } + + fn consolidate(self) -> Option { + // TODO: Not clear what the time of the batch should be here -- in Spine + // the batch will not be `minimum` as it's created through merges of all + // batches. + // + // In discussion with Leonid: We probably want to move consolidate out + // of the trace trait. + let mut builder = ::Builder::new_builder(Self::Time::minimum()); + + let mut cursor = self.cursor(); + while cursor.key_valid() { + while cursor.val_valid() { + let v = cursor.val().clone(); + let mut w = B::R::zero(); + cursor.map_times(|_t, cur_w| { + w.add_assign_by_ref(cur_w); + }); + let k = cursor.key().clone(); + + builder.push((Self::Batch::item_from(k, v), w)); + cursor.step_val(); + } + + cursor.step_key(); + } + + Some(builder.done()) + } + + fn insert(&mut self, batch: Self::Batch) { + assert!(batch.lower() != batch.upper()); + + // Ignore empty batches. + // Note: we may want to use empty batches to artificially force compaction. + if batch.is_empty() { + return; + } + + self.dirty = true; + self.lower = self.lower.as_ref().meet(batch.lower()); + self.upper = self.upper.as_ref().join(batch.upper()); + + self.add_batch_to_cf(batch); + } + + fn clear_dirty_flag(&mut self) { + self.dirty = false; + } + + fn dirty(&self) -> bool { + self.dirty + } +} + +impl PersistentTrace +where + B: Batch, +{ + fn add_batch_to_cf(&mut self, batch: B) { + use crate::trace::cursor::CursorDebug; + + let mut tmp_key = ReusableEncodeBuffer::default(); + let mut tmp_val = ReusableEncodeBuffer::default(); + + let mut sstable = WriteBatch::default(); + let mut batch_cursor = batch.cursor(); + while batch_cursor.key_valid() { + let key = batch_cursor.key(); + let encoded_key = tmp_key.encode(&key).expect("Can't encode `key`"); + let vals: Values = batch_cursor.val_to_vec(); + self.approximate_len += vals.len(); + let encoded_vals = tmp_val + .encode(&MergeOp::Insert(vals)) + .expect("Can't encode `vals`"); + sstable.merge_cf(&self.cf, encoded_key, encoded_vals); + + batch_cursor.step_key(); + } + + ROCKS_DB_INSTANCE + .write(sstable) + .expect("Could not write batch to db"); + } +}