From dc668e34d3a10a8bc6fa1db306151c2ee4635285 Mon Sep 17 00:00:00 2001 From: tompro Date: Mon, 14 Apr 2025 19:12:37 +0200 Subject: [PATCH 01/12] Init crate --- Cargo.lock | 406 ++++++++++++++++-- crates/nostr-postgresdb/Cargo.toml | 21 + crates/nostr-postgresdb/README.md | 19 + crates/nostr-postgresdb/diesel.toml | 10 + crates/nostr-postgresdb/migrations/.keep | 0 .../down.sql | 6 + .../up.sql | 36 ++ .../2025-04-11-095120_events/down.sql | 4 + .../2025-04-11-095120_events/up.sql | 30 ++ crates/nostr-postgresdb/src/lib.rs | 131 ++++++ crates/nostr-postgresdb/src/migrations.rs | 19 + crates/nostr-postgresdb/src/model.rs | 73 ++++ crates/nostr-postgresdb/src/postgres.rs | 192 +++++++++ crates/nostr-postgresdb/src/schema.rs | 34 ++ 14 files changed, 935 insertions(+), 46 deletions(-) create mode 100644 crates/nostr-postgresdb/Cargo.toml create mode 100644 crates/nostr-postgresdb/README.md create mode 100644 crates/nostr-postgresdb/diesel.toml create mode 100644 crates/nostr-postgresdb/migrations/.keep create mode 100644 crates/nostr-postgresdb/migrations/00000000000000_diesel_initial_setup/down.sql create mode 100644 crates/nostr-postgresdb/migrations/00000000000000_diesel_initial_setup/up.sql create mode 100644 crates/nostr-postgresdb/migrations/2025-04-11-095120_events/down.sql create mode 100644 crates/nostr-postgresdb/migrations/2025-04-11-095120_events/up.sql create mode 100644 crates/nostr-postgresdb/src/lib.rs create mode 100644 crates/nostr-postgresdb/src/migrations.rs create mode 100644 crates/nostr-postgresdb/src/model.rs create mode 100644 crates/nostr-postgresdb/src/postgres.rs create mode 100644 crates/nostr-postgresdb/src/schema.rs diff --git a/Cargo.lock b/Cargo.lock index 98fbeca97..0ea659a87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,7 +60,7 @@ dependencies = [ "cfg-if", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -206,7 +206,7 @@ dependencies = [ "libc", "once_cell", "postage", - "rand", + "rand 0.8.5", "safelog", "serde", "thiserror 2.0.8", @@ -914,7 +914,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "subtle", "zeroize", ] @@ -926,7 +926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "typenum", ] @@ -1063,6 +1063,26 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +[[package]] +name = "deadpool" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ed5957ff93768adf7a65ab167a17835c3d2c3c50d084fe305174c112f468e2f" +dependencies = [ + "deadpool-runtime", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +dependencies = [ + "tokio", +] + [[package]] name = "delegate-display" version = "2.1.1" @@ -1254,6 +1274,68 @@ dependencies = [ "zeroize", ] +[[package]] +name = "diesel" +version = "2.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34d3950690ba3a6910126162b47e775e203006d4242a15de912bec6c0a695153" +dependencies = [ + "bitflags 2.9.0", + "byteorder", + "diesel_derives", + "itoa", + "pq-sys", + "serde_json", +] + +[[package]] +name = "diesel-async" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51a307ac00f7c23f526a04a77761a0519b9f0eb2838ebf5b905a58580095bdcb" +dependencies = [ + "async-trait", + "deadpool", + "diesel", + "futures-util", + "scoped-futures", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "diesel_derives" +version = "2.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a93958254b70bea63b4187ff73d10180599d9d8d177071b7f91e6da4e0c0ad55" +dependencies = [ + "diesel_table_macro_syntax", + "dsl_auto_type", + "proc-macro2", + "quote", + "syn 2.0.90", +] + +[[package]] +name = "diesel_migrations" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a73ce704bad4231f001bff3314d91dce4aba0770cee8b233991859abc15c1f6" +dependencies = [ + "diesel", + "migrations_internals", + "migrations_macros", +] + +[[package]] +name = "diesel_table_macro_syntax" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" +dependencies = [ + "syn 2.0.90", +] + [[package]] name = "digest" version = "0.10.7" @@ -1364,6 +1446,20 @@ dependencies = [ "phf", ] +[[package]] +name = "dsl_auto_type" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139ae9aca7527f85f26dd76483eb38533fd84bd571065da1739656ef71c5ff5b" +dependencies = [ + "darling 0.20.10", + "either", + "heck", + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "dyn-clone" version = "1.0.17" @@ -1403,7 +1499,7 @@ dependencies = [ "curve25519-dalek", "ed25519", "merlin", - "rand_core", + "rand_core 0.6.4", "serde", "sha2", "subtle", @@ -1441,7 +1537,7 @@ dependencies = [ "generic-array", "group", "pkcs8", - "rand_core", + "rand_core 0.6.4", "sec1", "subtle", "zeroize", @@ -1518,6 +1614,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -1565,7 +1667,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" dependencies = [ - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -1867,7 +1969,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -2425,7 +2527,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -2561,6 +2663,16 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.4" @@ -2584,10 +2696,31 @@ checksum = "58c38e2799fc0978b65dfff8023ec7843e2330bb462f19198840b34b6582397d" dependencies = [ "byteorder", "keccak", - "rand_core", + "rand_core 0.6.4", "zeroize", ] +[[package]] +name = "migrations_internals" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd01039851e82f8799046eabbb354056283fb265c8ec0996af940f4e85a380ff" +dependencies = [ + "serde", + "toml", +] + +[[package]] +name = "migrations_macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb161cc72176cb37aa47f1fc520d3ef02263d67d661f44f05d05a079e1237fd" +dependencies = [ + "migrations_internals", + "proc-macro2", + "quote", +] + [[package]] name = "mime" version = "0.3.17" @@ -2800,6 +2933,19 @@ dependencies = [ "ureq", ] +[[package]] +name = "nostr-postgresdb" +version = "0.40.0" +dependencies = [ + "deadpool", + "diesel", + "diesel-async", + "diesel_migrations", + "nostr", + "nostr-database", + "tracing", +] + [[package]] name = "nostr-relay-builder" version = "0.40.0" @@ -2925,7 +3071,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "smallvec", "zeroize", ] @@ -2966,6 +3112,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + [[package]] name = "num_enum" version = "0.7.3" @@ -3134,7 +3290,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "rand_core", + "rand_core 0.6.4", "sha2", ] @@ -3184,7 +3340,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ "base64ct", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -3236,7 +3392,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] @@ -3352,6 +3508,35 @@ dependencies = [ "thiserror 1.0.64", ] +[[package]] +name = "postgres-protocol" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "hmac", + "md-5", + "memchr", + "rand 0.9.0", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "postgres-protocol", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -3364,7 +3549,17 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", +] + +[[package]] +name = "pq-sys" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41c852911b98f5981956037b2ca976660612e548986c30af075e753107bc3400" +dependencies = [ + "libc", + "vcpkg", ] [[package]] @@ -3466,7 +3661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", - "rand", + "rand 0.8.5", "ring 0.17.14", "rustc-hash 2.0.0", "rustls", @@ -3511,8 +3706,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "zerocopy 0.8.24", ] [[package]] @@ -3522,7 +3728,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -3534,6 +3750,15 @@ dependencies = [ "getrandom 0.2.15", ] +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.1", +] + [[package]] name = "rangemap" version = "1.5.1" @@ -3717,7 +3942,7 @@ dependencies = [ "num-traits", "pkcs1", "pkcs8", - "rand_core", + "rand_core 0.6.4", "sha2", "signature", "spki", @@ -3732,7 +3957,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ "bitflags 2.9.0", - "fallible-iterator", + "fallible-iterator 0.3.0", "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", @@ -3903,6 +4128,15 @@ dependencies = [ "regex", ] +[[package]] +name = "scoped-futures" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b24aae2d0636530f359e9d5ef0c04669d11c5e756699b27a6a6d845d8329091" +dependencies = [ + "pin-project-lite", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -3941,7 +4175,7 @@ version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113" dependencies = [ - "rand", + "rand 0.8.5", "secp256k1-sys", "serde", ] @@ -4188,7 +4422,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -4297,7 +4531,7 @@ dependencies = [ "p256", "p384", "p521", - "rand_core", + "rand_core 0.6.4", "rsa", "sec1", "sha2", @@ -4314,6 +4548,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.10.0" @@ -4579,6 +4824,32 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "tokio-postgres" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c95d533c83082bb6490e0189acaa0bbeef9084e60471b696ca6988cd0541fb0" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.9.0", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + [[package]] name = "tokio-rustls" version = "0.26.0" @@ -4693,8 +4964,8 @@ dependencies = [ "itertools 0.14.0", "libc", "paste", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "serde", "slab", "smallvec", @@ -4733,7 +5004,7 @@ dependencies = [ "derive_more 2.0.1", "educe", "paste", - "rand", + "rand 0.8.5", "smallvec", "thiserror 2.0.8", "tor-basic-utils", @@ -4778,7 +5049,7 @@ dependencies = [ "futures", "oneshot-fused-workaround", "postage", - "rand", + "rand 0.8.5", "safelog", "serde", "thiserror 2.0.8", @@ -4832,7 +5103,7 @@ dependencies = [ "once_cell", "oneshot-fused-workaround", "pin-project", - "rand", + "rand 0.8.5", "retry-error", "safelog", "serde", @@ -4973,7 +5244,7 @@ dependencies = [ "oneshot-fused-workaround", "paste", "postage", - "rand", + "rand 0.8.5", "rusqlite", "safelog", "scopeguard", @@ -5050,7 +5321,7 @@ dependencies = [ "oneshot-fused-workaround", "pin-project", "postage", - "rand", + "rand 0.8.5", "safelog", "serde", "strum", @@ -5086,7 +5357,7 @@ dependencies = [ "itertools 0.14.0", "oneshot-fused-workaround", "postage", - "rand", + "rand 0.8.5", "retry-error", "safelog", "slotmap-careful", @@ -5127,7 +5398,7 @@ dependencies = [ "digest", "itertools 0.14.0", "paste", - "rand", + "rand 0.8.5", "safelog", "signature", "subtle", @@ -5195,8 +5466,8 @@ dependencies = [ "once_cell", "oneshot-fused-workaround", "postage", - "rand", - "rand_core", + "rand 0.8.5", + "rand_core 0.6.4", "retry-error", "safelog", "serde", @@ -5238,7 +5509,7 @@ dependencies = [ "derive_more 2.0.1", "downcast-rs", "paste", - "rand", + "rand 0.8.5", "signature", "ssh-key", "thiserror 2.0.8", @@ -5268,7 +5539,7 @@ dependencies = [ "humantime", "inventory", "itertools 0.14.0", - "rand", + "rand 0.8.5", "serde", "signature", "ssh-key", @@ -5332,7 +5603,7 @@ dependencies = [ "educe", "getrandom 0.2.15", "hex", - "rand_core", + "rand_core 0.6.4", "rsa", "safelog", "serde", @@ -5407,7 +5678,7 @@ dependencies = [ "humantime", "itertools 0.14.0", "num_enum", - "rand", + "rand 0.8.5", "serde", "static_assertions", "strum", @@ -5444,7 +5715,7 @@ dependencies = [ "itertools 0.14.0", "once_cell", "phf", - "rand", + "rand 0.8.5", "serde", "serde_with", "signature", @@ -5522,8 +5793,8 @@ dependencies = [ "hmac", "oneshot-fused-workaround", "pin-project", - "rand", - "rand_core", + "rand 0.8.5", + "rand_core 0.6.4", "safelog", "slotmap-careful", "static_assertions", @@ -5570,7 +5841,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e41754428684bd62892df2c74c2d11128cfbf3f1a8a9aaa1b920fcb90e04961a" dependencies = [ - "rand", + "rand 0.8.5", "serde", "tor-basic-utils", "tor-linkspec", @@ -5770,7 +6041,7 @@ dependencies = [ "http", "httparse", "log", - "rand", + "rand 0.8.5", "rustls", "rustls-pki-types", "sha1", @@ -5829,6 +6100,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -6006,6 +6283,12 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasix" version = "0.12.21" @@ -6149,6 +6432,17 @@ dependencies = [ "rustix", ] +[[package]] +name = "whoami" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" +dependencies = [ + "redox_syscall", + "wasite", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" @@ -6467,7 +6761,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ "curve25519-dalek", - "rand_core", + "rand_core 0.6.4", "serde", "zeroize", ] @@ -6495,7 +6789,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive 0.8.24", ] [[package]] @@ -6509,6 +6812,17 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "zeroize" version = "1.8.1" diff --git a/crates/nostr-postgresdb/Cargo.toml b/crates/nostr-postgresdb/Cargo.toml new file mode 100644 index 000000000..810ac0323 --- /dev/null +++ b/crates/nostr-postgresdb/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "nostr-postgresdb" +version = "0.40.0" +edition = "2021" +description = "Postgres storage backend for Nostr apps" +authors.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +readme = "README.md" +rust-version.workspace = true +keywords = ["nostr", "database", "postgres"] + +[dependencies] +nostr = { workspace = true, features = ["std"] } +nostr-database = { workspace = true, features = ["flatbuf"] } +tracing.workspace = true +diesel = { version = "2", features = ["postgres", "serde_json"] } +diesel-async = { version = "0.5", features = ["postgres", "deadpool"] } +diesel_migrations = { version = "2", features = ["postgres"] } +deadpool = { version = "0.12", features = ["managed", "rt_tokio_1"] } diff --git a/crates/nostr-postgresdb/README.md b/crates/nostr-postgresdb/README.md new file mode 100644 index 000000000..8ba6786ed --- /dev/null +++ b/crates/nostr-postgresdb/README.md @@ -0,0 +1,19 @@ +# Nostr Postgres database backend + +Postgres storage backend for nostr apps + +## State + +**This library is in an ALPHA state**, things that are implemented generally +work but the API will change in breaking ways. + +## Donations + +`rust-nostr` is free and open-source. This means we do not earn any revenue by +selling it. Instead, we rely on your financial support. If you actively use any +of the `rust-nostr` libs/software/services, then please [donate](https://rust-nostr.org/donate). + +## License + +This project is distributed under the MIT software license - see the +[LICENSE](../../LICENSE) file for details diff --git a/crates/nostr-postgresdb/diesel.toml b/crates/nostr-postgresdb/diesel.toml new file mode 100644 index 000000000..1c5febda4 --- /dev/null +++ b/crates/nostr-postgresdb/diesel.toml @@ -0,0 +1,10 @@ +# For documentation on how to configure this file, +# see https://diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "src/schema.rs" +custom_type_derives = ["diesel::query_builder::QueryId", "Clone"] +schema = "nostr" + +[migrations_directory] +dir = "migrations" diff --git a/crates/nostr-postgresdb/migrations/.keep b/crates/nostr-postgresdb/migrations/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/crates/nostr-postgresdb/migrations/00000000000000_diesel_initial_setup/down.sql b/crates/nostr-postgresdb/migrations/00000000000000_diesel_initial_setup/down.sql new file mode 100644 index 000000000..a9f526091 --- /dev/null +++ b/crates/nostr-postgresdb/migrations/00000000000000_diesel_initial_setup/down.sql @@ -0,0 +1,6 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + +DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); +DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/crates/nostr-postgresdb/migrations/00000000000000_diesel_initial_setup/up.sql b/crates/nostr-postgresdb/migrations/00000000000000_diesel_initial_setup/up.sql new file mode 100644 index 000000000..d68895b1a --- /dev/null +++ b/crates/nostr-postgresdb/migrations/00000000000000_diesel_initial_setup/up.sql @@ -0,0 +1,36 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + + + + +-- Sets up a trigger for the given table to automatically set a column called +-- `updated_at` whenever the row is modified (unless `updated_at` was included +-- in the modified columns) +-- +-- # Example +-- +-- ```sql +-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); +-- +-- SELECT diesel_manage_updated_at('users'); +-- ``` +CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ +BEGIN + EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s + FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ +BEGIN + IF ( + NEW IS DISTINCT FROM OLD AND + NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at + ) THEN + NEW.updated_at := current_timestamp; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/crates/nostr-postgresdb/migrations/2025-04-11-095120_events/down.sql b/crates/nostr-postgresdb/migrations/2025-04-11-095120_events/down.sql new file mode 100644 index 000000000..8e6a73889 --- /dev/null +++ b/crates/nostr-postgresdb/migrations/2025-04-11-095120_events/down.sql @@ -0,0 +1,4 @@ +-- This file should undo anything in `up.sql` +DROP TABLE nostr.event_tags; +DROP TABLE nostr.events; +DROP SCHEMA nostr; diff --git a/crates/nostr-postgresdb/migrations/2025-04-11-095120_events/up.sql b/crates/nostr-postgresdb/migrations/2025-04-11-095120_events/up.sql new file mode 100644 index 000000000..1fabcf294 --- /dev/null +++ b/crates/nostr-postgresdb/migrations/2025-04-11-095120_events/up.sql @@ -0,0 +1,30 @@ +-- Init the schema +CREATE SCHEMA IF NOT EXISTS nostr; + +-- The actual event data +CREATE TABLE nostr.events ( + id VARCHAR(64) PRIMARY KEY, + pubkey VARCHAR(64) NOT NULL, + created_at BIGINT NOT NULL, + kind BIGINT NOT NULL, + payload BYTEA NOT NULL, + signature VARCHAR(128) NOT NULL, + deleted BOOLEAN NOT NULL +); + +-- Direct indexes +CREATE INDEX event_pubkey ON nostr.events (pubkey); +CREATE INDEX event_date ON nostr.events (created_at); +CREATE INDEX event_kind ON nostr.events (kind); +CREATE INDEX event_deleted ON nostr.events (deleted); + +-- The tag index, the primary will give us the index automatically +CREATE TABLE nostr.event_tags ( + tag TEXT NOT NULL, + tag_value TEXT NOT NULL, + event_id VARCHAR(64) NOT NULL + REFERENCES nostr.events (id) + ON DELETE CASCADE + ON UPDATE CASCADE, + PRIMARY KEY (tag, tag_value, event_id) +); diff --git a/crates/nostr-postgresdb/src/lib.rs b/crates/nostr-postgresdb/src/lib.rs new file mode 100644 index 000000000..4a3b77a02 --- /dev/null +++ b/crates/nostr-postgresdb/src/lib.rs @@ -0,0 +1,131 @@ +mod migrations; +mod model; +mod postgres; +mod schema; + +use diesel::prelude::*; +use diesel_async::RunQueryDsl; +use model::{EventDataDb, EventDb}; +use nostr::{event::*, filter::Filter, types::Timestamp, util::BoxedFuture}; +use nostr_database::*; +use postgres::{build_filter_query, with_limit}; +use schema::nostr::events; + +pub use migrations::run_migrations; +pub use postgres::{postgres_connection_pool, NostrPostgres}; + +impl NostrDatabase for NostrPostgres { + fn backend(&self) -> Backend { + Backend::Custom("Postgres".to_string()) + } +} + +impl NostrEventsDatabase for NostrPostgres { + /// Save [`Event`] into store + /// + /// **This method assumes that [`Event`] was already verified** + fn save_event<'a>( + &'a self, + event: &'a Event, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { self.save(EventDataDb::try_from(event)?).await }) + } + + /// Check event status by ID + /// + /// Check if the event is saved, deleted or not existent. + fn check_id<'a>( + &'a self, + event_id: &'a EventId, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { + let status = match self.event_by_id(event_id).await? { + Some(e) if e.deleted => DatabaseEventStatus::Deleted, + Some(_) => DatabaseEventStatus::Saved, + None => DatabaseEventStatus::NotExistent, + }; + Ok(status) + }) + } + + /// Coordinate feature is not supported yet + fn has_coordinate_been_deleted<'a>( + &'a self, + _coordinate: &'a nostr::nips::nip01::CoordinateBorrow<'a>, + _timestamp: &'a Timestamp, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { Ok(false) }) + } + + /// Get [`Event`] by [`EventId`] + fn event_by_id<'a>( + &'a self, + _event_id: &'a EventId, + ) -> BoxedFuture<'a, Result, DatabaseError>> { + Box::pin(async move { + let event = match self.event_by_id(_event_id).await? { + Some(e) => Some(Event::decode(&e.payload).map_err(DatabaseError::backend)?), + None => None, + }; + Ok(event) + }) + } + + /// Count the number of events found with [`Filter`]. + /// + /// Use `Filter::new()` or `Filter::default()` to count all events. + fn count(&self, filter: Filter) -> BoxedFuture> { + Box::pin(async move { + let res: i64 = build_filter_query(filter) + .count() + .get_result(&mut self.get_connection().await?) + .await + .map_err(DatabaseError::backend)?; + Ok(res as usize) + }) + } + + /// Query stored events. + fn query(&self, filter: Filter) -> BoxedFuture> { + let filter = with_limit(filter, 10000); + Box::pin(async move { + let mut events = Events::new(&filter); + let result = build_filter_query(filter.clone()) + .select(EventDb::as_select()) + .load(&mut self.get_connection().await?) + .await + .map_err(DatabaseError::backend)?; + + for item in result.into_iter() { + if let Ok(event) = Event::decode(&item.payload) { + events.insert(event); + } + } + Ok(events) + }) + } + + /// Delete all events that match the [Filter] + fn delete(&self, filter: Filter) -> BoxedFuture> { + let filter = with_limit(filter, 999); + Box::pin(async move { + let filter = build_filter_query(filter); + diesel::update(events::table) + .set(events::deleted.eq(true)) + .filter(events::id.eq_any(filter.select(events::id))) + .execute(&mut self.get_connection().await?) + .await + .map_err(DatabaseError::backend)?; + + Ok(()) + }) + } +} + +/// For now we want to avoid wiping the database +impl NostrDatabaseWipe for NostrPostgres { + #[inline] + fn wipe(&self) -> BoxedFuture> { + Box::pin(async move { Err(DatabaseError::NotSupported) }) + } +} diff --git a/crates/nostr-postgresdb/src/migrations.rs b/crates/nostr-postgresdb/src/migrations.rs new file mode 100644 index 000000000..9d7e6f7f2 --- /dev/null +++ b/crates/nostr-postgresdb/src/migrations.rs @@ -0,0 +1,19 @@ +use diesel::{Connection, PgConnection}; +use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; +use nostr_database::DatabaseError; +use tracing::info; + +const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations"); + +/// programatically run the db migrations +pub fn run_migrations(connection_string: &str) -> Result<(), DatabaseError> { + info!("Running db migrations in postgres database",); + let mut connection = + PgConnection::establish(connection_string).map_err(DatabaseError::backend)?; + + let res = connection + .run_pending_migrations(MIGRATIONS) + .map_err(DatabaseError::Backend)?; + info!("Successfully executed postgres db migrations {:?}", res); + Ok(()) +} diff --git a/crates/nostr-postgresdb/src/model.rs b/crates/nostr-postgresdb/src/model.rs new file mode 100644 index 000000000..1f592c6a6 --- /dev/null +++ b/crates/nostr-postgresdb/src/model.rs @@ -0,0 +1,73 @@ +use crate::schema::nostr::event_tags; +use crate::schema::nostr::events; +use diesel::prelude::*; +use nostr::event::Event; +use nostr_database::DatabaseError; +use nostr_database::FlatBufferBuilder; +use nostr_database::FlatBufferEncode; + +/// DB representation of [`Event`] +#[derive(Queryable, Selectable, Insertable, AsChangeset, Debug, Clone)] +#[diesel(table_name = events, check_for_backend(diesel::pg::Pg))] +pub struct EventDb { + pub id: String, + pub pubkey: String, + pub created_at: i64, + pub kind: i64, + pub payload: Vec, + pub signature: String, + pub deleted: bool, +} + +/// DB representation of [`EventTag`] +#[derive(Queryable, Selectable, Insertable, AsChangeset, Debug, Clone)] +#[diesel(table_name = event_tags, check_for_backend(diesel::pg::Pg))] +pub struct EventTagDb { + pub tag: String, + pub tag_value: String, + pub event_id: String, +} + +/// A data container for extracting data from [`Event`] and its tags +#[derive(Debug, Clone)] +pub struct EventDataDb { + pub event: EventDb, + pub tags: Vec, +} + +impl TryFrom<&Event> for EventDataDb { + type Error = DatabaseError; + fn try_from(value: &Event) -> Result { + let serialized = value.encode(&mut FlatBufferBuilder::new()).to_vec(); + Ok(Self { + event: EventDb { + id: value.id.to_string(), + pubkey: value.pubkey.to_string(), + created_at: value.created_at.as_u64() as i64, + kind: value.kind.as_u16() as i64, + payload: serialized, + signature: value.sig.to_string(), + deleted: false, + }, + tags: extract_tags(value), + }) + } +} + +fn extract_tags(event: &Event) -> Vec { + event + .tags + .iter() + .filter_map(|tag| { + if let (kind, Some(content)) = (tag.kind(), tag.content()) { + Some(EventTagDb { + tag: kind.to_string(), + tag_value: content.to_string(), + event_id: event.id.to_string(), + }) + } else { + None + } + }) + .collect() +} diff --git a/crates/nostr-postgresdb/src/postgres.rs b/crates/nostr-postgresdb/src/postgres.rs new file mode 100644 index 000000000..5c7df213e --- /dev/null +++ b/crates/nostr-postgresdb/src/postgres.rs @@ -0,0 +1,192 @@ +use deadpool::managed::{Object, Pool}; +use diesel::{ + QueryResult, + dsl::{Eq, Filter as DieselFilter, InnerJoin, IntoBoxed}, + pg::Pg, + prelude::*, + result::{DatabaseErrorKind, Error as DieselError}, +}; +use diesel_async::{ + AsyncConnection, AsyncPgConnection, RunQueryDsl, + pooled_connection::AsyncDieselConnectionManager, scoped_futures::ScopedFutureExt, +}; +use super::model::{EventDataDb, EventDb}; +use nostr::{event::*, filter::Filter}; +use nostr_database::*; +use super::schema::nostr::{event_tags, events}; + +/// Shorthand for a database connection pool type +pub type PostgresConnectionPool = Pool>; +pub type PostgresConnection = Object>; + +#[derive(Clone)] +pub struct NostrPostgres { + pool: PostgresConnectionPool, +} + +impl NostrPostgres { + /// Create a new [`NostrPostgres`] instance + pub async fn new(connection_string: C) -> Result + where + C: AsRef, + { + let pool = postgres_connection_pool(connection_string).await?; + Ok(Self { pool }) + } + + pub (crate) async fn get_connection(&self) -> Result { + self.pool.get().await.map_err(DatabaseError::backend) + } + + pub (crate) async fn save(&self, event_data: EventDataDb) -> Result { + let mut db = self.get_connection().await?; + let result: QueryResult = db + .transaction(|c| { + async move { + diesel::insert_into(events::table) + .values(&event_data.event) + .execute(c) + .await?; + + diesel::insert_into(event_tags::table) + .values(&event_data.tags) + .execute(c) + .await?; + + Ok(true) + } + .scope_boxed() + }) + .await; + + match result { + Ok(_) => Ok(SaveEventStatus::Success), + Err(e) => match e { + DieselError::DatabaseError(DatabaseErrorKind::UniqueViolation, _) => { + Ok(SaveEventStatus::Rejected(RejectedReason::Duplicate)) + } + e => Err(DatabaseError::backend(e)), + }, + } + } + + pub (crate) async fn event_by_id(&self, event_id: &EventId) -> Result, DatabaseError> { + let event_id = event_id.to_hex(); + let res = events::table + .select(EventDb::as_select()) + .filter(events::id.eq(event_id)) + .first(&mut self.get_connection().await?) + .await + .optional() + .map_err(DatabaseError::backend)?; + Ok(res) + } +} + +/// Create a new [`NostrPostgres`] instance from an existing connection pool +impl From for NostrPostgres { + fn from(pool: PostgresConnectionPool) -> Self { + Self { pool } + } +} + +/// Create a connection pool for a Postgres database with the given connection string. +pub async fn postgres_connection_pool( + connection_string: C, +) -> Result +where + C: AsRef, +{ + let config = AsyncDieselConnectionManager::::new(connection_string.as_ref()); + let pool: PostgresConnectionPool = Pool::builder(config) + .build() + .map_err(|e| DatabaseError::Backend(Box::new(e)))?; + Ok(pool) +} + +impl std::fmt::Debug for NostrPostgres { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NostrPostgres") + .field("pool", &self.pool.status()) + .finish() + } +} + +/// sets the given default limit on a Nostr filter if not set +pub fn with_limit(filter: Filter, default_limit: usize) -> Filter { + if filter.limit.is_none() { + return filter.limit(default_limit); + } + filter +} + +// filter type of a join query. +type QuerySetJoinType<'a> = IntoBoxed< + 'a, + DieselFilter< + InnerJoin, + Eq>, + >, + Pg, +>; + +pub fn build_filter_query<'a>(filter: Filter) -> QuerySetJoinType<'a> { + let mut query = events::table + .distinct_on(events::id) + .inner_join(event_tags::table) + .into_boxed(); + + if let Some(limit) = filter.limit { + query = query.limit(limit as i64); + } + + if !has_filters(&filter) { + return query; + } + if let Some(ids) = filter.ids.clone() { + let values = ids.iter().map(|id| id.to_hex()).collect::>(); + query = query.filter(events::id.eq_any(values)); + } + + if let Some(authors) = filter.authors.clone() { + let values = authors.iter().map(|a| a.to_hex()).collect::>(); + query = query.filter(events::pubkey.eq_any(values)); + } + + if let Some(kinds) = filter.kinds.clone() { + let values = kinds.iter().map(|k| k.as_u16() as i64).collect::>(); + query = query.filter(events::kind.eq_any(values)); + } + + if let Some(since) = filter.since { + query = query.filter(events::created_at.ge(since.as_u64() as i64)); + } + + if let Some(until) = filter.until { + query = query.filter(events::created_at.le(until.as_u64() as i64)); + } + + if !filter.generic_tags.is_empty() { + for (tag, values) in filter.generic_tags.into_iter() { + let values = values.iter().map(|v| v.to_string()).collect::>(); + query = query.filter( + event_tags::tag + .eq(tag.to_string()) + .and(event_tags::tag_value.eq_any(values)), + ); + } + } + + query +} + +// determine if the filter has any filters set +fn has_filters(filter: &Filter) -> bool { + filter.ids.is_some() + || filter.authors.is_some() + || filter.kinds.is_some() + || filter.since.is_some() + || filter.until.is_some() + || !filter.generic_tags.is_empty() + || filter.limit.is_some() +} diff --git a/crates/nostr-postgresdb/src/schema.rs b/crates/nostr-postgresdb/src/schema.rs new file mode 100644 index 000000000..77229aea9 --- /dev/null +++ b/crates/nostr-postgresdb/src/schema.rs @@ -0,0 +1,34 @@ +// @generated automatically by Diesel CLI. + +pub mod nostr { + diesel::table! { + nostr.event_tags (tag, tag_value, event_id) { + tag -> Text, + tag_value -> Text, + #[max_length = 64] + event_id -> Varchar, + } + } + + diesel::table! { + nostr.events (id) { + #[max_length = 64] + id -> Varchar, + #[max_length = 64] + pubkey -> Varchar, + created_at -> Int8, + kind -> Int8, + payload -> Bytea, + #[max_length = 128] + signature -> Varchar, + deleted -> Bool, + } + } + + diesel::joinable!(event_tags -> events (event_id)); + + diesel::allow_tables_to_appear_in_same_query!( + event_tags, + events, + ); +} From 2dd01b4d1f4640323ee47823ee51bf81a73737ce Mon Sep 17 00:00:00 2001 From: tompro Date: Tue, 15 Apr 2025 18:41:55 +0200 Subject: [PATCH 02/12] Added example --- Cargo.lock | 5 ++- crates/nostr-postgresdb/Cargo.toml | 10 ++++- .../examples/postgres-relay.rs | 37 +++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 crates/nostr-postgresdb/examples/postgres-relay.rs diff --git a/Cargo.lock b/Cargo.lock index 3712e02bc..a6aacbccc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2935,7 +2935,7 @@ dependencies = [ [[package]] name = "nostr-postgresdb" -version = "0.40.0" +version = "0.41.0" dependencies = [ "deadpool", "diesel", @@ -2943,7 +2943,10 @@ dependencies = [ "diesel_migrations", "nostr", "nostr-database", + "nostr-relay-builder", + "tokio", "tracing", + "tracing-subscriber", ] [[package]] diff --git a/crates/nostr-postgresdb/Cargo.toml b/crates/nostr-postgresdb/Cargo.toml index 810ac0323..555176780 100644 --- a/crates/nostr-postgresdb/Cargo.toml +++ b/crates/nostr-postgresdb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nostr-postgresdb" -version = "0.40.0" +version = "0.41.0" edition = "2021" description = "Postgres storage backend for Nostr apps" authors.workspace = true @@ -19,3 +19,11 @@ diesel = { version = "2", features = ["postgres", "serde_json"] } diesel-async = { version = "0.5", features = ["postgres", "deadpool"] } diesel_migrations = { version = "2", features = ["postgres"] } deadpool = { version = "0.12", features = ["managed", "rt_tokio_1"] } + +[dev-dependencies] +tokio.workspace = true +nostr-relay-builder = { workspace = true } +tracing-subscriber = { workspace = true } + +[[example]] +name = "postgres-relay" diff --git a/crates/nostr-postgresdb/examples/postgres-relay.rs b/crates/nostr-postgresdb/examples/postgres-relay.rs new file mode 100644 index 000000000..a52431b6f --- /dev/null +++ b/crates/nostr-postgresdb/examples/postgres-relay.rs @@ -0,0 +1,37 @@ +// Copyright (c) 2025 Protom +// Distributed under the MIT software license + +use std::time::Duration; + +use nostr_database::prelude::*; +use nostr_postgresdb::NostrPostgres; +use nostr_relay_builder::prelude::*; + +// Your database URL +const DB_URL: &str = "postgres://postgres:password@localhost:5432"; + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt::init(); + + // This will programatically run pending db migrations + nostr_postgresdb::run_migrations(DB_URL)?; + + // Create a conncetion pool + let pool = nostr_postgresdb::postgres_connection_pool(DB_URL).await?; + + // Create a nostr db instance + let db: NostrPostgres = pool.into(); + + // Add db to builder + let builder = RelayBuilder::default().database(db); + + // Create local relay + let relay = LocalRelay::run(builder).await?; + println!("Url: {}", relay.url()); + + // Keep up the program + loop { + tokio::time::sleep(Duration::from_secs(60)).await; + } +} From cd828008e8d3adb68e4d7fe32d938c97f8f65869 Mon Sep 17 00:00:00 2001 From: tompro Date: Tue, 15 Apr 2025 19:01:21 +0200 Subject: [PATCH 03/12] nostr-postgresdb formatting --- crates/nostr-postgresdb/src/lib.rs | 10 ++++--- crates/nostr-postgresdb/src/model.rs | 8 ++---- crates/nostr-postgresdb/src/postgres.rs | 37 ++++++++++++++----------- crates/nostr-postgresdb/src/schema.rs | 5 +--- 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/crates/nostr-postgresdb/src/lib.rs b/crates/nostr-postgresdb/src/lib.rs index 4a3b77a02..83622343a 100644 --- a/crates/nostr-postgresdb/src/lib.rs +++ b/crates/nostr-postgresdb/src/lib.rs @@ -5,14 +5,16 @@ mod schema; use diesel::prelude::*; use diesel_async::RunQueryDsl; +pub use migrations::run_migrations; use model::{EventDataDb, EventDb}; -use nostr::{event::*, filter::Filter, types::Timestamp, util::BoxedFuture}; +use nostr::event::*; +use nostr::filter::Filter; +use nostr::types::Timestamp; +use nostr::util::BoxedFuture; use nostr_database::*; use postgres::{build_filter_query, with_limit}; -use schema::nostr::events; - -pub use migrations::run_migrations; pub use postgres::{postgres_connection_pool, NostrPostgres}; +use schema::nostr::events; impl NostrDatabase for NostrPostgres { fn backend(&self) -> Backend { diff --git a/crates/nostr-postgresdb/src/model.rs b/crates/nostr-postgresdb/src/model.rs index 1f592c6a6..cc87fe7b9 100644 --- a/crates/nostr-postgresdb/src/model.rs +++ b/crates/nostr-postgresdb/src/model.rs @@ -1,10 +1,8 @@ -use crate::schema::nostr::event_tags; -use crate::schema::nostr::events; use diesel::prelude::*; use nostr::event::Event; -use nostr_database::DatabaseError; -use nostr_database::FlatBufferBuilder; -use nostr_database::FlatBufferEncode; +use nostr_database::{DatabaseError, FlatBufferBuilder, FlatBufferEncode}; + +use crate::schema::nostr::{event_tags, events}; /// DB representation of [`Event`] #[derive(Queryable, Selectable, Insertable, AsChangeset, Debug, Clone)] diff --git a/crates/nostr-postgresdb/src/postgres.rs b/crates/nostr-postgresdb/src/postgres.rs index 5c7df213e..0bc460c95 100644 --- a/crates/nostr-postgresdb/src/postgres.rs +++ b/crates/nostr-postgresdb/src/postgres.rs @@ -1,18 +1,17 @@ use deadpool::managed::{Object, Pool}; -use diesel::{ - QueryResult, - dsl::{Eq, Filter as DieselFilter, InnerJoin, IntoBoxed}, - pg::Pg, - prelude::*, - result::{DatabaseErrorKind, Error as DieselError}, -}; -use diesel_async::{ - AsyncConnection, AsyncPgConnection, RunQueryDsl, - pooled_connection::AsyncDieselConnectionManager, scoped_futures::ScopedFutureExt, -}; -use super::model::{EventDataDb, EventDb}; -use nostr::{event::*, filter::Filter}; +use diesel::dsl::{Eq, Filter as DieselFilter, InnerJoin, IntoBoxed}; +use diesel::pg::Pg; +use diesel::prelude::*; +use diesel::result::{DatabaseErrorKind, Error as DieselError}; +use diesel::QueryResult; +use diesel_async::pooled_connection::AsyncDieselConnectionManager; +use diesel_async::scoped_futures::ScopedFutureExt; +use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl}; +use nostr::event::*; +use nostr::filter::Filter; use nostr_database::*; + +use super::model::{EventDataDb, EventDb}; use super::schema::nostr::{event_tags, events}; /// Shorthand for a database connection pool type @@ -34,11 +33,14 @@ impl NostrPostgres { Ok(Self { pool }) } - pub (crate) async fn get_connection(&self) -> Result { + pub(crate) async fn get_connection(&self) -> Result { self.pool.get().await.map_err(DatabaseError::backend) } - pub (crate) async fn save(&self, event_data: EventDataDb) -> Result { + pub(crate) async fn save( + &self, + event_data: EventDataDb, + ) -> Result { let mut db = self.get_connection().await?; let result: QueryResult = db .transaction(|c| { @@ -70,7 +72,10 @@ impl NostrPostgres { } } - pub (crate) async fn event_by_id(&self, event_id: &EventId) -> Result, DatabaseError> { + pub(crate) async fn event_by_id( + &self, + event_id: &EventId, + ) -> Result, DatabaseError> { let event_id = event_id.to_hex(); let res = events::table .select(EventDb::as_select()) diff --git a/crates/nostr-postgresdb/src/schema.rs b/crates/nostr-postgresdb/src/schema.rs index 77229aea9..0ca95c955 100644 --- a/crates/nostr-postgresdb/src/schema.rs +++ b/crates/nostr-postgresdb/src/schema.rs @@ -27,8 +27,5 @@ pub mod nostr { diesel::joinable!(event_tags -> events (event_id)); - diesel::allow_tables_to_appear_in_same_query!( - event_tags, - events, - ); + diesel::allow_tables_to_appear_in_same_query!(event_tags, events,); } From 0846696be7f0673535c2b0d671ef8a6644ee098c Mon Sep 17 00:00:00 2001 From: tompro Date: Wed, 16 Apr 2025 14:50:01 +0200 Subject: [PATCH 04/12] Deleted filters --- crates/nostr-postgresdb/src/lib.rs | 6 ++++-- crates/nostr-postgresdb/src/postgres.rs | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/nostr-postgresdb/src/lib.rs b/crates/nostr-postgresdb/src/lib.rs index 83622343a..ac079b751 100644 --- a/crates/nostr-postgresdb/src/lib.rs +++ b/crates/nostr-postgresdb/src/lib.rs @@ -66,8 +66,10 @@ impl NostrEventsDatabase for NostrPostgres { ) -> BoxedFuture<'a, Result, DatabaseError>> { Box::pin(async move { let event = match self.event_by_id(_event_id).await? { - Some(e) => Some(Event::decode(&e.payload).map_err(DatabaseError::backend)?), - None => None, + Some(e) if !e.deleted => { + Some(Event::decode(&e.payload).map_err(DatabaseError::backend)?) + } + _ => None, }; Ok(event) }) diff --git a/crates/nostr-postgresdb/src/postgres.rs b/crates/nostr-postgresdb/src/postgres.rs index 0bc460c95..9f2d0d5c6 100644 --- a/crates/nostr-postgresdb/src/postgres.rs +++ b/crates/nostr-postgresdb/src/postgres.rs @@ -139,6 +139,7 @@ pub fn build_filter_query<'a>(filter: Filter) -> QuerySetJoinType<'a> { let mut query = events::table .distinct_on(events::id) .inner_join(event_tags::table) + .filter(events::deleted.eq(false)) .into_boxed(); if let Some(limit) = filter.limit { From 88d7f895ddf56b627e75af4afcc45d0fabad6a9f Mon Sep 17 00:00:00 2001 From: tompro Date: Fri, 18 Apr 2025 12:35:52 +0200 Subject: [PATCH 05/12] Rename crate --- crates/nostr-postgresdb/src/schema.rs | 31 ------------------ .../Cargo.toml | 0 .../README.md | 0 .../diesel.toml | 1 - .../examples/postgres-relay.rs | 0 .../migrations/.keep | 0 .../down.sql | 0 .../up.sql | 0 .../2025-04-11-095120_events/down.sql | 1 - .../2025-04-11-095120_events/up.sql | 17 ++++------ .../src/lib.rs | 2 +- .../src/migrations.rs | 0 .../src/model.rs | 2 +- .../src/postgres.rs | 2 +- crates/nostr-sqldb/src/schema.rs | 32 +++++++++++++++++++ 15 files changed, 42 insertions(+), 46 deletions(-) delete mode 100644 crates/nostr-postgresdb/src/schema.rs rename crates/{nostr-postgresdb => nostr-sqldb}/Cargo.toml (100%) rename crates/{nostr-postgresdb => nostr-sqldb}/README.md (100%) rename crates/{nostr-postgresdb => nostr-sqldb}/diesel.toml (93%) rename crates/{nostr-postgresdb => nostr-sqldb}/examples/postgres-relay.rs (100%) rename crates/{nostr-postgresdb => nostr-sqldb}/migrations/.keep (100%) rename crates/{nostr-postgresdb => nostr-sqldb}/migrations/00000000000000_diesel_initial_setup/down.sql (100%) rename crates/{nostr-postgresdb => nostr-sqldb}/migrations/00000000000000_diesel_initial_setup/up.sql (100%) rename crates/{nostr-postgresdb => nostr-sqldb}/migrations/2025-04-11-095120_events/down.sql (84%) rename crates/{nostr-postgresdb => nostr-sqldb}/migrations/2025-04-11-095120_events/up.sql (58%) rename crates/{nostr-postgresdb => nostr-sqldb}/src/lib.rs (99%) rename crates/{nostr-postgresdb => nostr-sqldb}/src/migrations.rs (100%) rename crates/{nostr-postgresdb => nostr-sqldb}/src/model.rs (97%) rename crates/{nostr-postgresdb => nostr-sqldb}/src/postgres.rs (99%) create mode 100644 crates/nostr-sqldb/src/schema.rs diff --git a/crates/nostr-postgresdb/src/schema.rs b/crates/nostr-postgresdb/src/schema.rs deleted file mode 100644 index 0ca95c955..000000000 --- a/crates/nostr-postgresdb/src/schema.rs +++ /dev/null @@ -1,31 +0,0 @@ -// @generated automatically by Diesel CLI. - -pub mod nostr { - diesel::table! { - nostr.event_tags (tag, tag_value, event_id) { - tag -> Text, - tag_value -> Text, - #[max_length = 64] - event_id -> Varchar, - } - } - - diesel::table! { - nostr.events (id) { - #[max_length = 64] - id -> Varchar, - #[max_length = 64] - pubkey -> Varchar, - created_at -> Int8, - kind -> Int8, - payload -> Bytea, - #[max_length = 128] - signature -> Varchar, - deleted -> Bool, - } - } - - diesel::joinable!(event_tags -> events (event_id)); - - diesel::allow_tables_to_appear_in_same_query!(event_tags, events,); -} diff --git a/crates/nostr-postgresdb/Cargo.toml b/crates/nostr-sqldb/Cargo.toml similarity index 100% rename from crates/nostr-postgresdb/Cargo.toml rename to crates/nostr-sqldb/Cargo.toml diff --git a/crates/nostr-postgresdb/README.md b/crates/nostr-sqldb/README.md similarity index 100% rename from crates/nostr-postgresdb/README.md rename to crates/nostr-sqldb/README.md diff --git a/crates/nostr-postgresdb/diesel.toml b/crates/nostr-sqldb/diesel.toml similarity index 93% rename from crates/nostr-postgresdb/diesel.toml rename to crates/nostr-sqldb/diesel.toml index 1c5febda4..a0d61bf48 100644 --- a/crates/nostr-postgresdb/diesel.toml +++ b/crates/nostr-sqldb/diesel.toml @@ -4,7 +4,6 @@ [print_schema] file = "src/schema.rs" custom_type_derives = ["diesel::query_builder::QueryId", "Clone"] -schema = "nostr" [migrations_directory] dir = "migrations" diff --git a/crates/nostr-postgresdb/examples/postgres-relay.rs b/crates/nostr-sqldb/examples/postgres-relay.rs similarity index 100% rename from crates/nostr-postgresdb/examples/postgres-relay.rs rename to crates/nostr-sqldb/examples/postgres-relay.rs diff --git a/crates/nostr-postgresdb/migrations/.keep b/crates/nostr-sqldb/migrations/.keep similarity index 100% rename from crates/nostr-postgresdb/migrations/.keep rename to crates/nostr-sqldb/migrations/.keep diff --git a/crates/nostr-postgresdb/migrations/00000000000000_diesel_initial_setup/down.sql b/crates/nostr-sqldb/migrations/00000000000000_diesel_initial_setup/down.sql similarity index 100% rename from crates/nostr-postgresdb/migrations/00000000000000_diesel_initial_setup/down.sql rename to crates/nostr-sqldb/migrations/00000000000000_diesel_initial_setup/down.sql diff --git a/crates/nostr-postgresdb/migrations/00000000000000_diesel_initial_setup/up.sql b/crates/nostr-sqldb/migrations/00000000000000_diesel_initial_setup/up.sql similarity index 100% rename from crates/nostr-postgresdb/migrations/00000000000000_diesel_initial_setup/up.sql rename to crates/nostr-sqldb/migrations/00000000000000_diesel_initial_setup/up.sql diff --git a/crates/nostr-postgresdb/migrations/2025-04-11-095120_events/down.sql b/crates/nostr-sqldb/migrations/2025-04-11-095120_events/down.sql similarity index 84% rename from crates/nostr-postgresdb/migrations/2025-04-11-095120_events/down.sql rename to crates/nostr-sqldb/migrations/2025-04-11-095120_events/down.sql index 8e6a73889..4383998aa 100644 --- a/crates/nostr-postgresdb/migrations/2025-04-11-095120_events/down.sql +++ b/crates/nostr-sqldb/migrations/2025-04-11-095120_events/down.sql @@ -1,4 +1,3 @@ -- This file should undo anything in `up.sql` DROP TABLE nostr.event_tags; DROP TABLE nostr.events; -DROP SCHEMA nostr; diff --git a/crates/nostr-postgresdb/migrations/2025-04-11-095120_events/up.sql b/crates/nostr-sqldb/migrations/2025-04-11-095120_events/up.sql similarity index 58% rename from crates/nostr-postgresdb/migrations/2025-04-11-095120_events/up.sql rename to crates/nostr-sqldb/migrations/2025-04-11-095120_events/up.sql index 1fabcf294..3f3b7c090 100644 --- a/crates/nostr-postgresdb/migrations/2025-04-11-095120_events/up.sql +++ b/crates/nostr-sqldb/migrations/2025-04-11-095120_events/up.sql @@ -1,8 +1,5 @@ --- Init the schema -CREATE SCHEMA IF NOT EXISTS nostr; - -- The actual event data -CREATE TABLE nostr.events ( +CREATE TABLE events ( id VARCHAR(64) PRIMARY KEY, pubkey VARCHAR(64) NOT NULL, created_at BIGINT NOT NULL, @@ -13,17 +10,17 @@ CREATE TABLE nostr.events ( ); -- Direct indexes -CREATE INDEX event_pubkey ON nostr.events (pubkey); -CREATE INDEX event_date ON nostr.events (created_at); -CREATE INDEX event_kind ON nostr.events (kind); -CREATE INDEX event_deleted ON nostr.events (deleted); +CREATE INDEX event_pubkey ON events (pubkey); +CREATE INDEX event_date ON events (created_at); +CREATE INDEX event_kind ON events (kind); +CREATE INDEX event_deleted ON events (deleted); -- The tag index, the primary will give us the index automatically -CREATE TABLE nostr.event_tags ( +CREATE TABLE event_tags ( tag TEXT NOT NULL, tag_value TEXT NOT NULL, event_id VARCHAR(64) NOT NULL - REFERENCES nostr.events (id) + REFERENCES events (id) ON DELETE CASCADE ON UPDATE CASCADE, PRIMARY KEY (tag, tag_value, event_id) diff --git a/crates/nostr-postgresdb/src/lib.rs b/crates/nostr-sqldb/src/lib.rs similarity index 99% rename from crates/nostr-postgresdb/src/lib.rs rename to crates/nostr-sqldb/src/lib.rs index ac079b751..3eca946d1 100644 --- a/crates/nostr-postgresdb/src/lib.rs +++ b/crates/nostr-sqldb/src/lib.rs @@ -14,7 +14,7 @@ use nostr::util::BoxedFuture; use nostr_database::*; use postgres::{build_filter_query, with_limit}; pub use postgres::{postgres_connection_pool, NostrPostgres}; -use schema::nostr::events; +use schema::events; impl NostrDatabase for NostrPostgres { fn backend(&self) -> Backend { diff --git a/crates/nostr-postgresdb/src/migrations.rs b/crates/nostr-sqldb/src/migrations.rs similarity index 100% rename from crates/nostr-postgresdb/src/migrations.rs rename to crates/nostr-sqldb/src/migrations.rs diff --git a/crates/nostr-postgresdb/src/model.rs b/crates/nostr-sqldb/src/model.rs similarity index 97% rename from crates/nostr-postgresdb/src/model.rs rename to crates/nostr-sqldb/src/model.rs index cc87fe7b9..b749af901 100644 --- a/crates/nostr-postgresdb/src/model.rs +++ b/crates/nostr-sqldb/src/model.rs @@ -2,7 +2,7 @@ use diesel::prelude::*; use nostr::event::Event; use nostr_database::{DatabaseError, FlatBufferBuilder, FlatBufferEncode}; -use crate::schema::nostr::{event_tags, events}; +use crate::schema::{event_tags, events}; /// DB representation of [`Event`] #[derive(Queryable, Selectable, Insertable, AsChangeset, Debug, Clone)] diff --git a/crates/nostr-postgresdb/src/postgres.rs b/crates/nostr-sqldb/src/postgres.rs similarity index 99% rename from crates/nostr-postgresdb/src/postgres.rs rename to crates/nostr-sqldb/src/postgres.rs index 9f2d0d5c6..1fdca36f7 100644 --- a/crates/nostr-postgresdb/src/postgres.rs +++ b/crates/nostr-sqldb/src/postgres.rs @@ -12,7 +12,7 @@ use nostr::filter::Filter; use nostr_database::*; use super::model::{EventDataDb, EventDb}; -use super::schema::nostr::{event_tags, events}; +use super::schema::{event_tags, events}; /// Shorthand for a database connection pool type pub type PostgresConnectionPool = Pool>; diff --git a/crates/nostr-sqldb/src/schema.rs b/crates/nostr-sqldb/src/schema.rs new file mode 100644 index 000000000..668608add --- /dev/null +++ b/crates/nostr-sqldb/src/schema.rs @@ -0,0 +1,32 @@ +// @generated automatically by Diesel CLI. + +diesel::table! { + event_tags (tag, tag_value, event_id) { + tag -> Text, + tag_value -> Text, + #[max_length = 64] + event_id -> Varchar, + } +} + +diesel::table! { + events (id) { + #[max_length = 64] + id -> Varchar, + #[max_length = 64] + pubkey -> Varchar, + created_at -> Int8, + kind -> Int8, + payload -> Bytea, + #[max_length = 128] + signature -> Varchar, + deleted -> Bool, + } +} + +diesel::joinable!(event_tags -> events (event_id)); + +diesel::allow_tables_to_appear_in_same_query!( + event_tags, + events, +); From dbf7e73a50583ecf495e986684f0e1d79799238a Mon Sep 17 00:00:00 2001 From: tompro Date: Fri, 18 Apr 2025 12:38:17 +0200 Subject: [PATCH 06/12] Formatting --- crates/nostr-sqldb/src/schema.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/nostr-sqldb/src/schema.rs b/crates/nostr-sqldb/src/schema.rs index 668608add..8cbb360e3 100644 --- a/crates/nostr-sqldb/src/schema.rs +++ b/crates/nostr-sqldb/src/schema.rs @@ -26,7 +26,4 @@ diesel::table! { diesel::joinable!(event_tags -> events (event_id)); -diesel::allow_tables_to_appear_in_same_query!( - event_tags, - events, -); +diesel::allow_tables_to_appear_in_same_query!(event_tags, events,); From 21819f86126576d0224227486b8d83b2c15bcd48 Mon Sep 17 00:00:00 2001 From: tompro Date: Tue, 22 Apr 2025 10:41:25 +0200 Subject: [PATCH 07/12] Multi db features --- Cargo.lock | 260 +++++++++++++++++- crates/nostr-sqldb/Cargo.toml | 22 +- .../2025-04-11-095120_events/down.sql | 3 - crates/nostr-sqldb/migrations/mysql/.keep | 0 .../mysql/2025-04-11-095120_events/down.sql | 3 + .../mysql/2025-04-11-095120_events/up.sql | 27 ++ .../down.sql | 0 .../up.sql | 0 .../2025-04-11-095120_events/down.sql | 3 + .../2025-04-11-095120_events/up.sql | 0 crates/nostr-sqldb/migrations/sqlite/.keep | 0 .../sqlite/2025-04-11-095120_events/down.sql | 3 + .../sqlite/2025-04-11-095120_events/up.sql | 27 ++ .../nostr-sqldb/{diesel.toml => mysql.toml} | 4 +- crates/nostr-sqldb/postgres.toml | 9 + crates/nostr-sqldb/sqlite.toml | 9 + crates/nostr-sqldb/src/lib.rs | 11 +- crates/nostr-sqldb/src/migrations/mod.rs | 8 + crates/nostr-sqldb/src/migrations/mysql.rs | 19 ++ .../{migrations.rs => migrations/postgres.rs} | 2 +- crates/nostr-sqldb/src/migrations/sqlite.rs | 19 ++ crates/nostr-sqldb/src/model.rs | 13 +- crates/nostr-sqldb/src/postgres.rs | 4 +- crates/nostr-sqldb/src/schema/mod.rs | 8 + crates/nostr-sqldb/src/schema/mysql.rs | 31 +++ .../src/{schema.rs => schema/postgres.rs} | 0 crates/nostr-sqldb/src/schema/sqlite.rs | 28 ++ crates/nostr-sqldb/src/types.rs | 31 +++ 28 files changed, 526 insertions(+), 18 deletions(-) delete mode 100644 crates/nostr-sqldb/migrations/2025-04-11-095120_events/down.sql create mode 100644 crates/nostr-sqldb/migrations/mysql/.keep create mode 100644 crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/down.sql create mode 100644 crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/up.sql rename crates/nostr-sqldb/migrations/{ => postgres}/00000000000000_diesel_initial_setup/down.sql (100%) rename crates/nostr-sqldb/migrations/{ => postgres}/00000000000000_diesel_initial_setup/up.sql (100%) create mode 100644 crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/down.sql rename crates/nostr-sqldb/migrations/{ => postgres}/2025-04-11-095120_events/up.sql (100%) create mode 100644 crates/nostr-sqldb/migrations/sqlite/.keep create mode 100644 crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/down.sql create mode 100644 crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/up.sql rename crates/nostr-sqldb/{diesel.toml => mysql.toml} (79%) create mode 100644 crates/nostr-sqldb/postgres.toml create mode 100644 crates/nostr-sqldb/sqlite.toml create mode 100644 crates/nostr-sqldb/src/migrations/mod.rs create mode 100644 crates/nostr-sqldb/src/migrations/mysql.rs rename crates/nostr-sqldb/src/{migrations.rs => migrations/postgres.rs} (97%) create mode 100644 crates/nostr-sqldb/src/migrations/sqlite.rs create mode 100644 crates/nostr-sqldb/src/schema/mod.rs create mode 100644 crates/nostr-sqldb/src/schema/mysql.rs rename crates/nostr-sqldb/src/{schema.rs => schema/postgres.rs} (100%) create mode 100644 crates/nostr-sqldb/src/schema/sqlite.rs create mode 100644 crates/nostr-sqldb/src/types.rs diff --git a/Cargo.lock b/Cargo.lock index a6aacbccc..d030be929 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,6 +72,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "amplify" version = "4.6.1" @@ -577,6 +583,15 @@ dependencies = [ "serde", ] +[[package]] +name = "btoi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd6407f73a9b8b6162d8a2ef999fe6afd7cc15902ebf42c5cd296addf17e0ad" +dependencies = [ + "num-traits", +] + [[package]] name = "bumpalo" version = "3.14.0" @@ -628,6 +643,8 @@ version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -766,6 +783,15 @@ dependencies = [ "error-code", ] +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + [[package]] name = "coarsetime" version = "0.1.34" @@ -892,6 +918,47 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -1284,8 +1351,13 @@ dependencies = [ "byteorder", "diesel_derives", "itoa", + "libsqlite3-sys", + "mysqlclient-sys", + "percent-encoding", "pq-sys", "serde_json", + "time", + "url", ] [[package]] @@ -1297,7 +1369,10 @@ dependencies = [ "async-trait", "deadpool", "diesel", + "futures-channel", "futures-util", + "mysql_async", + "mysql_common", "scoped-futures", "tokio", "tokio-postgres", @@ -1734,6 +1809,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -2000,6 +2081,17 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashlink" version = "0.9.1" @@ -2437,6 +2529,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.70" @@ -2465,6 +2566,15 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "keyed_priority_queue" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" +dependencies = [ + "indexmap 2.5.0", +] + [[package]] name = "keyring" version = "3.6.2" @@ -2601,6 +2711,15 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "lru" version = "0.13.0" @@ -2765,6 +2884,78 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mysql_async" +version = "0.34.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0b66e411c31265e879d9814d03721f2daa7ad07337b6308cb4bb0cde7e6fd47" +dependencies = [ + "bytes", + "crossbeam", + "flate2", + "futures-core", + "futures-sink", + "futures-util", + "keyed_priority_queue", + "lru 0.12.5", + "mysql_common", + "pem", + "percent-encoding", + "pin-project", + "rand 0.8.5", + "serde", + "serde_json", + "socket2", + "thiserror 1.0.64", + "tokio", + "tokio-util", + "twox-hash", + "url", +] + +[[package]] +name = "mysql_common" +version = "0.32.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478b0ff3f7d67b79da2b96f56f334431aef65e15ba4b29dd74a4236e29582bdc" +dependencies = [ + "base64 0.21.7", + "bindgen", + "bitflags 2.9.0", + "btoi", + "byteorder", + "bytes", + "cc", + "cmake", + "crc32fast", + "flate2", + "lazy_static", + "num-bigint", + "num-traits", + "rand 0.8.5", + "regex", + "saturating", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "subprocess", + "thiserror 1.0.64", + "uuid", + "zstd", +] + +[[package]] +name = "mysqlclient-sys" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f29e21174d84e2622ceb7b0146a9187d36458a3a9ee9a66c9cac22e96493ef9" +dependencies = [ + "pkg-config", + "vcpkg", +] + [[package]] name = "ndk-context" version = "0.1.1" @@ -2873,7 +3064,7 @@ name = "nostr-database" version = "0.41.0" dependencies = [ "flatbuffers", - "lru", + "lru 0.13.0", "nostr", "tokio", ] @@ -2974,7 +3165,7 @@ dependencies = [ "async-utility", "async-wsocket", "atomic-destructor", - "lru", + "lru 0.13.0", "negentropy 0.3.1", "negentropy 0.5.0", "nostr", @@ -3363,6 +3554,16 @@ dependencies = [ "hmac", ] +[[package]] +name = "pem" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" +dependencies = [ + "base64 0.22.1", + "serde", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -4131,6 +4332,12 @@ dependencies = [ "regex", ] +[[package]] +name = "saturating" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" + [[package]] name = "scoped-futures" version = "0.1.4" @@ -4596,6 +4803,16 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "subprocess" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "subtle" version = "2.6.1" @@ -6052,6 +6269,17 @@ dependencies = [ "utf-8", ] +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "rand 0.8.5", + "static_assertions", +] + [[package]] name = "typed-index-collections" version = "3.1.0" @@ -6845,3 +7073,31 @@ dependencies = [ "quote", "syn 2.0.90", ] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/crates/nostr-sqldb/Cargo.toml b/crates/nostr-sqldb/Cargo.toml index 555176780..6a0de30ab 100644 --- a/crates/nostr-sqldb/Cargo.toml +++ b/crates/nostr-sqldb/Cargo.toml @@ -15,11 +15,27 @@ keywords = ["nostr", "database", "postgres"] nostr = { workspace = true, features = ["std"] } nostr-database = { workspace = true, features = ["flatbuf"] } tracing.workspace = true -diesel = { version = "2", features = ["postgres", "serde_json"] } -diesel-async = { version = "0.5", features = ["postgres", "deadpool"] } -diesel_migrations = { version = "2", features = ["postgres"] } +diesel = { version = "2", features = ["serde_json"] } +diesel-async = { version = "0.5", features = ["deadpool"] } +diesel_migrations = { version = "2" } deadpool = { version = "0.12", features = ["managed", "rt_tokio_1"] } +[features] +default = ["postgres"] +postgres = [ + "diesel/postgres", + "diesel-async/postgres", + "diesel_migrations/postgres", +] + +mysql = ["diesel/mysql", "diesel-async/mysql", "diesel_migrations/mysql"] +sqlite = [ + "diesel/sqlite", + "diesel-async/sqlite", + "diesel_migrations/sqlite", + "diesel/returning_clauses_for_sqlite_3_35", +] + [dev-dependencies] tokio.workspace = true nostr-relay-builder = { workspace = true } diff --git a/crates/nostr-sqldb/migrations/2025-04-11-095120_events/down.sql b/crates/nostr-sqldb/migrations/2025-04-11-095120_events/down.sql deleted file mode 100644 index 4383998aa..000000000 --- a/crates/nostr-sqldb/migrations/2025-04-11-095120_events/down.sql +++ /dev/null @@ -1,3 +0,0 @@ --- This file should undo anything in `up.sql` -DROP TABLE nostr.event_tags; -DROP TABLE nostr.events; diff --git a/crates/nostr-sqldb/migrations/mysql/.keep b/crates/nostr-sqldb/migrations/mysql/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/down.sql b/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/down.sql new file mode 100644 index 000000000..01cb13d3d --- /dev/null +++ b/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/down.sql @@ -0,0 +1,3 @@ +-- This file should undo anything in `up.sql` +DROP TABLE IF EXISTS event_tags; +DROP TABLE IF EXISTS events; diff --git a/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/up.sql b/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/up.sql new file mode 100644 index 000000000..90cac4a08 --- /dev/null +++ b/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/up.sql @@ -0,0 +1,27 @@ +-- The actual event data +CREATE TABLE IF NOT EXISTS events ( + id VARCHAR(64) PRIMARY KEY, + pubkey VARCHAR(64) NOT NULL, + created_at BIGINT NOT NULL, + kind BIGINT NOT NULL, + payload BLOB NOT NULL, + signature VARCHAR(128) NOT NULL, + deleted BOOLEAN NOT NULL +); + +-- Direct indexes +CREATE INDEX event_pubkey ON events (pubkey); +CREATE INDEX event_date ON events (created_at); +CREATE INDEX event_kind ON events (kind); +CREATE INDEX event_deleted ON events (deleted); + +-- The tag index, the primary will give us the index automatically +CREATE TABLE IF NOT EXISTS event_tags ( + tag VARCHAR(64) NOT NULL, + tag_value VARCHAR(512) NOT NULL, + event_id VARCHAR(64) NOT NULL + REFERENCES events (id) + ON DELETE CASCADE + ON UPDATE CASCADE, + PRIMARY KEY (tag, tag_value, event_id) +); diff --git a/crates/nostr-sqldb/migrations/00000000000000_diesel_initial_setup/down.sql b/crates/nostr-sqldb/migrations/postgres/00000000000000_diesel_initial_setup/down.sql similarity index 100% rename from crates/nostr-sqldb/migrations/00000000000000_diesel_initial_setup/down.sql rename to crates/nostr-sqldb/migrations/postgres/00000000000000_diesel_initial_setup/down.sql diff --git a/crates/nostr-sqldb/migrations/00000000000000_diesel_initial_setup/up.sql b/crates/nostr-sqldb/migrations/postgres/00000000000000_diesel_initial_setup/up.sql similarity index 100% rename from crates/nostr-sqldb/migrations/00000000000000_diesel_initial_setup/up.sql rename to crates/nostr-sqldb/migrations/postgres/00000000000000_diesel_initial_setup/up.sql diff --git a/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/down.sql b/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/down.sql new file mode 100644 index 000000000..ab16f6bdf --- /dev/null +++ b/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/down.sql @@ -0,0 +1,3 @@ +-- This file should undo anything in `up.sql` +DROP TABLE event_tags; +DROP TABLE events; diff --git a/crates/nostr-sqldb/migrations/2025-04-11-095120_events/up.sql b/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/up.sql similarity index 100% rename from crates/nostr-sqldb/migrations/2025-04-11-095120_events/up.sql rename to crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/up.sql diff --git a/crates/nostr-sqldb/migrations/sqlite/.keep b/crates/nostr-sqldb/migrations/sqlite/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/down.sql b/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/down.sql new file mode 100644 index 000000000..ab16f6bdf --- /dev/null +++ b/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/down.sql @@ -0,0 +1,3 @@ +-- This file should undo anything in `up.sql` +DROP TABLE event_tags; +DROP TABLE events; diff --git a/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/up.sql b/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/up.sql new file mode 100644 index 000000000..9744e25fb --- /dev/null +++ b/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/up.sql @@ -0,0 +1,27 @@ +-- The actual event data +CREATE TABLE events ( + id VARCHAR(64) PRIMARY KEY, + pubkey VARCHAR(64) NOT NULL, + created_at BIGINT NOT NULL, + kind BIGINT NOT NULL, + payload BLOB NOT NULL, + signature VARCHAR(128) NOT NULL, + deleted BOOLEAN NOT NULL +); + +-- Direct indexes +CREATE INDEX event_pubkey ON events (pubkey); +CREATE INDEX event_date ON events (created_at); +CREATE INDEX event_kind ON events (kind); +CREATE INDEX event_deleted ON events (deleted); + +-- The tag index, the primary will give us the index automatically +CREATE TABLE event_tags ( + tag TEXT NOT NULL, + tag_value TEXT NOT NULL, + event_id VARCHAR(64) NOT NULL + REFERENCES events (id) + ON DELETE CASCADE + ON UPDATE CASCADE, + PRIMARY KEY (tag, tag_value, event_id) +); diff --git a/crates/nostr-sqldb/diesel.toml b/crates/nostr-sqldb/mysql.toml similarity index 79% rename from crates/nostr-sqldb/diesel.toml rename to crates/nostr-sqldb/mysql.toml index a0d61bf48..75175050b 100644 --- a/crates/nostr-sqldb/diesel.toml +++ b/crates/nostr-sqldb/mysql.toml @@ -2,8 +2,8 @@ # see https://diesel.rs/guides/configuring-diesel-cli [print_schema] -file = "src/schema.rs" +file = "src/schema/mysql.rs" custom_type_derives = ["diesel::query_builder::QueryId", "Clone"] [migrations_directory] -dir = "migrations" +dir = "migrations/mysql" diff --git a/crates/nostr-sqldb/postgres.toml b/crates/nostr-sqldb/postgres.toml new file mode 100644 index 000000000..285cb79ad --- /dev/null +++ b/crates/nostr-sqldb/postgres.toml @@ -0,0 +1,9 @@ +# For documentation on how to configure this file, +# see https://diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "src/schema/postgres.rs" +custom_type_derives = ["diesel::query_builder::QueryId", "Clone"] + +[migrations_directory] +dir = "migrations/postgres" diff --git a/crates/nostr-sqldb/sqlite.toml b/crates/nostr-sqldb/sqlite.toml new file mode 100644 index 000000000..e1c7f21c0 --- /dev/null +++ b/crates/nostr-sqldb/sqlite.toml @@ -0,0 +1,9 @@ +# For documentation on how to configure this file, +# see https://diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "src/schema/sqlite.rs" +custom_type_derives = ["diesel::query_builder::QueryId", "Clone"] + +[migrations_directory] +dir = "migrations/sqlite" diff --git a/crates/nostr-sqldb/src/lib.rs b/crates/nostr-sqldb/src/lib.rs index 3eca946d1..4518dcdf9 100644 --- a/crates/nostr-sqldb/src/lib.rs +++ b/crates/nostr-sqldb/src/lib.rs @@ -2,10 +2,10 @@ mod migrations; mod model; mod postgres; mod schema; +mod types; use diesel::prelude::*; use diesel_async::RunQueryDsl; -pub use migrations::run_migrations; use model::{EventDataDb, EventDb}; use nostr::event::*; use nostr::filter::Filter; @@ -13,8 +13,15 @@ use nostr::types::Timestamp; use nostr::util::BoxedFuture; use nostr_database::*; use postgres::{build_filter_query, with_limit}; + +#[cfg(feature = "postgres")] pub use postgres::{postgres_connection_pool, NostrPostgres}; -use schema::events; + +#[cfg(feature = "postgres")] +pub use migrations::postgres::run_migrations; + +#[cfg(feature = "postgres")] +use schema::postgres::{event_tags, events}; impl NostrDatabase for NostrPostgres { fn backend(&self) -> Backend { diff --git a/crates/nostr-sqldb/src/migrations/mod.rs b/crates/nostr-sqldb/src/migrations/mod.rs new file mode 100644 index 000000000..13c12d36d --- /dev/null +++ b/crates/nostr-sqldb/src/migrations/mod.rs @@ -0,0 +1,8 @@ +#[cfg(feature = "mysql")] +pub mod mysql; + +#[cfg(feature = "postgres")] +pub mod postgres; + +#[cfg(feature = "sqlite")] +pub mod sqlite; diff --git a/crates/nostr-sqldb/src/migrations/mysql.rs b/crates/nostr-sqldb/src/migrations/mysql.rs new file mode 100644 index 000000000..ae3f171c8 --- /dev/null +++ b/crates/nostr-sqldb/src/migrations/mysql.rs @@ -0,0 +1,19 @@ +use diesel::{Connection, MysqlConnection}; +use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; +use nostr_database::DatabaseError; +use tracing::info; + +const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations/mysql"); + +/// programatically run the db migrations +pub fn run_migrations(connection_string: &str) -> Result<(), DatabaseError> { + info!("Running db migrations in mysql database",); + let mut connection = + MysqlConnection::establish(connection_string).map_err(DatabaseError::backend)?; + + let res = connection + .run_pending_migrations(MIGRATIONS) + .map_err(DatabaseError::Backend)?; + info!("Successfully executed mysql db migrations {:?}", res); + Ok(()) +} diff --git a/crates/nostr-sqldb/src/migrations.rs b/crates/nostr-sqldb/src/migrations/postgres.rs similarity index 97% rename from crates/nostr-sqldb/src/migrations.rs rename to crates/nostr-sqldb/src/migrations/postgres.rs index 9d7e6f7f2..f54b32080 100644 --- a/crates/nostr-sqldb/src/migrations.rs +++ b/crates/nostr-sqldb/src/migrations/postgres.rs @@ -3,7 +3,7 @@ use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; use nostr_database::DatabaseError; use tracing::info; -const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations"); +const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations/postgres"); /// programatically run the db migrations pub fn run_migrations(connection_string: &str) -> Result<(), DatabaseError> { diff --git a/crates/nostr-sqldb/src/migrations/sqlite.rs b/crates/nostr-sqldb/src/migrations/sqlite.rs new file mode 100644 index 000000000..34a3e943d --- /dev/null +++ b/crates/nostr-sqldb/src/migrations/sqlite.rs @@ -0,0 +1,19 @@ +use diesel::{Connection, SqliteConnection}; +use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; +use nostr_database::DatabaseError; +use tracing::info; + +const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations/sqlite"); + +/// programatically run the db migrations +pub fn run_migrations(connection_string: &str) -> Result<(), DatabaseError> { + info!("Running db migrations in sqlite database",); + let mut connection = + SqliteConnection::establish(connection_string).map_err(DatabaseError::backend)?; + + let res = connection + .run_pending_migrations(MIGRATIONS) + .map_err(DatabaseError::Backend)?; + info!("Successfully executed sqlite db migrations {:?}", res); + Ok(()) +} diff --git a/crates/nostr-sqldb/src/model.rs b/crates/nostr-sqldb/src/model.rs index b749af901..9dd7c69fd 100644 --- a/crates/nostr-sqldb/src/model.rs +++ b/crates/nostr-sqldb/src/model.rs @@ -2,11 +2,18 @@ use diesel::prelude::*; use nostr::event::Event; use nostr_database::{DatabaseError, FlatBufferBuilder, FlatBufferEncode}; -use crate::schema::{event_tags, events}; +#[cfg(feature = "postgres")] +use crate::schema::postgres::{event_tags, events}; + +#[cfg(feature = "mysql")] +use crate::schema::mysql::{event_tags, events}; + +#[cfg(feature = "sqlite")] +use crate::schema::sqlite::{event_tags, events}; /// DB representation of [`Event`] #[derive(Queryable, Selectable, Insertable, AsChangeset, Debug, Clone)] -#[diesel(table_name = events, check_for_backend(diesel::pg::Pg))] +#[diesel(table_name = events)] pub struct EventDb { pub id: String, pub pubkey: String, @@ -19,7 +26,7 @@ pub struct EventDb { /// DB representation of [`EventTag`] #[derive(Queryable, Selectable, Insertable, AsChangeset, Debug, Clone)] -#[diesel(table_name = event_tags, check_for_backend(diesel::pg::Pg))] +#[diesel(table_name = event_tags)] pub struct EventTagDb { pub tag: String, pub tag_value: String, diff --git a/crates/nostr-sqldb/src/postgres.rs b/crates/nostr-sqldb/src/postgres.rs index 1fdca36f7..eca376504 100644 --- a/crates/nostr-sqldb/src/postgres.rs +++ b/crates/nostr-sqldb/src/postgres.rs @@ -12,7 +12,7 @@ use nostr::filter::Filter; use nostr_database::*; use super::model::{EventDataDb, EventDb}; -use super::schema::{event_tags, events}; +use super::schema::postgres::{event_tags, events}; /// Shorthand for a database connection pool type pub type PostgresConnectionPool = Pool>; @@ -137,7 +137,7 @@ type QuerySetJoinType<'a> = IntoBoxed< pub fn build_filter_query<'a>(filter: Filter) -> QuerySetJoinType<'a> { let mut query = events::table - .distinct_on(events::id) + .distinct() .inner_join(event_tags::table) .filter(events::deleted.eq(false)) .into_boxed(); diff --git a/crates/nostr-sqldb/src/schema/mod.rs b/crates/nostr-sqldb/src/schema/mod.rs new file mode 100644 index 000000000..13f8538b0 --- /dev/null +++ b/crates/nostr-sqldb/src/schema/mod.rs @@ -0,0 +1,8 @@ +#[cfg(feature = "postgres")] +pub mod postgres; + +#[cfg(feature = "mysql")] +pub mod mysql; + +#[cfg(feature = "sqlite")] +pub mod sqlite; diff --git a/crates/nostr-sqldb/src/schema/mysql.rs b/crates/nostr-sqldb/src/schema/mysql.rs new file mode 100644 index 000000000..3259a7c6b --- /dev/null +++ b/crates/nostr-sqldb/src/schema/mysql.rs @@ -0,0 +1,31 @@ +// @generated automatically by Diesel CLI. + +diesel::table! { + event_tags (tag, tag_value, event_id) { + #[max_length = 64] + tag -> Varchar, + #[max_length = 512] + tag_value -> Varchar, + #[max_length = 64] + event_id -> Varchar, + } +} + +diesel::table! { + events (id) { + #[max_length = 64] + id -> Varchar, + #[max_length = 64] + pubkey -> Varchar, + created_at -> Bigint, + kind -> Bigint, + payload -> Blob, + #[max_length = 128] + signature -> Varchar, + deleted -> Bool, + } +} + +diesel::joinable!(event_tags -> events (event_id)); + +diesel::allow_tables_to_appear_in_same_query!(event_tags, events,); diff --git a/crates/nostr-sqldb/src/schema.rs b/crates/nostr-sqldb/src/schema/postgres.rs similarity index 100% rename from crates/nostr-sqldb/src/schema.rs rename to crates/nostr-sqldb/src/schema/postgres.rs diff --git a/crates/nostr-sqldb/src/schema/sqlite.rs b/crates/nostr-sqldb/src/schema/sqlite.rs new file mode 100644 index 000000000..8b80828f8 --- /dev/null +++ b/crates/nostr-sqldb/src/schema/sqlite.rs @@ -0,0 +1,28 @@ +// @generated automatically by Diesel CLI. + +diesel::table! { + event_tags (tag, tag_value, event_id) { + tag -> Text, + tag_value -> Text, + event_id -> Text, + } +} + +diesel::table! { + events (id) { + id -> Nullable, + pubkey -> Text, + created_at -> BigInt, + kind -> BigInt, + payload -> Binary, + signature -> Text, + deleted -> Bool, + } +} + +diesel::joinable!(event_tags -> events (event_id)); + +diesel::allow_tables_to_appear_in_same_query!( + event_tags, + events, +); diff --git a/crates/nostr-sqldb/src/types.rs b/crates/nostr-sqldb/src/types.rs new file mode 100644 index 000000000..0f6954402 --- /dev/null +++ b/crates/nostr-sqldb/src/types.rs @@ -0,0 +1,31 @@ +use deadpool::managed::{Object, Pool}; +use diesel_async::pooled_connection::AsyncDieselConnectionManager; + +#[cfg(feature = "postgres")] +use diesel_async::AsyncPgConnection; +#[cfg(feature = "postgres")] +pub type DbConnectionPool = Pool>; +#[cfg(feature = "postgres")] +pub type DbConnection = Object>; + +#[cfg(feature = "mysql")] +use diesel_async::AsyncMysqlConnection; +#[cfg(feature = "mysql")] +pub type DbConnectionPool = Pool>; +#[cfg(feature = "mysql")] +pub type DbConnection = Object>; + +#[cfg(feature = "sqlite")] +use diesel::sqlite::SqliteConnection; +#[cfg(feature = "sqlite")] +use diesel_async::sync_connection_wrapper::SyncConnectionWrapper; +#[cfg(feature = "sqlite")] +pub type DbConnection = SyncConnectionWrapper; + +// #[cfg(feature = "sqlite")] +// async fn get_connection() -> Result, DatabaseError> { +// let mut conn = SyncConnectionWrapper::::establish(&database_url) +// .await +// .unwrap(); +// pool.get().await.map_err(DatabaseError::backend) +// } From 68ed5b30a8e784529fb8e09df5be86395eb87072 Mon Sep 17 00:00:00 2001 From: tompro Date: Wed, 23 Apr 2025 18:08:44 +0200 Subject: [PATCH 08/12] Generic schema and migrations, separate postgres --- crates/nostr-sqldb/src/lib.rs | 140 +--------------- crates/nostr-sqldb/src/model.rs | 6 - crates/nostr-sqldb/src/postgres.rs | 202 ++++++++++++++---------- crates/nostr-sqldb/src/query.rs | 117 ++++++++++++++ crates/nostr-sqldb/src/schema/sqlite.rs | 5 +- crates/nostr-sqldb/src/types.rs | 31 ---- 6 files changed, 244 insertions(+), 257 deletions(-) create mode 100644 crates/nostr-sqldb/src/query.rs delete mode 100644 crates/nostr-sqldb/src/types.rs diff --git a/crates/nostr-sqldb/src/lib.rs b/crates/nostr-sqldb/src/lib.rs index 4518dcdf9..c4dddb18e 100644 --- a/crates/nostr-sqldb/src/lib.rs +++ b/crates/nostr-sqldb/src/lib.rs @@ -1,142 +1,16 @@ mod migrations; mod model; -mod postgres; +mod query; mod schema; -mod types; - -use diesel::prelude::*; -use diesel_async::RunQueryDsl; -use model::{EventDataDb, EventDb}; -use nostr::event::*; -use nostr::filter::Filter; -use nostr::types::Timestamp; -use nostr::util::BoxedFuture; -use nostr_database::*; -use postgres::{build_filter_query, with_limit}; #[cfg(feature = "postgres")] -pub use postgres::{postgres_connection_pool, NostrPostgres}; +mod postgres; +#[cfg(feature = "mysql")] +pub use migrations::mysql::run_migrations; #[cfg(feature = "postgres")] pub use migrations::postgres::run_migrations; - +#[cfg(feature = "sqlite")] +pub use migrations::sqlite::run_migrations; #[cfg(feature = "postgres")] -use schema::postgres::{event_tags, events}; - -impl NostrDatabase for NostrPostgres { - fn backend(&self) -> Backend { - Backend::Custom("Postgres".to_string()) - } -} - -impl NostrEventsDatabase for NostrPostgres { - /// Save [`Event`] into store - /// - /// **This method assumes that [`Event`] was already verified** - fn save_event<'a>( - &'a self, - event: &'a Event, - ) -> BoxedFuture<'a, Result> { - Box::pin(async move { self.save(EventDataDb::try_from(event)?).await }) - } - - /// Check event status by ID - /// - /// Check if the event is saved, deleted or not existent. - fn check_id<'a>( - &'a self, - event_id: &'a EventId, - ) -> BoxedFuture<'a, Result> { - Box::pin(async move { - let status = match self.event_by_id(event_id).await? { - Some(e) if e.deleted => DatabaseEventStatus::Deleted, - Some(_) => DatabaseEventStatus::Saved, - None => DatabaseEventStatus::NotExistent, - }; - Ok(status) - }) - } - - /// Coordinate feature is not supported yet - fn has_coordinate_been_deleted<'a>( - &'a self, - _coordinate: &'a nostr::nips::nip01::CoordinateBorrow<'a>, - _timestamp: &'a Timestamp, - ) -> BoxedFuture<'a, Result> { - Box::pin(async move { Ok(false) }) - } - - /// Get [`Event`] by [`EventId`] - fn event_by_id<'a>( - &'a self, - _event_id: &'a EventId, - ) -> BoxedFuture<'a, Result, DatabaseError>> { - Box::pin(async move { - let event = match self.event_by_id(_event_id).await? { - Some(e) if !e.deleted => { - Some(Event::decode(&e.payload).map_err(DatabaseError::backend)?) - } - _ => None, - }; - Ok(event) - }) - } - - /// Count the number of events found with [`Filter`]. - /// - /// Use `Filter::new()` or `Filter::default()` to count all events. - fn count(&self, filter: Filter) -> BoxedFuture> { - Box::pin(async move { - let res: i64 = build_filter_query(filter) - .count() - .get_result(&mut self.get_connection().await?) - .await - .map_err(DatabaseError::backend)?; - Ok(res as usize) - }) - } - - /// Query stored events. - fn query(&self, filter: Filter) -> BoxedFuture> { - let filter = with_limit(filter, 10000); - Box::pin(async move { - let mut events = Events::new(&filter); - let result = build_filter_query(filter.clone()) - .select(EventDb::as_select()) - .load(&mut self.get_connection().await?) - .await - .map_err(DatabaseError::backend)?; - - for item in result.into_iter() { - if let Ok(event) = Event::decode(&item.payload) { - events.insert(event); - } - } - Ok(events) - }) - } - - /// Delete all events that match the [Filter] - fn delete(&self, filter: Filter) -> BoxedFuture> { - let filter = with_limit(filter, 999); - Box::pin(async move { - let filter = build_filter_query(filter); - diesel::update(events::table) - .set(events::deleted.eq(true)) - .filter(events::id.eq_any(filter.select(events::id))) - .execute(&mut self.get_connection().await?) - .await - .map_err(DatabaseError::backend)?; - - Ok(()) - }) - } -} - -/// For now we want to avoid wiping the database -impl NostrDatabaseWipe for NostrPostgres { - #[inline] - fn wipe(&self) -> BoxedFuture> { - Box::pin(async move { Err(DatabaseError::NotSupported) }) - } -} +pub use postgres::{postgres_connection_pool, NostrPostgres}; diff --git a/crates/nostr-sqldb/src/model.rs b/crates/nostr-sqldb/src/model.rs index 9dd7c69fd..0e888bb70 100644 --- a/crates/nostr-sqldb/src/model.rs +++ b/crates/nostr-sqldb/src/model.rs @@ -5,12 +5,6 @@ use nostr_database::{DatabaseError, FlatBufferBuilder, FlatBufferEncode}; #[cfg(feature = "postgres")] use crate::schema::postgres::{event_tags, events}; -#[cfg(feature = "mysql")] -use crate::schema::mysql::{event_tags, events}; - -#[cfg(feature = "sqlite")] -use crate::schema::sqlite::{event_tags, events}; - /// DB representation of [`Event`] #[derive(Queryable, Selectable, Insertable, AsChangeset, Debug, Clone)] #[diesel(table_name = events)] diff --git a/crates/nostr-sqldb/src/postgres.rs b/crates/nostr-sqldb/src/postgres.rs index eca376504..b1d78c278 100644 --- a/crates/nostr-sqldb/src/postgres.rs +++ b/crates/nostr-sqldb/src/postgres.rs @@ -1,6 +1,4 @@ use deadpool::managed::{Object, Pool}; -use diesel::dsl::{Eq, Filter as DieselFilter, InnerJoin, IntoBoxed}; -use diesel::pg::Pg; use diesel::prelude::*; use diesel::result::{DatabaseErrorKind, Error as DieselError}; use diesel::QueryResult; @@ -9,10 +7,13 @@ use diesel_async::scoped_futures::ScopedFutureExt; use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl}; use nostr::event::*; use nostr::filter::Filter; +use nostr::types::Timestamp; use nostr_database::*; +use prelude::BoxedFuture; use super::model::{EventDataDb, EventDb}; use super::schema::postgres::{event_tags, events}; +use crate::query::{build_filter_query, event_by_id, with_limit}; /// Shorthand for a database connection pool type pub type PostgresConnectionPool = Pool>; @@ -76,10 +77,7 @@ impl NostrPostgres { &self, event_id: &EventId, ) -> Result, DatabaseError> { - let event_id = event_id.to_hex(); - let res = events::table - .select(EventDb::as_select()) - .filter(events::id.eq(event_id)) + let res = event_by_id(event_id) .first(&mut self.get_connection().await?) .await .optional() @@ -88,6 +86,116 @@ impl NostrPostgres { } } +impl NostrEventsDatabase for NostrPostgres { + /// Save [`Event`] into store + /// + /// **This method assumes that [`Event`] was already verified** + fn save_event<'a>( + &'a self, + event: &'a Event, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { self.save(EventDataDb::try_from(event)?).await }) + } + + /// Check event status by ID + /// + /// Check if the event is saved, deleted or not existent. + fn check_id<'a>( + &'a self, + event_id: &'a EventId, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { + let status = match self.event_by_id(event_id).await? { + Some(e) if e.deleted => DatabaseEventStatus::Deleted, + Some(_) => DatabaseEventStatus::Saved, + None => DatabaseEventStatus::NotExistent, + }; + Ok(status) + }) + } + + /// Coordinate feature is not supported yet + fn has_coordinate_been_deleted<'a>( + &'a self, + _coordinate: &'a nostr::nips::nip01::CoordinateBorrow<'a>, + _timestamp: &'a Timestamp, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { Ok(false) }) + } + + /// Get [`Event`] by [`EventId`] + fn event_by_id<'a>( + &'a self, + event_id: &'a EventId, + ) -> BoxedFuture<'a, Result, DatabaseError>> { + Box::pin(async move { + let event = match self.event_by_id(event_id).await? { + Some(e) if !e.deleted => { + Some(Event::decode(&e.payload).map_err(DatabaseError::backend)?) + } + _ => None, + }; + Ok(event) + }) + } + + /// Count the number of events found with [`Filter`]. + /// + /// Use `Filter::new()` or `Filter::default()` to count all events. + fn count(&self, filter: Filter) -> BoxedFuture> { + Box::pin(async move { + let res: i64 = build_filter_query(filter) + .count() + .get_result(&mut self.get_connection().await?) + .await + .map_err(DatabaseError::backend)?; + Ok(res as usize) + }) + } + + /// Query stored events. + fn query(&self, filter: Filter) -> BoxedFuture> { + let filter = with_limit(filter, 10000); + Box::pin(async move { + let mut events = Events::new(&filter); + let result = build_filter_query(filter.clone()) + .select(EventDb::as_select()) + .load(&mut self.get_connection().await?) + .await + .map_err(DatabaseError::backend)?; + + for item in result.into_iter() { + if let Ok(event) = Event::decode(&item.payload) { + events.insert(event); + } + } + Ok(events) + }) + } + + /// Delete all events that match the [Filter] + fn delete(&self, filter: Filter) -> BoxedFuture> { + let filter = with_limit(filter, 999); + Box::pin(async move { + let filter = build_filter_query(filter); + diesel::update(events::table) + .set(events::deleted.eq(true)) + .filter(events::id.eq_any(filter.select(events::id))) + .execute(&mut self.get_connection().await?) + .await + .map_err(DatabaseError::backend)?; + + Ok(()) + }) + } +} + +impl NostrDatabase for NostrPostgres { + fn backend(&self) -> Backend { + Backend::Custom("Postgres".to_string()) + } +} + /// Create a new [`NostrPostgres`] instance from an existing connection pool impl From for NostrPostgres { fn from(pool: PostgresConnectionPool) -> Self { @@ -117,82 +225,10 @@ impl std::fmt::Debug for NostrPostgres { } } -/// sets the given default limit on a Nostr filter if not set -pub fn with_limit(filter: Filter, default_limit: usize) -> Filter { - if filter.limit.is_none() { - return filter.limit(default_limit); +/// For now we want to avoid wiping the database +impl NostrDatabaseWipe for NostrPostgres { + #[inline] + fn wipe(&self) -> BoxedFuture> { + Box::pin(async move { Err(DatabaseError::NotSupported) }) } - filter -} - -// filter type of a join query. -type QuerySetJoinType<'a> = IntoBoxed< - 'a, - DieselFilter< - InnerJoin, - Eq>, - >, - Pg, ->; - -pub fn build_filter_query<'a>(filter: Filter) -> QuerySetJoinType<'a> { - let mut query = events::table - .distinct() - .inner_join(event_tags::table) - .filter(events::deleted.eq(false)) - .into_boxed(); - - if let Some(limit) = filter.limit { - query = query.limit(limit as i64); - } - - if !has_filters(&filter) { - return query; - } - if let Some(ids) = filter.ids.clone() { - let values = ids.iter().map(|id| id.to_hex()).collect::>(); - query = query.filter(events::id.eq_any(values)); - } - - if let Some(authors) = filter.authors.clone() { - let values = authors.iter().map(|a| a.to_hex()).collect::>(); - query = query.filter(events::pubkey.eq_any(values)); - } - - if let Some(kinds) = filter.kinds.clone() { - let values = kinds.iter().map(|k| k.as_u16() as i64).collect::>(); - query = query.filter(events::kind.eq_any(values)); - } - - if let Some(since) = filter.since { - query = query.filter(events::created_at.ge(since.as_u64() as i64)); - } - - if let Some(until) = filter.until { - query = query.filter(events::created_at.le(until.as_u64() as i64)); - } - - if !filter.generic_tags.is_empty() { - for (tag, values) in filter.generic_tags.into_iter() { - let values = values.iter().map(|v| v.to_string()).collect::>(); - query = query.filter( - event_tags::tag - .eq(tag.to_string()) - .and(event_tags::tag_value.eq_any(values)), - ); - } - } - - query -} - -// determine if the filter has any filters set -fn has_filters(filter: &Filter) -> bool { - filter.ids.is_some() - || filter.authors.is_some() - || filter.kinds.is_some() - || filter.since.is_some() - || filter.until.is_some() - || !filter.generic_tags.is_empty() - || filter.limit.is_some() } diff --git a/crates/nostr-sqldb/src/query.rs b/crates/nostr-sqldb/src/query.rs new file mode 100644 index 000000000..183f3493d --- /dev/null +++ b/crates/nostr-sqldb/src/query.rs @@ -0,0 +1,117 @@ +use diesel::dsl::{AsSelect, Eq, Filter as DieselFilter, InnerJoin, IntoBoxed, SqlTypeOf}; +use diesel::prelude::*; +use nostr::event::*; +use nostr::filter::Filter; +use nostr_database::*; + +use super::model::EventDb; +#[cfg(feature = "mysql")] +use super::schema::mysql::{event_tags, events}; +#[cfg(feature = "postgres")] +use super::schema::postgres::{event_tags, events}; +#[cfg(feature = "sqlite")] +use super::schema::sqlite::{event_tags, events}; + +// filter type of a join query. +type QuerySetJoinTypeDb<'a, DB> = IntoBoxed< + 'a, + DieselFilter< + InnerJoin, + Eq>, + >, + DB, +>; +type SelectEventTypeDb = SqlTypeOf>; +type BoxedEventQueryDb<'a, DB> = events::BoxedQuery<'a, DB, SelectEventTypeDb>; + +#[cfg(feature = "postgres")] +type QuerySetJoinType<'a> = QuerySetJoinTypeDb<'a, diesel::pg::Pg>; +#[cfg(feature = "postgres")] +type BoxedEventQuery<'a> = BoxedEventQueryDb<'a, diesel::pg::Pg>; +#[cfg(feature = "sqlite")] +type QuerySetJoinType<'a> = QuerySetJoinTypeDb<'a, diesel::sqlite::Sqlite>; +#[cfg(feature = "sqlite")] +type BoxedEventQuery<'a> = BoxedEventQueryDb<'a, diesel::sqlite::Sqlite>; +#[cfg(feature = "mysql")] +type QuerySetJoinType<'a> = QuerySetJoinTypeDb<'a, diesel::mysql::Mysql>; +#[cfg(feature = "mysql")] +type BoxedEventQuery<'a> = BoxedEventQueryDb<'a, diesel::mysql::Mysql>; + +pub fn build_filter_query<'a>(filter: Filter) -> QuerySetJoinType<'a> { + let mut query = events::table + .distinct() + .inner_join(event_tags::table) + .filter(events::deleted.eq(false)) + .order_by(events::created_at.desc()) + .into_boxed(); + + if let Some(limit) = filter.limit { + query = query.limit(limit as i64); + } + + if !has_filters(&filter) { + return query; + } + if let Some(ids) = filter.ids.clone() { + let values = ids.iter().map(|id| id.to_hex()).collect::>(); + query = query.filter(events::id.eq_any(values)); + } + + if let Some(authors) = filter.authors.clone() { + let values = authors.iter().map(|a| a.to_hex()).collect::>(); + query = query.filter(events::pubkey.eq_any(values)); + } + + if let Some(kinds) = filter.kinds.clone() { + let values = kinds.iter().map(|k| k.as_u16() as i64).collect::>(); + query = query.filter(events::kind.eq_any(values)); + } + + if let Some(since) = filter.since { + query = query.filter(events::created_at.ge(since.as_u64() as i64)); + } + + if let Some(until) = filter.until { + query = query.filter(events::created_at.le(until.as_u64() as i64)); + } + + if !filter.generic_tags.is_empty() { + for (tag, values) in filter.generic_tags.into_iter() { + let values = values.iter().map(|v| v.to_string()).collect::>(); + query = query.filter( + event_tags::tag + .eq(tag.to_string()) + .and(event_tags::tag_value.eq_any(values)), + ); + } + } + + query +} + +/// sets the given default limit on a Nostr filter if not set +pub fn with_limit(filter: Filter, default_limit: usize) -> Filter { + if filter.limit.is_none() { + return filter.limit(default_limit); + } + filter +} + +pub fn event_by_id<'a>(event_id: &EventId) -> BoxedEventQuery<'a> { + let event_id = event_id.to_hex(); + events::table + .select(EventDb::as_select()) + .filter(events::id.eq(event_id)) + .into_boxed() +} + +// determine if the filter has any filters set +fn has_filters(filter: &Filter) -> bool { + filter.ids.is_some() + || filter.authors.is_some() + || filter.kinds.is_some() + || filter.since.is_some() + || filter.until.is_some() + || !filter.generic_tags.is_empty() + || filter.limit.is_some() +} diff --git a/crates/nostr-sqldb/src/schema/sqlite.rs b/crates/nostr-sqldb/src/schema/sqlite.rs index 8b80828f8..7e806221b 100644 --- a/crates/nostr-sqldb/src/schema/sqlite.rs +++ b/crates/nostr-sqldb/src/schema/sqlite.rs @@ -22,7 +22,4 @@ diesel::table! { diesel::joinable!(event_tags -> events (event_id)); -diesel::allow_tables_to_appear_in_same_query!( - event_tags, - events, -); +diesel::allow_tables_to_appear_in_same_query!(event_tags, events,); diff --git a/crates/nostr-sqldb/src/types.rs b/crates/nostr-sqldb/src/types.rs deleted file mode 100644 index 0f6954402..000000000 --- a/crates/nostr-sqldb/src/types.rs +++ /dev/null @@ -1,31 +0,0 @@ -use deadpool::managed::{Object, Pool}; -use diesel_async::pooled_connection::AsyncDieselConnectionManager; - -#[cfg(feature = "postgres")] -use diesel_async::AsyncPgConnection; -#[cfg(feature = "postgres")] -pub type DbConnectionPool = Pool>; -#[cfg(feature = "postgres")] -pub type DbConnection = Object>; - -#[cfg(feature = "mysql")] -use diesel_async::AsyncMysqlConnection; -#[cfg(feature = "mysql")] -pub type DbConnectionPool = Pool>; -#[cfg(feature = "mysql")] -pub type DbConnection = Object>; - -#[cfg(feature = "sqlite")] -use diesel::sqlite::SqliteConnection; -#[cfg(feature = "sqlite")] -use diesel_async::sync_connection_wrapper::SyncConnectionWrapper; -#[cfg(feature = "sqlite")] -pub type DbConnection = SyncConnectionWrapper; - -// #[cfg(feature = "sqlite")] -// async fn get_connection() -> Result, DatabaseError> { -// let mut conn = SyncConnectionWrapper::::establish(&database_url) -// .await -// .unwrap(); -// pool.get().await.map_err(DatabaseError::backend) -// } From 0bbb01dbe91e267f2966d61bbe085cdac7abccee Mon Sep 17 00:00:00 2001 From: tompro Date: Wed, 23 Apr 2025 18:15:00 +0200 Subject: [PATCH 09/12] Cargo name --- Cargo.lock | 32 ++++++++++++++++---------------- crates/nostr-sqldb/Cargo.toml | 6 +++--- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d030be929..bd13ed6d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3124,22 +3124,6 @@ dependencies = [ "ureq", ] -[[package]] -name = "nostr-postgresdb" -version = "0.41.0" -dependencies = [ - "deadpool", - "diesel", - "diesel-async", - "diesel_migrations", - "nostr", - "nostr-database", - "nostr-relay-builder", - "tokio", - "tracing", - "tracing-subscriber", -] - [[package]] name = "nostr-relay-builder" version = "0.41.0" @@ -3193,6 +3177,22 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "nostr-sqldb" +version = "0.41.0" +dependencies = [ + "deadpool", + "diesel", + "diesel-async", + "diesel_migrations", + "nostr", + "nostr-database", + "nostr-relay-builder", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "nostrdb" version = "0.6.1" diff --git a/crates/nostr-sqldb/Cargo.toml b/crates/nostr-sqldb/Cargo.toml index 6a0de30ab..2406dfa2e 100644 --- a/crates/nostr-sqldb/Cargo.toml +++ b/crates/nostr-sqldb/Cargo.toml @@ -1,15 +1,15 @@ [package] -name = "nostr-postgresdb" +name = "nostr-sqldb" version = "0.41.0" edition = "2021" -description = "Postgres storage backend for Nostr apps" +description = "SQL storage backend for Nostr apps" authors.workspace = true homepage.workspace = true repository.workspace = true license.workspace = true readme = "README.md" rust-version.workspace = true -keywords = ["nostr", "database", "postgres"] +keywords = ["nostr", "database", "postgres", "mysql", "sqlite"] [dependencies] nostr = { workspace = true, features = ["std"] } From 81009ce3295f5ae5a5cede5782e2441bc80d2e06 Mon Sep 17 00:00:00 2001 From: tompro Date: Wed, 23 Apr 2025 18:40:50 +0200 Subject: [PATCH 10/12] Fix example --- crates/nostr-sqldb/examples/postgres-relay.rs | 12 +++--------- crates/nostr-sqldb/src/postgres.rs | 1 + 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/crates/nostr-sqldb/examples/postgres-relay.rs b/crates/nostr-sqldb/examples/postgres-relay.rs index a52431b6f..72fbfef91 100644 --- a/crates/nostr-sqldb/examples/postgres-relay.rs +++ b/crates/nostr-sqldb/examples/postgres-relay.rs @@ -4,8 +4,8 @@ use std::time::Duration; use nostr_database::prelude::*; -use nostr_postgresdb::NostrPostgres; use nostr_relay_builder::prelude::*; +use nostr_sqldb::NostrPostgres; // Your database URL const DB_URL: &str = "postgres://postgres:password@localhost:5432"; @@ -14,14 +14,8 @@ const DB_URL: &str = "postgres://postgres:password@localhost:5432"; async fn main() -> Result<()> { tracing_subscriber::fmt::init(); - // This will programatically run pending db migrations - nostr_postgresdb::run_migrations(DB_URL)?; - - // Create a conncetion pool - let pool = nostr_postgresdb::postgres_connection_pool(DB_URL).await?; - - // Create a nostr db instance - let db: NostrPostgres = pool.into(); + // Create a nostr db instance and run pending db migrations if any + let db = NostrPostgres::new(DB_URL).await?; // Add db to builder let builder = RelayBuilder::default().database(db); diff --git a/crates/nostr-sqldb/src/postgres.rs b/crates/nostr-sqldb/src/postgres.rs index b1d78c278..257ae49d3 100644 --- a/crates/nostr-sqldb/src/postgres.rs +++ b/crates/nostr-sqldb/src/postgres.rs @@ -30,6 +30,7 @@ impl NostrPostgres { where C: AsRef, { + crate::migrations::postgres::run_migrations(connection_string.as_ref())?; let pool = postgres_connection_pool(connection_string).await?; Ok(Self { pool }) } From bd86247c93741b368e638787605dd5437181bab2 Mon Sep 17 00:00:00 2001 From: tompro Date: Fri, 25 Apr 2025 10:31:12 +0200 Subject: [PATCH 11/12] Reuse FlatBufferBuilder --- crates/nostr-sqldb/src/model.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/nostr-sqldb/src/model.rs b/crates/nostr-sqldb/src/model.rs index 0e888bb70..bdd6e1e0d 100644 --- a/crates/nostr-sqldb/src/model.rs +++ b/crates/nostr-sqldb/src/model.rs @@ -1,3 +1,5 @@ +use std::sync::{Mutex, OnceLock}; + use diesel::prelude::*; use nostr::event::Event; use nostr_database::{DatabaseError, FlatBufferBuilder, FlatBufferEncode}; @@ -37,14 +39,13 @@ pub struct EventDataDb { impl TryFrom<&Event> for EventDataDb { type Error = DatabaseError; fn try_from(value: &Event) -> Result { - let serialized = value.encode(&mut FlatBufferBuilder::new()).to_vec(); Ok(Self { event: EventDb { id: value.id.to_string(), pubkey: value.pubkey.to_string(), created_at: value.created_at.as_u64() as i64, kind: value.kind.as_u16() as i64, - payload: serialized, + payload: encode_payload(value), signature: value.sig.to_string(), deleted: false, }, @@ -53,6 +54,20 @@ impl TryFrom<&Event> for EventDataDb { } } +fn encode_payload(value: &Event) -> Vec { + static FB_BUILDER: OnceLock> = OnceLock::new(); + match FB_BUILDER + .get_or_init(|| Mutex::new(FlatBufferBuilder::new())) + .lock() + { + Ok(mut fb_builder) => { + fb_builder.reset(); + value.encode(&mut fb_builder).to_vec() + } + Err(_) => value.encode(&mut FlatBufferBuilder::new()).to_vec(), + } +} + fn extract_tags(event: &Event) -> Vec { event .tags From 84d1bfb0d981da82146ff6f51b88d6f7ed9aeb9c Mon Sep 17 00:00:00 2001 From: tompro Date: Tue, 29 Apr 2025 17:07:21 +0200 Subject: [PATCH 12/12] Remove signature, use binary columns --- crates/nostr-sqldb/README.md | 4 ++-- .../mysql/2025-04-11-095120_events/up.sql | 1 - .../postgres/2025-04-11-095120_events/up.sql | 7 +++---- .../sqlite/2025-04-11-095120_events/up.sql | 1 - crates/nostr-sqldb/src/model.rs | 14 ++++++-------- crates/nostr-sqldb/src/query.rs | 14 ++++++++++---- crates/nostr-sqldb/src/schema/mysql.rs | 2 -- crates/nostr-sqldb/src/schema/postgres.rs | 11 +++-------- crates/nostr-sqldb/src/schema/sqlite.rs | 1 - 9 files changed, 24 insertions(+), 31 deletions(-) diff --git a/crates/nostr-sqldb/README.md b/crates/nostr-sqldb/README.md index 8ba6786ed..54543c63f 100644 --- a/crates/nostr-sqldb/README.md +++ b/crates/nostr-sqldb/README.md @@ -1,6 +1,6 @@ -# Nostr Postgres database backend +# Nostr SQL database backend -Postgres storage backend for nostr apps +SQL storage backend for nostr apps working with Postgres, SQLite and MySQL. ## State diff --git a/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/up.sql b/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/up.sql index 90cac4a08..5cb675f30 100644 --- a/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/up.sql +++ b/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/up.sql @@ -5,7 +5,6 @@ CREATE TABLE IF NOT EXISTS events ( created_at BIGINT NOT NULL, kind BIGINT NOT NULL, payload BLOB NOT NULL, - signature VARCHAR(128) NOT NULL, deleted BOOLEAN NOT NULL ); diff --git a/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/up.sql b/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/up.sql index 3f3b7c090..a0acea534 100644 --- a/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/up.sql +++ b/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/up.sql @@ -1,11 +1,10 @@ -- The actual event data CREATE TABLE events ( - id VARCHAR(64) PRIMARY KEY, - pubkey VARCHAR(64) NOT NULL, + id BYTEA PRIMARY KEY, + pubkey BYTEA NOT NULL, created_at BIGINT NOT NULL, kind BIGINT NOT NULL, payload BYTEA NOT NULL, - signature VARCHAR(128) NOT NULL, deleted BOOLEAN NOT NULL ); @@ -19,7 +18,7 @@ CREATE INDEX event_deleted ON events (deleted); CREATE TABLE event_tags ( tag TEXT NOT NULL, tag_value TEXT NOT NULL, - event_id VARCHAR(64) NOT NULL + event_id BYTEA NOT NULL REFERENCES events (id) ON DELETE CASCADE ON UPDATE CASCADE, diff --git a/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/up.sql b/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/up.sql index 9744e25fb..4fe621738 100644 --- a/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/up.sql +++ b/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/up.sql @@ -5,7 +5,6 @@ CREATE TABLE events ( created_at BIGINT NOT NULL, kind BIGINT NOT NULL, payload BLOB NOT NULL, - signature VARCHAR(128) NOT NULL, deleted BOOLEAN NOT NULL ); diff --git a/crates/nostr-sqldb/src/model.rs b/crates/nostr-sqldb/src/model.rs index bdd6e1e0d..735107a77 100644 --- a/crates/nostr-sqldb/src/model.rs +++ b/crates/nostr-sqldb/src/model.rs @@ -11,12 +11,11 @@ use crate::schema::postgres::{event_tags, events}; #[derive(Queryable, Selectable, Insertable, AsChangeset, Debug, Clone)] #[diesel(table_name = events)] pub struct EventDb { - pub id: String, - pub pubkey: String, + pub id: Vec, + pub pubkey: Vec, pub created_at: i64, pub kind: i64, pub payload: Vec, - pub signature: String, pub deleted: bool, } @@ -26,7 +25,7 @@ pub struct EventDb { pub struct EventTagDb { pub tag: String, pub tag_value: String, - pub event_id: String, + pub event_id: Vec, } /// A data container for extracting data from [`Event`] and its tags @@ -41,12 +40,11 @@ impl TryFrom<&Event> for EventDataDb { fn try_from(value: &Event) -> Result { Ok(Self { event: EventDb { - id: value.id.to_string(), - pubkey: value.pubkey.to_string(), + id: value.id.as_bytes().to_vec(), + pubkey: value.pubkey.as_bytes().to_vec(), created_at: value.created_at.as_u64() as i64, kind: value.kind.as_u16() as i64, payload: encode_payload(value), - signature: value.sig.to_string(), deleted: false, }, tags: extract_tags(value), @@ -77,7 +75,7 @@ fn extract_tags(event: &Event) -> Vec { Some(EventTagDb { tag: kind.to_string(), tag_value: content.to_string(), - event_id: event.id.to_string(), + event_id: event.id.as_bytes().to_vec(), }) } else { None diff --git a/crates/nostr-sqldb/src/query.rs b/crates/nostr-sqldb/src/query.rs index 183f3493d..0f795eb7c 100644 --- a/crates/nostr-sqldb/src/query.rs +++ b/crates/nostr-sqldb/src/query.rs @@ -17,7 +17,7 @@ type QuerySetJoinTypeDb<'a, DB> = IntoBoxed< 'a, DieselFilter< InnerJoin, - Eq>, + Eq>, >, DB, >; @@ -53,12 +53,18 @@ pub fn build_filter_query<'a>(filter: Filter) -> QuerySetJoinType<'a> { return query; } if let Some(ids) = filter.ids.clone() { - let values = ids.iter().map(|id| id.to_hex()).collect::>(); + let values = ids + .iter() + .map(|id| id.as_bytes().to_vec()) + .collect::>(); query = query.filter(events::id.eq_any(values)); } if let Some(authors) = filter.authors.clone() { - let values = authors.iter().map(|a| a.to_hex()).collect::>(); + let values = authors + .iter() + .map(|a| a.as_bytes().to_vec()) + .collect::>(); query = query.filter(events::pubkey.eq_any(values)); } @@ -98,7 +104,7 @@ pub fn with_limit(filter: Filter, default_limit: usize) -> Filter { } pub fn event_by_id<'a>(event_id: &EventId) -> BoxedEventQuery<'a> { - let event_id = event_id.to_hex(); + let event_id = event_id.as_bytes().to_vec(); events::table .select(EventDb::as_select()) .filter(events::id.eq(event_id)) diff --git a/crates/nostr-sqldb/src/schema/mysql.rs b/crates/nostr-sqldb/src/schema/mysql.rs index 3259a7c6b..4a0d56e4a 100644 --- a/crates/nostr-sqldb/src/schema/mysql.rs +++ b/crates/nostr-sqldb/src/schema/mysql.rs @@ -20,8 +20,6 @@ diesel::table! { created_at -> Bigint, kind -> Bigint, payload -> Blob, - #[max_length = 128] - signature -> Varchar, deleted -> Bool, } } diff --git a/crates/nostr-sqldb/src/schema/postgres.rs b/crates/nostr-sqldb/src/schema/postgres.rs index 8cbb360e3..bf991b16b 100644 --- a/crates/nostr-sqldb/src/schema/postgres.rs +++ b/crates/nostr-sqldb/src/schema/postgres.rs @@ -4,22 +4,17 @@ diesel::table! { event_tags (tag, tag_value, event_id) { tag -> Text, tag_value -> Text, - #[max_length = 64] - event_id -> Varchar, + event_id -> Bytea, } } diesel::table! { events (id) { - #[max_length = 64] - id -> Varchar, - #[max_length = 64] - pubkey -> Varchar, + id -> Bytea, + pubkey -> Bytea, created_at -> Int8, kind -> Int8, payload -> Bytea, - #[max_length = 128] - signature -> Varchar, deleted -> Bool, } } diff --git a/crates/nostr-sqldb/src/schema/sqlite.rs b/crates/nostr-sqldb/src/schema/sqlite.rs index 7e806221b..49ed89d26 100644 --- a/crates/nostr-sqldb/src/schema/sqlite.rs +++ b/crates/nostr-sqldb/src/schema/sqlite.rs @@ -15,7 +15,6 @@ diesel::table! { created_at -> BigInt, kind -> BigInt, payload -> Binary, - signature -> Text, deleted -> Bool, } }