diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1302e10b..15d2bfe09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,7 @@ jobs: - nostr-database - nostr-gossip - nostr-gossip-memory + - nostr-gossip-sqlite - nostr-gossip-test-suite - nostr-lmdb - nostr-indexeddb --target wasm32-unknown-unknown diff --git a/Cargo.lock b/Cargo.lock index 91184a4dc..f0679973d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,6 +72,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "amplify" version = "4.6.1" @@ -366,6 +372,15 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic" version = "0.5.3" @@ -868,6 +883,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.4.2" @@ -1316,6 +1346,12 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "downcast-rs" version = "2.0.1" @@ -1394,6 +1430,9 @@ name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +dependencies = [ + "serde", +] [[package]] name = "elliptic-curve" @@ -1462,6 +1501,17 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "event-listener" version = "5.3.1" @@ -1598,6 +1648,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1702,6 +1758,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -1864,6 +1931,17 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashbrown" version = "0.16.0" @@ -1879,6 +1957,15 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "heck" version = "0.4.1" @@ -2453,6 +2540,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ + "bindgen", "cc", "pkg-config", "vcpkg", @@ -2569,6 +2657,16 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.4" @@ -2782,6 +2880,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "nostr-gossip-sqlite" +version = "0.44.0" +dependencies = [ + "nostr", + "nostr-gossip", + "nostr-gossip-test-suite", + "sqlx", + "tempfile", +] + [[package]] name = "nostr-gossip-test-suite" version = "0.1.0" @@ -3797,7 +3906,7 @@ dependencies = [ "bitflags 2.9.0", "fallible-iterator", "fallible-streaming-iterator", - "hashlink", + "hashlink 0.9.1", "libsqlite3-sys", "smallvec", "time", @@ -4290,6 +4399,9 @@ name = "smallvec" version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +dependencies = [ + "serde", +] [[package]] name = "socket2" @@ -4326,6 +4438,194 @@ dependencies = [ "der", ] +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink 0.10.0", + "indexmap 2.12.0", + "log", + "memchr", + "once_cell", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.8", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.90", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.90", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.9.0", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.8", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.9.0", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.8", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror 2.0.8", + "tracing", + "url", +] + [[package]] name = "ssh-cipher" version = "0.2.0" @@ -4373,6 +4673,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.10.0" @@ -4651,6 +4962,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.26.1" @@ -5764,6 +6086,7 @@ version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -5914,6 +6237,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -6087,6 +6416,12 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasix" version = "0.12.21" @@ -6230,6 +6565,16 @@ dependencies = [ "rustix 0.38.44", ] +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index ef296b258..b978fcdba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ # Gossip "gossip/nostr-gossip", "gossip/nostr-gossip-memory", + "gossip/nostr-gossip-sqlite", "gossip/nostr-gossip-test-suite", # Remote File Storage implementations @@ -46,6 +47,7 @@ nostr-connect = { version = "0.44", path = "./signer/nostr-connect", default-fea nostr-database = { version = "0.44", path = "./database/nostr-database", default-features = false } nostr-gossip = { version = "0.44", path = "./gossip/nostr-gossip", default-features = false } nostr-gossip-memory = { version = "0.44", path = "./gossip/nostr-gossip-memory", default-features = false } +nostr-gossip-sqlite = { version = "0.44", path = "./gossip/nostr-gossip-sqlite", default-features = false } nostr-gossip-test-suite = { path = "./gossip/nostr-gossip-test-suite" } nostr-lmdb = { version = "0.44", path = "./database/nostr-lmdb", default-features = false } nostr-ndb = { version = "0.44", path = "./database/nostr-ndb", default-features = false } diff --git a/README.md b/README.md index 784399459..0857f1460 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ The project is split up into several crates: - [**nostr-indexeddb**](./database/nostr-indexeddb): IndexedDB storage backend - [**nostr-gossip**](./gossip/nostr-gossip): Gossip traits - [**nostr-gossip-memory**](./gossip/nostr-gossip-memory): In-memory gossip database + - [**nostr-gossip-sqlite**](./gossip/nostr-gossip-sqlite): SQLite storage for gossip - Remote File Storage implementations: - [**nostr-blossom**](./rfs/nostr-blossom): A library for interacting with the Blossom protocol - [**nostr-http-file-storage**](./rfs/nostr-http-file-storage): HTTP File Storage client (NIP-96) diff --git a/contrib/scripts/check-crates.sh b/contrib/scripts/check-crates.sh index f2b1107e1..4f1dae6fa 100755 --- a/contrib/scripts/check-crates.sh +++ b/contrib/scripts/check-crates.sh @@ -36,6 +36,7 @@ buildargs=( "-p nostr-database" "-p nostr-gossip" "-p nostr-gossip-memory" + "-p nostr-gossip-sqlite" "-p nostr-gossip-test-suite" "-p nostr-lmdb" "-p nostr-indexeddb --target wasm32-unknown-unknown" diff --git a/crates/nostr/src/nips/nip17.rs b/crates/nostr/src/nips/nip17.rs index b4469998f..48f606064 100644 --- a/crates/nostr/src/nips/nip17.rs +++ b/crates/nostr/src/nips/nip17.rs @@ -6,37 +6,26 @@ //! //! -use alloc::boxed::Box; -use core::iter; - -use crate::{Event, Kind, RelayUrl, TagStandard}; +use crate::{Event, RelayUrl, TagStandard}; /// Extracts the relay list -pub fn extract_relay_list<'a>(event: &'a Event) -> Box + 'a> { - if event.kind != Kind::InboxRelays { - return Box::new(iter::empty()); - } - - Box::new(event.tags.iter().filter_map(|tag| { +pub fn extract_relay_list(event: &Event) -> impl Iterator { + event.tags.iter().filter_map(|tag| { if let Some(TagStandard::Relay(url)) = tag.as_standardized() { Some(url) } else { None } - })) + }) } /// Extracts the relay list -pub fn extract_owned_relay_list(event: Event) -> Box> { - if event.kind != Kind::InboxRelays { - return Box::new(iter::empty()); - } - - Box::new(event.tags.into_iter().filter_map(|tag| { +pub fn extract_owned_relay_list(event: Event) -> impl Iterator { + event.tags.into_iter().filter_map(|tag| { if let Some(TagStandard::Relay(url)) = tag.to_standardized() { Some(url) } else { None } - })) + }) } diff --git a/crates/nostr/src/types/time/mod.rs b/crates/nostr/src/types/time/mod.rs index dc61987c4..f55b236a7 100644 --- a/crates/nostr/src/types/time/mod.rs +++ b/crates/nostr/src/types/time/mod.rs @@ -42,6 +42,16 @@ impl Timestamp { Self(secs) } + /// Construct from seconds + #[inline] + pub const fn from_i64_secs(secs: i64) -> Self { + if secs <= 0 { + Self::zero() + } else { + Self::from_secs(secs as u64) + } + } + /// Compose `0` timestamp #[inline] pub const fn zero() -> Self { diff --git a/gossip/nostr-gossip-sqlite/CHANGELOG.md b/gossip/nostr-gossip-sqlite/CHANGELOG.md new file mode 100644 index 000000000..f489f773b --- /dev/null +++ b/gossip/nostr-gossip-sqlite/CHANGELOG.md @@ -0,0 +1,28 @@ +# Changelog + + + + + + + + +## Unreleased + +First release. diff --git a/gossip/nostr-gossip-sqlite/Cargo.toml b/gossip/nostr-gossip-sqlite/Cargo.toml new file mode 100644 index 000000000..c623b1785 --- /dev/null +++ b/gossip/nostr-gossip-sqlite/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "nostr-gossip-sqlite" +version = "0.44.0" +edition = "2021" +description = "Nostr SQLite gossip backend" +authors.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +readme = "README.md" +rust-version.workspace = true +keywords = ["nostr", "gossip", "sqlite"] + +[features] +default = [] +unbundled = ["sqlx/sqlite-unbundled"] + +[dependencies] +nostr = { workspace = true, features = ["std"] } +nostr-gossip.workspace = true +sqlx = { version = "0.8", features = ["migrate", "runtime-tokio", "sqlite"] } + +[dev-dependencies] +nostr-gossip-test-suite.workspace = true +tempfile.workspace = true diff --git a/gossip/nostr-gossip-sqlite/README.md b/gossip/nostr-gossip-sqlite/README.md new file mode 100644 index 000000000..b9b9c8c59 --- /dev/null +++ b/gossip/nostr-gossip-sqlite/README.md @@ -0,0 +1,17 @@ +# Nostr SQLite gossip backend + +## Changelog + +All notable changes to this library are documented in the [CHANGELOG.md](CHANGELOG.md). + +## State + +**This library is in an ALPHA state**, things that are implemented generally work but the API will change in breaking ways. + +## Donations + +`rust-nostr` is free and open-source. This means we do not earn any revenue by selling it. Instead, we rely on your financial support. If you actively use any of the `rust-nostr` libs/software/services, then please [donate](https://rust-nostr.org/donate). + +## License + +This project is distributed under the MIT software license - see the [LICENSE](../../LICENSE) file for details diff --git a/gossip/nostr-gossip-sqlite/doc/database.md b/gossip/nostr-gossip-sqlite/doc/database.md new file mode 100644 index 000000000..43e7774ff --- /dev/null +++ b/gossip/nostr-gossip-sqlite/doc/database.md @@ -0,0 +1,26 @@ +# Database schema + +## Public keys table + +- `id`: Public Key ID +- `public_key`: Public Key 32-byte array + +## Lists table + +- `public_key_id`: Public Key ID +- `event_kind`: The event kind of the list (i.e., 10050, 10002) +- `event_created_at`: UNIX timestamp of when the event list has been created +- `last_checked_at`: UNIX timestamp of the last check + +## Relays table + +- `id`: Relay ID +- `url`: Relay URL + +## Relays-per-user table + +- `public_key_id`: Public Key ID +- `relay_id`: Relay ID +- `bitflags`: flags of the relay (read, write, hint, etc.) +- `received_events`: number of received events from the relay for that user +- `last_received_event`: UNIX timestamp of the last received event from the relay for that user diff --git a/gossip/nostr-gossip-sqlite/migrations/001_init.sql b/gossip/nostr-gossip-sqlite/migrations/001_init.sql new file mode 100644 index 000000000..a4fcb7c92 --- /dev/null +++ b/gossip/nostr-gossip-sqlite/migrations/001_init.sql @@ -0,0 +1,44 @@ +PRAGMA foreign_keys = ON; + +CREATE TABLE public_keys( + id INTEGER PRIMARY KEY AUTOINCREMENT, + public_key BLOB NOT NULL UNIQUE, + CHECK (length(public_key) = 32) +); + +CREATE INDEX idx_public_keys_public_key ON public_keys(public_key); + +CREATE TABLE lists( + id INTEGER PRIMARY KEY AUTOINCREMENT, + public_key_id INTEGER NOT NULL, + event_kind INTEGER NOT NULL, + event_created_at BIGINT DEFAULT NULL, + last_checked_at BIGINT DEFAULT NULL, + UNIQUE(public_key_id, event_kind), + FOREIGN KEY (public_key_id) REFERENCES public_keys(id) ON DELETE CASCADE +); + +CREATE INDEX idx_lists_pub_kind ON lists(public_key_id, event_kind); + +CREATE TABLE relays( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT NOT NULL UNIQUE +); + +CREATE INDEX idx_relays_relay ON relays(url); + +CREATE TABLE relays_per_user( + id INTEGER PRIMARY KEY AUTOINCREMENT, + public_key_id INTEGER NOT NULL, + relay_id INTEGER NOT NULL, + bitflags INTEGER NOT NULL DEFAULT 0, + received_events INTEGER NOT NULL DEFAULT 0, + last_received_event BIGINT NOT NULL DEFAULT 0, + UNIQUE(public_key_id, relay_id), + FOREIGN KEY (public_key_id) REFERENCES public_keys(id) ON DELETE CASCADE, + FOREIGN KEY (relay_id) REFERENCES relays(id) ON DELETE CASCADE +); + +CREATE INDEX idx_rpu_pub_relay ON relays_per_user(public_key_id, relay_id); +CREATE INDEX idx_rpu_pub_rank ON relays_per_user(public_key_id, received_events DESC); +CREATE INDEX idx_rpu_relay ON relays_per_user(relay_id); diff --git a/gossip/nostr-gossip-sqlite/migrations/README.md b/gossip/nostr-gossip-sqlite/migrations/README.md new file mode 100644 index 000000000..c771d7308 --- /dev/null +++ b/gossip/nostr-gossip-sqlite/migrations/README.md @@ -0,0 +1,13 @@ +# Migrations + +## Notes + +SQLx creates a checksum of the migrations and compares it to the database. +This means that also comments are included in the checksum. If you change +comments, the hash will change and will break the migrations! + +## SQL file format + +- Use a tab for indentation +- Leave an empty line at the end of the file +- **DON'T use `--` comments** (schema comments are documented [here](../doc/database.md)) diff --git a/gossip/nostr-gossip-sqlite/src/constant.rs b/gossip/nostr-gossip-sqlite/src/constant.rs new file mode 100644 index 000000000..182d5788a --- /dev/null +++ b/gossip/nostr-gossip-sqlite/src/constant.rs @@ -0,0 +1,15 @@ +// Copyright (c) 2022-2023 Yuki Kishimoto +// Copyright (c) 2023-2025 Rust Nostr Developers +// Distributed under the MIT software license + +use std::time::Duration; + +use nostr_gossip::flags::GossipFlags; + +pub(super) const PUBKEY_METADATA_OUTDATED_AFTER: Duration = Duration::from_secs(60 * 60); // 60 min + +pub(super) const READ_WRITE_FLAGS: GossipFlags = { + let mut flags = GossipFlags::READ; + flags.add(GossipFlags::WRITE); + flags +}; diff --git a/gossip/nostr-gossip-sqlite/src/error.rs b/gossip/nostr-gossip-sqlite/src/error.rs new file mode 100644 index 000000000..a164828e6 --- /dev/null +++ b/gossip/nostr-gossip-sqlite/src/error.rs @@ -0,0 +1,35 @@ +//! Gossip SQLite error + +use std::fmt; + +/// Gossip SQLite error +#[derive(Debug)] +pub enum Error { + /// SQLx error + Sqlx(sqlx::Error), + /// SQLx migration error + Migrate(sqlx::migrate::MigrateError), +} + +impl std::error::Error for Error {} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Sqlx(e) => e.fmt(f), + Self::Migrate(e) => e.fmt(f), + } + } +} + +impl From for Error { + fn from(err: sqlx::Error) -> Self { + Self::Sqlx(err) + } +} + +impl From for Error { + fn from(err: sqlx::migrate::MigrateError) -> Self { + Self::Migrate(err) + } +} diff --git a/gossip/nostr-gossip-sqlite/src/lib.rs b/gossip/nostr-gossip-sqlite/src/lib.rs new file mode 100644 index 000000000..0e4930ddd --- /dev/null +++ b/gossip/nostr-gossip-sqlite/src/lib.rs @@ -0,0 +1,16 @@ +// Copyright (c) 2022-2023 Yuki Kishimoto +// Copyright (c) 2023-2025 Rust Nostr Developers +// Distributed under the MIT software license + +//! Nostr Gossip SQLite store. + +#![forbid(unsafe_code)] +#![warn(missing_docs)] +#![warn(rustdoc::bare_urls)] +#![warn(clippy::large_futures)] + +mod constant; +pub mod error; +mod model; +pub mod prelude; +pub mod store; diff --git a/gossip/nostr-gossip-sqlite/src/model.rs b/gossip/nostr-gossip-sqlite/src/model.rs new file mode 100644 index 000000000..b1e101791 --- /dev/null +++ b/gossip/nostr-gossip-sqlite/src/model.rs @@ -0,0 +1,7 @@ +use sqlx::FromRow; + +#[derive(FromRow)] +pub(super) struct ListRow { + pub(super) event_created_at: Option, + pub(super) last_checked_at: Option, +} diff --git a/gossip/nostr-gossip-sqlite/src/prelude.rs b/gossip/nostr-gossip-sqlite/src/prelude.rs new file mode 100644 index 000000000..4f72afe7b --- /dev/null +++ b/gossip/nostr-gossip-sqlite/src/prelude.rs @@ -0,0 +1,14 @@ +// Copyright (c) 2022-2023 Yuki Kishimoto +// Copyright (c) 2023-2025 Rust Nostr Developers +// Distributed under the MIT software license + +//! Prelude + +#![allow(unknown_lints)] +#![allow(ambiguous_glob_reexports)] +#![doc(hidden)] + +pub use nostr::prelude::*; + +pub use crate::error::*; +pub use crate::store::*; diff --git a/gossip/nostr-gossip-sqlite/src/store.rs b/gossip/nostr-gossip-sqlite/src/store.rs new file mode 100644 index 000000000..347fd3342 --- /dev/null +++ b/gossip/nostr-gossip-sqlite/src/store.rs @@ -0,0 +1,568 @@ +//! Nostr gossip SQLite store. + +use std::collections::HashSet; +use std::path::Path; + +use nostr::nips::nip17; +use nostr::nips::nip65::{self, RelayMetadata}; +use nostr::util::BoxedFuture; +use nostr::{Event, Kind, PublicKey, RelayUrl, TagKind, TagStandard, Timestamp}; +use nostr_gossip::error::GossipError; +use nostr_gossip::flags::GossipFlags; +use nostr_gossip::{BestRelaySelection, GossipListKind, GossipPublicKeyStatus, NostrGossip}; +use sqlx::migrate::Migrator; +use sqlx::sqlite::SqliteConnectOptions; +use sqlx::{Executor, Sqlite, SqlitePool, Transaction}; + +use crate::constant::{PUBKEY_METADATA_OUTDATED_AFTER, READ_WRITE_FLAGS}; +use crate::error::Error; +use crate::model::ListRow; + +/// Nostr Gossip SQLite store. +#[derive(Debug, Clone)] +pub struct NostrGossipSqlite { + pool: SqlitePool, +} + +impl NostrGossipSqlite { + async fn new(opts: SqliteConnectOptions) -> Result { + // Create a connection pool. + let pool: SqlitePool = SqlitePool::connect_with(opts).await?; + + // Run migrations + let migrator: Migrator = sqlx::migrate!(); + migrator.run(&pool).await?; + + // Construct + Ok(Self { pool }) + } + + /// Open a persistent database + pub async fn open

(path: P) -> Result + where + P: AsRef, + { + // Built options + let opts: SqliteConnectOptions = SqliteConnectOptions::new() + .create_if_missing(true) + .filename(path); + + // Create instance + Self::new(opts).await + } + + // TODO: at the moment seems that the migrations don't work with the in-memory mode + // /// Open an in-memory database + // pub async fn in_memory() -> Result { + // // Built options + // let opts: SqliteConnectOptions = SqliteConnectOptions::new().in_memory(true).shared_cache(true); + // + // // Create instance + // Self::new(opts).await + // } + + async fn process_event( + &self, + event: &Event, + relay_url: Option<&RelayUrl>, + ) -> Result<(), Error> { + // Beings a new transaction + let mut tx = self.pool.begin().await?; + + // Save public key and get ID + let pk_id: i32 = get_or_save_public_key(&mut tx, &event.pubkey).await?; + + // Check the event kind + match &event.kind { + // Extract NIP-65 relays + Kind::RelayList => { + update_nip65_relays(&mut tx, pk_id, nip65::extract_relay_list(event)).await? + } + // Extract NIP-17 relays + Kind::InboxRelays => { + update_nip17_relays(&mut tx, pk_id, nip17::extract_relay_list(event)).await? + } + // Extract hints + _ => update_hints(&mut tx, event).await?, + } + + if let Some(relay_url) = relay_url { + update_relay_per_user(&mut tx, pk_id, relay_url, GossipFlags::RECEIVED).await?; + } + + // Commit the transaction + tx.commit().await?; + + Ok(()) + } + + async fn get_status( + &self, + public_key: &PublicKey, + list: GossipListKind, + ) -> Result { + // Get public key ID + match get_id_by_public_key(&self.pool, public_key).await? { + Some(pk_id) => { + let row: Option = sqlx::query_as( + "SELECT event_created_at, last_checked_at FROM lists WHERE public_key_id = $1 AND event_kind = $2", + ) + .bind(pk_id) + .bind(list.to_event_kind().as_u16()) + .fetch_optional(&self.pool) + .await?; + + match row { + Some(row) => { + let now: Timestamp = Timestamp::now(); + let last: Timestamp = + Timestamp::from_i64_secs(row.last_checked_at.unwrap_or(0)); + + if last + PUBKEY_METADATA_OUTDATED_AFTER < now { + Ok(GossipPublicKeyStatus::Outdated { + created_at: row.event_created_at.map(Timestamp::from_i64_secs), + }) + } else { + Ok(GossipPublicKeyStatus::Updated) + } + } + None => Ok(GossipPublicKeyStatus::Outdated { created_at: None }), + } + } + None => Ok(GossipPublicKeyStatus::Outdated { created_at: None }), + } + } + + async fn _update_fetch_attempt( + &self, + public_key: &PublicKey, + list: GossipListKind, + ) -> Result<(), Error> { + // Beings a new transaction + let mut tx = self.pool.begin().await?; + + // Save public key and get ID + let pk_id: i32 = get_or_save_public_key(&mut tx, public_key).await?; + + let now: i64 = Timestamp::now().as_secs() as i64; + + sqlx::query( + r#" + INSERT INTO lists (public_key_id, event_kind, last_checked_at) + VALUES ($1, $2, $3) + ON CONFLICT (public_key_id, event_kind) + DO UPDATE SET last_checked_at = excluded.last_checked_at + "#, + ) + .bind(pk_id) + .bind(list.to_event_kind().as_u16()) + .bind(now) + .execute(&mut *tx) + .await?; + + // Write changes + tx.commit().await?; + + Ok(()) + } + + async fn _get_best_relays( + &self, + public_key: &PublicKey, + selection: BestRelaySelection, + ) -> Result, Error> { + let mut relays: HashSet = HashSet::new(); + + match selection { + BestRelaySelection::All { + read, + write, + hints, + most_received, + } => { + // Get read relays + relays.extend( + self.get_relays_by_flag(public_key, GossipFlags::READ, read) + .await?, + ); + + // Get write relays + relays.extend( + self.get_relays_by_flag(public_key, GossipFlags::WRITE, write) + .await?, + ); + + // Get hint relays + relays.extend( + self.get_relays_by_flag(public_key, GossipFlags::HINT, hints) + .await?, + ); + + // Get most received relays + relays.extend( + self.get_relays_by_flag(public_key, GossipFlags::RECEIVED, most_received) + .await?, + ); + } + BestRelaySelection::Read { limit } => { + relays.extend( + self.get_relays_by_flag(public_key, GossipFlags::READ, limit) + .await?, + ); + } + BestRelaySelection::Write { limit } => { + relays.extend( + self.get_relays_by_flag(public_key, GossipFlags::WRITE, limit) + .await?, + ); + } + BestRelaySelection::PrivateMessage { limit } => { + relays.extend( + self.get_relays_by_flag(public_key, GossipFlags::PRIVATE_MESSAGE, limit) + .await?, + ); + } + BestRelaySelection::Hints { limit } => { + relays.extend( + self.get_relays_by_flag(public_key, GossipFlags::HINT, limit) + .await?, + ); + } + BestRelaySelection::MostReceived { limit } => { + relays.extend( + self.get_relays_by_flag(public_key, GossipFlags::RECEIVED, limit) + .await?, + ); + } + } + + Ok(relays) + } + + async fn get_relays_by_flag( + &self, + public_key: &PublicKey, + flag: GossipFlags, + limit: u8, + ) -> Result, Error> { + let query = r#" + SELECT r.url + FROM relays_per_user rpu + INNER JOIN relays r ON rpu.relay_id = r.id + INNER JOIN public_keys pk ON rpu.public_key_id = pk.id + WHERE pk.public_key = $1 AND (rpu.bitflags & $2) = $2 + ORDER BY rpu.received_events DESC, rpu.last_received_event DESC + LIMIT $3 + "#; + + let rows: Vec<(String,)> = sqlx::query_as(query) + .bind(public_key.as_bytes().as_slice()) + .bind(flag.as_u32()) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + let mut relays = Vec::with_capacity(rows.len()); + for (url,) in rows.into_iter() { + if let Ok(relay_url) = RelayUrl::parse(&url) { + relays.push(relay_url); + } + } + + Ok(relays) + } +} + +async fn get_or_save_public_key( + tx: &mut Transaction<'_, Sqlite>, + public_key: &PublicKey, +) -> Result { + match get_id_by_public_key(&mut **tx, public_key).await? { + Some(id) => Ok(id), + None => save_public_key(tx, public_key).await, + } +} + +async fn get_id_by_public_key<'a, E>( + executor: E, + public_key: &PublicKey, +) -> Result, Error> +where + E: Executor<'a, Database = Sqlite>, +{ + let pk_id: Option<(i32,)> = sqlx::query_as("SELECT id FROM public_keys WHERE public_key = $1") + .bind(public_key.as_bytes().as_slice()) + .fetch_optional(executor) + .await?; + Ok(pk_id.map(|(p,)| p)) +} + +async fn save_public_key( + tx: &mut Transaction<'_, Sqlite>, + public_key: &PublicKey, +) -> Result { + let pk_id: (i32,) = sqlx::query_as("INSERT INTO public_keys (public_key) VALUES ($1) ON CONFLICT (public_key) DO NOTHING RETURNING id") + .bind(public_key.as_bytes().as_slice()) + .fetch_one(&mut **tx) + .await?; + Ok(pk_id.0) +} + +async fn get_or_save_relay_url( + tx: &mut Transaction<'_, Sqlite>, + relay_url: &RelayUrl, +) -> Result { + match get_id_by_relay_url(tx, relay_url).await? { + Some(id) => Ok(id), + None => save_relay_url(tx, relay_url).await, + } +} + +async fn get_id_by_relay_url( + tx: &mut Transaction<'_, Sqlite>, + relay_url: &RelayUrl, +) -> Result, Error> { + let pk_id: Option<(i32,)> = sqlx::query_as("SELECT id FROM relays WHERE url = $1") + .bind(relay_url.as_str_without_trailing_slash()) + .fetch_optional(&mut **tx) + .await?; + Ok(pk_id.map(|(p,)| p)) +} + +async fn save_relay_url( + tx: &mut Transaction<'_, Sqlite>, + relay_url: &RelayUrl, +) -> Result { + let pk_id: (i32,) = sqlx::query_as( + "INSERT INTO relays (url) VALUES ($1) ON CONFLICT (url) DO NOTHING RETURNING id", + ) + .bind(relay_url.as_str_without_trailing_slash()) + .fetch_one(&mut **tx) + .await?; + Ok(pk_id.0) +} + +async fn remove_flag_from_user_relays( + tx: &mut Transaction<'_, Sqlite>, + public_key_id: i32, + flags_to_remove: GossipFlags, +) -> Result<(), Error> { + sqlx::query("UPDATE relays_per_user SET bitflags = (bitflags & ~$1) WHERE public_key_id = $2") + .bind(flags_to_remove.as_u32()) + .bind(public_key_id) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Add relay per user or update the received events and bitflags. +async fn update_relay_per_user( + tx: &mut Transaction<'_, Sqlite>, + public_key_id: i32, + relay_url: &RelayUrl, + flags: GossipFlags, +) -> Result<(), Error> { + let relay_id: i32 = get_or_save_relay_url(tx, relay_url).await?; + + let now: u64 = Timestamp::now().as_secs(); + + sqlx::query( + r#" + INSERT INTO relays_per_user (public_key_id, relay_id, bitflags, received_events, last_received_event) + VALUES ($1, $2, $3, 1, $4) + ON CONFLICT (public_key_id, relay_id) + DO UPDATE SET + bitflags = bitflags | excluded.bitflags, + received_events = received_events + 1, + last_received_event = excluded.last_received_event + "#) + .bind(public_key_id) + .bind(relay_id) + .bind(flags.as_u32()) + .bind(now as i64) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn update_nip65_relays<'a, I>( + tx: &mut Transaction<'_, Sqlite>, + public_key_id: i32, + iter: I, +) -> Result<(), Error> +where + I: IntoIterator)>, +{ + // Remove all READ and WRITE flags from the relays of the public key + remove_flag_from_user_relays(tx, public_key_id, READ_WRITE_FLAGS).await?; + + // Extract relay list + for (relay_url, metadata) in iter { + // Save relay and get ID + let relay_id: i32 = get_or_save_relay_url(tx, relay_url).await?; + + // New bitflag for the relay + let bitflag: GossipFlags = match metadata { + Some(RelayMetadata::Read) => GossipFlags::READ, + Some(RelayMetadata::Write) => GossipFlags::WRITE, + None => READ_WRITE_FLAGS, + }; + + // Update bitflag + sqlx::query( + r#" + INSERT INTO relays_per_user (public_key_id, relay_id, bitflags) + VALUES ($1, $2, $3) + ON CONFLICT (public_key_id, relay_id) + DO UPDATE SET + bitflags = bitflags | excluded.bitflags + "#, + ) + .bind(public_key_id) + .bind(relay_id) + .bind(bitflag.as_u32()) + .execute(&mut **tx) + .await?; + } + + Ok(()) +} + +async fn update_nip17_relays<'a, I>( + tx: &mut Transaction<'_, Sqlite>, + public_key_id: i32, + iter: I, +) -> Result<(), Error> +where + I: IntoIterator, +{ + // Remove all PRIVATE_MESSAGE flag from the relays of the public key + remove_flag_from_user_relays(tx, public_key_id, GossipFlags::PRIVATE_MESSAGE).await?; + + // Extract relay list + for relay_url in iter { + let relay_id: i32 = get_or_save_relay_url(tx, relay_url).await?; + + sqlx::query( + r#" + INSERT INTO relays_per_user (public_key_id, relay_id, bitflags) + VALUES ($1, $2, $3) + ON CONFLICT (public_key_id, relay_id) + DO UPDATE SET + bitflags = bitflags | excluded.bitflags + "#, + ) + .bind(public_key_id) + .bind(relay_id) + .bind(GossipFlags::PRIVATE_MESSAGE.as_u32()) + .execute(&mut **tx) + .await?; + } + + Ok(()) +} + +async fn update_hints(tx: &mut Transaction<'_, Sqlite>, event: &Event) -> Result<(), Error> { + for tag in event.tags.filter_standardized(TagKind::p()) { + if let TagStandard::PublicKey { + public_key, + relay_url: Some(relay_url), + .. + } = tag + { + let p_tag_pk_id: i32 = get_or_save_public_key(tx, public_key).await?; + update_relay_per_user(tx, p_tag_pk_id, relay_url, GossipFlags::HINT).await?; + } + } + + Ok(()) +} + +impl NostrGossip for NostrGossipSqlite { + fn process<'a>( + &'a self, + event: &'a Event, + relay_url: Option<&'a RelayUrl>, + ) -> BoxedFuture<'a, Result<(), GossipError>> { + Box::pin(async move { + self.process_event(event, relay_url) + .await + .map_err(GossipError::backend) + }) + } + + fn status<'a>( + &'a self, + public_key: &'a PublicKey, + list: GossipListKind, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { + self.get_status(public_key, list) + .await + .map_err(GossipError::backend) + }) + } + + fn update_fetch_attempt<'a>( + &'a self, + public_key: &'a PublicKey, + list: GossipListKind, + ) -> BoxedFuture<'a, Result<(), GossipError>> { + Box::pin(async move { + self._update_fetch_attempt(public_key, list) + .await + .map_err(GossipError::backend) + }) + } + + fn get_best_relays<'a>( + &'a self, + public_key: &'a PublicKey, + selection: BestRelaySelection, + ) -> BoxedFuture<'a, Result, GossipError>> { + Box::pin(async move { + self._get_best_relays(public_key, selection) + .await + .map_err(GossipError::backend) + }) + } +} + +#[cfg(test)] +mod tests { + use std::ops::Deref; + + use nostr_gossip_test_suite::gossip_unit_tests; + use tempfile::TempDir; + + use super::*; + + #[derive(Debug)] + struct NostrGossipSqliteUnitTest { + store: NostrGossipSqlite, + _temp_dir: TempDir, + } + + impl Deref for NostrGossipSqliteUnitTest { + type Target = NostrGossipSqlite; + + fn deref(&self) -> &Self::Target { + &self.store + } + } + + async fn setup() -> NostrGossipSqliteUnitTest { + let temp_dir = TempDir::new().expect("Failed to create temp directory"); + let path = temp_dir.path().join("test.db"); + + let store = NostrGossipSqlite::open(path).await.unwrap(); + + NostrGossipSqliteUnitTest { + store, + _temp_dir: temp_dir, + } + } + + gossip_unit_tests!(NostrGossipSqliteUnitTest, setup); +}