database: improve performance

* Remove `rayon` (causes very slow queries in some cases)
* Add indexes for IDs, replaceable and param. replaceable events
* Add support to query result order (ASC, DESC)
* Refactoring
* Improve unit tests

Author: yukibtc

Cargo.lock

@@ -6,7 +6,7 @@ use std::ops::Deref;
 use std::sync::Arc;
 
 use nostr_ffi::{Event, EventId, Filter, PublicKey};
-use nostr_sdk::database::{DynNostrDatabase, IntoNostrDatabase, NostrDatabaseExt};
+use nostr_sdk::database::{DynNostrDatabase, IntoNostrDatabase, NostrDatabaseExt, Order};
 use nostr_sdk::{block_on, SQLiteDatabase};
 use uniffi::Object;
 
@@ -78,7 +78,7 @@ impl NostrDatabase {
                 .collect();
             Ok(self
                 .inner
-                .query(filters)
+                .query(filters, Order::Desc)
                 .await?
                 .into_iter()
                 .map(|e| Arc::new(e.into()))

bindings/nostr-sdk-ffi/src/database.rs

@@ -9,7 +9,7 @@ use nostr_js::error::{into_err, Result};
 use nostr_js::event::{JsEvent, JsEventArray, JsEventId};
 use nostr_js::key::JsPublicKey;
 use nostr_js::message::JsFilter;
-use nostr_sdk::database::{DynNostrDatabase, IntoNostrDatabase, NostrDatabaseExt};
+use nostr_sdk::database::{DynNostrDatabase, IntoNostrDatabase, NostrDatabaseExt, Order};
 use nostr_sdk::WebDatabase;
 use wasm_bindgen::prelude::*;
 
@@ -94,7 +94,7 @@ impl JsNostrDatabase {
         let filters = filters.into_iter().map(|f| f.inner()).collect();
         Ok(self
             .inner
-            .query(filters)
+            .query(filters, Order::Desc)
             .await
             .map_err(into_err)?
             .into_iter()

bindings/nostr-sdk-js/src/database.rs

@@ -0,0 +1 @@
+too-many-arguments-threshold = 13

clippy.toml

@@ -20,7 +20,6 @@ flatbuf = ["dep:flatbuffers"]
 async-trait.workspace = true
 flatbuffers = { version = "23.5", optional = true }
 nostr = { workspace = true, features = ["std"] }
-rayon = "1.8"
 thiserror.workspace = true
 tokio = { workspace = true, features = ["sync"] }
 tracing = { workspace = true, features = ["std", "attributes"] }

crates/nostr-database/Cargo.toml

@@ -1,5 +1,3 @@
-all: build
-
 flatbuf:
 	flatc --rust -o ./src/flatbuffers ./fbs/event.fbs
 	flatc --rust -o ./src/flatbuffers ./fbs/event_seen_by.fbs

crates/nostr-database/Makefile

@@ -3,7 +3,7 @@
 // Distributed under the MIT software license
 
 use nostr::prelude::*;
-use nostr_database::DatabaseIndexes;
+use nostr_database::{DatabaseIndexes, Order};
 use tracing_subscriber::fmt::format::FmtSpan;
 
 #[tokio::main]
@@ -59,12 +59,15 @@ async fn main() {
     }
 
     let ids = index
-        .query(vec![Filter::new()
-            .kinds(vec![Kind::Metadata, Kind::Custom(123), Kind::TextNote])
-            .limit(20)
-            //.kind(Kind::Custom(123))
-            //.identifier("myid5000")
-            .author(keys_a.public_key())])
+        .query(
+            vec![Filter::new()
+                .kinds(vec![Kind::Metadata, Kind::Custom(123), Kind::TextNote])
+                .limit(20)
+                //.kind(Kind::Custom(123))
+                //.identifier("myid5000")
+                .author(keys_a.public_key())],
+            Order::Desc,
+        )
         .await;
     println!("Got {} ids", ids.len());
 

crates/nostr-database/examples/indexes.rs

@@ -5,7 +5,7 @@
 use nostr::prelude::*;
 use nostr::{EventBuilder, Filter, Keys, Kind, Metadata, Tag};
 use nostr_database::memory::MemoryDatabase;
-use nostr_database::{DatabaseOptions, NostrDatabase};
+use nostr_database::{DatabaseOptions, NostrDatabase, Order};
 use tracing_subscriber::fmt::format::FmtSpan;
 
 #[tokio::main]
@@ -63,12 +63,15 @@ async fn main() {
     }
 
     let events = database
-        .query(vec![Filter::new()
-            .kinds(vec![Kind::Metadata, Kind::Custom(123), Kind::TextNote])
-            .limit(20)
-            //.kind(Kind::Custom(123))
-            //.identifier("myid5000")
-            .author(keys_a.public_key())])
+        .query(
+            vec![Filter::new()
+                .kinds(vec![Kind::Metadata, Kind::Custom(123), Kind::TextNote])
+                .limit(20)
+                //.kind(Kind::Custom(123))
+                //.identifier("myid5000")
+                .author(keys_a.public_key())],
+            Order::Desc,
+        )
         .await
         .unwrap();
     println!("Got {} events", events.len());