chore: improve & fix post filtering

Author: dpc

crates/rostra-client-db/src/social.rs

@@ -21,6 +21,16 @@ pub struct EventPaginationCursor {
     pub event_id: ShortEventId,
 }
 
+impl EventPaginationCursor {
+    pub const ZERO: Self = Self {
+        ts: Timestamp::ZERO,
+        event_id: ShortEventId::ZERO,
+    };
+    pub const MAX: Self = Self {
+        ts: Timestamp::MAX,
+        event_id: ShortEventId::MAX,
+    };
+}
 #[derive(Clone)]
 pub struct SocialPostRecord<C> {
     pub ts: Timestamp,
@@ -105,7 +115,10 @@ impl Database {
         upper_bound: Option<EventPaginationCursor>,
         limit: usize,
         filter_fn: impl Fn(&SocialPostRecord<SocialPost>) -> bool + Send + 'static,
-    ) -> Vec<SocialPostRecord<content_kind::SocialPost>> {
+    ) -> (
+        Vec<SocialPostRecord<content_kind::SocialPost>>,
+        EventPaginationCursor,
+    ) {
         let upper_bound = upper_bound
             .map(|b| (b.ts, b.event_id))
             .unwrap_or((Timestamp::MAX, ShortEventId::MAX));
@@ -116,6 +129,7 @@ impl Database {
             let events_content_table = tx.open_table(&events_content::TABLE)?;
 
             let mut ret = vec![];
+            let mut last = EventPaginationCursor { ts: Timestamp::ZERO, event_id: ShortEventId::ZERO };
 
             for event in social_posts_by_time_table
                 .range(&(Timestamp::ZERO, ShortEventId::ZERO)..&upper_bound)?
@@ -127,6 +141,7 @@ impl Database {
                 let (k, _) = event?;
                 let (ts, event_id) = k.value();
 
+                last = EventPaginationCursor { ts, event_id};
 
                 let Some(content_state) =
                     Database::get_event_content_tx(event_id, &events_content_table)?
@@ -168,7 +183,7 @@ impl Database {
                 ret.push(social_post_record);
             }
 
-            Ok(ret)
+            Ok((ret, last))
         })
         .await
         .expect("Storage error")
@@ -179,7 +194,10 @@ impl Database {
         lower_bound: Option<EventPaginationCursor>,
         limit: usize,
         filter_fn: impl Fn(&SocialPostRecord<SocialPost>) -> bool + Send + 'static,
-    ) -> Vec<SocialPostRecord<content_kind::SocialPost>> {
+    ) -> (
+        Vec<SocialPostRecord<content_kind::SocialPost>>,
+        EventPaginationCursor,
+    ) {
         let lower_bound = lower_bound
             .map(|b| (b.ts, b.event_id))
             .unwrap_or((Timestamp::ZERO, ShortEventId::ZERO));
@@ -190,6 +208,7 @@ impl Database {
             let events_content_table = tx.open_table(&events_content::TABLE)?;
 
             let mut ret = vec![];
+            let mut last = EventPaginationCursor { ts: Timestamp::MAX, event_id: ShortEventId::MAX };
 
             for event in social_posts_by_time_table
                 .range(&lower_bound..&(Timestamp::MAX, ShortEventId::MAX))?
@@ -201,6 +220,12 @@ impl Database {
                 let (k, _) = event?;
                 let (ts, event_id) = k.value();
 
+                // Since we don't have exclusive lower bound range, we need to skip it manually
+                if (ts, event_id)  == lower_bound {
+                    continue;
+                }
+
+                last = EventPaginationCursor { ts, event_id};
 
                 let Some(content_state) =
                     Database::get_event_content_tx(event_id, &events_content_table)?
@@ -243,7 +268,7 @@ impl Database {
 
             }
 
-            Ok(ret)
+            Ok((ret, last))
         })
         .await
         .expect("Storage error")

crates/rostra-web-ui/src/routes/timeline.rs

@@ -274,8 +274,9 @@ impl UiState {
             TimelineMode::Notifications => {
                 if let Some(latest_event) = client
                     .db()
-                    .paginate_social_posts_with_filter(None, 1, |_| true)
+                    .paginate_social_posts_rev_with_filter(None, 1, |_| true)
                     .await
+                    .0
                     .into_iter()
                     .next()
                 {
@@ -289,12 +290,13 @@ impl UiState {
             TimelineMode::Followees | TimelineMode::Network => {
                 let pending_len = client
                     .db()
-                    .paginate_social_posts_rev_with_filter(
+                    .paginate_social_posts_with_filter(
                         cookies.get_last_seen(),
                         10,
                         TimelineMode::Notifications.to_filter_fn(client).await,
                     )
                     .await
+                    .0
                     .len();
 
                 Ok(Some(pending_len))
@@ -374,11 +376,10 @@ impl UiState {
 
         let filter_fn = mode.to_filter_fn(&client_ref).await;
 
-        let post_page = client_ref
+        let (filtered_posts, last_seen) = client_ref
             .db()
-            .paginate_social_posts_rev(pagination, mode.db_page_size())
+            .paginate_social_posts_rev_with_filter(pagination, 30, filter_fn)
             .await;
-        let filtered_posts: Vec<_> = post_page.iter().filter(|post| (filter_fn)(post)).collect();
 
         let parents = self
             .client(session.id())
@@ -458,13 +459,13 @@ impl UiState {
                                 .call().await?)
                     }
                 }
-                @if let Some(last) = post_page.last() {
+                @if last_seen != EventPaginationCursor::ZERO {
                     div ."o-mainBarTimeline__rest -empty"
                         hx-get=(
                             format!("{}?ts={}&event_id={}",
                                 mode.to_path(),
-                                last.ts,
-                                last.event_id)
+                                last_seen.ts,
+                                last_seen.event_id)
                         )
                         hx-select=".o-mainBarTimeline__item, .o-mainBarTimeline__rest, script.mathjax"
                         hx-trigger="intersect once, threshold:0.5"
@@ -671,17 +672,6 @@ pub(crate) enum TimelineMode {
 }
 
 impl TimelineMode {
-    /// How many records in a page to query, to have something reasonable to
-    /// show
-    fn db_page_size(self) -> usize {
-        match self {
-            TimelineMode::Followees => 100,
-            TimelineMode::Network => 50,
-            TimelineMode::Notifications => 200,
-            TimelineMode::Profile(_) => 200,
-        }
-    }
-
     fn to_path(self) -> String {
         match self {
             TimelineMode::Followees => "/ui/followees".to_string(),