chore: rostra dev db-dump

Author: dpc

Cargo.lock

@@ -38,6 +38,7 @@ rostra-util-fmt = { workspace = true }
 dedup-chan = { workspace = true }
 rostra-util = { workspace = true }
 serde = { workspace = true }
+serde_json = { workspace = true }
 snafu = { workspace = true }
 rand = { workspace = true }
 tokio = { workspace = true }

crates/rostra-client-db/Cargo.toml

@@ -4,6 +4,7 @@ mod models;
 mod process_event_content_ops;
 mod process_event_ops;
 pub mod social;
+mod table_ops;
 mod tables;
 mod tx_ops;
 
@@ -78,6 +79,13 @@ impl WriteTransactionCtx {
     }
 }
 
+#[derive(Debug, Snafu)]
+pub enum TableDumpError {
+    #[snafu(display("Unknown table `{name}`"))]
+    UnknownTable { name: String },
+}
+pub type TableDumpResult<T> = std::result::Result<T, TableDumpError>;
+
 #[derive(Debug, Snafu)]
 pub enum DbError {
     Database {
@@ -149,6 +157,7 @@ pub struct Database {
 }
 
 impl Database {
+    const MAX_CONTENT_LEN: u32 = 1_000_000u32;
     pub async fn mk_db_path(
         data_dir: &Path,
         self_id: RostraId,
@@ -234,7 +243,30 @@ impl Database {
         tokio::task::block_in_place(|| self.inner.as_raw_mut().compact())
     }
 
-    const MAX_CONTENT_LEN: u32 = 1_000_000u32;
+    pub async fn dump_table(&self, name: &str) -> TableDumpResult<()> {
+        self.read_with(|tx| {
+            match name {
+                "events" => Self::dump_table_dbtx(tx, &tables::events::TABLE)?,
+                "events_content" => Self::dump_table_dbtx(tx, &tables::events_content::TABLE)?,
+                "social_posts" => Self::dump_table_dbtx(tx, &tables::social_posts::TABLE)?,
+                "social_posts_replies" => {
+                    Self::dump_table_dbtx(tx, &tables::social_posts_replies::TABLE)?
+                }
+                "social_posts_reactions" => {
+                    Self::dump_table_dbtx(tx, &tables::social_posts_reactions::TABLE)?
+                }
+                _ => {
+                    return Ok(Err(UnknownTableSnafu {
+                        name: name.to_string(),
+                    }
+                    .build()));
+                }
+            }
+            Ok(Ok(()))
+        })
+        .await
+        .expect("Database panic")
+    }
 
     pub fn self_followees_subscribe(
         &self,

crates/rostra-client-db/src/lib.rs

@@ -0,0 +1,25 @@
+use redb_bincode::ReadTransaction;
+
+use crate::{Database, DbResult};
+
+impl Database {
+    pub(crate) fn dump_table_dbtx<K, V>(
+        dbtx: &ReadTransaction,
+        def: &redb_bincode::TableDefinition<'_, K, V>,
+    ) -> DbResult<()>
+    where
+        V: bincode::Decode + bincode::Encode + serde::Serialize,
+        K: bincode::Decode + bincode::Encode + serde::Serialize,
+    {
+        let tbl = dbtx.open_table(def)?;
+        for record in tbl.range(..)? {
+            let (k, v) = record?;
+            println!(
+                "{} => {}",
+                serde_json::to_string(&k.value()).expect("Can't fail"),
+                serde_json::to_string(&v.value()).expect("Can't fail")
+            )
+        }
+        Ok(())
+    }
+}

crates/rostra-client-db/src/table_ops.rs

@@ -6,6 +6,7 @@ use ids::{IdsFolloweesRecord, IdsFollowersRecord, IdsPersonaRecord, IdsUnfollowe
 use rostra_core::event::{IrohNodeId, PersonaId};
 use rostra_core::id::{RestRostraId, RostraId, ShortRostraId};
 use rostra_core::{ShortEventId, Timestamp};
+use serde::Serialize;
 
 pub use self::event::EventsHeadsTableRecord;
 pub(crate) mod event;
@@ -75,9 +76,9 @@ pub struct Latest<T> {
     pub inner: T,
 }
 
-#[derive(Debug, Encode, Decode, Clone, Copy)]
+#[derive(Debug, Encode, Serialize, Decode, Clone, Copy)]
 pub struct SocialPostsRepliesRecord;
-#[derive(Debug, Encode, Decode, Clone, Copy)]
+#[derive(Debug, Encode, Serialize, Decode, Clone, Copy)]
 pub struct SocialPostsReactionsRecord;
 
 #[derive(Debug, Encode, Decode, Clone)]
@@ -114,6 +115,7 @@ pub struct SocialPostRecordV0 {
     Debug,
     Encode,
     Decode,
+    Serialize,
     Clone,
     // Note: needs to be default so we can track number of replies even before we get what was
     // replied to

crates/rostra-client-db/src/tables.rs

@@ -3,8 +3,9 @@ use std::borrow::Cow;
 use bincode::{Decode, Encode};
 use rostra_core::event::{EventContentUnsized, EventExt, SignedEvent};
 use rostra_core::ShortEventId;
+use serde::Serialize;
 
-#[derive(Debug, Encode, Decode, Clone)]
+#[derive(Debug, Encode, Decode, Clone, Serialize)]
 pub enum EventContentState<'a> {
     /// The event content is present and we processed it without problems
     Present(Cow<'a, EventContentUnsized>),
@@ -27,7 +28,7 @@ pub enum EventContentState<'a> {
 
 pub type EventContentStateOwned = EventContentState<'static>;
 
-#[derive(Debug, Encode, Decode, Clone)]
+#[derive(Debug, Encode, Decode, Clone, Serialize)]
 pub struct EventRecord {
     pub signed: SignedEvent,
 }

crates/rostra-client-db/src/tables/event.rs

@@ -1,12 +1,19 @@
 use std::borrow::Borrow;
 use std::ops;
 use std::sync::Arc;
+#[cfg(feature = "serde")]
+mod serde;
 
 #[derive(Debug)]
 #[cfg_attr(feature = "bincode", derive(::bincode::Encode))]
 #[repr(transparent)]
 pub struct EventContentUnsized([u8]);
 
+impl std::fmt::Display for EventContentUnsized {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        data_encoding::HEXLOWER.encode_write(self.as_slice(), f)
+    }
+}
 impl EventContentUnsized {
     pub fn as_slice(&self) -> &[u8] {
         &self.0

crates/rostra-core/src/event/content.rs

@@ -0,0 +1,16 @@
+use serde::Serialize;
+
+use super::EventContentUnsized;
+
+impl Serialize for EventContentUnsized {
+    fn serialize<S>(&self, s: S) -> Result<S::Ok, S::Error>
+    where
+        S: ::serde::Serializer,
+    {
+        if s.is_human_readable() {
+            s.serialize_str(&self.to_string())
+        } else {
+            s.serialize_bytes(&self.0)
+        }
+    }
+}

crates/rostra-core/src/event/content/serde.rs

@@ -131,4 +131,11 @@ pub enum DevCmd {
     },
     /// Run tests
     Test,
+    /// Dump database
+    DbDump {
+        #[arg(long)]
+        table: String,
+        #[arg(long)]
+        rostra_id: RostraId,
+    },
 }

crates/rostra/src/cli.rs

@@ -10,11 +10,11 @@ use futures::future::pending;
 use rostra_client::error::{ConnectError, IdResolveError, IdSecretReadError, InitError, PostError};
 use rostra_client::multiclient::MultiClient;
 use rostra_client::Client;
-use rostra_client_db::DbError;
+use rostra_client_db::{Database, DbError};
 use rostra_core::id::RostraIdSecretKey;
 use rostra_p2p::connection::{Connection, PingRequest, PingResponse};
 use rostra_p2p::RpcError;
-use rostra_util_error::FmtCompact as _;
+use rostra_util_error::{BoxedError, FmtCompact as _};
 use rostra_web_ui::{Server, WebUiServerError};
 use snafu::{FromString, ResultExt, Snafu, Whatever};
 use tokio::time::Instant;
@@ -49,6 +49,8 @@ pub enum CliError {
     DataDir { source: io::Error },
     #[snafu(display("Database error: {source}"))]
     Database { source: DbError },
+    #[snafu(display("Miscellaneous error: {source}"))]
+    Other { source: BoxedError },
 }
 
 pub type CliResult<T> = std::result::Result<T, CliError>;
@@ -157,6 +159,20 @@ async fn handle_cmd(opts: Opts) -> CliResult<serde_json::Value> {
 
                 serde_json::to_value(&resp).expect("Can't fail")
             }
+            cli::DevCmd::DbDump {
+                rostra_id: id,
+                table,
+            } => {
+                let db_path = Database::mk_db_path(opts.global.data_dir(), id)
+                    .await
+                    .context(DataDirSnafu)?;
+
+                let db = Database::open(&db_path, id).await.context(DatabaseSnafu)?;
+
+                db.dump_table(&table).await.boxed().context(OtherSnafu)?;
+
+                serde_json::to_value(serde_json::Value::Null).expect("Can't fail")
+            }
         },
         cli::OptsCmd::Serve { secret_file } => {
             let secret_id = if let Some(secret_file) = secret_file {