chore: save folder snapshot with custom config (#1670)

appflowy · web-flow · commit a0b5f09b060a · 2023-01-06T14:11:35.000+08:00
diff --git a/frontend/rust-lib/flowy-database/migrations/2023-01-06-032859_folder_rev_snapshot/down.sql b/frontend/rust-lib/flowy-database/migrations/2023-01-06-032859_folder_rev_snapshot/down.sql
@@ -0,0 +1,2 @@
+-- This file should undo anything in `up.sql`
+DROP TABLE folder_rev_snapshot;
diff --git a/frontend/rust-lib/flowy-database/migrations/2023-01-06-032859_folder_rev_snapshot/up.sql b/frontend/rust-lib/flowy-database/migrations/2023-01-06-032859_folder_rev_snapshot/up.sql
@@ -0,0 +1,9 @@
+-- Your SQL goes here
+CREATE TABLE folder_rev_snapshot (
+   snapshot_id TEXT NOT NULL PRIMARY KEY DEFAULT '',
+   object_id TEXT NOT NULL DEFAULT '',
+   rev_id BIGINT NOT NULL DEFAULT 0,
+   base_rev_id BIGINT NOT NULL DEFAULT 0,
+   timestamp BIGINT NOT NULL DEFAULT 0,
+   data BLOB NOT NULL DEFAULT (x'')
+);
diff --git a/frontend/rust-lib/flowy-database/src/schema.rs b/frontend/rust-lib/flowy-database/src/schema.rs
@@ -1,6 +1,4 @@
-// @generated automatically by Diesel CLI.
-
-diesel::table! {
+table! {
     app_table (id) {
         id -> Text,
         workspace_id -> Text,
@@ -15,7 +13,7 @@ diesel::table! {
     }
 }
 
-diesel::table! {
+table! {
     document_rev_table (id) {
         id -> Integer,
         document_id -> Text,
@@ -26,14 +24,25 @@ diesel::table! {
     }
 }
 
-diesel::table! {
+table! {
+    folder_rev_snapshot (snapshot_id) {
+        snapshot_id -> Text,
+        object_id -> Text,
+        rev_id -> BigInt,
+        base_rev_id -> BigInt,
+        timestamp -> BigInt,
+        data -> Binary,
+    }
+}
+
+table! {
     grid_block_index_table (row_id) {
         row_id -> Text,
         block_id -> Text,
     }
 }
 
-diesel::table! {
+table! {
     grid_meta_rev_table (id) {
         id -> Integer,
         object_id -> Text,
@@ -44,7 +53,7 @@ diesel::table! {
     }
 }
 
-diesel::table! {
+table! {
     grid_rev_snapshot (snapshot_id) {
         snapshot_id -> Text,
         object_id -> Text,
@@ -55,7 +64,7 @@ diesel::table! {
     }
 }
 
-diesel::table! {
+table! {
     grid_rev_table (id) {
         id -> Integer,
         object_id -> Text,
@@ -66,7 +75,7 @@ diesel::table! {
     }
 }
 
-diesel::table! {
+table! {
     grid_view_rev_table (id) {
         id -> Integer,
         object_id -> Text,
@@ -77,14 +86,14 @@ diesel::table! {
     }
 }
 
-diesel::table! {
+table! {
     kv_table (key) {
         key -> Text,
         value -> Binary,
     }
 }
 
-diesel::table! {
+table! {
     rev_snapshot (id) {
         id -> Integer,
         object_id -> Text,
@@ -93,7 +102,7 @@ diesel::table! {
     }
 }
 
-diesel::table! {
+table! {
     rev_table (id) {
         id -> Integer,
         doc_id -> Text,
@@ -105,7 +114,7 @@ diesel::table! {
     }
 }
 
-diesel::table! {
+table! {
     trash_table (id) {
         id -> Text,
         name -> Text,
@@ -116,7 +125,7 @@ diesel::table! {
     }
 }
 
-diesel::table! {
+table! {
     user_table (id) {
         id -> Text,
         name -> Text,
@@ -127,7 +136,7 @@ diesel::table! {
     }
 }
 
-diesel::table! {
+table! {
     view_table (id) {
         id -> Text,
         belong_to_id -> Text,
@@ -143,7 +152,7 @@ diesel::table! {
     }
 }
 
-diesel::table! {
+table! {
     workspace_table (id) {
         id -> Text,
         name -> Text,
@@ -155,9 +164,10 @@ diesel::table! {
     }
 }
 
-diesel::allow_tables_to_appear_in_same_query!(
+allow_tables_to_appear_in_same_query!(
     app_table,
     document_rev_table,
+    folder_rev_snapshot,
     grid_block_index_table,
     grid_meta_rev_table,
     grid_rev_snapshot,
diff --git a/frontend/rust-lib/flowy-folder/src/manager.rs b/frontend/rust-lib/flowy-folder/src/manager.rs
@@ -23,7 +23,9 @@ use lazy_static::lazy_static;
 use lib_infra::future::FutureResult;
 
 use crate::services::clear_current_workspace;
-use crate::services::persistence::rev_sqlite::SQLiteFolderRevisionPersistence;
+use crate::services::persistence::rev_sqlite::{
+    SQLiteFolderRevisionPersistence, SQLiteFolderRevisionSnapshotPersistence,
+};
 use flowy_http_model::ws_data::ServerRevisionWSData;
 use flowy_sync::client_folder::FolderPad;
 use std::convert::TryFrom;
@@ -174,12 +176,15 @@ impl FolderManager {
         let configuration = RevisionPersistenceConfiguration::new(100, false);
         let rev_persistence = RevisionPersistence::new(user_id, object_id, disk_cache, configuration);
         let rev_compactor = FolderRevisionMergeable();
+
+        let snapshot_object_id = format!("folder:{}", object_id);
+        let snapshot_persistence = SQLiteFolderRevisionSnapshotPersistence::new(&snapshot_object_id, pool);
         let rev_manager = RevisionManager::new(
             user_id,
             folder_id.as_ref(),
             rev_persistence,
             rev_compactor,
-            PhantomSnapshotPersistence(),
+            snapshot_persistence,
         );
 
         let folder_editor = FolderEditor::new(user_id, &folder_id, token, rev_manager, self.web_socket.clone()).await?;
diff --git a/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_snapshot_sqlite_impl.rs b/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/folder_snapshot_sqlite_impl.rs
@@ -0,0 +1,96 @@
+use bytes::Bytes;
+use flowy_database::{
+    prelude::*,
+    schema::{folder_rev_snapshot, folder_rev_snapshot::dsl},
+    ConnectionPool,
+};
+use flowy_error::{internal_error, FlowyResult};
+use flowy_revision::{RevisionSnapshot, RevisionSnapshotDiskCache};
+use lib_infra::util::timestamp;
+use std::sync::Arc;
+
+pub struct SQLiteFolderRevisionSnapshotPersistence {
+    object_id: String,
+    pool: Arc<ConnectionPool>,
+}
+
+impl SQLiteFolderRevisionSnapshotPersistence {
+    pub fn new(object_id: &str, pool: Arc<ConnectionPool>) -> Self {
+        Self {
+            object_id: object_id.to_string(),
+            pool,
+        }
+    }
+
+    fn gen_snapshot_id(&self, rev_id: i64) -> String {
+        format!("{}:{}", self.object_id, rev_id)
+    }
+}
+
+impl RevisionSnapshotDiskCache for SQLiteFolderRevisionSnapshotPersistence {
+    fn should_generate_snapshot_from_range(&self, start_rev_id: i64, current_rev_id: i64) -> bool {
+        (current_rev_id - start_rev_id) >= 2
+    }
+
+    fn write_snapshot(&self, rev_id: i64, data: Vec<u8>) -> FlowyResult<()> {
+        let conn = self.pool.get().map_err(internal_error)?;
+        let snapshot_id = self.gen_snapshot_id(rev_id);
+        let timestamp = timestamp();
+        let record = (
+            dsl::snapshot_id.eq(&snapshot_id),
+            dsl::object_id.eq(&self.object_id),
+            dsl::rev_id.eq(rev_id),
+            dsl::base_rev_id.eq(rev_id),
+            dsl::timestamp.eq(timestamp),
+            dsl::data.eq(data),
+        );
+        let _ = insert_or_ignore_into(dsl::folder_rev_snapshot)
+            .values(record)
+            .execute(&*conn)?;
+        Ok(())
+    }
+
+    fn read_snapshot(&self, rev_id: i64) -> FlowyResult<Option<RevisionSnapshot>> {
+        let conn = self.pool.get().map_err(internal_error)?;
+        let snapshot_id = self.gen_snapshot_id(rev_id);
+        let record = dsl::folder_rev_snapshot
+            .filter(dsl::snapshot_id.eq(&snapshot_id))
+            .first::<FolderSnapshotRecord>(&*conn)?;
+
+        Ok(Some(record.into()))
+    }
+
+    fn read_last_snapshot(&self) -> FlowyResult<Option<RevisionSnapshot>> {
+        let conn = self.pool.get().map_err(internal_error)?;
+        let latest_record = dsl::folder_rev_snapshot
+            .filter(dsl::object_id.eq(&self.object_id))
+            .order(dsl::rev_id.desc())
+            // .select(max(dsl::rev_id))
+            // .select((dsl::id, dsl::object_id, dsl::rev_id, dsl::data))
+            .first::<FolderSnapshotRecord>(&*conn)?;
+        Ok(Some(latest_record.into()))
+    }
+}
+
+#[derive(PartialEq, Clone, Debug, Queryable, Identifiable, Insertable, Associations)]
+#[table_name = "folder_rev_snapshot"]
+#[primary_key("snapshot_id")]
+struct FolderSnapshotRecord {
+    snapshot_id: String,
+    object_id: String,
+    rev_id: i64,
+    base_rev_id: i64,
+    timestamp: i64,
+    data: Vec<u8>,
+}
+
+impl std::convert::From<FolderSnapshotRecord> for RevisionSnapshot {
+    fn from(record: FolderSnapshotRecord) -> Self {
+        RevisionSnapshot {
+            rev_id: record.rev_id,
+            base_rev_id: record.base_rev_id,
+            timestamp: record.timestamp,
+            data: Bytes::from(record.data),
+        }
+    }
+}
diff --git a/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/mod.rs b/frontend/rust-lib/flowy-folder/src/services/persistence/rev_sqlite/mod.rs
@@ -1,2 +1,5 @@
 mod folder_rev_sqlite;
+mod folder_snapshot_sqlite_impl;
+
 pub use folder_rev_sqlite::*;
+pub use folder_snapshot_sqlite_impl::*;
diff --git a/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_snapshot_sqlite_impl.rs b/frontend/rust-lib/flowy-grid/src/services/persistence/rev_sqlite/grid_snapshot_sqlite_impl.rs
@@ -115,35 +115,3 @@ impl std::convert::From<GridSnapshotRecord> for RevisionSnapshot {
         }
     }
 }
-
-// pub(crate) fn get_latest_rev_id_from(rev_ids: Vec<i64>, anchor: i64) -> Option<i64> {
-//     let mut target_rev_id = None;
-//     let mut old_step: Option<i64> = None;
-//     for rev_id in rev_ids {
-//         let step = (rev_id - anchor).abs();
-//         if let Some(old_step) = &mut old_step {
-//             if *old_step > step {
-//                 *old_step = step;
-//                 target_rev_id = Some(rev_id);
-//             }
-//         } else {
-//             old_step = Some(step);
-//             target_rev_id = Some(rev_id);
-//         }
-//     }
-//     target_rev_id
-// }
-
-// #[cfg(test)]
-// mod tests {
-//     use crate::services::persistence::rev_sqlite::get_latest_rev_id_from;
-//
-//     #[test]
-//     fn test_latest_rev_id() {
-//         let ids = vec![1, 2, 3, 4, 5, 6];
-//         for (anchor, expected_value) in vec![(3, 3), (7, 6), (1, 1)] {
-//             let value = get_latest_rev_id_from(ids.clone(), anchor).unwrap();
-//             assert_eq!(value, expected_value);
-//         }
-//     }
-// }
diff --git a/frontend/rust-lib/flowy-revision/src/rev_snapshot.rs b/frontend/rust-lib/flowy-revision/src/rev_snapshot.rs

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+-- This file should undo anything in `up.sql`
	`2`	`+DROP TABLE folder_rev_snapshot;`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,4 @@`
`1`		`-// @generated automatically by Diesel CLI.`
`2`		`-`
`3`		`-diesel::table! {`
	`1`	`+table! {`
`4`	`2`	`app_table (id) {`
`5`	`3`	`id -> Text,`
`6`	`4`	`workspace_id -> Text,`
`@@ -15,7 +13,7 @@ diesel::table! {`
`15`	`13`	`}`
`16`	`14`	`}`
`17`	`15`
`18`		`-diesel::table! {`
	`16`	`+table! {`
`19`	`17`	`document_rev_table (id) {`
`20`	`18`	`id -> Integer,`
`21`	`19`	`document_id -> Text,`
`@@ -26,14 +24,25 @@ diesel::table! {`
`26`	`24`	`}`
`27`	`25`	`}`
`28`	`26`
`29`		`-diesel::table! {`
	`27`	`+table! {`
	`28`	`+ folder_rev_snapshot (snapshot_id) {`
	`29`	`+ snapshot_id -> Text,`
	`30`	`+ object_id -> Text,`
	`31`	`+ rev_id -> BigInt,`
	`32`	`+ base_rev_id -> BigInt,`
	`33`	`+ timestamp -> BigInt,`
	`34`	`+ data -> Binary,`
	`35`	`+ }`
	`36`	`+}`
	`37`	`+`
	`38`	`+table! {`
`30`	`39`	`grid_block_index_table (row_id) {`
`31`	`40`	`row_id -> Text,`
`32`	`41`	`block_id -> Text,`
`33`	`42`	`}`
`34`	`43`	`}`
`35`	`44`
`36`		`-diesel::table! {`
	`45`	`+table! {`
`37`	`46`	`grid_meta_rev_table (id) {`
`38`	`47`	`id -> Integer,`
`39`	`48`	`object_id -> Text,`
`@@ -44,7 +53,7 @@ diesel::table! {`
`44`	`53`	`}`
`45`	`54`	`}`
`46`	`55`
`47`		`-diesel::table! {`
	`56`	`+table! {`
`48`	`57`	`grid_rev_snapshot (snapshot_id) {`
`49`	`58`	`snapshot_id -> Text,`
`50`	`59`	`object_id -> Text,`
`@@ -55,7 +64,7 @@ diesel::table! {`
`55`	`64`	`}`
`56`	`65`	`}`
`57`	`66`
`58`		`-diesel::table! {`
	`67`	`+table! {`
`59`	`68`	`grid_rev_table (id) {`
`60`	`69`	`id -> Integer,`
`61`	`70`	`object_id -> Text,`
`@@ -66,7 +75,7 @@ diesel::table! {`
`66`	`75`	`}`
`67`	`76`	`}`
`68`	`77`
`69`		`-diesel::table! {`
	`78`	`+table! {`
`70`	`79`	`grid_view_rev_table (id) {`
`71`	`80`	`id -> Integer,`
`72`	`81`	`object_id -> Text,`
`@@ -77,14 +86,14 @@ diesel::table! {`
`77`	`86`	`}`
`78`	`87`	`}`
`79`	`88`
`80`		`-diesel::table! {`
	`89`	`+table! {`
`81`	`90`	`kv_table (key) {`
`82`	`91`	`key -> Text,`
`83`	`92`	`value -> Binary,`
`84`	`93`	`}`
`85`	`94`	`}`
`86`	`95`
`87`		`-diesel::table! {`
	`96`	`+table! {`
`88`	`97`	`rev_snapshot (id) {`
`89`	`98`	`id -> Integer,`
`90`	`99`	`object_id -> Text,`
`@@ -93,7 +102,7 @@ diesel::table! {`
`93`	`102`	`}`
`94`	`103`	`}`
`95`	`104`
`96`		`-diesel::table! {`
	`105`	`+table! {`
`97`	`106`	`rev_table (id) {`
`98`	`107`	`id -> Integer,`
`99`	`108`	`doc_id -> Text,`
`@@ -105,7 +114,7 @@ diesel::table! {`
`105`	`114`	`}`
`106`	`115`	`}`
`107`	`116`
`108`		`-diesel::table! {`
	`117`	`+table! {`
`109`	`118`	`trash_table (id) {`
`110`	`119`	`id -> Text,`
`111`	`120`	`name -> Text,`
`@@ -116,7 +125,7 @@ diesel::table! {`
`116`	`125`	`}`
`117`	`126`	`}`
`118`	`127`
`119`		`-diesel::table! {`
	`128`	`+table! {`
`120`	`129`	`user_table (id) {`
`121`	`130`	`id -> Text,`
`122`	`131`	`name -> Text,`
`@@ -127,7 +136,7 @@ diesel::table! {`
`127`	`136`	`}`
`128`	`137`	`}`
`129`	`138`
`130`		`-diesel::table! {`
	`139`	`+table! {`
`131`	`140`	`view_table (id) {`
`132`	`141`	`id -> Text,`
`133`	`142`	`belong_to_id -> Text,`
`@@ -143,7 +152,7 @@ diesel::table! {`
`143`	`152`	`}`
`144`	`153`	`}`
`145`	`154`
`146`		`-diesel::table! {`
	`155`	`+table! {`
`147`	`156`	`workspace_table (id) {`
`148`	`157`	`id -> Text,`
`149`	`158`	`name -> Text,`
`@@ -155,9 +164,10 @@ diesel::table! {`
`155`	`164`	`}`
`156`	`165`	`}`
`157`	`166`
`158`		`-diesel::allow_tables_to_appear_in_same_query!(`
	`167`	`+allow_tables_to_appear_in_same_query!(`
`159`	`168`	`app_table,`
`160`	`169`	`document_rev_table,`
	`170`	`+ folder_rev_snapshot,`
`161`	`171`	`grid_block_index_table,`
`162`	`172`	`grid_meta_rev_table,`
`163`	`173`	`grid_rev_snapshot,`