IndexedDB: prepare matrix-sdk-indexeddb crate for updating indexed_db_futures to latest version

crates/matrix-sdk-indexeddb/src/crypto_store/migrations/mod.rs

@@ -25,6 +25,7 @@ mod v0_to_v5;
 mod v10_to_v11;
 mod v11_to_v12;
 mod v12_to_v13;
+mod v13_to_v15;
 mod v5_to_v7;
 mod v7;
 mod v7_to_v8;
@@ -156,13 +157,22 @@ pub async fn open_and_upgrade_db(
     }
 
     if old_version < 12 {
-        v11_to_v12::schema_add(name).await?;
+        v11_to_v12::schema_bump(name).await?;
     }
 
     if old_version < 13 {
         v12_to_v13::schema_add(name).await?;
     }
 
+    if old_version < 14 {
+        v13_to_v15::schema_add(name).await?;
+    }
+
+    if old_version < 15 {
+        v13_to_v15::data_migrate(name, serializer).await?;
+        v13_to_v15::schema_delete(name).await?;
+    }
+
     // If you add more migrations here, you'll need to update
     // `tests::EXPECTED_SCHEMA_VERSION`.
 
@@ -190,13 +200,12 @@ type OldVersion = u32;
 /// * `name` - name of the indexeddb database to be upgraded.
 /// * `version` - version we are upgrading to.
 /// * `f` - closure which will be called if the database is below the version
-///   given. It will be called with three arguments `(db, txn, oldver)`, where:
+///   given. It will be called with two arguments `(db, oldver)`, where:
 ///   * `db` - the [`IdbDatabase`]
-///   * `txn` - the database transaction: a [`IdbTransaction`]
 ///   * `oldver` - the version number before the upgrade.
 async fn do_schema_upgrade<F>(name: &str, version: u32, f: F) -> Result<(), DomException>
 where
-    F: Fn(&IdbDatabase, IdbTransaction<'_>, OldVersion) -> Result<(), JsValue> + 'static,
+    F: Fn(&IdbDatabase, OldVersion) -> Result<(), JsValue> + 'static,
 {
     info!("IndexeddbCryptoStore upgrade schema -> v{version} starting");
     let mut db_req: OpenDbRequest = IdbDatabase::open_u32(name, version)?;
@@ -208,7 +217,7 @@ where
         let old_version = evt.old_version() as u32;
 
         // Run the upgrade code we were supplied
-        f(evt.db(), evt.transaction(), old_version)
+        f(evt.db(), old_version)
     }));
 
     let db = db_req.await?;
@@ -267,7 +276,7 @@ mod tests {
     wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
 
     /// The schema version we expect after we open the store.
-    const EXPECTED_SCHEMA_VERSION: u32 = 13;
+    const EXPECTED_SCHEMA_VERSION: u32 = 15;
 
     /// Adjust this to test do a more comprehensive perf test
     const NUM_RECORDS_FOR_PERF: usize = 2_000;
@@ -488,25 +497,22 @@ mod tests {
     /// Test migrating `inbound_group_sessions` data from store v5 to latest,
     /// on a store with encryption disabled.
     #[async_test]
-    async fn test_v8_v10_v12_migration_unencrypted() {
-        test_v8_v10_v12_migration_with_cipher("test_v8_migration_unencrypted", None).await
+    async fn test_v8_v15_migration_unencrypted() {
+        test_v8_v15_migration_with_cipher("test_v8_migration_unencrypted", None).await
     }
 
     /// Test migrating `inbound_group_sessions` data from store v5 to store v8,
     /// on a store with encryption enabled.
     #[async_test]
-    async fn test_v8_v10_v12_migration_encrypted() {
+    async fn test_v8_v15_migration_encrypted() {
         let cipher = StoreCipher::new().unwrap();
-        test_v8_v10_v12_migration_with_cipher(
-            "test_v8_migration_encrypted",
-            Some(Arc::new(cipher)),
-        )
-        .await;
+        test_v8_v15_migration_with_cipher("test_v8_migration_encrypted", Some(Arc::new(cipher)))
+            .await;
     }
 
     /// Helper function for `test_v8_v10_v12_migration_{un,}encrypted`: test
     /// migrating `inbound_group_sessions` data from store v5 to store v12.
-    async fn test_v8_v10_v12_migration_with_cipher(
+    async fn test_v8_v15_migration_with_cipher(
         db_prefix: &str,
         store_cipher: Option<Arc<StoreCipher>>,
     ) {
@@ -549,65 +555,43 @@ mod tests {
         assert!(fetched_backed_up_session.backed_up());
         assert!(!fetched_not_backed_up_session.backed_up());
 
-        // For v10: they have the backed_up_to property and it is indexed
-        assert_matches_v10_schema(&db_name, &store, &fetched_backed_up_session).await;
-
-        // For v12: they have the session_id, sender_key and sender_data_type properties
+        // For v15: they have the backed_up_to property and it is indexed and
+        // they have the session_id, sender_key and sender_data_type properties
         // and they are indexed
-        assert_matches_v12_schema(&db_name, &store, &fetched_backed_up_session).await;
-    }
-
-    async fn assert_matches_v10_schema(
-        db_name: &str,
-        store: &IndexeddbCryptoStore,
-        fetched_backed_up_session: &InboundGroupSession,
-    ) {
-        let db = IdbDatabase::open(&db_name).unwrap().await.unwrap();
-        assert!(db.version() >= 10.0);
-        let transaction = db.transaction_on_one("inbound_group_sessions3").unwrap();
-        let raw_store = transaction.object_store("inbound_group_sessions3").unwrap();
-        let key = store.serializer.encode_key(
-            keys::INBOUND_GROUP_SESSIONS_V3,
-            (fetched_backed_up_session.room_id(), fetched_backed_up_session.session_id()),
-        );
-        let idb_object: InboundGroupSessionIndexedDbObject =
-            serde_wasm_bindgen::from_value(raw_store.get(&key).unwrap().await.unwrap().unwrap())
-                .unwrap();
-
-        assert_eq!(idb_object.backed_up_to, -1);
-        assert!(raw_store.index_names().find(|idx| idx == "backed_up_to").is_some());
-
-        db.close();
+        assert_matches_v15_schema(&db_name, &store, &fetched_backed_up_session).await;
     }
 
-    async fn assert_matches_v12_schema(
+    async fn assert_matches_v15_schema(
         db_name: &str,
         store: &IndexeddbCryptoStore,
         session: &InboundGroupSession,
     ) {
         let db = IdbDatabase::open(&db_name).unwrap().await.unwrap();
-        assert!(db.version() >= 12.0);
-        let transaction = db.transaction_on_one("inbound_group_sessions3").unwrap();
-        let raw_store = transaction.object_store("inbound_group_sessions3").unwrap();
+        assert!(db.version() >= 15.0);
+        let transaction = db.transaction_on_one(keys::INBOUND_GROUP_SESSIONS_V4).unwrap();
+        let raw_store = transaction.object_store(keys::INBOUND_GROUP_SESSIONS_V4).unwrap();
         let key = store
             .serializer
-            .encode_key(keys::INBOUND_GROUP_SESSIONS_V3, (session.room_id(), session.session_id()));
+            .encode_key(keys::INBOUND_GROUP_SESSIONS_V4, (session.room_id(), session.session_id()));
         let idb_object: InboundGroupSessionIndexedDbObject =
             serde_wasm_bindgen::from_value(raw_store.get(&key).unwrap().await.unwrap().unwrap())
                 .unwrap();
 
+        assert_eq!(idb_object.backed_up_to, -1);
+        assert!(raw_store.index_names().find(|idx| idx == "backed_up_to").is_some());
+
         assert_eq!(
             idb_object.session_id,
             Some(
                 store
                     .serializer
-                    .encode_key_as_string(keys::INBOUND_GROUP_SESSIONS_V3, session.session_id())
+                    .encode_key_as_string(keys::INBOUND_GROUP_SESSIONS_V4, session.session_id())
             )
         );
         assert_eq!(
             idb_object.sender_key,
             Some(store.serializer.encode_key_as_string(
-                keys::INBOUND_GROUP_SESSIONS_V3,
+                keys::INBOUND_GROUP_SESSIONS_V4,
                 session.sender_key().to_base64()
             ))
         );

crates/matrix-sdk-indexeddb/src/crypto_store/migrations/old_keys.rs

@@ -24,5 +24,7 @@ pub const INBOUND_GROUP_SESSIONS_V1: &str = "inbound_group_sessions";
 /// Also lacked the `backed_up_to` property+index.
 pub const INBOUND_GROUP_SESSIONS_V2: &str = "inbound_group_sessions2";
 
+pub const INBOUND_GROUP_SESSIONS_V3: &str = "inbound_group_sessions3";
+
 /// An old name for [`BACKUP_VERSION_V1`].
 pub const BACKUP_KEY_V1: &str = "backup_key_v1";

crates/matrix-sdk-indexeddb/src/crypto_store/migrations/v0_to_v5.rs

@@ -26,7 +26,7 @@ use crate::crypto_store::{
 
 /// Perform schema migrations as needed, up to schema version 5.
 pub(crate) async fn schema_add(name: &str) -> Result<(), DomException> {
-    do_schema_upgrade(name, 5, |db, _, old_version| {
+    do_schema_upgrade(name, 5, |db, old_version| {
         // An old_version of 1 could either mean actually the first version of the
         // schema, or a completely empty schema that has been created with a
         // call to `IdbDatabase::open` with no explicit "version". So, to determine

crates/matrix-sdk-indexeddb/src/crypto_store/migrations/v10_to_v11.rs

@@ -59,5 +59,5 @@ pub(crate) async fn data_migrate(
 pub(crate) async fn schema_bump(name: &str) -> crate::crypto_store::Result<(), DomException> {
     // Just bump the version number to 11 to demonstrate that we have run the data
     // changes from data_migrate.
-    do_schema_upgrade(name, 11, |_, _, _| Ok(())).await
+    do_schema_upgrade(name, 11, |_, _| Ok(())).await
 }

crates/matrix-sdk-indexeddb/src/crypto_store/migrations/v11_to_v12.rs

@@ -12,23 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use indexed_db_futures::IdbKeyPath;
 use web_sys::DomException;
 
-use crate::crypto_store::{keys, migrations::do_schema_upgrade, Result};
+use crate::crypto_store::{migrations::do_schema_upgrade, Result};
 
-/// Perform the schema upgrade v11 to v12, adding an index on
-/// `(curve_key, sender_data_type, session_id)` to `inbound_group_sessions3`.
-pub(crate) async fn schema_add(name: &str) -> Result<(), DomException> {
-    do_schema_upgrade(name, 12, |_, transaction, _| {
-        let object_store = transaction.object_store(keys::INBOUND_GROUP_SESSIONS_V3)?;
-
-        object_store.create_index(
-            keys::INBOUND_GROUP_SESSIONS_SENDER_KEY_INDEX,
-            &IdbKeyPath::str_sequence(&["sender_key", "sender_data_type", "session_id"]),
-        )?;
-
-        Ok(())
-    })
-    .await
+/// Perform the schema upgrade v11 to v12, just bumping the schema version
+pub(crate) async fn schema_bump(name: &str) -> Result<(), DomException> {
+    do_schema_upgrade(name, 12, |_, _| Ok(())).await
 }

crates/matrix-sdk-indexeddb/src/crypto_store/migrations/v12_to_v13.rs

@@ -21,7 +21,7 @@ use crate::crypto_store::{keys, migrations::do_schema_upgrade, Result};
 /// Perform the schema upgrade v12 to v13, adding the
 /// `received_room_key_bundles` store.
 pub(crate) async fn schema_add(name: &str) -> Result<(), DomException> {
-    do_schema_upgrade(name, 13, |db, _, _| {
+    do_schema_upgrade(name, 13, |db, _| {
         db.create_object_store(keys::RECEIVED_ROOM_KEY_BUNDLES)?;
         Ok(())
     })

crates/matrix-sdk-indexeddb/src/crypto_store/migrations/v13_to_v15.rs

@@ -0,0 +1,112 @@
+// Copyright 2024 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use indexed_db_futures::{IdbKeyPath, IdbQuerySource};
+use tracing::{debug, info};
+use web_sys::{DomException, IdbTransactionMode};
+
+use crate::{
+    crypto_store::{
+        deserialize_inbound_group_session, keys,
+        migrations::{do_schema_upgrade, old_keys, v8_to_v10, MigrationDb},
+        serialize_inbound_group_session, Result,
+    },
+    serializer::IndexeddbSerializer,
+};
+
+/// Perform the schema upgrade v13 to v14
+///
+/// This creates an identical object store to `inbound_group_sessions3`, but
+/// adds index on `(curve_key, sender_data_type, session_id)`.
+pub(crate) async fn schema_add(name: &str) -> Result<(), DomException> {
+    do_schema_upgrade(name, 14, |db, _| {
+        let object_store = db.create_object_store(keys::INBOUND_GROUP_SESSIONS_V4)?;
+        v8_to_v10::index_add(&object_store)?;
+        object_store.create_index(
+            keys::INBOUND_GROUP_SESSIONS_SENDER_KEY_INDEX,
+            &IdbKeyPath::str_sequence(&["sender_key", "sender_data_type", "session_id"]),
+        )?;
+        Ok(())
+    })
+    .await
+}
+
+/// Migrate data from `inbound_group_sessions3` into `inbound_group_sessions4`.
+pub(crate) async fn data_migrate(name: &str, serializer: &IndexeddbSerializer) -> Result<()> {
+    let db = MigrationDb::new(name, 15).await?;
+
+    let txn = db.transaction_on_multi_with_mode(
+        &[old_keys::INBOUND_GROUP_SESSIONS_V3, keys::INBOUND_GROUP_SESSIONS_V4],
+        IdbTransactionMode::Readwrite,
+    )?;
+
+    let inbound_group_sessions3 = txn.object_store(old_keys::INBOUND_GROUP_SESSIONS_V3)?;
+    let inbound_group_sessions4 = txn.object_store(keys::INBOUND_GROUP_SESSIONS_V4)?;
+
+    let row_count = inbound_group_sessions3.count()?.await?;
+    info!(row_count, "Shrinking inbound_group_session records");
+
+    // Iterate through all rows
+    if let Some(cursor) = inbound_group_sessions3.open_cursor()?.await? {
+        let mut idx = 0;
+        loop {
+            idx += 1;
+
+            if idx % 100 == 0 {
+                debug!("Migrating session {idx} of {row_count}");
+            }
+
+            // Deserialize the session from the old store
+            let session = deserialize_inbound_group_session(cursor.value(), serializer)?;
+
+            // Calculate its key in the new table
+            let new_key = serializer.encode_key(
+                keys::INBOUND_GROUP_SESSIONS_V4,
+                (&session.room_id, session.session_id()),
+            );
+
+            // Serialize the session in the new format
+            let new_session = serialize_inbound_group_session(&session, serializer).await?;
+
+            // Write it to the new store
+            inbound_group_sessions4.add_key_val(&new_key, &new_session)?;
+
+            // We are done with the original data, so delete it now.
+            cursor.delete()?;
+
+            // Continue to the next record, or stop if we're done
+            if !cursor.continue_cursor()?.await? {
+                debug!("Migrated {idx} sessions.");
+                break;
+            }
+        }
+    }
+
+    // We have finished with the old store. Clear it, since it is faster to
+    // clear+delete than just delete. See https://www.artificialworlds.net/blog/2024/02/02/deleting-an-indexed-db-store-can-be-incredibly-slow-on-firefox/
+    // for more details.
+    inbound_group_sessions3.clear()?.await?;
+
+    txn.await.into_result()?;
+    Ok(())
+}
+
+/// Perform the schema upgrade v14 to v15, deleting `inbound_group_sessions3`.
+pub(crate) async fn schema_delete(name: &str) -> Result<(), DomException> {
+    do_schema_upgrade(name, 15, |db, _| {
+        db.delete_object_store(old_keys::INBOUND_GROUP_SESSIONS_V3)?;
+        Ok(())
+    })
+    .await
+}

crates/matrix-sdk-indexeddb/src/crypto_store/migrations/v5_to_v7.rs

@@ -36,7 +36,7 @@ use crate::{
 
 /// Perform the schema upgrade v5 to v6, creating `inbound_group_sessions2`.
 pub(crate) async fn schema_add(name: &str) -> Result<(), DomException> {
-    do_schema_upgrade(name, 6, |db, _, _| {
+    do_schema_upgrade(name, 6, |db, _| {
         let object_store = db.create_object_store(old_keys::INBOUND_GROUP_SESSIONS_V2)?;
 
         add_nonunique_index(
@@ -109,7 +109,7 @@ pub(crate) async fn data_migrate(name: &str, serializer: &IndexeddbSerializer) -
 
 /// Perform the schema upgrade v6 to v7, deleting `inbound_group_sessions`.
 pub(crate) async fn schema_delete(name: &str) -> Result<(), DomException> {
-    do_schema_upgrade(name, 7, |db, _, _| {
+    do_schema_upgrade(name, 7, |db, _| {
         db.delete_object_store(old_keys::INBOUND_GROUP_SESSIONS_V1)?;
         Ok(())
     })

crates/matrix-sdk-indexeddb/src/crypto_store/migrations/v7_to_v8.rs

@@ -116,7 +116,7 @@ pub(crate) async fn data_migrate(name: &str, serializer: &IndexeddbSerializer) -
 
 /// Perform the schema upgrade v7 to v8, Just bumping the schema version.
 pub(crate) async fn schema_bump(name: &str) -> Result<(), DomException> {
-    do_schema_upgrade(name, 8, |_, _, _| {
+    do_schema_upgrade(name, 8, |_, _| {
         // Just bump the version number to 8 to demonstrate that we have run the data
         // changes from prepare_data_for_v8.
         Ok(())