Improve the list_blob_ids / list_chain_ids (#4886)

## Motivation

Several changes suggested by the work on the storage are integrated
here.

## Proposal

The following changes have been made:
* The `BLOB_ID_LENGTH` / `CHAIN_ID_LENGTH` are eliminated from the code
just as in TestNet Conway.
* The `list_blob_ids` /  `list_chain_ids` are part of `DbStorage`.
* The functions are then integrated into the storage trait.

## Test Plan

The CI.

## Release Plan

- Nothing to do / These changes follow the usual release cycle.

## Links

None.

Author: MathieuDutSik

CLI.md

@@ -71,6 +71,7 @@ This document contains the help content for the `linera` command-line program.
 * [`linera storage list-namespaces`↴](#linera-storage-list-namespaces)
 * [`linera storage list-blob-ids`↴](#linera-storage-list-blob-ids)
 * [`linera storage list-chain-ids`↴](#linera-storage-list-chain-ids)
+* [`linera storage list-event-ids`↴](#linera-storage-list-event-ids)
 
 ## `linera`
 
@@ -1360,6 +1361,7 @@ Operation on the storage
 * `list-namespaces` — List the namespaces in the database
 * `list-blob-ids` — List the blob IDs in the database
 * `list-chain-ids` — List the chain IDs in the database
+* `list-event-ids` — List the event IDs in the database
 
 
 
@@ -1423,6 +1425,14 @@ List the chain IDs in the database
 
 
 
+## `linera storage list-event-ids`
+
+List the event IDs in the database
+
+**Usage:** `linera storage list-event-ids`
+
+
+
 <hr/>
 
 <small><i>

linera-base/src/identifiers.rs

@@ -622,6 +622,12 @@ pub struct EventId {
     pub index: u32,
 }
 
+impl fmt::Display for EventId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}:{}:{}", self.chain_id, self.stream_id, self.index)
+    }
+}
+
 impl StreamName {
     /// Turns the stream name into bytes.
     pub fn into_bytes(self) -> Vec<u8> {

linera-service/src/cli/command.rs

@@ -1130,6 +1130,9 @@ pub enum DatabaseToolCommand {
 
     /// List the chain IDs in the database
     ListChainIds,
+
+    /// List the event IDs in the database
+    ListEventIds,
 }
 
 #[allow(clippy::large_enum_variant)]

linera-service/src/cli/main.rs

@@ -2100,15 +2100,19 @@ impl RunnableWithStore for DatabaseToolJob<'_> {
                 }
             }
             DatabaseToolCommand::ListBlobIds => {
-                let blob_ids = DbStorage::<D, _>::list_blob_ids(&config, &namespace).await?;
+                let storage =
+                    DbStorage::<D, _>::maybe_create_and_connect(&config, &namespace, None).await?;
+                let blob_ids = storage.list_blob_ids().await?;
                 info!("Blob IDs listed in {} ms", start_time.elapsed().as_millis());
                 info!("The list of blob IDs is:");
                 for id in blob_ids {
                     println!("{}", id);
                 }
             }
             DatabaseToolCommand::ListChainIds => {
-                let chain_ids = DbStorage::<D, _>::list_chain_ids(&config, &namespace).await?;
+                let storage =
+                    DbStorage::<D, _>::maybe_create_and_connect(&config, &namespace, None).await?;
+                let chain_ids = storage.list_chain_ids().await?;
                 info!(
                     "Chain IDs listed in {} ms",
                     start_time.elapsed().as_millis()
@@ -2118,6 +2122,19 @@ impl RunnableWithStore for DatabaseToolJob<'_> {
                     println!("{}", id);
                 }
             }
+            DatabaseToolCommand::ListEventIds => {
+                let storage =
+                    DbStorage::<D, _>::maybe_create_and_connect(&config, &namespace, None).await?;
+                let event_ids = storage.list_event_ids().await?;
+                info!(
+                    "Event IDs listed in {} ms",
+                    start_time.elapsed().as_millis()
+                );
+                info!("The list of event IDs is:");
+                for id in event_ids {
+                    println!("{}", id);
+                }
+            }
         }
         Ok(0)
     }

linera-storage/src/db_storage.rs

@@ -1,7 +1,7 @@
 // Copyright (c) Zefchain Labs, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-use std::{fmt::Debug, sync::Arc};
+use std::{fmt::Debug, ops::Deref, sync::Arc};
 
 use async_trait::async_trait;
 #[cfg(with_metrics)]
@@ -337,16 +337,28 @@ enum RootKey {
     NetworkDescription,
 }
 
+const CHAIN_ID_TAG: u8 = 0;
+const BLOB_ID_TAG: u8 = 2;
+const EVENT_ID_TAG: u8 = 3;
+
 impl RootKey {
     fn bytes(&self) -> Vec<u8> {
         bcs::to_bytes(self).unwrap()
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+struct RestrictedEventId {
+    pub stream_id: StreamId,
+    pub index: u32,
+}
+
 fn event_key(event_id: &EventId) -> Vec<u8> {
-    let mut key = bcs::to_bytes(&event_id.stream_id).unwrap();
-    key.extend(bcs::to_bytes(&event_id.index).unwrap());
-    key
+    let restricted_event_id = RestrictedEventId {
+        stream_id: event_id.stream_id.clone(),
+        index: event_id.index,
+    };
+    bcs::to_bytes(&restricted_event_id).unwrap()
 }
 
 fn is_chain_state(root_key: &[u8]) -> bool {
@@ -356,11 +368,6 @@ fn is_chain_state(root_key: &[u8]) -> bool {
     root_key[0] == CHAIN_ID_TAG
 }
 
-const CHAIN_ID_TAG: u8 = 0;
-const BLOB_ID_TAG: u8 = 2;
-const CHAIN_ID_LENGTH: usize = std::mem::size_of::<ChainId>();
-const BLOB_ID_LENGTH: usize = std::mem::size_of::<BlobId>();
-
 #[cfg(test)]
 mod tests {
     use linera_base::{
@@ -371,9 +378,7 @@ mod tests {
         },
     };
 
-    use crate::db_storage::{
-        event_key, RootKey, BLOB_ID_LENGTH, BLOB_ID_TAG, CHAIN_ID_LENGTH, CHAIN_ID_TAG,
-    };
+    use crate::db_storage::{event_key, RootKey, BLOB_ID_TAG, CHAIN_ID_TAG, EVENT_ID_TAG};
 
     // Several functionalities of the storage rely on the way that the serialization
     // is done. Thus we need to check that the serialization works in the way that
@@ -388,7 +393,7 @@ mod tests {
         let blob_id = BlobId::new(hash, blob_type);
         let root_key = RootKey::Blob(blob_id).bytes();
         assert_eq!(root_key[0], BLOB_ID_TAG);
-        assert_eq!(root_key.len(), 1 + BLOB_ID_LENGTH);
+        assert_eq!(bcs::from_bytes::<BlobId>(&root_key[1..]).unwrap(), blob_id);
     }
 
     // The listing of the chains in `list_chain_ids` depends on the serialization
@@ -399,7 +404,10 @@ mod tests {
         let chain_id = ChainId(hash);
         let root_key = RootKey::ChainState(chain_id).bytes();
         assert_eq!(root_key[0], CHAIN_ID_TAG);
-        assert_eq!(root_key.len(), 1 + CHAIN_ID_LENGTH);
+        assert_eq!(
+            bcs::from_bytes::<ChainId>(&root_key[1..]).unwrap(),
+            chain_id
+        );
     }
 
     // The listing of the events in `read_events_from_index` depends on the
@@ -426,6 +434,12 @@ mod tests {
         };
         let key = event_key(&event_id);
         assert!(key.starts_with(&prefix));
+        let root_key = RootKey::Event(chain_id).bytes();
+        assert_eq!(root_key[0], EVENT_ID_TAG);
+        assert_eq!(
+            bcs::from_bytes::<ChainId>(&root_key[1..]).unwrap(),
+            chain_id
+        );
     }
 }
 
@@ -985,6 +999,56 @@ where
         let store = self.database.open_exclusive(&root_key)?;
         Ok(ViewContext::create_root_context(store, block_exporter_id).await?)
     }
+
+    async fn list_blob_ids(&self) -> Result<Vec<BlobId>, ViewError> {
+        let root_keys = self.database.list_root_keys().await?;
+        let mut blob_ids = Vec::new();
+        for root_key in root_keys {
+            if !root_key.is_empty() && root_key[0] == BLOB_ID_TAG {
+                let root_key_red = &root_key[1..];
+                let blob_id = bcs::from_bytes(root_key_red)?;
+                blob_ids.push(blob_id);
+            }
+        }
+        Ok(blob_ids)
+    }
+
+    async fn list_chain_ids(&self) -> Result<Vec<ChainId>, ViewError> {
+        let root_keys = self.database.list_root_keys().await?;
+        let mut chain_ids = Vec::new();
+        for root_key in root_keys {
+            if !root_key.is_empty() && root_key[0] == CHAIN_ID_TAG {
+                let root_key_red = &root_key[1..];
+                let chain_id = bcs::from_bytes(root_key_red)?;
+                chain_ids.push(chain_id);
+            }
+        }
+        Ok(chain_ids)
+    }
+
+    async fn list_event_ids(&self) -> Result<Vec<EventId>, ViewError> {
+        let database = self.database.deref();
+        let root_keys = database.list_root_keys().await?;
+        let mut event_ids = Vec::new();
+        for root_key in root_keys {
+            if !root_key.is_empty() && root_key[0] == EVENT_ID_TAG {
+                let root_key_red = &root_key[1..];
+                let chain_id = bcs::from_bytes(root_key_red)?;
+                let store = database.open_shared(&root_key)?;
+                let keys = store.find_keys_by_prefix(&[]).await?;
+                for key in keys {
+                    let restricted_event_id = bcs::from_bytes::<RestrictedEventId>(&key)?;
+                    let event_id = EventId {
+                        chain_id,
+                        stream_id: restricted_event_id.stream_id,
+                        index: restricted_event_id.index,
+                    };
+                    event_ids.push(event_id);
+                }
+            }
+        }
+        Ok(event_ids)
+    }
 }
 
 impl<Database, C> DbStorage<Database, C>
@@ -1085,48 +1149,6 @@ where
         let database = Database::connect(config, namespace).await?;
         Ok(Self::new(database, wasm_runtime, WallClock))
     }
-
-    /// Lists the blob IDs of the storage.
-    pub async fn list_blob_ids(
-        config: &Database::Config,
-        namespace: &str,
-    ) -> Result<Vec<BlobId>, ViewError> {
-        let database = Database::connect(config, namespace).await?;
-        let root_keys = database.list_root_keys().await?;
-        let mut blob_ids = Vec::new();
-        for root_key in root_keys {
-            if root_key.len() == 1 + BLOB_ID_LENGTH && root_key[0] == BLOB_ID_TAG {
-                let root_key_red = &root_key[1..=BLOB_ID_LENGTH];
-                let blob_id = bcs::from_bytes(root_key_red)?;
-                blob_ids.push(blob_id);
-            }
-        }
-        Ok(blob_ids)
-    }
-}
-
-impl<Database> DbStorage<Database, WallClock>
-where
-    Database: KeyValueDatabase + Clone + Send + Sync + 'static,
-    Database::Error: Send + Sync,
-{
-    /// Lists the chain IDs of the storage.
-    pub async fn list_chain_ids(
-        config: &Database::Config,
-        namespace: &str,
-    ) -> Result<Vec<ChainId>, ViewError> {
-        let database = Database::connect(config, namespace).await?;
-        let root_keys = database.list_root_keys().await?;
-        let mut chain_ids = Vec::new();
-        for root_key in root_keys {
-            if root_key.len() == 1 + CHAIN_ID_LENGTH && root_key[0] == CHAIN_ID_TAG {
-                let root_key_red = &root_key[1..=CHAIN_ID_LENGTH];
-                let chain_id = bcs::from_bytes(root_key_red)?;
-                chain_ids.push(chain_id);
-            }
-        }
-        Ok(chain_ids)
-    }
 }
 
 #[cfg(with_testing)]

linera-storage/src/lib.rs

@@ -400,6 +400,15 @@ pub trait Storage: Sized {
         &self,
         block_exporter_id: u32,
     ) -> Result<Self::BlockExporterContext, ViewError>;
+
+    /// Lists the blob IDs in storage.
+    async fn list_blob_ids(&self) -> Result<Vec<BlobId>, ViewError>;
+
+    /// Lists the chain IDs in storage.
+    async fn list_chain_ids(&self) -> Result<Vec<ChainId>, ViewError>;
+
+    /// Lists the event IDs in storage.
+    async fn list_event_ids(&self) -> Result<Vec<EventId>, ViewError>;
 }
 
 /// The result of processing the obtained read certificates.