Merge pull request #1077 from openmina/feat/archive-redundancy

Archive redundancy with precomputed blocks storages

Author: adonagy

Cargo.lock

@@ -17,7 +17,7 @@ use node::p2p::identity::SecretKey;
 use node::service::Recorder;
 use node::SnarkerStrategy;
 
-use openmina_node_native::{tracing, NodeBuilder};
+use openmina_node_native::{archive::config::ArchiveStorageOptions, tracing, NodeBuilder};
 
 /// Openmina node
 #[derive(Debug, clap::Args)]
@@ -138,9 +138,39 @@ pub struct Node {
     #[arg(short = 'c', long, env)]
     pub config: Option<PathBuf>,
 
-    /// Enable archive mode (seding blocks to the archive process).
+    /// Enable local precomputed storage.
+    ///
+    /// This option requires the following environment variables to be set:
+    /// - OPENMINA_ARCHIVE_LOCAL_STORAGE_PATH (otherwise the path to the working directory will be used)
+    #[arg(long, env)]
+    pub archive_local_storage: bool,
+
+    /// Enable archiver process.
+    ///
+    /// This requires the following environment variables to be set:
+    /// - OPENMINA_ARCHIVE_ADDRESS
+    #[arg(long, env)]
+    pub archive_archiver_process: bool,
+
+    /// Enable GCP precomputed storage.
+    ///
+    /// This requires the following environment variables to be set:
+    /// - GCP_CREDENTIALS_JSON
+    /// - GCP_BUCKET_NAME
+    ///
     #[arg(long, env)]
-    pub archive_address: Option<Url>,
+    pub archive_gcp_storage: bool,
+
+    /// Enable AWS precomputed storage.
+    ///
+    /// This requires the following environment variables to be set:
+    /// - AWS_ACCESS_KEY_ID
+    /// - AWS_SECRET_ACCESS_KEY
+    /// - AWS_SESSION_TOKEN
+    /// - AWS_DEFAULT_REGION
+    /// - OPENMINA_AWS_BUCKET_NAME
+    #[arg(long, env)]
+    pub archive_aws_storage: bool,
 }
 
 impl Node {
@@ -272,23 +302,44 @@ impl Node {
             }
         }
 
-        if let Some(address) = self.archive_address {
-            openmina_core::IS_ARCHIVE
-                .set(true)
-                .expect("IS_ARCHIVE already set");
+        let archive_storage_options = ArchiveStorageOptions::from_iter(
+            [
+                (
+                    self.archive_local_storage,
+                    ArchiveStorageOptions::LOCAL_PRECOMPUTED_STORAGE,
+                ),
+                (
+                    self.archive_archiver_process,
+                    ArchiveStorageOptions::ARCHIVER_PROCESS,
+                ),
+                (
+                    self.archive_gcp_storage,
+                    ArchiveStorageOptions::GCP_PRECOMPUTED_STORAGE,
+                ),
+                (
+                    self.archive_aws_storage,
+                    ArchiveStorageOptions::AWS_PRECOMPUTED_STORAGE,
+                ),
+            ]
+            .iter()
+            .filter(|(enabled, _)| *enabled)
+            .map(|(_, option)| option.clone()),
+        );
+
+        if archive_storage_options.is_enabled() {
             node::core::info!(
                 summary = "Archive mode enabled",
-                address = address.to_string()
+                local_storage = archive_storage_options.uses_local_precomputed_storage(),
+                archiver_process = archive_storage_options.uses_archiver_process(),
+                gcp_storage = archive_storage_options.uses_gcp_precomputed_storage(),
+                aws_storage = archive_storage_options.uses_aws_precomputed_storage(),
             );
-            // Convert URL to SocketAddr
-            let socket_addrs = address.socket_addrs(|| None).expect("Invalid URL");
 
-            let socket_addr = socket_addrs.first().expect("No socket address found");
-            node_builder.archive(*socket_addr);
-        } else {
-            openmina_core::IS_ARCHIVE
-                .set(false)
-                .expect("IS_ARCHIVE already set");
+            archive_storage_options
+                .validate_env_vars()
+                .map_err(|e| anyhow::anyhow!(e))?;
+
+            node_builder.archive(archive_storage_options, work_dir.clone());
         }
 
         if let Some(sec_key) = self.run_snarker {

cli/src/commands/node/mod.rs

@@ -24,7 +24,6 @@ pub mod transaction;
 pub mod consensus;
 
 mod substate;
-use std::sync::OnceLock;
 
 pub use substate::{Substate, SubstateAccess, SubstateResult};
 
@@ -37,9 +36,6 @@ pub use chain_id::*;
 pub mod encrypted_key;
 pub use encrypted_key::*;
 
-// FIXME(#1043): refactor
-pub static IS_ARCHIVE: OnceLock<bool> = OnceLock::new();
-
 mod work_dir {
     use once_cell::sync::OnceCell;
     use std::path::PathBuf;

core/src/lib.rs

@@ -137,9 +137,11 @@ services:
   node-openmina:
     image: adrnagy/openmina:archive-test
     container_name: node-openmina
+    environment:
+      OPENMINA_ARCHIVE_ADDRESS: http://archive-openmina:3087
     command: >
       node
-      --archive-address http://archive-openmina:3087
+      --archive-archiver-process
     ports:
       - "3000:3000"
     depends_on:

docker-compose.archive.devnet.compare.yml

@@ -63,9 +63,11 @@ services:
   node-openmina:
     image: openmina/openmina:latest
     container_name: node-openmina
+    environment:
+      - OPENMINA_ARCHIVE_ADDRESS=http://archive-openmina:3086
     command: >
       node
-      --archive-address http://archive-openmina:3086
+      --archive-archiver-process
     ports:
       - "127.0.0.1:3000:3000"
     depends_on:

docker-compose.archive.devnet.yml

@@ -2,15 +2,61 @@
 
 This guide is intended for setting up archive nodes on **Mina Devnet** only. Do not use this guide for Mina Mainnet
 
+## Archive Mode Configuration
+
+We start archive mode in openmina by setting one of the following flags along with their associated environment variables:
+
+### Archiver Process (`--archive-archiver-process`)
+
+Stores blocks in a database by receiving them directly from the openmina node
+
+**Required Environment Variables**:
+- `OPENMINA_ARCHIVE_ADDRESS`: Network address for the archiver service
+
+### Local Storage (`--archive-local-storage`)
+
+Stores blocks in the local filesystem
+
+**Required Environment Variables**:
+- (None)
+
+**Optional Environment Variables**:
+- `OPENMINA_ARCHIVE_LOCAL_STORAGE_PATH`: Custom path for block storage (default: ~/.openmina/archive-precomputed)
+
+### GCP Storage (`--archive-gcp-storage`)
+
+Uploads blocks to a Google Cloud Platform bucket
+
+**Required Environment Variables**:
+- `GCP_CREDENTIALS_JSON`: Service account credentials JSON
+- `GCP_BUCKET_NAME`: Target storage bucket name
+
+### AWS Storage (`--archive-aws-storage`)
+
+Uploads blocks to an AWS S3 bucket
+
+**Required Environment Variables**:
+- `AWS_ACCESS_KEY_ID`: IAM user access key
+- `AWS_SECRET_ACCESS_KEY`: IAM user secret key
+- `AWS_DEFAULT_REGION`: AWS region name
+- `AWS_SESSION_TOKEN`: Temporary session token for temporary credentials
+- `OPENMINA_AWS_BUCKET_NAME`: Target S3 bucket name
+
+## Redundancy
+
+The archive mode is designed to be redundant. We can combine the flags to have multiple options running simultaneously.
+
 ## Prerequisites
 
 Ensure Docker and Docker Compose are installed on your system - [Docker Installation Guide](./docker-installation.md)
 
-## Docker compose setup
+## Docker compose setup (with archiver process)
+
+The compose file sets up a PG database, the archiver process and the openmina node. The archiver process is responsible for storing the blocks in the database by receiving the blocks from the openmina node.
 
-The compose file sets up a PG database, the archiver process and the openmina node. The archiver process is responsible for storing the blocks in the database by receiving the blocks from the openmina node. We start the archive mode in openmina by setting the `--archive-mode` flag to the address fo archiver process. See [docker-compose.archive.devnet.yml](../docker-compose.archive.devnet.yml) for more details.
+See [docker-compose.archive.devnet.yml](../docker-compose.archive.devnet.yml) for more details.
 
-## Starting the setup
+### Starting the setup
 
 ```bash
 docker compose -f docker-compose.archive.devnet.yml up -d

docs/archive-node-guide.md

@@ -300,7 +300,12 @@ impl LedgerIntf for Mask {
     /// public API of this module.
     /// This should *NOT* be used to create a ledger for other purposes.
     fn create_masked(&self) -> Self {
-        let mask = Mask::new_unattached(self.depth() as usize);
+        let mut mask = Mask::new_unattached(self.depth() as usize);
+
+        if self.has_token_owners() {
+            mask.set_token_owners();
+        }
+
         // We don't register the mask here. This is only used in transaction logic,
         // where we don't want to unregister. Transaction logic is also
         // synchronous, so we don't need to worry that our mask will be reparented.

ledger/src/base.rs

@@ -61,6 +61,20 @@ impl Database<V2> {
         Self::create_with_dir(depth, None)
     }
 
+    pub fn create_with_token_owners(depth: u8) -> Self {
+        let mut db = Self::create_with_dir(depth, None);
+        db.set_token_owners();
+        db
+    }
+
+    pub fn set_token_owners(&mut self) {
+        self.with(|this| this.set_token_owners());
+    }
+
+    pub fn unset_token_owners(&mut self) {
+        self.with(|this| this.unset_token_owners());
+    }
+
     pub fn root_hash(&mut self) -> Fp {
         self.with(|this| this.root_hash())
     }
@@ -105,6 +119,10 @@ impl Database<V2> {
         self.with(|this| this.transfert_hashes(hashes))
     }
 
+    pub fn has_token_owners(&self) -> bool {
+        self.with(|this| this.has_token_owners())
+    }
+
     #[cfg(test)]
     pub fn test_matrix(&self) -> HashesMatrix {
         self.with(|this| this.hashes_matrix.clone())

ledger/src/database/database.rs

@@ -6,7 +6,6 @@ use std::{
 
 use mina_hasher::Fp;
 use mina_signer::CompressedPubKey;
-use openmina_core::IS_ARCHIVE;
 
 use crate::{
     next_uuid, Account, AccountId, AccountIndex, AccountLegacy, Address, AddressIterator,
@@ -279,6 +278,10 @@ impl DatabaseImpl<V2> {
     pub fn transfert_hashes(&mut self, hashes: HashesMatrix) {
         self.hashes_matrix.transfert_hashes(hashes)
     }
+
+    pub fn has_token_owners(&self) -> bool {
+        self.token_owners.is_some()
+    }
 }
 
 impl DatabaseImpl<V1> {
@@ -311,11 +314,12 @@ impl DatabaseImpl<V1> {
 }
 
 impl DatabaseImpl<V2> {
+    const NACCOUNTS: usize = 10_000;
+    const NTOKENS: usize = 10;
+
     pub fn create_with_dir(depth: u8, dir_name: Option<PathBuf>) -> Self {
         assert!((1..0xfe).contains(&depth));
 
-        const NACCOUNTS: usize = 10_000;
-
         let uuid = next_uuid();
 
         let path = match dir_name {
@@ -339,19 +343,13 @@ impl DatabaseImpl<V2> {
 
         // std::fs::create_dir_all(&path).ok();
 
-        let token_owners = if IS_ARCHIVE.get().cloned().unwrap_or_default() {
-            Some(HashMap::with_capacity(NACCOUNTS))
-        } else {
-            None
-        };
-
         Self {
             depth,
-            accounts: Vec::with_capacity(NACCOUNTS),
+            accounts: Vec::with_capacity(Self::NACCOUNTS),
             last_location: None,
             naccounts: 0,
-            id_to_addr: HashMap::with_capacity(NACCOUNTS),
-            token_owners,
+            id_to_addr: HashMap::with_capacity(Self::NACCOUNTS),
+            token_owners: None,
             uuid,
             directory: path,
             hashes_matrix: HashesMatrix::new(depth as usize),
@@ -363,6 +361,22 @@ impl DatabaseImpl<V2> {
         Self::create_with_dir(depth, None)
     }
 
+    pub fn create_with_token_owners(depth: u8) -> Self {
+        let mut db = Self::create_with_dir(depth, None);
+        db.set_token_owners();
+        db
+    }
+
+    pub fn set_token_owners(&mut self) {
+        if self.token_owners.is_none() {
+            self.token_owners = Some(HashMap::with_capacity(Self::NTOKENS));
+        }
+    }
+
+    pub fn unset_token_owners(&mut self) {
+        self.token_owners = None;
+    }
+
     pub fn root_hash(&mut self) -> Fp {
         self.emulate_tree_to_get_hash_at(Address::root())
     }