WIP: archive send precomputed blocks - AWS

Author: Adrian Nagy

cli/src/commands/node/mod.rs

@@ -17,7 +17,7 @@ use node::p2p::identity::SecretKey;
 use node::service::Recorder;
 use node::SnarkerStrategy;
 
-use openmina_node_native::{tracing, NodeBuilder};
+use openmina_node_native::{archive::ArchiveStorageOptions, tracing, NodeBuilder};
 
 /// Openmina node
 #[derive(Debug, clap::Args)]
@@ -141,6 +141,24 @@ pub struct Node {
     /// Enable archive mode (seding blocks to the archive process).
     #[arg(long, env)]
     pub archive_address: Option<Url>,
+
+    /// Enable local precomputed storage.
+    #[arg(long, env)]
+    pub archive_local_storage: bool,
+
+    // TODO(adonagy): Sort out this... Do we want to support the ocaml options 1:1?
+    // we could move the addrss to an env var, just like gcp and aws
+    /// Enable archiver process.
+    #[arg(long, env)]
+    pub archive_archiver_process: bool,
+
+    /// Enable GCP precomputed storage.
+    #[arg(long, env)]
+    pub archive_gcp_storage: bool,
+
+    /// Enable AWS precomputed storage.
+    #[arg(long, env)]
+    pub archive_aws_storage: bool,
 }
 
 impl Node {
@@ -272,6 +290,30 @@ impl Node {
             }
         }
 
+        let archive_storage_options = ArchiveStorageOptions::from_iter(
+            [
+                (
+                    self.archive_local_storage,
+                    ArchiveStorageOptions::LOCAL_PRECOMPUTED_STORAGE,
+                ),
+                (
+                    self.archive_archiver_process,
+                    ArchiveStorageOptions::ARCHIVER_PROCESS,
+                ),
+                (
+                    self.archive_gcp_storage,
+                    ArchiveStorageOptions::GCP_PRECOMPUTED_STORAGE,
+                ),
+                (
+                    self.archive_aws_storage,
+                    ArchiveStorageOptions::AWS_PRECOMPUTED_STORAGE,
+                ),
+            ]
+            .iter()
+            .filter(|(enabled, _)| *enabled)
+            .map(|(_, option)| option.clone()),
+        );
+
         if let Some(address) = self.archive_address {
             node::core::info!(
                 summary = "Archive mode enabled",
@@ -281,7 +323,8 @@ impl Node {
             let socket_addrs = address.socket_addrs(|| None).expect("Invalid URL");
 
             let socket_addr = socket_addrs.first().expect("No socket address found");
-            node_builder.archive(*socket_addr);
+            // TODO(adonagy): add options
+            node_builder.archive(*socket_addr, archive_storage_options);
         }
 
         if let Some(sec_key) = self.run_snarker {

mina-p2p-messages/src/v2/manual.rs

@@ -741,23 +741,41 @@ pub enum ArchiveTransitionFronntierDiff {
 }
 
 impl ArchiveTransitionFronntierDiff {
-    pub fn accounts_accessed(&self) -> List<(crate::number::UInt64, MinaBaseAccountBinableArgStableV2)> {
+    pub fn block(&self) -> Option<MinaBlockBlockStableV2> {
         match self {
-            ArchiveTransitionFronntierDiff::BreadcrumbAdded { accounts_accessed, .. } => accounts_accessed.clone(),
+            // TODO(adonagy): maybe we should use Arc here instead of cloning
+            ArchiveTransitionFronntierDiff::BreadcrumbAdded { block, .. } => Some(block.0.clone()),
+            _ => None,
+        }
+    }
+
+    pub fn accounts_accessed(
+        &self,
+    ) -> List<(crate::number::UInt64, MinaBaseAccountBinableArgStableV2)> {
+        match self {
+            ArchiveTransitionFronntierDiff::BreadcrumbAdded {
+                accounts_accessed, ..
+            } => accounts_accessed.clone(),
             _ => List::new(),
         }
     }
 
     pub fn accounts_created(&self) -> List<(MinaBaseAccountIdStableV2, CurrencyFeeStableV1)> {
         match self {
-            ArchiveTransitionFronntierDiff::BreadcrumbAdded { accounts_created, .. } => accounts_created.clone(),
+            ArchiveTransitionFronntierDiff::BreadcrumbAdded {
+                accounts_created, ..
+            } => accounts_created.clone(),
             _ => List::new(),
         }
     }
 
-    pub fn tokens_used(&self) -> List<(MinaBaseTokenIdStableV2, Option<MinaBaseAccountIdStableV2>)> {
+    pub fn tokens_used(
+        &self,
+    ) -> List<(MinaBaseTokenIdStableV2, Option<MinaBaseAccountIdStableV2>)> {
         match self {
-            ArchiveTransitionFronntierDiff::BreadcrumbAdded { tokens_used, .. } => tokens_used.clone(),
+            ArchiveTransitionFronntierDiff::BreadcrumbAdded { tokens_used, .. } => {
+                tokens_used.clone()
+            }
             _ => List::new(),
         }
     }
@@ -836,6 +854,39 @@ pub struct PrecomputedBlockData {
     pub data: PrecomputedBlock,
 }
 
+impl PrecomputedBlock {
+    pub fn with_version(&self, version: u32) -> PrecomputedBlockData {
+        PrecomputedBlockData {
+            version,
+            data: self.clone(),
+        }
+    }
+}
+
+impl TryFrom<ArchiveTransitionFronntierDiff> for PrecomputedBlock {
+    type Error = String;
+
+    fn try_from(value: ArchiveTransitionFronntierDiff) -> Result<Self, Self::Error> {
+        let block = value
+            .block()
+            .ok_or("Block not found in archive transition frontier diff")?;
+        let res = Self {
+            scheduled_time: block.header.protocol_state.body.blockchain_state.timestamp,
+            protocol_state: block.header.protocol_state.clone(),
+            protocol_state_proof: block.header.protocol_state_proof.as_ref().clone().into(),
+            staged_ledger_diff: block.body.staged_ledger_diff,
+            // TODO(adonagy): add the actual delta transition chain proof
+            delta_transition_chain_proof: (LedgerHash::zero(), List::new()),
+            protocol_version: block.header.current_protocol_version.clone(),
+            proposed_protocol_version: None,
+            accounts_accessed: value.accounts_accessed(),
+            accounts_created: value.accounts_created(),
+            tokens_used: value.tokens_used(),
+        };
+        Ok(res)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::fmt::Debug;

node/common/Cargo.toml

@@ -21,6 +21,7 @@ sha3 = "0.10.8"
 ark-ff = { workspace = true }
 binprot = { git = "https://github.com/openmina/binprot-rs", rev = "400b52c" }
 binprot_derive = { git = "https://github.com/openmina/binprot-rs", rev = "400b52c" }
+bitflags = "2.8.0"
 
 node = { path = "../../node", features = ["replay"] }
 openmina-core = { path = "../../core" }
@@ -41,6 +42,8 @@ tracing-appender = "0.2.3"
 libp2p-identity = { version = "=0.2.7", features = ["ed25519", "rand", "serde"] }
 mio = { version = "1.0.2", features = ["os-poll", "net"] }
 reqwest = { version = "0.12.8", features = ["blocking", "json"] }
+aws-config = { version = "1.1.7", features = ["behavior-version-latest"] }
+aws-sdk-s3 = "1.73.0"
 
 [features]
 p2p-webrtc = ["node/p2p-webrtc"]