perf(node): load prover indexes in parallel with bootstrap to reduce startup time

closes: #751

Author: binier

cli/src/commands/node/mod.rs

@@ -226,10 +226,12 @@ impl Node {
 
         if let Some(producer_key_path) = self.producer_key {
             let password = &self.producer_key_password;
-            node::core::info!(node::core::log::system_time(); summary = "loading provers index");
-            let provers = BlockProver::make(Some(block_verifier_index), Some(work_verifier_index));
-            node::core::info!(node::core::log::system_time(); summary = "loaded provers index");
-            node_builder.block_producer_from_file(provers, producer_key_path, password)?;
+            openmina_core::thread::spawn(|| {
+                node::core::info!(node::core::log::system_time(); summary = "loading provers index");
+                BlockProver::make(Some(block_verifier_index), Some(work_verifier_index));
+                node::core::info!(node::core::log::system_time(); summary = "loaded provers index");
+            });
+            node_builder.block_producer_from_file(producer_key_path, password, None)?;
 
             if let Some(pub_key) = self.coinbase_receiver {
                 node_builder

core/src/thread.rs

@@ -73,6 +73,24 @@ mod main_thread {
         }));
         rx.await.ok()
     }
+
+    pub fn run_async_fn_in_main_thread_blocking<F, FU, T>(f: F) -> Option<T>
+    where
+        T: 'static + Send,
+        FU: Future<Output = T>,
+        F: 'static + Send + FnOnce() -> FU,
+    {
+        let sender = MAIN_THREAD_TASK_SENDER
+            .get()
+            .expect("main thread not initialized");
+        let (tx, rx) = oneshot::channel();
+        let _ = sender.send(Box::pin(async move {
+            wasm_bindgen_futures::spawn_local(async move {
+                let _ = tx.send(f().await);
+            })
+        }));
+        rx.blocking_recv().ok()
+    }
 }
 #[cfg(target_family = "wasm")]
 pub use main_thread::*;

ledger/src/proofs/circuit_blobs.rs

@@ -36,6 +36,15 @@ mod http {
             _get_bytes(url).await
         }
     }
+
+    pub fn get_bytes_blocking(url: &str) -> std::io::Result<Vec<u8>> {
+        let url = url.to_owned();
+        if thread::is_web_worker_thread() {
+            thread::run_async_fn_in_main_thread_blocking(move || _get_bytes(url)).expect("failed to run task in the main thread! Maybe main thread crashed or not initialized?")
+        } else {
+            panic!("can't do blocking requests from main browser thread");
+        }
+    }
 }
 
 #[cfg(not(target_family = "wasm"))]
@@ -53,7 +62,7 @@ fn git_release_url(filename: &impl AsRef<Path>) -> String {
 }
 
 #[cfg(not(target_family = "wasm"))]
-pub fn fetch(filename: &impl AsRef<Path>) -> std::io::Result<Vec<u8>> {
+pub fn fetch_blocking(filename: &impl AsRef<Path>) -> std::io::Result<Vec<u8>> {
     use std::path::PathBuf;
 
     fn try_base_dir<P: Into<PathBuf>>(base_dir: P, filename: &impl AsRef<Path>) -> Option<PathBuf> {
@@ -124,3 +133,11 @@ pub async fn fetch(filename: &impl AsRef<Path>) -> std::io::Result<Vec<u8>> {
     http::get_bytes(&url).await
     // http::get_bytes(&git_release_url(filename)).await
 }
+
+#[cfg(target_family = "wasm")]
+pub fn fetch_blocking(filename: &impl AsRef<Path>) -> std::io::Result<Vec<u8>> {
+    let prefix =
+        option_env!("CIRCUIT_BLOBS_HTTP_PREFIX").unwrap_or("/assets/webnode/circuit-blobs");
+    let url = format!("{prefix}/{}", filename.as_ref().to_str().unwrap());
+    http::get_bytes_blocking(&url)
+}

ledger/src/proofs/provers.rs

@@ -50,23 +50,13 @@ fn decode_gates_file<F: FieldWitness>(
     Ok(data.gates)
 }
 
-#[cfg(not(target_family = "wasm"))]
 fn read_gates_file<F: FieldWitness>(
     filename: &impl AsRef<Path>,
 ) -> std::io::Result<Vec<CircuitGate<F>>> {
-    let bytes = circuit_blobs::fetch(filename)?;
+    let bytes = circuit_blobs::fetch_blocking(filename)?;
     decode_gates_file(bytes.as_slice())
 }
 
-#[cfg(target_family = "wasm")]
-async fn read_gates_file<F: FieldWitness>(
-    filepath: &impl AsRef<Path>,
-) -> std::io::Result<Vec<CircuitGate<F>>> {
-    let resp = circuit_blobs::fetch(filepath).await?;
-    decode_gates_file(&mut resp.as_slice())
-}
-
-#[cfg(not(target_family = "wasm"))]
 fn make_gates<F: FieldWitness>(
     filename: &str,
 ) -> (
@@ -88,30 +78,6 @@ fn make_gates<F: FieldWitness>(
     (internal_vars_path, rows_rev_path, gates)
 }
 
-#[cfg(target_family = "wasm")]
-async fn make_gates<F: FieldWitness>(
-    filename: &str,
-) -> (
-    HashMap<usize, (Vec<(F, V)>, Option<F>)>,
-    Vec<Vec<Option<V>>>,
-    Vec<CircuitGate<F>>,
-) {
-    let circuits_config = openmina_core::NetworkConfig::global().circuits_config;
-    let base_dir = Path::new(circuits_config.directory_name);
-
-    let internal_vars_path = base_dir.join(format!("{}_internal_vars.bin", filename));
-    let rows_rev_path = base_dir.join(format!("{}_rows_rev.bin", filename));
-    let gates_path = base_dir.join(format!("{}_gates.json", filename));
-
-    let gates: Vec<CircuitGate<F>> = read_gates_file(&gates_path).await.unwrap();
-    let (internal_vars_path, rows_rev_path) =
-        read_constraints_data::<F>(&internal_vars_path, &rows_rev_path)
-            .await
-            .unwrap();
-
-    (internal_vars_path, rows_rev_path, gates)
-}
-
 macro_rules! get_or_make {
     ($constant: ident, $type: ty, $filename: expr) => {{
         get_or_make!($constant, $type, None, $filename)
@@ -121,13 +87,7 @@ macro_rules! get_or_make {
             return prover.clone();
         }
 
-        let (internal_vars, rows_rev, gates) = {
-            #[cfg(not(target_family = "wasm"))]
-            let res = make_gates($filename);
-            #[cfg(target_family = "wasm")]
-            let res = make_gates($filename).await;
-            res
-        };
+        let (internal_vars, rows_rev, gates) = make_gates($filename);
 
         let index = make_prover_index::<$type, _>(gates, $verifier_index);
         let prover = Prover {
@@ -153,7 +113,6 @@ fn default_circuits_config() -> &'static CircuitsConfig {
     openmina_core::NetworkConfig::global().circuits_config
 }
 
-#[cfg(not(target_family = "wasm"))]
 mod prover_makers {
     use super::*;
 
@@ -274,128 +233,6 @@ mod prover_makers {
     }
 }
 
-#[cfg(target_family = "wasm")]
-mod prover_makers {
-    use super::*;
-
-    async fn get_or_make_tx_step_prover(config: &CircuitsConfig) -> Arc<Prover<Fp>> {
-        get_or_make!(
-            TX_STEP_PROVER,
-            StepTransactionProof,
-            config.step_transaction_gates
-        )
-    }
-    async fn get_or_make_tx_wrap_prover(
-        config: &CircuitsConfig,
-        verifier_index: Option<TransactionVerifier>,
-    ) -> Arc<Prover<Fq>> {
-        get_or_make!(
-            TX_WRAP_PROVER,
-            WrapTransactionProof,
-            verifier_index.map(Into::into),
-            config.wrap_transaction_gates
-        )
-    }
-    async fn get_or_make_merge_step_prover(config: &CircuitsConfig) -> Arc<Prover<Fp>> {
-        get_or_make!(MERGE_STEP_PROVER, StepMergeProof, config.step_merge_gates)
-    }
-    async fn get_or_make_block_step_prover(config: &CircuitsConfig) -> Arc<Prover<Fp>> {
-        get_or_make!(
-            BLOCK_STEP_PROVER,
-            StepBlockProof,
-            config.step_blockchain_gates
-        )
-    }
-    async fn get_or_make_block_wrap_prover(
-        config: &CircuitsConfig,
-        verifier_index: Option<BlockVerifier>,
-    ) -> Arc<Prover<Fq>> {
-        get_or_make!(
-            BLOCK_WRAP_PROVER,
-            WrapBlockProof,
-            verifier_index.map(Into::into),
-            config.wrap_blockchain_gates
-        )
-    }
-    async fn get_or_make_zkapp_step_opt_signed_opt_signed_prover(
-        config: &CircuitsConfig,
-    ) -> Arc<Prover<Fp>> {
-        get_or_make!(
-            ZKAPP_STEP_OPT_SIGNED_OPT_SIGNED_PROVER,
-            StepZkappOptSignedOptSignedProof,
-            config.step_transaction_opt_signed_opt_signed_gates
-        )
-    }
-    async fn get_or_make_zkapp_step_opt_signed_prover(config: &CircuitsConfig) -> Arc<Prover<Fp>> {
-        get_or_make!(
-            ZKAPP_STEP_OPT_SIGNED_PROVER,
-            StepZkappOptSignedProof,
-            config.step_transaction_opt_signed_gates
-        )
-    }
-    async fn get_or_make_zkapp_step_proof_prover(config: &CircuitsConfig) -> Arc<Prover<Fp>> {
-        get_or_make!(
-            ZKAPP_STEP_PROOF_PROVER,
-            StepZkappProvedProof,
-            config.step_transaction_proved_gates
-        )
-    }
-
-    impl BlockProver {
-        pub async fn make(
-            block_verifier_index: Option<BlockVerifier>,
-            tx_verifier_index: Option<TransactionVerifier>,
-        ) -> Self {
-            let config = default_circuits_config();
-            let block_step_prover = get_or_make_block_step_prover(config).await;
-            let block_wrap_prover =
-                get_or_make_block_wrap_prover(config, block_verifier_index).await;
-            let tx_wrap_prover = get_or_make_tx_wrap_prover(config, tx_verifier_index).await;
-
-            Self {
-                block_step_prover,
-                block_wrap_prover,
-                tx_wrap_prover,
-            }
-        }
-    }
-
-    impl TransactionProver {
-        pub async fn make(tx_verifier_index: Option<TransactionVerifier>) -> Self {
-            let config = default_circuits_config();
-            let tx_step_prover = get_or_make_tx_step_prover(config).await;
-            let tx_wrap_prover = get_or_make_tx_wrap_prover(config, tx_verifier_index).await;
-            let merge_step_prover = get_or_make_merge_step_prover(config).await;
-
-            Self {
-                tx_step_prover,
-                tx_wrap_prover,
-                merge_step_prover,
-            }
-        }
-    }
-
-    impl ZkappProver {
-        pub async fn make(tx_verifier_index: Option<TransactionVerifier>) -> Self {
-            let config = default_circuits_config();
-            let tx_wrap_prover = get_or_make_tx_wrap_prover(config, tx_verifier_index).await;
-            let merge_step_prover = get_or_make_merge_step_prover(config).await;
-            let step_opt_signed_opt_signed_prover =
-                get_or_make_zkapp_step_opt_signed_opt_signed_prover(config).await;
-            let step_opt_signed_prover = get_or_make_zkapp_step_opt_signed_prover(config).await;
-            let step_proof_prover = get_or_make_zkapp_step_proof_prover(config).await;
-
-            Self {
-                tx_wrap_prover,
-                merge_step_prover,
-                step_opt_signed_opt_signed_prover,
-                step_opt_signed_prover,
-                step_proof_prover,
-            }
-        }
-    }
-}
-
 #[derive(Clone)]
 pub struct BlockProver {
     pub block_step_prover: Arc<Prover<Fp>>,
@@ -419,6 +256,18 @@ pub struct ZkappProver {
     pub step_proof_prover: Arc<Prover<Fp>>,
 }
 
+impl BlockProver {
+    /// Called must make sure that before calling this,
+    /// `BlockProver::make` call was made.
+    pub fn get_once_made() -> Self {
+        Self {
+            block_step_prover: BLOCK_STEP_PROVER.wait().clone(),
+            block_wrap_prover: BLOCK_WRAP_PROVER.wait().clone(),
+            tx_wrap_prover: TX_WRAP_PROVER.wait().clone(),
+        }
+    }
+}
+
 fn decode_constraints_data<F: FieldWitness>(
     mut internal_vars: &[u8],
     mut rows_rev: &[u8],
@@ -471,31 +320,15 @@ fn decode_constraints_data<F: FieldWitness>(
     Some((internal_vars, rows_rev))
 }
 
-#[cfg(not(target_family = "wasm"))]
 fn read_constraints_data<F: FieldWitness>(
     internal_vars_path: &Path,
     rows_rev_path: &Path,
 ) -> Option<(InternalVars<F>, Vec<Vec<Option<V>>>)> {
     // ((Fp.t * V.t) list * Fp.t option)
-    let internal_vars = circuit_blobs::fetch(&internal_vars_path).ok()?;
+    let internal_vars = circuit_blobs::fetch_blocking(&internal_vars_path).ok()?;
 
     // V.t option array list
-    let rows_rev = circuit_blobs::fetch(&rows_rev_path).ok()?;
+    let rows_rev = circuit_blobs::fetch_blocking(&rows_rev_path).ok()?;
 
     decode_constraints_data(internal_vars.as_slice(), rows_rev.as_slice())
 }
-
-#[cfg(target_family = "wasm")]
-async fn read_constraints_data<F: FieldWitness>(
-    internal_vars_path: &Path,
-    rows_rev_path: &Path,
-) -> Option<(InternalVars<F>, Vec<Vec<Option<V>>>)> {
-    // ((Fp.t * V.t) list * Fp.t option)
-    let internal_vars = circuit_blobs::fetch(&internal_vars_path).await.ok()?;
-
-    // V.t option array list
-    let rows_rev = circuit_blobs::fetch(&rows_rev_path).await.ok()?;
-    // std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("rows_rev.bin")).unwrap();
-
-    decode_constraints_data(internal_vars.as_ref(), rows_rev.as_ref())
-}

ledger/src/proofs/verifiers.rs

@@ -71,7 +71,7 @@ fn cache_path(kind: Kind) -> Option<PathBuf> {
 macro_rules! read_cache {
     ($kind: expr, $digest: expr) => {{
         #[cfg(not(target_family = "wasm"))]
-        let data = super::circuit_blobs::fetch(&cache_filename($kind))
+        let data = super::circuit_blobs::fetch_blocking(&cache_filename($kind))
             .context("fetching verifier index failed")?;
         #[cfg(target_family = "wasm")]
         let data = super::circuit_blobs::fetch(&cache_filename($kind))

node/common/src/service/block_producer/mod.rs

@@ -16,16 +16,16 @@ use node::{
 use crate::EventSender;
 
 pub struct BlockProducerService {
-    provers: BlockProver,
+    provers: Option<BlockProver>,
     keypair: AccountSecretKey,
     vrf_evaluation_sender: mpsc::UnboundedSender<VrfEvaluatorInput>,
 }
 
 impl BlockProducerService {
     pub fn new(
-        provers: BlockProver,
         keypair: AccountSecretKey,
         vrf_evaluation_sender: mpsc::UnboundedSender<VrfEvaluatorInput>,
+        provers: Option<BlockProver>,
     ) -> Self {
         Self {
             provers,
@@ -35,9 +35,9 @@ impl BlockProducerService {
     }
 
     pub fn start(
-        provers: BlockProver,
         event_sender: EventSender,
         keypair: AccountSecretKey,
+        provers: Option<BlockProver>,
     ) -> Self {
         let (vrf_evaluation_sender, vrf_evaluation_receiver) =
             mpsc::unbounded_channel::<VrfEvaluatorInput>();
@@ -54,7 +54,7 @@ impl BlockProducerService {
             })
             .unwrap();
 
-        BlockProducerService::new(provers, keypair, vrf_evaluation_sender)
+        BlockProducerService::new(keypair, vrf_evaluation_sender, provers)
     }
 
     pub fn keypair(&self) -> AccountSecretKey {
@@ -103,6 +103,7 @@ impl node::service::BlockProducerService for crate::NodeService {
             .expect("provers shouldn't be needed if block producer isn't initialized")
             .provers
             .clone()
+            .unwrap_or_else(|| BlockProver::get_once_made())
     }
 
     fn prove(&mut self, block_hash: StateHash, input: Box<ProverExtendBlockchainInputStableV2>) {