Include chunked phase2 script

phase1-cli/scripts/phase1_chunked.sh

@@ -3,9 +3,9 @@
 rm -f challenge* response* new_challenge* new_response* new_new_challenge_* processed* initial_ceremony* response_list* combined* seed*
 
 PROVING_SYSTEM=$1
-POWER=10
-BATCH=64
-CHUNK_SIZE=512
+POWER=18
+BATCH=131072
+CHUNK_SIZE=131072
 if [ "$PROVING_SYSTEM" == "groth16" ]; then
   MAX_CHUNK_INDEX=3 # we have 4 chunks, since we have a total of 2^11-1 powers
 else

setup2/Cargo.toml

@@ -29,6 +29,7 @@ hex-literal = { version = "0.3.1", optional = true }
 memmap = { version = "0.7.0", optional = true }
 rand = { version = "0.8" }
 rand_chacha = { version = "0.3" }
+serde = { version = "1.0", features = ["derive"] }
 thiserror = { version = "1.0.22" }
 tracing-subscriber = { version = "0.2.3" }
 

setup2/scripts/phase2_chunked.sh

@@ -0,0 +1,67 @@
+#!/bin/bash -e
+
+rm -f challenge* response* new_challenge* new_response* new_new_challenge_* processed* initial_ceremony* response_list* combined* seed* chunk*
+
+export RUSTFLAGS="-C target-feature=+bmi2,+adx"
+CARGO_VER=""
+PROVING_SYSTEM=groth16
+POWER=18
+BATCH=131072
+CHUNK_SIZE=131072
+CURVE="bw6"
+PATH_PHASE1="../../phase1-cli/scripts/combined" 
+SEED1=$(tr -dc 'A-F0-9' < /dev/urandom | head -c32)
+echo $SEED1 > seed1
+SEED2=$(tr -dc 'A-F0-9' < /dev/urandom | head -c32)
+echo $SEED2 > seed2
+
+function check_hash() {
+  test "`xxd -p -c 64 $1.hash`" = "`b2sum $1 | awk '{print $1}'`"
+}
+
+cargo $CARGO_VER build --release --bin setup2
+
+phase2_new="cargo run --release --features cli  -- new --curve-type $CURVE --chunk-size $CHUNK_SIZE --batch-size $BATCH --contribution-mode full"
+phase2_chunked="cargo run --release --bin setup2 --features cli  -- --curve-type $CURVE --chunk-size $CHUNK_SIZE --batch-size $BATCH --contribution-mode full --proving-system $PROVING_SYSTEM"
+phase2_1="cargo run --release --bin setup2 --features cli  -- --curve-type $CURVE --batch-size $BATCH --contribution-mode chunked --chunk-size $CHUNK_SIZE --seed seed1 --proving-system $PROVING_SYSTEM"
+phase2_2="cargo run --release --bin setup2 --features cli  -- --curve-type $CURVE --batch-size $BATCH --contribution-mode chunked --chunk-size $CHUNK_SIZE --seed seed2 --proving-system $PROVING_SYSTEM"
+####### Phase 2
+
+MAX_CHUNK_INDEX=1
+
+pwd
+
+ls $PATH_PHASE1
+
+$phase2_new --challenge-fname challenge --challenge-hash-fname challenge.verified.hash --phase1-fname $PATH_PHASE1 --phase1-powers $POWER --num-validators 1 --num-epochs 1
+for i in $(seq 0 $(($MAX_CHUNK_INDEX/2))); do
+  echo "Contributing and verifying chunk $i..."
+  $phase2_1 --chunk-index $i contribute --challenge-fname challenge.$i --challenge-hash-fname challenge.$i.hash --response-fname response_$i --response-hash-fname response_$i.hash
+  check_hash challenge.$i
+  check_hash response_$i
+  $phase2_1 --chunk-index $i verify --challenge-fname challenge.$i --challenge-hash-fname challenge_$i.verified.hash --response-fname response_$i --response-hash-fname response_$i.verified.hash
+  rm response_$i.hash
+  $phase2_2 --chunk-index $i contribute --challenge-fname response_$i --challenge-hash-fname response_$i.hash --response-fname new_response_$i --response-hash-fname new_response_$i.hash
+  check_hash new_response_$i
+  $phase2_2 --chunk-index $i verify --challenge-fname response_$i  --challenge-hash-fname response_$i.verified.hash --response-fname new_response_$i --response-hash-fname new_response_$i.verified.hash
+  rm challenge.$i response_$i # no longer needed
+  echo new_response_$i >> response_list
+done
+
+for i in $(seq $(($MAX_CHUNK_INDEX/2 + 1)) $MAX_CHUNK_INDEX); do
+  echo "Contributing and verifying chunk $i..."
+  $phase2_2 --chunk-index $i contribute --challenge-fname challenge.$i --challenge-hash-fname challenge.$i.hash --response-fname response_$i --response-hash-fname response_$i.hash
+  check_hash challenge.$i
+  check_hash response_$i
+  $phase2_2 --chunk-index $i verify --challenge-fname challenge.$i --challenge-hash-fname challenge_$i.verified.hash --response-fname response_$i --response-hash-fname response_$i.verified.hash
+  rm response_$i.hash
+  $phase2_1 --chunk-index $i contribute --challenge-fname response_$i --challenge-hash-fname response_$i.hash --response-fname new_response_$i --response-hash-fname new_response_$i.hash
+  check_hash new_response_$i
+  $phase2_1 --chunk-index $i verify --challenge-fname response_$i  --challenge-hash-fname response_$i.verified.hash --response-fname new_response_$i --response-hash-fname new_response_$i.verified.hash
+  rm challenge.$i response_$i # no longer needed
+  echo new_response_$i >> response_list
+done
+
+$phase2_chunked combine --response-list-fname response_list --initial-query-fname challenge.query --initial-full-fname challenge.full --combined-fname combined
+
+echo "Done!"

setup2/src/cli/new.rs

@@ -1,5 +1,5 @@
 use phase2::parameters::MPCParameters;
-use setup_utils::{log_2, CheckForCorrectness, Groth16Params, UseCompression};
+use setup_utils::{log_2, CheckForCorrectness, UseCompression};
 use snarkvm_algorithms::{SNARK, SRS};
 use snarkvm_curves::PairingEngine;
 use snarkvm_dpc::{
@@ -8,13 +8,15 @@ use snarkvm_dpc::{
 };
 use snarkvm_fields::Field;
 use snarkvm_r1cs::{ConstraintCounter, ConstraintSynthesizer};
+use snarkvm_utilities::CanonicalSerialize;
 
 use gumdrop::Options;
 use memmap::MmapOptions;
-use phase2::parameters::circuit_to_qap;
 use rand::{RngCore, SeedableRng};
 use rand_chacha::ChaChaRng;
-use std::fs::OpenOptions;
+use serde::{Deserialize, Serialize};
+use setup_utils::calculate_hash;
+use std::{fs::OpenOptions, io::Write};
 
 type AleoInner = <Testnet2Parameters as Parameters>::InnerCurve;
 type AleoOuter = <Testnet2Parameters as Parameters>::OuterCurve;
@@ -39,11 +41,24 @@ pub fn curve_from_str(src: &str) -> std::result::Result<CurveKind, String> {
     Ok(curve)
 }
 
+#[derive(Clone, PartialEq, Eq, Debug, Copy, Serialize, Deserialize)]
+pub enum ContributionMode {
+    Full,
+    Chunked,
+}
+
+pub fn contribution_mode_from_str(src: &str) -> Result<ContributionMode, String> {
+    let mode = match src.to_lowercase().as_str() {
+        "full" => ContributionMode::Full,
+        "chunked" => ContributionMode::Chunked,
+        _ => return Err("unsupported contribution mode. Currently supported: full, chunked".to_string()),
+    };
+    Ok(mode)
+}
+
 #[derive(Debug, Options, Clone)]
 pub struct NewOpts {
     help: bool,
-    #[options(help = "the path to the phase1 parameters", default = "phase1")]
-    pub phase1: String,
     #[options(help = "the total number of coefficients (in powers of 2) which were created after processing phase 1")]
     pub phase1_size: u32,
     #[options(help = "the challenge file name to be created", default = "challenge")]
@@ -56,47 +71,45 @@ pub struct NewOpts {
     )]
     pub curve_type: CurveKind,
 
-    #[options(help = "setup the inner or the outer circuit?")]
-    pub is_inner: bool,
-}
+    #[options(
+        help = "the contribution mode",
+        default = "chunked",
+        parse(try_from_str = "contribution_mode_from_str")
+    )]
+    pub contribution_mode: ContributionMode,
 
-pub fn new(opt: &NewOpts) -> anyhow::Result<()> {
-    if opt.is_inner {
-        let circuit = InnerCircuit::<Testnet2Parameters>::blank();
-        generate_params::<AleoInner, _>(opt, circuit)
-    } else {
-        let mut seed: Seed = [0; SEED_LENGTH];
-        rand::thread_rng().fill_bytes(&mut seed[..]);
-        let rng = &mut ChaChaRng::from_seed(seed);
-        let dpc = Testnet2DPC::load(false)?;
+    #[options(help = "the chunk size")]
+    pub chunk_size: usize,
 
-        let noop_circuit = dpc
-            .noop_program
-            .find_circuit_by_index(0)
-            .ok_or(DPCError::MissingNoopCircuit)?;
-        let private_program_input = dpc.noop_program.execute_blank(noop_circuit.circuit_id())?;
+    #[options(help = "the size of batches to process", default = "256")]
+    pub batch_size: usize,
 
-        let inner_snark_parameters = <Testnet2Parameters as Parameters>::InnerSNARK::setup(
-            &InnerCircuit::<Testnet2Parameters>::blank(),
-            &mut SRS::CircuitSpecific(rng),
-        )?;
+    #[options(help = "setup the inner or the outer circuit?", default = "true")]
+    pub is_inner: String,
 
-        let inner_snark_vk: <<Testnet2Parameters as Parameters>::InnerSNARK as SNARK>::VerifyingKey =
-            inner_snark_parameters.1.clone().into();
-        let inner_snark_proof = <Testnet2Parameters as Parameters>::InnerSNARK::prove(
-            &inner_snark_parameters.0,
-            &InnerCircuit::<Testnet2Parameters>::blank(),
-            rng,
-        )?;
-
-        let circuit =
-            OuterCircuit::<Testnet2Parameters>::blank(inner_snark_vk, inner_snark_proof, private_program_input);
-        generate_params::<AleoOuter, _>(opt, circuit)
-    }
+    #[options(help = "the provided challenge file", default = "challenge")]
+    pub challenge_fname: String,
+    #[options(help = "the new challenge file hash", default = "challenge.verified.hash")]
+    pub challenge_hash_fname: String,
+    #[options(help = "the provided response file which will be verified", default = "response")]
+    pub response_fname: String,
+    #[options(
+        help = "the new challenge file which will be generated in response",
+        default = "new_challenge"
+    )]
+    pub new_challenge_fname: String,
+    #[options(help = "phase 1 file name", default = "phase1")]
+    pub phase1_fname: String,
+    #[options(help = "phase 1 powers")]
+    pub phase1_powers: usize,
+    #[options(help = "number of validators")]
+    pub num_validators: usize,
+    #[options(help = "number of epochs")]
+    pub num_epochs: usize,
 }
 
-pub fn new_challenge(opt: &NewOpts) -> anyhow::Result<()> {
-    if opt.is_inner {
+pub fn new(opt: &NewOpts) -> anyhow::Result<()> {
+    if opt.is_inner == "true" {
         let circuit = InnerCircuit::<Testnet2Parameters>::blank();
         generate_params_chunked::<AleoInner, _>(opt, circuit)
     } else {
@@ -156,47 +169,6 @@ fn ceremony_size<F: Field, C: Clone + ConstraintSynthesizer<F>>(circuit: &C) ->
     }
 }
 
-pub fn generate_params<E, C>(opt: &NewOpts, circuit: C) -> anyhow::Result<()>
-where
-    E: PairingEngine,
-    C: Clone + ConstraintSynthesizer<E::Fr>,
-{
-    let phase1_transcript = OpenOptions::new()
-        .read(true)
-        .write(true)
-        .open(&opt.phase1)
-        .expect("could not read phase 1 transcript file");
-    let mut phase1_transcript = unsafe {
-        MmapOptions::new()
-            .map_mut(&phase1_transcript)
-            .expect("unable to create a memory map for input")
-    };
-    let mut output = OpenOptions::new()
-        .read(false)
-        .write(true)
-        .create_new(true)
-        .open(&opt.output)
-        .expect("could not open file for writing the MPC parameters ");
-
-    let phase2_size = ceremony_size(&circuit);
-    let keypair = circuit_to_qap::<E, _>(circuit)?;
-    // Read `num_constraints` Lagrange coefficients from the Phase1 Powers of Tau which were
-    // prepared for this step. This will fail if Phase 1 was too small.
-    let phase1 = Groth16Params::<E>::read(
-        &mut phase1_transcript,
-        COMPRESSION,
-        CheckForCorrectness::No, // No need to check for correctness, since this has been processed by the coordinator.
-        2usize.pow(opt.phase1_size),
-        phase2_size,
-    )?;
-
-    // Generate the initial transcript
-    let mpc = MPCParameters::new(keypair, phase1)?;
-    mpc.write(&mut output)?;
-
-    Ok(())
-}
-
 pub fn generate_params_chunked<E, C>(opt: &NewOpts, circuit: C) -> anyhow::Result<()>
 where
     E: PairingEngine,
@@ -205,50 +177,72 @@ where
     let phase1_transcript = OpenOptions::new()
         .read(true)
         .write(true)
-        .open(&opt.phase1)
+        .open(&opt.phase1_fname)
         .expect("could not read phase 1 transcript file");
     let mut phase1_transcript = unsafe {
         MmapOptions::new()
             .map_mut(&phase1_transcript)
             .expect("unable to create a memory map for input")
     };
-    let mut output = OpenOptions::new()
-        .read(false)
-        .write(true)
-        .create_new(true)
-        .open(&opt.output)
-        .expect("could not open file for writing the MPC parameters ");
-
     let phase2_size = ceremony_size(&circuit);
     // Read `num_constraints` Lagrange coefficients from the Phase1 Powers of Tau which were
     // prepared for this step. This will fail if Phase 1 was too small.
-    let phase1 = Groth16Params::<E>::read(
-        &mut phase1_transcript,
-        COMPRESSION,
-        CheckForCorrectness::No, // No need to check for correctness, since this has been processed by the coordinator.
-        2usize.pow(opt.phase1_size),
-        phase2_size,
-    )?;
 
-    let compressed = UseCompression::Yes;
-    let mut writer = vec![];
-    phase1.write(&mut writer, compressed).unwrap();
-    let chunk_size = phase2_size / 3;
-
-    let (full_mpc_parameters, _, _) = MPCParameters::<E>::new_from_buffer_chunked(
+    let (full_mpc_parameters, query_parameters, all_mpc_parameters) = MPCParameters::<E>::new_from_buffer_chunked(
         circuit,
-        writer.as_mut(),
+        &mut phase1_transcript,
         UseCompression::No,
         CheckForCorrectness::No,
-        2usize.pow(opt.phase1_size),
+        1 << opt.phase1_powers,
         phase2_size,
-        chunk_size,
+        opt.chunk_size,
     )
     .unwrap();
 
-    // Generate the initial transcript
-    //let mpc = MPCParameters::new(keypair, phase1)?;
-    full_mpc_parameters.write(&mut output)?;
+    let mut serialized_mpc_parameters = vec![];
+    full_mpc_parameters.write(&mut serialized_mpc_parameters).unwrap();
+
+    let mut serialized_query_parameters = vec![];
+    match COMPRESSION {
+        UseCompression::No => query_parameters.serialize(&mut serialized_query_parameters),
+        UseCompression::Yes => query_parameters.serialize(&mut serialized_query_parameters),
+    }
+    .unwrap();
+
+    let contribution_hash = {
+        std::fs::File::create(format!("{}.full", opt.challenge_fname))
+            .expect("unable to open new challenge hash file")
+            .write_all(&serialized_mpc_parameters)
+            .expect("unable to write serialized mpc parameters");
+        // Get the hash of the contribution, so the user can compare later
+        calculate_hash(&serialized_mpc_parameters)
+    };
+
+    std::fs::File::create(format!("{}.query", opt.challenge_fname))
+        .expect("unable to open new challenge hash file")
+        .write_all(&serialized_query_parameters)
+        .expect("unable to write serialized mpc parameters");
+
+    let mut challenge_list_file = std::fs::File::create("phase1").expect("unable to open new challenge list file");
+
+    for (i, chunk) in all_mpc_parameters.iter().enumerate() {
+        let mut serialized_chunk = vec![];
+        chunk.write(&mut serialized_chunk).expect("unable to write chunk");
+        std::fs::File::create(format!("{}.{}", opt.challenge_fname, i))
+            .expect("unable to open new challenge hash file")
+            .write_all(&serialized_chunk)
+            .expect("unable to write serialized mpc parameters");
+        challenge_list_file
+            .write(format!("{}.{}\n", opt.challenge_fname, i).as_bytes())
+            .expect("unable to write challenge list");
+    }
+
+    std::fs::File::create(format!("{}.{}\n", opt.challenge_hash_fname, "query"))
+        .expect("unable to open new challenge hash file")
+        .write_all(&contribution_hash)
+        .expect("unable to write new challenge hash");
+
+    println!("Wrote a fresh accumulator to challenge file");
 
     Ok(())
 }