From 230e30409d16711cc4e15970d8613a698bee7df3 Mon Sep 17 00:00:00 2001 From: saitima Date: Sat, 11 Jan 2025 13:12:52 +0300 Subject: [PATCH 01/18] feat(fflonk): changes for better compression --- crates/fflonk/src/context.rs | 29 +++++++++++++++++----- crates/fflonk/src/convenience.rs | 4 ++-- crates/fflonk/src/lib.rs | 4 +++- crates/fflonk/src/setup.rs | 41 ++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 9 deletions(-) diff --git a/crates/fflonk/src/context.rs b/crates/fflonk/src/context.rs index 38b14a1..fec31e1 100644 --- a/crates/fflonk/src/context.rs +++ b/crates/fflonk/src/context.rs @@ -55,7 +55,7 @@ pub(crate) fn init_tmp_mempool() { unsafe { _TMP_MEMPOOL = Some(bc_mem_pool::new(DEFAULT_DEVICE_ID).unwrap()); } - let num_tmp_bytes = 1_100_000_000; + let num_tmp_bytes = 3 << 29; //1.5GB let stream = bc_stream::new().unwrap(); DVec::::allocate_on(num_tmp_bytes, _tmp_mempool(), stream); } @@ -146,9 +146,19 @@ const POWERS_OF_COSET_OMEGA_COARSE_LOG_COUNT: u32 = 14; pub type DeviceContextWithSingleDevice = DeviceContext<1>; impl DeviceContext { + pub fn init_from_preloaded_crs( + domain_size: usize, + crs: Crs, + ) -> CudaResult { + let context = Self::init_no_msm(domain_size)?; + Self::init_msm_on_static_memory(domain_size, Some(crs))?; + + Ok(context) + } + pub fn init(domain_size: usize) -> CudaResult { let context = Self::init_no_msm(domain_size)?; - Self::init_msm_on_static_memory(domain_size)?; + Self::init_msm_on_static_memory(domain_size, None)?; // Self::init_msm_on_pool(domain_size)?; Ok(context) @@ -179,8 +189,11 @@ impl DeviceContext { Ok(DeviceContext) } - fn init_msm_on_static_memory(domain_size: usize) -> CudaResult<()> { - Self::inner_init_msm(domain_size, None, None)?; + fn init_msm_on_static_memory( + domain_size: usize, + crs: Option>, + ) -> CudaResult<()> { + Self::inner_init_msm(domain_size, crs, None, None)?; Ok(()) } @@ -188,13 +201,14 @@ impl DeviceContext { unsafe fn init_msm_on_pool(domain_size: usize) -> CudaResult<()> { let pool = _msm_bases_mempool(); let stream = bc_stream::new().unwrap(); - Self::inner_init_msm(domain_size, Some(pool), Some(stream))?; + Self::inner_init_msm(domain_size, None, Some(pool), Some(stream))?; stream.sync().unwrap(); Ok(()) } fn inner_init_msm( domain_size: usize, + crs: Option>, pool: Option, stream: Option, ) -> CudaResult<()> { @@ -205,7 +219,10 @@ impl DeviceContext { init_msm_result_mempool(); // MSM impl requires bases to be located in a buffer that is // multiple of the domain_size - let crs = init_compact_crs(&bellman::worker::Worker::new(), domain_size); + let crs = match crs { + Some(preloaded_crs) => preloaded_crs, + None => init_compact_crs(&bellman::worker::Worker::new(), domain_size), + }; let num_bases = MAX_COMBINED_DEGREE_FACTOR * domain_size; assert!(crs.g1_bases.len() >= num_bases); let bases = match (pool, stream) { diff --git a/crates/fflonk/src/convenience.rs b/crates/fflonk/src/convenience.rs index 4e6bc91..7affafc 100644 --- a/crates/fflonk/src/convenience.rs +++ b/crates/fflonk/src/convenience.rs @@ -14,8 +14,8 @@ use circuit_definitions::circuit_definitions::aux_layer::{ }; use fflonk::{FflonkAssembly, L1_VERIFIER_DOMAIN_SIZE_LOG}; -pub type FflonkSnarkVerifierCircuitDeviceSetup = - FflonkDeviceSetup; +pub type FflonkSnarkVerifierCircuitDeviceSetup = + FflonkDeviceSetup; use super::*; diff --git a/crates/fflonk/src/lib.rs b/crates/fflonk/src/lib.rs index 02256f5..3669e99 100644 --- a/crates/fflonk/src/lib.rs +++ b/crates/fflonk/src/lib.rs @@ -63,7 +63,7 @@ pub use gpu_ffi; use gpu_ffi::{bc_event, bc_mem_pool, bc_stream}; use std::alloc::Allocator; -pub use context::{DeviceContext, DeviceContextWithSingleDevice}; +pub use context::{init_compact_crs, DeviceContext, DeviceContextWithSingleDevice}; pub use fflonk::MAX_COMBINED_DEGREE_FACTOR; pub use convenience::FflonkSnarkVerifierCircuitDeviceSetup; @@ -74,3 +74,5 @@ pub type FflonkSnarkVerifierCircuit = ZkSyncSnarkWrapperCircuitNoLookupCustomGat pub type FflonkSnarkVerifierCircuitVK = FflonkVerificationKey; pub type FflonkSnarkVerifierCircuitProof = FflonkProof; pub type FflonkSnarkVerifierCircuitSetup = FflonkSetup; + +pub use allocator::{GlobalHost, HostAllocator}; diff --git a/crates/fflonk/src/setup.rs b/crates/fflonk/src/setup.rs index bb8e3d3..31557d1 100644 --- a/crates/fflonk/src/setup.rs +++ b/crates/fflonk/src/setup.rs @@ -307,3 +307,44 @@ impl, A: HostAllocator> FflonkDeviceSetup { write_curve_affine(&self.g2_elems[1], writer) } } + +pub fn read_raw_fr_vec( + mut src: R, +) -> std::io::Result> { + use byteorder::{BigEndian, ReadBytesExt}; + let num_values = src.read_u32::()? as usize; + let mut values = Vec::with_capacity_in(num_values, A::default()); + unsafe { + values.set_len(num_values); + let buf = std::slice::from_raw_parts_mut( + values.as_mut_ptr() as *mut u8, + num_values * std::mem::size_of::(), + ); + // src.read_exact(buf)?; + let mut dst = &mut buf[..]; + std::io::copy(&mut src, &mut dst)?; + } + + Ok(values) +} + +pub fn write_raw_fr_slice( + src_values: &[F], + mut dst: W, +) -> std::io::Result<()> { + use byteorder::{BigEndian, WriteBytesExt}; + let num_values = src_values.len(); + assert!(num_values < u32::MAX as usize); + dst.write_u32::(num_values as u32)?; + unsafe { + let buf = std::slice::from_raw_parts_mut( + src_values.as_ptr() as *mut u8, + num_values * std::mem::size_of::(), + ); + let mut reader = &buf[..]; + // dst.write_all(buf)?; + std::io::copy(&mut reader, &mut dst)?; + } + + Ok(()) +} From 57aaf7d3009cc7db2d8749304a9037b87b17cf99 Mon Sep 17 00:00:00 2001 From: saitima Date: Tue, 7 Jan 2025 20:13:34 +0300 Subject: [PATCH 02/18] feat(proof-compression): new interface for entire chain --- crates/proof-compression/src/artifacts.rs | 165 +++++++ crates/proof-compression/src/chain.rs | 200 ++++++++ crates/proof-compression/src/compression.rs | 187 ++++++++ crates/proof-compression/src/context.rs | 7 + crates/proof-compression/src/lib.rs | 61 ++- crates/proof-compression/src/proof_system.rs | 440 ++++++++++++++++++ crates/proof-compression/src/serialization.rs | 104 +++++ crates/proof-compression/src/snark_wrapper.rs | 138 ++++++ crates/proof-compression/src/step.rs | 46 ++ 9 files changed, 1343 insertions(+), 5 deletions(-) create mode 100644 crates/proof-compression/src/artifacts.rs create mode 100644 crates/proof-compression/src/chain.rs create mode 100644 crates/proof-compression/src/compression.rs create mode 100644 crates/proof-compression/src/context.rs create mode 100644 crates/proof-compression/src/proof_system.rs create mode 100644 crates/proof-compression/src/serialization.rs create mode 100644 crates/proof-compression/src/snark_wrapper.rs create mode 100644 crates/proof-compression/src/step.rs diff --git a/crates/proof-compression/src/artifacts.rs b/crates/proof-compression/src/artifacts.rs new file mode 100644 index 0000000..c9d56a3 --- /dev/null +++ b/crates/proof-compression/src/artifacts.rs @@ -0,0 +1,165 @@ +use std::io::Read; + +use boojum::cs::implementations::fast_serialization::MemcopySerializable; +use boojum::cs::implementations::setup::FinalizationHintsForProver; +use boojum::cs::traits::gate::FinalizationHintSerialized; +use circuit_definitions::circuit_definitions::aux_layer::compression_modes::CompressionTreeHasherForWrapper; +use circuit_definitions::circuit_definitions::aux_layer::{ + CompressionProofsTreeHasher, ZkSyncCompressionVerificationKey, +}; +use circuit_definitions::circuit_definitions::recursion_layer::ZkSyncRecursionVerificationKey; +use shivini::cs::GpuSetup as BoojumDeviceSetup; + +use super::*; + +pub trait BlobStorage { + fn save(artifact: T) + where + T: MemcopySerializable; +} + +pub struct AsyncHandler { + precomputation: std::sync::mpsc::Receiver, // This is indeed a receiver +} + +impl AsyncHandler { + pub fn into_inner(self) -> T { + todo!() + } +} +pub trait ArtifactLoader: Sized { + fn init(bs: BS) -> Self + where + BS: BlobStorage; + fn load_scheduler_finalization_hint(&self) -> FinalizationHintsForProver; + fn read_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box; + fn load_compression_layer_finalization_hint( + &self, + circuit_id: u8, + ) -> FinalizationHintsForProver; + fn load_compression_wrapper_finalization_hint( + &self, + circuit_id: u8, + ) -> FinalizationHintSerialized; + fn read_compression_wrapper_finalization_hint(&self, circuit_id: u8) -> Box; + fn load_plonk_finalization_hint(&self) -> usize; + + fn load_scheduler_vk(&self) -> ZkSyncRecursionVerificationKey; + fn load_compression_layer_vk(&self, circuit_id: u8) -> ZkSyncCompressionVerificationKey; + fn read_scheduler_vk(&self) -> Box; + fn read_compression_layer_vk(&self, circuit_id: u8) -> Box; + fn read_compression_wrapper_vk(&self, circuit_id: u8) -> Box; + fn read_plonk_vk(&self) -> Box; + fn read_fflonk_vk(&self) -> Box; + fn get_compression_layer_precomputation( + &self, + circuit_id: u8, + ) -> AsyncHandler>; + fn get_compression_wrapper_precomputation( + &self, + circuit_id: u8, + ) -> AsyncHandler>; + fn get_plonk_snark_wrapper_precomputation( + &self, + circuit_id: u8, + ) -> AsyncHandler; + fn get_fflonk_snark_wrapper_precomputation( + &self, + ) -> AsyncHandler; +} + +pub struct SimpleArtifactLoader; + +impl ArtifactLoader for SimpleArtifactLoader { + fn init(bs: BS) -> Self + where + BS: BlobStorage, + { + todo!() + } + + fn load_scheduler_finalization_hint(&self) -> FinalizationHintsForProver { + todo!() + } + + fn read_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box { + todo!() + } + + fn load_compression_layer_finalization_hint( + &self, + circuit_id: u8, + ) -> FinalizationHintsForProver { + todo!() + } + + fn load_compression_wrapper_finalization_hint( + &self, + circuit_id: u8, + ) -> FinalizationHintSerialized { + todo!() + } + + fn read_compression_wrapper_finalization_hint(&self, circuit_id: u8) -> Box { + todo!() + } + + fn load_plonk_finalization_hint(&self) -> usize { + todo!() + } + + fn load_scheduler_vk(&self) -> ZkSyncRecursionVerificationKey { + todo!() + } + + fn load_compression_layer_vk(&self, circuit_id: u8) -> ZkSyncCompressionVerificationKey { + todo!() + } + + fn read_scheduler_vk(&self) -> Box { + todo!() + } + + fn read_compression_layer_vk(&self, circuit_id: u8) -> Box { + todo!() + } + + fn read_compression_wrapper_vk(&self, circuit_id: u8) -> Box { + todo!() + } + + fn read_plonk_vk(&self) -> Box { + todo!() + } + + fn read_fflonk_vk(&self) -> Box { + todo!() + } + + fn get_compression_layer_precomputation( + &self, + circuit_id: u8, + ) -> AsyncHandler> { + todo!() + } + + fn get_compression_wrapper_precomputation( + &self, + circuit_id: u8, + ) -> AsyncHandler> { + todo!() + } + + fn get_plonk_snark_wrapper_precomputation( + &self, + circuit_id: u8, + ) -> AsyncHandler { + todo!() + } + + fn get_fflonk_snark_wrapper_precomputation( + &self, + ) -> AsyncHandler { + todo!() + } +} diff --git a/crates/proof-compression/src/chain.rs b/crates/proof-compression/src/chain.rs new file mode 100644 index 0000000..286c7b8 --- /dev/null +++ b/crates/proof-compression/src/chain.rs @@ -0,0 +1,200 @@ +use circuit_definitions::circuit_definitions::aux_layer::{ + compression::ProofCompressionFunction, + compression_modes::{ + CompressionMode1, CompressionMode1ForWrapper, CompressionMode2, CompressionMode3, + CompressionMode4, CompressionMode5ForWrapper, + }, +}; + +use super::*; + +pub struct CompressionChain(std::marker::PhantomData<(C, U)>); + +impl CompressionChain +where + C: CompressionStep, +{ + pub fn new(step: C) -> CompressionChain + where + U: CompressionStep, + { + todo!() + } + + pub fn next(self) -> U { + todo!() + } +} + +impl StepDefinition for CompressionChain +where + C: CompressionStep, +{ + type PreviousProofSystem = C::PreviousProofSystem; + type ThisProofSystem = C::ThisProofSystem; +} + +impl ProofCompressionFunction for CompressionChain +where + C: CompressionStep, +{ + type PreviousLayerPoW = C::PreviousLayerPoW; + + type ThisLayerPoW = C::ThisLayerPoW; + + type ThisLayerHasher = C::ThisLayerHasher; + + type ThisLayerTranscript = C::ThisLayerTranscript; + + fn this_layer_transcript_parameters( + ) -> >::TransciptParameters { + C::this_layer_transcript_parameters() + } + + fn description_for_compression_step() -> String { + C::description_for_compression_step() + } + + fn size_hint_for_compression_step() -> (usize, usize) { + C::size_hint_for_compression_step() + } + + fn geometry_for_compression_step() -> boojum::cs::CSGeometry { + C::geometry_for_compression_step() + } + + fn lookup_parameters_for_compression_step() -> boojum::cs::LookupParameters { + C::lookup_parameters_for_compression_step() + } + + fn configure_builder_for_compression_step< + T: boojum::cs::cs_builder::CsBuilderImpl, + GC: boojum::cs::GateConfigurationHolder, + TB: boojum::cs::StaticToolboxHolder, + >( + builder: boojum::cs::cs_builder::CsBuilder, + ) -> boojum::cs::cs_builder::CsBuilder< + T, + GoldilocksField, + impl boojum::cs::GateConfigurationHolder, + impl boojum::cs::StaticToolboxHolder, + > { + C::configure_builder_for_compression_step(builder) + } + + fn previous_step_builder_for_compression< + CS: boojum::cs::traits::cs::ConstraintSystem + 'static, + >() -> Box< + dyn boojum::cs::traits::circuit::ErasedBuilderForRecursiveVerifier< + GoldilocksField, + GoldilocksExt2, + CS, + >, + > { + C::previous_step_builder_for_compression() + } + + fn proof_config_for_compression_step() -> boojum::cs::implementations::prover::ProofConfig { + C::proof_config_for_compression_step() + } +} + +impl CompressionStep for CompressionChain +where + C: CompressionStep, +{ + const MODE: u8 = C::MODE; + const IS_WRAPPER: bool = C::IS_WRAPPER; +} + +pub struct CompressionChainBuilder(std::marker::PhantomData); + +impl CompressionChainBuilder +where + C: CompressionStep, +{ + pub fn new(step: C) -> CompressionChainBuilder { + todo!() + } +} + +impl CompressionChainBuilder +where + C: CompressionStep, +{ + pub fn link_compression_step( + self, + step: CC, + ) -> CompressionChainBuilder> + where + CC: CompressionStep, + { + todo!() + } + + pub fn build(self) -> C { + todo!() + } +} + +pub fn build_full_chain_with_fflonk() -> CompressionChain< + CompressionChain< + CompressionChain, CompressionMode3>, + CompressionMode4, + >, + CompressionMode5ForWrapper, +> { + let chain = CompressionChainBuilder::new(CompressionMode1) + .link_compression_step(CompressionMode2) + .link_compression_step(CompressionMode3) + .link_compression_step(CompressionMode4) + .link_compression_step(CompressionMode5ForWrapper) + .build(); + + chain +} + +pub fn build_full_chain_with_plonk() -> CompressionChain { + todo!() +} + +pub fn run_step_chain_with_fflonk( + input_proof: SchedulerProof, + blob_storage: BS, +) -> FflonkSnarkVerifierCircuitProof +where + BS: BlobStorage, +{ + let chain = build_full_chain_with_fflonk(); + run_step_chain::<_, _, FflonkSnarkWrapper, _>(chain, input_proof, blob_storage) +} + +pub fn run_step_chain_with_plonk( + input_proof: SchedulerProof, + blob_storage: BS, +) -> PlonkSnarkVerifierCircuitProof +where + BS: BlobStorage, +{ + let chain = build_full_chain_with_plonk(); + todo!() +} + +pub fn run_step_chain( + chain: CompressionChain, + input_proof: SchedulerProof, + blob_storage: BS, +) -> ::Proof +where + C1: CompressionStep, + C2: CompressionStep, + SW: SnarkWrapperStep, + BS: BlobStorage, +{ + let artifact_loader = SimpleArtifactLoader::init(blob_storage); + + let next_step = chain.next(); + todo!() +} diff --git a/crates/proof-compression/src/compression.rs b/crates/proof-compression/src/compression.rs new file mode 100644 index 0000000..90c57c9 --- /dev/null +++ b/crates/proof-compression/src/compression.rs @@ -0,0 +1,187 @@ +use super::*; + +use circuit_definitions::circuit_definitions::{ + aux_layer::{ + compression::ProofCompressionFunction, + compression_modes::{ + CompressionMode1, CompressionMode1ForWrapper, CompressionMode2, CompressionMode3, + CompressionMode4, CompressionMode5ForWrapper, CompressionTreeHasherForWrapper, + }, + CompressionProofsTreeHasher, + }, + recursion_layer::RecursiveProofsTreeHasher, +}; + +pub trait CompressionStep: + ProofCompressionFunction + StepDefinition +{ + const MODE: u8; + const IS_WRAPPER: bool; + fn load_finalization_hint( + artifact_loader: &AL, + ) -> ::FinalizationHint + where + AL: ArtifactLoader, + { + let reader = if Self::IS_WRAPPER { + artifact_loader.read_compression_wrapper_finalization_hint(Self::MODE) + } else { + artifact_loader.read_compression_layer_finalization_hint(Self::MODE) + }; + serde_json::from_reader(reader).unwrap() + } + + fn load_previous_vk( + artifact_loader: &AL, + ) -> ::VK + where + AL: ArtifactLoader, + { + assert!(Self::MODE >= 1); + + let reader = if Self::MODE == 1 { + artifact_loader.read_scheduler_vk() + } else if Self::IS_WRAPPER { + artifact_loader.read_compression_wrapper_vk(Self::MODE) + } else { + artifact_loader.read_compression_layer_vk(Self::MODE - 1) + }; + + serde_json::from_reader(reader).unwrap() + } + + fn load_this_vk( + artifact_loader: &AL, + ) -> ::VK + where + AL: ArtifactLoader, + { + let reader = if Self::IS_WRAPPER { + artifact_loader.read_compression_wrapper_vk(Self::MODE) + } else { + artifact_loader.read_compression_layer_vk(Self::MODE) + }; + + serde_json::from_reader(reader).unwrap() + } + + fn get_precomputation( + artifact_loader: &AL, + ) -> AsyncHandler<::Precomputation> + where + AL: ArtifactLoader, + { + todo!() + } + + fn prove_compression_step( + input_proof: ::Proof, + artifact_loader: &AL, + ) -> ::Proof + where + AL: ArtifactLoader, + CI: ContextInitializator, + { + let input_vk = Self::load_previous_vk(artifact_loader); + let precomputation = Self::get_precomputation(artifact_loader); + let config = ::get_context_config(); + let ctx = CI::init::(config); + let finalization_hint = Self::load_finalization_hint(artifact_loader); + let proving_assembly = + ::synthesize_for_proving::< + Self::PreviousProofSystem, + >(input_vk, input_proof, Self::MODE); + let proof_config = + ::proof_config_for_compression_step::< + Self, + >(); + Self::prove_step( + ctx, + proving_assembly, + proof_config, + precomputation, + finalization_hint, + ) + } +} + +pub trait CompressionStepExt: CompressionStep { + fn run_precomputation_for_compression( + artifact_loader: &AL, + ) -> ::Proof + where + AL: ArtifactLoader, + { + let input_vk = Self::load_previous_vk(artifact_loader); + let finalization_hint = Self::load_finalization_hint(artifact_loader); + let setup_assembly = + ::synthesize_for_setup::< + Self::PreviousProofSystem, + >(input_vk, Self::MODE); + ::generate_precomputation_and_vk( + setup_assembly, + finalization_hint, + ); + todo!() + } +} + +impl StepDefinition for CompressionMode1 { + type PreviousProofSystem = BoojumProofSystem; + type ThisProofSystem = BoojumProofSystem; +} + +impl CompressionStep for CompressionMode1 { + const MODE: u8 = 1; + const IS_WRAPPER: bool = false; +} + +impl StepDefinition for CompressionMode2 { + type PreviousProofSystem = BoojumProofSystem; + type ThisProofSystem = BoojumProofSystem; +} + +impl CompressionStep for CompressionMode2 { + const MODE: u8 = 2; + const IS_WRAPPER: bool = false; +} + +impl StepDefinition for CompressionMode3 { + type PreviousProofSystem = BoojumProofSystem; + type ThisProofSystem = BoojumProofSystem; +} + +impl CompressionStep for CompressionMode3 { + const MODE: u8 = 3; + const IS_WRAPPER: bool = false; +} + +impl StepDefinition for CompressionMode4 { + type PreviousProofSystem = BoojumProofSystem; + type ThisProofSystem = BoojumProofSystem; +} + +impl CompressionStep for CompressionMode4 { + const MODE: u8 = 4; + const IS_WRAPPER: bool = false; +} + +impl StepDefinition for CompressionMode1ForWrapper { + type PreviousProofSystem = BoojumProofSystem; + type ThisProofSystem = BoojumProofSystem; +} + +impl CompressionStep for CompressionMode1ForWrapper { + const MODE: u8 = 1; + const IS_WRAPPER: bool = true; +} + +impl StepDefinition for CompressionMode5ForWrapper { + type PreviousProofSystem = BoojumProofSystem; + type ThisProofSystem = BoojumProofSystem; +} + +impl CompressionStep for CompressionMode5ForWrapper { + const MODE: u8 = 5; + const IS_WRAPPER: bool = true; +} diff --git a/crates/proof-compression/src/context.rs b/crates/proof-compression/src/context.rs new file mode 100644 index 0000000..ab226f5 --- /dev/null +++ b/crates/proof-compression/src/context.rs @@ -0,0 +1,7 @@ +use crate::{AsyncHandler, ProofSystemDefinition}; + +pub trait ContextInitializator { + fn init

(config: P::ContextConfig) -> AsyncHandler + where + P: ProofSystemDefinition; +} diff --git a/crates/proof-compression/src/lib.rs b/crates/proof-compression/src/lib.rs index ad2b976..4383218 100644 --- a/crates/proof-compression/src/lib.rs +++ b/crates/proof-compression/src/lib.rs @@ -1,10 +1,38 @@ #![allow(incomplete_features)] #![feature(generic_const_exprs)] #![feature(allocator_api)] +#![feature(associated_type_defaults)] // TODO + +mod artifacts; +use artifacts::*; + +mod chain; +use chain::*; mod common; use common::*; +mod compression; +use compression::*; + +mod context; +use context::*; + +mod proof_system; +use proof_system::*; + +mod snark_wrapper; +use snark_wrapper::*; + +mod step; +use step::*; + +mod serialization; +use serialization::*; + +mod task; +use task::*; + cfg_if::cfg_if! { if #[cfg(feature = "gpu")] { mod gpu; @@ -15,18 +43,41 @@ cfg_if::cfg_if! { pub use cpu::*; } } +use bellman::worker::Worker; use boojum::pairing::bn256::{Bn256, Fr}; -use circuit_definitions::circuit_definitions::aux_layer::ZkSyncSnarkWrapperCircuitNoLookupCustomGate; +use circuit_definitions::circuit_definitions::{ + aux_layer::{ZkSyncSnarkWrapperCircuit, ZkSyncSnarkWrapperCircuitNoLookupCustomGate}, + recursion_layer::RecursiveProofsTreeHasher, +}; use fflonk::{ + bellman::plonk::better_better_cs::{ + cs::VerificationKey as PlonkVerificationKey, proof::Proof as PlonkProof, + setup::Setup as PlonkSetup, + }, fflonk::{FflonkProof, FflonkSetup, FflonkVerificationKey}, fflonk_cpu::franklin_crypto, + FflonkSnarkVerifierCircuitDeviceSetup, }; use franklin_crypto::bellman; -use shivini::boojum; - -use bellman::worker::Worker; +use gpu_prover::AsyncSetup; +use shivini::boojum::{self, field::goldilocks::GoldilocksExt2}; pub type FflonkSnarkVerifierCircuit = ZkSyncSnarkWrapperCircuitNoLookupCustomGate; pub type FflonkSnarkVerifierCircuitVK = FflonkVerificationKey; pub type FflonkSnarkVerifierCircuitProof = FflonkProof; -pub type FflonkSnarkVerifierCircuitSetup = FflonkSetup; + +pub type PlonkSnarkVerifierCircuit = ZkSyncSnarkWrapperCircuit; +pub type PlonkSnarkVerifierCircuitVK = PlonkVerificationKey; +pub type PlonkSnarkVerifierCircuitProof = PlonkProof; +pub type PlonkSnarkVerifierCircuitDeviceSetup = AsyncSetup; + +pub use fflonk::{GlobalHost, HostAllocator}; +use std::alloc::Global; + +use shivini::boojum::field::goldilocks::GoldilocksField; + +pub type SchedulerProof = boojum::cs::implementations::proof::Proof< + GoldilocksField, + RecursiveProofsTreeHasher, + GoldilocksExt2, +>; diff --git a/crates/proof-compression/src/proof_system.rs b/crates/proof-compression/src/proof_system.rs new file mode 100644 index 0000000..7604cc3 --- /dev/null +++ b/crates/proof-compression/src/proof_system.rs @@ -0,0 +1,440 @@ +use super::*; +use bellman::pairing::compact_bn256::G1Affine as CompactG1Affine; +use bellman::plonk::better_better_cs::cs::SynthesisModeGenerateSetup; +use bellman::plonk::better_better_cs::{ + cs::{Assembly, PlonkCsWidth3Params, PlonkCsWidth4WithNextStepParams, SynthesisModeProve}, + gates::{ + naive_main_gate::NaiveMainGate, + selector_optimized_with_d_next::SelectorOptimizedWidth4MainGateWithDNext, + }, +}; +use boojum::config::SetupCSConfig; +use boojum::cs::implementations::prover::ProofConfig; +use boojum::cs::oracle::TreeHasher; +use boojum::{ + config::ProvingCSConfig, + cs::implementations::{ + fast_serialization::MemcopySerializable, proof::Proof, reference_cs::CSReferenceAssembly, + setup::FinalizationHintsForProver, verifier::VerificationKey, + }, + field::goldilocks::GoldilocksExt2, +}; +use circuit_definitions::circuit_definitions::aux_layer::compression::ProofCompressionFunction; +use fflonk::DeviceContextWithSingleDevice; +use gpu_prover::{DeviceMemoryManager, ManagerConfigs}; +use shivini::ProverContextConfig; +use shivini::{cs::GpuSetup, GpuTreeHasher, ProverContext}; + +// We can't have circuit interface as part of the proof system +// definition due to type dependency between step inputs +// circuit(input_vk, input_proof), however we can synthesize it +// through aux interface +pub trait ProofSystemDefinition: Sized { + type FieldElement; + type Precomputation: MemcopySerializable; + type Proof: serde::Serialize + serde::de::DeserializeOwned; + type VK: serde::Serialize + serde::de::DeserializeOwned; + type FinalizationHint: serde::Serialize + serde::de::DeserializeOwned; + type Allocator: std::alloc::Allocator; + type ProvingAssembly: Sized + Send + Sync + 'static; + type ContextConfig; + type Context; + type ProofConfig: 'static; + fn get_context_config() -> Self::ContextConfig { + todo!() + } + fn init_context(config: Self::ContextConfig) -> AsyncHandler; + fn build_proving_assembly() -> Self::ProvingAssembly; + + fn prove( + _: Self::ProvingAssembly, + _: Self::Precomputation, + _: Self::FinalizationHint, + _: Self::ProofConfig, + ) -> Self::Proof; + + fn take_witnesses( + proving_assembly: &mut Self::ProvingAssembly, + ) -> Vec; + + fn prove_from_witnesses( + _: Vec, + _: Self::Precomputation, + _: Self::FinalizationHint, + _: Self::ProofConfig, + ) -> Self::Proof; + + fn verify(_: &Self::Proof, _: &Self::VK) -> bool; +} + +pub trait CompressionProofSystem: ProofSystemDefinition { + fn proof_config_for_compression_step() -> Self::ProofConfig + where + CF: ProofCompressionFunction; + + fn synthesize_for_proving

( + input_vk: P::VK, + input_proof: P::Proof, + compression_mode: u8, + ) -> Self::ProvingAssembly + where + P: ProofSystemDefinition, + { + todo!() + } +} + +pub trait CompressionProofSystemExt: ProofSystemExt { + fn synthesize_for_setup

(input_vk: P::VK, mode: u8) -> Self::SetupAssembly + where + P: ProofSystemDefinition; +} + +pub trait SnarkWrapperProofSystem: ProofSystemDefinition { + fn proof_config() -> Self::ProofConfig; + fn synthesize_for_proving

(input_vk: P::VK, input_proof: P::Proof) -> Self::ProvingAssembly + where + P: ProofSystemDefinition; +} + +pub trait SnarkWrapperProofSystemExt: ProofSystemExt { + fn synthesize_for_setup

(input_vk: P::VK) -> Self::SetupAssembly + where + P: ProofSystemDefinition; +} + +pub trait ProofSystemExt: ProofSystemDefinition { + type SetupAssembly: Sized + Send + Sync + 'static; + fn generate_precomputation_and_vk( + _: Self::SetupAssembly, + _: Self::FinalizationHint, + ) -> (AsyncHandler, Self::VK); +} +pub struct BoojumProofSystem> { + _hasher: std::marker::PhantomData, +} + +type BoojumAssembly = + CSReferenceAssembly; + +pub struct TreeHasherCompatibleGpuSetup>( + std::marker::PhantomData, +); + +impl ProofSystemDefinition for BoojumProofSystem +where + H: TreeHasher, +{ + type FieldElement = GoldilocksField; + type Precomputation = TreeHasherCompatibleGpuSetup; + type Proof = Proof; + type VK = VerificationKey; + type FinalizationHint = FinalizationHintsForProver; + type Allocator = std::alloc::Global; + type ProvingAssembly = BoojumAssembly; + type ContextConfig = usize; // domain_size + type Context = ProverContext; + type ProofConfig = ProofConfig; + // type Circuit = ZkSyncCompressionLayerCircuit; + fn init_context(domain_size: Self::ContextConfig) -> AsyncHandler { + let config = + ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); + let context = Self::Context::create_with_config(config).expect("gpu prover context"); + todo!() + } + + fn build_proving_assembly() -> Self::ProvingAssembly { + todo!() + } + + fn prove( + _: Self::ProvingAssembly, + _: Self::Precomputation, + _: Self::FinalizationHint, + _: Self::ProofConfig, + ) -> Self::Proof { + todo!() + } + fn prove_from_witnesses( + _: Vec, + _: Self::Precomputation, + _: Self::FinalizationHint, + _: Self::ProofConfig, + ) -> Self::Proof { + todo!() + } + + fn verify(_: &Self::Proof, _: &Self::VK) -> bool { + todo!() + } + + fn take_witnesses( + proving_assembly: &mut Self::ProvingAssembly, + ) -> Vec { + todo!() + } +} + +impl> CompressionProofSystem for BoojumProofSystem +where + H: TreeHasher, +{ + fn proof_config_for_compression_step() -> Self::ProofConfig + where + CF: ProofCompressionFunction, + { + CF::proof_config_for_compression_step() + } + + fn synthesize_for_proving

( + input_vk: P::VK, + input_proof: P::Proof, + compression_mode: u8, + ) -> Self::ProvingAssembly + where + P: ProofSystemDefinition, + { + todo!() + } +} + +impl ProofSystemExt for BoojumProofSystem +where + H: TreeHasher, +{ + type SetupAssembly = BoojumAssembly; + fn generate_precomputation_and_vk( + _: Self::SetupAssembly, + _: Self::FinalizationHint, + ) -> (AsyncHandler, Self::VK) { + todo!() + } +} + +pub struct PlonkProofSystem; + +type PlonkAssembly = Assembly< + Bn256, + PlonkCsWidth4WithNextStepParams, + SelectorOptimizedWidth4MainGateWithDNext, + CSConfig, + A, +>; + +pub struct PlonkProverDeviceMemoryManager; + +impl ManagerConfigs for PlonkProverDeviceMemoryManager { + const NUM_GPUS_LOG: usize = 0; + const FULL_SLOT_SIZE_LOG: usize = 24; + const NUM_SLOTS: usize = 29; + const NUM_HOST_SLOTS: usize = 2; +} + +impl ProofSystemDefinition for PlonkProofSystem { + type FieldElement = Fr; + type Precomputation = PlonkSnarkVerifierCircuitDeviceSetupWrapper; + type Proof = PlonkSnarkVerifierCircuitProof; + type VK = PlonkSnarkVerifierCircuitVK; + type FinalizationHint = usize; + type Allocator = GlobalHost; + type ProvingAssembly = PlonkAssembly; + type ContextConfig = (Vec, Vec); + type Context = DeviceMemoryManager; + type ProofConfig = (); + fn init_context(config: Self::ContextConfig) -> AsyncHandler { + let (device_ids, compact_crs) = config; + let ctx = Self::Context::init(&device_ids, &compact_crs[..]); + todo!() + } + + fn build_proving_assembly() -> Self::ProvingAssembly { + todo!() + } + + fn prove( + _: Self::ProvingAssembly, + _: Self::Precomputation, + _: Self::FinalizationHint, + _: Self::ProofConfig, + ) -> Self::Proof { + todo!() + } + + fn take_witnesses( + proving_assembly: &mut Self::ProvingAssembly, + ) -> Vec { + todo!() + } + + fn prove_from_witnesses( + _: Vec, + _: Self::Precomputation, + _: Self::FinalizationHint, + _: Self::ProofConfig, + ) -> Self::Proof { + todo!() + } + + fn verify(_: &Self::Proof, _: &Self::VK) -> bool { + todo!() + } +} + +impl SnarkWrapperProofSystem for PlonkProofSystem { + fn synthesize_for_proving

(input_vk: P::VK, input_proof: P::Proof) -> Self::ProvingAssembly + where + P: ProofSystemDefinition, + { + todo!() + } + + fn proof_config() -> Self::ProofConfig { + todo!() + } +} + +impl ProofSystemExt for PlonkProofSystem { + type SetupAssembly = PlonkAssembly; + + fn generate_precomputation_and_vk( + _: Self::SetupAssembly, + _: Self::FinalizationHint, + ) -> (AsyncHandler, Self::VK) { + todo!() + } +} + +pub struct FflonkProofSystem; + +type FflonkAssembly = Assembly; + +impl ProofSystemDefinition for FflonkProofSystem { + type FieldElement = Fr; + type Precomputation = FflonkSnarkVerifierCircuitDeviceSetupWrapper; + type Proof = FflonkSnarkVerifierCircuitProof; + type VK = FflonkSnarkVerifierCircuitVK; + type FinalizationHint = usize; + type Allocator = GlobalHost; + type ProvingAssembly = FflonkAssembly; + type ContextConfig = usize; // domain_size + type Context = DeviceContextWithSingleDevice; + type ProofConfig = (); + // type Circuit = (); + fn get_context_config() -> Self::ContextConfig { + todo!() + } + fn init_context(log_domain_size: Self::ContextConfig) -> AsyncHandler { + let domain_size = 1 << log_domain_size; + let context = Self::Context::init(domain_size).unwrap(); + + todo!() + } + + fn build_proving_assembly() -> Self::ProvingAssembly { + todo!() + } + + fn prove( + _: Self::ProvingAssembly, + _: Self::Precomputation, + _: Self::FinalizationHint, + _: Self::ProofConfig, + ) -> Self::Proof { + todo!() + } + + fn take_witnesses( + proving_assembly: &mut Self::ProvingAssembly, + ) -> Vec { + todo!() + } + + fn prove_from_witnesses( + _: Vec, + _: Self::Precomputation, + _: Self::FinalizationHint, + _: Self::ProofConfig, + ) -> Self::Proof { + todo!() + } + + fn verify(_: &Self::Proof, _: &Self::VK) -> bool { + todo!() + } +} + +impl SnarkWrapperProofSystem for FflonkProofSystem { + fn synthesize_for_proving

(input_vk: P::VK, input_proof: P::Proof) -> Self::ProvingAssembly + where + P: ProofSystemDefinition, + { + todo!() + } + + fn proof_config() -> Self::ProofConfig { + todo!() + } +} + +impl ProofSystemExt for FflonkProofSystem { + type SetupAssembly = FflonkAssembly; + fn generate_precomputation_and_vk( + _: Self::SetupAssembly, + _: Self::FinalizationHint, + ) -> (AsyncHandler, Self::VK) { + todo!() + } +} + +pub struct MarkerProofSystem; +impl ProofSystemDefinition for MarkerProofSystem { + type FieldElement = (); + type Precomputation = MarkerPrecomputation; + type Proof = (); + type VK = (); + type FinalizationHint = (); + type Allocator = std::alloc::Global; + type ProvingAssembly = (); + type ContextConfig = (); + type Context = (); + type ProofConfig = (); + // type Circuit = (); + fn get_context_config() -> Self::ContextConfig { + todo!() + } + fn init_context(config: Self::ContextConfig) -> AsyncHandler { + todo!() + } + + fn build_proving_assembly() -> Self::ProvingAssembly { + todo!() + } + + fn prove( + _: Self::ProvingAssembly, + _: Self::Precomputation, + _: Self::FinalizationHint, + _: Self::ProofConfig, + ) -> Self::Proof { + todo!() + } + + fn prove_from_witnesses( + _: Vec, + _: Self::Precomputation, + _: Self::FinalizationHint, + _: Self::ProofConfig, + ) -> Self::Proof { + todo!() + } + + fn verify(_: &Self::Proof, _: &Self::VK) -> bool { + todo!() + } + + fn take_witnesses( + proving_assembly: &mut Self::ProvingAssembly, + ) -> Vec { + todo!() + } +} diff --git a/crates/proof-compression/src/serialization.rs b/crates/proof-compression/src/serialization.rs new file mode 100644 index 0000000..e978e02 --- /dev/null +++ b/crates/proof-compression/src/serialization.rs @@ -0,0 +1,104 @@ +use super::*; + +use boojum::cs::oracle::TreeHasher; +use fflonk::FflonkSnarkVerifierCircuitDeviceSetup; +use shivini::boojum::cs::implementations::fast_serialization::MemcopySerializable; + +use crate::{PlonkSnarkVerifierCircuitDeviceSetup, TreeHasherCompatibleGpuSetup}; + +pub struct PlonkSnarkVerifierCircuitDeviceSetupWrapper(PlonkSnarkVerifierCircuitDeviceSetup); + +impl MemcopySerializable for PlonkSnarkVerifierCircuitDeviceSetupWrapper { + fn write_into_buffer( + &self, + dst: W, + ) -> Result<(), Box> { + todo!() + } + + fn read_from_buffer(src: R) -> Result> { + todo!() + } +} + +impl serde::Serialize for PlonkSnarkVerifierCircuitDeviceSetupWrapper { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + +impl<'de> serde::Deserialize<'de> for PlonkSnarkVerifierCircuitDeviceSetupWrapper { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + todo!() + } +} + +pub struct FflonkSnarkVerifierCircuitDeviceSetupWrapper(FflonkSnarkVerifierCircuitDeviceSetup); + +impl MemcopySerializable for FflonkSnarkVerifierCircuitDeviceSetupWrapper { + fn write_into_buffer( + &self, + dst: W, + ) -> Result<(), Box> { + todo!() + } + + fn read_from_buffer(src: R) -> Result> { + todo!() + } +} + +impl serde::Serialize for FflonkSnarkVerifierCircuitDeviceSetupWrapper { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + +impl<'de> serde::Deserialize<'de> for FflonkSnarkVerifierCircuitDeviceSetupWrapper { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + todo!() + } +} + +#[derive(serde::Serialize, serde::Deserialize)] +pub struct MarkerPrecomputation; +impl MemcopySerializable for MarkerPrecomputation { + fn write_into_buffer( + &self, + dst: W, + ) -> Result<(), Box> { + todo!() + } + + fn read_from_buffer(src: R) -> Result> { + todo!() + } +} + +impl MemcopySerializable for TreeHasherCompatibleGpuSetup +where + H: TreeHasher, +{ + fn write_into_buffer( + &self, + dst: W, + ) -> Result<(), Box> { + todo!() + } + + fn read_from_buffer(src: R) -> Result> { + todo!() + } +} diff --git a/crates/proof-compression/src/snark_wrapper.rs b/crates/proof-compression/src/snark_wrapper.rs new file mode 100644 index 0000000..115d0c3 --- /dev/null +++ b/crates/proof-compression/src/snark_wrapper.rs @@ -0,0 +1,138 @@ +use circuit_definitions::circuit_definitions::aux_layer::compression_modes::CompressionTreeHasherForWrapper; + +use super::*; + +pub trait SnarkWrapperStep: StepDefinition { + const IS_PLONK: bool; + const IS_FFLONK: bool; + const LAST_COMPRESSION_MODE: u8; + + fn load_finalization_hint( + artifact_loader: &AL, + ) -> ::FinalizationHint + where + AL: ArtifactLoader, + { + assert!(Self::IS_FFLONK ^ Self::IS_PLONK); + let hint = if Self::IS_PLONK { &[26u8] } else { &[24] }; + serde_json::from_reader(&hint[..]).unwrap() + } + + fn load_previous_vk( + artifact_loader: &AL, + ) -> ::VK + where + AL: ArtifactLoader, + { + assert!(Self::IS_FFLONK ^ Self::IS_PLONK); + let reader = artifact_loader.read_compression_wrapper_vk(Self::LAST_COMPRESSION_MODE); + serde_json::from_reader(reader).unwrap() + } + + fn load_this_vk( + artifact_loader: &AL, + ) -> ::VK + where + AL: ArtifactLoader, + { + assert!(Self::IS_FFLONK ^ Self::IS_PLONK); + let reader = if Self::IS_FFLONK { + assert_eq!(Self::IS_PLONK, false); + artifact_loader.read_fflonk_vk() + } else { + assert_eq!(Self::IS_PLONK, true); + artifact_loader.read_plonk_vk() + }; + + serde_json::from_reader(reader).unwrap() + } + + fn get_precomputation( + artifact_loader: &AL, + ) -> AsyncHandler<::Precomputation> + where + AL: ArtifactLoader, + { + todo!() + } + + fn prove_snark_wrapper_step( + input_proof: ::Proof, + artifact_loader: &AL, + ) -> ::Proof + where + AL: ArtifactLoader, + CI: ContextInitializator, + { + assert!(Self::IS_FFLONK ^ Self::IS_PLONK); + let input_vk = Self::load_previous_vk(artifact_loader); + let precomputation = Self::get_precomputation(artifact_loader); + let config = ::get_context_config(); + let ctx = CI::init::(config); + let finalization_hint = Self::load_finalization_hint(artifact_loader); + let proving_assembly = + ::synthesize_for_proving::< + Self::PreviousProofSystem, + >(input_vk, input_proof); + let proof_config = ::proof_config(); + Self::prove_step( + ctx, + proving_assembly, + proof_config, + precomputation, + finalization_hint, + ) + } +} + +pub trait SnarkWrapperStepExt: + SnarkWrapperStep +{ + fn run_precomputation_for_compression( + artifact_loader: &AL, + ) -> ( + ::Precomputation, + ::VK, + ) + where + AL: ArtifactLoader, + { + let input_vk = Self::load_previous_vk(artifact_loader); + let finalization_hint = Self::load_finalization_hint(artifact_loader); + let setup_assembly = + ::synthesize_for_setup::< + Self::PreviousProofSystem, + >(input_vk); + let (precomputation, vk) = + ::generate_precomputation_and_vk( + setup_assembly, + finalization_hint, + ); + (precomputation.into_inner(), vk) + } +} + +pub struct FflonkSnarkWrapper; + +impl StepDefinition for FflonkSnarkWrapper { + type PreviousProofSystem = BoojumProofSystem; + type ThisProofSystem = FflonkProofSystem; +} + +impl SnarkWrapperStep for FflonkSnarkWrapper { + const IS_PLONK: bool = false; + const IS_FFLONK: bool = true; + const LAST_COMPRESSION_MODE: u8 = 5; +} +pub struct PlonkSnarkWrapper; + +impl StepDefinition for PlonkSnarkWrapper { + type PreviousProofSystem = BoojumProofSystem; + type ThisProofSystem = PlonkProofSystem; +} + +impl SnarkWrapperStep for PlonkSnarkWrapper { + const IS_PLONK: bool = true; + const IS_FFLONK: bool = false; + const LAST_COMPRESSION_MODE: u8 = 1; +} diff --git a/crates/proof-compression/src/step.rs b/crates/proof-compression/src/step.rs new file mode 100644 index 0000000..223f4ef --- /dev/null +++ b/crates/proof-compression/src/step.rs @@ -0,0 +1,46 @@ +use super::*; + +pub trait StepDefinition: Sized { + type PreviousProofSystem: ProofSystemDefinition; + type ThisProofSystem: ProofSystemDefinition; + + fn prove_step( + ctx: AsyncHandler<::Context>, + proving_assembly: ::ProvingAssembly, + proof_config: ::ProofConfig, + precomputation: AsyncHandler< + ::Precomputation, + >, + finalization_hint: ::FinalizationHint, + ) -> ::Proof { + let _ = ctx.into_inner(); + let proof = ::prove( + proving_assembly, + precomputation.into_inner(), + finalization_hint, + proof_config, + ); + + proof + } +} + +pub trait StepDefinitionExt: StepDefinition { + fn generate_precomputation_and_vk( + ctx: AsyncHandler<::Context>, + setup_assembly: ::SetupAssembly, + proof_config: ::ProofConfig, + finalization_hint: ::FinalizationHint, + ) -> ( + AsyncHandler<::Precomputation>, + ::VK, + ) { + let _ = ctx.into_inner(); + let (precomputation, vk) = + ::generate_precomputation_and_vk( + setup_assembly, + finalization_hint, + ); + (precomputation, vk) + } +} From e5df8c0c69b69acbb47065ef26d2f087171aa7f6 Mon Sep 17 00:00:00 2001 From: saitima Date: Wed, 8 Jan 2025 22:15:44 +0300 Subject: [PATCH 03/18] feat(proof-compression): implement proof systems --- crates/proof-compression/src/artifacts.rs | 57 +- crates/proof-compression/src/chain.rs | 236 ++----- crates/proof-compression/src/common.rs | 3 +- crates/proof-compression/src/compression.rs | 231 ++++--- crates/proof-compression/src/context.rs | 16 + crates/proof-compression/src/gpu.rs | 1 - crates/proof-compression/src/lib.rs | 10 +- crates/proof-compression/src/proof_system.rs | 599 +++++++++++------- crates/proof-compression/src/serialization.rs | 33 +- crates/proof-compression/src/snark_wrapper.rs | 131 ++-- crates/proof-compression/src/step.rs | 45 -- 11 files changed, 716 insertions(+), 646 deletions(-) diff --git a/crates/proof-compression/src/artifacts.rs b/crates/proof-compression/src/artifacts.rs index c9d56a3..37171bb 100644 --- a/crates/proof-compression/src/artifacts.rs +++ b/crates/proof-compression/src/artifacts.rs @@ -3,31 +3,44 @@ use std::io::Read; use boojum::cs::implementations::fast_serialization::MemcopySerializable; use boojum::cs::implementations::setup::FinalizationHintsForProver; use boojum::cs::traits::gate::FinalizationHintSerialized; -use circuit_definitions::circuit_definitions::aux_layer::compression_modes::CompressionTreeHasherForWrapper; -use circuit_definitions::circuit_definitions::aux_layer::{ - CompressionProofsTreeHasher, ZkSyncCompressionVerificationKey, -}; +use circuit_definitions::circuit_definitions::aux_layer::ZkSyncCompressionVerificationKey; use circuit_definitions::circuit_definitions::recursion_layer::ZkSyncRecursionVerificationKey; -use shivini::cs::GpuSetup as BoojumDeviceSetup; use super::*; pub trait BlobStorage { - fn save(artifact: T) + fn save(data: W) where - T: MemcopySerializable; + W: std::io::Write; } pub struct AsyncHandler { - precomputation: std::sync::mpsc::Receiver, // This is indeed a receiver + receiver: std::sync::mpsc::Receiver, } -impl AsyncHandler { - pub fn into_inner(self) -> T { - todo!() +impl AsyncHandler +where + T: Send + Sync + 'static, +{ + pub fn spawn(f: F) -> Self + where + F: FnOnce() -> std::sync::mpsc::Receiver + Send + Sync + 'static, + { + let receiver = std::thread::spawn(f); + + Self { + receiver: receiver.join().unwrap(), + } + } + + pub fn wait(self) -> T { + self.receiver.recv().unwrap() } } -pub trait ArtifactLoader: Sized { + +use shivini::cs::GpuSetup; + +pub trait ArtifactLoader: Sized + Send + Sync { fn init(bs: BS) -> Self where BS: BlobStorage; @@ -54,18 +67,16 @@ pub trait ArtifactLoader: Sized { fn get_compression_layer_precomputation( &self, circuit_id: u8, - ) -> AsyncHandler>; + ) -> Box; fn get_compression_wrapper_precomputation( &self, circuit_id: u8, - ) -> AsyncHandler>; + ) -> Box; fn get_plonk_snark_wrapper_precomputation( &self, circuit_id: u8, - ) -> AsyncHandler; - fn get_fflonk_snark_wrapper_precomputation( - &self, - ) -> AsyncHandler; + ) -> Box; + fn get_fflonk_snark_wrapper_precomputation(&self) -> Box; } pub struct SimpleArtifactLoader; @@ -139,27 +150,25 @@ impl ArtifactLoader for SimpleArtifactLoader { fn get_compression_layer_precomputation( &self, circuit_id: u8, - ) -> AsyncHandler> { + ) -> Box { todo!() } fn get_compression_wrapper_precomputation( &self, circuit_id: u8, - ) -> AsyncHandler> { + ) -> Box { todo!() } fn get_plonk_snark_wrapper_precomputation( &self, circuit_id: u8, - ) -> AsyncHandler { + ) -> Box { todo!() } - fn get_fflonk_snark_wrapper_precomputation( - &self, - ) -> AsyncHandler { + fn get_fflonk_snark_wrapper_precomputation(&self) -> Box { todo!() } } diff --git a/crates/proof-compression/src/chain.rs b/crates/proof-compression/src/chain.rs index 286c7b8..b3a15b2 100644 --- a/crates/proof-compression/src/chain.rs +++ b/crates/proof-compression/src/chain.rs @@ -1,200 +1,88 @@ -use circuit_definitions::circuit_definitions::aux_layer::{ - compression::ProofCompressionFunction, - compression_modes::{ - CompressionMode1, CompressionMode1ForWrapper, CompressionMode2, CompressionMode3, - CompressionMode4, CompressionMode5ForWrapper, - }, -}; - use super::*; -pub struct CompressionChain(std::marker::PhantomData<(C, U)>); - -impl CompressionChain -where - C: CompressionStep, -{ - pub fn new(step: C) -> CompressionChain - where - U: CompressionStep, - { - todo!() - } - - pub fn next(self) -> U { - todo!() - } +pub enum SnarkWrapper { + Plonk, + FFfonk, } - -impl StepDefinition for CompressionChain -where - C: CompressionStep, -{ - type PreviousProofSystem = C::PreviousProofSystem; - type ThisProofSystem = C::ThisProofSystem; +pub enum SnarkWrapperProof { + Plonk(PlonkSnarkVerifierCircuitProof), + FFfonk(FflonkSnarkVerifierCircuitProof), } - -impl ProofCompressionFunction for CompressionChain -where - C: CompressionStep, -{ - type PreviousLayerPoW = C::PreviousLayerPoW; - - type ThisLayerPoW = C::ThisLayerPoW; - - type ThisLayerHasher = C::ThisLayerHasher; - - type ThisLayerTranscript = C::ThisLayerTranscript; - - fn this_layer_transcript_parameters( - ) -> >::TransciptParameters { - C::this_layer_transcript_parameters() - } - - fn description_for_compression_step() -> String { - C::description_for_compression_step() - } - - fn size_hint_for_compression_step() -> (usize, usize) { - C::size_hint_for_compression_step() - } - - fn geometry_for_compression_step() -> boojum::cs::CSGeometry { - C::geometry_for_compression_step() - } - - fn lookup_parameters_for_compression_step() -> boojum::cs::LookupParameters { - C::lookup_parameters_for_compression_step() - } - - fn configure_builder_for_compression_step< - T: boojum::cs::cs_builder::CsBuilderImpl, - GC: boojum::cs::GateConfigurationHolder, - TB: boojum::cs::StaticToolboxHolder, - >( - builder: boojum::cs::cs_builder::CsBuilder, - ) -> boojum::cs::cs_builder::CsBuilder< - T, - GoldilocksField, - impl boojum::cs::GateConfigurationHolder, - impl boojum::cs::StaticToolboxHolder, - > { - C::configure_builder_for_compression_step(builder) - } - - fn previous_step_builder_for_compression< - CS: boojum::cs::traits::cs::ConstraintSystem + 'static, - >() -> Box< - dyn boojum::cs::traits::circuit::ErasedBuilderForRecursiveVerifier< - GoldilocksField, - GoldilocksExt2, - CS, - >, - > { - C::previous_step_builder_for_compression() - } - - fn proof_config_for_compression_step() -> boojum::cs::implementations::prover::ProofConfig { - C::proof_config_for_compression_step() - } -} - -impl CompressionStep for CompressionChain -where - C: CompressionStep, -{ - const MODE: u8 = C::MODE; - const IS_WRAPPER: bool = C::IS_WRAPPER; -} - -pub struct CompressionChainBuilder(std::marker::PhantomData); - -impl CompressionChainBuilder -where - C: CompressionStep, -{ - pub fn new(step: C) -> CompressionChainBuilder { - todo!() - } -} - -impl CompressionChainBuilder -where - C: CompressionStep, -{ - pub fn link_compression_step( - self, - step: CC, - ) -> CompressionChainBuilder> - where - CC: CompressionStep, - { - todo!() - } - - pub fn build(self) -> C { - todo!() - } -} - -pub fn build_full_chain_with_fflonk() -> CompressionChain< - CompressionChain< - CompressionChain, CompressionMode3>, - CompressionMode4, - >, - CompressionMode5ForWrapper, -> { - let chain = CompressionChainBuilder::new(CompressionMode1) - .link_compression_step(CompressionMode2) - .link_compression_step(CompressionMode3) - .link_compression_step(CompressionMode4) - .link_compression_step(CompressionMode5ForWrapper) - .build(); - - chain -} - -pub fn build_full_chain_with_plonk() -> CompressionChain { - todo!() -} - -pub fn run_step_chain_with_fflonk( +pub fn wrap_proof( input_proof: SchedulerProof, + snark_wrapper: SnarkWrapper, blob_storage: BS, -) -> FflonkSnarkVerifierCircuitProof +) -> SnarkWrapperProof where BS: BlobStorage, { - let chain = build_full_chain_with_fflonk(); - run_step_chain::<_, _, FflonkSnarkWrapper, _>(chain, input_proof, blob_storage) + match snark_wrapper { + SnarkWrapper::Plonk => { + let proof = run_step_chain_with_plonk(input_proof, blob_storage); + SnarkWrapperProof::Plonk(proof) + } + SnarkWrapper::FFfonk => { + let proof = run_step_chain_with_fflonk(input_proof, blob_storage); + SnarkWrapperProof::FFfonk(proof) + } + } } -pub fn run_step_chain_with_plonk( +pub(crate) fn run_step_chain_with_fflonk( input_proof: SchedulerProof, blob_storage: BS, -) -> PlonkSnarkVerifierCircuitProof +) -> FflonkSnarkVerifierCircuitProof where BS: BlobStorage, { - let chain = build_full_chain_with_plonk(); - todo!() + let artifact_loader = SimpleArtifactLoader::init(blob_storage); + let context_initializor = SimpelContextInitializor::new(); + + let next_proof = CompressionMode1::prove_compression_step::<_, SimpelContextInitializor>( + input_proof, + &artifact_loader, + ); + let next_proof = CompressionMode2::prove_compression_step::<_, SimpelContextInitializor>( + next_proof, + &artifact_loader, + ); + let next_proof = CompressionMode3::prove_compression_step::<_, SimpelContextInitializor>( + next_proof, + &artifact_loader, + ); + let next_proof = CompressionMode4::prove_compression_step::<_, SimpelContextInitializor>( + next_proof, + &artifact_loader, + ); + let next_proof = CompressionMode5ForWrapper::prove_compression_step::< + _, + SimpelContextInitializor, + >(next_proof, &artifact_loader); + let final_proof = FflonkSnarkWrapper::prove_snark_wrapper_step::<_, SimpelContextInitializor>( + next_proof, + &artifact_loader, + ); + + final_proof } -pub fn run_step_chain( - chain: CompressionChain, +pub(crate) fn run_step_chain_with_plonk( input_proof: SchedulerProof, blob_storage: BS, -) -> ::Proof +) -> PlonkSnarkVerifierCircuitProof where - C1: CompressionStep, - C2: CompressionStep, - SW: SnarkWrapperStep, BS: BlobStorage, { let artifact_loader = SimpleArtifactLoader::init(blob_storage); + let next_proof = CompressionMode1ForWrapper::prove_compression_step::< + _, + SimpelContextInitializor, + >(input_proof, &artifact_loader); + + let final_proof = PlonkSnarkWrapper::prove_snark_wrapper_step::<_, SimpelContextInitializor>( + next_proof, + &artifact_loader, + ); - let next_step = chain.next(); - todo!() + final_proof } diff --git a/crates/proof-compression/src/common.rs b/crates/proof-compression/src/common.rs index 2a047a9..68c780a 100644 --- a/crates/proof-compression/src/common.rs +++ b/crates/proof-compression/src/common.rs @@ -1,7 +1,6 @@ +use super::*; use circuit_definitions::circuit_definitions::{ aux_layer::{ - compression::{CompressionLayerCircuit, ProofCompressionFunction}, - compression_modes::CompressionTranscriptForWrapper, CompressionProofsTreeHasher, CompressionProofsTreeHasherForWrapper, ZkSyncCompressionForWrapperCircuit, ZkSyncCompressionLayerCircuit, ZkSyncCompressionProof, ZkSyncCompressionProofForWrapper, ZkSyncCompressionVerificationKey, diff --git a/crates/proof-compression/src/compression.rs b/crates/proof-compression/src/compression.rs index 90c57c9..239e2b2 100644 --- a/crates/proof-compression/src/compression.rs +++ b/crates/proof-compression/src/compression.rs @@ -1,25 +1,30 @@ use super::*; +use boojum::cs::{ + implementations::{ + fast_serialization::MemcopySerializable, proof::Proof, verifier::VerificationKey, + }, + oracle::TreeHasher, +}; use circuit_definitions::circuit_definitions::{ aux_layer::{ - compression::ProofCompressionFunction, - compression_modes::{ - CompressionMode1, CompressionMode1ForWrapper, CompressionMode2, CompressionMode3, - CompressionMode4, CompressionMode5ForWrapper, CompressionTreeHasherForWrapper, - }, - CompressionProofsTreeHasher, + compression::{CompressionLayerCircuit, ProofCompressionFunction}, + ZkSyncCompressionForWrapperCircuit, ZkSyncCompressionLayerCircuit, }, recursion_layer::RecursiveProofsTreeHasher, }; -pub trait CompressionStep: - ProofCompressionFunction + StepDefinition -{ +pub trait CompressionStep: CompressionProofSystem { + type PreviousStepTreeHasher: TreeHasher< + GoldilocksField, + Output: serde::Serialize + serde::de::DeserializeOwned, + >; + const MODE: u8; const IS_WRAPPER: bool; fn load_finalization_hint( artifact_loader: &AL, - ) -> ::FinalizationHint + ) -> ::FinalizationHint where AL: ArtifactLoader, { @@ -33,7 +38,7 @@ pub trait CompressionStep: fn load_previous_vk( artifact_loader: &AL, - ) -> ::VK + ) -> VerificationKey where AL: ArtifactLoader, { @@ -50,9 +55,7 @@ pub trait CompressionStep: serde_json::from_reader(reader).unwrap() } - fn load_this_vk( - artifact_loader: &AL, - ) -> ::VK + fn load_this_vk(artifact_loader: &AL) -> ::VK where AL: ArtifactLoader, { @@ -67,121 +70,157 @@ pub trait CompressionStep: fn get_precomputation( artifact_loader: &AL, - ) -> AsyncHandler<::Precomputation> + ) -> AsyncHandler<::Precomputation> where AL: ArtifactLoader, { - todo!() + // TODO + let reader = if Self::IS_WRAPPER { + artifact_loader.get_compression_layer_precomputation(Self::MODE) + } else { + artifact_loader.get_compression_layer_precomputation(Self::MODE) + }; + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + + let precomputation = + <::Precomputation as MemcopySerializable>::read_from_buffer( + reader, + ) + .unwrap(); + + sender.send(precomputation).unwrap(); + receiver + }; + + AsyncHandler::spawn(f) } fn prove_compression_step( - input_proof: ::Proof, + input_proof: Proof, artifact_loader: &AL, - ) -> ::Proof + ) -> ::Proof where AL: ArtifactLoader, CI: ContextInitializator, { let input_vk = Self::load_previous_vk(artifact_loader); + let vk = Self::load_this_vk(artifact_loader); let precomputation = Self::get_precomputation(artifact_loader); - let config = ::get_context_config(); - let ctx = CI::init::(config); + let config = ::get_context_config(); + let ctx = CI::init::(config); let finalization_hint = Self::load_finalization_hint(artifact_loader); - let proving_assembly = - ::synthesize_for_proving::< - Self::PreviousProofSystem, - >(input_vk, input_proof, Self::MODE); - let proof_config = - ::proof_config_for_compression_step::< - Self, - >(); - Self::prove_step( + let circuit = Self::build_circuit(input_vk, Some(input_proof)); + let proving_assembly = ::synthesize_for_proving( + circuit, + finalization_hint.clone(), + ); + let aux_config = + ::aux_config_from_assembly(&proving_assembly); + ::prove( ctx, proving_assembly, - proof_config, + aux_config, precomputation, finalization_hint, + vk, ) } + + // CompressionLayerCircuit is unified type for both compression circuits + fn build_circuit( + input_vk: VerificationKey, + input_proof: Option>, + ) -> CompressionLayerCircuit; } -pub trait CompressionStepExt: CompressionStep { +pub trait CompressionStepExt: CompressionProofSystemExt + CompressionStep { fn run_precomputation_for_compression( artifact_loader: &AL, - ) -> ::Proof + ) -> ( + ::Precomputation, + ::VK, + ) where AL: ArtifactLoader, { let input_vk = Self::load_previous_vk(artifact_loader); - let finalization_hint = Self::load_finalization_hint(artifact_loader); - let setup_assembly = - ::synthesize_for_setup::< - Self::PreviousProofSystem, - >(input_vk, Self::MODE); - ::generate_precomputation_and_vk( + let circuit = Self::build_circuit(input_vk, None); + let (finalization_hint, setup_assembly) = + ::synthesize_for_setup(circuit); + let data = ::generate_precomputation_and_vk( setup_assembly, finalization_hint, ); - todo!() + data.wait() } } -impl StepDefinition for CompressionMode1 { - type PreviousProofSystem = BoojumProofSystem; - type ThisProofSystem = BoojumProofSystem; -} - -impl CompressionStep for CompressionMode1 { - const MODE: u8 = 1; - const IS_WRAPPER: bool = false; -} - -impl StepDefinition for CompressionMode2 { - type PreviousProofSystem = BoojumProofSystem; - type ThisProofSystem = BoojumProofSystem; +macro_rules! impl_compression_circuit { + ($type:ty,$mode:expr, $is_wrapper:expr, $enum:ident::$variant:ident, $hasher:ty) => { + impl CompressionStep for $type { + const MODE: u8 = $mode; + const IS_WRAPPER: bool = $is_wrapper; + type PreviousStepTreeHasher = $hasher; + fn build_circuit( + input_vk: VerificationKey, + input_proof: Option< + Proof, + >, + ) -> CompressionLayerCircuit { + let circuit = $enum::from_witness_and_vk(input_proof, input_vk, Self::MODE); + match circuit { + $enum::$variant(compression_layer_circuit) => compression_layer_circuit, + _ => unreachable!(), + } + } + } + }; } -impl CompressionStep for CompressionMode2 { - const MODE: u8 = 2; - const IS_WRAPPER: bool = false; -} - -impl StepDefinition for CompressionMode3 { - type PreviousProofSystem = BoojumProofSystem; - type ThisProofSystem = BoojumProofSystem; -} - -impl CompressionStep for CompressionMode3 { - const MODE: u8 = 3; - const IS_WRAPPER: bool = false; -} - -impl StepDefinition for CompressionMode4 { - type PreviousProofSystem = BoojumProofSystem; - type ThisProofSystem = BoojumProofSystem; -} - -impl CompressionStep for CompressionMode4 { - const MODE: u8 = 4; - const IS_WRAPPER: bool = false; -} - -impl StepDefinition for CompressionMode1ForWrapper { - type PreviousProofSystem = BoojumProofSystem; - type ThisProofSystem = BoojumProofSystem; -} - -impl CompressionStep for CompressionMode1ForWrapper { - const MODE: u8 = 1; - const IS_WRAPPER: bool = true; -} - -impl StepDefinition for CompressionMode5ForWrapper { - type PreviousProofSystem = BoojumProofSystem; - type ThisProofSystem = BoojumProofSystem; -} - -impl CompressionStep for CompressionMode5ForWrapper { - const MODE: u8 = 5; - const IS_WRAPPER: bool = true; -} +impl_compression_circuit!( + CompressionMode1, + 1, + false, + ZkSyncCompressionLayerCircuit::CompressionMode1Circuit, + RecursiveProofsTreeHasher +); +impl_compression_circuit!( + CompressionMode2, + 2, + false, + ZkSyncCompressionLayerCircuit::CompressionMode2Circuit, + ::ThisLayerHasher +); + +impl_compression_circuit!( + CompressionMode3, + 3, + false, + ZkSyncCompressionLayerCircuit::CompressionMode3Circuit, + ::ThisLayerHasher +); + +impl_compression_circuit!( + CompressionMode4, + 4, + false, + ZkSyncCompressionLayerCircuit::CompressionMode4Circuit, + ::ThisLayerHasher +); + +impl_compression_circuit!( + CompressionMode1ForWrapper, + 1, + true, + ZkSyncCompressionForWrapperCircuit::CompressionMode1Circuit, + RecursiveProofsTreeHasher +); + +impl_compression_circuit!( + CompressionMode5ForWrapper, + 5, + true, + ZkSyncCompressionForWrapperCircuit::CompressionMode5Circuit, + ::ThisLayerHasher +); diff --git a/crates/proof-compression/src/context.rs b/crates/proof-compression/src/context.rs index ab226f5..f54750d 100644 --- a/crates/proof-compression/src/context.rs +++ b/crates/proof-compression/src/context.rs @@ -5,3 +5,19 @@ pub trait ContextInitializator { where P: ProofSystemDefinition; } + +pub struct SimpelContextInitializor; + +impl SimpelContextInitializor { + pub fn new() -> Self { + todo!() + } +} +impl ContextInitializator for SimpelContextInitializor { + fn init

(config: P::ContextConfig) -> AsyncHandler + where + P: ProofSystemDefinition, + { + todo!() + } +} diff --git a/crates/proof-compression/src/gpu.rs b/crates/proof-compression/src/gpu.rs index 24329e3..df1a5fa 100644 --- a/crates/proof-compression/src/gpu.rs +++ b/crates/proof-compression/src/gpu.rs @@ -9,7 +9,6 @@ use shivini::circuit_definitions::boojum; use shivini::circuit_definitions::circuit_definitions::{ aux_layer::{ compression::{CompressionLayerCircuit, ProofCompressionFunction}, - compression_modes::{CompressionTranscriptForWrapper, CompressionTreeHasherForWrapper}, CompressionProofsTreeHasher, ZkSyncCompressionForWrapperCircuit, ZkSyncCompressionLayerCircuit, ZkSyncCompressionProof, ZkSyncCompressionProofForWrapper, ZkSyncCompressionVerificationKey, ZkSyncCompressionVerificationKeyForWrapper, diff --git a/crates/proof-compression/src/lib.rs b/crates/proof-compression/src/lib.rs index 4383218..c625d3a 100644 --- a/crates/proof-compression/src/lib.rs +++ b/crates/proof-compression/src/lib.rs @@ -1,7 +1,6 @@ #![allow(incomplete_features)] #![feature(generic_const_exprs)] #![feature(allocator_api)] -#![feature(associated_type_defaults)] // TODO mod artifacts; use artifacts::*; @@ -81,3 +80,12 @@ pub type SchedulerProof = boojum::cs::implementations::proof::Proof< RecursiveProofsTreeHasher, GoldilocksExt2, >; + +use circuit_definitions::circuit_definitions::aux_layer::{ + compression::{CompressionLayerCircuit, ProofCompressionFunction}, + compression_modes::{ + CompressionMode1, CompressionMode1ForWrapper, CompressionMode2, CompressionMode3, + CompressionMode4, CompressionMode5ForWrapper, CompressionTranscriptForWrapper, + CompressionTreeHasherForWrapper, + }, +}; diff --git a/crates/proof-compression/src/proof_system.rs b/crates/proof-compression/src/proof_system.rs index 7604cc3..448e144 100644 --- a/crates/proof-compression/src/proof_system.rs +++ b/crates/proof-compression/src/proof_system.rs @@ -8,9 +8,13 @@ use bellman::plonk::better_better_cs::{ selector_optimized_with_d_next::SelectorOptimizedWidth4MainGateWithDNext, }, }; +use bellman::plonk::commitments::transcript::keccak_transcript::RollingKeccakTranscript; use boojum::config::SetupCSConfig; -use boojum::cs::implementations::prover::ProofConfig; -use boojum::cs::oracle::TreeHasher; +use boojum::cs::implementations::transcript::Transcript; +use boojum::cs::implementations::witness::WitnessVec; +use boojum::cs::traits::circuit::CircuitBuilderProxy; +use boojum::cs::traits::GoodAllocator; +use boojum::worker::Worker; use boojum::{ config::ProvingCSConfig, cs::implementations::{ @@ -19,200 +23,305 @@ use boojum::{ }, field::goldilocks::GoldilocksExt2, }; -use circuit_definitions::circuit_definitions::aux_layer::compression::ProofCompressionFunction; -use fflonk::DeviceContextWithSingleDevice; +use circuit_definitions::circuit_definitions::aux_layer::compression::{ + CompressionLayerCircuit, ProofCompressionFunction, +}; + +use fflonk::bellman::plonk::better_better_cs::cs::Circuit; +use fflonk::{CombinedMonomialDeviceStorage, DeviceContextWithSingleDevice}; use gpu_prover::{DeviceMemoryManager, ManagerConfigs}; -use shivini::ProverContextConfig; +use shivini::gpu_proof_config::GpuProofConfig; use shivini::{cs::GpuSetup, GpuTreeHasher, ProverContext}; - -// We can't have circuit interface as part of the proof system -// definition due to type dependency between step inputs -// circuit(input_vk, input_proof), however we can synthesize it -// through aux interface +use shivini::{ + CacheStrategy, CommitmentCacheStrategy, GPUPoWRunner, PolynomialsCacheStrategy, + ProverContextConfig, +}; pub trait ProofSystemDefinition: Sized { type FieldElement; - type Precomputation: MemcopySerializable; + type ExternalWitnessData; + type Precomputation: MemcopySerializable + Send + Sync + 'static; type Proof: serde::Serialize + serde::de::DeserializeOwned; - type VK: serde::Serialize + serde::de::DeserializeOwned; - type FinalizationHint: serde::Serialize + serde::de::DeserializeOwned; + type VK: serde::Serialize + serde::de::DeserializeOwned + Send + Sync + Clone + 'static; + type FinalizationHint: serde::Serialize + serde::de::DeserializeOwned + Clone; type Allocator: std::alloc::Allocator; type ProvingAssembly: Sized + Send + Sync + 'static; type ContextConfig; type Context; - type ProofConfig: 'static; - fn get_context_config() -> Self::ContextConfig { - todo!() - } + fn get_context_config() -> Self::ContextConfig; fn init_context(config: Self::ContextConfig) -> AsyncHandler; - fn build_proving_assembly() -> Self::ProvingAssembly; + fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData; + fn verify(_: &Self::Proof, _: &Self::VK) -> bool; +} + +pub trait CompressionProofSystem: + ProofCompressionFunction< + ThisLayerHasher: GpuTreeHasher, + ThisLayerTranscript: Transcript, + ThisLayerPoW: GPUPoWRunner, + > + ProofSystemDefinition +{ + type AuxConfig; + + fn aux_config_from_assembly(proving_assembly: &Self::ProvingAssembly) -> Self::AuxConfig; + + fn synthesize_for_proving( + circuit: CompressionLayerCircuit, + finalization_hint: Self::FinalizationHint, + ) -> Self::ProvingAssembly; fn prove( + _: AsyncHandler, _: Self::ProvingAssembly, - _: Self::Precomputation, + _: Self::AuxConfig, + _: AsyncHandler, _: Self::FinalizationHint, - _: Self::ProofConfig, + _: Self::VK, ) -> Self::Proof; - fn take_witnesses( - proving_assembly: &mut Self::ProvingAssembly, - ) -> Vec; - fn prove_from_witnesses( - _: Vec, - _: Self::Precomputation, + _: AsyncHandler, + _: Self::ExternalWitnessData, + _: Self::AuxConfig, + _: AsyncHandler, _: Self::FinalizationHint, - _: Self::ProofConfig, + _: Self::VK, ) -> Self::Proof; - - fn verify(_: &Self::Proof, _: &Self::VK) -> bool; -} - -pub trait CompressionProofSystem: ProofSystemDefinition { - fn proof_config_for_compression_step() -> Self::ProofConfig - where - CF: ProofCompressionFunction; - - fn synthesize_for_proving

( - input_vk: P::VK, - input_proof: P::Proof, - compression_mode: u8, - ) -> Self::ProvingAssembly - where - P: ProofSystemDefinition, - { - todo!() - } } -pub trait CompressionProofSystemExt: ProofSystemExt { - fn synthesize_for_setup

(input_vk: P::VK, mode: u8) -> Self::SetupAssembly - where - P: ProofSystemDefinition; +pub trait CompressionProofSystemExt: CompressionProofSystem { + type SetupAssembly; + fn generate_precomputation_and_vk( + _: Self::SetupAssembly, + _: Self::FinalizationHint, + ) -> AsyncHandler<(Self::Precomputation, Self::VK)>; + fn synthesize_for_setup( + circuit: CompressionLayerCircuit, + ) -> (Self::FinalizationHint, Self::SetupAssembly); } pub trait SnarkWrapperProofSystem: ProofSystemDefinition { - fn proof_config() -> Self::ProofConfig; - fn synthesize_for_proving

(input_vk: P::VK, input_proof: P::Proof) -> Self::ProvingAssembly - where - P: ProofSystemDefinition; -} + type Circuit; + fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly; + fn prove( + _: AsyncHandler, + _: Self::ProvingAssembly, + _: AsyncHandler, + _: Self::FinalizationHint, + ) -> Self::Proof; -pub trait SnarkWrapperProofSystemExt: ProofSystemExt { - fn synthesize_for_setup

(input_vk: P::VK) -> Self::SetupAssembly - where - P: ProofSystemDefinition; + fn prove_from_witnesses( + _: AsyncHandler, + _: Vec, + _: AsyncHandler, + _: Self::FinalizationHint, + ) -> Self::Proof; } -pub trait ProofSystemExt: ProofSystemDefinition { - type SetupAssembly: Sized + Send + Sync + 'static; +pub trait SnarkWrapperProofSystemExt: SnarkWrapperProofSystem { + type SetupAssembly; + fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly; fn generate_precomputation_and_vk( _: Self::SetupAssembly, _: Self::FinalizationHint, - ) -> (AsyncHandler, Self::VK); -} -pub struct BoojumProofSystem> { - _hasher: std::marker::PhantomData, + ) -> AsyncHandler<(Self::Precomputation, Self::VK)>; } type BoojumAssembly = CSReferenceAssembly; -pub struct TreeHasherCompatibleGpuSetup>( - std::marker::PhantomData, -); - -impl ProofSystemDefinition for BoojumProofSystem +impl ProofSystemDefinition for CF where - H: TreeHasher, + CF: ProofCompressionFunction< + ThisLayerHasher: GpuTreeHasher, + ThisLayerTranscript: Transcript< + GoldilocksField, + TransciptParameters = (), + CompatibleCap: Send + Sync + 'static, + >, + ThisLayerPoW: GPUPoWRunner, + > + 'static, { type FieldElement = GoldilocksField; - type Precomputation = TreeHasherCompatibleGpuSetup; - type Proof = Proof; - type VK = VerificationKey; + type ExternalWitnessData = WitnessVec; + type Precomputation = GpuSetup; + type Proof = Proof; + type VK = VerificationKey; type FinalizationHint = FinalizationHintsForProver; type Allocator = std::alloc::Global; type ProvingAssembly = BoojumAssembly; type ContextConfig = usize; // domain_size type Context = ProverContext; - type ProofConfig = ProofConfig; - // type Circuit = ZkSyncCompressionLayerCircuit; - fn init_context(domain_size: Self::ContextConfig) -> AsyncHandler { - let config = - ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); - let context = Self::Context::create_with_config(config).expect("gpu prover context"); - todo!() + fn get_context_config() -> Self::ContextConfig { + todo!("domain size") } - fn build_proving_assembly() -> Self::ProvingAssembly { - todo!() - } + fn init_context(domain_size: Self::ContextConfig) -> AsyncHandler { + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let config = + ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); + let context = Self::Context::create_with_config(config).expect("gpu prover context"); + sender.send(context).unwrap(); - fn prove( - _: Self::ProvingAssembly, - _: Self::Precomputation, - _: Self::FinalizationHint, - _: Self::ProofConfig, - ) -> Self::Proof { - todo!() - } - fn prove_from_witnesses( - _: Vec, - _: Self::Precomputation, - _: Self::FinalizationHint, - _: Self::ProofConfig, - ) -> Self::Proof { - todo!() + receiver + }; + + AsyncHandler::spawn(f) } - fn verify(_: &Self::Proof, _: &Self::VK) -> bool { - todo!() + fn verify(proof: &Self::Proof, vk: &Self::VK) -> bool { + let verifier_builder = + CircuitBuilderProxy::>::dyn_verifier_builder::(); + let verifier = verifier_builder.create_verifier(); + verifier.verify::( + (), + vk, + proof, + ) } - fn take_witnesses( - proving_assembly: &mut Self::ProvingAssembly, - ) -> Vec { - todo!() + fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { + proving_assembly.witness.take().unwrap() } } -impl> CompressionProofSystem for BoojumProofSystem +impl CompressionProofSystem for CF where - H: TreeHasher, + CF: ProofCompressionFunction< + ThisLayerHasher: GpuTreeHasher, + ThisLayerTranscript: Transcript< + GoldilocksField, + TransciptParameters = (), + CompatibleCap: Send + Sync + 'static, + >, + ThisLayerPoW: GPUPoWRunner, + > + 'static, { - fn proof_config_for_compression_step() -> Self::ProofConfig - where - CF: ProofCompressionFunction, - { - CF::proof_config_for_compression_step() - } - - fn synthesize_for_proving

( - input_vk: P::VK, - input_proof: P::Proof, - compression_mode: u8, - ) -> Self::ProvingAssembly - where - P: ProofSystemDefinition, - { - todo!() + type AuxConfig = GpuProofConfig; + + fn aux_config_from_assembly(proving_assembly: &Self::ProvingAssembly) -> Self::AuxConfig { + GpuProofConfig::from_assembly(proving_assembly) + } + + fn synthesize_for_proving( + circuit: CompressionLayerCircuit, + finalization_hint: Self::FinalizationHint, + ) -> Self::ProvingAssembly { + synthesize_circuit_for_proving(circuit, &finalization_hint) + } + + fn prove( + ctx: AsyncHandler, + proving_assembly: Self::ProvingAssembly, + aux_config: Self::AuxConfig, + precomputation: AsyncHandler, + finalization_hint: Self::FinalizationHint, + vk: Self::VK, + ) -> Self::Proof { + Self::prove_from_witnesses( + ctx, + proving_assembly.witness.unwrap(), + aux_config, + precomputation, + finalization_hint, + vk, + ) + } + + fn prove_from_witnesses( + ctx: AsyncHandler, + witness: Self::ExternalWitnessData, + aux_config: Self::AuxConfig, + precomputation: AsyncHandler, + finalization_hint: Self::FinalizationHint, + vk: Self::VK, + ) -> Self::Proof { + let domain_size = vk.fixed_parameters.domain_size as usize; + assert_eq!(finalization_hint.final_trace_len, domain_size); + let cache_strategy = CacheStrategy { + setup_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, + trace_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, + other_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, + commitment: CommitmentCacheStrategy::CacheCosetCaps, + }; + let worker = Worker::new(); + let precomputation = precomputation.wait(); + let ctx = ctx.wait(); + let gpu_proof = shivini::gpu_prove_from_external_witness_data_with_cache_strategy::< + CF::ThisLayerTranscript, + CF::ThisLayerHasher, + CF::ThisLayerPoW, + Self::Allocator, + >( + &aux_config, + &witness, + CF::proof_config_for_compression_step(), + &precomputation, + &vk, + (), + &worker, + cache_strategy, + ) + .expect("gpu proof"); + drop(ctx); + let proof = gpu_proof.into(); + + proof } } -impl ProofSystemExt for BoojumProofSystem +impl CompressionProofSystemExt for CF where - H: TreeHasher, + CF: ProofCompressionFunction< + ThisLayerHasher: GpuTreeHasher, + ThisLayerTranscript: Transcript< + GoldilocksField, + TransciptParameters = (), + CompatibleCap: Send + Sync + 'static, + >, + ThisLayerPoW: GPUPoWRunner, + > + 'static, + Self::Allocator: GoodAllocator, { type SetupAssembly = BoojumAssembly; fn generate_precomputation_and_vk( - _: Self::SetupAssembly, - _: Self::FinalizationHint, - ) -> (AsyncHandler, Self::VK) { - todo!() + setup_assembly: Self::SetupAssembly, + finalization_hint: Self::FinalizationHint, + ) -> AsyncHandler<(Self::Precomputation, Self::VK)> { + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let worker = Worker::new(); + let proof_config = CF::proof_config_for_compression_step(); + let (setup_base, vk_params, vars_hint, wits_hint) = setup_assembly.get_light_setup( + &worker, + proof_config.fri_lde_factor, + proof_config.merkle_tree_cap_size, + ); + let domain_size = vk_params.domain_size as usize; + assert_eq!(finalization_hint.final_trace_len, domain_size); + let config = + ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); + let _ctx = ProverContext::create_with_config(config).expect("gpu prover context"); + let (device_setup, vk) = + shivini::cs::gpu_setup_and_vk_from_base_setup_vk_params_and_hints::< + CF::ThisLayerHasher, + _, + >(setup_base, vk_params, vars_hint, wits_hint, &worker) + .unwrap(); + sender.send((device_setup, vk)).unwrap(); + + receiver + }; + + AsyncHandler::spawn(f) + } + fn synthesize_for_setup( + circuit: CompressionLayerCircuit, + ) -> (Self::FinalizationHint, Self::SetupAssembly) { + synthesize_circuit_for_setup(circuit) } } -pub struct PlonkProofSystem; - type PlonkAssembly = Assembly< Bn256, PlonkCsWidth4WithNextStepParams, @@ -230,9 +339,10 @@ impl ManagerConfigs for PlonkProverDeviceMemoryManager { const NUM_HOST_SLOTS: usize = 2; } -impl ProofSystemDefinition for PlonkProofSystem { +impl ProofSystemDefinition for PlonkSnarkWrapper { type FieldElement = Fr; type Precomputation = PlonkSnarkVerifierCircuitDeviceSetupWrapper; + type ExternalWitnessData = Vec; type Proof = PlonkSnarkVerifierCircuitProof; type VK = PlonkSnarkVerifierCircuitVK; type FinalizationHint = usize; @@ -240,77 +350,73 @@ impl ProofSystemDefinition for PlonkProofSystem { type ProvingAssembly = PlonkAssembly; type ContextConfig = (Vec, Vec); type Context = DeviceMemoryManager; - type ProofConfig = (); + fn get_context_config() -> Self::ContextConfig { + todo!() + } fn init_context(config: Self::ContextConfig) -> AsyncHandler { let (device_ids, compact_crs) = config; let ctx = Self::Context::init(&device_ids, &compact_crs[..]); todo!() } + fn take_witnesses( + proving_assembly: &mut Self::ProvingAssembly, + ) -> Vec { + todo!() + } + fn verify(_: &Self::Proof, _: &Self::VK) -> bool { + todo!() + } +} - fn build_proving_assembly() -> Self::ProvingAssembly { +impl SnarkWrapperProofSystem for PlonkSnarkWrapper { + type Circuit = PlonkSnarkVerifierCircuit; + fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly { todo!() } fn prove( + _: AsyncHandler, _: Self::ProvingAssembly, - _: Self::Precomputation, + _: AsyncHandler, _: Self::FinalizationHint, - _: Self::ProofConfig, ) -> Self::Proof { todo!() } - fn take_witnesses( - proving_assembly: &mut Self::ProvingAssembly, - ) -> Vec { - todo!() - } - fn prove_from_witnesses( + _: AsyncHandler, _: Vec, - _: Self::Precomputation, + _: AsyncHandler, _: Self::FinalizationHint, - _: Self::ProofConfig, ) -> Self::Proof { todo!() } - - fn verify(_: &Self::Proof, _: &Self::VK) -> bool { - todo!() - } } -impl SnarkWrapperProofSystem for PlonkProofSystem { - fn synthesize_for_proving

(input_vk: P::VK, input_proof: P::Proof) -> Self::ProvingAssembly - where - P: ProofSystemDefinition, - { - todo!() - } +impl SnarkWrapperProofSystemExt for PlonkSnarkWrapper { + type SetupAssembly = PlonkAssembly; - fn proof_config() -> Self::ProofConfig { + fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly { todo!() } -} - -impl ProofSystemExt for PlonkProofSystem { - type SetupAssembly = PlonkAssembly; fn generate_precomputation_and_vk( _: Self::SetupAssembly, _: Self::FinalizationHint, - ) -> (AsyncHandler, Self::VK) { + ) -> AsyncHandler<(Self::Precomputation, Self::VK)> { todo!() } } -pub struct FflonkProofSystem; - type FflonkAssembly = Assembly; -impl ProofSystemDefinition for FflonkProofSystem { +impl ProofSystemDefinition for FflonkSnarkWrapper { type FieldElement = Fr; type Precomputation = FflonkSnarkVerifierCircuitDeviceSetupWrapper; + type ExternalWitnessData = ( + Vec, + Vec, + ); type Proof = FflonkSnarkVerifierCircuitProof; type VK = FflonkSnarkVerifierCircuitVK; type FinalizationHint = usize; @@ -318,71 +424,123 @@ impl ProofSystemDefinition for FflonkProofSystem { type ProvingAssembly = FflonkAssembly; type ContextConfig = usize; // domain_size type Context = DeviceContextWithSingleDevice; - type ProofConfig = (); - // type Circuit = (); + fn get_context_config() -> Self::ContextConfig { - todo!() + fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG } fn init_context(log_domain_size: Self::ContextConfig) -> AsyncHandler { - let domain_size = 1 << log_domain_size; - let context = Self::Context::init(domain_size).unwrap(); + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let domain_size = 1 << log_domain_size; + let context = Self::Context::init(domain_size).unwrap(); + sender.send(context).unwrap(); - todo!() + receiver + }; + + AsyncHandler::spawn(f) } + fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { + let input_assignments = + std::mem::replace(&mut proving_assembly.input_assingments, Vec::new()); + let aux_assignments = std::mem::replace( + &mut proving_assembly.aux_assingments, + Vec::new_in(Self::Allocator::default()), + ); - fn build_proving_assembly() -> Self::ProvingAssembly { - todo!() + (input_assignments, aux_assignments) } - fn prove( - _: Self::ProvingAssembly, - _: Self::Precomputation, - _: Self::FinalizationHint, - _: Self::ProofConfig, - ) -> Self::Proof { - todo!() + fn verify(proof: &Self::Proof, vk: &Self::VK) -> bool { + fflonk::fflonk_cpu::verify::<_, FflonkSnarkVerifierCircuit, RollingKeccakTranscript>( + vk, proof, None, + ) + .unwrap() } +} - fn take_witnesses( - proving_assembly: &mut Self::ProvingAssembly, - ) -> Vec { - todo!() +impl SnarkWrapperProofSystem for FflonkSnarkWrapper { + type Circuit = FflonkSnarkVerifierCircuit; + fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly { + let mut proving_assembly = FflonkAssembly::::new(); + circuit + .synthesize(&mut proving_assembly) + .expect("must work"); + proving_assembly + } + + fn prove( + ctx: AsyncHandler, + mut proving_assembly: Self::ProvingAssembly, + precomputation: AsyncHandler, + finalization_hint: Self::FinalizationHint, + ) -> Self::Proof { + assert!(proving_assembly.is_satisfied()); + let raw_trace_len = proving_assembly.n(); + proving_assembly.finalize_to_size_log_2(1 << finalization_hint); + let domain_size = proving_assembly.n() + 1; + assert!(domain_size.is_power_of_two()); + assert!(domain_size <= 1 << Self::get_context_config()); + + let ctx = ctx.wait(); + let precomputation = precomputation.wait().into_inner(); + let proof = fflonk::create_proof::< + _, + _, + _, + RollingKeccakTranscript<_>, + CombinedMonomialDeviceStorage, + _, + >(&proving_assembly, &precomputation, raw_trace_len) + .unwrap(); + drop(ctx); + proof } fn prove_from_witnesses( + _: AsyncHandler, _: Vec, - _: Self::Precomputation, + _: AsyncHandler, _: Self::FinalizationHint, - _: Self::ProofConfig, ) -> Self::Proof { - todo!() - } - - fn verify(_: &Self::Proof, _: &Self::VK) -> bool { - todo!() + unimplemented!() } } -impl SnarkWrapperProofSystem for FflonkProofSystem { - fn synthesize_for_proving

(input_vk: P::VK, input_proof: P::Proof) -> Self::ProvingAssembly - where - P: ProofSystemDefinition, - { - todo!() - } +impl SnarkWrapperProofSystemExt for FflonkSnarkWrapper { + type SetupAssembly = FflonkAssembly; - fn proof_config() -> Self::ProofConfig { - todo!() + fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly { + let mut setup_assembly = + FflonkAssembly::::new(); + circuit.synthesize(&mut setup_assembly).unwrap(); + + setup_assembly } -} -impl ProofSystemExt for FflonkProofSystem { - type SetupAssembly = FflonkAssembly; fn generate_precomputation_and_vk( - _: Self::SetupAssembly, - _: Self::FinalizationHint, - ) -> (AsyncHandler, Self::VK) { - todo!() + setup_assembly: Self::SetupAssembly, + finalization_hint: Self::FinalizationHint, + ) -> AsyncHandler<(Self::Precomputation, Self::VK)> { + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let device_setup = + FflonkSnarkVerifierCircuitDeviceSetup::create_setup_from_assembly_on_device( + &setup_assembly, + ) + .unwrap(); + let vk = device_setup.get_verification_key(); + sender + .send(( + FflonkSnarkVerifierCircuitDeviceSetupWrapper(device_setup), + vk, + )) + .unwrap(); + + receiver + }; + + AsyncHandler::spawn(f) } } @@ -392,13 +550,12 @@ impl ProofSystemDefinition for MarkerProofSystem { type Precomputation = MarkerPrecomputation; type Proof = (); type VK = (); + type ExternalWitnessData = (); type FinalizationHint = (); type Allocator = std::alloc::Global; type ProvingAssembly = (); type ContextConfig = (); type Context = (); - type ProofConfig = (); - // type Circuit = (); fn get_context_config() -> Self::ContextConfig { todo!() } @@ -406,35 +563,11 @@ impl ProofSystemDefinition for MarkerProofSystem { todo!() } - fn build_proving_assembly() -> Self::ProvingAssembly { - todo!() - } - - fn prove( - _: Self::ProvingAssembly, - _: Self::Precomputation, - _: Self::FinalizationHint, - _: Self::ProofConfig, - ) -> Self::Proof { - todo!() - } - - fn prove_from_witnesses( - _: Vec, - _: Self::Precomputation, - _: Self::FinalizationHint, - _: Self::ProofConfig, - ) -> Self::Proof { - todo!() - } - fn verify(_: &Self::Proof, _: &Self::VK) -> bool { todo!() } - fn take_witnesses( - proving_assembly: &mut Self::ProvingAssembly, - ) -> Vec { + fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { todo!() } } diff --git a/crates/proof-compression/src/serialization.rs b/crates/proof-compression/src/serialization.rs index e978e02..c7c8c34 100644 --- a/crates/proof-compression/src/serialization.rs +++ b/crates/proof-compression/src/serialization.rs @@ -1,10 +1,9 @@ use super::*; -use boojum::cs::oracle::TreeHasher; use fflonk::FflonkSnarkVerifierCircuitDeviceSetup; use shivini::boojum::cs::implementations::fast_serialization::MemcopySerializable; -use crate::{PlonkSnarkVerifierCircuitDeviceSetup, TreeHasherCompatibleGpuSetup}; +use crate::PlonkSnarkVerifierCircuitDeviceSetup; pub struct PlonkSnarkVerifierCircuitDeviceSetupWrapper(PlonkSnarkVerifierCircuitDeviceSetup); @@ -39,7 +38,13 @@ impl<'de> serde::Deserialize<'de> for PlonkSnarkVerifierCircuitDeviceSetupWrappe } } -pub struct FflonkSnarkVerifierCircuitDeviceSetupWrapper(FflonkSnarkVerifierCircuitDeviceSetup); +impl PlonkSnarkVerifierCircuitDeviceSetupWrapper { + pub fn into_inner(self) -> PlonkSnarkVerifierCircuitDeviceSetup { + self.0 + } +} + +pub struct FflonkSnarkVerifierCircuitDeviceSetupWrapper(pub FflonkSnarkVerifierCircuitDeviceSetup); impl MemcopySerializable for FflonkSnarkVerifierCircuitDeviceSetupWrapper { fn write_into_buffer( @@ -72,25 +77,15 @@ impl<'de> serde::Deserialize<'de> for FflonkSnarkVerifierCircuitDeviceSetupWrapp } } -#[derive(serde::Serialize, serde::Deserialize)] -pub struct MarkerPrecomputation; -impl MemcopySerializable for MarkerPrecomputation { - fn write_into_buffer( - &self, - dst: W, - ) -> Result<(), Box> { - todo!() - } - - fn read_from_buffer(src: R) -> Result> { - todo!() +impl FflonkSnarkVerifierCircuitDeviceSetupWrapper { + pub fn into_inner(self) -> FflonkSnarkVerifierCircuitDeviceSetup { + self.0 } } -impl MemcopySerializable for TreeHasherCompatibleGpuSetup -where - H: TreeHasher, -{ +#[derive(serde::Serialize, serde::Deserialize)] +pub struct MarkerPrecomputation; +impl MemcopySerializable for MarkerPrecomputation { fn write_into_buffer( &self, dst: W, diff --git a/crates/proof-compression/src/snark_wrapper.rs b/crates/proof-compression/src/snark_wrapper.rs index 115d0c3..80504f1 100644 --- a/crates/proof-compression/src/snark_wrapper.rs +++ b/crates/proof-compression/src/snark_wrapper.rs @@ -1,15 +1,24 @@ -use circuit_definitions::circuit_definitions::aux_layer::compression_modes::CompressionTreeHasherForWrapper; +use boojum::cs::{ + implementations::{proof::Proof, verifier::VerificationKey}, + oracle::TreeHasher, +}; +use circuit_definitions::circuit_definitions::aux_layer::{ + compression::ProofCompressionFunction, wrapper::ZkSyncCompressionWrapper, +}; use super::*; -pub trait SnarkWrapperStep: StepDefinition { +pub trait SnarkWrapperStep: SnarkWrapperProofSystem { const IS_PLONK: bool; const IS_FFLONK: bool; - const LAST_COMPRESSION_MODE: u8; - + const PREVIOUS_COMPRESSION_MODE: u8; + type PreviousStepTreeHasher: TreeHasher< + GoldilocksField, + Output: serde::Serialize + serde::de::DeserializeOwned, + >; fn load_finalization_hint( artifact_loader: &AL, - ) -> ::FinalizationHint + ) -> ::FinalizationHint where AL: ArtifactLoader, { @@ -20,18 +29,17 @@ pub trait SnarkWrapperStep: StepDefinition( artifact_loader: &AL, - ) -> ::VK + ) -> VerificationKey where AL: ArtifactLoader, { assert!(Self::IS_FFLONK ^ Self::IS_PLONK); - let reader = artifact_loader.read_compression_wrapper_vk(Self::LAST_COMPRESSION_MODE); + let previous_compression_mode = Self::PREVIOUS_COMPRESSION_MODE; + let reader = artifact_loader.read_compression_wrapper_vk(previous_compression_mode); serde_json::from_reader(reader).unwrap() } - fn load_this_vk( - artifact_loader: &AL, - ) -> ::VK + fn load_this_vk(artifact_loader: &AL) -> ::VK where AL: ArtifactLoader, { @@ -49,7 +57,7 @@ pub trait SnarkWrapperStep: StepDefinition( artifact_loader: &AL, - ) -> AsyncHandler<::Precomputation> + ) -> AsyncHandler<::Precomputation> where AL: ArtifactLoader, { @@ -57,9 +65,9 @@ pub trait SnarkWrapperStep: StepDefinition( - input_proof: ::Proof, + input_proof: Proof, artifact_loader: &AL, - ) -> ::Proof + ) -> ::Proof where AL: ArtifactLoader, CI: ContextInitializator, @@ -67,72 +75,93 @@ pub trait SnarkWrapperStep: StepDefinition::get_context_config(); - let ctx = CI::init::(config); + let config = ::get_context_config(); + let ctx = CI::init::(config); let finalization_hint = Self::load_finalization_hint(artifact_loader); - let proving_assembly = - ::synthesize_for_proving::< - Self::PreviousProofSystem, - >(input_vk, input_proof); - let proof_config = ::proof_config(); - Self::prove_step( + let circuit = Self::build_circuit(input_vk, Some(input_proof)); + let proving_assembly = ::synthesize_for_proving(circuit); + ::prove( ctx, proving_assembly, - proof_config, precomputation, finalization_hint, ) } + + fn build_circuit( + input_vk: VerificationKey, + input_proof: Option>, + ) -> Self::Circuit; } -pub trait SnarkWrapperStepExt: - SnarkWrapperStep -{ +pub trait SnarkWrapperStepExt: SnarkWrapperProofSystemExt + SnarkWrapperStep { fn run_precomputation_for_compression( artifact_loader: &AL, ) -> ( - ::Precomputation, - ::VK, + ::Precomputation, + ::VK, ) where AL: ArtifactLoader, + ::VK: 'static, { let input_vk = Self::load_previous_vk(artifact_loader); let finalization_hint = Self::load_finalization_hint(artifact_loader); - let setup_assembly = - ::synthesize_for_setup::< - Self::PreviousProofSystem, - >(input_vk); - let (precomputation, vk) = - ::generate_precomputation_and_vk( - setup_assembly, - finalization_hint, - ); - (precomputation.into_inner(), vk) + let circuit = Self::build_circuit(input_vk, None); + let setup_assembly = ::synthesize_for_setup(circuit); + let data = ::generate_precomputation_and_vk( + setup_assembly, + finalization_hint, + ); + + data.wait() } } pub struct FflonkSnarkWrapper; - -impl StepDefinition for FflonkSnarkWrapper { - type PreviousProofSystem = BoojumProofSystem; - type ThisProofSystem = FflonkProofSystem; -} - impl SnarkWrapperStep for FflonkSnarkWrapper { const IS_PLONK: bool = false; const IS_FFLONK: bool = true; - const LAST_COMPRESSION_MODE: u8 = 5; + const PREVIOUS_COMPRESSION_MODE: u8 = 5; + type PreviousStepTreeHasher = + ::ThisLayerHasher; + fn build_circuit( + input_vk: VerificationKey, + input_proof: Option>, + ) -> Self::Circuit { + let fixed_parameters = input_vk.fixed_parameters.clone(); + FflonkSnarkVerifierCircuit { + witness: input_proof, + vk: input_vk, + fixed_parameters, + transcript_params: (), + wrapper_function: ZkSyncCompressionWrapper::from_numeric_circuit_type( + Self::PREVIOUS_COMPRESSION_MODE, + ), + } + } } pub struct PlonkSnarkWrapper; - -impl StepDefinition for PlonkSnarkWrapper { - type PreviousProofSystem = BoojumProofSystem; - type ThisProofSystem = PlonkProofSystem; -} - impl SnarkWrapperStep for PlonkSnarkWrapper { const IS_PLONK: bool = true; const IS_FFLONK: bool = false; - const LAST_COMPRESSION_MODE: u8 = 1; + + const PREVIOUS_COMPRESSION_MODE: u8 = 1; + type PreviousStepTreeHasher = + ::ThisLayerHasher; + fn build_circuit( + input_vk: VerificationKey, + input_proof: Option>, + ) -> Self::Circuit { + let fixed_parameters = input_vk.fixed_parameters.clone(); + PlonkSnarkVerifierCircuit { + witness: input_proof, + vk: input_vk, + fixed_parameters, + transcript_params: (), + wrapper_function: ZkSyncCompressionWrapper::from_numeric_circuit_type( + Self::PREVIOUS_COMPRESSION_MODE, + ), + } + } } diff --git a/crates/proof-compression/src/step.rs b/crates/proof-compression/src/step.rs index 223f4ef..8b13789 100644 --- a/crates/proof-compression/src/step.rs +++ b/crates/proof-compression/src/step.rs @@ -1,46 +1 @@ -use super::*; -pub trait StepDefinition: Sized { - type PreviousProofSystem: ProofSystemDefinition; - type ThisProofSystem: ProofSystemDefinition; - - fn prove_step( - ctx: AsyncHandler<::Context>, - proving_assembly: ::ProvingAssembly, - proof_config: ::ProofConfig, - precomputation: AsyncHandler< - ::Precomputation, - >, - finalization_hint: ::FinalizationHint, - ) -> ::Proof { - let _ = ctx.into_inner(); - let proof = ::prove( - proving_assembly, - precomputation.into_inner(), - finalization_hint, - proof_config, - ); - - proof - } -} - -pub trait StepDefinitionExt: StepDefinition { - fn generate_precomputation_and_vk( - ctx: AsyncHandler<::Context>, - setup_assembly: ::SetupAssembly, - proof_config: ::ProofConfig, - finalization_hint: ::FinalizationHint, - ) -> ( - AsyncHandler<::Precomputation>, - ::VK, - ) { - let _ = ctx.into_inner(); - let (precomputation, vk) = - ::generate_precomputation_and_vk( - setup_assembly, - finalization_hint, - ); - (precomputation, vk) - } -} From d516b89bc3649f49ba4f720b243e9bf41d269187 Mon Sep 17 00:00:00 2001 From: saitima Date: Thu, 9 Jan 2025 22:57:16 +0300 Subject: [PATCH 04/18] feat(proof-compression): testing precomputation part --- .../data/scheduler_recursive_vk.json | 490 +++++++------- crates/proof-compression/src/artifacts.rs | 174 ----- crates/proof-compression/src/blob_storage.rs | 277 ++++++++ crates/proof-compression/src/chain.rs | 105 ++- crates/proof-compression/src/compression.rs | 117 ++-- crates/proof-compression/src/context.rs | 32 +- crates/proof-compression/src/cpu.rs | 425 ------------ crates/proof-compression/src/gpu.rs | 616 ------------------ crates/proof-compression/src/lib.rs | 17 +- crates/proof-compression/src/proof_system.rs | 217 +++--- crates/proof-compression/src/serialization.rs | 42 +- crates/proof-compression/src/snark_wrapper.rs | 125 ++-- crates/proof-compression/src/test.rs | 41 ++ 13 files changed, 951 insertions(+), 1727 deletions(-) delete mode 100644 crates/proof-compression/src/artifacts.rs create mode 100644 crates/proof-compression/src/blob_storage.rs delete mode 100644 crates/proof-compression/src/cpu.rs delete mode 100644 crates/proof-compression/src/gpu.rs create mode 100644 crates/proof-compression/src/test.rs diff --git a/crates/proof-compression/data/scheduler_recursive_vk.json b/crates/proof-compression/data/scheduler_recursive_vk.json index 4c328cb..2817f8c 100644 --- a/crates/proof-compression/data/scheduler_recursive_vk.json +++ b/crates/proof-compression/data/scheduler_recursive_vk.json @@ -1,270 +1,268 @@ { - "SchedulerCircuit": { - "fixed_parameters": { - "parameters": { - "num_columns_under_copy_permutation": 130, - "num_witness_columns": 0, - "num_constant_columns": 4, - "max_allowed_constraint_degree": 8 - }, - "lookup_parameters": { - "UseSpecializedColumnsWithTableIdAsConstant": { - "width": 3, - "num_repetitions": 4, - "share_table_id": true - } - }, - "domain_size": 1048576, - "total_tables_len": 132096, - "public_inputs_locations": [ - [ - 0, - 1043851 - ], - [ - 1, - 1043851 - ], - [ - 2, - 1043851 - ], - [ - 3, - 1043851 - ] + "fixed_parameters": { + "parameters": { + "num_columns_under_copy_permutation": 130, + "num_witness_columns": 0, + "num_constant_columns": 4, + "max_allowed_constraint_degree": 8 + }, + "lookup_parameters": { + "UseSpecializedColumnsWithTableIdAsConstant": { + "width": 3, + "num_repetitions": 4, + "share_table_id": true + } + }, + "domain_size": 1048576, + "total_tables_len": 132096, + "public_inputs_locations": [ + [ + 0, + 1043851 ], - "extra_constant_polys_for_selectors": 4, - "table_ids_column_idxes": [ - 8 + [ + 1, + 1043851 ], - "quotient_degree": 8, - "selectors_placement": { - "Fork": { - "left": { - "GateOnly": { - "gate_idx": 1, - "num_constants": 0, - "degree": 7, - "needs_selector": true, - "is_lookup": false - } - }, - "right": { - "Fork": { - "left": { - "Fork": { - "left": { - "Fork": { - "left": { - "GateOnly": { - "gate_idx": 4, - "num_constants": 4, - "degree": 3, - "needs_selector": true, - "is_lookup": false - } - }, - "right": { - "Fork": { - "left": { - "Fork": { - "left": { - "GateOnly": { - "gate_idx": 5, - "num_constants": 1, - "degree": 2, - "needs_selector": true, - "is_lookup": false - } - }, - "right": { - "GateOnly": { - "gate_idx": 6, - "num_constants": 0, - "degree": 2, - "needs_selector": true, - "is_lookup": false - } + [ + 2, + 1043851 + ], + [ + 3, + 1043851 + ] + ], + "extra_constant_polys_for_selectors": 4, + "table_ids_column_idxes": [ + 8 + ], + "quotient_degree": 8, + "selectors_placement": { + "Fork": { + "left": { + "GateOnly": { + "gate_idx": 1, + "num_constants": 0, + "degree": 7, + "needs_selector": true, + "is_lookup": false + } + }, + "right": { + "Fork": { + "left": { + "Fork": { + "left": { + "Fork": { + "left": { + "GateOnly": { + "gate_idx": 4, + "num_constants": 4, + "degree": 3, + "needs_selector": true, + "is_lookup": false + } + }, + "right": { + "Fork": { + "left": { + "Fork": { + "left": { + "GateOnly": { + "gate_idx": 5, + "num_constants": 1, + "degree": 2, + "needs_selector": true, + "is_lookup": false + } + }, + "right": { + "GateOnly": { + "gate_idx": 6, + "num_constants": 0, + "degree": 2, + "needs_selector": true, + "is_lookup": false } } - }, - "right": { - "Fork": { - "left": { - "GateOnly": { - "gate_idx": 2, - "num_constants": 0, - "degree": 2, - "needs_selector": true, - "is_lookup": false - } - }, - "right": { - "GateOnly": { - "gate_idx": 7, - "num_constants": 0, - "degree": 2, - "needs_selector": true, - "is_lookup": false - } + } + }, + "right": { + "Fork": { + "left": { + "GateOnly": { + "gate_idx": 2, + "num_constants": 0, + "degree": 2, + "needs_selector": true, + "is_lookup": false + } + }, + "right": { + "GateOnly": { + "gate_idx": 7, + "num_constants": 0, + "degree": 2, + "needs_selector": true, + "is_lookup": false } } } } } } - }, - "right": { - "Fork": { - "left": { - "GateOnly": { - "gate_idx": 9, - "num_constants": 4, - "degree": 2, - "needs_selector": true, - "is_lookup": false - } - }, - "right": { - "GateOnly": { - "gate_idx": 0, - "num_constants": 4, - "degree": 1, - "needs_selector": true, - "is_lookup": false - } + } + }, + "right": { + "Fork": { + "left": { + "GateOnly": { + "gate_idx": 9, + "num_constants": 4, + "degree": 2, + "needs_selector": true, + "is_lookup": false + } + }, + "right": { + "GateOnly": { + "gate_idx": 0, + "num_constants": 4, + "degree": 1, + "needs_selector": true, + "is_lookup": false } } } } - }, - "right": { - "Fork": { - "left": { - "GateOnly": { - "gate_idx": 3, - "num_constants": 2, - "degree": 3, - "needs_selector": true, - "is_lookup": false - } - }, - "right": { - "GateOnly": { - "gate_idx": 8, - "num_constants": 0, - "degree": 0, - "needs_selector": true, - "is_lookup": false - } + } + }, + "right": { + "Fork": { + "left": { + "GateOnly": { + "gate_idx": 3, + "num_constants": 2, + "degree": 3, + "needs_selector": true, + "is_lookup": false + } + }, + "right": { + "GateOnly": { + "gate_idx": 8, + "num_constants": 0, + "degree": 0, + "needs_selector": true, + "is_lookup": false } } } } } } - }, - "fri_lde_factor": 2, - "cap_size": 16 + } }, - "setup_merkle_tree_cap": [ - [ - 2680192913777199386, - 7877900777764568562, - 7967270885539056261, - 11491786516879257714 - ], - [ - 1576848689219001454, - 2538042691131197824, - 16789498574115229290, - 3214129711903181558 - ], - [ - 856301905705619734, - 4331213335266799158, - 15267490766684530921, - 3265714654258242220 - ], - [ - 8865784570897245270, - 2362765988103793581, - 6943670874402562853, - 14632996114278721596 - ], - [ - 63247458005995468, - 12539771084927052853, - 13041512411442114569, - 9742813247561592554 - ], - [ - 16743936557271219178, - 14841453989210747254, - 12724413787690930702, - 10592542358880202219 - ], - [ - 16695338323889693576, - 8527536001711027994, - 13212045085202022064, - 11071462626939596790 - ], - [ - 18060750313558946749, - 15824434706098663517, - 775292596891170912, - 18445377984966327048 - ], - [ - 3549745875383468285, - 2238890537215251462, - 4591889095789072384, - 13012706980710418598 - ], - [ - 14771394899136640222, - 13143304103596416048, - 14456129193020560275, - 5740433968684323698 - ], - [ - 11651473654699970526, - 4694969877986805556, - 7029204199916750383, - 6916614362901685796 - ], - [ - 4368206191480113515, - 9562279231528697429, - 1907048590194817686, - 13209277185471975687 - ], - [ - 14438342866286439870, - 383769026263703315, - 1077241575478137065, - 1158227982301730574 - ], - [ - 10868817472877525981, - 11920954565057859026, - 10684659491915725994, - 15343028344024922569 - ], - [ - 4969179907509861760, - 3560160134545277440, - 11797495979614319546, - 13436348584120593030 - ], - [ - 8873263215018682993, - 13828390019511310487, - 12329030402425507188, - 18004618114160314165 - ] + "fri_lde_factor": 2, + "cap_size": 16 + }, + "setup_merkle_tree_cap": [ + [ + 2680192913777199386, + 7877900777764568562, + 7967270885539056261, + 11491786516879257714 + ], + [ + 1576848689219001454, + 2538042691131197824, + 16789498574115229290, + 3214129711903181558 + ], + [ + 856301905705619734, + 4331213335266799158, + 15267490766684530921, + 3265714654258242220 + ], + [ + 8865784570897245270, + 2362765988103793581, + 6943670874402562853, + 14632996114278721596 + ], + [ + 63247458005995468, + 12539771084927052853, + 13041512411442114569, + 9742813247561592554 + ], + [ + 16743936557271219178, + 14841453989210747254, + 12724413787690930702, + 10592542358880202219 + ], + [ + 16695338323889693576, + 8527536001711027994, + 13212045085202022064, + 11071462626939596790 + ], + [ + 18060750313558946749, + 15824434706098663517, + 775292596891170912, + 18445377984966327048 + ], + [ + 3549745875383468285, + 2238890537215251462, + 4591889095789072384, + 13012706980710418598 + ], + [ + 14771394899136640222, + 13143304103596416048, + 14456129193020560275, + 5740433968684323698 + ], + [ + 11651473654699970526, + 4694969877986805556, + 7029204199916750383, + 6916614362901685796 + ], + [ + 4368206191480113515, + 9562279231528697429, + 1907048590194817686, + 13209277185471975687 + ], + [ + 14438342866286439870, + 383769026263703315, + 1077241575478137065, + 1158227982301730574 + ], + [ + 10868817472877525981, + 11920954565057859026, + 10684659491915725994, + 15343028344024922569 + ], + [ + 4969179907509861760, + 3560160134545277440, + 11797495979614319546, + 13436348584120593030 + ], + [ + 8873263215018682993, + 13828390019511310487, + 12329030402425507188, + 18004618114160314165 ] - } + ] } \ No newline at end of file diff --git a/crates/proof-compression/src/artifacts.rs b/crates/proof-compression/src/artifacts.rs deleted file mode 100644 index 37171bb..0000000 --- a/crates/proof-compression/src/artifacts.rs +++ /dev/null @@ -1,174 +0,0 @@ -use std::io::Read; - -use boojum::cs::implementations::fast_serialization::MemcopySerializable; -use boojum::cs::implementations::setup::FinalizationHintsForProver; -use boojum::cs::traits::gate::FinalizationHintSerialized; -use circuit_definitions::circuit_definitions::aux_layer::ZkSyncCompressionVerificationKey; -use circuit_definitions::circuit_definitions::recursion_layer::ZkSyncRecursionVerificationKey; - -use super::*; - -pub trait BlobStorage { - fn save(data: W) - where - W: std::io::Write; -} - -pub struct AsyncHandler { - receiver: std::sync::mpsc::Receiver, -} - -impl AsyncHandler -where - T: Send + Sync + 'static, -{ - pub fn spawn(f: F) -> Self - where - F: FnOnce() -> std::sync::mpsc::Receiver + Send + Sync + 'static, - { - let receiver = std::thread::spawn(f); - - Self { - receiver: receiver.join().unwrap(), - } - } - - pub fn wait(self) -> T { - self.receiver.recv().unwrap() - } -} - -use shivini::cs::GpuSetup; - -pub trait ArtifactLoader: Sized + Send + Sync { - fn init(bs: BS) -> Self - where - BS: BlobStorage; - fn load_scheduler_finalization_hint(&self) -> FinalizationHintsForProver; - fn read_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box; - fn load_compression_layer_finalization_hint( - &self, - circuit_id: u8, - ) -> FinalizationHintsForProver; - fn load_compression_wrapper_finalization_hint( - &self, - circuit_id: u8, - ) -> FinalizationHintSerialized; - fn read_compression_wrapper_finalization_hint(&self, circuit_id: u8) -> Box; - fn load_plonk_finalization_hint(&self) -> usize; - - fn load_scheduler_vk(&self) -> ZkSyncRecursionVerificationKey; - fn load_compression_layer_vk(&self, circuit_id: u8) -> ZkSyncCompressionVerificationKey; - fn read_scheduler_vk(&self) -> Box; - fn read_compression_layer_vk(&self, circuit_id: u8) -> Box; - fn read_compression_wrapper_vk(&self, circuit_id: u8) -> Box; - fn read_plonk_vk(&self) -> Box; - fn read_fflonk_vk(&self) -> Box; - fn get_compression_layer_precomputation( - &self, - circuit_id: u8, - ) -> Box; - fn get_compression_wrapper_precomputation( - &self, - circuit_id: u8, - ) -> Box; - fn get_plonk_snark_wrapper_precomputation( - &self, - circuit_id: u8, - ) -> Box; - fn get_fflonk_snark_wrapper_precomputation(&self) -> Box; -} - -pub struct SimpleArtifactLoader; - -impl ArtifactLoader for SimpleArtifactLoader { - fn init(bs: BS) -> Self - where - BS: BlobStorage, - { - todo!() - } - - fn load_scheduler_finalization_hint(&self) -> FinalizationHintsForProver { - todo!() - } - - fn read_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box { - todo!() - } - - fn load_compression_layer_finalization_hint( - &self, - circuit_id: u8, - ) -> FinalizationHintsForProver { - todo!() - } - - fn load_compression_wrapper_finalization_hint( - &self, - circuit_id: u8, - ) -> FinalizationHintSerialized { - todo!() - } - - fn read_compression_wrapper_finalization_hint(&self, circuit_id: u8) -> Box { - todo!() - } - - fn load_plonk_finalization_hint(&self) -> usize { - todo!() - } - - fn load_scheduler_vk(&self) -> ZkSyncRecursionVerificationKey { - todo!() - } - - fn load_compression_layer_vk(&self, circuit_id: u8) -> ZkSyncCompressionVerificationKey { - todo!() - } - - fn read_scheduler_vk(&self) -> Box { - todo!() - } - - fn read_compression_layer_vk(&self, circuit_id: u8) -> Box { - todo!() - } - - fn read_compression_wrapper_vk(&self, circuit_id: u8) -> Box { - todo!() - } - - fn read_plonk_vk(&self) -> Box { - todo!() - } - - fn read_fflonk_vk(&self) -> Box { - todo!() - } - - fn get_compression_layer_precomputation( - &self, - circuit_id: u8, - ) -> Box { - todo!() - } - - fn get_compression_wrapper_precomputation( - &self, - circuit_id: u8, - ) -> Box { - todo!() - } - - fn get_plonk_snark_wrapper_precomputation( - &self, - circuit_id: u8, - ) -> Box { - todo!() - } - - fn get_fflonk_snark_wrapper_precomputation(&self) -> Box { - todo!() - } -} diff --git a/crates/proof-compression/src/blob_storage.rs b/crates/proof-compression/src/blob_storage.rs new file mode 100644 index 0000000..0469a54 --- /dev/null +++ b/crates/proof-compression/src/blob_storage.rs @@ -0,0 +1,277 @@ +use std::io::{Read, Write}; + +use super::*; + +pub trait BlobStorage: Send + Sync { + fn read_scheduler_vk(&self) -> Box; + fn read_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box; + fn read_compression_layer_vk(&self, circuit_id: u8) -> Box; + fn read_compression_layer_precomputation(&self, circuit_id: u8) -> Box; + + fn read_compression_wrapper_finalization_hint(&self, circuit_id: u8) -> Box; + fn read_compression_wrapper_vk(&self, circuit_id: u8) -> Box; + fn read_compression_wrapper_precomputation( + &self, + circuit_id: u8, + ) -> Box; + + fn read_fflonk_vk(&self) -> Box; + fn read_fflonk_precomputation(&self) -> Box; + + fn read_plonk_vk(&self) -> Box; + fn read_plonk_precomputation(&self) -> Box; +} + +pub trait BlobStorageExt: BlobStorage { + fn write_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box; + fn write_compression_layer_vk(&self, circuit_id: u8) -> Box; + fn write_compression_layer_precomputation(&self, circuit_id: u8) -> Box; + + fn write_compression_wrapper_finalization_hint(&self, circuit_id: u8) -> Box; + fn write_compression_wrapper_vk(&self, circuit_id: u8) -> Box; + fn write_compression_wrapper_precomputation(&self, circuit_id: u8) -> Box; + + fn write_fflonk_vk(&self) -> Box; + fn write_fflonk_precomputation(&self) -> Box; + + fn write_plonk_vk(&self) -> Box; + fn write_plonk_precomputation(&self) -> Box; +} + +pub struct FileSystemBlobStorage; + +impl FileSystemBlobStorage { + const DATA_DIR_PATH: &str = "./data"; + const SCHEDULER_PREFIX: &str = "scheduler_recursive"; + const COMPRESSION_LAYER_PREFIX: &str = "compression"; + const COMPRESSION_WRAPPER_PREFIX: &str = "compression_wrapper"; + const FFLONK_PREFIX: &str = "fflonk"; + const PLONK_PREFIX: &str = "plonk"; + + fn open_file(path: &str) -> Box { + let file = std::fs::File::open(path).unwrap(); + Box::new(file) + } + + fn create_file(path: &str) -> Box { + let file = std::fs::File::create(path).unwrap(); + Box::new(file) + } +} + +impl BlobStorage for FileSystemBlobStorage { + fn read_scheduler_vk(&self) -> Box { + let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::SCHEDULER_PREFIX,); + println!("Reading compression layer finalization at path {}", path); + Self::open_file(&path) + } + + fn read_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_hint.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_LAYER_PREFIX, + circuit_id + ); + println!("Reading compression layer finalization at path {}", path); + Self::open_file(&path) + } + + fn read_compression_layer_vk(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_vk.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_LAYER_PREFIX, + circuit_id + ); + println!("Reading compression layer finalization at path {}", path); + Self::open_file(&path) + } + + fn read_compression_layer_precomputation(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_setup_data.bin", + Self::DATA_DIR_PATH, + Self::COMPRESSION_LAYER_PREFIX, + circuit_id + ); + println!("Reading compression layer finalization at path {}", path); + Self::open_file(&path) + } + + fn read_compression_wrapper_finalization_hint(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_hint.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_WRAPPER_PREFIX, + circuit_id + ); + println!("Reading compression layer finalization at path {}", path); + Self::open_file(&path) + } + + fn read_compression_wrapper_vk(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_vk.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_WRAPPER_PREFIX, + circuit_id + ); + println!("Reading compression wrapper vk at path {}", path); + Self::open_file(&path) + } + + fn read_compression_wrapper_precomputation( + &self, + circuit_id: u8, + ) -> Box { + let path = format!( + "{}/{}_{}_setup_data.bin", + Self::DATA_DIR_PATH, + Self::COMPRESSION_WRAPPER_PREFIX, + circuit_id + ); + println!("Reading compression layer finalization at path {}", path); + Self::open_file(&path) + } + + fn read_fflonk_vk(&self) -> Box { + let path = format!("{}/{}_setup_.bin", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); + println!("Reading compression layer finalization at path {}", path); + Self::open_file(&path) + } + + fn read_fflonk_precomputation(&self) -> Box { + let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); + println!("Reading compression layer finalization at path {}", path); + Self::open_file(&path) + } + + fn read_plonk_precomputation(&self) -> Box { + let path = format!("{}/{}_setup_.bin", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); + println!("Reading compression layer finalization at path {}", path); + Self::open_file(&path) + } + + fn read_plonk_vk(&self) -> Box { + let path = format!("{}/{}_vk.bin", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); + println!("Reading compression layer finalization at path {}", path); + Self::open_file(&path) + } +} + +impl BlobStorageExt for FileSystemBlobStorage { + fn write_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_hint.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_LAYER_PREFIX, + circuit_id + ); + println!("Writing compression layer finalization at path {}", path); + Self::create_file(&path) + } + + fn write_compression_layer_vk(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_vk.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_LAYER_PREFIX, + circuit_id + ); + println!("Writeing compression layer finalization at path {}", path); + Self::create_file(&path) + } + + fn write_compression_layer_precomputation(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_setup_data.bin", + Self::DATA_DIR_PATH, + Self::COMPRESSION_LAYER_PREFIX, + circuit_id + ); + println!("Writeing compression layer finalization at path {}", path); + Self::create_file(&path) + } + + fn write_compression_wrapper_finalization_hint(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_hint.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_WRAPPER_PREFIX, + circuit_id + ); + println!("Writeing compression layer finalization at path {}", path); + Self::create_file(&path) + } + + fn write_compression_wrapper_vk(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_vk.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_WRAPPER_PREFIX, + circuit_id + ); + println!("Writeing compression layer finalization at path {}", path); + Self::create_file(&path) + } + + fn write_compression_wrapper_precomputation(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_setup_data.bin", + Self::DATA_DIR_PATH, + Self::COMPRESSION_WRAPPER_PREFIX, + circuit_id + ); + println!("Writeing compression layer finalization at path {}", path); + Self::create_file(&path) + } + + fn write_fflonk_vk(&self) -> Box { + let path = format!("{}/{}_setup_.bin", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); + println!("Writeing compression layer finalization at path {}", path); + Self::create_file(&path) + } + + fn write_fflonk_precomputation(&self) -> Box { + let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); + println!("Writeing compression layer finalization at path {}", path); + Self::create_file(&path) + } + + fn write_plonk_precomputation(&self) -> Box { + let path = format!("{}/{}_setup_.bin", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); + println!("Writeing compression layer finalization at path {}", path); + Self::create_file(&path) + } + + fn write_plonk_vk(&self) -> Box { + let path = format!("{}/{}_vk.bin", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); + println!("Writeing compression layer finalization at path {}", path); + Self::create_file(&path) + } +} + +pub struct AsyncHandler { + receiver: std::sync::mpsc::Receiver, +} + +impl AsyncHandler +where + T: Send + Sync + 'static, +{ + pub fn spawn(f: F) -> Self + where + F: FnOnce() -> std::sync::mpsc::Receiver + Send + Sync + 'static, + { + let receiver = std::thread::spawn(f); + + Self { + receiver: receiver.join().unwrap(), + } + } + + pub fn wait(self) -> T { + self.receiver.recv().unwrap() + } +} diff --git a/crates/proof-compression/src/chain.rs b/crates/proof-compression/src/chain.rs index b3a15b2..7c409a8 100644 --- a/crates/proof-compression/src/chain.rs +++ b/crates/proof-compression/src/chain.rs @@ -8,81 +8,120 @@ pub enum SnarkWrapperProof { Plonk(PlonkSnarkVerifierCircuitProof), FFfonk(FflonkSnarkVerifierCircuitProof), } -pub fn wrap_proof( + +pub fn run_proof_chain( input_proof: SchedulerProof, snark_wrapper: SnarkWrapper, - blob_storage: BS, + blob_storage: &BS, ) -> SnarkWrapperProof where BS: BlobStorage, { match snark_wrapper { SnarkWrapper::Plonk => { - let proof = run_step_chain_with_plonk(input_proof, blob_storage); + let proof = run_proof_chain_with_plonk(input_proof, blob_storage); SnarkWrapperProof::Plonk(proof) } SnarkWrapper::FFfonk => { - let proof = run_step_chain_with_fflonk(input_proof, blob_storage); + let proof = run_proof_chain_with_fflonk(input_proof, blob_storage); SnarkWrapperProof::FFfonk(proof) } } } -pub(crate) fn run_step_chain_with_fflonk( +pub(crate) fn run_proof_chain_with_fflonk( input_proof: SchedulerProof, - blob_storage: BS, + blob_storage: &BS, ) -> FflonkSnarkVerifierCircuitProof where BS: BlobStorage, { - let artifact_loader = SimpleArtifactLoader::init(blob_storage); - let context_initializor = SimpelContextInitializor::new(); + let context_manager = SimpleContextManager::new(); - let next_proof = CompressionMode1::prove_compression_step::<_, SimpelContextInitializor>( - input_proof, - &artifact_loader, + let next_proof = + CompressionMode1::prove_compression_step(input_proof, blob_storage, &context_manager); + let next_proof = CompressionMode2::prove_compression_step::<_, SimpleContextManager>( + next_proof, + blob_storage, + &context_manager, ); - let next_proof = CompressionMode2::prove_compression_step::<_, SimpelContextInitializor>( + let next_proof = CompressionMode3::prove_compression_step::<_, SimpleContextManager>( next_proof, - &artifact_loader, + blob_storage, + &context_manager, ); - let next_proof = CompressionMode3::prove_compression_step::<_, SimpelContextInitializor>( + let next_proof = CompressionMode4::prove_compression_step::<_, SimpleContextManager>( next_proof, - &artifact_loader, + blob_storage, + &context_manager, ); - let next_proof = CompressionMode4::prove_compression_step::<_, SimpelContextInitializor>( + let next_proof = CompressionMode5ForWrapper::prove_compression_step::<_, SimpleContextManager>( next_proof, - &artifact_loader, + blob_storage, + &context_manager, ); - let next_proof = CompressionMode5ForWrapper::prove_compression_step::< - _, - SimpelContextInitializor, - >(next_proof, &artifact_loader); - let final_proof = FflonkSnarkWrapper::prove_snark_wrapper_step::<_, SimpelContextInitializor>( + let final_proof = FflonkSnarkWrapper::prove_snark_wrapper_step::<_, SimpleContextManager>( next_proof, - &artifact_loader, + blob_storage, + &context_manager, ); final_proof } -pub(crate) fn run_step_chain_with_plonk( +pub(crate) fn precompute_proof_chain_with_fflonk(blob_storage: &BS) +where + BS: BlobStorageExt, +{ + let context_manager = SimpleContextManager::new(); + println!("Precomputing step 1"); + CompressionMode1::precomputae_and_store_compression_circuits(blob_storage, &context_manager); + println!("Precomputing step 2"); + CompressionMode2::precomputae_and_store_compression_circuits(blob_storage, &context_manager); + println!("Precomputing step 3"); + CompressionMode3::precomputae_and_store_compression_circuits(blob_storage, &context_manager); + println!("Precomputing step 4"); + CompressionMode4::precomputae_and_store_compression_circuits(blob_storage, &context_manager); + println!("Precomputing step 5"); + CompressionMode5ForWrapper::precomputae_and_store_compression_circuits( + blob_storage, + &context_manager, + ); + println!("Precomputing fflonk"); + FflonkSnarkWrapper::precompute_and_store_snark_wrapper_circuit(blob_storage, &context_manager); + println!("All steps in this approach precomputed and saved into blob storage"); +} + +pub(crate) fn run_proof_chain_with_plonk( input_proof: SchedulerProof, - blob_storage: BS, + blob_storage: &BS, ) -> PlonkSnarkVerifierCircuitProof where BS: BlobStorage, { - let artifact_loader = SimpleArtifactLoader::init(blob_storage); - let next_proof = CompressionMode1ForWrapper::prove_compression_step::< - _, - SimpelContextInitializor, - >(input_proof, &artifact_loader); + let context_manager = SimpleContextManager::new(); - let final_proof = PlonkSnarkWrapper::prove_snark_wrapper_step::<_, SimpelContextInitializor>( - next_proof, - &artifact_loader, + let next_proof = CompressionMode1ForWrapper::prove_compression_step( + input_proof, + blob_storage, + &context_manager, ); + let final_proof = + PlonkSnarkWrapper::prove_snark_wrapper_step(next_proof, blob_storage, &context_manager); + final_proof } + +pub(crate) fn precompute_proof_chain_with_plonk(blob_storage: &BS) +where + BS: BlobStorageExt, +{ + let context_manager = SimpleContextManager::new(); + CompressionMode1ForWrapper::precomputae_and_store_compression_circuits( + blob_storage, + &context_manager, + ); + PlonkSnarkWrapper::precompute_and_store_snark_wrapper_circuit(blob_storage, &context_manager); + println!("All steps in this approach precomputed and saved into blob storage"); +} diff --git a/crates/proof-compression/src/compression.rs b/crates/proof-compression/src/compression.rs index 239e2b2..67cf718 100644 --- a/crates/proof-compression/src/compression.rs +++ b/crates/proof-compression/src/compression.rs @@ -22,63 +22,60 @@ pub trait CompressionStep: CompressionProofSystem { const MODE: u8; const IS_WRAPPER: bool; - fn load_finalization_hint( - artifact_loader: &AL, + fn load_finalization_hint( + blob_storage: &BS, ) -> ::FinalizationHint where - AL: ArtifactLoader, + BS: BlobStorage, { let reader = if Self::IS_WRAPPER { - artifact_loader.read_compression_wrapper_finalization_hint(Self::MODE) + blob_storage.read_compression_wrapper_finalization_hint(Self::MODE) } else { - artifact_loader.read_compression_layer_finalization_hint(Self::MODE) + blob_storage.read_compression_layer_finalization_hint(Self::MODE) }; serde_json::from_reader(reader).unwrap() } - fn load_previous_vk( - artifact_loader: &AL, + fn load_previous_vk( + blob_storage: &BS, ) -> VerificationKey where - AL: ArtifactLoader, + BS: BlobStorage, { assert!(Self::MODE >= 1); let reader = if Self::MODE == 1 { - artifact_loader.read_scheduler_vk() - } else if Self::IS_WRAPPER { - artifact_loader.read_compression_wrapper_vk(Self::MODE) + blob_storage.read_scheduler_vk() } else { - artifact_loader.read_compression_layer_vk(Self::MODE - 1) + blob_storage.read_compression_layer_vk(Self::MODE - 1) }; serde_json::from_reader(reader).unwrap() } - fn load_this_vk(artifact_loader: &AL) -> ::VK + fn load_this_vk(blob_storage: &BS) -> ::VK where - AL: ArtifactLoader, + BS: BlobStorage, { let reader = if Self::IS_WRAPPER { - artifact_loader.read_compression_wrapper_vk(Self::MODE) + blob_storage.read_compression_wrapper_vk(Self::MODE) } else { - artifact_loader.read_compression_layer_vk(Self::MODE) + blob_storage.read_compression_layer_vk(Self::MODE) }; serde_json::from_reader(reader).unwrap() } - fn get_precomputation( - artifact_loader: &AL, + fn get_precomputation( + blob_storage: &BS, ) -> AsyncHandler<::Precomputation> where - AL: ArtifactLoader, + BS: BlobStorage, { - // TODO let reader = if Self::IS_WRAPPER { - artifact_loader.get_compression_layer_precomputation(Self::MODE) + blob_storage.read_compression_layer_precomputation(Self::MODE) } else { - artifact_loader.get_compression_layer_precomputation(Self::MODE) + blob_storage.read_compression_wrapper_precomputation(Self::MODE) }; let f = move || { let (sender, receiver) = std::sync::mpsc::channel(); @@ -96,20 +93,20 @@ pub trait CompressionStep: CompressionProofSystem { AsyncHandler::spawn(f) } - fn prove_compression_step( + fn prove_compression_step( input_proof: Proof, - artifact_loader: &AL, + blob_storage: &BS, + context_handler: &CI, ) -> ::Proof where - AL: ArtifactLoader, - CI: ContextInitializator, + BS: BlobStorage, + CI: ContextManagerInterface, { - let input_vk = Self::load_previous_vk(artifact_loader); - let vk = Self::load_this_vk(artifact_loader); - let precomputation = Self::get_precomputation(artifact_loader); - let config = ::get_context_config(); - let ctx = CI::init::(config); - let finalization_hint = Self::load_finalization_hint(artifact_loader); + let input_vk = Self::load_previous_vk(blob_storage); + let vk = Self::load_this_vk(blob_storage); + let precomputation = Self::get_precomputation(blob_storage); + let ctx = context_handler.init_context::(); + let finalization_hint = Self::load_finalization_hint(blob_storage); let circuit = Self::build_circuit(input_vk, Some(input_proof)); let proving_assembly = ::synthesize_for_proving( circuit, @@ -117,14 +114,18 @@ pub trait CompressionStep: CompressionProofSystem { ); let aux_config = ::aux_config_from_assembly(&proving_assembly); - ::prove( + let proof = ::prove( ctx, proving_assembly, aux_config, precomputation, finalization_hint, - vk, - ) + &vk, + ); + + assert!(::verify(&proof, &vk)); + + proof } // CompressionLayerCircuit is unified type for both compression circuits @@ -135,24 +136,44 @@ pub trait CompressionStep: CompressionProofSystem { } pub trait CompressionStepExt: CompressionProofSystemExt + CompressionStep { - fn run_precomputation_for_compression( - artifact_loader: &AL, - ) -> ( - ::Precomputation, - ::VK, - ) + fn precomputae_and_store_compression_circuits(blob_storage: &BS, context_manager: &CM) where - AL: ArtifactLoader, + BS: BlobStorageExt, + CM: ContextManagerInterface, { - let input_vk = Self::load_previous_vk(artifact_loader); + let input_vk = Self::load_previous_vk(blob_storage); let circuit = Self::build_circuit(input_vk, None); + let ctx = context_manager.init_context::(); let (finalization_hint, setup_assembly) = ::synthesize_for_setup(circuit); - let data = ::generate_precomputation_and_vk( - setup_assembly, - finalization_hint, + let (precomputation, vk) = + ::generate_precomputation_and_vk( + ctx, + setup_assembly, + &finalization_hint, + ); + let (precompuatation_writer, vk_writer, hint_writer) = if Self::IS_WRAPPER { + ( + blob_storage.write_compression_wrapper_precomputation(Self::MODE), + blob_storage.write_compression_wrapper_vk(Self::MODE), + blob_storage.write_compression_wrapper_finalization_hint(Self::MODE), + ) + } else { + ( + blob_storage.write_compression_layer_precomputation(Self::MODE), + blob_storage.write_compression_layer_vk(Self::MODE), + blob_storage.write_compression_layer_finalization_hint(Self::MODE), + ) + }; + precomputation + .write_into_buffer(precompuatation_writer) + .unwrap(); + serde_json::to_writer(vk_writer, &vk).unwrap(); + serde_json::to_writer(hint_writer, &finalization_hint).unwrap(); + println!( + "Precomputation and vk of compression circuit {} saved into blob storage", + Self::MODE ); - data.wait() } } @@ -175,6 +196,8 @@ macro_rules! impl_compression_circuit { } } } + + impl CompressionStepExt for $type {} }; } diff --git a/crates/proof-compression/src/context.rs b/crates/proof-compression/src/context.rs index f54750d..2b6940b 100644 --- a/crates/proof-compression/src/context.rs +++ b/crates/proof-compression/src/context.rs @@ -1,23 +1,39 @@ +use std::sync::atomic::AtomicBool; + use crate::{AsyncHandler, ProofSystemDefinition}; -pub trait ContextInitializator { - fn init

(config: P::ContextConfig) -> AsyncHandler +pub trait ContextManagerInterface { + fn init_context

(&self) -> AsyncHandler where P: ProofSystemDefinition; } -pub struct SimpelContextInitializor; +pub struct SimpleContextManager(std::sync::Arc); -impl SimpelContextInitializor { +impl SimpleContextManager { pub fn new() -> Self { - todo!() + SimpleContextManager(std::sync::Arc::new(AtomicBool::new(false))) } } -impl ContextInitializator for SimpelContextInitializor { - fn init

(config: P::ContextConfig) -> AsyncHandler +impl ContextManagerInterface for SimpleContextManager { + fn init_context

(&self) -> AsyncHandler where P: ProofSystemDefinition, { - todo!() + assert!(self.0.load(std::sync::atomic::Ordering::Relaxed) == false); + // load next context + let flag = self.0.clone(); + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let config = P::get_context_config(); + let context = P::init_context(config); + // mark status + flag.store(false, std::sync::atomic::Ordering::Relaxed); + sender.send(context).unwrap(); + + receiver + }; + + AsyncHandler::spawn(f) } } diff --git a/crates/proof-compression/src/cpu.rs b/crates/proof-compression/src/cpu.rs deleted file mode 100644 index 7e9f8e1..0000000 --- a/crates/proof-compression/src/cpu.rs +++ /dev/null @@ -1,425 +0,0 @@ -use super::*; -use bellman::plonk::better_better_cs::cs::ProvingAssembly; -use bellman::plonk::better_better_cs::{ - cs::{Circuit, PlonkCsWidth3Params, SetupAssembly, TrivialAssembly}, - gates::naive_main_gate::NaiveMainGate, -}; -use circuit_definitions::circuit_definitions::recursion_layer::{ - ZkSyncRecursionProof, ZkSyncRecursionVerificationKey, -}; -use circuit_definitions::{ - circuit_definitions::aux_layer::{ - compression::{CompressionLayerCircuit, ProofCompressionFunction}, - ZkSyncCompressionForWrapperCircuit, ZkSyncCompressionLayerCircuit, ZkSyncCompressionProof, - ZkSyncCompressionProofForWrapper, ZkSyncCompressionVerificationKey, - ZkSyncCompressionVerificationKeyForWrapper, - }, - snark_wrapper::franklin_crypto::bellman::plonk::commitments::transcript::keccak_transcript::RollingKeccakTranscript, -}; - -use franklin_crypto::boojum::{ - config::{CSConfig, DevCSConfig}, - cs::{ - cs_builder::new_builder, - cs_builder_reference::CsReferenceImplementationBuilder, - implementations::{ - prover::ProofConfig, reference_cs::CSReferenceAssembly, - setup::FinalizationHintsForProver, - }, - }, - field::goldilocks::{GoldilocksExt2, GoldilocksField}, -}; - -use franklin_crypto::boojum::cs::implementations::proof::Proof as BoojumProof; -use franklin_crypto::boojum::cs::implementations::verifier::VerificationKey as BoojumVK; -use franklin_crypto::boojum::worker::Worker as BoojumWorker; - -pub const L1_VERIFIER_DOMAIN_SIZE_LOG: usize = 23; -pub const MAX_COMBINED_DEGREE_FACTOR: usize = 9; -type F = GoldilocksField; -type EXT = GoldilocksExt2; - -pub fn precompute_and_save_setup_for_fflonk_snark_circuit( - circuit: &FflonkSnarkVerifierCircuit, - worker: &Worker, - output_blob_path: &str, -) { - let compression_wrapper_mode = circuit.wrapper_function.numeric_circuit_type(); - println!("Compression mode: {compression_wrapper_mode}"); - let mut setup_assembly = SetupAssembly::::new(); - circuit.synthesize(&mut setup_assembly).expect("must work"); - assert!(setup_assembly.is_satisfied()); - setup_assembly.finalize(); - println!("Finalized assembly contains {} gates", setup_assembly.n()); - - let domain_size = setup_assembly.n() + 1; - assert!(domain_size.is_power_of_two()); - assert!(domain_size <= 1 << L1_VERIFIER_DOMAIN_SIZE_LOG); - - let mon_crs = fflonk::init_crs(&worker, domain_size); - let setup: FflonkSnarkVerifierCircuitSetup = - FflonkSetup::create_setup(&setup_assembly, &worker, &mon_crs).expect("fflonk setup"); - let vk = FflonkVerificationKey::from_setup(&setup, &mon_crs).unwrap(); - - save_fflonk_setup_and_vk_into_file(&setup, &vk, output_blob_path); -} - -pub fn prove_fflonk_snark_verifier_circuit_with_precomputation( - circuit: &FflonkSnarkVerifierCircuit, - precomputed_setup: &FflonkSnarkVerifierCircuitSetup, - vk: &FflonkSnarkVerifierCircuitVK, - worker: &Worker, -) -> FflonkSnarkVerifierCircuitProof { - let compression_wrapper_mode = circuit.wrapper_function.numeric_circuit_type(); - let mut assembly = ProvingAssembly::::new(); - circuit.synthesize(&mut assembly).expect("must work"); - assert!(assembly.is_satisfied()); - assembly.finalize(); - - let domain_size = assembly.n() + 1; - println!( - "Trace log length {} for compression mode {}", - domain_size.trailing_zeros(), - compression_wrapper_mode - ); - assert!(domain_size.is_power_of_two()); - - assert!(domain_size <= 1 << L1_VERIFIER_DOMAIN_SIZE_LOG); - - let mon_crs = fflonk::init_crs(&worker, domain_size); - - let proof = fflonk::fflonk_cpu::prover::create_proof::< - _, - FflonkSnarkVerifierCircuit, - _, - _, - _, - RollingKeccakTranscript, - >(&assembly, &worker, &precomputed_setup, &mon_crs, None) - .expect("proof"); - let valid = - fflonk::fflonk_cpu::verify::<_, _, RollingKeccakTranscript>(&vk, &proof, None).unwrap(); - assert!(valid, "proof verification fails"); - - proof -} - -pub fn prove_fflonk_snark_verifier_circuit_single_shot( - circuit: &FflonkSnarkVerifierCircuit, - worker: &Worker, -) -> ( - FflonkSnarkVerifierCircuitProof, - FflonkSnarkVerifierCircuitVK, -) { - let compression_wrapper_mode = circuit.wrapper_function.numeric_circuit_type(); - let mut assembly = TrivialAssembly::::new(); - circuit.synthesize(&mut assembly).expect("must work"); - assert!(assembly.is_satisfied()); - assembly.finalize(); - let domain_size = assembly.n() + 1; - assert!(domain_size.is_power_of_two()); - assert!(domain_size <= 1 << L1_VERIFIER_DOMAIN_SIZE_LOG); - println!( - "Trace log length {} for compression mode {}", - domain_size.trailing_zeros(), - compression_wrapper_mode - ); - - let max_combined_degree = fflonk::fflonk::compute_max_combined_degree_from_assembly::< - _, - _, - _, - _, - FflonkSnarkVerifierCircuit, - >(&assembly); - println!("Max degree is {}", max_combined_degree); - let mon_crs = fflonk::init_crs(&worker, domain_size); - let setup = FflonkSetup::create_setup(&assembly, &worker, &mon_crs).expect("setup"); - let vk = FflonkVerificationKey::from_setup(&setup, &mon_crs).unwrap(); - - let proof = fflonk::fflonk_cpu::prover::create_proof::< - _, - FflonkSnarkVerifierCircuit, - _, - _, - _, - RollingKeccakTranscript, - >(&assembly, &worker, &setup, &mon_crs, None) - .expect("proof"); - let valid = - fflonk::fflonk_cpu::verify::<_, _, RollingKeccakTranscript>(&vk, &proof, None).unwrap(); - assert!(valid, "proof verification fails"); - - (proof, vk) -} - -#[test] -#[ignore] -fn run_proof_compression_by_schedule() { - let path = if let Ok(path) = std::env::var("BLOB_PATH") { - path.to_string() - } else { - "./data".to_string() - }; - let (scheduler_proof, scheduler_vk) = load_scheduler_proof_and_vk(&path); - process_steps( - scheduler_proof, - scheduler_vk, - CompressionSchedule::hard(), - &path, - ); -} -pub fn process_steps( - proof: ZkSyncRecursionProof, - vk: ZkSyncRecursionVerificationKey, - schedule: CompressionSchedule, - path: &str, -) { - let worker = BoojumWorker::new(); - let mut input = CompressionInput::Recursion(Some(proof), vk, CompressionMode::One); - - dbg!(&schedule); - let CompressionSchedule { - name: compression_schedule_name, - compression_steps, - } = schedule; - - let last_compression_wrapping_mode = - CompressionMode::from_compression_mode(compression_steps.last().unwrap().clone() as u8 + 1); - dbg!(&last_compression_wrapping_mode); - - /* - This illustrates how compression enforced for the "hardest" strategy - - input compression verifier output compression wrapper - _____________________________ ____________ ___________ __________ ___________________ - scheduler proof vk 1 scheduler -> compressed1 compressed2 - compressed1 proof vk 2 compressed1 -> compressed2 compressed3 - compressed2 proof vk 3 compressed2 -> compressed3 compressed4 - compressed3 proof vk 4 compressed3 -> compressed4 compressed5 - - - compressed5 proof vk - compression wrapper5 -> fflonk proof - */ - - let num_compression_steps = compression_steps.len(); - let mut compression_modes_iter = compression_steps.into_iter(); - for step_idx in 0..num_compression_steps { - let compression_mode = compression_modes_iter.next().unwrap(); - let proof_file_path = format!("{path}/compression_{}_proof.json", compression_mode as u8); - let proof_file_path = std::path::Path::new(&proof_file_path); - let vk_file_path = format!("{path}/compression_{}_vk.json", compression_mode as u8); - let vk_file_path = std::path::Path::new(&vk_file_path); - if proof_file_path.exists() && vk_file_path.exists() { - println!( - "Compression {compression_schedule_name}/{} proof and vk already exist ignoring", - compression_mode as u8 - ); - let proof_file = std::fs::File::open(proof_file_path).unwrap(); - let proof = serde_json::from_reader(&proof_file).unwrap(); - let vk_file = std::fs::File::open(vk_file_path).unwrap(); - let vk = serde_json::from_reader(&vk_file).unwrap(); - if step_idx + 1 == num_compression_steps { - input = - CompressionInput::CompressionWrapper(proof, vk, last_compression_wrapping_mode) - } else { - input = CompressionInput::Compression( - proof, - vk, - CompressionMode::from_compression_mode(compression_mode as u8 + 1), - ) - } - - continue; - } - let compression_circuit = input.into_compression_circuit(); - let circuit_type = compression_circuit.numeric_circuit_type(); - println!( - "Proving compression {compression_schedule_name}/{}", - compression_mode as u8 - ); - let (proof, vk) = inner_prove_compression_layer_circuit(compression_circuit, &worker); - println!( - "Proof for compression {compression_schedule_name}/{} is generated!", - compression_mode as u8 - ); - - save_compression_proof_and_vk_into_file(&proof, &vk, compression_mode as u8, path); - if step_idx + 1 == num_compression_steps { - input = CompressionInput::CompressionWrapper( - Some(proof), - vk, - last_compression_wrapping_mode, - ); - } else { - input = CompressionInput::Compression( - Some(proof), - vk, - CompressionMode::from_compression_mode(compression_mode as u8 + 1), - ); - } - } - - // last wrapping step - let proof_file_path = format!( - "{path}/compression_wrapper_{}_proof.json", - last_compression_wrapping_mode as u8 - ); - let proof_file_path = std::path::Path::new(&proof_file_path); - let vk_file_path = format!( - "{path}/compression_wrapper_{}_vk.json", - last_compression_wrapping_mode as u8 - ); - let vk_file_path = std::path::Path::new(&vk_file_path); - println!( - "Compression for wrapper level {}", - last_compression_wrapping_mode as u8 - ); - if proof_file_path.exists() && vk_file_path.exists() { - println!( - "Compression {compression_schedule_name}/{} for wrapper proof and vk already exist ignoring", - last_compression_wrapping_mode as u8 - ); - } else { - println!( - "Proving compression {compression_schedule_name}/{} for wrapper", - last_compression_wrapping_mode as u8 - ); - let compression_circuit = input.into_compression_wrapper_circuit(); - let (proof, vk) = inner_prove_compression_wrapper_circuit(compression_circuit, &worker); - println!( - "Proof for compression wrapper {compression_schedule_name}/{} is generated!", - last_compression_wrapping_mode as u8 - ); - save_compression_wrapper_proof_and_vk_into_file( - &proof, - &vk, - last_compression_wrapping_mode as u8, - path, - ); - println!( - "Compression wrapper proof and vk for {compression_schedule_name}/{} saved", - last_compression_wrapping_mode as u8 - ); - } - - // final wrapping step - let final_proof_file_path = format!("{}/final_proof.json", path); - let final_proof_file_path = std::path::Path::new(&final_proof_file_path); - let final_vk_file_path = format!("{}/final_vk.json", path,); - let final_vk_file_path = std::path::Path::new(&final_vk_file_path); - - if final_proof_file_path.exists() == false || final_vk_file_path.exists() == false { - let (compression_wrapper_proof, compression_wrapper_vk) = - load_compression_wrapper_proof_and_vk_from_file( - path, - last_compression_wrapping_mode as u8, - ); - let wrapper_circuit = fflonk::init_snark_wrapper_circuit_from_inputs( - last_compression_wrapping_mode as u8, - compression_wrapper_proof, - compression_wrapper_vk, - ); - - let (final_proof, final_vk) = - prove_fflonk_snark_verifier_circuit_single_shot(&wrapper_circuit, &Worker::new()); - let final_proof_file = std::fs::File::create(final_proof_file_path).unwrap(); - serde_json::to_writer(&final_proof_file, &final_proof).unwrap(); - println!( - "final snark proof saved into {}", - final_proof_file_path.to_string_lossy() - ); - save_fflonk_proof_and_vk_into_file(&final_proof, &final_vk, &path); - } else { - println!( - "final proof already exists {}", - final_proof_file_path.to_string_lossy() - ); - } -} - -pub fn inner_prove_compression_layer_circuit( - circuit: ZkSyncCompressionLayerCircuit, - worker: &BoojumWorker, -) -> (ZkSyncCompressionProof, ZkSyncCompressionVerificationKey) { - let proof_config = circuit.proof_config_for_compression_step(); - let verifier_builder = circuit.into_dyn_verifier_builder(); - let verifier = verifier_builder.create_verifier(); - - let (proof, vk, is_proof_valid) = match circuit { - ZkSyncCompressionLayerCircuit::CompressionMode1Circuit(inner) => { - let (proof, vk) = - prove_compression_circuit_with_precomputations(inner.clone(), proof_config, worker); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode2Circuit(inner) => { - let (proof, vk) = - prove_compression_circuit_with_precomputations(inner.clone(), proof_config, worker); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode3Circuit(inner) => { - let (proof, vk) = - prove_compression_circuit_with_precomputations(inner.clone(), proof_config, worker); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode4Circuit(inner) => { - let (proof, vk) = - prove_compression_circuit_with_precomputations(inner.clone(), proof_config, worker); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode5Circuit(_inner) => { - unreachable!("Only 4 modes of compression is allowed") - } - }; - if is_proof_valid == false { - println!("Proof is invalid"); - } - (proof, vk) -} - -pub fn inner_prove_compression_wrapper_circuit( - circuit: ZkSyncCompressionForWrapperCircuit, - worker: &BoojumWorker, -) -> ( - ZkSyncCompressionProofForWrapper, - ZkSyncCompressionVerificationKeyForWrapper, -) { - let proof_config = circuit.proof_config_for_compression_step(); - let verifier_builder = circuit.into_dyn_verifier_builder(); - let verifier = verifier_builder.create_verifier(); - - let (proof, vk, is_proof_valid) = match circuit { - ZkSyncCompressionForWrapperCircuit::CompressionMode5Circuit(inner) => { - let (proof, vk) = - prove_compression_circuit_with_precomputations(inner.clone(), proof_config, worker); - let is_proof_valid = verify_compression_wrapper_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - _ => unreachable!("Only mode 5 is supported for wrapper"), - }; - if is_proof_valid == false { - println!("Proof is invalid"); - } - - (proof, vk) -} - -pub fn prove_compression_circuit_with_precomputations( - circuit: CompressionLayerCircuit, - proof_config: ProofConfig, - worker: &BoojumWorker, -) -> ( - BoojumProof, - BoojumVK::ThisLayerHasher>, -) { - let cs = synthesize_circuit_for_dev(circuit); - cs.prove_one_shot::<_, CF::ThisLayerTranscript, CF::ThisLayerHasher, CF::ThisLayerPoW>( - worker, - proof_config, - CF::this_layer_transcript_parameters(), - ) -} diff --git a/crates/proof-compression/src/gpu.rs b/crates/proof-compression/src/gpu.rs deleted file mode 100644 index df1a5fa..0000000 --- a/crates/proof-compression/src/gpu.rs +++ /dev/null @@ -1,616 +0,0 @@ -use super::*; -use ::fflonk::*; -use boojum::cs::implementations::prover::ProofConfig; -use boojum::field::goldilocks::GoldilocksField as F; -use boojum::worker::Worker; -use shivini::boojum::cs::implementations::setup::FinalizationHintsForProver; -use shivini::boojum::cs::implementations::verifier::VerificationKey; -use shivini::circuit_definitions::boojum; -use shivini::circuit_definitions::circuit_definitions::{ - aux_layer::{ - compression::{CompressionLayerCircuit, ProofCompressionFunction}, - CompressionProofsTreeHasher, ZkSyncCompressionForWrapperCircuit, - ZkSyncCompressionLayerCircuit, ZkSyncCompressionProof, ZkSyncCompressionProofForWrapper, - ZkSyncCompressionVerificationKey, ZkSyncCompressionVerificationKeyForWrapper, - }, - recursion_layer::{ZkSyncRecursionProof, ZkSyncRecursionVerificationKey}, -}; -use shivini::cs::GpuSetup; -use shivini::gpu_proof_config::GpuProofConfig; -use shivini::{ - gpu_prove_from_external_witness_data_with_cache_strategy, CacheStrategy, - CommitmentCacheStrategy, GPUPoWRunner, GpuTreeHasher, PolynomialsCacheStrategy, ProverContext, - ProverContextConfig, -}; -use std::alloc::Global; - -#[test] -#[ignore] -fn run_proof_compression_by_schedule() { - let path = if let Ok(path) = std::env::var("BLOB_PATH") { - path.to_string() - } else { - "./data".to_string() - }; - let (scheduler_proof, scheduler_vk) = load_scheduler_proof_and_vk(&path); - process_steps( - scheduler_proof, - scheduler_vk, - CompressionSchedule::hard(), - &path, - ); -} - -pub fn process_steps( - proof: ZkSyncRecursionProof, - vk: ZkSyncRecursionVerificationKey, - schedule: CompressionSchedule, - path: &str, -) { - let worker = Worker::new(); - let mut input = CompressionInput::Recursion(Some(proof), vk, CompressionMode::One); - - dbg!(&schedule); - let CompressionSchedule { - name: compression_schedule_name, - compression_steps, - } = schedule; - - let last_compression_wrapping_mode = - CompressionMode::from_compression_mode(*compression_steps.last().unwrap() as u8 + 1); - dbg!(&last_compression_wrapping_mode); - - /* - This illustrates how compression enforced for the "hardest" strategy - - input compression verifier output compression wrapper - _____________________________ ____________ ___________ __________ ___________________ - scheduler proof vk 1 scheduler -> compressed1 compressed2 - compressed1 proof vk 2 compressed1 -> compressed2 compressed3 - compressed2 proof vk 3 compressed2 -> compressed3 compressed4 - compressed3 proof vk 4 compressed3 -> compressed4 compressed5 - - - compressed5 proof vk - compression wrapper5 -> fflonk proof - */ - - let num_compression_steps = compression_steps.len(); - let mut compression_modes_iter = compression_steps.into_iter(); - for step_idx in 0..num_compression_steps { - let compression_mode = compression_modes_iter.next().unwrap(); - let proof_file_path = format!("{}/compression_{}_proof.json", path, compression_mode as u8); - let proof_file_path = std::path::Path::new(&proof_file_path); - let vk_file_path = format!("{}/compression_{}_vk.json", path, compression_mode as u8); - let vk_file_path = std::path::Path::new(&vk_file_path); - if proof_file_path.exists() && vk_file_path.exists() { - println!( - "Compression {compression_schedule_name}/{} proof and vk already exist ignoring", - compression_mode as u8 - ); - let proof_file = std::fs::File::open(proof_file_path).unwrap(); - let input_proof = serde_json::from_reader(&proof_file).unwrap(); - let vk_file = std::fs::File::open(vk_file_path).unwrap(); - let input_vk = serde_json::from_reader(&vk_file).unwrap(); - if step_idx + 1 == num_compression_steps { - input = CompressionInput::CompressionWrapper( - input_proof, - input_vk, - last_compression_wrapping_mode, - ) - } else { - input = CompressionInput::Compression( - input_proof, - input_vk, - CompressionMode::from_compression_mode(compression_mode as u8 + 1), - ) - } - - continue; - } - - let compression_circuit = input.into_compression_circuit(); - let circuit_type = compression_circuit.numeric_circuit_type(); - println!( - "Proving compression {compression_schedule_name}/{}", - compression_mode as u8 - ); - let setup_file_path = format!("{path}/compression_{}_setup.bin", compression_mode as u8); - let setup_file_path = std::path::Path::new(&setup_file_path); - let (device_setup, vk, finalization_hint) = if setup_file_path.exists() { - load_compression_precomputations(compression_mode as u8, path) - } else { - let proof_config = compression_circuit.proof_config_for_compression_step(); - precompute_and_save_compression_layer_circuit_setup_and_vk::( - compression_circuit.clone(), - &worker, - proof_config, - path, - ) - }; - let (output_proof, output_vk) = prove_compression_layer_circuit_with_precomputations( - compression_circuit.clone(), - &device_setup, - finalization_hint, - vk, - &worker, - ); - println!( - "Proof for compression {compression_schedule_name}/{} is generated!", - compression_mode as u8 - ); - - save_compression_proof_and_vk_into_file(&output_proof, &output_vk, circuit_type, path); - - if step_idx + 1 == num_compression_steps { - input = CompressionInput::CompressionWrapper( - Some(output_proof), - output_vk, - last_compression_wrapping_mode, - ); - } else { - input = CompressionInput::Compression( - Some(output_proof), - output_vk, - CompressionMode::from_compression_mode(compression_mode as u8 + 1), - ); - } - } - - // last wrapping step - let proof_file_path = format!( - "{}/compression_wrapper_{}_proof.json", - path, last_compression_wrapping_mode as u8 - ); - let compression_wrapper_proof_file_path = std::path::Path::new(&proof_file_path); - let vk_file_path = format!( - "{}/compression_wrapper_{}_vk.json", - path, last_compression_wrapping_mode as u8 - ); - let compression_wrapper_vk_file_path = std::path::Path::new(&vk_file_path); - println!( - "Compression for wrapper level {}", - last_compression_wrapping_mode as u8 - ); - if compression_wrapper_proof_file_path.exists() && compression_wrapper_vk_file_path.exists() { - println!( - "Compression {compression_schedule_name}/{} for wrapper proof and vk already exist ignoring", - last_compression_wrapping_mode as u8 - ); - } else { - let compression_circuit = input.into_compression_wrapper_circuit(); - let setup_file_path = format!( - "{path}/compression_wrapper_{}_setup.bin", - last_compression_wrapping_mode as u8 - ); - let setup_file_path = std::path::Path::new(&setup_file_path); - let (device_setup, vk, finalization_hint) = if setup_file_path.exists() { - load_compression_precomputations(last_compression_wrapping_mode as u8, path) - } else { - let proof_config = compression_circuit.proof_config_for_compression_step(); - precompute_and_save_compression_wrapper_circuit_setup_and_vk::( - compression_circuit.clone(), - &worker, - proof_config, - path, - ) - }; - println!( - "Proving compression {compression_schedule_name}/{} for wrapper", - last_compression_wrapping_mode as u8 - ); - let (compression_wrapper_output_proof, compression_wrapper_output_vk) = - prove_compression_wrapper_circuit_with_precomputations( - compression_circuit, - &device_setup, - finalization_hint, - vk, - &worker, - ); - println!( - "Proof for compression wrapper {compression_schedule_name}/{} is generated!", - last_compression_wrapping_mode as u8 - ); - save_compression_wrapper_proof_and_vk_into_file( - &compression_wrapper_output_proof, - &compression_wrapper_output_vk, - last_compression_wrapping_mode as u8, - path, - ); - println!( - "Compression wrapper proof and vk for {compression_schedule_name}/{} saved", - last_compression_wrapping_mode as u8 - ); - } - // final wrapping step - let final_proof_file_path = format!("{}/final_proof.json", path); - let final_proof_file_path = std::path::Path::new(&final_proof_file_path); - let final_vk_file_path = format!("{}/final_vk.json", path,); - let final_vk_file_path = std::path::Path::new(&final_vk_file_path); - - if final_proof_file_path.exists() == false || final_vk_file_path.exists() == false { - let (compression_wrapper_proof, compression_wrapper_vk) = - load_compression_wrapper_proof_and_vk_from_file( - path, - last_compression_wrapping_mode as u8, - ); - let wrapper_circuit = init_snark_wrapper_circuit_from_inputs( - last_compression_wrapping_mode as u8, - compression_wrapper_proof, - compression_wrapper_vk, - ); - - let (final_proof, final_vk) = - ::fflonk::gpu_prove_fflonk_snark_verifier_circuit_single_shot(&wrapper_circuit); - let final_proof_file = std::fs::File::create(final_proof_file_path).unwrap(); - serde_json::to_writer(&final_proof_file, &final_proof).unwrap(); - println!( - "final snark proof saved into {}", - final_proof_file_path.to_string_lossy() - ); - save_fflonk_proof_and_vk_into_file(&final_proof, &final_vk, &path); - } else { - println!( - "final proof already exists {}", - final_proof_file_path.to_string_lossy() - ); - } -} - -pub fn prove_compression_layer_circuit_with_precomputations( - circuit: ZkSyncCompressionLayerCircuit, - device_setup: &GpuSetup, - finalization_hint: FinalizationHintsForProver, - vk: VerificationKey, - worker: &Worker, -) -> (ZkSyncCompressionProof, ZkSyncCompressionVerificationKey) { - let proof_config = circuit.proof_config_for_compression_step(); - let verifier_builder = circuit.into_dyn_verifier_builder(); - let verifier = verifier_builder.create_verifier(); - let gpu_proof_config = GpuProofConfig::from_compression_layer_circuit(&circuit); - - let (proof, vk, is_proof_valid) = match circuit { - ZkSyncCompressionLayerCircuit::CompressionMode1Circuit(inner) => { - let (proof, vk) = inner_prove_compression_layer_circuit( - inner.clone(), - device_setup, - finalization_hint, - vk, - proof_config, - gpu_proof_config, - worker, - ); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode2Circuit(inner) => { - let (proof, vk) = inner_prove_compression_layer_circuit( - inner.clone(), - device_setup, - finalization_hint, - vk, - proof_config, - gpu_proof_config, - worker, - ); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode3Circuit(inner) => { - let (proof, vk) = inner_prove_compression_layer_circuit( - inner.clone(), - device_setup, - finalization_hint, - vk, - proof_config, - gpu_proof_config, - worker, - ); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode4Circuit(inner) => { - let (proof, vk) = inner_prove_compression_layer_circuit( - inner.clone(), - device_setup, - finalization_hint, - vk, - proof_config, - gpu_proof_config, - worker, - ); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - _ => unreachable!("Only 4 modes of compression is allowed"), - }; - if !is_proof_valid { - panic!("Proof is invalid"); - } - - (proof, vk) -} - -pub fn prove_compression_wrapper_circuit_with_precomputations( - circuit: ZkSyncCompressionForWrapperCircuit, - device_setup: &GpuSetup, - finalization_hint: FinalizationHintsForProver, - vk: VerificationKey, - worker: &Worker, -) -> ( - ZkSyncCompressionProofForWrapper, - ZkSyncCompressionVerificationKeyForWrapper, -) { - let proof_config = circuit.proof_config_for_compression_step(); - let verifier_builder = circuit.into_dyn_verifier_builder(); - let verifier = verifier_builder.create_verifier(); - let gpu_proof_config = GpuProofConfig::from_compression_wrapper_circuit(&circuit); - - let (proof, vk, is_proof_valid) = match circuit { - ZkSyncCompressionForWrapperCircuit::CompressionMode5Circuit(inner) => { - let (proof, vk) = inner_prove_compression_wrapper_circuit( - inner.clone(), - device_setup, - finalization_hint, - vk, - proof_config, - gpu_proof_config, - worker, - ); - let is_proof_valid = verify_compression_wrapper_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - _ => unreachable!("Only mode 5 is supported for wrapper"), - }; - if !is_proof_valid { - panic!("Proof is invalid"); - } - - (proof, vk) -} - -pub fn inner_prove_compression_layer_circuit< - CF: ProofCompressionFunction, ->( - circuit: CompressionLayerCircuit, - device_setup: &GpuSetup, - finalization_hint: FinalizationHintsForProver, - vk: VerificationKey, - proof_cfg: ProofConfig, - gpu_cfg: GpuProofConfig, - worker: &Worker, -) -> (ZkSyncCompressionProof, ZkSyncCompressionVerificationKey) { - let proving_cs = synthesize_circuit_for_proving(circuit, &finalization_hint); - let witness = proving_cs.witness.as_ref().unwrap(); - let domain_size = vk.fixed_parameters.domain_size as usize; - assert_eq!(finalization_hint.final_trace_len, domain_size); - let config = ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); - let ctx = ProverContext::create_with_config(config).expect("gpu prover context"); - let cache_strategy = CacheStrategy { - setup_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - trace_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - other_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - commitment: CommitmentCacheStrategy::CacheCosetCaps, - }; - let gpu_proof = gpu_prove_from_external_witness_data_with_cache_strategy::< - CompressionProofsTranscript, - CompressionProofsTreeHasher, - CF::ThisLayerPoW, - Global, - >( - &gpu_cfg, - witness, - proof_cfg.clone(), - &device_setup, - &vk, - (), - worker, - cache_strategy, - ) - .expect("gpu proof"); - drop(ctx); - let proof = gpu_proof.into(); - (proof, vk) -} - -pub fn inner_prove_compression_wrapper_circuit< - CF: ProofCompressionFunction, ->( - circuit: CompressionLayerCircuit, - device_setup: &GpuSetup, - finalization_hint: FinalizationHintsForProver, - vk: VerificationKey, - proof_cfg: ProofConfig, - gpu_cfg: GpuProofConfig, - worker: &Worker, -) -> ( - ZkSyncCompressionProofForWrapper, - ZkSyncCompressionVerificationKeyForWrapper, -) { - let proving_cs = synthesize_circuit_for_proving(circuit, &finalization_hint); - let witness = proving_cs.witness.as_ref().unwrap(); - let domain_size = vk.fixed_parameters.domain_size as usize; - assert_eq!(finalization_hint.final_trace_len, domain_size); - let config = ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); - let ctx = ProverContext::create_with_config(config).expect("gpu prover context"); - let cache_strategy = CacheStrategy { - setup_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - trace_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - other_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - commitment: CommitmentCacheStrategy::CacheCosetCaps, - }; - let gpu_proof = gpu_prove_from_external_witness_data_with_cache_strategy::< - CompressionTranscriptForWrapper, - CompressionTreeHasherForWrapper, - CF::ThisLayerPoW, - Global, - >( - &gpu_cfg, - witness, - proof_cfg.clone(), - device_setup, - &vk, - (), - worker, - cache_strategy, - ) - .expect("gpu proof"); - drop(ctx); - (gpu_proof.into(), vk) -} - -pub fn precompute_and_save_compression_layer_circuit_setup_and_vk( - circuit: ZkSyncCompressionLayerCircuit, - worker: &Worker, - proof_cfg: ProofConfig, - path: &str, -) -> ( - GpuSetup, - VerificationKey, - FinalizationHintsForProver, -) { - match circuit { - ZkSyncCompressionLayerCircuit::CompressionMode1Circuit(compression_layer_circuit) => { - precompute_and_save_compression_circuit_setup_and_vk::< - _, - CompressionProofsTreeHasher, - SAVE, - >(compression_layer_circuit, 1, worker, proof_cfg, path) - } - ZkSyncCompressionLayerCircuit::CompressionMode2Circuit(compression_layer_circuit) => { - precompute_and_save_compression_circuit_setup_and_vk::< - _, - CompressionProofsTreeHasher, - SAVE, - >(compression_layer_circuit, 2, worker, proof_cfg, path) - } - ZkSyncCompressionLayerCircuit::CompressionMode3Circuit(compression_layer_circuit) => { - precompute_and_save_compression_circuit_setup_and_vk::< - _, - CompressionProofsTreeHasher, - SAVE, - >(compression_layer_circuit, 3, worker, proof_cfg, path) - } - ZkSyncCompressionLayerCircuit::CompressionMode4Circuit(compression_layer_circuit) => { - precompute_and_save_compression_circuit_setup_and_vk::< - _, - CompressionProofsTreeHasher, - SAVE, - >(compression_layer_circuit, 4, worker, proof_cfg, path) - } - _ => unreachable!("Only 4 modes of compression is allowed"), - } -} - -pub fn precompute_and_save_compression_wrapper_circuit_setup_and_vk( - circuit: ZkSyncCompressionForWrapperCircuit, - worker: &Worker, - proof_cfg: ProofConfig, - path: &str, -) -> ( - GpuSetup, - VerificationKey, - FinalizationHintsForProver, -) { - match circuit { - ZkSyncCompressionForWrapperCircuit::CompressionMode5Circuit( - compression_wrapper_circuit, - ) => precompute_and_save_compression_circuit_setup_and_vk::<_, _, SAVE>( - compression_wrapper_circuit, - 5, - worker, - proof_cfg, - path, - ), - _ => unreachable!("Only mode 5 is supported for wrapper"), - } -} - -pub fn precompute_and_save_compression_circuit_setup_and_vk< - CF: ProofCompressionFunction, - H: GpuTreeHasher, - const SAVE: bool, ->( - circuit: CompressionLayerCircuit, - compression_mode: usize, - worker: &Worker, - proof_cfg: ProofConfig, - path: &str, -) -> ( - GpuSetup, - VerificationKey, - FinalizationHintsForProver, -) { - let (finalization_hint, setup_assembly) = synthesize_circuit_for_setup::(circuit); - let (setup_base, vk_params, vars_hint, wits_hint) = setup_assembly.get_light_setup( - &worker, - proof_cfg.fri_lde_factor, - proof_cfg.merkle_tree_cap_size, - ); - let domain_size = vk_params.domain_size as usize; - assert_eq!(finalization_hint.final_trace_len, domain_size); - let config = ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); - let _ctx = ProverContext::create_with_config(config).expect("gpu prover context"); - let (device_setup, vk) = shivini::cs::gpu_setup_and_vk_from_base_setup_vk_params_and_hints::< - H, - _, - >(setup_base, vk_params, vars_hint, wits_hint, worker) - .unwrap(); - if SAVE { - let base_path = if compression_mode == 5 { - format!("{path}/compression_wrapper") - } else { - format!("{path}/compression") - }; - let setup_file_path = format!("{}_device_setup.bin", base_path); - println!("Saving setup into file {setup_file_path}"); - let setup_file = std::fs::File::create(&setup_file_path).unwrap(); - bincode::serialize_into(&setup_file, &device_setup).unwrap(); - println!("fflonk device setup saved into {}", setup_file_path); - - let vk_file_path = format!("{}_vk.json", base_path); - let vk_file = std::fs::File::create(&vk_file_path).unwrap(); - serde_json::to_writer(&vk_file, &vk).unwrap(); - println!("fflonk vk saved into {}", vk_file_path); - - let finalization_hint_file_path = format!("{}_hint.json", base_path); - let finalization_hint_file = std::fs::File::create(&finalization_hint_file_path).unwrap(); - serde_json::to_writer(&finalization_hint_file, &finalization_hint).unwrap(); - println!( - "finalization hint saved into {}", - finalization_hint_file_path - ); - } - - (device_setup, vk, finalization_hint) -} - -pub fn load_compression_precomputations( - compression_mode: u8, - path: &str, -) -> ( - GpuSetup, - VerificationKey, - FinalizationHintsForProver, -) { - let base_path = if compression_mode == 5 { - format!("{path}/compression_wrapper") - } else { - format!("{path}/compression") - }; - let setup_file_path = format!("{base_path}_device_setup.bin",); - println!("Loading device setup from file {setup_file_path}"); - let setup_file = std::fs::File::open(&setup_file_path).unwrap(); - let device_setup: GpuSetup = bincode::deserialize_from(&setup_file).unwrap(); - - let vk_file_path = format!("{base_path}_vk.json"); - println!("Loading vk from file {vk_file_path}"); - let vk_file = std::fs::File::open(&vk_file_path).unwrap(); - let vk = serde_json::from_reader(&vk_file).unwrap(); - - let finalization_hint_file_path = format!("{base_path}_hint.json"); - println!("Loading finalization hint from file {finalization_hint_file_path}"); - let finalization_hint_file = std::fs::File::open(&finalization_hint_file_path).unwrap(); - let finalization_hint = serde_json::from_reader(&finalization_hint_file).unwrap(); - - (device_setup, vk, finalization_hint) -} diff --git a/crates/proof-compression/src/lib.rs b/crates/proof-compression/src/lib.rs index c625d3a..1774f07 100644 --- a/crates/proof-compression/src/lib.rs +++ b/crates/proof-compression/src/lib.rs @@ -2,15 +2,12 @@ #![feature(generic_const_exprs)] #![feature(allocator_api)] -mod artifacts; -use artifacts::*; +mod blob_storage; +use blob_storage::*; mod chain; use chain::*; -mod common; -use common::*; - mod compression; use compression::*; @@ -29,17 +26,15 @@ use step::*; mod serialization; use serialization::*; -mod task; -use task::*; +#[cfg(test)] +mod test; cfg_if::cfg_if! { if #[cfg(feature = "gpu")] { - mod gpu; - pub use gpu::*; + // } else { - mod cpu; - pub use cpu::*; + // } } use bellman::worker::Worker; diff --git a/crates/proof-compression/src/proof_system.rs b/crates/proof-compression/src/proof_system.rs index 448e144..3f90b39 100644 --- a/crates/proof-compression/src/proof_system.rs +++ b/crates/proof-compression/src/proof_system.rs @@ -46,9 +46,9 @@ pub trait ProofSystemDefinition: Sized { type Allocator: std::alloc::Allocator; type ProvingAssembly: Sized + Send + Sync + 'static; type ContextConfig; - type Context; + type Context: Send + Sync + 'static; fn get_context_config() -> Self::ContextConfig; - fn init_context(config: Self::ContextConfig) -> AsyncHandler; + fn init_context(config: Self::ContextConfig) -> Self::Context; fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData; fn verify(_: &Self::Proof, _: &Self::VK) -> bool; } @@ -75,7 +75,7 @@ pub trait CompressionProofSystem: _: Self::AuxConfig, _: AsyncHandler, _: Self::FinalizationHint, - _: Self::VK, + _: &Self::VK, ) -> Self::Proof; fn prove_from_witnesses( @@ -84,16 +84,17 @@ pub trait CompressionProofSystem: _: Self::AuxConfig, _: AsyncHandler, _: Self::FinalizationHint, - _: Self::VK, + _: &Self::VK, ) -> Self::Proof; } pub trait CompressionProofSystemExt: CompressionProofSystem { type SetupAssembly; fn generate_precomputation_and_vk( + _: AsyncHandler, _: Self::SetupAssembly, - _: Self::FinalizationHint, - ) -> AsyncHandler<(Self::Precomputation, Self::VK)>; + _: &Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK); fn synthesize_for_setup( circuit: CompressionLayerCircuit, ) -> (Self::FinalizationHint, Self::SetupAssembly); @@ -107,6 +108,7 @@ pub trait SnarkWrapperProofSystem: ProofSystemDefinition { _: Self::ProvingAssembly, _: AsyncHandler, _: Self::FinalizationHint, + _: &Self::VK, ) -> Self::Proof; fn prove_from_witnesses( @@ -114,6 +116,7 @@ pub trait SnarkWrapperProofSystem: ProofSystemDefinition { _: Vec, _: AsyncHandler, _: Self::FinalizationHint, + _: &Self::VK, ) -> Self::Proof; } @@ -121,9 +124,10 @@ pub trait SnarkWrapperProofSystemExt: SnarkWrapperProofSystem { type SetupAssembly; fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly; fn generate_precomputation_and_vk( + _: AsyncHandler, _: Self::SetupAssembly, _: Self::FinalizationHint, - ) -> AsyncHandler<(Self::Precomputation, Self::VK)>; + ) -> (Self::Precomputation, Self::VK); } type BoojumAssembly = @@ -152,21 +156,17 @@ where type ContextConfig = usize; // domain_size type Context = ProverContext; fn get_context_config() -> Self::ContextConfig { - todo!("domain size") + // TODO + println!("Using hardcoded domain size 2^17 for compression step"); + 1 << 17 } - fn init_context(domain_size: Self::ContextConfig) -> AsyncHandler { - let f = move || { - let (sender, receiver) = std::sync::mpsc::channel(); - let config = - ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); - let context = Self::Context::create_with_config(config).expect("gpu prover context"); - sender.send(context).unwrap(); - - receiver - }; + fn init_context(domain_size: Self::ContextConfig) -> Self::Context { + let config = + ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); + let context = Self::Context::create_with_config(config).expect("gpu prover context"); - AsyncHandler::spawn(f) + context } fn verify(proof: &Self::Proof, vk: &Self::VK) -> bool { @@ -216,7 +216,7 @@ where aux_config: Self::AuxConfig, precomputation: AsyncHandler, finalization_hint: Self::FinalizationHint, - vk: Self::VK, + vk: &Self::VK, ) -> Self::Proof { Self::prove_from_witnesses( ctx, @@ -234,7 +234,7 @@ where aux_config: Self::AuxConfig, precomputation: AsyncHandler, finalization_hint: Self::FinalizationHint, - vk: Self::VK, + vk: &Self::VK, ) -> Self::Proof { let domain_size = vk.fixed_parameters.domain_size as usize; assert_eq!(finalization_hint.final_trace_len, domain_size); @@ -285,35 +285,28 @@ where { type SetupAssembly = BoojumAssembly; fn generate_precomputation_and_vk( + ctx: AsyncHandler, setup_assembly: Self::SetupAssembly, - finalization_hint: Self::FinalizationHint, - ) -> AsyncHandler<(Self::Precomputation, Self::VK)> { - let f = move || { - let (sender, receiver) = std::sync::mpsc::channel(); - let worker = Worker::new(); - let proof_config = CF::proof_config_for_compression_step(); - let (setup_base, vk_params, vars_hint, wits_hint) = setup_assembly.get_light_setup( - &worker, - proof_config.fri_lde_factor, - proof_config.merkle_tree_cap_size, - ); - let domain_size = vk_params.domain_size as usize; - assert_eq!(finalization_hint.final_trace_len, domain_size); - let config = - ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); - let _ctx = ProverContext::create_with_config(config).expect("gpu prover context"); - let (device_setup, vk) = - shivini::cs::gpu_setup_and_vk_from_base_setup_vk_params_and_hints::< - CF::ThisLayerHasher, - _, - >(setup_base, vk_params, vars_hint, wits_hint, &worker) - .unwrap(); - sender.send((device_setup, vk)).unwrap(); - - receiver - }; - - AsyncHandler::spawn(f) + finalization_hint: &Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK) { + let worker = Worker::new(); + let proof_config = CF::proof_config_for_compression_step(); + let (setup_base, vk_params, vars_hint, wits_hint) = setup_assembly.get_light_setup( + &worker, + proof_config.fri_lde_factor, + proof_config.merkle_tree_cap_size, + ); + let domain_size = vk_params.domain_size as usize; + assert_eq!(finalization_hint.final_trace_len, domain_size); + let ctx = ctx.wait(); + let (precomputation, vk) = + shivini::cs::gpu_setup_and_vk_from_base_setup_vk_params_and_hints::< + CF::ThisLayerHasher, + _, + >(setup_base, vk_params, vars_hint, wits_hint, &worker) + .unwrap(); + drop(ctx); + (precomputation, vk) } fn synthesize_for_setup( circuit: CompressionLayerCircuit, @@ -330,9 +323,14 @@ type PlonkAssembly = Assembly< A, >; -pub struct PlonkProverDeviceMemoryManager; +pub struct UnsafePlonkProverDeviceMemoryManagerWrapper( + DeviceMemoryManager, +); +unsafe impl Send for UnsafePlonkProverDeviceMemoryManagerWrapper {} +unsafe impl Sync for UnsafePlonkProverDeviceMemoryManagerWrapper {} +pub struct PlonkProverDeviceMemoryManagerConfig; -impl ManagerConfigs for PlonkProverDeviceMemoryManager { +impl ManagerConfigs for PlonkProverDeviceMemoryManagerConfig { const NUM_GPUS_LOG: usize = 0; const FULL_SLOT_SIZE_LOG: usize = 24; const NUM_SLOTS: usize = 29; @@ -349,14 +347,14 @@ impl ProofSystemDefinition for PlonkSnarkWrapper { type Allocator = GlobalHost; type ProvingAssembly = PlonkAssembly; type ContextConfig = (Vec, Vec); - type Context = DeviceMemoryManager; + type Context = UnsafePlonkProverDeviceMemoryManagerWrapper; fn get_context_config() -> Self::ContextConfig { todo!() } - fn init_context(config: Self::ContextConfig) -> AsyncHandler { + fn init_context(config: Self::ContextConfig) -> Self::Context { let (device_ids, compact_crs) = config; - let ctx = Self::Context::init(&device_ids, &compact_crs[..]); - todo!() + let manager = DeviceMemoryManager::init(&device_ids, &compact_crs[..]).unwrap(); + UnsafePlonkProverDeviceMemoryManagerWrapper(manager) } fn take_witnesses( proving_assembly: &mut Self::ProvingAssembly, @@ -379,6 +377,7 @@ impl SnarkWrapperProofSystem for PlonkSnarkWrapper { _: Self::ProvingAssembly, _: AsyncHandler, _: Self::FinalizationHint, + _: &Self::VK, ) -> Self::Proof { todo!() } @@ -388,6 +387,7 @@ impl SnarkWrapperProofSystem for PlonkSnarkWrapper { _: Vec, _: AsyncHandler, _: Self::FinalizationHint, + _: &Self::VK, ) -> Self::Proof { todo!() } @@ -401,9 +401,10 @@ impl SnarkWrapperProofSystemExt for PlonkSnarkWrapper { } fn generate_precomputation_and_vk( + _: AsyncHandler, _: Self::SetupAssembly, _: Self::FinalizationHint, - ) -> AsyncHandler<(Self::Precomputation, Self::VK)> { + ) -> (Self::Precomputation, Self::VK) { todo!() } } @@ -428,17 +429,11 @@ impl ProofSystemDefinition for FflonkSnarkWrapper { fn get_context_config() -> Self::ContextConfig { fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG } - fn init_context(log_domain_size: Self::ContextConfig) -> AsyncHandler { - let f = move || { - let (sender, receiver) = std::sync::mpsc::channel(); - let domain_size = 1 << log_domain_size; - let context = Self::Context::init(domain_size).unwrap(); - sender.send(context).unwrap(); - - receiver - }; + fn init_context(log_domain_size: Self::ContextConfig) -> Self::Context { + let domain_size = 1 << log_domain_size; + let context = Self::Context::init(domain_size).unwrap(); - AsyncHandler::spawn(f) + context } fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { let input_assignments = @@ -474,6 +469,7 @@ impl SnarkWrapperProofSystem for FflonkSnarkWrapper { mut proving_assembly: Self::ProvingAssembly, precomputation: AsyncHandler, finalization_hint: Self::FinalizationHint, + vk: &Self::VK, ) -> Self::Proof { assert!(proving_assembly.is_satisfied()); let raw_trace_len = proving_assembly.n(); @@ -502,6 +498,7 @@ impl SnarkWrapperProofSystem for FflonkSnarkWrapper { _: Vec, _: AsyncHandler, _: Self::FinalizationHint, + _: &Self::VK, ) -> Self::Proof { unimplemented!() } @@ -519,28 +516,22 @@ impl SnarkWrapperProofSystemExt for FflonkSnarkWrapper { } fn generate_precomputation_and_vk( + ctx: AsyncHandler, setup_assembly: Self::SetupAssembly, - finalization_hint: Self::FinalizationHint, - ) -> AsyncHandler<(Self::Precomputation, Self::VK)> { - let f = move || { - let (sender, receiver) = std::sync::mpsc::channel(); - let device_setup = - FflonkSnarkVerifierCircuitDeviceSetup::create_setup_from_assembly_on_device( - &setup_assembly, - ) - .unwrap(); - let vk = device_setup.get_verification_key(); - sender - .send(( - FflonkSnarkVerifierCircuitDeviceSetupWrapper(device_setup), - vk, - )) - .unwrap(); - - receiver - }; - - AsyncHandler::spawn(f) + _finalization_hint: Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK) { + let ctx = ctx.wait(); + let device_setup = + FflonkSnarkVerifierCircuitDeviceSetup::create_setup_from_assembly_on_device( + &setup_assembly, + ) + .unwrap(); + let vk = device_setup.get_verification_key(); + drop(ctx); + ( + FflonkSnarkVerifierCircuitDeviceSetupWrapper(device_setup), + vk, + ) } } @@ -559,7 +550,7 @@ impl ProofSystemDefinition for MarkerProofSystem { fn get_context_config() -> Self::ContextConfig { todo!() } - fn init_context(config: Self::ContextConfig) -> AsyncHandler { + fn init_context(config: Self::ContextConfig) -> Self::Context { todo!() } @@ -571,3 +562,53 @@ impl ProofSystemDefinition for MarkerProofSystem { todo!() } } + +pub fn synthesize_circuit_for_setup( + circuit: CompressionLayerCircuit, +) -> ( + FinalizationHintsForProver, + CSReferenceAssembly, +) { + let geometry = circuit.geometry(); + let (max_trace_len, num_vars) = circuit.size_hint(); + + let builder_impl = boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< + GoldilocksField, + GoldilocksField, + SetupCSConfig, + >::new(geometry, max_trace_len.unwrap()); + let builder = boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); + + let builder = circuit.configure_builder_proxy(builder); + let mut cs = builder.build(num_vars.unwrap()); + circuit.add_tables(&mut cs); + circuit.synthesize_into_cs(&mut cs); + let (_domain_size, finalization_hint) = cs.pad_and_shrink(); + let cs = cs.into_assembly::(); + + (finalization_hint, cs) +} + +pub fn synthesize_circuit_for_proving( + circuit: CompressionLayerCircuit, + finalization_hint: &FinalizationHintsForProver, +) -> CSReferenceAssembly { + let geometry = circuit.geometry(); + let (max_trace_len, num_vars) = circuit.size_hint(); + + let builder_impl = boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< + GoldilocksField, + GoldilocksField, + ProvingCSConfig, + >::new(geometry, max_trace_len.unwrap()); + let builder = boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); + + let builder = circuit.configure_builder_proxy(builder); + let mut cs = builder.build(num_vars.unwrap()); + circuit.add_tables(&mut cs); + circuit.synthesize_into_cs(&mut cs); + let _ = cs.pad_and_shrink_using_hint(&finalization_hint); + let cs = cs.into_assembly::(); + + cs +} diff --git a/crates/proof-compression/src/serialization.rs b/crates/proof-compression/src/serialization.rs index c7c8c34..1dc4973 100644 --- a/crates/proof-compression/src/serialization.rs +++ b/crates/proof-compression/src/serialization.rs @@ -20,24 +20,6 @@ impl MemcopySerializable for PlonkSnarkVerifierCircuitDeviceSetupWrapper { } } -impl serde::Serialize for PlonkSnarkVerifierCircuitDeviceSetupWrapper { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - todo!() - } -} - -impl<'de> serde::Deserialize<'de> for PlonkSnarkVerifierCircuitDeviceSetupWrapper { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - todo!() - } -} - impl PlonkSnarkVerifierCircuitDeviceSetupWrapper { pub fn into_inner(self) -> PlonkSnarkVerifierCircuitDeviceSetup { self.0 @@ -51,29 +33,13 @@ impl MemcopySerializable for FflonkSnarkVerifierCircuitDeviceSetupWrapper { &self, dst: W, ) -> Result<(), Box> { - todo!() + Ok(self.0.write(dst).unwrap()) } fn read_from_buffer(src: R) -> Result> { - todo!() - } -} - -impl serde::Serialize for FflonkSnarkVerifierCircuitDeviceSetupWrapper { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - todo!() - } -} - -impl<'de> serde::Deserialize<'de> for FflonkSnarkVerifierCircuitDeviceSetupWrapper { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - todo!() + Ok(Self( + FflonkSnarkVerifierCircuitDeviceSetup::read(src).unwrap(), + )) } } diff --git a/crates/proof-compression/src/snark_wrapper.rs b/crates/proof-compression/src/snark_wrapper.rs index 80504f1..89a67f7 100644 --- a/crates/proof-compression/src/snark_wrapper.rs +++ b/crates/proof-compression/src/snark_wrapper.rs @@ -1,5 +1,7 @@ use boojum::cs::{ - implementations::{proof::Proof, verifier::VerificationKey}, + implementations::{ + fast_serialization::MemcopySerializable, proof::Proof, verifier::VerificationKey, + }, oracle::TreeHasher, }; use circuit_definitions::circuit_definitions::aux_layer::{ @@ -16,76 +18,100 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { GoldilocksField, Output: serde::Serialize + serde::de::DeserializeOwned, >; - fn load_finalization_hint( - artifact_loader: &AL, + fn load_finalization_hint( + blob_storage: &BS, ) -> ::FinalizationHint where - AL: ArtifactLoader, + BS: BlobStorage, { assert!(Self::IS_FFLONK ^ Self::IS_PLONK); let hint = if Self::IS_PLONK { &[26u8] } else { &[24] }; serde_json::from_reader(&hint[..]).unwrap() } - fn load_previous_vk( - artifact_loader: &AL, + fn load_previous_vk( + blob_storage: &BS, ) -> VerificationKey where - AL: ArtifactLoader, + BS: BlobStorage, { assert!(Self::IS_FFLONK ^ Self::IS_PLONK); let previous_compression_mode = Self::PREVIOUS_COMPRESSION_MODE; - let reader = artifact_loader.read_compression_wrapper_vk(previous_compression_mode); + let reader = blob_storage.read_compression_wrapper_vk(previous_compression_mode); serde_json::from_reader(reader).unwrap() } - fn load_this_vk(artifact_loader: &AL) -> ::VK + fn load_this_vk(blob_storage: &BS) -> ::VK where - AL: ArtifactLoader, + BS: BlobStorage, { assert!(Self::IS_FFLONK ^ Self::IS_PLONK); let reader = if Self::IS_FFLONK { assert_eq!(Self::IS_PLONK, false); - artifact_loader.read_fflonk_vk() + blob_storage.read_fflonk_vk() } else { assert_eq!(Self::IS_PLONK, true); - artifact_loader.read_plonk_vk() + blob_storage.read_plonk_vk() }; serde_json::from_reader(reader).unwrap() } - fn get_precomputation( - artifact_loader: &AL, + fn get_precomputation( + blob_storage: &BS, ) -> AsyncHandler<::Precomputation> where - AL: ArtifactLoader, + BS: BlobStorage, { - todo!() + let reader = if Self::IS_FFLONK { + blob_storage.read_fflonk_precomputation() + } else { + blob_storage.read_plonk_precomputation() + }; + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + + let precomputation = + <::Precomputation as MemcopySerializable>::read_from_buffer( + reader, + ) + .unwrap(); + + sender.send(precomputation).unwrap(); + receiver + }; + + AsyncHandler::spawn(f) } - fn prove_snark_wrapper_step( + fn prove_snark_wrapper_step( input_proof: Proof, - artifact_loader: &AL, + blob_storage: &BS, + context_handler: &CI, ) -> ::Proof where - AL: ArtifactLoader, - CI: ContextInitializator, + BS: BlobStorage, + CI: ContextManagerInterface, { assert!(Self::IS_FFLONK ^ Self::IS_PLONK); - let input_vk = Self::load_previous_vk(artifact_loader); - let precomputation = Self::get_precomputation(artifact_loader); - let config = ::get_context_config(); - let ctx = CI::init::(config); - let finalization_hint = Self::load_finalization_hint(artifact_loader); + let input_vk = Self::load_previous_vk(blob_storage); + let vk = Self::load_this_vk(blob_storage); + let precomputation = Self::get_precomputation(blob_storage); + let ctx = context_handler.init_context::(); + let finalization_hint = Self::load_finalization_hint(blob_storage); let circuit = Self::build_circuit(input_vk, Some(input_proof)); let proving_assembly = ::synthesize_for_proving(circuit); - ::prove( + let proof = ::prove( ctx, proving_assembly, precomputation, finalization_hint, - ) + &vk, + ); + + assert!(::verify(&proof, &vk)); + + proof } fn build_circuit( @@ -95,26 +121,40 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { } pub trait SnarkWrapperStepExt: SnarkWrapperProofSystemExt + SnarkWrapperStep { - fn run_precomputation_for_compression( - artifact_loader: &AL, - ) -> ( - ::Precomputation, - ::VK, - ) + fn precompute_and_store_snark_wrapper_circuit(blob_storage: &BS, context_manager: &CM) where - AL: ArtifactLoader, + BS: BlobStorageExt, + CM: ContextManagerInterface, ::VK: 'static, { - let input_vk = Self::load_previous_vk(artifact_loader); - let finalization_hint = Self::load_finalization_hint(artifact_loader); + let input_vk = Self::load_previous_vk(blob_storage); + let finalization_hint = Self::load_finalization_hint(blob_storage); let circuit = Self::build_circuit(input_vk, None); + let ctx = context_manager.init_context::(); let setup_assembly = ::synthesize_for_setup(circuit); - let data = ::generate_precomputation_and_vk( - setup_assembly, - finalization_hint, - ); - data.wait() + let (precomputation, vk) = + ::generate_precomputation_and_vk( + ctx, + setup_assembly, + finalization_hint, + ); + let (precompuatation_writer, vk_writer) = if Self::IS_FFLONK { + ( + blob_storage.write_fflonk_precomputation(), + blob_storage.write_fflonk_vk(), + ) + } else { + ( + blob_storage.write_plonk_precomputation(), + blob_storage.write_plonk_vk(), + ) + }; + precomputation + .write_into_buffer(precompuatation_writer) + .unwrap(); + serde_json::to_writer(vk_writer, &vk).unwrap(); + println!("Pecomputation and vk of snark wrapper circuit saved into blob storage"); } } @@ -141,6 +181,8 @@ impl SnarkWrapperStep for FflonkSnarkWrapper { } } } +impl SnarkWrapperStepExt for FflonkSnarkWrapper {} + pub struct PlonkSnarkWrapper; impl SnarkWrapperStep for PlonkSnarkWrapper { const IS_PLONK: bool = true; @@ -165,3 +207,4 @@ impl SnarkWrapperStep for PlonkSnarkWrapper { } } } +impl SnarkWrapperStepExt for PlonkSnarkWrapper {} diff --git a/crates/proof-compression/src/test.rs b/crates/proof-compression/src/test.rs new file mode 100644 index 0000000..31aa26e --- /dev/null +++ b/crates/proof-compression/src/test.rs @@ -0,0 +1,41 @@ +use circuit_definitions::circuit_definitions::recursion_layer::ZkSyncRecursionLayerProof; + +use super::*; + +pub fn test_proof_chain_with_fflonk() { + let scheduler_proof_file = + std::fs::File::open("./data/scheduler_recursive_proof.json").unwrap(); + let scheduler_proof: ZkSyncRecursionLayerProof = + serde_json::from_reader(&scheduler_proof_file).unwrap(); + let scheduler_proof = scheduler_proof.into_inner(); + + let simple_blob_storage = FileSystemBlobStorage; + let proof = run_proof_chain_with_fflonk(scheduler_proof, &simple_blob_storage); + + let proof_file_path = format!("./data/final_fflonk_proof.json"); + let proof_file = std::fs::File::create(&proof_file_path).unwrap(); + serde_json::to_writer(proof_file, &proof).unwrap(); + println!("Final fflonk snark wrapper roof saved at {proof_file_path}"); +} + +pub fn test_proof_chain_with_plonk() { + let scheduler_proof_file = + std::fs::File::open("./data/scheduler_recursive_proof.json").unwrap(); + let scheduler_proof: ZkSyncRecursionLayerProof = + serde_json::from_reader(&scheduler_proof_file).unwrap(); + let scheduler_proof = scheduler_proof.into_inner(); + + let simple_blob_storage = FileSystemBlobStorage; + let proof = run_proof_chain_with_fflonk(scheduler_proof, &simple_blob_storage); + + let proof_file_path = format!("./data/final_fflonk_proof.json"); + let proof_file = std::fs::File::create(&proof_file_path).unwrap(); + serde_json::to_writer(proof_file, &proof).unwrap(); + println!("Final fflonk snark wrapper roof saved at {proof_file_path}"); +} + +#[test] +fn test_precompute_compression_chain_artifacts() { + let simple_blob_storage = FileSystemBlobStorage; + precompute_proof_chain_with_fflonk(&simple_blob_storage); +} From 4af45aa9ace9d30ff5fa1c828aa65235afa87a45 Mon Sep 17 00:00:00 2001 From: saitima Date: Fri, 10 Jan 2025 19:48:37 +0300 Subject: [PATCH 05/18] feat(proof-compression): testing proving --- crates/proof-compression/Cargo.toml | 3 + crates/proof-compression/src/blob_storage.rs | 127 ++++--- crates/proof-compression/src/chain.rs | 63 +++- crates/proof-compression/src/compression.rs | 14 +- crates/proof-compression/src/context.rs | 82 ++++- crates/proof-compression/src/lib.rs | 6 +- crates/proof-compression/src/proof_system.rs | 323 ++++++++++++------ crates/proof-compression/src/serialization.rs | 55 ++- crates/proof-compression/src/snark_wrapper.rs | 25 +- crates/proof-compression/src/step.rs | 1 - crates/proof-compression/src/test.rs | 1 + crates/proof-compression/src/utils.rs | 28 +- 12 files changed, 511 insertions(+), 217 deletions(-) delete mode 100644 crates/proof-compression/src/step.rs diff --git a/crates/proof-compression/Cargo.toml b/crates/proof-compression/Cargo.toml index c3bb25d..22e8a33 100644 --- a/crates/proof-compression/Cargo.toml +++ b/crates/proof-compression/Cargo.toml @@ -15,6 +15,7 @@ exclude = ["/data"] circuit_definitions.workspace = true fflonk.workspace = true shivini.workspace = true +gpu-prover.workspace = true serde = "1" serde_json = "1" bincode = "1.3" @@ -25,3 +26,5 @@ byteorder = "1" default = ["gpu"] gpu = [] cpu = [] + +#TODO: define allocator diff --git a/crates/proof-compression/src/blob_storage.rs b/crates/proof-compression/src/blob_storage.rs index 0469a54..fb2bf3f 100644 --- a/crates/proof-compression/src/blob_storage.rs +++ b/crates/proof-compression/src/blob_storage.rs @@ -1,7 +1,5 @@ use std::io::{Read, Write}; -use super::*; - pub trait BlobStorage: Send + Sync { fn read_scheduler_vk(&self) -> Box; fn read_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box; @@ -17,9 +15,11 @@ pub trait BlobStorage: Send + Sync { fn read_fflonk_vk(&self) -> Box; fn read_fflonk_precomputation(&self) -> Box; + fn read_fflonk_crs(&self) -> Box; fn read_plonk_vk(&self) -> Box; fn read_plonk_precomputation(&self) -> Box; + fn read_plonk_crs(&self) -> Box; } pub trait BlobStorageExt: BlobStorage { @@ -33,9 +33,11 @@ pub trait BlobStorageExt: BlobStorage { fn write_fflonk_vk(&self) -> Box; fn write_fflonk_precomputation(&self) -> Box; + fn write_fflonk_crs(&self) -> Box; fn write_plonk_vk(&self) -> Box; fn write_plonk_precomputation(&self) -> Box; + fn write_plonk_crs(&self) -> Box; } pub struct FileSystemBlobStorage; @@ -62,10 +64,10 @@ impl FileSystemBlobStorage { impl BlobStorage for FileSystemBlobStorage { fn read_scheduler_vk(&self) -> Box { let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::SCHEDULER_PREFIX,); - println!("Reading compression layer finalization at path {}", path); + println!("Reading scheduler vk at path {}", path); Self::open_file(&path) } - + fn read_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box { let path = format!( "{}/{}_{}_hint.json", @@ -84,18 +86,18 @@ impl BlobStorage for FileSystemBlobStorage { Self::COMPRESSION_LAYER_PREFIX, circuit_id ); - println!("Reading compression layer finalization at path {}", path); + println!("Reading compression layer vk at path {}", path); Self::open_file(&path) } fn read_compression_layer_precomputation(&self, circuit_id: u8) -> Box { let path = format!( - "{}/{}_{}_setup_data.bin", + "{}/{}_{}_setup.bin", Self::DATA_DIR_PATH, Self::COMPRESSION_LAYER_PREFIX, circuit_id ); - println!("Reading compression layer finalization at path {}", path); + println!("Reading compression layer precomputation at path {}", path); Self::open_file(&path) } @@ -106,7 +108,7 @@ impl BlobStorage for FileSystemBlobStorage { Self::COMPRESSION_WRAPPER_PREFIX, circuit_id ); - println!("Reading compression layer finalization at path {}", path); + println!("Reading compression wrapper finalization at path {}", path); Self::open_file(&path) } @@ -126,36 +128,59 @@ impl BlobStorage for FileSystemBlobStorage { circuit_id: u8, ) -> Box { let path = format!( - "{}/{}_{}_setup_data.bin", + "{}/{}_{}_setup.bin", Self::DATA_DIR_PATH, Self::COMPRESSION_WRAPPER_PREFIX, circuit_id ); - println!("Reading compression layer finalization at path {}", path); + println!( + "Reading compression wrapper precomputation at path {}", + path + ); Self::open_file(&path) } fn read_fflonk_vk(&self) -> Box { - let path = format!("{}/{}_setup_.bin", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); - println!("Reading compression layer finalization at path {}", path); + let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); + println!("Reading fflonk vk at path {}", path); Self::open_file(&path) } fn read_fflonk_precomputation(&self) -> Box { - let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); - println!("Reading compression layer finalization at path {}", path); + let path = format!("{}/{}_setup.bin", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); + println!("Reading fflonk precomputation at path {}", path); Self::open_file(&path) } fn read_plonk_precomputation(&self) -> Box { - let path = format!("{}/{}_setup_.bin", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); - println!("Reading compression layer finalization at path {}", path); + let path = format!("{}/{}_setup.bin", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); + println!("Reading plonk precomputation at path {}", path); Self::open_file(&path) } fn read_plonk_vk(&self) -> Box { - let path = format!("{}/{}_vk.bin", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); - println!("Reading compression layer finalization at path {}", path); + let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); + println!("Reading plonk vk at path {}", path); + Self::open_file(&path) + } + + fn read_fflonk_crs(&self) -> Box { + let path = format!( + "{}/{}_compact_crs.key.raw", + Self::DATA_DIR_PATH, + Self::FFLONK_PREFIX + ); + println!("Reading fflonk CRS at path {}", path); + Self::open_file(&path) + } + + fn read_plonk_crs(&self) -> Box { + let path = format!( + "{}/{}_compact_crs.key.raw", + Self::DATA_DIR_PATH, + Self::PLONK_PREFIX + ); + println!("Reading fflonk CRS at path {}", path); Self::open_file(&path) } } @@ -179,18 +204,18 @@ impl BlobStorageExt for FileSystemBlobStorage { Self::COMPRESSION_LAYER_PREFIX, circuit_id ); - println!("Writeing compression layer finalization at path {}", path); + println!("Writeing compression layer vk at path {}", path); Self::create_file(&path) } fn write_compression_layer_precomputation(&self, circuit_id: u8) -> Box { let path = format!( - "{}/{}_{}_setup_data.bin", + "{}/{}_{}_setup.bin", Self::DATA_DIR_PATH, Self::COMPRESSION_LAYER_PREFIX, circuit_id ); - println!("Writeing compression layer finalization at path {}", path); + println!("Writeing compression layer precomputation at path {}", path); Self::create_file(&path) } @@ -201,7 +226,7 @@ impl BlobStorageExt for FileSystemBlobStorage { Self::COMPRESSION_WRAPPER_PREFIX, circuit_id ); - println!("Writeing compression layer finalization at path {}", path); + println!("Writeing compression wrapper finalization at path {}", path); Self::create_file(&path) } @@ -212,48 +237,71 @@ impl BlobStorageExt for FileSystemBlobStorage { Self::COMPRESSION_WRAPPER_PREFIX, circuit_id ); - println!("Writeing compression layer finalization at path {}", path); + println!("Writeing compression wrapper vk at path {}", path); Self::create_file(&path) } fn write_compression_wrapper_precomputation(&self, circuit_id: u8) -> Box { let path = format!( - "{}/{}_{}_setup_data.bin", + "{}/{}_{}_setup.bin", Self::DATA_DIR_PATH, Self::COMPRESSION_WRAPPER_PREFIX, circuit_id ); - println!("Writeing compression layer finalization at path {}", path); + println!( + "Writeing compression wrapper precomputation at path {}", + path + ); Self::create_file(&path) } fn write_fflonk_vk(&self) -> Box { - let path = format!("{}/{}_setup_.bin", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); - println!("Writeing compression layer finalization at path {}", path); + let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); + println!("Writeing fflonk vk at path {}", path); Self::create_file(&path) } fn write_fflonk_precomputation(&self) -> Box { - let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); - println!("Writeing compression layer finalization at path {}", path); + let path = format!("{}/{}_setup.bin", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); + println!("Writeing fflonk precomputation at path {}", path); Self::create_file(&path) } fn write_plonk_precomputation(&self) -> Box { - let path = format!("{}/{}_setup_.bin", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); - println!("Writeing compression layer finalization at path {}", path); + let path = format!("{}/{}_setup.bin", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); + println!("Writeing plonk precomputation at path {}", path); Self::create_file(&path) } fn write_plonk_vk(&self) -> Box { - let path = format!("{}/{}_vk.bin", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); - println!("Writeing compression layer finalization at path {}", path); + let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); + println!("Writeing plonk vk at path {}", path); + Self::create_file(&path) + } + + fn write_fflonk_crs(&self) -> Box { + let path = format!( + "{}/{}_compact_crs.key.raw", + Self::DATA_DIR_PATH, + Self::PLONK_PREFIX + ); + println!("Writeing fflonk CRS at path {}", path); + Self::create_file(&path) + } + + fn write_plonk_crs(&self) -> Box { + let path = format!( + "{}/{}_compact_crs.key.raw", + Self::DATA_DIR_PATH, + Self::PLONK_PREFIX + ); + println!("Writeing plonk CRS at path {}", path); Self::create_file(&path) } } -pub struct AsyncHandler { - receiver: std::sync::mpsc::Receiver, +pub struct AsyncHandler { + receiver: std::thread::JoinHandle>, } impl AsyncHandler @@ -266,12 +314,13 @@ where { let receiver = std::thread::spawn(f); - Self { - receiver: receiver.join().unwrap(), - } + Self { receiver } } pub fn wait(self) -> T { - self.receiver.recv().unwrap() + self.receiver.join().unwrap().recv().unwrap() } } + +unsafe impl Send for AsyncHandler where T: Send + Sync {} +unsafe impl Sync for AsyncHandler where T: Send + Sync {} diff --git a/crates/proof-compression/src/chain.rs b/crates/proof-compression/src/chain.rs index 7c409a8..d264718 100644 --- a/crates/proof-compression/src/chain.rs +++ b/crates/proof-compression/src/chain.rs @@ -37,7 +37,9 @@ where BS: BlobStorage, { let context_manager = SimpleContextManager::new(); - + let start = std::time::Instant::now(); + let snark_context_config = + context_manager.initialize_snark_context_config::(); let next_proof = CompressionMode1::prove_compression_step(input_proof, blob_storage, &context_manager); let next_proof = CompressionMode2::prove_compression_step::<_, SimpleContextManager>( @@ -60,12 +62,20 @@ where blob_storage, &context_manager, ); + println!( + "Proving entire compression chain took {}s", + start.elapsed().as_secs() + ); let final_proof = FflonkSnarkWrapper::prove_snark_wrapper_step::<_, SimpleContextManager>( + snark_context_config, next_proof, blob_storage, &context_manager, ); - + println!( + "Proving entire chain with snark wrapper took {}s", + start.elapsed().as_secs() + ); final_proof } @@ -74,22 +84,30 @@ where BS: BlobStorageExt, { let context_manager = SimpleContextManager::new(); - println!("Precomputing step 1"); + let snark_context_config = + context_manager.initialize_snark_context_config::(); + let start = std::time::Instant::now(); CompressionMode1::precomputae_and_store_compression_circuits(blob_storage, &context_manager); - println!("Precomputing step 2"); CompressionMode2::precomputae_and_store_compression_circuits(blob_storage, &context_manager); - println!("Precomputing step 3"); CompressionMode3::precomputae_and_store_compression_circuits(blob_storage, &context_manager); - println!("Precomputing step 4"); CompressionMode4::precomputae_and_store_compression_circuits(blob_storage, &context_manager); - println!("Precomputing step 5"); CompressionMode5ForWrapper::precomputae_and_store_compression_circuits( blob_storage, &context_manager, ); - println!("Precomputing fflonk"); - FflonkSnarkWrapper::precompute_and_store_snark_wrapper_circuit(blob_storage, &context_manager); - println!("All steps in this approach precomputed and saved into blob storage"); + println!( + "Precomputation of compression chain took {}s", + start.elapsed().as_secs() + ); + FflonkSnarkWrapper::precompute_and_store_snark_wrapper_circuit( + snark_context_config, + blob_storage, + &context_manager, + ); + println!( + "Precomputation of entire chain took {}s", + start.elapsed().as_secs() + ); } pub(crate) fn run_proof_chain_with_plonk( @@ -100,16 +118,25 @@ where BS: BlobStorage, { let context_manager = SimpleContextManager::new(); - + let snark_context_config = + context_manager.initialize_snark_context_config::(); + let start = std::time::Instant::now(); let next_proof = CompressionMode1ForWrapper::prove_compression_step( input_proof, blob_storage, &context_manager, ); - let final_proof = - PlonkSnarkWrapper::prove_snark_wrapper_step(next_proof, blob_storage, &context_manager); - + let final_proof = PlonkSnarkWrapper::prove_snark_wrapper_step( + snark_context_config, + next_proof, + blob_storage, + &context_manager, + ); + println!( + "Entire compression chain took {}s", + start.elapsed().as_secs() + ); final_proof } @@ -118,10 +145,16 @@ where BS: BlobStorageExt, { let context_manager = SimpleContextManager::new(); + let snark_context_config = + context_manager.initialize_snark_context_config::(); CompressionMode1ForWrapper::precomputae_and_store_compression_circuits( blob_storage, &context_manager, ); - PlonkSnarkWrapper::precompute_and_store_snark_wrapper_circuit(blob_storage, &context_manager); + PlonkSnarkWrapper::precompute_and_store_snark_wrapper_circuit( + snark_context_config, + blob_storage, + &context_manager, + ); println!("All steps in this approach precomputed and saved into blob storage"); } diff --git a/crates/proof-compression/src/compression.rs b/crates/proof-compression/src/compression.rs index 67cf718..e0df74a 100644 --- a/crates/proof-compression/src/compression.rs +++ b/crates/proof-compression/src/compression.rs @@ -73,13 +73,12 @@ pub trait CompressionStep: CompressionProofSystem { BS: BlobStorage, { let reader = if Self::IS_WRAPPER { - blob_storage.read_compression_layer_precomputation(Self::MODE) - } else { blob_storage.read_compression_wrapper_precomputation(Self::MODE) + } else { + blob_storage.read_compression_layer_precomputation(Self::MODE) }; let f = move || { let (sender, receiver) = std::sync::mpsc::channel(); - let precomputation = <::Precomputation as MemcopySerializable>::read_from_buffer( reader, @@ -105,8 +104,9 @@ pub trait CompressionStep: CompressionProofSystem { let input_vk = Self::load_previous_vk(blob_storage); let vk = Self::load_this_vk(blob_storage); let precomputation = Self::get_precomputation(blob_storage); - let ctx = context_handler.init_context::(); let finalization_hint = Self::load_finalization_hint(blob_storage); + let ctx_config = Self::get_context_config_from_hint(&finalization_hint); + let ctx = context_handler.init_compression_context::(ctx_config); let circuit = Self::build_circuit(input_vk, Some(input_proof)); let proving_assembly = ::synthesize_for_proving( circuit, @@ -122,7 +122,6 @@ pub trait CompressionStep: CompressionProofSystem { finalization_hint, &vk, ); - assert!(::verify(&proof, &vk)); proof @@ -143,7 +142,10 @@ pub trait CompressionStepExt: CompressionProofSystemExt + CompressionStep { { let input_vk = Self::load_previous_vk(blob_storage); let circuit = Self::build_circuit(input_vk, None); - let ctx = context_manager.init_context::(); + // Workaround: trace length is not known at this point, so thats totally fine + // to use a hardcoded trace length + let ctx_config = Self::get_context_config(); + let ctx = context_manager.init_compression_context::(ctx_config); let (finalization_hint, setup_assembly) = ::synthesize_for_setup(circuit); let (precomputation, vk) = diff --git a/crates/proof-compression/src/context.rs b/crates/proof-compression/src/context.rs index 2b6940b..1d084d0 100644 --- a/crates/proof-compression/src/context.rs +++ b/crates/proof-compression/src/context.rs @@ -1,36 +1,94 @@ use std::sync::atomic::AtomicBool; -use crate::{AsyncHandler, ProofSystemDefinition}; +use crate::{AsyncHandler, CompressionProofSystem, SnarkWrapperProofSystem}; pub trait ContextManagerInterface { - fn init_context

(&self) -> AsyncHandler + fn init_compression_context

(&self, config: P::ContextConfig) -> AsyncHandler where - P: ProofSystemDefinition; + P: CompressionProofSystem; + + fn initialize_snark_context_config(&self) -> AsyncHandler + where + S: SnarkWrapperProofSystem; + + fn init_snark_context( + &self, + config: AsyncHandler, + ) -> AsyncHandler + where + S: SnarkWrapperProofSystem; } -pub struct SimpleContextManager(std::sync::Arc); +pub struct SimpleContextManager { + context_status: std::sync::Arc, +} impl SimpleContextManager { pub fn new() -> Self { - SimpleContextManager(std::sync::Arc::new(AtomicBool::new(false))) + Self { + context_status: std::sync::Arc::new(AtomicBool::new(false)), + } } } + impl ContextManagerInterface for SimpleContextManager { - fn init_context

(&self) -> AsyncHandler + fn init_compression_context

(&self, config: P::ContextConfig) -> AsyncHandler where - P: ProofSystemDefinition, + P: CompressionProofSystem, { - assert!(self.0.load(std::sync::atomic::Ordering::Relaxed) == false); + assert!( + self.context_status + .load(std::sync::atomic::Ordering::Relaxed) + == false + ); // load next context - let flag = self.0.clone(); + let flag = self.context_status.clone(); let f = move || { let (sender, receiver) = std::sync::mpsc::channel(); - let config = P::get_context_config(); let context = P::init_context(config); - // mark status - flag.store(false, std::sync::atomic::Ordering::Relaxed); sender.send(context).unwrap(); + flag.store(false, std::sync::atomic::Ordering::Relaxed); + receiver + }; + + AsyncHandler::spawn(f) + } + + fn initialize_snark_context_config(&self) -> AsyncHandler + where + S: SnarkWrapperProofSystem, + { + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let start = std::time::Instant::now(); + let context_config = S::get_context_config(); + println!("CRS loading takes {}s", start.elapsed().as_secs()); + sender.send(context_config).unwrap(); + + receiver + }; + AsyncHandler::spawn(f) + } + fn init_snark_context( + &self, + config: AsyncHandler, + ) -> AsyncHandler + where + S: SnarkWrapperProofSystem, + { + assert!( + self.context_status + .load(std::sync::atomic::Ordering::Relaxed) + == false + ); + // load next context + let flag = self.context_status.clone(); + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let context = S::init_context(config.wait()); + sender.send(context).unwrap(); + flag.store(false, std::sync::atomic::Ordering::Relaxed); receiver }; diff --git a/crates/proof-compression/src/lib.rs b/crates/proof-compression/src/lib.rs index 1774f07..c409723 100644 --- a/crates/proof-compression/src/lib.rs +++ b/crates/proof-compression/src/lib.rs @@ -20,12 +20,12 @@ use proof_system::*; mod snark_wrapper; use snark_wrapper::*; -mod step; -use step::*; - mod serialization; use serialization::*; +mod utils; +use utils::*; + #[cfg(test)] mod test; diff --git a/crates/proof-compression/src/proof_system.rs b/crates/proof-compression/src/proof_system.rs index 3f90b39..b515115 100644 --- a/crates/proof-compression/src/proof_system.rs +++ b/crates/proof-compression/src/proof_system.rs @@ -1,4 +1,7 @@ +use std::alloc::Allocator; + use super::*; +use bellman::kate_commitment::{Crs, CrsForMonomialForm}; use bellman::pairing::compact_bn256::G1Affine as CompactG1Affine; use bellman::plonk::better_better_cs::cs::SynthesisModeGenerateSetup; use bellman::plonk::better_better_cs::{ @@ -9,6 +12,7 @@ use bellman::plonk::better_better_cs::{ }, }; use bellman::plonk::commitments::transcript::keccak_transcript::RollingKeccakTranscript; +use bellman::CurveAffine; use boojum::config::SetupCSConfig; use boojum::cs::implementations::transcript::Transcript; use boojum::cs::implementations::witness::WitnessVec; @@ -23,12 +27,14 @@ use boojum::{ }, field::goldilocks::GoldilocksExt2, }; + use circuit_definitions::circuit_definitions::aux_layer::compression::{ CompressionLayerCircuit, ProofCompressionFunction, }; use fflonk::bellman::plonk::better_better_cs::cs::Circuit; -use fflonk::{CombinedMonomialDeviceStorage, DeviceContextWithSingleDevice}; +use fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG; +use fflonk::{init_compact_crs, CombinedMonomialDeviceStorage, DeviceContextWithSingleDevice}; use gpu_prover::{DeviceMemoryManager, ManagerConfigs}; use shivini::gpu_proof_config::GpuProofConfig; use shivini::{cs::GpuSetup, GpuTreeHasher, ProverContext}; @@ -45,10 +51,6 @@ pub trait ProofSystemDefinition: Sized { type FinalizationHint: serde::Serialize + serde::de::DeserializeOwned + Clone; type Allocator: std::alloc::Allocator; type ProvingAssembly: Sized + Send + Sync + 'static; - type ContextConfig; - type Context: Send + Sync + 'static; - fn get_context_config() -> Self::ContextConfig; - fn init_context(config: Self::ContextConfig) -> Self::Context; fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData; fn verify(_: &Self::Proof, _: &Self::VK) -> bool; } @@ -60,8 +62,12 @@ pub trait CompressionProofSystem: ThisLayerPoW: GPUPoWRunner, > + ProofSystemDefinition { + type ContextConfig: Send + Sync + 'static; + type Context: Send + Sync + 'static; type AuxConfig; - + fn get_context_config() -> Self::ContextConfig; + fn get_context_config_from_hint(_: &Self::FinalizationHint) -> Self::ContextConfig; + fn init_context(config: Self::ContextConfig) -> Self::Context; fn aux_config_from_assembly(proving_assembly: &Self::ProvingAssembly) -> Self::AuxConfig; fn synthesize_for_proving( @@ -101,7 +107,13 @@ pub trait CompressionProofSystemExt: CompressionProofSystem { } pub trait SnarkWrapperProofSystem: ProofSystemDefinition { + type CRS; type Circuit; + type ContextConfig: Send + Sync + 'static; + type Context: Send + Sync + 'static; + fn get_context_config() -> Self::ContextConfig; + fn init_context(config: Self::ContextConfig) -> Self::Context; + fn load_crs(_: Self::FinalizationHint) -> Self::CRS; fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly; fn prove( _: AsyncHandler, @@ -128,6 +140,7 @@ pub trait SnarkWrapperProofSystemExt: SnarkWrapperProofSystem { _: Self::SetupAssembly, _: Self::FinalizationHint, ) -> (Self::Precomputation, Self::VK); + fn create_crs(_: Self::FinalizationHint); } type BoojumAssembly = @@ -153,21 +166,6 @@ where type FinalizationHint = FinalizationHintsForProver; type Allocator = std::alloc::Global; type ProvingAssembly = BoojumAssembly; - type ContextConfig = usize; // domain_size - type Context = ProverContext; - fn get_context_config() -> Self::ContextConfig { - // TODO - println!("Using hardcoded domain size 2^17 for compression step"); - 1 << 17 - } - - fn init_context(domain_size: Self::ContextConfig) -> Self::Context { - let config = - ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); - let context = Self::Context::create_with_config(config).expect("gpu prover context"); - - context - } fn verify(proof: &Self::Proof, vk: &Self::VK) -> bool { let verifier_builder = @@ -198,7 +196,24 @@ where > + 'static, { type AuxConfig = GpuProofConfig; + type ContextConfig = usize; // domain_size + type Context = ProverContext; + + fn get_context_config() -> Self::ContextConfig { + // println!("Using hardcoded domain size 2^17 for compression step"); + 1 << 17 + } + + fn init_context(domain_size: Self::ContextConfig) -> Self::Context { + let config = + ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); + let context = Self::Context::create_with_config(config).expect("gpu prover context"); + context + } + fn get_context_config_from_hint(hint: &Self::FinalizationHint) -> Self::ContextConfig { + hint.final_trace_len + } fn aux_config_from_assembly(proving_assembly: &Self::ProvingAssembly) -> Self::AuxConfig { GpuProofConfig::from_assembly(proving_assembly) } @@ -207,7 +222,24 @@ where circuit: CompressionLayerCircuit, finalization_hint: Self::FinalizationHint, ) -> Self::ProvingAssembly { - synthesize_circuit_for_proving(circuit, &finalization_hint) + let geometry = circuit.geometry(); + let (max_trace_len, num_vars) = circuit.size_hint(); + + let builder_impl = boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< + GoldilocksField, + GoldilocksField, + ProvingCSConfig, + >::new(geometry, max_trace_len.unwrap()); + let builder = boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); + + let builder = circuit.configure_builder_proxy(builder); + let mut cs = builder.build(num_vars.unwrap()); + circuit.add_tables(&mut cs); + circuit.synthesize_into_cs(&mut cs); + let _ = cs.pad_and_shrink_using_hint(&finalization_hint); + let cs = cs.into_assembly::(); + + cs } fn prove( @@ -311,7 +343,24 @@ where fn synthesize_for_setup( circuit: CompressionLayerCircuit, ) -> (Self::FinalizationHint, Self::SetupAssembly) { - synthesize_circuit_for_setup(circuit) + let geometry = circuit.geometry(); + let (max_trace_len, num_vars) = circuit.size_hint(); + + let builder_impl = boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< + GoldilocksField, + GoldilocksField, + SetupCSConfig, + >::new(geometry, max_trace_len.unwrap()); + let builder = boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); + + let builder = circuit.configure_builder_proxy(builder); + let mut cs = builder.build(num_vars.unwrap()); + circuit.add_tables(&mut cs); + circuit.synthesize_into_cs(&mut cs); + let (_domain_size, finalization_hint) = cs.pad_and_shrink(); + let cs = cs.into_assembly::(); + + (finalization_hint, cs) } } @@ -346,16 +395,6 @@ impl ProofSystemDefinition for PlonkSnarkWrapper { type FinalizationHint = usize; type Allocator = GlobalHost; type ProvingAssembly = PlonkAssembly; - type ContextConfig = (Vec, Vec); - type Context = UnsafePlonkProverDeviceMemoryManagerWrapper; - fn get_context_config() -> Self::ContextConfig { - todo!() - } - fn init_context(config: Self::ContextConfig) -> Self::Context { - let (device_ids, compact_crs) = config; - let manager = DeviceMemoryManager::init(&device_ids, &compact_crs[..]).unwrap(); - UnsafePlonkProverDeviceMemoryManagerWrapper(manager) - } fn take_witnesses( proving_assembly: &mut Self::ProvingAssembly, ) -> Vec { @@ -367,7 +406,16 @@ impl ProofSystemDefinition for PlonkSnarkWrapper { } impl SnarkWrapperProofSystem for PlonkSnarkWrapper { + type CRS = bellman::kate_commitment::Crs; type Circuit = PlonkSnarkVerifierCircuit; + type ContextConfig = (Vec, Vec); + type Context = UnsafePlonkProverDeviceMemoryManagerWrapper; + fn init_context(config: Self::ContextConfig) -> Self::Context { + let (device_ids, compact_crs) = config; + let manager = DeviceMemoryManager::init(&device_ids, &compact_crs[..]).unwrap(); + UnsafePlonkProverDeviceMemoryManagerWrapper(manager) + } + fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly { todo!() } @@ -391,6 +439,18 @@ impl SnarkWrapperProofSystem for PlonkSnarkWrapper { ) -> Self::Proof { todo!() } + + fn get_context_config() -> Self::ContextConfig { + todo!() + } + + fn load_crs(domain_size: Self::FinalizationHint) -> Self::CRS { + let raw_compact_crs_file_path = std::env::var("PLONK_COMPACT_RAW_CRS_FILE").unwrap(); + let raw_compact_crs_file = std::fs::File::open(raw_compact_crs_file_path).unwrap(); + let num_points = domain_size * L1_VERIFIER_DOMAIN_SIZE_LOG; + read_crs_from_raw_compact_form::<_, Self::Allocator>(raw_compact_crs_file, num_points) + .unwrap() + } } impl SnarkWrapperProofSystemExt for PlonkSnarkWrapper { @@ -407,13 +467,17 @@ impl SnarkWrapperProofSystemExt for PlonkSnarkWrapper { ) -> (Self::Precomputation, Self::VK) { todo!() } + + fn create_crs(_: Self::FinalizationHint) { + todo!() + } } type FflonkAssembly = Assembly; impl ProofSystemDefinition for FflonkSnarkWrapper { type FieldElement = Fr; - type Precomputation = FflonkSnarkVerifierCircuitDeviceSetupWrapper; + type Precomputation = FflonkSnarkVerifierCircuitDeviceSetupWrapper; type ExternalWitnessData = ( Vec, Vec, @@ -421,20 +485,9 @@ impl ProofSystemDefinition for FflonkSnarkWrapper { type Proof = FflonkSnarkVerifierCircuitProof; type VK = FflonkSnarkVerifierCircuitVK; type FinalizationHint = usize; - type Allocator = GlobalHost; + // type Allocator = GlobalHost; // TODO need global host with preallocated host memory + type Allocator = std::alloc::Global; type ProvingAssembly = FflonkAssembly; - type ContextConfig = usize; // domain_size - type Context = DeviceContextWithSingleDevice; - - fn get_context_config() -> Self::ContextConfig { - fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG - } - fn init_context(log_domain_size: Self::ContextConfig) -> Self::Context { - let domain_size = 1 << log_domain_size; - let context = Self::Context::init(domain_size).unwrap(); - - context - } fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { let input_assignments = std::mem::replace(&mut proving_assembly.input_assingments, Vec::new()); @@ -455,7 +508,33 @@ impl ProofSystemDefinition for FflonkSnarkWrapper { } impl SnarkWrapperProofSystem for FflonkSnarkWrapper { + type CRS = bellman::kate_commitment::Crs; type Circuit = FflonkSnarkVerifierCircuit; + type ContextConfig = (usize, Self::CRS); + type Context = DeviceContextWithSingleDevice; + + fn load_crs(domain_size: Self::FinalizationHint) -> Self::CRS { + // let raw_compact_crs_file_path = std::env::var("FFLONK_COMPACT_RAW_CRS_FILE").unwrap(); + // let raw_compact_crs_file = std::fs::File::open(raw_compact_crs_file_path).unwrap(); + // let num_points = domain_size * fflonk::MAX_COMBINED_DEGREE_FACTOR; + // read_crs_from_raw_compact_form::<_, Self::Allocator>(raw_compact_crs_file, num_points) + // .unwrap() + init_compact_crs(&bellman::worker::Worker::new(), domain_size) + } + + fn get_context_config() -> Self::ContextConfig { + let domain_size = 1 << fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG; + let crs = Self::load_crs(domain_size); + (domain_size, crs) + } + + fn init_context(config: Self::ContextConfig) -> Self::Context { + let (domain_size, crs) = config; + let context = Self::Context::init_from_preloaded_crs(domain_size, crs).unwrap(); + + context + } + fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly { let mut proving_assembly = FflonkAssembly::::new(); circuit @@ -469,14 +548,15 @@ impl SnarkWrapperProofSystem for FflonkSnarkWrapper { mut proving_assembly: Self::ProvingAssembly, precomputation: AsyncHandler, finalization_hint: Self::FinalizationHint, - vk: &Self::VK, + _vk: &Self::VK, ) -> Self::Proof { assert!(proving_assembly.is_satisfied()); let raw_trace_len = proving_assembly.n(); - proving_assembly.finalize_to_size_log_2(1 << finalization_hint); + assert!(finalization_hint.is_power_of_two()); + proving_assembly.finalize_to_size_log_2(finalization_hint.trailing_zeros() as usize); let domain_size = proving_assembly.n() + 1; assert!(domain_size.is_power_of_two()); - assert!(domain_size <= 1 << Self::get_context_config()); + assert_eq!(domain_size, finalization_hint); let ctx = ctx.wait(); let precomputation = precomputation.wait().into_inner(); @@ -521,11 +601,12 @@ impl SnarkWrapperProofSystemExt for FflonkSnarkWrapper { _finalization_hint: Self::FinalizationHint, ) -> (Self::Precomputation, Self::VK) { let ctx = ctx.wait(); - let device_setup = - FflonkSnarkVerifierCircuitDeviceSetup::create_setup_from_assembly_on_device( - &setup_assembly, - ) - .unwrap(); + let device_setup = fflonk::FflonkDeviceSetup::< + Bn256, + FflonkSnarkVerifierCircuit, + Self::Allocator, + >::create_setup_from_assembly_on_device(&setup_assembly) + .unwrap(); let vk = device_setup.get_verification_key(); drop(ctx); ( @@ -533,6 +614,18 @@ impl SnarkWrapperProofSystemExt for FflonkSnarkWrapper { vk, ) } + + fn create_crs(domain_size: Self::FinalizationHint) { + assert!(domain_size < fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG); + let num_points = fflonk::MAX_COMBINED_DEGREE_FACTOR * domain_size; + let original_crs = make_fflonk_crs_from_ignition_transcripts(num_points); + let raw_compact_crs_file_path = std::env::var("FFLONK_COMPACT_RAW_CRS_FILE").unwrap(); + assert!(!std::path::Path::exists(std::path::Path::new( + &raw_compact_crs_file_path + ))); + let raw_compact_crs_file = std::fs::File::create(raw_compact_crs_file_path).unwrap(); + write_crs_into_raw_compact_form(original_crs, raw_compact_crs_file, num_points).unwrap(); + } } pub struct MarkerProofSystem; @@ -545,70 +638,78 @@ impl ProofSystemDefinition for MarkerProofSystem { type FinalizationHint = (); type Allocator = std::alloc::Global; type ProvingAssembly = (); - type ContextConfig = (); - type Context = (); - fn get_context_config() -> Self::ContextConfig { - todo!() - } - fn init_context(config: Self::ContextConfig) -> Self::Context { - todo!() - } fn verify(_: &Self::Proof, _: &Self::VK) -> bool { - todo!() + unreachable!() } fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { - todo!() + unreachable!() } } -pub fn synthesize_circuit_for_setup( - circuit: CompressionLayerCircuit, -) -> ( - FinalizationHintsForProver, - CSReferenceAssembly, -) { - let geometry = circuit.geometry(); - let (max_trace_len, num_vars) = circuit.size_hint(); - - let builder_impl = boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< - GoldilocksField, - GoldilocksField, - SetupCSConfig, - >::new(geometry, max_trace_len.unwrap()); - let builder = boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); - - let builder = circuit.configure_builder_proxy(builder); - let mut cs = builder.build(num_vars.unwrap()); - circuit.add_tables(&mut cs); - circuit.synthesize_into_cs(&mut cs); - let (_domain_size, finalization_hint) = cs.pad_and_shrink(); - let cs = cs.into_assembly::(); - - (finalization_hint, cs) +pub fn write_crs_into_raw_compact_form( + original_crs: Crs, + mut dst_raw_compact_crs: W, + num_points: usize, +) -> std::io::Result<()> { + assert!(num_points <= original_crs.g1_bases.len()); + use bellman::{PrimeField, PrimeFieldRepr}; + use byteorder::{BigEndian, WriteBytesExt}; + assert!(num_points < u32::MAX as usize); + dst_raw_compact_crs.write_u32::(num_points as u32)?; + for g1_base in original_crs.g1_bases.iter() { + let (x, y) = g1_base.as_xy(); + x.into_raw_repr().write_be(&mut dst_raw_compact_crs)?; + y.into_raw_repr().write_be(&mut dst_raw_compact_crs)?; + } + for g2_base in original_crs.g2_monomial_bases.iter() { + let (x, y) = g2_base.as_xy(); + x.c0.into_raw_repr().write_be(&mut dst_raw_compact_crs)?; + x.c1.into_raw_repr().write_be(&mut dst_raw_compact_crs)?; + y.c0.into_raw_repr().write_be(&mut dst_raw_compact_crs)?; + y.c1.into_raw_repr().write_be(&mut dst_raw_compact_crs)?; + } + + Ok(()) } -pub fn synthesize_circuit_for_proving( - circuit: CompressionLayerCircuit, - finalization_hint: &FinalizationHintsForProver, -) -> CSReferenceAssembly { - let geometry = circuit.geometry(); - let (max_trace_len, num_vars) = circuit.size_hint(); - - let builder_impl = boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< - GoldilocksField, - GoldilocksField, - ProvingCSConfig, - >::new(geometry, max_trace_len.unwrap()); - let builder = boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); - - let builder = circuit.configure_builder_proxy(builder); - let mut cs = builder.build(num_vars.unwrap()); - circuit.add_tables(&mut cs); - circuit.synthesize_into_cs(&mut cs); - let _ = cs.pad_and_shrink_using_hint(&finalization_hint); - let cs = cs.into_assembly::(); - - cs +// TODO: Crs doesn't allow bases located in a custom allocator +pub fn read_crs_from_raw_compact_form( + mut src_raw_compact_crs: R, + num_points: usize, +) -> std::io::Result> { + println!("Reading Raw compact CRS"); + use byteorder::{BigEndian, ReadBytesExt}; + let actual_num_points = src_raw_compact_crs.read_u32::()? as usize; + assert!(num_points <= actual_num_points as usize); + use bellman::{PrimeField, PrimeFieldRepr}; + // let mut g1_bases = Vec::with_capacity_in(num_points, A::default()); + println!("Reading G1 points"); + let mut g1_bases = Vec::with_capacity(num_points); + unsafe { + g1_bases.set_len(num_points); + let buf = std::slice::from_raw_parts_mut( + g1_bases.as_mut_ptr() as *mut u8, + num_points * std::mem::size_of::(), + ); + src_raw_compact_crs.read_exact(buf)?; + } + let num_g2_points = 2; + // let mut g2_bases = Vec::with_capacity_in(num_g2_points, A::default()); + println!("Reading G2 points"); + let mut g2_bases = Vec::with_capacity(num_g2_points); + unsafe { + g2_bases.set_len(num_g2_points); + let buf = std::slice::from_raw_parts_mut( + g2_bases.as_mut_ptr() as *mut u8, + num_g2_points * std::mem::size_of::(), + ); + src_raw_compact_crs.read_exact(buf)?; + } + let mut compact_crs = Crs::<_, CrsForMonomialForm>::dummy_crs(1); + compact_crs.g1_bases = std::sync::Arc::new(g1_bases); + compact_crs.g2_monomial_bases = std::sync::Arc::new(g2_bases); + + Ok(compact_crs) } diff --git a/crates/proof-compression/src/serialization.rs b/crates/proof-compression/src/serialization.rs index 1dc4973..ad0cdcf 100644 --- a/crates/proof-compression/src/serialization.rs +++ b/crates/proof-compression/src/serialization.rs @@ -1,10 +1,30 @@ use super::*; -use fflonk::FflonkSnarkVerifierCircuitDeviceSetup; -use shivini::boojum::cs::implementations::fast_serialization::MemcopySerializable; +use gpu_prover::ManagerConfigs; +use shivini::{ + boojum::cs::implementations::fast_serialization::MemcopySerializable, cs::GpuSetup, + GpuTreeHasher, +}; use crate::PlonkSnarkVerifierCircuitDeviceSetup; +pub struct BoojumDeviceSetupWrapper(GpuSetup); + +impl boojum::cs::implementations::fast_serialization::MemcopySerializable + for BoojumDeviceSetupWrapper +{ + fn write_into_buffer( + &self, + dst: W, + ) -> Result<(), Box> { + Ok(bincode::serialize_into(dst, self.0).unwrap()) + } + + fn read_from_buffer(src: R) -> Result> { + Ok(Self(bincode::deserialize_from(src).unwrap())) + } +} + pub struct PlonkSnarkVerifierCircuitDeviceSetupWrapper(PlonkSnarkVerifierCircuitDeviceSetup); impl MemcopySerializable for PlonkSnarkVerifierCircuitDeviceSetupWrapper { @@ -12,11 +32,16 @@ impl MemcopySerializable for PlonkSnarkVerifierCircuitDeviceSetupWrapper { &self, dst: W, ) -> Result<(), Box> { - todo!() + self.0.write(dst).unwrap(); + Ok(()) } fn read_from_buffer(src: R) -> Result> { - todo!() + let mut precomputation = PlonkSnarkVerifierCircuitDeviceSetup::allocate( + 1 << PlonkProverDeviceMemoryManagerConfig::FULL_SLOT_SIZE_LOG, + ); + precomputation.read(src).unwrap(); + Ok(Self(precomputation)) } } @@ -26,9 +51,14 @@ impl PlonkSnarkVerifierCircuitDeviceSetupWrapper { } } -pub struct FflonkSnarkVerifierCircuitDeviceSetupWrapper(pub FflonkSnarkVerifierCircuitDeviceSetup); +pub struct FflonkSnarkVerifierCircuitDeviceSetupWrapper( + pub fflonk::FflonkDeviceSetup, +); -impl MemcopySerializable for FflonkSnarkVerifierCircuitDeviceSetupWrapper { +impl MemcopySerializable for FflonkSnarkVerifierCircuitDeviceSetupWrapper +where + A: HostAllocator, +{ fn write_into_buffer( &self, dst: W, @@ -37,14 +67,17 @@ impl MemcopySerializable for FflonkSnarkVerifierCircuitDeviceSetupWrapper { } fn read_from_buffer(src: R) -> Result> { - Ok(Self( - FflonkSnarkVerifierCircuitDeviceSetup::read(src).unwrap(), - )) + let precomputation = fflonk::FflonkDeviceSetup::::read(src).unwrap(); + + Ok(Self(precomputation)) } } -impl FflonkSnarkVerifierCircuitDeviceSetupWrapper { - pub fn into_inner(self) -> FflonkSnarkVerifierCircuitDeviceSetup { +impl FflonkSnarkVerifierCircuitDeviceSetupWrapper +where + A: HostAllocator, +{ + pub fn into_inner(self) -> fflonk::FflonkDeviceSetup { self.0 } } diff --git a/crates/proof-compression/src/snark_wrapper.rs b/crates/proof-compression/src/snark_wrapper.rs index 89a67f7..f8e2d23 100644 --- a/crates/proof-compression/src/snark_wrapper.rs +++ b/crates/proof-compression/src/snark_wrapper.rs @@ -25,8 +25,12 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { BS: BlobStorage, { assert!(Self::IS_FFLONK ^ Self::IS_PLONK); - let hint = if Self::IS_PLONK { &[26u8] } else { &[24] }; - serde_json::from_reader(&hint[..]).unwrap() + let hint = if Self::IS_PLONK { + (1 << ::FULL_SLOT_SIZE_LOG).to_string() + } else { + (1 << fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG).to_string() + }; + serde_json::from_str(&hint).unwrap() } fn load_previous_vk( @@ -85,6 +89,7 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { } fn prove_snark_wrapper_step( + ctx_config: AsyncHandler, input_proof: Proof, blob_storage: &BS, context_handler: &CI, @@ -95,12 +100,13 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { { assert!(Self::IS_FFLONK ^ Self::IS_PLONK); let input_vk = Self::load_previous_vk(blob_storage); - let vk = Self::load_this_vk(blob_storage); + let precomputation = Self::get_precomputation(blob_storage); - let ctx = context_handler.init_context::(); + let ctx = context_handler.init_snark_context::(ctx_config); let finalization_hint = Self::load_finalization_hint(blob_storage); let circuit = Self::build_circuit(input_vk, Some(input_proof)); let proving_assembly = ::synthesize_for_proving(circuit); + let vk = Self::load_this_vk(blob_storage); let proof = ::prove( ctx, proving_assembly, @@ -109,7 +115,7 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { &vk, ); - assert!(::verify(&proof, &vk)); + // assert!(::verify(&proof, &vk)); proof } @@ -121,8 +127,11 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { } pub trait SnarkWrapperStepExt: SnarkWrapperProofSystemExt + SnarkWrapperStep { - fn precompute_and_store_snark_wrapper_circuit(blob_storage: &BS, context_manager: &CM) - where + fn precompute_and_store_snark_wrapper_circuit( + ctx_config: AsyncHandler, + blob_storage: &BS, + context_manager: &CM, + ) where BS: BlobStorageExt, CM: ContextManagerInterface, ::VK: 'static, @@ -130,7 +139,7 @@ pub trait SnarkWrapperStepExt: SnarkWrapperProofSystemExt + SnarkWrapperStep { let input_vk = Self::load_previous_vk(blob_storage); let finalization_hint = Self::load_finalization_hint(blob_storage); let circuit = Self::build_circuit(input_vk, None); - let ctx = context_manager.init_context::(); + let ctx = context_manager.init_snark_context::(ctx_config); let setup_assembly = ::synthesize_for_setup(circuit); let (precomputation, vk) = diff --git a/crates/proof-compression/src/step.rs b/crates/proof-compression/src/step.rs deleted file mode 100644 index 8b13789..0000000 --- a/crates/proof-compression/src/step.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/proof-compression/src/test.rs b/crates/proof-compression/src/test.rs index 31aa26e..550884f 100644 --- a/crates/proof-compression/src/test.rs +++ b/crates/proof-compression/src/test.rs @@ -2,6 +2,7 @@ use circuit_definitions::circuit_definitions::recursion_layer::ZkSyncRecursionLa use super::*; +#[test] pub fn test_proof_chain_with_fflonk() { let scheduler_proof_file = std::fs::File::open("./data/scheduler_recursive_proof.json").unwrap(); diff --git a/crates/proof-compression/src/utils.rs b/crates/proof-compression/src/utils.rs index c16945d..5ad0a33 100644 --- a/crates/proof-compression/src/utils.rs +++ b/crates/proof-compression/src/utils.rs @@ -1,6 +1,11 @@ use super::*; +use bellman::{ + kate_commitment::{Crs, CrsForMonomialForm}, + CurveAffine, Engine, Field, PrimeField, +}; +use byteorder::{BigEndian, ReadBytesExt}; -pub fn make_crs_from_ignition_transcript + ?Sized>( +pub fn create_crs_from_ignition_transcript + ?Sized>( path: &S, num_chunks: usize, ) -> Result< @@ -169,10 +174,13 @@ pub fn make_crs_from_ignition_transcript + ?Sized>( Ok(new) } -pub fn transform_ignition_transcripts(domain_size: usize) { - let transcripts_dir = std::env::var("IGNITION_TRANSCRIPT_PATH").unwrap_or("./".to_string()); +pub fn make_fflonk_crs_from_ignition_transcripts( + num_points: usize, +) -> Crs { + let transcripts_dir = + std::env::var("IGNITION_TRANSCRIPT_PATH").expect("IGNITION_TRANSCRIPT_PATH env variable"); let chunk_size = 5_040_000usize; - let num_chunks = domain_size.div_ceil(chunk_size); + let num_chunks = num_points.div_ceil(chunk_size); // Check transcript files already downloaded from "https://aztec-ignition.s3.eu-west-2.amazonaws.com/MAIN+IGNITION/sealed/transcript{idx}.dat"; for idx in 0..num_chunks { @@ -182,21 +190,19 @@ pub fn transform_ignition_transcripts(domain_size: usize) { } // transform - let crs = make_crs_from_ignition_transcript(&transcripts_dir, num_chunks).unwrap(); + let crs = create_crs_from_ignition_transcript(&transcripts_dir, num_chunks).unwrap(); let out_path = format!("{}/full_ignition.key", &transcripts_dir); let out_file = std::fs::File::create(&out_path).unwrap(); - let Crs { + let bellman::kate_commitment::Crs { g1_bases, g2_monomial_bases, .. } = crs; - assert!(g1_bases.len() >= domain_size); + assert!(g1_bases.len() >= num_points); let mut g1_bases = std::sync::Arc::try_unwrap(g1_bases).unwrap(); let g2_monomial_bases = std::sync::Arc::try_unwrap(g2_monomial_bases).unwrap(); - g1_bases.truncate(domain_size); + g1_bases.truncate(num_points); - let crs: Crs = Crs::new(g1_bases, g2_monomial_bases); - crs.write(&out_file).unwrap(); - println!("full ignition ceremony saved into {out_path}"); + Crs::new(g1_bases, g2_monomial_bases) } From fc9944761c3c79fa929cc3312a735908c64fc312 Mon Sep 17 00:00:00 2001 From: saitima Date: Sat, 11 Jan 2025 12:51:02 +0300 Subject: [PATCH 06/18] feat(proof-compression): faster CRS loading with raw values --- crates/proof-compression/src/proof_system.rs | 77 +++++++++---------- crates/proof-compression/src/serialization.rs | 39 ++++++++-- 2 files changed, 71 insertions(+), 45 deletions(-) diff --git a/crates/proof-compression/src/proof_system.rs b/crates/proof-compression/src/proof_system.rs index b515115..9adeae0 100644 --- a/crates/proof-compression/src/proof_system.rs +++ b/crates/proof-compression/src/proof_system.rs @@ -34,14 +34,14 @@ use circuit_definitions::circuit_definitions::aux_layer::compression::{ use fflonk::bellman::plonk::better_better_cs::cs::Circuit; use fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG; -use fflonk::{init_compact_crs, CombinedMonomialDeviceStorage, DeviceContextWithSingleDevice}; +use fflonk::{CombinedMonomialDeviceStorage, DeviceContextWithSingleDevice}; use gpu_prover::{DeviceMemoryManager, ManagerConfigs}; use shivini::gpu_proof_config::GpuProofConfig; -use shivini::{cs::GpuSetup, GpuTreeHasher, ProverContext}; use shivini::{ CacheStrategy, CommitmentCacheStrategy, GPUPoWRunner, PolynomialsCacheStrategy, ProverContextConfig, }; +use shivini::{GpuTreeHasher, ProverContext}; pub trait ProofSystemDefinition: Sized { type FieldElement; type ExternalWitnessData; @@ -160,7 +160,7 @@ where { type FieldElement = GoldilocksField; type ExternalWitnessData = WitnessVec; - type Precomputation = GpuSetup; + type Precomputation = BoojumDeviceSetupWrapper; type Proof = Proof; type VK = VerificationKey; type FinalizationHint = FinalizationHintsForProver; @@ -277,7 +277,7 @@ where commitment: CommitmentCacheStrategy::CacheCosetCaps, }; let worker = Worker::new(); - let precomputation = precomputation.wait(); + let precomputation = precomputation.wait().into_inner(); let ctx = ctx.wait(); let gpu_proof = shivini::gpu_prove_from_external_witness_data_with_cache_strategy::< CF::ThisLayerTranscript, @@ -338,7 +338,7 @@ where >(setup_base, vk_params, vars_hint, wits_hint, &worker) .unwrap(); drop(ctx); - (precomputation, vk) + (BoojumDeviceSetupWrapper::from_inner(precomputation), vk) } fn synthesize_for_setup( circuit: CompressionLayerCircuit, @@ -406,7 +406,11 @@ impl ProofSystemDefinition for PlonkSnarkWrapper { } impl SnarkWrapperProofSystem for PlonkSnarkWrapper { - type CRS = bellman::kate_commitment::Crs; + type CRS = bellman::kate_commitment::Crs< + bellman::compact_bn256::Bn256, + CrsForMonomialForm, + Self::Allocator, + >; type Circuit = PlonkSnarkVerifierCircuit; type ContextConfig = (Vec, Vec); type Context = UnsafePlonkProverDeviceMemoryManagerWrapper; @@ -508,18 +512,21 @@ impl ProofSystemDefinition for FflonkSnarkWrapper { } impl SnarkWrapperProofSystem for FflonkSnarkWrapper { - type CRS = bellman::kate_commitment::Crs; + type CRS = bellman::kate_commitment::Crs< + bellman::compact_bn256::Bn256, + CrsForMonomialForm, + Self::Allocator, + >; type Circuit = FflonkSnarkVerifierCircuit; type ContextConfig = (usize, Self::CRS); type Context = DeviceContextWithSingleDevice; fn load_crs(domain_size: Self::FinalizationHint) -> Self::CRS { - // let raw_compact_crs_file_path = std::env::var("FFLONK_COMPACT_RAW_CRS_FILE").unwrap(); - // let raw_compact_crs_file = std::fs::File::open(raw_compact_crs_file_path).unwrap(); - // let num_points = domain_size * fflonk::MAX_COMBINED_DEGREE_FACTOR; - // read_crs_from_raw_compact_form::<_, Self::Allocator>(raw_compact_crs_file, num_points) - // .unwrap() - init_compact_crs(&bellman::worker::Worker::new(), domain_size) + let raw_compact_crs_file_path = std::env::var("FFLONK_COMPACT_RAW_CRS_FILE").unwrap(); + let raw_compact_crs_file = std::fs::File::open(raw_compact_crs_file_path).unwrap(); + let num_points = domain_size * fflonk::MAX_COMBINED_DEGREE_FACTOR; + read_crs_from_raw_compact_form::<_, Self::Allocator>(raw_compact_crs_file, num_points) + .unwrap() } fn get_context_config() -> Self::ContextConfig { @@ -531,7 +538,6 @@ impl SnarkWrapperProofSystem for FflonkSnarkWrapper { fn init_context(config: Self::ContextConfig) -> Self::Context { let (domain_size, crs) = config; let context = Self::Context::init_from_preloaded_crs(domain_size, crs).unwrap(); - context } @@ -624,7 +630,7 @@ impl SnarkWrapperProofSystemExt for FflonkSnarkWrapper { &raw_compact_crs_file_path ))); let raw_compact_crs_file = std::fs::File::create(raw_compact_crs_file_path).unwrap(); - write_crs_into_raw_compact_form(original_crs, raw_compact_crs_file, num_points).unwrap(); + write_crs_into_raw_compact_form(&original_crs, raw_compact_crs_file, num_points).unwrap(); } } @@ -649,7 +655,7 @@ impl ProofSystemDefinition for MarkerProofSystem { } pub fn write_crs_into_raw_compact_form( - original_crs: Crs, + original_crs: &Crs, mut dst_raw_compact_crs: W, num_points: usize, ) -> std::io::Result<()> { @@ -660,15 +666,16 @@ pub fn write_crs_into_raw_compact_form( dst_raw_compact_crs.write_u32::(num_points as u32)?; for g1_base in original_crs.g1_bases.iter() { let (x, y) = g1_base.as_xy(); - x.into_raw_repr().write_be(&mut dst_raw_compact_crs)?; - y.into_raw_repr().write_be(&mut dst_raw_compact_crs)?; + x.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + y.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; } + assert_eq!(original_crs.g2_monomial_bases.len(), 2); for g2_base in original_crs.g2_monomial_bases.iter() { let (x, y) = g2_base.as_xy(); - x.c0.into_raw_repr().write_be(&mut dst_raw_compact_crs)?; - x.c1.into_raw_repr().write_be(&mut dst_raw_compact_crs)?; - y.c0.into_raw_repr().write_be(&mut dst_raw_compact_crs)?; - y.c1.into_raw_repr().write_be(&mut dst_raw_compact_crs)?; + x.c0.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + x.c1.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + y.c0.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + y.c1.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; } Ok(()) @@ -677,28 +684,22 @@ pub fn write_crs_into_raw_compact_form( // TODO: Crs doesn't allow bases located in a custom allocator pub fn read_crs_from_raw_compact_form( mut src_raw_compact_crs: R, - num_points: usize, -) -> std::io::Result> { - println!("Reading Raw compact CRS"); + num_g1_points: usize, +) -> std::io::Result> { use byteorder::{BigEndian, ReadBytesExt}; let actual_num_points = src_raw_compact_crs.read_u32::()? as usize; - assert!(num_points <= actual_num_points as usize); - use bellman::{PrimeField, PrimeFieldRepr}; - // let mut g1_bases = Vec::with_capacity_in(num_points, A::default()); - println!("Reading G1 points"); - let mut g1_bases = Vec::with_capacity(num_points); + assert!(num_g1_points <= actual_num_points as usize); + let mut g1_bases = Vec::with_capacity_in(num_g1_points, A::default()); unsafe { - g1_bases.set_len(num_points); + g1_bases.set_len(num_g1_points); let buf = std::slice::from_raw_parts_mut( g1_bases.as_mut_ptr() as *mut u8, - num_points * std::mem::size_of::(), + num_g1_points * std::mem::size_of::(), ); src_raw_compact_crs.read_exact(buf)?; } let num_g2_points = 2; - // let mut g2_bases = Vec::with_capacity_in(num_g2_points, A::default()); - println!("Reading G2 points"); - let mut g2_bases = Vec::with_capacity(num_g2_points); + let mut g2_bases = Vec::with_capacity_in(num_g2_points, A::default()); unsafe { g2_bases.set_len(num_g2_points); let buf = std::slice::from_raw_parts_mut( @@ -707,9 +708,5 @@ pub fn read_crs_from_raw_compact_form( ); src_raw_compact_crs.read_exact(buf)?; } - let mut compact_crs = Crs::<_, CrsForMonomialForm>::dummy_crs(1); - compact_crs.g1_bases = std::sync::Arc::new(g1_bases); - compact_crs.g2_monomial_bases = std::sync::Arc::new(g2_bases); - - Ok(compact_crs) + Ok(Crs::<_, CrsForMonomialForm, A>::new_in(g1_bases, g2_bases)) } diff --git a/crates/proof-compression/src/serialization.rs b/crates/proof-compression/src/serialization.rs index ad0cdcf..19f6579 100644 --- a/crates/proof-compression/src/serialization.rs +++ b/crates/proof-compression/src/serialization.rs @@ -6,9 +6,28 @@ use shivini::{ GpuTreeHasher, }; +pub trait SerializationWrapper: Sized { + type Inner; + fn into_inner(self) -> Self::Inner; + fn from_inner(inner: Self::Inner) -> Self; +} + use crate::PlonkSnarkVerifierCircuitDeviceSetup; pub struct BoojumDeviceSetupWrapper(GpuSetup); +impl SerializationWrapper for BoojumDeviceSetupWrapper +where + H: GpuTreeHasher, +{ + type Inner = GpuSetup; + fn into_inner(self) -> Self::Inner { + self.0 + } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } +} impl boojum::cs::implementations::fast_serialization::MemcopySerializable for BoojumDeviceSetupWrapper @@ -17,7 +36,7 @@ impl boojum::cs::implementations::fast_serialization::MemcopyS &self, dst: W, ) -> Result<(), Box> { - Ok(bincode::serialize_into(dst, self.0).unwrap()) + Ok(bincode::serialize_into(dst, &self.0).unwrap()) } fn read_from_buffer(src: R) -> Result> { @@ -45,10 +64,15 @@ impl MemcopySerializable for PlonkSnarkVerifierCircuitDeviceSetupWrapper { } } -impl PlonkSnarkVerifierCircuitDeviceSetupWrapper { - pub fn into_inner(self) -> PlonkSnarkVerifierCircuitDeviceSetup { +impl SerializationWrapper for PlonkSnarkVerifierCircuitDeviceSetupWrapper { + type Inner = PlonkSnarkVerifierCircuitDeviceSetup; + fn into_inner(self) -> Self::Inner { self.0 } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } } pub struct FflonkSnarkVerifierCircuitDeviceSetupWrapper( @@ -73,13 +97,18 @@ where } } -impl FflonkSnarkVerifierCircuitDeviceSetupWrapper +impl SerializationWrapper for FflonkSnarkVerifierCircuitDeviceSetupWrapper where A: HostAllocator, { - pub fn into_inner(self) -> fflonk::FflonkDeviceSetup { + type Inner = fflonk::FflonkDeviceSetup; + fn into_inner(self) -> Self::Inner { self.0 } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } } #[derive(serde::Serialize, serde::Deserialize)] From 64923f38e053d33ee65f9aadeb5152e5ad644c2f Mon Sep 17 00:00:00 2001 From: saitima Date: Sat, 11 Jan 2025 22:33:32 +0300 Subject: [PATCH 07/18] chore(proof-compressio): cleanup --- crates/proof-compression/Cargo.toml | 9 +- .../scripts/download-ignition-transcripts.sh | 29 + .../{ => scripts}/setup-instance.sh | 0 crates/proof-compression/src/blob_storage.rs | 56 +- crates/proof-compression/src/chain.rs | 60 +- crates/proof-compression/src/common.rs | 358 --------- crates/proof-compression/src/context.rs | 39 +- crates/proof-compression/src/lib.rs | 69 +- crates/proof-compression/src/proof_system.rs | 712 ------------------ .../src/proof_system/boojum.rs | 247 ++++++ .../src/{utils.rs => proof_system/crs.rs} | 143 +++- .../src/proof_system/fflonk.rs | 159 ++++ .../src/proof_system/interface.rs | 129 ++++ .../proof-compression/src/proof_system/mod.rs | 18 + .../src/proof_system/plonk.rs | 237 ++++++ crates/proof-compression/src/serialization.rs | 23 +- .../src/{ => step}/compression.rs | 20 +- crates/proof-compression/src/step/mod.rs | 7 + .../src/{ => step}/snark_wrapper.rs | 59 +- crates/proof-compression/src/test.rs | 29 +- 20 files changed, 1088 insertions(+), 1315 deletions(-) create mode 100644 crates/proof-compression/scripts/download-ignition-transcripts.sh rename crates/proof-compression/{ => scripts}/setup-instance.sh (100%) delete mode 100644 crates/proof-compression/src/common.rs delete mode 100644 crates/proof-compression/src/proof_system.rs create mode 100644 crates/proof-compression/src/proof_system/boojum.rs rename crates/proof-compression/src/{utils.rs => proof_system/crs.rs} (66%) create mode 100644 crates/proof-compression/src/proof_system/fflonk.rs create mode 100644 crates/proof-compression/src/proof_system/interface.rs create mode 100644 crates/proof-compression/src/proof_system/mod.rs create mode 100644 crates/proof-compression/src/proof_system/plonk.rs rename crates/proof-compression/src/{ => step}/compression.rs (95%) create mode 100644 crates/proof-compression/src/step/mod.rs rename crates/proof-compression/src/{ => step}/snark_wrapper.rs (81%) diff --git a/crates/proof-compression/Cargo.toml b/crates/proof-compression/Cargo.toml index 22e8a33..4b3c1a0 100644 --- a/crates/proof-compression/Cargo.toml +++ b/crates/proof-compression/Cargo.toml @@ -13,9 +13,9 @@ exclude = ["/data"] [dependencies] circuit_definitions.workspace = true -fflonk.workspace = true +fflonk = {workspace = true, optional = true} shivini.workspace = true -gpu-prover.workspace = true +gpu-prover = { workspace = true, optional = true} serde = "1" serde_json = "1" bincode = "1.3" @@ -23,8 +23,7 @@ cfg-if = "1.0" byteorder = "1" [features] -default = ["gpu"] +default = ["gpu", "fflonk", "gpu-prover"] gpu = [] cpu = [] - -#TODO: define allocator +allocator = ["fflonk/allocator", "gpu-prover/allocator"] diff --git a/crates/proof-compression/scripts/download-ignition-transcripts.sh b/crates/proof-compression/scripts/download-ignition-transcripts.sh new file mode 100644 index 0000000..842d1ab --- /dev/null +++ b/crates/proof-compression/scripts/download-ignition-transcripts.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Configuration: adjust these variables as needed +BASE_URL="https://aztec-ignition.s3.eu-west-2.amazonaws.com/MAIN+IGNITION/sealed/transcript" # Base URL before the index +FILE_EXT=".dat" # File extension (if any) +START_INDEX=0 # Starting index +END_INDEX=15 # Ending index + +# Loop over the desired range of indexes +for ((i=START_INDEX; i<=END_INDEX; i++)); do + # Format the index as two digits (e.g., 01, 02, ..., 10) + FORMATTED_INDEX=$(printf "%02d" "$i") + + # Construct the full URL for the current file using the formatted index + FILE_URL="${BASE_URL}${FORMATTED_INDEX}${FILE_EXT}" + + echo "Downloading ${FILE_URL} ..." + + # Download the file using wget + wget "$FILE_URL" + + # Check if the download succeeded; if not, optionally handle errors + if [[ $? -ne 0 ]]; then + echo "Failed to download ${FILE_URL}" + # Optionally break or continue based on your needs + fi +done + +echo "Download complete." \ No newline at end of file diff --git a/crates/proof-compression/setup-instance.sh b/crates/proof-compression/scripts/setup-instance.sh similarity index 100% rename from crates/proof-compression/setup-instance.sh rename to crates/proof-compression/scripts/setup-instance.sh diff --git a/crates/proof-compression/src/blob_storage.rs b/crates/proof-compression/src/blob_storage.rs index fb2bf3f..ca791f7 100644 --- a/crates/proof-compression/src/blob_storage.rs +++ b/crates/proof-compression/src/blob_storage.rs @@ -15,11 +15,10 @@ pub trait BlobStorage: Send + Sync { fn read_fflonk_vk(&self) -> Box; fn read_fflonk_precomputation(&self) -> Box; - fn read_fflonk_crs(&self) -> Box; fn read_plonk_vk(&self) -> Box; fn read_plonk_precomputation(&self) -> Box; - fn read_plonk_crs(&self) -> Box; + fn read_compact_raw_crs(&self) -> Box; } pub trait BlobStorageExt: BlobStorage { @@ -33,14 +32,14 @@ pub trait BlobStorageExt: BlobStorage { fn write_fflonk_vk(&self) -> Box; fn write_fflonk_precomputation(&self) -> Box; - fn write_fflonk_crs(&self) -> Box; fn write_plonk_vk(&self) -> Box; fn write_plonk_precomputation(&self) -> Box; - fn write_plonk_crs(&self) -> Box; + + fn write_compact_raw_crs(&self) -> Box; } -pub struct FileSystemBlobStorage; +pub(crate) struct FileSystemBlobStorage; impl FileSystemBlobStorage { const DATA_DIR_PATH: &str = "./data"; @@ -163,24 +162,9 @@ impl BlobStorage for FileSystemBlobStorage { println!("Reading plonk vk at path {}", path); Self::open_file(&path) } - - fn read_fflonk_crs(&self) -> Box { - let path = format!( - "{}/{}_compact_crs.key.raw", - Self::DATA_DIR_PATH, - Self::FFLONK_PREFIX - ); - println!("Reading fflonk CRS at path {}", path); - Self::open_file(&path) - } - - fn read_plonk_crs(&self) -> Box { - let path = format!( - "{}/{}_compact_crs.key.raw", - Self::DATA_DIR_PATH, - Self::PLONK_PREFIX - ); - println!("Reading fflonk CRS at path {}", path); + fn read_compact_raw_crs(&self) -> Box { + let path = format!("{}/compact_raw_crs.key", Self::DATA_DIR_PATH,); + println!("Reading CRS at path {}", path); Self::open_file(&path) } } @@ -279,28 +263,14 @@ impl BlobStorageExt for FileSystemBlobStorage { Self::create_file(&path) } - fn write_fflonk_crs(&self) -> Box { - let path = format!( - "{}/{}_compact_crs.key.raw", - Self::DATA_DIR_PATH, - Self::PLONK_PREFIX - ); - println!("Writeing fflonk CRS at path {}", path); - Self::create_file(&path) - } - - fn write_plonk_crs(&self) -> Box { - let path = format!( - "{}/{}_compact_crs.key.raw", - Self::DATA_DIR_PATH, - Self::PLONK_PREFIX - ); - println!("Writeing plonk CRS at path {}", path); + fn write_compact_raw_crs(&self) -> Box { + let path = format!("{}/compact_raw_crs.key", Self::DATA_DIR_PATH); + println!("Writeing compact raw CRS at path {}", path); Self::create_file(&path) } } -pub struct AsyncHandler { +pub(crate) struct AsyncHandler { receiver: std::thread::JoinHandle>, } @@ -308,7 +278,7 @@ impl AsyncHandler where T: Send + Sync + 'static, { - pub fn spawn(f: F) -> Self + pub(crate) fn spawn(f: F) -> Self where F: FnOnce() -> std::sync::mpsc::Receiver + Send + Sync + 'static, { @@ -317,7 +287,7 @@ where Self { receiver } } - pub fn wait(self) -> T { + pub(crate) fn wait(self) -> T { self.receiver.join().unwrap().recv().unwrap() } } diff --git a/crates/proof-compression/src/chain.rs b/crates/proof-compression/src/chain.rs index d264718..3a60aa5 100644 --- a/crates/proof-compression/src/chain.rs +++ b/crates/proof-compression/src/chain.rs @@ -1,3 +1,8 @@ +use circuit_definitions::circuit_definitions::aux_layer::compression_modes::{ + CompressionMode1, CompressionMode1ForWrapper, CompressionMode2, CompressionMode3, + CompressionMode4, CompressionMode5ForWrapper, +}; + use super::*; pub enum SnarkWrapper { @@ -9,6 +14,12 @@ pub enum SnarkWrapperProof { FFfonk(FflonkSnarkVerifierCircuitProof), } +pub type SchedulerProof = franklin_crypto::boojum::cs::implementations::proof::Proof< + GoldilocksField, + circuit_definitions::circuit_definitions::recursion_layer::RecursiveProofsTreeHasher, + GoldilocksExt2, +>; + pub fn run_proof_chain( input_proof: SchedulerProof, snark_wrapper: SnarkWrapper, @@ -29,7 +40,7 @@ where } } -pub(crate) fn run_proof_chain_with_fflonk( +pub fn run_proof_chain_with_fflonk( input_proof: SchedulerProof, blob_storage: &BS, ) -> FflonkSnarkVerifierCircuitProof @@ -38,8 +49,10 @@ where { let context_manager = SimpleContextManager::new(); let start = std::time::Instant::now(); - let snark_context_config = - context_manager.initialize_snark_context_config::(); + let compact_raw_crs = + ::load_compact_raw_crs(blob_storage); + let fflonk_precomputation = FflonkSnarkWrapper::get_precomputation(blob_storage); + let next_proof = CompressionMode1::prove_compression_step(input_proof, blob_storage, &context_manager); let next_proof = CompressionMode2::prove_compression_step::<_, SimpleContextManager>( @@ -67,7 +80,8 @@ where start.elapsed().as_secs() ); let final_proof = FflonkSnarkWrapper::prove_snark_wrapper_step::<_, SimpleContextManager>( - snark_context_config, + compact_raw_crs, + fflonk_precomputation, next_proof, blob_storage, &context_manager, @@ -79,13 +93,14 @@ where final_proof } -pub(crate) fn precompute_proof_chain_with_fflonk(blob_storage: &BS) +pub fn precompute_proof_chain_with_fflonk(blob_storage: &BS) where BS: BlobStorageExt, { let context_manager = SimpleContextManager::new(); - let snark_context_config = - context_manager.initialize_snark_context_config::(); + let compact_raw_crs = + ::load_compact_raw_crs(blob_storage); + let start = std::time::Instant::now(); CompressionMode1::precomputae_and_store_compression_circuits(blob_storage, &context_manager); CompressionMode2::precomputae_and_store_compression_circuits(blob_storage, &context_manager); @@ -100,17 +115,17 @@ where start.elapsed().as_secs() ); FflonkSnarkWrapper::precompute_and_store_snark_wrapper_circuit( - snark_context_config, + compact_raw_crs, blob_storage, &context_manager, ); println!( - "Precomputation of entire chain took {}s", + "Precomputation of entire chain with fflonk took {}s", start.elapsed().as_secs() ); } -pub(crate) fn run_proof_chain_with_plonk( +pub fn run_proof_chain_with_plonk( input_proof: SchedulerProof, blob_storage: &BS, ) -> PlonkSnarkVerifierCircuitProof @@ -118,9 +133,11 @@ where BS: BlobStorage, { let context_manager = SimpleContextManager::new(); - let snark_context_config = - context_manager.initialize_snark_context_config::(); let start = std::time::Instant::now(); + let compact_raw_crs = + ::load_compact_raw_crs(blob_storage); + let plonk_precomputation = PlonkSnarkWrapper::get_precomputation(blob_storage); + let next_proof = CompressionMode1ForWrapper::prove_compression_step( input_proof, blob_storage, @@ -128,33 +145,38 @@ where ); let final_proof = PlonkSnarkWrapper::prove_snark_wrapper_step( - snark_context_config, + compact_raw_crs, + plonk_precomputation, next_proof, blob_storage, &context_manager, ); println!( - "Entire compression chain took {}s", + "Entire compression chain with plonk took {}s", start.elapsed().as_secs() ); final_proof } -pub(crate) fn precompute_proof_chain_with_plonk(blob_storage: &BS) +pub fn precompute_proof_chain_with_plonk(blob_storage: &BS) where BS: BlobStorageExt, { let context_manager = SimpleContextManager::new(); - let snark_context_config = - context_manager.initialize_snark_context_config::(); + let start = std::time::Instant::now(); + let compact_raw_crs = + ::load_compact_raw_crs(blob_storage); CompressionMode1ForWrapper::precomputae_and_store_compression_circuits( blob_storage, &context_manager, ); PlonkSnarkWrapper::precompute_and_store_snark_wrapper_circuit( - snark_context_config, + compact_raw_crs, blob_storage, &context_manager, ); - println!("All steps in this approach precomputed and saved into blob storage"); + println!( + "Precomputation of entire chain with fflonk took {}s", + start.elapsed().as_secs() + ); } diff --git a/crates/proof-compression/src/common.rs b/crates/proof-compression/src/common.rs deleted file mode 100644 index 68c780a..0000000 --- a/crates/proof-compression/src/common.rs +++ /dev/null @@ -1,358 +0,0 @@ -use super::*; -use circuit_definitions::circuit_definitions::{ - aux_layer::{ - CompressionProofsTreeHasher, CompressionProofsTreeHasherForWrapper, - ZkSyncCompressionForWrapperCircuit, ZkSyncCompressionLayerCircuit, ZkSyncCompressionProof, - ZkSyncCompressionProofForWrapper, ZkSyncCompressionVerificationKey, - ZkSyncCompressionVerificationKeyForWrapper, - }, - recursion_layer::{ - ZkSyncRecursionLayerProof, ZkSyncRecursionLayerVerificationKey, ZkSyncRecursionProof, - ZkSyncRecursionVerificationKey, - }, -}; -use fflonk::{ - FflonkSnarkVerifierCircuitProof, FflonkSnarkVerifierCircuitSetup, FflonkSnarkVerifierCircuitVK, -}; -use shivini::boojum::{ - algebraic_props::{round_function::AbsorptionModeOverwrite, sponge::GoldilocksPoseidon2Sponge}, - config::{CSConfig, DevCSConfig, ProvingCSConfig, SetupCSConfig}, - cs::{ - cs_builder::new_builder, - cs_builder_reference::CsReferenceImplementationBuilder, - implementations::{ - proof::Proof, reference_cs::CSReferenceAssembly, setup::FinalizationHintsForProver, - transcript::GoldilocksPoisedon2Transcript, verifier::Verifier, - }, - }, - field::goldilocks::{GoldilocksExt2, GoldilocksField}, -}; - -type F = GoldilocksField; -type EXT = GoldilocksExt2; -type DefaultTreeHasher = GoldilocksPoseidon2Sponge; -type DefaultTranscript = GoldilocksPoisedon2Transcript; - -pub type ZksyncProof = Proof; -pub type CompressionProofsTranscript = GoldilocksPoisedon2Transcript; - -pub fn load_scheduler_proof_and_vk( - path: &str, -) -> (ZkSyncRecursionProof, ZkSyncRecursionVerificationKey) { - let scheduler_vk_file = - std::fs::File::open(format!("{}/scheduler_recursive_vk.json", path)).unwrap(); - let scheduler_vk: ZkSyncRecursionLayerVerificationKey = - serde_json::from_reader(&scheduler_vk_file).unwrap(); - let scheduler_proof_file = - std::fs::File::open(format!("{}/scheduler_recursive_proof.json", path)).unwrap(); - let scheduler_proof: ZkSyncRecursionLayerProof = - serde_json::from_reader(&scheduler_proof_file).unwrap(); - - (scheduler_proof.into_inner(), scheduler_vk.into_inner()) -} - -#[derive(Copy, Clone, Debug)] -pub enum CompressionMode { - One = 1, - Two = 2, - Three = 3, - Four = 4, - Five = 5, -} - -impl CompressionMode { - pub fn from_compression_mode(compression_mode: u8) -> Self { - match compression_mode { - 1 => CompressionMode::One, - 2 => CompressionMode::Two, - 3 => CompressionMode::Three, - 4 => CompressionMode::Four, - 5 => CompressionMode::Five, - _ => unreachable!(), - } - } -} - -#[derive(Debug)] -pub struct CompressionSchedule { - pub name: &'static str, - pub compression_steps: Vec, -} - -impl CompressionSchedule { - pub fn name(&self) -> &'static str { - self.name - } - pub fn hard() -> Self { - CompressionSchedule { - name: "hard", - compression_steps: vec![ - CompressionMode::One, - CompressionMode::Two, - CompressionMode::Three, - CompressionMode::Four, - ], - } - } -} - -pub enum CompressionInput { - Recursion( - Option, - ZkSyncRecursionVerificationKey, - CompressionMode, - ), - Compression( - Option, - ZkSyncCompressionVerificationKey, - CompressionMode, - ), - CompressionWrapper( - Option, - ZkSyncCompressionVerificationKey, - CompressionMode, - ), -} - -impl CompressionInput { - pub fn into_compression_circuit(self) -> ZkSyncCompressionLayerCircuit { - match self { - CompressionInput::Recursion(proof, vk, compression_mode) => { - assert_eq!(compression_mode as u8, 1); - ZkSyncCompressionLayerCircuit::from_witness_and_vk(proof, vk, 1) - } - CompressionInput::Compression(proof, vk, compression_mode) => { - ZkSyncCompressionLayerCircuit::from_witness_and_vk( - proof, - vk, - compression_mode as u8, - ) - } - CompressionInput::CompressionWrapper(_, _, _) => { - unreachable!() - } - } - } - - pub fn into_compression_wrapper_circuit(self) -> ZkSyncCompressionForWrapperCircuit { - match self { - CompressionInput::Recursion(_, _, _) => { - unreachable!() - } - CompressionInput::Compression(_, _, _) => { - unreachable!() - } - CompressionInput::CompressionWrapper(proof, vk, compression_mode) => { - ZkSyncCompressionForWrapperCircuit::from_witness_and_vk( - proof, - vk, - compression_mode as u8, - ) - } - } - } -} - -pub fn save_compression_proof_and_vk_into_file( - proof: &ZkSyncCompressionProof, - vk: &ZkSyncCompressionVerificationKey, - compression_mode: u8, - path: &str, -) { - let proof_file = std::fs::File::create(format!( - "{}/compression_{}_proof.json", - path, compression_mode - )) - .unwrap(); - serde_json::to_writer(proof_file, &proof).unwrap(); - let vk_file = - std::fs::File::create(format!("{}/compression_{}_vk.json", path, compression_mode)) - .unwrap(); - serde_json::to_writer(vk_file, &vk).unwrap(); -} - -pub fn save_compression_wrapper_proof_and_vk_into_file( - proof: &ZkSyncCompressionProofForWrapper, - vk: &ZkSyncCompressionVerificationKeyForWrapper, - compression_mode: u8, - path: &str, -) { - let proof_file = std::fs::File::create(format!( - "{}/compression_wrapper_{}_proof.json", - path, compression_mode - )) - .unwrap(); - serde_json::to_writer(proof_file, &proof).unwrap(); - let vk_file = std::fs::File::create(format!( - "{}/compression_wrapper_{}_vk.json", - path, compression_mode - )) - .unwrap(); - serde_json::to_writer(vk_file, &vk).unwrap(); -} - -pub fn load_compression_wrapper_proof_and_vk_from_file( - blob_path: &str, - compression_mode: u8, -) -> ( - ZkSyncCompressionProofForWrapper, - ZkSyncCompressionVerificationKeyForWrapper, -) { - let proof_file = std::fs::File::open(format!( - "{}/compression_wrapper_{}_proof.json", - blob_path, compression_mode - )) - .unwrap(); - let proof = serde_json::from_reader(proof_file).unwrap(); - let vk_file = std::fs::File::open(format!( - "{}/compression_wrapper_{}_vk.json", - blob_path, compression_mode - )) - .unwrap(); - let vk = serde_json::from_reader(vk_file).unwrap(); - - (proof, vk) -} - -pub fn verify_compression_layer_circuit( - _circuit: CompressionLayerCircuit, - proof: &ZkSyncCompressionProof, - vk: &ZkSyncCompressionVerificationKey, - verifier: Verifier, -) -> bool { - verifier.verify::( - (), - vk, - proof, - ) -} - -pub fn verify_compression_wrapper_circuit( - _circuit: CompressionLayerCircuit, - proof: &ZkSyncCompressionProofForWrapper, - vk: &ZkSyncCompressionVerificationKeyForWrapper, - verifier: Verifier, -) -> bool { - verifier.verify::( - (), - vk, - proof, - ) -} - -pub fn save_fflonk_proof_and_vk_into_file( - proof: &FflonkSnarkVerifierCircuitProof, - vk: &FflonkSnarkVerifierCircuitVK, - output_blob_path: &str, -) { - let proof_file_path = format!("{}/final_proof.json", output_blob_path); - let proof_file = std::fs::File::create(&proof_file_path).unwrap(); - serde_json::to_writer(proof_file, &proof).unwrap(); - println!("proof saved at {proof_file_path}"); - let vk_file_path = format!("{}/final_vk.json", output_blob_path); - let vk_file = std::fs::File::create(&vk_file_path).unwrap(); - serde_json::to_writer(vk_file, &vk).unwrap(); - println!("vk saved at {vk_file_path}"); -} - -pub fn save_fflonk_setup_and_vk_into_file( - setup: &FflonkSnarkVerifierCircuitSetup, - vk: &FflonkSnarkVerifierCircuitVK, - output_blob_path: &str, -) { - let setup_file_path = format!("{}/fflonk_snark_setup.json", output_blob_path); - let setup_file = std::fs::File::create(&setup_file_path).unwrap(); - setup.write(&setup_file).unwrap(); - println!("fflonk precomputed setup saved into file at {setup_file_path}"); - let vk_file_path = format!("{}/fflonk_snark_setup.json", output_blob_path); - let vk_file = std::fs::File::create(&vk_file_path).unwrap(); - vk.write(&vk_file).unwrap(); - println!("fflonk VK saved into file at {vk_file_path}"); -} - -pub fn load_fflonk_setup_and_vk_from_file( - output_blob_path: &str, -) -> ( - FflonkSnarkVerifierCircuitSetup, - FflonkSnarkVerifierCircuitVK, -) { - let setup_file_path = format!("{}/fflonk_snark_setup.json", output_blob_path); - println!("reading fflonk precomputed setup from file at {setup_file_path}"); - let setup_file = std::fs::File::open(&setup_file_path).unwrap(); - let setup = FflonkSnarkVerifierCircuitSetup::read(&setup_file).unwrap(); - let vk_file_path = format!("{}/fflonk_snark_setup.json", output_blob_path); - println!("reading fflonk VK saved from file at {vk_file_path}"); - let vk_file = std::fs::File::open(&vk_file_path).unwrap(); - let vk = FflonkSnarkVerifierCircuitVK::read(&vk_file).unwrap(); - - (setup, vk) -} - -pub fn synthesize_circuit_for_setup( - circuit: CompressionLayerCircuit, -) -> ( - FinalizationHintsForProver, - CSReferenceAssembly, -) { - let geometry = circuit.geometry(); - let (max_trace_len, num_vars) = circuit.size_hint(); - - let builder_impl = CsReferenceImplementationBuilder::::new( - geometry, - max_trace_len.unwrap(), - ); - let builder = new_builder::<_, GoldilocksField>(builder_impl); - - let builder = circuit.configure_builder_proxy(builder); - let mut cs = builder.build(num_vars.unwrap()); - circuit.add_tables(&mut cs); - circuit.synthesize_into_cs(&mut cs); - let (_domain_size, finalization_hint) = cs.pad_and_shrink(); - let cs = cs.into_assembly::(); - - (finalization_hint, cs) -} - -pub fn synthesize_circuit_for_proving( - circuit: CompressionLayerCircuit, - finalization_hint: &FinalizationHintsForProver, -) -> CSReferenceAssembly { - let geometry = circuit.geometry(); - let (max_trace_len, num_vars) = circuit.size_hint(); - - let builder_impl = CsReferenceImplementationBuilder::::new( - geometry, - max_trace_len.unwrap(), - ); - let builder = new_builder::<_, GoldilocksField>(builder_impl); - - let builder = circuit.configure_builder_proxy(builder); - let mut cs = builder.build(num_vars.unwrap()); - circuit.add_tables(&mut cs); - circuit.synthesize_into_cs(&mut cs); - let _ = cs.pad_and_shrink_using_hint(&finalization_hint); - let cs = cs.into_assembly::(); - - cs -} -pub fn synthesize_circuit_for_dev( - circuit: CompressionLayerCircuit, -) -> CSReferenceAssembly { - let geometry = circuit.geometry(); - let (max_trace_len, num_vars) = circuit.size_hint(); - - let builder_impl = CsReferenceImplementationBuilder::::new( - geometry, - max_trace_len.unwrap(), - ); - let builder = new_builder::<_, GoldilocksField>(builder_impl); - - let builder = circuit.configure_builder_proxy(builder); - let mut cs = builder.build(num_vars.unwrap()); - circuit.add_tables(&mut cs); - circuit.synthesize_into_cs(&mut cs); - let _ = cs.pad_and_shrink(); - let cs = cs.into_assembly::(); - - cs -} diff --git a/crates/proof-compression/src/context.rs b/crates/proof-compression/src/context.rs index 1d084d0..9bd5510 100644 --- a/crates/proof-compression/src/context.rs +++ b/crates/proof-compression/src/context.rs @@ -1,25 +1,18 @@ use std::sync::atomic::AtomicBool; -use crate::{AsyncHandler, CompressionProofSystem, SnarkWrapperProofSystem}; +use crate::{AsyncHandler, CompressionProofSystem, SnarkWrapperProofSystem, SnarkWrapperStep}; -pub trait ContextManagerInterface { +pub(crate) trait ContextManagerInterface { fn init_compression_context

(&self, config: P::ContextConfig) -> AsyncHandler where P: CompressionProofSystem; - fn initialize_snark_context_config(&self) -> AsyncHandler + fn init_snark_context(&self, crs: AsyncHandler) -> AsyncHandler where - S: SnarkWrapperProofSystem; - - fn init_snark_context( - &self, - config: AsyncHandler, - ) -> AsyncHandler - where - S: SnarkWrapperProofSystem; + S: SnarkWrapperStep; } -pub struct SimpleContextManager { +pub(crate) struct SimpleContextManager { context_status: std::sync::Arc, } @@ -41,7 +34,6 @@ impl ContextManagerInterface for SimpleContextManager { .load(std::sync::atomic::Ordering::Relaxed) == false ); - // load next context let flag = self.context_status.clone(); let f = move || { let (sender, receiver) = std::sync::mpsc::channel(); @@ -53,26 +45,9 @@ impl ContextManagerInterface for SimpleContextManager { AsyncHandler::spawn(f) } - - fn initialize_snark_context_config(&self) -> AsyncHandler - where - S: SnarkWrapperProofSystem, - { - let f = move || { - let (sender, receiver) = std::sync::mpsc::channel(); - let start = std::time::Instant::now(); - let context_config = S::get_context_config(); - println!("CRS loading takes {}s", start.elapsed().as_secs()); - sender.send(context_config).unwrap(); - - receiver - }; - AsyncHandler::spawn(f) - } - fn init_snark_context( &self, - config: AsyncHandler, + compact_raw_crs: AsyncHandler, ) -> AsyncHandler where S: SnarkWrapperProofSystem, @@ -86,7 +61,7 @@ impl ContextManagerInterface for SimpleContextManager { let flag = self.context_status.clone(); let f = move || { let (sender, receiver) = std::sync::mpsc::channel(); - let context = S::init_context(config.wait()); + let context = S::init_context(compact_raw_crs); sender.send(context).unwrap(); flag.store(false, std::sync::atomic::Ordering::Relaxed); receiver diff --git a/crates/proof-compression/src/lib.rs b/crates/proof-compression/src/lib.rs index c409723..9057d4d 100644 --- a/crates/proof-compression/src/lib.rs +++ b/crates/proof-compression/src/lib.rs @@ -1,4 +1,3 @@ -#![allow(incomplete_features)] #![feature(generic_const_exprs)] #![feature(allocator_api)] @@ -6,10 +5,7 @@ mod blob_storage; use blob_storage::*; mod chain; -use chain::*; - -mod compression; -use compression::*; +pub use chain::*; mod context; use context::*; @@ -17,70 +13,11 @@ use context::*; mod proof_system; use proof_system::*; -mod snark_wrapper; -use snark_wrapper::*; - mod serialization; use serialization::*; -mod utils; -use utils::*; +mod step; +use step::*; #[cfg(test)] mod test; - -cfg_if::cfg_if! { - if #[cfg(feature = "gpu")] { - // - - } else { - // - } -} -use bellman::worker::Worker; -use boojum::pairing::bn256::{Bn256, Fr}; -use circuit_definitions::circuit_definitions::{ - aux_layer::{ZkSyncSnarkWrapperCircuit, ZkSyncSnarkWrapperCircuitNoLookupCustomGate}, - recursion_layer::RecursiveProofsTreeHasher, -}; -use fflonk::{ - bellman::plonk::better_better_cs::{ - cs::VerificationKey as PlonkVerificationKey, proof::Proof as PlonkProof, - setup::Setup as PlonkSetup, - }, - fflonk::{FflonkProof, FflonkSetup, FflonkVerificationKey}, - fflonk_cpu::franklin_crypto, - FflonkSnarkVerifierCircuitDeviceSetup, -}; -use franklin_crypto::bellman; -use gpu_prover::AsyncSetup; -use shivini::boojum::{self, field::goldilocks::GoldilocksExt2}; - -pub type FflonkSnarkVerifierCircuit = ZkSyncSnarkWrapperCircuitNoLookupCustomGate; -pub type FflonkSnarkVerifierCircuitVK = FflonkVerificationKey; -pub type FflonkSnarkVerifierCircuitProof = FflonkProof; - -pub type PlonkSnarkVerifierCircuit = ZkSyncSnarkWrapperCircuit; -pub type PlonkSnarkVerifierCircuitVK = PlonkVerificationKey; -pub type PlonkSnarkVerifierCircuitProof = PlonkProof; -pub type PlonkSnarkVerifierCircuitDeviceSetup = AsyncSetup; - -pub use fflonk::{GlobalHost, HostAllocator}; -use std::alloc::Global; - -use shivini::boojum::field::goldilocks::GoldilocksField; - -pub type SchedulerProof = boojum::cs::implementations::proof::Proof< - GoldilocksField, - RecursiveProofsTreeHasher, - GoldilocksExt2, ->; - -use circuit_definitions::circuit_definitions::aux_layer::{ - compression::{CompressionLayerCircuit, ProofCompressionFunction}, - compression_modes::{ - CompressionMode1, CompressionMode1ForWrapper, CompressionMode2, CompressionMode3, - CompressionMode4, CompressionMode5ForWrapper, CompressionTranscriptForWrapper, - CompressionTreeHasherForWrapper, - }, -}; diff --git a/crates/proof-compression/src/proof_system.rs b/crates/proof-compression/src/proof_system.rs deleted file mode 100644 index 9adeae0..0000000 --- a/crates/proof-compression/src/proof_system.rs +++ /dev/null @@ -1,712 +0,0 @@ -use std::alloc::Allocator; - -use super::*; -use bellman::kate_commitment::{Crs, CrsForMonomialForm}; -use bellman::pairing::compact_bn256::G1Affine as CompactG1Affine; -use bellman::plonk::better_better_cs::cs::SynthesisModeGenerateSetup; -use bellman::plonk::better_better_cs::{ - cs::{Assembly, PlonkCsWidth3Params, PlonkCsWidth4WithNextStepParams, SynthesisModeProve}, - gates::{ - naive_main_gate::NaiveMainGate, - selector_optimized_with_d_next::SelectorOptimizedWidth4MainGateWithDNext, - }, -}; -use bellman::plonk::commitments::transcript::keccak_transcript::RollingKeccakTranscript; -use bellman::CurveAffine; -use boojum::config::SetupCSConfig; -use boojum::cs::implementations::transcript::Transcript; -use boojum::cs::implementations::witness::WitnessVec; -use boojum::cs::traits::circuit::CircuitBuilderProxy; -use boojum::cs::traits::GoodAllocator; -use boojum::worker::Worker; -use boojum::{ - config::ProvingCSConfig, - cs::implementations::{ - fast_serialization::MemcopySerializable, proof::Proof, reference_cs::CSReferenceAssembly, - setup::FinalizationHintsForProver, verifier::VerificationKey, - }, - field::goldilocks::GoldilocksExt2, -}; - -use circuit_definitions::circuit_definitions::aux_layer::compression::{ - CompressionLayerCircuit, ProofCompressionFunction, -}; - -use fflonk::bellman::plonk::better_better_cs::cs::Circuit; -use fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG; -use fflonk::{CombinedMonomialDeviceStorage, DeviceContextWithSingleDevice}; -use gpu_prover::{DeviceMemoryManager, ManagerConfigs}; -use shivini::gpu_proof_config::GpuProofConfig; -use shivini::{ - CacheStrategy, CommitmentCacheStrategy, GPUPoWRunner, PolynomialsCacheStrategy, - ProverContextConfig, -}; -use shivini::{GpuTreeHasher, ProverContext}; -pub trait ProofSystemDefinition: Sized { - type FieldElement; - type ExternalWitnessData; - type Precomputation: MemcopySerializable + Send + Sync + 'static; - type Proof: serde::Serialize + serde::de::DeserializeOwned; - type VK: serde::Serialize + serde::de::DeserializeOwned + Send + Sync + Clone + 'static; - type FinalizationHint: serde::Serialize + serde::de::DeserializeOwned + Clone; - type Allocator: std::alloc::Allocator; - type ProvingAssembly: Sized + Send + Sync + 'static; - fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData; - fn verify(_: &Self::Proof, _: &Self::VK) -> bool; -} - -pub trait CompressionProofSystem: - ProofCompressionFunction< - ThisLayerHasher: GpuTreeHasher, - ThisLayerTranscript: Transcript, - ThisLayerPoW: GPUPoWRunner, - > + ProofSystemDefinition -{ - type ContextConfig: Send + Sync + 'static; - type Context: Send + Sync + 'static; - type AuxConfig; - fn get_context_config() -> Self::ContextConfig; - fn get_context_config_from_hint(_: &Self::FinalizationHint) -> Self::ContextConfig; - fn init_context(config: Self::ContextConfig) -> Self::Context; - fn aux_config_from_assembly(proving_assembly: &Self::ProvingAssembly) -> Self::AuxConfig; - - fn synthesize_for_proving( - circuit: CompressionLayerCircuit, - finalization_hint: Self::FinalizationHint, - ) -> Self::ProvingAssembly; - - fn prove( - _: AsyncHandler, - _: Self::ProvingAssembly, - _: Self::AuxConfig, - _: AsyncHandler, - _: Self::FinalizationHint, - _: &Self::VK, - ) -> Self::Proof; - - fn prove_from_witnesses( - _: AsyncHandler, - _: Self::ExternalWitnessData, - _: Self::AuxConfig, - _: AsyncHandler, - _: Self::FinalizationHint, - _: &Self::VK, - ) -> Self::Proof; -} - -pub trait CompressionProofSystemExt: CompressionProofSystem { - type SetupAssembly; - fn generate_precomputation_and_vk( - _: AsyncHandler, - _: Self::SetupAssembly, - _: &Self::FinalizationHint, - ) -> (Self::Precomputation, Self::VK); - fn synthesize_for_setup( - circuit: CompressionLayerCircuit, - ) -> (Self::FinalizationHint, Self::SetupAssembly); -} - -pub trait SnarkWrapperProofSystem: ProofSystemDefinition { - type CRS; - type Circuit; - type ContextConfig: Send + Sync + 'static; - type Context: Send + Sync + 'static; - fn get_context_config() -> Self::ContextConfig; - fn init_context(config: Self::ContextConfig) -> Self::Context; - fn load_crs(_: Self::FinalizationHint) -> Self::CRS; - fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly; - fn prove( - _: AsyncHandler, - _: Self::ProvingAssembly, - _: AsyncHandler, - _: Self::FinalizationHint, - _: &Self::VK, - ) -> Self::Proof; - - fn prove_from_witnesses( - _: AsyncHandler, - _: Vec, - _: AsyncHandler, - _: Self::FinalizationHint, - _: &Self::VK, - ) -> Self::Proof; -} - -pub trait SnarkWrapperProofSystemExt: SnarkWrapperProofSystem { - type SetupAssembly; - fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly; - fn generate_precomputation_and_vk( - _: AsyncHandler, - _: Self::SetupAssembly, - _: Self::FinalizationHint, - ) -> (Self::Precomputation, Self::VK); - fn create_crs(_: Self::FinalizationHint); -} - -type BoojumAssembly = - CSReferenceAssembly; - -impl ProofSystemDefinition for CF -where - CF: ProofCompressionFunction< - ThisLayerHasher: GpuTreeHasher, - ThisLayerTranscript: Transcript< - GoldilocksField, - TransciptParameters = (), - CompatibleCap: Send + Sync + 'static, - >, - ThisLayerPoW: GPUPoWRunner, - > + 'static, -{ - type FieldElement = GoldilocksField; - type ExternalWitnessData = WitnessVec; - type Precomputation = BoojumDeviceSetupWrapper; - type Proof = Proof; - type VK = VerificationKey; - type FinalizationHint = FinalizationHintsForProver; - type Allocator = std::alloc::Global; - type ProvingAssembly = BoojumAssembly; - - fn verify(proof: &Self::Proof, vk: &Self::VK) -> bool { - let verifier_builder = - CircuitBuilderProxy::>::dyn_verifier_builder::(); - let verifier = verifier_builder.create_verifier(); - verifier.verify::( - (), - vk, - proof, - ) - } - - fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { - proving_assembly.witness.take().unwrap() - } -} - -impl CompressionProofSystem for CF -where - CF: ProofCompressionFunction< - ThisLayerHasher: GpuTreeHasher, - ThisLayerTranscript: Transcript< - GoldilocksField, - TransciptParameters = (), - CompatibleCap: Send + Sync + 'static, - >, - ThisLayerPoW: GPUPoWRunner, - > + 'static, -{ - type AuxConfig = GpuProofConfig; - type ContextConfig = usize; // domain_size - type Context = ProverContext; - - fn get_context_config() -> Self::ContextConfig { - // println!("Using hardcoded domain size 2^17 for compression step"); - 1 << 17 - } - - fn init_context(domain_size: Self::ContextConfig) -> Self::Context { - let config = - ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); - let context = Self::Context::create_with_config(config).expect("gpu prover context"); - - context - } - fn get_context_config_from_hint(hint: &Self::FinalizationHint) -> Self::ContextConfig { - hint.final_trace_len - } - fn aux_config_from_assembly(proving_assembly: &Self::ProvingAssembly) -> Self::AuxConfig { - GpuProofConfig::from_assembly(proving_assembly) - } - - fn synthesize_for_proving( - circuit: CompressionLayerCircuit, - finalization_hint: Self::FinalizationHint, - ) -> Self::ProvingAssembly { - let geometry = circuit.geometry(); - let (max_trace_len, num_vars) = circuit.size_hint(); - - let builder_impl = boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< - GoldilocksField, - GoldilocksField, - ProvingCSConfig, - >::new(geometry, max_trace_len.unwrap()); - let builder = boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); - - let builder = circuit.configure_builder_proxy(builder); - let mut cs = builder.build(num_vars.unwrap()); - circuit.add_tables(&mut cs); - circuit.synthesize_into_cs(&mut cs); - let _ = cs.pad_and_shrink_using_hint(&finalization_hint); - let cs = cs.into_assembly::(); - - cs - } - - fn prove( - ctx: AsyncHandler, - proving_assembly: Self::ProvingAssembly, - aux_config: Self::AuxConfig, - precomputation: AsyncHandler, - finalization_hint: Self::FinalizationHint, - vk: &Self::VK, - ) -> Self::Proof { - Self::prove_from_witnesses( - ctx, - proving_assembly.witness.unwrap(), - aux_config, - precomputation, - finalization_hint, - vk, - ) - } - - fn prove_from_witnesses( - ctx: AsyncHandler, - witness: Self::ExternalWitnessData, - aux_config: Self::AuxConfig, - precomputation: AsyncHandler, - finalization_hint: Self::FinalizationHint, - vk: &Self::VK, - ) -> Self::Proof { - let domain_size = vk.fixed_parameters.domain_size as usize; - assert_eq!(finalization_hint.final_trace_len, domain_size); - let cache_strategy = CacheStrategy { - setup_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - trace_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - other_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - commitment: CommitmentCacheStrategy::CacheCosetCaps, - }; - let worker = Worker::new(); - let precomputation = precomputation.wait().into_inner(); - let ctx = ctx.wait(); - let gpu_proof = shivini::gpu_prove_from_external_witness_data_with_cache_strategy::< - CF::ThisLayerTranscript, - CF::ThisLayerHasher, - CF::ThisLayerPoW, - Self::Allocator, - >( - &aux_config, - &witness, - CF::proof_config_for_compression_step(), - &precomputation, - &vk, - (), - &worker, - cache_strategy, - ) - .expect("gpu proof"); - drop(ctx); - let proof = gpu_proof.into(); - - proof - } -} - -impl CompressionProofSystemExt for CF -where - CF: ProofCompressionFunction< - ThisLayerHasher: GpuTreeHasher, - ThisLayerTranscript: Transcript< - GoldilocksField, - TransciptParameters = (), - CompatibleCap: Send + Sync + 'static, - >, - ThisLayerPoW: GPUPoWRunner, - > + 'static, - Self::Allocator: GoodAllocator, -{ - type SetupAssembly = BoojumAssembly; - fn generate_precomputation_and_vk( - ctx: AsyncHandler, - setup_assembly: Self::SetupAssembly, - finalization_hint: &Self::FinalizationHint, - ) -> (Self::Precomputation, Self::VK) { - let worker = Worker::new(); - let proof_config = CF::proof_config_for_compression_step(); - let (setup_base, vk_params, vars_hint, wits_hint) = setup_assembly.get_light_setup( - &worker, - proof_config.fri_lde_factor, - proof_config.merkle_tree_cap_size, - ); - let domain_size = vk_params.domain_size as usize; - assert_eq!(finalization_hint.final_trace_len, domain_size); - let ctx = ctx.wait(); - let (precomputation, vk) = - shivini::cs::gpu_setup_and_vk_from_base_setup_vk_params_and_hints::< - CF::ThisLayerHasher, - _, - >(setup_base, vk_params, vars_hint, wits_hint, &worker) - .unwrap(); - drop(ctx); - (BoojumDeviceSetupWrapper::from_inner(precomputation), vk) - } - fn synthesize_for_setup( - circuit: CompressionLayerCircuit, - ) -> (Self::FinalizationHint, Self::SetupAssembly) { - let geometry = circuit.geometry(); - let (max_trace_len, num_vars) = circuit.size_hint(); - - let builder_impl = boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< - GoldilocksField, - GoldilocksField, - SetupCSConfig, - >::new(geometry, max_trace_len.unwrap()); - let builder = boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); - - let builder = circuit.configure_builder_proxy(builder); - let mut cs = builder.build(num_vars.unwrap()); - circuit.add_tables(&mut cs); - circuit.synthesize_into_cs(&mut cs); - let (_domain_size, finalization_hint) = cs.pad_and_shrink(); - let cs = cs.into_assembly::(); - - (finalization_hint, cs) - } -} - -type PlonkAssembly = Assembly< - Bn256, - PlonkCsWidth4WithNextStepParams, - SelectorOptimizedWidth4MainGateWithDNext, - CSConfig, - A, ->; - -pub struct UnsafePlonkProverDeviceMemoryManagerWrapper( - DeviceMemoryManager, -); -unsafe impl Send for UnsafePlonkProverDeviceMemoryManagerWrapper {} -unsafe impl Sync for UnsafePlonkProverDeviceMemoryManagerWrapper {} -pub struct PlonkProverDeviceMemoryManagerConfig; - -impl ManagerConfigs for PlonkProverDeviceMemoryManagerConfig { - const NUM_GPUS_LOG: usize = 0; - const FULL_SLOT_SIZE_LOG: usize = 24; - const NUM_SLOTS: usize = 29; - const NUM_HOST_SLOTS: usize = 2; -} - -impl ProofSystemDefinition for PlonkSnarkWrapper { - type FieldElement = Fr; - type Precomputation = PlonkSnarkVerifierCircuitDeviceSetupWrapper; - type ExternalWitnessData = Vec; - type Proof = PlonkSnarkVerifierCircuitProof; - type VK = PlonkSnarkVerifierCircuitVK; - type FinalizationHint = usize; - type Allocator = GlobalHost; - type ProvingAssembly = PlonkAssembly; - fn take_witnesses( - proving_assembly: &mut Self::ProvingAssembly, - ) -> Vec { - todo!() - } - fn verify(_: &Self::Proof, _: &Self::VK) -> bool { - todo!() - } -} - -impl SnarkWrapperProofSystem for PlonkSnarkWrapper { - type CRS = bellman::kate_commitment::Crs< - bellman::compact_bn256::Bn256, - CrsForMonomialForm, - Self::Allocator, - >; - type Circuit = PlonkSnarkVerifierCircuit; - type ContextConfig = (Vec, Vec); - type Context = UnsafePlonkProverDeviceMemoryManagerWrapper; - fn init_context(config: Self::ContextConfig) -> Self::Context { - let (device_ids, compact_crs) = config; - let manager = DeviceMemoryManager::init(&device_ids, &compact_crs[..]).unwrap(); - UnsafePlonkProverDeviceMemoryManagerWrapper(manager) - } - - fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly { - todo!() - } - - fn prove( - _: AsyncHandler, - _: Self::ProvingAssembly, - _: AsyncHandler, - _: Self::FinalizationHint, - _: &Self::VK, - ) -> Self::Proof { - todo!() - } - - fn prove_from_witnesses( - _: AsyncHandler, - _: Vec, - _: AsyncHandler, - _: Self::FinalizationHint, - _: &Self::VK, - ) -> Self::Proof { - todo!() - } - - fn get_context_config() -> Self::ContextConfig { - todo!() - } - - fn load_crs(domain_size: Self::FinalizationHint) -> Self::CRS { - let raw_compact_crs_file_path = std::env::var("PLONK_COMPACT_RAW_CRS_FILE").unwrap(); - let raw_compact_crs_file = std::fs::File::open(raw_compact_crs_file_path).unwrap(); - let num_points = domain_size * L1_VERIFIER_DOMAIN_SIZE_LOG; - read_crs_from_raw_compact_form::<_, Self::Allocator>(raw_compact_crs_file, num_points) - .unwrap() - } -} - -impl SnarkWrapperProofSystemExt for PlonkSnarkWrapper { - type SetupAssembly = PlonkAssembly; - - fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly { - todo!() - } - - fn generate_precomputation_and_vk( - _: AsyncHandler, - _: Self::SetupAssembly, - _: Self::FinalizationHint, - ) -> (Self::Precomputation, Self::VK) { - todo!() - } - - fn create_crs(_: Self::FinalizationHint) { - todo!() - } -} - -type FflonkAssembly = Assembly; - -impl ProofSystemDefinition for FflonkSnarkWrapper { - type FieldElement = Fr; - type Precomputation = FflonkSnarkVerifierCircuitDeviceSetupWrapper; - type ExternalWitnessData = ( - Vec, - Vec, - ); - type Proof = FflonkSnarkVerifierCircuitProof; - type VK = FflonkSnarkVerifierCircuitVK; - type FinalizationHint = usize; - // type Allocator = GlobalHost; // TODO need global host with preallocated host memory - type Allocator = std::alloc::Global; - type ProvingAssembly = FflonkAssembly; - fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { - let input_assignments = - std::mem::replace(&mut proving_assembly.input_assingments, Vec::new()); - let aux_assignments = std::mem::replace( - &mut proving_assembly.aux_assingments, - Vec::new_in(Self::Allocator::default()), - ); - - (input_assignments, aux_assignments) - } - - fn verify(proof: &Self::Proof, vk: &Self::VK) -> bool { - fflonk::fflonk_cpu::verify::<_, FflonkSnarkVerifierCircuit, RollingKeccakTranscript>( - vk, proof, None, - ) - .unwrap() - } -} - -impl SnarkWrapperProofSystem for FflonkSnarkWrapper { - type CRS = bellman::kate_commitment::Crs< - bellman::compact_bn256::Bn256, - CrsForMonomialForm, - Self::Allocator, - >; - type Circuit = FflonkSnarkVerifierCircuit; - type ContextConfig = (usize, Self::CRS); - type Context = DeviceContextWithSingleDevice; - - fn load_crs(domain_size: Self::FinalizationHint) -> Self::CRS { - let raw_compact_crs_file_path = std::env::var("FFLONK_COMPACT_RAW_CRS_FILE").unwrap(); - let raw_compact_crs_file = std::fs::File::open(raw_compact_crs_file_path).unwrap(); - let num_points = domain_size * fflonk::MAX_COMBINED_DEGREE_FACTOR; - read_crs_from_raw_compact_form::<_, Self::Allocator>(raw_compact_crs_file, num_points) - .unwrap() - } - - fn get_context_config() -> Self::ContextConfig { - let domain_size = 1 << fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG; - let crs = Self::load_crs(domain_size); - (domain_size, crs) - } - - fn init_context(config: Self::ContextConfig) -> Self::Context { - let (domain_size, crs) = config; - let context = Self::Context::init_from_preloaded_crs(domain_size, crs).unwrap(); - context - } - - fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly { - let mut proving_assembly = FflonkAssembly::::new(); - circuit - .synthesize(&mut proving_assembly) - .expect("must work"); - proving_assembly - } - - fn prove( - ctx: AsyncHandler, - mut proving_assembly: Self::ProvingAssembly, - precomputation: AsyncHandler, - finalization_hint: Self::FinalizationHint, - _vk: &Self::VK, - ) -> Self::Proof { - assert!(proving_assembly.is_satisfied()); - let raw_trace_len = proving_assembly.n(); - assert!(finalization_hint.is_power_of_two()); - proving_assembly.finalize_to_size_log_2(finalization_hint.trailing_zeros() as usize); - let domain_size = proving_assembly.n() + 1; - assert!(domain_size.is_power_of_two()); - assert_eq!(domain_size, finalization_hint); - - let ctx = ctx.wait(); - let precomputation = precomputation.wait().into_inner(); - let proof = fflonk::create_proof::< - _, - _, - _, - RollingKeccakTranscript<_>, - CombinedMonomialDeviceStorage, - _, - >(&proving_assembly, &precomputation, raw_trace_len) - .unwrap(); - drop(ctx); - proof - } - - fn prove_from_witnesses( - _: AsyncHandler, - _: Vec, - _: AsyncHandler, - _: Self::FinalizationHint, - _: &Self::VK, - ) -> Self::Proof { - unimplemented!() - } -} - -impl SnarkWrapperProofSystemExt for FflonkSnarkWrapper { - type SetupAssembly = FflonkAssembly; - - fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly { - let mut setup_assembly = - FflonkAssembly::::new(); - circuit.synthesize(&mut setup_assembly).unwrap(); - - setup_assembly - } - - fn generate_precomputation_and_vk( - ctx: AsyncHandler, - setup_assembly: Self::SetupAssembly, - _finalization_hint: Self::FinalizationHint, - ) -> (Self::Precomputation, Self::VK) { - let ctx = ctx.wait(); - let device_setup = fflonk::FflonkDeviceSetup::< - Bn256, - FflonkSnarkVerifierCircuit, - Self::Allocator, - >::create_setup_from_assembly_on_device(&setup_assembly) - .unwrap(); - let vk = device_setup.get_verification_key(); - drop(ctx); - ( - FflonkSnarkVerifierCircuitDeviceSetupWrapper(device_setup), - vk, - ) - } - - fn create_crs(domain_size: Self::FinalizationHint) { - assert!(domain_size < fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG); - let num_points = fflonk::MAX_COMBINED_DEGREE_FACTOR * domain_size; - let original_crs = make_fflonk_crs_from_ignition_transcripts(num_points); - let raw_compact_crs_file_path = std::env::var("FFLONK_COMPACT_RAW_CRS_FILE").unwrap(); - assert!(!std::path::Path::exists(std::path::Path::new( - &raw_compact_crs_file_path - ))); - let raw_compact_crs_file = std::fs::File::create(raw_compact_crs_file_path).unwrap(); - write_crs_into_raw_compact_form(&original_crs, raw_compact_crs_file, num_points).unwrap(); - } -} - -pub struct MarkerProofSystem; -impl ProofSystemDefinition for MarkerProofSystem { - type FieldElement = (); - type Precomputation = MarkerPrecomputation; - type Proof = (); - type VK = (); - type ExternalWitnessData = (); - type FinalizationHint = (); - type Allocator = std::alloc::Global; - type ProvingAssembly = (); - - fn verify(_: &Self::Proof, _: &Self::VK) -> bool { - unreachable!() - } - - fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { - unreachable!() - } -} - -pub fn write_crs_into_raw_compact_form( - original_crs: &Crs, - mut dst_raw_compact_crs: W, - num_points: usize, -) -> std::io::Result<()> { - assert!(num_points <= original_crs.g1_bases.len()); - use bellman::{PrimeField, PrimeFieldRepr}; - use byteorder::{BigEndian, WriteBytesExt}; - assert!(num_points < u32::MAX as usize); - dst_raw_compact_crs.write_u32::(num_points as u32)?; - for g1_base in original_crs.g1_bases.iter() { - let (x, y) = g1_base.as_xy(); - x.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; - y.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; - } - assert_eq!(original_crs.g2_monomial_bases.len(), 2); - for g2_base in original_crs.g2_monomial_bases.iter() { - let (x, y) = g2_base.as_xy(); - x.c0.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; - x.c1.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; - y.c0.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; - y.c1.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; - } - - Ok(()) -} - -// TODO: Crs doesn't allow bases located in a custom allocator -pub fn read_crs_from_raw_compact_form( - mut src_raw_compact_crs: R, - num_g1_points: usize, -) -> std::io::Result> { - use byteorder::{BigEndian, ReadBytesExt}; - let actual_num_points = src_raw_compact_crs.read_u32::()? as usize; - assert!(num_g1_points <= actual_num_points as usize); - let mut g1_bases = Vec::with_capacity_in(num_g1_points, A::default()); - unsafe { - g1_bases.set_len(num_g1_points); - let buf = std::slice::from_raw_parts_mut( - g1_bases.as_mut_ptr() as *mut u8, - num_g1_points * std::mem::size_of::(), - ); - src_raw_compact_crs.read_exact(buf)?; - } - let num_g2_points = 2; - let mut g2_bases = Vec::with_capacity_in(num_g2_points, A::default()); - unsafe { - g2_bases.set_len(num_g2_points); - let buf = std::slice::from_raw_parts_mut( - g2_bases.as_mut_ptr() as *mut u8, - num_g2_points * std::mem::size_of::(), - ); - src_raw_compact_crs.read_exact(buf)?; - } - Ok(Crs::<_, CrsForMonomialForm, A>::new_in(g1_bases, g2_bases)) -} diff --git a/crates/proof-compression/src/proof_system/boojum.rs b/crates/proof-compression/src/proof_system/boojum.rs new file mode 100644 index 0000000..2dadf6b --- /dev/null +++ b/crates/proof-compression/src/proof_system/boojum.rs @@ -0,0 +1,247 @@ +use shivini::{ + boojum::{ + config::{ProvingCSConfig, SetupCSConfig}, + cs::{ + implementations::{ + proof::Proof, reference_cs::CSReferenceAssembly, setup::FinalizationHintsForProver, + transcript::Transcript, verifier::VerificationKey, witness::WitnessVec, + }, + traits::{circuit::CircuitBuilderProxy, GoodAllocator}, + }, + worker::Worker, + }, + gpu_proof_config::GpuProofConfig, + CacheStrategy, CommitmentCacheStrategy, GPUPoWRunner, GpuTreeHasher, PolynomialsCacheStrategy, + ProverContext, ProverContextConfig, +}; + +use circuit_definitions::circuit_definitions::aux_layer::compression::{ + CompressionLayerCircuit, ProofCompressionFunction, +}; + +use super::*; +pub(crate) use shivini::boojum; +pub(crate) use shivini::boojum::field::goldilocks::{GoldilocksExt2, GoldilocksField}; + +type BoojumAssembly = + CSReferenceAssembly; + +impl ProofSystemDefinition for CF +where + CF: ProofCompressionFunction< + ThisLayerHasher: GpuTreeHasher, + ThisLayerTranscript: Transcript< + GoldilocksField, + TransciptParameters = (), + CompatibleCap: Send + Sync + 'static, + >, + ThisLayerPoW: GPUPoWRunner, + > + 'static, +{ + type FieldElement = GoldilocksField; + type ExternalWitnessData = WitnessVec; + type Precomputation = BoojumDeviceSetupWrapper; + type Proof = Proof; + type VK = VerificationKey; + type FinalizationHint = FinalizationHintsForProver; + type Allocator = std::alloc::Global; + type ProvingAssembly = BoojumAssembly; + type Transcript = CF::ThisLayerTranscript; + fn verify(proof: &Self::Proof, vk: &Self::VK) -> bool { + let verifier_builder = + CircuitBuilderProxy::>::dyn_verifier_builder::(); + let verifier = verifier_builder.create_verifier(); + verifier.verify::( + (), + vk, + proof, + ) + } + + fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { + proving_assembly.witness.take().unwrap() + } +} + +impl CompressionProofSystem for CF +where + CF: ProofCompressionFunction< + ThisLayerHasher: GpuTreeHasher, + ThisLayerTranscript: Transcript< + GoldilocksField, + TransciptParameters = (), + CompatibleCap: Send + Sync + 'static, + >, + ThisLayerPoW: GPUPoWRunner, + > + 'static, +{ + type AuxConfig = GpuProofConfig; + type ContextConfig = usize; // domain_size + type Context = ProverContext; + + fn get_context_config() -> Self::ContextConfig { + // println!("Using hardcoded domain size 2^17 for compression step"); + 1 << 17 + } + + fn init_context(domain_size: Self::ContextConfig) -> Self::Context { + let config = + ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); + let context = Self::Context::create_with_config(config).expect("gpu prover context"); + + context + } + fn get_context_config_from_hint(hint: &Self::FinalizationHint) -> Self::ContextConfig { + hint.final_trace_len + } + fn aux_config_from_assembly(proving_assembly: &Self::ProvingAssembly) -> Self::AuxConfig { + GpuProofConfig::from_assembly(proving_assembly) + } + + fn synthesize_for_proving( + circuit: CompressionLayerCircuit, + finalization_hint: Self::FinalizationHint, + ) -> Self::ProvingAssembly { + let geometry = circuit.geometry(); + let (max_trace_len, num_vars) = circuit.size_hint(); + + let builder_impl = boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< + GoldilocksField, + GoldilocksField, + ProvingCSConfig, + >::new(geometry, max_trace_len.unwrap()); + let builder = boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); + + let builder = circuit.configure_builder_proxy(builder); + let mut cs = builder.build(num_vars.unwrap()); + circuit.add_tables(&mut cs); + circuit.synthesize_into_cs(&mut cs); + let _ = cs.pad_and_shrink_using_hint(&finalization_hint); + let cs = cs.into_assembly::(); + + cs + } + + fn prove( + ctx: AsyncHandler, + proving_assembly: Self::ProvingAssembly, + aux_config: Self::AuxConfig, + precomputation: AsyncHandler, + finalization_hint: Self::FinalizationHint, + vk: &Self::VK, + ) -> Self::Proof { + Self::prove_from_witnesses( + ctx, + proving_assembly.witness.unwrap(), + aux_config, + precomputation, + finalization_hint, + vk, + ) + } + + fn prove_from_witnesses( + ctx: AsyncHandler, + witness: Self::ExternalWitnessData, + aux_config: Self::AuxConfig, + precomputation: AsyncHandler, + finalization_hint: Self::FinalizationHint, + vk: &Self::VK, + ) -> Self::Proof { + let domain_size = vk.fixed_parameters.domain_size as usize; + assert_eq!(finalization_hint.final_trace_len, domain_size); + let cache_strategy = CacheStrategy { + setup_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, + trace_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, + other_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, + commitment: CommitmentCacheStrategy::CacheCosetCaps, + }; + let worker = Worker::new(); + let precomputation = precomputation.wait().into_inner(); + let ctx = ctx.wait(); + let gpu_proof = shivini::gpu_prove_from_external_witness_data_with_cache_strategy::< + CF::ThisLayerTranscript, + CF::ThisLayerHasher, + CF::ThisLayerPoW, + Self::Allocator, + >( + &aux_config, + &witness, + CF::proof_config_for_compression_step(), + &precomputation, + &vk, + (), + &worker, + cache_strategy, + ) + .expect("gpu proof"); + drop(ctx); + let proof = gpu_proof.into(); + + proof + } +} + +impl CompressionProofSystemExt for CF +where + CF: ProofCompressionFunction< + ThisLayerHasher: GpuTreeHasher, + ThisLayerTranscript: Transcript< + GoldilocksField, + TransciptParameters = (), + CompatibleCap: Send + Sync + 'static, + >, + ThisLayerPoW: GPUPoWRunner, + > + 'static, + Self::Allocator: GoodAllocator, +{ + type SetupAssembly = BoojumAssembly; + fn generate_precomputation_and_vk( + ctx: AsyncHandler, + setup_assembly: Self::SetupAssembly, + finalization_hint: &Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK) { + let worker = Worker::new(); + let proof_config = CF::proof_config_for_compression_step(); + let (setup_base, vk_params, vars_hint, wits_hint) = setup_assembly.get_light_setup( + &worker, + proof_config.fri_lde_factor, + proof_config.merkle_tree_cap_size, + ); + let domain_size = vk_params.domain_size as usize; + assert_eq!(finalization_hint.final_trace_len, domain_size); + let ctx = ctx.wait(); + let (precomputation, vk) = + shivini::cs::gpu_setup_and_vk_from_base_setup_vk_params_and_hints::< + CF::ThisLayerHasher, + _, + >(setup_base, vk_params, vars_hint, wits_hint, &worker) + .unwrap(); + drop(ctx); + (BoojumDeviceSetupWrapper::from_inner(precomputation), vk) + } + fn synthesize_for_setup( + circuit: CompressionLayerCircuit, + ) -> (Self::FinalizationHint, Self::SetupAssembly) { + let geometry = circuit.geometry(); + let (max_trace_len, num_vars) = circuit.size_hint(); + + let builder_impl = + boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< + GoldilocksField, + GoldilocksField, + SetupCSConfig, + >::new(geometry, max_trace_len.unwrap()); + let builder = + boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); + + let builder = circuit.configure_builder_proxy(builder); + let mut cs = builder.build(num_vars.unwrap()); + circuit.add_tables(&mut cs); + circuit.synthesize_into_cs(&mut cs); + let (_domain_size, finalization_hint) = cs.pad_and_shrink(); + let cs = cs.into_assembly::(); + + (finalization_hint, cs) + } +} diff --git a/crates/proof-compression/src/utils.rs b/crates/proof-compression/src/proof_system/crs.rs similarity index 66% rename from crates/proof-compression/src/utils.rs rename to crates/proof-compression/src/proof_system/crs.rs index 5ad0a33..e63292f 100644 --- a/crates/proof-compression/src/utils.rs +++ b/crates/proof-compression/src/proof_system/crs.rs @@ -1,11 +1,115 @@ use super::*; -use bellman::{ +use ::fflonk::bellman::{ + self, kate_commitment::{Crs, CrsForMonomialForm}, - CurveAffine, Engine, Field, PrimeField, }; +use bellman::{bn256::Bn256, CurveAffine, Engine, Field, PrimeField}; use byteorder::{BigEndian, ReadBytesExt}; +use gpu_prover::ManagerConfigs; -pub fn create_crs_from_ignition_transcript + ?Sized>( +pub(crate) fn write_crs_into_raw_compact_form( + original_crs: &Crs, + mut dst_raw_compact_crs: W, + num_points: usize, +) -> std::io::Result<()> { + use bellman::CurveAffine; + assert!(num_points <= original_crs.g1_bases.len()); + use bellman::{PrimeField, PrimeFieldRepr}; + use byteorder::{BigEndian, WriteBytesExt}; + assert!(num_points < u32::MAX as usize); + dst_raw_compact_crs.write_u32::(num_points as u32)?; + for g1_base in original_crs.g1_bases.iter() { + let (x, y) = g1_base.as_xy(); + x.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + y.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + } + assert_eq!(original_crs.g2_monomial_bases.len(), 2); + for g2_base in original_crs.g2_monomial_bases.iter() { + let (x, y) = g2_base.as_xy(); + x.c0.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + x.c1.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + y.c0.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + y.c1.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + } + + Ok(()) +} + +pub(crate) fn read_crs_from_raw_compact_form( + mut src_raw_compact_crs: R, + num_g1_points: usize, +) -> std::io::Result> { + use byteorder::{BigEndian, ReadBytesExt}; + let actual_num_points = src_raw_compact_crs.read_u32::()? as usize; + assert!(num_g1_points <= actual_num_points as usize); + let mut g1_bases = Vec::with_capacity_in(num_g1_points, A::default()); + unsafe { + g1_bases.set_len(num_g1_points); + let buf = std::slice::from_raw_parts_mut( + g1_bases.as_mut_ptr() as *mut u8, + num_g1_points * std::mem::size_of::(), + ); + src_raw_compact_crs.read_exact(buf)?; + } + let num_g2_points = 2; + let mut g2_bases = Vec::with_capacity_in(num_g2_points, A::default()); + unsafe { + g2_bases.set_len(num_g2_points); + let buf = std::slice::from_raw_parts_mut( + g2_bases.as_mut_ptr() as *mut u8, + num_g2_points * std::mem::size_of::(), + ); + src_raw_compact_crs.read_exact(buf)?; + } + Ok(Crs::<_, CrsForMonomialForm, A>::new_in(g1_bases, g2_bases)) +} + +pub(crate) fn create_compact_raw_crs(dst: W) { + let num_points = [ + ::fflonk::MAX_COMBINED_DEGREE_FACTOR << ::fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG, + ::FULL_SLOT_SIZE, + ] + .into_iter() + .max() + .unwrap(); + let original_crs = make_crs_from_ignition_transcripts(num_points); + write_crs_into_raw_compact_form(&original_crs, dst, num_points).unwrap(); +} + +fn make_crs_from_ignition_transcripts(num_points: usize) -> Crs { + let transcripts_dir = + std::env::var("IGNITION_TRANSCRIPT_PATH").expect("IGNITION_TRANSCRIPT_PATH env variable"); + let chunk_size = 5_040_000usize; + let num_chunks = num_points.div_ceil(chunk_size); + + // Check transcript files already downloaded from "https://aztec-ignition.s3.eu-west-2.amazonaws.com/MAIN+IGNITION/sealed/transcript{idx}.dat"; + for idx in 0..num_chunks { + let transcript_file_path = format!("{}/transcript{:02}.dat", transcripts_dir, idx); + let transcript_file_path = std::path::Path::new(&transcript_file_path); + assert!( + transcript_file_path.exists(), + "CRS transcript file {:?} couldn't found.", + transcript_file_path + ); + } + + // transform + let crs = create_crs_from_ignition_transcript(&transcripts_dir, num_chunks).unwrap(); + + let bellman::kate_commitment::Crs { + g1_bases, + g2_monomial_bases, + .. + } = crs; + assert!(g1_bases.len() >= num_points); + let mut g1_bases = std::sync::Arc::try_unwrap(g1_bases).unwrap(); + let g2_monomial_bases = std::sync::Arc::try_unwrap(g2_monomial_bases).unwrap(); + g1_bases.truncate(num_points); + + Crs::new(g1_bases, g2_monomial_bases) +} + +fn create_crs_from_ignition_transcript + ?Sized>( path: &S, num_chunks: usize, ) -> Result< @@ -173,36 +277,3 @@ pub fn create_crs_from_ignition_transcript + ?Sized>( Ok(new) } - -pub fn make_fflonk_crs_from_ignition_transcripts( - num_points: usize, -) -> Crs { - let transcripts_dir = - std::env::var("IGNITION_TRANSCRIPT_PATH").expect("IGNITION_TRANSCRIPT_PATH env variable"); - let chunk_size = 5_040_000usize; - let num_chunks = num_points.div_ceil(chunk_size); - - // Check transcript files already downloaded from "https://aztec-ignition.s3.eu-west-2.amazonaws.com/MAIN+IGNITION/sealed/transcript{idx}.dat"; - for idx in 0..num_chunks { - let transcript_file_path = format!("{}/transcript{:02}.dat", transcripts_dir, idx); - let transcript_file_path = std::path::Path::new(&transcript_file_path); - assert!(transcript_file_path.exists()); - } - - // transform - let crs = create_crs_from_ignition_transcript(&transcripts_dir, num_chunks).unwrap(); - let out_path = format!("{}/full_ignition.key", &transcripts_dir); - let out_file = std::fs::File::create(&out_path).unwrap(); - - let bellman::kate_commitment::Crs { - g1_bases, - g2_monomial_bases, - .. - } = crs; - assert!(g1_bases.len() >= num_points); - let mut g1_bases = std::sync::Arc::try_unwrap(g1_bases).unwrap(); - let g2_monomial_bases = std::sync::Arc::try_unwrap(g2_monomial_bases).unwrap(); - g1_bases.truncate(num_points); - - Crs::new(g1_bases, g2_monomial_bases) -} diff --git a/crates/proof-compression/src/proof_system/fflonk.rs b/crates/proof-compression/src/proof_system/fflonk.rs new file mode 100644 index 0000000..7b2a12e --- /dev/null +++ b/crates/proof-compression/src/proof_system/fflonk.rs @@ -0,0 +1,159 @@ +use ::fflonk::fflonk_cpu::{FflonkProof, FflonkVerificationKey}; +use ::fflonk::{ + CombinedMonomialDeviceStorage, DeviceContextWithSingleDevice, + FflonkSnarkVerifierCircuitDeviceSetup, +}; +use bellman::bn256::{Bn256, Fr}; +use bellman::{ + kate_commitment::CrsForMonomialForm, + plonk::{ + better_better_cs::{ + cs::{ + Assembly, Circuit, PlonkCsWidth3Params, SynthesisModeGenerateSetup, + SynthesisModeProve, + }, + gates::naive_main_gate::NaiveMainGate, + }, + commitments::transcript::keccak_transcript::RollingKeccakTranscript, + }, +}; +use circuit_definitions::circuit_definitions::aux_layer::ZkSyncSnarkWrapperCircuitNoLookupCustomGate; + +use super::*; +pub(crate) use ::fflonk::HostAllocator; +pub(crate) type FflonkSnarkVerifierCircuit = ZkSyncSnarkWrapperCircuitNoLookupCustomGate; +pub(crate) type FflonkSnarkVerifierCircuitVK = FflonkVerificationKey; +pub(crate) type FflonkSnarkVerifierCircuitProof = FflonkProof; +type FflonkAssembly = Assembly; +pub(crate) struct FflonkSnarkWrapper; + +impl ProofSystemDefinition for FflonkSnarkWrapper { + type FieldElement = Fr; + type Precomputation = FflonkSnarkVerifierCircuitDeviceSetupWrapper; + type ExternalWitnessData = ( + Vec, + Vec, + ); + type Proof = FflonkSnarkVerifierCircuitProof; + type VK = FflonkSnarkVerifierCircuitVK; + type FinalizationHint = usize; + // type Allocator = GlobalHost; // TODO need global host with preallocated host memory + type Allocator = std::alloc::Global; + type ProvingAssembly = FflonkAssembly; + type Transcript = RollingKeccakTranscript; + fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { + let input_assignments = + std::mem::replace(&mut proving_assembly.input_assingments, Vec::new()); + let aux_assignments = std::mem::replace( + &mut proving_assembly.aux_assingments, + Vec::new_in(Self::Allocator::default()), + ); + + (input_assignments, aux_assignments) + } + + fn verify(proof: &Self::Proof, vk: &Self::VK) -> bool { + ::fflonk::fflonk_cpu::verify::<_, FflonkSnarkVerifierCircuit, Self::Transcript>( + vk, proof, None, + ) + .unwrap() + } +} + +impl SnarkWrapperProofSystem for FflonkSnarkWrapper { + type Circuit = FflonkSnarkVerifierCircuit; + type Context = DeviceContextWithSingleDevice; + type CRS = bellman::kate_commitment::Crs< + bellman::compact_bn256::Bn256, + CrsForMonomialForm, + Self::Allocator, + >; + fn load_compact_raw_crs(src: R) -> Self::CRS { + let domain_size = 1 << ::fflonk::fflonk_cpu::L1_VERIFIER_DOMAIN_SIZE_LOG; + let num_g1_bases_for_crs = ::fflonk::fflonk_cpu::MAX_COMBINED_DEGREE_FACTOR * domain_size; + read_crs_from_raw_compact_form(src, num_g1_bases_for_crs).unwrap() + } + + fn init_context(compact_raw_crs: AsyncHandler) -> Self::Context { + let compact_raw_crs = compact_raw_crs.wait(); + let domain_size = 1 << ::fflonk::fflonk_cpu::L1_VERIFIER_DOMAIN_SIZE_LOG; + let context = Self::Context::init_from_preloaded_crs(domain_size, compact_raw_crs).unwrap(); + context + } + + fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly { + let mut proving_assembly = FflonkAssembly::::new(); + circuit + .synthesize(&mut proving_assembly) + .expect("must work"); + proving_assembly + } + + fn prove( + ctx: AsyncHandler, + mut proving_assembly: Self::ProvingAssembly, + precomputation: AsyncHandler, + finalization_hint: Self::FinalizationHint, + ) -> Self::Proof { + assert!(proving_assembly.is_satisfied()); + let raw_trace_len = proving_assembly.n(); + assert!(finalization_hint.is_power_of_two()); + proving_assembly.finalize_to_size_log_2(finalization_hint.trailing_zeros() as usize); + let domain_size = proving_assembly.n() + 1; + assert!(domain_size.is_power_of_two()); + assert_eq!(domain_size, finalization_hint); + let ctx = ctx.wait(); + let precomputation = precomputation.wait().into_inner(); + let proof = ::fflonk::create_proof::< + _, + _, + _, + RollingKeccakTranscript<_>, + CombinedMonomialDeviceStorage, + _, + >(&proving_assembly, &precomputation, raw_trace_len) + .unwrap(); + drop(ctx); + proof + } + + fn prove_from_witnesses( + _: AsyncHandler, + _: Vec, + _: AsyncHandler, + _: Self::FinalizationHint, + ) -> Self::Proof { + unimplemented!() + } +} + +impl SnarkWrapperProofSystemExt for FflonkSnarkWrapper { + type SetupAssembly = FflonkAssembly; + + fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly { + let mut setup_assembly = + FflonkAssembly::::new(); + circuit.synthesize(&mut setup_assembly).unwrap(); + + setup_assembly + } + + fn generate_precomputation_and_vk( + ctx: AsyncHandler, + setup_assembly: Self::SetupAssembly, + _hardcoded_finalization_hint: Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK) { + let ctx = ctx.wait(); + let device_setup = + FflonkSnarkVerifierCircuitDeviceSetup::::create_setup_from_assembly_on_device( + &setup_assembly, + ) + .unwrap(); + let vk = device_setup.get_verification_key(); + drop(ctx); + ( + FflonkSnarkVerifierCircuitDeviceSetupWrapper(device_setup), + vk, + ) + } +} diff --git a/crates/proof-compression/src/proof_system/interface.rs b/crates/proof-compression/src/proof_system/interface.rs new file mode 100644 index 0000000..119ec1f --- /dev/null +++ b/crates/proof-compression/src/proof_system/interface.rs @@ -0,0 +1,129 @@ +use super::*; + +use circuit_definitions::circuit_definitions::aux_layer::compression::{ + CompressionLayerCircuit, ProofCompressionFunction, +}; +use shivini::{ + boojum::cs::implementations::{ + fast_serialization::MemcopySerializable, transcript::Transcript, + }, + GPUPoWRunner, GpuTreeHasher, +}; + +pub(crate) trait ProofSystemDefinition: Sized { + type FieldElement; + type ExternalWitnessData; + type Precomputation: MemcopySerializable + Send + Sync + 'static; + type Proof: serde::Serialize + serde::de::DeserializeOwned; + type VK: serde::Serialize + serde::de::DeserializeOwned + Send + Sync + Clone + 'static; + type FinalizationHint: serde::Serialize + serde::de::DeserializeOwned + Clone; + type Allocator: std::alloc::Allocator + Default; + type ProvingAssembly: Sized + Send + Sync + 'static; + type Transcript; + fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData; + fn verify(_: &Self::Proof, _: &Self::VK) -> bool; +} + +pub(crate) trait CompressionProofSystem: + ProofCompressionFunction< + ThisLayerHasher: GpuTreeHasher, + ThisLayerTranscript: Transcript, + ThisLayerPoW: GPUPoWRunner, + > + ProofSystemDefinition +{ + type ContextConfig: Send + Sync + 'static; + type Context: Send + Sync + 'static; + type AuxConfig; + fn get_context_config() -> Self::ContextConfig; + fn get_context_config_from_hint(_: &Self::FinalizationHint) -> Self::ContextConfig; + fn init_context(config: Self::ContextConfig) -> Self::Context; + fn aux_config_from_assembly(proving_assembly: &Self::ProvingAssembly) -> Self::AuxConfig; + + fn synthesize_for_proving( + circuit: CompressionLayerCircuit, + finalization_hint: Self::FinalizationHint, + ) -> Self::ProvingAssembly; + + fn prove( + _: AsyncHandler, + _: Self::ProvingAssembly, + _: Self::AuxConfig, + _: AsyncHandler, + _: Self::FinalizationHint, + _: &Self::VK, + ) -> Self::Proof; + + fn prove_from_witnesses( + _: AsyncHandler, + _: Self::ExternalWitnessData, + _: Self::AuxConfig, + _: AsyncHandler, + _: Self::FinalizationHint, + _: &Self::VK, + ) -> Self::Proof; +} + +pub(crate) trait CompressionProofSystemExt: CompressionProofSystem { + type SetupAssembly; + fn generate_precomputation_and_vk( + _: AsyncHandler, + _: Self::SetupAssembly, + _: &Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK); + fn synthesize_for_setup( + circuit: CompressionLayerCircuit, + ) -> (Self::FinalizationHint, Self::SetupAssembly); +} + +pub(crate) trait SnarkWrapperProofSystem: ProofSystemDefinition { + type Circuit; + type Context: Send + Sync + 'static; + type CRS: Send + Sync + 'static; + fn init_context(crs: AsyncHandler) -> Self::Context; + fn load_compact_raw_crs(src: R) -> Self::CRS; + fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly; + fn prove( + _: AsyncHandler, + _: Self::ProvingAssembly, + _: AsyncHandler, + _: Self::FinalizationHint, + ) -> Self::Proof; + + fn prove_from_witnesses( + _: AsyncHandler, + _: Vec, + _: AsyncHandler, + _: Self::FinalizationHint, + ) -> Self::Proof; +} + +pub(crate) trait SnarkWrapperProofSystemExt: SnarkWrapperProofSystem { + type SetupAssembly; + fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly; + fn generate_precomputation_and_vk( + _: AsyncHandler, + _: Self::SetupAssembly, + _: Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK); +} + +pub(crate) struct MarkerProofSystem; + +impl ProofSystemDefinition for MarkerProofSystem { + type FieldElement = (); + type Precomputation = MarkerPrecomputation; + type Proof = (); + type VK = (); + type ExternalWitnessData = (); + type FinalizationHint = (); + type Allocator = std::alloc::Global; + type ProvingAssembly = (); + type Transcript = (); + fn verify(_: &Self::Proof, _: &Self::VK) -> bool { + unreachable!() + } + + fn take_witnesses(_proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { + unreachable!() + } +} diff --git a/crates/proof-compression/src/proof_system/mod.rs b/crates/proof-compression/src/proof_system/mod.rs new file mode 100644 index 0000000..0841e5d --- /dev/null +++ b/crates/proof-compression/src/proof_system/mod.rs @@ -0,0 +1,18 @@ +use std::alloc::Allocator; + +use super::*; + +mod boojum; +pub(crate) use boojum::*; +mod crs; +pub(crate) use crs::*; +mod fflonk; +pub(crate) use fflonk::*; +mod interface; +pub(crate) use interface::*; +mod plonk; +pub(crate) use plonk::*; + + +pub(crate) use ::fflonk::fflonk_cpu::franklin_crypto; +pub(crate) use franklin_crypto::bellman; \ No newline at end of file diff --git a/crates/proof-compression/src/proof_system/plonk.rs b/crates/proof-compression/src/proof_system/plonk.rs new file mode 100644 index 0000000..2fcbd52 --- /dev/null +++ b/crates/proof-compression/src/proof_system/plonk.rs @@ -0,0 +1,237 @@ +use bellman::{ + kate_commitment::{Crs, CrsForMonomialForm}, + plonk::{ + better_better_cs::{ + cs::{ + Assembly, Circuit, PlonkCsWidth4WithNextStepAndCustomGatesParams, + SynthesisModeGenerateSetup, SynthesisModeProve, + }, + gates::selector_optimized_with_d_next::SelectorOptimizedWidth4MainGateWithDNext, + }, + commitments::transcript::keccak_transcript::RollingKeccakTranscript, + }, +}; +use circuit_definitions::circuit_definitions::aux_layer::ZkSyncSnarkWrapperCircuit; +use gpu_prover::{AsyncSetup, DeviceMemoryManager, ManagerConfigs}; + +use super::*; + +use bellman::plonk::better_better_cs::{ + cs::VerificationKey as PlonkVerificationKey, proof::Proof as PlonkProof, +}; + +use bellman::bn256::{Bn256, Fr}; + +pub(crate) type PlonkSnarkVerifierCircuit = ZkSyncSnarkWrapperCircuit; +pub(crate) type PlonkSnarkVerifierCircuitVK = + PlonkVerificationKey; +pub(crate) type PlonkSnarkVerifierCircuitProof = PlonkProof; +pub(crate) type PlonkSnarkVerifierCircuitDeviceSetup = AsyncSetup; + +type PlonkAssembly = Assembly< + Bn256, + PlonkCsWidth4WithNextStepAndCustomGatesParams, + SelectorOptimizedWidth4MainGateWithDNext, + CSConfig, + A, +>; + +pub(crate) struct UnsafePlonkProverDeviceMemoryManagerWrapper( + DeviceMemoryManager, +); +impl GenericWrapper for UnsafePlonkProverDeviceMemoryManagerWrapper { + type Inner = DeviceMemoryManager; + + fn into_inner(self) -> Self::Inner { + self.0 + } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } +} +unsafe impl Send for UnsafePlonkProverDeviceMemoryManagerWrapper {} +unsafe impl Sync for UnsafePlonkProverDeviceMemoryManagerWrapper {} +pub(crate) struct PlonkProverDeviceMemoryManagerConfig; + +impl ManagerConfigs for PlonkProverDeviceMemoryManagerConfig { + const NUM_GPUS_LOG: usize = 0; + const FULL_SLOT_SIZE_LOG: usize = 24; + const NUM_SLOTS: usize = 29; + const NUM_HOST_SLOTS: usize = 2; +} + +pub(crate) struct PlonkSnarkWrapper; + +impl ProofSystemDefinition for PlonkSnarkWrapper { + type FieldElement = Fr; + type Precomputation = PlonkSnarkVerifierCircuitDeviceSetupWrapper; + type ExternalWitnessData = Vec; + type Proof = PlonkSnarkVerifierCircuitProof; + type VK = PlonkSnarkVerifierCircuitVK; + type FinalizationHint = usize; + #[cfg(feature = "allocator")] + type Allocator = gpu_prover::cuda_bindings::CudaAllocator; + #[cfg(not(feature = "allocator"))] + type Allocator = std::alloc::Global; + type ProvingAssembly = PlonkAssembly; + type Transcript = RollingKeccakTranscript; + fn take_witnesses( + _proving_assembly: &mut Self::ProvingAssembly, + ) -> Vec { + // let input_assignments = + // std::mem::replace(&mut proving_assembly.input_assingments, Vec::new()); + // let aux_assignments = std::mem::replace( + // &mut proving_assembly.aux_assingments, + // Vec::new_in(Self::Allocator::default()), + // ); + + todo!() + } + fn verify(proof: &Self::Proof, vk: &Self::VK) -> bool { + bellman::plonk::better_better_cs::verifier::verify::<_, _, Self::Transcript>( + vk, proof, None, + ) + .unwrap() + } +} + +impl SnarkWrapperProofSystem for PlonkSnarkWrapper { + type Circuit = PlonkSnarkVerifierCircuit; + type Context = ( + UnsafePlonkProverDeviceMemoryManagerWrapper, + Vec, + ); + + type CRS = bellman::kate_commitment::Crs< + bellman::compact_bn256::Bn256, + CrsForMonomialForm, + Self::Allocator, + >; + fn load_compact_raw_crs(src: R) -> Self::CRS { + let num_g1_points_for_crs = 1 << PlonkProverDeviceMemoryManagerConfig::FULL_SLOT_SIZE_LOG; + read_crs_from_raw_compact_form(src, num_g1_points_for_crs).unwrap() + } + + fn init_context(compact_raw_crs: AsyncHandler) -> Self::Context { + let device_ids: Vec<_> = + (0..::NUM_GPUS).collect(); + let compact_raw_crs = compact_raw_crs.wait(); + let g2_bases = compact_raw_crs.g2_monomial_bases.as_ref().to_vec(); + let manager = DeviceMemoryManager::init(&device_ids, &compact_raw_crs.g1_bases).unwrap(); + ( + UnsafePlonkProverDeviceMemoryManagerWrapper(manager), + g2_bases, + ) + } + + fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly { + let mut proving_assembly = PlonkAssembly::::new(); + circuit + .synthesize(&mut proving_assembly) + .expect("must work"); + proving_assembly + } + + fn prove( + ctx: AsyncHandler, + mut proving_assembly: Self::ProvingAssembly, + precomputation: AsyncHandler, + finalization_hint: Self::FinalizationHint, + ) -> Self::Proof { + assert!(proving_assembly.is_satisfied()); + assert!(finalization_hint.is_power_of_two()); + proving_assembly.finalize_to_size_log_2(finalization_hint.trailing_zeros() as usize); + let domain_size = proving_assembly.n() + 1; + assert!(domain_size.is_power_of_two()); + assert_eq!(domain_size, finalization_hint); + + let (ctx, _) = ctx.wait(); + let mut ctx = ctx.into_inner(); + let mut precomputation = precomputation.wait().into_inner(); + let worker = bellman::worker::Worker::new(); + let proof = gpu_prover::create_proof::<_, _, Self::Transcript, _>( + &proving_assembly, + &mut ctx, + &worker, + &mut precomputation, + None, + ) + .unwrap(); + + ctx.free_all_slots(); + + proof + } + + fn prove_from_witnesses( + _: AsyncHandler, + _: Vec, + _: AsyncHandler, + _: Self::FinalizationHint, + ) -> Self::Proof { + unimplemented!() + } +} + +impl SnarkWrapperProofSystemExt for PlonkSnarkWrapper { + type SetupAssembly = PlonkAssembly; + + fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly { + let mut setup_assembly = + PlonkAssembly::::new(); + circuit.synthesize(&mut setup_assembly).expect("must work"); + setup_assembly + } + + fn generate_precomputation_and_vk( + ctx: AsyncHandler, + mut setup_assembly: Self::SetupAssembly, + hardcoded_finalization_hint: Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK) { + assert!(setup_assembly.is_satisfied()); + assert!(hardcoded_finalization_hint.is_power_of_two()); + setup_assembly + .finalize_to_size_log_2(hardcoded_finalization_hint.trailing_zeros() as usize); + let domain_size = setup_assembly.n() + 1; + assert!(domain_size.is_power_of_two()); + assert_eq!(domain_size, hardcoded_finalization_hint); + + let (ctx, compact_g2_bases) = ctx.wait(); + let mut ctx = ctx.into_inner(); + let worker = bellman::worker::Worker::new(); + #[cfg(feature = "allocator")] + let mut precomputation = + AsyncSetup::::allocate(hardcoded_finalization_hint); + #[cfg(not(feature = "allocator"))] + let mut precomputation = AsyncSetup::allocate(hardcoded_finalization_hint); + precomputation + .generate_from_assembly(&worker, &setup_assembly, &mut ctx) + .unwrap(); + + let mut dummy_crs = Crs::::dummy_crs(1); + let mut g2_bases = vec![]; + use bellman::CurveAffine; + for base in compact_g2_bases.into_iter() { + let (x, y) = base.into_xy_unchecked(); + let base = bellman::bn256::G2Affine::from_xy_unchecked(x, y); + g2_bases.push(base); + } + + dummy_crs.g2_monomial_bases = std::sync::Arc::new(g2_bases); + let vk = gpu_prover::compute_vk_from_assembly::< + _, + _, + PlonkCsWidth4WithNextStepAndCustomGatesParams, + SynthesisModeGenerateSetup, + >(&mut ctx, &setup_assembly, &dummy_crs) + .unwrap(); + + ctx.free_all_slots(); + + ( + PlonkSnarkVerifierCircuitDeviceSetupWrapper::from_inner(precomputation), + vk, + ) + } +} diff --git a/crates/proof-compression/src/serialization.rs b/crates/proof-compression/src/serialization.rs index 19f6579..17c3967 100644 --- a/crates/proof-compression/src/serialization.rs +++ b/crates/proof-compression/src/serialization.rs @@ -1,12 +1,13 @@ use super::*; +use fflonk::bellman::bn256::Bn256; use gpu_prover::ManagerConfigs; use shivini::{ boojum::cs::implementations::fast_serialization::MemcopySerializable, cs::GpuSetup, GpuTreeHasher, }; -pub trait SerializationWrapper: Sized { +pub(crate) trait GenericWrapper: Sized { type Inner; fn into_inner(self) -> Self::Inner; fn from_inner(inner: Self::Inner) -> Self; @@ -14,8 +15,8 @@ pub trait SerializationWrapper: Sized { use crate::PlonkSnarkVerifierCircuitDeviceSetup; -pub struct BoojumDeviceSetupWrapper(GpuSetup); -impl SerializationWrapper for BoojumDeviceSetupWrapper +pub(crate) struct BoojumDeviceSetupWrapper(GpuSetup); +impl GenericWrapper for BoojumDeviceSetupWrapper where H: GpuTreeHasher, { @@ -29,7 +30,7 @@ where } } -impl boojum::cs::implementations::fast_serialization::MemcopySerializable +impl shivini::boojum::cs::implementations::fast_serialization::MemcopySerializable for BoojumDeviceSetupWrapper { fn write_into_buffer( @@ -44,7 +45,7 @@ impl boojum::cs::implementations::fast_serialization::MemcopyS } } -pub struct PlonkSnarkVerifierCircuitDeviceSetupWrapper(PlonkSnarkVerifierCircuitDeviceSetup); +pub(crate) struct PlonkSnarkVerifierCircuitDeviceSetupWrapper(PlonkSnarkVerifierCircuitDeviceSetup); impl MemcopySerializable for PlonkSnarkVerifierCircuitDeviceSetupWrapper { fn write_into_buffer( @@ -64,7 +65,7 @@ impl MemcopySerializable for PlonkSnarkVerifierCircuitDeviceSetupWrapper { } } -impl SerializationWrapper for PlonkSnarkVerifierCircuitDeviceSetupWrapper { +impl GenericWrapper for PlonkSnarkVerifierCircuitDeviceSetupWrapper { type Inner = PlonkSnarkVerifierCircuitDeviceSetup; fn into_inner(self) -> Self::Inner { self.0 @@ -75,7 +76,7 @@ impl SerializationWrapper for PlonkSnarkVerifierCircuitDeviceSetupWrapper { } } -pub struct FflonkSnarkVerifierCircuitDeviceSetupWrapper( +pub(crate) struct FflonkSnarkVerifierCircuitDeviceSetupWrapper( pub fflonk::FflonkDeviceSetup, ); @@ -97,7 +98,7 @@ where } } -impl SerializationWrapper for FflonkSnarkVerifierCircuitDeviceSetupWrapper +impl GenericWrapper for FflonkSnarkVerifierCircuitDeviceSetupWrapper where A: HostAllocator, { @@ -112,16 +113,16 @@ where } #[derive(serde::Serialize, serde::Deserialize)] -pub struct MarkerPrecomputation; +pub(crate) struct MarkerPrecomputation; impl MemcopySerializable for MarkerPrecomputation { fn write_into_buffer( &self, - dst: W, + _dst: W, ) -> Result<(), Box> { todo!() } - fn read_from_buffer(src: R) -> Result> { + fn read_from_buffer(_src: R) -> Result> { todo!() } } diff --git a/crates/proof-compression/src/compression.rs b/crates/proof-compression/src/step/compression.rs similarity index 95% rename from crates/proof-compression/src/compression.rs rename to crates/proof-compression/src/step/compression.rs index e0df74a..4ae502c 100644 --- a/crates/proof-compression/src/compression.rs +++ b/crates/proof-compression/src/step/compression.rs @@ -1,20 +1,24 @@ use super::*; -use boojum::cs::{ - implementations::{ - fast_serialization::MemcopySerializable, proof::Proof, verifier::VerificationKey, - }, - oracle::TreeHasher, -}; use circuit_definitions::circuit_definitions::{ aux_layer::{ compression::{CompressionLayerCircuit, ProofCompressionFunction}, + compression_modes::{ + CompressionMode1, CompressionMode1ForWrapper, CompressionMode2, CompressionMode3, + CompressionMode4, CompressionMode5ForWrapper, + }, ZkSyncCompressionForWrapperCircuit, ZkSyncCompressionLayerCircuit, }, recursion_layer::RecursiveProofsTreeHasher, }; +use franklin_crypto::boojum::cs::{ + implementations::{ + fast_serialization::MemcopySerializable, proof::Proof, verifier::VerificationKey, + }, + oracle::TreeHasher, +}; -pub trait CompressionStep: CompressionProofSystem { +pub(crate)trait CompressionStep: CompressionProofSystem { type PreviousStepTreeHasher: TreeHasher< GoldilocksField, Output: serde::Serialize + serde::de::DeserializeOwned, @@ -134,7 +138,7 @@ pub trait CompressionStep: CompressionProofSystem { ) -> CompressionLayerCircuit; } -pub trait CompressionStepExt: CompressionProofSystemExt + CompressionStep { +pub(crate)trait CompressionStepExt: CompressionProofSystemExt + CompressionStep { fn precomputae_and_store_compression_circuits(blob_storage: &BS, context_manager: &CM) where BS: BlobStorageExt, diff --git a/crates/proof-compression/src/step/mod.rs b/crates/proof-compression/src/step/mod.rs new file mode 100644 index 0000000..10d9ea8 --- /dev/null +++ b/crates/proof-compression/src/step/mod.rs @@ -0,0 +1,7 @@ +use super::*; + +mod compression; +pub(crate) use compression::*; + +mod snark_wrapper; +pub(crate) use snark_wrapper::*; diff --git a/crates/proof-compression/src/snark_wrapper.rs b/crates/proof-compression/src/step/snark_wrapper.rs similarity index 81% rename from crates/proof-compression/src/snark_wrapper.rs rename to crates/proof-compression/src/step/snark_wrapper.rs index f8e2d23..09bb98d 100644 --- a/crates/proof-compression/src/snark_wrapper.rs +++ b/crates/proof-compression/src/step/snark_wrapper.rs @@ -1,16 +1,18 @@ -use boojum::cs::{ +use circuit_definitions::circuit_definitions::aux_layer::{ + compression::ProofCompressionFunction, + compression_modes::{CompressionMode1ForWrapper, CompressionMode5ForWrapper}, + wrapper::ZkSyncCompressionWrapper, +}; +use franklin_crypto::boojum::cs::{ implementations::{ fast_serialization::MemcopySerializable, proof::Proof, verifier::VerificationKey, }, oracle::TreeHasher, }; -use circuit_definitions::circuit_definitions::aux_layer::{ - compression::ProofCompressionFunction, wrapper::ZkSyncCompressionWrapper, -}; use super::*; -pub trait SnarkWrapperStep: SnarkWrapperProofSystem { +pub(crate)trait SnarkWrapperStep: SnarkWrapperProofSystem { const IS_PLONK: bool; const IS_FFLONK: bool; const PREVIOUS_COMPRESSION_MODE: u8; @@ -19,7 +21,7 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { Output: serde::Serialize + serde::de::DeserializeOwned, >; fn load_finalization_hint( - blob_storage: &BS, + _blob_storage: &BS, ) -> ::FinalizationHint where BS: BlobStorage, @@ -61,6 +63,28 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { serde_json::from_reader(reader).unwrap() } + fn load_compact_raw_crs(blob_storage: &BS) -> AsyncHandler + where + BS: BlobStorage, + Self::Allocator: Send + Sync + 'static, + { + assert!(Self::IS_FFLONK ^ Self::IS_PLONK); + let reader = blob_storage.read_compact_raw_crs(); + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let start = std::time::Instant::now(); + let compact_raw_crs = ::load_compact_raw_crs(reader); + println!( + "Compact raw CRS loading takes {}s", + start.elapsed().as_secs() + ); + sender.send(compact_raw_crs).unwrap(); + receiver + }; + + AsyncHandler::spawn(f) + } + fn get_precomputation( blob_storage: &BS, ) -> AsyncHandler<::Precomputation> @@ -74,13 +98,16 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { }; let f = move || { let (sender, receiver) = std::sync::mpsc::channel(); - + let start = std::time::Instant::now(); let precomputation = <::Precomputation as MemcopySerializable>::read_from_buffer( reader, ) .unwrap(); - + println!( + "Snark wrapper device setup loading takes {}s", + start.elapsed().as_secs() + ); sender.send(precomputation).unwrap(); receiver }; @@ -89,7 +116,8 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { } fn prove_snark_wrapper_step( - ctx_config: AsyncHandler, + ctx_config: AsyncHandler, + precomputation: AsyncHandler, input_proof: Proof, blob_storage: &BS, context_handler: &CI, @@ -101,21 +129,20 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { assert!(Self::IS_FFLONK ^ Self::IS_PLONK); let input_vk = Self::load_previous_vk(blob_storage); - let precomputation = Self::get_precomputation(blob_storage); let ctx = context_handler.init_snark_context::(ctx_config); let finalization_hint = Self::load_finalization_hint(blob_storage); let circuit = Self::build_circuit(input_vk, Some(input_proof)); let proving_assembly = ::synthesize_for_proving(circuit); let vk = Self::load_this_vk(blob_storage); + let proof = ::prove( ctx, proving_assembly, precomputation, finalization_hint, - &vk, ); - // assert!(::verify(&proof, &vk)); + assert!(::verify(&proof, &vk)); proof } @@ -126,9 +153,9 @@ pub trait SnarkWrapperStep: SnarkWrapperProofSystem { ) -> Self::Circuit; } -pub trait SnarkWrapperStepExt: SnarkWrapperProofSystemExt + SnarkWrapperStep { +pub(crate)trait SnarkWrapperStepExt: SnarkWrapperProofSystemExt + SnarkWrapperStep { fn precompute_and_store_snark_wrapper_circuit( - ctx_config: AsyncHandler, + compact_raw_crs: AsyncHandler, blob_storage: &BS, context_manager: &CM, ) where @@ -139,7 +166,7 @@ pub trait SnarkWrapperStepExt: SnarkWrapperProofSystemExt + SnarkWrapperStep { let input_vk = Self::load_previous_vk(blob_storage); let finalization_hint = Self::load_finalization_hint(blob_storage); let circuit = Self::build_circuit(input_vk, None); - let ctx = context_manager.init_snark_context::(ctx_config); + let ctx = context_manager.init_snark_context::(compact_raw_crs); let setup_assembly = ::synthesize_for_setup(circuit); let (precomputation, vk) = @@ -167,7 +194,6 @@ pub trait SnarkWrapperStepExt: SnarkWrapperProofSystemExt + SnarkWrapperStep { } } -pub struct FflonkSnarkWrapper; impl SnarkWrapperStep for FflonkSnarkWrapper { const IS_PLONK: bool = false; const IS_FFLONK: bool = true; @@ -192,7 +218,6 @@ impl SnarkWrapperStep for FflonkSnarkWrapper { } impl SnarkWrapperStepExt for FflonkSnarkWrapper {} -pub struct PlonkSnarkWrapper; impl SnarkWrapperStep for PlonkSnarkWrapper { const IS_PLONK: bool = true; const IS_FFLONK: bool = false; diff --git a/crates/proof-compression/src/test.rs b/crates/proof-compression/src/test.rs index 550884f..ca303ff 100644 --- a/crates/proof-compression/src/test.rs +++ b/crates/proof-compression/src/test.rs @@ -1,9 +1,8 @@ -use circuit_definitions::circuit_definitions::recursion_layer::ZkSyncRecursionLayerProof; - use super::*; +use circuit_definitions::circuit_definitions::recursion_layer::ZkSyncRecursionLayerProof; #[test] -pub fn test_proof_chain_with_fflonk() { +fn test_proof_chain_with_fflonk() { let scheduler_proof_file = std::fs::File::open("./data/scheduler_recursive_proof.json").unwrap(); let scheduler_proof: ZkSyncRecursionLayerProof = @@ -19,7 +18,8 @@ pub fn test_proof_chain_with_fflonk() { println!("Final fflonk snark wrapper roof saved at {proof_file_path}"); } -pub fn test_proof_chain_with_plonk() { +#[test] +fn test_proof_chain_with_plonk() { let scheduler_proof_file = std::fs::File::open("./data/scheduler_recursive_proof.json").unwrap(); let scheduler_proof: ZkSyncRecursionLayerProof = @@ -27,16 +27,29 @@ pub fn test_proof_chain_with_plonk() { let scheduler_proof = scheduler_proof.into_inner(); let simple_blob_storage = FileSystemBlobStorage; - let proof = run_proof_chain_with_fflonk(scheduler_proof, &simple_blob_storage); + let proof = run_proof_chain_with_plonk(scheduler_proof, &simple_blob_storage); - let proof_file_path = format!("./data/final_fflonk_proof.json"); + let proof_file_path = format!("./data/final_plonk_proof.json"); let proof_file = std::fs::File::create(&proof_file_path).unwrap(); serde_json::to_writer(proof_file, &proof).unwrap(); - println!("Final fflonk snark wrapper roof saved at {proof_file_path}"); + println!("Final plonk snark wrapper roof saved at {proof_file_path}"); } #[test] -fn test_precompute_compression_chain_artifacts() { +fn test_precompute_compression_chain_artifacts_with_fflonk() { let simple_blob_storage = FileSystemBlobStorage; precompute_proof_chain_with_fflonk(&simple_blob_storage); } + +#[test] +fn test_precompute_compression_chain_artifacts_with_plonk() { + let simple_blob_storage = FileSystemBlobStorage; + precompute_proof_chain_with_plonk(&simple_blob_storage); +} + +#[test] +fn test_create_compact_raw_crs() { + let blob_storage = FileSystemBlobStorage; + let writer = blob_storage.write_compact_raw_crs(); + create_compact_raw_crs(writer); +} From eab05b82ef947930c150bd63192207bd1a84814e Mon Sep 17 00:00:00 2001 From: saitima Date: Sat, 11 Jan 2025 22:34:40 +0300 Subject: [PATCH 08/18] chore(fflonk): allocator feature --- crates/fflonk/Cargo.toml | 5 ++- crates/fflonk/src/convenience.rs | 9 ++-- crates/fflonk/src/setup.rs | 43 +++++++++---------- .../src/proof_system/boojum.rs | 14 +++--- .../src/proof_system/fflonk.rs | 3 +- .../proof-compression/src/proof_system/mod.rs | 3 +- .../proof-compression/src/step/compression.rs | 4 +- .../src/step/snark_wrapper.rs | 4 +- 8 files changed, 41 insertions(+), 44 deletions(-) diff --git a/crates/fflonk/Cargo.toml b/crates/fflonk/Cargo.toml index 46638b1..3236024 100644 --- a/crates/fflonk/Cargo.toml +++ b/crates/fflonk/Cargo.toml @@ -12,7 +12,7 @@ description = "CUDA implementation of the fflonk prover" exclude = ["/data"] [dependencies] -fflonk-cpu = {workspace = true} +fflonk-cpu = {workspace = true, optional = true} circuit_definitions.workspace = true gpu-ffi.workspace = true rand = "0.4" @@ -24,5 +24,6 @@ serde_json = "1" serde_derive = "1" [features] -default = [] +default = ["fflonk-cpu"] sanity = [] +allocator = ["fflonk-cpu/allocator"] diff --git a/crates/fflonk/src/convenience.rs b/crates/fflonk/src/convenience.rs index 7affafc..c0c95e0 100644 --- a/crates/fflonk/src/convenience.rs +++ b/crates/fflonk/src/convenience.rs @@ -2,9 +2,7 @@ use bellman::{ bn256::{Bn256, Fr}, kate_commitment::{Crs, CrsForMonomialForm}, plonk::{ - better_better_cs::cs::{ - Circuit, SynthesisModeGenerateSetup, SynthesisModeProve, SynthesisModeTesting, - }, + better_better_cs::cs::{Circuit, SynthesisModeProve, SynthesisModeTesting}, commitments::transcript::keccak_transcript::RollingKeccakTranscript, }, }; @@ -14,7 +12,7 @@ use circuit_definitions::circuit_definitions::aux_layer::{ }; use fflonk::{FflonkAssembly, L1_VERIFIER_DOMAIN_SIZE_LOG}; -pub type FflonkSnarkVerifierCircuitDeviceSetup = +pub type FflonkSnarkVerifierCircuitDeviceSetup = FflonkDeviceSetup; use super::*; @@ -190,7 +188,8 @@ pub fn precompute_and_save_setup_and_vk_for_fflonk_snark_circuit( println!("Generating fflonk setup data on the device"); let device_setup = - FflonkSnarkVerifierCircuitDeviceSetup::create_setup_on_device(&circuit).unwrap(); + FflonkSnarkVerifierCircuitDeviceSetup::::create_setup_on_device(&circuit) + .unwrap(); let setup_file_path = format!("{}/final_snark_device_setup.bin", path); println!("Saving setup into file {setup_file_path}"); let device_setup_file = std::fs::File::create(&setup_file_path).unwrap(); diff --git a/crates/fflonk/src/setup.rs b/crates/fflonk/src/setup.rs index 31557d1..839d779 100644 --- a/crates/fflonk/src/setup.rs +++ b/crates/fflonk/src/setup.rs @@ -254,12 +254,7 @@ impl, A: HostAllocator> FflonkDeviceSetup { assert_eq!(num_polys, 5); let mut main_gate_selector_monomials = vec![]; for _ in 0..num_polys { - let num_values = reader.read_u64::()?; - let mut coeffs = Vec::with_capacity_in(num_values as usize, A::default()); - for _ in 0..num_values { - let el = read_fr(&mut reader)?; - coeffs.push(el); - } + let coeffs = read_raw_fr_vec::<_, _, A>(&mut reader)?; main_gate_selector_monomials.push(coeffs); } @@ -267,12 +262,16 @@ impl, A: HostAllocator> FflonkDeviceSetup { assert_eq!(num_polys, 3); let mut variable_indexes = vec![]; for _ in 0..num_polys { - let num_values = reader.read_u64::()?; - let mut indexes = Vec::with_capacity_in(num_values as usize, A::default()); - for _ in 0..num_values { - let el = reader.read_u32::()?; - indexes.push(el); - } + let num_values = reader.read_u64::()? as usize; + let mut indexes = Vec::with_capacity_in(num_values, A::default()); + let indexes_buf = unsafe { + indexes.set_len(num_values); + std::slice::from_raw_parts_mut( + indexes.as_mut_ptr() as *mut u8, + num_values * std::mem::size_of::(), + ) + }; + reader.read_exact(indexes_buf)?; variable_indexes.push(indexes); } @@ -293,14 +292,18 @@ impl, A: HostAllocator> FflonkDeviceSetup { use byteorder::{BigEndian, WriteBytesExt}; writer.write_u64::(self.main_gate_selector_monomials.len() as u64)?; for mon in self.main_gate_selector_monomials.iter() { - write_fr_vec(&mon, &mut writer)?; + write_raw_fr_slice(&mon, &mut writer)?; } writer.write_u64::(self.variable_indexes.len() as u64)?; for col in self.variable_indexes.iter() { writer.write_u64::(col.len() as u64)?; - for el in col { - writer.write_u32::(*el)?; - } + let buf = unsafe { + std::slice::from_raw_parts( + col.as_ptr() as *mut u8, + col.len() * std::mem::size_of::(), + ) + }; + writer.write_all(buf)?; } write_curve_affine(&self.c0_commitment, &mut writer)?; write_curve_affine(&self.g2_elems[0], &mut writer)?; @@ -320,9 +323,7 @@ pub fn read_raw_fr_vec( values.as_mut_ptr() as *mut u8, num_values * std::mem::size_of::(), ); - // src.read_exact(buf)?; - let mut dst = &mut buf[..]; - std::io::copy(&mut src, &mut dst)?; + src.read_exact(buf)?; } Ok(values) @@ -341,9 +342,7 @@ pub fn write_raw_fr_slice( src_values.as_ptr() as *mut u8, num_values * std::mem::size_of::(), ); - let mut reader = &buf[..]; - // dst.write_all(buf)?; - std::io::copy(&mut reader, &mut dst)?; + dst.write_all(buf)?; } Ok(()) diff --git a/crates/proof-compression/src/proof_system/boojum.rs b/crates/proof-compression/src/proof_system/boojum.rs index 2dadf6b..d59440e 100644 --- a/crates/proof-compression/src/proof_system/boojum.rs +++ b/crates/proof-compression/src/proof_system/boojum.rs @@ -226,14 +226,12 @@ where let geometry = circuit.geometry(); let (max_trace_len, num_vars) = circuit.size_hint(); - let builder_impl = - boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< - GoldilocksField, - GoldilocksField, - SetupCSConfig, - >::new(geometry, max_trace_len.unwrap()); - let builder = - boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); + let builder_impl = boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< + GoldilocksField, + GoldilocksField, + SetupCSConfig, + >::new(geometry, max_trace_len.unwrap()); + let builder = boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); let builder = circuit.configure_builder_proxy(builder); let mut cs = builder.build(num_vars.unwrap()); diff --git a/crates/proof-compression/src/proof_system/fflonk.rs b/crates/proof-compression/src/proof_system/fflonk.rs index 7b2a12e..c7af650 100644 --- a/crates/proof-compression/src/proof_system/fflonk.rs +++ b/crates/proof-compression/src/proof_system/fflonk.rs @@ -22,7 +22,8 @@ use circuit_definitions::circuit_definitions::aux_layer::ZkSyncSnarkWrapperCircu use super::*; pub(crate) use ::fflonk::HostAllocator; pub(crate) type FflonkSnarkVerifierCircuit = ZkSyncSnarkWrapperCircuitNoLookupCustomGate; -pub(crate) type FflonkSnarkVerifierCircuitVK = FflonkVerificationKey; +pub(crate) type FflonkSnarkVerifierCircuitVK = + FflonkVerificationKey; pub(crate) type FflonkSnarkVerifierCircuitProof = FflonkProof; type FflonkAssembly = Assembly; pub(crate) struct FflonkSnarkWrapper; diff --git a/crates/proof-compression/src/proof_system/mod.rs b/crates/proof-compression/src/proof_system/mod.rs index 0841e5d..e47347a 100644 --- a/crates/proof-compression/src/proof_system/mod.rs +++ b/crates/proof-compression/src/proof_system/mod.rs @@ -13,6 +13,5 @@ pub(crate) use interface::*; mod plonk; pub(crate) use plonk::*; - pub(crate) use ::fflonk::fflonk_cpu::franklin_crypto; -pub(crate) use franklin_crypto::bellman; \ No newline at end of file +pub(crate) use franklin_crypto::bellman; diff --git a/crates/proof-compression/src/step/compression.rs b/crates/proof-compression/src/step/compression.rs index 4ae502c..6086c3f 100644 --- a/crates/proof-compression/src/step/compression.rs +++ b/crates/proof-compression/src/step/compression.rs @@ -18,7 +18,7 @@ use franklin_crypto::boojum::cs::{ oracle::TreeHasher, }; -pub(crate)trait CompressionStep: CompressionProofSystem { +pub(crate) trait CompressionStep: CompressionProofSystem { type PreviousStepTreeHasher: TreeHasher< GoldilocksField, Output: serde::Serialize + serde::de::DeserializeOwned, @@ -138,7 +138,7 @@ pub(crate)trait CompressionStep: CompressionProofSystem { ) -> CompressionLayerCircuit; } -pub(crate)trait CompressionStepExt: CompressionProofSystemExt + CompressionStep { +pub(crate) trait CompressionStepExt: CompressionProofSystemExt + CompressionStep { fn precomputae_and_store_compression_circuits(blob_storage: &BS, context_manager: &CM) where BS: BlobStorageExt, diff --git a/crates/proof-compression/src/step/snark_wrapper.rs b/crates/proof-compression/src/step/snark_wrapper.rs index 09bb98d..ca3faec 100644 --- a/crates/proof-compression/src/step/snark_wrapper.rs +++ b/crates/proof-compression/src/step/snark_wrapper.rs @@ -12,7 +12,7 @@ use franklin_crypto::boojum::cs::{ use super::*; -pub(crate)trait SnarkWrapperStep: SnarkWrapperProofSystem { +pub(crate) trait SnarkWrapperStep: SnarkWrapperProofSystem { const IS_PLONK: bool; const IS_FFLONK: bool; const PREVIOUS_COMPRESSION_MODE: u8; @@ -153,7 +153,7 @@ pub(crate)trait SnarkWrapperStep: SnarkWrapperProofSystem { ) -> Self::Circuit; } -pub(crate)trait SnarkWrapperStepExt: SnarkWrapperProofSystemExt + SnarkWrapperStep { +pub(crate) trait SnarkWrapperStepExt: SnarkWrapperProofSystemExt + SnarkWrapperStep { fn precompute_and_store_snark_wrapper_circuit( compact_raw_crs: AsyncHandler, blob_storage: &BS, From ab43d6179d1c0950151f34fc6f6e08ec577ae274 Mon Sep 17 00:00:00 2001 From: saitima Date: Mon, 13 Jan 2025 18:30:53 +0300 Subject: [PATCH 09/18] pinned memory for fflonk combined monomials --- crates/proof-compression/src/chain.rs | 130 ++++++++++++++---- .../proof-compression/src/proof_system/crs.rs | 31 ++--- .../src/proof_system/fflonk.rs | 16 ++- .../src/proof_system/interface.rs | 3 +- .../src/proof_system/plonk.rs | 36 ++--- .../src/step/snark_wrapper.rs | 4 + crates/proof-compression/src/test.rs | 31 +---- 7 files changed, 154 insertions(+), 97 deletions(-) diff --git a/crates/proof-compression/src/chain.rs b/crates/proof-compression/src/chain.rs index 3a60aa5..fffeaa7 100644 --- a/crates/proof-compression/src/chain.rs +++ b/crates/proof-compression/src/chain.rs @@ -1,10 +1,74 @@ -use circuit_definitions::circuit_definitions::aux_layer::compression_modes::{ - CompressionMode1, CompressionMode1ForWrapper, CompressionMode2, CompressionMode3, - CompressionMode4, CompressionMode5ForWrapper, +use circuit_definitions::circuit_definitions::aux_layer::{ + compression_modes::{ + CompressionMode1, CompressionMode1ForWrapper, CompressionMode2, CompressionMode3, + CompressionMode4, CompressionMode5ForWrapper, + }, + ZkSyncCompressionForWrapperProof, ZkSyncCompressionLayerProof, }; +use fflonk::FflonkSnarkVerifierCircuitProof; use super::*; +pub trait ProofStorage { + fn get_scheduler_proof(&self) -> SchedulerProof; + fn save_compression_layer_proof(&mut self, circuit_id: u8, proof: ZkSyncCompressionLayerProof); + fn save_compression_wrapper_proof( + &mut self, + circuit_id: u8, + proof: ZkSyncCompressionForWrapperProof, + ); + fn save_plonk_proof(&mut self, proof: PlonkSnarkVerifierCircuitProof); + fn save_fflonk_proof(&mut self, proof: FflonkSnarkVerifierCircuitProof); +} + +pub struct SimpleProofStorage { + compression_layer_storage: std::collections::HashMap, + compression_wrapper_storage: std::collections::HashMap, + fflonk: Option, + plonk: Option, +} + +impl SimpleProofStorage { + pub fn new() -> Self { + Self { + compression_layer_storage: std::collections::HashMap::new(), + compression_wrapper_storage: std::collections::HashMap::new(), + fflonk: None, + plonk: None, + } + } +} + +impl ProofStorage for SimpleProofStorage { + fn get_scheduler_proof(&self) -> SchedulerProof { + let scheduler_proof_file = + std::fs::File::open("./data/scheduler_recursive_proof.json").unwrap(); + let scheduler_proof: circuit_definitions::circuit_definitions::recursion_layer::ZkSyncRecursionLayerProof = + serde_json::from_reader(&scheduler_proof_file).unwrap(); + let scheduler_proof = scheduler_proof.into_inner(); + scheduler_proof + } + fn save_compression_layer_proof(&mut self, circuit_id: u8, proof: ZkSyncCompressionLayerProof) { + self.compression_layer_storage.insert(circuit_id, proof); + } + + fn save_compression_wrapper_proof( + &mut self, + circuit_id: u8, + proof: ZkSyncCompressionForWrapperProof, + ) { + self.compression_wrapper_storage.insert(circuit_id, proof); + } + + fn save_plonk_proof(&mut self, proof: PlonkSnarkVerifierCircuitProof) { + self.plonk = Some(proof); + } + + fn save_fflonk_proof(&mut self, proof: FflonkSnarkVerifierCircuitProof) { + self.fflonk = Some(proof); + } +} + pub enum SnarkWrapper { Plonk, FFfonk, @@ -20,61 +84,72 @@ pub type SchedulerProof = franklin_crypto::boojum::cs::implementations::proof::P GoldilocksExt2, >; -pub fn run_proof_chain( - input_proof: SchedulerProof, +pub fn run_proof_chain( snark_wrapper: SnarkWrapper, blob_storage: &BS, -) -> SnarkWrapperProof -where + proof_storage: &mut PS, +) where BS: BlobStorage, + PS: ProofStorage, { match snark_wrapper { - SnarkWrapper::Plonk => { - let proof = run_proof_chain_with_plonk(input_proof, blob_storage); - SnarkWrapperProof::Plonk(proof) - } - SnarkWrapper::FFfonk => { - let proof = run_proof_chain_with_fflonk(input_proof, blob_storage); - SnarkWrapperProof::FFfonk(proof) - } + SnarkWrapper::Plonk => run_proof_chain_with_plonk(blob_storage, proof_storage), + SnarkWrapper::FFfonk => run_proof_chain_with_fflonk(blob_storage, proof_storage), } } -pub fn run_proof_chain_with_fflonk( - input_proof: SchedulerProof, - blob_storage: &BS, -) -> FflonkSnarkVerifierCircuitProof +pub fn run_proof_chain_with_fflonk(blob_storage: &BS, proof_storage: &mut PS) where BS: BlobStorage, + PS: ProofStorage, { let context_manager = SimpleContextManager::new(); let start = std::time::Instant::now(); + ::run_pre_initialization_tasks(); let compact_raw_crs = ::load_compact_raw_crs(blob_storage); let fflonk_precomputation = FflonkSnarkWrapper::get_precomputation(blob_storage); + let input_proof = proof_storage.get_scheduler_proof(); + let next_proof = CompressionMode1::prove_compression_step(input_proof, blob_storage, &context_manager); + let compression_proof_1 = ZkSyncCompressionLayerProof::from_inner(1, next_proof.clone()); + proof_storage.save_compression_layer_proof(1, compression_proof_1.clone()); + let next_proof = CompressionMode2::prove_compression_step::<_, SimpleContextManager>( next_proof, blob_storage, &context_manager, ); + let compression_proof_2 = ZkSyncCompressionLayerProof::from_inner(2, next_proof.clone()); + proof_storage.save_compression_layer_proof(2, compression_proof_2.clone()); + let next_proof = CompressionMode3::prove_compression_step::<_, SimpleContextManager>( next_proof, blob_storage, &context_manager, ); + let compression_proof_3 = ZkSyncCompressionLayerProof::from_inner(3, next_proof.clone()); + proof_storage.save_compression_layer_proof(3, compression_proof_3.clone()); + let next_proof = CompressionMode4::prove_compression_step::<_, SimpleContextManager>( next_proof, blob_storage, &context_manager, ); + let compression_proof_4 = ZkSyncCompressionLayerProof::from_inner(4, next_proof.clone()); + proof_storage.save_compression_layer_proof(4, compression_proof_4.clone()); + let next_proof = CompressionMode5ForWrapper::prove_compression_step::<_, SimpleContextManager>( next_proof, blob_storage, &context_manager, ); + proof_storage.save_compression_wrapper_proof( + 5, + ZkSyncCompressionForWrapperProof::from_inner(5, next_proof.clone()), + ); println!( "Proving entire compression chain took {}s", start.elapsed().as_secs() @@ -90,7 +165,7 @@ where "Proving entire chain with snark wrapper took {}s", start.elapsed().as_secs() ); - final_proof + proof_storage.save_fflonk_proof(final_proof); } pub fn precompute_proof_chain_with_fflonk(blob_storage: &BS) @@ -98,6 +173,7 @@ where BS: BlobStorageExt, { let context_manager = SimpleContextManager::new(); + ::run_pre_initialization_tasks(); let compact_raw_crs = ::load_compact_raw_crs(blob_storage); @@ -125,12 +201,10 @@ where ); } -pub fn run_proof_chain_with_plonk( - input_proof: SchedulerProof, - blob_storage: &BS, -) -> PlonkSnarkVerifierCircuitProof +pub fn run_proof_chain_with_plonk(blob_storage: &BS, proof_storage: &mut PS) where BS: BlobStorage, + PS: ProofStorage, { let context_manager = SimpleContextManager::new(); let start = std::time::Instant::now(); @@ -138,11 +212,17 @@ where ::load_compact_raw_crs(blob_storage); let plonk_precomputation = PlonkSnarkWrapper::get_precomputation(blob_storage); + let input_proof = proof_storage.get_scheduler_proof(); + let next_proof = CompressionMode1ForWrapper::prove_compression_step( input_proof, blob_storage, &context_manager, ); + proof_storage.save_compression_wrapper_proof( + 1, + ZkSyncCompressionForWrapperProof::from_inner(1, next_proof.clone()), + ); let final_proof = PlonkSnarkWrapper::prove_snark_wrapper_step( compact_raw_crs, @@ -155,7 +235,7 @@ where "Entire compression chain with plonk took {}s", start.elapsed().as_secs() ); - final_proof + proof_storage.save_plonk_proof(final_proof); } pub fn precompute_proof_chain_with_plonk(blob_storage: &BS) diff --git a/crates/proof-compression/src/proof_system/crs.rs b/crates/proof-compression/src/proof_system/crs.rs index e63292f..8cf0cec 100644 --- a/crates/proof-compression/src/proof_system/crs.rs +++ b/crates/proof-compression/src/proof_system/crs.rs @@ -1,7 +1,7 @@ use super::*; -use ::fflonk::bellman::{ - self, - kate_commitment::{Crs, CrsForMonomialForm}, +use ::fflonk::{ + bellman::kate_commitment::{Crs, CrsForMonomialForm}, + hardcoded_g2_bases, }; use bellman::{bn256::Bn256, CurveAffine, Engine, Field, PrimeField}; use byteorder::{BigEndian, ReadBytesExt}; @@ -10,13 +10,11 @@ use gpu_prover::ManagerConfigs; pub(crate) fn write_crs_into_raw_compact_form( original_crs: &Crs, mut dst_raw_compact_crs: W, - num_points: usize, ) -> std::io::Result<()> { use bellman::CurveAffine; - assert!(num_points <= original_crs.g1_bases.len()); use bellman::{PrimeField, PrimeFieldRepr}; use byteorder::{BigEndian, WriteBytesExt}; - assert!(num_points < u32::MAX as usize); + let num_points = original_crs.g1_bases.len(); dst_raw_compact_crs.write_u32::(num_points as u32)?; for g1_base in original_crs.g1_bases.iter() { let (x, y) = g1_base.as_xy(); @@ -39,6 +37,7 @@ pub(crate) fn read_crs_from_raw_compact_form std::io::Result> { + // requested number of bases can be smaller than the available bases use byteorder::{BigEndian, ReadBytesExt}; let actual_num_points = src_raw_compact_crs.read_u32::()? as usize; assert!(num_g1_points <= actual_num_points as usize); @@ -51,16 +50,9 @@ pub(crate) fn read_crs_from_raw_compact_form(), - ); - src_raw_compact_crs.read_exact(buf)?; - } + + let g2_bases = hardcoded_g2_bases::().to_vec_in(A::default()); + Ok(Crs::<_, CrsForMonomialForm, A>::new_in(g1_bases, g2_bases)) } @@ -73,7 +65,8 @@ pub(crate) fn create_compact_raw_crs(dst: W) { .max() .unwrap(); let original_crs = make_crs_from_ignition_transcripts(num_points); - write_crs_into_raw_compact_form(&original_crs, dst, num_points).unwrap(); + assert_eq!(original_crs.g1_bases.len(), num_points); + write_crs_into_raw_compact_form(&original_crs, dst).unwrap(); } fn make_crs_from_ignition_transcripts(num_points: usize) -> Crs { @@ -277,3 +270,7 @@ fn create_crs_from_ignition_transcript + ?Sized>( Ok(new) } + +pub(crate) fn hardcoded_canonical_g2_bases() -> [bellman::bn256::G2Affine; 2] { + ::fflonk::hardcoded_g2_bases::() +} diff --git a/crates/proof-compression/src/proof_system/fflonk.rs b/crates/proof-compression/src/proof_system/fflonk.rs index c7af650..a442b88 100644 --- a/crates/proof-compression/src/proof_system/fflonk.rs +++ b/crates/proof-compression/src/proof_system/fflonk.rs @@ -38,7 +38,7 @@ impl ProofSystemDefinition for FflonkSnarkWrapper { type Proof = FflonkSnarkVerifierCircuitProof; type VK = FflonkSnarkVerifierCircuitVK; type FinalizationHint = usize; - // type Allocator = GlobalHost; // TODO need global host with preallocated host memory + // Pinned memory with small allocations is expensive e.g Assembly storage type Allocator = std::alloc::Global; type ProvingAssembly = FflonkAssembly; type Transcript = RollingKeccakTranscript; @@ -69,6 +69,12 @@ impl SnarkWrapperProofSystem for FflonkSnarkWrapper { CrsForMonomialForm, Self::Allocator, >; + + fn pre_init() { + let domain_size = ::fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG; + Self::Context::init_pinned_memory(domain_size).unwrap(); + } + fn load_compact_raw_crs(src: R) -> Self::CRS { let domain_size = 1 << ::fflonk::fflonk_cpu::L1_VERIFIER_DOMAIN_SIZE_LOG; let num_g1_bases_for_crs = ::fflonk::fflonk_cpu::MAX_COMBINED_DEGREE_FACTOR * domain_size; @@ -78,7 +84,11 @@ impl SnarkWrapperProofSystem for FflonkSnarkWrapper { fn init_context(compact_raw_crs: AsyncHandler) -> Self::Context { let compact_raw_crs = compact_raw_crs.wait(); let domain_size = 1 << ::fflonk::fflonk_cpu::L1_VERIFIER_DOMAIN_SIZE_LOG; - let context = Self::Context::init_from_preloaded_crs(domain_size, compact_raw_crs).unwrap(); + let context = DeviceContextWithSingleDevice::init_from_preloaded_crs::( + domain_size, + compact_raw_crs, + ) + .unwrap(); context } @@ -120,7 +130,7 @@ impl SnarkWrapperProofSystem for FflonkSnarkWrapper { fn prove_from_witnesses( _: AsyncHandler, - _: Vec, + _: Self::ExternalWitnessData, _: AsyncHandler, _: Self::FinalizationHint, ) -> Self::Proof { diff --git a/crates/proof-compression/src/proof_system/interface.rs b/crates/proof-compression/src/proof_system/interface.rs index 119ec1f..724f76b 100644 --- a/crates/proof-compression/src/proof_system/interface.rs +++ b/crates/proof-compression/src/proof_system/interface.rs @@ -79,6 +79,7 @@ pub(crate) trait SnarkWrapperProofSystem: ProofSystemDefinition { type Circuit; type Context: Send + Sync + 'static; type CRS: Send + Sync + 'static; + fn pre_init(); fn init_context(crs: AsyncHandler) -> Self::Context; fn load_compact_raw_crs(src: R) -> Self::CRS; fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly; @@ -91,7 +92,7 @@ pub(crate) trait SnarkWrapperProofSystem: ProofSystemDefinition { fn prove_from_witnesses( _: AsyncHandler, - _: Vec, + _: Self::ExternalWitnessData, _: AsyncHandler, _: Self::FinalizationHint, ) -> Self::Proof; diff --git a/crates/proof-compression/src/proof_system/plonk.rs b/crates/proof-compression/src/proof_system/plonk.rs index 2fcbd52..69bcc71 100644 --- a/crates/proof-compression/src/proof_system/plonk.rs +++ b/crates/proof-compression/src/proof_system/plonk.rs @@ -70,10 +70,7 @@ impl ProofSystemDefinition for PlonkSnarkWrapper { type Proof = PlonkSnarkVerifierCircuitProof; type VK = PlonkSnarkVerifierCircuitVK; type FinalizationHint = usize; - #[cfg(feature = "allocator")] type Allocator = gpu_prover::cuda_bindings::CudaAllocator; - #[cfg(not(feature = "allocator"))] - type Allocator = std::alloc::Global; type ProvingAssembly = PlonkAssembly; type Transcript = RollingKeccakTranscript; fn take_witnesses( @@ -98,16 +95,17 @@ impl ProofSystemDefinition for PlonkSnarkWrapper { impl SnarkWrapperProofSystem for PlonkSnarkWrapper { type Circuit = PlonkSnarkVerifierCircuit; - type Context = ( - UnsafePlonkProverDeviceMemoryManagerWrapper, - Vec, - ); + type Context = UnsafePlonkProverDeviceMemoryManagerWrapper; type CRS = bellman::kate_commitment::Crs< bellman::compact_bn256::Bn256, CrsForMonomialForm, Self::Allocator, >; + fn pre_init() { + // TODO: initialize static pinned memory + } + fn load_compact_raw_crs(src: R) -> Self::CRS { let num_g1_points_for_crs = 1 << PlonkProverDeviceMemoryManagerConfig::FULL_SLOT_SIZE_LOG; read_crs_from_raw_compact_form(src, num_g1_points_for_crs).unwrap() @@ -117,12 +115,8 @@ impl SnarkWrapperProofSystem for PlonkSnarkWrapper { let device_ids: Vec<_> = (0..::NUM_GPUS).collect(); let compact_raw_crs = compact_raw_crs.wait(); - let g2_bases = compact_raw_crs.g2_monomial_bases.as_ref().to_vec(); let manager = DeviceMemoryManager::init(&device_ids, &compact_raw_crs.g1_bases).unwrap(); - ( - UnsafePlonkProverDeviceMemoryManagerWrapper(manager), - g2_bases, - ) + UnsafePlonkProverDeviceMemoryManagerWrapper(manager) } fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly { @@ -146,7 +140,7 @@ impl SnarkWrapperProofSystem for PlonkSnarkWrapper { assert!(domain_size.is_power_of_two()); assert_eq!(domain_size, finalization_hint); - let (ctx, _) = ctx.wait(); + let ctx = ctx.wait(); let mut ctx = ctx.into_inner(); let mut precomputation = precomputation.wait().into_inner(); let worker = bellman::worker::Worker::new(); @@ -197,28 +191,18 @@ impl SnarkWrapperProofSystemExt for PlonkSnarkWrapper { assert!(domain_size.is_power_of_two()); assert_eq!(domain_size, hardcoded_finalization_hint); - let (ctx, compact_g2_bases) = ctx.wait(); + let ctx = ctx.wait(); let mut ctx = ctx.into_inner(); let worker = bellman::worker::Worker::new(); - #[cfg(feature = "allocator")] let mut precomputation = AsyncSetup::::allocate(hardcoded_finalization_hint); - #[cfg(not(feature = "allocator"))] - let mut precomputation = AsyncSetup::allocate(hardcoded_finalization_hint); precomputation .generate_from_assembly(&worker, &setup_assembly, &mut ctx) .unwrap(); + let hardcoded_g2_bases = hardcoded_canonical_g2_bases(); let mut dummy_crs = Crs::::dummy_crs(1); - let mut g2_bases = vec![]; - use bellman::CurveAffine; - for base in compact_g2_bases.into_iter() { - let (x, y) = base.into_xy_unchecked(); - let base = bellman::bn256::G2Affine::from_xy_unchecked(x, y); - g2_bases.push(base); - } - - dummy_crs.g2_monomial_bases = std::sync::Arc::new(g2_bases); + dummy_crs.g2_monomial_bases = std::sync::Arc::new(hardcoded_g2_bases.to_vec()); let vk = gpu_prover::compute_vk_from_assembly::< _, _, diff --git a/crates/proof-compression/src/step/snark_wrapper.rs b/crates/proof-compression/src/step/snark_wrapper.rs index ca3faec..cf465b5 100644 --- a/crates/proof-compression/src/step/snark_wrapper.rs +++ b/crates/proof-compression/src/step/snark_wrapper.rs @@ -115,6 +115,10 @@ pub(crate) trait SnarkWrapperStep: SnarkWrapperProofSystem { AsyncHandler::spawn(f) } + fn run_pre_initialization_tasks() { + Self::pre_init(); + } + fn prove_snark_wrapper_step( ctx_config: AsyncHandler, precomputation: AsyncHandler, diff --git a/crates/proof-compression/src/test.rs b/crates/proof-compression/src/test.rs index ca303ff..bc9f1cf 100644 --- a/crates/proof-compression/src/test.rs +++ b/crates/proof-compression/src/test.rs @@ -1,38 +1,19 @@ use super::*; -use circuit_definitions::circuit_definitions::recursion_layer::ZkSyncRecursionLayerProof; - #[test] fn test_proof_chain_with_fflonk() { - let scheduler_proof_file = - std::fs::File::open("./data/scheduler_recursive_proof.json").unwrap(); - let scheduler_proof: ZkSyncRecursionLayerProof = - serde_json::from_reader(&scheduler_proof_file).unwrap(); - let scheduler_proof = scheduler_proof.into_inner(); - let simple_blob_storage = FileSystemBlobStorage; - let proof = run_proof_chain_with_fflonk(scheduler_proof, &simple_blob_storage); - - let proof_file_path = format!("./data/final_fflonk_proof.json"); - let proof_file = std::fs::File::create(&proof_file_path).unwrap(); - serde_json::to_writer(proof_file, &proof).unwrap(); - println!("Final fflonk snark wrapper roof saved at {proof_file_path}"); + let mut simple_proof_storage = SimpleProofStorage::new(); + run_proof_chain_with_fflonk(&simple_blob_storage, &mut simple_proof_storage); + println!("Final fflonk snark wrapper proof has successfully generated"); } #[test] fn test_proof_chain_with_plonk() { - let scheduler_proof_file = - std::fs::File::open("./data/scheduler_recursive_proof.json").unwrap(); - let scheduler_proof: ZkSyncRecursionLayerProof = - serde_json::from_reader(&scheduler_proof_file).unwrap(); - let scheduler_proof = scheduler_proof.into_inner(); - let simple_blob_storage = FileSystemBlobStorage; - let proof = run_proof_chain_with_plonk(scheduler_proof, &simple_blob_storage); + let mut simple_proof_storage = SimpleProofStorage::new(); + run_proof_chain_with_plonk(&simple_blob_storage, &mut simple_proof_storage); - let proof_file_path = format!("./data/final_plonk_proof.json"); - let proof_file = std::fs::File::create(&proof_file_path).unwrap(); - serde_json::to_writer(proof_file, &proof).unwrap(); - println!("Final plonk snark wrapper roof saved at {proof_file_path}"); + println!("Final plonk snark wrapper proof has successfully generated"); } #[test] From e8d959b44c6e34c212e3ee86a079fed6822ce228 Mon Sep 17 00:00:00 2001 From: saitima Date: Mon, 13 Jan 2025 18:35:57 +0300 Subject: [PATCH 10/18] resolve conflicts --- crates/fflonk/src/allocator/bitmap.rs | 161 +++++++++++++++++++ crates/fflonk/src/allocator/mod.rs | 8 + crates/fflonk/src/allocator/pinned.rs | 104 +++++++++--- crates/fflonk/src/allocator/static_device.rs | 161 +++---------------- crates/fflonk/src/context.rs | 114 +++++++++---- crates/fflonk/src/convenience.rs | 74 ++++++++- crates/fflonk/src/lib.rs | 2 +- crates/fflonk/src/setup.rs | 13 +- crates/fflonk/src/storage.rs | 2 +- crates/fflonk/src/test.rs | 4 +- 10 files changed, 436 insertions(+), 207 deletions(-) create mode 100644 crates/fflonk/src/allocator/bitmap.rs diff --git a/crates/fflonk/src/allocator/bitmap.rs b/crates/fflonk/src/allocator/bitmap.rs new file mode 100644 index 0000000..0c2cd42 --- /dev/null +++ b/crates/fflonk/src/allocator/bitmap.rs @@ -0,0 +1,161 @@ +use std::{ + ptr::NonNull, + sync::{Arc, Mutex}, +}; + +use super::*; + +#[derive(Clone)] +pub(crate) struct UnsafeNonNullPtr(pub(crate) Arc>); +unsafe impl Send for UnsafeNonNullPtr {} +unsafe impl Sync for UnsafeNonNullPtr {} + +impl UnsafeNonNullPtr { + pub(crate) fn new(ptr: NonNull<[u8]>) -> Self { + Self(Arc::new(ptr)) + } + + pub(crate) fn as_ptr(&self) -> *const u8 { + self.0.as_ptr().cast() + } + pub(crate) fn as_mut_ptr(&mut self) -> *mut u8 { + self.0.as_ptr().cast() + } +} + +#[derive(Clone)] +pub(crate) struct StaticBitmapAllocator { + pub(crate) memory: UnsafeNonNullPtr, + pub(crate) memory_size: usize, + pub(crate) block_size_in_bytes: usize, + pub(crate) bitmap: Arc>>, +} + +impl StaticBitmapAllocator { + pub(crate) fn init( + memory: NonNull<[u8]>, + num_blocks: usize, + block_size_in_bytes: usize, + ) -> Self { + let memory_size_in_bytes = num_blocks * block_size_in_bytes; + Self { + memory: UnsafeNonNullPtr::new(memory), + memory_size: memory_size_in_bytes, + block_size_in_bytes, + bitmap: Arc::new(Mutex::new(vec![false; num_blocks])), + } + } + + pub(crate) fn as_ptr(&self) -> *const u8 { + self.memory.as_ptr().cast() + } + + pub(crate) fn find_free_block(&self) -> Option { + for (idx, entry) in self.bitmap.lock().unwrap().iter_mut().enumerate() { + if !*entry { + *entry = true; + return Some(idx); + } + } + None + } + + #[allow(unreachable_code)] + pub(crate) fn find_adjacent_free_blocks( + &self, + requested_num_blocks: usize, + ) -> Option> { + let mut bitmap = self.bitmap.lock().unwrap(); + if requested_num_blocks > bitmap.len() { + return None; + } + let _range_of_blocks_found = false; + let _found_range = 0..0; + + let mut start = 0; + let mut end = requested_num_blocks; + let mut busy_block_idx = 0; + loop { + let mut has_busy_block = false; + for (idx, sub_entry) in bitmap[start..end].iter().copied().enumerate() { + if sub_entry { + has_busy_block = true; + busy_block_idx = start + idx; + } + } + if !has_busy_block { + for entry in bitmap[start..end].iter_mut() { + *entry = true; + } + return Some(start..end); + } else { + start = busy_block_idx + 1; + end = start + requested_num_blocks; + if end > bitmap.len() { + break; + } + } + } + // panic!("not found block {} {} {}", start, end, self.bitmap.len()); + None + } + + pub(crate) fn free_blocks(&self, index: usize, num_blocks: usize) { + assert!(num_blocks > 0); + let mut guard = self.bitmap.lock().unwrap(); + for i in index..index + num_blocks { + guard[i] = false; + } + } + + pub(crate) fn allocate( + &self, + layout: std::alloc::Layout, + ) -> CudaResult> { + let size = layout.size(); + assert!(size > 0); + assert_eq!(size % self.block_size_in_bytes, 0); + let num_blocks = size / self.block_size_in_bytes; + + if size > self.block_size_in_bytes { + if let Some(range) = self.find_adjacent_free_blocks(num_blocks) { + let index = range.start; + let offset = index * self.block_size_in_bytes; + let ptr = unsafe { self.as_ptr().add(offset) }; + let ptr = unsafe { NonNull::new_unchecked(ptr as _) }; + return Ok(NonNull::slice_from_raw_parts(ptr, size)); + } + panic!("allocation of {} blocks has failed", num_blocks); + // return Err(CudaError::AllocationError(format!( + // "allocation of {} blocks has failed", + // num_blocks + // ))); + } + + if let Some(index) = self.find_free_block() { + let offset = index * self.block_size_in_bytes; + let ptr = unsafe { self.as_ptr().add(offset) }; + let ptr = unsafe { NonNull::new_unchecked(ptr as _) }; + Ok(NonNull::slice_from_raw_parts(ptr, size)) + } else { + panic!("allocation of 1 block has failed"); + // return Err(CudaError::AllocationError(format!( + // "allocation of 1 block has failed", + // ))); + } + } + + pub(crate) fn deallocate(&self, ptr: std::ptr::NonNull, layout: std::alloc::Layout) { + let size = layout.size(); + assert!(size > 0); + assert_eq!(size % self.block_size_in_bytes, 0); + let offset = unsafe { ptr.as_ptr().offset_from(self.as_ptr()) } as usize; + if offset >= self.memory_size { + return; + } + assert_eq!(offset % self.block_size_in_bytes, 0); + let index = offset / self.block_size_in_bytes; + let num_blocks = size / self.block_size_in_bytes; + self.free_blocks(index, num_blocks); + } +} diff --git a/crates/fflonk/src/allocator/mod.rs b/crates/fflonk/src/allocator/mod.rs index 5e363fd..e47ab43 100644 --- a/crates/fflonk/src/allocator/mod.rs +++ b/crates/fflonk/src/allocator/mod.rs @@ -1,4 +1,8 @@ use super::*; + +mod bitmap; +use bitmap::*; + mod pinned; pub use pinned::*; @@ -7,3 +11,7 @@ pub use pool::*; mod static_device; pub use static_device::*; + + +use std::ptr::NonNull; +use bellman::bn256::Fr; \ No newline at end of file diff --git a/crates/fflonk/src/allocator/pinned.rs b/crates/fflonk/src/allocator/pinned.rs index ac51586..d6303b5 100644 --- a/crates/fflonk/src/allocator/pinned.rs +++ b/crates/fflonk/src/allocator/pinned.rs @@ -3,42 +3,110 @@ use super::*; // Both assembly and device setup has an ability to store data on the pinned memory // - Assembly uses for the variables(7487741), state and setup columns // - Device setup uses variable indexes and gate selectors -static mut _STATIC_HOST_ALLOC: Option = None; +static mut _STATIC_HOST_ALLOC: Option = None; -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct GlobalHost; +pub(crate) fn _static_host_alloc() -> GlobalStaticHost { + unsafe { + _STATIC_HOST_ALLOC + .as_ref() + .expect("initialize static host allocator") + .clone() + } +} -impl GlobalHost { - pub fn init(domain_size: usize) -> CudaResult { - let num_variables = 0; - let num_cols = 3; +pub(crate) fn init_static_host_alloc(domain_size: usize) { + unsafe { + // Pinned memory could be initialized before device initialization + if _STATIC_HOST_ALLOC.is_some() { + println!("fflonk pinned memory already initialized, ignoring"); + return; + } + } + // Bitmap allocator with small block size and high number of allocations doesn't make + // sense, and doesn't give good runtime performance compared to default allocator. + // However it provides satisfying improvement for 3 combined monomials, since prover + // transfers them them back and forth in case of L4 devices. + let num_blocks = 3; + let block_size_in_bytes = 9 * 32 * domain_size; + let allocator = GlobalStaticHost::init(num_blocks, block_size_in_bytes) + .expect("initialize static allocator"); - let size_of_indexes_in_bytes = 8 * num_cols * domain_size; - let size_of_vars_in_bytes = 32 * num_variables; + unsafe { _STATIC_HOST_ALLOC = Some(allocator) } +} - let total_size_in_bytes = size_of_indexes_in_bytes + size_of_vars_in_bytes; +pub(crate) fn free_static_host_alloc() { + unsafe { + if let Some(alloc) = _STATIC_HOST_ALLOC.take() { + alloc.free().expect("Couldn't free static allocator"); + } + } +} - todo!() +#[derive(Clone)] +pub struct GlobalStaticHost(StaticBitmapAllocator); + +impl Default for GlobalStaticHost { + fn default() -> Self { + _static_host_alloc() } } pub trait HostAllocator: Allocator + Default + Clone + Send + Sync + 'static {} -unsafe impl Allocator for GlobalHost { +impl GlobalStaticHost { + pub fn init(num_blocks: usize, block_size_in_bytes: usize) -> CudaResult { + assert_ne!(num_blocks, 0); + + let memory_size_in_bytes = num_blocks * block_size_in_bytes; + let memory = host_allocate(memory_size_in_bytes) + .map(|ptr| unsafe { std::ptr::NonNull::new_unchecked(ptr as _) }) + .map(|ptr| std::ptr::NonNull::slice_from_raw_parts(ptr, memory_size_in_bytes))?; + println!("allocated {memory_size_in_bytes} bytes on pinned host memory"); + let allocator = StaticBitmapAllocator::init(memory, num_blocks, block_size_in_bytes); + + Ok(Self(allocator)) + } + + pub(crate) fn free(self) -> CudaResult<()> { + println!("freeing static cuda allocation"); + assert_eq!(std::sync::Arc::weak_count(&self.0.memory.0), 0); + // TODO + // assert_eq!(Arc::strong_count(&self.memory), 1); + let StaticBitmapAllocator { mut memory, .. } = self.0; + // let memory = Arc::try_unwrap(memory).expect("exclusive access"); + host_dealloc(memory.as_mut_ptr().cast()) + } +} + +unsafe impl Allocator for GlobalStaticHost { fn allocate( &self, layout: std::alloc::Layout, + ) -> Result, std::alloc::AllocError> { + self.0.allocate(layout).map_err(|_| std::alloc::AllocError) + } + + fn allocate_zeroed( + &self, + layout: std::alloc::Layout, ) -> Result, std::alloc::AllocError> { - host_allocate(layout.size()) - .map(|ptr| unsafe { std::ptr::NonNull::new_unchecked(ptr as _) }) - .map(|ptr| std::ptr::NonNull::slice_from_raw_parts(ptr, layout.size())) - .map_err(|_| std::alloc::AllocError) + let ptr = self.allocate(layout)?; + let num_bytes = layout.size(); + unsafe { + std::ptr::write_bytes(ptr.as_ptr() as *mut u8, 0, layout.size()); + let result = gpu_ffi::bc_memset(ptr.as_ptr().cast(), 0, num_bytes as u64); + if result != 0 { + panic!("Couldn't allocate zeroed buffer") + } + } + + Ok(ptr) } unsafe fn deallocate(&self, ptr: std::ptr::NonNull, layout: std::alloc::Layout) { - host_dealloc(ptr.as_ptr().cast()).expect("deallocate static buffer") + self.0.deallocate(ptr, layout); } } -impl HostAllocator for GlobalHost {} +impl HostAllocator for GlobalStaticHost {} impl HostAllocator for std::alloc::Global {} diff --git a/crates/fflonk/src/allocator/static_device.rs b/crates/fflonk/src/allocator/static_device.rs index d7f3f7c..55ba165 100644 --- a/crates/fflonk/src/allocator/static_device.rs +++ b/crates/fflonk/src/allocator/static_device.rs @@ -1,7 +1,3 @@ -use std::{cell::RefCell, ptr::NonNull, rc::Rc}; - -use bellman::bn256::Fr; - use super::*; static mut _STATIC_ALLOC: Option = None; @@ -17,8 +13,9 @@ pub(crate) fn _static_alloc() -> GlobalDeviceStatic { pub(crate) fn init_static_alloc(domain_size: usize) { let num_blocks = Device::static_alloc_num_blocks(); - let allocator = - GlobalDeviceStatic::init(num_blocks, domain_size).expect("initialize static allocator"); + let block_size_in_bytes = std::mem::size_of::() * domain_size; + let allocator = GlobalDeviceStatic::init(num_blocks, block_size_in_bytes) + .expect("initialize static allocator"); unsafe { _STATIC_ALLOC = Some(allocator) } } @@ -34,112 +31,32 @@ pub(crate) fn free_static_alloc() { } #[derive(Clone)] -pub struct GlobalDeviceStatic { - memory: Rc>, - memory_size: usize, - block_size_in_bytes: usize, - bitmap: Rc>>, -} +pub struct GlobalDeviceStatic(StaticBitmapAllocator); impl GlobalDeviceStatic { - fn init_bitmap(num_blocks: usize) -> Vec { - vec![false; num_blocks] - } - - pub fn init(num_blocks: usize, block_size: usize) -> CudaResult { + pub fn init(num_blocks: usize, block_size_in_bytes: usize) -> CudaResult { assert_ne!(num_blocks, 0); - assert!(block_size.is_power_of_two()); - - let memory_size = num_blocks * block_size; - let memory_size_in_bytes = memory_size * size_of::(); - let block_size_in_bytes = block_size * size_of::(); + assert!(block_size_in_bytes.is_power_of_two()); + let memory_size_in_bytes = num_blocks * block_size_in_bytes; let memory = allocate(memory_size_in_bytes) - .map(|ptr| unsafe { std::ptr::NonNull::new_unchecked(ptr as _) }) - .map(|ptr| std::ptr::NonNull::slice_from_raw_parts(ptr, memory_size_in_bytes))?; + .map(|ptr| unsafe { NonNull::new_unchecked(ptr as _) }) + .map(|ptr| NonNull::slice_from_raw_parts(ptr, memory_size_in_bytes))?; println!("allocated {memory_size_in_bytes} bytes on device"); - let alloc = GlobalDeviceStatic { - memory: Rc::new(memory), - memory_size: memory_size_in_bytes, - block_size_in_bytes, - bitmap: Rc::new(RefCell::new(Self::init_bitmap(num_blocks))), - }; - - return Ok(alloc); - } - fn as_ptr(&self) -> *const u8 { - self.memory.as_ptr().cast() + let allocator = StaticBitmapAllocator::init(memory, num_blocks, block_size_in_bytes); + Ok(Self(allocator)) } - fn find_free_block(&self) -> Option { - for (idx, entry) in self.bitmap.borrow_mut().iter_mut().enumerate() { - if !*entry { - *entry = true; - return Some(idx); - } - } - None - } - - #[allow(unreachable_code)] - fn find_adjacent_free_blocks( - &self, - requested_num_blocks: usize, - ) -> Option> { - let mut bitmap = self.bitmap.borrow_mut(); - if requested_num_blocks > bitmap.len() { - return None; - } - let _range_of_blocks_found = false; - let _found_range = 0..0; - - let mut start = 0; - let mut end = requested_num_blocks; - let mut busy_block_idx = 0; - loop { - let mut has_busy_block = false; - for (idx, sub_entry) in bitmap[start..end].iter().copied().enumerate() { - if sub_entry { - has_busy_block = true; - busy_block_idx = start + idx; - } - } - if !has_busy_block { - for entry in bitmap[start..end].iter_mut() { - *entry = true; - } - return Some(start..end); - } else { - start = busy_block_idx + 1; - end = start + requested_num_blocks; - if end > bitmap.len() { - break; - } - } - } - // panic!("not found block {} {} {}", start, end, self.bitmap.len()); - None - } - - fn free_blocks(&self, index: usize, num_blocks: usize) { - assert!(num_blocks > 0); - let mut guard = self.bitmap.borrow_mut(); - for i in index..index + num_blocks { - guard[i] = false; - } - } - - pub fn free(self) -> CudaResult<()> { + pub(crate) fn free(self) -> CudaResult<()> { println!("freeing static cuda allocation"); - assert_eq!(Rc::weak_count(&self.memory), 0); + assert_eq!(std::sync::Arc::weak_count(&self.0.memory.0), 0); // TODO - // assert_eq!(Rc::strong_count(&self.memory), 1); - let Self { memory, .. } = self; - // let memory = Rc::try_unwrap(memory).expect("exclusive access"); - dealloc(memory.as_ptr().cast())?; - Ok(()) + // assert_eq!(Arc::strong_count(&self.memory), 1); + let StaticBitmapAllocator { mut memory, .. } = self.0; + // let memory = Arc::try_unwrap(memory).expect("exclusive access"); + dealloc(memory.as_mut_ptr().cast()) } } @@ -151,37 +68,7 @@ impl Default for GlobalDeviceStatic { impl DeviceAllocator for GlobalDeviceStatic { fn allocate(&self, layout: std::alloc::Layout) -> CudaResult> { - let size = layout.size(); - assert!(size > 0); - assert_eq!(size % self.block_size_in_bytes, 0); - let num_blocks = size / self.block_size_in_bytes; - - if size > self.block_size_in_bytes { - if let Some(range) = self.find_adjacent_free_blocks(num_blocks) { - let index = range.start; - let offset = index * self.block_size_in_bytes; - let ptr = unsafe { self.as_ptr().add(offset) }; - let ptr = unsafe { NonNull::new_unchecked(ptr as _) }; - return Ok(NonNull::slice_from_raw_parts(ptr, size)); - } - panic!("allocation of {} blocks has failed", num_blocks); - // return Err(CudaError::AllocationError(format!( - // "allocation of {} blocks has failed", - // num_blocks - // ))); - } - - if let Some(index) = self.find_free_block() { - let offset = index * self.block_size_in_bytes; - let ptr = unsafe { self.as_ptr().add(offset) }; - let ptr = unsafe { NonNull::new_unchecked(ptr as _) }; - Ok(NonNull::slice_from_raw_parts(ptr, size)) - } else { - panic!("allocation of 1 block has failed"); - // return Err(CudaError::AllocationError(format!( - // "allocation of 1 block has failed", - // ))); - } + self.0.allocate(layout) } fn allocate_zeroed(&self, layout: std::alloc::Layout) -> CudaResult> { @@ -198,17 +85,7 @@ impl DeviceAllocator for GlobalDeviceStatic { } fn deallocate(&self, ptr: std::ptr::NonNull, layout: std::alloc::Layout) { - let size = layout.size(); - assert!(size > 0); - assert_eq!(size % self.block_size_in_bytes, 0); - let offset = unsafe { ptr.as_ptr().offset_from(self.as_ptr()) } as usize; - if offset >= self.memory_size { - return; - } - assert_eq!(offset % self.block_size_in_bytes, 0); - let index = offset / self.block_size_in_bytes; - let num_blocks = size / self.block_size_in_bytes; - self.free_blocks(index, num_blocks); + self.0.deallocate(ptr, layout); } fn allocate_async( diff --git a/crates/fflonk/src/context.rs b/crates/fflonk/src/context.rs index fec31e1..8324e03 100644 --- a/crates/fflonk/src/context.rs +++ b/crates/fflonk/src/context.rs @@ -2,6 +2,8 @@ use std::mem::ManuallyDrop; use super::*; use bellman::compact_bn256::Bn256 as CompactBn256; +use bellman::compact_bn256::{G1Affine as CompactG1Affine, G2Affine as CompactG2Affine}; +use bellman::CurveAffine; use gpu_ffi::bc_mem_pool; static mut _MSM_BASES_MEMPOOL: Option = None; @@ -146,10 +148,19 @@ const POWERS_OF_COSET_OMEGA_COARSE_LOG_COUNT: u32 = 14; pub type DeviceContextWithSingleDevice = DeviceContext<1>; impl DeviceContext { - pub fn init_from_preloaded_crs( + pub fn init_pinned_memory(domain_size: usize) -> CudaResult<()> { + init_static_host_alloc(domain_size); + + Ok(()) + } + + pub fn init_from_preloaded_crs( domain_size: usize, - crs: Crs, - ) -> CudaResult { + crs: Crs, + ) -> CudaResult + where + A: HostAllocator, + { let context = Self::init_no_msm(domain_size)?; Self::init_msm_on_static_memory(domain_size, Some(crs))?; @@ -158,7 +169,7 @@ impl DeviceContext { pub fn init(domain_size: usize) -> CudaResult { let context = Self::init_no_msm(domain_size)?; - Self::init_msm_on_static_memory(domain_size, None)?; + Self::init_msm_on_static_memory::(domain_size, None)?; // Self::init_msm_on_pool(domain_size)?; Ok(context) @@ -189,29 +200,38 @@ impl DeviceContext { Ok(DeviceContext) } - fn init_msm_on_static_memory( + fn init_msm_on_static_memory( domain_size: usize, - crs: Option>, - ) -> CudaResult<()> { + crs: Option>, + ) -> CudaResult<()> + where + A: HostAllocator, + { Self::inner_init_msm(domain_size, crs, None, None)?; Ok(()) } // In reality we keep bases on a statically allocated buffer. - unsafe fn init_msm_on_pool(domain_size: usize) -> CudaResult<()> { + unsafe fn init_msm_on_pool(domain_size: usize) -> CudaResult<()> + where + A: HostAllocator, + { let pool = _msm_bases_mempool(); let stream = bc_stream::new().unwrap(); - Self::inner_init_msm(domain_size, None, Some(pool), Some(stream))?; + Self::inner_init_msm::(domain_size, None, Some(pool), Some(stream))?; stream.sync().unwrap(); Ok(()) } - fn inner_init_msm( + fn inner_init_msm( domain_size: usize, - crs: Option>, + crs: Option>, pool: Option, stream: Option, - ) -> CudaResult<()> { + ) -> CudaResult<()> + where + A: HostAllocator, + { assert!( is_msm_context_initialized() == false, "MSM context is already initialized" @@ -221,7 +241,7 @@ impl DeviceContext { // multiple of the domain_size let crs = match crs { Some(preloaded_crs) => preloaded_crs, - None => init_compact_crs(&bellman::worker::Worker::new(), domain_size), + None => init_compact_crs::(domain_size), }; let num_bases = MAX_COMBINED_DEGREE_FACTOR * domain_size; assert!(crs.g1_bases.len() >= num_bases); @@ -255,13 +275,14 @@ impl DeviceContext { } pub fn init_allocations(domain_size: usize) { - init_static_alloc(domain_size); + init_static_alloc(domain_size); init_small_scalar_mempool(); init_tmp_mempool(); } pub fn free_allocations() { free_static_alloc(); + free_static_host_alloc(); destroy_small_scalar_mempool(); destroy_tmp_mempool(); } @@ -302,25 +323,56 @@ pub(crate) fn _bases() -> &'static DSlice { use bellman::kate_commitment::{Crs, CrsForMonomialForm}; -pub fn init_compact_crs( - worker: &bellman::worker::Worker, - domain_size: usize, -) -> Crs { +pub fn init_compact_crs(domain_size: usize) -> Crs +where + A: HostAllocator, +{ assert!(domain_size <= 1 << fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG); let num_points = MAX_COMBINED_DEGREE_FACTOR * domain_size; - let mon_crs = if let Ok(crs_file_path) = std::env::var("COMPACT_CRS_FILE") { - println!("using crs file at {crs_file_path}"); - let crs_file = - std::fs::File::open(&crs_file_path).expect(&format!("crs file at {}", crs_file_path)); - let mon_crs = Crs::::read(crs_file) - .expect(&format!("read crs file at {}", crs_file_path)); - assert!(num_points <= mon_crs.g1_bases.len()); - - mon_crs - } else { - println!("Using dummy CRS"); - Crs::::non_power_of_two_crs_42(num_points, &worker) - }; + let crs_file_path = std::env::var("COMPACT_RAW_CRS_FILE").unwrap(); + println!("using crs file at {crs_file_path}"); + let crs_file = + std::fs::File::open(&crs_file_path).expect(&format!("crs file at {}", crs_file_path)); + let (g1_bases, g2_bases) = + read_bases(crs_file).expect(&format!("read crs file at {}", crs_file_path)); + let mon_crs = Crs::new_in(g1_bases, g2_bases); + assert!(num_points <= mon_crs.g1_bases.len()); mon_crs } + +pub fn read_bases( + mut reader: R, +) -> std::io::Result<(Vec, Vec)> { + use bellman::pairing::EncodedPoint; + use byteorder::{BigEndian, ReadBytesExt}; + let mut g1_repr = ::Uncompressed::empty(); + let mut g2_repr = ::Uncompressed::empty(); + + let num_g1 = reader.read_u64::()?; + + let mut g1_bases = Vec::with_capacity_in(num_g1 as usize, A::default()); + + for _ in 0..num_g1 { + reader.read_exact(g1_repr.as_mut())?; + let p = g1_repr + .into_affine() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + g1_bases.push(p); + } + + let num_g2 = reader.read_u64::()?; + assert!(num_g2 == 2u64); + + let mut g2_bases = Vec::with_capacity_in(num_g2 as usize, A::default()); + + for _ in 0..num_g2 { + reader.read_exact(g2_repr.as_mut())?; + let p = g2_repr + .into_affine() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + g2_bases.push(p); + } + + Ok((g1_bases, g2_bases)) +} diff --git a/crates/fflonk/src/convenience.rs b/crates/fflonk/src/convenience.rs index c0c95e0..26f9ff3 100644 --- a/crates/fflonk/src/convenience.rs +++ b/crates/fflonk/src/convenience.rs @@ -5,6 +5,7 @@ use bellman::{ better_better_cs::cs::{Circuit, SynthesisModeProve, SynthesisModeTesting}, commitments::transcript::keccak_transcript::RollingKeccakTranscript, }, + CurveAffine, PrimeFieldRepr, }; use circuit_definitions::circuit_definitions::aux_layer::{ wrapper::ZkSyncCompressionWrapper, ZkSyncCompressionProofForWrapper, @@ -12,7 +13,7 @@ use circuit_definitions::circuit_definitions::aux_layer::{ }; use fflonk::{FflonkAssembly, L1_VERIFIER_DOMAIN_SIZE_LOG}; -pub type FflonkSnarkVerifierCircuitDeviceSetup = +pub type FflonkSnarkVerifierCircuitDeviceSetup = FflonkDeviceSetup; use super::*; @@ -106,7 +107,7 @@ pub fn gpu_prove_fflonk_snark_verifier_circuit_single_shot( FflonkSnarkVerifierCircuitProof, FflonkSnarkVerifierCircuitVK, ) { - let mut assembly = FflonkAssembly::::new(); + let mut assembly = FflonkAssembly::::new(); circuit.synthesize(&mut assembly).expect("must work"); assert!(assembly.is_satisfied()); let raw_trace_len = assembly.n(); @@ -115,8 +116,8 @@ pub fn gpu_prove_fflonk_snark_verifier_circuit_single_shot( .expect("Couldn't create fflonk GPU Context"); let setup = - FflonkDeviceSetup::<_, FflonkSnarkVerifierCircuit, GlobalHost>::create_setup_from_assembly_on_device( - &assembly, + FflonkDeviceSetup::<_, FflonkSnarkVerifierCircuit, GlobalStaticHost>::create_setup_from_assembly_on_device( + &assembly ) .unwrap(); let vk = setup.get_verification_key(); @@ -150,7 +151,7 @@ pub fn gpu_prove_fflonk_snark_verifier_circuit_with_precomputation( vk: &FflonkSnarkVerifierCircuitVK, ) -> FflonkSnarkVerifierCircuitProof { println!("Synthesizing for fflonk proving"); - let mut proving_assembly = FflonkAssembly::::new(); + let mut proving_assembly = FflonkAssembly::::new(); circuit .synthesize(&mut proving_assembly) .expect("must work"); @@ -188,7 +189,7 @@ pub fn precompute_and_save_setup_and_vk_for_fflonk_snark_circuit( println!("Generating fflonk setup data on the device"); let device_setup = - FflonkSnarkVerifierCircuitDeviceSetup::::create_setup_on_device(&circuit) + FflonkSnarkVerifierCircuitDeviceSetup::::create_setup_on_device(&circuit) .unwrap(); let setup_file_path = format!("{}/final_snark_device_setup.bin", path); println!("Saving setup into file {setup_file_path}"); @@ -220,3 +221,64 @@ pub fn load_device_setup_and_vk_of_fflonk_snark_circuit( (device_setup, vk) } + +pub fn hardcoded_g2_bases() -> [E::G2Affine; 2] { + use bellman::compact_bn256::{Fq, Fq2, FqRepr}; + use bellman::PrimeField; + + let encoding = [ + 38, 32, 188, 2, 209, 181, 131, 142, 114, 1, 123, 73, 53, 25, 235, 220, 223, 26, 129, 151, + 71, 38, 184, 251, 59, 80, 150, 175, 65, 56, 87, 25, 64, 97, 76, 168, 125, 115, 180, 175, + 196, 216, 2, 88, 90, 221, 67, 96, 134, 47, 160, 82, 252, 80, 233, 9, 107, 123, 234, 58, + 131, 240, 254, 20, 246, 233, 107, 136, 157, 250, 157, 97, 120, 155, 158, 245, 151, 210, + 127, 254, 254, 125, 27, 35, 98, 26, 158, 255, 6, 66, 158, 174, 235, 126, 253, 40, 238, 86, + 24, 199, 86, 91, 9, 100, 187, 60, 125, 50, 34, 249, 87, 220, 118, 16, 53, 51, 190, 53, 249, + 85, 130, 100, 253, 147, 230, 160, 164, 13, 182, 244, 28, 71, 70, 29, 100, 233, 208, 232, + 16, 57, 87, 156, 156, 214, 74, 82, 220, 27, 161, 197, 77, 15, 210, 9, 240, 189, 172, 71, + 111, 30, 128, 0, 56, 200, 24, 253, 23, 165, 145, 209, 42, 156, 197, 0, 120, 91, 50, 140, + 234, 115, 242, 5, 158, 54, 169, 218, 26, 132, 98, 22, 43, 23, 206, 192, 48, 61, 48, 232, + 255, 223, 42, 230, 151, 216, 68, 117, 123, 28, 236, 116, 246, 140, 245, 170, 54, 180, 144, + 56, 190, 222, 70, 225, 237, 33, 140, 112, 147, 119, 162, 146, 63, 229, 0, 167, 154, 82, 99, + 93, 178, 109, 44, 185, 245, 226, 45, 73, 216, 99, 172, 7, 183, 178, 189, 54, 218, 31, + ]; + assert_eq!(encoding.len(), 256); + let mut src = &encoding[..]; + + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let x0c0 = Fq::from_raw_repr(repr).unwrap(); + + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let x0c1 = Fq::from_raw_repr(repr).unwrap(); + let x = unsafe { *(&Fq2 { c0: x0c0, c1: x0c1 } as *const Fq2 as *const E::Fqe) }; + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let y0c0 = Fq::from_raw_repr(repr).unwrap(); + + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let y0c1 = Fq::from_raw_repr(repr).unwrap(); + let y = unsafe { *(&Fq2 { c0: y0c0, c1: y0c1 } as *const Fq2 as *const E::Fqe) }; + let p0 = E::G2Affine::from_xy_checked(x, y).unwrap(); + + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let x1c0 = Fq::from_raw_repr(repr).unwrap(); + + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let x1c1 = Fq::from_raw_repr(repr).unwrap(); + let x = unsafe { *(&Fq2 { c0: x1c0, c1: x1c1 } as *const Fq2 as *const E::Fqe) }; + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let y1c0 = Fq::from_raw_repr(repr).unwrap(); + + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let y1c1 = Fq::from_raw_repr(repr).unwrap(); + let y = unsafe { *(&Fq2 { c0: y1c0, c1: y1c1 } as *const Fq2 as *const E::Fqe) }; + let p1 = E::G2Affine::from_xy_checked(x, y).unwrap(); + + [p0, p1] +} diff --git a/crates/fflonk/src/lib.rs b/crates/fflonk/src/lib.rs index 3669e99..32f718d 100644 --- a/crates/fflonk/src/lib.rs +++ b/crates/fflonk/src/lib.rs @@ -75,4 +75,4 @@ pub type FflonkSnarkVerifierCircuitVK = FflonkVerificationKey; pub type FflonkSnarkVerifierCircuitSetup = FflonkSetup; -pub use allocator::{GlobalHost, HostAllocator}; +pub use allocator::{GlobalStaticHost, HostAllocator}; diff --git a/crates/fflonk/src/setup.rs b/crates/fflonk/src/setup.rs index 839d779..fb70564 100644 --- a/crates/fflonk/src/setup.rs +++ b/crates/fflonk/src/setup.rs @@ -5,7 +5,7 @@ use bellman::{ Assembly, Circuit, GateInternal, MainGate, PlonkConstraintSystemParams, PolyIdentifier, Setup, SynthesisMode, SynthesisModeGenerateSetup, }, - better_cs::keys::{read_curve_affine, read_fr, write_curve_affine, write_fr_vec}, + better_cs::keys::{read_curve_affine, write_curve_affine}, }, worker::Worker, }; @@ -15,7 +15,7 @@ use super::*; use crate::HostAllocator; -pub struct FflonkDeviceSetup, A: HostAllocator = GlobalHost> { +pub struct FflonkDeviceSetup, A: HostAllocator = GlobalStaticHost> { pub main_gate_selector_monomials: [Vec; 5], pub variable_indexes: [Vec; 3], pub c0_commitment: E::G1Affine, @@ -148,14 +148,13 @@ where )?; println!("Computing preprocessing combined commitment on the device"); let c0_commitment = msm::(combined_monomial.as_ref(), domain_size, stream)?; - let g2_elems = get_g2_elems_from_compact_crs::(); d2h_events.into_iter().for_each(|e| e.sync().unwrap()); stream.sync().unwrap(); Ok(Self { variable_indexes: h_all_transformed_variables.try_into().unwrap(), main_gate_selector_monomials: h_main_gate_selectors.try_into().unwrap(), c0_commitment, - g2_elems, + g2_elems: hardcoded_g2_bases::(), _c: std::marker::PhantomData, }) } @@ -257,13 +256,15 @@ impl, A: HostAllocator> FflonkDeviceSetup { let coeffs = read_raw_fr_vec::<_, _, A>(&mut reader)?; main_gate_selector_monomials.push(coeffs); } - + let domain_size = main_gate_selector_monomials[0].len(); let num_polys = reader.read_u64::()?; assert_eq!(num_polys, 3); let mut variable_indexes = vec![]; for _ in 0..num_polys { let num_values = reader.read_u64::()? as usize; - let mut indexes = Vec::with_capacity_in(num_values, A::default()); + // Block size of pinned memory allocator requires each allocation + // to be same length + let mut indexes = Vec::with_capacity_in(domain_size, A::default()); let indexes_buf = unsafe { indexes.set_len(num_values); std::slice::from_raw_parts_mut( diff --git a/crates/fflonk/src/storage.rs b/crates/fflonk/src/storage.rs index e4a6d13..573da02 100644 --- a/crates/fflonk/src/storage.rs +++ b/crates/fflonk/src/storage.rs @@ -126,7 +126,7 @@ where } } -pub struct CombinedMonomialHostStorage { +pub struct CombinedMonomialHostStorage { pub(crate) combined_monomials: [Vec; 3], pub(crate) events: [bc_event; 3], } diff --git a/crates/fflonk/src/test.rs b/crates/fflonk/src/test.rs index 5cc93fd..28eaae8 100644 --- a/crates/fflonk/src/test.rs +++ b/crates/fflonk/src/test.rs @@ -35,7 +35,7 @@ fn test_simple_circuit_with_naive_main_gate() { type C = FflonkTestCircuit; let circuit = C {}; - let mut assembly = FflonkAssembly::::new(); + let mut assembly = FflonkAssembly::::new(); circuit.synthesize(&mut assembly).expect("must work"); assert!(assembly.is_satisfied()); let raw_trace_len = assembly.n(); @@ -57,7 +57,7 @@ fn test_simple_circuit_with_naive_main_gate() { _, RollingKeccakTranscript, CombinedMonomialDeviceStorage, - GlobalHost, + GlobalStaticHost, >(&assembly, &setup, raw_trace_len) .expect("proof"); From b21d8888f07dc432a7b63b437a6e6543317b4977 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Mon, 13 Jan 2025 19:06:04 +0200 Subject: [PATCH 11/18] make create_crs public --- crates/proof-compression/src/proof_system/crs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proof-compression/src/proof_system/crs.rs b/crates/proof-compression/src/proof_system/crs.rs index 8cf0cec..8158db5 100644 --- a/crates/proof-compression/src/proof_system/crs.rs +++ b/crates/proof-compression/src/proof_system/crs.rs @@ -56,7 +56,7 @@ pub(crate) fn read_crs_from_raw_compact_form::new_in(g1_bases, g2_bases)) } -pub(crate) fn create_compact_raw_crs(dst: W) { +pub fn create_compact_raw_crs(dst: W) { let num_points = [ ::fflonk::MAX_COMBINED_DEGREE_FACTOR << ::fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG, ::FULL_SLOT_SIZE, From 470a376e634992e00f0190c13025f71aa8208221 Mon Sep 17 00:00:00 2001 From: saitima Date: Tue, 14 Jan 2025 10:15:15 +0300 Subject: [PATCH 12/18] expose crs creation --- Cargo.toml | 21 ++++++++++++------- crates/proof-compression/src/lib.rs | 3 ++- .../proof-compression/src/proof_system/crs.rs | 2 +- .../proof-compression/src/proof_system/mod.rs | 1 + 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index db92b2a..de30ebe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,14 +29,21 @@ fflonk = { version = "=0.152.10", path = "crates/fflonk", package = "fflonk-cuda # These dependencies should be shared by all the crates. # zksync-crypto repository -boojum = "=0.30.12" -fflonk-cpu = {package = "fflonk", version = "=0.30.12"} -franklin-crypto = "=0.30.12" -rescue_poseidon = "=0.30.12" -snark_wrapper = "=0.30.12" +# boojum = "=0.30.9" +# fflonk-cpu = {package = "fflonk", version = "=0.30.9"} +# franklin-crypto = "=0.30.9" +# rescue_poseidon = "=0.30.9" +# snark_wrapper = "=0.30.9" +boojum = { path = "../zksync-crypto/crates/boojum"} +fflonk-cpu = { path = "../zksync-crypto/crates/fflonk", package = "fflonk", features = ["allocator"]} +franklin-crypto = { path = "../zksync-crypto/crates/franklin-crypto", package = "franklin-crypto"} +rescue_poseidon = { path = "../zksync-crypto/crates/rescue-poseidon", package = "rescue_poseidon"} +snark_wrapper = { path = "../zksync-crypto/crates/snark-wrapper", package = "snark_wrapper"} # zksync-protocol repository -circuit_definitions = { version = "=0.150.19" } -zkevm_test_harness = { version = "=0.150.19" } +# circuit_definitions = { version = "=0.150.16" } +# zkevm_test_harness = { version = "=0.150.16" } +circuit_definitions = { path = "../zksync-protocol/crates/circuit_definitions"} +zkevm_test_harness = { path = "../zksync-protocol/crates/zkevm_test_harness"} [profile.release] debug = "line-tables-only" diff --git a/crates/proof-compression/src/lib.rs b/crates/proof-compression/src/lib.rs index 9057d4d..f78c714 100644 --- a/crates/proof-compression/src/lib.rs +++ b/crates/proof-compression/src/lib.rs @@ -10,7 +10,8 @@ pub use chain::*; mod context; use context::*; -mod proof_system; +pub mod proof_system; +pub use proof_system::create_compact_raw_crs; use proof_system::*; mod serialization; diff --git a/crates/proof-compression/src/proof_system/crs.rs b/crates/proof-compression/src/proof_system/crs.rs index 8cf0cec..8158db5 100644 --- a/crates/proof-compression/src/proof_system/crs.rs +++ b/crates/proof-compression/src/proof_system/crs.rs @@ -56,7 +56,7 @@ pub(crate) fn read_crs_from_raw_compact_form::new_in(g1_bases, g2_bases)) } -pub(crate) fn create_compact_raw_crs(dst: W) { +pub fn create_compact_raw_crs(dst: W) { let num_points = [ ::fflonk::MAX_COMBINED_DEGREE_FACTOR << ::fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG, ::FULL_SLOT_SIZE, diff --git a/crates/proof-compression/src/proof_system/mod.rs b/crates/proof-compression/src/proof_system/mod.rs index e47347a..c1832ca 100644 --- a/crates/proof-compression/src/proof_system/mod.rs +++ b/crates/proof-compression/src/proof_system/mod.rs @@ -6,6 +6,7 @@ mod boojum; pub(crate) use boojum::*; mod crs; pub(crate) use crs::*; +pub use crs::*; mod fflonk; pub(crate) use fflonk::*; mod interface; From 5eb4da945cfbe503f80280cb4d7e8fb4d5948632 Mon Sep 17 00:00:00 2001 From: afo Date: Tue, 14 Jan 2025 16:00:40 +0000 Subject: [PATCH 13/18] change the interface, expose some functions --- crates/proof-compression/src/chain.rs | 49 +++++++-------------------- crates/proof-compression/src/lib.rs | 2 +- 2 files changed, 13 insertions(+), 38 deletions(-) diff --git a/crates/proof-compression/src/chain.rs b/crates/proof-compression/src/chain.rs index fffeaa7..3c83ce0 100644 --- a/crates/proof-compression/src/chain.rs +++ b/crates/proof-compression/src/chain.rs @@ -84,24 +84,23 @@ pub type SchedulerProof = franklin_crypto::boojum::cs::implementations::proof::P GoldilocksExt2, >; -pub fn run_proof_chain( +pub fn run_proof_chain( snark_wrapper: SnarkWrapper, blob_storage: &BS, - proof_storage: &mut PS, -) where + scheduler_proof: SchedulerProof, +) -> SnarkWrapperProof +where BS: BlobStorage, - PS: ProofStorage, { match snark_wrapper { - SnarkWrapper::Plonk => run_proof_chain_with_plonk(blob_storage, proof_storage), - SnarkWrapper::FFfonk => run_proof_chain_with_fflonk(blob_storage, proof_storage), + SnarkWrapper::Plonk => run_proof_chain_with_plonk(blob_storage, scheduler_proof), + SnarkWrapper::FFfonk => run_proof_chain_with_fflonk(blob_storage, scheduler_proof), } } -pub fn run_proof_chain_with_fflonk(blob_storage: &BS, proof_storage: &mut PS) +pub fn run_proof_chain_with_fflonk(blob_storage: &BS, scheduler_proof: SchedulerProof) -> SnarkWrapperProof where BS: BlobStorage, - PS: ProofStorage, { let context_manager = SimpleContextManager::new(); let start = std::time::Instant::now(); @@ -109,47 +108,30 @@ where let compact_raw_crs = ::load_compact_raw_crs(blob_storage); let fflonk_precomputation = FflonkSnarkWrapper::get_precomputation(blob_storage); - - let input_proof = proof_storage.get_scheduler_proof(); - let next_proof = - CompressionMode1::prove_compression_step(input_proof, blob_storage, &context_manager); - let compression_proof_1 = ZkSyncCompressionLayerProof::from_inner(1, next_proof.clone()); - proof_storage.save_compression_layer_proof(1, compression_proof_1.clone()); + CompressionMode1::prove_compression_step(scheduler_proof, blob_storage, &context_manager); let next_proof = CompressionMode2::prove_compression_step::<_, SimpleContextManager>( next_proof, blob_storage, &context_manager, ); - let compression_proof_2 = ZkSyncCompressionLayerProof::from_inner(2, next_proof.clone()); - proof_storage.save_compression_layer_proof(2, compression_proof_2.clone()); - let next_proof = CompressionMode3::prove_compression_step::<_, SimpleContextManager>( next_proof, blob_storage, &context_manager, ); - let compression_proof_3 = ZkSyncCompressionLayerProof::from_inner(3, next_proof.clone()); - proof_storage.save_compression_layer_proof(3, compression_proof_3.clone()); let next_proof = CompressionMode4::prove_compression_step::<_, SimpleContextManager>( next_proof, blob_storage, &context_manager, ); - let compression_proof_4 = ZkSyncCompressionLayerProof::from_inner(4, next_proof.clone()); - proof_storage.save_compression_layer_proof(4, compression_proof_4.clone()); - let next_proof = CompressionMode5ForWrapper::prove_compression_step::<_, SimpleContextManager>( next_proof, blob_storage, &context_manager, ); - proof_storage.save_compression_wrapper_proof( - 5, - ZkSyncCompressionForWrapperProof::from_inner(5, next_proof.clone()), - ); println!( "Proving entire compression chain took {}s", start.elapsed().as_secs() @@ -165,7 +147,7 @@ where "Proving entire chain with snark wrapper took {}s", start.elapsed().as_secs() ); - proof_storage.save_fflonk_proof(final_proof); + SnarkWrapperProof::FFfonk(final_proof) } pub fn precompute_proof_chain_with_fflonk(blob_storage: &BS) @@ -201,10 +183,9 @@ where ); } -pub fn run_proof_chain_with_plonk(blob_storage: &BS, proof_storage: &mut PS) +pub fn run_proof_chain_with_plonk(blob_storage: &BS, scheduler_proof: SchedulerProof) -> SnarkWrapperProof where BS: BlobStorage, - PS: ProofStorage, { let context_manager = SimpleContextManager::new(); let start = std::time::Instant::now(); @@ -212,17 +193,11 @@ where ::load_compact_raw_crs(blob_storage); let plonk_precomputation = PlonkSnarkWrapper::get_precomputation(blob_storage); - let input_proof = proof_storage.get_scheduler_proof(); - let next_proof = CompressionMode1ForWrapper::prove_compression_step( - input_proof, + scheduler_proof, blob_storage, &context_manager, ); - proof_storage.save_compression_wrapper_proof( - 1, - ZkSyncCompressionForWrapperProof::from_inner(1, next_proof.clone()), - ); let final_proof = PlonkSnarkWrapper::prove_snark_wrapper_step( compact_raw_crs, @@ -235,7 +210,7 @@ where "Entire compression chain with plonk took {}s", start.elapsed().as_secs() ); - proof_storage.save_plonk_proof(final_proof); + SnarkWrapperProof::Plonk(final_proof) } pub fn precompute_proof_chain_with_plonk(blob_storage: &BS) diff --git a/crates/proof-compression/src/lib.rs b/crates/proof-compression/src/lib.rs index f78c714..816e95d 100644 --- a/crates/proof-compression/src/lib.rs +++ b/crates/proof-compression/src/lib.rs @@ -2,7 +2,7 @@ #![feature(allocator_api)] mod blob_storage; -use blob_storage::*; +pub use blob_storage::*; mod chain; pub use chain::*; From 234de686ca6c05ddbcd93ab395816bcde5162302 Mon Sep 17 00:00:00 2001 From: saitima Date: Tue, 14 Jan 2025 19:21:28 +0300 Subject: [PATCH 14/18] apply missing pinned allocator to combined monomials --- crates/fflonk/src/allocator/pinned.rs | 1 + crates/fflonk/src/convenience.rs | 44 +++++++++---------- crates/fflonk/src/prover.rs | 4 +- crates/fflonk/src/setup.rs | 2 +- crates/fflonk/src/storage.rs | 5 ++- crates/fflonk/src/test.rs | 15 +++---- .../src/proof_system/fflonk.rs | 17 ++++--- .../src/proof_system/plonk.rs | 3 +- 8 files changed, 44 insertions(+), 47 deletions(-) diff --git a/crates/fflonk/src/allocator/pinned.rs b/crates/fflonk/src/allocator/pinned.rs index d6303b5..9e2a2ce 100644 --- a/crates/fflonk/src/allocator/pinned.rs +++ b/crates/fflonk/src/allocator/pinned.rs @@ -22,6 +22,7 @@ pub(crate) fn init_static_host_alloc(domain_size: usize) { return; } } + assert!(domain_size.is_power_of_two()); // Bitmap allocator with small block size and high number of allocations doesn't make // sense, and doesn't give good runtime performance compared to default allocator. // However it provides satisfying improvement for 3 combined monomials, since prover diff --git a/crates/fflonk/src/convenience.rs b/crates/fflonk/src/convenience.rs index 26f9ff3..1df9b20 100644 --- a/crates/fflonk/src/convenience.rs +++ b/crates/fflonk/src/convenience.rs @@ -13,7 +13,7 @@ use circuit_definitions::circuit_definitions::aux_layer::{ }; use fflonk::{FflonkAssembly, L1_VERIFIER_DOMAIN_SIZE_LOG}; -pub type FflonkSnarkVerifierCircuitDeviceSetup = +pub type FflonkSnarkVerifierCircuitDeviceSetup = FflonkDeviceSetup; use super::*; @@ -107,17 +107,18 @@ pub fn gpu_prove_fflonk_snark_verifier_circuit_single_shot( FflonkSnarkVerifierCircuitProof, FflonkSnarkVerifierCircuitVK, ) { - let mut assembly = FflonkAssembly::::new(); + let mut assembly = FflonkAssembly::::new(); circuit.synthesize(&mut assembly).expect("must work"); assert!(assembly.is_satisfied()); let raw_trace_len = assembly.n(); let domain_size = (raw_trace_len + 1).next_power_of_two(); + DeviceContextWithSingleDevice::init_pinned_memory(domain_size).unwrap(); let _context = DeviceContextWithSingleDevice::init(domain_size) .expect("Couldn't create fflonk GPU Context"); let setup = - FflonkDeviceSetup::<_, FflonkSnarkVerifierCircuit, GlobalStaticHost>::create_setup_from_assembly_on_device( - &assembly + FflonkDeviceSetup::<_, FflonkSnarkVerifierCircuit>::create_setup_from_assembly_on_device( + &assembly, ) .unwrap(); let vk = setup.get_verification_key(); @@ -128,14 +129,11 @@ pub fn gpu_prove_fflonk_snark_verifier_circuit_single_shot( assert!(domain_size <= 1 << L1_VERIFIER_DOMAIN_SIZE_LOG); let start = std::time::Instant::now(); - let proof = create_proof::< - _, - FflonkSnarkVerifierCircuit, - _, - RollingKeccakTranscript<_>, - CombinedMonomialDeviceStorage, - _, - >(&assembly, &setup, raw_trace_len) + let proof = create_proof::<_, FflonkSnarkVerifierCircuit, _, RollingKeccakTranscript<_>, _>( + &assembly, + &setup, + raw_trace_len, + ) .unwrap(); println!("proof generation takes {} ms", start.elapsed().as_millis()); @@ -151,7 +149,8 @@ pub fn gpu_prove_fflonk_snark_verifier_circuit_with_precomputation( vk: &FflonkSnarkVerifierCircuitVK, ) -> FflonkSnarkVerifierCircuitProof { println!("Synthesizing for fflonk proving"); - let mut proving_assembly = FflonkAssembly::::new(); + let mut proving_assembly = + FflonkAssembly::::new(); circuit .synthesize(&mut proving_assembly) .expect("must work"); @@ -163,14 +162,11 @@ pub fn gpu_prove_fflonk_snark_verifier_circuit_with_precomputation( assert!(domain_size <= 1 << L1_VERIFIER_DOMAIN_SIZE_LOG); let start = std::time::Instant::now(); - let proof = create_proof::< - _, - FflonkSnarkVerifierCircuit, - _, - RollingKeccakTranscript<_>, - CombinedMonomialDeviceStorage, - _, - >(&proving_assembly, setup, raw_trace_len) + let proof = create_proof::<_, FflonkSnarkVerifierCircuit, _, RollingKeccakTranscript<_>, _>( + &proving_assembly, + setup, + raw_trace_len, + ) .unwrap(); println!("proof generation takes {} ms", start.elapsed().as_millis()); @@ -189,8 +185,10 @@ pub fn precompute_and_save_setup_and_vk_for_fflonk_snark_circuit( println!("Generating fflonk setup data on the device"); let device_setup = - FflonkSnarkVerifierCircuitDeviceSetup::::create_setup_on_device(&circuit) - .unwrap(); + FflonkSnarkVerifierCircuitDeviceSetup::::create_setup_on_device( + &circuit, + ) + .unwrap(); let setup_file_path = format!("{}/final_snark_device_setup.bin", path); println!("Saving setup into file {setup_file_path}"); let device_setup_file = std::fs::File::create(&setup_file_path).unwrap(); diff --git a/crates/fflonk/src/prover.rs b/crates/fflonk/src/prover.rs index 4bfc35c..98bd204 100644 --- a/crates/fflonk/src/prover.rs +++ b/crates/fflonk/src/prover.rs @@ -12,7 +12,7 @@ use fflonk::{ commit_point_as_xy, compute_generators, horner_evaluation, FflonkAssembly, FflonkProof, }; -pub fn create_proof( +pub fn create_proof( assembly: &FflonkAssembly, setup: &FflonkDeviceSetup, raw_trace_len: usize, @@ -58,7 +58,7 @@ where let common_combined_degree = MAX_COMBINED_DEGREE_FACTOR * domain_size; let device = Device::model(); let mut combined_monomial_storage = - GenericCombinedStorage::::allocate_on(&device, domain_size)?; + GenericCombinedStorage::::allocate_on(&device, domain_size)?; let start = std::time::Instant::now(); let ([c1_commitment, c2_commitment], h_all_evaluations, h_aux_evaluations, challenges) = prove_arguments( diff --git a/crates/fflonk/src/setup.rs b/crates/fflonk/src/setup.rs index fb70564..340e6be 100644 --- a/crates/fflonk/src/setup.rs +++ b/crates/fflonk/src/setup.rs @@ -15,7 +15,7 @@ use super::*; use crate::HostAllocator; -pub struct FflonkDeviceSetup, A: HostAllocator = GlobalStaticHost> { +pub struct FflonkDeviceSetup, A: HostAllocator = std::alloc::Global> { pub main_gate_selector_monomials: [Vec; 5], pub variable_indexes: [Vec; 3], pub c0_commitment: E::G1Affine, diff --git a/crates/fflonk/src/storage.rs b/crates/fflonk/src/storage.rs index 573da02..7efd4ad 100644 --- a/crates/fflonk/src/storage.rs +++ b/crates/fflonk/src/storage.rs @@ -13,7 +13,7 @@ pub trait CombinedMonomialStorage { ) -> CudaResult<()>; } -pub enum GenericCombinedStorage +pub enum GenericCombinedStorage where F: PrimeField, A: HostAllocator, @@ -46,9 +46,10 @@ where } } -impl CombinedMonomialStorage for GenericCombinedStorage +impl CombinedMonomialStorage for GenericCombinedStorage where F: PrimeField, + A: HostAllocator, { type Poly = Poly; diff --git a/crates/fflonk/src/test.rs b/crates/fflonk/src/test.rs index 28eaae8..3590677 100644 --- a/crates/fflonk/src/test.rs +++ b/crates/fflonk/src/test.rs @@ -35,7 +35,7 @@ fn test_simple_circuit_with_naive_main_gate() { type C = FflonkTestCircuit; let circuit = C {}; - let mut assembly = FflonkAssembly::::new(); + let mut assembly = FflonkAssembly::::new(); circuit.synthesize(&mut assembly).expect("must work"); assert!(assembly.is_satisfied()); let raw_trace_len = assembly.n(); @@ -51,14 +51,11 @@ fn test_simple_circuit_with_naive_main_gate() { let vk = setup.get_verification_key(); assert_eq!(vk.n + 1, domain_size); - let proof = create_proof::< - _, - C, - _, - RollingKeccakTranscript, - CombinedMonomialDeviceStorage, - GlobalStaticHost, - >(&assembly, &setup, raw_trace_len) + let proof = create_proof::<_, C, _, RollingKeccakTranscript, std::alloc::Global>( + &assembly, + &setup, + raw_trace_len, + ) .expect("proof"); let valid = fflonk::verify::<_, _, RollingKeccakTranscript>(&vk, &proof, None).unwrap(); diff --git a/crates/proof-compression/src/proof_system/fflonk.rs b/crates/proof-compression/src/proof_system/fflonk.rs index a442b88..f4e1f0c 100644 --- a/crates/proof-compression/src/proof_system/fflonk.rs +++ b/crates/proof-compression/src/proof_system/fflonk.rs @@ -71,7 +71,7 @@ impl SnarkWrapperProofSystem for FflonkSnarkWrapper { >; fn pre_init() { - let domain_size = ::fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG; + let domain_size = 1 << ::fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG; Self::Context::init_pinned_memory(domain_size).unwrap(); } @@ -115,15 +115,14 @@ impl SnarkWrapperProofSystem for FflonkSnarkWrapper { assert_eq!(domain_size, finalization_hint); let ctx = ctx.wait(); let precomputation = precomputation.wait().into_inner(); - let proof = ::fflonk::create_proof::< - _, - _, - _, - RollingKeccakTranscript<_>, - CombinedMonomialDeviceStorage, - _, - >(&proving_assembly, &precomputation, raw_trace_len) + let start = std::time::Instant::now(); + let proof = ::fflonk::create_proof::<_, _, _, RollingKeccakTranscript<_>, _>( + &proving_assembly, + &precomputation, + raw_trace_len, + ) .unwrap(); + println!("fflonk proving takes {} s", start.elapsed().as_secs()); drop(ctx); proof } diff --git a/crates/proof-compression/src/proof_system/plonk.rs b/crates/proof-compression/src/proof_system/plonk.rs index 69bcc71..6be7fe7 100644 --- a/crates/proof-compression/src/proof_system/plonk.rs +++ b/crates/proof-compression/src/proof_system/plonk.rs @@ -144,6 +144,7 @@ impl SnarkWrapperProofSystem for PlonkSnarkWrapper { let mut ctx = ctx.into_inner(); let mut precomputation = precomputation.wait().into_inner(); let worker = bellman::worker::Worker::new(); + let start = std::time::Instant::now(); let proof = gpu_prover::create_proof::<_, _, Self::Transcript, _>( &proving_assembly, &mut ctx, @@ -152,7 +153,7 @@ impl SnarkWrapperProofSystem for PlonkSnarkWrapper { None, ) .unwrap(); - + println!("plonk proving takes {} s", start.elapsed().as_secs()); ctx.free_all_slots(); proof From 5aa792d7dbb5b6229969071a557dabee5bc46912 Mon Sep 17 00:00:00 2001 From: afo Date: Wed, 15 Jan 2025 11:02:18 +0000 Subject: [PATCH 15/18] bump dependencies --- Cargo.toml | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index de30ebe..af6dc3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,21 +29,15 @@ fflonk = { version = "=0.152.10", path = "crates/fflonk", package = "fflonk-cuda # These dependencies should be shared by all the crates. # zksync-crypto repository -# boojum = "=0.30.9" -# fflonk-cpu = {package = "fflonk", version = "=0.30.9"} -# franklin-crypto = "=0.30.9" -# rescue_poseidon = "=0.30.9" -# snark_wrapper = "=0.30.9" -boojum = { path = "../zksync-crypto/crates/boojum"} -fflonk-cpu = { path = "../zksync-crypto/crates/fflonk", package = "fflonk", features = ["allocator"]} -franklin-crypto = { path = "../zksync-crypto/crates/franklin-crypto", package = "franklin-crypto"} -rescue_poseidon = { path = "../zksync-crypto/crates/rescue-poseidon", package = "rescue_poseidon"} -snark_wrapper = { path = "../zksync-crypto/crates/snark-wrapper", package = "snark_wrapper"} +boojum = "=0.30.13" +fflonk-cpu = {package = "fflonk", version = "=0.30.13"} +franklin-crypto = "=0.30.13" +rescue_poseidon = "=0.30.13" +snark_wrapper = "=0.30.13" + # zksync-protocol repository -# circuit_definitions = { version = "=0.150.16" } -# zkevm_test_harness = { version = "=0.150.16" } -circuit_definitions = { path = "../zksync-protocol/crates/circuit_definitions"} -zkevm_test_harness = { path = "../zksync-protocol/crates/zkevm_test_harness"} +circuit_definitions = { version = "=0.150.20" } +zkevm_test_harness = { version = "=0.150.20" } [profile.release] debug = "line-tables-only" From f756ae913fcc5151b1855c713e73ecd13d20155f Mon Sep 17 00:00:00 2001 From: afo Date: Wed, 15 Jan 2025 13:01:14 +0000 Subject: [PATCH 16/18] fmt --- crates/fflonk/src/allocator/mod.rs | 3 +-- crates/fflonk/src/context.rs | 2 +- crates/proof-compression/src/chain.rs | 10 ++++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/fflonk/src/allocator/mod.rs b/crates/fflonk/src/allocator/mod.rs index e47ab43..5cf0870 100644 --- a/crates/fflonk/src/allocator/mod.rs +++ b/crates/fflonk/src/allocator/mod.rs @@ -12,6 +12,5 @@ pub use pool::*; mod static_device; pub use static_device::*; - +use bellman::bn256::Fr; use std::ptr::NonNull; -use bellman::bn256::Fr; \ No newline at end of file diff --git a/crates/fflonk/src/context.rs b/crates/fflonk/src/context.rs index 8324e03..de83340 100644 --- a/crates/fflonk/src/context.rs +++ b/crates/fflonk/src/context.rs @@ -275,7 +275,7 @@ impl DeviceContext { } pub fn init_allocations(domain_size: usize) { - init_static_alloc(domain_size); + init_static_alloc(domain_size); init_small_scalar_mempool(); init_tmp_mempool(); } diff --git a/crates/proof-compression/src/chain.rs b/crates/proof-compression/src/chain.rs index 3c83ce0..0277be7 100644 --- a/crates/proof-compression/src/chain.rs +++ b/crates/proof-compression/src/chain.rs @@ -98,7 +98,10 @@ where } } -pub fn run_proof_chain_with_fflonk(blob_storage: &BS, scheduler_proof: SchedulerProof) -> SnarkWrapperProof +pub fn run_proof_chain_with_fflonk( + blob_storage: &BS, + scheduler_proof: SchedulerProof, +) -> SnarkWrapperProof where BS: BlobStorage, { @@ -183,7 +186,10 @@ where ); } -pub fn run_proof_chain_with_plonk(blob_storage: &BS, scheduler_proof: SchedulerProof) -> SnarkWrapperProof +pub fn run_proof_chain_with_plonk( + blob_storage: &BS, + scheduler_proof: SchedulerProof, +) -> SnarkWrapperProof where BS: BlobStorage, { From 082071bbfc62646090c7816b02c0d2014118a981 Mon Sep 17 00:00:00 2001 From: afo Date: Wed, 15 Jan 2025 13:32:22 +0000 Subject: [PATCH 17/18] write pretty VKs --- crates/proof-compression/src/step/compression.rs | 4 ++-- crates/proof-compression/src/step/snark_wrapper.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/proof-compression/src/step/compression.rs b/crates/proof-compression/src/step/compression.rs index 6086c3f..c5de5bc 100644 --- a/crates/proof-compression/src/step/compression.rs +++ b/crates/proof-compression/src/step/compression.rs @@ -174,8 +174,8 @@ pub(crate) trait CompressionStepExt: CompressionProofSystemExt + CompressionStep precomputation .write_into_buffer(precompuatation_writer) .unwrap(); - serde_json::to_writer(vk_writer, &vk).unwrap(); - serde_json::to_writer(hint_writer, &finalization_hint).unwrap(); + serde_json::to_writer_pretty(vk_writer, &vk).unwrap(); + serde_json::to_writer_pretty(hint_writer, &finalization_hint).unwrap(); println!( "Precomputation and vk of compression circuit {} saved into blob storage", Self::MODE diff --git a/crates/proof-compression/src/step/snark_wrapper.rs b/crates/proof-compression/src/step/snark_wrapper.rs index cf465b5..d07a99d 100644 --- a/crates/proof-compression/src/step/snark_wrapper.rs +++ b/crates/proof-compression/src/step/snark_wrapper.rs @@ -193,7 +193,7 @@ pub(crate) trait SnarkWrapperStepExt: SnarkWrapperProofSystemExt + SnarkWrapperS precomputation .write_into_buffer(precompuatation_writer) .unwrap(); - serde_json::to_writer(vk_writer, &vk).unwrap(); + serde_json::to_writer_pretty(vk_writer, &vk).unwrap(); println!("Pecomputation and vk of snark wrapper circuit saved into blob storage"); } } From cdf0c2fb320021e111ee1fdfc3d9ca403613fdee Mon Sep 17 00:00:00 2001 From: afo Date: Wed, 15 Jan 2025 15:34:44 +0000 Subject: [PATCH 18/18] make allocator the default feature for fflonk-cuda --- crates/fflonk/Cargo.toml | 3 +-- crates/proof-compression/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/fflonk/Cargo.toml b/crates/fflonk/Cargo.toml index 3236024..4c61a55 100644 --- a/crates/fflonk/Cargo.toml +++ b/crates/fflonk/Cargo.toml @@ -24,6 +24,5 @@ serde_json = "1" serde_derive = "1" [features] -default = ["fflonk-cpu"] +default = ["fflonk-cpu/allocator"] sanity = [] -allocator = ["fflonk-cpu/allocator"] diff --git a/crates/proof-compression/Cargo.toml b/crates/proof-compression/Cargo.toml index 4b3c1a0..cb9594f 100644 --- a/crates/proof-compression/Cargo.toml +++ b/crates/proof-compression/Cargo.toml @@ -26,4 +26,4 @@ byteorder = "1" default = ["gpu", "fflonk", "gpu-prover"] gpu = [] cpu = [] -allocator = ["fflonk/allocator", "gpu-prover/allocator"] +allocator = ["fflonk", "gpu-prover/allocator"]