diff --git a/Cargo.toml b/Cargo.toml index db92b2a..af6dc3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,14 +29,15 @@ fflonk = { version = "=0.152.10", path = "crates/fflonk", package = "fflonk-cuda # These dependencies should be shared by all the crates. # zksync-crypto repository -boojum = "=0.30.12" -fflonk-cpu = {package = "fflonk", version = "=0.30.12"} -franklin-crypto = "=0.30.12" -rescue_poseidon = "=0.30.12" -snark_wrapper = "=0.30.12" +boojum = "=0.30.13" +fflonk-cpu = {package = "fflonk", version = "=0.30.13"} +franklin-crypto = "=0.30.13" +rescue_poseidon = "=0.30.13" +snark_wrapper = "=0.30.13" + # zksync-protocol repository -circuit_definitions = { version = "=0.150.19" } -zkevm_test_harness = { version = "=0.150.19" } +circuit_definitions = { version = "=0.150.20" } +zkevm_test_harness = { version = "=0.150.20" } [profile.release] debug = "line-tables-only" diff --git a/crates/fflonk/Cargo.toml b/crates/fflonk/Cargo.toml index 46638b1..4c61a55 100644 --- a/crates/fflonk/Cargo.toml +++ b/crates/fflonk/Cargo.toml @@ -12,7 +12,7 @@ description = "CUDA implementation of the fflonk prover" exclude = ["/data"] [dependencies] -fflonk-cpu = {workspace = true} +fflonk-cpu = {workspace = true, optional = true} circuit_definitions.workspace = true gpu-ffi.workspace = true rand = "0.4" @@ -24,5 +24,5 @@ serde_json = "1" serde_derive = "1" [features] -default = [] +default = ["fflonk-cpu/allocator"] sanity = [] diff --git a/crates/fflonk/src/allocator/bitmap.rs b/crates/fflonk/src/allocator/bitmap.rs new file mode 100644 index 0000000..0c2cd42 --- /dev/null +++ b/crates/fflonk/src/allocator/bitmap.rs @@ -0,0 +1,161 @@ +use std::{ + ptr::NonNull, + sync::{Arc, Mutex}, +}; + +use super::*; + +#[derive(Clone)] +pub(crate) struct UnsafeNonNullPtr(pub(crate) Arc>); +unsafe impl Send for UnsafeNonNullPtr {} +unsafe impl Sync for UnsafeNonNullPtr {} + +impl UnsafeNonNullPtr { + pub(crate) fn new(ptr: NonNull<[u8]>) -> Self { + Self(Arc::new(ptr)) + } + + pub(crate) fn as_ptr(&self) -> *const u8 { + self.0.as_ptr().cast() + } + pub(crate) fn as_mut_ptr(&mut self) -> *mut u8 { + self.0.as_ptr().cast() + } +} + +#[derive(Clone)] +pub(crate) struct StaticBitmapAllocator { + pub(crate) memory: UnsafeNonNullPtr, + pub(crate) memory_size: usize, + pub(crate) block_size_in_bytes: usize, + pub(crate) bitmap: Arc>>, +} + +impl StaticBitmapAllocator { + pub(crate) fn init( + memory: NonNull<[u8]>, + num_blocks: usize, + block_size_in_bytes: usize, + ) -> Self { + let memory_size_in_bytes = num_blocks * block_size_in_bytes; + Self { + memory: UnsafeNonNullPtr::new(memory), + memory_size: memory_size_in_bytes, + block_size_in_bytes, + bitmap: Arc::new(Mutex::new(vec![false; num_blocks])), + } + } + + pub(crate) fn as_ptr(&self) -> *const u8 { + self.memory.as_ptr().cast() + } + + pub(crate) fn find_free_block(&self) -> Option { + for (idx, entry) in self.bitmap.lock().unwrap().iter_mut().enumerate() { + if !*entry { + *entry = true; + return Some(idx); + } + } + None + } + + #[allow(unreachable_code)] + pub(crate) fn find_adjacent_free_blocks( + &self, + requested_num_blocks: usize, + ) -> Option> { + let mut bitmap = self.bitmap.lock().unwrap(); + if requested_num_blocks > bitmap.len() { + return None; + } + let _range_of_blocks_found = false; + let _found_range = 0..0; + + let mut start = 0; + let mut end = requested_num_blocks; + let mut busy_block_idx = 0; + loop { + let mut has_busy_block = false; + for (idx, sub_entry) in bitmap[start..end].iter().copied().enumerate() { + if sub_entry { + has_busy_block = true; + busy_block_idx = start + idx; + } + } + if !has_busy_block { + for entry in bitmap[start..end].iter_mut() { + *entry = true; + } + return Some(start..end); + } else { + start = busy_block_idx + 1; + end = start + requested_num_blocks; + if end > bitmap.len() { + break; + } + } + } + // panic!("not found block {} {} {}", start, end, self.bitmap.len()); + None + } + + pub(crate) fn free_blocks(&self, index: usize, num_blocks: usize) { + assert!(num_blocks > 0); + let mut guard = self.bitmap.lock().unwrap(); + for i in index..index + num_blocks { + guard[i] = false; + } + } + + pub(crate) fn allocate( + &self, + layout: std::alloc::Layout, + ) -> CudaResult> { + let size = layout.size(); + assert!(size > 0); + assert_eq!(size % self.block_size_in_bytes, 0); + let num_blocks = size / self.block_size_in_bytes; + + if size > self.block_size_in_bytes { + if let Some(range) = self.find_adjacent_free_blocks(num_blocks) { + let index = range.start; + let offset = index * self.block_size_in_bytes; + let ptr = unsafe { self.as_ptr().add(offset) }; + let ptr = unsafe { NonNull::new_unchecked(ptr as _) }; + return Ok(NonNull::slice_from_raw_parts(ptr, size)); + } + panic!("allocation of {} blocks has failed", num_blocks); + // return Err(CudaError::AllocationError(format!( + // "allocation of {} blocks has failed", + // num_blocks + // ))); + } + + if let Some(index) = self.find_free_block() { + let offset = index * self.block_size_in_bytes; + let ptr = unsafe { self.as_ptr().add(offset) }; + let ptr = unsafe { NonNull::new_unchecked(ptr as _) }; + Ok(NonNull::slice_from_raw_parts(ptr, size)) + } else { + panic!("allocation of 1 block has failed"); + // return Err(CudaError::AllocationError(format!( + // "allocation of 1 block has failed", + // ))); + } + } + + pub(crate) fn deallocate(&self, ptr: std::ptr::NonNull, layout: std::alloc::Layout) { + let size = layout.size(); + assert!(size > 0); + assert_eq!(size % self.block_size_in_bytes, 0); + let offset = unsafe { ptr.as_ptr().offset_from(self.as_ptr()) } as usize; + if offset >= self.memory_size { + return; + } + assert_eq!(offset % self.block_size_in_bytes, 0); + let index = offset / self.block_size_in_bytes; + let num_blocks = size / self.block_size_in_bytes; + self.free_blocks(index, num_blocks); + } +} diff --git a/crates/fflonk/src/allocator/mod.rs b/crates/fflonk/src/allocator/mod.rs index 5e363fd..5cf0870 100644 --- a/crates/fflonk/src/allocator/mod.rs +++ b/crates/fflonk/src/allocator/mod.rs @@ -1,4 +1,8 @@ use super::*; + +mod bitmap; +use bitmap::*; + mod pinned; pub use pinned::*; @@ -7,3 +11,6 @@ pub use pool::*; mod static_device; pub use static_device::*; + +use bellman::bn256::Fr; +use std::ptr::NonNull; diff --git a/crates/fflonk/src/allocator/pinned.rs b/crates/fflonk/src/allocator/pinned.rs index ac51586..9e2a2ce 100644 --- a/crates/fflonk/src/allocator/pinned.rs +++ b/crates/fflonk/src/allocator/pinned.rs @@ -3,42 +3,111 @@ use super::*; // Both assembly and device setup has an ability to store data on the pinned memory // - Assembly uses for the variables(7487741), state and setup columns // - Device setup uses variable indexes and gate selectors -static mut _STATIC_HOST_ALLOC: Option = None; +static mut _STATIC_HOST_ALLOC: Option = None; -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct GlobalHost; +pub(crate) fn _static_host_alloc() -> GlobalStaticHost { + unsafe { + _STATIC_HOST_ALLOC + .as_ref() + .expect("initialize static host allocator") + .clone() + } +} -impl GlobalHost { - pub fn init(domain_size: usize) -> CudaResult { - let num_variables = 0; - let num_cols = 3; +pub(crate) fn init_static_host_alloc(domain_size: usize) { + unsafe { + // Pinned memory could be initialized before device initialization + if _STATIC_HOST_ALLOC.is_some() { + println!("fflonk pinned memory already initialized, ignoring"); + return; + } + } + assert!(domain_size.is_power_of_two()); + // Bitmap allocator with small block size and high number of allocations doesn't make + // sense, and doesn't give good runtime performance compared to default allocator. + // However it provides satisfying improvement for 3 combined monomials, since prover + // transfers them them back and forth in case of L4 devices. + let num_blocks = 3; + let block_size_in_bytes = 9 * 32 * domain_size; + let allocator = GlobalStaticHost::init(num_blocks, block_size_in_bytes) + .expect("initialize static allocator"); - let size_of_indexes_in_bytes = 8 * num_cols * domain_size; - let size_of_vars_in_bytes = 32 * num_variables; + unsafe { _STATIC_HOST_ALLOC = Some(allocator) } +} - let total_size_in_bytes = size_of_indexes_in_bytes + size_of_vars_in_bytes; +pub(crate) fn free_static_host_alloc() { + unsafe { + if let Some(alloc) = _STATIC_HOST_ALLOC.take() { + alloc.free().expect("Couldn't free static allocator"); + } + } +} - todo!() +#[derive(Clone)] +pub struct GlobalStaticHost(StaticBitmapAllocator); + +impl Default for GlobalStaticHost { + fn default() -> Self { + _static_host_alloc() } } pub trait HostAllocator: Allocator + Default + Clone + Send + Sync + 'static {} -unsafe impl Allocator for GlobalHost { +impl GlobalStaticHost { + pub fn init(num_blocks: usize, block_size_in_bytes: usize) -> CudaResult { + assert_ne!(num_blocks, 0); + + let memory_size_in_bytes = num_blocks * block_size_in_bytes; + let memory = host_allocate(memory_size_in_bytes) + .map(|ptr| unsafe { std::ptr::NonNull::new_unchecked(ptr as _) }) + .map(|ptr| std::ptr::NonNull::slice_from_raw_parts(ptr, memory_size_in_bytes))?; + println!("allocated {memory_size_in_bytes} bytes on pinned host memory"); + let allocator = StaticBitmapAllocator::init(memory, num_blocks, block_size_in_bytes); + + Ok(Self(allocator)) + } + + pub(crate) fn free(self) -> CudaResult<()> { + println!("freeing static cuda allocation"); + assert_eq!(std::sync::Arc::weak_count(&self.0.memory.0), 0); + // TODO + // assert_eq!(Arc::strong_count(&self.memory), 1); + let StaticBitmapAllocator { mut memory, .. } = self.0; + // let memory = Arc::try_unwrap(memory).expect("exclusive access"); + host_dealloc(memory.as_mut_ptr().cast()) + } +} + +unsafe impl Allocator for GlobalStaticHost { fn allocate( &self, layout: std::alloc::Layout, + ) -> Result, std::alloc::AllocError> { + self.0.allocate(layout).map_err(|_| std::alloc::AllocError) + } + + fn allocate_zeroed( + &self, + layout: std::alloc::Layout, ) -> Result, std::alloc::AllocError> { - host_allocate(layout.size()) - .map(|ptr| unsafe { std::ptr::NonNull::new_unchecked(ptr as _) }) - .map(|ptr| std::ptr::NonNull::slice_from_raw_parts(ptr, layout.size())) - .map_err(|_| std::alloc::AllocError) + let ptr = self.allocate(layout)?; + let num_bytes = layout.size(); + unsafe { + std::ptr::write_bytes(ptr.as_ptr() as *mut u8, 0, layout.size()); + let result = gpu_ffi::bc_memset(ptr.as_ptr().cast(), 0, num_bytes as u64); + if result != 0 { + panic!("Couldn't allocate zeroed buffer") + } + } + + Ok(ptr) } unsafe fn deallocate(&self, ptr: std::ptr::NonNull, layout: std::alloc::Layout) { - host_dealloc(ptr.as_ptr().cast()).expect("deallocate static buffer") + self.0.deallocate(ptr, layout); } } -impl HostAllocator for GlobalHost {} +impl HostAllocator for GlobalStaticHost {} impl HostAllocator for std::alloc::Global {} diff --git a/crates/fflonk/src/allocator/static_device.rs b/crates/fflonk/src/allocator/static_device.rs index d7f3f7c..55ba165 100644 --- a/crates/fflonk/src/allocator/static_device.rs +++ b/crates/fflonk/src/allocator/static_device.rs @@ -1,7 +1,3 @@ -use std::{cell::RefCell, ptr::NonNull, rc::Rc}; - -use bellman::bn256::Fr; - use super::*; static mut _STATIC_ALLOC: Option = None; @@ -17,8 +13,9 @@ pub(crate) fn _static_alloc() -> GlobalDeviceStatic { pub(crate) fn init_static_alloc(domain_size: usize) { let num_blocks = Device::static_alloc_num_blocks(); - let allocator = - GlobalDeviceStatic::init(num_blocks, domain_size).expect("initialize static allocator"); + let block_size_in_bytes = std::mem::size_of::() * domain_size; + let allocator = GlobalDeviceStatic::init(num_blocks, block_size_in_bytes) + .expect("initialize static allocator"); unsafe { _STATIC_ALLOC = Some(allocator) } } @@ -34,112 +31,32 @@ pub(crate) fn free_static_alloc() { } #[derive(Clone)] -pub struct GlobalDeviceStatic { - memory: Rc>, - memory_size: usize, - block_size_in_bytes: usize, - bitmap: Rc>>, -} +pub struct GlobalDeviceStatic(StaticBitmapAllocator); impl GlobalDeviceStatic { - fn init_bitmap(num_blocks: usize) -> Vec { - vec![false; num_blocks] - } - - pub fn init(num_blocks: usize, block_size: usize) -> CudaResult { + pub fn init(num_blocks: usize, block_size_in_bytes: usize) -> CudaResult { assert_ne!(num_blocks, 0); - assert!(block_size.is_power_of_two()); - - let memory_size = num_blocks * block_size; - let memory_size_in_bytes = memory_size * size_of::(); - let block_size_in_bytes = block_size * size_of::(); + assert!(block_size_in_bytes.is_power_of_two()); + let memory_size_in_bytes = num_blocks * block_size_in_bytes; let memory = allocate(memory_size_in_bytes) - .map(|ptr| unsafe { std::ptr::NonNull::new_unchecked(ptr as _) }) - .map(|ptr| std::ptr::NonNull::slice_from_raw_parts(ptr, memory_size_in_bytes))?; + .map(|ptr| unsafe { NonNull::new_unchecked(ptr as _) }) + .map(|ptr| NonNull::slice_from_raw_parts(ptr, memory_size_in_bytes))?; println!("allocated {memory_size_in_bytes} bytes on device"); - let alloc = GlobalDeviceStatic { - memory: Rc::new(memory), - memory_size: memory_size_in_bytes, - block_size_in_bytes, - bitmap: Rc::new(RefCell::new(Self::init_bitmap(num_blocks))), - }; - - return Ok(alloc); - } - fn as_ptr(&self) -> *const u8 { - self.memory.as_ptr().cast() + let allocator = StaticBitmapAllocator::init(memory, num_blocks, block_size_in_bytes); + Ok(Self(allocator)) } - fn find_free_block(&self) -> Option { - for (idx, entry) in self.bitmap.borrow_mut().iter_mut().enumerate() { - if !*entry { - *entry = true; - return Some(idx); - } - } - None - } - - #[allow(unreachable_code)] - fn find_adjacent_free_blocks( - &self, - requested_num_blocks: usize, - ) -> Option> { - let mut bitmap = self.bitmap.borrow_mut(); - if requested_num_blocks > bitmap.len() { - return None; - } - let _range_of_blocks_found = false; - let _found_range = 0..0; - - let mut start = 0; - let mut end = requested_num_blocks; - let mut busy_block_idx = 0; - loop { - let mut has_busy_block = false; - for (idx, sub_entry) in bitmap[start..end].iter().copied().enumerate() { - if sub_entry { - has_busy_block = true; - busy_block_idx = start + idx; - } - } - if !has_busy_block { - for entry in bitmap[start..end].iter_mut() { - *entry = true; - } - return Some(start..end); - } else { - start = busy_block_idx + 1; - end = start + requested_num_blocks; - if end > bitmap.len() { - break; - } - } - } - // panic!("not found block {} {} {}", start, end, self.bitmap.len()); - None - } - - fn free_blocks(&self, index: usize, num_blocks: usize) { - assert!(num_blocks > 0); - let mut guard = self.bitmap.borrow_mut(); - for i in index..index + num_blocks { - guard[i] = false; - } - } - - pub fn free(self) -> CudaResult<()> { + pub(crate) fn free(self) -> CudaResult<()> { println!("freeing static cuda allocation"); - assert_eq!(Rc::weak_count(&self.memory), 0); + assert_eq!(std::sync::Arc::weak_count(&self.0.memory.0), 0); // TODO - // assert_eq!(Rc::strong_count(&self.memory), 1); - let Self { memory, .. } = self; - // let memory = Rc::try_unwrap(memory).expect("exclusive access"); - dealloc(memory.as_ptr().cast())?; - Ok(()) + // assert_eq!(Arc::strong_count(&self.memory), 1); + let StaticBitmapAllocator { mut memory, .. } = self.0; + // let memory = Arc::try_unwrap(memory).expect("exclusive access"); + dealloc(memory.as_mut_ptr().cast()) } } @@ -151,37 +68,7 @@ impl Default for GlobalDeviceStatic { impl DeviceAllocator for GlobalDeviceStatic { fn allocate(&self, layout: std::alloc::Layout) -> CudaResult> { - let size = layout.size(); - assert!(size > 0); - assert_eq!(size % self.block_size_in_bytes, 0); - let num_blocks = size / self.block_size_in_bytes; - - if size > self.block_size_in_bytes { - if let Some(range) = self.find_adjacent_free_blocks(num_blocks) { - let index = range.start; - let offset = index * self.block_size_in_bytes; - let ptr = unsafe { self.as_ptr().add(offset) }; - let ptr = unsafe { NonNull::new_unchecked(ptr as _) }; - return Ok(NonNull::slice_from_raw_parts(ptr, size)); - } - panic!("allocation of {} blocks has failed", num_blocks); - // return Err(CudaError::AllocationError(format!( - // "allocation of {} blocks has failed", - // num_blocks - // ))); - } - - if let Some(index) = self.find_free_block() { - let offset = index * self.block_size_in_bytes; - let ptr = unsafe { self.as_ptr().add(offset) }; - let ptr = unsafe { NonNull::new_unchecked(ptr as _) }; - Ok(NonNull::slice_from_raw_parts(ptr, size)) - } else { - panic!("allocation of 1 block has failed"); - // return Err(CudaError::AllocationError(format!( - // "allocation of 1 block has failed", - // ))); - } + self.0.allocate(layout) } fn allocate_zeroed(&self, layout: std::alloc::Layout) -> CudaResult> { @@ -198,17 +85,7 @@ impl DeviceAllocator for GlobalDeviceStatic { } fn deallocate(&self, ptr: std::ptr::NonNull, layout: std::alloc::Layout) { - let size = layout.size(); - assert!(size > 0); - assert_eq!(size % self.block_size_in_bytes, 0); - let offset = unsafe { ptr.as_ptr().offset_from(self.as_ptr()) } as usize; - if offset >= self.memory_size { - return; - } - assert_eq!(offset % self.block_size_in_bytes, 0); - let index = offset / self.block_size_in_bytes; - let num_blocks = size / self.block_size_in_bytes; - self.free_blocks(index, num_blocks); + self.0.deallocate(ptr, layout); } fn allocate_async( diff --git a/crates/fflonk/src/context.rs b/crates/fflonk/src/context.rs index 38b14a1..de83340 100644 --- a/crates/fflonk/src/context.rs +++ b/crates/fflonk/src/context.rs @@ -2,6 +2,8 @@ use std::mem::ManuallyDrop; use super::*; use bellman::compact_bn256::Bn256 as CompactBn256; +use bellman::compact_bn256::{G1Affine as CompactG1Affine, G2Affine as CompactG2Affine}; +use bellman::CurveAffine; use gpu_ffi::bc_mem_pool; static mut _MSM_BASES_MEMPOOL: Option = None; @@ -55,7 +57,7 @@ pub(crate) fn init_tmp_mempool() { unsafe { _TMP_MEMPOOL = Some(bc_mem_pool::new(DEFAULT_DEVICE_ID).unwrap()); } - let num_tmp_bytes = 1_100_000_000; + let num_tmp_bytes = 3 << 29; //1.5GB let stream = bc_stream::new().unwrap(); DVec::::allocate_on(num_tmp_bytes, _tmp_mempool(), stream); } @@ -146,9 +148,28 @@ const POWERS_OF_COSET_OMEGA_COARSE_LOG_COUNT: u32 = 14; pub type DeviceContextWithSingleDevice = DeviceContext<1>; impl DeviceContext { + pub fn init_pinned_memory(domain_size: usize) -> CudaResult<()> { + init_static_host_alloc(domain_size); + + Ok(()) + } + + pub fn init_from_preloaded_crs( + domain_size: usize, + crs: Crs, + ) -> CudaResult + where + A: HostAllocator, + { + let context = Self::init_no_msm(domain_size)?; + Self::init_msm_on_static_memory(domain_size, Some(crs))?; + + Ok(context) + } + pub fn init(domain_size: usize) -> CudaResult { let context = Self::init_no_msm(domain_size)?; - Self::init_msm_on_static_memory(domain_size)?; + Self::init_msm_on_static_memory::(domain_size, None)?; // Self::init_msm_on_pool(domain_size)?; Ok(context) @@ -179,25 +200,38 @@ impl DeviceContext { Ok(DeviceContext) } - fn init_msm_on_static_memory(domain_size: usize) -> CudaResult<()> { - Self::inner_init_msm(domain_size, None, None)?; + fn init_msm_on_static_memory( + domain_size: usize, + crs: Option>, + ) -> CudaResult<()> + where + A: HostAllocator, + { + Self::inner_init_msm(domain_size, crs, None, None)?; Ok(()) } // In reality we keep bases on a statically allocated buffer. - unsafe fn init_msm_on_pool(domain_size: usize) -> CudaResult<()> { + unsafe fn init_msm_on_pool(domain_size: usize) -> CudaResult<()> + where + A: HostAllocator, + { let pool = _msm_bases_mempool(); let stream = bc_stream::new().unwrap(); - Self::inner_init_msm(domain_size, Some(pool), Some(stream))?; + Self::inner_init_msm::(domain_size, None, Some(pool), Some(stream))?; stream.sync().unwrap(); Ok(()) } - fn inner_init_msm( + fn inner_init_msm( domain_size: usize, + crs: Option>, pool: Option, stream: Option, - ) -> CudaResult<()> { + ) -> CudaResult<()> + where + A: HostAllocator, + { assert!( is_msm_context_initialized() == false, "MSM context is already initialized" @@ -205,7 +239,10 @@ impl DeviceContext { init_msm_result_mempool(); // MSM impl requires bases to be located in a buffer that is // multiple of the domain_size - let crs = init_compact_crs(&bellman::worker::Worker::new(), domain_size); + let crs = match crs { + Some(preloaded_crs) => preloaded_crs, + None => init_compact_crs::(domain_size), + }; let num_bases = MAX_COMBINED_DEGREE_FACTOR * domain_size; assert!(crs.g1_bases.len() >= num_bases); let bases = match (pool, stream) { @@ -245,6 +282,7 @@ pub fn init_allocations(domain_size: usize) { pub fn free_allocations() { free_static_alloc(); + free_static_host_alloc(); destroy_small_scalar_mempool(); destroy_tmp_mempool(); } @@ -285,25 +323,56 @@ pub(crate) fn _bases() -> &'static DSlice { use bellman::kate_commitment::{Crs, CrsForMonomialForm}; -pub fn init_compact_crs( - worker: &bellman::worker::Worker, - domain_size: usize, -) -> Crs { +pub fn init_compact_crs(domain_size: usize) -> Crs +where + A: HostAllocator, +{ assert!(domain_size <= 1 << fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG); let num_points = MAX_COMBINED_DEGREE_FACTOR * domain_size; - let mon_crs = if let Ok(crs_file_path) = std::env::var("COMPACT_CRS_FILE") { - println!("using crs file at {crs_file_path}"); - let crs_file = - std::fs::File::open(&crs_file_path).expect(&format!("crs file at {}", crs_file_path)); - let mon_crs = Crs::::read(crs_file) - .expect(&format!("read crs file at {}", crs_file_path)); - assert!(num_points <= mon_crs.g1_bases.len()); - - mon_crs - } else { - println!("Using dummy CRS"); - Crs::::non_power_of_two_crs_42(num_points, &worker) - }; + let crs_file_path = std::env::var("COMPACT_RAW_CRS_FILE").unwrap(); + println!("using crs file at {crs_file_path}"); + let crs_file = + std::fs::File::open(&crs_file_path).expect(&format!("crs file at {}", crs_file_path)); + let (g1_bases, g2_bases) = + read_bases(crs_file).expect(&format!("read crs file at {}", crs_file_path)); + let mon_crs = Crs::new_in(g1_bases, g2_bases); + assert!(num_points <= mon_crs.g1_bases.len()); mon_crs } + +pub fn read_bases( + mut reader: R, +) -> std::io::Result<(Vec, Vec)> { + use bellman::pairing::EncodedPoint; + use byteorder::{BigEndian, ReadBytesExt}; + let mut g1_repr = ::Uncompressed::empty(); + let mut g2_repr = ::Uncompressed::empty(); + + let num_g1 = reader.read_u64::()?; + + let mut g1_bases = Vec::with_capacity_in(num_g1 as usize, A::default()); + + for _ in 0..num_g1 { + reader.read_exact(g1_repr.as_mut())?; + let p = g1_repr + .into_affine() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + g1_bases.push(p); + } + + let num_g2 = reader.read_u64::()?; + assert!(num_g2 == 2u64); + + let mut g2_bases = Vec::with_capacity_in(num_g2 as usize, A::default()); + + for _ in 0..num_g2 { + reader.read_exact(g2_repr.as_mut())?; + let p = g2_repr + .into_affine() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + g2_bases.push(p); + } + + Ok((g1_bases, g2_bases)) +} diff --git a/crates/fflonk/src/convenience.rs b/crates/fflonk/src/convenience.rs index 4e6bc91..1df9b20 100644 --- a/crates/fflonk/src/convenience.rs +++ b/crates/fflonk/src/convenience.rs @@ -2,11 +2,10 @@ use bellman::{ bn256::{Bn256, Fr}, kate_commitment::{Crs, CrsForMonomialForm}, plonk::{ - better_better_cs::cs::{ - Circuit, SynthesisModeGenerateSetup, SynthesisModeProve, SynthesisModeTesting, - }, + better_better_cs::cs::{Circuit, SynthesisModeProve, SynthesisModeTesting}, commitments::transcript::keccak_transcript::RollingKeccakTranscript, }, + CurveAffine, PrimeFieldRepr, }; use circuit_definitions::circuit_definitions::aux_layer::{ wrapper::ZkSyncCompressionWrapper, ZkSyncCompressionProofForWrapper, @@ -14,8 +13,8 @@ use circuit_definitions::circuit_definitions::aux_layer::{ }; use fflonk::{FflonkAssembly, L1_VERIFIER_DOMAIN_SIZE_LOG}; -pub type FflonkSnarkVerifierCircuitDeviceSetup = - FflonkDeviceSetup; +pub type FflonkSnarkVerifierCircuitDeviceSetup = + FflonkDeviceSetup; use super::*; @@ -108,16 +107,17 @@ pub fn gpu_prove_fflonk_snark_verifier_circuit_single_shot( FflonkSnarkVerifierCircuitProof, FflonkSnarkVerifierCircuitVK, ) { - let mut assembly = FflonkAssembly::::new(); + let mut assembly = FflonkAssembly::::new(); circuit.synthesize(&mut assembly).expect("must work"); assert!(assembly.is_satisfied()); let raw_trace_len = assembly.n(); let domain_size = (raw_trace_len + 1).next_power_of_two(); + DeviceContextWithSingleDevice::init_pinned_memory(domain_size).unwrap(); let _context = DeviceContextWithSingleDevice::init(domain_size) .expect("Couldn't create fflonk GPU Context"); let setup = - FflonkDeviceSetup::<_, FflonkSnarkVerifierCircuit, GlobalHost>::create_setup_from_assembly_on_device( + FflonkDeviceSetup::<_, FflonkSnarkVerifierCircuit>::create_setup_from_assembly_on_device( &assembly, ) .unwrap(); @@ -129,14 +129,11 @@ pub fn gpu_prove_fflonk_snark_verifier_circuit_single_shot( assert!(domain_size <= 1 << L1_VERIFIER_DOMAIN_SIZE_LOG); let start = std::time::Instant::now(); - let proof = create_proof::< - _, - FflonkSnarkVerifierCircuit, - _, - RollingKeccakTranscript<_>, - CombinedMonomialDeviceStorage, - _, - >(&assembly, &setup, raw_trace_len) + let proof = create_proof::<_, FflonkSnarkVerifierCircuit, _, RollingKeccakTranscript<_>, _>( + &assembly, + &setup, + raw_trace_len, + ) .unwrap(); println!("proof generation takes {} ms", start.elapsed().as_millis()); @@ -152,7 +149,8 @@ pub fn gpu_prove_fflonk_snark_verifier_circuit_with_precomputation( vk: &FflonkSnarkVerifierCircuitVK, ) -> FflonkSnarkVerifierCircuitProof { println!("Synthesizing for fflonk proving"); - let mut proving_assembly = FflonkAssembly::::new(); + let mut proving_assembly = + FflonkAssembly::::new(); circuit .synthesize(&mut proving_assembly) .expect("must work"); @@ -164,14 +162,11 @@ pub fn gpu_prove_fflonk_snark_verifier_circuit_with_precomputation( assert!(domain_size <= 1 << L1_VERIFIER_DOMAIN_SIZE_LOG); let start = std::time::Instant::now(); - let proof = create_proof::< - _, - FflonkSnarkVerifierCircuit, - _, - RollingKeccakTranscript<_>, - CombinedMonomialDeviceStorage, - _, - >(&proving_assembly, setup, raw_trace_len) + let proof = create_proof::<_, FflonkSnarkVerifierCircuit, _, RollingKeccakTranscript<_>, _>( + &proving_assembly, + setup, + raw_trace_len, + ) .unwrap(); println!("proof generation takes {} ms", start.elapsed().as_millis()); @@ -190,7 +185,10 @@ pub fn precompute_and_save_setup_and_vk_for_fflonk_snark_circuit( println!("Generating fflonk setup data on the device"); let device_setup = - FflonkSnarkVerifierCircuitDeviceSetup::create_setup_on_device(&circuit).unwrap(); + FflonkSnarkVerifierCircuitDeviceSetup::::create_setup_on_device( + &circuit, + ) + .unwrap(); let setup_file_path = format!("{}/final_snark_device_setup.bin", path); println!("Saving setup into file {setup_file_path}"); let device_setup_file = std::fs::File::create(&setup_file_path).unwrap(); @@ -221,3 +219,64 @@ pub fn load_device_setup_and_vk_of_fflonk_snark_circuit( (device_setup, vk) } + +pub fn hardcoded_g2_bases() -> [E::G2Affine; 2] { + use bellman::compact_bn256::{Fq, Fq2, FqRepr}; + use bellman::PrimeField; + + let encoding = [ + 38, 32, 188, 2, 209, 181, 131, 142, 114, 1, 123, 73, 53, 25, 235, 220, 223, 26, 129, 151, + 71, 38, 184, 251, 59, 80, 150, 175, 65, 56, 87, 25, 64, 97, 76, 168, 125, 115, 180, 175, + 196, 216, 2, 88, 90, 221, 67, 96, 134, 47, 160, 82, 252, 80, 233, 9, 107, 123, 234, 58, + 131, 240, 254, 20, 246, 233, 107, 136, 157, 250, 157, 97, 120, 155, 158, 245, 151, 210, + 127, 254, 254, 125, 27, 35, 98, 26, 158, 255, 6, 66, 158, 174, 235, 126, 253, 40, 238, 86, + 24, 199, 86, 91, 9, 100, 187, 60, 125, 50, 34, 249, 87, 220, 118, 16, 53, 51, 190, 53, 249, + 85, 130, 100, 253, 147, 230, 160, 164, 13, 182, 244, 28, 71, 70, 29, 100, 233, 208, 232, + 16, 57, 87, 156, 156, 214, 74, 82, 220, 27, 161, 197, 77, 15, 210, 9, 240, 189, 172, 71, + 111, 30, 128, 0, 56, 200, 24, 253, 23, 165, 145, 209, 42, 156, 197, 0, 120, 91, 50, 140, + 234, 115, 242, 5, 158, 54, 169, 218, 26, 132, 98, 22, 43, 23, 206, 192, 48, 61, 48, 232, + 255, 223, 42, 230, 151, 216, 68, 117, 123, 28, 236, 116, 246, 140, 245, 170, 54, 180, 144, + 56, 190, 222, 70, 225, 237, 33, 140, 112, 147, 119, 162, 146, 63, 229, 0, 167, 154, 82, 99, + 93, 178, 109, 44, 185, 245, 226, 45, 73, 216, 99, 172, 7, 183, 178, 189, 54, 218, 31, + ]; + assert_eq!(encoding.len(), 256); + let mut src = &encoding[..]; + + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let x0c0 = Fq::from_raw_repr(repr).unwrap(); + + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let x0c1 = Fq::from_raw_repr(repr).unwrap(); + let x = unsafe { *(&Fq2 { c0: x0c0, c1: x0c1 } as *const Fq2 as *const E::Fqe) }; + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let y0c0 = Fq::from_raw_repr(repr).unwrap(); + + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let y0c1 = Fq::from_raw_repr(repr).unwrap(); + let y = unsafe { *(&Fq2 { c0: y0c0, c1: y0c1 } as *const Fq2 as *const E::Fqe) }; + let p0 = E::G2Affine::from_xy_checked(x, y).unwrap(); + + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let x1c0 = Fq::from_raw_repr(repr).unwrap(); + + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let x1c1 = Fq::from_raw_repr(repr).unwrap(); + let x = unsafe { *(&Fq2 { c0: x1c0, c1: x1c1 } as *const Fq2 as *const E::Fqe) }; + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let y1c0 = Fq::from_raw_repr(repr).unwrap(); + + let mut repr = FqRepr::default(); + repr.read_le(&mut src).unwrap(); + let y1c1 = Fq::from_raw_repr(repr).unwrap(); + let y = unsafe { *(&Fq2 { c0: y1c0, c1: y1c1 } as *const Fq2 as *const E::Fqe) }; + let p1 = E::G2Affine::from_xy_checked(x, y).unwrap(); + + [p0, p1] +} diff --git a/crates/fflonk/src/lib.rs b/crates/fflonk/src/lib.rs index 02256f5..32f718d 100644 --- a/crates/fflonk/src/lib.rs +++ b/crates/fflonk/src/lib.rs @@ -63,7 +63,7 @@ pub use gpu_ffi; use gpu_ffi::{bc_event, bc_mem_pool, bc_stream}; use std::alloc::Allocator; -pub use context::{DeviceContext, DeviceContextWithSingleDevice}; +pub use context::{init_compact_crs, DeviceContext, DeviceContextWithSingleDevice}; pub use fflonk::MAX_COMBINED_DEGREE_FACTOR; pub use convenience::FflonkSnarkVerifierCircuitDeviceSetup; @@ -74,3 +74,5 @@ pub type FflonkSnarkVerifierCircuit = ZkSyncSnarkWrapperCircuitNoLookupCustomGat pub type FflonkSnarkVerifierCircuitVK = FflonkVerificationKey; pub type FflonkSnarkVerifierCircuitProof = FflonkProof; pub type FflonkSnarkVerifierCircuitSetup = FflonkSetup; + +pub use allocator::{GlobalStaticHost, HostAllocator}; diff --git a/crates/fflonk/src/prover.rs b/crates/fflonk/src/prover.rs index 4bfc35c..98bd204 100644 --- a/crates/fflonk/src/prover.rs +++ b/crates/fflonk/src/prover.rs @@ -12,7 +12,7 @@ use fflonk::{ commit_point_as_xy, compute_generators, horner_evaluation, FflonkAssembly, FflonkProof, }; -pub fn create_proof( +pub fn create_proof( assembly: &FflonkAssembly, setup: &FflonkDeviceSetup, raw_trace_len: usize, @@ -58,7 +58,7 @@ where let common_combined_degree = MAX_COMBINED_DEGREE_FACTOR * domain_size; let device = Device::model(); let mut combined_monomial_storage = - GenericCombinedStorage::::allocate_on(&device, domain_size)?; + GenericCombinedStorage::::allocate_on(&device, domain_size)?; let start = std::time::Instant::now(); let ([c1_commitment, c2_commitment], h_all_evaluations, h_aux_evaluations, challenges) = prove_arguments( diff --git a/crates/fflonk/src/setup.rs b/crates/fflonk/src/setup.rs index bb8e3d3..340e6be 100644 --- a/crates/fflonk/src/setup.rs +++ b/crates/fflonk/src/setup.rs @@ -5,7 +5,7 @@ use bellman::{ Assembly, Circuit, GateInternal, MainGate, PlonkConstraintSystemParams, PolyIdentifier, Setup, SynthesisMode, SynthesisModeGenerateSetup, }, - better_cs::keys::{read_curve_affine, read_fr, write_curve_affine, write_fr_vec}, + better_cs::keys::{read_curve_affine, write_curve_affine}, }, worker::Worker, }; @@ -15,7 +15,7 @@ use super::*; use crate::HostAllocator; -pub struct FflonkDeviceSetup, A: HostAllocator = GlobalHost> { +pub struct FflonkDeviceSetup, A: HostAllocator = std::alloc::Global> { pub main_gate_selector_monomials: [Vec; 5], pub variable_indexes: [Vec; 3], pub c0_commitment: E::G1Affine, @@ -148,14 +148,13 @@ where )?; println!("Computing preprocessing combined commitment on the device"); let c0_commitment = msm::(combined_monomial.as_ref(), domain_size, stream)?; - let g2_elems = get_g2_elems_from_compact_crs::(); d2h_events.into_iter().for_each(|e| e.sync().unwrap()); stream.sync().unwrap(); Ok(Self { variable_indexes: h_all_transformed_variables.try_into().unwrap(), main_gate_selector_monomials: h_main_gate_selectors.try_into().unwrap(), c0_commitment, - g2_elems, + g2_elems: hardcoded_g2_bases::(), _c: std::marker::PhantomData, }) } @@ -254,25 +253,26 @@ impl, A: HostAllocator> FflonkDeviceSetup { assert_eq!(num_polys, 5); let mut main_gate_selector_monomials = vec![]; for _ in 0..num_polys { - let num_values = reader.read_u64::()?; - let mut coeffs = Vec::with_capacity_in(num_values as usize, A::default()); - for _ in 0..num_values { - let el = read_fr(&mut reader)?; - coeffs.push(el); - } + let coeffs = read_raw_fr_vec::<_, _, A>(&mut reader)?; main_gate_selector_monomials.push(coeffs); } - + let domain_size = main_gate_selector_monomials[0].len(); let num_polys = reader.read_u64::()?; assert_eq!(num_polys, 3); let mut variable_indexes = vec![]; for _ in 0..num_polys { - let num_values = reader.read_u64::()?; - let mut indexes = Vec::with_capacity_in(num_values as usize, A::default()); - for _ in 0..num_values { - let el = reader.read_u32::()?; - indexes.push(el); - } + let num_values = reader.read_u64::()? as usize; + // Block size of pinned memory allocator requires each allocation + // to be same length + let mut indexes = Vec::with_capacity_in(domain_size, A::default()); + let indexes_buf = unsafe { + indexes.set_len(num_values); + std::slice::from_raw_parts_mut( + indexes.as_mut_ptr() as *mut u8, + num_values * std::mem::size_of::(), + ) + }; + reader.read_exact(indexes_buf)?; variable_indexes.push(indexes); } @@ -293,17 +293,58 @@ impl, A: HostAllocator> FflonkDeviceSetup { use byteorder::{BigEndian, WriteBytesExt}; writer.write_u64::(self.main_gate_selector_monomials.len() as u64)?; for mon in self.main_gate_selector_monomials.iter() { - write_fr_vec(&mon, &mut writer)?; + write_raw_fr_slice(&mon, &mut writer)?; } writer.write_u64::(self.variable_indexes.len() as u64)?; for col in self.variable_indexes.iter() { writer.write_u64::(col.len() as u64)?; - for el in col { - writer.write_u32::(*el)?; - } + let buf = unsafe { + std::slice::from_raw_parts( + col.as_ptr() as *mut u8, + col.len() * std::mem::size_of::(), + ) + }; + writer.write_all(buf)?; } write_curve_affine(&self.c0_commitment, &mut writer)?; write_curve_affine(&self.g2_elems[0], &mut writer)?; write_curve_affine(&self.g2_elems[1], writer) } } + +pub fn read_raw_fr_vec( + mut src: R, +) -> std::io::Result> { + use byteorder::{BigEndian, ReadBytesExt}; + let num_values = src.read_u32::()? as usize; + let mut values = Vec::with_capacity_in(num_values, A::default()); + unsafe { + values.set_len(num_values); + let buf = std::slice::from_raw_parts_mut( + values.as_mut_ptr() as *mut u8, + num_values * std::mem::size_of::(), + ); + src.read_exact(buf)?; + } + + Ok(values) +} + +pub fn write_raw_fr_slice( + src_values: &[F], + mut dst: W, +) -> std::io::Result<()> { + use byteorder::{BigEndian, WriteBytesExt}; + let num_values = src_values.len(); + assert!(num_values < u32::MAX as usize); + dst.write_u32::(num_values as u32)?; + unsafe { + let buf = std::slice::from_raw_parts_mut( + src_values.as_ptr() as *mut u8, + num_values * std::mem::size_of::(), + ); + dst.write_all(buf)?; + } + + Ok(()) +} diff --git a/crates/fflonk/src/storage.rs b/crates/fflonk/src/storage.rs index e4a6d13..7efd4ad 100644 --- a/crates/fflonk/src/storage.rs +++ b/crates/fflonk/src/storage.rs @@ -13,7 +13,7 @@ pub trait CombinedMonomialStorage { ) -> CudaResult<()>; } -pub enum GenericCombinedStorage +pub enum GenericCombinedStorage where F: PrimeField, A: HostAllocator, @@ -46,9 +46,10 @@ where } } -impl CombinedMonomialStorage for GenericCombinedStorage +impl CombinedMonomialStorage for GenericCombinedStorage where F: PrimeField, + A: HostAllocator, { type Poly = Poly; @@ -126,7 +127,7 @@ where } } -pub struct CombinedMonomialHostStorage { +pub struct CombinedMonomialHostStorage { pub(crate) combined_monomials: [Vec; 3], pub(crate) events: [bc_event; 3], } diff --git a/crates/fflonk/src/test.rs b/crates/fflonk/src/test.rs index 5cc93fd..3590677 100644 --- a/crates/fflonk/src/test.rs +++ b/crates/fflonk/src/test.rs @@ -35,7 +35,7 @@ fn test_simple_circuit_with_naive_main_gate() { type C = FflonkTestCircuit; let circuit = C {}; - let mut assembly = FflonkAssembly::::new(); + let mut assembly = FflonkAssembly::::new(); circuit.synthesize(&mut assembly).expect("must work"); assert!(assembly.is_satisfied()); let raw_trace_len = assembly.n(); @@ -51,14 +51,11 @@ fn test_simple_circuit_with_naive_main_gate() { let vk = setup.get_verification_key(); assert_eq!(vk.n + 1, domain_size); - let proof = create_proof::< - _, - C, - _, - RollingKeccakTranscript, - CombinedMonomialDeviceStorage, - GlobalHost, - >(&assembly, &setup, raw_trace_len) + let proof = create_proof::<_, C, _, RollingKeccakTranscript, std::alloc::Global>( + &assembly, + &setup, + raw_trace_len, + ) .expect("proof"); let valid = fflonk::verify::<_, _, RollingKeccakTranscript>(&vk, &proof, None).unwrap(); diff --git a/crates/proof-compression/Cargo.toml b/crates/proof-compression/Cargo.toml index c3bb25d..cb9594f 100644 --- a/crates/proof-compression/Cargo.toml +++ b/crates/proof-compression/Cargo.toml @@ -13,8 +13,9 @@ exclude = ["/data"] [dependencies] circuit_definitions.workspace = true -fflonk.workspace = true +fflonk = {workspace = true, optional = true} shivini.workspace = true +gpu-prover = { workspace = true, optional = true} serde = "1" serde_json = "1" bincode = "1.3" @@ -22,6 +23,7 @@ cfg-if = "1.0" byteorder = "1" [features] -default = ["gpu"] +default = ["gpu", "fflonk", "gpu-prover"] gpu = [] cpu = [] +allocator = ["fflonk", "gpu-prover/allocator"] diff --git a/crates/proof-compression/data/scheduler_recursive_vk.json b/crates/proof-compression/data/scheduler_recursive_vk.json index 4c328cb..2817f8c 100644 --- a/crates/proof-compression/data/scheduler_recursive_vk.json +++ b/crates/proof-compression/data/scheduler_recursive_vk.json @@ -1,270 +1,268 @@ { - "SchedulerCircuit": { - "fixed_parameters": { - "parameters": { - "num_columns_under_copy_permutation": 130, - "num_witness_columns": 0, - "num_constant_columns": 4, - "max_allowed_constraint_degree": 8 - }, - "lookup_parameters": { - "UseSpecializedColumnsWithTableIdAsConstant": { - "width": 3, - "num_repetitions": 4, - "share_table_id": true - } - }, - "domain_size": 1048576, - "total_tables_len": 132096, - "public_inputs_locations": [ - [ - 0, - 1043851 - ], - [ - 1, - 1043851 - ], - [ - 2, - 1043851 - ], - [ - 3, - 1043851 - ] + "fixed_parameters": { + "parameters": { + "num_columns_under_copy_permutation": 130, + "num_witness_columns": 0, + "num_constant_columns": 4, + "max_allowed_constraint_degree": 8 + }, + "lookup_parameters": { + "UseSpecializedColumnsWithTableIdAsConstant": { + "width": 3, + "num_repetitions": 4, + "share_table_id": true + } + }, + "domain_size": 1048576, + "total_tables_len": 132096, + "public_inputs_locations": [ + [ + 0, + 1043851 ], - "extra_constant_polys_for_selectors": 4, - "table_ids_column_idxes": [ - 8 + [ + 1, + 1043851 ], - "quotient_degree": 8, - "selectors_placement": { - "Fork": { - "left": { - "GateOnly": { - "gate_idx": 1, - "num_constants": 0, - "degree": 7, - "needs_selector": true, - "is_lookup": false - } - }, - "right": { - "Fork": { - "left": { - "Fork": { - "left": { - "Fork": { - "left": { - "GateOnly": { - "gate_idx": 4, - "num_constants": 4, - "degree": 3, - "needs_selector": true, - "is_lookup": false - } - }, - "right": { - "Fork": { - "left": { - "Fork": { - "left": { - "GateOnly": { - "gate_idx": 5, - "num_constants": 1, - "degree": 2, - "needs_selector": true, - "is_lookup": false - } - }, - "right": { - "GateOnly": { - "gate_idx": 6, - "num_constants": 0, - "degree": 2, - "needs_selector": true, - "is_lookup": false - } + [ + 2, + 1043851 + ], + [ + 3, + 1043851 + ] + ], + "extra_constant_polys_for_selectors": 4, + "table_ids_column_idxes": [ + 8 + ], + "quotient_degree": 8, + "selectors_placement": { + "Fork": { + "left": { + "GateOnly": { + "gate_idx": 1, + "num_constants": 0, + "degree": 7, + "needs_selector": true, + "is_lookup": false + } + }, + "right": { + "Fork": { + "left": { + "Fork": { + "left": { + "Fork": { + "left": { + "GateOnly": { + "gate_idx": 4, + "num_constants": 4, + "degree": 3, + "needs_selector": true, + "is_lookup": false + } + }, + "right": { + "Fork": { + "left": { + "Fork": { + "left": { + "GateOnly": { + "gate_idx": 5, + "num_constants": 1, + "degree": 2, + "needs_selector": true, + "is_lookup": false + } + }, + "right": { + "GateOnly": { + "gate_idx": 6, + "num_constants": 0, + "degree": 2, + "needs_selector": true, + "is_lookup": false } } - }, - "right": { - "Fork": { - "left": { - "GateOnly": { - "gate_idx": 2, - "num_constants": 0, - "degree": 2, - "needs_selector": true, - "is_lookup": false - } - }, - "right": { - "GateOnly": { - "gate_idx": 7, - "num_constants": 0, - "degree": 2, - "needs_selector": true, - "is_lookup": false - } + } + }, + "right": { + "Fork": { + "left": { + "GateOnly": { + "gate_idx": 2, + "num_constants": 0, + "degree": 2, + "needs_selector": true, + "is_lookup": false + } + }, + "right": { + "GateOnly": { + "gate_idx": 7, + "num_constants": 0, + "degree": 2, + "needs_selector": true, + "is_lookup": false } } } } } } - }, - "right": { - "Fork": { - "left": { - "GateOnly": { - "gate_idx": 9, - "num_constants": 4, - "degree": 2, - "needs_selector": true, - "is_lookup": false - } - }, - "right": { - "GateOnly": { - "gate_idx": 0, - "num_constants": 4, - "degree": 1, - "needs_selector": true, - "is_lookup": false - } + } + }, + "right": { + "Fork": { + "left": { + "GateOnly": { + "gate_idx": 9, + "num_constants": 4, + "degree": 2, + "needs_selector": true, + "is_lookup": false + } + }, + "right": { + "GateOnly": { + "gate_idx": 0, + "num_constants": 4, + "degree": 1, + "needs_selector": true, + "is_lookup": false } } } } - }, - "right": { - "Fork": { - "left": { - "GateOnly": { - "gate_idx": 3, - "num_constants": 2, - "degree": 3, - "needs_selector": true, - "is_lookup": false - } - }, - "right": { - "GateOnly": { - "gate_idx": 8, - "num_constants": 0, - "degree": 0, - "needs_selector": true, - "is_lookup": false - } + } + }, + "right": { + "Fork": { + "left": { + "GateOnly": { + "gate_idx": 3, + "num_constants": 2, + "degree": 3, + "needs_selector": true, + "is_lookup": false + } + }, + "right": { + "GateOnly": { + "gate_idx": 8, + "num_constants": 0, + "degree": 0, + "needs_selector": true, + "is_lookup": false } } } } } } - }, - "fri_lde_factor": 2, - "cap_size": 16 + } }, - "setup_merkle_tree_cap": [ - [ - 2680192913777199386, - 7877900777764568562, - 7967270885539056261, - 11491786516879257714 - ], - [ - 1576848689219001454, - 2538042691131197824, - 16789498574115229290, - 3214129711903181558 - ], - [ - 856301905705619734, - 4331213335266799158, - 15267490766684530921, - 3265714654258242220 - ], - [ - 8865784570897245270, - 2362765988103793581, - 6943670874402562853, - 14632996114278721596 - ], - [ - 63247458005995468, - 12539771084927052853, - 13041512411442114569, - 9742813247561592554 - ], - [ - 16743936557271219178, - 14841453989210747254, - 12724413787690930702, - 10592542358880202219 - ], - [ - 16695338323889693576, - 8527536001711027994, - 13212045085202022064, - 11071462626939596790 - ], - [ - 18060750313558946749, - 15824434706098663517, - 775292596891170912, - 18445377984966327048 - ], - [ - 3549745875383468285, - 2238890537215251462, - 4591889095789072384, - 13012706980710418598 - ], - [ - 14771394899136640222, - 13143304103596416048, - 14456129193020560275, - 5740433968684323698 - ], - [ - 11651473654699970526, - 4694969877986805556, - 7029204199916750383, - 6916614362901685796 - ], - [ - 4368206191480113515, - 9562279231528697429, - 1907048590194817686, - 13209277185471975687 - ], - [ - 14438342866286439870, - 383769026263703315, - 1077241575478137065, - 1158227982301730574 - ], - [ - 10868817472877525981, - 11920954565057859026, - 10684659491915725994, - 15343028344024922569 - ], - [ - 4969179907509861760, - 3560160134545277440, - 11797495979614319546, - 13436348584120593030 - ], - [ - 8873263215018682993, - 13828390019511310487, - 12329030402425507188, - 18004618114160314165 - ] + "fri_lde_factor": 2, + "cap_size": 16 + }, + "setup_merkle_tree_cap": [ + [ + 2680192913777199386, + 7877900777764568562, + 7967270885539056261, + 11491786516879257714 + ], + [ + 1576848689219001454, + 2538042691131197824, + 16789498574115229290, + 3214129711903181558 + ], + [ + 856301905705619734, + 4331213335266799158, + 15267490766684530921, + 3265714654258242220 + ], + [ + 8865784570897245270, + 2362765988103793581, + 6943670874402562853, + 14632996114278721596 + ], + [ + 63247458005995468, + 12539771084927052853, + 13041512411442114569, + 9742813247561592554 + ], + [ + 16743936557271219178, + 14841453989210747254, + 12724413787690930702, + 10592542358880202219 + ], + [ + 16695338323889693576, + 8527536001711027994, + 13212045085202022064, + 11071462626939596790 + ], + [ + 18060750313558946749, + 15824434706098663517, + 775292596891170912, + 18445377984966327048 + ], + [ + 3549745875383468285, + 2238890537215251462, + 4591889095789072384, + 13012706980710418598 + ], + [ + 14771394899136640222, + 13143304103596416048, + 14456129193020560275, + 5740433968684323698 + ], + [ + 11651473654699970526, + 4694969877986805556, + 7029204199916750383, + 6916614362901685796 + ], + [ + 4368206191480113515, + 9562279231528697429, + 1907048590194817686, + 13209277185471975687 + ], + [ + 14438342866286439870, + 383769026263703315, + 1077241575478137065, + 1158227982301730574 + ], + [ + 10868817472877525981, + 11920954565057859026, + 10684659491915725994, + 15343028344024922569 + ], + [ + 4969179907509861760, + 3560160134545277440, + 11797495979614319546, + 13436348584120593030 + ], + [ + 8873263215018682993, + 13828390019511310487, + 12329030402425507188, + 18004618114160314165 ] - } + ] } \ No newline at end of file diff --git a/crates/proof-compression/scripts/download-ignition-transcripts.sh b/crates/proof-compression/scripts/download-ignition-transcripts.sh new file mode 100644 index 0000000..842d1ab --- /dev/null +++ b/crates/proof-compression/scripts/download-ignition-transcripts.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Configuration: adjust these variables as needed +BASE_URL="https://aztec-ignition.s3.eu-west-2.amazonaws.com/MAIN+IGNITION/sealed/transcript" # Base URL before the index +FILE_EXT=".dat" # File extension (if any) +START_INDEX=0 # Starting index +END_INDEX=15 # Ending index + +# Loop over the desired range of indexes +for ((i=START_INDEX; i<=END_INDEX; i++)); do + # Format the index as two digits (e.g., 01, 02, ..., 10) + FORMATTED_INDEX=$(printf "%02d" "$i") + + # Construct the full URL for the current file using the formatted index + FILE_URL="${BASE_URL}${FORMATTED_INDEX}${FILE_EXT}" + + echo "Downloading ${FILE_URL} ..." + + # Download the file using wget + wget "$FILE_URL" + + # Check if the download succeeded; if not, optionally handle errors + if [[ $? -ne 0 ]]; then + echo "Failed to download ${FILE_URL}" + # Optionally break or continue based on your needs + fi +done + +echo "Download complete." \ No newline at end of file diff --git a/crates/proof-compression/setup-instance.sh b/crates/proof-compression/scripts/setup-instance.sh similarity index 100% rename from crates/proof-compression/setup-instance.sh rename to crates/proof-compression/scripts/setup-instance.sh diff --git a/crates/proof-compression/src/blob_storage.rs b/crates/proof-compression/src/blob_storage.rs new file mode 100644 index 0000000..ca791f7 --- /dev/null +++ b/crates/proof-compression/src/blob_storage.rs @@ -0,0 +1,296 @@ +use std::io::{Read, Write}; + +pub trait BlobStorage: Send + Sync { + fn read_scheduler_vk(&self) -> Box; + fn read_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box; + fn read_compression_layer_vk(&self, circuit_id: u8) -> Box; + fn read_compression_layer_precomputation(&self, circuit_id: u8) -> Box; + + fn read_compression_wrapper_finalization_hint(&self, circuit_id: u8) -> Box; + fn read_compression_wrapper_vk(&self, circuit_id: u8) -> Box; + fn read_compression_wrapper_precomputation( + &self, + circuit_id: u8, + ) -> Box; + + fn read_fflonk_vk(&self) -> Box; + fn read_fflonk_precomputation(&self) -> Box; + + fn read_plonk_vk(&self) -> Box; + fn read_plonk_precomputation(&self) -> Box; + fn read_compact_raw_crs(&self) -> Box; +} + +pub trait BlobStorageExt: BlobStorage { + fn write_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box; + fn write_compression_layer_vk(&self, circuit_id: u8) -> Box; + fn write_compression_layer_precomputation(&self, circuit_id: u8) -> Box; + + fn write_compression_wrapper_finalization_hint(&self, circuit_id: u8) -> Box; + fn write_compression_wrapper_vk(&self, circuit_id: u8) -> Box; + fn write_compression_wrapper_precomputation(&self, circuit_id: u8) -> Box; + + fn write_fflonk_vk(&self) -> Box; + fn write_fflonk_precomputation(&self) -> Box; + + fn write_plonk_vk(&self) -> Box; + fn write_plonk_precomputation(&self) -> Box; + + fn write_compact_raw_crs(&self) -> Box; +} + +pub(crate) struct FileSystemBlobStorage; + +impl FileSystemBlobStorage { + const DATA_DIR_PATH: &str = "./data"; + const SCHEDULER_PREFIX: &str = "scheduler_recursive"; + const COMPRESSION_LAYER_PREFIX: &str = "compression"; + const COMPRESSION_WRAPPER_PREFIX: &str = "compression_wrapper"; + const FFLONK_PREFIX: &str = "fflonk"; + const PLONK_PREFIX: &str = "plonk"; + + fn open_file(path: &str) -> Box { + let file = std::fs::File::open(path).unwrap(); + Box::new(file) + } + + fn create_file(path: &str) -> Box { + let file = std::fs::File::create(path).unwrap(); + Box::new(file) + } +} + +impl BlobStorage for FileSystemBlobStorage { + fn read_scheduler_vk(&self) -> Box { + let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::SCHEDULER_PREFIX,); + println!("Reading scheduler vk at path {}", path); + Self::open_file(&path) + } + + fn read_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_hint.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_LAYER_PREFIX, + circuit_id + ); + println!("Reading compression layer finalization at path {}", path); + Self::open_file(&path) + } + + fn read_compression_layer_vk(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_vk.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_LAYER_PREFIX, + circuit_id + ); + println!("Reading compression layer vk at path {}", path); + Self::open_file(&path) + } + + fn read_compression_layer_precomputation(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_setup.bin", + Self::DATA_DIR_PATH, + Self::COMPRESSION_LAYER_PREFIX, + circuit_id + ); + println!("Reading compression layer precomputation at path {}", path); + Self::open_file(&path) + } + + fn read_compression_wrapper_finalization_hint(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_hint.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_WRAPPER_PREFIX, + circuit_id + ); + println!("Reading compression wrapper finalization at path {}", path); + Self::open_file(&path) + } + + fn read_compression_wrapper_vk(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_vk.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_WRAPPER_PREFIX, + circuit_id + ); + println!("Reading compression wrapper vk at path {}", path); + Self::open_file(&path) + } + + fn read_compression_wrapper_precomputation( + &self, + circuit_id: u8, + ) -> Box { + let path = format!( + "{}/{}_{}_setup.bin", + Self::DATA_DIR_PATH, + Self::COMPRESSION_WRAPPER_PREFIX, + circuit_id + ); + println!( + "Reading compression wrapper precomputation at path {}", + path + ); + Self::open_file(&path) + } + + fn read_fflonk_vk(&self) -> Box { + let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); + println!("Reading fflonk vk at path {}", path); + Self::open_file(&path) + } + + fn read_fflonk_precomputation(&self) -> Box { + let path = format!("{}/{}_setup.bin", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); + println!("Reading fflonk precomputation at path {}", path); + Self::open_file(&path) + } + + fn read_plonk_precomputation(&self) -> Box { + let path = format!("{}/{}_setup.bin", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); + println!("Reading plonk precomputation at path {}", path); + Self::open_file(&path) + } + + fn read_plonk_vk(&self) -> Box { + let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); + println!("Reading plonk vk at path {}", path); + Self::open_file(&path) + } + fn read_compact_raw_crs(&self) -> Box { + let path = format!("{}/compact_raw_crs.key", Self::DATA_DIR_PATH,); + println!("Reading CRS at path {}", path); + Self::open_file(&path) + } +} + +impl BlobStorageExt for FileSystemBlobStorage { + fn write_compression_layer_finalization_hint(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_hint.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_LAYER_PREFIX, + circuit_id + ); + println!("Writing compression layer finalization at path {}", path); + Self::create_file(&path) + } + + fn write_compression_layer_vk(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_vk.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_LAYER_PREFIX, + circuit_id + ); + println!("Writeing compression layer vk at path {}", path); + Self::create_file(&path) + } + + fn write_compression_layer_precomputation(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_setup.bin", + Self::DATA_DIR_PATH, + Self::COMPRESSION_LAYER_PREFIX, + circuit_id + ); + println!("Writeing compression layer precomputation at path {}", path); + Self::create_file(&path) + } + + fn write_compression_wrapper_finalization_hint(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_hint.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_WRAPPER_PREFIX, + circuit_id + ); + println!("Writeing compression wrapper finalization at path {}", path); + Self::create_file(&path) + } + + fn write_compression_wrapper_vk(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_vk.json", + Self::DATA_DIR_PATH, + Self::COMPRESSION_WRAPPER_PREFIX, + circuit_id + ); + println!("Writeing compression wrapper vk at path {}", path); + Self::create_file(&path) + } + + fn write_compression_wrapper_precomputation(&self, circuit_id: u8) -> Box { + let path = format!( + "{}/{}_{}_setup.bin", + Self::DATA_DIR_PATH, + Self::COMPRESSION_WRAPPER_PREFIX, + circuit_id + ); + println!( + "Writeing compression wrapper precomputation at path {}", + path + ); + Self::create_file(&path) + } + + fn write_fflonk_vk(&self) -> Box { + let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); + println!("Writeing fflonk vk at path {}", path); + Self::create_file(&path) + } + + fn write_fflonk_precomputation(&self) -> Box { + let path = format!("{}/{}_setup.bin", Self::DATA_DIR_PATH, Self::FFLONK_PREFIX); + println!("Writeing fflonk precomputation at path {}", path); + Self::create_file(&path) + } + + fn write_plonk_precomputation(&self) -> Box { + let path = format!("{}/{}_setup.bin", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); + println!("Writeing plonk precomputation at path {}", path); + Self::create_file(&path) + } + + fn write_plonk_vk(&self) -> Box { + let path = format!("{}/{}_vk.json", Self::DATA_DIR_PATH, Self::PLONK_PREFIX); + println!("Writeing plonk vk at path {}", path); + Self::create_file(&path) + } + + fn write_compact_raw_crs(&self) -> Box { + let path = format!("{}/compact_raw_crs.key", Self::DATA_DIR_PATH); + println!("Writeing compact raw CRS at path {}", path); + Self::create_file(&path) + } +} + +pub(crate) struct AsyncHandler { + receiver: std::thread::JoinHandle>, +} + +impl AsyncHandler +where + T: Send + Sync + 'static, +{ + pub(crate) fn spawn(f: F) -> Self + where + F: FnOnce() -> std::sync::mpsc::Receiver + Send + Sync + 'static, + { + let receiver = std::thread::spawn(f); + + Self { receiver } + } + + pub(crate) fn wait(self) -> T { + self.receiver.join().unwrap().recv().unwrap() + } +} + +unsafe impl Send for AsyncHandler where T: Send + Sync {} +unsafe impl Sync for AsyncHandler where T: Send + Sync {} diff --git a/crates/proof-compression/src/chain.rs b/crates/proof-compression/src/chain.rs new file mode 100644 index 0000000..0277be7 --- /dev/null +++ b/crates/proof-compression/src/chain.rs @@ -0,0 +1,243 @@ +use circuit_definitions::circuit_definitions::aux_layer::{ + compression_modes::{ + CompressionMode1, CompressionMode1ForWrapper, CompressionMode2, CompressionMode3, + CompressionMode4, CompressionMode5ForWrapper, + }, + ZkSyncCompressionForWrapperProof, ZkSyncCompressionLayerProof, +}; +use fflonk::FflonkSnarkVerifierCircuitProof; + +use super::*; + +pub trait ProofStorage { + fn get_scheduler_proof(&self) -> SchedulerProof; + fn save_compression_layer_proof(&mut self, circuit_id: u8, proof: ZkSyncCompressionLayerProof); + fn save_compression_wrapper_proof( + &mut self, + circuit_id: u8, + proof: ZkSyncCompressionForWrapperProof, + ); + fn save_plonk_proof(&mut self, proof: PlonkSnarkVerifierCircuitProof); + fn save_fflonk_proof(&mut self, proof: FflonkSnarkVerifierCircuitProof); +} + +pub struct SimpleProofStorage { + compression_layer_storage: std::collections::HashMap, + compression_wrapper_storage: std::collections::HashMap, + fflonk: Option, + plonk: Option, +} + +impl SimpleProofStorage { + pub fn new() -> Self { + Self { + compression_layer_storage: std::collections::HashMap::new(), + compression_wrapper_storage: std::collections::HashMap::new(), + fflonk: None, + plonk: None, + } + } +} + +impl ProofStorage for SimpleProofStorage { + fn get_scheduler_proof(&self) -> SchedulerProof { + let scheduler_proof_file = + std::fs::File::open("./data/scheduler_recursive_proof.json").unwrap(); + let scheduler_proof: circuit_definitions::circuit_definitions::recursion_layer::ZkSyncRecursionLayerProof = + serde_json::from_reader(&scheduler_proof_file).unwrap(); + let scheduler_proof = scheduler_proof.into_inner(); + scheduler_proof + } + fn save_compression_layer_proof(&mut self, circuit_id: u8, proof: ZkSyncCompressionLayerProof) { + self.compression_layer_storage.insert(circuit_id, proof); + } + + fn save_compression_wrapper_proof( + &mut self, + circuit_id: u8, + proof: ZkSyncCompressionForWrapperProof, + ) { + self.compression_wrapper_storage.insert(circuit_id, proof); + } + + fn save_plonk_proof(&mut self, proof: PlonkSnarkVerifierCircuitProof) { + self.plonk = Some(proof); + } + + fn save_fflonk_proof(&mut self, proof: FflonkSnarkVerifierCircuitProof) { + self.fflonk = Some(proof); + } +} + +pub enum SnarkWrapper { + Plonk, + FFfonk, +} +pub enum SnarkWrapperProof { + Plonk(PlonkSnarkVerifierCircuitProof), + FFfonk(FflonkSnarkVerifierCircuitProof), +} + +pub type SchedulerProof = franklin_crypto::boojum::cs::implementations::proof::Proof< + GoldilocksField, + circuit_definitions::circuit_definitions::recursion_layer::RecursiveProofsTreeHasher, + GoldilocksExt2, +>; + +pub fn run_proof_chain( + snark_wrapper: SnarkWrapper, + blob_storage: &BS, + scheduler_proof: SchedulerProof, +) -> SnarkWrapperProof +where + BS: BlobStorage, +{ + match snark_wrapper { + SnarkWrapper::Plonk => run_proof_chain_with_plonk(blob_storage, scheduler_proof), + SnarkWrapper::FFfonk => run_proof_chain_with_fflonk(blob_storage, scheduler_proof), + } +} + +pub fn run_proof_chain_with_fflonk( + blob_storage: &BS, + scheduler_proof: SchedulerProof, +) -> SnarkWrapperProof +where + BS: BlobStorage, +{ + let context_manager = SimpleContextManager::new(); + let start = std::time::Instant::now(); + ::run_pre_initialization_tasks(); + let compact_raw_crs = + ::load_compact_raw_crs(blob_storage); + let fflonk_precomputation = FflonkSnarkWrapper::get_precomputation(blob_storage); + let next_proof = + CompressionMode1::prove_compression_step(scheduler_proof, blob_storage, &context_manager); + + let next_proof = CompressionMode2::prove_compression_step::<_, SimpleContextManager>( + next_proof, + blob_storage, + &context_manager, + ); + let next_proof = CompressionMode3::prove_compression_step::<_, SimpleContextManager>( + next_proof, + blob_storage, + &context_manager, + ); + + let next_proof = CompressionMode4::prove_compression_step::<_, SimpleContextManager>( + next_proof, + blob_storage, + &context_manager, + ); + let next_proof = CompressionMode5ForWrapper::prove_compression_step::<_, SimpleContextManager>( + next_proof, + blob_storage, + &context_manager, + ); + println!( + "Proving entire compression chain took {}s", + start.elapsed().as_secs() + ); + let final_proof = FflonkSnarkWrapper::prove_snark_wrapper_step::<_, SimpleContextManager>( + compact_raw_crs, + fflonk_precomputation, + next_proof, + blob_storage, + &context_manager, + ); + println!( + "Proving entire chain with snark wrapper took {}s", + start.elapsed().as_secs() + ); + SnarkWrapperProof::FFfonk(final_proof) +} + +pub fn precompute_proof_chain_with_fflonk(blob_storage: &BS) +where + BS: BlobStorageExt, +{ + let context_manager = SimpleContextManager::new(); + ::run_pre_initialization_tasks(); + let compact_raw_crs = + ::load_compact_raw_crs(blob_storage); + + let start = std::time::Instant::now(); + CompressionMode1::precomputae_and_store_compression_circuits(blob_storage, &context_manager); + CompressionMode2::precomputae_and_store_compression_circuits(blob_storage, &context_manager); + CompressionMode3::precomputae_and_store_compression_circuits(blob_storage, &context_manager); + CompressionMode4::precomputae_and_store_compression_circuits(blob_storage, &context_manager); + CompressionMode5ForWrapper::precomputae_and_store_compression_circuits( + blob_storage, + &context_manager, + ); + println!( + "Precomputation of compression chain took {}s", + start.elapsed().as_secs() + ); + FflonkSnarkWrapper::precompute_and_store_snark_wrapper_circuit( + compact_raw_crs, + blob_storage, + &context_manager, + ); + println!( + "Precomputation of entire chain with fflonk took {}s", + start.elapsed().as_secs() + ); +} + +pub fn run_proof_chain_with_plonk( + blob_storage: &BS, + scheduler_proof: SchedulerProof, +) -> SnarkWrapperProof +where + BS: BlobStorage, +{ + let context_manager = SimpleContextManager::new(); + let start = std::time::Instant::now(); + let compact_raw_crs = + ::load_compact_raw_crs(blob_storage); + let plonk_precomputation = PlonkSnarkWrapper::get_precomputation(blob_storage); + + let next_proof = CompressionMode1ForWrapper::prove_compression_step( + scheduler_proof, + blob_storage, + &context_manager, + ); + + let final_proof = PlonkSnarkWrapper::prove_snark_wrapper_step( + compact_raw_crs, + plonk_precomputation, + next_proof, + blob_storage, + &context_manager, + ); + println!( + "Entire compression chain with plonk took {}s", + start.elapsed().as_secs() + ); + SnarkWrapperProof::Plonk(final_proof) +} + +pub fn precompute_proof_chain_with_plonk(blob_storage: &BS) +where + BS: BlobStorageExt, +{ + let context_manager = SimpleContextManager::new(); + let start = std::time::Instant::now(); + let compact_raw_crs = + ::load_compact_raw_crs(blob_storage); + CompressionMode1ForWrapper::precomputae_and_store_compression_circuits( + blob_storage, + &context_manager, + ); + PlonkSnarkWrapper::precompute_and_store_snark_wrapper_circuit( + compact_raw_crs, + blob_storage, + &context_manager, + ); + println!( + "Precomputation of entire chain with fflonk took {}s", + start.elapsed().as_secs() + ); +} diff --git a/crates/proof-compression/src/common.rs b/crates/proof-compression/src/common.rs deleted file mode 100644 index 2a047a9..0000000 --- a/crates/proof-compression/src/common.rs +++ /dev/null @@ -1,359 +0,0 @@ -use circuit_definitions::circuit_definitions::{ - aux_layer::{ - compression::{CompressionLayerCircuit, ProofCompressionFunction}, - compression_modes::CompressionTranscriptForWrapper, - CompressionProofsTreeHasher, CompressionProofsTreeHasherForWrapper, - ZkSyncCompressionForWrapperCircuit, ZkSyncCompressionLayerCircuit, ZkSyncCompressionProof, - ZkSyncCompressionProofForWrapper, ZkSyncCompressionVerificationKey, - ZkSyncCompressionVerificationKeyForWrapper, - }, - recursion_layer::{ - ZkSyncRecursionLayerProof, ZkSyncRecursionLayerVerificationKey, ZkSyncRecursionProof, - ZkSyncRecursionVerificationKey, - }, -}; -use fflonk::{ - FflonkSnarkVerifierCircuitProof, FflonkSnarkVerifierCircuitSetup, FflonkSnarkVerifierCircuitVK, -}; -use shivini::boojum::{ - algebraic_props::{round_function::AbsorptionModeOverwrite, sponge::GoldilocksPoseidon2Sponge}, - config::{CSConfig, DevCSConfig, ProvingCSConfig, SetupCSConfig}, - cs::{ - cs_builder::new_builder, - cs_builder_reference::CsReferenceImplementationBuilder, - implementations::{ - proof::Proof, reference_cs::CSReferenceAssembly, setup::FinalizationHintsForProver, - transcript::GoldilocksPoisedon2Transcript, verifier::Verifier, - }, - }, - field::goldilocks::{GoldilocksExt2, GoldilocksField}, -}; - -type F = GoldilocksField; -type EXT = GoldilocksExt2; -type DefaultTreeHasher = GoldilocksPoseidon2Sponge; -type DefaultTranscript = GoldilocksPoisedon2Transcript; - -pub type ZksyncProof = Proof; -pub type CompressionProofsTranscript = GoldilocksPoisedon2Transcript; - -pub fn load_scheduler_proof_and_vk( - path: &str, -) -> (ZkSyncRecursionProof, ZkSyncRecursionVerificationKey) { - let scheduler_vk_file = - std::fs::File::open(format!("{}/scheduler_recursive_vk.json", path)).unwrap(); - let scheduler_vk: ZkSyncRecursionLayerVerificationKey = - serde_json::from_reader(&scheduler_vk_file).unwrap(); - let scheduler_proof_file = - std::fs::File::open(format!("{}/scheduler_recursive_proof.json", path)).unwrap(); - let scheduler_proof: ZkSyncRecursionLayerProof = - serde_json::from_reader(&scheduler_proof_file).unwrap(); - - (scheduler_proof.into_inner(), scheduler_vk.into_inner()) -} - -#[derive(Copy, Clone, Debug)] -pub enum CompressionMode { - One = 1, - Two = 2, - Three = 3, - Four = 4, - Five = 5, -} - -impl CompressionMode { - pub fn from_compression_mode(compression_mode: u8) -> Self { - match compression_mode { - 1 => CompressionMode::One, - 2 => CompressionMode::Two, - 3 => CompressionMode::Three, - 4 => CompressionMode::Four, - 5 => CompressionMode::Five, - _ => unreachable!(), - } - } -} - -#[derive(Debug)] -pub struct CompressionSchedule { - pub name: &'static str, - pub compression_steps: Vec, -} - -impl CompressionSchedule { - pub fn name(&self) -> &'static str { - self.name - } - pub fn hard() -> Self { - CompressionSchedule { - name: "hard", - compression_steps: vec![ - CompressionMode::One, - CompressionMode::Two, - CompressionMode::Three, - CompressionMode::Four, - ], - } - } -} - -pub enum CompressionInput { - Recursion( - Option, - ZkSyncRecursionVerificationKey, - CompressionMode, - ), - Compression( - Option, - ZkSyncCompressionVerificationKey, - CompressionMode, - ), - CompressionWrapper( - Option, - ZkSyncCompressionVerificationKey, - CompressionMode, - ), -} - -impl CompressionInput { - pub fn into_compression_circuit(self) -> ZkSyncCompressionLayerCircuit { - match self { - CompressionInput::Recursion(proof, vk, compression_mode) => { - assert_eq!(compression_mode as u8, 1); - ZkSyncCompressionLayerCircuit::from_witness_and_vk(proof, vk, 1) - } - CompressionInput::Compression(proof, vk, compression_mode) => { - ZkSyncCompressionLayerCircuit::from_witness_and_vk( - proof, - vk, - compression_mode as u8, - ) - } - CompressionInput::CompressionWrapper(_, _, _) => { - unreachable!() - } - } - } - - pub fn into_compression_wrapper_circuit(self) -> ZkSyncCompressionForWrapperCircuit { - match self { - CompressionInput::Recursion(_, _, _) => { - unreachable!() - } - CompressionInput::Compression(_, _, _) => { - unreachable!() - } - CompressionInput::CompressionWrapper(proof, vk, compression_mode) => { - ZkSyncCompressionForWrapperCircuit::from_witness_and_vk( - proof, - vk, - compression_mode as u8, - ) - } - } - } -} - -pub fn save_compression_proof_and_vk_into_file( - proof: &ZkSyncCompressionProof, - vk: &ZkSyncCompressionVerificationKey, - compression_mode: u8, - path: &str, -) { - let proof_file = std::fs::File::create(format!( - "{}/compression_{}_proof.json", - path, compression_mode - )) - .unwrap(); - serde_json::to_writer(proof_file, &proof).unwrap(); - let vk_file = - std::fs::File::create(format!("{}/compression_{}_vk.json", path, compression_mode)) - .unwrap(); - serde_json::to_writer(vk_file, &vk).unwrap(); -} - -pub fn save_compression_wrapper_proof_and_vk_into_file( - proof: &ZkSyncCompressionProofForWrapper, - vk: &ZkSyncCompressionVerificationKeyForWrapper, - compression_mode: u8, - path: &str, -) { - let proof_file = std::fs::File::create(format!( - "{}/compression_wrapper_{}_proof.json", - path, compression_mode - )) - .unwrap(); - serde_json::to_writer(proof_file, &proof).unwrap(); - let vk_file = std::fs::File::create(format!( - "{}/compression_wrapper_{}_vk.json", - path, compression_mode - )) - .unwrap(); - serde_json::to_writer(vk_file, &vk).unwrap(); -} - -pub fn load_compression_wrapper_proof_and_vk_from_file( - blob_path: &str, - compression_mode: u8, -) -> ( - ZkSyncCompressionProofForWrapper, - ZkSyncCompressionVerificationKeyForWrapper, -) { - let proof_file = std::fs::File::open(format!( - "{}/compression_wrapper_{}_proof.json", - blob_path, compression_mode - )) - .unwrap(); - let proof = serde_json::from_reader(proof_file).unwrap(); - let vk_file = std::fs::File::open(format!( - "{}/compression_wrapper_{}_vk.json", - blob_path, compression_mode - )) - .unwrap(); - let vk = serde_json::from_reader(vk_file).unwrap(); - - (proof, vk) -} - -pub fn verify_compression_layer_circuit( - _circuit: CompressionLayerCircuit, - proof: &ZkSyncCompressionProof, - vk: &ZkSyncCompressionVerificationKey, - verifier: Verifier, -) -> bool { - verifier.verify::( - (), - vk, - proof, - ) -} - -pub fn verify_compression_wrapper_circuit( - _circuit: CompressionLayerCircuit, - proof: &ZkSyncCompressionProofForWrapper, - vk: &ZkSyncCompressionVerificationKeyForWrapper, - verifier: Verifier, -) -> bool { - verifier.verify::( - (), - vk, - proof, - ) -} - -pub fn save_fflonk_proof_and_vk_into_file( - proof: &FflonkSnarkVerifierCircuitProof, - vk: &FflonkSnarkVerifierCircuitVK, - output_blob_path: &str, -) { - let proof_file_path = format!("{}/final_proof.json", output_blob_path); - let proof_file = std::fs::File::create(&proof_file_path).unwrap(); - serde_json::to_writer(proof_file, &proof).unwrap(); - println!("proof saved at {proof_file_path}"); - let vk_file_path = format!("{}/final_vk.json", output_blob_path); - let vk_file = std::fs::File::create(&vk_file_path).unwrap(); - serde_json::to_writer(vk_file, &vk).unwrap(); - println!("vk saved at {vk_file_path}"); -} - -pub fn save_fflonk_setup_and_vk_into_file( - setup: &FflonkSnarkVerifierCircuitSetup, - vk: &FflonkSnarkVerifierCircuitVK, - output_blob_path: &str, -) { - let setup_file_path = format!("{}/fflonk_snark_setup.json", output_blob_path); - let setup_file = std::fs::File::create(&setup_file_path).unwrap(); - setup.write(&setup_file).unwrap(); - println!("fflonk precomputed setup saved into file at {setup_file_path}"); - let vk_file_path = format!("{}/fflonk_snark_setup.json", output_blob_path); - let vk_file = std::fs::File::create(&vk_file_path).unwrap(); - vk.write(&vk_file).unwrap(); - println!("fflonk VK saved into file at {vk_file_path}"); -} - -pub fn load_fflonk_setup_and_vk_from_file( - output_blob_path: &str, -) -> ( - FflonkSnarkVerifierCircuitSetup, - FflonkSnarkVerifierCircuitVK, -) { - let setup_file_path = format!("{}/fflonk_snark_setup.json", output_blob_path); - println!("reading fflonk precomputed setup from file at {setup_file_path}"); - let setup_file = std::fs::File::open(&setup_file_path).unwrap(); - let setup = FflonkSnarkVerifierCircuitSetup::read(&setup_file).unwrap(); - let vk_file_path = format!("{}/fflonk_snark_setup.json", output_blob_path); - println!("reading fflonk VK saved from file at {vk_file_path}"); - let vk_file = std::fs::File::open(&vk_file_path).unwrap(); - let vk = FflonkSnarkVerifierCircuitVK::read(&vk_file).unwrap(); - - (setup, vk) -} - -pub fn synthesize_circuit_for_setup( - circuit: CompressionLayerCircuit, -) -> ( - FinalizationHintsForProver, - CSReferenceAssembly, -) { - let geometry = circuit.geometry(); - let (max_trace_len, num_vars) = circuit.size_hint(); - - let builder_impl = CsReferenceImplementationBuilder::::new( - geometry, - max_trace_len.unwrap(), - ); - let builder = new_builder::<_, GoldilocksField>(builder_impl); - - let builder = circuit.configure_builder_proxy(builder); - let mut cs = builder.build(num_vars.unwrap()); - circuit.add_tables(&mut cs); - circuit.synthesize_into_cs(&mut cs); - let (_domain_size, finalization_hint) = cs.pad_and_shrink(); - let cs = cs.into_assembly::(); - - (finalization_hint, cs) -} - -pub fn synthesize_circuit_for_proving( - circuit: CompressionLayerCircuit, - finalization_hint: &FinalizationHintsForProver, -) -> CSReferenceAssembly { - let geometry = circuit.geometry(); - let (max_trace_len, num_vars) = circuit.size_hint(); - - let builder_impl = CsReferenceImplementationBuilder::::new( - geometry, - max_trace_len.unwrap(), - ); - let builder = new_builder::<_, GoldilocksField>(builder_impl); - - let builder = circuit.configure_builder_proxy(builder); - let mut cs = builder.build(num_vars.unwrap()); - circuit.add_tables(&mut cs); - circuit.synthesize_into_cs(&mut cs); - let _ = cs.pad_and_shrink_using_hint(&finalization_hint); - let cs = cs.into_assembly::(); - - cs -} -pub fn synthesize_circuit_for_dev( - circuit: CompressionLayerCircuit, -) -> CSReferenceAssembly { - let geometry = circuit.geometry(); - let (max_trace_len, num_vars) = circuit.size_hint(); - - let builder_impl = CsReferenceImplementationBuilder::::new( - geometry, - max_trace_len.unwrap(), - ); - let builder = new_builder::<_, GoldilocksField>(builder_impl); - - let builder = circuit.configure_builder_proxy(builder); - let mut cs = builder.build(num_vars.unwrap()); - circuit.add_tables(&mut cs); - circuit.synthesize_into_cs(&mut cs); - let _ = cs.pad_and_shrink(); - let cs = cs.into_assembly::(); - - cs -} diff --git a/crates/proof-compression/src/context.rs b/crates/proof-compression/src/context.rs new file mode 100644 index 0000000..9bd5510 --- /dev/null +++ b/crates/proof-compression/src/context.rs @@ -0,0 +1,72 @@ +use std::sync::atomic::AtomicBool; + +use crate::{AsyncHandler, CompressionProofSystem, SnarkWrapperProofSystem, SnarkWrapperStep}; + +pub(crate) trait ContextManagerInterface { + fn init_compression_context

(&self, config: P::ContextConfig) -> AsyncHandler + where + P: CompressionProofSystem; + + fn init_snark_context(&self, crs: AsyncHandler) -> AsyncHandler + where + S: SnarkWrapperStep; +} + +pub(crate) struct SimpleContextManager { + context_status: std::sync::Arc, +} + +impl SimpleContextManager { + pub fn new() -> Self { + Self { + context_status: std::sync::Arc::new(AtomicBool::new(false)), + } + } +} + +impl ContextManagerInterface for SimpleContextManager { + fn init_compression_context

(&self, config: P::ContextConfig) -> AsyncHandler + where + P: CompressionProofSystem, + { + assert!( + self.context_status + .load(std::sync::atomic::Ordering::Relaxed) + == false + ); + let flag = self.context_status.clone(); + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let context = P::init_context(config); + sender.send(context).unwrap(); + flag.store(false, std::sync::atomic::Ordering::Relaxed); + receiver + }; + + AsyncHandler::spawn(f) + } + fn init_snark_context( + &self, + compact_raw_crs: AsyncHandler, + ) -> AsyncHandler + where + S: SnarkWrapperProofSystem, + { + assert!( + self.context_status + .load(std::sync::atomic::Ordering::Relaxed) + == false + ); + // load next context + let flag = self.context_status.clone(); + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let context = S::init_context(compact_raw_crs); + sender.send(context).unwrap(); + flag.store(false, std::sync::atomic::Ordering::Relaxed); + receiver + }; + + AsyncHandler::spawn(f) + } +} diff --git a/crates/proof-compression/src/cpu.rs b/crates/proof-compression/src/cpu.rs deleted file mode 100644 index 7e9f8e1..0000000 --- a/crates/proof-compression/src/cpu.rs +++ /dev/null @@ -1,425 +0,0 @@ -use super::*; -use bellman::plonk::better_better_cs::cs::ProvingAssembly; -use bellman::plonk::better_better_cs::{ - cs::{Circuit, PlonkCsWidth3Params, SetupAssembly, TrivialAssembly}, - gates::naive_main_gate::NaiveMainGate, -}; -use circuit_definitions::circuit_definitions::recursion_layer::{ - ZkSyncRecursionProof, ZkSyncRecursionVerificationKey, -}; -use circuit_definitions::{ - circuit_definitions::aux_layer::{ - compression::{CompressionLayerCircuit, ProofCompressionFunction}, - ZkSyncCompressionForWrapperCircuit, ZkSyncCompressionLayerCircuit, ZkSyncCompressionProof, - ZkSyncCompressionProofForWrapper, ZkSyncCompressionVerificationKey, - ZkSyncCompressionVerificationKeyForWrapper, - }, - snark_wrapper::franklin_crypto::bellman::plonk::commitments::transcript::keccak_transcript::RollingKeccakTranscript, -}; - -use franklin_crypto::boojum::{ - config::{CSConfig, DevCSConfig}, - cs::{ - cs_builder::new_builder, - cs_builder_reference::CsReferenceImplementationBuilder, - implementations::{ - prover::ProofConfig, reference_cs::CSReferenceAssembly, - setup::FinalizationHintsForProver, - }, - }, - field::goldilocks::{GoldilocksExt2, GoldilocksField}, -}; - -use franklin_crypto::boojum::cs::implementations::proof::Proof as BoojumProof; -use franklin_crypto::boojum::cs::implementations::verifier::VerificationKey as BoojumVK; -use franklin_crypto::boojum::worker::Worker as BoojumWorker; - -pub const L1_VERIFIER_DOMAIN_SIZE_LOG: usize = 23; -pub const MAX_COMBINED_DEGREE_FACTOR: usize = 9; -type F = GoldilocksField; -type EXT = GoldilocksExt2; - -pub fn precompute_and_save_setup_for_fflonk_snark_circuit( - circuit: &FflonkSnarkVerifierCircuit, - worker: &Worker, - output_blob_path: &str, -) { - let compression_wrapper_mode = circuit.wrapper_function.numeric_circuit_type(); - println!("Compression mode: {compression_wrapper_mode}"); - let mut setup_assembly = SetupAssembly::::new(); - circuit.synthesize(&mut setup_assembly).expect("must work"); - assert!(setup_assembly.is_satisfied()); - setup_assembly.finalize(); - println!("Finalized assembly contains {} gates", setup_assembly.n()); - - let domain_size = setup_assembly.n() + 1; - assert!(domain_size.is_power_of_two()); - assert!(domain_size <= 1 << L1_VERIFIER_DOMAIN_SIZE_LOG); - - let mon_crs = fflonk::init_crs(&worker, domain_size); - let setup: FflonkSnarkVerifierCircuitSetup = - FflonkSetup::create_setup(&setup_assembly, &worker, &mon_crs).expect("fflonk setup"); - let vk = FflonkVerificationKey::from_setup(&setup, &mon_crs).unwrap(); - - save_fflonk_setup_and_vk_into_file(&setup, &vk, output_blob_path); -} - -pub fn prove_fflonk_snark_verifier_circuit_with_precomputation( - circuit: &FflonkSnarkVerifierCircuit, - precomputed_setup: &FflonkSnarkVerifierCircuitSetup, - vk: &FflonkSnarkVerifierCircuitVK, - worker: &Worker, -) -> FflonkSnarkVerifierCircuitProof { - let compression_wrapper_mode = circuit.wrapper_function.numeric_circuit_type(); - let mut assembly = ProvingAssembly::::new(); - circuit.synthesize(&mut assembly).expect("must work"); - assert!(assembly.is_satisfied()); - assembly.finalize(); - - let domain_size = assembly.n() + 1; - println!( - "Trace log length {} for compression mode {}", - domain_size.trailing_zeros(), - compression_wrapper_mode - ); - assert!(domain_size.is_power_of_two()); - - assert!(domain_size <= 1 << L1_VERIFIER_DOMAIN_SIZE_LOG); - - let mon_crs = fflonk::init_crs(&worker, domain_size); - - let proof = fflonk::fflonk_cpu::prover::create_proof::< - _, - FflonkSnarkVerifierCircuit, - _, - _, - _, - RollingKeccakTranscript, - >(&assembly, &worker, &precomputed_setup, &mon_crs, None) - .expect("proof"); - let valid = - fflonk::fflonk_cpu::verify::<_, _, RollingKeccakTranscript>(&vk, &proof, None).unwrap(); - assert!(valid, "proof verification fails"); - - proof -} - -pub fn prove_fflonk_snark_verifier_circuit_single_shot( - circuit: &FflonkSnarkVerifierCircuit, - worker: &Worker, -) -> ( - FflonkSnarkVerifierCircuitProof, - FflonkSnarkVerifierCircuitVK, -) { - let compression_wrapper_mode = circuit.wrapper_function.numeric_circuit_type(); - let mut assembly = TrivialAssembly::::new(); - circuit.synthesize(&mut assembly).expect("must work"); - assert!(assembly.is_satisfied()); - assembly.finalize(); - let domain_size = assembly.n() + 1; - assert!(domain_size.is_power_of_two()); - assert!(domain_size <= 1 << L1_VERIFIER_DOMAIN_SIZE_LOG); - println!( - "Trace log length {} for compression mode {}", - domain_size.trailing_zeros(), - compression_wrapper_mode - ); - - let max_combined_degree = fflonk::fflonk::compute_max_combined_degree_from_assembly::< - _, - _, - _, - _, - FflonkSnarkVerifierCircuit, - >(&assembly); - println!("Max degree is {}", max_combined_degree); - let mon_crs = fflonk::init_crs(&worker, domain_size); - let setup = FflonkSetup::create_setup(&assembly, &worker, &mon_crs).expect("setup"); - let vk = FflonkVerificationKey::from_setup(&setup, &mon_crs).unwrap(); - - let proof = fflonk::fflonk_cpu::prover::create_proof::< - _, - FflonkSnarkVerifierCircuit, - _, - _, - _, - RollingKeccakTranscript, - >(&assembly, &worker, &setup, &mon_crs, None) - .expect("proof"); - let valid = - fflonk::fflonk_cpu::verify::<_, _, RollingKeccakTranscript>(&vk, &proof, None).unwrap(); - assert!(valid, "proof verification fails"); - - (proof, vk) -} - -#[test] -#[ignore] -fn run_proof_compression_by_schedule() { - let path = if let Ok(path) = std::env::var("BLOB_PATH") { - path.to_string() - } else { - "./data".to_string() - }; - let (scheduler_proof, scheduler_vk) = load_scheduler_proof_and_vk(&path); - process_steps( - scheduler_proof, - scheduler_vk, - CompressionSchedule::hard(), - &path, - ); -} -pub fn process_steps( - proof: ZkSyncRecursionProof, - vk: ZkSyncRecursionVerificationKey, - schedule: CompressionSchedule, - path: &str, -) { - let worker = BoojumWorker::new(); - let mut input = CompressionInput::Recursion(Some(proof), vk, CompressionMode::One); - - dbg!(&schedule); - let CompressionSchedule { - name: compression_schedule_name, - compression_steps, - } = schedule; - - let last_compression_wrapping_mode = - CompressionMode::from_compression_mode(compression_steps.last().unwrap().clone() as u8 + 1); - dbg!(&last_compression_wrapping_mode); - - /* - This illustrates how compression enforced for the "hardest" strategy - - input compression verifier output compression wrapper - _____________________________ ____________ ___________ __________ ___________________ - scheduler proof vk 1 scheduler -> compressed1 compressed2 - compressed1 proof vk 2 compressed1 -> compressed2 compressed3 - compressed2 proof vk 3 compressed2 -> compressed3 compressed4 - compressed3 proof vk 4 compressed3 -> compressed4 compressed5 - - - compressed5 proof vk - compression wrapper5 -> fflonk proof - */ - - let num_compression_steps = compression_steps.len(); - let mut compression_modes_iter = compression_steps.into_iter(); - for step_idx in 0..num_compression_steps { - let compression_mode = compression_modes_iter.next().unwrap(); - let proof_file_path = format!("{path}/compression_{}_proof.json", compression_mode as u8); - let proof_file_path = std::path::Path::new(&proof_file_path); - let vk_file_path = format!("{path}/compression_{}_vk.json", compression_mode as u8); - let vk_file_path = std::path::Path::new(&vk_file_path); - if proof_file_path.exists() && vk_file_path.exists() { - println!( - "Compression {compression_schedule_name}/{} proof and vk already exist ignoring", - compression_mode as u8 - ); - let proof_file = std::fs::File::open(proof_file_path).unwrap(); - let proof = serde_json::from_reader(&proof_file).unwrap(); - let vk_file = std::fs::File::open(vk_file_path).unwrap(); - let vk = serde_json::from_reader(&vk_file).unwrap(); - if step_idx + 1 == num_compression_steps { - input = - CompressionInput::CompressionWrapper(proof, vk, last_compression_wrapping_mode) - } else { - input = CompressionInput::Compression( - proof, - vk, - CompressionMode::from_compression_mode(compression_mode as u8 + 1), - ) - } - - continue; - } - let compression_circuit = input.into_compression_circuit(); - let circuit_type = compression_circuit.numeric_circuit_type(); - println!( - "Proving compression {compression_schedule_name}/{}", - compression_mode as u8 - ); - let (proof, vk) = inner_prove_compression_layer_circuit(compression_circuit, &worker); - println!( - "Proof for compression {compression_schedule_name}/{} is generated!", - compression_mode as u8 - ); - - save_compression_proof_and_vk_into_file(&proof, &vk, compression_mode as u8, path); - if step_idx + 1 == num_compression_steps { - input = CompressionInput::CompressionWrapper( - Some(proof), - vk, - last_compression_wrapping_mode, - ); - } else { - input = CompressionInput::Compression( - Some(proof), - vk, - CompressionMode::from_compression_mode(compression_mode as u8 + 1), - ); - } - } - - // last wrapping step - let proof_file_path = format!( - "{path}/compression_wrapper_{}_proof.json", - last_compression_wrapping_mode as u8 - ); - let proof_file_path = std::path::Path::new(&proof_file_path); - let vk_file_path = format!( - "{path}/compression_wrapper_{}_vk.json", - last_compression_wrapping_mode as u8 - ); - let vk_file_path = std::path::Path::new(&vk_file_path); - println!( - "Compression for wrapper level {}", - last_compression_wrapping_mode as u8 - ); - if proof_file_path.exists() && vk_file_path.exists() { - println!( - "Compression {compression_schedule_name}/{} for wrapper proof and vk already exist ignoring", - last_compression_wrapping_mode as u8 - ); - } else { - println!( - "Proving compression {compression_schedule_name}/{} for wrapper", - last_compression_wrapping_mode as u8 - ); - let compression_circuit = input.into_compression_wrapper_circuit(); - let (proof, vk) = inner_prove_compression_wrapper_circuit(compression_circuit, &worker); - println!( - "Proof for compression wrapper {compression_schedule_name}/{} is generated!", - last_compression_wrapping_mode as u8 - ); - save_compression_wrapper_proof_and_vk_into_file( - &proof, - &vk, - last_compression_wrapping_mode as u8, - path, - ); - println!( - "Compression wrapper proof and vk for {compression_schedule_name}/{} saved", - last_compression_wrapping_mode as u8 - ); - } - - // final wrapping step - let final_proof_file_path = format!("{}/final_proof.json", path); - let final_proof_file_path = std::path::Path::new(&final_proof_file_path); - let final_vk_file_path = format!("{}/final_vk.json", path,); - let final_vk_file_path = std::path::Path::new(&final_vk_file_path); - - if final_proof_file_path.exists() == false || final_vk_file_path.exists() == false { - let (compression_wrapper_proof, compression_wrapper_vk) = - load_compression_wrapper_proof_and_vk_from_file( - path, - last_compression_wrapping_mode as u8, - ); - let wrapper_circuit = fflonk::init_snark_wrapper_circuit_from_inputs( - last_compression_wrapping_mode as u8, - compression_wrapper_proof, - compression_wrapper_vk, - ); - - let (final_proof, final_vk) = - prove_fflonk_snark_verifier_circuit_single_shot(&wrapper_circuit, &Worker::new()); - let final_proof_file = std::fs::File::create(final_proof_file_path).unwrap(); - serde_json::to_writer(&final_proof_file, &final_proof).unwrap(); - println!( - "final snark proof saved into {}", - final_proof_file_path.to_string_lossy() - ); - save_fflonk_proof_and_vk_into_file(&final_proof, &final_vk, &path); - } else { - println!( - "final proof already exists {}", - final_proof_file_path.to_string_lossy() - ); - } -} - -pub fn inner_prove_compression_layer_circuit( - circuit: ZkSyncCompressionLayerCircuit, - worker: &BoojumWorker, -) -> (ZkSyncCompressionProof, ZkSyncCompressionVerificationKey) { - let proof_config = circuit.proof_config_for_compression_step(); - let verifier_builder = circuit.into_dyn_verifier_builder(); - let verifier = verifier_builder.create_verifier(); - - let (proof, vk, is_proof_valid) = match circuit { - ZkSyncCompressionLayerCircuit::CompressionMode1Circuit(inner) => { - let (proof, vk) = - prove_compression_circuit_with_precomputations(inner.clone(), proof_config, worker); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode2Circuit(inner) => { - let (proof, vk) = - prove_compression_circuit_with_precomputations(inner.clone(), proof_config, worker); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode3Circuit(inner) => { - let (proof, vk) = - prove_compression_circuit_with_precomputations(inner.clone(), proof_config, worker); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode4Circuit(inner) => { - let (proof, vk) = - prove_compression_circuit_with_precomputations(inner.clone(), proof_config, worker); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode5Circuit(_inner) => { - unreachable!("Only 4 modes of compression is allowed") - } - }; - if is_proof_valid == false { - println!("Proof is invalid"); - } - (proof, vk) -} - -pub fn inner_prove_compression_wrapper_circuit( - circuit: ZkSyncCompressionForWrapperCircuit, - worker: &BoojumWorker, -) -> ( - ZkSyncCompressionProofForWrapper, - ZkSyncCompressionVerificationKeyForWrapper, -) { - let proof_config = circuit.proof_config_for_compression_step(); - let verifier_builder = circuit.into_dyn_verifier_builder(); - let verifier = verifier_builder.create_verifier(); - - let (proof, vk, is_proof_valid) = match circuit { - ZkSyncCompressionForWrapperCircuit::CompressionMode5Circuit(inner) => { - let (proof, vk) = - prove_compression_circuit_with_precomputations(inner.clone(), proof_config, worker); - let is_proof_valid = verify_compression_wrapper_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - _ => unreachable!("Only mode 5 is supported for wrapper"), - }; - if is_proof_valid == false { - println!("Proof is invalid"); - } - - (proof, vk) -} - -pub fn prove_compression_circuit_with_precomputations( - circuit: CompressionLayerCircuit, - proof_config: ProofConfig, - worker: &BoojumWorker, -) -> ( - BoojumProof, - BoojumVK::ThisLayerHasher>, -) { - let cs = synthesize_circuit_for_dev(circuit); - cs.prove_one_shot::<_, CF::ThisLayerTranscript, CF::ThisLayerHasher, CF::ThisLayerPoW>( - worker, - proof_config, - CF::this_layer_transcript_parameters(), - ) -} diff --git a/crates/proof-compression/src/gpu.rs b/crates/proof-compression/src/gpu.rs deleted file mode 100644 index 24329e3..0000000 --- a/crates/proof-compression/src/gpu.rs +++ /dev/null @@ -1,617 +0,0 @@ -use super::*; -use ::fflonk::*; -use boojum::cs::implementations::prover::ProofConfig; -use boojum::field::goldilocks::GoldilocksField as F; -use boojum::worker::Worker; -use shivini::boojum::cs::implementations::setup::FinalizationHintsForProver; -use shivini::boojum::cs::implementations::verifier::VerificationKey; -use shivini::circuit_definitions::boojum; -use shivini::circuit_definitions::circuit_definitions::{ - aux_layer::{ - compression::{CompressionLayerCircuit, ProofCompressionFunction}, - compression_modes::{CompressionTranscriptForWrapper, CompressionTreeHasherForWrapper}, - CompressionProofsTreeHasher, ZkSyncCompressionForWrapperCircuit, - ZkSyncCompressionLayerCircuit, ZkSyncCompressionProof, ZkSyncCompressionProofForWrapper, - ZkSyncCompressionVerificationKey, ZkSyncCompressionVerificationKeyForWrapper, - }, - recursion_layer::{ZkSyncRecursionProof, ZkSyncRecursionVerificationKey}, -}; -use shivini::cs::GpuSetup; -use shivini::gpu_proof_config::GpuProofConfig; -use shivini::{ - gpu_prove_from_external_witness_data_with_cache_strategy, CacheStrategy, - CommitmentCacheStrategy, GPUPoWRunner, GpuTreeHasher, PolynomialsCacheStrategy, ProverContext, - ProverContextConfig, -}; -use std::alloc::Global; - -#[test] -#[ignore] -fn run_proof_compression_by_schedule() { - let path = if let Ok(path) = std::env::var("BLOB_PATH") { - path.to_string() - } else { - "./data".to_string() - }; - let (scheduler_proof, scheduler_vk) = load_scheduler_proof_and_vk(&path); - process_steps( - scheduler_proof, - scheduler_vk, - CompressionSchedule::hard(), - &path, - ); -} - -pub fn process_steps( - proof: ZkSyncRecursionProof, - vk: ZkSyncRecursionVerificationKey, - schedule: CompressionSchedule, - path: &str, -) { - let worker = Worker::new(); - let mut input = CompressionInput::Recursion(Some(proof), vk, CompressionMode::One); - - dbg!(&schedule); - let CompressionSchedule { - name: compression_schedule_name, - compression_steps, - } = schedule; - - let last_compression_wrapping_mode = - CompressionMode::from_compression_mode(*compression_steps.last().unwrap() as u8 + 1); - dbg!(&last_compression_wrapping_mode); - - /* - This illustrates how compression enforced for the "hardest" strategy - - input compression verifier output compression wrapper - _____________________________ ____________ ___________ __________ ___________________ - scheduler proof vk 1 scheduler -> compressed1 compressed2 - compressed1 proof vk 2 compressed1 -> compressed2 compressed3 - compressed2 proof vk 3 compressed2 -> compressed3 compressed4 - compressed3 proof vk 4 compressed3 -> compressed4 compressed5 - - - compressed5 proof vk - compression wrapper5 -> fflonk proof - */ - - let num_compression_steps = compression_steps.len(); - let mut compression_modes_iter = compression_steps.into_iter(); - for step_idx in 0..num_compression_steps { - let compression_mode = compression_modes_iter.next().unwrap(); - let proof_file_path = format!("{}/compression_{}_proof.json", path, compression_mode as u8); - let proof_file_path = std::path::Path::new(&proof_file_path); - let vk_file_path = format!("{}/compression_{}_vk.json", path, compression_mode as u8); - let vk_file_path = std::path::Path::new(&vk_file_path); - if proof_file_path.exists() && vk_file_path.exists() { - println!( - "Compression {compression_schedule_name}/{} proof and vk already exist ignoring", - compression_mode as u8 - ); - let proof_file = std::fs::File::open(proof_file_path).unwrap(); - let input_proof = serde_json::from_reader(&proof_file).unwrap(); - let vk_file = std::fs::File::open(vk_file_path).unwrap(); - let input_vk = serde_json::from_reader(&vk_file).unwrap(); - if step_idx + 1 == num_compression_steps { - input = CompressionInput::CompressionWrapper( - input_proof, - input_vk, - last_compression_wrapping_mode, - ) - } else { - input = CompressionInput::Compression( - input_proof, - input_vk, - CompressionMode::from_compression_mode(compression_mode as u8 + 1), - ) - } - - continue; - } - - let compression_circuit = input.into_compression_circuit(); - let circuit_type = compression_circuit.numeric_circuit_type(); - println!( - "Proving compression {compression_schedule_name}/{}", - compression_mode as u8 - ); - let setup_file_path = format!("{path}/compression_{}_setup.bin", compression_mode as u8); - let setup_file_path = std::path::Path::new(&setup_file_path); - let (device_setup, vk, finalization_hint) = if setup_file_path.exists() { - load_compression_precomputations(compression_mode as u8, path) - } else { - let proof_config = compression_circuit.proof_config_for_compression_step(); - precompute_and_save_compression_layer_circuit_setup_and_vk::( - compression_circuit.clone(), - &worker, - proof_config, - path, - ) - }; - let (output_proof, output_vk) = prove_compression_layer_circuit_with_precomputations( - compression_circuit.clone(), - &device_setup, - finalization_hint, - vk, - &worker, - ); - println!( - "Proof for compression {compression_schedule_name}/{} is generated!", - compression_mode as u8 - ); - - save_compression_proof_and_vk_into_file(&output_proof, &output_vk, circuit_type, path); - - if step_idx + 1 == num_compression_steps { - input = CompressionInput::CompressionWrapper( - Some(output_proof), - output_vk, - last_compression_wrapping_mode, - ); - } else { - input = CompressionInput::Compression( - Some(output_proof), - output_vk, - CompressionMode::from_compression_mode(compression_mode as u8 + 1), - ); - } - } - - // last wrapping step - let proof_file_path = format!( - "{}/compression_wrapper_{}_proof.json", - path, last_compression_wrapping_mode as u8 - ); - let compression_wrapper_proof_file_path = std::path::Path::new(&proof_file_path); - let vk_file_path = format!( - "{}/compression_wrapper_{}_vk.json", - path, last_compression_wrapping_mode as u8 - ); - let compression_wrapper_vk_file_path = std::path::Path::new(&vk_file_path); - println!( - "Compression for wrapper level {}", - last_compression_wrapping_mode as u8 - ); - if compression_wrapper_proof_file_path.exists() && compression_wrapper_vk_file_path.exists() { - println!( - "Compression {compression_schedule_name}/{} for wrapper proof and vk already exist ignoring", - last_compression_wrapping_mode as u8 - ); - } else { - let compression_circuit = input.into_compression_wrapper_circuit(); - let setup_file_path = format!( - "{path}/compression_wrapper_{}_setup.bin", - last_compression_wrapping_mode as u8 - ); - let setup_file_path = std::path::Path::new(&setup_file_path); - let (device_setup, vk, finalization_hint) = if setup_file_path.exists() { - load_compression_precomputations(last_compression_wrapping_mode as u8, path) - } else { - let proof_config = compression_circuit.proof_config_for_compression_step(); - precompute_and_save_compression_wrapper_circuit_setup_and_vk::( - compression_circuit.clone(), - &worker, - proof_config, - path, - ) - }; - println!( - "Proving compression {compression_schedule_name}/{} for wrapper", - last_compression_wrapping_mode as u8 - ); - let (compression_wrapper_output_proof, compression_wrapper_output_vk) = - prove_compression_wrapper_circuit_with_precomputations( - compression_circuit, - &device_setup, - finalization_hint, - vk, - &worker, - ); - println!( - "Proof for compression wrapper {compression_schedule_name}/{} is generated!", - last_compression_wrapping_mode as u8 - ); - save_compression_wrapper_proof_and_vk_into_file( - &compression_wrapper_output_proof, - &compression_wrapper_output_vk, - last_compression_wrapping_mode as u8, - path, - ); - println!( - "Compression wrapper proof and vk for {compression_schedule_name}/{} saved", - last_compression_wrapping_mode as u8 - ); - } - // final wrapping step - let final_proof_file_path = format!("{}/final_proof.json", path); - let final_proof_file_path = std::path::Path::new(&final_proof_file_path); - let final_vk_file_path = format!("{}/final_vk.json", path,); - let final_vk_file_path = std::path::Path::new(&final_vk_file_path); - - if final_proof_file_path.exists() == false || final_vk_file_path.exists() == false { - let (compression_wrapper_proof, compression_wrapper_vk) = - load_compression_wrapper_proof_and_vk_from_file( - path, - last_compression_wrapping_mode as u8, - ); - let wrapper_circuit = init_snark_wrapper_circuit_from_inputs( - last_compression_wrapping_mode as u8, - compression_wrapper_proof, - compression_wrapper_vk, - ); - - let (final_proof, final_vk) = - ::fflonk::gpu_prove_fflonk_snark_verifier_circuit_single_shot(&wrapper_circuit); - let final_proof_file = std::fs::File::create(final_proof_file_path).unwrap(); - serde_json::to_writer(&final_proof_file, &final_proof).unwrap(); - println!( - "final snark proof saved into {}", - final_proof_file_path.to_string_lossy() - ); - save_fflonk_proof_and_vk_into_file(&final_proof, &final_vk, &path); - } else { - println!( - "final proof already exists {}", - final_proof_file_path.to_string_lossy() - ); - } -} - -pub fn prove_compression_layer_circuit_with_precomputations( - circuit: ZkSyncCompressionLayerCircuit, - device_setup: &GpuSetup, - finalization_hint: FinalizationHintsForProver, - vk: VerificationKey, - worker: &Worker, -) -> (ZkSyncCompressionProof, ZkSyncCompressionVerificationKey) { - let proof_config = circuit.proof_config_for_compression_step(); - let verifier_builder = circuit.into_dyn_verifier_builder(); - let verifier = verifier_builder.create_verifier(); - let gpu_proof_config = GpuProofConfig::from_compression_layer_circuit(&circuit); - - let (proof, vk, is_proof_valid) = match circuit { - ZkSyncCompressionLayerCircuit::CompressionMode1Circuit(inner) => { - let (proof, vk) = inner_prove_compression_layer_circuit( - inner.clone(), - device_setup, - finalization_hint, - vk, - proof_config, - gpu_proof_config, - worker, - ); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode2Circuit(inner) => { - let (proof, vk) = inner_prove_compression_layer_circuit( - inner.clone(), - device_setup, - finalization_hint, - vk, - proof_config, - gpu_proof_config, - worker, - ); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode3Circuit(inner) => { - let (proof, vk) = inner_prove_compression_layer_circuit( - inner.clone(), - device_setup, - finalization_hint, - vk, - proof_config, - gpu_proof_config, - worker, - ); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - ZkSyncCompressionLayerCircuit::CompressionMode4Circuit(inner) => { - let (proof, vk) = inner_prove_compression_layer_circuit( - inner.clone(), - device_setup, - finalization_hint, - vk, - proof_config, - gpu_proof_config, - worker, - ); - let is_proof_valid = verify_compression_layer_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - _ => unreachable!("Only 4 modes of compression is allowed"), - }; - if !is_proof_valid { - panic!("Proof is invalid"); - } - - (proof, vk) -} - -pub fn prove_compression_wrapper_circuit_with_precomputations( - circuit: ZkSyncCompressionForWrapperCircuit, - device_setup: &GpuSetup, - finalization_hint: FinalizationHintsForProver, - vk: VerificationKey, - worker: &Worker, -) -> ( - ZkSyncCompressionProofForWrapper, - ZkSyncCompressionVerificationKeyForWrapper, -) { - let proof_config = circuit.proof_config_for_compression_step(); - let verifier_builder = circuit.into_dyn_verifier_builder(); - let verifier = verifier_builder.create_verifier(); - let gpu_proof_config = GpuProofConfig::from_compression_wrapper_circuit(&circuit); - - let (proof, vk, is_proof_valid) = match circuit { - ZkSyncCompressionForWrapperCircuit::CompressionMode5Circuit(inner) => { - let (proof, vk) = inner_prove_compression_wrapper_circuit( - inner.clone(), - device_setup, - finalization_hint, - vk, - proof_config, - gpu_proof_config, - worker, - ); - let is_proof_valid = verify_compression_wrapper_circuit(inner, &proof, &vk, verifier); - (proof, vk, is_proof_valid) - } - _ => unreachable!("Only mode 5 is supported for wrapper"), - }; - if !is_proof_valid { - panic!("Proof is invalid"); - } - - (proof, vk) -} - -pub fn inner_prove_compression_layer_circuit< - CF: ProofCompressionFunction, ->( - circuit: CompressionLayerCircuit, - device_setup: &GpuSetup, - finalization_hint: FinalizationHintsForProver, - vk: VerificationKey, - proof_cfg: ProofConfig, - gpu_cfg: GpuProofConfig, - worker: &Worker, -) -> (ZkSyncCompressionProof, ZkSyncCompressionVerificationKey) { - let proving_cs = synthesize_circuit_for_proving(circuit, &finalization_hint); - let witness = proving_cs.witness.as_ref().unwrap(); - let domain_size = vk.fixed_parameters.domain_size as usize; - assert_eq!(finalization_hint.final_trace_len, domain_size); - let config = ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); - let ctx = ProverContext::create_with_config(config).expect("gpu prover context"); - let cache_strategy = CacheStrategy { - setup_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - trace_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - other_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - commitment: CommitmentCacheStrategy::CacheCosetCaps, - }; - let gpu_proof = gpu_prove_from_external_witness_data_with_cache_strategy::< - CompressionProofsTranscript, - CompressionProofsTreeHasher, - CF::ThisLayerPoW, - Global, - >( - &gpu_cfg, - witness, - proof_cfg.clone(), - &device_setup, - &vk, - (), - worker, - cache_strategy, - ) - .expect("gpu proof"); - drop(ctx); - let proof = gpu_proof.into(); - (proof, vk) -} - -pub fn inner_prove_compression_wrapper_circuit< - CF: ProofCompressionFunction, ->( - circuit: CompressionLayerCircuit, - device_setup: &GpuSetup, - finalization_hint: FinalizationHintsForProver, - vk: VerificationKey, - proof_cfg: ProofConfig, - gpu_cfg: GpuProofConfig, - worker: &Worker, -) -> ( - ZkSyncCompressionProofForWrapper, - ZkSyncCompressionVerificationKeyForWrapper, -) { - let proving_cs = synthesize_circuit_for_proving(circuit, &finalization_hint); - let witness = proving_cs.witness.as_ref().unwrap(); - let domain_size = vk.fixed_parameters.domain_size as usize; - assert_eq!(finalization_hint.final_trace_len, domain_size); - let config = ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); - let ctx = ProverContext::create_with_config(config).expect("gpu prover context"); - let cache_strategy = CacheStrategy { - setup_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - trace_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - other_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, - commitment: CommitmentCacheStrategy::CacheCosetCaps, - }; - let gpu_proof = gpu_prove_from_external_witness_data_with_cache_strategy::< - CompressionTranscriptForWrapper, - CompressionTreeHasherForWrapper, - CF::ThisLayerPoW, - Global, - >( - &gpu_cfg, - witness, - proof_cfg.clone(), - device_setup, - &vk, - (), - worker, - cache_strategy, - ) - .expect("gpu proof"); - drop(ctx); - (gpu_proof.into(), vk) -} - -pub fn precompute_and_save_compression_layer_circuit_setup_and_vk( - circuit: ZkSyncCompressionLayerCircuit, - worker: &Worker, - proof_cfg: ProofConfig, - path: &str, -) -> ( - GpuSetup, - VerificationKey, - FinalizationHintsForProver, -) { - match circuit { - ZkSyncCompressionLayerCircuit::CompressionMode1Circuit(compression_layer_circuit) => { - precompute_and_save_compression_circuit_setup_and_vk::< - _, - CompressionProofsTreeHasher, - SAVE, - >(compression_layer_circuit, 1, worker, proof_cfg, path) - } - ZkSyncCompressionLayerCircuit::CompressionMode2Circuit(compression_layer_circuit) => { - precompute_and_save_compression_circuit_setup_and_vk::< - _, - CompressionProofsTreeHasher, - SAVE, - >(compression_layer_circuit, 2, worker, proof_cfg, path) - } - ZkSyncCompressionLayerCircuit::CompressionMode3Circuit(compression_layer_circuit) => { - precompute_and_save_compression_circuit_setup_and_vk::< - _, - CompressionProofsTreeHasher, - SAVE, - >(compression_layer_circuit, 3, worker, proof_cfg, path) - } - ZkSyncCompressionLayerCircuit::CompressionMode4Circuit(compression_layer_circuit) => { - precompute_and_save_compression_circuit_setup_and_vk::< - _, - CompressionProofsTreeHasher, - SAVE, - >(compression_layer_circuit, 4, worker, proof_cfg, path) - } - _ => unreachable!("Only 4 modes of compression is allowed"), - } -} - -pub fn precompute_and_save_compression_wrapper_circuit_setup_and_vk( - circuit: ZkSyncCompressionForWrapperCircuit, - worker: &Worker, - proof_cfg: ProofConfig, - path: &str, -) -> ( - GpuSetup, - VerificationKey, - FinalizationHintsForProver, -) { - match circuit { - ZkSyncCompressionForWrapperCircuit::CompressionMode5Circuit( - compression_wrapper_circuit, - ) => precompute_and_save_compression_circuit_setup_and_vk::<_, _, SAVE>( - compression_wrapper_circuit, - 5, - worker, - proof_cfg, - path, - ), - _ => unreachable!("Only mode 5 is supported for wrapper"), - } -} - -pub fn precompute_and_save_compression_circuit_setup_and_vk< - CF: ProofCompressionFunction, - H: GpuTreeHasher, - const SAVE: bool, ->( - circuit: CompressionLayerCircuit, - compression_mode: usize, - worker: &Worker, - proof_cfg: ProofConfig, - path: &str, -) -> ( - GpuSetup, - VerificationKey, - FinalizationHintsForProver, -) { - let (finalization_hint, setup_assembly) = synthesize_circuit_for_setup::(circuit); - let (setup_base, vk_params, vars_hint, wits_hint) = setup_assembly.get_light_setup( - &worker, - proof_cfg.fri_lde_factor, - proof_cfg.merkle_tree_cap_size, - ); - let domain_size = vk_params.domain_size as usize; - assert_eq!(finalization_hint.final_trace_len, domain_size); - let config = ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); - let _ctx = ProverContext::create_with_config(config).expect("gpu prover context"); - let (device_setup, vk) = shivini::cs::gpu_setup_and_vk_from_base_setup_vk_params_and_hints::< - H, - _, - >(setup_base, vk_params, vars_hint, wits_hint, worker) - .unwrap(); - if SAVE { - let base_path = if compression_mode == 5 { - format!("{path}/compression_wrapper") - } else { - format!("{path}/compression") - }; - let setup_file_path = format!("{}_device_setup.bin", base_path); - println!("Saving setup into file {setup_file_path}"); - let setup_file = std::fs::File::create(&setup_file_path).unwrap(); - bincode::serialize_into(&setup_file, &device_setup).unwrap(); - println!("fflonk device setup saved into {}", setup_file_path); - - let vk_file_path = format!("{}_vk.json", base_path); - let vk_file = std::fs::File::create(&vk_file_path).unwrap(); - serde_json::to_writer(&vk_file, &vk).unwrap(); - println!("fflonk vk saved into {}", vk_file_path); - - let finalization_hint_file_path = format!("{}_hint.json", base_path); - let finalization_hint_file = std::fs::File::create(&finalization_hint_file_path).unwrap(); - serde_json::to_writer(&finalization_hint_file, &finalization_hint).unwrap(); - println!( - "finalization hint saved into {}", - finalization_hint_file_path - ); - } - - (device_setup, vk, finalization_hint) -} - -pub fn load_compression_precomputations( - compression_mode: u8, - path: &str, -) -> ( - GpuSetup, - VerificationKey, - FinalizationHintsForProver, -) { - let base_path = if compression_mode == 5 { - format!("{path}/compression_wrapper") - } else { - format!("{path}/compression") - }; - let setup_file_path = format!("{base_path}_device_setup.bin",); - println!("Loading device setup from file {setup_file_path}"); - let setup_file = std::fs::File::open(&setup_file_path).unwrap(); - let device_setup: GpuSetup = bincode::deserialize_from(&setup_file).unwrap(); - - let vk_file_path = format!("{base_path}_vk.json"); - println!("Loading vk from file {vk_file_path}"); - let vk_file = std::fs::File::open(&vk_file_path).unwrap(); - let vk = serde_json::from_reader(&vk_file).unwrap(); - - let finalization_hint_file_path = format!("{base_path}_hint.json"); - println!("Loading finalization hint from file {finalization_hint_file_path}"); - let finalization_hint_file = std::fs::File::open(&finalization_hint_file_path).unwrap(); - let finalization_hint = serde_json::from_reader(&finalization_hint_file).unwrap(); - - (device_setup, vk, finalization_hint) -} diff --git a/crates/proof-compression/src/lib.rs b/crates/proof-compression/src/lib.rs index ad2b976..816e95d 100644 --- a/crates/proof-compression/src/lib.rs +++ b/crates/proof-compression/src/lib.rs @@ -1,32 +1,24 @@ -#![allow(incomplete_features)] #![feature(generic_const_exprs)] #![feature(allocator_api)] -mod common; -use common::*; +mod blob_storage; +pub use blob_storage::*; -cfg_if::cfg_if! { - if #[cfg(feature = "gpu")] { - mod gpu; - pub use gpu::*; +mod chain; +pub use chain::*; - } else { - mod cpu; - pub use cpu::*; - } -} -use boojum::pairing::bn256::{Bn256, Fr}; -use circuit_definitions::circuit_definitions::aux_layer::ZkSyncSnarkWrapperCircuitNoLookupCustomGate; -use fflonk::{ - fflonk::{FflonkProof, FflonkSetup, FflonkVerificationKey}, - fflonk_cpu::franklin_crypto, -}; -use franklin_crypto::bellman; -use shivini::boojum; +mod context; +use context::*; -use bellman::worker::Worker; +pub mod proof_system; +pub use proof_system::create_compact_raw_crs; +use proof_system::*; -pub type FflonkSnarkVerifierCircuit = ZkSyncSnarkWrapperCircuitNoLookupCustomGate; -pub type FflonkSnarkVerifierCircuitVK = FflonkVerificationKey; -pub type FflonkSnarkVerifierCircuitProof = FflonkProof; -pub type FflonkSnarkVerifierCircuitSetup = FflonkSetup; +mod serialization; +use serialization::*; + +mod step; +use step::*; + +#[cfg(test)] +mod test; diff --git a/crates/proof-compression/src/proof_system/boojum.rs b/crates/proof-compression/src/proof_system/boojum.rs new file mode 100644 index 0000000..d59440e --- /dev/null +++ b/crates/proof-compression/src/proof_system/boojum.rs @@ -0,0 +1,245 @@ +use shivini::{ + boojum::{ + config::{ProvingCSConfig, SetupCSConfig}, + cs::{ + implementations::{ + proof::Proof, reference_cs::CSReferenceAssembly, setup::FinalizationHintsForProver, + transcript::Transcript, verifier::VerificationKey, witness::WitnessVec, + }, + traits::{circuit::CircuitBuilderProxy, GoodAllocator}, + }, + worker::Worker, + }, + gpu_proof_config::GpuProofConfig, + CacheStrategy, CommitmentCacheStrategy, GPUPoWRunner, GpuTreeHasher, PolynomialsCacheStrategy, + ProverContext, ProverContextConfig, +}; + +use circuit_definitions::circuit_definitions::aux_layer::compression::{ + CompressionLayerCircuit, ProofCompressionFunction, +}; + +use super::*; +pub(crate) use shivini::boojum; +pub(crate) use shivini::boojum::field::goldilocks::{GoldilocksExt2, GoldilocksField}; + +type BoojumAssembly = + CSReferenceAssembly; + +impl ProofSystemDefinition for CF +where + CF: ProofCompressionFunction< + ThisLayerHasher: GpuTreeHasher, + ThisLayerTranscript: Transcript< + GoldilocksField, + TransciptParameters = (), + CompatibleCap: Send + Sync + 'static, + >, + ThisLayerPoW: GPUPoWRunner, + > + 'static, +{ + type FieldElement = GoldilocksField; + type ExternalWitnessData = WitnessVec; + type Precomputation = BoojumDeviceSetupWrapper; + type Proof = Proof; + type VK = VerificationKey; + type FinalizationHint = FinalizationHintsForProver; + type Allocator = std::alloc::Global; + type ProvingAssembly = BoojumAssembly; + type Transcript = CF::ThisLayerTranscript; + fn verify(proof: &Self::Proof, vk: &Self::VK) -> bool { + let verifier_builder = + CircuitBuilderProxy::>::dyn_verifier_builder::(); + let verifier = verifier_builder.create_verifier(); + verifier.verify::( + (), + vk, + proof, + ) + } + + fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { + proving_assembly.witness.take().unwrap() + } +} + +impl CompressionProofSystem for CF +where + CF: ProofCompressionFunction< + ThisLayerHasher: GpuTreeHasher, + ThisLayerTranscript: Transcript< + GoldilocksField, + TransciptParameters = (), + CompatibleCap: Send + Sync + 'static, + >, + ThisLayerPoW: GPUPoWRunner, + > + 'static, +{ + type AuxConfig = GpuProofConfig; + type ContextConfig = usize; // domain_size + type Context = ProverContext; + + fn get_context_config() -> Self::ContextConfig { + // println!("Using hardcoded domain size 2^17 for compression step"); + 1 << 17 + } + + fn init_context(domain_size: Self::ContextConfig) -> Self::Context { + let config = + ProverContextConfig::default().with_smallest_supported_domain_size(domain_size); + let context = Self::Context::create_with_config(config).expect("gpu prover context"); + + context + } + fn get_context_config_from_hint(hint: &Self::FinalizationHint) -> Self::ContextConfig { + hint.final_trace_len + } + fn aux_config_from_assembly(proving_assembly: &Self::ProvingAssembly) -> Self::AuxConfig { + GpuProofConfig::from_assembly(proving_assembly) + } + + fn synthesize_for_proving( + circuit: CompressionLayerCircuit, + finalization_hint: Self::FinalizationHint, + ) -> Self::ProvingAssembly { + let geometry = circuit.geometry(); + let (max_trace_len, num_vars) = circuit.size_hint(); + + let builder_impl = boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< + GoldilocksField, + GoldilocksField, + ProvingCSConfig, + >::new(geometry, max_trace_len.unwrap()); + let builder = boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); + + let builder = circuit.configure_builder_proxy(builder); + let mut cs = builder.build(num_vars.unwrap()); + circuit.add_tables(&mut cs); + circuit.synthesize_into_cs(&mut cs); + let _ = cs.pad_and_shrink_using_hint(&finalization_hint); + let cs = cs.into_assembly::(); + + cs + } + + fn prove( + ctx: AsyncHandler, + proving_assembly: Self::ProvingAssembly, + aux_config: Self::AuxConfig, + precomputation: AsyncHandler, + finalization_hint: Self::FinalizationHint, + vk: &Self::VK, + ) -> Self::Proof { + Self::prove_from_witnesses( + ctx, + proving_assembly.witness.unwrap(), + aux_config, + precomputation, + finalization_hint, + vk, + ) + } + + fn prove_from_witnesses( + ctx: AsyncHandler, + witness: Self::ExternalWitnessData, + aux_config: Self::AuxConfig, + precomputation: AsyncHandler, + finalization_hint: Self::FinalizationHint, + vk: &Self::VK, + ) -> Self::Proof { + let domain_size = vk.fixed_parameters.domain_size as usize; + assert_eq!(finalization_hint.final_trace_len, domain_size); + let cache_strategy = CacheStrategy { + setup_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, + trace_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, + other_polynomials: PolynomialsCacheStrategy::CacheMonomialsAndFirstCoset, + commitment: CommitmentCacheStrategy::CacheCosetCaps, + }; + let worker = Worker::new(); + let precomputation = precomputation.wait().into_inner(); + let ctx = ctx.wait(); + let gpu_proof = shivini::gpu_prove_from_external_witness_data_with_cache_strategy::< + CF::ThisLayerTranscript, + CF::ThisLayerHasher, + CF::ThisLayerPoW, + Self::Allocator, + >( + &aux_config, + &witness, + CF::proof_config_for_compression_step(), + &precomputation, + &vk, + (), + &worker, + cache_strategy, + ) + .expect("gpu proof"); + drop(ctx); + let proof = gpu_proof.into(); + + proof + } +} + +impl CompressionProofSystemExt for CF +where + CF: ProofCompressionFunction< + ThisLayerHasher: GpuTreeHasher, + ThisLayerTranscript: Transcript< + GoldilocksField, + TransciptParameters = (), + CompatibleCap: Send + Sync + 'static, + >, + ThisLayerPoW: GPUPoWRunner, + > + 'static, + Self::Allocator: GoodAllocator, +{ + type SetupAssembly = BoojumAssembly; + fn generate_precomputation_and_vk( + ctx: AsyncHandler, + setup_assembly: Self::SetupAssembly, + finalization_hint: &Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK) { + let worker = Worker::new(); + let proof_config = CF::proof_config_for_compression_step(); + let (setup_base, vk_params, vars_hint, wits_hint) = setup_assembly.get_light_setup( + &worker, + proof_config.fri_lde_factor, + proof_config.merkle_tree_cap_size, + ); + let domain_size = vk_params.domain_size as usize; + assert_eq!(finalization_hint.final_trace_len, domain_size); + let ctx = ctx.wait(); + let (precomputation, vk) = + shivini::cs::gpu_setup_and_vk_from_base_setup_vk_params_and_hints::< + CF::ThisLayerHasher, + _, + >(setup_base, vk_params, vars_hint, wits_hint, &worker) + .unwrap(); + drop(ctx); + (BoojumDeviceSetupWrapper::from_inner(precomputation), vk) + } + fn synthesize_for_setup( + circuit: CompressionLayerCircuit, + ) -> (Self::FinalizationHint, Self::SetupAssembly) { + let geometry = circuit.geometry(); + let (max_trace_len, num_vars) = circuit.size_hint(); + + let builder_impl = boojum::cs::cs_builder_reference::CsReferenceImplementationBuilder::< + GoldilocksField, + GoldilocksField, + SetupCSConfig, + >::new(geometry, max_trace_len.unwrap()); + let builder = boojum::cs::cs_builder::new_builder::<_, GoldilocksField>(builder_impl); + + let builder = circuit.configure_builder_proxy(builder); + let mut cs = builder.build(num_vars.unwrap()); + circuit.add_tables(&mut cs); + circuit.synthesize_into_cs(&mut cs); + let (_domain_size, finalization_hint) = cs.pad_and_shrink(); + let cs = cs.into_assembly::(); + + (finalization_hint, cs) + } +} diff --git a/crates/proof-compression/src/utils.rs b/crates/proof-compression/src/proof_system/crs.rs similarity index 61% rename from crates/proof-compression/src/utils.rs rename to crates/proof-compression/src/proof_system/crs.rs index c16945d..8158db5 100644 --- a/crates/proof-compression/src/utils.rs +++ b/crates/proof-compression/src/proof_system/crs.rs @@ -1,6 +1,108 @@ use super::*; +use ::fflonk::{ + bellman::kate_commitment::{Crs, CrsForMonomialForm}, + hardcoded_g2_bases, +}; +use bellman::{bn256::Bn256, CurveAffine, Engine, Field, PrimeField}; +use byteorder::{BigEndian, ReadBytesExt}; +use gpu_prover::ManagerConfigs; + +pub(crate) fn write_crs_into_raw_compact_form( + original_crs: &Crs, + mut dst_raw_compact_crs: W, +) -> std::io::Result<()> { + use bellman::CurveAffine; + use bellman::{PrimeField, PrimeFieldRepr}; + use byteorder::{BigEndian, WriteBytesExt}; + let num_points = original_crs.g1_bases.len(); + dst_raw_compact_crs.write_u32::(num_points as u32)?; + for g1_base in original_crs.g1_bases.iter() { + let (x, y) = g1_base.as_xy(); + x.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + y.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + } + assert_eq!(original_crs.g2_monomial_bases.len(), 2); + for g2_base in original_crs.g2_monomial_bases.iter() { + let (x, y) = g2_base.as_xy(); + x.c0.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + x.c1.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + y.c0.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + y.c1.into_raw_repr().write_le(&mut dst_raw_compact_crs)?; + } + + Ok(()) +} + +pub(crate) fn read_crs_from_raw_compact_form( + mut src_raw_compact_crs: R, + num_g1_points: usize, +) -> std::io::Result> { + // requested number of bases can be smaller than the available bases + use byteorder::{BigEndian, ReadBytesExt}; + let actual_num_points = src_raw_compact_crs.read_u32::()? as usize; + assert!(num_g1_points <= actual_num_points as usize); + let mut g1_bases = Vec::with_capacity_in(num_g1_points, A::default()); + unsafe { + g1_bases.set_len(num_g1_points); + let buf = std::slice::from_raw_parts_mut( + g1_bases.as_mut_ptr() as *mut u8, + num_g1_points * std::mem::size_of::(), + ); + src_raw_compact_crs.read_exact(buf)?; + } + + let g2_bases = hardcoded_g2_bases::().to_vec_in(A::default()); + + Ok(Crs::<_, CrsForMonomialForm, A>::new_in(g1_bases, g2_bases)) +} + +pub fn create_compact_raw_crs(dst: W) { + let num_points = [ + ::fflonk::MAX_COMBINED_DEGREE_FACTOR << ::fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG, + ::FULL_SLOT_SIZE, + ] + .into_iter() + .max() + .unwrap(); + let original_crs = make_crs_from_ignition_transcripts(num_points); + assert_eq!(original_crs.g1_bases.len(), num_points); + write_crs_into_raw_compact_form(&original_crs, dst).unwrap(); +} + +fn make_crs_from_ignition_transcripts(num_points: usize) -> Crs { + let transcripts_dir = + std::env::var("IGNITION_TRANSCRIPT_PATH").expect("IGNITION_TRANSCRIPT_PATH env variable"); + let chunk_size = 5_040_000usize; + let num_chunks = num_points.div_ceil(chunk_size); + + // Check transcript files already downloaded from "https://aztec-ignition.s3.eu-west-2.amazonaws.com/MAIN+IGNITION/sealed/transcript{idx}.dat"; + for idx in 0..num_chunks { + let transcript_file_path = format!("{}/transcript{:02}.dat", transcripts_dir, idx); + let transcript_file_path = std::path::Path::new(&transcript_file_path); + assert!( + transcript_file_path.exists(), + "CRS transcript file {:?} couldn't found.", + transcript_file_path + ); + } + + // transform + let crs = create_crs_from_ignition_transcript(&transcripts_dir, num_chunks).unwrap(); -pub fn make_crs_from_ignition_transcript + ?Sized>( + let bellman::kate_commitment::Crs { + g1_bases, + g2_monomial_bases, + .. + } = crs; + assert!(g1_bases.len() >= num_points); + let mut g1_bases = std::sync::Arc::try_unwrap(g1_bases).unwrap(); + let g2_monomial_bases = std::sync::Arc::try_unwrap(g2_monomial_bases).unwrap(); + g1_bases.truncate(num_points); + + Crs::new(g1_bases, g2_monomial_bases) +} + +fn create_crs_from_ignition_transcript + ?Sized>( path: &S, num_chunks: usize, ) -> Result< @@ -169,34 +271,6 @@ pub fn make_crs_from_ignition_transcript + ?Sized>( Ok(new) } -pub fn transform_ignition_transcripts(domain_size: usize) { - let transcripts_dir = std::env::var("IGNITION_TRANSCRIPT_PATH").unwrap_or("./".to_string()); - let chunk_size = 5_040_000usize; - let num_chunks = domain_size.div_ceil(chunk_size); - - // Check transcript files already downloaded from "https://aztec-ignition.s3.eu-west-2.amazonaws.com/MAIN+IGNITION/sealed/transcript{idx}.dat"; - for idx in 0..num_chunks { - let transcript_file_path = format!("{}/transcript{:02}.dat", transcripts_dir, idx); - let transcript_file_path = std::path::Path::new(&transcript_file_path); - assert!(transcript_file_path.exists()); - } - - // transform - let crs = make_crs_from_ignition_transcript(&transcripts_dir, num_chunks).unwrap(); - let out_path = format!("{}/full_ignition.key", &transcripts_dir); - let out_file = std::fs::File::create(&out_path).unwrap(); - - let Crs { - g1_bases, - g2_monomial_bases, - .. - } = crs; - assert!(g1_bases.len() >= domain_size); - let mut g1_bases = std::sync::Arc::try_unwrap(g1_bases).unwrap(); - let g2_monomial_bases = std::sync::Arc::try_unwrap(g2_monomial_bases).unwrap(); - g1_bases.truncate(domain_size); - - let crs: Crs = Crs::new(g1_bases, g2_monomial_bases); - crs.write(&out_file).unwrap(); - println!("full ignition ceremony saved into {out_path}"); +pub(crate) fn hardcoded_canonical_g2_bases() -> [bellman::bn256::G2Affine; 2] { + ::fflonk::hardcoded_g2_bases::() } diff --git a/crates/proof-compression/src/proof_system/fflonk.rs b/crates/proof-compression/src/proof_system/fflonk.rs new file mode 100644 index 0000000..f4e1f0c --- /dev/null +++ b/crates/proof-compression/src/proof_system/fflonk.rs @@ -0,0 +1,169 @@ +use ::fflonk::fflonk_cpu::{FflonkProof, FflonkVerificationKey}; +use ::fflonk::{ + CombinedMonomialDeviceStorage, DeviceContextWithSingleDevice, + FflonkSnarkVerifierCircuitDeviceSetup, +}; +use bellman::bn256::{Bn256, Fr}; +use bellman::{ + kate_commitment::CrsForMonomialForm, + plonk::{ + better_better_cs::{ + cs::{ + Assembly, Circuit, PlonkCsWidth3Params, SynthesisModeGenerateSetup, + SynthesisModeProve, + }, + gates::naive_main_gate::NaiveMainGate, + }, + commitments::transcript::keccak_transcript::RollingKeccakTranscript, + }, +}; +use circuit_definitions::circuit_definitions::aux_layer::ZkSyncSnarkWrapperCircuitNoLookupCustomGate; + +use super::*; +pub(crate) use ::fflonk::HostAllocator; +pub(crate) type FflonkSnarkVerifierCircuit = ZkSyncSnarkWrapperCircuitNoLookupCustomGate; +pub(crate) type FflonkSnarkVerifierCircuitVK = + FflonkVerificationKey; +pub(crate) type FflonkSnarkVerifierCircuitProof = FflonkProof; +type FflonkAssembly = Assembly; +pub(crate) struct FflonkSnarkWrapper; + +impl ProofSystemDefinition for FflonkSnarkWrapper { + type FieldElement = Fr; + type Precomputation = FflonkSnarkVerifierCircuitDeviceSetupWrapper; + type ExternalWitnessData = ( + Vec, + Vec, + ); + type Proof = FflonkSnarkVerifierCircuitProof; + type VK = FflonkSnarkVerifierCircuitVK; + type FinalizationHint = usize; + // Pinned memory with small allocations is expensive e.g Assembly storage + type Allocator = std::alloc::Global; + type ProvingAssembly = FflonkAssembly; + type Transcript = RollingKeccakTranscript; + fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { + let input_assignments = + std::mem::replace(&mut proving_assembly.input_assingments, Vec::new()); + let aux_assignments = std::mem::replace( + &mut proving_assembly.aux_assingments, + Vec::new_in(Self::Allocator::default()), + ); + + (input_assignments, aux_assignments) + } + + fn verify(proof: &Self::Proof, vk: &Self::VK) -> bool { + ::fflonk::fflonk_cpu::verify::<_, FflonkSnarkVerifierCircuit, Self::Transcript>( + vk, proof, None, + ) + .unwrap() + } +} + +impl SnarkWrapperProofSystem for FflonkSnarkWrapper { + type Circuit = FflonkSnarkVerifierCircuit; + type Context = DeviceContextWithSingleDevice; + type CRS = bellman::kate_commitment::Crs< + bellman::compact_bn256::Bn256, + CrsForMonomialForm, + Self::Allocator, + >; + + fn pre_init() { + let domain_size = 1 << ::fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG; + Self::Context::init_pinned_memory(domain_size).unwrap(); + } + + fn load_compact_raw_crs(src: R) -> Self::CRS { + let domain_size = 1 << ::fflonk::fflonk_cpu::L1_VERIFIER_DOMAIN_SIZE_LOG; + let num_g1_bases_for_crs = ::fflonk::fflonk_cpu::MAX_COMBINED_DEGREE_FACTOR * domain_size; + read_crs_from_raw_compact_form(src, num_g1_bases_for_crs).unwrap() + } + + fn init_context(compact_raw_crs: AsyncHandler) -> Self::Context { + let compact_raw_crs = compact_raw_crs.wait(); + let domain_size = 1 << ::fflonk::fflonk_cpu::L1_VERIFIER_DOMAIN_SIZE_LOG; + let context = DeviceContextWithSingleDevice::init_from_preloaded_crs::( + domain_size, + compact_raw_crs, + ) + .unwrap(); + context + } + + fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly { + let mut proving_assembly = FflonkAssembly::::new(); + circuit + .synthesize(&mut proving_assembly) + .expect("must work"); + proving_assembly + } + + fn prove( + ctx: AsyncHandler, + mut proving_assembly: Self::ProvingAssembly, + precomputation: AsyncHandler, + finalization_hint: Self::FinalizationHint, + ) -> Self::Proof { + assert!(proving_assembly.is_satisfied()); + let raw_trace_len = proving_assembly.n(); + assert!(finalization_hint.is_power_of_two()); + proving_assembly.finalize_to_size_log_2(finalization_hint.trailing_zeros() as usize); + let domain_size = proving_assembly.n() + 1; + assert!(domain_size.is_power_of_two()); + assert_eq!(domain_size, finalization_hint); + let ctx = ctx.wait(); + let precomputation = precomputation.wait().into_inner(); + let start = std::time::Instant::now(); + let proof = ::fflonk::create_proof::<_, _, _, RollingKeccakTranscript<_>, _>( + &proving_assembly, + &precomputation, + raw_trace_len, + ) + .unwrap(); + println!("fflonk proving takes {} s", start.elapsed().as_secs()); + drop(ctx); + proof + } + + fn prove_from_witnesses( + _: AsyncHandler, + _: Self::ExternalWitnessData, + _: AsyncHandler, + _: Self::FinalizationHint, + ) -> Self::Proof { + unimplemented!() + } +} + +impl SnarkWrapperProofSystemExt for FflonkSnarkWrapper { + type SetupAssembly = FflonkAssembly; + + fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly { + let mut setup_assembly = + FflonkAssembly::::new(); + circuit.synthesize(&mut setup_assembly).unwrap(); + + setup_assembly + } + + fn generate_precomputation_and_vk( + ctx: AsyncHandler, + setup_assembly: Self::SetupAssembly, + _hardcoded_finalization_hint: Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK) { + let ctx = ctx.wait(); + let device_setup = + FflonkSnarkVerifierCircuitDeviceSetup::::create_setup_from_assembly_on_device( + &setup_assembly, + ) + .unwrap(); + let vk = device_setup.get_verification_key(); + drop(ctx); + ( + FflonkSnarkVerifierCircuitDeviceSetupWrapper(device_setup), + vk, + ) + } +} diff --git a/crates/proof-compression/src/proof_system/interface.rs b/crates/proof-compression/src/proof_system/interface.rs new file mode 100644 index 0000000..724f76b --- /dev/null +++ b/crates/proof-compression/src/proof_system/interface.rs @@ -0,0 +1,130 @@ +use super::*; + +use circuit_definitions::circuit_definitions::aux_layer::compression::{ + CompressionLayerCircuit, ProofCompressionFunction, +}; +use shivini::{ + boojum::cs::implementations::{ + fast_serialization::MemcopySerializable, transcript::Transcript, + }, + GPUPoWRunner, GpuTreeHasher, +}; + +pub(crate) trait ProofSystemDefinition: Sized { + type FieldElement; + type ExternalWitnessData; + type Precomputation: MemcopySerializable + Send + Sync + 'static; + type Proof: serde::Serialize + serde::de::DeserializeOwned; + type VK: serde::Serialize + serde::de::DeserializeOwned + Send + Sync + Clone + 'static; + type FinalizationHint: serde::Serialize + serde::de::DeserializeOwned + Clone; + type Allocator: std::alloc::Allocator + Default; + type ProvingAssembly: Sized + Send + Sync + 'static; + type Transcript; + fn take_witnesses(proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData; + fn verify(_: &Self::Proof, _: &Self::VK) -> bool; +} + +pub(crate) trait CompressionProofSystem: + ProofCompressionFunction< + ThisLayerHasher: GpuTreeHasher, + ThisLayerTranscript: Transcript, + ThisLayerPoW: GPUPoWRunner, + > + ProofSystemDefinition +{ + type ContextConfig: Send + Sync + 'static; + type Context: Send + Sync + 'static; + type AuxConfig; + fn get_context_config() -> Self::ContextConfig; + fn get_context_config_from_hint(_: &Self::FinalizationHint) -> Self::ContextConfig; + fn init_context(config: Self::ContextConfig) -> Self::Context; + fn aux_config_from_assembly(proving_assembly: &Self::ProvingAssembly) -> Self::AuxConfig; + + fn synthesize_for_proving( + circuit: CompressionLayerCircuit, + finalization_hint: Self::FinalizationHint, + ) -> Self::ProvingAssembly; + + fn prove( + _: AsyncHandler, + _: Self::ProvingAssembly, + _: Self::AuxConfig, + _: AsyncHandler, + _: Self::FinalizationHint, + _: &Self::VK, + ) -> Self::Proof; + + fn prove_from_witnesses( + _: AsyncHandler, + _: Self::ExternalWitnessData, + _: Self::AuxConfig, + _: AsyncHandler, + _: Self::FinalizationHint, + _: &Self::VK, + ) -> Self::Proof; +} + +pub(crate) trait CompressionProofSystemExt: CompressionProofSystem { + type SetupAssembly; + fn generate_precomputation_and_vk( + _: AsyncHandler, + _: Self::SetupAssembly, + _: &Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK); + fn synthesize_for_setup( + circuit: CompressionLayerCircuit, + ) -> (Self::FinalizationHint, Self::SetupAssembly); +} + +pub(crate) trait SnarkWrapperProofSystem: ProofSystemDefinition { + type Circuit; + type Context: Send + Sync + 'static; + type CRS: Send + Sync + 'static; + fn pre_init(); + fn init_context(crs: AsyncHandler) -> Self::Context; + fn load_compact_raw_crs(src: R) -> Self::CRS; + fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly; + fn prove( + _: AsyncHandler, + _: Self::ProvingAssembly, + _: AsyncHandler, + _: Self::FinalizationHint, + ) -> Self::Proof; + + fn prove_from_witnesses( + _: AsyncHandler, + _: Self::ExternalWitnessData, + _: AsyncHandler, + _: Self::FinalizationHint, + ) -> Self::Proof; +} + +pub(crate) trait SnarkWrapperProofSystemExt: SnarkWrapperProofSystem { + type SetupAssembly; + fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly; + fn generate_precomputation_and_vk( + _: AsyncHandler, + _: Self::SetupAssembly, + _: Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK); +} + +pub(crate) struct MarkerProofSystem; + +impl ProofSystemDefinition for MarkerProofSystem { + type FieldElement = (); + type Precomputation = MarkerPrecomputation; + type Proof = (); + type VK = (); + type ExternalWitnessData = (); + type FinalizationHint = (); + type Allocator = std::alloc::Global; + type ProvingAssembly = (); + type Transcript = (); + fn verify(_: &Self::Proof, _: &Self::VK) -> bool { + unreachable!() + } + + fn take_witnesses(_proving_assembly: &mut Self::ProvingAssembly) -> Self::ExternalWitnessData { + unreachable!() + } +} diff --git a/crates/proof-compression/src/proof_system/mod.rs b/crates/proof-compression/src/proof_system/mod.rs new file mode 100644 index 0000000..c1832ca --- /dev/null +++ b/crates/proof-compression/src/proof_system/mod.rs @@ -0,0 +1,18 @@ +use std::alloc::Allocator; + +use super::*; + +mod boojum; +pub(crate) use boojum::*; +mod crs; +pub(crate) use crs::*; +pub use crs::*; +mod fflonk; +pub(crate) use fflonk::*; +mod interface; +pub(crate) use interface::*; +mod plonk; +pub(crate) use plonk::*; + +pub(crate) use ::fflonk::fflonk_cpu::franklin_crypto; +pub(crate) use franklin_crypto::bellman; diff --git a/crates/proof-compression/src/proof_system/plonk.rs b/crates/proof-compression/src/proof_system/plonk.rs new file mode 100644 index 0000000..6be7fe7 --- /dev/null +++ b/crates/proof-compression/src/proof_system/plonk.rs @@ -0,0 +1,222 @@ +use bellman::{ + kate_commitment::{Crs, CrsForMonomialForm}, + plonk::{ + better_better_cs::{ + cs::{ + Assembly, Circuit, PlonkCsWidth4WithNextStepAndCustomGatesParams, + SynthesisModeGenerateSetup, SynthesisModeProve, + }, + gates::selector_optimized_with_d_next::SelectorOptimizedWidth4MainGateWithDNext, + }, + commitments::transcript::keccak_transcript::RollingKeccakTranscript, + }, +}; +use circuit_definitions::circuit_definitions::aux_layer::ZkSyncSnarkWrapperCircuit; +use gpu_prover::{AsyncSetup, DeviceMemoryManager, ManagerConfigs}; + +use super::*; + +use bellman::plonk::better_better_cs::{ + cs::VerificationKey as PlonkVerificationKey, proof::Proof as PlonkProof, +}; + +use bellman::bn256::{Bn256, Fr}; + +pub(crate) type PlonkSnarkVerifierCircuit = ZkSyncSnarkWrapperCircuit; +pub(crate) type PlonkSnarkVerifierCircuitVK = + PlonkVerificationKey; +pub(crate) type PlonkSnarkVerifierCircuitProof = PlonkProof; +pub(crate) type PlonkSnarkVerifierCircuitDeviceSetup = AsyncSetup; + +type PlonkAssembly = Assembly< + Bn256, + PlonkCsWidth4WithNextStepAndCustomGatesParams, + SelectorOptimizedWidth4MainGateWithDNext, + CSConfig, + A, +>; + +pub(crate) struct UnsafePlonkProverDeviceMemoryManagerWrapper( + DeviceMemoryManager, +); +impl GenericWrapper for UnsafePlonkProverDeviceMemoryManagerWrapper { + type Inner = DeviceMemoryManager; + + fn into_inner(self) -> Self::Inner { + self.0 + } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } +} +unsafe impl Send for UnsafePlonkProverDeviceMemoryManagerWrapper {} +unsafe impl Sync for UnsafePlonkProverDeviceMemoryManagerWrapper {} +pub(crate) struct PlonkProverDeviceMemoryManagerConfig; + +impl ManagerConfigs for PlonkProverDeviceMemoryManagerConfig { + const NUM_GPUS_LOG: usize = 0; + const FULL_SLOT_SIZE_LOG: usize = 24; + const NUM_SLOTS: usize = 29; + const NUM_HOST_SLOTS: usize = 2; +} + +pub(crate) struct PlonkSnarkWrapper; + +impl ProofSystemDefinition for PlonkSnarkWrapper { + type FieldElement = Fr; + type Precomputation = PlonkSnarkVerifierCircuitDeviceSetupWrapper; + type ExternalWitnessData = Vec; + type Proof = PlonkSnarkVerifierCircuitProof; + type VK = PlonkSnarkVerifierCircuitVK; + type FinalizationHint = usize; + type Allocator = gpu_prover::cuda_bindings::CudaAllocator; + type ProvingAssembly = PlonkAssembly; + type Transcript = RollingKeccakTranscript; + fn take_witnesses( + _proving_assembly: &mut Self::ProvingAssembly, + ) -> Vec { + // let input_assignments = + // std::mem::replace(&mut proving_assembly.input_assingments, Vec::new()); + // let aux_assignments = std::mem::replace( + // &mut proving_assembly.aux_assingments, + // Vec::new_in(Self::Allocator::default()), + // ); + + todo!() + } + fn verify(proof: &Self::Proof, vk: &Self::VK) -> bool { + bellman::plonk::better_better_cs::verifier::verify::<_, _, Self::Transcript>( + vk, proof, None, + ) + .unwrap() + } +} + +impl SnarkWrapperProofSystem for PlonkSnarkWrapper { + type Circuit = PlonkSnarkVerifierCircuit; + type Context = UnsafePlonkProverDeviceMemoryManagerWrapper; + + type CRS = bellman::kate_commitment::Crs< + bellman::compact_bn256::Bn256, + CrsForMonomialForm, + Self::Allocator, + >; + fn pre_init() { + // TODO: initialize static pinned memory + } + + fn load_compact_raw_crs(src: R) -> Self::CRS { + let num_g1_points_for_crs = 1 << PlonkProverDeviceMemoryManagerConfig::FULL_SLOT_SIZE_LOG; + read_crs_from_raw_compact_form(src, num_g1_points_for_crs).unwrap() + } + + fn init_context(compact_raw_crs: AsyncHandler) -> Self::Context { + let device_ids: Vec<_> = + (0..::NUM_GPUS).collect(); + let compact_raw_crs = compact_raw_crs.wait(); + let manager = DeviceMemoryManager::init(&device_ids, &compact_raw_crs.g1_bases).unwrap(); + UnsafePlonkProverDeviceMemoryManagerWrapper(manager) + } + + fn synthesize_for_proving(circuit: Self::Circuit) -> Self::ProvingAssembly { + let mut proving_assembly = PlonkAssembly::::new(); + circuit + .synthesize(&mut proving_assembly) + .expect("must work"); + proving_assembly + } + + fn prove( + ctx: AsyncHandler, + mut proving_assembly: Self::ProvingAssembly, + precomputation: AsyncHandler, + finalization_hint: Self::FinalizationHint, + ) -> Self::Proof { + assert!(proving_assembly.is_satisfied()); + assert!(finalization_hint.is_power_of_two()); + proving_assembly.finalize_to_size_log_2(finalization_hint.trailing_zeros() as usize); + let domain_size = proving_assembly.n() + 1; + assert!(domain_size.is_power_of_two()); + assert_eq!(domain_size, finalization_hint); + + let ctx = ctx.wait(); + let mut ctx = ctx.into_inner(); + let mut precomputation = precomputation.wait().into_inner(); + let worker = bellman::worker::Worker::new(); + let start = std::time::Instant::now(); + let proof = gpu_prover::create_proof::<_, _, Self::Transcript, _>( + &proving_assembly, + &mut ctx, + &worker, + &mut precomputation, + None, + ) + .unwrap(); + println!("plonk proving takes {} s", start.elapsed().as_secs()); + ctx.free_all_slots(); + + proof + } + + fn prove_from_witnesses( + _: AsyncHandler, + _: Vec, + _: AsyncHandler, + _: Self::FinalizationHint, + ) -> Self::Proof { + unimplemented!() + } +} + +impl SnarkWrapperProofSystemExt for PlonkSnarkWrapper { + type SetupAssembly = PlonkAssembly; + + fn synthesize_for_setup(circuit: Self::Circuit) -> Self::SetupAssembly { + let mut setup_assembly = + PlonkAssembly::::new(); + circuit.synthesize(&mut setup_assembly).expect("must work"); + setup_assembly + } + + fn generate_precomputation_and_vk( + ctx: AsyncHandler, + mut setup_assembly: Self::SetupAssembly, + hardcoded_finalization_hint: Self::FinalizationHint, + ) -> (Self::Precomputation, Self::VK) { + assert!(setup_assembly.is_satisfied()); + assert!(hardcoded_finalization_hint.is_power_of_two()); + setup_assembly + .finalize_to_size_log_2(hardcoded_finalization_hint.trailing_zeros() as usize); + let domain_size = setup_assembly.n() + 1; + assert!(domain_size.is_power_of_two()); + assert_eq!(domain_size, hardcoded_finalization_hint); + + let ctx = ctx.wait(); + let mut ctx = ctx.into_inner(); + let worker = bellman::worker::Worker::new(); + let mut precomputation = + AsyncSetup::::allocate(hardcoded_finalization_hint); + precomputation + .generate_from_assembly(&worker, &setup_assembly, &mut ctx) + .unwrap(); + + let hardcoded_g2_bases = hardcoded_canonical_g2_bases(); + let mut dummy_crs = Crs::::dummy_crs(1); + dummy_crs.g2_monomial_bases = std::sync::Arc::new(hardcoded_g2_bases.to_vec()); + let vk = gpu_prover::compute_vk_from_assembly::< + _, + _, + PlonkCsWidth4WithNextStepAndCustomGatesParams, + SynthesisModeGenerateSetup, + >(&mut ctx, &setup_assembly, &dummy_crs) + .unwrap(); + + ctx.free_all_slots(); + + ( + PlonkSnarkVerifierCircuitDeviceSetupWrapper::from_inner(precomputation), + vk, + ) + } +} diff --git a/crates/proof-compression/src/serialization.rs b/crates/proof-compression/src/serialization.rs new file mode 100644 index 0000000..17c3967 --- /dev/null +++ b/crates/proof-compression/src/serialization.rs @@ -0,0 +1,128 @@ +use super::*; + +use fflonk::bellman::bn256::Bn256; +use gpu_prover::ManagerConfigs; +use shivini::{ + boojum::cs::implementations::fast_serialization::MemcopySerializable, cs::GpuSetup, + GpuTreeHasher, +}; + +pub(crate) trait GenericWrapper: Sized { + type Inner; + fn into_inner(self) -> Self::Inner; + fn from_inner(inner: Self::Inner) -> Self; +} + +use crate::PlonkSnarkVerifierCircuitDeviceSetup; + +pub(crate) struct BoojumDeviceSetupWrapper(GpuSetup); +impl GenericWrapper for BoojumDeviceSetupWrapper +where + H: GpuTreeHasher, +{ + type Inner = GpuSetup; + fn into_inner(self) -> Self::Inner { + self.0 + } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } +} + +impl shivini::boojum::cs::implementations::fast_serialization::MemcopySerializable + for BoojumDeviceSetupWrapper +{ + fn write_into_buffer( + &self, + dst: W, + ) -> Result<(), Box> { + Ok(bincode::serialize_into(dst, &self.0).unwrap()) + } + + fn read_from_buffer(src: R) -> Result> { + Ok(Self(bincode::deserialize_from(src).unwrap())) + } +} + +pub(crate) struct PlonkSnarkVerifierCircuitDeviceSetupWrapper(PlonkSnarkVerifierCircuitDeviceSetup); + +impl MemcopySerializable for PlonkSnarkVerifierCircuitDeviceSetupWrapper { + fn write_into_buffer( + &self, + dst: W, + ) -> Result<(), Box> { + self.0.write(dst).unwrap(); + Ok(()) + } + + fn read_from_buffer(src: R) -> Result> { + let mut precomputation = PlonkSnarkVerifierCircuitDeviceSetup::allocate( + 1 << PlonkProverDeviceMemoryManagerConfig::FULL_SLOT_SIZE_LOG, + ); + precomputation.read(src).unwrap(); + Ok(Self(precomputation)) + } +} + +impl GenericWrapper for PlonkSnarkVerifierCircuitDeviceSetupWrapper { + type Inner = PlonkSnarkVerifierCircuitDeviceSetup; + fn into_inner(self) -> Self::Inner { + self.0 + } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } +} + +pub(crate) struct FflonkSnarkVerifierCircuitDeviceSetupWrapper( + pub fflonk::FflonkDeviceSetup, +); + +impl MemcopySerializable for FflonkSnarkVerifierCircuitDeviceSetupWrapper +where + A: HostAllocator, +{ + fn write_into_buffer( + &self, + dst: W, + ) -> Result<(), Box> { + Ok(self.0.write(dst).unwrap()) + } + + fn read_from_buffer(src: R) -> Result> { + let precomputation = fflonk::FflonkDeviceSetup::::read(src).unwrap(); + + Ok(Self(precomputation)) + } +} + +impl GenericWrapper for FflonkSnarkVerifierCircuitDeviceSetupWrapper +where + A: HostAllocator, +{ + type Inner = fflonk::FflonkDeviceSetup; + fn into_inner(self) -> Self::Inner { + self.0 + } + + fn from_inner(inner: Self::Inner) -> Self { + Self(inner) + } +} + +#[derive(serde::Serialize, serde::Deserialize)] +pub(crate) struct MarkerPrecomputation; +impl MemcopySerializable for MarkerPrecomputation { + fn write_into_buffer( + &self, + _dst: W, + ) -> Result<(), Box> { + todo!() + } + + fn read_from_buffer(_src: R) -> Result> { + todo!() + } +} diff --git a/crates/proof-compression/src/step/compression.rs b/crates/proof-compression/src/step/compression.rs new file mode 100644 index 0000000..c5de5bc --- /dev/null +++ b/crates/proof-compression/src/step/compression.rs @@ -0,0 +1,255 @@ +use super::*; + +use circuit_definitions::circuit_definitions::{ + aux_layer::{ + compression::{CompressionLayerCircuit, ProofCompressionFunction}, + compression_modes::{ + CompressionMode1, CompressionMode1ForWrapper, CompressionMode2, CompressionMode3, + CompressionMode4, CompressionMode5ForWrapper, + }, + ZkSyncCompressionForWrapperCircuit, ZkSyncCompressionLayerCircuit, + }, + recursion_layer::RecursiveProofsTreeHasher, +}; +use franklin_crypto::boojum::cs::{ + implementations::{ + fast_serialization::MemcopySerializable, proof::Proof, verifier::VerificationKey, + }, + oracle::TreeHasher, +}; + +pub(crate) trait CompressionStep: CompressionProofSystem { + type PreviousStepTreeHasher: TreeHasher< + GoldilocksField, + Output: serde::Serialize + serde::de::DeserializeOwned, + >; + + const MODE: u8; + const IS_WRAPPER: bool; + fn load_finalization_hint( + blob_storage: &BS, + ) -> ::FinalizationHint + where + BS: BlobStorage, + { + let reader = if Self::IS_WRAPPER { + blob_storage.read_compression_wrapper_finalization_hint(Self::MODE) + } else { + blob_storage.read_compression_layer_finalization_hint(Self::MODE) + }; + serde_json::from_reader(reader).unwrap() + } + + fn load_previous_vk( + blob_storage: &BS, + ) -> VerificationKey + where + BS: BlobStorage, + { + assert!(Self::MODE >= 1); + + let reader = if Self::MODE == 1 { + blob_storage.read_scheduler_vk() + } else { + blob_storage.read_compression_layer_vk(Self::MODE - 1) + }; + + serde_json::from_reader(reader).unwrap() + } + + fn load_this_vk(blob_storage: &BS) -> ::VK + where + BS: BlobStorage, + { + let reader = if Self::IS_WRAPPER { + blob_storage.read_compression_wrapper_vk(Self::MODE) + } else { + blob_storage.read_compression_layer_vk(Self::MODE) + }; + + serde_json::from_reader(reader).unwrap() + } + + fn get_precomputation( + blob_storage: &BS, + ) -> AsyncHandler<::Precomputation> + where + BS: BlobStorage, + { + let reader = if Self::IS_WRAPPER { + blob_storage.read_compression_wrapper_precomputation(Self::MODE) + } else { + blob_storage.read_compression_layer_precomputation(Self::MODE) + }; + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let precomputation = + <::Precomputation as MemcopySerializable>::read_from_buffer( + reader, + ) + .unwrap(); + + sender.send(precomputation).unwrap(); + receiver + }; + + AsyncHandler::spawn(f) + } + + fn prove_compression_step( + input_proof: Proof, + blob_storage: &BS, + context_handler: &CI, + ) -> ::Proof + where + BS: BlobStorage, + CI: ContextManagerInterface, + { + let input_vk = Self::load_previous_vk(blob_storage); + let vk = Self::load_this_vk(blob_storage); + let precomputation = Self::get_precomputation(blob_storage); + let finalization_hint = Self::load_finalization_hint(blob_storage); + let ctx_config = Self::get_context_config_from_hint(&finalization_hint); + let ctx = context_handler.init_compression_context::(ctx_config); + let circuit = Self::build_circuit(input_vk, Some(input_proof)); + let proving_assembly = ::synthesize_for_proving( + circuit, + finalization_hint.clone(), + ); + let aux_config = + ::aux_config_from_assembly(&proving_assembly); + let proof = ::prove( + ctx, + proving_assembly, + aux_config, + precomputation, + finalization_hint, + &vk, + ); + assert!(::verify(&proof, &vk)); + + proof + } + + // CompressionLayerCircuit is unified type for both compression circuits + fn build_circuit( + input_vk: VerificationKey, + input_proof: Option>, + ) -> CompressionLayerCircuit; +} + +pub(crate) trait CompressionStepExt: CompressionProofSystemExt + CompressionStep { + fn precomputae_and_store_compression_circuits(blob_storage: &BS, context_manager: &CM) + where + BS: BlobStorageExt, + CM: ContextManagerInterface, + { + let input_vk = Self::load_previous_vk(blob_storage); + let circuit = Self::build_circuit(input_vk, None); + // Workaround: trace length is not known at this point, so thats totally fine + // to use a hardcoded trace length + let ctx_config = Self::get_context_config(); + let ctx = context_manager.init_compression_context::(ctx_config); + let (finalization_hint, setup_assembly) = + ::synthesize_for_setup(circuit); + let (precomputation, vk) = + ::generate_precomputation_and_vk( + ctx, + setup_assembly, + &finalization_hint, + ); + let (precompuatation_writer, vk_writer, hint_writer) = if Self::IS_WRAPPER { + ( + blob_storage.write_compression_wrapper_precomputation(Self::MODE), + blob_storage.write_compression_wrapper_vk(Self::MODE), + blob_storage.write_compression_wrapper_finalization_hint(Self::MODE), + ) + } else { + ( + blob_storage.write_compression_layer_precomputation(Self::MODE), + blob_storage.write_compression_layer_vk(Self::MODE), + blob_storage.write_compression_layer_finalization_hint(Self::MODE), + ) + }; + precomputation + .write_into_buffer(precompuatation_writer) + .unwrap(); + serde_json::to_writer_pretty(vk_writer, &vk).unwrap(); + serde_json::to_writer_pretty(hint_writer, &finalization_hint).unwrap(); + println!( + "Precomputation and vk of compression circuit {} saved into blob storage", + Self::MODE + ); + } +} + +macro_rules! impl_compression_circuit { + ($type:ty,$mode:expr, $is_wrapper:expr, $enum:ident::$variant:ident, $hasher:ty) => { + impl CompressionStep for $type { + const MODE: u8 = $mode; + const IS_WRAPPER: bool = $is_wrapper; + type PreviousStepTreeHasher = $hasher; + fn build_circuit( + input_vk: VerificationKey, + input_proof: Option< + Proof, + >, + ) -> CompressionLayerCircuit { + let circuit = $enum::from_witness_and_vk(input_proof, input_vk, Self::MODE); + match circuit { + $enum::$variant(compression_layer_circuit) => compression_layer_circuit, + _ => unreachable!(), + } + } + } + + impl CompressionStepExt for $type {} + }; +} + +impl_compression_circuit!( + CompressionMode1, + 1, + false, + ZkSyncCompressionLayerCircuit::CompressionMode1Circuit, + RecursiveProofsTreeHasher +); +impl_compression_circuit!( + CompressionMode2, + 2, + false, + ZkSyncCompressionLayerCircuit::CompressionMode2Circuit, + ::ThisLayerHasher +); + +impl_compression_circuit!( + CompressionMode3, + 3, + false, + ZkSyncCompressionLayerCircuit::CompressionMode3Circuit, + ::ThisLayerHasher +); + +impl_compression_circuit!( + CompressionMode4, + 4, + false, + ZkSyncCompressionLayerCircuit::CompressionMode4Circuit, + ::ThisLayerHasher +); + +impl_compression_circuit!( + CompressionMode1ForWrapper, + 1, + true, + ZkSyncCompressionForWrapperCircuit::CompressionMode1Circuit, + RecursiveProofsTreeHasher +); + +impl_compression_circuit!( + CompressionMode5ForWrapper, + 5, + true, + ZkSyncCompressionForWrapperCircuit::CompressionMode5Circuit, + ::ThisLayerHasher +); diff --git a/crates/proof-compression/src/step/mod.rs b/crates/proof-compression/src/step/mod.rs new file mode 100644 index 0000000..10d9ea8 --- /dev/null +++ b/crates/proof-compression/src/step/mod.rs @@ -0,0 +1,7 @@ +use super::*; + +mod compression; +pub(crate) use compression::*; + +mod snark_wrapper; +pub(crate) use snark_wrapper::*; diff --git a/crates/proof-compression/src/step/snark_wrapper.rs b/crates/proof-compression/src/step/snark_wrapper.rs new file mode 100644 index 0000000..d07a99d --- /dev/null +++ b/crates/proof-compression/src/step/snark_wrapper.rs @@ -0,0 +1,248 @@ +use circuit_definitions::circuit_definitions::aux_layer::{ + compression::ProofCompressionFunction, + compression_modes::{CompressionMode1ForWrapper, CompressionMode5ForWrapper}, + wrapper::ZkSyncCompressionWrapper, +}; +use franklin_crypto::boojum::cs::{ + implementations::{ + fast_serialization::MemcopySerializable, proof::Proof, verifier::VerificationKey, + }, + oracle::TreeHasher, +}; + +use super::*; + +pub(crate) trait SnarkWrapperStep: SnarkWrapperProofSystem { + const IS_PLONK: bool; + const IS_FFLONK: bool; + const PREVIOUS_COMPRESSION_MODE: u8; + type PreviousStepTreeHasher: TreeHasher< + GoldilocksField, + Output: serde::Serialize + serde::de::DeserializeOwned, + >; + fn load_finalization_hint( + _blob_storage: &BS, + ) -> ::FinalizationHint + where + BS: BlobStorage, + { + assert!(Self::IS_FFLONK ^ Self::IS_PLONK); + let hint = if Self::IS_PLONK { + (1 << ::FULL_SLOT_SIZE_LOG).to_string() + } else { + (1 << fflonk::fflonk::L1_VERIFIER_DOMAIN_SIZE_LOG).to_string() + }; + serde_json::from_str(&hint).unwrap() + } + + fn load_previous_vk( + blob_storage: &BS, + ) -> VerificationKey + where + BS: BlobStorage, + { + assert!(Self::IS_FFLONK ^ Self::IS_PLONK); + let previous_compression_mode = Self::PREVIOUS_COMPRESSION_MODE; + let reader = blob_storage.read_compression_wrapper_vk(previous_compression_mode); + serde_json::from_reader(reader).unwrap() + } + + fn load_this_vk(blob_storage: &BS) -> ::VK + where + BS: BlobStorage, + { + assert!(Self::IS_FFLONK ^ Self::IS_PLONK); + let reader = if Self::IS_FFLONK { + assert_eq!(Self::IS_PLONK, false); + blob_storage.read_fflonk_vk() + } else { + assert_eq!(Self::IS_PLONK, true); + blob_storage.read_plonk_vk() + }; + + serde_json::from_reader(reader).unwrap() + } + + fn load_compact_raw_crs(blob_storage: &BS) -> AsyncHandler + where + BS: BlobStorage, + Self::Allocator: Send + Sync + 'static, + { + assert!(Self::IS_FFLONK ^ Self::IS_PLONK); + let reader = blob_storage.read_compact_raw_crs(); + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let start = std::time::Instant::now(); + let compact_raw_crs = ::load_compact_raw_crs(reader); + println!( + "Compact raw CRS loading takes {}s", + start.elapsed().as_secs() + ); + sender.send(compact_raw_crs).unwrap(); + receiver + }; + + AsyncHandler::spawn(f) + } + + fn get_precomputation( + blob_storage: &BS, + ) -> AsyncHandler<::Precomputation> + where + BS: BlobStorage, + { + let reader = if Self::IS_FFLONK { + blob_storage.read_fflonk_precomputation() + } else { + blob_storage.read_plonk_precomputation() + }; + let f = move || { + let (sender, receiver) = std::sync::mpsc::channel(); + let start = std::time::Instant::now(); + let precomputation = + <::Precomputation as MemcopySerializable>::read_from_buffer( + reader, + ) + .unwrap(); + println!( + "Snark wrapper device setup loading takes {}s", + start.elapsed().as_secs() + ); + sender.send(precomputation).unwrap(); + receiver + }; + + AsyncHandler::spawn(f) + } + + fn run_pre_initialization_tasks() { + Self::pre_init(); + } + + fn prove_snark_wrapper_step( + ctx_config: AsyncHandler, + precomputation: AsyncHandler, + input_proof: Proof, + blob_storage: &BS, + context_handler: &CI, + ) -> ::Proof + where + BS: BlobStorage, + CI: ContextManagerInterface, + { + assert!(Self::IS_FFLONK ^ Self::IS_PLONK); + let input_vk = Self::load_previous_vk(blob_storage); + + let ctx = context_handler.init_snark_context::(ctx_config); + let finalization_hint = Self::load_finalization_hint(blob_storage); + let circuit = Self::build_circuit(input_vk, Some(input_proof)); + let proving_assembly = ::synthesize_for_proving(circuit); + let vk = Self::load_this_vk(blob_storage); + + let proof = ::prove( + ctx, + proving_assembly, + precomputation, + finalization_hint, + ); + + assert!(::verify(&proof, &vk)); + + proof + } + + fn build_circuit( + input_vk: VerificationKey, + input_proof: Option>, + ) -> Self::Circuit; +} + +pub(crate) trait SnarkWrapperStepExt: SnarkWrapperProofSystemExt + SnarkWrapperStep { + fn precompute_and_store_snark_wrapper_circuit( + compact_raw_crs: AsyncHandler, + blob_storage: &BS, + context_manager: &CM, + ) where + BS: BlobStorageExt, + CM: ContextManagerInterface, + ::VK: 'static, + { + let input_vk = Self::load_previous_vk(blob_storage); + let finalization_hint = Self::load_finalization_hint(blob_storage); + let circuit = Self::build_circuit(input_vk, None); + let ctx = context_manager.init_snark_context::(compact_raw_crs); + let setup_assembly = ::synthesize_for_setup(circuit); + + let (precomputation, vk) = + ::generate_precomputation_and_vk( + ctx, + setup_assembly, + finalization_hint, + ); + let (precompuatation_writer, vk_writer) = if Self::IS_FFLONK { + ( + blob_storage.write_fflonk_precomputation(), + blob_storage.write_fflonk_vk(), + ) + } else { + ( + blob_storage.write_plonk_precomputation(), + blob_storage.write_plonk_vk(), + ) + }; + precomputation + .write_into_buffer(precompuatation_writer) + .unwrap(); + serde_json::to_writer_pretty(vk_writer, &vk).unwrap(); + println!("Pecomputation and vk of snark wrapper circuit saved into blob storage"); + } +} + +impl SnarkWrapperStep for FflonkSnarkWrapper { + const IS_PLONK: bool = false; + const IS_FFLONK: bool = true; + const PREVIOUS_COMPRESSION_MODE: u8 = 5; + type PreviousStepTreeHasher = + ::ThisLayerHasher; + fn build_circuit( + input_vk: VerificationKey, + input_proof: Option>, + ) -> Self::Circuit { + let fixed_parameters = input_vk.fixed_parameters.clone(); + FflonkSnarkVerifierCircuit { + witness: input_proof, + vk: input_vk, + fixed_parameters, + transcript_params: (), + wrapper_function: ZkSyncCompressionWrapper::from_numeric_circuit_type( + Self::PREVIOUS_COMPRESSION_MODE, + ), + } + } +} +impl SnarkWrapperStepExt for FflonkSnarkWrapper {} + +impl SnarkWrapperStep for PlonkSnarkWrapper { + const IS_PLONK: bool = true; + const IS_FFLONK: bool = false; + + const PREVIOUS_COMPRESSION_MODE: u8 = 1; + type PreviousStepTreeHasher = + ::ThisLayerHasher; + fn build_circuit( + input_vk: VerificationKey, + input_proof: Option>, + ) -> Self::Circuit { + let fixed_parameters = input_vk.fixed_parameters.clone(); + PlonkSnarkVerifierCircuit { + witness: input_proof, + vk: input_vk, + fixed_parameters, + transcript_params: (), + wrapper_function: ZkSyncCompressionWrapper::from_numeric_circuit_type( + Self::PREVIOUS_COMPRESSION_MODE, + ), + } + } +} +impl SnarkWrapperStepExt for PlonkSnarkWrapper {} diff --git a/crates/proof-compression/src/test.rs b/crates/proof-compression/src/test.rs new file mode 100644 index 0000000..bc9f1cf --- /dev/null +++ b/crates/proof-compression/src/test.rs @@ -0,0 +1,36 @@ +use super::*; +#[test] +fn test_proof_chain_with_fflonk() { + let simple_blob_storage = FileSystemBlobStorage; + let mut simple_proof_storage = SimpleProofStorage::new(); + run_proof_chain_with_fflonk(&simple_blob_storage, &mut simple_proof_storage); + println!("Final fflonk snark wrapper proof has successfully generated"); +} + +#[test] +fn test_proof_chain_with_plonk() { + let simple_blob_storage = FileSystemBlobStorage; + let mut simple_proof_storage = SimpleProofStorage::new(); + run_proof_chain_with_plonk(&simple_blob_storage, &mut simple_proof_storage); + + println!("Final plonk snark wrapper proof has successfully generated"); +} + +#[test] +fn test_precompute_compression_chain_artifacts_with_fflonk() { + let simple_blob_storage = FileSystemBlobStorage; + precompute_proof_chain_with_fflonk(&simple_blob_storage); +} + +#[test] +fn test_precompute_compression_chain_artifacts_with_plonk() { + let simple_blob_storage = FileSystemBlobStorage; + precompute_proof_chain_with_plonk(&simple_blob_storage); +} + +#[test] +fn test_create_compact_raw_crs() { + let blob_storage = FileSystemBlobStorage; + let writer = blob_storage.write_compact_raw_crs(); + create_compact_raw_crs(writer); +}