Merge remote-tracking branch 'origin/merge-train/barretenberg' into ad/bbapi/vk-simplified-buffer

Author: ludamad

.test_patterns.yml

@@ -258,6 +258,11 @@ tests:
     owners:
       - *mitch
 
+  - regex: "src/e2e_multi_eoa.test.ts"
+    error_regex: "TypeError: Cannot convert undefined to a BigInt"
+    owners:
+      - *alex
+
   # Nightly GKE tests
   - regex: "spartan/bootstrap.sh"
     owners:

barretenberg/cpp/CLAUDE.md

@@ -0,0 +1,83 @@
+succint aztec-packages cheat sheet.
+
+THE PROJECT ROOT IS AT TWO LEVELS ABOVE THIS FOLDER. Typically, the repository is at ~/aztec-packages. all advice is from the root.
+
+Run ./bootstrap.sh at the top-level to be sure the repo fully builds.
+Bootstrap scripts can be called with relative paths e.g. ../barretenberg/bootstrap.sh
+You can use DISABLE_AVM=1 to bootstrap things generally.
+
+# Working on modules:
+
+## barretenberg/
+The core proving system library. Focus development is in barretenberg/cpp.
+
+### cpp/ => cpp code for prover library
+Bootstrap modes:
+- `./bootstrap.sh` => full build, needed for other components
+- `./bootstrap.sh build` => standard build
+- `DISABLE_AVM=1 ./bootstrap.sh build_native` => quick build without AVM. Good for verifying compilation works. Needed to build ts/
+Development commands:
+- cmake --preset build-no-avm
+  cd build-no-avm
+  ninja <test>
+    NOTE: DO NOT add the -j flag, default is optimal.
+    where test is based on what you're working on:
+    - `./bin/ultra_honk_tests` - Ultra Honk circuit tests
+    - `./bin/client_ivc_tests` - Client IVC tests
+    - `./bin/api_tests` - API/CLI tests
+    - `./bin/stdlib_*_tests` - Standard library tests
+    - `./bin/crypto_*_tests` - Cryptographic primitive tests
+
+### Barretenberg module components:
+- **commitment_schemes/** - Polynomial commitment schemes (KZG, IPA)
+- **crypto/** - Cryptographic primitives (hashes, merkle trees, fields)
+- **ecc/** - Elliptic curve operations
+- **flavor/** - Circuit proving system flavors (Ultra, Mega)
+- **honk/** - The Honk proving system implementation
+- **stdlib/** - Circuit-friendly implementations of primitives
+- **ultra_honk/** - Ultra Honk prover/verifier
+- **client_ivc/** - Client-side IVC (Incremental Verifiable Computation)
+- **vm2/** - AVM implementation (not enabled, but might need to be fixed for compilation issues in root ./bootstrap.sh)
+- **bbapi/** - BB API for external interaction. If changing here, we will also want to update the ts/ folder because bb.js consumes this. (first build ninja bb in build/)
+- **dsl/** - ACIR definition in C++. This is dictated by the serialization in noir/, so refactor should generally not change the structure without confirming that the user is changing noir.
+
+### ts/ => typescript code for bb.js
+Bootstrap modes:
+- `./bootstrap.sh` => generate TypeScript bindings and build. See package.json for more fine-grained commands.
+Other commands:
+- `yarn build:esm` => the quickest way to rebuild, if only changes inside ts/ folder, and only testing yarn-project.
+
+## noir/
+### noir-repo/ => clone of noir programming language git repo
+Bootstrap modes:
+- `./bootstrap.sh` => standard build
+
+## avm-transpiler:
+Transpiles Noir to AVM bytecode
+Bootstrap modes:
+- `./bootstrap.sh` => standard build
+
+## Integration testing:
+The focus is on barretenberg/cpp development. Other components need to work with barretenberg changes:
+
+### yarn-project/end-to-end - E2E tests that verify the full stack
+Run end-to-end tests from the root directory:
+```bash
+# Run specific e2e tests
+yarn-project/end-to-end/scripts/run_test.sh simple e2e_block_building
+# To run this you CANNOT USE DISABLE_AVM=1. Only run this if the user asks (e.g. 'run the prover full test') You first need to confirm with the user that they want to build without AVM.
+yarn-project/end-to-end/scripts/run_test.sh simple e2e_prover/full
+
+### yarn-project IVC integration tests
+Run IVC (Incremental Verifiable Computation) integration tests from the root:
+```bash
+# Run specific IVC tests
+yarn-project/scripts/run_test.sh ivc-integration/src/native_client_ivc_integration.test.ts
+yarn-project/scripts/run_test.sh ivc-integration/src/wasm_client_ivc_integration.test.ts
+yarn-project/scripts/run_test.sh ivc-integration/src/browser_client_ivc_integration.test.ts
+
+# Run rollup IVC tests (with verbose logging)
+BB_VERBOSE=1 yarn-project/scripts/run_test.sh ivc-integration/src/rollup_ivc_integration.test.ts
+```
+
+When making barretenberg changes, ensure these tests still pass.

barretenberg/cpp/pil/vm2/alu.pil

@@ -39,7 +39,7 @@ sel = 0;
 pol commit cf;
 
 // Generic helper column
-// Current use: EQ (inverse of a-b) & DIV (remainder)
+// Current use: EQ (inverse of a-b), DIV (remainder), and SHL (2**ib)
 pol commit helper1;
 
 // maximum bits the number can hold (i.e. 8 for a u8):
@@ -95,7 +95,7 @@ execution.sel_execute_alu {
 
 // IS_FF CHECKING
 
-pol CHECK_TAG_FF = sel_op_div + sel_op_fdiv + sel_op_lt + sel_op_lte + sel_op_not;
+pol CHECK_TAG_FF = sel_op_div + sel_op_fdiv + sel_op_lt + sel_op_lte + sel_op_not + sel_shift_ops;
 // We prove that sel_is_ff == 1 <==> ia_tag == MEM_TAG_FF
 pol TAG_FF_DIFF = ia_tag - constants.MEM_TAG_FF;
 pol commit tag_ff_diff_inv;
@@ -113,7 +113,7 @@ CHECK_TAG_U128 * (TAG_U128_DIFF * (sel_is_u128 * (1 - tag_u128_diff_inv) + tag_u
 
 // TAG CHECKING
 
-pol EXPECTED_C_TAG = (sel_op_add + sel_op_sub + sel_op_mul + sel_op_div + sel_op_truncate + sel_op_shr + sel_op_shl) * ia_tag + (sel_op_eq + sel_op_lt + sel_op_lte) * constants.MEM_TAG_U1 + sel_op_fdiv * constants.MEM_TAG_FF;
+pol EXPECTED_C_TAG = (sel_op_add + sel_op_sub + sel_op_mul + sel_op_div + sel_op_truncate + sel_shift_ops) * ia_tag + (sel_op_eq + sel_op_lt + sel_op_lte) * constants.MEM_TAG_U1 + sel_op_fdiv * constants.MEM_TAG_FF;
 
 // The tag of c is generated by the opcode and is never wrong.
 // Gating with (1 - sel_tag_err) is necessary because when an error occurs, we have to set the tag to 0,
@@ -125,22 +125,22 @@ pol commit sel_tag_err;
 sel_tag_err * (1 - sel_tag_err) = 0;
 
 // Tag errors currently have cases:
-// 1. Input tagged as a field for NOT or DIV operations or non-field for FDIV operation
+// 1. Input tagged as a field for NOT, DIV, or shift operations or non-field for FDIV operation
 // 2. Mismatched tags for inputs a and b for all opcodes apart from TRUNC
 // 1 is handled by checking FF_TAG_ERR in TAG_ERR_CHECK and 2 is handled in AB_TAGS_CHECK.
-pol FF_TAG_ERR = (sel_op_div + sel_op_not) * sel_is_ff + sel_op_fdiv * IS_NOT_FF;
+pol FF_TAG_ERR = (sel_op_div + sel_op_not + sel_shift_ops) * sel_is_ff + sel_op_fdiv * IS_NOT_FF;
 pol commit sel_ab_tag_mismatch;
 sel_ab_tag_mismatch * (1 - sel_ab_tag_mismatch) = 0;
 
-// TODO(MW): It's technically possible to have BOTH cases be true if we perform a DIV with FF ib and integer ia,
+// TODO(MW): It's technically possible to have BOTH cases be true if we perform a DIV or shift with FF ib and integer ia,
 // so for now I take sel_ab_tag_mismatch * FF_TAG_ERR.
 #[TAG_ERR_CHECK]
 sel_tag_err = sel_ab_tag_mismatch + FF_TAG_ERR - sel_ab_tag_mismatch * FF_TAG_ERR;
 
 // For NOT opcode, an error occurs if the tag of a is FF. In this case, tracegen will set
 // b's tag as 0 which, while it would currently pass the checks below, is not a tag inequality we
 // want to throw with sel_ab_tag_mismatch:
-pol CHECK_AB_TAGS = 1 - sel_op_not * sel_is_ff - sel_op_truncate - sel_op_shr - sel_op_shl; // note: shifts are temporary, they should be subject to AB checks
+pol CHECK_AB_TAGS = 1 - sel_op_not * sel_is_ff - sel_op_truncate;
 pol AB_TAGS_EQ = 1 - sel_ab_tag_mismatch;
 pol commit ab_tags_diff_inv;
 // Prove that sel_ab_tag_mismatch = 1 <==> we have a disallowed inequality between the tags:
@@ -150,6 +150,52 @@ CHECK_AB_TAGS * ( (ia_tag - ib_tag) * ( AB_TAGS_EQ * (1 - ab_tags_diff_inv) + ab
 #[TAG_MAX_BITS_VALUE]
 sel { ia_tag, max_bits, max_value } in precomputed.sel_tag_parameters { precomputed.clk, precomputed.tag_max_bits, precomputed.tag_max_value };
 
+// BIT DECOMPOSITION
+
+// We use the below to prove correct decomposition of limbs. Currently used by MUL, DIV, SHL, and SHR.
+pol commit sel_decompose_a;
+// #[OP_ID_CHECK] ensures selectors are mutually exclusive:
+sel_decompose_a = sel_mul_div_u128 + sel_shift_ops * IS_NOT_FF;
+// Currently, sel_decompose_b would just equal sel_mul_div_u128, so no need for another column.
+pol commit a_lo, a_hi, b_lo, b_hi;
+pol TWO_POW_64 = 2 ** 64;
+
+// Reusing columns for decomposition (#[OP_ID_CHECK] ensures selectors are mutually exclusive):
+pol DECOMPOSED_A = ((sel_mul_u128 + sel_shift_ops_no_overflow) * ia) + (sel_shift_ops - sel_shift_ops_no_overflow) * (ib - max_bits) + (sel_is_u128 * sel_op_div * (1 - sel_tag_err) * ic);
+pol DECOMPOSED_B = ib;
+// For MUL and DIV, we decompose into 64 bit limbs. For shifts, we have one limb of b bits and one limb of max_bits - b bits.
+pol LIMB_SIZE = sel_mul_div_u128 * TWO_POW_64 + sel_shift_ops * two_pow_shift_lo_bits;
+
+#[A_DECOMPOSITION]
+sel_decompose_a * (DECOMPOSED_A - (a_lo + LIMB_SIZE * a_hi)) = 0;
+#[B_DECOMPOSITION]
+sel_mul_div_u128 * (DECOMPOSED_B - (b_lo + LIMB_SIZE * b_hi)) = 0;
+
+// Note: the only current use for decomposition of b has 64 bit limbs, so no need for b_lo/hi_bits.
+pol commit a_lo_bits, a_hi_bits;
+// TODO: Once lookups support expression in tuple, we can inline constant_64 into the lookup.
+// Note: only currently used for MUL/DIV u128, so gated by sel_mul_div_u128:
+pol commit constant_64;
+sel_mul_div_u128 * (64 - constant_64) = 0;
+
+#[A_LO_BITS]
+a_lo_bits - sel_mul_div_u128 * constant_64 - sel_shift_ops * shift_lo_bits = 0;
+
+#[A_HI_BITS]
+a_hi_bits - sel_mul_div_u128 * constant_64 - sel_shift_ops * SHIFT_HI_BITS = 0;
+
+#[RANGE_CHECK_DECOMPOSITION_A_LO]
+sel_decompose_a { a_lo, a_lo_bits } in range_check.sel { range_check.value, range_check.rng_chk_bits };
+
+#[RANGE_CHECK_DECOMPOSITION_A_HI]
+sel_decompose_a { a_hi, a_hi_bits } in range_check.sel { range_check.value, range_check.rng_chk_bits };
+
+#[RANGE_CHECK_DECOMPOSITION_B_LO]
+sel_mul_div_u128 { b_lo, constant_64 } in range_check.sel { range_check.value, range_check.rng_chk_bits };
+
+#[RANGE_CHECK_DECOMPOSITION_B_HI]
+sel_mul_div_u128 { b_hi, constant_64 } in range_check.sel { range_check.value, range_check.rng_chk_bits };
+
 
 // ADD
 
@@ -194,21 +240,6 @@ sel_mul_u128 = sel_is_u128 * sel_op_mul;
 // a * b_l + a_l * b_h * 2^64 = (cf * 2^64 + c_hi) * 2^128 + c
 // => no need for a_h in final relation
 
-pol commit a_lo;
-pol commit a_hi;
-pol commit b_lo;
-pol commit b_hi;
-pol TWO_POW_64 = 2 ** 64;
-
-// Reusing columns for decomposition (#[OP_ID_CHECK] ensures selectors are mutually exclusive):
-pol DECOMPOSED_A = (sel_mul_u128 * ia) + (sel_is_u128 * sel_op_div * (1 - sel_tag_err) * ic);
-pol DECOMPOSED_B = ib;
-
-#[A_DECOMPOSITION]
-sel_mul_div_u128 * (DECOMPOSED_A - (a_lo + TWO_POW_64 * a_hi)) = 0;
-#[B_DECOMPOSITION]
-sel_mul_div_u128 * (DECOMPOSED_B - (b_lo + TWO_POW_64 * b_hi)) = 0;
-
 #[ALU_MUL_U128]
 sel_mul_u128 * (1 - sel_tag_err)
     * (
@@ -217,23 +248,6 @@ sel_mul_u128 * (1 - sel_tag_err)
         - (max_value + 1) * (cf * TWO_POW_64 + c_hi)    // c_hi * 2^128 + (cf ? 2^192 : 0)
     ) = 0;
 
-// TODO: Once lookups support expression in tuple, we can inline constant_64 into the lookup.
-// Note: only used for MUL/DIV u128, so gated by sel_mul_div_u128
-pol commit constant_64;
-sel_mul_div_u128 * (64 - constant_64) = 0;
-
-#[RANGE_CHECK_MUL_U128_A_LO]
-sel_mul_div_u128 { a_lo, constant_64 } in range_check.sel { range_check.value, range_check.rng_chk_bits };
-
-#[RANGE_CHECK_MUL_U128_A_HI]
-sel_mul_div_u128 { a_hi, constant_64 } in range_check.sel { range_check.value, range_check.rng_chk_bits };
-
-#[RANGE_CHECK_MUL_U128_B_LO]
-sel_mul_div_u128 { b_lo, constant_64 } in range_check.sel { range_check.value, range_check.rng_chk_bits };
-
-#[RANGE_CHECK_MUL_U128_B_HI]
-sel_mul_div_u128 { b_hi, constant_64 } in range_check.sel { range_check.value, range_check.rng_chk_bits };
-
 // No need to range_check c_hi for cases other than u128 because we know a and b's size from the tags and have looked
 // up max_value. i.e. we cannot provide a malicious c, c_hi such that a + b - c_hi * 2^n = c passes for n < 128.
 // No need to range_check c_lo = ic because the memory write will ensure ic <= max_value.
@@ -392,6 +406,87 @@ sel_op_not * (1 - sel_op_not) = 0;
 #[NOT_OP_MAIN]
 sel_op_not * (1 - sel_tag_err) * (ia + ib - max_value) = 0;
 
+// SHIFTS - Taken from vm1:
+// Given (1) an input a, within the range [0, 2**128-1],
+//       (2) a value s, the amount of bits to shift a by (stored in ib),
+//       (3) and a memory tag, mem_tag that supports a maximum of t bits (stored in max_bits).
+// Split input a into Big Endian hi and lo limbs, (we re-use the a_hi and a_lo columns we used for the MUL/DIV u128 operators)
+// a_hi and a_lo, and the number of bits represented by the memory tag, t.
+// If we are shifting by more than the bit length represented by the memory tag, the result is trivially zero.
+
+// SHL
+
+// === Steps when performing SHL
+// (1) Prove the correct decomposition: a_hi * 2**(t-s) + a_lo = a ---> see #[A_DECOMPOSITION]
+// (2) Range check a_hi < 2**s && a_lo < 2**(t-s)                  ---> see #[RANGE_CHECK_DECOMPOSITION_A_LO/HI]
+// (3) Return a_lo * 2**s                                          ---> see #[ALU_SHL]
+//
+//  <-- s bits -->   | <-- (t-s) bits -->
+// ------------------|-------------------
+// |      a_hi       |      a_lo        | --> a
+// --------------------------------------
+//
+// Use of helper1 for SHL:
+//  We have: s (=ib), t (=max_bits), 2**(t-s) (=two_pow_shift_lo_bits), and 2**t (=max_value + 1)
+//  We want: 2**s (=2**ib), ideally without another precomputed.power_of_2 lookup
+//  Injecting 2**s (=helper1), we can check that 2**t == 2**(t-s) * 2**s:
+#[SHL_TWO_POW_SHIFT]
+sel_op_shl * sel_shift_ops_no_overflow * (1 - sel_tag_err) * (max_value + 1 - two_pow_shift_lo_bits * helper1) = 0;
+
+#[ALU_SHL]
+sel_op_shl * (1 - sel_tag_err) * (ic - sel_shift_ops_no_overflow * a_lo * helper1 ) = 0;
+
+// SHR
+
+// === Steps when performing SHR
+// (1) Prove the correct decomposition: a_hi * 2**s + a_lo = a ---> see #[A_DECOMPOSITION]
+// (2) Range check a_hi < 2**(t-s) && a_lo < 2**s              ---> see #[RANGE_CHECK_DECOMPOSITION_A_LO/HI]
+// (3) Return a_hi                                             ---> see #[ALU_SHR]
+//
+//  <--(t-s) bits --> |   <-- s bits -->
+// -------------------|-------------------
+// |      a_hi        |       a_lo       | --> a
+// ---------------------------------------
+
+#[ALU_SHR]
+sel_op_shr * (1 - sel_tag_err) * (ic - sel_shift_ops_no_overflow * a_hi) = 0;
+
+// SHL & SHR - Shared relations:
+
+pol commit sel_shift_ops;
+// sel_op_shl || sel_op_shr:
+sel_shift_ops = sel_op_shl + sel_op_shr;
+
+pol commit sel_shift_ops_no_overflow;
+// sel_shift_ops_no_overflow = 1 ==> sel_shift_ops = 1:
+sel_shift_ops_no_overflow * (1 - sel_shift_ops) = 0;
+// (sel_op_shl || sel_op_shr) & b < max_bits: see below* for constraining.
+pol SHIFT_OVERFLOW = sel_shift_ops * (1 - sel_shift_ops_no_overflow);
+
+// The bit size of the lo limb used by the shift:
+pol commit shift_lo_bits;
+pol commit two_pow_shift_lo_bits;
+
+// *For SHL and SHR, when the shift (b) > max_bits we want SHIFT_OVERFLOW == 1 and c == 0:
+//  SHL: a_lo_bits = max_bits - b -> will underflow
+//  SHR: a_hi_bits = max_bits - b -> will underflow
+//  so instead set a_lo = b - max_bits and shift_lo_bits = max_bits for both SHL and SHR (see DECOMPOSED_A) and reuse the range check
+//  RANGE_CHECK_DECOMPOSITION_A_LO to prove that b > max_bits <==> SHIFT_OVERFLOW = 1 <==> c = 0.
+// Note: sel_decompose_a is gated by IS_NOT_FF, so no gating for the FF tag error case is required below.
+
+#[SHIFTS_LO_BITS]
+shift_lo_bits
+    - sel_shift_ops_no_overflow * (sel_op_shl * (max_bits - ib) + sel_op_shr * ib)
+    - SHIFT_OVERFLOW * max_bits
+    = 0;
+
+// Set shift_hi_bits = max_bits in the overflow case, so RANGE_CHECK_DECOMPOSITION_A_HI passes. Since we set c == 0 in this case,
+// we don't need to constrain that a_hi is within a certain limb size.
+pol SHIFT_HI_BITS = max_bits - sel_shift_ops_no_overflow * shift_lo_bits;
+
+#[SHIFTS_TWO_POW]
+sel_shift_ops_no_overflow { shift_lo_bits, two_pow_shift_lo_bits } in precomputed.sel_range_8 { precomputed.clk, precomputed.power_of_2 };
+
 // TRUNCATE (ALU part for opcodes CAST and SET)
 // Input of truncation value is sent to ia, destination tag in ia_tag and output is computed as ic.
 // We have one dispatching lookup from execution specific to CAST and another one for SET, as

barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 source $(git rev-parse --show-toplevel)/ci3/source
 
+# export bb as it is needed when using exported functions
+export bb=$(./find-bb)
 cd ..
 
 # NOTE: We pin the captured IVC inputs to a known master commit, exploiting that there won't be frequent changes.
@@ -63,9 +65,9 @@ function check_circuit_vks {
   local flow_folder="$inputs_tmp_dir/$1"
 
   if [[ "${2:-}" == "--update_inputs" ]]; then
-    ./build/bin/bb check --update_inputs --scheme client_ivc --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" || { echo_stderr "Error: Likely VK change detected in $flow_folder! Updating inputs."; exit 1; }
+    $bb check --update_inputs --scheme client_ivc --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" || { echo_stderr "Error: Likely VK change detected in $flow_folder! Updating inputs."; exit 1; }
   else
-    ./build/bin/bb check --scheme client_ivc --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" || { echo_stderr "Error: Likely VK change detected in $flow_folder!"; exit 1; }
+    $bb check --scheme client_ivc --ivc_inputs_path "$flow_folder/ivc-inputs.msgpack" || { echo_stderr "Error: Likely VK change detected in $flow_folder!"; exit 1; }
   fi
 }
 

barretenberg/cpp/src/barretenberg/bb/CMakeLists.txt

@@ -1,28 +1,23 @@
-# Used also by bb_cli_bench
-barretenberg_module(
-    bb-cli-lib
-    barretenberg
-    env
-    api
-    circuit_checker
-    ${TRACY_LIBS}
-    libdeflate::libdeflate_static)
-
-if(NOT DISABLE_AZTEC_VM)
-    add_dependencies(bb-cli-lib vm2)
-endif()
-
 if (NOT(FUZZING))
     add_executable(
         bb
         main.cpp
+        cli.cpp
     )
     target_link_libraries(
         bb
         PRIVATE
-        bb-cli-lib
+        barretenberg
+        env
+        api
+        circuit_checker
+        ${TRACY_LIBS}
+        libdeflate::libdeflate_static
         tracy_mem
     )
+    if(NOT DISABLE_AZTEC_VM)
+        add_dependencies(bb vm2)
+    endif()
     if(CHECK_CIRCUIT_STACKTRACES OR ENABLE_STACKTRACES)
         target_link_libraries(
             bb