feat: merge-train/barretenberg (#17987)

BEGIN_COMMIT_OVERRIDE
chore: Simplify default PairingPoints construction and update
AcirProgram metadata (#17912)
refactor: Remove unused domain_start template parameter from Univariate
fix(ci): set AVM_TRANSPILER="" in ci-barretenberg (#17991)
feat!: Aggregate multiple pairing points at once (#17664)
feat!: Databus consistency checks in CIVC verification (#17559)
fix: restore BB_WASM_PATH handling in yarn-project (#17990)
fix: Update Mega proof length (#17995)
fix(cmake): backwards headers (#17993)
chore: Disallow dangerous usage of add_variable and from_witness
functions with non-field types (updated) (#17994)
fix: Fix merge train failure (#18001)
END_COMMIT_OVERRIDE

Author: AztecBot

barretenberg/cpp/bootstrap.sh

@@ -61,7 +61,11 @@ function inject_version {
 function build_preset() {
   local preset=$1
   shift
-  cmake --fresh --preset "$preset"
+  local avm_transpiler_flag=""
+  if [ "${AVM_TRANSPILER:-1}" -eq 0 ]; then
+    avm_transpiler_flag="-DAVM_TRANSPILER_LIB="
+  fi
+  cmake --fresh --preset "$preset" $avm_transpiler_flag
   cmake --build --preset "$preset" "$@"
 }
 

barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh

@@ -13,7 +13,7 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-civc-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-civc-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_short_hash="6c1efddb"
+pinned_short_hash="bd98634a"
 pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-${pinned_short_hash}.tar.gz"
 
 function compress_and_upload {

barretenberg/cpp/src/CMakeLists.txt

@@ -49,7 +49,7 @@ if(WASM)
     add_link_options(-Wl,--export-memory,--import-memory,--stack-first,-z,stack-size=1048576,--max-memory=4294967296)
 endif()
 
-include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${MSGPACK_INCLUDE} ${TRACY_INCLUDE} ${LMDB_INCLUDE} ${LIBDEFLATE_INCLUDE} ${HTTPLIB_INCLUDE} ${NLOHMANN_JSON_INCLUDE})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${MSGPACK_INCLUDE} ${TRACY_INCLUDE} ${LMDB_INCLUDE} ${LIBDEFLATE_INCLUDE} ${HTTPLIB_INCLUDE} ${BACKWARD_INCLUDE} ${NLOHMANN_JSON_INCLUDE})
 
 # Add avm-transpiler include path when library is provided
 if(AVM_TRANSPILER_LIB)

barretenberg/cpp/src/barretenberg/bbapi/bbapi_ultra_honk.cpp

@@ -34,20 +34,9 @@ namespace bb::bbapi {
 
 template <typename Flavor> acir_format::ProgramMetadata _create_program_metadata()
 {
-    uint32_t honk_recursion = 0;
+    bool constexpr has_ipa_claim = IsAnyOf<Flavor, UltraRollupFlavor>;
 
-    if constexpr (IsAnyOf<Flavor, UltraFlavor, UltraZKFlavor, UltraKeccakFlavor, UltraKeccakZKFlavor>) {
-        honk_recursion = 1;
-    } else if constexpr (IsAnyOf<Flavor, UltraRollupFlavor>) {
-        honk_recursion = 2;
-    }
-#ifdef STARKNET_GARAGA_FLAVORS
-    if constexpr (IsAnyOf<Flavor, UltraStarknetFlavor, UltraStarknetZKFlavor>) {
-        honk_recursion = 1;
-    }
-#endif
-
-    return acir_format::ProgramMetadata{ .honk_recursion = honk_recursion };
+    return acir_format::ProgramMetadata{ .has_ipa_claim = has_ipa_claim };
 }
 
 template <typename Flavor, typename Circuit = typename Flavor::CircuitBuilder>

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_circuits.hpp

@@ -18,7 +18,7 @@ namespace bb::mock_circuits {
  */
 template <typename Builder> void generate_basic_arithmetic_circuit(Builder& builder, size_t log2_num_gates)
 {
-    stdlib::recursion::PairingPoints<Builder>::add_default_to_public_inputs(builder);
+    stdlib::recursion::honk::DefaultIO<Builder>::add_default(builder);
 
     stdlib::field_t a(stdlib::witness_t(&builder, fr::random_element()));
     stdlib::field_t b(stdlib::witness_t(&builder, fr::random_element()));

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_dynamic_array.test.cpp

@@ -37,7 +37,7 @@ TEST(boomerang_stdlib_dynamic_array, graph_description_dynamic_array_method_resi
 
     field_ct next_size = field_ct(witness_ct(&builder, (uint256_t)(max_size - 1)));
     for (size_t i = 0; i < max_size; ++i) {
-        array.push(field_ct::from_witness(&builder, i));
+        array.push(field_ct::from_witness(&builder, typename field_ct::native(i)));
     }
 
     array.resize(next_size, 7);
@@ -67,7 +67,7 @@ TEST(boomerang_stdlib_dynamic_array, graph_description_dynamic_array_consistency
     DynamicArray_ct array(&builder, max_size);
 
     for (size_t i = 0; i < max_size; ++i) {
-        array.push(field_ct::from_witness(&builder, i));
+        array.push(field_ct::from_witness(&builder, typename field_ct::native(i)));
     }
 
     for (size_t i = 0; i < max_size; ++i) {

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_ipa_recursive.test.cpp

@@ -5,8 +5,8 @@
 #include "barretenberg/commitment_schemes/shplonk/shplemini.hpp"
 #include "barretenberg/stdlib/eccvm_verifier/verifier_commitment_key.hpp"
 #include "barretenberg/stdlib/primitives/curves/grumpkin.hpp"
-#include "barretenberg/stdlib/primitives/pairing_points.hpp"
 #include "barretenberg/stdlib/proof/proof.hpp"
+#include "barretenberg/stdlib/special_public_inputs/special_public_inputs.hpp"
 #include "barretenberg/transcript/transcript.hpp"
 #include "barretenberg/ultra_honk/prover_instance.hpp"
 
@@ -100,7 +100,7 @@ class BoomerangIPARecursiveTests : public CommitmentTest<NativeCurve> {
         auto [stdlib_transcript, stdlib_claim] = create_ipa_claim<log_poly_length>(builder, poly, x);
 
         RecursiveIPA::reduce_verify(stdlib_claim, stdlib_transcript);
-        stdlib::recursion::PairingPoints<Builder>::add_default_to_public_inputs(builder);
+        stdlib::recursion::honk::DefaultIO<Builder>::add_default(builder);
         builder.finalize_circuit(/*ensure_nonzero=*/true);
         return builder;
     }

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_ultra_recursive_verifier.test.cpp

@@ -78,13 +78,10 @@ template <typename RecursiveFlavor> class BoomerangRecursiveVerifierTest : publi
 
             builder.create_big_add_gate({ a_idx, b_idx, c_idx, d_idx, fr(1), fr(1), fr(1), fr(-1), fr(0) });
         }
-        PairingPoints<InnerBuilder>::add_default_to_public_inputs(builder);
-
         if constexpr (HasIPAAccumulator<RecursiveFlavor>) {
-            auto [stdlib_opening_claim, ipa_proof] =
-                IPA<grumpkin<InnerBuilder>>::create_random_valid_ipa_claim_and_proof(builder);
-            stdlib_opening_claim.set_public();
-            builder.ipa_proof = ipa_proof;
+            RollupIO::add_default(builder);
+        } else {
+            DefaultIO<InnerBuilder>::add_default(builder);
         }
         return builder;
     };

barretenberg/cpp/src/barretenberg/circuit_checker/ultra_circuit_builder.test.cpp

@@ -811,10 +811,10 @@ TEST(UltraCircuitBuilder, Rom)
         builder.set_ROM_element(rom_id, i, rom_values[i]);
     }
 
-    uint32_t a_idx = builder.read_ROM_array(rom_id, builder.add_variable(5));
+    uint32_t a_idx = builder.read_ROM_array(rom_id, builder.add_variable(fr(5)));
     EXPECT_EQ(a_idx != rom_values[5], true);
-    uint32_t b_idx = builder.read_ROM_array(rom_id, builder.add_variable(4));
-    uint32_t c_idx = builder.read_ROM_array(rom_id, builder.add_variable(1));
+    uint32_t b_idx = builder.read_ROM_array(rom_id, builder.add_variable(fr(4)));
+    uint32_t c_idx = builder.read_ROM_array(rom_id, builder.add_variable(fr(1)));
 
     const auto d_value = builder.get_variable(a_idx) + builder.get_variable(b_idx) + builder.get_variable(c_idx);
     uint32_t d_idx = builder.add_variable(d_value);
@@ -850,7 +850,7 @@ TEST(UltraCircuitBuilder, RamSimple)
     builder.init_RAM_element(ram_id, /*index_value=*/0, ram_value_idx);
 
     // Read from the RAM array we just created (at the 0th index)
-    uint32_t read_idx = builder.add_variable(0);
+    uint32_t read_idx = builder.add_variable(fr(0));
     uint32_t a_idx = builder.read_RAM_array(ram_id, read_idx);
 
     // Use the result in a simple arithmetic gate
@@ -886,14 +886,14 @@ TEST(UltraCircuitBuilder, Ram)
         builder.init_RAM_element(ram_id, i, ram_values[i]);
     }
 
-    uint32_t a_idx = builder.read_RAM_array(ram_id, builder.add_variable(5));
+    uint32_t a_idx = builder.read_RAM_array(ram_id, builder.add_variable(fr(5)));
     EXPECT_EQ(a_idx != ram_values[5], true);
 
-    uint32_t b_idx = builder.read_RAM_array(ram_id, builder.add_variable(4));
-    uint32_t c_idx = builder.read_RAM_array(ram_id, builder.add_variable(1));
+    uint32_t b_idx = builder.read_RAM_array(ram_id, builder.add_variable(fr(4)));
+    uint32_t c_idx = builder.read_RAM_array(ram_id, builder.add_variable(fr(1)));
 
-    builder.write_RAM_array(ram_id, builder.add_variable(4), builder.add_variable(500));
-    uint32_t d_idx = builder.read_RAM_array(ram_id, builder.add_variable(4));
+    builder.write_RAM_array(ram_id, builder.add_variable(fr(4)), builder.add_variable(fr(500)));
+    uint32_t d_idx = builder.read_RAM_array(ram_id, builder.add_variable(fr(4)));
 
     EXPECT_EQ(builder.get_variable(d_idx), 500);
 
@@ -943,10 +943,10 @@ TEST(UltraCircuitBuilder, RangeChecksOnDuplicates)
 {
     UltraCircuitBuilder builder = UltraCircuitBuilder();
 
-    uint32_t a = builder.add_variable(100);
-    uint32_t b = builder.add_variable(100);
-    uint32_t c = builder.add_variable(100);
-    uint32_t d = builder.add_variable(100);
+    uint32_t a = builder.add_variable(fr(100));
+    uint32_t b = builder.add_variable(fr(100));
+    uint32_t c = builder.add_variable(fr(100));
+    uint32_t d = builder.add_variable(fr(100));
 
     builder.assert_equal(a, b);
     builder.assert_equal(a, c);
@@ -979,8 +979,8 @@ TEST(UltraCircuitBuilder, CheckCircuitShowcase)
     UltraCircuitBuilder builder = UltraCircuitBuilder();
     // check_circuit allows us to check correctness on the go
 
-    uint32_t a = builder.add_variable(0xdead);
-    uint32_t b = builder.add_variable(0xbeef);
+    uint32_t a = builder.add_variable(fr(0xdead));
+    uint32_t b = builder.add_variable(fr(0xbeef));
     // Let's create 2 gates that will bind these 2 variables to be one these two values
     builder.create_poly_gate(
         { a, a, builder.zero_idx(), fr(1), -fr(0xdead) - fr(0xbeef), 0, 0, fr(0xdead) * fr(0xbeef) });
@@ -1003,7 +1003,7 @@ TEST(UltraCircuitBuilder, CheckCircuitShowcase)
     EXPECT_EQ(CircuitChecker::check(builder), false);
 
     // But if we force them both back to be 0xbeef...
-    uint32_t c = builder.add_variable(0xbeef);
+    uint32_t c = builder.add_variable(fr(0xbeef));
     builder.assert_equal(c, b);
 
     // The circuit will magically pass again

barretenberg/cpp/src/barretenberg/client_ivc/sumcheck_client_ivc.cpp

@@ -90,7 +90,7 @@ void SumcheckClientIVC::instantiate_stdlib_verification_queue(
  * tables as read from the proof by the Merge verifier
  */
 std::tuple<std::optional<SumcheckClientIVC::RecursiveVerifierAccumulator>,
-           SumcheckClientIVC::PairingPoints,
+           std::vector<SumcheckClientIVC::PairingPoints>,
            SumcheckClientIVC::TableCommitments>
 SumcheckClientIVC::perform_recursive_verification_and_databus_consistency_checks(
     ClientCircuit& circuit,
@@ -101,8 +101,8 @@ SumcheckClientIVC::perform_recursive_verification_and_databus_consistency_checks
 {
     using MergeCommitments = Goblin::MergeRecursiveVerifier::InputCommitments;
 
-    // The pairing points produced by the verification of the decider proof
-    PairingPoints decider_pairing_points;
+    // PairingPoints to be returned for aggregation
+    std::vector<PairingPoints> pairing_points;
 
     // Input commitments to be passed to the merge recursive verification
     MergeCommitments merge_commitments{ .T_prev_commitments = T_prev_commitments };
@@ -111,8 +111,6 @@ SumcheckClientIVC::perform_recursive_verification_and_databus_consistency_checks
 
     std::optional<RecursiveVerifierAccumulator> output_verifier_accumulator;
     std::optional<StdlibFF> prev_accum_hash = std::nullopt;
-    // The decider proof exists if the tail kernel has been accumulated
-    bool is_hiding_kernel = !decider_proof.empty();
 
     // Update previous accumulator hash so that we can check it against the one extracted from the public inputs
     if (verifier_inputs.is_kernel && verifier_inputs.type != QUEUE_TYPE::OINK) {
@@ -152,7 +150,7 @@ SumcheckClientIVC::perform_recursive_verification_and_databus_consistency_checks
 
         RecursiveDeciderVerifier decider_verifier(accumulation_recursive_transcript);
         StdlibProof stdlib_decider_proof(circuit, decider_proof);
-        decider_pairing_points = decider_verifier.verify_proof(final_verifier_accumulator, stdlib_decider_proof);
+        pairing_points.emplace_back(decider_verifier.verify_proof(final_verifier_accumulator, stdlib_decider_proof));
 
         BB_ASSERT_EQ(output_verifier_accumulator.has_value(), false);
         break;
@@ -166,13 +164,12 @@ SumcheckClientIVC::perform_recursive_verification_and_databus_consistency_checks
     WitnessCommitments witness_commitments = std::move(verifier_instance->witness_commitments);
     std::vector<StdlibFF> public_inputs = std::move(verifier_instance->public_inputs);
 
-    PairingPoints nested_pairing_points; // to be extracted from public inputs of app or kernel proof just verified
-
     if (verifier_inputs.is_kernel) {
         // Reconstruct the input from the previous kernel from its public inputs
         KernelIO kernel_input; // pairing points, databus return data commitments
         kernel_input.reconstruct_from_public(public_inputs);
-        nested_pairing_points = kernel_input.pairing_inputs;
+        // Add pairing points for aggregation
+        pairing_points.emplace_back(kernel_input.pairing_inputs);
         // Perform databus consistency checks
         kernel_input.kernel_return_data.incomplete_assert_equal(witness_commitments.calldata);
         kernel_input.app_return_data.incomplete_assert_equal(witness_commitments.secondary_calldata);
@@ -190,15 +187,14 @@ SumcheckClientIVC::perform_recursive_verification_and_databus_consistency_checks
         BB_ASSERT(prev_accum_hash.has_value());
         kernel_input.output_pg_accum_hash.assert_equal(*prev_accum_hash);
 
-        if (!is_hiding_kernel) {
-            // The hiding kernel has no return data; it uses the traditional public-inputs mechanism
-            bus_depot.set_kernel_return_data_commitment(witness_commitments.return_data);
-        }
+        // Set the kernel return data commitment to be propagated via the public inputs
+        bus_depot.set_kernel_return_data_commitment(witness_commitments.return_data);
     } else {
         // Reconstruct the input from the previous app from its public inputs
         AppIO app_input; // pairing points
         app_input.reconstruct_from_public(public_inputs);
-        nested_pairing_points = app_input.pairing_inputs;
+        // Add pairing points for aggregation
+        pairing_points.emplace_back(app_input.pairing_inputs);
 
         // Set the app return data commitment to be propagated via the public inputs
         bus_depot.set_app_return_data_commitment(witness_commitments.return_data);
@@ -208,16 +204,9 @@ SumcheckClientIVC::perform_recursive_verification_and_databus_consistency_checks
     merge_commitments.t_commitments = witness_commitments.get_ecc_op_wires().get_copy();
 
     // Recursively verify the corresponding merge proof
-    auto [pairing_points, merged_table_commitments] =
+    auto [merge_pairing_points, merged_table_commitments] =
         goblin.recursively_verify_merge(circuit, merge_commitments, accumulation_recursive_transcript);
-
-    pairing_points.aggregate(nested_pairing_points);
-    if (is_hiding_kernel) {
-        pairing_points.aggregate(decider_pairing_points);
-        // Add randomness at the end of the hiding kernel (whose ecc ops fall right at the end of the op queue table) to
-        // ensure the CIVC proof doesn't leak information about the actual content of the op queue
-        hide_op_queue_content_in_hiding(circuit);
-    }
+    pairing_points.emplace_back(merge_pairing_points);
 
     return { output_verifier_accumulator, pairing_points, merged_table_commitments };
 }
@@ -268,7 +257,7 @@ void SumcheckClientIVC::complete_kernel_circuit_logic(ClientCircuit& circuit)
     circuit.queue_ecc_eq();
 
     // Perform Oink/PG and Merge recursive verification + databus consistency checks for each entry in the queue
-    PairingPoints points_accumulator;
+    std::vector<PairingPoints> points_accumulator;
     std::optional<RecursiveVerifierAccumulator> current_stdlib_verifier_accumulator;
     if (!is_init_kernel) {
         current_stdlib_verifier_accumulator = RecursiveVerifierAccumulator::stdlib_from_native<RecursiveFlavor::Curve>(
@@ -283,26 +272,36 @@ void SumcheckClientIVC::complete_kernel_circuit_logic(ClientCircuit& circuit)
                                                                           current_stdlib_verifier_accumulator,
                                                                           T_prev_commitments,
                                                                           accumulation_recursive_transcript);
-        points_accumulator.aggregate(pairing_points);
+        points_accumulator.insert(points_accumulator.end(), pairing_points.begin(), pairing_points.end());
         // Update commitment to the status of the op_queue
         T_prev_commitments = merged_table_commitments;
         // Update the output verifier accumulator
         current_stdlib_verifier_accumulator = output_stdlib_verifier_accumulator;
 
         stdlib_verification_queue.pop_front();
     }
+
+    // PairingPoint aggregation
+    PairingPoints pairing_points_aggregator = PairingPoints::aggregate_multiple(points_accumulator);
+
     // Set the kernel output data to be propagated via the public inputs
     if (is_hiding_kernel) {
         BB_ASSERT_EQ(current_stdlib_verifier_accumulator.has_value(), false);
-        HidingKernelIO hiding_output{ points_accumulator, T_prev_commitments };
+        // Add randomness at the end of the hiding kernel (whose ecc ops fall right at the end of the op queue table) to
+        // ensure the CIVC proof doesn't leak information about the actual content of the op queue
+        hide_op_queue_content_in_hiding(circuit);
+
+        HidingKernelIO hiding_output{ pairing_points_aggregator,
+                                      bus_depot.get_kernel_return_data_commitment(circuit),
+                                      T_prev_commitments };
         hiding_output.set_public();
     } else {
         BB_ASSERT_NEQ(current_stdlib_verifier_accumulator.has_value(), false);
         // Extract native verifier accumulator from the stdlib accum for use on the next round
         recursive_verifier_native_accum = current_stdlib_verifier_accumulator->get_value<VerifierAccumulator>();
 
         KernelIO kernel_output;
-        kernel_output.pairing_inputs = points_accumulator;
+        kernel_output.pairing_inputs = pairing_points_aggregator;
         kernel_output.kernel_return_data = bus_depot.get_kernel_return_data_commitment(circuit);
         kernel_output.app_return_data = bus_depot.get_app_return_data_commitment(circuit);
         kernel_output.ecc_op_tables = T_prev_commitments;
@@ -569,8 +568,12 @@ bool SumcheckClientIVC::verify(const Proof& proof, const VerificationKey& vk)
     std::shared_ptr<Goblin::Transcript> civc_verifier_transcript = std::make_shared<Goblin::Transcript>();
     // Verify the hiding circuit proof
     MegaZKVerifier verifier{ vk.mega, /*ipa_verification_key=*/{}, civc_verifier_transcript };
-    auto [mega_verified, T_prev_commitments] = verifier.template verify_proof<bb::HidingKernelIO>(proof.mega_proof);
+    auto [mega_verified, kernel_return_data, T_prev_commitments] =
+        verifier.template verify_proof<bb::HidingKernelIO>(proof.mega_proof);
     vinfo("Mega verified: ", mega_verified);
+    // Perform databus consistency checks
+    bool databus_consistency_verified = kernel_return_data == verifier.verifier_instance->witness_commitments.calldata;
+    vinfo("Databus consistency verified: ", databus_consistency_verified);
     // Extract the commitments to the subtable corresponding to the incoming circuit
     TableCommitments t_commitments = verifier.verifier_instance->witness_commitments.get_ecc_op_wires().get_copy();
 
@@ -580,7 +583,7 @@ bool SumcheckClientIVC::verify(const Proof& proof, const VerificationKey& vk)
     vinfo("Goblin verified: ", goblin_verified);
 
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1396): State tracking in CIVC verifiers.
-    return goblin_verified && mega_verified;
+    return goblin_verified && mega_verified && databus_consistency_verified;
 }
 
 // Proof methods