feat: merge-train/barretenberg (#16454)

BEGIN_COMMIT_OVERRIDE
chore: Clean up public input propagation in `acir_format` (#16411)
feat: merge via append ecc ops in hiding kernel (#16338)
chore: install clang20, upgrade wasi-sdk to 27 (#16414)
chore: make clang20 default and clean up presets (#16306)
chore: stdlib sha256 without packed_byte_array  (#15912)
fix: Revert "chore: stdlib sha256 without packed_byte_array  (#15912)"
fix: Correct active range for databus (#16381)
fix: Revert "fix: Correct active range for databus (#16381)"
feat: fixed size decider without padding (#16318)
Revert "chore: make clang20 default and clean up presets (#16306)"
Revert "chore: install clang20, upgrade wasi-sdk to 27 (#16414)"
chore: document the ECCVM builders. (#15913)
END_COMMIT_OVERRIDE

Author: AztecBot

barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh

@@ -11,7 +11,7 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-civc-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-civc-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_short_hash="ca12eef6"
+pinned_short_hash="251cc432"
 pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-${pinned_short_hash}.tar.gz"
 
 function compress_and_upload {

barretenberg/cpp/src/barretenberg/api/api_client_ivc.cpp

@@ -71,6 +71,7 @@ std::vector<uint8_t> write_civc_vk(const std::string& output_format,
         throw_or_abort("Unsupported output format for ClientIVC vk: " + output_format);
     }
     // compute the hiding kernel's vk
+    info("ClientIVC: computing IVC vk for hiding kernel circuit");
     auto response = bbapi::ClientIvcComputeIvcVk{
         .circuit{ .name = "standalone_circuit", .bytecode = std::move(bytecode) }
     }.execute({ .trace_settings = {} });
@@ -95,7 +96,7 @@ void ClientIVCAPI::prove(const Flags& flags,
     std::vector<PrivateExecutionStepRaw> raw_steps = PrivateExecutionStepRaw::load_and_decompress(input_path);
 
     bbapi::ClientIvcStart{ .num_circuits = raw_steps.size() - 1 }.execute(request);
-
+    info("ClientIVC: starting with ", raw_steps.size(), " circuits");
     for (size_t i = 0; i < raw_steps.size() - 1; ++i) {
         const auto& step = raw_steps[i];
         bbapi::ClientIvcLoad{
@@ -268,7 +269,7 @@ void gate_count_for_ivc(const std::string& bytecode_path, bool include_gates_per
 void write_arbitrary_valid_client_ivc_proof_and_vk_to_file(const std::filesystem::path& output_dir)
 {
 
-    const size_t NUM_CIRCUITS = 2;
+    const size_t NUM_CIRCUITS = 6;
     ClientIVC ivc{ NUM_CIRCUITS, { AZTEC_TRACE_STRUCTURE } };
 
     // Construct and accumulate a series of mocked private function execution circuits

barretenberg/cpp/src/barretenberg/api/api_client_ivc.test.cpp

@@ -37,6 +37,8 @@ std::filesystem::path get_test_dir(const std::string_view& test_name)
     return temp_dir / test_name;
 }
 
+// TODO(https://github.com/AztecProtocol/barretenberg/issues/1509): expand these test to accomodate a more realistic
+// CIVC flow in order to re-enable the ProveAndVerify* tests in this file
 void create_test_private_execution_steps(const std::filesystem::path& output_path)
 {
     using namespace acir_format;
@@ -194,6 +196,31 @@ TEST_F(ClientIVCAPITests, WriteVkFieldsSmokeTest)
     EXPECT_NE(vk_str.find(']'), std::string::npos);
 }
 
+TEST_F(ClientIVCAPITests, WriteIVCVkSmokeTest)
+{
+    // Create a simple circuit bytecode
+    auto [bytecode, witness_data] = acir_bincode_mocks::create_simple_circuit_bytecode();
+
+    // Compress and write bytecode to file
+    std::filesystem::path bytecode_path = test_dir / "circuit.acir";
+    write_file(bytecode_path, bb::compress(bytecode));
+
+    // Set flags for VK generation
+    ClientIVCAPI::Flags flags;
+    flags.verifier_type = "ivc";
+    flags.output_format = "bytes";
+
+    // Call write_vk
+    ClientIVCAPI api;
+    api.write_vk(flags, bytecode_path, test_dir);
+
+    // Check that VK file exists and is non-empty
+    std::filesystem::path vk_path = test_dir / "vk";
+    ASSERT_TRUE(std::filesystem::exists(vk_path));
+    auto vk_data = read_file(vk_path);
+    ASSERT_FALSE(vk_data.empty());
+}
+
 // TODO(https://github.com/AztecProtocol/barretenberg/issues/1461): Make this test actually test # gates
 TEST_F(ClientIVCAPITests, GatesCommandSmokeTest)
 {
@@ -233,7 +260,7 @@ TEST_F(ClientIVCAPITests, GatesCommandSmokeTest)
 }
 
 // Test prove_and_verify for our example IVC flow.
-TEST_F(ClientIVCAPITests, ProveAndVerifyCommand)
+TEST_F(ClientIVCAPITests, DISABLED_ProveAndVerifyCommand)
 {
     // Create test input file
     std::filesystem::path input_path = test_dir / "input.msgpack";

barretenberg/cpp/src/barretenberg/benchmark/client_ivc_bench/client_ivc.bench.cpp

@@ -32,15 +32,12 @@ class ClientIVCBench : public benchmark::Fixture {
  */
 BENCHMARK_DEFINE_F(ClientIVCBench, VerificationOnly)(benchmark::State& state)
 {
-    ClientIVC ivc{ /*num_circuits=*/2, { AZTEC_TRACE_STRUCTURE } };
+    ClientIVC ivc{ /*num_circuits=*/4, { AZTEC_TRACE_STRUCTURE } };
 
     PrivateFunctionExecutionMockCircuitProducer circuit_producer;
-
-    // Initialize the IVC with an arbitrary circuit
-    circuit_producer.construct_and_accumulate_next_circuit(ivc);
-
-    // Create another circuit and accumulate
-    circuit_producer.construct_and_accumulate_next_circuit(ivc);
+    for (size_t idx = 0; idx < 4; ++idx) {
+        circuit_producer.construct_and_accumulate_next_circuit(ivc);
+    }
 
     auto proof = ivc.prove();
 

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_goblin.test.cpp

@@ -55,8 +55,7 @@ class BoomerangGoblinRecursiveVerifierTests : public testing::Test {
         Goblin goblin_final;
         goblin_final.op_queue = goblin.op_queue;
         MegaCircuitBuilder builder{ goblin_final.op_queue };
-        builder.queue_ecc_no_op();
-        GoblinMockCircuits::construct_simple_circuit(builder);
+        GoblinMockCircuits::construct_simple_circuit(builder, /*last_circuit=*/true);
         goblin_final.op_queue->merge();
         // Subtable values and commitments - needed for (Recursive)MergeVerifier
         MergeCommitments merge_commitments;

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_ultra_recursive_verifier.test.cpp

@@ -114,6 +114,8 @@ template <typename RecursiveFlavor> class BoomerangRecursiveVerifierTest : publi
         RecursiveVerifier verifier{ &outer_circuit, stdlib_vk_and_hash };
         verifier.key->vk_and_hash->vk->num_public_inputs.fix_witness();
         verifier.key->vk_and_hash->vk->pub_inputs_offset.fix_witness();
+        // It's currently un-used
+        verifier.key->vk_and_hash->vk->log_circuit_size.fix_witness();
 
         StdlibProof stdlib_inner_proof(outer_circuit, inner_proof);
         VerifierOutput output = verifier.template verify_proof<DefaultIO<OuterBuilder>>(stdlib_inner_proof);

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -238,13 +238,25 @@ void ClientIVC::complete_kernel_circuit_logic(ClientCircuit& circuit)
     bool is_hiding_kernel =
         stdlib_verification_queue.size() == 1 && (stdlib_verification_queue.front().type == QUEUE_TYPE::PG_FINAL);
 
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1511): this should check the queue is of size one and
+    // contains an entry of type PG_TAIL, currently it might contain several entries in case it's a minimal transaction
+    // produced by our tests which is not exactly realistic
+    bool is_tail_kernel = std::any_of(stdlib_verification_queue.begin(),
+                                      stdlib_verification_queue.end(),
+                                      [](const auto& entry) { return entry.type == QUEUE_TYPE::PG_TAIL; });
     // If the incoming circuit is a kernel, start its subtable with an eq and reset operation to ensure a
     // neighbouring misconfigured subtable coming from an app cannot affect the operations in the
-    // current subtable. We don't do this for the hiding kernel as it succeeds another kernel and because the hiding
-    // kernel has to start with a no-op for the correct functioning of translator.
+    // current subtable. We don't do this for the hiding kernel as it succeeds another kernel.
     if (!is_hiding_kernel) {
+        if (is_tail_kernel) {
+            // Add a no-op at the beginning of the tail kernel (the last circuit whose ecc ops subtable is prepended)
+            // to ensure the wires representing the op queue in translator circuit are shiftable polynomials, i.e.
+            // their 0th coefficient is 0.
+            circuit.queue_ecc_no_op();
+        }
         circuit.queue_ecc_eq();
     }
+
     // Perform Oink/PG and Merge recursive verification + databus consistency checks for each entry in the queue
     PairingPoints points_accumulator;
     while (!stdlib_verification_queue.empty()) {
@@ -419,10 +431,6 @@ std::pair<ClientIVC::PairingPoints, ClientIVC::TableCommitments> ClientIVC::comp
     // share a transcript
     std::shared_ptr<RecursiveTranscript> pg_merge_transcript = std::make_shared<RecursiveTranscript>();
 
-    // Add a no-op at the beginning of the hiding circuit to ensure the wires representing the op queue in translator
-    // circuit are shiftable polynomials, i.e. their 0th coefficient is equal to 0.
-    circuit.queue_ecc_no_op();
-
     hide_op_queue_accumulation_result(circuit);
 
     // Construct stdlib accumulator, decider vkey and folding proof
@@ -515,8 +523,13 @@ ClientIVC::Proof ClientIVC::prove()
     // A transcript is shared between the Hiding circuit prover and the Goblin prover
     goblin.transcript = transcript;
 
-    // Prove ECCVM and Translator
-    return { mega_proof, goblin.prove() };
+    // Returns a proof for the hiding circuit and the Goblin proof. The latter consists of Translator and ECCVM proof
+    // for the whole ecc op table and the merge proof for appending the subtable coming from the hiding circuit. The
+    // final merging is done via appending to facilitate creating a zero-knowledge merge proof. This enables us to add
+    // randomness to the beginning of the tail kernel and the end of the hiding kernel, hiding the commitments and
+    // evaluations of both the previous table and the incoming subtable.
+    // https://github.com/AztecProtocol/barretenberg/issues/1360
+    return { mega_proof, goblin.prove(MergeSettings::APPEND) };
 };
 
 bool ClientIVC::verify(const Proof& proof, const VerificationKey& vk)
@@ -532,8 +545,8 @@ bool ClientIVC::verify(const Proof& proof, const VerificationKey& vk)
     TableCommitments t_commitments = verifier.verification_key->witness_commitments.get_ecc_op_wires().get_copy();
 
     // Goblin verification (final merge, eccvm, translator)
-    bool goblin_verified =
-        Goblin::verify(proof.goblin_proof, { t_commitments, T_prev_commitments }, civc_verifier_transcript);
+    bool goblin_verified = Goblin::verify(
+        proof.goblin_proof, { t_commitments, T_prev_commitments }, civc_verifier_transcript, MergeSettings::APPEND);
     vinfo("Goblin verified: ", goblin_verified);
 
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1396): State tracking in CIVC verifiers.