feat: fixed size decider without padding (#16318)

Closes AztecProtocol/barretenberg#1158
Closes AztecProtocol/barretenberg#1310
Closes AztecProtocol/barretenberg#1283

---------

Co-authored-by: ledwards2225 <[email protected]>
Co-authored-by: Maddiaa <[email protected]>

Author: 3 people

barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh

@@ -11,7 +11,7 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-civc-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-civc-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_short_hash="1710c0ac"
+pinned_short_hash="251cc432"
 pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-${pinned_short_hash}.tar.gz"
 
 function compress_and_upload {

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_ultra_recursive_verifier.test.cpp

@@ -114,6 +114,8 @@ template <typename RecursiveFlavor> class BoomerangRecursiveVerifierTest : publi
         RecursiveVerifier verifier{ &outer_circuit, stdlib_vk_and_hash };
         verifier.key->vk_and_hash->vk->num_public_inputs.fix_witness();
         verifier.key->vk_and_hash->vk->pub_inputs_offset.fix_witness();
+        // It's currently un-used
+        verifier.key->vk_and_hash->vk->log_circuit_size.fix_witness();
 
         StdlibProof stdlib_inner_proof(outer_circuit, inner_proof);
         VerifierOutput output = verifier.template verify_proof<DefaultIO<OuterBuilder>>(stdlib_inner_proof);

barretenberg/cpp/src/barretenberg/commitment_schemes/gemini/gemini.hpp

@@ -317,7 +317,8 @@ template <typename Curve> class GeminiProver_ {
 
     static std::vector<Polynomial> compute_fold_polynomials(const size_t log_n,
                                                             std::span<const Fr> multilinear_challenge,
-                                                            const Polynomial& A_0);
+                                                            const Polynomial& A_0,
+                                                            const bool& has_zk = false);
 
     static std::pair<Polynomial, Polynomial> compute_partially_evaluated_batch_polynomials(
         const size_t log_n,
@@ -549,7 +550,8 @@ template <typename Curve> class GeminiVerifier_ {
      *
      * @param padding_indicator_array An array with first log_n entries equal to 1, and the remaining entries are 0.
      * @param batched_evaluation The evaluation of the batched polynomial at \f$ (u_0, \ldots, u_{d-1})\f$.
-     * @param evaluation_point Evaluation point \f$ (u_0, \ldots, u_{d-1}) \f$ padded to CONST_PROOF_SIZE_LOG_N.
+     * @param evaluation_point Evaluation point \f$ (u_0, \ldots, u_{d-1}) \f$. Depending on the context, might be
+     * padded to `virtual_log_n` size.
      * @param challenge_powers Powers of \f$ r \f$, \f$ r^2 \), ..., \( r^{2^{d-1}} \f$.
      * @param fold_neg_evals  Evaluations \f$ A_{i-1}(-r^{2^{i-1}}) \f$.
      * @return \f A_{i}}(r^{2^{i}})\f$ \f$ i = 0, \ldots, \text{virtual_log_n} - 1 \f$.

barretenberg/cpp/src/barretenberg/commitment_schemes/gemini/gemini.test.cpp

@@ -10,11 +10,15 @@ template <class Curve> class GeminiTest : public CommitmentTest<Curve> {
     using GeminiVerifier = GeminiVerifier_<Curve>;
     using Fr = typename Curve::ScalarField;
     using Commitment = typename Curve::AffineElement;
+    using ClaimBatcher = ClaimBatcher_<Curve>;
+    using ClaimBatch = ClaimBatcher::Batch;
 
   public:
     static constexpr size_t log_n = 4;
     static constexpr size_t n = 1UL << log_n;
 
+    static constexpr size_t virtual_log_n = 6;
+
     using CK = CommitmentKey<Curve>;
     using VK = VerifierCommitmentKey<Curve>;
 
@@ -82,7 +86,80 @@ template <class Curve> class GeminiTest : public CommitmentTest<Curve> {
             ASSERT_EQ(prover_claim.opening_pair, verifier_claim.opening_pair);
         }
     }
-};
+
+    void open_extension_by_zero()
+    {
+        auto prover_transcript = NativeTranscript::prover_init_empty();
+
+        auto u = this->random_evaluation_point(virtual_log_n);
+
+        Polynomial<Fr> poly((1UL << log_n));
+
+        poly.at(0) = 1;
+        poly.at(1) = 2;
+        poly.at(2) = 3;
+
+        typename GeminiProver::PolynomialBatcher poly_batcher(1UL << log_n);
+        poly_batcher.set_unshifted(RefVector(poly));
+
+        // As we are opening `poly` extended by zero from `log_n` dimensions to `virtual_log_n` dimensions, it needs to
+        // be multiplied by appropriate scalars.
+        Fr eval = poly.evaluate_mle(std::span(u).subspan(0, log_n)) * (Fr(1) - u[virtual_log_n - 1]) *
+                  (Fr(1) - u[virtual_log_n - 2]);
+        auto comm = ck.commit(poly);
+        auto claim_batcher = ClaimBatcher{ .unshifted = ClaimBatch{ RefVector(comm), RefVector(eval) } };
+
+        // Compute:
+        // - (d+1) opening pairs: {r, \hat{a}_0}, {-r^{2^i}, a_i}, i = 0, ..., d-1
+        // - (d+1) Fold polynomials Fold_{r}^(0), Fold_{-r}^(0), and Fold^(i), i = 0, ..., d-1
+        auto prover_output = GeminiProver::prove(1UL << log_n, poly_batcher, u, ck, prover_transcript);
+
+        // The prover output needs to be completed by adding the "positive" Fold claims, i.e. evaluations of
+        // Fold^(i) at r^{2^i} for i=1, ..., d-1. Although here we are copying polynomials, it is not the case when
+        // GeminiProver is combined with ShplonkProver.
+        std::vector<ProverOpeningClaim<Curve>> prover_claims_with_pos_evals;
+        // `prover_output` consists of d+1 opening claims, we add another d-1 claims for each positive evaluation
+        // Fold^i(r^{2^i}) for i = 1, ..., d-1
+        const size_t total_num_claims = 2 * virtual_log_n;
+        prover_claims_with_pos_evals.reserve(total_num_claims);
+
+        for (auto& claim : prover_output) {
+            if (claim.gemini_fold) {
+                if (claim.gemini_fold) {
+                    // "positive" evaluation challenge r^{2^i} for i = 1, ..., d-1
+                    const Fr evaluation_challenge = -claim.opening_pair.challenge;
+                    // Fold^(i) at r^{2^i} for i=1, ..., d-1
+                    const Fr pos_evaluation = claim.polynomial.evaluate(evaluation_challenge);
+
+                    // Add the positive Fold claims to the vector of claims
+                    ProverOpeningClaim<Curve> pos_fold_claim = { .polynomial = claim.polynomial,
+                                                                 .opening_pair = { .challenge = evaluation_challenge,
+                                                                                   .evaluation = pos_evaluation } };
+                    prover_claims_with_pos_evals.emplace_back(pos_fold_claim);
+                }
+            }
+            prover_claims_with_pos_evals.emplace_back(claim);
+        }
+
+        // Check that the Fold polynomials have been evaluated correctly in the prover
+        this->verify_batch_opening_pair(prover_claims_with_pos_evals);
+
+        auto verifier_transcript = NativeTranscript::verifier_init_empty(prover_transcript);
+
+        // Compute:
+        // - d opening pairs: {r^{2^i}, \hat{a}_i} for i = 0, ..., d-1
+        // - 2 partially evaluated Fold polynomial commitments [Fold_{r}^(0)] and [Fold_{-r}^(0)]
+        // Aggregate: 2d opening pairs and 2d Fold poly commitments into verifier claim
+        auto verifier_claims = GeminiVerifier::reduce_verification(u, claim_batcher, verifier_transcript);
+        // Check equality of the opening pairs computed by prover and verifier
+        for (auto [prover_claim, verifier_claim] : zip_view(prover_claims_with_pos_evals, verifier_claims)) {
+            this->verify_opening_claim(verifier_claim, prover_claim.polynomial, ck);
+            ASSERT_EQ(prover_claim.opening_pair, verifier_claim.opening_pair);
+        }
+    }
+}
+
+;
 
 using ParamsTypes = ::testing::Types<curve::BN254, curve::Grumpkin>;
 TYPED_TEST_SUITE(GeminiTest, ParamsTypes);
@@ -144,6 +221,10 @@ TYPED_TEST(GeminiTest, DoubleWithShiftAndInterleaving)
     this->execute_gemini_and_verify_claims(u, mock_claims);
 }
 
+TYPED_TEST(GeminiTest, OpenExtensionByZero)
+{
+    TestFixture::open_extension_by_zero();
+}
 /**
  * @brief Implementation of the [attack described by Ariel](https://hackmd.io/zm5SDfBqTKKXGpI-zQHtpA?view).
  *
@@ -247,8 +328,8 @@ TYPED_TEST(GeminiTest, SoundnessRegression)
         EXPECT_TRUE(prover_opening_claims[idx].opening_pair == verifier_claims[idx].opening_pair);
     }
 
-    // The mismatch in claims below leads to Gemini and Shplemini Verifier rejecting the tampered proof and confirms the
-    // necessity of opening `fold_i` at r^{2^i} for i = 1, ..., log_n - 1.
+    // The mismatch in claims below leads to Gemini and Shplemini Verifier rejecting the tampered proof and confirms
+    // the necessity of opening `fold_i` at r^{2^i} for i = 1, ..., log_n - 1.
     for (auto idx : mismatching_claim_indices) {
         EXPECT_FALSE(prover_opening_claims[idx].opening_pair == verifier_claims[idx].opening_pair);
     }

barretenberg/cpp/src/barretenberg/commitment_schemes/gemini/gemini_impl.hpp

@@ -80,16 +80,13 @@ std::vector<typename GeminiProver_<Curve>::Claim> GeminiProver_<Curve>::prove(
     Polynomial A_0 = polynomial_batcher.compute_batched(rho, running_scalar);
 
     // Construct the d-1 Gemini foldings of A₀(X)
-    std::vector<Polynomial> fold_polynomials = compute_fold_polynomials(log_n, multilinear_challenge, A_0);
+    std::vector<Polynomial> fold_polynomials = compute_fold_polynomials(log_n, multilinear_challenge, A_0, has_zk);
 
     // If virtual_log_n >= log_n, pad the fold commitments with dummy group elements [1]_1.
     for (size_t l = 0; l < virtual_log_n - 1; l++) {
         std::string label = "Gemini:FOLD_" + std::to_string(l + 1);
-        if (l < log_n - 1) {
-            transcript->send_to_verifier(label, commitment_key.commit(fold_polynomials[l]));
-        } else {
-            transcript->send_to_verifier(label, Commitment::one());
-        }
+        // When has_zk is true, we are sending commitments to 0. Seems to work, but maybe brittle.
+        transcript->send_to_verifier(label, commitment_key.commit(fold_polynomials[l]));
     }
     const Fr r_challenge = transcript->template get_challenge<Fr>("Gemini:r");
 
@@ -106,16 +103,11 @@ std::vector<typename GeminiProver_<Curve>::Claim> GeminiProver_<Curve>::prove(
     auto [A_0_pos, A_0_neg] = polynomial_batcher.compute_partially_evaluated_batch_polynomials(r_challenge);
     // Construct claims for the d + 1 univariate evaluations A₀₊(r), A₀₋(-r), and Foldₗ(−r^{2ˡ}), l = 1, ..., d-1
     std::vector<Claim> claims = construct_univariate_opening_claims(
-        log_n, std::move(A_0_pos), std::move(A_0_neg), std::move(fold_polynomials), r_challenge);
+        virtual_log_n, std::move(A_0_pos), std::move(A_0_neg), std::move(fold_polynomials), r_challenge);
 
-    // If virtual_log_n >= log_n, pad the negative fold evaluations with zeroes.
     for (size_t l = 1; l <= virtual_log_n; l++) {
         std::string label = "Gemini:a_" + std::to_string(l);
-        if (l <= log_n) {
-            transcript->send_to_verifier(label, claims[l].opening_pair.evaluation);
-        } else {
-            transcript->send_to_verifier(label, Fr::zero());
-        }
+        transcript->send_to_verifier(label, claims[l].opening_pair.evaluation);
     }
 
     // If running Gemini for the Translator VM polynomials, A₀(r) = A₀₊(r) + P₊(rˢ) and A₀(-r) = A₀₋(-r) + P₋(rˢ)
@@ -145,15 +137,18 @@ std::vector<typename GeminiProver_<Curve>::Claim> GeminiProver_<Curve>::prove(
  */
 template <typename Curve>
 std::vector<typename GeminiProver_<Curve>::Polynomial> GeminiProver_<Curve>::compute_fold_polynomials(
-    const size_t log_n, std::span<const Fr> multilinear_challenge, const Polynomial& A_0)
+    const size_t log_n, std::span<const Fr> multilinear_challenge, const Polynomial& A_0, const bool& has_zk)
 {
     const size_t num_threads = get_num_cpus_pow2();
+
+    const size_t virtual_log_n = multilinear_challenge.size();
+
     constexpr size_t efficient_operations_per_thread = 64; // A guess of the number of operation for which there
                                                            // would be a point in sending them to a separate thread
 
     // Reserve and allocate space for m-1 Fold polynomials, the foldings of the full batched polynomial A₀
     std::vector<Polynomial> fold_polynomials;
-    fold_polynomials.reserve(log_n - 1);
+    fold_polynomials.reserve(virtual_log_n - 1);
     for (size_t l = 0; l < log_n - 1; ++l) {
         // size of the previous polynomial/2
         const size_t n_l = 1 << (log_n - l - 1);
@@ -198,6 +193,28 @@ std::vector<typename GeminiProver_<Curve>::Polynomial> GeminiProver_<Curve>::com
         A_l = A_l_fold;
     }
 
+    // Perform virtual rounds.
+    // After the first `log_n - 1` rounds, the prover's `fold` univariates stabilize. With ZK, the verifier multiplies
+    // the evaluations by 0, otherwise, when `virtual_log_n > log_n`, the prover honestly computes and sends the
+    // constant folds.
+    const auto& last = fold_polynomials.back();
+    const Fr u_last = multilinear_challenge[log_n - 1];
+    const Fr final_eval = last.at(0) + u_last * (last.at(1) - last.at(0));
+    Polynomial const_fold(1);
+    // Temporary fix: when we're running a zk proof, the verifier uses a `padding_indicator_array`. So the evals in
+    // rounds past `log_n - 1` will be ignored. Hence the prover also needs to ignore them, otherwise Shplonk will fail.
+    const_fold.at(0) = final_eval * Fr(static_cast<int>(!has_zk));
+    fold_polynomials.emplace_back(const_fold);
+
+    // FOLD_{log_n+1}, ..., FOLD_{d_v-1}
+    Fr tail = Fr(1);
+    for (size_t k = log_n; k < virtual_log_n - 1; ++k) {
+        tail *= (Fr(1) - multilinear_challenge[k]); // multiply by (1 - u_k)
+        Polynomial next_const(1);
+        next_const.at(0) = final_eval * tail * Fr(static_cast<int>(!has_zk));
+        fold_polynomials.emplace_back(next_const);
+    }
+
     return fold_polynomials;
 };
 

barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.test.cpp

@@ -288,7 +288,7 @@ TYPED_TEST(ShpleminiTest, ShpleminiZKNoSumcheckOpenings)
     // Generate Libra polynomials, compute masked concatenated Libra polynomial, commit to it
     ZKData zk_sumcheck_data(this->log_n, prover_transcript, ck);
 
-    // Generate multivariate challenge of size CONST_PROOF_SIZE_LOG_N
+    // Generate multivariate challenge
     std::vector<Fr> mle_opening_point = this->random_evaluation_point(this->log_n);
 
     // Generate random prover polynomials, compute their evaluations and commitments

barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplonk.hpp

@@ -93,8 +93,8 @@ template <typename Curve> class ShplonkProver_ {
             current_nu *= nu;
         }
         // We use the same batching challenge for Gemini and Libra opening claims. The number of the claims
-        // batched before adding Libra commitments and evaluations is bounded by 2 * CONST_PROOF_SIZE_LOG_N + 2, where
-        // 2 * CONST_PROOF_SIZE_LOG_N is the number of fold claims including the dummy ones, and +2 is reserved for
+        // batched before adding Libra commitments and evaluations is bounded by 2 * `virtual_log_n` + 2, where
+        // 2 * `virtual_log_n` is the number of fold claims including the dummy ones, and +2 is reserved for
         // interleaving.
         if (!libra_opening_claims.empty()) {
             current_nu = nu.pow(2 * virtual_log_n + NUM_INTERLEAVING_CLAIMS);

barretenberg/cpp/src/barretenberg/commitment_schemes/utils/mock_witness_generator.hpp

@@ -87,15 +87,36 @@ template <typename Curve> struct MockClaimGenerator {
         , polynomial_batcher(poly_size)
 
     {
+        size_t log_size = numeric::get_msb(poly_size);
+        // If the size of the opening point is bigger than the log of the poly size, we assume that the prover is
+        // extending all of its polynomials by zero outside of the hypercube of size 2^{log_size}.
+        bool has_virtual_rounds = (mle_opening_point.size() > log_size);
+
+        std::span<const Fr> challenge;
+
+        if (has_virtual_rounds) {
+            // The evaluation on the full domain can be obtain by scaling by extension-by-zero factor `ebz_factor`
+            // computed below.
+            challenge = std::span<const Fr>(mle_opening_point).subspan(0, log_size);
+        } else {
+            challenge = std::span<const Fr>(mle_opening_point);
+        }
+
         const size_t total_num_to_be_shifted = num_to_be_shifted + num_to_be_right_shifted_by_k;
         BB_ASSERT_GTE(num_polynomials, total_num_to_be_shifted);
         const size_t num_not_to_be_shifted = num_polynomials - total_num_to_be_shifted;
 
+        Fr ebz_factor = 1;
+
+        for (size_t idx = log_size; idx < mle_opening_point.size(); idx++) {
+            ebz_factor *= (Fr(1) - mle_opening_point[idx]);
+        }
+
         // Construct claim data for polynomials that are NOT to be shifted
         for (size_t idx = 0; idx < num_not_to_be_shifted; idx++) {
             Polynomial poly = Polynomial::random(poly_size);
             unshifted.commitments.push_back(ck.commit(poly));
-            unshifted.evals.push_back(poly.evaluate_mle(mle_opening_point));
+            unshifted.evals.push_back(poly.evaluate_mle(challenge) * ebz_factor);
             unshifted.polys.push_back(std::move(poly));
         }
 
@@ -104,11 +125,11 @@ template <typename Curve> struct MockClaimGenerator {
             Polynomial poly = Polynomial::random(poly_size, /*shiftable*/ 1);
             Commitment commitment = ck.commit(poly);
             to_be_shifted.commitments.push_back(commitment);
-            to_be_shifted.evals.push_back(poly.shifted().evaluate_mle(mle_opening_point));
+            to_be_shifted.evals.push_back(poly.shifted().evaluate_mle(challenge) * ebz_factor);
             to_be_shifted.polys.push_back(poly.share());
             // Populate the unshifted counterpart in the unshifted claims
             unshifted.commitments.push_back(commitment);
-            unshifted.evals.push_back(poly.evaluate_mle(mle_opening_point));
+            unshifted.evals.push_back(poly.evaluate_mle(challenge) * ebz_factor);
             unshifted.polys.push_back(std::move(poly));
         }
 
@@ -117,11 +138,12 @@ template <typename Curve> struct MockClaimGenerator {
             Polynomial poly = Polynomial::random(poly_size - k_magnitude, poly_size, 0);
             Commitment commitment = ck.commit(poly);
             to_be_right_shifted_by_k.commitments.push_back(commitment);
-            to_be_right_shifted_by_k.evals.push_back(poly.right_shifted(k_magnitude).evaluate_mle(mle_opening_point));
+            to_be_right_shifted_by_k.evals.push_back(poly.right_shifted(k_magnitude).evaluate_mle(challenge) *
+                                                     ebz_factor);
             to_be_right_shifted_by_k.polys.push_back(poly.share());
             // Populate the unshifted counterpart in the unshifted claims
             unshifted.commitments.push_back(commitment);
-            unshifted.evals.push_back(poly.evaluate_mle(mle_opening_point));
+            unshifted.evals.push_back(poly.evaluate_mle(challenge) * ebz_factor);
             unshifted.polys.push_back(std::move(poly));
         }
 

barretenberg/cpp/src/barretenberg/commitment_schemes_recursion/shplemini.test.cpp

@@ -48,14 +48,11 @@ TEST(ShpleminiRecursionTest, ProveAndVerifySingle)
 
     bb::srs::init_file_crs_factory(bb::srs::bb_crs_path());
     auto run_shplemini = [](size_t log_circuit_size) {
-        using diff_t = std::vector<NativeFr>::difference_type;
-
         size_t N = 1 << log_circuit_size;
-        const auto padding_indicator_array =
-            stdlib::compute_padding_indicator_array<Curve, CONST_PROOF_SIZE_LOG_N>(log_circuit_size);
+        const std::vector<Fr> padding_indicator_array(CONST_PROOF_SIZE_LOG_N, 1);
         constexpr size_t NUM_POLYS = 5;
         constexpr size_t NUM_SHIFTED = 2;
-        constexpr size_t NUM_RIGHT_SHIFTED_BY_K = 1;
+        constexpr size_t NUM_RIGHT_SHIFTED_BY_K = 0;
 
         CommitmentKey commitment_key(16384);
 
@@ -65,12 +62,8 @@ TEST(ShpleminiRecursionTest, ProveAndVerifySingle)
             u_challenge.emplace_back(NativeFr::random_element(&shplemini_engine));
         };
 
-        // Truncate to real size to create mock claims.
-        std::vector<NativeFr> truncated_u_challenge(u_challenge.begin(),
-                                                    u_challenge.begin() + static_cast<diff_t>(log_circuit_size));
         // Construct mock multivariate polynomial opening claims
-        MockClaimGen mock_claims(
-            N, NUM_POLYS, NUM_SHIFTED, NUM_RIGHT_SHIFTED_BY_K, truncated_u_challenge, commitment_key);
+        MockClaimGen mock_claims(N, NUM_POLYS, NUM_SHIFTED, NUM_RIGHT_SHIFTED_BY_K, u_challenge, commitment_key);
 
         // Initialize an empty NativeTranscript
         auto prover_transcript = NativeTranscript::prover_init_empty();