Skip to content

Commit f60d2c5

Browse files
authored
feat: merge-train/barretenberg (#18105)
BEGIN_COMMIT_OVERRIDE chore!: improve poly allocation in ZK setting (#18067) END_COMMIT_OVERRIDE
2 parents fca9b22 + d53bd3d commit f60d2c5

File tree

8 files changed

+130
-120
lines changed

8 files changed

+130
-120
lines changed

barretenberg/cpp/bootstrap.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,8 +305,10 @@ function build_bench {
305305
function bench_cmds {
306306
prefix="$hash:CPUS=8"
307307
echo "$prefix barretenberg/cpp/scripts/run_bench.sh native bb-micro-bench/native/ultra_honk build/bin/ultra_honk_bench construct_proof_ultrahonk_power_of_2/20$"
308+
echo "$prefix barretenberg/cpp/scripts/run_bench.sh native bb-micro-bench/native/ultra_honk_zk build/bin/ultra_honk_bench construct_proof_ultrahonk_zk_power_of_2/20$"
308309
echo "$prefix barretenberg/cpp/scripts/run_bench.sh native bb-micro-bench/native/chonk build/bin/chonk_bench ChonkBench/Full/5$"
309310
echo "$prefix barretenberg/cpp/scripts/run_bench.sh wasm bb-micro-bench/wasm/ultra_honk build-wasm-threads/bin/ultra_honk_bench construct_proof_ultrahonk_power_of_2/20$"
311+
echo "$prefix barretenberg/cpp/scripts/run_bench.sh wasm bb-micro-bench/wasm/ultra_honk_zk build-wasm-threads/bin/ultra_honk_bench construct_proof_ultrahonk_zk_power_of_2/20$"
310312
echo "$prefix barretenberg/cpp/scripts/run_bench.sh wasm bb-micro-bench/wasm/chonk build-wasm-threads/bin/chonk_bench ChonkBench/Full/5$"
311313
prefix="$hash:CPUS=1"
312314
echo "$prefix barretenberg/cpp/scripts/run_bench.sh native bb-micro-bench/native/chonk_verify build/bin/chonk_bench VerificationOnly$"

barretenberg/cpp/scripts/test_chonk_standalone_vks_havent_changed.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ cd ..
1313
# - Generate a hash for versioning: sha256sum bb-chonk-inputs.tar.gz
1414
# - Upload the compressed results: aws s3 cp bb-chonk-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-chonk-inputs-[hash(0:8)].tar.gz
1515
# Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
16-
pinned_short_hash="84c467db"
16+
pinned_short_hash="07b5558d"
1717
pinned_chonk_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-chonk-inputs-${pinned_short_hash}.tar.gz"
1818

1919
function compress_and_upload {

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/ultra_honk.bench.cpp

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,16 @@ static void construct_proof_ultrahonk_power_of_2(State& state) noexcept
2727
state, &bb::mock_circuits::generate_basic_arithmetic_circuit<UltraCircuitBuilder>, log2_of_gates);
2828
}
2929

30+
/**
31+
* @brief Benchmark: Construction of a Ultra Honk ZK proof with 2**n gates
32+
*/
33+
static void construct_proof_ultrahonk_zk_power_of_2(State& state) noexcept
34+
{
35+
auto log2_of_gates = static_cast<size_t>(state.range(0));
36+
bb::mock_circuits::construct_proof_with_specified_num_iterations<UltraZKProver>(
37+
state, &bb::mock_circuits::generate_basic_arithmetic_circuit<UltraCircuitBuilder>, log2_of_gates);
38+
}
39+
3040
// Define benchmarks
3141
BENCHMARK_CAPTURE(construct_proof_ultrahonk, sha256, &generate_sha256_test_circuit<UltraCircuitBuilder>)
3242
->Unit(kMillisecond);
@@ -42,4 +52,9 @@ BENCHMARK(construct_proof_ultrahonk_power_of_2)
4252
->DenseRange(15, 20)
4353
->Unit(kMillisecond);
4454

55+
BENCHMARK(construct_proof_ultrahonk_zk_power_of_2)
56+
// 2**15 gates to 2**20 gates
57+
->DenseRange(15, 20)
58+
->Unit(kMillisecond);
59+
4560
BENCHMARK_MAIN();

barretenberg/cpp/src/barretenberg/honk/composer/composer_lib.hpp

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -14,33 +14,27 @@
1414

1515
namespace bb {
1616

17+
/**
18+
* @brief Construct polynomials containing the concatenation of all lookup tables used in the circuit.
19+
* @details The first three polynomials contain the table data, while the fourth polynomial contains a table index that
20+
* simply reflects the order in which the tables were added to the circuit.
21+
*
22+
* @tparam Flavor
23+
* @param table_polynomials Polynomials to populate with table data
24+
* @param circuit
25+
*/
1726
template <typename Flavor>
1827
void construct_lookup_table_polynomials(const RefArray<typename Flavor::Polynomial, 4>& table_polynomials,
19-
const typename Flavor::CircuitBuilder& circuit,
20-
const size_t dyadic_circuit_size,
21-
const size_t additional_offset = 0)
28+
const typename Flavor::CircuitBuilder& circuit)
2229
{
23-
// Create lookup selector polynomials which interpolate each table column.
24-
// Our selector polys always need to interpolate the full subgroup size, so here we offset so as to
25-
// put the table column's values at the end. (The first gates are for non-lookup constraints).
26-
// [0, ..., 0, ...table, 0, 0, 0, x]
27-
// ^^^^^^^^^ ^^^^^^^^ ^^^^^^^ ^nonzero to ensure uniqueness and to avoid infinity commitments
28-
// | table randomness
29-
// ignored, as used for regular constraints and padding to the next power of 2.
30-
// TODO(https://github.com/AztecProtocol/barretenberg/issues/1033): construct tables and counts at top of trace
31-
const size_t tables_size = circuit.get_tables_size();
32-
BB_ASSERT_GT(dyadic_circuit_size, tables_size + additional_offset);
33-
size_t offset = circuit.blocks.lookup.trace_offset();
34-
30+
size_t offset = 0;
3531
for (const auto& table : circuit.lookup_tables) {
36-
const fr table_index(table.table_index);
37-
3832
for (size_t i = 0; i < table.size(); ++i) {
3933
table_polynomials[0].at(offset) = table.column_1[i];
4034
table_polynomials[1].at(offset) = table.column_2[i];
4135
table_polynomials[2].at(offset) = table.column_3[i];
42-
table_polynomials[3].at(offset) = table_index;
43-
++offset;
36+
table_polynomials[3].at(offset) = table.table_index;
37+
offset++;
4438
}
4539
}
4640
}
@@ -55,13 +49,10 @@ void construct_lookup_table_polynomials(const RefArray<typename Flavor::Polynomi
5549
template <typename Flavor>
5650
void construct_lookup_read_counts(typename Flavor::Polynomial& read_counts,
5751
typename Flavor::Polynomial& read_tags,
58-
typename Flavor::CircuitBuilder& circuit,
59-
[[maybe_unused]] const size_t dyadic_circuit_size)
52+
typename Flavor::CircuitBuilder& circuit)
6053
{
61-
// TODO(https://github.com/AztecProtocol/barretenberg/issues/1033): construct tables and counts at top of trace
62-
size_t table_offset = circuit.blocks.lookup.trace_offset();
63-
6454
// loop over all tables used in the circuit; each table contains data about the lookups made on it
55+
size_t table_offset = 0;
6556
for (auto& table : circuit.lookup_tables) {
6657
table.initialize_index_map();
6758

barretenberg/cpp/src/barretenberg/honk/composer/composer_lib.test.cpp

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,24 +53,21 @@ TEST_F(ComposerLibTests, LookupReadCounts)
5353
Polynomial read_tags{ circuit_size };
5454

5555
builder.blocks.compute_offsets();
56-
construct_lookup_read_counts<Flavor>(read_counts, read_tags, builder, circuit_size);
57-
58-
// The table polys are constructed at the start of the lookup gates block, thus so to are the counts/tags
59-
size_t offset = builder.blocks.lookup.trace_offset();
56+
construct_lookup_read_counts<Flavor>(read_counts, read_tags, builder);
6057

6158
// The uint32 XOR lookup table is constructed for 6 bit operands via double for loop that iterates through the left
6259
// operand externally (0 to 63) then the right operand internally (0 to 63). Computing (1 XOR 5) will thus result in
6360
// 1 lookup from the (1*64 + 5)th index in the table and 4 lookups from the (0*64 + 0)th index (for the remaining 4
6461
// 6-bits limbs that are all 0) and one lookup from second table from the (64 * 64 + 0) index (for last 2 bits).
6562
// The counts and tags at all other indices should be zero.
6663
for (auto [idx, count, tag] : zip_polys(read_counts, read_tags)) {
67-
if (idx == (0 + offset)) {
64+
if (idx == (0)) {
6865
EXPECT_EQ(count, 4);
6966
EXPECT_EQ(tag, 1);
70-
} else if (idx == (69 + offset)) {
67+
} else if (idx == (69)) {
7168
EXPECT_EQ(count, 1);
7269
EXPECT_EQ(tag, 1);
73-
} else if (idx == (64 * 64 + offset)) {
70+
} else if (idx == (64 * 64)) {
7471
EXPECT_EQ(count, 1);
7572
EXPECT_EQ(tag, 1);
7673
} else {

barretenberg/cpp/src/barretenberg/ultra_honk/prover_instance.cpp

Lines changed: 64 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,26 @@
1414
namespace bb {
1515

1616
/**
17-
* @brief Helper method to compute quantities like total number of gates and dyadic circuit size
17+
* @brief Compute the minimum dyadic (power-of-2) circuit size
18+
* @details The dyadic circuit size is the smallest power of two which can accommodate all polynomials required for the
19+
* proving system. This size must account for the execution trace itself, i.e. the wires/selectors, but also any
20+
* auxiliary polynomials like those that store the table data for lookup arguments.
1821
*
1922
* @tparam Flavor
2023
* @param circuit
2124
*/
2225
template <IsUltraOrMegaHonk Flavor> size_t ProverInstance_<Flavor>::compute_dyadic_size(Circuit& circuit)
2326
{
24-
// for the lookup argument the circuit size must be at least as large as the sum of all tables used
25-
const size_t min_size_due_to_lookups = circuit.get_tables_size();
27+
// For the lookup argument the circuit size must be at least as large as the sum of all tables used
28+
const size_t tables_size = circuit.get_tables_size();
2629

2730
// minimum size of execution trace due to everything else
2831
size_t min_size_of_execution_trace = circuit.blocks.get_total_content_size();
2932

3033
// The number of gates is the maximum required by the lookup argument or everything else, plus an optional zero row
3134
// to allow for shifts.
3235
size_t total_num_gates =
33-
NUM_DISABLED_ROWS_IN_SUMCHECK + num_zero_rows + std::max(min_size_due_to_lookups, min_size_of_execution_trace);
36+
NUM_DISABLED_ROWS_IN_SUMCHECK + num_zero_rows + std::max(tables_size, min_size_of_execution_trace);
3437

3538
// Next power of 2 (dyadic circuit size)
3639
return circuit.get_circuit_subgroup_size(total_num_gates);
@@ -40,24 +43,29 @@ template <IsUltraOrMegaHonk Flavor> void ProverInstance_<Flavor>::allocate_wires
4043
{
4144
BB_BENCH_NAME("allocate_wires");
4245

43-
// Allocate only enough memory for the active range; wires are zero outside this range
46+
// If no ZK, allocate only the active range of the trace; else allocate full dyadic size to allow for blinding
47+
const size_t wire_size = Flavor::HasZK ? dyadic_size() : trace_active_range_size();
48+
4449
for (auto& wire : polynomials.get_wires()) {
45-
wire = Polynomial::shiftable(final_active_wire_idx + 1, dyadic_size());
50+
wire = Polynomial::shiftable(wire_size, dyadic_size());
4651
}
4752
}
4853

4954
template <IsUltraOrMegaHonk Flavor> void ProverInstance_<Flavor>::allocate_permutation_argument_polynomials()
5055
{
5156
BB_BENCH_NAME("allocate_permutation_argument_polynomials");
5257

53-
// Allocate only enough memory for the active range; permutation polynomials are zero outside this range
58+
// Sigma and ID polynomials are zero outside the active trace range
5459
for (auto& sigma : polynomials.get_sigmas()) {
55-
sigma = Polynomial::shiftable(final_active_wire_idx + 1, dyadic_size());
60+
sigma = Polynomial::shiftable(trace_active_range_size(), dyadic_size());
5661
}
5762
for (auto& id : polynomials.get_ids()) {
58-
id = Polynomial::shiftable(final_active_wire_idx + 1, dyadic_size());
63+
id = Polynomial::shiftable(trace_active_range_size(), dyadic_size());
5964
}
60-
polynomials.z_perm = Polynomial::shiftable(final_active_wire_idx + 1, dyadic_size());
65+
66+
// If no ZK, allocate only the active range of the trace; else allocate full dyadic size to allow for blinding
67+
const size_t z_perm_size = Flavor::HasZK ? dyadic_size() : trace_active_range_size();
68+
polynomials.z_perm = Polynomial::shiftable(z_perm_size, dyadic_size());
6169
}
6270

6371
template <IsUltraOrMegaHonk Flavor> void ProverInstance_<Flavor>::allocate_lagrange_polynomials()
@@ -91,31 +99,27 @@ void ProverInstance_<Flavor>::allocate_table_lookup_polynomials(const Circuit& c
9199
{
92100
BB_BENCH_NAME("allocate_table_lookup_and_lookup_read_polynomials");
93101

94-
size_t table_offset = circuit.blocks.lookup.trace_offset();
95-
const size_t tables_size = circuit.get_tables_size(); // cumulative size of all lookup tables used in the circuit
96-
BB_ASSERT_GT(dyadic_size(), tables_size);
102+
const size_t tables_size = circuit.get_tables_size(); // cumulative size of all lookup tables
97103

98-
// Allocate the polynomials containing the actual table data
99-
for (auto& poly : polynomials.get_tables()) {
100-
poly = Polynomial(tables_size, dyadic_size(), table_offset);
104+
// Allocate polynomials containing the actual table data; offset to align with the lookup gate block
105+
BB_ASSERT_GT(dyadic_size(), tables_size);
106+
for (auto& table_poly : polynomials.get_tables()) {
107+
table_poly = Polynomial(tables_size, dyadic_size());
101108
}
102109

103-
// Allocate the read counts and tags polynomials
104-
polynomials.lookup_read_counts = Polynomial(tables_size, dyadic_size(), table_offset);
105-
polynomials.lookup_read_tags = Polynomial(tables_size, dyadic_size(), table_offset);
110+
// Read counts and tags: track which table entries have been read
111+
// For non-ZK, allocate just the table size; for ZK: allocate fulll dyadic_size
112+
const size_t counts_and_tags_size = Flavor::HasZK ? dyadic_size() : tables_size;
113+
polynomials.lookup_read_counts = Polynomial(counts_and_tags_size, dyadic_size());
114+
polynomials.lookup_read_tags = Polynomial(counts_and_tags_size, dyadic_size());
106115

107-
// Determine end index for the lookup block and the tables themselves
108-
// Note that the start of the tables is aligned with the start of the lookup block in the trace
109-
const size_t lookup_block_end =
110-
static_cast<size_t>(circuit.blocks.lookup.trace_offset()) + circuit.blocks.lookup.size();
111-
const auto tables_end = circuit.blocks.lookup.trace_offset() + tables_size;
116+
// Lookup inverses: used in the log-derivative lookup argument
117+
// Must cover both the lookup gate block (where reads occur) and the table data itself
118+
const size_t lookup_block_end = circuit.blocks.lookup.trace_offset() + circuit.blocks.lookup.size();
119+
const size_t lookup_inverses_end = std::max(lookup_block_end, tables_size);
112120

113-
// Allocate the lookup_inverses polynomial
114-
const size_t lookup_inverses_start = table_offset;
115-
const size_t lookup_inverses_end = std::max(lookup_block_end, tables_end);
116-
117-
polynomials.lookup_inverses =
118-
Polynomial(lookup_inverses_end - lookup_inverses_start, dyadic_size(), lookup_inverses_start);
121+
const size_t lookup_inverses_size = (Flavor::HasZK ? dyadic_size() : lookup_inverses_end);
122+
polynomials.lookup_inverses = Polynomial(lookup_inverses_size, dyadic_size());
119123
}
120124

121125
template <IsUltraOrMegaHonk Flavor>
@@ -125,6 +129,7 @@ void ProverInstance_<Flavor>::allocate_ecc_op_polynomials(const Circuit& circuit
125129
BB_BENCH_NAME("allocate_ecc_op_polynomials");
126130

127131
// Allocate the ecc op wires and selector
132+
// Note: ECC op wires are not blinded directly so we do not need to allocate full dyadic size for ZK
128133
const size_t ecc_op_block_size = circuit.blocks.ecc_op.size();
129134
for (auto& wire : polynomials.get_ecc_op_wires()) {
130135
wire = Polynomial(ecc_op_block_size, dyadic_size());
@@ -139,36 +144,41 @@ void ProverInstance_<Flavor>::allocate_databus_polynomials(const Circuit& circui
139144
BB_BENCH_NAME("allocate_databus_and_lookup_inverse_polynomials");
140145

141146
const size_t calldata_size = circuit.get_calldata().size();
142-
const size_t secondary_calldata_size = circuit.get_secondary_calldata().size();
147+
const size_t sec_calldata_size = circuit.get_secondary_calldata().size();
143148
const size_t return_data_size = circuit.get_return_data().size();
144149

145-
polynomials.calldata = Polynomial(calldata_size, dyadic_size());
146-
polynomials.calldata_read_counts = Polynomial(calldata_size, dyadic_size());
147-
polynomials.calldata_read_tags = Polynomial(calldata_size, dyadic_size());
150+
// Allocate only enough space for the databus data; for ZK, allocate full dyadic size
151+
const size_t calldata_poly_size = Flavor::HasZK ? dyadic_size() : calldata_size;
152+
const size_t sec_calldata_poly_size = Flavor::HasZK ? dyadic_size() : sec_calldata_size;
153+
const size_t return_data_poly_size = Flavor::HasZK ? dyadic_size() : return_data_size;
148154

149-
polynomials.secondary_calldata = Polynomial(secondary_calldata_size, dyadic_size());
150-
polynomials.secondary_calldata_read_counts = Polynomial(secondary_calldata_size, dyadic_size());
151-
polynomials.secondary_calldata_read_tags = Polynomial(secondary_calldata_size, dyadic_size());
155+
polynomials.calldata = Polynomial(calldata_poly_size, dyadic_size());
156+
polynomials.calldata_read_counts = Polynomial(calldata_poly_size, dyadic_size());
157+
polynomials.calldata_read_tags = Polynomial(calldata_poly_size, dyadic_size());
152158

153-
polynomials.return_data = Polynomial(return_data_size, dyadic_size());
154-
polynomials.return_data_read_counts = Polynomial(return_data_size, dyadic_size());
155-
polynomials.return_data_read_tags = Polynomial(return_data_size, dyadic_size());
159+
polynomials.secondary_calldata = Polynomial(sec_calldata_poly_size, dyadic_size());
160+
polynomials.secondary_calldata_read_counts = Polynomial(sec_calldata_poly_size, dyadic_size());
161+
polynomials.secondary_calldata_read_tags = Polynomial(sec_calldata_poly_size, dyadic_size());
156162

157-
// Allocate log derivative lookup argument inverse polynomials
158-
const size_t q_busread_end = circuit.blocks.busread.trace_offset() + circuit.blocks.busread.size();
163+
polynomials.return_data = Polynomial(return_data_poly_size, dyadic_size());
164+
polynomials.return_data_read_counts = Polynomial(return_data_poly_size, dyadic_size());
165+
polynomials.return_data_read_tags = Polynomial(return_data_poly_size, dyadic_size());
159166

160-
// TODO(https://github.com/AztecProtocol/barretenberg/issues/1555): Size of databus_id can always be set to max size
161-
// between the three databus columns. It currently uses dyadic_size because its values are later set based on its
162-
// size(). This means when we naively construct all ProverPolynomials with dyadic size (e.g. for ZK), we get a
163-
// different databus_id polynomial and therefore a different VK.
164-
polynomials.databus_id = Polynomial(dyadic_size(), dyadic_size());
165-
// polynomials.databus_id = Polynomial(std::max({ calldata_size, secondary_calldata_size, return_data_size,
166-
// q_busread_end }), dyadic_size());
167-
168-
polynomials.calldata_inverses = Polynomial(std::max(calldata_size, q_busread_end), dyadic_size());
169-
polynomials.secondary_calldata_inverses =
170-
Polynomial(std::max(secondary_calldata_size, q_busread_end), dyadic_size());
171-
polynomials.return_data_inverses = Polynomial(std::max(return_data_size, q_busread_end), dyadic_size());
167+
// Databus lookup inverses: used in the log-derivative lookup argument
168+
// Must cover both the databus gate block (where reads occur) and the databus data itself
169+
const size_t q_busread_end = circuit.blocks.busread.trace_offset() + circuit.blocks.busread.size();
170+
size_t calldata_inverses_size = Flavor::HasZK ? dyadic_size() : std::max(calldata_size, q_busread_end);
171+
size_t sec_calldata_inverses_size = Flavor::HasZK ? dyadic_size() : std::max(sec_calldata_size, q_busread_end);
172+
size_t return_data_inverses_size = Flavor::HasZK ? dyadic_size() : std::max(return_data_size, q_busread_end);
173+
174+
polynomials.calldata_inverses = Polynomial(calldata_inverses_size, dyadic_size());
175+
polynomials.secondary_calldata_inverses = Polynomial(sec_calldata_inverses_size, dyadic_size());
176+
polynomials.return_data_inverses = Polynomial(return_data_inverses_size, dyadic_size());
177+
178+
// TODO(https://github.com/AztecProtocol/barretenberg/issues/1555): Allocate minimum size >1 to avoid point at
179+
// infinity commitment.
180+
const size_t max_databus_column_size = std::max({ calldata_size, sec_calldata_size, return_data_size, 2UL });
181+
polynomials.databus_id = Polynomial(max_databus_column_size, dyadic_size());
172182
}
173183

174184
/**

0 commit comments

Comments
 (0)