Skip to content

Commit f2279e5

Browse files
AztecBotAztecBot
authored
feat(avm): separate accumulate instantiations to own cpp (#16480)
This PR separates all template instantiations of `Relation::accummulate`​ to their own cpp file. This improves paralellizability and removes the load on files like `prover.cpp`​, `verifier.cpp`​, etc. For relations - `generated/relations/{{relation}}.hpp`​ has only the declaration of `accumulate`​. - `generated/relations/{{relation}}_impl.hpp` has only the definition of `accumulate`​. - `generated/relations/{{relation}}.cpp`​ has (almost all) the instantiations of `accumulate`​. - `generated/relations/relation_impls.hpp` includes all impls (used only in benches and possibly tests). For lookups - `constraining/relations/interactions_base.hpp` has only the declaration. - `constraining/relations/interactions_base_impl.hpp` has only the definition. - `generated/relations/lookups_{{relation}}.cpp`​ instantiates the definition. For permutations (to be done, I have to mirror what I did for lookups). Results - ARM compilation time goes down from 21m to 10m (see note later) - x86 time gets reduced by ~1m - Memory during compilation now peaks briefly at 58GB, but otherwise mostly doesnt use much memory. For ARM, now the poseidon perm cpp becomes the bottleneck, but this will be solved by Ilyas’ PR. So the time will go down to 9m. If we find some other improvements for the recursive verifier, we might be able to take it down to 5m and ~parity with x86. ![image.png](https://app.graphite.dev/user-attachments/assets/61c88e85-8487-42ae-b586-62a76806bb8e.png)
2 parents 5692041 + 749d1ab commit f2279e5

File tree

245 files changed

+19566
-15059
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

245 files changed

+19566
-15059
lines changed

‎barretenberg/cpp/src/barretenberg/vm2/constraining/benchmark/relations_acc.bench.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
#include "barretenberg/vm2/constraining/flavor.hpp"
1212
#include "barretenberg/vm2/generated/columns.hpp"
1313

14+
#include "barretenberg/vm2/constraining/relations/interactions_base_impl.hpp"
15+
#include "barretenberg/vm2/generated/relations/relations_impls.hpp"
16+
1417
using namespace benchmark;
1518
using namespace bb::avm2;
1619

‎barretenberg/cpp/src/barretenberg/vm2/constraining/recursion/recursive_flavor.hpp‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,19 @@
66
#include "barretenberg/stdlib/transcript/transcript.hpp"
77
#include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp"
88
#include "barretenberg/vm2/constraining/flavor.hpp"
9+
#include "barretenberg/vm2/constraining/recursion/recursive_flavor_settings.hpp"
910

1011
namespace bb::avm2 {
1112

1213
class AvmRecursiveFlavor {
1314
public:
14-
using CircuitBuilder = MegaCircuitBuilder;
15-
using Curve = stdlib::bn254<CircuitBuilder>;
16-
using PCS = KZG<Curve>;
17-
using GroupElement = typename Curve::Element;
18-
using Commitment = typename Curve::AffineElement;
19-
using FF = typename Curve::ScalarField;
20-
using BF = typename Curve::BaseField;
15+
using CircuitBuilder = AvmRecursiveFlavorSettings::CircuitBuilder;
16+
using Curve = AvmRecursiveFlavorSettings::Curve;
17+
using PCS = AvmRecursiveFlavorSettings::PCS;
18+
using GroupElement = AvmRecursiveFlavorSettings::GroupElement;
19+
using Commitment = AvmRecursiveFlavorSettings::Commitment;
20+
using FF = AvmRecursiveFlavorSettings::FF;
21+
using BF = AvmRecursiveFlavorSettings::BF;
2122

2223
using NativeFlavor = avm2::AvmFlavor;
2324
using NativeVerificationKey = NativeFlavor::VerificationKey;

‎barretenberg/cpp/src/barretenberg/vm2/constraining/recursion/recursive_flavor_settings.hpp‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#pragma once
2+
3+
#include "barretenberg/commitment_schemes/kzg/kzg.hpp"
4+
#include "barretenberg/stdlib/primitives/curves/bn254.hpp"
5+
#include "barretenberg/stdlib_circuit_builders/mega_circuit_builder.hpp"
6+
7+
namespace bb::avm2 {
8+
9+
struct AvmRecursiveFlavorSettings {
10+
using CircuitBuilder = MegaCircuitBuilder;
11+
using Curve = stdlib::bn254<CircuitBuilder>;
12+
using PCS = KZG<Curve>;
13+
using GroupElement = typename Curve::Element;
14+
using Commitment = typename Curve::AffineElement;
15+
using FF = typename Curve::ScalarField;
16+
using BF = typename Curve::BaseField;
17+
};
18+
19+
} // namespace bb::avm2

‎barretenberg/cpp/src/barretenberg/vm2/constraining/relations/interactions_base.hpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,13 @@ template <typename FF_, typename Settings_> struct lookup_relation_base : public
7171
return (in.get(static_cast<ColumnAndShifts>(Settings::INVERSES))).is_zero();
7272
}
7373

74+
// This obscures the definition from GenericLookupRelation, so that we can instantiate it somewhere else.
75+
template <typename ContainerOverSubrelations, typename AllEntities, typename Parameters>
76+
static void accumulate(ContainerOverSubrelations& accumulator,
77+
const AllEntities& in,
78+
const Parameters& params,
79+
const FF_& scaling_factor);
80+
7481
static std::string get_subrelation_label(size_t index)
7582
{
7683
switch (index) {

‎barretenberg/cpp/src/barretenberg/vm2/constraining/relations/interactions_base_impl.hpp‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#pragma once
2+
3+
#include "barretenberg/vm2/constraining/relations/interactions_base.hpp"
4+
5+
namespace bb::avm2 {
6+
7+
/////////////////// LOOKUPS ///////////////////
8+
9+
template <typename FF_, typename Settings_>
10+
template <typename ContainerOverSubrelations, typename AllEntities, typename Parameters>
11+
void lookup_relation_base<FF_, Settings_>::accumulate(ContainerOverSubrelations& accumulator,
12+
const AllEntities& in,
13+
const Parameters& params,
14+
const FF_& scaling_factor)
15+
{
16+
GenericLookupRelationImpl<Settings_, FF_>::accumulate(accumulator, in, params, scaling_factor);
17+
}
18+
19+
} // namespace bb::avm2

‎barretenberg/cpp/src/barretenberg/vm2/generated/relations/address_derivation.cpp‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// AUTOGENERATED FILE
2+
#include "barretenberg/flavor/relation_definitions.hpp"
3+
#include "barretenberg/vm2/constraining/flavor.hpp"
4+
#include "barretenberg/vm2/constraining/full_row.hpp"
5+
#include "barretenberg/vm2/constraining/recursion/recursive_flavor.hpp"
6+
#include "barretenberg/vm2/generated/relations/address_derivation_impl.hpp"
7+
8+
#define AvmCheckCircuitEdge(Flavor) Flavor::PolynomialEntitiesAtFixedRow<Flavor::ProverPolynomials>
9+
#define AvmCheckRelationEdge(Flavor) ::bb::avm2::AvmFullRowProxy
10+
11+
namespace bb::avm2 {
12+
13+
template class address_derivationImpl<AvmFlavorSettings::FF>;
14+
ACCUMULATE(address_derivationImpl, AvmFlavor, SumcheckTupleOfUnivariatesOverSubrelations, ExtendedEdge); // Prover.
15+
ACCUMULATE(address_derivationImpl, AvmFlavor, SumcheckArrayOfValuesOverSubrelations, EvaluationEdge); // Verifier.
16+
ACCUMULATE(address_derivationImpl,
17+
AvmFlavor,
18+
SumcheckArrayOfValuesOverSubrelations,
19+
AvmCheckCircuitEdge); // Check circuit.
20+
ACCUMULATE(address_derivationImpl,
21+
AvmFlavor,
22+
SumcheckArrayOfValuesOverSubrelations,
23+
AvmCheckRelationEdge); // Check relation (tests).
24+
25+
template class address_derivationImpl<AvmRecursiveFlavor::FF>;
26+
ACCUMULATE(address_derivationImpl,
27+
AvmRecursiveFlavor,
28+
SumcheckArrayOfValuesOverSubrelations,
29+
EvaluationEdge); // Verifier.
30+
31+
} // namespace bb::avm2

‎barretenberg/cpp/src/barretenberg/vm2/generated/relations/address_derivation.hpp‎

Lines changed: 1 addition & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -27,63 +27,7 @@ template <typename FF_> class address_derivationImpl {
2727
void static accumulate(ContainerOverSubrelations& evals,
2828
const AllEntities& in,
2929
[[maybe_unused]] const RelationParameters<FF>&,
30-
[[maybe_unused]] const FF& scaling_factor)
31-
{
32-
using C = ColumnAndShifts;
33-
34-
PROFILE_THIS_NAME("accumulate/address_derivation");
35-
36-
const auto constants_GRUMPKIN_ONE_X = FF(1);
37-
const auto constants_GRUMPKIN_ONE_Y =
38-
FF(uint256_t{ 9457493854555940652UL, 3253583849847263892UL, 14921373847124204899UL, 2UL });
39-
const auto constants_GENERATOR_INDEX__CONTRACT_ADDRESS_V1 = FF(15);
40-
const auto constants_GENERATOR_INDEX__PARTIAL_ADDRESS = FF(27);
41-
const auto constants_GENERATOR_INDEX__PUBLIC_KEYS_HASH = FF(52);
42-
43-
{
44-
using Accumulator = typename std::tuple_element_t<0, ContainerOverSubrelations>;
45-
auto tmp = in.get(C::address_derivation_sel) * (FF(1) - in.get(C::address_derivation_sel));
46-
tmp *= scaling_factor;
47-
std::get<0>(evals) += typename Accumulator::View(tmp);
48-
}
49-
{
50-
using Accumulator = typename std::tuple_element_t<1, ContainerOverSubrelations>;
51-
auto tmp =
52-
in.get(C::address_derivation_sel) * (in.get(C::address_derivation_partial_address_domain_separator) -
53-
constants_GENERATOR_INDEX__PARTIAL_ADDRESS);
54-
tmp *= scaling_factor;
55-
std::get<1>(evals) += typename Accumulator::View(tmp);
56-
}
57-
{
58-
using Accumulator = typename std::tuple_element_t<2, ContainerOverSubrelations>;
59-
auto tmp =
60-
in.get(C::address_derivation_sel) * (in.get(C::address_derivation_public_keys_hash_domain_separator) -
61-
constants_GENERATOR_INDEX__PUBLIC_KEYS_HASH);
62-
tmp *= scaling_factor;
63-
std::get<2>(evals) += typename Accumulator::View(tmp);
64-
}
65-
{
66-
using Accumulator = typename std::tuple_element_t<3, ContainerOverSubrelations>;
67-
auto tmp = in.get(C::address_derivation_sel) * (in.get(C::address_derivation_preaddress_domain_separator) -
68-
constants_GENERATOR_INDEX__CONTRACT_ADDRESS_V1);
69-
tmp *= scaling_factor;
70-
std::get<3>(evals) += typename Accumulator::View(tmp);
71-
}
72-
{
73-
using Accumulator = typename std::tuple_element_t<4, ContainerOverSubrelations>;
74-
auto tmp =
75-
in.get(C::address_derivation_sel) * (in.get(C::address_derivation_g1_x) - constants_GRUMPKIN_ONE_X);
76-
tmp *= scaling_factor;
77-
std::get<4>(evals) += typename Accumulator::View(tmp);
78-
}
79-
{
80-
using Accumulator = typename std::tuple_element_t<5, ContainerOverSubrelations>;
81-
auto tmp =
82-
in.get(C::address_derivation_sel) * (in.get(C::address_derivation_g1_y) - constants_GRUMPKIN_ONE_Y);
83-
tmp *= scaling_factor;
84-
std::get<5>(evals) += typename Accumulator::View(tmp);
85-
}
86-
}
30+
[[maybe_unused]] const FF& scaling_factor);
8731
};
8832

8933
template <typename FF> class address_derivation : public Relation<address_derivationImpl<FF>> {

‎barretenberg/cpp/src/barretenberg/vm2/generated/relations/address_derivation_impl.hpp‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
// AUTOGENERATED FILE
2+
#pragma once
3+
4+
#include "barretenberg/vm2/generated/relations/address_derivation.hpp"
5+
6+
namespace bb::avm2 {
7+
8+
template <typename FF_>
9+
template <typename ContainerOverSubrelations, typename AllEntities>
10+
void address_derivationImpl<FF_>::accumulate(ContainerOverSubrelations& evals,
11+
const AllEntities& in,
12+
[[maybe_unused]] const RelationParameters<FF_>&,
13+
[[maybe_unused]] const FF_& scaling_factor)
14+
{
15+
using C = ColumnAndShifts;
16+
17+
PROFILE_THIS_NAME("accumulate/address_derivation");
18+
19+
const auto constants_GRUMPKIN_ONE_X = FF(1);
20+
const auto constants_GRUMPKIN_ONE_Y =
21+
FF(uint256_t{ 9457493854555940652UL, 3253583849847263892UL, 14921373847124204899UL, 2UL });
22+
const auto constants_GENERATOR_INDEX__CONTRACT_ADDRESS_V1 = FF(15);
23+
const auto constants_GENERATOR_INDEX__PARTIAL_ADDRESS = FF(27);
24+
const auto constants_GENERATOR_INDEX__PUBLIC_KEYS_HASH = FF(52);
25+
26+
{
27+
using Accumulator = typename std::tuple_element_t<0, ContainerOverSubrelations>;
28+
auto tmp = in.get(C::address_derivation_sel) * (FF(1) - in.get(C::address_derivation_sel));
29+
tmp *= scaling_factor;
30+
std::get<0>(evals) += typename Accumulator::View(tmp);
31+
}
32+
{
33+
using Accumulator = typename std::tuple_element_t<1, ContainerOverSubrelations>;
34+
auto tmp = in.get(C::address_derivation_sel) * (in.get(C::address_derivation_partial_address_domain_separator) -
35+
constants_GENERATOR_INDEX__PARTIAL_ADDRESS);
36+
tmp *= scaling_factor;
37+
std::get<1>(evals) += typename Accumulator::View(tmp);
38+
}
39+
{
40+
using Accumulator = typename std::tuple_element_t<2, ContainerOverSubrelations>;
41+
auto tmp =
42+
in.get(C::address_derivation_sel) * (in.get(C::address_derivation_public_keys_hash_domain_separator) -
43+
constants_GENERATOR_INDEX__PUBLIC_KEYS_HASH);
44+
tmp *= scaling_factor;
45+
std::get<2>(evals) += typename Accumulator::View(tmp);
46+
}
47+
{
48+
using Accumulator = typename std::tuple_element_t<3, ContainerOverSubrelations>;
49+
auto tmp = in.get(C::address_derivation_sel) * (in.get(C::address_derivation_preaddress_domain_separator) -
50+
constants_GENERATOR_INDEX__CONTRACT_ADDRESS_V1);
51+
tmp *= scaling_factor;
52+
std::get<3>(evals) += typename Accumulator::View(tmp);
53+
}
54+
{
55+
using Accumulator = typename std::tuple_element_t<4, ContainerOverSubrelations>;
56+
auto tmp = in.get(C::address_derivation_sel) * (in.get(C::address_derivation_g1_x) - constants_GRUMPKIN_ONE_X);
57+
tmp *= scaling_factor;
58+
std::get<4>(evals) += typename Accumulator::View(tmp);
59+
}
60+
{
61+
using Accumulator = typename std::tuple_element_t<5, ContainerOverSubrelations>;
62+
auto tmp = in.get(C::address_derivation_sel) * (in.get(C::address_derivation_g1_y) - constants_GRUMPKIN_ONE_Y);
63+
tmp *= scaling_factor;
64+
std::get<5>(evals) += typename Accumulator::View(tmp);
65+
}
66+
}
67+
68+
} // namespace bb::avm2

‎barretenberg/cpp/src/barretenberg/vm2/generated/relations/addressing.cpp‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// AUTOGENERATED FILE
2+
#include "barretenberg/flavor/relation_definitions.hpp"
3+
#include "barretenberg/vm2/constraining/flavor.hpp"
4+
#include "barretenberg/vm2/constraining/full_row.hpp"
5+
#include "barretenberg/vm2/constraining/recursion/recursive_flavor.hpp"
6+
#include "barretenberg/vm2/generated/relations/addressing_impl.hpp"
7+
8+
#define AvmCheckCircuitEdge(Flavor) Flavor::PolynomialEntitiesAtFixedRow<Flavor::ProverPolynomials>
9+
#define AvmCheckRelationEdge(Flavor) ::bb::avm2::AvmFullRowProxy
10+
11+
namespace bb::avm2 {
12+
13+
template class addressingImpl<AvmFlavorSettings::FF>;
14+
ACCUMULATE(addressingImpl, AvmFlavor, SumcheckTupleOfUnivariatesOverSubrelations, ExtendedEdge); // Prover.
15+
ACCUMULATE(addressingImpl, AvmFlavor, SumcheckArrayOfValuesOverSubrelations, EvaluationEdge); // Verifier.
16+
ACCUMULATE(addressingImpl, AvmFlavor, SumcheckArrayOfValuesOverSubrelations, AvmCheckCircuitEdge); // Check circuit.
17+
ACCUMULATE(addressingImpl,
18+
AvmFlavor,
19+
SumcheckArrayOfValuesOverSubrelations,
20+
AvmCheckRelationEdge); // Check relation (tests).
21+
22+
template class addressingImpl<AvmRecursiveFlavor::FF>;
23+
ACCUMULATE(addressingImpl, AvmRecursiveFlavor, SumcheckArrayOfValuesOverSubrelations, EvaluationEdge); // Verifier.
24+
25+
} // namespace bb::avm2

0 commit comments

Comments
 (0)