Skip to content

Commit 4f8958d

Browse files
sipasipa
committed
clusterlin: add PostLinearize + benchmarks + fuzz tests
1 parent 0e2812d commit 4f8958d

File tree

3 files changed

+391
-0
lines changed

3 files changed

+391
-0
lines changed

‎src/bench/cluster_linearize.cpp

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,17 @@ void BenchLinearizeNoItersWorstCaseLIMO(ClusterIndex ntx, benchmark::Bench& benc
169169
});
170170
}
171171

172+
template<typename SetType>
173+
void BenchPostLinearizeWorstCase(ClusterIndex ntx, benchmark::Bench& bench)
174+
{
175+
DepGraph<SetType> depgraph = MakeWideGraph<SetType>(ntx);
176+
std::vector<ClusterIndex> lin(ntx);
177+
bench.run([&] {
178+
for (ClusterIndex i = 0; i < ntx; ++i) lin[i] = i;
179+
PostLinearize(depgraph, lin);
180+
});
181+
}
182+
172183
} // namespace
173184

174185
static void LinearizePerIter16TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase<BitSet<16>>(16, bench); }
@@ -192,6 +203,13 @@ static void LinearizeNoIters64TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLi
192203
static void LinearizeNoIters75TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO<BitSet<75>>(75, bench); }
193204
static void LinearizeNoIters99TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO<BitSet<99>>(99, bench); }
194205

206+
static void PostLinearize16TxWorstCase(benchmark::Bench& bench) { BenchPostLinearizeWorstCase<BitSet<16>>(16, bench); }
207+
static void PostLinearize32TxWorstCase(benchmark::Bench& bench) { BenchPostLinearizeWorstCase<BitSet<32>>(32, bench); }
208+
static void PostLinearize48TxWorstCase(benchmark::Bench& bench) { BenchPostLinearizeWorstCase<BitSet<48>>(48, bench); }
209+
static void PostLinearize64TxWorstCase(benchmark::Bench& bench) { BenchPostLinearizeWorstCase<BitSet<64>>(64, bench); }
210+
static void PostLinearize75TxWorstCase(benchmark::Bench& bench) { BenchPostLinearizeWorstCase<BitSet<75>>(75, bench); }
211+
static void PostLinearize99TxWorstCase(benchmark::Bench& bench) { BenchPostLinearizeWorstCase<BitSet<99>>(99, bench); }
212+
195213
BENCHMARK(LinearizePerIter16TxWorstCase, benchmark::PriorityLevel::HIGH);
196214
BENCHMARK(LinearizePerIter32TxWorstCase, benchmark::PriorityLevel::HIGH);
197215
BENCHMARK(LinearizePerIter48TxWorstCase, benchmark::PriorityLevel::HIGH);
@@ -212,3 +230,10 @@ BENCHMARK(LinearizeNoIters48TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH);
212230
BENCHMARK(LinearizeNoIters64TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH);
213231
BENCHMARK(LinearizeNoIters75TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH);
214232
BENCHMARK(LinearizeNoIters99TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH);
233+
234+
BENCHMARK(PostLinearize16TxWorstCase, benchmark::PriorityLevel::HIGH);
235+
BENCHMARK(PostLinearize32TxWorstCase, benchmark::PriorityLevel::HIGH);
236+
BENCHMARK(PostLinearize48TxWorstCase, benchmark::PriorityLevel::HIGH);
237+
BENCHMARK(PostLinearize64TxWorstCase, benchmark::PriorityLevel::HIGH);
238+
BENCHMARK(PostLinearize75TxWorstCase, benchmark::PriorityLevel::HIGH);
239+
BENCHMARK(PostLinearize99TxWorstCase, benchmark::PriorityLevel::HIGH);

‎src/cluster_linearize.h

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ class DepGraph
122122
auto TxCount() const noexcept { return entries.size(); }
123123
/** Get the feerate of a given transaction i. Complexity: O(1). */
124124
const FeeFrac& FeeRate(ClusterIndex i) const noexcept { return entries[i].feerate; }
125+
/** Get the mutable feerate of a given transaction i. Complexity: O(1). */
126+
FeeFrac& FeeRate(ClusterIndex i) noexcept { return entries[i].feerate; }
125127
/** Get the ancestors of a given transaction i. Complexity: O(1). */
126128
const SetType& Ancestors(ClusterIndex i) const noexcept { return entries[i].ancestors; }
127129
/** Get the descendants of a given transaction i. Complexity: O(1). */
@@ -782,6 +784,207 @@ std::pair<std::vector<ClusterIndex>, bool> Linearize(const DepGraph<SetType>& de
782784
return {std::move(linearization), optimal};
783785
}
784786

787+
/** Improve a given linearization.
788+
*
789+
* @param[in] depgraph Dependency graph of the cluster being linearized.
790+
* @param[in,out] linearization On input, an existing linearization for depgraph. On output, a
791+
* potentially better linearization for the same graph.
792+
*
793+
* Postlinearization guarantees:
794+
* - The resulting chunks are connected.
795+
* - If the input has a tree shape (either all transactions have at most one child, or all
796+
* transactions have at most one parent), the result is optimal.
797+
* - Given a linearization L1 and a leaf transaction T in it. Let L2 be L1 with T moved to the end,
798+
* optionally with its fee increased. Let L3 be the postlinearization of L2. L3 will be at least
799+
* as good as L1. This means that replacing transactions with same-size higher-fee transactions
800+
* will not worsen linearizations through a "drop conflicts, append new transactions,
801+
* postlinearize" process.
802+
*/
803+
template<typename SetType>
804+
void PostLinearize(const DepGraph<SetType>& depgraph, Span<ClusterIndex> linearization)
805+
{
806+
// This algorithm performs a number of passes (currently 2); the even ones operate from back to
807+
// front, the odd ones from front to back. Each results in an equal-or-better linearization
808+
// than the one started from.
809+
// - One pass in either direction guarantees that the resulting chunks are connected.
810+
// - Each direction corresponds to one shape of tree being linearized optimally (forward passes
811+
// guarantee this for graphs where each transaction has at most one child; backward passes
812+
// guarantee this for graphs where each transaction has at most one parent).
813+
// - Starting with a backward pass guarantees the moved-tree property.
814+
//
815+
// During an odd (forward) pass, the high-level operation is:
816+
// - Start with an empty list of groups L=[].
817+
// - For every transaction i in the old linearization, from front to back:
818+
// - Append a new group C=[i], containing just i, to the back of L.
819+
// - While L has at least one group before C, and the group immediately before C has feerate
820+
// lower than C:
821+
// - If C depends on P:
822+
// - Merge P into C, making C the concatenation of P+C, continuing with the combined C.
823+
// - Otherwise:
824+
// - Swap P with C, continuing with the now-moved C.
825+
// - The output linearization is the concatenation of the groups in L.
826+
//
827+
// During even (backward) passes, i iterates from the back to the front of the existing
828+
// linearization, and new groups are prepended instead of appended to the list L. To enable
829+
// more code reuse, both passes append groups, but during even passes the meanings of
830+
// parent/child, and of high/low feerate are reversed, and the final concatenation is reversed
831+
// on output.
832+
//
833+
// In the implementation below, the groups are represented by singly-linked lists (pointing
834+
// from the back to the front), which are themselves organized in a singly-linked circular
835+
// list (each group pointing to its predecessor, with a special sentinel group at the front
836+
// that points back to the last group).
837+
//
838+
// Information about transaction t is stored in entries[t + 1], while the sentinel is in
839+
// entries[0].
840+
841+
/** Index of the sentinel in the entries array below. */
842+
static constexpr ClusterIndex SENTINEL{0};
843+
/** Indicator that a group has no previous transaction. */
844+
static constexpr ClusterIndex NO_PREV_TX{0};
845+
846+
847+
/** Data structure per transaction entry. */
848+
struct TxEntry
849+
{
850+
/** The index of the previous transaction in this group; NO_PREV_TX if this is the first
851+
* entry of a group. */
852+
ClusterIndex prev_tx;
853+
854+
// The fields below are only used for transactions that are the last one in a group
855+
// (referred to as tail transactions below).
856+
857+
/** Index of the first transaction in this group, possibly itself. */
858+
ClusterIndex first_tx;
859+
/** Index of the last transaction in the previous group. The first group (the sentinel)
860+
* points back to the last group here, making it a singly-linked circular list. */
861+
ClusterIndex prev_group;
862+
/** All transactions in the group. Empty for the sentinel. */
863+
SetType group;
864+
/** All dependencies of the group (descendants in even passes; ancestors in odd ones). */
865+
SetType deps;
866+
/** The combined fee/size of transactions in the group. Fee is negated in even passes. */
867+
FeeFrac feerate;
868+
};
869+
870+
// As an example, consider the state corresponding to the linearization [1,0,3,2], with
871+
// groups [1,0,3] and [2], in an odd pass. The linked lists would be:
872+
//
873+
// +-----+
874+
// 0<-P-- | 0 S | ---\ Legend:
875+
// +-----+ |
876+
// ^ | - digit in box: entries index
877+
// /--------------F---------+ G | (note: one more than tx value)
878+
// v \ | | - S: sentinel group
879+
// +-----+ +-----+ +-----+ | (empty feerate)
880+
// 0<-P-- | 2 | <--P-- | 1 | <--P-- | 4 T | | - T: tail transaction, contains
881+
// +-----+ +-----+ +-----+ | fields beyond prev_tv.
882+
// ^ | - P: prev_tx reference
883+
// G G - F: first_tx reference
884+
// | | - G: prev_group reference
885+
// +-----+ |
886+
// 0<-P-- | 3 T | <--/
887+
// +-----+
888+
// ^ |
889+
// \-F-/
890+
//
891+
// During an even pass, the diagram above would correspond to linearization [2,3,0,1], with
892+
// groups [2] and [3,0,1].
893+
894+
std::vector<TxEntry> entries(linearization.size() + 1);
895+
896+
// Perform two passes over the linearization.
897+
for (int pass = 0; pass < 2; ++pass) {
898+
int rev = !(pass & 1);
899+
// Construct a sentinel group, identifying the start of the list.
900+
entries[SENTINEL].prev_group = SENTINEL;
901+
Assume(entries[SENTINEL].feerate.IsEmpty());
902+
903+
// Iterate over all elements in the existing linearization.
904+
for (ClusterIndex i = 0; i < linearization.size(); ++i) {
905+
// Even passes are from back to front; odd passes from front to back.
906+
ClusterIndex idx = linearization[rev ? linearization.size() - 1 - i : i];
907+
// Construct a new group containing just idx. In even passes, the meaning of
908+
// parent/child and high/low feerate are swapped.
909+
ClusterIndex cur_group = idx + 1;
910+
entries[cur_group].group = SetType::Singleton(idx);
911+
entries[cur_group].deps = rev ? depgraph.Descendants(idx): depgraph.Ancestors(idx);
912+
entries[cur_group].feerate = depgraph.FeeRate(idx);
913+
if (rev) entries[cur_group].feerate.fee = -entries[cur_group].feerate.fee;
914+
entries[cur_group].prev_tx = NO_PREV_TX; // No previous transaction in group.
915+
entries[cur_group].first_tx = cur_group; // Transaction itself is first of group.
916+
// Insert the new group at the back of the groups linked list.
917+
entries[cur_group].prev_group = entries[SENTINEL].prev_group;
918+
entries[SENTINEL].prev_group = cur_group;
919+
920+
// Start merge/swap cycle.
921+
ClusterIndex next_group = SENTINEL; // We inserted at the end, so next group is sentinel.
922+
ClusterIndex prev_group = entries[cur_group].prev_group;
923+
// Continue as long as the current group has higher feerate than the previous one.
924+
while (entries[cur_group].feerate >> entries[prev_group].feerate) {
925+
// prev_group/cur_group/next_group refer to (the last transactions of) 3
926+
// consecutive entries in groups list.
927+
Assume(cur_group == entries[next_group].prev_group);
928+
Assume(prev_group == entries[cur_group].prev_group);
929+
// The sentinel has empty feerate, which is neither higher or lower than other
930+
// feerates. Thus, the while loop we are in here guarantees that cur_group and
931+
// prev_group are not the sentinel.
932+
Assume(cur_group != SENTINEL);
933+
Assume(prev_group != SENTINEL);
934+
if (entries[cur_group].deps.Overlaps(entries[prev_group].group)) {
935+
// There is a dependency between cur_group and prev_group; merge prev_group
936+
// into cur_group. The group/deps/feerate fields of prev_group remain unchanged
937+
// but become unused.
938+
entries[cur_group].group |= entries[prev_group].group;
939+
entries[cur_group].deps |= entries[prev_group].deps;
940+
entries[cur_group].feerate += entries[prev_group].feerate;
941+
// Make the first of the current group point to the tail of the previous group.
942+
entries[entries[cur_group].first_tx].prev_tx = prev_group;
943+
// The first of the previous group becomes the first of the newly-merged group.
944+
entries[cur_group].first_tx = entries[prev_group].first_tx;
945+
// The previous group becomes whatever group was before the former one.
946+
prev_group = entries[prev_group].prev_group;
947+
entries[cur_group].prev_group = prev_group;
948+
} else {
949+
// There is no dependency between cur_group and prev_group; swap them.
950+
ClusterIndex preprev_group = entries[prev_group].prev_group;
951+
// If PP, P, C, N were the old preprev, prev, cur, next groups, then the new
952+
// layout becomes [PP, C, P, N]. Update prev_groups to reflect that order.
953+
entries[next_group].prev_group = prev_group;
954+
entries[prev_group].prev_group = cur_group;
955+
entries[cur_group].prev_group = preprev_group;
956+
// The current group remains the same, but the groups before/after it have
957+
// changed.
958+
next_group = prev_group;
959+
prev_group = preprev_group;
960+
}
961+
}
962+
}
963+
964+
// Convert the entries back to linearization (overwriting the existing one).
965+
ClusterIndex cur_group = entries[0].prev_group;
966+
ClusterIndex done = 0;
967+
while (cur_group != SENTINEL) {
968+
ClusterIndex cur_tx = cur_group;
969+
// Traverse the transactions of cur_group (from back to front), and write them in the
970+
// same order during odd passes, and reversed (front to back) in even passes.
971+
if (rev) {
972+
do {
973+
*(linearization.begin() + (done++)) = cur_tx - 1;
974+
cur_tx = entries[cur_tx].prev_tx;
975+
} while (cur_tx != NO_PREV_TX);
976+
} else {
977+
do {
978+
*(linearization.end() - (++done)) = cur_tx - 1;
979+
cur_tx = entries[cur_tx].prev_tx;
980+
} while (cur_tx != NO_PREV_TX);
981+
}
982+
cur_group = entries[cur_group].prev_group;
983+
}
984+
Assume(done == linearization.size());
985+
}
986+
}
987+
785988
} // namespace cluster_linearize
786989

787990
#endif // BITCOIN_CLUSTER_LINEARIZE_H

0 commit comments

Comments
 (0)