txgraph: Expose ability to compare transactions (feature)

In order to make it possible for higher layers to compare transaction quality
(ordering within the implicit total ordering on the mempool), expose a comparison
function and test it.

Author: sipa

src/test/fuzz/txgraph.cpp

@@ -508,13 +508,79 @@ FUZZ_TARGET(txgraph)
                 // cause it to be re-evaluated in TxGraph).
                 sims.back().oversized = std::nullopt;
                 break;
+            } else if (!main_sim.IsOversized() && command-- == 0) {
+                // CompareMainOrder.
+                auto ref_a = pick_fn();
+                auto ref_b = pick_fn();
+                auto sim_a = main_sim.Find(ref_a);
+                auto sim_b = main_sim.Find(ref_b);
+                // Both transactions must exist in the main graph.
+                if (sim_a == SimTxGraph::MISSING || sim_b == SimTxGraph::MISSING) break;
+                auto cmp = real->CompareMainOrder(*ref_a, *ref_b);
+                // Distinct transactions have distinct places.
+                if (sim_a != sim_b) assert(cmp != 0);
+                // Ancestors go before descendants.
+                if (main_sim.graph.Ancestors(sim_a)[sim_b]) assert(cmp >= 0);
+                if (main_sim.graph.Descendants(sim_a)[sim_b]) assert(cmp <= 0);
+                // Do not verify consistency with chunk feerates, as we cannot easily determine
+                // these here without making more calls to real, which could affect its internal
+                // state. A full comparison is done at the end.
+                break;
             }
         }
     }
 
     // After running all modifications, perform an internal sanity check (before invoking
     // inspectors that may modify the internal state).
     real->SanityCheck();
+
+    if (!sims[0].IsOversized()) {
+        // If the main graph is not oversized, verify the total ordering implied by
+        // CompareMainOrder.
+        // First construct two distinct randomized permutations of the positions in sims[0].
+        std::vector<SimTxGraph::Pos> vec1;
+        for (auto i : sims[0].graph.Positions()) vec1.push_back(i);
+        std::shuffle(vec1.begin(), vec1.end(), rng);
+        auto vec2 = vec1;
+        std::shuffle(vec2.begin(), vec2.end(), rng);
+        if (vec1 == vec2) std::next_permutation(vec2.begin(), vec2.end());
+        // Sort both according to CompareMainOrder. By having randomized starting points, the order
+        // of CompareMainOrder invocations is somewhat randomized as well.
+        auto cmp = [&](SimTxGraph::Pos a, SimTxGraph::Pos b) noexcept {
+            return real->CompareMainOrder(*sims[0].GetRef(a), *sims[0].GetRef(b)) < 0;
+        };
+        std::sort(vec1.begin(), vec1.end(), cmp);
+        std::sort(vec2.begin(), vec2.end(), cmp);
+
+        // Verify the resulting orderings are identical. This could only fail if the ordering was
+        // not total.
+        assert(vec1 == vec2);
+
+        // Verify that the ordering is topological.
+        auto todo = sims[0].graph.Positions();
+        for (auto i : vec1) {
+            todo.Reset(i);
+            assert(!sims[0].graph.Ancestors(i).Overlaps(todo));
+        }
+        assert(todo.None());
+
+        // For every transaction in the total ordering, find a random one before it and after it,
+        // and compare their chunk feerates, which must be consistent with the ordering.
+        for (size_t pos = 0; pos < vec1.size(); ++pos) {
+            auto pos_feerate = real->GetMainChunkFeerate(*sims[0].GetRef(vec1[pos]));
+            if (pos > 0) {
+                size_t before = rng.randrange<size_t>(pos);
+                auto before_feerate = real->GetMainChunkFeerate(*sims[0].GetRef(vec1[before]));
+                assert(FeeRateCompare(before_feerate, pos_feerate) >= 0);
+            }
+            if (pos + 1 < vec1.size()) {
+                size_t after = pos + 1 + rng.randrange<size_t>(vec1.size() - 1 - pos);
+                auto after_feerate = real->GetMainChunkFeerate(*sims[0].GetRef(vec1[after]));
+                assert(FeeRateCompare(after_feerate, pos_feerate) <= 0);
+            }
+        }
+    }
+
     assert(real->HaveStaging() == (sims.size() > 1));
 
     // Try to run a full comparison, for both main_only=false and main_only=true in TxGraph

src/txgraph.cpp

@@ -303,6 +303,8 @@ class TxGraphImpl final : public TxGraph
         Locator m_locator[MAX_LEVELS];
         /** The chunk feerate of this transaction in main (if present in m_locator[0]). */
         FeePerWeight m_main_chunk_feerate;
+        /** The position this transaction has in the main linearization (if present). */
+        LinearizationIndex m_main_lin_index;
     };
 
     /** The set of all transactions (in all levels combined). GraphIndex values index into this. */
@@ -447,6 +449,7 @@ class TxGraphImpl final : public TxGraph
     std::vector<Ref*> GetDescendants(const Ref& arg, bool main_only = false) noexcept final;
     GraphIndex GetTransactionCount(bool main_only = false) noexcept final;
     bool IsOversized(bool main_only = false) noexcept final;
+    std::strong_ordering CompareMainOrder(const Ref& a, const Ref& b) noexcept final;
 
     void SanityCheck() const final;
 };
@@ -499,9 +502,10 @@ void Cluster::Updated(TxGraphImpl& graph) noexcept
         entry.m_locator[m_level].SetPresent(this, idx);
     }
     // If this is for the main graph (level = 0), and the Cluster's quality is ACCEPTABLE or
-    // OPTIMAL, compute its chunking and store its information in the Entry's m_main_chunk_feerate.
-    // These fields are only accessed after making the entire graph ACCEPTABLE, so it is pointless
-    // to compute these if we haven't reached that quality level yet.
+    // OPTIMAL, compute its chunking and store its information in the Entry's m_main_lin_index
+    // and m_main_chunk_feerate. These fields are only accessed after making the entire graph
+    // ACCEPTABLE, so it is pointless to compute these if we haven't reached that quality level
+    // yet.
     if (m_level == 0 && IsAcceptable()) {
         LinearizationChunking chunking(m_depgraph, m_linearization);
         LinearizationIndex lin_idx{0};
@@ -511,9 +515,10 @@ void Cluster::Updated(TxGraphImpl& graph) noexcept
             Assume(chunk.transactions.Any());
             // Iterate over the transactions in the linearization, which must match those in chunk.
             do {
-                DepGraphIndex idx = m_linearization[lin_idx++];
+                DepGraphIndex idx = m_linearization[lin_idx];
                 GraphIndex graph_idx = m_mapping[idx];
                 auto& entry = graph.m_entries[graph_idx];
+                entry.m_main_lin_index = lin_idx++;
                 entry.m_main_chunk_feerate = FeePerWeight::FromFeeFrac(chunk.feerate);
                 Assume(chunk.transactions[idx]);
                 chunk.transactions.Reset(idx);
@@ -594,6 +599,10 @@ void Cluster::ApplyRemovals(TxGraphImpl& graph, std::span<GraphIndex>& to_remove
         //   are just never accessed, but set it to -1 here to increase the ability to detect a bug
         //   that causes it to be accessed regardless.
         m_mapping[locator.index] = GraphIndex(-1);
+        // - Remove its linearization index from the Entry (if in main).
+        if (m_level == 0) {
+            entry.m_main_lin_index = LinearizationIndex(-1);
+        }
         // - Mark it as missing/removed in the Entry's locator.
         graph.ClearLocator(m_level, idx);
         to_remove = to_remove.subspan(1);
@@ -1730,6 +1739,33 @@ void TxGraphImpl::SetTransactionFee(const Ref& ref, int64_t fee) noexcept
     }
 }
 
+std::strong_ordering TxGraphImpl::CompareMainOrder(const Ref& a, const Ref& b) noexcept
+{
+    // The references must not be empty.
+    Assume(GetRefGraph(a) == this);
+    Assume(GetRefGraph(b) == this);
+    // Apply dependencies in main.
+    ApplyDependencies(0);
+    Assume(m_main_clusterset.m_deps_to_add.empty());
+    // Make both involved Clusters acceptable, so chunk feerates are relevant.
+    const auto& entry_a = m_entries[GetRefIndex(a)];
+    const auto& entry_b = m_entries[GetRefIndex(b)];
+    const auto& locator_a = entry_a.m_locator[0];
+    const auto& locator_b = entry_b.m_locator[0];
+    Assume(locator_a.IsPresent());
+    Assume(locator_b.IsPresent());
+    MakeAcceptable(*locator_a.cluster);
+    MakeAcceptable(*locator_b.cluster);
+    // Compare chunk feerates, and return result if it differs.
+    auto feerate_cmp = FeeRateCompare(entry_b.m_main_chunk_feerate, entry_a.m_main_chunk_feerate);
+    if (feerate_cmp < 0) return std::strong_ordering::less;
+    if (feerate_cmp > 0) return std::strong_ordering::greater;
+    // Compare Cluster* as tie-break for equal chunk feerates.
+    if (locator_a.cluster != locator_b.cluster) return locator_a.cluster <=> locator_b.cluster;
+    // As final tie-break, compare position within cluster linearization.
+    return entry_a.m_main_lin_index <=> entry_b.m_main_lin_index;
+}
+
 void Cluster::SanityCheck(const TxGraphImpl& graph, int level) const
 {
     // There must be an m_mapping for each m_depgraph position (including holes).
@@ -1747,6 +1783,7 @@ void Cluster::SanityCheck(const TxGraphImpl& graph, int level) const
 
     // Verify m_linearization.
     SetType m_done;
+    LinearizationIndex linindex{0};
     assert(m_depgraph.IsAcyclic());
     for (auto lin_pos : m_linearization) {
         assert(lin_pos < m_mapping.size());
@@ -1759,6 +1796,8 @@ void Cluster::SanityCheck(const TxGraphImpl& graph, int level) const
         assert(entry.m_locator[level].index == lin_pos);
         // For main-level entries, check linearization position and chunk feerate.
         if (level == 0 && IsAcceptable()) {
+            assert(entry.m_main_lin_index == linindex);
+            ++linindex;
             if (!linchunking.GetChunk(0).transactions[lin_pos]) {
                 linchunking.MarkDone(linchunking.GetChunk(0).transactions);
             }

src/txgraph.h

@@ -141,6 +141,9 @@ class TxGraph
      *  graph exists, it is queried; otherwise the main graph is queried. This is available even
      *  for oversized graphs. */
     virtual GraphIndex GetTransactionCount(bool main_only = false) noexcept = 0;
+    /** Compare two transactions according to their order in the main graph. Both transactions must
+     *  be in the main graph. The main graph must not be oversized. */
+    virtual std::strong_ordering CompareMainOrder(const Ref& a, const Ref& b) noexcept = 0;
 
     /** Perform an internal consistency check on this object. */
     virtual void SanityCheck() const = 0;