test: accept unordered tracepoints in...

...interface_usdt_utxocache.py

We have encountered an instance where the tracepoints were not collected
in the same order they were fired (#31951).

Tracepoint ordering is not guaranteed in userspace for a number of
reasons.

As this test does not require a strict collection/processing order
collect `expected` and `actual` events into dicts and compare them.

This will gracefully handle both the number of events, and out-of-order
events should they reoccur in the future.

Author: willcl-ark

test/functional/interface_usdt_utxocache.py

@@ -265,14 +265,6 @@ def test_add_spent(self):
         actual_utxocache_adds = []
         actual_utxocache_spents = []
 
-        def compare_utxo_with_event(utxo, event):
-            """Compare a utxo dict to the event produced by BPF"""
-            assert_equal(utxo["txid"], bytes(event.txid[::-1]).hex())
-            assert_equal(utxo["index"], event.index)
-            assert_equal(utxo["height"], event.height)
-            assert_equal(utxo["value"], event.value)
-            assert_equal(utxo["is_coinbase"], event.is_coinbase)
-
         def handle_utxocache_add(_, data, __):
             event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents
             self.log.info(f"handle_utxocache_add(): {event}")
@@ -324,9 +316,28 @@ def handle_utxocache_spent(_, data, __):
 
         self.log.info(
             f"check that we successfully traced {EXPECTED_HANDLE_ADD_SUCCESS} adds and {EXPECTED_HANDLE_SPENT_SUCCESS} spent")
-        for expected_utxo, actual_event in zip(expected_utxocache_adds + expected_utxocache_spents,
-                                               actual_utxocache_adds + actual_utxocache_spents):
-            compare_utxo_with_event(expected_utxo, actual_event)
+
+        # Check that all expected tracepoints are recieved, but not the order they were recieved in.
+        # Tracepoint ordering is not strictly guaranteed, so this comparison avoids intermittent failures in the test.
+        def cache_event_to_key(event):
+            return (
+                bytes(event.txid[::-1]).hex(),
+                event.index,
+                event.height,
+                event.value,
+                event.is_coinbase
+            )
+
+        expected_add_keys = {(e["txid"], e["index"], e["height"], e["value"], e["is_coinbase"])
+                             for e in expected_utxocache_adds}
+        expected_spent_keys = {(e["txid"], e["index"], e["height"], e["value"], e["is_coinbase"])
+                              for e in expected_utxocache_spents}
+
+        actual_add_keys = {cache_event_to_key(e) for e in actual_utxocache_adds}
+        actual_spent_keys = {cache_event_to_key(e) for e in actual_utxocache_spents}
+
+        assert_equal(expected_add_keys, actual_add_keys)
+        assert_equal(expected_spent_keys, actual_spent_keys)
 
         bpf.cleanup()