Merge commit 'a8a30e3541a9074e2e5ca85e7869c90f61448b8b' into catchup/long_lived_datalayer_merkle_blob_from_main_a8a30e3541a9074e2e5ca85e7869c90f61448b8b

Author: altendky

benchmarks/address_manager_store.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+import asyncio
+import io
+import random
+import tempfile
+import time
+from datetime import datetime
+from ipaddress import IPv4Address
+from pathlib import Path
+
+import aiofiles
+from chia_rs.sized_ints import uint16, uint64
+
+from chia.server.address_manager import (
+    NEW_BUCKETS_PER_ADDRESS,
+    AddressManager,
+    ExtendedPeerInfo,
+)
+from chia.types.peer_info import TimestampedPeerInfo
+from chia.util.files import write_file_async
+
+
+def generate_random_ip(rand: random.Random) -> str:
+    return str(IPv4Address(rand.getrandbits(32)))
+
+
+def populate_address_manager(num_new: int = 500000, num_tried: int = 200000) -> AddressManager:
+    rand = random.Random()
+    rand.seed(1337)
+    am = AddressManager()
+    current_time = int(datetime.now().timestamp())
+    total = num_new + num_tried
+
+    for i in range(total):
+        host = generate_random_ip(rand)
+        port = rand.randint(1024, 65535)
+        timestamp = current_time - rand.randint(0, 100000)
+
+        # Construct TimestampedPeerInfo
+        tpi = TimestampedPeerInfo(host=host, port=uint16(port), timestamp=uint64(timestamp))
+
+        # Create the ExtendedPeerInfo
+        epi = ExtendedPeerInfo(
+            addr=tpi,
+            src_peer=None,  # will default to itself inside constructor
+        )
+
+        am.tried_count += 1  # why do we even have `assert tried_ids != tried_count`?
+        node_id = am.id_count
+        am.id_count += 1
+        epi.random_pos = len(am.random_pos)
+        am.map_info[node_id] = epi
+        am.map_addr[epi.peer_info.host] = node_id
+        am.random_pos.append(node_id)
+
+        if i >= num_new:
+            # make a tried_table entry
+            epi.is_tried = True
+            epi.last_success = timestamp
+            epi.last_try = timestamp - rand.randint(0, 1000)
+            bucket = epi.get_tried_bucket(am.key)
+            pos = epi.get_bucket_position(am.key, False, bucket)
+            if am.tried_matrix[bucket][pos] == -1:
+                am.tried_matrix[bucket][pos] = node_id
+                am.tried_count += 1
+        else:
+            # make a new_table entry
+            ref_count = rand.randint(1, NEW_BUCKETS_PER_ADDRESS)
+            epi.ref_count = ref_count
+            assigned = False
+            for _ in range(ref_count):
+                bucket = epi.get_new_bucket(am.key)
+                pos = epi.get_bucket_position(am.key, True, bucket)
+                if am.new_matrix[bucket][pos] == -1:
+                    am.new_matrix[bucket][pos] = node_id
+                    am.new_count += 1
+                    assigned = True
+                    break
+            if not assigned:
+                # fallback if no bucket available
+                epi.ref_count = 0
+
+    return am
+
+
+async def benchmark_serialize_deserialize(iterations: int = 5) -> None:
+    """
+    Benchmarks the serialization and deserialization of peer data.
+    """
+
+    total_serialize_time = 0.0
+    total_deserialize_time = 0.0
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        peers_file_path = Path(tmpdir) / "peers.dat"
+
+        for i in range(iterations):
+            address_manager: AddressManager = populate_address_manager()
+            print(f"--- Benchmark Run {i + 1} ---")
+
+            # Benchmark serialize
+            start_serialize = time.perf_counter()
+
+            serialised_bytes = address_manager.serialize_bytes()
+            await write_file_async(peers_file_path, serialised_bytes, file_mode=0o644)
+            end_serialize = time.perf_counter()
+            serialize_duration = end_serialize - start_serialize
+            total_serialize_time += serialize_duration
+            print(f"Serialize time: {serialize_duration:.6f} seconds")
+
+            # Benchmark deserialize
+            async with aiofiles.open(peers_file_path, "rb") as f:
+                data = io.BytesIO(await f.read())
+            start_deserialize = time.perf_counter()
+            _ = AddressManager.deserialize_bytes(data)
+            end_deserialize = time.perf_counter()
+            deserialize_duration = end_deserialize - start_deserialize
+            total_deserialize_time += deserialize_duration
+            print(f"Deserialize time: {deserialize_duration:.6f} seconds")
+
+        print(f"\n=== Benchmark Summary ({iterations} iterations) ===")
+        print(f"Average serialize time:   {total_serialize_time / iterations:.6f} seconds")
+        print(f"Average deserialize time: {total_deserialize_time / iterations:.6f} seconds")
+
+
+async def main() -> None:
+    await benchmark_serialize_deserialize(iterations=10)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

benchmarks/block_store.py

@@ -7,7 +7,18 @@
 from pathlib import Path
 from time import monotonic
 
-from chia_rs import Foliage, FoliageBlockData, FoliageTransactionBlock, PoolTarget, RewardChainBlock, TransactionsInfo
+from chia_rs import (
+    BlockRecord,
+    Foliage,
+    FoliageBlockData,
+    FoliageTransactionBlock,
+    FullBlock,
+    PoolTarget,
+    ProofOfSpace,
+    RewardChainBlock,
+    SubEpochSummary,
+    TransactionsInfo,
+)
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint8, uint32, uint64, uint128
 
@@ -23,12 +34,8 @@
     rand_vdf_proof,
     rewards,
 )
-from chia.consensus.block_record import BlockRecord
 from chia.full_node.block_store import BlockStore
-from chia.types.blockchain_format.proof_of_space import ProofOfSpace
 from chia.types.blockchain_format.serialized_program import SerializedProgram
-from chia.types.blockchain_format.sub_epoch_summary import SubEpochSummary
-from chia.types.full_block import FullBlock
 
 # to run this benchmark:
 # python -m benchmarks.coin_store

benchmarks/mempool-long-lived.py

@@ -6,7 +6,7 @@
 from time import monotonic
 from typing import Optional
 
-from chia_rs import G2Element
+from chia_rs import CoinSpend, G2Element, SpendBundle
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint32, uint64
 from clvm.casts import int_to_bytes
@@ -16,10 +16,8 @@
 from chia.types.blockchain_format.coin import Coin
 from chia.types.blockchain_format.serialized_program import SerializedProgram
 from chia.types.coin_record import CoinRecord
-from chia.types.coin_spend import CoinSpend
 from chia.types.condition_opcodes import ConditionOpcode
-from chia.types.eligible_coin_spends import UnspentLineageInfo
-from chia.types.spend_bundle import SpendBundle
+from chia.types.mempool_item import UnspentLineageInfo
 
 # this is one week worth of blocks
 NUM_ITERS = 32256

benchmarks/mempool.py

@@ -9,6 +9,7 @@
 from time import monotonic
 from typing import Optional
 
+from chia_rs import SpendBundle
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint32, uint64
 
@@ -18,9 +19,8 @@
 from chia.simulator.wallet_tools import WalletTool
 from chia.types.blockchain_format.coin import Coin
 from chia.types.coin_record import CoinRecord
-from chia.types.eligible_coin_spends import UnspentLineageInfo
 from chia.types.mempool_inclusion_status import MempoolInclusionStatus
-from chia.types.spend_bundle import SpendBundle
+from chia.types.mempool_item import UnspentLineageInfo
 from chia.util.batches import to_batches
 from chia.util.task_referencer import create_referenced_task
 
@@ -243,9 +243,7 @@ async def add_spend_bundles(spend_bundles: list[SpendBundle]) -> None:
         with enable_profiler(True, f"create-{suffix}"):
             start = monotonic()
             for _ in range(10):
-                await mempool.create_block_generator(
-                    last_tb_header_hash=rec.header_hash,
-                )
+                mempool.create_block_generator(last_tb_header_hash=rec.header_hash)
             stop = monotonic()
         print(f"  time: {stop - start:0.4f}s")
         print(f"  per call: {(stop - start) / 10 * 1000:0.2f}ms")
@@ -254,7 +252,7 @@ async def add_spend_bundles(spend_bundles: list[SpendBundle]) -> None:
         with enable_profiler(True, f"create2-{suffix}"):
             start = monotonic()
             for _ in range(10):
-                await mempool.create_block_generator2(last_tb_header_hash=rec.header_hash)
+                mempool.create_block_generator2(last_tb_header_hash=rec.header_hash)
             stop = monotonic()
         print(f"  time: {stop - start:0.4f}s")
         print(f"  per call: {(stop - start) / 10 * 1000:0.2f}ms")

benchmarks/streamable.py

@@ -9,12 +9,12 @@
 from typing import Any, Callable, Optional, TextIO, Union
 
 import click
+from chia_rs import FullBlock
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint8, uint64
 
 from benchmarks.utils import EnumType, get_commit_hash
 from chia._tests.util.benchmarks import rand_bytes, rand_full_block, rand_hash
-from chia.types.full_block import FullBlock
 from chia.util.streamable import Streamable, streamable
 
 # to run this benchmark:

chia/_tests/blockchain/blockchain_test_utils.py

@@ -2,14 +2,13 @@
 
 from typing import Optional
 
-from chia_rs import SpendBundleConditions
+from chia_rs import FullBlock, SpendBundleConditions
 from chia_rs.sized_ints import uint32, uint64
 
 from chia.consensus.block_body_validation import ForkInfo
 from chia.consensus.blockchain import AddBlockResult, Blockchain
 from chia.consensus.difficulty_adjustment import get_next_sub_slot_iters_and_difficulty
 from chia.consensus.multiprocess_validation import PreValidationResult, pre_validate_block
-from chia.types.full_block import FullBlock
 from chia.types.validation_state import ValidationState
 from chia.util.augmented_chain import AugmentedBlockchain
 from chia.util.errors import Err

chia/_tests/blockchain/test_augmented_chain.py

@@ -4,14 +4,13 @@
 from typing import TYPE_CHECKING, ClassVar, Optional, cast
 
 import pytest
+from chia_rs import BlockRecord, FullBlock
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint32
 
 from chia._tests.blockchain.blockchain_test_utils import _validate_and_add_block
 from chia._tests.util.blockchain import create_blockchain
-from chia.consensus.block_record import BlockRecord
 from chia.simulator.block_tools import BlockTools
-from chia.types.full_block import FullBlock
 from chia.util.augmented_chain import AugmentedBlockchain
 from chia.util.errors import Err
 

chia/_tests/blockchain/test_blockchain.py

@@ -12,11 +12,16 @@
 import pytest
 from chia_rs import (
     AugSchemeMPL,
+    BlockRecord,
     ConsensusConstants,
+    EndOfSubSlotBundle,
+    FullBlock,
     G2Element,
     InfusedChallengeChainSubSlot,
     MerkleSet,
+    SpendBundle,
     TransactionsInfo,
+    UnfinishedBlock,
 )
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint8, uint32, uint64
@@ -34,7 +39,6 @@
 from chia._tests.util.get_name_puzzle_conditions import get_name_puzzle_conditions
 from chia.consensus.block_body_validation import ForkInfo
 from chia.consensus.block_header_validation import validate_finished_header_block
-from chia.consensus.block_record import BlockRecord
 from chia.consensus.block_rewards import calculate_base_farmer_reward
 from chia.consensus.blockchain import AddBlockResult, Blockchain
 from chia.consensus.coinbase import create_farmer_coin
@@ -52,11 +56,7 @@
 from chia.types.blockchain_format.vdf import VDFInfo, VDFProof, validate_vdf
 from chia.types.condition_opcodes import ConditionOpcode
 from chia.types.condition_with_args import ConditionWithArgs
-from chia.types.end_of_slot_bundle import EndOfSubSlotBundle
-from chia.types.full_block import FullBlock
 from chia.types.generator_types import BlockGenerator
-from chia.types.spend_bundle import SpendBundle
-from chia.types.unfinished_block import UnfinishedBlock
 from chia.types.validation_state import ValidationState
 from chia.util.augmented_chain import AugmentedBlockchain
 from chia.util.errors import Err

chia/_tests/blockchain/test_blockchain_transactions.py

@@ -3,6 +3,7 @@
 import logging
 
 import pytest
+from chia_rs import SpendBundle
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint32, uint64
 from clvm.casts import int_to_bytes
@@ -18,7 +19,7 @@
 from chia.simulator.wallet_tools import WalletTool
 from chia.types.condition_opcodes import ConditionOpcode
 from chia.types.condition_with_args import ConditionWithArgs
-from chia.types.spend_bundle import SpendBundle, estimate_fees
+from chia.types.spend_bundle import estimate_fees
 from chia.util.errors import Err
 from chia.wallet.conditions import AssertCoinAnnouncement, AssertPuzzleAnnouncement
 
@@ -65,7 +66,7 @@ async def test_basic_blockchain_tx(
         assert sb == spend_bundle
 
         last_block = blocks[-1]
-        result = await full_node_1.mempool_manager.create_bundle_from_mempool(last_block.header_hash)
+        result = full_node_1.mempool_manager.create_bundle_from_mempool(last_block.header_hash)
         assert result is not None
         next_spendbundle, _ = result
 

chia/_tests/blockchain/test_build_chains.py

@@ -3,11 +3,10 @@
 from typing import Optional
 
 import pytest
-from chia_rs import Coin, ConsensusConstants, additions_and_removals, get_flags_for_height_and_constants
+from chia_rs import Coin, ConsensusConstants, FullBlock, additions_and_removals, get_flags_for_height_and_constants
 from chia_rs.sized_ints import uint64
 
 from chia.simulator.block_tools import BlockTools
-from chia.types.full_block import FullBlock
 
 # These test targets are used to trigger a build of the test chains.
 # On CI we clone the test-cache repository to load the chains from, so they